diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..e48fa823 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,17 @@ +.git +.github +default.etcd +*.gz +*.tar.gz +*.bzip2 +*.zip +browser/node_modules +node_modules +docs/debugging/s3-verify/s3-verify +docs/debugging/xl-meta/xl-meta +docs/debugging/s3-check-md5/s3-check-md5 +docs/debugging/hash-set/hash-set +docs/debugging/healing-bin/healing-bin +docs/debugging/inspect/inspect +docs/debugging/pprofgoparser/pprofgoparser +docs/debugging/reorder-disks/reorder-disks diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md new file mode 100644 index 00000000..978d9085 --- /dev/null +++ b/.github/ISSUE_TEMPLATE.md @@ -0,0 +1,51 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: community, triage +assignees: '' + +--- + +## NOTE +All GitHub issues are addressed on a best-effort basis at MinIO's sole discretion. There are no Service Level Agreements (SLA) or Objectives (SLO). Remember our [Code of Conduct](https://github.com/minio/minio/blob/master/code_of_conduct.md) when engaging with MinIO Engineers and the larger community. + +For urgent issues (e.g. production down, etc.), subscribe to [SUBNET](https://min.io/pricing?jmp=github) for direct to engineering support. + + + +## Expected Behavior + + + +## Current Behavior + + + +## Possible Solution + + + +## Steps to Reproduce (for bugs) + + + + +1. +2. +3. +4. + +## Context + + + +## Regression + + + +## Your Environment + +* Version used (`minio --version`): +* Server setup and configuration: +* Operating System and version (`uname -a`): diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..cd3f4153 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,54 @@ +--- +name: Bug report +about: Report a bug in MinIO (community edition is source-only) +title: '' +labels: community, triage +assignees: '' + +--- + +## IMPORTANT NOTES + +**Community Edition**: MinIO community edition is now source-only. Install via `go install github.com/minio/minio@latest` + +**Feature Requests**: We are no longer accepting feature requests for the community edition. For feature requests and enterprise support, please subscribe to [MinIO Enterprise Support](https://min.io/pricing). + +**Urgent Issues**: If this case is urgent or affects production, please subscribe to [SUBNET](https://min.io/pricing) for 24/7 enterprise support. + + + +## Expected Behavior + + + +## Current Behavior + + + +## Possible Solution + + + +## Steps to Reproduce (for bugs) + + + + +1. +2. +3. +4. + +## Context + + + +## Regression + + + +## Your Environment + +* Version used (`minio --version`): +* Server setup and configuration: +* Operating System and version (`uname -a`): diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml new file mode 100644 index 00000000..28ed32af --- /dev/null +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -0,0 +1,8 @@ +blank_issues_enabled: false +contact_links: + - name: MinIO Community Support + url: https://slack.min.io + about: Community support via Slack - for questions and discussions + - name: MinIO Enterprise Support (SUBNET) + url: https://min.io/pricing + about: Enterprise support with SLA - for production deployments and feature requests diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md new file mode 100644 index 00000000..2ff68465 --- /dev/null +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -0,0 +1,26 @@ +## Community Contribution License +All community contributions in this pull request are licensed to the project maintainers +under the terms of the [Apache 2 license](https://www.apache.org/licenses/LICENSE-2.0). +By creating this pull request I represent that I have the right to license the +contributions to the project maintainers under the Apache 2 license. + +## Description + + +## Motivation and Context + + +## How to test this PR? + + +## Types of changes +- [ ] Bug fix (non-breaking change which fixes an issue) +- [ ] New feature (non-breaking change which adds functionality) +- [ ] Optimization (provides speedup with no functional changes) +- [ ] Breaking change (fix or feature that would cause existing functionality to change) + +## Checklist: +- [ ] Fixes a regression (If yes, please add `commit-id` or `PR #` here) +- [ ] Unit tests added/updated +- [ ] Internal documentation updated +- [ ] Create a documentation update request [here](https://github.com/minio/docs/issues/new?label=doc-change,title=Doc+Updated+Needed+For+PR+github.com%2fminio%2fminio%2fpull%2fNNNNN) diff --git a/.github/logo.svg b/.github/logo.svg new file mode 100644 index 00000000..e928e786 --- /dev/null +++ b/.github/logo.svg @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/.github/stale.yml b/.github/stale.yml new file mode 100644 index 00000000..1085be6b --- /dev/null +++ b/.github/stale.yml @@ -0,0 +1,60 @@ +# Configuration for probot-stale - https://github.com/probot/stale + +# Number of days of inactivity before an Issue or Pull Request becomes stale +daysUntilStale: 30 + +# Number of days of inactivity before an Issue or Pull Request with the stale label is closed. +# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale. +daysUntilClose: 15 + +# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled) +onlyLabels: [] + +# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable +exemptLabels: + - "security" + - "pending discussion" + - "do-not-close" + +# Set to true to ignore issues in a project (defaults to false) +exemptProjects: false + +# Set to true to ignore issues in a milestone (defaults to false) +exemptMilestones: false + +# Set to true to ignore issues with an assignee (defaults to false) +exemptAssignees: false + +# Label to use when marking as stale +staleLabel: stale + +# Comment to post when marking as stale. Set to `false` to disable +markComment: >- + This issue has been automatically marked as stale because it has not had + recent activity. It will be closed after 15 days if no further activity + occurs. Thank you for your contributions. +# Comment to post when removing the stale label. +# unmarkComment: > +# Your comment here. + +# Comment to post when closing a stale Issue or Pull Request. +# closeComment: > +# Your comment here. + +# Limit the number of actions per hour, from 1-30. Default is 30 +limitPerRun: 1 + +# Limit to only `issues` or `pulls` +# only: issues + +# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls': +# pulls: +# daysUntilStale: 30 +# markComment: > +# This pull request has been automatically marked as stale because it has not had +# recent activity. It will be closed if no further activity occurs. Thank you +# for your contributions. + +# issues: +# exemptLabels: +# - confirmed diff --git a/.github/workflows/depsreview.yaml b/.github/workflows/depsreview.yaml new file mode 100644 index 00000000..b9d6d20f --- /dev/null +++ b/.github/workflows/depsreview.yaml @@ -0,0 +1,14 @@ +name: 'Dependency Review' +on: [pull_request] + +permissions: + contents: read + +jobs: + dependency-review: + runs-on: ubuntu-latest + steps: + - name: 'Checkout Repository' + uses: actions/checkout@v4 + - name: 'Dependency Review' + uses: actions/dependency-review-action@v4 diff --git a/.github/workflows/go-cross.yml b/.github/workflows/go-cross.yml new file mode 100644 index 00000000..324af3e9 --- /dev/null +++ b/.github/workflows/go-cross.yml @@ -0,0 +1,39 @@ +name: Crosscompile + +on: + pull_request: + branches: + - master + +# This ensures that previous jobs for the PR are canceled when the PR is +# updated. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + build: + name: Build Tests with Go ${{ matrix.go-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + go-version: [1.24.x] + os: [ubuntu-latest] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + check-latest: true + - name: Build on ${{ matrix.os }} + if: matrix.os == 'ubuntu-latest' + env: + CGO_ENABLED: 0 + GO111MODULE: on + run: | + sudo sysctl net.ipv6.conf.all.disable_ipv6=0 + sudo sysctl net.ipv6.conf.default.disable_ipv6=0 + make crosscompile diff --git a/.github/workflows/go-healing.yml b/.github/workflows/go-healing.yml new file mode 100644 index 00000000..0ea25051 --- /dev/null +++ b/.github/workflows/go-healing.yml @@ -0,0 +1,44 @@ +name: Healing Functional Tests + +on: + pull_request: + branches: + - master + +# This ensures that previous jobs for the PR are canceled when the PR is +# updated. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + build: + name: Go ${{ matrix.go-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + go-version: [1.24.x] + os: [ubuntu-latest] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + check-latest: true + - name: Build on ${{ matrix.os }} + if: matrix.os == 'ubuntu-latest' + env: + CGO_ENABLED: 0 + GO111MODULE: on + MINIO_KMS_SECRET_KEY: "my-minio-key:oyArl7zlPECEduNbB1KXgdzDn2Bdpvvw0l8VO51HQnY=" + MINIO_KMS_AUTO_ENCRYPTION: on + run: | + sudo sysctl net.ipv6.conf.all.disable_ipv6=0 + sudo sysctl net.ipv6.conf.default.disable_ipv6=0 + make verify-healing + make verify-healing-inconsistent-versions + make verify-healing-with-root-disks + make verify-healing-with-rewrite diff --git a/.github/workflows/go-lint.yml b/.github/workflows/go-lint.yml new file mode 100644 index 00000000..6b43086a --- /dev/null +++ b/.github/workflows/go-lint.yml @@ -0,0 +1,42 @@ +name: Linters and Tests + +on: + pull_request: + branches: + - master + +# This ensures that previous jobs for the PR are canceled when the PR is +# updated. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + build: + name: Go ${{ matrix.go-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + go-version: [1.24.x] + os: [ubuntu-latest] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + check-latest: true + - name: Build on ${{ matrix.os }} + if: matrix.os == 'ubuntu-latest' + env: + CGO_ENABLED: 0 + GO111MODULE: on + run: | + sudo apt install jq -y + sudo sysctl net.ipv6.conf.all.disable_ipv6=0 + sudo sysctl net.ipv6.conf.default.disable_ipv6=0 + make + make test + make test-race diff --git a/.github/workflows/go-resiliency.yml b/.github/workflows/go-resiliency.yml new file mode 100644 index 00000000..4eb2bbb9 --- /dev/null +++ b/.github/workflows/go-resiliency.yml @@ -0,0 +1,39 @@ +name: Resiliency Functional Tests + +on: + pull_request: + branches: + - master + +# This ensures that previous jobs for the PR are canceled when the PR is +# updated. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + build: + name: Go ${{ matrix.go-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + go-version: [1.24.x] + os: [ubuntu-latest] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + check-latest: true + - name: Build on ${{ matrix.os }} + if: matrix.os == 'ubuntu-latest' + env: + CGO_ENABLED: 0 + GO111MODULE: on + run: | + sudo sysctl net.ipv6.conf.all.disable_ipv6=0 + sudo sysctl net.ipv6.conf.default.disable_ipv6=0 + make test-resiliency diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml new file mode 100644 index 00000000..9e4b9d07 --- /dev/null +++ b/.github/workflows/go.yml @@ -0,0 +1,42 @@ +name: Functional Tests + +on: + pull_request: + branches: + - master + +# This ensures that previous jobs for the PR are canceled when the PR is +# updated. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + build: + name: Go ${{ matrix.go-version }} on ${{ matrix.os }} - healing + runs-on: ${{ matrix.os }} + strategy: + matrix: + go-version: [1.24.x] + os: [ubuntu-latest] + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + check-latest: true + - name: Build on ${{ matrix.os }} + if: matrix.os == 'ubuntu-latest' + env: + CGO_ENABLED: 0 + GO111MODULE: on + MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw=" + MINIO_KMS_AUTO_ENCRYPTION: on + run: | + sudo sysctl net.ipv6.conf.all.disable_ipv6=0 + sudo sysctl net.ipv6.conf.default.disable_ipv6=0 + make verify + make test-timeout diff --git a/.github/workflows/helm-lint.yml b/.github/workflows/helm-lint.yml new file mode 100644 index 00000000..f444106f --- /dev/null +++ b/.github/workflows/helm-lint.yml @@ -0,0 +1,30 @@ +name: Helm Chart linting + +on: + pull_request: + branches: + - master + +# This ensures that previous jobs for the PR are canceled when the PR is +# updated. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + release: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install Helm + uses: azure/setup-helm@v4 + + - name: Run helm lint + run: | + cd helm/minio + helm lint . diff --git a/.github/workflows/iam-integrations.yaml b/.github/workflows/iam-integrations.yaml new file mode 100644 index 00000000..cc57a77a --- /dev/null +++ b/.github/workflows/iam-integrations.yaml @@ -0,0 +1,161 @@ +name: IAM integration + +on: + pull_request: + branches: + - master + +# This ensures that previous jobs for the PR are canceled when the PR is +# updated. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + iam-matrix-test: + name: "[Go=${{ matrix.go-version }}|ldap=${{ matrix.ldap }}|etcd=${{ matrix.etcd }}|openid=${{ matrix.openid }}]" + runs-on: ubuntu-latest + + services: + openldap: + image: quay.io/minio/openldap + ports: + - "389:389" + - "636:636" + env: + LDAP_ORGANIZATION: "MinIO Inc" + LDAP_DOMAIN: "min.io" + LDAP_ADMIN_PASSWORD: "admin" + etcd: + image: "quay.io/coreos/etcd:v3.5.1" + env: + ETCD_LISTEN_CLIENT_URLS: "http://0.0.0.0:2379" + ETCD_ADVERTISE_CLIENT_URLS: "http://0.0.0.0:2379" + ports: + - "2379:2379" + options: >- + --health-cmd "etcdctl endpoint health" + --health-interval 10s + --health-timeout 5s + --health-retries 5 + openid: + image: quay.io/minio/dex + ports: + - "5556:5556" + env: + DEX_LDAP_SERVER: "openldap:389" + openid2: + image: quay.io/minio/dex + ports: + - "5557:5557" + env: + DEX_LDAP_SERVER: "openldap:389" + DEX_ISSUER: "http://127.0.0.1:5557/dex" + DEX_WEB_HTTP: "0.0.0.0:5557" + + strategy: + # When ldap, etcd or openid vars are empty below, those external servers + # are turned off - i.e. if ldap="", then ldap server is not enabled for + # the tests. + matrix: + go-version: [1.24.x] + ldap: ["", "localhost:389"] + etcd: ["", "http://localhost:2379"] + openid: ["", "http://127.0.0.1:5556/dex"] + exclude: + # exclude combos where all are empty. + - ldap: "" + etcd: "" + openid: "" + # exclude combos where both ldap and openid IDPs are specified. + - ldap: "localhost:389" + openid: "http://127.0.0.1:5556/dex" + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + check-latest: true + - name: Test LDAP/OpenID/Etcd combo + env: + _MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }} + _MINIO_ETCD_TEST_SERVER: ${{ matrix.etcd }} + _MINIO_OPENID_TEST_SERVER: ${{ matrix.openid }} + run: | + sudo sysctl net.ipv6.conf.all.disable_ipv6=0 + sudo sysctl net.ipv6.conf.default.disable_ipv6=0 + make test-iam + - name: Test with multiple OpenID providers + if: matrix.openid == 'http://127.0.0.1:5556/dex' + env: + _MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }} + _MINIO_ETCD_TEST_SERVER: ${{ matrix.etcd }} + _MINIO_OPENID_TEST_SERVER: ${{ matrix.openid }} + _MINIO_OPENID_TEST_SERVER_2: "http://127.0.0.1:5557/dex" + run: | + sudo sysctl net.ipv6.conf.all.disable_ipv6=0 + sudo sysctl net.ipv6.conf.default.disable_ipv6=0 + make test-iam + - name: Test with Access Management Plugin enabled + env: + _MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }} + _MINIO_ETCD_TEST_SERVER: ${{ matrix.etcd }} + _MINIO_OPENID_TEST_SERVER: ${{ matrix.openid }} + _MINIO_POLICY_PLUGIN_TEST_ENDPOINT: "http://127.0.0.1:8080" + run: | + sudo sysctl net.ipv6.conf.all.disable_ipv6=0 + sudo sysctl net.ipv6.conf.default.disable_ipv6=0 + go run docs/iam/access-manager-plugin.go & + make test-iam + - name: Test MinIO Old Version data to IAM import current version + if: matrix.ldap == 'ldaphost:389' + env: + _MINIO_LDAP_TEST_SERVER: ${{ matrix.ldap }} + run: | + make test-iam-ldap-upgrade-import + - name: Test LDAP for automatic site replication + if: matrix.ldap == 'localhost:389' + run: | + make test-site-replication-ldap + - name: Test OIDC for automatic site replication + if: matrix.openid == 'http://127.0.0.1:5556/dex' + run: | + make test-site-replication-oidc + iam-import-with-missing-entities: + name: Test IAM import in new cluster with missing entities + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + check-latest: true + - name: Checkout minio-iam-testing + uses: actions/checkout@v4 + with: + repository: minio/minio-iam-testing + path: minio-iam-testing + - name: Test import of IAM artifacts when in fresh cluster there are missing groups etc + run: | + make test-iam-import-with-missing-entities + iam-import-with-openid: + name: Test IAM import in new cluster with opendid configurations + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + check-latest: true + - name: Checkout minio-iam-testing + uses: actions/checkout@v4 + with: + repository: minio/minio-iam-testing + path: minio-iam-testing + - name: Test import of IAM artifacts when in fresh cluster with openid configurations + run: | + make test-iam-import-with-openid diff --git a/.github/workflows/issues.yaml b/.github/workflows/issues.yaml new file mode 100644 index 00000000..d91950ae --- /dev/null +++ b/.github/workflows/issues.yaml @@ -0,0 +1,18 @@ +# @format + +name: Issue Workflow + +on: + issues: + types: + - opened + +jobs: + add-to-project: + name: Add issue to project + runs-on: ubuntu-latest + steps: + - uses: actions/add-to-project@v0.5.0 + with: + project-url: https://github.com/orgs/miniohq/projects/2 + github-token: ${{ secrets.BOT_PAT }} diff --git a/.github/workflows/lock.yml b/.github/workflows/lock.yml new file mode 100644 index 00000000..da3afe3c --- /dev/null +++ b/.github/workflows/lock.yml @@ -0,0 +1,24 @@ +name: 'Lock Threads' + +on: + schedule: + - cron: '0 0 * * *' + workflow_dispatch: + +permissions: + issues: write + +concurrency: + group: lock + +jobs: + action: + runs-on: ubuntu-latest + steps: + - uses: dessant/lock-threads@v3 + with: + github-token: ${{ github.token }} + issue-inactive-days: '365' + exclude-any-issue-labels: 'do-not-close' + issue-lock-reason: 'resolved' + log-output: true diff --git a/.github/workflows/mint.yml b/.github/workflows/mint.yml new file mode 100644 index 00000000..77dc498c --- /dev/null +++ b/.github/workflows/mint.yml @@ -0,0 +1,81 @@ +name: Mint Tests + +on: + pull_request: + branches: + - master + +# This ensures that previous jobs for the PR are canceled when the PR is +# updated. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + mint-test: + runs-on: mint + timeout-minutes: 120 + steps: + - name: cleanup #https://github.com/actions/checkout/issues/273 + run: | + sudo -S rm -rf ${GITHUB_WORKSPACE} + mkdir ${GITHUB_WORKSPACE} + - name: checkout-step + uses: actions/checkout@v4 + + - name: setup-go-step + uses: actions/setup-go@v5 + with: + go-version: 1.24.x + + - name: github sha short + id: vars + run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT + + - name: build-minio + run: | + TAG="quay.io/minio/minio:${{ steps.vars.outputs.sha_short }}" make docker + + - name: multipart uploads test + run: | + ${GITHUB_WORKSPACE}/.github/workflows/multipart/migrate.sh "${{ steps.vars.outputs.sha_short }}" + + - name: compress and encrypt + run: | + ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "compress-encrypt" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}" + + - name: multiple pools + run: | + ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "pools" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}" + + - name: standalone erasure + run: | + ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "erasure" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}" + + # FIXME: renable this back when we have a valid way to add deadlines for PUT()s (internode CreateFile) + # - name: resiliency + # run: | + # ${GITHUB_WORKSPACE}/.github/workflows/run-mint.sh "resiliency" "minio" "minio123" "${{ steps.vars.outputs.sha_short }}" + + - name: The job must cleanup + if: ${{ always() }} + run: | + export JOB_NAME=${{ steps.vars.outputs.sha_short }} + for mode in $(echo compress-encrypt pools erasure); do + docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml down || true + docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/mint/minio-${mode}.yaml rm || true + done + + docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/multipart/docker-compose-site1.yaml rm -s -f || true + docker-compose -f ${GITHUB_WORKSPACE}/.github/workflows/multipart/docker-compose-site2.yaml rm -s -f || true + for volume in $(docker volume ls -q | grep minio); do + docker volume rm ${volume} || true + done + + docker rmi -f quay.io/minio/minio:${{ steps.vars.outputs.sha_short }} + docker system prune -f || true + docker volume prune -f || true + docker volume rm $(docker volume ls -q -f dangling=true) || true diff --git a/.github/workflows/mint/minio-compress-encrypt.yaml b/.github/workflows/mint/minio-compress-encrypt.yaml new file mode 100644 index 00000000..fe1238e8 --- /dev/null +++ b/.github/workflows/mint/minio-compress-encrypt.yaml @@ -0,0 +1,80 @@ +version: '3.7' + +# Settings and configurations that are common for all containers +x-minio-common: &minio-common + image: quay.io/minio/minio:${JOB_NAME} + command: server --console-address ":9001" http://minio{1...4}/cdata{1...2} + expose: + - "9000" + - "9001" + environment: + MINIO_CI_CD: "on" + MINIO_ROOT_USER: "minio" + MINIO_ROOT_PASSWORD: "minio123" + MINIO_COMPRESSION_ENABLE: "on" + MINIO_COMPRESSION_MIME_TYPES: "*" + MINIO_COMPRESSION_ALLOW_ENCRYPTION: "on" + MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw=" + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 5s + retries: 5 + +# starts 4 docker containers running minio server instances. +# using nginx reverse proxy, load balancing, you can access +# it through port 9000. +services: + minio1: + <<: *minio-common + hostname: minio1 + volumes: + - cdata1-1:/cdata1 + - cdata1-2:/cdata2 + + minio2: + <<: *minio-common + hostname: minio2 + volumes: + - cdata2-1:/cdata1 + - cdata2-2:/cdata2 + + minio3: + <<: *minio-common + hostname: minio3 + volumes: + - cdata3-1:/cdata1 + - cdata3-2:/cdata2 + + minio4: + <<: *minio-common + hostname: minio4 + volumes: + - cdata4-1:/cdata1 + - cdata4-2:/cdata2 + + nginx: + image: nginx:1.19.2-alpine + hostname: nginx + volumes: + - ./nginx-4-node.conf:/etc/nginx/nginx.conf:ro + ports: + - "9000:9000" + - "9001:9001" + depends_on: + - minio1 + - minio2 + - minio3 + - minio4 + +## By default this config uses default local driver, +## For custom volumes replace with volume driver configuration. +volumes: + cdata1-1: + cdata1-2: + cdata2-1: + cdata2-2: + cdata3-1: + cdata3-2: + cdata4-1: + cdata4-2: diff --git a/.github/workflows/mint/minio-erasure.yaml b/.github/workflows/mint/minio-erasure.yaml new file mode 100644 index 00000000..a3fd6dbc --- /dev/null +++ b/.github/workflows/mint/minio-erasure.yaml @@ -0,0 +1,51 @@ +version: '3.7' + +# Settings and configurations that are common for all containers +x-minio-common: &minio-common + image: quay.io/minio/minio:${JOB_NAME} + command: server --console-address ":9001" edata{1...4} + expose: + - "9000" + - "9001" + environment: + MINIO_CI_CD: "on" + MINIO_ROOT_USER: "minio" + MINIO_ROOT_PASSWORD: "minio123" + MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw=" + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 5s + retries: 5 + +# starts 4 docker containers running minio server instances. +# using nginx reverse proxy, load balancing, you can access +# it through port 9000. +services: + minio1: + <<: *minio-common + hostname: minio1 + volumes: + - edata1-1:/edata1 + - edata1-2:/edata2 + - edata1-3:/edata3 + - edata1-4:/edata4 + + nginx: + image: nginx:1.19.2-alpine + hostname: nginx + volumes: + - ./nginx-1-node.conf:/etc/nginx/nginx.conf:ro + ports: + - "9000:9000" + - "9001:9001" + depends_on: + - minio1 + +## By default this config uses default local driver, +## For custom volumes replace with volume driver configuration. +volumes: + edata1-1: + edata1-2: + edata1-3: + edata1-4: diff --git a/.github/workflows/mint/minio-pools.yaml b/.github/workflows/mint/minio-pools.yaml new file mode 100644 index 00000000..bd79fdae --- /dev/null +++ b/.github/workflows/mint/minio-pools.yaml @@ -0,0 +1,117 @@ +version: '3.7' + +# Settings and configurations that are common for all containers +x-minio-common: &minio-common + image: quay.io/minio/minio:${JOB_NAME} + command: server --console-address ":9001" http://minio{1...4}/pdata{1...2} http://minio{5...8}/pdata{1...2} + expose: + - "9000" + - "9001" + environment: + MINIO_CI_CD: "on" + MINIO_ROOT_USER: "minio" + MINIO_ROOT_PASSWORD: "minio123" + MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw=" + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 5s + retries: 5 + +# starts 4 docker containers running minio server instances. +# using nginx reverse proxy, load balancing, you can access +# it through port 9000. +services: + minio1: + <<: *minio-common + hostname: minio1 + volumes: + - pdata1-1:/pdata1 + - pdata1-2:/pdata2 + + minio2: + <<: *minio-common + hostname: minio2 + volumes: + - pdata2-1:/pdata1 + - pdata2-2:/pdata2 + + minio3: + <<: *minio-common + hostname: minio3 + volumes: + - pdata3-1:/pdata1 + - pdata3-2:/pdata2 + + minio4: + <<: *minio-common + hostname: minio4 + volumes: + - pdata4-1:/pdata1 + - pdata4-2:/pdata2 + + minio5: + <<: *minio-common + hostname: minio5 + volumes: + - pdata5-1:/pdata1 + - pdata5-2:/pdata2 + + minio6: + <<: *minio-common + hostname: minio6 + volumes: + - pdata6-1:/pdata1 + - pdata6-2:/pdata2 + + minio7: + <<: *minio-common + hostname: minio7 + volumes: + - pdata7-1:/pdata1 + - pdata7-2:/pdata2 + + minio8: + <<: *minio-common + hostname: minio8 + volumes: + - pdata8-1:/pdata1 + - pdata8-2:/pdata2 + + nginx: + image: nginx:1.19.2-alpine + hostname: nginx + volumes: + - ./nginx-8-node.conf:/etc/nginx/nginx.conf:ro + ports: + - "9000:9000" + - "9001:9001" + depends_on: + - minio1 + - minio2 + - minio3 + - minio4 + - minio5 + - minio6 + - minio7 + - minio8 + +## By default this config uses default local driver, +## For custom volumes replace with volume driver configuration. +volumes: + pdata1-1: + pdata1-2: + pdata2-1: + pdata2-2: + pdata3-1: + pdata3-2: + pdata4-1: + pdata4-2: + pdata5-1: + pdata5-2: + pdata6-1: + pdata6-2: + pdata7-1: + pdata7-2: + pdata8-1: + pdata8-2: diff --git a/.github/workflows/mint/minio-resiliency.yaml b/.github/workflows/mint/minio-resiliency.yaml new file mode 100644 index 00000000..9d569c59 --- /dev/null +++ b/.github/workflows/mint/minio-resiliency.yaml @@ -0,0 +1,78 @@ +version: '3.7' + +# Settings and configurations that are common for all containers +x-minio-common: &minio-common + image: quay.io/minio/minio:${JOB_NAME} + command: server --console-address ":9001" http://minio{1...4}/rdata{1...2} + expose: + - "9000" + - "9001" + environment: + MINIO_CI_CD: "on" + MINIO_ROOT_USER: "minio" + MINIO_ROOT_PASSWORD: "minio123" + MINIO_KMS_SECRET_KEY: "my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw=" + MINIO_DRIVE_MAX_TIMEOUT: "5s" + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 5s + retries: 5 + +# starts 4 docker containers running minio server instances. +# using nginx reverse proxy, load balancing, you can access +# it through port 9000. +services: + minio1: + <<: *minio-common + hostname: minio1 + volumes: + - rdata1-1:/rdata1 + - rdata1-2:/rdata2 + + minio2: + <<: *minio-common + hostname: minio2 + volumes: + - rdata2-1:/rdata1 + - rdata2-2:/rdata2 + + minio3: + <<: *minio-common + hostname: minio3 + volumes: + - rdata3-1:/rdata1 + - rdata3-2:/rdata2 + + minio4: + <<: *minio-common + hostname: minio4 + volumes: + - rdata4-1:/rdata1 + - rdata4-2:/rdata2 + + nginx: + image: nginx:1.19.2-alpine + hostname: nginx + volumes: + - ./nginx-4-node.conf:/etc/nginx/nginx.conf:ro + ports: + - "9000:9000" + - "9001:9001" + depends_on: + - minio1 + - minio2 + - minio3 + - minio4 + +## By default this config uses default local driver, +## For custom volumes replace with volume driver configuration. +volumes: + rdata1-1: + rdata1-2: + rdata2-1: + rdata2-2: + rdata3-1: + rdata3-2: + rdata4-1: + rdata4-2: diff --git a/.github/workflows/mint/nginx-1-node.conf b/.github/workflows/mint/nginx-1-node.conf new file mode 100644 index 00000000..bb13bde2 --- /dev/null +++ b/.github/workflows/mint/nginx-1-node.conf @@ -0,0 +1,100 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 4096; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + sendfile on; + keepalive_timeout 65; + + # include /etc/nginx/conf.d/*.conf; + + upstream minio { + server minio1:9000; + } + + upstream console { + ip_hash; + server minio1:9001; + } + + server { + listen 9000; + listen [::]:9000; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio; + } + } + + server { + listen 9001; + listen [::]:9001; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-NginX-Proxy true; + + # This is necessary to pass the correct IP to be hashed + real_ip_header X-Real-IP; + + proxy_connect_timeout 300; + + # To support websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + chunked_transfer_encoding off; + + proxy_pass http://console; + } + } +} diff --git a/.github/workflows/mint/nginx-4-node.conf b/.github/workflows/mint/nginx-4-node.conf new file mode 100644 index 00000000..b849940d --- /dev/null +++ b/.github/workflows/mint/nginx-4-node.conf @@ -0,0 +1,105 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 4096; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + sendfile on; + keepalive_timeout 65; + + # include /etc/nginx/conf.d/*.conf; + + upstream minio { + server minio1:9000 max_fails=1 fail_timeout=10s; + server minio2:9000 max_fails=1 fail_timeout=10s; + server minio3:9000 max_fails=1 fail_timeout=10s; + } + + upstream console { + ip_hash; + server minio1:9001; + server minio2:9001; + server minio3:9001; + server minio4:9001; + } + + server { + listen 9000; + listen [::]:9000; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio; + } + } + + server { + listen 9001; + listen [::]:9001; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-NginX-Proxy true; + + # This is necessary to pass the correct IP to be hashed + real_ip_header X-Real-IP; + + proxy_connect_timeout 300; + + # To support websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + chunked_transfer_encoding off; + + proxy_pass http://console; + } + } +} diff --git a/.github/workflows/mint/nginx-8-node.conf b/.github/workflows/mint/nginx-8-node.conf new file mode 100644 index 00000000..278b5ae7 --- /dev/null +++ b/.github/workflows/mint/nginx-8-node.conf @@ -0,0 +1,114 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 4096; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + sendfile on; + keepalive_timeout 65; + + # include /etc/nginx/conf.d/*.conf; + + upstream minio { + server minio1:9000 max_fails=1 fail_timeout=10s; + server minio2:9000 max_fails=1 fail_timeout=10s; + server minio3:9000 max_fails=1 fail_timeout=10s; + server minio4:9000 max_fails=1 fail_timeout=10s; + server minio5:9000 max_fails=1 fail_timeout=10s; + server minio6:9000 max_fails=1 fail_timeout=10s; + server minio7:9000 max_fails=1 fail_timeout=10s; + server minio8:9000 max_fails=1 fail_timeout=10s; + } + + upstream console { + ip_hash; + server minio1:9001; + server minio2:9001; + server minio3:9001; + server minio4:9001; + server minio5:9001; + server minio6:9001; + server minio7:9001; + server minio8:9001; + } + + server { + listen 9000; + listen [::]:9000; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio; + } + } + + server { + listen 9001; + listen [::]:9001; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-NginX-Proxy true; + + # This is necessary to pass the correct IP to be hashed + real_ip_header X-Real-IP; + + proxy_connect_timeout 300; + + # To support websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + chunked_transfer_encoding off; + + proxy_pass http://console; + } + } +} diff --git a/.github/workflows/mint/nginx.conf b/.github/workflows/mint/nginx.conf new file mode 100644 index 00000000..1455a3e2 --- /dev/null +++ b/.github/workflows/mint/nginx.conf @@ -0,0 +1,106 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 4096; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + sendfile on; + keepalive_timeout 65; + + # include /etc/nginx/conf.d/*.conf; + + upstream minio { + server minio1:9000 max_fails=1 fail_timeout=10s; + server minio2:9000 max_fails=1 fail_timeout=10s; + server minio3:9000 max_fails=1 fail_timeout=10s; + server minio4:9000 max_fails=1 fail_timeout=10s; + } + + upstream console { + ip_hash; + server minio1:9001; + server minio2:9001; + server minio3:9001; + server minio4:9001; + } + + server { + listen 9000; + listen [::]:9000; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio; + } + } + + server { + listen 9001; + listen [::]:9001; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-NginX-Proxy true; + + # This is necessary to pass the correct IP to be hashed + real_ip_header X-Real-IP; + + proxy_connect_timeout 300; + + # To support websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + chunked_transfer_encoding off; + + proxy_pass http://console; + } + } +} diff --git a/.github/workflows/multipart/docker-compose-site1.yaml b/.github/workflows/multipart/docker-compose-site1.yaml new file mode 100644 index 00000000..b4a665ef --- /dev/null +++ b/.github/workflows/multipart/docker-compose-site1.yaml @@ -0,0 +1,66 @@ +version: '3.7' + +# Settings and configurations that are common for all containers +x-minio-common: &minio-common + image: quay.io/minio/minio:${RELEASE} + command: server http://site1-minio{1...4}/data{1...2} + environment: + - MINIO_PROMETHEUS_AUTH_TYPE=public + - CI=true + +# starts 4 docker containers running minio server instances. +# using nginx reverse proxy, load balancing, you can access +# it through port 9000. +services: + site1-minio1: + <<: *minio-common + hostname: site1-minio1 + volumes: + - site1-data1-1:/data1 + - site1-data1-2:/data2 + + site1-minio2: + <<: *minio-common + hostname: site1-minio2 + volumes: + - site1-data2-1:/data1 + - site1-data2-2:/data2 + + site1-minio3: + <<: *minio-common + hostname: site1-minio3 + volumes: + - site1-data3-1:/data1 + - site1-data3-2:/data2 + + site1-minio4: + <<: *minio-common + hostname: site1-minio4 + volumes: + - site1-data4-1:/data1 + - site1-data4-2:/data2 + + site1-nginx: + image: nginx:1.19.2-alpine + hostname: site1-nginx + volumes: + - ./nginx-site1.conf:/etc/nginx/nginx.conf:ro + ports: + - "9001:9001" + depends_on: + - site1-minio1 + - site1-minio2 + - site1-minio3 + - site1-minio4 + +## By default this config uses default local driver, +## For custom volumes replace with volume driver configuration. +volumes: + site1-data1-1: + site1-data1-2: + site1-data2-1: + site1-data2-2: + site1-data3-1: + site1-data3-2: + site1-data4-1: + site1-data4-2: diff --git a/.github/workflows/multipart/docker-compose-site2.yaml b/.github/workflows/multipart/docker-compose-site2.yaml new file mode 100644 index 00000000..31a1f7a8 --- /dev/null +++ b/.github/workflows/multipart/docker-compose-site2.yaml @@ -0,0 +1,66 @@ +version: '3.7' + +# Settings and configurations that are common for all containers +x-minio-common: &minio-common + image: quay.io/minio/minio:${RELEASE} + command: server http://site2-minio{1...4}/data{1...2} + environment: + - MINIO_PROMETHEUS_AUTH_TYPE=public + - CI=true + +# starts 4 docker containers running minio server instances. +# using nginx reverse proxy, load balancing, you can access +# it through port 9000. +services: + site2-minio1: + <<: *minio-common + hostname: site2-minio1 + volumes: + - site2-data1-1:/data1 + - site2-data1-2:/data2 + + site2-minio2: + <<: *minio-common + hostname: site2-minio2 + volumes: + - site2-data2-1:/data1 + - site2-data2-2:/data2 + + site2-minio3: + <<: *minio-common + hostname: site2-minio3 + volumes: + - site2-data3-1:/data1 + - site2-data3-2:/data2 + + site2-minio4: + <<: *minio-common + hostname: site2-minio4 + volumes: + - site2-data4-1:/data1 + - site2-data4-2:/data2 + + site2-nginx: + image: nginx:1.19.2-alpine + hostname: site2-nginx + volumes: + - ./nginx-site2.conf:/etc/nginx/nginx.conf:ro + ports: + - "9002:9002" + depends_on: + - site2-minio1 + - site2-minio2 + - site2-minio3 + - site2-minio4 + +## By default this config uses default local driver, +## For custom volumes replace with volume driver configuration. +volumes: + site2-data1-1: + site2-data1-2: + site2-data2-1: + site2-data2-2: + site2-data3-1: + site2-data3-2: + site2-data4-1: + site2-data4-2: diff --git a/.github/workflows/multipart/migrate.sh b/.github/workflows/multipart/migrate.sh new file mode 100755 index 00000000..6058b008 --- /dev/null +++ b/.github/workflows/multipart/migrate.sh @@ -0,0 +1,147 @@ +#!/bin/bash + +set -x + +## change working directory +cd .github/workflows/multipart/ + +function cleanup() { + docker-compose -f docker-compose-site1.yaml rm -s -f || true + docker-compose -f docker-compose-site2.yaml rm -s -f || true + for volume in $(docker volume ls -q | grep minio); do + docker volume rm ${volume} || true + done + + docker system prune -f || true + docker volume prune -f || true + docker volume rm $(docker volume ls -q -f dangling=true) || true +} + +cleanup + +if [ ! -f ./mc ]; then + wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc +fi + +export RELEASE=RELEASE.2023-08-29T23-07-35Z + +docker-compose -f docker-compose-site1.yaml up -d +docker-compose -f docker-compose-site2.yaml up -d + +sleep 30s + +./mc alias set site1 http://site1-nginx:9001 minioadmin minioadmin --api s3v4 +./mc alias set site2 http://site2-nginx:9002 minioadmin minioadmin --api s3v4 + +./mc ready site1/ +./mc ready site2/ + +./mc admin replicate add site1 site2 +./mc mb site1/testbucket/ +./mc cp -r --quiet /usr/bin site1/testbucket/ + +sleep 5 + +./s3-check-md5 -h + +failed_count_site1=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l) +failed_count_site2=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l) + +if [ $failed_count_site1 -ne 0 ]; then + echo "failed with multipart on site1 uploads" + exit 1 +fi + +if [ $failed_count_site2 -ne 0 ]; then + echo "failed with multipart on site2 uploads" + exit 1 +fi + +./mc cp -r --quiet /usr/bin site1/testbucket/ + +sleep 5 + +failed_count_site1=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l) +failed_count_site2=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l) + +## we do not need to fail here, since we are going to test +## upgrading to master, healing and being able to recover +## the last version. +if [ $failed_count_site1 -ne 0 ]; then + echo "failed with multipart on site1 uploads ${failed_count_site1}" +fi + +if [ $failed_count_site2 -ne 0 ]; then + echo "failed with multipart on site2 uploads ${failed_count_site2}" +fi + +export RELEASE=${1} + +docker-compose -f docker-compose-site1.yaml up -d +docker-compose -f docker-compose-site2.yaml up -d + +./mc ready site1/ +./mc ready site2/ + +for i in $(seq 1 10); do + # mc admin heal -r --remove when used against a LB endpoint + # behaves flaky, let this run 10 times before giving up + ./mc admin heal -r --remove --json site1/ 2>&1 >/dev/null + ./mc admin heal -r --remove --json site2/ 2>&1 >/dev/null +done + +failed_count_site1=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site1-nginx:9001 -bucket testbucket 2>&1 | grep FAILED | wc -l) +failed_count_site2=$(./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://site2-nginx:9002 -bucket testbucket 2>&1 | grep FAILED | wc -l) + +if [ $failed_count_site1 -ne 0 ]; then + echo "failed with multipart on site1 uploads" + exit 1 +fi + +if [ $failed_count_site2 -ne 0 ]; then + echo "failed with multipart on site2 uploads" + exit 1 +fi + +# Add user group test +./mc admin user add site1 site-replication-issue-user site-replication-issue-password +./mc admin group add site1 site-replication-issue-group site-replication-issue-user + +max_wait_attempts=30 +wait_interval=5 + +attempt=1 +while true; do + diff <(./mc admin group info site1 site-replication-issue-group) <(./mc admin group info site2 site-replication-issue-group) + + if [[ $? -eq 0 ]]; then + echo "Outputs are consistent." + break + fi + + remaining_attempts=$((max_wait_attempts - attempt)) + if ((attempt >= max_wait_attempts)); then + echo "Outputs remain inconsistent after $max_wait_attempts attempts. Exiting with error." + exit 1 + else + echo "Outputs are inconsistent. Waiting for $wait_interval seconds (attempt $attempt/$max_wait_attempts)." + sleep $wait_interval + fi + + ((attempt++)) +done + +status=$(./mc admin group info site1 site-replication-issue-group --json | jq .groupStatus | tr -d '"') + +if [[ $status == "enabled" ]]; then + echo "Success" +else + echo "Expected status: enabled, actual status: $status" + exit 1 +fi + +cleanup + +## change working directory +cd ../../../ diff --git a/.github/workflows/multipart/nginx-site1.conf b/.github/workflows/multipart/nginx-site1.conf new file mode 100644 index 00000000..1a37b9d2 --- /dev/null +++ b/.github/workflows/multipart/nginx-site1.conf @@ -0,0 +1,61 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 4096; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + sendfile on; + keepalive_timeout 65; + + # include /etc/nginx/conf.d/*.conf; + + upstream minio { + server site1-minio1:9000; + server site1-minio2:9000; + server site1-minio3:9000; + server site1-minio4:9000; + } + + server { + listen 9001; + listen [::]:9001; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio; + } + } +} diff --git a/.github/workflows/multipart/nginx-site2.conf b/.github/workflows/multipart/nginx-site2.conf new file mode 100644 index 00000000..b93bba24 --- /dev/null +++ b/.github/workflows/multipart/nginx-site2.conf @@ -0,0 +1,61 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 4096; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + sendfile on; + keepalive_timeout 65; + + # include /etc/nginx/conf.d/*.conf; + + upstream minio { + server site2-minio1:9000; + server site2-minio2:9000; + server site2-minio3:9000; + server site2-minio4:9000; + } + + server { + listen 9002; + listen [::]:9002; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio; + } + } +} diff --git a/.github/workflows/replication.yaml b/.github/workflows/replication.yaml new file mode 100644 index 00000000..18c32774 --- /dev/null +++ b/.github/workflows/replication.yaml @@ -0,0 +1,79 @@ +name: MinIO advanced tests + +on: + pull_request: + branches: + - master + +# This ensures that previous jobs for the PR are canceled when the PR is +# updated. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + replication-test: + name: Advanced Tests with Go ${{ matrix.go-version }} + runs-on: ubuntu-latest + + strategy: + matrix: + go-version: [1.24.x] + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + check-latest: true + - name: Test Decom + run: | + sudo sysctl net.ipv6.conf.all.disable_ipv6=0 + sudo sysctl net.ipv6.conf.default.disable_ipv6=0 + make test-decom + + - name: Test ILM + run: | + sudo sysctl net.ipv6.conf.all.disable_ipv6=0 + sudo sysctl net.ipv6.conf.default.disable_ipv6=0 + make test-ilm + make test-ilm-transition + + - name: Test PBAC + run: | + sudo sysctl net.ipv6.conf.all.disable_ipv6=0 + sudo sysctl net.ipv6.conf.default.disable_ipv6=0 + make test-pbac + + - name: Test Config File + run: | + sudo sysctl net.ipv6.conf.all.disable_ipv6=0 + sudo sysctl net.ipv6.conf.default.disable_ipv6=0 + make test-configfile + + - name: Test Replication + run: | + sudo sysctl net.ipv6.conf.all.disable_ipv6=0 + sudo sysctl net.ipv6.conf.default.disable_ipv6=0 + make test-replication + + - name: Test MinIO IDP for automatic site replication + run: | + sudo sysctl net.ipv6.conf.all.disable_ipv6=0 + sudo sysctl net.ipv6.conf.default.disable_ipv6=0 + make test-site-replication-minio + + - name: Test Versioning + run: | + sudo sysctl net.ipv6.conf.all.disable_ipv6=0 + sudo sysctl net.ipv6.conf.default.disable_ipv6=0 + make test-versioning + + - name: Test Multipart upload with failures + run: | + sudo sysctl net.ipv6.conf.all.disable_ipv6=0 + sudo sysctl net.ipv6.conf.default.disable_ipv6=0 + make test-multipart diff --git a/.github/workflows/root-disable.yml b/.github/workflows/root-disable.yml new file mode 100644 index 00000000..c08fb8b1 --- /dev/null +++ b/.github/workflows/root-disable.yml @@ -0,0 +1,34 @@ +name: Root lockdown tests + +on: + pull_request: + branches: + - master + +# This ensures that previous jobs for the PR are canceled when the PR is +# updated. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + build: + name: Go ${{ matrix.go-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + go-version: [1.24.x] + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + check-latest: true + - name: Start root lockdown tests + run: | + make test-root-disable diff --git a/.github/workflows/root.cert b/.github/workflows/root.cert new file mode 100644 index 00000000..5f220f79 --- /dev/null +++ b/.github/workflows/root.cert @@ -0,0 +1,9 @@ +-----BEGIN CERTIFICATE----- +MIIBKDCB26ADAgECAhB6vebGMUfKnmBKyqoApRSOMAUGAytlcDAbMRkwFwYDVQQD +DBByb290QHBsYXkubWluLmlvMB4XDTIwMDQzMDE1MjIyNVoXDTI1MDQyOTE1MjIy +NVowGzEZMBcGA1UEAwwQcm9vdEBwbGF5Lm1pbi5pbzAqMAUGAytlcAMhALzn735W +fmSH/ghKs+4iPWziZMmWdiWr/sqvqeW+WwSxozUwMzAOBgNVHQ8BAf8EBAMCB4Aw +EwYDVR0lBAwwCgYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAFBgMrZXADQQDZOrGK +b2ATkDlu2pTcP3LyhSBDpYh7V4TvjRkBTRgjkacCzwFLm+mh+7US8V4dBpIDsJ4u +uWoF0y6vbLVGIlkG +-----END CERTIFICATE----- diff --git a/.github/workflows/root.key b/.github/workflows/root.key new file mode 100644 index 00000000..53a47e25 --- /dev/null +++ b/.github/workflows/root.key @@ -0,0 +1,3 @@ +-----BEGIN PRIVATE KEY----- +MC4CAQAwBQYDK2VwBCIEID9E7FSYWrMD+VjhI6q545cYT9YOyFxZb7UnjEepYDRc +-----END PRIVATE KEY----- diff --git a/.github/workflows/run-mint.sh b/.github/workflows/run-mint.sh new file mode 100755 index 00000000..0bbc1cba --- /dev/null +++ b/.github/workflows/run-mint.sh @@ -0,0 +1,64 @@ +#!/bin/bash + +set -ex + +export MODE="$1" +export ACCESS_KEY="$2" +export SECRET_KEY="$3" +export JOB_NAME="$4" +export MINT_MODE="full" + +docker system prune -f || true +docker volume prune -f || true +docker volume rm $(docker volume ls -f dangling=true) || true + +## change working directory +cd .github/workflows/mint + +## always pull latest +docker pull docker.io/minio/mint:edge + +docker-compose -f minio-${MODE}.yaml up -d +sleep 1m + +docker system prune -f || true +docker volume prune -f || true +docker volume rm $(docker volume ls -q -f dangling=true) || true + +# Stop two nodes, one of each pool, to check that all S3 calls work while quorum is still there +[ "${MODE}" == "pools" ] && docker-compose -f minio-${MODE}.yaml stop minio2 +[ "${MODE}" == "pools" ] && docker-compose -f minio-${MODE}.yaml stop minio6 + +# Pause one node, to check that all S3 calls work while one node goes wrong +[ "${MODE}" == "resiliency" ] && docker-compose -f minio-${MODE}.yaml pause minio4 + +docker run --rm --net=mint_default \ + --name="mint-${MODE}-${JOB_NAME}" \ + -e SERVER_ENDPOINT="nginx:9000" \ + -e ACCESS_KEY="${ACCESS_KEY}" \ + -e SECRET_KEY="${SECRET_KEY}" \ + -e ENABLE_HTTPS=0 \ + -e MINT_MODE="${MINT_MODE}" \ + docker.io/minio/mint:edge + +# FIXME: enable this after fixing aws-sdk-java-v2 tests +# # unpause the node, to check that all S3 calls work while one node goes wrong +# [ "${MODE}" == "resiliency" ] && docker-compose -f minio-${MODE}.yaml unpause minio4 +# [ "${MODE}" == "resiliency" ] && docker run --rm --net=mint_default \ +# --name="mint-${MODE}-${JOB_NAME}" \ +# -e SERVER_ENDPOINT="nginx:9000" \ +# -e ACCESS_KEY="${ACCESS_KEY}" \ +# -e SECRET_KEY="${SECRET_KEY}" \ +# -e ENABLE_HTTPS=0 \ +# -e MINT_MODE="${MINT_MODE}" \ +# docker.io/minio/mint:edge + +docker-compose -f minio-${MODE}.yaml down || true +sleep 10s + +docker system prune -f || true +docker volume prune -f || true +docker volume rm $(docker volume ls -q -f dangling=true) || true + +## change working directory +cd ../../../ diff --git a/.github/workflows/shfmt.yml b/.github/workflows/shfmt.yml new file mode 100644 index 00000000..3e446306 --- /dev/null +++ b/.github/workflows/shfmt.yml @@ -0,0 +1,22 @@ +name: Shell formatting checks + +on: + pull_request: + branches: + - master + +permissions: + contents: read + +jobs: + build: + name: runner / shfmt + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: luizm/action-sh-checker@master + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SHFMT_OPTS: "-s" + with: + sh_checker_shellcheck_disable: true # disable for now diff --git a/.github/workflows/typos.yml b/.github/workflows/typos.yml new file mode 100644 index 00000000..7addb314 --- /dev/null +++ b/.github/workflows/typos.yml @@ -0,0 +1,15 @@ +--- +name: Spelling +on: [pull_request] + +jobs: + run: + name: Spell Check with Typos + runs-on: ubuntu-latest + steps: + - name: Checkout Actions Repository + uses: actions/checkout@v4 + + - name: Check spelling of repo + uses: crate-ci/typos@master + diff --git a/.github/workflows/upgrade-ci-cd.yaml b/.github/workflows/upgrade-ci-cd.yaml new file mode 100644 index 00000000..d3e71ff5 --- /dev/null +++ b/.github/workflows/upgrade-ci-cd.yaml @@ -0,0 +1,34 @@ +name: Upgrade old version tests + +on: + pull_request: + branches: + - master + +# This ensures that previous jobs for the PR are canceled when the PR is +# updated. +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + build: + name: Go ${{ matrix.go-version }} on ${{ matrix.os }} + runs-on: ${{ matrix.os }} + strategy: + matrix: + go-version: [1.24.x] + os: [ubuntu-latest] + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 + with: + go-version: ${{ matrix.go-version }} + check-latest: true + - name: Start upgrade tests + run: | + make test-upgrade diff --git a/.github/workflows/vulncheck.yml b/.github/workflows/vulncheck.yml new file mode 100644 index 00000000..5dfcde04 --- /dev/null +++ b/.github/workflows/vulncheck.yml @@ -0,0 +1,31 @@ +name: VulnCheck +on: + pull_request: + branches: + - master + + push: + branches: + - master + +permissions: + contents: read # to fetch code (actions/checkout) + +jobs: + vulncheck: + name: Analysis + runs-on: ubuntu-latest + steps: + - name: Check out code into the Go module directory + uses: actions/checkout@v4 + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version: 1.24.x + cached: false + - name: Get official govulncheck + run: go install golang.org/x/vuln/cmd/govulncheck@latest + shell: bash + - name: Run govulncheck + run: govulncheck -show verbose ./... + shell: bash diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..3a51a0b2 --- /dev/null +++ b/.gitignore @@ -0,0 +1,55 @@ +**/*.swp +cover.out +*~ +minio +!*/ +site/ +**/*.test +**/*.sublime-workspace +/.idea/ +/Minio.iml +**/access.log +vendor/ +.DS_Store +*.syso +coverage.txt +.vscode/ +*.tar.bz2 +parts/ +prime/ +stage/ +.sia_temp/ +config.json +node_modules/ +mc.* +s3-check-md5* +xl-meta* +healing-* +inspect*.zip +200M* +hash-set +minio.RELEASE* +mc +nancy +inspects/* +.bin/ +*.gz +docs/debugging/s3-verify/s3-verify +docs/debugging/xl-meta/xl-meta +docs/debugging/s3-check-md5/s3-check-md5 +docs/debugging/hash-set/hash-set +docs/debugging/healing-bin/healing-bin +docs/debugging/inspect/inspect +docs/debugging/pprofgoparser/pprofgoparser +docs/debugging/reorder-disks/reorder-disks +docs/debugging/populate-hard-links/populate-hardlinks +docs/debugging/xattr/xattr +hash-set +healing-bin +inspect +pprofgoparser +reorder-disks +s3-check-md5 +s3-verify +xattr +xl-meta diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 00000000..0533d7cd --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,64 @@ +version: "2" +linters: + default: none + enable: + - durationcheck + - forcetypeassert + - gocritic + - gomodguard + - govet + - ineffassign + - misspell + - revive + - staticcheck + - unconvert + - unused + - usetesting + - whitespace + settings: + misspell: + locale: US + staticcheck: + checks: + - all + - -SA1008 + - -SA1019 + - -SA4000 + - -SA9004 + - -ST1000 + - -ST1005 + - -ST1016 + - -U1000 + exclusions: + generated: lax + rules: + - linters: + - forcetypeassert + path: _test\.go + - path: (.+)\.go$ + text: 'empty-block:' + - path: (.+)\.go$ + text: 'unused-parameter:' + - path: (.+)\.go$ + text: 'dot-imports:' + - path: (.+)\.go$ + text: should have a package comment + - path: (.+)\.go$ + text: error strings should not be capitalized or end with punctuation or a newline + paths: + - third_party$ + - builtin$ + - examples$ +issues: + max-issues-per-linter: 100 + max-same-issues: 100 +formatters: + enable: + - gofumpt + - goimports + exclusions: + generated: lax + paths: + - third_party$ + - builtin$ + - examples$ diff --git a/.mailmap b/.mailmap new file mode 100644 index 00000000..4adf1be9 --- /dev/null +++ b/.mailmap @@ -0,0 +1,18 @@ +# Generate CONTRIBUTORS.md: contributors.sh + +# Tip for finding duplicates (besides scanning the output of CONTRIBUTORS.md for name +# duplicates that aren't also email duplicates): scan the output of: +# git log --format='%aE - %aN' | sort -uf +# +# For explanation on this file format: man git-shortlog + +Anand Babu (AB) Periasamy Anand Babu (AB) Periasamy +Anand Babu (AB) Periasamy +Anis Elleuch +Frederick F. Kautz IV +Harshavardhana +Harshavardhana +Harshavardhana +Krishna Srinivas +Matthew Farrellee +Nate Rosenblum \ No newline at end of file diff --git a/.typos.toml b/.typos.toml new file mode 100644 index 00000000..3168e9dd --- /dev/null +++ b/.typos.toml @@ -0,0 +1,45 @@ +[files] +extend-exclude = [".git/", "docs/", "CREDITS", "go.mod", "go.sum"] +ignore-hidden = false + +[default] +extend-ignore-re = [ + "Patrick Collison", + "Copyright 2014 Unknwon", + "[0-9A-Za-z/+=]{64}", + "ZXJuZXQxDjAMBgNVBA-some-junk-Q4wDAYDVQQLEwVNaW5pbzEOMAwGA1UEAxMF", + "eyJmb28iOiJiYXIifQ", + "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.*", + "MIIDBTCCAe2gAwIBAgIQWHw7h.*", + 'http\.Header\{"X-Amz-Server-Side-Encryptio":', + "ZoEoZdLlzVbOlT9rbhD7ZN7TLyiYXSAlB79uGEge", + "ERRO:", + "(?Rm)^.*(#|//)\\s*spellchecker:disable-line$", # ignore line +] + +[default.extend-words] +"encrypter" = "encrypter" +"kms" = "kms" +"requestor" = "requestor" + +[default.extend-identifiers] +"HashiCorp" = "HashiCorp" + +[type.go.extend-identifiers] +"bui" = "bui" +"dm2nd" = "dm2nd" +"ot" = "ot" +"ParseND" = "ParseND" +"ParseNDStream" = "ParseNDStream" +"pn" = "pn" +"TestGetPartialObjectMisAligned" = "TestGetPartialObjectMisAligned" +"thr" = "thr" +"toi" = "toi" + +[type.go] +extend-ignore-identifiers-re = [ + # Variants of `typ` used to mean `type` in golang as it is otherwise a + # keyword - some of these (like typ1 -> type1) can be fixed, but probably + # not worth the effort. + "[tT]yp[0-9]*", +] diff --git a/CNAME b/CNAME new file mode 100644 index 00000000..f5ea8034 --- /dev/null +++ b/CNAME @@ -0,0 +1 @@ +charts.min.io diff --git a/COMPLIANCE.md b/COMPLIANCE.md new file mode 100644 index 00000000..40a58640 --- /dev/null +++ b/COMPLIANCE.md @@ -0,0 +1,7 @@ +# AGPLv3 Compliance + +We have designed MinIO as an Open Source software for the Open Source software community. This requires applications to consider whether their usage of MinIO is in compliance with the GNU AGPLv3 [license](https://github.com/minio/minio/blob/master/LICENSE). + +MinIO cannot make the determination as to whether your application's usage of MinIO is in compliance with the AGPLv3 license requirements. You should instead rely on your own legal counsel or licensing specialists to audit and ensure your application is in compliance with the licenses of MinIO and all other open-source projects with which your application integrates or interacts. We understand that AGPLv3 licensing is complex and nuanced. It is for that reason we strongly encourage using experts in licensing to make any such determinations around compliance instead of relying on apocryphal or anecdotal advice. + +[MinIO Commercial Licensing](https://min.io/pricing) is the best option for applications that trigger AGPLv3 obligations (e.g. open sourcing your application). Applications using MinIO - or any other OSS-licensed code - without validating their usage do so at their own risk. diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 00000000..c99df744 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,82 @@ +# MinIO Contribution Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) + +``MinIO`` community welcomes your contribution. To make the process as seamless as possible, we recommend you read this contribution guide. + +## Development Workflow + +Start by forking the MinIO GitHub repository, make changes in a branch and then send a pull request. We encourage pull requests to discuss code changes. Here are the steps in details: + +### Setup your MinIO GitHub Repository + +Fork [MinIO upstream](https://github.com/minio/minio/fork) source repository to your own personal repository. Copy the URL of your MinIO fork (you will need it for the `git clone` command below). + +```sh +git clone https://github.com/minio/minio +cd minio +go install -v +ls $(go env GOPATH)/bin/minio +``` + +### Set up git remote as ``upstream`` + +```sh +$ cd minio +$ git remote add upstream https://github.com/minio/minio +$ git fetch upstream +$ git merge upstream/master +... +``` + +### Create your feature branch + +Before making code changes, make sure you create a separate branch for these changes + +``` +git checkout -b my-new-feature +``` + +### Test MinIO server changes + +After your code changes, make sure + +- To add test cases for the new code. If you have questions about how to do it, please ask on our [Slack](https://slack.min.io) channel. +- To run `make verifiers` +- To squash your commits into a single commit. `git rebase -i`. It's okay to force update your pull request. +- To run `make test` and `make build` completes. + +### Commit changes + +After verification, commit your changes. This is a [great post](https://chris.beams.io/posts/git-commit/) on how to write useful commit messages + +``` +git commit -am 'Add some feature' +``` + +### Push to the branch + +Push your locally committed changes to the remote origin (your fork) + +``` +git push origin my-new-feature +``` + +### Create a Pull Request + +Pull requests can be created via GitHub. Refer to [this document](https://help.github.com/articles/creating-a-pull-request/) for detailed steps on how to create a pull request. After a Pull Request gets peer reviewed and approved, it will be merged. + +## FAQs + +### How does ``MinIO`` manage dependencies? + +``MinIO`` uses `go mod` to manage its dependencies. + +- Run `go get foo/bar` in the source folder to add the dependency to `go.mod` file. + +To remove a dependency + +- Edit your code and remove the import reference. +- Run `go mod tidy` in the source folder to remove dependency from `go.mod` file. + +### What are the coding guidelines for MinIO? + +``MinIO`` is fully conformant with Golang style. Refer: [Effective Go](https://github.com/golang/go/wiki/CodeReviewComments) article from Golang project. If you observe offending code, please feel free to send a pull request or ping us on [Slack](https://slack.min.io). diff --git a/CREDITS b/CREDITS new file mode 100644 index 00000000..c70aed83 --- /dev/null +++ b/CREDITS @@ -0,0 +1,35316 @@ +Go (the standard library) +https://golang.org/ +---------------------------------------------------------------- +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +aead.dev/mem +https://aead.dev/mem +---------------------------------------------------------------- +MIT License + +Copyright (c) 2022 Andreas Auernhammer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +aead.dev/minisign +https://aead.dev/minisign +---------------------------------------------------------------- +MIT License + +Copyright (c) 2021 Andreas Auernhammer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +cel.dev/expr +https://cel.dev/expr +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +cloud.google.com/go +https://cloud.google.com/go +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +cloud.google.com/go/auth +https://cloud.google.com/go/auth +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +cloud.google.com/go/auth/oauth2adapt +https://cloud.google.com/go/auth/oauth2adapt +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +cloud.google.com/go/compute/metadata +https://cloud.google.com/go/compute/metadata +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +cloud.google.com/go/iam +https://cloud.google.com/go/iam +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +cloud.google.com/go/logging +https://cloud.google.com/go/logging +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +cloud.google.com/go/longrunning +https://cloud.google.com/go/longrunning +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +cloud.google.com/go/monitoring +https://cloud.google.com/go/monitoring +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +cloud.google.com/go/storage +https://cloud.google.com/go/storage +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +cloud.google.com/go/trace +https://cloud.google.com/go/trace +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +filippo.io/edwards25519 +https://filippo.io/edwards25519 +---------------------------------------------------------------- +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/Azure/azure-sdk-for-go/sdk/azcore +https://github.com/Azure/azure-sdk-for-go/sdk/azcore +---------------------------------------------------------------- +MIT License + +Copyright (c) Microsoft Corporation. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE + +================================================================ + +github.com/Azure/azure-sdk-for-go/sdk/azidentity +https://github.com/Azure/azure-sdk-for-go/sdk/azidentity +---------------------------------------------------------------- +MIT License + +Copyright (c) Microsoft Corporation. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE + +================================================================ + +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache +https://github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache +---------------------------------------------------------------- +MIT License + +Copyright (c) Microsoft Corporation. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE + +================================================================ + +github.com/Azure/azure-sdk-for-go/sdk/internal +https://github.com/Azure/azure-sdk-for-go/sdk/internal +---------------------------------------------------------------- +MIT License + +Copyright (c) Microsoft Corporation. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE + +================================================================ + +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage +https://github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage +---------------------------------------------------------------- +MIT License + +Copyright (c) Microsoft Corporation. All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +================================================================ + +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob +https://github.com/Azure/azure-sdk-for-go/sdk/storage/azblob +---------------------------------------------------------------- + MIT License + + Copyright (c) Microsoft Corporation. All rights reserved. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE +================================================================ + +github.com/Azure/go-ntlmssp +https://github.com/Azure/go-ntlmssp +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2016 Microsoft + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache +https://github.com/AzureAD/microsoft-authentication-extensions-for-go/cache +---------------------------------------------------------------- + MIT License + + Copyright (c) Microsoft Corporation. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE + +================================================================ + +github.com/AzureAD/microsoft-authentication-library-for-go +https://github.com/AzureAD/microsoft-authentication-library-for-go +---------------------------------------------------------------- + MIT License + + Copyright (c) Microsoft Corporation. + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in all + copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + SOFTWARE + +================================================================ + +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp +https://github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric +https://github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock +https://github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping +https://github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/IBM/sarama +https://github.com/IBM/sarama +---------------------------------------------------------------- +# MIT License + +Copyright (c) 2013 Shopify + +Copyright (c) 2023 IBM Corporation + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================ + +github.com/VividCortex/ewma +https://github.com/VividCortex/ewma +---------------------------------------------------------------- +The MIT License + +Copyright (c) 2013 VividCortex + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +================================================================ + +github.com/acarl005/stripansi +https://github.com/acarl005/stripansi +---------------------------------------------------------------- +MIT License + +Copyright (c) 2018 Andrew Carlson + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/alecthomas/participle +https://github.com/alecthomas/participle +---------------------------------------------------------------- +Copyright (C) 2017 Alec Thomas + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/alexbrainman/sspi +https://github.com/alexbrainman/sspi +---------------------------------------------------------------- +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/apache/thrift +https://github.com/apache/thrift +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +-------------------------------------------------- +SOFTWARE DISTRIBUTED WITH THRIFT: + +The Apache Thrift software includes a number of subcomponents with +separate copyright notices and license terms. Your use of the source +code for the these subcomponents is subject to the terms and +conditions of the following licenses. + +-------------------------------------------------- +Portions of the following files are licensed under the MIT License: + + lib/erl/src/Makefile.am + +Please see doc/otp-base-license.txt for the full terms of this license. + +-------------------------------------------------- +For the aclocal/ax_boost_base.m4 and contrib/fb303/aclocal/ax_boost_base.m4 components: + +# Copyright (c) 2007 Thomas Porschberg +# +# Copying and distribution of this file, with or without +# modification, are permitted in any medium without royalty provided +# the copyright notice and this notice are preserved. + +-------------------------------------------------- +For the lib/nodejs/lib/thrift/json_parse.js: + +/* + json_parse.js + 2015-05-02 + Public Domain. + NO WARRANTY EXPRESSED OR IMPLIED. USE AT YOUR OWN RISK. + +*/ +(By Douglas Crockford ) + +-------------------------------------------------- +For lib/cpp/src/thrift/windows/SocketPair.cpp + +/* socketpair.c + * Copyright 2007 by Nathan C. Myers ; some rights reserved. + * This code is Free Software. It may be copied freely, in original or + * modified form, subject only to the restrictions that (1) the author is + * relieved from all responsibilities for any use for any purpose, and (2) + * this copyright notice must be retained, unchanged, in its entirety. If + * for any reason the author might be held responsible for any consequences + * of copying or use, license is withheld. + */ + + +-------------------------------------------------- +For lib/py/compat/win32/stdint.h + +// ISO C9x compliant stdint.h for Microsoft Visual Studio +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// +// Copyright (c) 2006-2008 Alexander Chemeris +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. The name of the author may be used to endorse or promote products +// derived from this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +/////////////////////////////////////////////////////////////////////////////// + + +-------------------------------------------------- +Codegen template in t_html_generator.h + +* Bootstrap v2.0.3 +* +* Copyright 2012 Twitter, Inc +* Licensed under the Apache License v2.0 +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Designed and built with all the love in the world @twitter by @mdo and @fat. + +--------------------------------------------------- +For t_cl_generator.cc + + * Copyright (c) 2008- Patrick Collison + * Copyright (c) 2006- Facebook + +--------------------------------------------------- + +================================================================ + +github.com/armon/go-metrics +https://github.com/armon/go-metrics +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2013 Armon Dadgar + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================ + +github.com/asaskevich/govalidator +https://github.com/asaskevich/govalidator +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2014-2020 Alex Saskevich + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +================================================================ + +github.com/aymanbagabas/go-osc52/v2 +https://github.com/aymanbagabas/go-osc52/v2 +---------------------------------------------------------------- +MIT License + +Copyright (c) 2022 Ayman Bagabas + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/beevik/ntp +https://github.com/beevik/ntp +---------------------------------------------------------------- +Copyright © 2015-2023 Brett Vickers. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY COPYRIGHT HOLDER ``AS IS'' AND ANY +EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL COPYRIGHT HOLDER OR +CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/beorn7/perks +https://github.com/beorn7/perks +---------------------------------------------------------------- +Copyright (C) 2013 Blake Mizerany + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================ + +github.com/buger/jsonparser +https://github.com/buger/jsonparser +---------------------------------------------------------------- +MIT License + +Copyright (c) 2016 Leonid Bugaev + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/cespare/xxhash/v2 +https://github.com/cespare/xxhash/v2 +---------------------------------------------------------------- +Copyright (c) 2016 Caleb Spare + +MIT License + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================ + +github.com/charmbracelet/bubbles +https://github.com/charmbracelet/bubbles +---------------------------------------------------------------- +MIT License + +Copyright (c) 2020-2023 Charmbracelet, Inc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/charmbracelet/bubbletea +https://github.com/charmbracelet/bubbletea +---------------------------------------------------------------- +MIT License + +Copyright (c) 2020-2023 Charmbracelet, Inc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/charmbracelet/lipgloss +https://github.com/charmbracelet/lipgloss +---------------------------------------------------------------- +MIT License + +Copyright (c) 2021-2023 Charmbracelet, Inc + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/charmbracelet/x/ansi +https://github.com/charmbracelet/x/ansi +---------------------------------------------------------------- +MIT License + +Copyright (c) 2023 Charmbracelet, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/charmbracelet/x/exp/golden +https://github.com/charmbracelet/x/exp/golden +---------------------------------------------------------------- +MIT License + +Copyright (c) 2023 Charmbracelet, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/charmbracelet/x/term +https://github.com/charmbracelet/x/term +---------------------------------------------------------------- +MIT License + +Copyright (c) 2023 Charmbracelet, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/cheggaaa/pb +https://github.com/cheggaaa/pb +---------------------------------------------------------------- +Copyright (c) 2012-2015, Sergey Cherepanov +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +* Neither the name of the author nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +================================================================ + +github.com/cncf/xds/go +https://github.com/cncf/xds/go +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/coreos/go-oidc/v3 +https://github.com/coreos/go-oidc/v3 +---------------------------------------------------------------- +Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +================================================================ + +github.com/coreos/go-semver +https://github.com/coreos/go-semver +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/coreos/go-systemd/v22 +https://github.com/coreos/go-systemd/v22 +---------------------------------------------------------------- +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/cosnicolaou/pbzip2 +https://github.com/cosnicolaou/pbzip2 +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/davecgh/go-spew +https://github.com/davecgh/go-spew +---------------------------------------------------------------- +ISC License + +Copyright (c) 2012-2016 Dave Collins + +Permission to use, copy, modify, and/or distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +================================================================ + +github.com/dchest/siphash +https://github.com/dchest/siphash +---------------------------------------------------------------- +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. + +================================================================ + +github.com/decred/dcrd/dcrec/secp256k1/v4 +https://github.com/decred/dcrd/dcrec/secp256k1/v4 +---------------------------------------------------------------- +ISC License + +Copyright (c) 2013-2017 The btcsuite developers +Copyright (c) 2015-2024 The Decred developers +Copyright (c) 2017 The Lightning Network Developers + +Permission to use, copy, modify, and distribute this software for any +purpose with or without fee is hereby granted, provided that the above +copyright notice and this permission notice appear in all copies. + +THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES +WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF +MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR +ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES +WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN +ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF +OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. + +================================================================ + +github.com/dgryski/go-rendezvous +https://github.com/dgryski/go-rendezvous +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2017-2020 Damian Gryski + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +================================================================ + +github.com/docker/go-units +https://github.com/docker/go-units +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + Copyright 2015 Docker, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/dustin/go-humanize +https://github.com/dustin/go-humanize +---------------------------------------------------------------- +Copyright (c) 2005-2008 Dustin Sallings + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + + +================================================================ + +github.com/eapache/go-resiliency +https://github.com/eapache/go-resiliency +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2014 Evan Huus + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +================================================================ + +github.com/eapache/go-xerial-snappy +https://github.com/eapache/go-xerial-snappy +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2016 Evan Huus + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/eapache/queue +https://github.com/eapache/queue +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2014 Evan Huus + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +================================================================ + +github.com/eclipse/paho.mqtt.golang +https://github.com/eclipse/paho.mqtt.golang +---------------------------------------------------------------- +Eclipse Public License - v 2.0 (EPL-2.0) + +This program and the accompanying materials +are made available under the terms of the Eclipse Public License v2.0 +and Eclipse Distribution License v1.0 which accompany this distribution. + +The Eclipse Public License is available at + https://www.eclipse.org/legal/epl-2.0/ +and the Eclipse Distribution License is available at + http://www.eclipse.org/org/documents/edl-v10.php. + +For an explanation of what dual-licensing means to you, see: +https://www.eclipse.org/legal/eplfaq.php#DUALLIC + +**** +The epl-2.0 is copied below in order to pass the pkg.go.dev license check (https://pkg.go.dev/license-policy). +**** +Eclipse Public License - v 2.0 + + THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE + PUBLIC LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION + OF THE PROGRAM CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. + +1. DEFINITIONS + +"Contribution" means: + + a) in the case of the initial Contributor, the initial content + Distributed under this Agreement, and + + b) in the case of each subsequent Contributor: + i) changes to the Program, and + ii) additions to the Program; + where such changes and/or additions to the Program originate from + and are Distributed by that particular Contributor. A Contribution + "originates" from a Contributor if it was added to the Program by + such Contributor itself or anyone acting on such Contributor's behalf. + Contributions do not include changes or additions to the Program that + are not Modified Works. + +"Contributor" means any person or entity that Distributes the Program. + +"Licensed Patents" mean patent claims licensable by a Contributor which +are necessarily infringed by the use or sale of its Contribution alone +or when combined with the Program. + +"Program" means the Contributions Distributed in accordance with this +Agreement. + +"Recipient" means anyone who receives the Program under this Agreement +or any Secondary License (as applicable), including Contributors. + +"Derivative Works" shall mean any work, whether in Source Code or other +form, that is based on (or derived from) the Program and for which the +editorial revisions, annotations, elaborations, or other modifications +represent, as a whole, an original work of authorship. + +"Modified Works" shall mean any work in Source Code or other form that +results from an addition to, deletion from, or modification of the +contents of the Program, including, for purposes of clarity any new file +in Source Code form that contains any contents of the Program. Modified +Works shall not include works that contain only declarations, +interfaces, types, classes, structures, or files of the Program solely +in each case in order to link to, bind by name, or subclass the Program +or Modified Works thereof. + +"Distribute" means the acts of a) distributing or b) making available +in any manner that enables the transfer of a copy. + +"Source Code" means the form of a Program preferred for making +modifications, including but not limited to software source code, +documentation source, and configuration files. + +"Secondary License" means either the GNU General Public License, +Version 2.0, or any later versions of that license, including any +exceptions or additional permissions as identified by the initial +Contributor. + +2. GRANT OF RIGHTS + + a) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free copyright + license to reproduce, prepare Derivative Works of, publicly display, + publicly perform, Distribute and sublicense the Contribution of such + Contributor, if any, and such Derivative Works. + + b) Subject to the terms of this Agreement, each Contributor hereby + grants Recipient a non-exclusive, worldwide, royalty-free patent + license under Licensed Patents to make, use, sell, offer to sell, + import and otherwise transfer the Contribution of such Contributor, + if any, in Source Code or other form. This patent license shall + apply to the combination of the Contribution and the Program if, at + the time the Contribution is added by the Contributor, such addition + of the Contribution causes such combination to be covered by the + Licensed Patents. The patent license shall not apply to any other + combinations which include the Contribution. No hardware per se is + licensed hereunder. + + c) Recipient understands that although each Contributor grants the + licenses to its Contributions set forth herein, no assurances are + provided by any Contributor that the Program does not infringe the + patent or other intellectual property rights of any other entity. + Each Contributor disclaims any liability to Recipient for claims + brought by any other entity based on infringement of intellectual + property rights or otherwise. As a condition to exercising the + rights and licenses granted hereunder, each Recipient hereby + assumes sole responsibility to secure any other intellectual + property rights needed, if any. For example, if a third party + patent license is required to allow Recipient to Distribute the + Program, it is Recipient's responsibility to acquire that license + before distributing the Program. + + d) Each Contributor represents that to its knowledge it has + sufficient copyright rights in its Contribution, if any, to grant + the copyright license set forth in this Agreement. + + e) Notwithstanding the terms of any Secondary License, no + Contributor makes additional grants to any Recipient (other than + those set forth in this Agreement) as a result of such Recipient's + receipt of the Program under the terms of a Secondary License + (if permitted under the terms of Section 3). + +3. REQUIREMENTS + +3.1 If a Contributor Distributes the Program in any form, then: + + a) the Program must also be made available as Source Code, in + accordance with section 3.2, and the Contributor must accompany + the Program with a statement that the Source Code for the Program + is available under this Agreement, and informs Recipients how to + obtain it in a reasonable manner on or through a medium customarily + used for software exchange; and + + b) the Contributor may Distribute the Program under a license + different than this Agreement, provided that such license: + i) effectively disclaims on behalf of all other Contributors all + warranties and conditions, express and implied, including + warranties or conditions of title and non-infringement, and + implied warranties or conditions of merchantability and fitness + for a particular purpose; + + ii) effectively excludes on behalf of all other Contributors all + liability for damages, including direct, indirect, special, + incidental and consequential damages, such as lost profits; + + iii) does not attempt to limit or alter the recipients' rights + in the Source Code under section 3.2; and + + iv) requires any subsequent distribution of the Program by any + party to be under a license that satisfies the requirements + of this section 3. + +3.2 When the Program is Distributed as Source Code: + + a) it must be made available under this Agreement, or if the + Program (i) is combined with other material in a separate file or + files made available under a Secondary License, and (ii) the initial + Contributor attached to the Source Code the notice described in + Exhibit A of this Agreement, then the Program may be made available + under the terms of such Secondary Licenses, and + + b) a copy of this Agreement must be included with each copy of + the Program. + +3.3 Contributors may not remove or alter any copyright, patent, +trademark, attribution notices, disclaimers of warranty, or limitations +of liability ("notices") contained within the Program from any copy of +the Program which they Distribute, provided that Contributors may add +their own appropriate notices. + +4. COMMERCIAL DISTRIBUTION + +Commercial distributors of software may accept certain responsibilities +with respect to end users, business partners and the like. While this +license is intended to facilitate the commercial use of the Program, +the Contributor who includes the Program in a commercial product +offering should do so in a manner which does not create potential +liability for other Contributors. Therefore, if a Contributor includes +the Program in a commercial product offering, such Contributor +("Commercial Contributor") hereby agrees to defend and indemnify every +other Contributor ("Indemnified Contributor") against any losses, +damages and costs (collectively "Losses") arising from claims, lawsuits +and other legal actions brought by a third party against the Indemnified +Contributor to the extent caused by the acts or omissions of such +Commercial Contributor in connection with its distribution of the Program +in a commercial product offering. The obligations in this section do not +apply to any claims or Losses relating to any actual or alleged +intellectual property infringement. In order to qualify, an Indemnified +Contributor must: a) promptly notify the Commercial Contributor in +writing of such claim, and b) allow the Commercial Contributor to control, +and cooperate with the Commercial Contributor in, the defense and any +related settlement negotiations. The Indemnified Contributor may +participate in any such claim at its own expense. + +For example, a Contributor might include the Program in a commercial +product offering, Product X. That Contributor is then a Commercial +Contributor. If that Commercial Contributor then makes performance +claims, or offers warranties related to Product X, those performance +claims and warranties are such Commercial Contributor's responsibility +alone. Under this section, the Commercial Contributor would have to +defend claims against the other Contributors related to those performance +claims and warranties, and if a court requires any other Contributor to +pay any damages as a result, the Commercial Contributor must pay +those damages. + +5. NO WARRANTY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT +PERMITTED BY APPLICABLE LAW, THE PROGRAM IS PROVIDED ON AN "AS IS" +BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER EXPRESS OR +IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR CONDITIONS OF +TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A PARTICULAR +PURPOSE. Each Recipient is solely responsible for determining the +appropriateness of using and distributing the Program and assumes all +risks associated with its exercise of rights under this Agreement, +including but not limited to the risks and costs of program errors, +compliance with applicable laws, damage to or loss of data, programs +or equipment, and unavailability or interruption of operations. + +6. DISCLAIMER OF LIABILITY + +EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, AND TO THE EXTENT +PERMITTED BY APPLICABLE LAW, NEITHER RECIPIENT NOR ANY CONTRIBUTORS +SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION LOST +PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE +EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGES. + +7. GENERAL + +If any provision of this Agreement is invalid or unenforceable under +applicable law, it shall not affect the validity or enforceability of +the remainder of the terms of this Agreement, and without further +action by the parties hereto, such provision shall be reformed to the +minimum extent necessary to make such provision valid and enforceable. + +If Recipient institutes patent litigation against any entity +(including a cross-claim or counterclaim in a lawsuit) alleging that the +Program itself (excluding combinations of the Program with other software +or hardware) infringes such Recipient's patent(s), then such Recipient's +rights granted under Section 2(b) shall terminate as of the date such +litigation is filed. + +All Recipient's rights under this Agreement shall terminate if it +fails to comply with any of the material terms or conditions of this +Agreement and does not cure such failure in a reasonable period of +time after becoming aware of such noncompliance. If all Recipient's +rights under this Agreement terminate, Recipient agrees to cease use +and distribution of the Program as soon as reasonably practicable. +However, Recipient's obligations under this Agreement and any licenses +granted by Recipient relating to the Program shall continue and survive. + +Everyone is permitted to copy and distribute copies of this Agreement, +but in order to avoid inconsistency the Agreement is copyrighted and +may only be modified in the following manner. The Agreement Steward +reserves the right to publish new versions (including revisions) of +this Agreement from time to time. No one other than the Agreement +Steward has the right to modify this Agreement. The Eclipse Foundation +is the initial Agreement Steward. The Eclipse Foundation may assign the +responsibility to serve as the Agreement Steward to a suitable separate +entity. Each new version of the Agreement will be given a distinguishing +version number. The Program (including Contributions) may always be +Distributed subject to the version of the Agreement under which it was +received. In addition, after a new version of the Agreement is published, +Contributor may elect to Distribute the Program (including its +Contributions) under the new version. + +Except as expressly stated in Sections 2(a) and 2(b) above, Recipient +receives no rights or licenses to the intellectual property of any +Contributor under this Agreement, whether expressly, by implication, +estoppel or otherwise. All rights in the Program not expressly granted +under this Agreement are reserved. Nothing in this Agreement is intended +to be enforceable by any entity that is not a Contributor or Recipient. +No third-party beneficiary rights are created under this Agreement. + +Exhibit A - Form of Secondary Licenses Notice + +"This Source Code may also be made available under the following +Secondary Licenses when the conditions for such availability set forth +in the Eclipse Public License, v. 2.0 are satisfied: {name license(s), +version(s), and exceptions or additional permissions here}." + + Simply including a copy of this Agreement, including this Exhibit A + is not sufficient to license the Source Code under Secondary Licenses. + + If it is not possible or desirable to put the notice in a particular + file, then You may include the notice in a location (such as a LICENSE + file in a relevant directory) where a recipient would be likely to + look for such a notice. + + You may add additional accurate notices of copyright ownership. + +================================================================ + +github.com/elastic/go-elasticsearch/v7 +https://github.com/elastic/go-elasticsearch/v7 +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2018 Elasticsearch BV + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/envoyproxy/go-control-plane +https://github.com/envoyproxy/go-control-plane +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/envoyproxy/go-control-plane/envoy +https://github.com/envoyproxy/go-control-plane/envoy +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/envoyproxy/go-control-plane/ratelimit +https://github.com/envoyproxy/go-control-plane/ratelimit +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/envoyproxy/protoc-gen-validate +https://github.com/envoyproxy/protoc-gen-validate +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/erikgeiser/coninput +https://github.com/erikgeiser/coninput +---------------------------------------------------------------- +MIT License + +Copyright (c) 2021 Erik G. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/fatih/color +https://github.com/fatih/color +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2013 Fatih Arslan + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================ + +github.com/fatih/structs +https://github.com/fatih/structs +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2014 Fatih Arslan + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. +================================================================ + +github.com/felixge/fgprof +https://github.com/felixge/fgprof +---------------------------------------------------------------- +The MIT License (MIT) +Copyright © 2020 Felix Geisendörfer + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================ + +github.com/felixge/httpsnoop +https://github.com/felixge/httpsnoop +---------------------------------------------------------------- +Copyright (c) 2016 Felix Geisendörfer (felix@debuggable.com) + + Permission is hereby granted, free of charge, to any person obtaining a copy + of this software and associated documentation files (the "Software"), to deal + in the Software without restriction, including without limitation the rights + to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + copies of the Software, and to permit persons to whom the Software is + furnished to do so, subject to the following conditions: + + The above copyright notice and this permission notice shall be included in + all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + THE SOFTWARE. + +================================================================ + +github.com/fortytw2/leaktest +https://github.com/fortytw2/leaktest +---------------------------------------------------------------- +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/fraugster/parquet-go +https://github.com/fraugster/parquet-go +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/go-asn1-ber/asn1-ber +https://github.com/go-asn1-ber/asn1-ber +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2011-2015 Michael Mitton (mmitton@gmail.com) +Portions copyright (c) 2015-2016 go-asn1-ber Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/go-ini/ini +https://github.com/go-ini/ini +---------------------------------------------------------------- +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright 2014 Unknwon + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/go-jose/go-jose/v4 +https://github.com/go-jose/go-jose/v4 +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/go-ldap/ldap/v3 +https://github.com/go-ldap/ldap/v3 +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2011-2015 Michael Mitton (mmitton@gmail.com) +Portions copyright (c) 2015-2024 go-ldap Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/go-logr/logr +https://github.com/go-logr/logr +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/go-logr/stdr +https://github.com/go-logr/stdr +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/go-ole/go-ole +https://github.com/go-ole/go-ole +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright © 2013-2017 Yasuhiro Matsumoto, + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the “Software”), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/go-openapi/analysis +https://github.com/go-openapi/analysis +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/go-openapi/errors +https://github.com/go-openapi/errors +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/go-openapi/jsonpointer +https://github.com/go-openapi/jsonpointer +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/go-openapi/jsonreference +https://github.com/go-openapi/jsonreference +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/go-openapi/loads +https://github.com/go-openapi/loads +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/go-openapi/runtime +https://github.com/go-openapi/runtime +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/go-openapi/spec +https://github.com/go-openapi/spec +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/go-openapi/strfmt +https://github.com/go-openapi/strfmt +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/go-openapi/swag +https://github.com/go-openapi/swag +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/go-openapi/validate +https://github.com/go-openapi/validate +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/go-sql-driver/mysql +https://github.com/go-sql-driver/mysql +---------------------------------------------------------------- +Mozilla Public License Version 2.0 +================================== + +1. Definitions +-------------- + +1.1. "Contributor" + means each individual or legal entity that creates, contributes to + the creation of, or owns Covered Software. + +1.2. "Contributor Version" + means the combination of the Contributions of others (if any) used + by a Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + means Source Code Form to which the initial Contributor has attached + the notice in Exhibit A, the Executable Form of such Source Code + Form, and Modifications of such Source Code Form, in each case + including portions thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + (a) that the initial Contributor has attached the notice described + in Exhibit B to the Covered Software; or + + (b) that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the + terms of a Secondary License. + +1.6. "Executable Form" + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + means a work that combines Covered Software with other material, in + a separate file or files, that is not Covered Software. + +1.8. "License" + means this document. + +1.9. "Licensable" + means having the right to grant, to the maximum extent possible, + whether at the time of the initial grant or subsequently, any and + all of the rights conveyed by this License. + +1.10. "Modifications" + means any of the following: + + (a) any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered + Software; or + + (b) any new file in Source Code Form that contains any Covered + Software. + +1.11. "Patent Claims" of a Contributor + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the + License, by the making, using, selling, offering for sale, having + made, import, or transfer of either its Contributions or its + Contributor Version. + +1.12. "Secondary License" + means either the GNU General Public License, Version 2.0, the GNU + Lesser General Public License, Version 2.1, the GNU Affero General + Public License, Version 3.0, or any later versions of those + licenses. + +1.13. "Source Code Form" + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that + controls, is controlled by, or is under common control with You. For + purposes of this definition, "control" means (a) the power, direct + or indirect, to cause the direction or management of such entity, + whether by contract or otherwise, or (b) ownership of more than + fifty percent (50%) of the outstanding shares or beneficial + ownership of such entity. + +2. License Grants and Conditions +-------------------------------- + +2.1. Grants + +Each Contributor hereby grants You a world-wide, royalty-free, +non-exclusive license: + +(a) under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + +(b) under Patent Claims of such Contributor to make, use, sell, offer + for sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + +The licenses granted in Section 2.1 with respect to any Contribution +become effective for each Contribution on the date the Contributor first +distributes such Contribution. + +2.3. Limitations on Grant Scope + +The licenses granted in this Section 2 are the only rights granted under +this License. No additional rights or licenses will be implied from the +distribution or licensing of Covered Software under this License. +Notwithstanding Section 2.1(b) above, no patent license is granted by a +Contributor: + +(a) for any code that a Contributor has removed from Covered Software; + or + +(b) for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + +(c) under Patent Claims infringed by Covered Software in the absence of + its Contributions. + +This License does not grant any rights in the trademarks, service marks, +or logos of any Contributor (except as may be necessary to comply with +the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + +No Contributor makes additional grants as a result of Your choice to +distribute the Covered Software under a subsequent version of this +License (see Section 10.2) or under the terms of a Secondary License (if +permitted under the terms of Section 3.3). + +2.5. Representation + +Each Contributor represents that the Contributor believes its +Contributions are its original creation(s) or it has sufficient rights +to grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + +This License is not intended to limit any rights You have under +applicable copyright doctrines of fair use, fair dealing, or other +equivalents. + +2.7. Conditions + +Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted +in Section 2.1. + +3. Responsibilities +------------------- + +3.1. Distribution of Source Form + +All distribution of Covered Software in Source Code Form, including any +Modifications that You create or to which You contribute, must be under +the terms of this License. You must inform recipients that the Source +Code Form of the Covered Software is governed by the terms of this +License, and how they can obtain a copy of this License. You may not +attempt to alter or restrict the recipients' rights in the Source Code +Form. + +3.2. Distribution of Executable Form + +If You distribute Covered Software in Executable Form then: + +(a) such Covered Software must also be made available in Source Code + Form, as described in Section 3.1, and You must inform recipients of + the Executable Form how they can obtain a copy of such Source Code + Form by reasonable means in a timely manner, at a charge no more + than the cost of distribution to the recipient; and + +(b) You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter + the recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + +You may create and distribute a Larger Work under terms of Your choice, +provided that You also comply with the requirements of this License for +the Covered Software. If the Larger Work is a combination of Covered +Software with a work governed by one or more Secondary Licenses, and the +Covered Software is not Incompatible With Secondary Licenses, this +License permits You to additionally distribute such Covered Software +under the terms of such Secondary License(s), so that the recipient of +the Larger Work may, at their option, further distribute the Covered +Software under the terms of either this License or such Secondary +License(s). + +3.4. Notices + +You may not remove or alter the substance of any license notices +(including copyright notices, patent notices, disclaimers of warranty, +or limitations of liability) contained within the Source Code Form of +the Covered Software, except that You may alter any license notices to +the extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + +You may choose to offer, and to charge a fee for, warranty, support, +indemnity or liability obligations to one or more recipients of Covered +Software. However, You may do so only on Your own behalf, and not on +behalf of any Contributor. You must make it absolutely clear that any +such warranty, support, indemnity, or liability obligation is offered by +You alone, and You hereby agree to indemnify every Contributor for any +liability incurred by such Contributor as a result of warranty, support, +indemnity or liability terms You offer. You may include additional +disclaimers of warranty and limitations of liability specific to any +jurisdiction. + +4. Inability to Comply Due to Statute or Regulation +--------------------------------------------------- + +If it is impossible for You to comply with any of the terms of this +License with respect to some or all of the Covered Software due to +statute, judicial order, or regulation then You must: (a) comply with +the terms of this License to the maximum extent possible; and (b) +describe the limitations and the code they affect. Such description must +be placed in a text file included with all distributions of the Covered +Software under this License. Except to the extent prohibited by statute +or regulation, such description must be sufficiently detailed for a +recipient of ordinary skill to be able to understand it. + +5. Termination +-------------- + +5.1. The rights granted under this License will terminate automatically +if You fail to comply with any of its terms. However, if You become +compliant, then the rights granted under this License from a particular +Contributor are reinstated (a) provisionally, unless and until such +Contributor explicitly and finally terminates Your grants, and (b) on an +ongoing basis, if such Contributor fails to notify You of the +non-compliance by some reasonable means prior to 60 days after You have +come back into compliance. Moreover, Your grants from a particular +Contributor are reinstated on an ongoing basis if such Contributor +notifies You of the non-compliance by some reasonable means, this is the +first time You have received notice of non-compliance with this License +from such Contributor, and You become compliant prior to 30 days after +Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent +infringement claim (excluding declaratory judgment actions, +counter-claims, and cross-claims) alleging that a Contributor Version +directly or indirectly infringes any patent, then the rights granted to +You by any and all Contributors for the Covered Software under Section +2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all +end user license agreements (excluding distributors and resellers) which +have been validly granted by You or Your distributors under this License +prior to termination shall survive termination. + +************************************************************************ +* * +* 6. Disclaimer of Warranty * +* ------------------------- * +* * +* Covered Software is provided under this License on an "as is" * +* basis, without warranty of any kind, either expressed, implied, or * +* statutory, including, without limitation, warranties that the * +* Covered Software is free of defects, merchantable, fit for a * +* particular purpose or non-infringing. The entire risk as to the * +* quality and performance of the Covered Software is with You. * +* Should any Covered Software prove defective in any respect, You * +* (not any Contributor) assume the cost of any necessary servicing, * +* repair, or correction. This disclaimer of warranty constitutes an * +* essential part of this License. No use of any Covered Software is * +* authorized under this License except under this disclaimer. * +* * +************************************************************************ + +************************************************************************ +* * +* 7. Limitation of Liability * +* -------------------------- * +* * +* Under no circumstances and under no legal theory, whether tort * +* (including negligence), contract, or otherwise, shall any * +* Contributor, or anyone who distributes Covered Software as * +* permitted above, be liable to You for any direct, indirect, * +* special, incidental, or consequential damages of any character * +* including, without limitation, damages for lost profits, loss of * +* goodwill, work stoppage, computer failure or malfunction, or any * +* and all other commercial damages or losses, even if such party * +* shall have been informed of the possibility of such damages. This * +* limitation of liability shall not apply to liability for death or * +* personal injury resulting from such party's negligence to the * +* extent applicable law prohibits such limitation. Some * +* jurisdictions do not allow the exclusion or limitation of * +* incidental or consequential damages, so this exclusion and * +* limitation may not apply to You. * +* * +************************************************************************ + +8. Litigation +------------- + +Any litigation relating to this License may be brought only in the +courts of a jurisdiction where the defendant maintains its principal +place of business and such litigation shall be governed by laws of that +jurisdiction, without reference to its conflict-of-law provisions. +Nothing in this Section shall prevent a party's ability to bring +cross-claims or counter-claims. + +9. Miscellaneous +---------------- + +This License represents the complete agreement concerning the subject +matter hereof. If any provision of this License is held to be +unenforceable, such provision shall be reformed only to the extent +necessary to make it enforceable. Any law or regulation which provides +that the language of a contract shall be construed against the drafter +shall not be used to construe this License against a Contributor. + +10. Versions of the License +--------------------------- + +10.1. New Versions + +Mozilla Foundation is the license steward. Except as provided in Section +10.3, no one other than the license steward has the right to modify or +publish new versions of this License. Each version will be given a +distinguishing version number. + +10.2. Effect of New Versions + +You may distribute the Covered Software under the terms of the version +of the License under which You originally received the Covered Software, +or under the terms of any subsequent version published by the license +steward. + +10.3. Modified Versions + +If you create software not governed by this License, and you want to +create a new license for such software, you may create and use a +modified version of this License if you rename the license and remove +any references to the name of the license steward (except to note that +such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary +Licenses + +If You choose to distribute Source Code Form that is Incompatible With +Secondary Licenses under the terms of this version of the License, the +notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice +------------------------------------------- + + This Source Code Form is subject to the terms of the Mozilla Public + License, v. 2.0. If a copy of the MPL was not distributed with this + file, You can obtain one at http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular +file, then You may include the notice in a location (such as a LICENSE +file in a relevant directory) where a recipient would be likely to look +for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice +--------------------------------------------------------- + + This Source Code Form is "Incompatible With Secondary Licenses", as + defined by the Mozilla Public License, v. 2.0. + +================================================================ + +github.com/gobwas/httphead +https://github.com/gobwas/httphead +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2017 Sergey Kamardin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/gobwas/pool +https://github.com/gobwas/pool +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2017-2019 Sergey Kamardin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/gobwas/ws +https://github.com/gobwas/ws +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2017-2021 Sergey Kamardin + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/goccy/go-json +https://github.com/goccy/go-json +---------------------------------------------------------------- +MIT License + +Copyright (c) 2020 Masaaki Goshima + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/gogo/protobuf +https://github.com/gogo/protobuf +---------------------------------------------------------------- +Copyright (c) 2013, The GoGo Authors. All rights reserved. + +Protocol Buffers for Go with Gadgets + +Go support for Protocol Buffers - Google's data interchange format + +Copyright 2010 The Go Authors. All rights reserved. +https://github.com/golang/protobuf + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +================================================================ + +github.com/golang-jwt/jwt/v4 +https://github.com/golang-jwt/jwt/v4 +---------------------------------------------------------------- +Copyright (c) 2012 Dave Grijalva +Copyright (c) 2021 golang-jwt maintainers + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +================================================================ + +github.com/golang-jwt/jwt/v5 +https://github.com/golang-jwt/jwt/v5 +---------------------------------------------------------------- +Copyright (c) 2012 Dave Grijalva +Copyright (c) 2021 golang-jwt maintainers + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + +================================================================ + +github.com/golang/groupcache +https://github.com/golang/groupcache +---------------------------------------------------------------- +Apache License +Version 2.0, January 2004 +http://www.apache.org/licenses/ + +TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + +1. Definitions. + +"License" shall mean the terms and conditions for use, reproduction, and +distribution as defined by Sections 1 through 9 of this document. + +"Licensor" shall mean the copyright owner or entity authorized by the copyright +owner that is granting the License. + +"Legal Entity" shall mean the union of the acting entity and all other entities +that control, are controlled by, or are under common control with that entity. +For the purposes of this definition, "control" means (i) the power, direct or +indirect, to cause the direction or management of such entity, whether by +contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the +outstanding shares, or (iii) beneficial ownership of such entity. + +"You" (or "Your") shall mean an individual or Legal Entity exercising +permissions granted by this License. + +"Source" form shall mean the preferred form for making modifications, including +but not limited to software source code, documentation source, and configuration +files. + +"Object" form shall mean any form resulting from mechanical transformation or +translation of a Source form, including but not limited to compiled object code, +generated documentation, and conversions to other media types. + +"Work" shall mean the work of authorship, whether in Source or Object form, made +available under the License, as indicated by a copyright notice that is included +in or attached to the work (an example is provided in the Appendix below). + +"Derivative Works" shall mean any work, whether in Source or Object form, that +is based on (or derived from) the Work and for which the editorial revisions, +annotations, elaborations, or other modifications represent, as a whole, an +original work of authorship. For the purposes of this License, Derivative Works +shall not include works that remain separable from, or merely link (or bind by +name) to the interfaces of, the Work and Derivative Works thereof. + +"Contribution" shall mean any work of authorship, including the original version +of the Work and any modifications or additions to that Work or Derivative Works +thereof, that is intentionally submitted to Licensor for inclusion in the Work +by the copyright owner or by an individual or Legal Entity authorized to submit +on behalf of the copyright owner. For the purposes of this definition, +"submitted" means any form of electronic, verbal, or written communication sent +to the Licensor or its representatives, including but not limited to +communication on electronic mailing lists, source code control systems, and +issue tracking systems that are managed by, or on behalf of, the Licensor for +the purpose of discussing and improving the Work, but excluding communication +that is conspicuously marked or otherwise designated in writing by the copyright +owner as "Not a Contribution." + +"Contributor" shall mean Licensor and any individual or Legal Entity on behalf +of whom a Contribution has been received by Licensor and subsequently +incorporated within the Work. + +2. Grant of Copyright License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable copyright license to reproduce, prepare Derivative Works of, +publicly display, publicly perform, sublicense, and distribute the Work and such +Derivative Works in Source or Object form. + +3. Grant of Patent License. + +Subject to the terms and conditions of this License, each Contributor hereby +grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, +irrevocable (except as stated in this section) patent license to make, have +made, use, offer to sell, sell, import, and otherwise transfer the Work, where +such license applies only to those patent claims licensable by such Contributor +that are necessarily infringed by their Contribution(s) alone or by combination +of their Contribution(s) with the Work to which such Contribution(s) was +submitted. If You institute patent litigation against any entity (including a +cross-claim or counterclaim in a lawsuit) alleging that the Work or a +Contribution incorporated within the Work constitutes direct or contributory +patent infringement, then any patent licenses granted to You under this License +for that Work shall terminate as of the date such litigation is filed. + +4. Redistribution. + +You may reproduce and distribute copies of the Work or Derivative Works thereof +in any medium, with or without modifications, and in Source or Object form, +provided that You meet the following conditions: + +You must give any other recipients of the Work or Derivative Works a copy of +this License; and +You must cause any modified files to carry prominent notices stating that You +changed the files; and +You must retain, in the Source form of any Derivative Works that You distribute, +all copyright, patent, trademark, and attribution notices from the Source form +of the Work, excluding those notices that do not pertain to any part of the +Derivative Works; and +If the Work includes a "NOTICE" text file as part of its distribution, then any +Derivative Works that You distribute must include a readable copy of the +attribution notices contained within such NOTICE file, excluding those notices +that do not pertain to any part of the Derivative Works, in at least one of the +following places: within a NOTICE text file distributed as part of the +Derivative Works; within the Source form or documentation, if provided along +with the Derivative Works; or, within a display generated by the Derivative +Works, if and wherever such third-party notices normally appear. The contents of +the NOTICE file are for informational purposes only and do not modify the +License. You may add Your own attribution notices within Derivative Works that +You distribute, alongside or as an addendum to the NOTICE text from the Work, +provided that such additional attribution notices cannot be construed as +modifying the License. +You may add Your own copyright statement to Your modifications and may provide +additional or different license terms and conditions for use, reproduction, or +distribution of Your modifications, or for any such Derivative Works as a whole, +provided Your use, reproduction, and distribution of the Work otherwise complies +with the conditions stated in this License. + +5. Submission of Contributions. + +Unless You explicitly state otherwise, any Contribution intentionally submitted +for inclusion in the Work by You to the Licensor shall be under the terms and +conditions of this License, without any additional terms or conditions. +Notwithstanding the above, nothing herein shall supersede or modify the terms of +any separate license agreement you may have executed with Licensor regarding +such Contributions. + +6. Trademarks. + +This License does not grant permission to use the trade names, trademarks, +service marks, or product names of the Licensor, except as required for +reasonable and customary use in describing the origin of the Work and +reproducing the content of the NOTICE file. + +7. Disclaimer of Warranty. + +Unless required by applicable law or agreed to in writing, Licensor provides the +Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, +including, without limitation, any warranties or conditions of TITLE, +NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are +solely responsible for determining the appropriateness of using or +redistributing the Work and assume any risks associated with Your exercise of +permissions under this License. + +8. Limitation of Liability. + +In no event and under no legal theory, whether in tort (including negligence), +contract, or otherwise, unless required by applicable law (such as deliberate +and grossly negligent acts) or agreed to in writing, shall any Contributor be +liable to You for damages, including any direct, indirect, special, incidental, +or consequential damages of any character arising as a result of this License or +out of the use or inability to use the Work (including but not limited to +damages for loss of goodwill, work stoppage, computer failure or malfunction, or +any and all other commercial damages or losses), even if such Contributor has +been advised of the possibility of such damages. + +9. Accepting Warranty or Additional Liability. + +While redistributing the Work or Derivative Works thereof, You may choose to +offer, and charge a fee for, acceptance of support, warranty, indemnity, or +other liability obligations and/or rights consistent with this License. However, +in accepting such obligations, You may act only on Your own behalf and on Your +sole responsibility, not on behalf of any other Contributor, and only if You +agree to indemnify, defend, and hold each Contributor harmless for any liability +incurred by, or claims asserted against, such Contributor by reason of your +accepting any such warranty or additional liability. + +END OF TERMS AND CONDITIONS + +APPENDIX: How to apply the Apache License to your work + +To apply the Apache License to your work, attach the following boilerplate +notice, with the fields enclosed by brackets "[]" replaced with your own +identifying information. (Don't include the brackets!) The text should be +enclosed in the appropriate comment syntax for the file format. We also +recommend that a file or class name and description of purpose be included on +the same "printed page" as the copyright notice for easier identification within +third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/golang/protobuf +https://github.com/golang/protobuf +---------------------------------------------------------------- +Copyright 2010 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +================================================================ + +github.com/golang/snappy +https://github.com/golang/snappy +---------------------------------------------------------------- +Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/gomodule/redigo +https://github.com/gomodule/redigo +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + +================================================================ + +github.com/google/go-cmp +https://github.com/google/go-cmp +---------------------------------------------------------------- +Copyright (c) 2017 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/google/martian/v3 +https://github.com/google/martian/v3 +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/google/pprof +https://github.com/google/pprof +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/google/s2a-go +https://github.com/google/s2a-go +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/google/shlex +https://github.com/google/shlex +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/google/uuid +https://github.com/google/uuid +---------------------------------------------------------------- +Copyright (c) 2009,2014 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/googleapis/enterprise-certificate-proxy +https://github.com/googleapis/enterprise-certificate-proxy +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/googleapis/gax-go/v2 +https://github.com/googleapis/gax-go/v2 +---------------------------------------------------------------- +Copyright 2016, Google Inc. +All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/gorilla/mux +https://github.com/gorilla/mux +---------------------------------------------------------------- +Copyright (c) 2023 The Gorilla Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/gorilla/websocket +https://github.com/gorilla/websocket +---------------------------------------------------------------- +Copyright (c) 2013 The Gorilla WebSocket Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/hashicorp/errwrap +https://github.com/hashicorp/errwrap +---------------------------------------------------------------- +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. “Contributor” + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. “Contributor Version” + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” + + means Covered Software of a particular Contributor. + +1.4. “Covered Software” + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. “Incompatible With Secondary Licenses” + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of version + 1.1 or earlier of the License, but not also under the terms of a + Secondary License. + +1.6. “Executable Form” + + means any form of the work other than Source Code Form. + +1.7. “Larger Work” + + means a work that combines Covered Software with other material, in a separate + file or files, that is not Covered Software. + +1.8. “License” + + means this document. + +1.9. “Licensable” + + means having the right to grant, to the maximum extent possible, whether at the + time of the initial grant or subsequently, any and all of the rights conveyed by + this License. + +1.10. “Modifications” + + means any of the following: + + a. any file in Source Code Form that results from an addition to, deletion + from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor + + means any patent claim(s), including without limitation, method, process, + and apparatus claims, in any patent Licensable by such Contributor that + would be infringed, but for the grant of the License, by the making, + using, selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. + +1.12. “Secondary License” + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. “Source Code Form” + + means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) + + means an individual or a legal entity exercising rights under this + License. For legal entities, “You” includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, “control” means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or as + part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its Contributions + or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution become + effective for each Contribution on the date the Contributor first distributes + such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under this + License. No additional rights or licenses will be implied from the distribution + or licensing of Covered Software under this License. Notwithstanding Section + 2.1(b) above, no patent license is granted by a Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party’s + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of its + Contributions. + + This License does not grant any rights in the trademarks, service marks, or + logos of any Contributor (except as may be necessary to comply with the + notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this License + (see Section 10.2) or under the terms of a Secondary License (if permitted + under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its Contributions + are its original creation(s) or it has sufficient rights to grant the + rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under applicable + copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under the + terms of this License. You must inform recipients that the Source Code Form + of the Covered Software is governed by the terms of this License, and how + they can obtain a copy of this License. You may not attempt to alter or + restrict the recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this License, + or sublicense it under different terms, provided that the license for + the Executable Form does not attempt to limit or alter the recipients’ + rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for the + Covered Software. If the Larger Work is a combination of Covered Software + with a work governed by one or more Secondary Licenses, and the Covered + Software is not Incompatible With Secondary Licenses, this License permits + You to additionally distribute such Covered Software under the terms of + such Secondary License(s), so that the recipient of the Larger Work may, at + their option, further distribute the Covered Software under the terms of + either this License or such Secondary License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices (including + copyright notices, patent notices, disclaimers of warranty, or limitations + of liability) contained within the Source Code Form of the Covered + Software, except that You may alter any license notices to the extent + required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on behalf + of any Contributor. You must make it absolutely clear that any such + warranty, support, indemnity, or liability obligation is offered by You + alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, judicial + order, or regulation then You must: (a) comply with the terms of this License + to the maximum extent possible; and (b) describe the limitations and the code + they affect. Such description must be placed in a text file included with all + distributions of the Covered Software under this License. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing basis, + if such Contributor fails to notify You of the non-compliance by some + reasonable means prior to 60 days after You have come back into compliance. + Moreover, Your grants from a particular Contributor are reinstated on an + ongoing basis if such Contributor notifies You of the non-compliance by + some reasonable means, this is the first time You have received notice of + non-compliance with this License from such Contributor, and You become + compliant prior to 30 days after Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, counter-claims, + and cross-claims) alleging that a Contributor Version directly or + indirectly infringes any patent, then the rights granted to You by any and + all Contributors for the Covered Software under Section 2.1 of this License + shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an “as is” basis, without + warranty of any kind, either expressed, implied, or statutory, including, + without limitation, warranties that the Covered Software is free of defects, + merchantable, fit for a particular purpose or non-infringing. The entire + risk as to the quality and performance of the Covered Software is with You. + Should any Covered Software prove defective in any respect, You (not any + Contributor) assume the cost of any necessary servicing, repair, or + correction. This disclaimer of warranty constitutes an essential part of this + License. No use of any Covered Software is authorized under this License + except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from such + party’s negligence to the extent applicable law prohibits such limitation. + Some jurisdictions do not allow the exclusion or limitation of incidental or + consequential damages, so this exclusion and limitation may not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts of + a jurisdiction where the defendant maintains its principal place of business + and such litigation shall be governed by laws of that jurisdiction, without + reference to its conflict-of-law provisions. Nothing in this Section shall + prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject matter + hereof. If any provision of this License is held to be unenforceable, such + provision shall be reformed only to the extent necessary to make it + enforceable. Any law or regulation which provides that the language of a + contract shall be construed against the drafter shall not be used to construe + this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version of + the License under which You originally received the Covered Software, or + under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + + This Source Code Form is “Incompatible + With Secondary Licenses”, as defined by + the Mozilla Public License, v. 2.0. + + +================================================================ + +github.com/hashicorp/go-hclog +https://github.com/hashicorp/go-hclog +---------------------------------------------------------------- +Copyright (c) 2017 HashiCorp, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/hashicorp/go-immutable-radix +https://github.com/hashicorp/go-immutable-radix +---------------------------------------------------------------- +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + + +================================================================ + +github.com/hashicorp/go-msgpack +https://github.com/hashicorp/go-msgpack +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2012-2015 Ugorji Nwoke. +All rights reserved. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/hashicorp/go-multierror +https://github.com/hashicorp/go-multierror +---------------------------------------------------------------- +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. “Contributor” + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. “Contributor Version” + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” + + means Covered Software of a particular Contributor. + +1.4. “Covered Software” + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. “Incompatible With Secondary Licenses” + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of version + 1.1 or earlier of the License, but not also under the terms of a + Secondary License. + +1.6. “Executable Form” + + means any form of the work other than Source Code Form. + +1.7. “Larger Work” + + means a work that combines Covered Software with other material, in a separate + file or files, that is not Covered Software. + +1.8. “License” + + means this document. + +1.9. “Licensable” + + means having the right to grant, to the maximum extent possible, whether at the + time of the initial grant or subsequently, any and all of the rights conveyed by + this License. + +1.10. “Modifications” + + means any of the following: + + a. any file in Source Code Form that results from an addition to, deletion + from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor + + means any patent claim(s), including without limitation, method, process, + and apparatus claims, in any patent Licensable by such Contributor that + would be infringed, but for the grant of the License, by the making, + using, selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. + +1.12. “Secondary License” + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. “Source Code Form” + + means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) + + means an individual or a legal entity exercising rights under this + License. For legal entities, “You” includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, “control” means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or as + part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its Contributions + or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution become + effective for each Contribution on the date the Contributor first distributes + such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under this + License. No additional rights or licenses will be implied from the distribution + or licensing of Covered Software under this License. Notwithstanding Section + 2.1(b) above, no patent license is granted by a Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party’s + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of its + Contributions. + + This License does not grant any rights in the trademarks, service marks, or + logos of any Contributor (except as may be necessary to comply with the + notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this License + (see Section 10.2) or under the terms of a Secondary License (if permitted + under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its Contributions + are its original creation(s) or it has sufficient rights to grant the + rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under applicable + copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under the + terms of this License. You must inform recipients that the Source Code Form + of the Covered Software is governed by the terms of this License, and how + they can obtain a copy of this License. You may not attempt to alter or + restrict the recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this License, + or sublicense it under different terms, provided that the license for + the Executable Form does not attempt to limit or alter the recipients’ + rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for the + Covered Software. If the Larger Work is a combination of Covered Software + with a work governed by one or more Secondary Licenses, and the Covered + Software is not Incompatible With Secondary Licenses, this License permits + You to additionally distribute such Covered Software under the terms of + such Secondary License(s), so that the recipient of the Larger Work may, at + their option, further distribute the Covered Software under the terms of + either this License or such Secondary License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices (including + copyright notices, patent notices, disclaimers of warranty, or limitations + of liability) contained within the Source Code Form of the Covered + Software, except that You may alter any license notices to the extent + required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on behalf + of any Contributor. You must make it absolutely clear that any such + warranty, support, indemnity, or liability obligation is offered by You + alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, judicial + order, or regulation then You must: (a) comply with the terms of this License + to the maximum extent possible; and (b) describe the limitations and the code + they affect. Such description must be placed in a text file included with all + distributions of the Covered Software under this License. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing basis, + if such Contributor fails to notify You of the non-compliance by some + reasonable means prior to 60 days after You have come back into compliance. + Moreover, Your grants from a particular Contributor are reinstated on an + ongoing basis if such Contributor notifies You of the non-compliance by + some reasonable means, this is the first time You have received notice of + non-compliance with this License from such Contributor, and You become + compliant prior to 30 days after Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, counter-claims, + and cross-claims) alleging that a Contributor Version directly or + indirectly infringes any patent, then the rights granted to You by any and + all Contributors for the Covered Software under Section 2.1 of this License + shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an “as is” basis, without + warranty of any kind, either expressed, implied, or statutory, including, + without limitation, warranties that the Covered Software is free of defects, + merchantable, fit for a particular purpose or non-infringing. The entire + risk as to the quality and performance of the Covered Software is with You. + Should any Covered Software prove defective in any respect, You (not any + Contributor) assume the cost of any necessary servicing, repair, or + correction. This disclaimer of warranty constitutes an essential part of this + License. No use of any Covered Software is authorized under this License + except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from such + party’s negligence to the extent applicable law prohibits such limitation. + Some jurisdictions do not allow the exclusion or limitation of incidental or + consequential damages, so this exclusion and limitation may not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts of + a jurisdiction where the defendant maintains its principal place of business + and such litigation shall be governed by laws of that jurisdiction, without + reference to its conflict-of-law provisions. Nothing in this Section shall + prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject matter + hereof. If any provision of this License is held to be unenforceable, such + provision shall be reformed only to the extent necessary to make it + enforceable. Any law or regulation which provides that the language of a + contract shall be construed against the drafter shall not be used to construe + this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version of + the License under which You originally received the Covered Software, or + under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + + This Source Code Form is “Incompatible + With Secondary Licenses”, as defined by + the Mozilla Public License, v. 2.0. + +================================================================ + +github.com/hashicorp/go-uuid +https://github.com/hashicorp/go-uuid +---------------------------------------------------------------- +Copyright © 2015-2022 HashiCorp, Inc. + +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + + +================================================================ + +github.com/hashicorp/golang-lru +https://github.com/hashicorp/golang-lru +---------------------------------------------------------------- +Copyright (c) 2014 HashiCorp, Inc. + +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + +================================================================ + +github.com/hashicorp/raft +https://github.com/hashicorp/raft +---------------------------------------------------------------- +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. “Contributor” + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. “Contributor Version” + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor’s Contribution. + +1.3. “Contribution” + + means Covered Software of a particular Contributor. + +1.4. “Covered Software” + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. “Incompatible With Secondary Licenses” + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of version + 1.1 or earlier of the License, but not also under the terms of a + Secondary License. + +1.6. “Executable Form” + + means any form of the work other than Source Code Form. + +1.7. “Larger Work” + + means a work that combines Covered Software with other material, in a separate + file or files, that is not Covered Software. + +1.8. “License” + + means this document. + +1.9. “Licensable” + + means having the right to grant, to the maximum extent possible, whether at the + time of the initial grant or subsequently, any and all of the rights conveyed by + this License. + +1.10. “Modifications” + + means any of the following: + + a. any file in Source Code Form that results from an addition to, deletion + from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. “Patent Claims” of a Contributor + + means any patent claim(s), including without limitation, method, process, + and apparatus claims, in any patent Licensable by such Contributor that + would be infringed, but for the grant of the License, by the making, + using, selling, offering for sale, having made, import, or transfer of + either its Contributions or its Contributor Version. + +1.12. “Secondary License” + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. “Source Code Form” + + means the form of the work preferred for making modifications. + +1.14. “You” (or “Your”) + + means an individual or a legal entity exercising rights under this + License. For legal entities, “You” includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, “control” means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or as + part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its Contributions + or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution become + effective for each Contribution on the date the Contributor first distributes + such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under this + License. No additional rights or licenses will be implied from the distribution + or licensing of Covered Software under this License. Notwithstanding Section + 2.1(b) above, no patent license is granted by a Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party’s + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of its + Contributions. + + This License does not grant any rights in the trademarks, service marks, or + logos of any Contributor (except as may be necessary to comply with the + notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this License + (see Section 10.2) or under the terms of a Secondary License (if permitted + under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its Contributions + are its original creation(s) or it has sufficient rights to grant the + rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under applicable + copyright doctrines of fair use, fair dealing, or other equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under the + terms of this License. You must inform recipients that the Source Code Form + of the Covered Software is governed by the terms of this License, and how + they can obtain a copy of this License. You may not attempt to alter or + restrict the recipients’ rights in the Source Code Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this License, + or sublicense it under different terms, provided that the license for + the Executable Form does not attempt to limit or alter the recipients’ + rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for the + Covered Software. If the Larger Work is a combination of Covered Software + with a work governed by one or more Secondary Licenses, and the Covered + Software is not Incompatible With Secondary Licenses, this License permits + You to additionally distribute such Covered Software under the terms of + such Secondary License(s), so that the recipient of the Larger Work may, at + their option, further distribute the Covered Software under the terms of + either this License or such Secondary License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices (including + copyright notices, patent notices, disclaimers of warranty, or limitations + of liability) contained within the Source Code Form of the Covered + Software, except that You may alter any license notices to the extent + required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on behalf + of any Contributor. You must make it absolutely clear that any such + warranty, support, indemnity, or liability obligation is offered by You + alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, judicial + order, or regulation then You must: (a) comply with the terms of this License + to the maximum extent possible; and (b) describe the limitations and the code + they affect. Such description must be placed in a text file included with all + distributions of the Covered Software under this License. Except to the + extent prohibited by statute or regulation, such description must be + sufficiently detailed for a recipient of ordinary skill to be able to + understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing basis, + if such Contributor fails to notify You of the non-compliance by some + reasonable means prior to 60 days after You have come back into compliance. + Moreover, Your grants from a particular Contributor are reinstated on an + ongoing basis if such Contributor notifies You of the non-compliance by + some reasonable means, this is the first time You have received notice of + non-compliance with this License from such Contributor, and You become + compliant prior to 30 days after Your receipt of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, counter-claims, + and cross-claims) alleging that a Contributor Version directly or + indirectly infringes any patent, then the rights granted to You by any and + all Contributors for the Covered Software under Section 2.1 of this License + shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an “as is” basis, without + warranty of any kind, either expressed, implied, or statutory, including, + without limitation, warranties that the Covered Software is free of defects, + merchantable, fit for a particular purpose or non-infringing. The entire + risk as to the quality and performance of the Covered Software is with You. + Should any Covered Software prove defective in any respect, You (not any + Contributor) assume the cost of any necessary servicing, repair, or + correction. This disclaimer of warranty constitutes an essential part of this + License. No use of any Covered Software is authorized under this License + except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from such + party’s negligence to the extent applicable law prohibits such limitation. + Some jurisdictions do not allow the exclusion or limitation of incidental or + consequential damages, so this exclusion and limitation may not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts of + a jurisdiction where the defendant maintains its principal place of business + and such litigation shall be governed by laws of that jurisdiction, without + reference to its conflict-of-law provisions. Nothing in this Section shall + prevent a party’s ability to bring cross-claims or counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject matter + hereof. If any provision of this License is held to be unenforceable, such + provision shall be reformed only to the extent necessary to make it + enforceable. Any law or regulation which provides that the language of a + contract shall be construed against the drafter shall not be used to construe + this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version of + the License under which You originally received the Covered Software, or + under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a modified + version of this License if you rename the license and remove any + references to the name of the license steward (except to note that such + modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary Licenses + If You choose to distribute Source Code Form that is Incompatible With + Secondary Licenses under the terms of this version of the License, the + notice described in Exhibit B of this License must be attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, then +You may include the notice in a location (such as a LICENSE file in a relevant +directory) where a recipient would be likely to look for such a notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - “Incompatible With Secondary Licenses” Notice + + This Source Code Form is “Incompatible + With Secondary Licenses”, as defined by + the Mozilla Public License, v. 2.0. + + +================================================================ + +github.com/inconshreveable/mousetrap +https://github.com/inconshreveable/mousetrap +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2022 Alan Shreve (@inconshreveable) + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/jcmturner/aescts/v2 +https://github.com/jcmturner/aescts/v2 +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/jcmturner/dnsutils/v2 +https://github.com/jcmturner/dnsutils/v2 +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/jcmturner/gofork +https://github.com/jcmturner/gofork +---------------------------------------------------------------- +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/jcmturner/goidentity/v6 +https://github.com/jcmturner/goidentity/v6 +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/jcmturner/gokrb5/v8 +https://github.com/jcmturner/gokrb5/v8 +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/jcmturner/rpc/v2 +https://github.com/jcmturner/rpc/v2 +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/jedib0t/go-pretty/v6 +https://github.com/jedib0t/go-pretty/v6 +---------------------------------------------------------------- +MIT License + +Copyright (c) 2018 jedib0t + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/jessevdk/go-flags +https://github.com/jessevdk/go-flags +---------------------------------------------------------------- +Copyright (c) 2012 Jesse van den Kieboom. All rights reserved. +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google Inc. nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/josharian/intern +https://github.com/josharian/intern +---------------------------------------------------------------- +MIT License + +Copyright (c) 2019 Josh Bleecher Snyder + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/json-iterator/go +https://github.com/json-iterator/go +---------------------------------------------------------------- +MIT License + +Copyright (c) 2016 json-iterator + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/juju/ratelimit +https://github.com/juju/ratelimit +---------------------------------------------------------------- +All files in this repository are licensed as follows. If you contribute +to this repository, it is assumed that you license your contribution +under the same license unless you state otherwise. + +All files Copyright (C) 2015 Canonical Ltd. unless otherwise specified in the file. + +This software is licensed under the LGPLv3, included below. + +As a special exception to the GNU Lesser General Public License version 3 +("LGPL3"), the copyright holders of this Library give you permission to +convey to a third party a Combined Work that links statically or dynamically +to this Library without providing any Minimal Corresponding Source or +Minimal Application Code as set out in 4d or providing the installation +information set out in section 4e, provided that you comply with the other +provisions of LGPL3 and provided that you meet, for the Application the +terms and conditions of the license(s) which apply to the Application. + +Except as stated in this special exception, the provisions of LGPL3 will +continue to comply in full to this Library. If you modify this Library, you +may apply this exception to your version of this Library, but you are not +obliged to do so. If you do not wish to do so, delete this exception +statement from your version. This exception does not (and cannot) modify any +license terms which apply to the Application, with which you must still +comply. + + + GNU LESSER GENERAL PUBLIC LICENSE + Version 3, 29 June 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + + This version of the GNU Lesser General Public License incorporates +the terms and conditions of version 3 of the GNU General Public +License, supplemented by the additional permissions listed below. + + 0. Additional Definitions. + + As used herein, "this License" refers to version 3 of the GNU Lesser +General Public License, and the "GNU GPL" refers to version 3 of the GNU +General Public License. + + "The Library" refers to a covered work governed by this License, +other than an Application or a Combined Work as defined below. + + An "Application" is any work that makes use of an interface provided +by the Library, but which is not otherwise based on the Library. +Defining a subclass of a class defined by the Library is deemed a mode +of using an interface provided by the Library. + + A "Combined Work" is a work produced by combining or linking an +Application with the Library. The particular version of the Library +with which the Combined Work was made is also called the "Linked +Version". + + The "Minimal Corresponding Source" for a Combined Work means the +Corresponding Source for the Combined Work, excluding any source code +for portions of the Combined Work that, considered in isolation, are +based on the Application, and not on the Linked Version. + + The "Corresponding Application Code" for a Combined Work means the +object code and/or source code for the Application, including any data +and utility programs needed for reproducing the Combined Work from the +Application, but excluding the System Libraries of the Combined Work. + + 1. Exception to Section 3 of the GNU GPL. + + You may convey a covered work under sections 3 and 4 of this License +without being bound by section 3 of the GNU GPL. + + 2. Conveying Modified Versions. + + If you modify a copy of the Library, and, in your modifications, a +facility refers to a function or data to be supplied by an Application +that uses the facility (other than as an argument passed when the +facility is invoked), then you may convey a copy of the modified +version: + + a) under this License, provided that you make a good faith effort to + ensure that, in the event an Application does not supply the + function or data, the facility still operates, and performs + whatever part of its purpose remains meaningful, or + + b) under the GNU GPL, with none of the additional permissions of + this License applicable to that copy. + + 3. Object Code Incorporating Material from Library Header Files. + + The object code form of an Application may incorporate material from +a header file that is part of the Library. You may convey such object +code under terms of your choice, provided that, if the incorporated +material is not limited to numerical parameters, data structure +layouts and accessors, or small macros, inline functions and templates +(ten or fewer lines in length), you do both of the following: + + a) Give prominent notice with each copy of the object code that the + Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the object code with a copy of the GNU GPL and this license + document. + + 4. Combined Works. + + You may convey a Combined Work under terms of your choice that, +taken together, effectively do not restrict modification of the +portions of the Library contained in the Combined Work and reverse +engineering for debugging such modifications, if you also do each of +the following: + + a) Give prominent notice with each copy of the Combined Work that + the Library is used in it and that the Library and its use are + covered by this License. + + b) Accompany the Combined Work with a copy of the GNU GPL and this license + document. + + c) For a Combined Work that displays copyright notices during + execution, include the copyright notice for the Library among + these notices, as well as a reference directing the user to the + copies of the GNU GPL and this license document. + + d) Do one of the following: + + 0) Convey the Minimal Corresponding Source under the terms of this + License, and the Corresponding Application Code in a form + suitable for, and under terms that permit, the user to + recombine or relink the Application with a modified version of + the Linked Version to produce a modified Combined Work, in the + manner specified by section 6 of the GNU GPL for conveying + Corresponding Source. + + 1) Use a suitable shared library mechanism for linking with the + Library. A suitable mechanism is one that (a) uses at run time + a copy of the Library already present on the user's computer + system, and (b) will operate properly with a modified version + of the Library that is interface-compatible with the Linked + Version. + + e) Provide Installation Information, but only if you would otherwise + be required to provide such information under section 6 of the + GNU GPL, and only to the extent that such information is + necessary to install and execute a modified version of the + Combined Work produced by recombining or relinking the + Application with a modified version of the Linked Version. (If + you use option 4d0, the Installation Information must accompany + the Minimal Corresponding Source and Corresponding Application + Code. If you use option 4d1, you must provide the Installation + Information in the manner specified by section 6 of the GNU GPL + for conveying Corresponding Source.) + + 5. Combined Libraries. + + You may place library facilities that are a work based on the +Library side by side in a single library together with other library +facilities that are not Applications and are not covered by this +License, and convey such a combined library under terms of your +choice, if you do both of the following: + + a) Accompany the combined library with a copy of the same work based + on the Library, uncombined with any other library facilities, + conveyed under the terms of this License. + + b) Give prominent notice with the combined library that part of it + is a work based on the Library, and explaining where to find the + accompanying uncombined form of the same work. + + 6. Revised Versions of the GNU Lesser General Public License. + + The Free Software Foundation may publish revised and/or new versions +of the GNU Lesser General Public License from time to time. Such new +versions will be similar in spirit to the present version, but may +differ in detail to address new problems or concerns. + + Each version is given a distinguishing version number. If the +Library as you received it specifies that a certain numbered version +of the GNU Lesser General Public License "or any later version" +applies to it, you have the option of following the terms and +conditions either of that published version or of any later version +published by the Free Software Foundation. If the Library as you +received it does not specify a version number of the GNU Lesser +General Public License, you may choose any version of the GNU Lesser +General Public License ever published by the Free Software Foundation. + + If the Library as you received it specifies that a proxy can decide +whether future versions of the GNU Lesser General Public License shall +apply, that proxy's public statement of acceptance of any version is +permanent authorization for you to choose that version for the +Library. + +================================================================ + +github.com/keybase/go-keychain +https://github.com/keybase/go-keychain +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2015 Keybase + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +================================================================ + +github.com/klauspost/compress +https://github.com/klauspost/compress +---------------------------------------------------------------- +Copyright (c) 2012 The Go Authors. All rights reserved. +Copyright (c) 2019 Klaus Post. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +------------------ + +Files: gzhttp/* + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016-2017 The New York Times Company + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +------------------ + +Files: s2/cmd/internal/readahead/* + +The MIT License (MIT) + +Copyright (c) 2015 Klaus Post + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +--------------------- +Files: snappy/* +Files: internal/snapref/* + +Copyright (c) 2011 The Snappy-Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +----------------- + +Files: s2/cmd/internal/filepathx/* + +Copyright 2016 The filepathx Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================ + +github.com/klauspost/cpuid/v2 +https://github.com/klauspost/cpuid/v2 +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2015 Klaus Post + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +================================================================ + +github.com/klauspost/filepathx +https://github.com/klauspost/filepathx +---------------------------------------------------------------- +Copyright 2016 The filepathx Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================ + +github.com/klauspost/pgzip +https://github.com/klauspost/pgzip +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2014 Klaus Post + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +================================================================ + +github.com/klauspost/readahead +https://github.com/klauspost/readahead +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2015 Klaus Post + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +================================================================ + +github.com/klauspost/reedsolomon +https://github.com/klauspost/reedsolomon +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2015 Klaus Post +Copyright (c) 2015 Backblaze + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +================================================================ + +github.com/kr/fs +https://github.com/kr/fs +---------------------------------------------------------------- +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/kr/pretty +https://github.com/kr/pretty +---------------------------------------------------------------- +Copyright 2012 Keith Rarick + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +================================================================ + +github.com/kr/text +https://github.com/kr/text +---------------------------------------------------------------- +Copyright 2012 Keith Rarick + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +================================================================ + +github.com/kylelemons/godebug +https://github.com/kylelemons/godebug +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/lestrrat-go/blackmagic +https://github.com/lestrrat-go/blackmagic +---------------------------------------------------------------- +MIT License + +Copyright (c) 2021 lestrrat-go + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/lestrrat-go/httpcc +https://github.com/lestrrat-go/httpcc +---------------------------------------------------------------- +MIT License + +Copyright (c) 2020 lestrrat-go + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/lestrrat-go/httprc +https://github.com/lestrrat-go/httprc +---------------------------------------------------------------- +MIT License + +Copyright (c) 2022 lestrrat + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/lestrrat-go/iter +https://github.com/lestrrat-go/iter +---------------------------------------------------------------- +MIT License + +Copyright (c) 2020 lestrrat-go + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/lestrrat-go/jwx/v2 +https://github.com/lestrrat-go/jwx/v2 +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2015 lestrrat + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +================================================================ + +github.com/lestrrat-go/option +https://github.com/lestrrat-go/option +---------------------------------------------------------------- +MIT License + +Copyright (c) 2021 lestrrat-go + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/lib/pq +https://github.com/lib/pq +---------------------------------------------------------------- +Copyright (c) 2011-2013, 'pq' Contributors +Portions Copyright (C) 2011 Blake Mizerany + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================ + +github.com/lithammer/shortuuid/v4 +https://github.com/lithammer/shortuuid/v4 +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2018 Peter Lithammer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/lucasb-eyer/go-colorful +https://github.com/lucasb-eyer/go-colorful +---------------------------------------------------------------- +Copyright (c) 2013 Lucas Beyer + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================ + +github.com/lufia/plan9stats +https://github.com/lufia/plan9stats +---------------------------------------------------------------- +BSD 3-Clause License + +Copyright (c) 2019, KADOTA, Kyohei +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/mailru/easyjson +https://github.com/mailru/easyjson +---------------------------------------------------------------- +Copyright (c) 2016 Mail.Ru Group + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================ + +github.com/mattn/go-colorable +https://github.com/mattn/go-colorable +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2016 Yasuhiro Matsumoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/mattn/go-ieproxy +https://github.com/mattn/go-ieproxy +---------------------------------------------------------------- +MIT License + +Copyright (c) 2014 mattn +Copyright (c) 2017 oliverpool +Copyright (c) 2019 Adele Reed + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/mattn/go-isatty +https://github.com/mattn/go-isatty +---------------------------------------------------------------- +Copyright (c) Yasuhiro MATSUMOTO + +MIT License (Expat) + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================ + +github.com/mattn/go-runewidth +https://github.com/mattn/go-runewidth +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2016 Yasuhiro Matsumoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/matttproud/golang_protobuf_extensions +https://github.com/matttproud/golang_protobuf_extensions +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/miekg/dns +https://github.com/miekg/dns +---------------------------------------------------------------- +BSD 3-Clause License + +Copyright (c) 2009, The Go Authors. Extensions copyright (c) 2011, Miek Gieben. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +3. Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/minio/cli +https://github.com/minio/cli +---------------------------------------------------------------- +MIT License + +Copyright (c) 2016 Jeremy Saenz & Contributors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/minio/console +https://github.com/minio/console +---------------------------------------------------------------- + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. + +================================================================ + +github.com/minio/crc64nvme +https://github.com/minio/crc64nvme +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/minio/dnscache +https://github.com/minio/dnscache +---------------------------------------------------------------- +MIT License + +Copyright (c) 2023 MinIO, Inc. +Copyright (c) 2018 Olivier Poitrey + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/minio/dperf +https://github.com/minio/dperf +---------------------------------------------------------------- + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. + +================================================================ + +github.com/minio/highwayhash +https://github.com/minio/highwayhash +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/minio/kms-go/kes +https://github.com/minio/kms-go/kes +---------------------------------------------------------------- + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. + +================================================================ + +github.com/minio/kms-go/kms +https://github.com/minio/kms-go/kms +---------------------------------------------------------------- + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published + by the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. + +================================================================ + +github.com/minio/madmin-go/v3 +https://github.com/minio/madmin-go/v3 +---------------------------------------------------------------- + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. + +================================================================ + +github.com/minio/mc +https://github.com/minio/mc +---------------------------------------------------------------- + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. + +================================================================ + +github.com/minio/md5-simd +https://github.com/minio/md5-simd +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/minio/minio-go/v7 +https://github.com/minio/minio-go/v7 +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/minio/mux +https://github.com/minio/mux +---------------------------------------------------------------- +Copyright (c) 2023 The Gorilla Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/minio/pkg/v3 +https://github.com/minio/pkg/v3 +---------------------------------------------------------------- + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. + +================================================================ + +github.com/minio/selfupdate +https://github.com/minio/selfupdate +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/minio/simdjson-go +https://github.com/minio/simdjson-go +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/minio/sio +https://github.com/minio/sio +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/minio/websocket +https://github.com/minio/websocket +---------------------------------------------------------------- +Copyright (c) 2013 The Gorilla WebSocket Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/minio/xxml +https://github.com/minio/xxml +---------------------------------------------------------------- +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/minio/zipindex +https://github.com/minio/zipindex +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/mitchellh/go-homedir +https://github.com/mitchellh/go-homedir +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2013 Mitchell Hashimoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +================================================================ + +github.com/mitchellh/mapstructure +https://github.com/mitchellh/mapstructure +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2013 Mitchell Hashimoto + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +================================================================ + +github.com/modern-go/concurrent +https://github.com/modern-go/concurrent +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/modern-go/reflect2 +https://github.com/modern-go/reflect2 +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/muesli/ansi +https://github.com/muesli/ansi +---------------------------------------------------------------- +MIT License + +Copyright (c) 2021 Christian Muehlhaeuser + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/muesli/cancelreader +https://github.com/muesli/cancelreader +---------------------------------------------------------------- +MIT License + +Copyright (c) 2022 Erik Geiser and Christian Muehlhaeuser + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/muesli/reflow +https://github.com/muesli/reflow +---------------------------------------------------------------- +MIT License + +Copyright (c) 2019 Christian Muehlhaeuser + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/muesli/termenv +https://github.com/muesli/termenv +---------------------------------------------------------------- +MIT License + +Copyright (c) 2019 Christian Muehlhaeuser + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/munnerz/goautoneg +https://github.com/munnerz/goautoneg +---------------------------------------------------------------- +Copyright (c) 2011, Open Knowledge Foundation Ltd. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + Neither the name of the Open Knowledge Foundation Ltd. nor the + names of its contributors may be used to endorse or promote + products derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/nats-io/jwt/v2 +https://github.com/nats-io/jwt/v2 +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/nats-io/nats-server/v2 +https://github.com/nats-io/nats-server/v2 +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/nats-io/nats-streaming-server +https://github.com/nats-io/nats-streaming-server +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +================================================================ + +github.com/nats-io/nats.go +https://github.com/nats-io/nats.go +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/nats-io/nkeys +https://github.com/nats-io/nkeys +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/nats-io/nuid +https://github.com/nats-io/nuid +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/nats-io/stan.go +https://github.com/nats-io/stan.go +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +================================================================ + +github.com/ncw/directio +https://github.com/ncw/directio +---------------------------------------------------------------- +Copyright (C) 2012 by Nick Craig-Wood http://www.craig-wood.com/nick/ + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + + +================================================================ + +github.com/nsqio/go-nsq +https://github.com/nsqio/go-nsq +---------------------------------------------------------------- +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +================================================================ + +github.com/oklog/ulid +https://github.com/oklog/ulid +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/olekukonko/tablewriter +https://github.com/olekukonko/tablewriter +---------------------------------------------------------------- +Copyright (C) 2014 by Oleku Konko + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +================================================================ + +github.com/philhofer/fwd +https://github.com/philhofer/fwd +---------------------------------------------------------------- +Copyright (c) 2014-2015, Philip Hofer + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +================================================================ + +github.com/pierrec/lz4/v4 +https://github.com/pierrec/lz4/v4 +---------------------------------------------------------------- +Copyright (c) 2015, Pierre Curto +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of xxHash nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +================================================================ + +github.com/pkg/browser +https://github.com/pkg/browser +---------------------------------------------------------------- +Copyright (c) 2014, Dave Cheney +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/pkg/errors +https://github.com/pkg/errors +---------------------------------------------------------------- +Copyright (c) 2015, Dave Cheney +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/pkg/sftp +https://github.com/pkg/sftp +---------------------------------------------------------------- +Copyright (c) 2013, Dave Cheney +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/pkg/xattr +https://github.com/pkg/xattr +---------------------------------------------------------------- +Copyright (c) 2012 Dave Cheney. All rights reserved. +Copyright (c) 2014 Kuba Podgórski. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/planetscale/vtprotobuf +https://github.com/planetscale/vtprotobuf +---------------------------------------------------------------- +Copyright (c) 2021, PlanetScale Inc. All rights reserved. +Copyright (c) 2013, The GoGo Authors. All rights reserved. +Copyright (c) 2018 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/pmezard/go-difflib +https://github.com/pmezard/go-difflib +---------------------------------------------------------------- +Copyright (c) 2013, Patrick Mezard +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + The names of its contributors may not be used to endorse or promote +products derived from this software without specific prior written +permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED +TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED +TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/posener/complete +https://github.com/posener/complete +---------------------------------------------------------------- +The MIT License + +Copyright (c) 2017 Eyal Posener + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. +================================================================ + +github.com/power-devops/perfstat +https://github.com/power-devops/perfstat +---------------------------------------------------------------- +MIT License + +Copyright (c) 2020 Power DevOps + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + + +================================================================ + +github.com/prometheus/client_golang +https://github.com/prometheus/client_golang +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/prometheus/client_model +https://github.com/prometheus/client_model +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/prometheus/common +https://github.com/prometheus/common +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/prometheus/procfs +https://github.com/prometheus/procfs +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/prometheus/prom2json +https://github.com/prometheus/prom2json +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/prometheus/prometheus +https://github.com/prometheus/prometheus +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/puzpuzpuz/xsync/v3 +https://github.com/puzpuzpuz/xsync/v3 +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/rabbitmq/amqp091-go +https://github.com/rabbitmq/amqp091-go +---------------------------------------------------------------- +AMQP 0-9-1 Go Client +Copyright (c) 2021 VMware, Inc. or its affiliates. All Rights Reserved. + +Copyright (c) 2012-2021, Sean Treadway, SoundCloud Ltd. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +Redistributions of source code must retain the above copyright notice, this +list of conditions and the following disclaimer. + +Redistributions in binary form must reproduce the above copyright notice, this +list of conditions and the following disclaimer in the documentation and/or +other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/rcrowley/go-metrics +https://github.com/rcrowley/go-metrics +---------------------------------------------------------------- +Copyright 2012 Richard Crowley. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + 1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + 2. Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + +THIS SOFTWARE IS PROVIDED BY RICHARD CROWLEY ``AS IS'' AND ANY EXPRESS +OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL RICHARD CROWLEY OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF +THE POSSIBILITY OF SUCH DAMAGE. + +The views and conclusions contained in the software and documentation +are those of the authors and should not be interpreted as representing +official policies, either expressed or implied, of Richard Crowley. + +================================================================ + +github.com/redis/go-redis/v9 +https://github.com/redis/go-redis/v9 +---------------------------------------------------------------- +Copyright (c) 2013 The github.com/redis/go-redis Authors. +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/rivo/uniseg +https://github.com/rivo/uniseg +---------------------------------------------------------------- +MIT License + +Copyright (c) 2019 Oliver Kuederle + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/rjeczalik/notify +https://github.com/rjeczalik/notify +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2014-2015 The Notify Authors + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/rogpeppe/go-internal +https://github.com/rogpeppe/go-internal +---------------------------------------------------------------- +Copyright (c) 2018 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/rs/cors +https://github.com/rs/cors +---------------------------------------------------------------- +Copyright (c) 2014 Olivier Poitrey + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is furnished +to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +================================================================ + +github.com/rs/xid +https://github.com/rs/xid +---------------------------------------------------------------- +Copyright (c) 2015 Olivier Poitrey + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is furnished +to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +================================================================ + +github.com/safchain/ethtool +https://github.com/safchain/ethtool +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright (c) 2015 The Ethtool Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +================================================================ + +github.com/secure-io/sio-go +https://github.com/secure-io/sio-go +---------------------------------------------------------------- +MIT License + +Copyright (c) 2019 SecureIO + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/segmentio/asm +https://github.com/segmentio/asm +---------------------------------------------------------------- +MIT License + +Copyright (c) 2021 Segment + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/shirou/gopsutil/v3 +https://github.com/shirou/gopsutil/v3 +---------------------------------------------------------------- +gopsutil is distributed under BSD license reproduced below. + +Copyright (c) 2014, WAKAYAMA Shirou +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + + * Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + * Neither the name of the gopsutil authors nor the names of its contributors + may be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +------- +internal/common/binary.go in the gopsutil is copied and modified from golang/encoding/binary.go. + + + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +================================================================ + +github.com/shoenig/go-m1cpu +https://github.com/shoenig/go-m1cpu +---------------------------------------------------------------- +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + + +================================================================ + +github.com/shoenig/test +https://github.com/shoenig/test +---------------------------------------------------------------- +Mozilla Public License, version 2.0 + +1. Definitions + +1.1. "Contributor" + + means each individual or legal entity that creates, contributes to the + creation of, or owns Covered Software. + +1.2. "Contributor Version" + + means the combination of the Contributions of others (if any) used by a + Contributor and that particular Contributor's Contribution. + +1.3. "Contribution" + + means Covered Software of a particular Contributor. + +1.4. "Covered Software" + + means Source Code Form to which the initial Contributor has attached the + notice in Exhibit A, the Executable Form of such Source Code Form, and + Modifications of such Source Code Form, in each case including portions + thereof. + +1.5. "Incompatible With Secondary Licenses" + means + + a. that the initial Contributor has attached the notice described in + Exhibit B to the Covered Software; or + + b. that the Covered Software was made available under the terms of + version 1.1 or earlier of the License, but not also under the terms of + a Secondary License. + +1.6. "Executable Form" + + means any form of the work other than Source Code Form. + +1.7. "Larger Work" + + means a work that combines Covered Software with other material, in a + separate file or files, that is not Covered Software. + +1.8. "License" + + means this document. + +1.9. "Licensable" + + means having the right to grant, to the maximum extent possible, whether + at the time of the initial grant or subsequently, any and all of the + rights conveyed by this License. + +1.10. "Modifications" + + means any of the following: + + a. any file in Source Code Form that results from an addition to, + deletion from, or modification of the contents of Covered Software; or + + b. any new file in Source Code Form that contains any Covered Software. + +1.11. "Patent Claims" of a Contributor + + means any patent claim(s), including without limitation, method, + process, and apparatus claims, in any patent Licensable by such + Contributor that would be infringed, but for the grant of the License, + by the making, using, selling, offering for sale, having made, import, + or transfer of either its Contributions or its Contributor Version. + +1.12. "Secondary License" + + means either the GNU General Public License, Version 2.0, the GNU Lesser + General Public License, Version 2.1, the GNU Affero General Public + License, Version 3.0, or any later versions of those licenses. + +1.13. "Source Code Form" + + means the form of the work preferred for making modifications. + +1.14. "You" (or "Your") + + means an individual or a legal entity exercising rights under this + License. For legal entities, "You" includes any entity that controls, is + controlled by, or is under common control with You. For purposes of this + definition, "control" means (a) the power, direct or indirect, to cause + the direction or management of such entity, whether by contract or + otherwise, or (b) ownership of more than fifty percent (50%) of the + outstanding shares or beneficial ownership of such entity. + + +2. License Grants and Conditions + +2.1. Grants + + Each Contributor hereby grants You a world-wide, royalty-free, + non-exclusive license: + + a. under intellectual property rights (other than patent or trademark) + Licensable by such Contributor to use, reproduce, make available, + modify, display, perform, distribute, and otherwise exploit its + Contributions, either on an unmodified basis, with Modifications, or + as part of a Larger Work; and + + b. under Patent Claims of such Contributor to make, use, sell, offer for + sale, have made, import, and otherwise transfer either its + Contributions or its Contributor Version. + +2.2. Effective Date + + The licenses granted in Section 2.1 with respect to any Contribution + become effective for each Contribution on the date the Contributor first + distributes such Contribution. + +2.3. Limitations on Grant Scope + + The licenses granted in this Section 2 are the only rights granted under + this License. No additional rights or licenses will be implied from the + distribution or licensing of Covered Software under this License. + Notwithstanding Section 2.1(b) above, no patent license is granted by a + Contributor: + + a. for any code that a Contributor has removed from Covered Software; or + + b. for infringements caused by: (i) Your and any other third party's + modifications of Covered Software, or (ii) the combination of its + Contributions with other software (except as part of its Contributor + Version); or + + c. under Patent Claims infringed by Covered Software in the absence of + its Contributions. + + This License does not grant any rights in the trademarks, service marks, + or logos of any Contributor (except as may be necessary to comply with + the notice requirements in Section 3.4). + +2.4. Subsequent Licenses + + No Contributor makes additional grants as a result of Your choice to + distribute the Covered Software under a subsequent version of this + License (see Section 10.2) or under the terms of a Secondary License (if + permitted under the terms of Section 3.3). + +2.5. Representation + + Each Contributor represents that the Contributor believes its + Contributions are its original creation(s) or it has sufficient rights to + grant the rights to its Contributions conveyed by this License. + +2.6. Fair Use + + This License is not intended to limit any rights You have under + applicable copyright doctrines of fair use, fair dealing, or other + equivalents. + +2.7. Conditions + + Sections 3.1, 3.2, 3.3, and 3.4 are conditions of the licenses granted in + Section 2.1. + + +3. Responsibilities + +3.1. Distribution of Source Form + + All distribution of Covered Software in Source Code Form, including any + Modifications that You create or to which You contribute, must be under + the terms of this License. You must inform recipients that the Source + Code Form of the Covered Software is governed by the terms of this + License, and how they can obtain a copy of this License. You may not + attempt to alter or restrict the recipients' rights in the Source Code + Form. + +3.2. Distribution of Executable Form + + If You distribute Covered Software in Executable Form then: + + a. such Covered Software must also be made available in Source Code Form, + as described in Section 3.1, and You must inform recipients of the + Executable Form how they can obtain a copy of such Source Code Form by + reasonable means in a timely manner, at a charge no more than the cost + of distribution to the recipient; and + + b. You may distribute such Executable Form under the terms of this + License, or sublicense it under different terms, provided that the + license for the Executable Form does not attempt to limit or alter the + recipients' rights in the Source Code Form under this License. + +3.3. Distribution of a Larger Work + + You may create and distribute a Larger Work under terms of Your choice, + provided that You also comply with the requirements of this License for + the Covered Software. If the Larger Work is a combination of Covered + Software with a work governed by one or more Secondary Licenses, and the + Covered Software is not Incompatible With Secondary Licenses, this + License permits You to additionally distribute such Covered Software + under the terms of such Secondary License(s), so that the recipient of + the Larger Work may, at their option, further distribute the Covered + Software under the terms of either this License or such Secondary + License(s). + +3.4. Notices + + You may not remove or alter the substance of any license notices + (including copyright notices, patent notices, disclaimers of warranty, or + limitations of liability) contained within the Source Code Form of the + Covered Software, except that You may alter any license notices to the + extent required to remedy known factual inaccuracies. + +3.5. Application of Additional Terms + + You may choose to offer, and to charge a fee for, warranty, support, + indemnity or liability obligations to one or more recipients of Covered + Software. However, You may do so only on Your own behalf, and not on + behalf of any Contributor. You must make it absolutely clear that any + such warranty, support, indemnity, or liability obligation is offered by + You alone, and You hereby agree to indemnify every Contributor for any + liability incurred by such Contributor as a result of warranty, support, + indemnity or liability terms You offer. You may include additional + disclaimers of warranty and limitations of liability specific to any + jurisdiction. + +4. Inability to Comply Due to Statute or Regulation + + If it is impossible for You to comply with any of the terms of this License + with respect to some or all of the Covered Software due to statute, + judicial order, or regulation then You must: (a) comply with the terms of + this License to the maximum extent possible; and (b) describe the + limitations and the code they affect. Such description must be placed in a + text file included with all distributions of the Covered Software under + this License. Except to the extent prohibited by statute or regulation, + such description must be sufficiently detailed for a recipient of ordinary + skill to be able to understand it. + +5. Termination + +5.1. The rights granted under this License will terminate automatically if You + fail to comply with any of its terms. However, if You become compliant, + then the rights granted under this License from a particular Contributor + are reinstated (a) provisionally, unless and until such Contributor + explicitly and finally terminates Your grants, and (b) on an ongoing + basis, if such Contributor fails to notify You of the non-compliance by + some reasonable means prior to 60 days after You have come back into + compliance. Moreover, Your grants from a particular Contributor are + reinstated on an ongoing basis if such Contributor notifies You of the + non-compliance by some reasonable means, this is the first time You have + received notice of non-compliance with this License from such + Contributor, and You become compliant prior to 30 days after Your receipt + of the notice. + +5.2. If You initiate litigation against any entity by asserting a patent + infringement claim (excluding declaratory judgment actions, + counter-claims, and cross-claims) alleging that a Contributor Version + directly or indirectly infringes any patent, then the rights granted to + You by any and all Contributors for the Covered Software under Section + 2.1 of this License shall terminate. + +5.3. In the event of termination under Sections 5.1 or 5.2 above, all end user + license agreements (excluding distributors and resellers) which have been + validly granted by You or Your distributors under this License prior to + termination shall survive termination. + +6. Disclaimer of Warranty + + Covered Software is provided under this License on an "as is" basis, + without warranty of any kind, either expressed, implied, or statutory, + including, without limitation, warranties that the Covered Software is free + of defects, merchantable, fit for a particular purpose or non-infringing. + The entire risk as to the quality and performance of the Covered Software + is with You. Should any Covered Software prove defective in any respect, + You (not any Contributor) assume the cost of any necessary servicing, + repair, or correction. This disclaimer of warranty constitutes an essential + part of this License. No use of any Covered Software is authorized under + this License except under this disclaimer. + +7. Limitation of Liability + + Under no circumstances and under no legal theory, whether tort (including + negligence), contract, or otherwise, shall any Contributor, or anyone who + distributes Covered Software as permitted above, be liable to You for any + direct, indirect, special, incidental, or consequential damages of any + character including, without limitation, damages for lost profits, loss of + goodwill, work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses, even if such party shall have been + informed of the possibility of such damages. This limitation of liability + shall not apply to liability for death or personal injury resulting from + such party's negligence to the extent applicable law prohibits such + limitation. Some jurisdictions do not allow the exclusion or limitation of + incidental or consequential damages, so this exclusion and limitation may + not apply to You. + +8. Litigation + + Any litigation relating to this License may be brought only in the courts + of a jurisdiction where the defendant maintains its principal place of + business and such litigation shall be governed by laws of that + jurisdiction, without reference to its conflict-of-law provisions. Nothing + in this Section shall prevent a party's ability to bring cross-claims or + counter-claims. + +9. Miscellaneous + + This License represents the complete agreement concerning the subject + matter hereof. If any provision of this License is held to be + unenforceable, such provision shall be reformed only to the extent + necessary to make it enforceable. Any law or regulation which provides that + the language of a contract shall be construed against the drafter shall not + be used to construe this License against a Contributor. + + +10. Versions of the License + +10.1. New Versions + + Mozilla Foundation is the license steward. Except as provided in Section + 10.3, no one other than the license steward has the right to modify or + publish new versions of this License. Each version will be given a + distinguishing version number. + +10.2. Effect of New Versions + + You may distribute the Covered Software under the terms of the version + of the License under which You originally received the Covered Software, + or under the terms of any subsequent version published by the license + steward. + +10.3. Modified Versions + + If you create software not governed by this License, and you want to + create a new license for such software, you may create and use a + modified version of this License if you rename the license and remove + any references to the name of the license steward (except to note that + such modified license differs from this License). + +10.4. Distributing Source Code Form that is Incompatible With Secondary + Licenses If You choose to distribute Source Code Form that is + Incompatible With Secondary Licenses under the terms of this version of + the License, the notice described in Exhibit B of this License must be + attached. + +Exhibit A - Source Code Form License Notice + + This Source Code Form is subject to the + terms of the Mozilla Public License, v. + 2.0. If a copy of the MPL was not + distributed with this file, You can + obtain one at + http://mozilla.org/MPL/2.0/. + +If it is not possible or desirable to put the notice in a particular file, +then You may include the notice in a location (such as a LICENSE file in a +relevant directory) where a recipient would be likely to look for such a +notice. + +You may add additional accurate notices of copyright ownership. + +Exhibit B - "Incompatible With Secondary Licenses" Notice + + This Source Code Form is "Incompatible + With Secondary Licenses", as defined by + the Mozilla Public License, v. 2.0. + + +================================================================ + +github.com/stretchr/testify +https://github.com/stretchr/testify +---------------------------------------------------------------- +MIT License + +Copyright (c) 2012-2020 Mat Ryer, Tyler Bunnell and contributors. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +================================================================ + +github.com/tidwall/gjson +https://github.com/tidwall/gjson +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2016 Josh Baker + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================ + +github.com/tidwall/match +https://github.com/tidwall/match +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2016 Josh Baker + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================ + +github.com/tidwall/pretty +https://github.com/tidwall/pretty +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2017 Josh Baker + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================ + +github.com/tinylib/msgp +https://github.com/tinylib/msgp +---------------------------------------------------------------- +Copyright (c) 2014 Philip Hofer +Portions Copyright (c) 2009 The Go Authors (license at http://golang.org) where indicated + +Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +================================================================ + +github.com/tklauser/go-sysconf +https://github.com/tklauser/go-sysconf +---------------------------------------------------------------- +BSD 3-Clause License + +Copyright (c) 2018-2022, Tobias Klauser +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +* Neither the name of the copyright holder nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +github.com/tklauser/numcpus +https://github.com/tklauser/numcpus +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +github.com/unrolled/secure +https://github.com/unrolled/secure +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2014 Cory Jacobsen + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================ + +github.com/valyala/bytebufferpool +https://github.com/valyala/bytebufferpool +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2016 Aliaksandr Valialkin, VertaMedia + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +================================================================ + +github.com/vbauerster/mpb/v8 +https://github.com/vbauerster/mpb/v8 +---------------------------------------------------------------- +This is free and unencumbered software released into the public domain. + +Anyone is free to copy, modify, publish, use, compile, sell, or +distribute this software, either in source code form or as a compiled +binary, for any purpose, commercial or non-commercial, and by any +means. + +In jurisdictions that recognize copyright laws, the author or authors +of this software dedicate any and all copyright interest in the +software to the public domain. We make this dedication for the benefit +of the public at large and to the detriment of our heirs and +successors. We intend this dedication to be an overt act of +relinquishment in perpetuity of all present and future rights to this +software under copyright law. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR +OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR +OTHER DEALINGS IN THE SOFTWARE. + +For more information, please refer to + +================================================================ + +github.com/xdg/scram +https://github.com/xdg/scram +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +================================================================ + +github.com/xdg/stringprep +https://github.com/xdg/stringprep +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + +================================================================ + +github.com/yusufpapurcu/wmi +https://github.com/yusufpapurcu/wmi +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2013 Stack Exchange + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================ + +github.com/zeebo/assert +https://github.com/zeebo/assert +---------------------------------------------------------------- +Creative Commons Legal Code + +CC0 1.0 Universal + + CREATIVE COMMONS CORPORATION IS NOT A LAW FIRM AND DOES NOT PROVIDE + LEGAL SERVICES. DISTRIBUTION OF THIS DOCUMENT DOES NOT CREATE AN + ATTORNEY-CLIENT RELATIONSHIP. CREATIVE COMMONS PROVIDES THIS + INFORMATION ON AN "AS-IS" BASIS. CREATIVE COMMONS MAKES NO WARRANTIES + REGARDING THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS + PROVIDED HEREUNDER, AND DISCLAIMS LIABILITY FOR DAMAGES RESULTING FROM + THE USE OF THIS DOCUMENT OR THE INFORMATION OR WORKS PROVIDED + HEREUNDER. + +Statement of Purpose + +The laws of most jurisdictions throughout the world automatically confer +exclusive Copyright and Related Rights (defined below) upon the creator +and subsequent owner(s) (each and all, an "owner") of an original work of +authorship and/or a database (each, a "Work"). + +Certain owners wish to permanently relinquish those rights to a Work for +the purpose of contributing to a commons of creative, cultural and +scientific works ("Commons") that the public can reliably and without fear +of later claims of infringement build upon, modify, incorporate in other +works, reuse and redistribute as freely as possible in any form whatsoever +and for any purposes, including without limitation commercial purposes. +These owners may contribute to the Commons to promote the ideal of a free +culture and the further production of creative, cultural and scientific +works, or to gain reputation or greater distribution for their Work in +part through the use and efforts of others. + +For these and/or other purposes and motivations, and without any +expectation of additional consideration or compensation, the person +associating CC0 with a Work (the "Affirmer"), to the extent that he or she +is an owner of Copyright and Related Rights in the Work, voluntarily +elects to apply CC0 to the Work and publicly distribute the Work under its +terms, with knowledge of his or her Copyright and Related Rights in the +Work and the meaning and intended legal effect of CC0 on those rights. + +1. Copyright and Related Rights. A Work made available under CC0 may be +protected by copyright and related or neighboring rights ("Copyright and +Related Rights"). Copyright and Related Rights include, but are not +limited to, the following: + + i. the right to reproduce, adapt, distribute, perform, display, + communicate, and translate a Work; + ii. moral rights retained by the original author(s) and/or performer(s); +iii. publicity and privacy rights pertaining to a person's image or + likeness depicted in a Work; + iv. rights protecting against unfair competition in regards to a Work, + subject to the limitations in paragraph 4(a), below; + v. rights protecting the extraction, dissemination, use and reuse of data + in a Work; + vi. database rights (such as those arising under Directive 96/9/EC of the + European Parliament and of the Council of 11 March 1996 on the legal + protection of databases, and under any national implementation + thereof, including any amended or successor version of such + directive); and +vii. other similar, equivalent or corresponding rights throughout the + world based on applicable law or treaty, and any national + implementations thereof. + +2. Waiver. To the greatest extent permitted by, but not in contravention +of, applicable law, Affirmer hereby overtly, fully, permanently, +irrevocably and unconditionally waives, abandons, and surrenders all of +Affirmer's Copyright and Related Rights and associated claims and causes +of action, whether now known or unknown (including existing as well as +future claims and causes of action), in the Work (i) in all territories +worldwide, (ii) for the maximum duration provided by applicable law or +treaty (including future time extensions), (iii) in any current or future +medium and for any number of copies, and (iv) for any purpose whatsoever, +including without limitation commercial, advertising or promotional +purposes (the "Waiver"). Affirmer makes the Waiver for the benefit of each +member of the public at large and to the detriment of Affirmer's heirs and +successors, fully intending that such Waiver shall not be subject to +revocation, rescission, cancellation, termination, or any other legal or +equitable action to disrupt the quiet enjoyment of the Work by the public +as contemplated by Affirmer's express Statement of Purpose. + +3. Public License Fallback. Should any part of the Waiver for any reason +be judged legally invalid or ineffective under applicable law, then the +Waiver shall be preserved to the maximum extent permitted taking into +account Affirmer's express Statement of Purpose. In addition, to the +extent the Waiver is so judged Affirmer hereby grants to each affected +person a royalty-free, non transferable, non sublicensable, non exclusive, +irrevocable and unconditional license to exercise Affirmer's Copyright and +Related Rights in the Work (i) in all territories worldwide, (ii) for the +maximum duration provided by applicable law or treaty (including future +time extensions), (iii) in any current or future medium and for any number +of copies, and (iv) for any purpose whatsoever, including without +limitation commercial, advertising or promotional purposes (the +"License"). The License shall be deemed effective as of the date CC0 was +applied by Affirmer to the Work. Should any part of the License for any +reason be judged legally invalid or ineffective under applicable law, such +partial invalidity or ineffectiveness shall not invalidate the remainder +of the License, and in such case Affirmer hereby affirms that he or she +will not (i) exercise any of his or her remaining Copyright and Related +Rights in the Work or (ii) assert any associated claims and causes of +action with respect to the Work, in either case contrary to Affirmer's +express Statement of Purpose. + +4. Limitations and Disclaimers. + + a. No trademark or patent rights held by Affirmer are waived, abandoned, + surrendered, licensed or otherwise affected by this document. + b. Affirmer offers the Work as-is and makes no representations or + warranties of any kind concerning the Work, express, implied, + statutory or otherwise, including without limitation warranties of + title, merchantability, fitness for a particular purpose, non + infringement, or the absence of latent or other defects, accuracy, or + the present or absence of errors, whether or not discoverable, all to + the greatest extent permissible under applicable law. + c. Affirmer disclaims responsibility for clearing rights of other persons + that may apply to the Work or any use thereof, including without + limitation any person's Copyright and Related Rights in the Work. + Further, Affirmer disclaims responsibility for obtaining any necessary + consents, permissions or other rights required for any use of the + Work. + d. Affirmer understands and acknowledges that Creative Commons is not a + party to this document and has no duty or obligation with respect to + this CC0 or use of the Work. + +================================================================ + +github.com/zeebo/xxh3 +https://github.com/zeebo/xxh3 +---------------------------------------------------------------- +xxHash Library +Copyright (c) 2012-2014, Yann Collet +Copyright (c) 2019, Jeff Wendling +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, +are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this + list of conditions and the following disclaimer in the documentation and/or + other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON +ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +go.etcd.io/bbolt +https://go.etcd.io/bbolt +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2013 Ben Johnson + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================ + +go.etcd.io/etcd/api/v3 +https://go.etcd.io/etcd/api/v3 +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +go.etcd.io/etcd/client/pkg/v3 +https://go.etcd.io/etcd/client/pkg/v3 +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +go.etcd.io/etcd/client/v3 +https://go.etcd.io/etcd/client/v3 +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +go.mongodb.org/mongo-driver +https://go.mongodb.org/mongo-driver +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +go.opencensus.io +https://go.opencensus.io +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +================================================================ + +go.opentelemetry.io/auto/sdk +https://go.opentelemetry.io/auto/sdk +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +go.opentelemetry.io/contrib/detectors/gcp +https://go.opentelemetry.io/contrib/detectors/gcp +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc +https://go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp +https://go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +go.opentelemetry.io/otel +https://go.opentelemetry.io/otel +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +go.opentelemetry.io/otel/metric +https://go.opentelemetry.io/otel/metric +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +go.opentelemetry.io/otel/sdk +https://go.opentelemetry.io/otel/sdk +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +go.opentelemetry.io/otel/sdk/metric +https://go.opentelemetry.io/otel/sdk/metric +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +go.opentelemetry.io/otel/trace +https://go.opentelemetry.io/otel/trace +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +go.uber.org/atomic +https://go.uber.org/atomic +---------------------------------------------------------------- +Copyright (c) 2016 Uber Technologies, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +================================================================ + +go.uber.org/goleak +https://go.uber.org/goleak +---------------------------------------------------------------- +The MIT License (MIT) + +Copyright (c) 2018 Uber Technologies, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +================================================================ + +go.uber.org/multierr +https://go.uber.org/multierr +---------------------------------------------------------------- +Copyright (c) 2017-2021 Uber Technologies, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +================================================================ + +go.uber.org/zap +https://go.uber.org/zap +---------------------------------------------------------------- +Copyright (c) 2016-2017 Uber Technologies, Inc. + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +================================================================ + +goftp.io/server/v2 +https://goftp.io/server/v2 +---------------------------------------------------------------- +Copyright (c) 2018 Goftp Authors + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +================================================================ + +golang.org/x/crypto +https://golang.org/x/crypto +---------------------------------------------------------------- +Copyright 2009 The Go Authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google LLC nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +golang.org/x/mod +https://golang.org/x/mod +---------------------------------------------------------------- +Copyright 2009 The Go Authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google LLC nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +golang.org/x/net +https://golang.org/x/net +---------------------------------------------------------------- +Copyright 2009 The Go Authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google LLC nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +golang.org/x/oauth2 +https://golang.org/x/oauth2 +---------------------------------------------------------------- +Copyright 2009 The Go Authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google LLC nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +golang.org/x/sync +https://golang.org/x/sync +---------------------------------------------------------------- +Copyright 2009 The Go Authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google LLC nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +golang.org/x/sys +https://golang.org/x/sys +---------------------------------------------------------------- +Copyright 2009 The Go Authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google LLC nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +golang.org/x/term +https://golang.org/x/term +---------------------------------------------------------------- +Copyright 2009 The Go Authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google LLC nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +golang.org/x/text +https://golang.org/x/text +---------------------------------------------------------------- +Copyright 2009 The Go Authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google LLC nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +golang.org/x/time +https://golang.org/x/time +---------------------------------------------------------------- +Copyright 2009 The Go Authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google LLC nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +golang.org/x/tools +https://golang.org/x/tools +---------------------------------------------------------------- +Copyright 2009 The Go Authors. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google LLC nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +google.golang.org/api +https://google.golang.org/api +---------------------------------------------------------------- +Copyright (c) 2011 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +google.golang.org/genproto +https://google.golang.org/genproto +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +google.golang.org/genproto/googleapis/api +https://google.golang.org/genproto/googleapis/api +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +google.golang.org/genproto/googleapis/rpc +https://google.golang.org/genproto/googleapis/rpc +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +google.golang.org/grpc +https://google.golang.org/grpc +---------------------------------------------------------------- + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +google.golang.org/protobuf +https://google.golang.org/protobuf +---------------------------------------------------------------- +Copyright (c) 2018 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +gopkg.in/check.v1 +https://gopkg.in/check.v1 +---------------------------------------------------------------- +Gocheck - A rich testing framework for Go + +Copyright (c) 2010-2013 Gustavo Niemeyer + +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +================================================================ + +gopkg.in/yaml.v2 +https://gopkg.in/yaml.v2 +---------------------------------------------------------------- + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "{}" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright {yyyy} {name of copyright owner} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + +================================================================ + +gopkg.in/yaml.v3 +https://gopkg.in/yaml.v3 +---------------------------------------------------------------- + +This project is covered by two different licenses: MIT and Apache. + +#### MIT License #### + +The following files were ported to Go from C files of libyaml, and thus +are still covered by their original MIT license, with the additional +copyright staring in 2011 when the project was ported over: + + apic.go emitterc.go parserc.go readerc.go scannerc.go + writerc.go yamlh.go yamlprivateh.go + +Copyright (c) 2006-2010 Kirill Simonov +Copyright (c) 2006-2011 Kirill Simonov + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + +### Apache License ### + +All the remaining project files are covered by the Apache license: + +Copyright (c) 2011-2019 Canonical Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +================================================================ + diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..693d814d --- /dev/null +++ b/Dockerfile @@ -0,0 +1,12 @@ +FROM minio/minio:latest + +RUN chmod -R 777 /usr/bin + +COPY ./minio /usr/bin/minio +COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh + +ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] + +VOLUME ["/data"] + +CMD ["minio"] diff --git a/Dockerfile.cicd b/Dockerfile.cicd new file mode 100644 index 00000000..118ab032 --- /dev/null +++ b/Dockerfile.cicd @@ -0,0 +1,3 @@ +FROM minio/minio:edge + +CMD ["minio", "server", "/data"] diff --git a/Dockerfile.hotfix b/Dockerfile.hotfix new file mode 100644 index 00000000..4dc428cf --- /dev/null +++ b/Dockerfile.hotfix @@ -0,0 +1,71 @@ +FROM golang:1.24-alpine as build + +ARG TARGETARCH +ARG RELEASE + +ENV GOPATH=/go +ENV CGO_ENABLED=0 + +# Install curl and minisign +RUN apk add -U --no-cache ca-certificates && \ + apk add -U --no-cache curl && \ + go install aead.dev/minisign/cmd/minisign@v0.2.1 + +# Download minio binary and signature files +RUN curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \ + curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \ + curl -s -q https://dl.min.io/server/minio/hotfixes/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /go/bin/minio.sha256sum && \ + chmod +x /go/bin/minio + +# Download mc binary and signature files +RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \ + curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \ + curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.sha256sum -o /go/bin/mc.sha256sum && \ + chmod +x /go/bin/mc + +RUN if [ "$TARGETARCH" = "amd64" ]; then \ + curl -L -s -q https://github.com/moparisthebest/static-curl/releases/latest/download/curl-${TARGETARCH} -o /go/bin/curl; \ + chmod +x /go/bin/curl; \ + fi + +# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN" +RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \ + minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav + +FROM registry.access.redhat.com/ubi9/ubi-micro:latest + +ARG RELEASE + +LABEL name="MinIO" \ + vendor="MinIO Inc " \ + maintainer="MinIO Inc " \ + version="${RELEASE}" \ + release="${RELEASE}" \ + summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \ + description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads." + +ENV MINIO_ACCESS_KEY_FILE=access_key \ + MINIO_SECRET_KEY_FILE=secret_key \ + MINIO_ROOT_USER_FILE=access_key \ + MINIO_ROOT_PASSWORD_FILE=secret_key \ + MINIO_KMS_SECRET_KEY_FILE=kms_master_key \ + MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \ + MINIO_CONFIG_ENV_FILE=config.env \ + MC_CONFIG_DIR=/tmp/.mc + +RUN chmod -R 777 /usr/bin + +COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=build /go/bin/minio* /usr/bin/ +COPY --from=build /go/bin/mc* /usr/bin/ +COPY --from=build /go/bin/cur* /usr/bin/ + +COPY CREDITS /licenses/CREDITS +COPY LICENSE /licenses/LICENSE +COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh + +EXPOSE 9000 +VOLUME ["/data"] + +ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] +CMD ["minio"] diff --git a/Dockerfile.release b/Dockerfile.release new file mode 100644 index 00000000..b2d05283 --- /dev/null +++ b/Dockerfile.release @@ -0,0 +1,73 @@ +FROM golang:1.24-alpine AS build + +ARG TARGETARCH +ARG RELEASE + +ENV GOPATH=/go +ENV CGO_ENABLED=0 + +WORKDIR /build + +# Install curl and minisign +RUN apk add -U --no-cache ca-certificates && \ + apk add -U --no-cache curl && \ + apk add -U --no-cache bash && \ + go install aead.dev/minisign/cmd/minisign@v0.2.1 + +# Download minio binary and signature files +RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \ + curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \ + curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /go/bin/minio.sha256sum && \ + chmod +x /go/bin/minio + +# Download mc binary and signature files +RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \ + curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \ + curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.sha256sum -o /go/bin/mc.sha256sum && \ + chmod +x /go/bin/mc + +# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN" +RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \ + minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav + +COPY dockerscripts/download-static-curl.sh /build/download-static-curl +RUN chmod +x /build/download-static-curl && \ + /build/download-static-curl + +FROM registry.access.redhat.com/ubi9/ubi-micro:latest + +ARG RELEASE + +LABEL name="MinIO" \ + vendor="MinIO Inc " \ + maintainer="MinIO Inc " \ + version="${RELEASE}" \ + release="${RELEASE}" \ + summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \ + description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads." + +ENV MINIO_ACCESS_KEY_FILE=access_key \ + MINIO_SECRET_KEY_FILE=secret_key \ + MINIO_ROOT_USER_FILE=access_key \ + MINIO_ROOT_PASSWORD_FILE=secret_key \ + MINIO_KMS_SECRET_KEY_FILE=kms_master_key \ + MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \ + MINIO_CONFIG_ENV_FILE=config.env \ + MC_CONFIG_DIR=/tmp/.mc + +RUN chmod -R 777 /usr/bin + +COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=build /go/bin/minio* /usr/bin/ +COPY --from=build /go/bin/mc* /usr/bin/ +COPY --from=build /go/bin/curl* /usr/bin/ + +COPY CREDITS /licenses/CREDITS +COPY LICENSE /licenses/LICENSE +COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh + +EXPOSE 9000 +VOLUME ["/data"] + +ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] +CMD ["minio"] diff --git a/Dockerfile.release.old_cpu b/Dockerfile.release.old_cpu new file mode 100644 index 00000000..5fc0f3e3 --- /dev/null +++ b/Dockerfile.release.old_cpu @@ -0,0 +1,71 @@ +FROM golang:1.24-alpine AS build + +ARG TARGETARCH +ARG RELEASE + +ENV GOPATH=/go +ENV CGO_ENABLED=0 + +# Install curl and minisign +RUN apk add -U --no-cache ca-certificates && \ + apk add -U --no-cache curl && \ + go install aead.dev/minisign/cmd/minisign@v0.2.1 + +# Download minio binary and signature files +RUN curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE} -o /go/bin/minio && \ + curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.minisig -o /go/bin/minio.minisig && \ + curl -s -q https://dl.min.io/server/minio/release/linux-${TARGETARCH}/archive/minio.${RELEASE}.sha256sum -o /go/bin/minio.sha256sum && \ + chmod +x /go/bin/minio + +# Download mc binary and signature files +RUN curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc -o /go/bin/mc && \ + curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.minisig -o /go/bin/mc.minisig && \ + curl -s -q https://dl.min.io/client/mc/release/linux-${TARGETARCH}/mc.sha256sum -o /go/bin/mc.sha256sum && \ + chmod +x /go/bin/mc + +RUN if [ "$TARGETARCH" = "amd64" ]; then \ + curl -L -s -q https://github.com/moparisthebest/static-curl/releases/latest/download/curl-${TARGETARCH} -o /go/bin/curl; \ + chmod +x /go/bin/curl; \ + fi + +# Verify binary signature using public key "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGavRUN" +RUN minisign -Vqm /go/bin/minio -x /go/bin/minio.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav && \ + minisign -Vqm /go/bin/mc -x /go/bin/mc.minisig -P RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav + +FROM registry.access.redhat.com/ubi8/ubi-micro:latest + +ARG RELEASE + +LABEL name="MinIO" \ + vendor="MinIO Inc " \ + maintainer="MinIO Inc " \ + version="${RELEASE}" \ + release="${RELEASE}" \ + summary="MinIO is a High Performance Object Storage, API compatible with Amazon S3 cloud storage service." \ + description="MinIO object storage is fundamentally different. Designed for performance and the S3 API, it is 100% open-source. MinIO is ideal for large, private cloud environments with stringent security requirements and delivers mission-critical availability across a diverse range of workloads." + +ENV MINIO_ACCESS_KEY_FILE=access_key \ + MINIO_SECRET_KEY_FILE=secret_key \ + MINIO_ROOT_USER_FILE=access_key \ + MINIO_ROOT_PASSWORD_FILE=secret_key \ + MINIO_KMS_SECRET_KEY_FILE=kms_master_key \ + MINIO_UPDATE_MINISIGN_PUBKEY="RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" \ + MINIO_CONFIG_ENV_FILE=config.env \ + MC_CONFIG_DIR=/tmp/.mc + +RUN chmod -R 777 /usr/bin + +COPY --from=build /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ +COPY --from=build /go/bin/minio* /usr/bin/ +COPY --from=build /go/bin/mc* /usr/bin/ +COPY --from=build /go/bin/cur* /usr/bin/ + +COPY CREDITS /licenses/CREDITS +COPY LICENSE /licenses/LICENSE +COPY dockerscripts/docker-entrypoint.sh /usr/bin/docker-entrypoint.sh + +EXPOSE 9000 +VOLUME ["/data"] + +ENTRYPOINT ["/usr/bin/docker-entrypoint.sh"] +CMD ["minio"] diff --git a/Dockerfile.scratch b/Dockerfile.scratch new file mode 100644 index 00000000..8074aa15 --- /dev/null +++ b/Dockerfile.scratch @@ -0,0 +1,5 @@ +FROM scratch + +COPY minio /minio + +CMD ["/minio"] diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..18fa743e --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,102 @@ +pipeline { + agent any + + post { + always { + deleteDir() /* clean up our workspace */ + } + success { + updateGitlabCommitStatus state: 'success' + } + failure { + updateGitlabCommitStatus state: 'failed' + step([$class: 'Mailer', notifyEveryUnstableBuild: true, recipients: 'support@confdroid.com', sendToIndividuals: true]) + } + } + + options { + gitLabConnection('gitlab.confdroid.com') + } + + stages { + + stage('SonarScan') { + steps { + withCredentials([string(credentialsId: 'sonar-token', variable: 'SONAR_TOKEN')]) { + sh ''' + /opt/sonar-scanner/bin/sonar-scanner \ + -Dsonar.projectKey=confdroid_minio \ + -Dsonar.sources=. \ + -Dsonar.host.url=https://sonarqube.confdroid.com \ + -Dsonar.token=$SONAR_TOKEN + ''' + } + } + } + +stage('build and push image to gitlab') { + steps { + withCredentials([usernamePassword(credentialsId: 'jenkins-gitlab-registry-token-2026', passwordVariable: 'pw', usernameVariable: 'un')]) { + sh ''' + set +xe + docker login gitlab.confdroid.com:5050 -u $un -p $pw + docker build --network=host -t gitlab.confdroid.com:5050/containers/confdroid_minio:1.0.0 . + docker build --network=host -t gitlab.confdroid.com:5050/containers/confdroid_minio:latest . + docker push gitlab.confdroid.com:5050/containers/confdroid_minio:1.0.0 + docker push gitlab.confdroid.com:5050/containers/confdroid_minio:latest + ''' + } + } + } + + stage('build and push image to gitea') { + steps { + withCredentials([usernamePassword(credentialsId: 'Gitea-package-token', passwordVariable: 'pw', usernameVariable: 'un')]) { + sh ''' + set +xe + docker login sourcecode.confdroid.com -u $un -p $pw + docker build --network=host -t sourcecode.confdroid.com/confdroid/confdroid_minio:1.0.0 . + docker push sourcecode.confdroid.com/confdroid/confdroid_minio:1.0.0 + docker build --network=host -t sourcecode.confdroid.com/confdroid/confdroid_minio:latest . + docker push sourcecode.confdroid.com/confdroid/confdroid_minio:latest + ''' + } + } + } + + stage('update repo') { + steps { + sshagent(['edd05eb6-26b5-4c7b-a5cc-ea2ab899f4fa']) { + sh ''' + git config user.name "Jenkins Server" + git config user.email jenkins@confdroid.com + git add -A && git commit -am "Recommit for updates in build $BUILD_NUMBER" || echo "No changes to commit" + git push origin HEAD:master + ''' + } + } + } + stage('Mirror to Gitea') { + steps { + withCredentials([usernamePassword( + credentialsId: 'Jenkins-gitea', + usernameVariable: 'GITEA_USER', + passwordVariable: 'GITEA_TOKEN')]) { + script { + // Checkout from GitLab (already done implicitly) + sh ''' + git checkout master + git pull origin master + git rm -f Jenkinsfile || echo "No Jenkinsfile to remove from git" + git rm -r --cached .vscode || echo "No .vscode to remove from git" + git commit --amend --no-edit --allow-empty + git remote add master https://sourcecode.confdroid.com/confdroid/confdroid_minio.git + git -c credential.helper="!f() { echo username=${GITEA_USER}; echo password=${GITEA_TOKEN}; }; f" \ + push master --mirror + ''' + } + } + } + } + } +} \ No newline at end of file diff --git a/LICENSE b/LICENSE new file mode 100644 index 00000000..be3f7b28 --- /dev/null +++ b/LICENSE @@ -0,0 +1,661 @@ + GNU AFFERO GENERAL PUBLIC LICENSE + Version 3, 19 November 2007 + + Copyright (C) 2007 Free Software Foundation, Inc. + Everyone is permitted to copy and distribute verbatim copies + of this license document, but changing it is not allowed. + + Preamble + + The GNU Affero General Public License is a free, copyleft license for +software and other kinds of works, specifically designed to ensure +cooperation with the community in the case of network server software. + + The licenses for most software and other practical works are designed +to take away your freedom to share and change the works. By contrast, +our General Public Licenses are intended to guarantee your freedom to +share and change all versions of a program--to make sure it remains free +software for all its users. + + When we speak of free software, we are referring to freedom, not +price. Our General Public Licenses are designed to make sure that you +have the freedom to distribute copies of free software (and charge for +them if you wish), that you receive source code or can get it if you +want it, that you can change the software or use pieces of it in new +free programs, and that you know you can do these things. + + Developers that use our General Public Licenses protect your rights +with two steps: (1) assert copyright on the software, and (2) offer +you this License which gives you legal permission to copy, distribute +and/or modify the software. + + A secondary benefit of defending all users' freedom is that +improvements made in alternate versions of the program, if they +receive widespread use, become available for other developers to +incorporate. Many developers of free software are heartened and +encouraged by the resulting cooperation. However, in the case of +software used on network servers, this result may fail to come about. +The GNU General Public License permits making a modified version and +letting the public access it on a server without ever releasing its +source code to the public. + + The GNU Affero General Public License is designed specifically to +ensure that, in such cases, the modified source code becomes available +to the community. It requires the operator of a network server to +provide the source code of the modified version running there to the +users of that server. Therefore, public use of a modified version, on +a publicly accessible server, gives the public access to the source +code of the modified version. + + An older license, called the Affero General Public License and +published by Affero, was designed to accomplish similar goals. This is +a different license, not a version of the Affero GPL, but Affero has +released a new version of the Affero GPL which permits relicensing under +this license. + + The precise terms and conditions for copying, distribution and +modification follow. + + TERMS AND CONDITIONS + + 0. Definitions. + + "This License" refers to version 3 of the GNU Affero General Public License. + + "Copyright" also means copyright-like laws that apply to other kinds of +works, such as semiconductor masks. + + "The Program" refers to any copyrightable work licensed under this +License. Each licensee is addressed as "you". "Licensees" and +"recipients" may be individuals or organizations. + + To "modify" a work means to copy from or adapt all or part of the work +in a fashion requiring copyright permission, other than the making of an +exact copy. The resulting work is called a "modified version" of the +earlier work or a work "based on" the earlier work. + + A "covered work" means either the unmodified Program or a work based +on the Program. + + To "propagate" a work means to do anything with it that, without +permission, would make you directly or secondarily liable for +infringement under applicable copyright law, except executing it on a +computer or modifying a private copy. Propagation includes copying, +distribution (with or without modification), making available to the +public, and in some countries other activities as well. + + To "convey" a work means any kind of propagation that enables other +parties to make or receive copies. Mere interaction with a user through +a computer network, with no transfer of a copy, is not conveying. + + An interactive user interface displays "Appropriate Legal Notices" +to the extent that it includes a convenient and prominently visible +feature that (1) displays an appropriate copyright notice, and (2) +tells the user that there is no warranty for the work (except to the +extent that warranties are provided), that licensees may convey the +work under this License, and how to view a copy of this License. If +the interface presents a list of user commands or options, such as a +menu, a prominent item in the list meets this criterion. + + 1. Source Code. + + The "source code" for a work means the preferred form of the work +for making modifications to it. "Object code" means any non-source +form of a work. + + A "Standard Interface" means an interface that either is an official +standard defined by a recognized standards body, or, in the case of +interfaces specified for a particular programming language, one that +is widely used among developers working in that language. + + The "System Libraries" of an executable work include anything, other +than the work as a whole, that (a) is included in the normal form of +packaging a Major Component, but which is not part of that Major +Component, and (b) serves only to enable use of the work with that +Major Component, or to implement a Standard Interface for which an +implementation is available to the public in source code form. A +"Major Component", in this context, means a major essential component +(kernel, window system, and so on) of the specific operating system +(if any) on which the executable work runs, or a compiler used to +produce the work, or an object code interpreter used to run it. + + The "Corresponding Source" for a work in object code form means all +the source code needed to generate, install, and (for an executable +work) run the object code and to modify the work, including scripts to +control those activities. However, it does not include the work's +System Libraries, or general-purpose tools or generally available free +programs which are used unmodified in performing those activities but +which are not part of the work. For example, Corresponding Source +includes interface definition files associated with source files for +the work, and the source code for shared libraries and dynamically +linked subprograms that the work is specifically designed to require, +such as by intimate data communication or control flow between those +subprograms and other parts of the work. + + The Corresponding Source need not include anything that users +can regenerate automatically from other parts of the Corresponding +Source. + + The Corresponding Source for a work in source code form is that +same work. + + 2. Basic Permissions. + + All rights granted under this License are granted for the term of +copyright on the Program, and are irrevocable provided the stated +conditions are met. This License explicitly affirms your unlimited +permission to run the unmodified Program. The output from running a +covered work is covered by this License only if the output, given its +content, constitutes a covered work. This License acknowledges your +rights of fair use or other equivalent, as provided by copyright law. + + You may make, run and propagate covered works that you do not +convey, without conditions so long as your license otherwise remains +in force. You may convey covered works to others for the sole purpose +of having them make modifications exclusively for you, or provide you +with facilities for running those works, provided that you comply with +the terms of this License in conveying all material for which you do +not control copyright. Those thus making or running the covered works +for you must do so exclusively on your behalf, under your direction +and control, on terms that prohibit them from making any copies of +your copyrighted material outside their relationship with you. + + Conveying under any other circumstances is permitted solely under +the conditions stated below. Sublicensing is not allowed; section 10 +makes it unnecessary. + + 3. Protecting Users' Legal Rights From Anti-Circumvention Law. + + No covered work shall be deemed part of an effective technological +measure under any applicable law fulfilling obligations under article +11 of the WIPO copyright treaty adopted on 20 December 1996, or +similar laws prohibiting or restricting circumvention of such +measures. + + When you convey a covered work, you waive any legal power to forbid +circumvention of technological measures to the extent such circumvention +is effected by exercising rights under this License with respect to +the covered work, and you disclaim any intention to limit operation or +modification of the work as a means of enforcing, against the work's +users, your or third parties' legal rights to forbid circumvention of +technological measures. + + 4. Conveying Verbatim Copies. + + You may convey verbatim copies of the Program's source code as you +receive it, in any medium, provided that you conspicuously and +appropriately publish on each copy an appropriate copyright notice; +keep intact all notices stating that this License and any +non-permissive terms added in accord with section 7 apply to the code; +keep intact all notices of the absence of any warranty; and give all +recipients a copy of this License along with the Program. + + You may charge any price or no price for each copy that you convey, +and you may offer support or warranty protection for a fee. + + 5. Conveying Modified Source Versions. + + You may convey a work based on the Program, or the modifications to +produce it from the Program, in the form of source code under the +terms of section 4, provided that you also meet all of these conditions: + + a) The work must carry prominent notices stating that you modified + it, and giving a relevant date. + + b) The work must carry prominent notices stating that it is + released under this License and any conditions added under section + 7. This requirement modifies the requirement in section 4 to + "keep intact all notices". + + c) You must license the entire work, as a whole, under this + License to anyone who comes into possession of a copy. This + License will therefore apply, along with any applicable section 7 + additional terms, to the whole of the work, and all its parts, + regardless of how they are packaged. This License gives no + permission to license the work in any other way, but it does not + invalidate such permission if you have separately received it. + + d) If the work has interactive user interfaces, each must display + Appropriate Legal Notices; however, if the Program has interactive + interfaces that do not display Appropriate Legal Notices, your + work need not make them do so. + + A compilation of a covered work with other separate and independent +works, which are not by their nature extensions of the covered work, +and which are not combined with it such as to form a larger program, +in or on a volume of a storage or distribution medium, is called an +"aggregate" if the compilation and its resulting copyright are not +used to limit the access or legal rights of the compilation's users +beyond what the individual works permit. Inclusion of a covered work +in an aggregate does not cause this License to apply to the other +parts of the aggregate. + + 6. Conveying Non-Source Forms. + + You may convey a covered work in object code form under the terms +of sections 4 and 5, provided that you also convey the +machine-readable Corresponding Source under the terms of this License, +in one of these ways: + + a) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by the + Corresponding Source fixed on a durable physical medium + customarily used for software interchange. + + b) Convey the object code in, or embodied in, a physical product + (including a physical distribution medium), accompanied by a + written offer, valid for at least three years and valid for as + long as you offer spare parts or customer support for that product + model, to give anyone who possesses the object code either (1) a + copy of the Corresponding Source for all the software in the + product that is covered by this License, on a durable physical + medium customarily used for software interchange, for a price no + more than your reasonable cost of physically performing this + conveying of source, or (2) access to copy the + Corresponding Source from a network server at no charge. + + c) Convey individual copies of the object code with a copy of the + written offer to provide the Corresponding Source. This + alternative is allowed only occasionally and noncommercially, and + only if you received the object code with such an offer, in accord + with subsection 6b. + + d) Convey the object code by offering access from a designated + place (gratis or for a charge), and offer equivalent access to the + Corresponding Source in the same way through the same place at no + further charge. You need not require recipients to copy the + Corresponding Source along with the object code. If the place to + copy the object code is a network server, the Corresponding Source + may be on a different server (operated by you or a third party) + that supports equivalent copying facilities, provided you maintain + clear directions next to the object code saying where to find the + Corresponding Source. Regardless of what server hosts the + Corresponding Source, you remain obligated to ensure that it is + available for as long as needed to satisfy these requirements. + + e) Convey the object code using peer-to-peer transmission, provided + you inform other peers where the object code and Corresponding + Source of the work are being offered to the general public at no + charge under subsection 6d. + + A separable portion of the object code, whose source code is excluded +from the Corresponding Source as a System Library, need not be +included in conveying the object code work. + + A "User Product" is either (1) a "consumer product", which means any +tangible personal property which is normally used for personal, family, +or household purposes, or (2) anything designed or sold for incorporation +into a dwelling. In determining whether a product is a consumer product, +doubtful cases shall be resolved in favor of coverage. For a particular +product received by a particular user, "normally used" refers to a +typical or common use of that class of product, regardless of the status +of the particular user or of the way in which the particular user +actually uses, or expects or is expected to use, the product. A product +is a consumer product regardless of whether the product has substantial +commercial, industrial or non-consumer uses, unless such uses represent +the only significant mode of use of the product. + + "Installation Information" for a User Product means any methods, +procedures, authorization keys, or other information required to install +and execute modified versions of a covered work in that User Product from +a modified version of its Corresponding Source. The information must +suffice to ensure that the continued functioning of the modified object +code is in no case prevented or interfered with solely because +modification has been made. + + If you convey an object code work under this section in, or with, or +specifically for use in, a User Product, and the conveying occurs as +part of a transaction in which the right of possession and use of the +User Product is transferred to the recipient in perpetuity or for a +fixed term (regardless of how the transaction is characterized), the +Corresponding Source conveyed under this section must be accompanied +by the Installation Information. But this requirement does not apply +if neither you nor any third party retains the ability to install +modified object code on the User Product (for example, the work has +been installed in ROM). + + The requirement to provide Installation Information does not include a +requirement to continue to provide support service, warranty, or updates +for a work that has been modified or installed by the recipient, or for +the User Product in which it has been modified or installed. Access to a +network may be denied when the modification itself materially and +adversely affects the operation of the network or violates the rules and +protocols for communication across the network. + + Corresponding Source conveyed, and Installation Information provided, +in accord with this section must be in a format that is publicly +documented (and with an implementation available to the public in +source code form), and must require no special password or key for +unpacking, reading or copying. + + 7. Additional Terms. + + "Additional permissions" are terms that supplement the terms of this +License by making exceptions from one or more of its conditions. +Additional permissions that are applicable to the entire Program shall +be treated as though they were included in this License, to the extent +that they are valid under applicable law. If additional permissions +apply only to part of the Program, that part may be used separately +under those permissions, but the entire Program remains governed by +this License without regard to the additional permissions. + + When you convey a copy of a covered work, you may at your option +remove any additional permissions from that copy, or from any part of +it. (Additional permissions may be written to require their own +removal in certain cases when you modify the work.) You may place +additional permissions on material, added by you to a covered work, +for which you have or can give appropriate copyright permission. + + Notwithstanding any other provision of this License, for material you +add to a covered work, you may (if authorized by the copyright holders of +that material) supplement the terms of this License with terms: + + a) Disclaiming warranty or limiting liability differently from the + terms of sections 15 and 16 of this License; or + + b) Requiring preservation of specified reasonable legal notices or + author attributions in that material or in the Appropriate Legal + Notices displayed by works containing it; or + + c) Prohibiting misrepresentation of the origin of that material, or + requiring that modified versions of such material be marked in + reasonable ways as different from the original version; or + + d) Limiting the use for publicity purposes of names of licensors or + authors of the material; or + + e) Declining to grant rights under trademark law for use of some + trade names, trademarks, or service marks; or + + f) Requiring indemnification of licensors and authors of that + material by anyone who conveys the material (or modified versions of + it) with contractual assumptions of liability to the recipient, for + any liability that these contractual assumptions directly impose on + those licensors and authors. + + All other non-permissive additional terms are considered "further +restrictions" within the meaning of section 10. If the Program as you +received it, or any part of it, contains a notice stating that it is +governed by this License along with a term that is a further +restriction, you may remove that term. If a license document contains +a further restriction but permits relicensing or conveying under this +License, you may add to a covered work material governed by the terms +of that license document, provided that the further restriction does +not survive such relicensing or conveying. + + If you add terms to a covered work in accord with this section, you +must place, in the relevant source files, a statement of the +additional terms that apply to those files, or a notice indicating +where to find the applicable terms. + + Additional terms, permissive or non-permissive, may be stated in the +form of a separately written license, or stated as exceptions; +the above requirements apply either way. + + 8. Termination. + + You may not propagate or modify a covered work except as expressly +provided under this License. Any attempt otherwise to propagate or +modify it is void, and will automatically terminate your rights under +this License (including any patent licenses granted under the third +paragraph of section 11). + + However, if you cease all violation of this License, then your +license from a particular copyright holder is reinstated (a) +provisionally, unless and until the copyright holder explicitly and +finally terminates your license, and (b) permanently, if the copyright +holder fails to notify you of the violation by some reasonable means +prior to 60 days after the cessation. + + Moreover, your license from a particular copyright holder is +reinstated permanently if the copyright holder notifies you of the +violation by some reasonable means, this is the first time you have +received notice of violation of this License (for any work) from that +copyright holder, and you cure the violation prior to 30 days after +your receipt of the notice. + + Termination of your rights under this section does not terminate the +licenses of parties who have received copies or rights from you under +this License. If your rights have been terminated and not permanently +reinstated, you do not qualify to receive new licenses for the same +material under section 10. + + 9. Acceptance Not Required for Having Copies. + + You are not required to accept this License in order to receive or +run a copy of the Program. Ancillary propagation of a covered work +occurring solely as a consequence of using peer-to-peer transmission +to receive a copy likewise does not require acceptance. However, +nothing other than this License grants you permission to propagate or +modify any covered work. These actions infringe copyright if you do +not accept this License. Therefore, by modifying or propagating a +covered work, you indicate your acceptance of this License to do so. + + 10. Automatic Licensing of Downstream Recipients. + + Each time you convey a covered work, the recipient automatically +receives a license from the original licensors, to run, modify and +propagate that work, subject to this License. You are not responsible +for enforcing compliance by third parties with this License. + + An "entity transaction" is a transaction transferring control of an +organization, or substantially all assets of one, or subdividing an +organization, or merging organizations. If propagation of a covered +work results from an entity transaction, each party to that +transaction who receives a copy of the work also receives whatever +licenses to the work the party's predecessor in interest had or could +give under the previous paragraph, plus a right to possession of the +Corresponding Source of the work from the predecessor in interest, if +the predecessor has it or can get it with reasonable efforts. + + You may not impose any further restrictions on the exercise of the +rights granted or affirmed under this License. For example, you may +not impose a license fee, royalty, or other charge for exercise of +rights granted under this License, and you may not initiate litigation +(including a cross-claim or counterclaim in a lawsuit) alleging that +any patent claim is infringed by making, using, selling, offering for +sale, or importing the Program or any portion of it. + + 11. Patents. + + A "contributor" is a copyright holder who authorizes use under this +License of the Program or a work on which the Program is based. The +work thus licensed is called the contributor's "contributor version". + + A contributor's "essential patent claims" are all patent claims +owned or controlled by the contributor, whether already acquired or +hereafter acquired, that would be infringed by some manner, permitted +by this License, of making, using, or selling its contributor version, +but do not include claims that would be infringed only as a +consequence of further modification of the contributor version. For +purposes of this definition, "control" includes the right to grant +patent sublicenses in a manner consistent with the requirements of +this License. + + Each contributor grants you a non-exclusive, worldwide, royalty-free +patent license under the contributor's essential patent claims, to +make, use, sell, offer for sale, import and otherwise run, modify and +propagate the contents of its contributor version. + + In the following three paragraphs, a "patent license" is any express +agreement or commitment, however denominated, not to enforce a patent +(such as an express permission to practice a patent or covenant not to +sue for patent infringement). To "grant" such a patent license to a +party means to make such an agreement or commitment not to enforce a +patent against the party. + + If you convey a covered work, knowingly relying on a patent license, +and the Corresponding Source of the work is not available for anyone +to copy, free of charge and under the terms of this License, through a +publicly available network server or other readily accessible means, +then you must either (1) cause the Corresponding Source to be so +available, or (2) arrange to deprive yourself of the benefit of the +patent license for this particular work, or (3) arrange, in a manner +consistent with the requirements of this License, to extend the patent +license to downstream recipients. "Knowingly relying" means you have +actual knowledge that, but for the patent license, your conveying the +covered work in a country, or your recipient's use of the covered work +in a country, would infringe one or more identifiable patents in that +country that you have reason to believe are valid. + + If, pursuant to or in connection with a single transaction or +arrangement, you convey, or propagate by procuring conveyance of, a +covered work, and grant a patent license to some of the parties +receiving the covered work authorizing them to use, propagate, modify +or convey a specific copy of the covered work, then the patent license +you grant is automatically extended to all recipients of the covered +work and works based on it. + + A patent license is "discriminatory" if it does not include within +the scope of its coverage, prohibits the exercise of, or is +conditioned on the non-exercise of one or more of the rights that are +specifically granted under this License. You may not convey a covered +work if you are a party to an arrangement with a third party that is +in the business of distributing software, under which you make payment +to the third party based on the extent of your activity of conveying +the work, and under which the third party grants, to any of the +parties who would receive the covered work from you, a discriminatory +patent license (a) in connection with copies of the covered work +conveyed by you (or copies made from those copies), or (b) primarily +for and in connection with specific products or compilations that +contain the covered work, unless you entered into that arrangement, +or that patent license was granted, prior to 28 March 2007. + + Nothing in this License shall be construed as excluding or limiting +any implied license or other defenses to infringement that may +otherwise be available to you under applicable patent law. + + 12. No Surrender of Others' Freedom. + + If conditions are imposed on you (whether by court order, agreement or +otherwise) that contradict the conditions of this License, they do not +excuse you from the conditions of this License. If you cannot convey a +covered work so as to satisfy simultaneously your obligations under this +License and any other pertinent obligations, then as a consequence you may +not convey it at all. For example, if you agree to terms that obligate you +to collect a royalty for further conveying from those to whom you convey +the Program, the only way you could satisfy both those terms and this +License would be to refrain entirely from conveying the Program. + + 13. Remote Network Interaction; Use with the GNU General Public License. + + Notwithstanding any other provision of this License, if you modify the +Program, your modified version must prominently offer all users +interacting with it remotely through a computer network (if your version +supports such interaction) an opportunity to receive the Corresponding +Source of your version by providing access to the Corresponding Source +from a network server at no charge, through some standard or customary +means of facilitating copying of software. This Corresponding Source +shall include the Corresponding Source for any work covered by version 3 +of the GNU General Public License that is incorporated pursuant to the +following paragraph. + + Notwithstanding any other provision of this License, you have +permission to link or combine any covered work with a work licensed +under version 3 of the GNU General Public License into a single +combined work, and to convey the resulting work. The terms of this +License will continue to apply to the part which is the covered work, +but the work with which it is combined will remain governed by version +3 of the GNU General Public License. + + 14. Revised Versions of this License. + + The Free Software Foundation may publish revised and/or new versions of +the GNU Affero General Public License from time to time. Such new versions +will be similar in spirit to the present version, but may differ in detail to +address new problems or concerns. + + Each version is given a distinguishing version number. If the +Program specifies that a certain numbered version of the GNU Affero General +Public License "or any later version" applies to it, you have the +option of following the terms and conditions either of that numbered +version or of any later version published by the Free Software +Foundation. If the Program does not specify a version number of the +GNU Affero General Public License, you may choose any version ever published +by the Free Software Foundation. + + If the Program specifies that a proxy can decide which future +versions of the GNU Affero General Public License can be used, that proxy's +public statement of acceptance of a version permanently authorizes you +to choose that version for the Program. + + Later license versions may give you additional or different +permissions. However, no additional obligations are imposed on any +author or copyright holder as a result of your choosing to follow a +later version. + + 15. Disclaimer of Warranty. + + THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY +APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT +HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY +OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, +THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM +IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF +ALL NECESSARY SERVICING, REPAIR OR CORRECTION. + + 16. Limitation of Liability. + + IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING +WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS +THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY +GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE +USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF +DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD +PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), +EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF +SUCH DAMAGES. + + 17. Interpretation of Sections 15 and 16. + + If the disclaimer of warranty and limitation of liability provided +above cannot be given local legal effect according to their terms, +reviewing courts shall apply local law that most closely approximates +an absolute waiver of all civil liability in connection with the +Program, unless a warranty or assumption of liability accompanies a +copy of the Program in return for a fee. + + END OF TERMS AND CONDITIONS + + How to Apply These Terms to Your New Programs + + If you develop a new program, and you want it to be of the greatest +possible use to the public, the best way to achieve this is to make it +free software which everyone can redistribute and change under these terms. + + To do so, attach the following notices to the program. It is safest +to attach them to the start of each source file to most effectively +state the exclusion of warranty; and each file should have at least +the "copyright" line and a pointer to where the full notice is found. + + + Copyright (C) + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU Affero General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU Affero General Public License for more details. + + You should have received a copy of the GNU Affero General Public License + along with this program. If not, see . + +Also add information on how to contact you by electronic and paper mail. + + If your software can interact with users remotely through a computer +network, you should also make sure that it provides a way for users to +get its source. For example, if your program is a web application, its +interface could display a "Source" link that leads users to an archive +of the code. There are many ways you could offer source, and different +solutions will be better for different programs; see section 13 for the +specific requirements. + + You should also get your employer (if you work as a programmer) or school, +if any, to sign a "copyright disclaimer" for the program, if necessary. +For more information on this, and how to apply and follow the GNU AGPL, see +. diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..c7ea4b27 --- /dev/null +++ b/Makefile @@ -0,0 +1,245 @@ +PWD := $(shell pwd) +GOPATH := $(shell go env GOPATH) +LDFLAGS := $(shell go run buildscripts/gen-ldflags.go) + +GOOS ?= $(shell go env GOOS) +GOARCH ?= $(shell go env GOARCH) + +VERSION ?= $(shell git describe --tags) +REPO ?= quay.io/minio +TAG ?= $(REPO)/minio:$(VERSION) + +GOLANGCI_DIR = .bin/golangci/$(GOLANGCI_VERSION) +GOLANGCI = $(GOLANGCI_DIR)/golangci-lint + +all: build + +checks: ## check dependencies + @echo "Checking dependencies" + @(env bash $(PWD)/buildscripts/checkdeps.sh) + +help: ## print this help + @grep -E '^[a-zA-Z_-]+:.*?## .*$$' Makefile | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-40s\033[0m %s\n", $$1, $$2}' + +getdeps: ## fetch necessary dependencies + @mkdir -p ${GOPATH}/bin + @echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOLANGCI_DIR) + +crosscompile: ## cross compile minio + @(env bash $(PWD)/buildscripts/cross-compile.sh) + +verifiers: lint check-gen + +check-gen: ## check for updated autogenerated files + @go generate ./... >/dev/null + @go mod tidy -compat=1.21 + @(! git diff --name-only | grep '_gen.go$$') || (echo "Non-committed changes in auto-generated code is detected, please commit them to proceed." && false) + @(! git diff --name-only | grep 'go.sum') || (echo "Non-committed changes in auto-generated go.sum is detected, please commit them to proceed." && false) + +lint: getdeps ## runs golangci-lint suite of linters + @echo "Running $@ check" + @$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml + @command typos && typos ./ || echo "typos binary is not found.. skipping.." + +lint-fix: getdeps ## runs golangci-lint suite of linters with automatic fixes + @echo "Running $@ check" + @$(GOLANGCI) run --build-tags kqueue --timeout=10m --config ./.golangci.yml --fix + +check: test +test: verifiers build ## builds minio, runs linters, tests + @echo "Running unit tests" + @MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -v -tags kqueue,dev ./... + +test-root-disable: install-race + @echo "Running minio root lockdown tests" + @env bash $(PWD)/buildscripts/disable-root.sh + +test-ilm: install-race + @echo "Running ILM tests" + @env bash $(PWD)/docs/bucket/replication/setup_ilm_expiry_replication.sh + +test-ilm-transition: install-race + @echo "Running ILM tiering tests with healing" + @env bash $(PWD)/docs/bucket/lifecycle/setup_ilm_transition.sh + +test-pbac: install-race + @echo "Running bucket policies tests" + @env bash $(PWD)/docs/iam/policies/pbac-tests.sh + +test-decom: install-race + @echo "Running minio decom tests" + @env bash $(PWD)/docs/distributed/decom.sh + @env bash $(PWD)/docs/distributed/decom-encrypted.sh + @env bash $(PWD)/docs/distributed/decom-encrypted-sse-s3.sh + @env bash $(PWD)/docs/distributed/decom-compressed-sse-s3.sh + @env bash $(PWD)/docs/distributed/decom-encrypted-kes.sh + +test-versioning: install-race + @echo "Running minio versioning tests" + @env bash $(PWD)/docs/bucket/versioning/versioning-tests.sh + +test-configfile: install-race + @env bash $(PWD)/docs/distributed/distributed-from-config-file.sh + +test-upgrade: install-race + @echo "Running minio upgrade tests" + @(env bash $(PWD)/buildscripts/minio-upgrade.sh) + +test-race: verifiers build ## builds minio, runs linters, tests (race) + @echo "Running unit tests under -race" + @(env bash $(PWD)/buildscripts/race.sh) + +test-iam: install-race ## verify IAM (external IDP, etcd backends) + @echo "Running tests for IAM (external IDP, etcd backends)" + @MINIO_API_REQUESTS_MAX=10000 CGO_ENABLED=0 go test -timeout 15m -tags kqueue,dev -v -run TestIAM* ./cmd + @echo "Running tests for IAM (external IDP, etcd backends) with -race" + @MINIO_API_REQUESTS_MAX=10000 GORACE=history_size=7 CGO_ENABLED=1 go test -timeout 15m -race -tags kqueue,dev -v -run TestIAM* ./cmd + +test-iam-ldap-upgrade-import: install-race ## verify IAM (external LDAP IDP) + @echo "Running upgrade tests for IAM (LDAP backend)" + @env bash $(PWD)/buildscripts/minio-iam-ldap-upgrade-import-test.sh + +test-iam-import-with-missing-entities: install-race ## test import of external iam config withg missing entities + @echo "Test IAM import configurations with missing entities" + @env bash $(PWD)/docs/distributed/iam-import-with-missing-entities.sh + +test-iam-import-with-openid: install-race + @echo "Test IAM import configurations with openid" + @env bash $(PWD)/docs/distributed/iam-import-with-openid.sh + +test-sio-error: + @(env bash $(PWD)/docs/bucket/replication/sio-error.sh) + +test-replication-2site: + @(env bash $(PWD)/docs/bucket/replication/setup_2site_existing_replication.sh) + +test-replication-3site: + @(env bash $(PWD)/docs/bucket/replication/setup_3site_replication.sh) + +test-delete-replication: + @(env bash $(PWD)/docs/bucket/replication/delete-replication.sh) + +test-delete-marker-proxying: + @(env bash $(PWD)/docs/bucket/replication/test_del_marker_proxying.sh) + +test-replication: install-race test-replication-2site test-replication-3site test-delete-replication test-sio-error test-delete-marker-proxying ## verify multi site replication + @echo "Running tests for replicating three sites" + +test-site-replication-ldap: install-race ## verify automatic site replication + @echo "Running tests for automatic site replication of IAM (with LDAP)" + @(env bash $(PWD)/docs/site-replication/run-multi-site-ldap.sh) + +test-site-replication-oidc: install-race ## verify automatic site replication + @echo "Running tests for automatic site replication of IAM (with OIDC)" + @(env bash $(PWD)/docs/site-replication/run-multi-site-oidc.sh) + +test-site-replication-minio: install-race ## verify automatic site replication + @echo "Running tests for automatic site replication of IAM (with MinIO IDP)" + @(env bash $(PWD)/docs/site-replication/run-multi-site-minio-idp.sh) + @echo "Running tests for automatic site replication of SSE-C objects" + @(env bash $(PWD)/docs/site-replication/run-ssec-object-replication.sh) + @echo "Running tests for automatic site replication of SSE-C objects with SSE-KMS enabled for bucket" + @(env bash $(PWD)/docs/site-replication/run-sse-kms-object-replication.sh) + @echo "Running tests for automatic site replication of SSE-C objects with compression enabled for site" + @(env bash $(PWD)/docs/site-replication/run-ssec-object-replication-with-compression.sh) + +test-multipart: install-race ## test multipart + @echo "Test multipart behavior when part files are missing" + @(env bash $(PWD)/buildscripts/multipart-quorum-test.sh) + +test-timeout: install-race ## test multipart + @echo "Test server timeout" + @(env bash $(PWD)/buildscripts/test-timeout.sh) + +verify: install-race ## verify minio various setups + @echo "Verifying build with race" + @(env bash $(PWD)/buildscripts/verify-build.sh) + +verify-healing: install-race ## verify healing and replacing disks with minio binary + @echo "Verify healing build with race" + @(env bash $(PWD)/buildscripts/verify-healing.sh) + @(env bash $(PWD)/buildscripts/verify-healing-empty-erasure-set.sh) + @(env bash $(PWD)/buildscripts/heal-inconsistent-versions.sh) + +verify-healing-with-root-disks: install-race ## verify healing root disks + @echo "Verify healing with root drives" + @(env bash $(PWD)/buildscripts/verify-healing-with-root-disks.sh) + +verify-healing-with-rewrite: install-race ## verify healing to rewrite old xl.meta -> new xl.meta + @echo "Verify healing with rewrite" + @(env bash $(PWD)/buildscripts/rewrite-old-new.sh) + +verify-healing-inconsistent-versions: install-race ## verify resolving inconsistent versions + @echo "Verify resolving inconsistent versions build with race" + @(env bash $(PWD)/buildscripts/resolve-right-versions.sh) + +build-debugging: + @(env bash $(PWD)/docs/debugging/build.sh) + +build: checks build-debugging ## builds minio to $(PWD) + @echo "Building minio binary to './minio'" + @CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) go build -tags kqueue -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null + +hotfix-vars: + $(eval LDFLAGS := $(shell MINIO_RELEASE="RELEASE" MINIO_HOTFIX="hotfix.$(shell git rev-parse --short HEAD)" go run buildscripts/gen-ldflags.go $(shell git describe --tags --abbrev=0 | \ + sed 's#RELEASE\.\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)T\([0-9]\+\)-\([0-9]\+\)-\([0-9]\+\)Z#\1-\2-\3T\4:\5:\6Z#'))) + $(eval VERSION := $(shell git describe --tags --abbrev=0).hotfix.$(shell git rev-parse --short HEAD)) + +hotfix: hotfix-vars clean install ## builds minio binary with hotfix tags + @wget -q -c https://github.com/minio/pkger/releases/download/v2.3.11/pkger_2.3.11_linux_amd64.deb + @wget -q -c https://raw.githubusercontent.com/minio/minio-service/v1.1.1/linux-systemd/distributed/minio.service + @sudo apt install ./pkger_2.3.11_linux_amd64.deb --yes + @mkdir -p minio-release/$(GOOS)-$(GOARCH)/archive + @cp -af ./minio minio-release/$(GOOS)-$(GOARCH)/minio + @cp -af ./minio minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION) + @minisign -qQSm minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION) -s "${CRED_DIR}/minisign.key" < "${CRED_DIR}/minisign-passphrase" + @sha256sum < minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION) | sed 's, -,minio.$(VERSION),g' > minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION).sha256sum + @cp -af minio-release/$(GOOS)-$(GOARCH)/minio.$(VERSION)* minio-release/$(GOOS)-$(GOARCH)/archive/ + @pkger -r $(VERSION) --ignore + +hotfix-push: hotfix + @scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-$(GOOS)/ + @scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-0.minio.io:~/releases/server/minio/hotfixes/linux-$(GOOS)/archive + @scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-$(GOOS)/ + @scp -q -r minio-release/$(GOOS)-$(GOARCH)/* minio@dl-1.minio.io:~/releases/server/minio/hotfixes/linux-$(GOOS)/archive + @echo "Published new hotfix binaries at https://dl.min.io/server/minio/hotfixes/linux-$(GOOS)/archive/minio.$(VERSION)" + +docker-hotfix-push: docker-hotfix + @docker push -q $(TAG) && echo "Published new container $(TAG)" + +docker-hotfix: hotfix-push checks ## builds minio docker container with hotfix tags + @echo "Building minio docker image '$(TAG)'" + @docker build -q --no-cache -t $(TAG) --build-arg RELEASE=$(VERSION) . -f Dockerfile.hotfix + +docker: build ## builds minio docker container + @echo "Building minio docker image '$(TAG)'" + @docker build -q --no-cache -t $(TAG) . -f Dockerfile + +test-resiliency: build + @echo "Running resiliency tests" + @(DOCKER_COMPOSE_FILE=$(PWD)/docs/resiliency/docker-compose.yaml env bash $(PWD)/docs/resiliency/resiliency-tests.sh) + +install-race: checks build-debugging ## builds minio to $(PWD) + @echo "Building minio binary with -race to './minio'" + @GORACE=history_size=7 CGO_ENABLED=1 go build -tags kqueue,dev -race -trimpath --ldflags "$(LDFLAGS)" -o $(PWD)/minio 1>/dev/null + @echo "Installing minio binary with -race to '$(GOPATH)/bin/minio'" + @mkdir -p $(GOPATH)/bin && cp -af $(PWD)/minio $(GOPATH)/bin/minio + +install: build ## builds minio and installs it to $GOPATH/bin. + @echo "Installing minio binary to '$(GOPATH)/bin/minio'" + @mkdir -p $(GOPATH)/bin && cp -af $(PWD)/minio $(GOPATH)/bin/minio + @echo "Installation successful. To learn more, try \"minio --help\"." + +clean: ## cleanup all generated assets + @echo "Cleaning up all the generated files" + @find . -name '*.test' | xargs rm -fv + @find . -name '*~' | xargs rm -fv + @find . -name '.#*#' | xargs rm -fv + @find . -name '#*#' | xargs rm -fv + @rm -rvf minio + @rm -rvf build + @rm -rvf release + @rm -rvf .verify* + @rm -rvf minio-release + @rm -rvf minio.RELEASE*.hotfix.* + @rm -rvf pkger_*.deb diff --git a/NOTICE b/NOTICE new file mode 100644 index 00000000..cee66a9c --- /dev/null +++ b/NOTICE @@ -0,0 +1,9 @@ +MinIO Project, (C) 2015-2023 MinIO, Inc. + +This product includes software developed at MinIO, Inc. +(https://min.io/). + +The MinIO project contains unmodified/modified subcomponents too with +separate copyright notices and license terms. Your use of the source +code for these subcomponents is subject to the terms and conditions +of GNU Affero General Public License 3.0. diff --git a/PULL_REQUESTS_ETIQUETTE.md b/PULL_REQUESTS_ETIQUETTE.md new file mode 100644 index 00000000..a4a9008f --- /dev/null +++ b/PULL_REQUESTS_ETIQUETTE.md @@ -0,0 +1,93 @@ +# MinIO Pull Request Guidelines + +These guidelines ensure high-quality commits in MinIO’s GitHub repositories, maintaining +a clear, valuable commit history for our open-source projects. They apply to all contributors, +fostering efficient reviews and robust code. + +## Why Pull Requests? + +Pull Requests (PRs) drive quality in MinIO’s codebase by: +- Enabling peer review without pair programming. +- Documenting changes for future reference. +- Ensuring commits tell a clear story of development. + +**A poor commit lasts forever, even if code is refactored.** + +## Crafting a Quality PR + +A strong MinIO PR: +- Delivers a complete, valuable change (feature, bug fix, or improvement). +- Has a concise title (e.g., `[S3] Fix bucket policy parsing #1234`) and a summary with context, referencing issues (e.g., `#1234`). +- Contains well-written, logical commits explaining *why* changes were made (e.g., “Add S3 bucket tagging support so that users can organize resources efficiently”). +- Is small, focused, and easy to review—ideally one commit, unless multiple commits better narrate complex work. +- Adheres to MinIO’s coding standards (e.g., Go style, error handling, testing). + +PRs must flow smoothly through review to reach production. Large PRs should be split into smaller, manageable ones. + +## Submitting PRs + +1. **Title and Summary**: + - Use a scannable title: `[Subsystem] Action Description #Issue` (e.g., `[IAM] Add role-based access control #567`). + - Include context in the summary: what changed, why, and any issue references. + - Use `[WIP]` for in-progress PRs to avoid premature merging or choose GitHub draft PRs. + +2. **Commits**: + - Write clear messages: what changed and why (e.g., “Refactor S3 API handler to reduce latency so that requests process 20% faster”). + - Rebase to tidy commits before submitting (e.g., `git rebase -i main` to squash typos or reword messages), unless multiple contributors worked on the branch. + - Keep PRs focused—one feature or fix. Split large changes into multiple PRs. + +3. **Testing**: + - Include unit tests for new functionality or bug fixes. + - Ensure existing tests pass (`make test`). + - Document testing steps in the PR summary if manual testing was performed. + +4. **Before Submitting**: + - Run `make verify` to check formatting, linting, and tests. + - Reference related issues (e.g., “Closes #1234”). + - Notify team members via GitHub `@mentions` if urgent or complex. + +## Reviewing PRs + +Reviewers ensure MinIO’s commit history remains a clear, reliable record. Responsibilities include: + +1. **Commit Quality**: + - Verify each commit explains *why* the change was made (e.g., “So that…”). + - Request rebasing if commits are unclear, redundant, or lack context (e.g., “Please squash typo fixes into the parent commit”). + +2. **Code Quality**: + - Check adherence to MinIO’s Go standards (e.g., error handling, documentation). + - Ensure tests cover new code and pass CI. + - Flag bugs or critical issues for immediate fixes; suggest non-blocking improvements as follow-up issues. + +3. **Flow**: + - Review promptly to avoid blocking progress. + - Balance quality and speed—minor issues can be addressed later via issues, not PR blocks. + - If unable to complete the review, tag another reviewer (e.g., `@username please take over`). + +4. **Shared Responsibility**: + - All MinIO contributors are reviewers. The first commenter on a PR owns the review unless they delegate. + - Multiple reviewers are encouraged for complex PRs. + +5. **No Self-Edits**: + - Don’t modify the PR directly (e.g., fixing bugs). Request changes from the submitter or create a follow-up PR. + - If you edit, you’re a collaborator, not a reviewer, and cannot merge. + +6. **Testing**: + - Assume the submitter tested the code. If testing is unclear, ask for details (e.g., “How was this tested?”). + - Reject untested PRs unless testing is infeasible, then assist with test setup. + +## Tips for Success + +- **Small PRs**: Easier to review, faster to merge. Split large changes logically. +- **Clear Commits**: Use `git rebase -i` to refine history before submitting. +- **Engage Early**: Discuss complex changes in issues or Slack (https://slack.min.io) before coding. +- **Be Responsive**: Address reviewer feedback promptly to keep PRs moving. +- **Learn from Reviews**: Use feedback to improve future contributions. + +## Resources + +- [MinIO Coding Standards](https://github.com/minio/minio/blob/master/CONTRIBUTING.md) +- [Effective Commit Messages](https://mislav.net/2014/02/hidden-documentation/) +- [GitHub PR Tips](https://github.com/blog/1943-how-to-write-the-perfect-pull-request) + +By following these guidelines, we ensure MinIO’s codebase remains high-quality, maintainable, and a joy to contribute to. Happy coding! diff --git a/README.md b/README.md index 1ee7aa6d..46929f90 100644 --- a/README.md +++ b/README.md @@ -1,93 +1,160 @@ -# confdroid_minio +# MinIO Quickstart Guide +[![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) [![license](https://img.shields.io/badge/license-AGPL%20V3-blue)](https://github.com/minio/minio/blob/master/LICENSE) +[![MinIO](https://raw.githubusercontent.com/minio/minio/master/.github/logo.svg?sanitize=true)](https://min.io) -## Getting started +MinIO is a high-performance, S3-compatible object storage solution released under the GNU AGPL v3.0 license. +Designed for speed and scalability, it powers AI/ML, analytics, and data-intensive workloads with industry-leading performance. -To make it easy for you to get started with GitLab, here's a list of recommended next steps. +- S3 API Compatible – Seamless integration with existing S3 tools +- Built for AI & Analytics – Optimized for large-scale data pipelines +- High Performance – Ideal for demanding storage workloads. -Already a pro? Just edit this README.md and make it your own. Want to make it easy? [Use the template at the bottom](#editing-this-readme)! +This README provides instructions for building MinIO from source and deploying onto baremetal hardware. +Use the [MinIO Documentation](https://github.com/minio/docs) project to build and host a local copy of the documentation. -## Add your files +## MinIO is Open Source Software -* [Create](https://docs.gitlab.com/user/project/repository/web_editor/#create-a-file) or [upload](https://docs.gitlab.com/user/project/repository/web_editor/#upload-a-file) files -* [Add files using the command line](https://docs.gitlab.com/topics/git/add_files/#add-files-to-a-git-repository) or push an existing Git repository with the following command: +We designed MinIO as Open Source software for the Open Source software community. We encourage the community to remix, redesign, and reshare MinIO under the terms of the AGPLv3 license. -``` -cd existing_repo -git remote add origin https://gitlab.confdroid.com/containers/confdroid_minio.git -git branch -M master -git push -uf origin master +All usage of MinIO in your application stack requires validation against AGPLv3 obligations, which include but are not limited to the release of modified code to the community from which you have benefited. Any commercial/proprietary usage of the AGPLv3 software, including repackaging or reselling services/features, is done at your own risk. + +The AGPLv3 provides no obligation by any party to support, maintain, or warranty the original or any modified work. +All support is provided on a best-effort basis through Github and our [Slack](https//slack.min.io) channel, and any member of the community is welcome to contribute and assist others in their usage of the software. + +MinIO [AIStor](https://www.min.io/product/aistor) includes enterprise-grade support and licensing for workloads which require commercial or proprietary usage and production-level SLA/SLO-backed support. For more information, [reach out for a quote](https://min.io/pricing). + +## Source-Only Distribution + +**Important:** The MinIO community edition is now distributed as source code only. We will no longer provide pre-compiled binary releases for the community version. + +### Installing Latest MinIO Community Edition + +To use MinIO community edition, you have two options: + +1. **Install from source** using `go install github.com/minio/minio@latest` (recommended) +2. **Build a Docker image** from the provided Dockerfile + +See the sections below for detailed instructions on each method. + +### Legacy Binary Releases + +Historical pre-compiled binary releases remain available for reference but are no longer maintained: +- GitHub Releases: https://github.com/minio/minio/releases +- Direct downloads: https://dl.min.io/server/minio/release/ + +**These legacy binaries will not receive updates.** We strongly recommend using source builds for access to the latest features, bug fixes, and security updates. + +## Install from Source + +Use the following commands to compile and run a standalone MinIO server from source. +If you do not have a working Golang environment, please follow [How to install Golang](https://golang.org/doc/install). Minimum version required is [go1.24](https://golang.org/dl/#stable) + +```sh +go install github.com/minio/minio@latest ``` -## Integrate with your tools +You can alternatively run `go build` and use the `GOOS` and `GOARCH` environment variables to control the OS and architecture target. +For example: -* [Set up project integrations](https://gitlab.confdroid.com/containers/confdroid_minio/-/settings/integrations) +``` +env GOOS=linux GOARCh=arm64 go build +``` -## Collaborate with your team +Start MinIO by running `minio server PATH` where `PATH` is any empty folder on your local filesystem. -* [Invite team members and collaborators](https://docs.gitlab.com/user/project/members/) -* [Create a new merge request](https://docs.gitlab.com/user/project/merge_requests/creating_merge_requests/) -* [Automatically close issues from merge requests](https://docs.gitlab.com/user/project/issues/managing_issues/#closing-issues-automatically) -* [Enable merge request approvals](https://docs.gitlab.com/user/project/merge_requests/approvals/) -* [Set auto-merge](https://docs.gitlab.com/user/project/merge_requests/auto_merge/) +The MinIO deployment starts using default root credentials `minioadmin:minioadmin`. +You can test the deployment using the MinIO Console, an embedded web-based object browser built into MinIO Server. +Point a web browser running on the host machine to and log in with the root credentials. +You can use the Browser to create buckets, upload objects, and browse the contents of the MinIO server. -## Test and Deploy +You can also connect using any S3-compatible tool, such as the MinIO Client `mc` commandline tool: -Use the built-in continuous integration in GitLab. +```sh +mc alias set local http://localhost:9000 minioadmin minioadmin +mc admin info local +``` -* [Get started with GitLab CI/CD](https://docs.gitlab.com/ci/quick_start/) -* [Analyze your code for known vulnerabilities with Static Application Security Testing (SAST)](https://docs.gitlab.com/user/application_security/sast/) -* [Deploy to Kubernetes, Amazon EC2, or Amazon ECS using Auto Deploy](https://docs.gitlab.com/topics/autodevops/requirements/) -* [Use pull-based deployments for improved Kubernetes management](https://docs.gitlab.com/user/clusters/agent/) -* [Set up protected environments](https://docs.gitlab.com/ci/environments/protected_environments/) +See [Test using MinIO Client `mc`](#test-using-minio-client-mc) for more information on using the `mc` commandline tool. +For application developers, see to view MinIO SDKs for supported languages. -*** +> [!NOTE] +> Production environments using compiled-from-source MinIO binaries do so at their own risk. +> The AGPLv3 license provides no warranties nor liabilites for any such usage. -# Editing this README +## Build Docker Image -When you're ready to make this README your own, just edit this file and use the handy template below (or feel free to structure it however you want - this is just a starting point!). Thanks to [makeareadme.com](https://www.makeareadme.com/) for this template. +You can use the `docker build .` command to build a Docker image on your local host machine. +You must first [build MinIO](#install-from-source) and ensure the `minio` binary exists in the project root. -## Suggestions for a good README +The following command builds the Docker image using the default `Dockerfile` in the root project directory with the repository and image tag `myminio:minio` -Every project is different, so consider which of these sections apply to yours. The sections used in the template are suggestions for most open source projects. Also keep in mind that while a README can be too long and detailed, too long is better than too short. If you think your README is too long, consider utilizing another form of documentation rather than cutting out information. +```sh +docker build -t myminio:minio . +``` -## Name -Choose a self-explaining name for your project. +Use `docker image ls` to confirm the image exists in your local repository. +You can run the server using standard Docker invocation: -## Description -Let people know what your project can do specifically. Provide context and add a link to any reference visitors might be unfamiliar with. A list of Features or a Background subsection can also be added here. If there are alternatives to your project, this is a good place to list differentiating factors. +```sh +docker run -p 9000:9000 -p 9001:9001 myminio:minio server /tmp/minio --console-address :9001 +``` -## Badges -On some READMEs, you may see small images that convey metadata, such as whether or not all the tests are passing for the project. You can use Shields to add some to your README. Many services also have instructions for adding a badge. +Complete documentation for building Docker containers, managing custom images, or loading images into orchestration platforms is out of scope for this documentation. +You can modify the `Dockerfile` and `dockerscripts/socker-entrypoint.sh` as-needed to reflect your specific image requirements. -## Visuals -Depending on what you are making, it can be a good idea to include screenshots or even a video (you'll frequently see GIFs rather than actual videos). Tools like ttygif can help, but check out Asciinema for a more sophisticated method. +See the [MinIO Container](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-as-a-container.html#deploy-minio-container) documentation for more guidance on running MinIO within a Container image. -## Installation -Within a particular ecosystem, there may be a common way of installing things, such as using Yarn, NuGet, or Homebrew. However, consider the possibility that whoever is reading your README is a novice and would like more guidance. Listing specific steps helps remove ambiguity and gets people to using your project as quickly as possible. If it only runs in a specific context like a particular programming language version or operating system or has dependencies that have to be installed manually, also add a Requirements subsection. +## Install using Helm Charts -## Usage -Use examples liberally, and show the expected output if you can. It's helpful to have inline the smallest example of usage that you can demonstrate, while providing links to more sophisticated examples if they are too long to reasonably include in the README. +There are two paths for installing MinIO onto Kubernetes infrastructure: -## Support -Tell people where they can go to for help. It can be any combination of an issue tracker, a chat room, an email address, etc. +- Use the [MinIO Operator](https://github.com/minio/operator) +- Use the community-maintained [Helm charts](https://github.com/minio/minio/tree/master/helm/minio) -## Roadmap -If you have ideas for releases in the future, it is a good idea to list them in the README. +See the [MinIO Documentation](https://docs.min.io/community/minio-object-store/operations/deployments/kubernetes.html) for guidance on deploying using the Operator. +The Community Helm chart has instructions in the folder-level README. -## Contributing -State if you are open to contributions and what your requirements are for accepting them. +## Test MinIO Connectivity -For people who want to make changes to your project, it's helpful to have some documentation on how to get started. Perhaps there is a script that they should run or some environment variables that they need to set. Make these steps explicit. These instructions could also be useful to your future self. +### Test using MinIO Console -You can also document commands to lint the code or run tests. These steps help to ensure high code quality and reduce the likelihood that the changes inadvertently break something. Having instructions for running tests is especially helpful if it requires external setup, such as starting a Selenium server for testing in a browser. +MinIO Server comes with an embedded web based object browser. +Point your web browser to to ensure your server has started successfully. -## Authors and acknowledgment -Show your appreciation to those who have contributed to the project. +> [!NOTE] +> MinIO runs console on random port by default, if you wish to choose a specific port use `--console-address` to pick a specific interface and port. + +### Test using MinIO Client `mc` + +`mc` provides a modern alternative to UNIX commands like ls, cat, cp, mirror, diff etc. It supports filesystems and Amazon S3 compatible cloud storage services. + +The following commands set a local alias, validate the server information, create a bucket, copy data to that bucket, and list the contents of the bucket. + +```sh +mc alias set local http://localhost:9000 minioadmin minioadmin +mc admin info +mc mb data +mc cp ~/Downloads/mydata data/ +mc ls data/ +``` + +Follow the MinIO Client [Quickstart Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart) for further instructions. + +## Explore Further + +- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html) +- [MinIO Erasure Code Overview](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html) +- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html) +- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/go/minio-go.html) + +## Contribute to MinIO Project + +Please follow MinIO [Contributor's Guide](https://github.com/minio/minio/blob/master/CONTRIBUTING.md) for guidance on making new contributions to the repository. ## License -For open source projects, say how it is licensed. -## Project status -If you have run out of energy or time for your project, put a note at the top of the README saying that development has slowed down or stopped completely. Someone may choose to fork your project or volunteer to step in as a maintainer or owner, allowing your project to keep going. You can also make an explicit request for maintainers. +- MinIO source is licensed under the [GNU AGPLv3](https://github.com/minio/minio/blob/master/LICENSE). +- MinIO [documentation](https://github.com/minio/minio/tree/master/docs) is licensed under [CC BY 4.0](https://creativecommons.org/licenses/by/4.0/). +- [License Compliance](https://github.com/minio/minio/blob/master/COMPLIANCE.md) diff --git a/SECURITY.md b/SECURITY.md new file mode 100644 index 00000000..496b2ef2 --- /dev/null +++ b/SECURITY.md @@ -0,0 +1,42 @@ +# Security Policy + +## Supported Versions + +We always provide security updates for the [latest release](https://github.com/minio/minio/releases/latest). +Whenever there is a security update you just need to upgrade to the latest version. + +## Reporting a Vulnerability + +All security bugs in [minio/minio](https://github,com/minio/minio) (or other minio/* repositories) +should be reported by email to security@min.io. Your email will be acknowledged within 48 hours, +and you'll receive a more detailed response to your email within 72 hours indicating the next steps +in handling your report. + +Please, provide a detailed explanation of the issue. In particular, outline the type of the security +issue (DoS, authentication bypass, information disclose, ...) and the assumptions you're making (e.g. do +you need access credentials for a successful exploit). + +If you have not received a reply to your email within 48 hours or you have not heard from the security team +for the past five days please contact the security team directly: + +- Primary security coordinator: aead@min.io +- Secondary coordinator: harsha@min.io +- If you receive no response: dev@min.io + +### Disclosure Process + +MinIO uses the following disclosure process: + +1. Once the security report is received one member of the security team tries to verify and reproduce + the issue and determines the impact it has. +2. A member of the security team will respond and either confirm or reject the security report. + If the report is rejected the response explains why. +3. Code is audited to find any potential similar problems. +4. Fixes are prepared for the latest release. +5. On the date that the fixes are applied a security advisory will be published on . + Please inform us in your report email whether MinIO should mention your contribution w.r.t. fixing + the security issue. By default MinIO will **not** publish this information to protect your privacy. + +This process can take some time, especially when coordination is required with maintainers of other projects. +Every effort will be made to handle the bug in as timely a manner as possible, however it's important that we +follow the process described above to ensure that disclosures are handled consistently. diff --git a/VULNERABILITY_REPORT.md b/VULNERABILITY_REPORT.md new file mode 100644 index 00000000..8fcace07 --- /dev/null +++ b/VULNERABILITY_REPORT.md @@ -0,0 +1,38 @@ +# Vulnerability Management Policy + +This document formally describes the process of addressing and managing a +reported vulnerability that has been found in the MinIO server code base, +any directly connected ecosystem component or a direct / indirect dependency +of the code base. + +## Scope + +The vulnerability management policy described in this document covers the +process of investigating, assessing and resolving a vulnerability report +opened by a MinIO employee or an external third party. + +Therefore, it lists pre-conditions and actions that should be performed to +resolve and fix a reported vulnerability. + +## Vulnerability Management Process + +The vulnerability management process requires that the vulnerability report +contains the following information: + +- The project / component that contains the reported vulnerability. +- A description of the vulnerability. In particular, the type of the + reported vulnerability and how it might be exploited. Alternatively, + a well-established vulnerability identifier, e.g. CVE number, can be + used instead. + +Based on the description mentioned above, a MinIO engineer or security team +member investigates: + +- Whether the reported vulnerability exists. +- The conditions that are required such that the vulnerability can be exploited. +- The steps required to fix the vulnerability. + +In general, if the vulnerability exists in one of the MinIO code bases +itself - not in a code dependency - then MinIO will, if possible, fix +the vulnerability or implement reasonable countermeasures such that the +vulnerability cannot be exploited anymore. diff --git a/_config.yml b/_config.yml new file mode 100644 index 00000000..2f7efbea --- /dev/null +++ b/_config.yml @@ -0,0 +1 @@ +theme: jekyll-theme-minimal \ No newline at end of file diff --git a/buildscripts/checkdeps.sh b/buildscripts/checkdeps.sh new file mode 100755 index 00000000..ed4f666e --- /dev/null +++ b/buildscripts/checkdeps.sh @@ -0,0 +1,143 @@ +#!/usr/bin/env bash +# + +_init() { + + shopt -s extglob + + ## Minimum required versions for build dependencies + GIT_VERSION="1.0" + GO_VERSION="1.16" + OSX_VERSION="10.8" + KNAME=$(uname -s) + ARCH=$(uname -m) + case "${KNAME}" in + SunOS) + ARCH=$(isainfo -k) + ;; + esac +} + +## FIXME: +## In OSX, 'readlink -f' option does not exist, hence +## we have our own readlink -f behavior here. +## Once OSX has the option, below function is good enough. +## +## readlink() { +## return /bin/readlink -f "$1" +## } +## +readlink() { + TARGET_FILE=$1 + + cd $(dirname $TARGET_FILE) + TARGET_FILE=$(basename $TARGET_FILE) + + # Iterate down a (possible) chain of symlinks + while [ -L "$TARGET_FILE" ]; do + TARGET_FILE=$(env readlink $TARGET_FILE) + cd $(dirname $TARGET_FILE) + TARGET_FILE=$(basename $TARGET_FILE) + done + + # Compute the canonicalized name by finding the physical path + # for the directory we're in and appending the target file. + PHYS_DIR=$(pwd -P) + RESULT=$PHYS_DIR/$TARGET_FILE + echo $RESULT +} + +## FIXME: +## In OSX, 'sort -V' option does not exist, hence +## we have our own version compare function. +## Once OSX has the option, below function is good enough. +## +## check_minimum_version() { +## versions=($(echo -e "$1\n$2" | sort -V)) +## return [ "$1" == "${versions[0]}" ] +## } +## +check_minimum_version() { + IFS='.' read -r -a varray1 <<<"$1" + IFS='.' read -r -a varray2 <<<"$2" + + for i in "${!varray1[@]}"; do + if [[ ${varray1[i]} -lt ${varray2[i]} ]]; then + return 0 + elif [[ ${varray1[i]} -gt ${varray2[i]} ]]; then + return 1 + fi + done + + return 0 +} + +assert_is_supported_arch() { + case "${ARCH}" in + x86_64 | amd64 | aarch64 | ppc64le | arm* | s390x | loong64 | loongarch64 | riscv64) + return + ;; + *) + echo "Arch '${ARCH}' is not supported. Supported Arch: [x86_64, amd64, aarch64, ppc64le, arm*, s390x, loong64, loongarch64, riscv64]" + exit 1 + ;; + esac +} + +assert_is_supported_os() { + case "${KNAME}" in + Linux | FreeBSD | OpenBSD | NetBSD | DragonFly | SunOS) + return + ;; + Darwin) + osx_host_version=$(env sw_vers -productVersion) + if ! check_minimum_version "${OSX_VERSION}" "${osx_host_version}"; then + echo "OSX version '${osx_host_version}' is not supported. Minimum supported version: ${OSX_VERSION}" + exit 1 + fi + return + ;; + *) + echo "OS '${KNAME}' is not supported. Supported OS: [Linux, FreeBSD, OpenBSD, NetBSD, Darwin, DragonFly]" + exit 1 + ;; + esac +} + +assert_check_golang_env() { + if ! which go >/dev/null 2>&1; then + echo "Cannot find go binary in your PATH configuration, please refer to Go installation document at https://golang.org/doc/install" + exit 1 + fi + + installed_go_version=$(go version | sed 's/^.* go\([0-9.]*\).*$/\1/') + if ! check_minimum_version "${GO_VERSION}" "${installed_go_version}"; then + echo "Go runtime version '${installed_go_version}' is unsupported. Minimum supported version: ${GO_VERSION} to compile." + exit 1 + fi +} + +assert_check_deps() { + # support unusual Git versions such as: 2.7.4 (Apple Git-66) + installed_git_version=$(git version | perl -ne '$_ =~ m/git version (.*?)( |$)/; print "$1\n";') + if ! check_minimum_version "${GIT_VERSION}" "${installed_git_version}"; then + echo "Git version '${installed_git_version}' is not supported. Minimum supported version: ${GIT_VERSION}" + exit 1 + fi +} + +main() { + ## Check for supported arch + assert_is_supported_arch + + ## Check for supported os + assert_is_supported_os + + ## Check for Go environment + assert_check_golang_env + + ## Check for dependencies + assert_check_deps +} + +_init && main "$@" diff --git a/buildscripts/cicd-corpus/disk1/bucket/testobj/2b4f7e41-df82-4a5e-a3c1-8df87f83332f/part.1 b/buildscripts/cicd-corpus/disk1/bucket/testobj/2b4f7e41-df82-4a5e-a3c1-8df87f83332f/part.1 new file mode 100644 index 00000000..afbb2da1 Binary files /dev/null and b/buildscripts/cicd-corpus/disk1/bucket/testobj/2b4f7e41-df82-4a5e-a3c1-8df87f83332f/part.1 differ diff --git a/buildscripts/cicd-corpus/disk2/bucket/testobj/2b4f7e41-df82-4a5e-a3c1-8df87f83332f/part.1 b/buildscripts/cicd-corpus/disk2/bucket/testobj/2b4f7e41-df82-4a5e-a3c1-8df87f83332f/part.1 new file mode 100644 index 00000000..1031cd2a Binary files /dev/null and b/buildscripts/cicd-corpus/disk2/bucket/testobj/2b4f7e41-df82-4a5e-a3c1-8df87f83332f/part.1 differ diff --git a/buildscripts/cicd-corpus/disk2/bucket/testobj/xl.meta b/buildscripts/cicd-corpus/disk2/bucket/testobj/xl.meta new file mode 100644 index 00000000..4e786994 Binary files /dev/null and b/buildscripts/cicd-corpus/disk2/bucket/testobj/xl.meta differ diff --git a/buildscripts/cicd-corpus/disk3/bucket/testobj/2b4f7e41-df82-4a5e-a3c1-8df87f83332f/part.1 b/buildscripts/cicd-corpus/disk3/bucket/testobj/2b4f7e41-df82-4a5e-a3c1-8df87f83332f/part.1 new file mode 100644 index 00000000..789a168c Binary files /dev/null and b/buildscripts/cicd-corpus/disk3/bucket/testobj/2b4f7e41-df82-4a5e-a3c1-8df87f83332f/part.1 differ diff --git a/buildscripts/cicd-corpus/disk3/bucket/testobj/xl.meta b/buildscripts/cicd-corpus/disk3/bucket/testobj/xl.meta new file mode 100644 index 00000000..cba1a347 Binary files /dev/null and b/buildscripts/cicd-corpus/disk3/bucket/testobj/xl.meta differ diff --git a/buildscripts/cicd-corpus/disk4/bucket/testobj/2b4f7e41-df82-4a5e-a3c1-8df87f83332f/part.1 b/buildscripts/cicd-corpus/disk4/bucket/testobj/2b4f7e41-df82-4a5e-a3c1-8df87f83332f/part.1 new file mode 100644 index 00000000..86700285 Binary files /dev/null and b/buildscripts/cicd-corpus/disk4/bucket/testobj/2b4f7e41-df82-4a5e-a3c1-8df87f83332f/part.1 differ diff --git a/buildscripts/cicd-corpus/disk4/bucket/testobj/a599bd9e-69fe-49b7-b6bf-fe53021039d5/part.1 b/buildscripts/cicd-corpus/disk4/bucket/testobj/a599bd9e-69fe-49b7-b6bf-fe53021039d5/part.1 new file mode 100644 index 00000000..86700285 Binary files /dev/null and b/buildscripts/cicd-corpus/disk4/bucket/testobj/a599bd9e-69fe-49b7-b6bf-fe53021039d5/part.1 differ diff --git a/buildscripts/cicd-corpus/disk4/bucket/testobj/xl.meta b/buildscripts/cicd-corpus/disk4/bucket/testobj/xl.meta new file mode 100644 index 00000000..54bbedd6 Binary files /dev/null and b/buildscripts/cicd-corpus/disk4/bucket/testobj/xl.meta differ diff --git a/buildscripts/cicd-corpus/disk5/bucket/testobj/2b4f7e41-df82-4a5e-a3c1-8df87f83332f/part.1 b/buildscripts/cicd-corpus/disk5/bucket/testobj/2b4f7e41-df82-4a5e-a3c1-8df87f83332f/part.1 new file mode 100644 index 00000000..d39542b5 Binary files /dev/null and b/buildscripts/cicd-corpus/disk5/bucket/testobj/2b4f7e41-df82-4a5e-a3c1-8df87f83332f/part.1 differ diff --git a/buildscripts/cicd-corpus/disk5/bucket/testobj/a599bd9e-69fe-49b7-b6bf-fe53021039d5/part.1 b/buildscripts/cicd-corpus/disk5/bucket/testobj/a599bd9e-69fe-49b7-b6bf-fe53021039d5/part.1 new file mode 100644 index 00000000..d39542b5 Binary files /dev/null and b/buildscripts/cicd-corpus/disk5/bucket/testobj/a599bd9e-69fe-49b7-b6bf-fe53021039d5/part.1 differ diff --git a/buildscripts/cicd-corpus/disk5/bucket/testobj/xl.meta b/buildscripts/cicd-corpus/disk5/bucket/testobj/xl.meta new file mode 100644 index 00000000..68f69a63 Binary files /dev/null and b/buildscripts/cicd-corpus/disk5/bucket/testobj/xl.meta differ diff --git a/buildscripts/cross-compile.sh b/buildscripts/cross-compile.sh new file mode 100755 index 00000000..0590aebb --- /dev/null +++ b/buildscripts/cross-compile.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +set -e +# Enable tracing if set. +[ -n "$BASH_XTRACEFD" ] && set -x + +function _init() { + ## All binaries are static make sure to disable CGO. + export CGO_ENABLED=0 + + ## List of architectures and OS to test coss compilation. + SUPPORTED_OSARCH="linux/ppc64le linux/mips64 linux/amd64 linux/arm64 linux/s390x darwin/arm64 darwin/amd64 freebsd/amd64 windows/amd64 linux/arm linux/386 netbsd/amd64 linux/mips openbsd/amd64 linux/riscv64" +} + +function _build() { + local osarch=$1 + IFS=/ read -r -a arr <<<"$osarch" + os="${arr[0]}" + arch="${arr[1]}" + package=$(go list -f '{{.ImportPath}}') + printf -- "--> %15s:%s\n" "${osarch}" "${package}" + + # go build -trimpath to build the binary. + export GOOS=$os + export GOARCH=$arch + export GO111MODULE=on + go build -trimpath -tags kqueue -o /dev/null +} + +function main() { + echo "Testing builds for OS/Arch: ${SUPPORTED_OSARCH}" + for each_osarch in ${SUPPORTED_OSARCH}; do + _build "${each_osarch}" + done +} + +_init && main "$@" diff --git a/buildscripts/disable-root.sh b/buildscripts/disable-root.sh new file mode 100755 index 00000000..c35c769f --- /dev/null +++ b/buildscripts/disable-root.sh @@ -0,0 +1,124 @@ +#!/bin/bash + +set -x + +export MINIO_CI_CD=1 +killall -9 minio + +rm -rf ${HOME}/tmp/dist + +scheme="http" +nr_servers=4 + +addr="localhost" +args="" +for ((i = 0; i < $((nr_servers)); i++)); do + args="$args $scheme://$addr:$((9100 + i))/${HOME}/tmp/dist/path1/$i" +done + +echo $args + +for ((i = 0; i < $((nr_servers)); i++)); do + (minio server --address ":$((9100 + i))" $args 2>&1 >/tmp/log$i.txt) & +done + +sleep 10s + +if [ ! -f ./mc ]; then + wget --quiet -O ./mc https://dl.minio.io/client/mc/release/linux-amd64/./mc && + chmod +x mc +fi + +set +e + +export MC_HOST_minioadm=http://minioadmin:minioadmin@localhost:9100/ +./mc ready minioadm + +./mc ls minioadm/ + +./mc admin config set minioadm/ api root_access=off + +sleep 3s # let things settle a little + +./mc ls minioadm/ +if [ $? -eq 0 ]; then + echo "listing succeeded, 'minioadmin' was not disabled" + exit 1 +fi + +set -e + +killall -9 minio + +export MINIO_API_ROOT_ACCESS=on +for ((i = 0; i < $((nr_servers)); i++)); do + (minio server --address ":$((9100 + i))" $args 2>&1 >/tmp/log$i.txt) & +done + +set +e + +./mc ready minioadm/ + +./mc ls minioadm/ +if [ $? -ne 0 ]; then + echo "listing failed, 'minioadmin' should be enabled" + exit 1 +fi + +killall -9 minio + +rm -rf /tmp/multisitea/ +rm -rf /tmp/multisiteb/ + +echo "Setup site-replication and then disable root credentials" + +minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 & +minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 & + +minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 & +minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 & + +export MC_HOST_sitea=http://minioadmin:minioadmin@127.0.0.1:9001 +export MC_HOST_siteb=http://minioadmin:minioadmin@127.0.0.1:9004 + +./mc ready sitea +./mc ready siteb + +./mc admin replicate add sitea siteb + +./mc admin user add sitea foobar foo12345 + +./mc admin policy attach sitea/ consoleAdmin --user=foobar + +./mc admin user info siteb foobar + +killall -9 minio + +echo "turning off root access, however site replication must continue" +export MINIO_API_ROOT_ACCESS=off + +minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 & +minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 & + +minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 & +minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 & + +export MC_HOST_sitea=http://foobar:foo12345@127.0.0.1:9001 +export MC_HOST_siteb=http://foobar:foo12345@127.0.0.1:9004 + +./mc ready sitea +./mc ready siteb + +./mc admin user add sitea foobar-admin foo12345 + +sleep 2s + +./mc admin user info siteb foobar-admin diff --git a/buildscripts/gen-ldflags.go b/buildscripts/gen-ldflags.go new file mode 100644 index 00000000..a7649056 --- /dev/null +++ b/buildscripts/gen-ldflags.go @@ -0,0 +1,121 @@ +//go:build ignore +// +build ignore + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package main + +import ( + "fmt" + "os" + "os/exec" + "strconv" + "strings" + "time" +) + +func genLDFlags(version string) string { + releaseTag, date := releaseTag(version) + copyrightYear := strconv.Itoa(date.Year()) + ldflagsStr := "-s -w" + ldflagsStr += " -X github.com/minio/minio/cmd.Version=" + version + ldflagsStr += " -X github.com/minio/minio/cmd.CopyrightYear=" + copyrightYear + ldflagsStr += " -X github.com/minio/minio/cmd.ReleaseTag=" + releaseTag + ldflagsStr += " -X github.com/minio/minio/cmd.CommitID=" + commitID() + ldflagsStr += " -X github.com/minio/minio/cmd.ShortCommitID=" + commitID()[:12] + ldflagsStr += " -X github.com/minio/minio/cmd.GOPATH=" + os.Getenv("GOPATH") + ldflagsStr += " -X github.com/minio/minio/cmd.GOROOT=" + os.Getenv("GOROOT") + return ldflagsStr +} + +// genReleaseTag prints release tag to the console for easy git tagging. +func releaseTag(version string) (string, time.Time) { + relPrefix := "DEVELOPMENT" + if prefix := os.Getenv("MINIO_RELEASE"); prefix != "" { + relPrefix = prefix + } + + relSuffix := "" + if hotfix := os.Getenv("MINIO_HOTFIX"); hotfix != "" { + relSuffix = hotfix + } + + relTag := strings.Replace(version, " ", "-", -1) + relTag = strings.Replace(relTag, ":", "-", -1) + t, err := time.Parse("2006-01-02T15-04-05Z", relTag) + if err != nil { + panic(err) + } + relTag = strings.Replace(relTag, ",", "", -1) + relTag = relPrefix + "." + relTag + if relSuffix != "" { + relTag += "." + relSuffix + } + + return relTag, t +} + +// commitID returns the abbreviated commit-id hash of the last commit. +func commitID() string { + // git log --format="%H" -n1 + var ( + commit []byte + err error + ) + cmdName := "git" + cmdArgs := []string{"log", "--format=%H", "-n1"} + if commit, err = exec.Command(cmdName, cmdArgs...).Output(); err != nil { + fmt.Fprintln(os.Stderr, "Error generating git commit-id: ", err) + os.Exit(1) + } + + return strings.TrimSpace(string(commit)) +} + +func commitTime() time.Time { + // git log --format=%cD -n1 + var ( + commitUnix []byte + err error + ) + cmdName := "git" + cmdArgs := []string{"log", "--format=%cI", "-n1"} + if commitUnix, err = exec.Command(cmdName, cmdArgs...).Output(); err != nil { + fmt.Fprintln(os.Stderr, "Error generating git commit-time: ", err) + os.Exit(1) + } + + t, err := time.Parse(time.RFC3339, strings.TrimSpace(string(commitUnix))) + if err != nil { + fmt.Fprintln(os.Stderr, "Error generating git commit-time: ", err) + os.Exit(1) + } + + return t.UTC() +} + +func main() { + var version string + if len(os.Args) > 1 { + version = os.Args[1] + } else { + version = commitTime().Format(time.RFC3339) + } + + fmt.Println(genLDFlags(version)) +} diff --git a/buildscripts/heal-inconsistent-versions.sh b/buildscripts/heal-inconsistent-versions.sh new file mode 100755 index 00000000..f4c918a6 --- /dev/null +++ b/buildscripts/heal-inconsistent-versions.sh @@ -0,0 +1,92 @@ +#!/bin/bash -e + +set -E +set -o pipefail +set -x + +WORK_DIR="$PWD/.verify-$RANDOM" +MINIO_CONFIG_DIR="$WORK_DIR/.minio" +MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server) + +if [ ! -x "$PWD/minio" ]; then + echo "minio executable binary not found in current directory" + exit 1 +fi + +if [ ! -x "$PWD/minio" ]; then + echo "minio executable binary not found in current directory" + exit 1 +fi + +function start_minio_4drive() { + start_port=$1 + + export MINIO_ROOT_USER=minio + export MINIO_ROOT_PASSWORD=minio123 + export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/" + unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects + export MINIO_CI_CD=1 + + mkdir ${WORK_DIR} + C_PWD=${PWD} + if [ ! -x "$PWD/mc" ]; then + MC_BUILD_DIR="mc-$RANDOM" + if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then + echo "failed to download https://github.com/minio/mc" + purge "${MC_BUILD_DIR}" + exit 1 + fi + + (cd "${MC_BUILD_DIR}" && go build -o "$C_PWD/mc") + + # remove mc source. + purge "${MC_BUILD_DIR}" + fi + + "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/disk{1...4}" >"${WORK_DIR}/server1.log" 2>&1 & + pid=$! + disown $pid + sleep 5 + + if ! ps -p ${pid} 1>&2 >/dev/null; then + echo "server1 log:" + cat "${WORK_DIR}/server1.log" + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi + + "${PWD}/mc" mb --with-versioning minio/bucket + + for i in $(seq 1 4); do + "${PWD}/mc" cp /etc/hosts minio/bucket/testobj + + sudo chown -R root. "${WORK_DIR}/disk${i}" + + "${PWD}/mc" cp /etc/hosts minio/bucket/testobj + + sudo chown -R ${USER}. "${WORK_DIR}/disk${i}" + done + + for vid in $("${PWD}/mc" ls --json --versions minio/bucket/testobj | jq -r .versionId); do + "${PWD}/mc" cat --vid "${vid}" minio/bucket/testobj | md5sum + done + + pkill minio + sleep 3 +} + +function main() { + start_port=$(shuf -i 10000-65000 -n 1) + + start_minio_4drive ${start_port} +} + +function purge() { + rm -rf "$1" +} + +(main "$@") +rv=$? +purge "$WORK_DIR" +exit "$rv" diff --git a/buildscripts/heal-manual.go b/buildscripts/heal-manual.go new file mode 100644 index 00000000..27c4183f --- /dev/null +++ b/buildscripts/heal-manual.go @@ -0,0 +1,86 @@ +//go:build ignore +// +build ignore + +// +// MinIO Object Storage (c) 2022 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +package main + +import ( + "context" + "encoding/json" + "fmt" + "log" + "os" + "time" + + "github.com/minio/madmin-go/v3" +) + +func main() { + // Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY are + // dummy values, please replace them with original values. + + // API requests are secure (HTTPS) if secure=true and insecure (HTTP) otherwise. + // New returns an MinIO Admin client object. + madmClnt, err := madmin.New(os.Args[1], os.Args[2], os.Args[3], false) + if err != nil { + log.Fatalln(err) + } + + opts := madmin.HealOpts{ + Recursive: true, // recursively heal all objects at 'prefix' + Remove: true, // remove content that has lost quorum and not recoverable + ScanMode: madmin.HealNormalScan, // by default do not do 'deep' scanning + } + + start, _, err := madmClnt.Heal(context.Background(), "healing-rewrite-bucket", "", opts, "", false, false) + if err != nil { + log.Fatalln(err) + } + fmt.Println("Healstart sequence ===") + enc := json.NewEncoder(os.Stdout) + if err = enc.Encode(&start); err != nil { + log.Fatalln(err) + } + + fmt.Println() + for { + _, status, err := madmClnt.Heal(context.Background(), "healing-rewrite-bucket", "", opts, start.ClientToken, false, false) + if status.Summary == "finished" { + fmt.Println("Healstatus on items ===") + for _, item := range status.Items { + if err = enc.Encode(&item); err != nil { + log.Fatalln(err) + } + } + break + } + if status.Summary == "stopped" { + fmt.Println("Healstatus on items ===") + fmt.Println("Heal failed with", status.FailureDetail) + break + } + + for _, item := range status.Items { + if err = enc.Encode(&item); err != nil { + log.Fatalln(err) + } + } + + time.Sleep(time.Second) + } +} diff --git a/buildscripts/minio-iam-ldap-upgrade-import-test.sh b/buildscripts/minio-iam-ldap-upgrade-import-test.sh new file mode 100755 index 00000000..e7da2b10 --- /dev/null +++ b/buildscripts/minio-iam-ldap-upgrade-import-test.sh @@ -0,0 +1,127 @@ +#!/bin/bash + +# This script is used to test the migration of IAM content from old minio +# instance to new minio instance. +# +# To run it locally, start the LDAP server in github.com/minio/minio-iam-testing +# repo (e.g. make podman-run), and then run this script. +# +# This script assumes that LDAP server is at: +# +# `localhost:389` +# +# if this is not the case, set the environment variable +# `_MINIO_LDAP_TEST_SERVER`. + +OLD_VERSION=RELEASE.2024-03-26T22-10-45Z +OLD_BINARY_LINK=https://dl.min.io/server/minio/release/linux-amd64/archive/minio.${OLD_VERSION} + +__init__() { + if which curl &>/dev/null; then + echo "curl is already installed" + else + echo "Installing curl:" + sudo apt install curl -y + fi + + export GOPATH=/tmp/gopath + export PATH="${PATH}":"${GOPATH}"/bin + + if which mc &>/dev/null; then + echo "mc is already installed" + else + echo "Installing mc:" + go install github.com/minio/mc@latest + fi + + if [ ! -x ./minio.${OLD_VERSION} ]; then + echo "Downloading minio.${OLD_VERSION} binary" + curl -o minio.${OLD_VERSION} ${OLD_BINARY_LINK} + chmod +x minio.${OLD_VERSION} + fi + + if [ -z "$_MINIO_LDAP_TEST_SERVER" ]; then + export _MINIO_LDAP_TEST_SERVER=localhost:389 + echo "Using default LDAP endpoint: $_MINIO_LDAP_TEST_SERVER" + fi + + rm -rf /tmp/data +} + +create_iam_content_in_old_minio() { + echo "Creating IAM content in old minio instance." + + MINIO_CI_CD=1 ./minio.${OLD_VERSION} server /tmp/data/{1...4} & + sleep 5 + + set -x + mc alias set old-minio http://localhost:9000 minioadmin minioadmin + mc ready old-minio + mc idp ldap add old-minio \ + server_addr=localhost:389 \ + server_insecure=on \ + lookup_bind_dn=cn=admin,dc=min,dc=io \ + lookup_bind_password=admin \ + user_dn_search_base_dn=dc=min,dc=io \ + user_dn_search_filter="(uid=%s)" \ + group_search_base_dn=ou=swengg,dc=min,dc=io \ + group_search_filter="(&(objectclass=groupOfNames)(member=%d))" + mc admin service restart old-minio + + mc idp ldap policy attach old-minio readwrite --user=UID=dillon,ou=people,ou=swengg,dc=min,dc=io + mc idp ldap policy attach old-minio readwrite --group=CN=project.c,ou=groups,ou=swengg,dc=min,dc=io + + mc idp ldap policy entities old-minio + + mc admin cluster iam export old-minio + set +x + + mc admin service stop old-minio +} + +import_iam_content_in_new_minio() { + echo "Importing IAM content in new minio instance." + # Assume current minio binary exists. + MINIO_CI_CD=1 ./minio server /tmp/data/{1...4} & + sleep 5 + + set -x + mc alias set new-minio http://localhost:9000 minioadmin minioadmin + echo "BEFORE IMPORT mappings:" + mc ready new-minio + mc idp ldap policy entities new-minio + mc admin cluster iam import new-minio ./old-minio-iam-info.zip + echo "AFTER IMPORT mappings:" + mc idp ldap policy entities new-minio + set +x + + # mc admin service stop new-minio +} + +verify_iam_content_in_new_minio() { + output=$(mc idp ldap policy entities new-minio --json) + + groups=$(echo "$output" | jq -r '.result.policyMappings[] | select(.policy == "readwrite") | .groups[]') + if [ "$groups" != "cn=project.c,ou=groups,ou=swengg,dc=min,dc=io" ]; then + echo "Failed to verify groups: $groups" + exit 1 + fi + + users=$(echo "$output" | jq -r '.result.policyMappings[] | select(.policy == "readwrite") | .users[]') + if [ "$users" != "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" ]; then + echo "Failed to verify users: $users" + exit 1 + fi + + mc admin service stop new-minio +} + +main() { + create_iam_content_in_old_minio + + import_iam_content_in_new_minio + + verify_iam_content_in_new_minio +} + +(__init__ "$@" && main "$@") diff --git a/buildscripts/minio-upgrade.sh b/buildscripts/minio-upgrade.sh new file mode 100755 index 00000000..deaaf460 --- /dev/null +++ b/buildscripts/minio-upgrade.sh @@ -0,0 +1,113 @@ +#!/bin/bash + +trap 'cleanup $LINENO' ERR + +# shellcheck disable=SC2120 +cleanup() { + MINIO_VERSION=dev /tmp/gopath/bin/docker-compose \ + -f "buildscripts/upgrade-tests/compose.yml" \ + down || true + + MINIO_VERSION=dev /tmp/gopath/bin/docker-compose \ + -f "buildscripts/upgrade-tests/compose.yml" \ + rm || true + + for volume in $(docker volume ls -q | grep upgrade); do + docker volume rm ${volume} || true + done + + docker volume prune -f + docker system prune -f || true + docker volume prune -f || true + docker volume rm $(docker volume ls -q -f dangling=true) || true +} + +verify_checksum_after_heal() { + local sum1 + sum1=$(curl -s "$2" | sha256sum) + mc admin heal --json -r "$1" >/dev/null # test after healing + local sum1_heal + sum1_heal=$(curl -s "$2" | sha256sum) + + if [ "${sum1_heal}" != "${sum1}" ]; then + echo "mismatch expected ${sum1_heal}, got ${sum1}" + exit 1 + fi +} + +verify_checksum_mc() { + local expected + expected=$(mc cat "$1" | sha256sum) + local got + got=$(mc cat "$2" | sha256sum) + + if [ "${expected}" != "${got}" ]; then + echo "mismatch - expected ${expected}, got ${got}" + exit 1 + fi + echo "matches - ${expected}, got ${got}" +} + +add_alias() { + for i in $(seq 1 4); do + echo "... attempting to add alias $i" + until (mc alias set minio http://127.0.0.1:9000 minioadmin minioadmin); do + echo "...waiting... for 5secs" && sleep 5 + done + done + + echo "Sleeping for nginx" + sleep 20 +} + +__init__() { + sudo apt install curl -y + export GOPATH=/tmp/gopath + export PATH=${PATH}:${GOPATH}/bin + + go install github.com/minio/mc@latest + + ## this is needed because github actions don't have + ## docker-compose on all runners + COMPOSE_VERSION=v2.35.1 + mkdir -p /tmp/gopath/bin/ + wget -O /tmp/gopath/bin/docker-compose https://github.com/docker/compose/releases/download/${COMPOSE_VERSION}/docker-compose-linux-x86_64 + chmod +x /tmp/gopath/bin/docker-compose + + cleanup + + TAG=minio/minio:dev make docker + + MINIO_VERSION=RELEASE.2019-12-19T22-52-26Z docker-compose \ + -f "buildscripts/upgrade-tests/compose.yml" \ + up -d --build + + add_alias + + mc mb minio/minio-test/ + mc cp ./minio minio/minio-test/to-read/ + mc cp /etc/hosts minio/minio-test/to-read/hosts + mc anonymous set download minio/minio-test + + verify_checksum_mc ./minio minio/minio-test/to-read/minio + + curl -s http://127.0.0.1:9000/minio-test/to-read/hosts | sha256sum + + MINIO_VERSION=dev /tmp/gopath/bin/docker-compose -f "buildscripts/upgrade-tests/compose.yml" stop +} + +main() { + MINIO_VERSION=dev /tmp/gopath/bin/docker-compose -f "buildscripts/upgrade-tests/compose.yml" up -d --build + + add_alias + + verify_checksum_after_heal minio/minio-test http://127.0.0.1:9000/minio-test/to-read/hosts + + verify_checksum_mc ./minio minio/minio-test/to-read/minio + + verify_checksum_mc /etc/hosts minio/minio-test/to-read/hosts + + cleanup +} + +(__init__ "$@" && main "$@") diff --git a/buildscripts/multipart-quorum-test.sh b/buildscripts/multipart-quorum-test.sh new file mode 100644 index 00000000..f226b0e5 --- /dev/null +++ b/buildscripts/multipart-quorum-test.sh @@ -0,0 +1,126 @@ +#!/bin/bash + +if [ -n "$TEST_DEBUG" ]; then + set -x +fi + +WORK_DIR="$PWD/.verify-$RANDOM" +MINIO_CONFIG_DIR="$WORK_DIR/.minio" +MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server) + +if [ ! -x "$PWD/minio" ]; then + echo "minio executable binary not found in current directory" + exit 1 +fi + +if [ ! -x "$PWD/minio" ]; then + echo "minio executable binary not found in current directory" + exit 1 +fi + +trap 'catch $LINENO' ERR + +function purge() { + rm -rf "$1" +} + +# shellcheck disable=SC2120 +catch() { + if [ $# -ne 0 ]; then + echo "error on line $1" + fi + + echo "Cleaning up instances of MinIO" + pkill minio || true + pkill -9 minio || true + purge "$WORK_DIR" + if [ $# -ne 0 ]; then + exit $# + fi +} + +catch + +function start_minio_10drive() { + start_port=$1 + + export MINIO_ROOT_USER=minio + export MINIO_ROOT_PASSWORD=minio123 + export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/" + unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects + export MINIO_CI_CD=1 + + mkdir ${WORK_DIR} + C_PWD=${PWD} + if [ ! -x "$PWD/mc" ]; then + MC_BUILD_DIR="mc-$RANDOM" + if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then + echo "failed to download https://github.com/minio/mc" + purge "${MC_BUILD_DIR}" + exit 1 + fi + + (cd "${MC_BUILD_DIR}" && go build -o "$C_PWD/mc") + + # remove mc source. + purge "${MC_BUILD_DIR}" + fi + + "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/disk{1...10}" >"${WORK_DIR}/server1.log" 2>&1 & + pid=$! + disown $pid + sleep 5 + + if ! ps -p ${pid} 1>&2 >/dev/null; then + echo "server1 log:" + cat "${WORK_DIR}/server1.log" + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi + + "${PWD}/mc" mb --with-versioning minio/bucket + + export AWS_ACCESS_KEY_ID=minio + export AWS_SECRET_ACCESS_KEY=minio123 + aws --endpoint-url http://localhost:"$start_port" s3api create-multipart-upload --bucket bucket --key obj-1 >upload-id.json + uploadId=$(jq -r '.UploadId' upload-id.json) + + truncate -s 5MiB file-5mib + for i in {1..2}; do + aws --endpoint-url http://localhost:"$start_port" s3api upload-part \ + --upload-id "$uploadId" --bucket bucket --key obj-1 \ + --part-number "$i" --body ./file-5mib + done + for i in {1..6}; do + find ${WORK_DIR}/disk${i}/.minio.sys/multipart/ -type f -name "part.1" -delete + done + cat <parts.json +{ + "Parts": [ + { + "PartNumber": 1, + "ETag": "5f363e0e58a95f06cbe9bbc662c5dfb6" + }, + { + "PartNumber": 2, + "ETag": "5f363e0e58a95f06cbe9bbc662c5dfb6" + } + ] +} +EOF + err=$(aws --endpoint-url http://localhost:"$start_port" s3api complete-multipart-upload --upload-id "$uploadId" --bucket bucket --key obj-1 --multipart-upload file://./parts.json 2>&1) + rv=$? + if [ $rv -eq 0 ]; then + echo "Failed to receive an error" + exit 1 + fi + echo "Received an error during complete-multipart as expected: $err" +} + +function main() { + start_port=$(shuf -i 10000-65000 -n 1) + start_minio_10drive ${start_port} +} + +main "$@" diff --git a/buildscripts/race.sh b/buildscripts/race.sh new file mode 100755 index 00000000..2931342d --- /dev/null +++ b/buildscripts/race.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +set -e + +export GORACE="history_size=7" +export MINIO_API_REQUESTS_MAX=10000 + +for d in $(go list ./...); do + CGO_ENABLED=1 go test -v -race --timeout 100m "$d" +done diff --git a/buildscripts/resolve-right-versions.sh b/buildscripts/resolve-right-versions.sh new file mode 100755 index 00000000..afbdbb1c --- /dev/null +++ b/buildscripts/resolve-right-versions.sh @@ -0,0 +1,72 @@ +#!/bin/bash -e + +set -E +set -o pipefail +set -x +set -e + +WORK_DIR="$PWD/.verify-$RANDOM" +MINIO_CONFIG_DIR="$WORK_DIR/.minio" +MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server) + +if [ ! -x "$PWD/minio" ]; then + echo "minio executable binary not found in current directory" + exit 1 +fi + +function start_minio_5drive() { + start_port=$1 + + export MINIO_ROOT_USER=minio + export MINIO_ROOT_PASSWORD=minio123 + export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/" + unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects + export MINIO_CI_CD=1 + + MC_BUILD_DIR="mc-$RANDOM" + if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then + echo "failed to download https://github.com/minio/mc" + purge "${MC_BUILD_DIR}" + exit 1 + fi + + (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc") + + # remove mc source. + purge "${MC_BUILD_DIR}" + + "${WORK_DIR}/mc" cp --quiet -r "buildscripts/cicd-corpus/" "${WORK_DIR}/cicd-corpus/" + + "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/cicd-corpus/disk{1...5}" >"${WORK_DIR}/server1.log" 2>&1 & + pid=$! + disown $pid + sleep 5 + + if ! ps -p ${pid} 1>&2 >/dev/null; then + echo "server1 log:" + cat "${WORK_DIR}/server1.log" + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi + + "${WORK_DIR}/mc" stat minio/bucket/testobj + + pkill minio + sleep 3 +} + +function main() { + start_port=$(shuf -i 10000-65000 -n 1) + + start_minio_5drive ${start_port} +} + +function purge() { + rm -rf "$1" +} + +(main "$@") +rv=$? +purge "$WORK_DIR" +exit "$rv" diff --git a/buildscripts/rewrite-old-new.sh b/buildscripts/rewrite-old-new.sh new file mode 100755 index 00000000..3527f3a6 --- /dev/null +++ b/buildscripts/rewrite-old-new.sh @@ -0,0 +1,157 @@ +#!/bin/bash -e + +set -E +set -o pipefail +set -x + +WORK_DIR="$PWD/.verify-$RANDOM" +MINIO_CONFIG_DIR="$WORK_DIR/.minio" +MINIO_OLD=("$PWD/minio.RELEASE.2020-10-28T08-16-50Z" --config-dir "$MINIO_CONFIG_DIR" server) +MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server) + +if [ ! -x "$PWD/minio" ]; then + echo "minio executable binary not found in current directory" + exit 1 +fi + +function download_old_release() { + if [ ! -f minio.RELEASE.2020-10-28T08-16-50Z ]; then + curl --silent -O https://dl.minio.io/server/minio/release/linux-amd64/archive/minio.RELEASE.2020-10-28T08-16-50Z + chmod a+x minio.RELEASE.2020-10-28T08-16-50Z + fi +} + +function verify_rewrite() { + start_port=$1 + + export MINIO_ACCESS_KEY=minio + export MINIO_SECRET_KEY=minio123 + export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/" + unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects + export MINIO_CI_CD=1 + + MC_BUILD_DIR="mc-$RANDOM" + if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then + echo "failed to download https://github.com/minio/mc" + purge "${MC_BUILD_DIR}" + exit 1 + fi + + (cd "${MC_BUILD_DIR}" && go build -o "$WORK_DIR/mc") + + # remove mc source. + purge "${MC_BUILD_DIR}" + + "${MINIO_OLD[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 & + pid=$! + disown $pid + + "${WORK_DIR}/mc" ready minio/ + + if ! ps -p ${pid} 1>&2 >/dev/null; then + echo "server1 log:" + cat "${WORK_DIR}/server1.log" + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi + + "${WORK_DIR}/mc" mb minio/healing-rewrite-bucket --quiet --with-lock + "${WORK_DIR}/mc" cp \ + buildscripts/verify-build.sh \ + minio/healing-rewrite-bucket/ \ + --disable-multipart --quiet + + "${WORK_DIR}/mc" cp \ + buildscripts/verify-build.sh \ + minio/healing-rewrite-bucket/ \ + --disable-multipart --quiet + + "${WORK_DIR}/mc" cp \ + buildscripts/verify-build.sh \ + minio/healing-rewrite-bucket/ \ + --disable-multipart --quiet + + kill ${pid} + sleep 3 + + "${MINIO[@]}" --address ":$start_port" "${WORK_DIR}/xl{1...16}" >"${WORK_DIR}/server1.log" 2>&1 & + pid=$! + disown $pid + + "${WORK_DIR}/mc" ready minio/ + + if ! ps -p ${pid} 1>&2 >/dev/null; then + echo "server1 log:" + cat "${WORK_DIR}/server1.log" + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi + + if ! ./s3-check-md5 \ + -debug \ + -versions \ + -access-key minio \ + -secret-key minio123 \ + -endpoint "http://127.0.0.1:${start_port}/" 2>&1 | grep INTACT; then + echo "server1 log:" + cat "${WORK_DIR}/server1.log" + echo "FAILED" + mkdir -p inspects + ( + cd inspects + "${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/** + ) + + "${WORK_DIR}/mc" mb play/inspects + "${WORK_DIR}/mc" mirror inspects play/inspects + + purge "$WORK_DIR" + exit 1 + fi + + go run ./buildscripts/heal-manual.go "127.0.0.1:${start_port}" "minio" "minio123" + sleep 1 + + if ! ./s3-check-md5 \ + -debug \ + -versions \ + -access-key minio \ + -secret-key minio123 \ + -endpoint http://127.0.0.1:${start_port}/ 2>&1 | grep INTACT; then + echo "server1 log:" + cat "${WORK_DIR}/server1.log" + echo "FAILED" + mkdir -p inspects + ( + cd inspects + "${WORK_DIR}/mc" admin inspect minio/healing-rewrite-bucket/verify-build.sh/** + ) + + "${WORK_DIR}/mc" mb play/inspects + "${WORK_DIR}/mc" mirror inspects play/inspects + + purge "$WORK_DIR" + exit 1 + fi + + kill ${pid} +} + +function main() { + download_old_release + + start_port=$(shuf -i 10000-65000 -n 1) + + verify_rewrite ${start_port} +} + +function purge() { + rm -rf "$1" +} + +(main "$@") +rv=$? +purge "$WORK_DIR" +exit "$rv" diff --git a/buildscripts/test-timeout.sh b/buildscripts/test-timeout.sh new file mode 100644 index 00000000..77a248a6 --- /dev/null +++ b/buildscripts/test-timeout.sh @@ -0,0 +1,137 @@ +#!/bin/bash + +if [ -n "$TEST_DEBUG" ]; then + set -x +fi + +WORK_DIR="$PWD/.verify-$RANDOM" +MINIO_CONFIG_DIR="$WORK_DIR/.minio" +MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server) + +if [ ! -x "$PWD/minio" ]; then + echo "minio executable binary not found in current directory" + exit 1 +fi + +if [ ! -x "$PWD/minio" ]; then + echo "minio executable binary not found in current directory" + exit 1 +fi + +trap 'catch $LINENO' ERR + +function purge() { + rm -rf "$1" +} + +# shellcheck disable=SC2120 +catch() { + if [ $# -ne 0 ]; then + echo "error on line $1" + fi + + echo "Cleaning up instances of MinIO" + pkill minio || true + pkill -9 minio || true + purge "$WORK_DIR" + if [ $# -ne 0 ]; then + exit $# + fi +} + +catch + +function gen_put_request() { + hdr_sleep=$1 + body_sleep=$2 + + echo "PUT /testbucket/testobject HTTP/1.1" + sleep $hdr_sleep + echo "Host: foo-header" + echo "User-Agent: curl/8.2.1" + echo "Accept: */*" + echo "Content-Length: 30" + echo "" + + sleep $body_sleep + echo "random line 0" + echo "random line 1" + echo "" + echo "" +} + +function send_put_object_request() { + hdr_timeout=$1 + body_timeout=$2 + + start=$(date +%s) + timeout 5m bash -c "gen_put_request $hdr_timeout $body_timeout | netcat 127.0.0.1 $start_port | read" || return -1 + [ $(($(date +%s) - start)) -gt $((srv_hdr_timeout + srv_idle_timeout + 1)) ] && return -1 + return 0 +} + +function test_minio_with_timeout() { + start_port=$1 + + export MINIO_ROOT_USER=minio + export MINIO_ROOT_PASSWORD=minio123 + export MC_HOST_minio="http://minio:minio123@127.0.0.1:${start_port}/" + export MINIO_CI_CD=1 + + mkdir ${WORK_DIR} + C_PWD=${PWD} + if [ ! -x "$PWD/mc" ]; then + MC_BUILD_DIR="mc-$RANDOM" + if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then + echo "failed to download https://github.com/minio/mc" + purge "${MC_BUILD_DIR}" + exit 1 + fi + + (cd "${MC_BUILD_DIR}" && go build -o "$C_PWD/mc") + + # remove mc source. + purge "${MC_BUILD_DIR}" + fi + + "${MINIO[@]}" --address ":$start_port" --read-header-timeout ${srv_hdr_timeout}s --idle-timeout ${srv_idle_timeout}s "${WORK_DIR}/disk/" >"${WORK_DIR}/server1.log" 2>&1 & + pid=$! + disown $pid + sleep 1 + + if ! ps -p ${pid} 1>&2 >/dev/null; then + echo "server1 log:" + cat "${WORK_DIR}/server1.log" + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi + + set -e + + "${PWD}/mc" mb minio/testbucket + "${PWD}/mc" anonymous set public minio/testbucket + + # slow header writing + send_put_object_request 20 0 && exit -1 + "${PWD}/mc" stat minio/testbucket/testobject && exit -1 + + # quick header write and slow bodywrite + send_put_object_request 0 40 && exit -1 + "${PWD}/mc" stat minio/testbucket/testobject && exit -1 + + # quick header and body write + send_put_object_request 1 1 || exit -1 + "${PWD}/mc" stat minio/testbucket/testobject || exit -1 +} + +function main() { + export start_port=$(shuf -i 10000-65000 -n 1) + export srv_hdr_timeout=5 + export srv_idle_timeout=5 + export -f gen_put_request + + test_minio_with_timeout ${start_port} +} + +main "$@" diff --git a/buildscripts/upgrade-tests/compose.yml b/buildscripts/upgrade-tests/compose.yml new file mode 100644 index 00000000..e820a6c5 --- /dev/null +++ b/buildscripts/upgrade-tests/compose.yml @@ -0,0 +1,74 @@ +# Settings and configurations that are common for all containers +x-minio-common: &minio-common + image: minio/minio:${MINIO_VERSION} + command: server http://minio{1...4}/data{1...3} + env_file: + - ./minio.env + expose: + - "9000" + - "9001" + +# starts 4 docker containers running minio server instances. +# using nginx reverse proxy, load balancing, you can access +# it through port 9000. +services: + minio1: + <<: *minio-common + hostname: minio1 + volumes: + - data1-1:/data1 + - data1-2:/data2 + - data1-3:/data3 + + minio2: + <<: *minio-common + hostname: minio2 + volumes: + - data2-1:/data1 + - data2-2:/data2 + - data2-3:/data3 + + minio3: + <<: *minio-common + hostname: minio3 + volumes: + - data3-1:/data1 + - data3-2:/data2 + - data3-3:/data3 + + minio4: + <<: *minio-common + hostname: minio4 + volumes: + - data4-1:/data1 + - data4-2:/data2 + - data4-3:/data3 + + nginx: + image: nginx:1.19.2-alpine + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + ports: + - "9000:9000" + - "9001:9001" + depends_on: + - minio1 + - minio2 + - minio3 + - minio4 + +## By default this config uses default local driver, +## For custom volumes replace with volume driver configuration. +volumes: + data1-1: + data1-2: + data1-3: + data2-1: + data2-2: + data2-3: + data3-1: + data3-2: + data3-3: + data4-1: + data4-2: + data4-3: diff --git a/buildscripts/upgrade-tests/minio.env b/buildscripts/upgrade-tests/minio.env new file mode 100644 index 00000000..a957766f --- /dev/null +++ b/buildscripts/upgrade-tests/minio.env @@ -0,0 +1,3 @@ +MINIO_ACCESS_KEY=minioadmin +MINIO_SECRET_KEY=minioadmin +MINIO_BROWSER=off diff --git a/buildscripts/upgrade-tests/nginx.conf b/buildscripts/upgrade-tests/nginx.conf new file mode 100644 index 00000000..0c46dc92 --- /dev/null +++ b/buildscripts/upgrade-tests/nginx.conf @@ -0,0 +1,68 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + # include /etc/nginx/conf.d/*.conf; + + upstream minio { + server minio1:9000; + server minio2:9000; + server minio3:9000; + server minio4:9000; + } + + # main minio + server { + listen 9000; + listen [::]:9000; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio; + } + } +} diff --git a/buildscripts/verify-build.sh b/buildscripts/verify-build.sh new file mode 100755 index 00000000..e15d6475 --- /dev/null +++ b/buildscripts/verify-build.sh @@ -0,0 +1,298 @@ +#!/bin/bash +# + +set -e +set -E +set -o pipefail + +if [ ! -x "$PWD/minio" ]; then + echo "minio executable binary not found in current directory" + exit 1 +fi + +WORK_DIR="$PWD/.verify-$RANDOM" + +export MINT_MODE=core +export MINT_DATA_DIR="$WORK_DIR/data" +export SERVER_ENDPOINT="127.0.0.1:9000" +export MC_HOST_verify="http://minio:minio123@${SERVER_ENDPOINT}/" +export MC_HOST_verify_ipv6="http://minio:minio123@[::1]:9000/" +export ACCESS_KEY="minio" +export SECRET_KEY="minio123" +export ENABLE_HTTPS=0 +export GO111MODULE=on +export GOGC=25 +export ENABLE_ADMIN=1 +export MINIO_CI_CD=1 + +MINIO_CONFIG_DIR="$WORK_DIR/.minio" +MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR") + +FILE_1_MB="$MINT_DATA_DIR/datafile-1-MB" +FILE_65_MB="$MINT_DATA_DIR/datafile-65-MB" + +FUNCTIONAL_TESTS="$WORK_DIR/functional-tests.sh" + +function start_minio_fs() { + export MINIO_ROOT_USER=$ACCESS_KEY + export MINIO_ROOT_PASSWORD=$SECRET_KEY + "${MINIO[@]}" server "${WORK_DIR}/fs-disk" >"$WORK_DIR/fs-minio.log" 2>&1 & + + "${WORK_DIR}/mc" ready verify +} + +function start_minio_erasure() { + "${MINIO[@]}" server "${WORK_DIR}/erasure-disk1" "${WORK_DIR}/erasure-disk2" "${WORK_DIR}/erasure-disk3" "${WORK_DIR}/erasure-disk4" >"$WORK_DIR/erasure-minio.log" 2>&1 & + + "${WORK_DIR}/mc" ready verify +} + +function start_minio_erasure_sets() { + export MINIO_ENDPOINTS="${WORK_DIR}/erasure-disk-sets{1...32}" + "${MINIO[@]}" server >"$WORK_DIR/erasure-minio-sets.log" 2>&1 & + + "${WORK_DIR}/mc" ready verify +} + +function start_minio_pool_erasure_sets() { + export MINIO_ROOT_USER=$ACCESS_KEY + export MINIO_ROOT_PASSWORD=$SECRET_KEY + export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/pool-disk-sets{1...4} http://127.0.0.1:9001${WORK_DIR}/pool-disk-sets{5...8}" + "${MINIO[@]}" server --address ":9000" >"$WORK_DIR/pool-minio-9000.log" 2>&1 & + "${MINIO[@]}" server --address ":9001" >"$WORK_DIR/pool-minio-9001.log" 2>&1 & + + "${WORK_DIR}/mc" ready verify +} + +function start_minio_pool_erasure_sets_ipv6() { + export MINIO_ROOT_USER=$ACCESS_KEY + export MINIO_ROOT_PASSWORD=$SECRET_KEY + export MINIO_ENDPOINTS="http://[::1]:9000${WORK_DIR}/pool-disk-sets-ipv6{1...4} http://[::1]:9001${WORK_DIR}/pool-disk-sets-ipv6{5...8}" + "${MINIO[@]}" server --address="[::1]:9000" >"$WORK_DIR/pool-minio-ipv6-9000.log" 2>&1 & + "${MINIO[@]}" server --address="[::1]:9001" >"$WORK_DIR/pool-minio-ipv6-9001.log" 2>&1 & + + "${WORK_DIR}/mc" ready verify_ipv6 +} + +function start_minio_dist_erasure() { + export MINIO_ROOT_USER=$ACCESS_KEY + export MINIO_ROOT_PASSWORD=$SECRET_KEY + export MINIO_ENDPOINTS="http://127.0.0.1:9000${WORK_DIR}/dist-disk1 http://127.0.0.1:9001${WORK_DIR}/dist-disk2 http://127.0.0.1:9002${WORK_DIR}/dist-disk3 http://127.0.0.1:9003${WORK_DIR}/dist-disk4" + for i in $(seq 0 3); do + "${MINIO[@]}" server --address ":900${i}" >"$WORK_DIR/dist-minio-900${i}.log" 2>&1 & + done + + "${WORK_DIR}/mc" ready verify +} + +function run_test_fs() { + start_minio_fs + + (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS") + rv=$? + + pkill minio + sleep 3 + + if [ "$rv" -ne 0 ]; then + cat "$WORK_DIR/fs-minio.log" + fi + rm -f "$WORK_DIR/fs-minio.log" + + return "$rv" +} + +function run_test_erasure_sets() { + start_minio_erasure_sets + + (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS") + rv=$? + + pkill minio + sleep 3 + + if [ "$rv" -ne 0 ]; then + cat "$WORK_DIR/erasure-minio-sets.log" + fi + rm -f "$WORK_DIR/erasure-minio-sets.log" + + return "$rv" +} + +function run_test_pool_erasure_sets() { + start_minio_pool_erasure_sets + + (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS") + rv=$? + + pkill minio + sleep 3 + + if [ "$rv" -ne 0 ]; then + for i in $(seq 0 1); do + echo "server$i log:" + cat "$WORK_DIR/pool-minio-900$i.log" + done + fi + + for i in $(seq 0 1); do + rm -f "$WORK_DIR/pool-minio-900$i.log" + done + + return "$rv" +} + +function run_test_pool_erasure_sets_ipv6() { + start_minio_pool_erasure_sets_ipv6 + + export SERVER_ENDPOINT="[::1]:9000" + + (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS") + rv=$? + + pkill minio + sleep 3 + + if [ "$rv" -ne 0 ]; then + for i in $(seq 0 1); do + echo "server$i log:" + cat "$WORK_DIR/pool-minio-ipv6-900$i.log" + done + fi + + for i in $(seq 0 1); do + rm -f "$WORK_DIR/pool-minio-ipv6-900$i.log" + done + + return "$rv" +} + +function run_test_erasure() { + start_minio_erasure + + (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS") + rv=$? + + pkill minio + sleep 3 + + if [ "$rv" -ne 0 ]; then + cat "$WORK_DIR/erasure-minio.log" + fi + rm -f "$WORK_DIR/erasure-minio.log" + + return "$rv" +} + +function run_test_dist_erasure() { + start_minio_dist_erasure + + (cd "$WORK_DIR" && "$FUNCTIONAL_TESTS") + rv=$? + + pkill minio + sleep 3 + + if [ "$rv" -ne 0 ]; then + echo "server1 log:" + cat "$WORK_DIR/dist-minio-9000.log" + echo "server2 log:" + cat "$WORK_DIR/dist-minio-9001.log" + echo "server3 log:" + cat "$WORK_DIR/dist-minio-9002.log" + echo "server4 log:" + cat "$WORK_DIR/dist-minio-9003.log" + fi + + rm -f "$WORK_DIR/dist-minio-9000.log" "$WORK_DIR/dist-minio-9001.log" "$WORK_DIR/dist-minio-9002.log" "$WORK_DIR/dist-minio-9003.log" + + return "$rv" +} + +function purge() { + rm -rf "$1" +} + +function __init__() { + echo "Initializing environment" + mkdir -p "$WORK_DIR" + mkdir -p "$MINIO_CONFIG_DIR" + mkdir -p "$MINT_DATA_DIR" + + MC_BUILD_DIR="mc-$RANDOM" + if ! git clone --quiet https://github.com/minio/mc "$MC_BUILD_DIR"; then + echo "failed to download https://github.com/minio/mc" + purge "${MC_BUILD_DIR}" + exit 1 + fi + + (cd "${MC_BUILD_DIR}" && go build -o "${WORK_DIR}/mc") + + # remove mc source. + purge "${MC_BUILD_DIR}" + + shred -n 1 -s 1M - 1>"$FILE_1_MB" 2>/dev/null + shred -n 1 -s 65M - 1>"$FILE_65_MB" 2>/dev/null + + ## version is purposefully set to '3' for minio to migrate configuration file + echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' >"$MINIO_CONFIG_DIR/config.json" + + if ! wget -q -O "$FUNCTIONAL_TESTS" https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh; then + echo "failed to download https://raw.githubusercontent.com/minio/mc/master/functional-tests.sh" + exit 1 + fi + + sed -i 's|-sS|-sSg|g' "$FUNCTIONAL_TESTS" + chmod a+x "$FUNCTIONAL_TESTS" +} + +function main() { + echo "Testing in FS setup" + if ! run_test_fs; then + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi + + echo "Testing in Erasure setup" + if ! run_test_erasure; then + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi + + echo "Testing in Distributed Erasure setup" + if ! run_test_dist_erasure; then + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi + + echo "Testing in Erasure setup as sets" + if ! run_test_erasure_sets; then + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi + + echo "Testing in Distributed Eraure expanded setup" + if ! run_test_pool_erasure_sets; then + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi + + echo "Testing in Distributed Erasure expanded setup with ipv6" + if ! run_test_pool_erasure_sets_ipv6; then + echo "FAILED" + purge "$WORK_DIR" + exit 1 + fi + + purge "$WORK_DIR" +} + +(__init__ "$@" && main "$@") +rv=$? +purge "$WORK_DIR" +exit "$rv" diff --git a/buildscripts/verify-healing-empty-erasure-set.sh b/buildscripts/verify-healing-empty-erasure-set.sh new file mode 100755 index 00000000..ddbbc1c0 --- /dev/null +++ b/buildscripts/verify-healing-empty-erasure-set.sh @@ -0,0 +1,151 @@ +#!/bin/bash -e +# + +set -E +set -o pipefail + +if [ ! -x "$PWD/minio" ]; then + echo "minio executable binary not found in current directory" + exit 1 +fi + +WORK_DIR="$PWD/.verify-$RANDOM" +MINIO_CONFIG_DIR="$WORK_DIR/.minio" +MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server) + +function start_minio_3_node() { + export MINIO_ROOT_USER=minio + export MINIO_ROOT_PASSWORD=minio123 + export MINIO_ERASURE_SET_DRIVE_COUNT=6 + export MINIO_CI_CD=1 + + start_port=$1 + args="" + for i in $(seq 1 3); do + args="$args http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/1/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/2/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/3/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/4/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/5/ http://127.0.0.1:$((start_port + i))${WORK_DIR}/$i/6/" + done + + "${MINIO[@]}" --address ":$((start_port + 1))" $args >"${WORK_DIR}/dist-minio-server1.log" 2>&1 & + pid1=$! + disown ${pid1} + + "${MINIO[@]}" --address ":$((start_port + 2))" $args >"${WORK_DIR}/dist-minio-server2.log" 2>&1 & + pid2=$! + disown $pid2 + + "${MINIO[@]}" --address ":$((start_port + 3))" $args >"${WORK_DIR}/dist-minio-server3.log" 2>&1 & + pid3=$! + disown $pid3 + + export MC_HOST_myminio="http://minio:minio123@127.0.0.1:$((start_port + 1))" + + timeout 15m /tmp/mc ready myminio || fail + + # Wait for all drives to be online and formatted + while [ $(/tmp/mc admin info --json myminio | jq '.info.servers[].drives[].state | select(. != "ok")' | wc -l) -gt 0 ]; do sleep 1; done + # Wait for all drives to be healed + while [ $(/tmp/mc admin info --json myminio | jq '.info.servers[].drives[].healing | select(. != null) | select(. == true)' | wc -l) -gt 0 ]; do sleep 1; done + + # Wait for Status: in MinIO output + while true; do + rv=$(check_online) + if [ "$rv" != "1" ]; then + # success + break + fi + + # Check if we should retry + retry=$((retry + 1)) + if [ $retry -le 20 ]; then + sleep 5 + continue + fi + + # Failure + fail + done + + if ! ps -p $pid1 1>&2 >/dev/null; then + echo "minio-server-1 is not running." && fail + fi + + if ! ps -p $pid2 1>&2 >/dev/null; then + echo "minio-server-2 is not running." && fail + fi + + if ! ps -p $pid3 1>&2 >/dev/null; then + echo "minio-server-3 is not running." && fail + fi + + if ! pkill minio; then + fail + fi + + sleep 1 + if pgrep minio; then + # forcibly killing, to proceed further properly. + if ! pkill -9 minio; then + echo "no minio process running anymore, proceed." + fi + fi +} + +function fail() { + for i in $(seq 1 3); do + echo "server$i log:" + cat "${WORK_DIR}/dist-minio-server$i.log" + done + echo "FAILED" + purge "$WORK_DIR" + exit 1 +} + +function check_online() { + if ! grep -q 'API:' ${WORK_DIR}/dist-minio-*.log; then + echo "1" + fi +} + +function purge() { + echo rm -rf "$1" +} + +function __init__() { + echo "Initializing environment" + mkdir -p "$WORK_DIR" + mkdir -p "$MINIO_CONFIG_DIR" + + ## version is purposefully set to '3' for minio to migrate configuration file + echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' >"$MINIO_CONFIG_DIR/config.json" + + if [ ! -f /tmp/mc ]; then + wget --quiet -O /tmp/mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x /tmp/mc + fi +} + +function perform_test() { + start_minio_3_node $2 + + echo "Testing Distributed Erasure setup healing of drives" + echo "Remove the contents of the disks belonging to '${1}' erasure set" + + rm -rf ${WORK_DIR}/${1}/*/ + + set -x + start_minio_3_node $2 +} + +function main() { + # use same ports for all tests + start_port=$(shuf -i 10000-65000 -n 1) + + perform_test "2" ${start_port} + perform_test "1" ${start_port} + perform_test "3" ${start_port} +} + +(__init__ "$@" && main "$@") +rv=$? +purge "$WORK_DIR" +exit "$rv" diff --git a/buildscripts/verify-healing-with-root-disks.sh b/buildscripts/verify-healing-with-root-disks.sh new file mode 100755 index 00000000..6166eddd --- /dev/null +++ b/buildscripts/verify-healing-with-root-disks.sh @@ -0,0 +1,96 @@ +#!/bin/bash -e + +set -E +set -o pipefail +set -x + +if [ ! -x "$PWD/minio" ]; then + echo "minio executable binary not found in current directory" + exit 1 +fi + +WORK_DIR="$(mktemp -d)" +MINIO_CONFIG_DIR="$WORK_DIR/.minio" +MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server) + +function start_minio() { + start_port=$1 + + export MINIO_ROOT_USER=minio + export MINIO_ROOT_PASSWORD=minio123 + unset MINIO_KMS_AUTO_ENCRYPTION # do not auto-encrypt objects + unset MINIO_CI_CD + unset CI + + args=() + for i in $(seq 1 4); do + args+=("http://localhost:$((start_port + i))${WORK_DIR}/mnt/disk$i/ ") + done + + for i in $(seq 1 4); do + "${MINIO[@]}" --address ":$((start_port + i))" ${args[@]} 2>&1 >"${WORK_DIR}/server$i.log" & + done + + # Wait until all nodes return 403 + for i in $(seq 1 4); do + while [ "$(curl -m 1 -s -o /dev/null -w "%{http_code}" http://localhost:$((start_port + i)))" -ne "403" ]; do + echo -n "." + sleep 1 + done + done + +} + +# Prepare fake disks with losetup +function prepare_block_devices() { + set -e + mkdir -p ${WORK_DIR}/disks/ ${WORK_DIR}/mnt/ + sudo modprobe loop + for i in 1 2 3 4; do + dd if=/dev/zero of=${WORK_DIR}/disks/img.${i} bs=1M count=2000 + device=$(sudo losetup --find --show ${WORK_DIR}/disks/img.${i}) + sudo mkfs.ext4 -F ${device} + mkdir -p ${WORK_DIR}/mnt/disk${i}/ + sudo mount ${device} ${WORK_DIR}/mnt/disk${i}/ + sudo chown "$(id -u):$(id -g)" ${device} ${WORK_DIR}/mnt/disk${i}/ + done + set +e +} + +# Start a distributed MinIO setup, unmount one disk and check if it is formatted +function main() { + start_port=$(shuf -i 10000-65000 -n 1) + start_minio ${start_port} + + # Unmount the disk, after the unmount the device id + # /tmp/xxx/mnt/disk4 will be the same as '/' and it + # will be detected as root disk + while [ "$u" != "0" ]; do + sudo umount ${WORK_DIR}/mnt/disk4/ + u=$? + sleep 1 + done + + # Wait until MinIO self heal kicks in + sleep 60 + + if [ -f ${WORK_DIR}/mnt/disk4/.minio.sys/format.json ]; then + echo "A root disk is formatted unexpectedely" + cat "${WORK_DIR}/server4.log" + exit -1 + fi +} + +function cleanup() { + pkill minio + sudo umount ${WORK_DIR}/mnt/disk{1..3}/ + sudo rm /dev/minio-loopdisk* + rm -rf "$WORK_DIR" +} + +(prepare_block_devices) +(main "$@") +rv=$? + +cleanup +exit "$rv" diff --git a/buildscripts/verify-healing.sh b/buildscripts/verify-healing.sh new file mode 100755 index 00000000..66778c17 --- /dev/null +++ b/buildscripts/verify-healing.sh @@ -0,0 +1,167 @@ +#!/bin/bash -e +# + +set -E +set -o pipefail + +if [ ! -x "$PWD/minio" ]; then + echo "minio executable binary not found in current directory" + exit 1 +fi + +WORK_DIR="$PWD/.verify-$RANDOM" +MINIO_CONFIG_DIR="$WORK_DIR/.minio" +MINIO=("$PWD/minio" --config-dir "$MINIO_CONFIG_DIR" server) +GOPATH=/tmp/gopath + +function start_minio_3_node() { + for i in $(seq 1 3); do + rm "${WORK_DIR}/dist-minio-server$i.log" + done + + export MINIO_ROOT_USER=minio + export MINIO_ROOT_PASSWORD=minio123 + export MINIO_ERASURE_SET_DRIVE_COUNT=6 + export MINIO_CI_CD=1 + + first_time=$(find ${WORK_DIR}/ | grep format.json | wc -l) + + start_port=$1 + args="" + for d in $(seq 1 3 5); do + args="$args http://127.0.0.1:$((start_port + 1))${WORK_DIR}/1/${d}/ http://127.0.0.1:$((start_port + 2))${WORK_DIR}/2/${d}/ http://127.0.0.1:$((start_port + 3))${WORK_DIR}/3/${d}/ " + d=$((d + 1)) + args="$args http://127.0.0.1:$((start_port + 1))${WORK_DIR}/1/${d}/ http://127.0.0.1:$((start_port + 2))${WORK_DIR}/2/${d}/ http://127.0.0.1:$((start_port + 3))${WORK_DIR}/3/${d}/ " + done + + "${MINIO[@]}" --address ":$((start_port + 1))" $args >"${WORK_DIR}/dist-minio-server1.log" 2>&1 & + pid1=$! + disown ${pid1} + + "${MINIO[@]}" --address ":$((start_port + 2))" $args >"${WORK_DIR}/dist-minio-server2.log" 2>&1 & + pid2=$! + disown $pid2 + + "${MINIO[@]}" --address ":$((start_port + 3))" $args >"${WORK_DIR}/dist-minio-server3.log" 2>&1 & + pid3=$! + disown $pid3 + + export MC_HOST_myminio="http://minio:minio123@127.0.0.1:$((start_port + 1))" + timeout 15m /tmp/mc ready myminio || fail + + [ ${first_time} -eq 0 ] && upload_objects + [ ${first_time} -ne 0 ] && sleep 120 + + if ! ps -p $pid1 1>&2 >/dev/null; then + echo "minio server 1 is not running" && fail + fi + + if ! ps -p $pid2 1>&2 >/dev/null; then + echo "minio server 2 is not running" && fail + fi + + if ! ps -p $pid3 1>&2 >/dev/null; then + echo "minio server 3 is not running" && fail + fi + + if ! pkill minio; then + fail + fi + + sleep 1 + if pgrep minio; then + # forcibly killing, to proceed further properly. + if ! pkill -9 minio; then + echo "no minio process running anymore, proceed." + fi + fi +} + +function check_heal() { + if ! grep -q 'API:' ${WORK_DIR}/dist-minio-*.log; then + return 1 + fi + + for ((i = 0; i < 20; i++)); do + test -f ${WORK_DIR}/$1/1/.minio.sys/format.json + v1=$? + nextInES=$(($1 + 1)) && [ $nextInES -gt 3 ] && nextInES=1 + foundFiles1=$(find ${WORK_DIR}/$1/1/ | grep -v .minio.sys | grep xl.meta | wc -l) + foundFiles2=$(find ${WORK_DIR}/$nextInES/1/ | grep -v .minio.sys | grep xl.meta | wc -l) + test $foundFiles1 -eq $foundFiles2 + v2=$? + [ $v1 == 0 -a $v2 == 0 ] && return 0 + sleep 10 + done + return 1 +} + +function purge() { + rm -rf "$1" +} + +function fail() { + for i in $(seq 1 3); do + echo "server$i log:" + cat "${WORK_DIR}/dist-minio-server$i.log" + done + pkill -9 minio + echo "FAILED" + purge "$WORK_DIR" + exit 1 +} + +function __init__() { + echo "Initializing environment" + mkdir -p "$WORK_DIR" + mkdir -p "$MINIO_CONFIG_DIR" + + ## version is purposefully set to '3' for minio to migrate configuration file + echo '{"version": "3", "credential": {"accessKey": "minio", "secretKey": "minio123"}, "region": "us-east-1"}' >"$MINIO_CONFIG_DIR/config.json" + + if [ ! -f /tmp/mc ]; then + wget --quiet -O /tmp/mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x /tmp/mc + fi +} + +function upload_objects() { + /tmp/mc mb myminio/testbucket/ + for ((i = 0; i < 20; i++)); do + echo "my content" | /tmp/mc pipe myminio/testbucket/file-$i + done +} + +function perform_test() { + start_port=$2 + + start_minio_3_node $start_port + + echo "Testing Distributed Erasure setup healing of drives" + echo "Remove the contents of the disks belonging to '${1}' node" + + rm -rf ${WORK_DIR}/${1}/*/ + + set -x + start_minio_3_node $start_port + + check_heal ${1} + rv=$? + if [ "$rv" == "1" ]; then + fail + fi +} + +function main() { + # use same ports for all tests + start_port=$(shuf -i 10000-65000 -n 1) + + perform_test "2" ${start_port} + perform_test "1" ${start_port} + perform_test "3" ${start_port} +} + +(__init__ "$@" && main "$@") +rv=$? +purge "$WORK_DIR" +exit "$rv" diff --git a/cmd/acl-handlers.go b/cmd/acl-handlers.go new file mode 100644 index 00000000..eb1f3c1e --- /dev/null +++ b/cmd/acl-handlers.go @@ -0,0 +1,280 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "encoding/xml" + "io" + "net/http" + + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" + "github.com/minio/mux" + "github.com/minio/pkg/v3/policy" +) + +// Data types used for returning dummy access control +// policy XML, these variables shouldn't be used elsewhere +// they are only defined to be used in this file alone. +type grantee struct { + XMLNS string `xml:"xmlns:xsi,attr"` + XMLXSI string `xml:"xsi:type,attr"` + Type string `xml:"Type"` + ID string `xml:"ID,omitempty"` + DisplayName string `xml:"DisplayName,omitempty"` + URI string `xml:"URI,omitempty"` +} + +type grant struct { + Grantee grantee `xml:"Grantee"` + Permission string `xml:"Permission"` +} + +type accessControlPolicy struct { + XMLName xml.Name `xml:"AccessControlPolicy"` + Owner Owner `xml:"Owner"` + AccessControlList struct { + Grants []grant `xml:"Grant"` + } `xml:"AccessControlList"` +} + +// PutBucketACLHandler - PUT Bucket ACL +// ----------------- +// This operation uses the ACL subresource +// to set ACL for a bucket, this is a dummy call +// only responds success if the ACL is private. +func (api objectAPIHandlers) PutBucketACLHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PutBucketACL") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + // Allow putBucketACL if policy action is set, since this is a dummy call + // we are simply re-purposing the bucketPolicyAction. + if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketPolicyAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Before proceeding validate if bucket exists. + _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + aclHeader := r.Header.Get(xhttp.AmzACL) + if aclHeader == "" { + acl := &accessControlPolicy{} + if err = xmlDecoder(r.Body, acl, r.ContentLength); err != nil { + if terr, ok := err.(*xml.SyntaxError); ok && terr.Msg == io.EOF.Error() { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), + r.URL) + return + } + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if len(acl.AccessControlList.Grants) == 0 { + writeErrorResponse(ctx, w, toAPIError(ctx, NotImplemented{}), r.URL) + return + } + + if acl.AccessControlList.Grants[0].Permission != "FULL_CONTROL" { + writeErrorResponse(ctx, w, toAPIError(ctx, NotImplemented{}), r.URL) + return + } + } + + if aclHeader != "" && aclHeader != "private" { + writeErrorResponse(ctx, w, toAPIError(ctx, NotImplemented{}), r.URL) + return + } +} + +// GetBucketACLHandler - GET Bucket ACL +// ----------------- +// This operation uses the ACL +// subresource to return the ACL of a specified bucket. +func (api objectAPIHandlers) GetBucketACLHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetBucketACL") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + // Allow getBucketACL if policy action is set, since this is a dummy call + // we are simply re-purposing the bucketPolicyAction. + if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Before proceeding validate if bucket exists. + _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + acl := &accessControlPolicy{} + acl.AccessControlList.Grants = append(acl.AccessControlList.Grants, grant{ + Grantee: grantee{ + XMLNS: "http://www.w3.org/2001/XMLSchema-instance", + XMLXSI: "CanonicalUser", + Type: "CanonicalUser", + }, + Permission: "FULL_CONTROL", + }) + + if err := xml.NewEncoder(w).Encode(acl); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } +} + +// PutObjectACLHandler - PUT Object ACL +// ----------------- +// This operation uses the ACL subresource +// to set ACL for a bucket, this is a dummy call +// only responds success if the ACL is private. +func (api objectAPIHandlers) PutObjectACLHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PutObjectACL") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + // Allow putObjectACL if policy action is set, since this is a dummy call + // we are simply re-purposing the bucketPolicyAction. + if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketPolicyAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Before proceeding validate if object exists. + _, err = objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{}) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + aclHeader := r.Header.Get(xhttp.AmzACL) + if aclHeader == "" { + acl := &accessControlPolicy{} + if err = xmlDecoder(r.Body, acl, r.ContentLength); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if len(acl.AccessControlList.Grants) == 0 { + writeErrorResponse(ctx, w, toAPIError(ctx, NotImplemented{}), r.URL) + return + } + + if acl.AccessControlList.Grants[0].Permission != "FULL_CONTROL" { + writeErrorResponse(ctx, w, toAPIError(ctx, NotImplemented{}), r.URL) + return + } + } + + if aclHeader != "" && aclHeader != "private" { + writeErrorResponse(ctx, w, toAPIError(ctx, NotImplemented{}), r.URL) + return + } +} + +// GetObjectACLHandler - GET Object ACL +// ----------------- +// This operation uses the ACL +// subresource to return the ACL of a specified object. +func (api objectAPIHandlers) GetObjectACLHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetObjectACL") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + // Allow getObjectACL if policy action is set, since this is a dummy call + // we are simply re-purposing the bucketPolicyAction. + if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Before proceeding validate if object exists. + _, err = objAPI.GetObjectInfo(ctx, bucket, object, ObjectOptions{}) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + acl := &accessControlPolicy{} + acl.AccessControlList.Grants = append(acl.AccessControlList.Grants, grant{ + Grantee: grantee{ + XMLNS: "http://www.w3.org/2001/XMLSchema-instance", + XMLXSI: "CanonicalUser", + Type: "CanonicalUser", + }, + Permission: "FULL_CONTROL", + }) + if err := xml.NewEncoder(w).Encode(acl); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } +} diff --git a/cmd/admin-bucket-handlers.go b/cmd/admin-bucket-handlers.go new file mode 100644 index 00000000..4ea93878 --- /dev/null +++ b/cmd/admin-bucket-handlers.go @@ -0,0 +1,1115 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "strings" + "time" + + jsoniter "github.com/json-iterator/go" + "github.com/klauspost/compress/zip" + "github.com/minio/kms-go/kes" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7/pkg/tags" + "github.com/minio/minio/internal/bucket/lifecycle" + objectlock "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/versioning" + "github.com/minio/minio/internal/event" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/kms" + "github.com/minio/mux" + "github.com/minio/pkg/v3/policy" +) + +const ( + bucketQuotaConfigFile = "quota.json" + bucketTargetsFile = "bucket-targets.json" +) + +// PutBucketQuotaConfigHandler - PUT Bucket quota configuration. +// ---------- +// Places a quota configuration on the specified bucket. The quota +// specified in the quota configuration will be applied by default +// to enforce total quota for the specified bucket. +func (a adminAPIHandlers) PutBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.SetBucketQuotaAdminAction) + if objectAPI == nil { + return + } + + vars := mux.Vars(r) + bucket := pathClean(vars["bucket"]) + + if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + + quotaConfig, err := parseBucketQuota(bucket, data) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, data) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + bucketMeta := madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaTypeQuotaConfig, + Bucket: bucket, + Quota: data, + UpdatedAt: updatedAt, + } + if quotaConfig.Size == 0 && quotaConfig.Quota == 0 { + bucketMeta.Quota = nil + } + + // Call site replication hook. + replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, bucketMeta)) + + // Write success response. + writeSuccessResponseHeadersOnly(w) +} + +// GetBucketQuotaConfigHandler - gets bucket quota configuration +func (a adminAPIHandlers) GetBucketQuotaConfigHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.GetBucketQuotaAdminAction) + if objectAPI == nil { + return + } + + vars := mux.Vars(r) + bucket := pathClean(vars["bucket"]) + + if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + config, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + configData, err := json.Marshal(config) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Write success response. + writeSuccessResponseJSON(w, configData) +} + +// SetRemoteTargetHandler - sets a remote target for bucket +func (a adminAPIHandlers) SetRemoteTargetHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + vars := mux.Vars(r) + bucket := pathClean(vars["bucket"]) + update := r.Form.Get("update") == "true" + + // Get current object layer instance. + objectAPI, _ := validateAdminReq(ctx, w, r, policy.SetBucketTargetAction) + if objectAPI == nil { + return + } + + // Check if bucket exists. + if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + cred, _, s3Err := validateAdminSignature(ctx, r, "") + if s3Err != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + password := cred.SecretKey + + reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength)) + if err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL) + return + } + var target madmin.BucketTarget + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err = json.Unmarshal(reqBytes, &target); err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL) + return + } + sameTarget, _ := isLocalHost(target.URL().Hostname(), target.URL().Port(), globalMinioPort) + if sameTarget && bucket == target.TargetBucket { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBucketRemoteIdenticalToSource), r.URL) + return + } + + target.SourceBucket = bucket + var ops []madmin.TargetUpdateType + if update { + ops = madmin.GetTargetUpdateOps(r.Form) + } else { + var exists bool // true if arn exists + target.Arn, exists = globalBucketTargetSys.getRemoteARN(bucket, &target, "") + if exists && target.Arn != "" { // return pre-existing ARN + data, err := json.Marshal(target.Arn) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + // Write success response. + writeSuccessResponseJSON(w, data) + return + } + } + if target.Arn == "" { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL) + return + } + if globalSiteReplicationSys.isEnabled() && !update { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrRemoteTargetDenyAddError, err), r.URL) + return + } + + if update { + // overlay the updates on existing target + tgt := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, bucket, target.Arn) + if tgt.Empty() { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrRemoteTargetNotFoundError, err), r.URL) + return + } + for _, op := range ops { + switch op { + case madmin.CredentialsUpdateType: + if !globalSiteReplicationSys.isEnabled() { + // credentials update is possible only in bucket replication. User will never + // know the site replicator creds. + tgt.Credentials = target.Credentials + tgt.TargetBucket = target.TargetBucket + tgt.Secure = target.Secure + tgt.Endpoint = target.Endpoint + } + case madmin.SyncUpdateType: + tgt.ReplicationSync = target.ReplicationSync + case madmin.ProxyUpdateType: + tgt.DisableProxy = target.DisableProxy + case madmin.PathUpdateType: + tgt.Path = target.Path + case madmin.BandwidthLimitUpdateType: + tgt.BandwidthLimit = target.BandwidthLimit + case madmin.HealthCheckDurationUpdateType: + tgt.HealthCheckDuration = target.HealthCheckDuration + } + } + target = tgt + } + + // enforce minimum bandwidth limit as 100MBps + if target.BandwidthLimit > 0 && target.BandwidthLimit < 100*1000*1000 { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationBandwidthLimitError, err), r.URL) + return + } + if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target, update); err != nil { + switch err.(type) { + case RemoteTargetConnectionErr: + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationRemoteConnectionError, err), r.URL) + default: + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + } + return + } + targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + tgtBytes, err := json.Marshal(&targets) + if err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL) + return + } + if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + data, err := json.Marshal(target.Arn) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + // Write success response. + writeSuccessResponseJSON(w, data) +} + +// ListRemoteTargetsHandler - lists remote target(s) for a bucket or gets a target +// for a particular ARN type +func (a adminAPIHandlers) ListRemoteTargetsHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + vars := mux.Vars(r) + bucket := pathClean(vars["bucket"]) + arnType := vars["type"] + + // Get current object layer instance. + objectAPI, _ := validateAdminReq(ctx, w, r, policy.GetBucketTargetAction) + if objectAPI == nil { + return + } + if bucket != "" { + // Check if bucket exists. + if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if _, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + } + targets := globalBucketTargetSys.ListTargets(ctx, bucket, arnType) + data, err := json.Marshal(targets) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + // Write success response. + writeSuccessResponseJSON(w, data) +} + +// RemoveRemoteTargetHandler - removes a remote target for bucket with specified ARN +func (a adminAPIHandlers) RemoveRemoteTargetHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + vars := mux.Vars(r) + bucket := pathClean(vars["bucket"]) + arn := vars["arn"] + + // Get current object layer instance. + objectAPI, _ := validateAdminReq(ctx, w, r, policy.SetBucketTargetAction) + if objectAPI == nil { + return + } + + // Check if bucket exists. + if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if err := globalBucketTargetSys.RemoveTarget(ctx, bucket, arn); err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + tgtBytes, err := json.Marshal(&targets) + if err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL) + return + } + if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Write success response. + writeSuccessNoContent(w) +} + +// ExportBucketMetadataHandler - exports all bucket metadata as a zipped file +func (a adminAPIHandlers) ExportBucketMetadataHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + bucket := pathClean(r.Form.Get("bucket")) + // Get current object layer instance. + objectAPI, _ := validateAdminReq(ctx, w, r, policy.ExportBucketMetadataAction) + if objectAPI == nil { + return + } + + var ( + buckets []BucketInfo + err error + ) + if bucket != "" { + // Check if bucket exists. + if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + buckets = append(buckets, BucketInfo{Name: bucket}) + } else { + buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{}) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + } + + // Initialize a zip writer which will provide a zipped content + // of bucket metadata + zipWriter := zip.NewWriter(w) + defer zipWriter.Close() + + rawDataFn := func(r io.Reader, filename string, sz int) { + header, zerr := zip.FileInfoHeader(dummyFileInfo{ + name: filename, + size: int64(sz), + mode: 0o600, + modTime: time.Now(), + isDir: false, + sys: nil, + }) + if zerr == nil { + header.Method = zip.Deflate + zwriter, zerr := zipWriter.CreateHeader(header) + if zerr == nil { + io.Copy(zwriter, r) + } + } + } + + cfgFiles := []string{ + bucketPolicyConfig, + bucketNotificationConfig, + bucketLifecycleConfig, + bucketSSEConfig, + bucketTaggingConfig, + bucketQuotaConfigFile, + objectLockConfig, + bucketVersioningConfig, + bucketReplicationConfig, + bucketTargetsFile, + } + for _, bi := range buckets { + for _, cfgFile := range cfgFiles { + cfgPath := pathJoin(bi.Name, cfgFile) + bucket := bi.Name + switch cfgFile { + case bucketPolicyConfig: + config, _, err := globalBucketMetadataSys.GetBucketPolicy(bucket) + if err != nil { + if errors.Is(err, BucketPolicyNotFound{Bucket: bucket}) { + continue + } + writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL) + return + } + configData, err := json.Marshal(config) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL) + return + } + rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)) + case bucketNotificationConfig: + config, err := globalBucketMetadataSys.GetNotificationConfig(bucket) + if err != nil { + adminLogIf(ctx, err) + writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL) + return + } + configData, err := xml.Marshal(config) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL) + return + } + rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)) + case bucketLifecycleConfig: + config, _, err := globalBucketMetadataSys.GetLifecycleConfig(bucket) + if err != nil { + if errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) { + continue + } + adminLogIf(ctx, err) + writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL) + return + } + configData, err := xml.Marshal(config) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL) + return + } + rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)) + case bucketQuotaConfigFile: + config, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket) + if err != nil { + if errors.Is(err, BucketQuotaConfigNotFound{Bucket: bucket}) { + continue + } + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + configData, err := json.Marshal(config) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)) + case bucketSSEConfig: + config, _, err := globalBucketMetadataSys.GetSSEConfig(bucket) + if err != nil { + if errors.Is(err, BucketSSEConfigNotFound{Bucket: bucket}) { + continue + } + writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL) + return + } + configData, err := xml.Marshal(config) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL) + return + } + rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)) + case bucketTaggingConfig: + config, _, err := globalBucketMetadataSys.GetTaggingConfig(bucket) + if err != nil { + if errors.Is(err, BucketTaggingNotFound{Bucket: bucket}) { + continue + } + writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL) + return + } + configData, err := xml.Marshal(config) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL) + return + } + rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)) + case objectLockConfig: + config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket) + if err != nil { + if errors.Is(err, BucketObjectLockConfigNotFound{Bucket: bucket}) { + continue + } + writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL) + return + } + + configData, err := xml.Marshal(config) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL) + return + } + rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)) + case bucketVersioningConfig: + config, _, err := globalBucketMetadataSys.GetVersioningConfig(bucket) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL) + return + } + // ignore empty versioning configs + if config.Status != versioning.Enabled && config.Status != versioning.Suspended { + continue + } + configData, err := xml.Marshal(config) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL) + return + } + rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)) + case bucketReplicationConfig: + config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket) + if err != nil { + if errors.Is(err, BucketReplicationConfigNotFound{Bucket: bucket}) { + continue + } + writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL) + return + } + configData, err := xml.Marshal(config) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL) + return + } + rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)) + case bucketTargetsFile: + config, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket) + if err != nil { + if errors.Is(err, BucketRemoteTargetNotFound{Bucket: bucket}) { + continue + } + + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + configData, err := xml.Marshal(config) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, cfgFile, bucket), r.URL) + return + } + rawDataFn(bytes.NewReader(configData), cfgPath, len(configData)) + } + } + } +} + +type importMetaReport struct { + madmin.BucketMetaImportErrs +} + +func (i *importMetaReport) SetStatus(bucket, fname string, err error) { + st := i.Buckets[bucket] + var errMsg string + if err != nil { + errMsg = err.Error() + } + switch fname { + case bucketPolicyConfig: + st.Policy = madmin.MetaStatus{IsSet: true, Err: errMsg} + case bucketNotificationConfig: + st.Notification = madmin.MetaStatus{IsSet: true, Err: errMsg} + case bucketLifecycleConfig: + st.Lifecycle = madmin.MetaStatus{IsSet: true, Err: errMsg} + case bucketSSEConfig: + st.SSEConfig = madmin.MetaStatus{IsSet: true, Err: errMsg} + case bucketTaggingConfig: + st.Tagging = madmin.MetaStatus{IsSet: true, Err: errMsg} + case bucketQuotaConfigFile: + st.Quota = madmin.MetaStatus{IsSet: true, Err: errMsg} + case objectLockConfig: + st.ObjectLock = madmin.MetaStatus{IsSet: true, Err: errMsg} + case bucketVersioningConfig: + st.Versioning = madmin.MetaStatus{IsSet: true, Err: errMsg} + default: + st.Err = errMsg + } + i.Buckets[bucket] = st +} + +// ImportBucketMetadataHandler - imports all bucket metadata from a zipped file and overwrite bucket metadata config +// There are some caveats regarding the following: +// 1. object lock config - object lock should have been specified at time of bucket creation. Only default retention settings are imported here. +// 2. Replication config - is omitted from import as remote target credentials are not available from exported data for security reasons. +// 3. lifecycle config - if transition rules are present, tier name needs to have been defined. +func (a adminAPIHandlers) ImportBucketMetadataHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Get current object layer instance. + objectAPI, _ := validateAdminReq(ctx, w, r, policy.ImportBucketMetadataAction) + if objectAPI == nil { + return + } + data, err := io.ReadAll(r.Body) + if err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + reader := bytes.NewReader(data) + zr, err := zip.NewReader(reader, int64(len(data))) + if err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + rpt := importMetaReport{ + madmin.BucketMetaImportErrs{ + Buckets: make(map[string]madmin.BucketStatus, len(zr.File)), + }, + } + + bucketMap := make(map[string]*BucketMetadata, len(zr.File)) + + updatedAt := UTCNow() + + for _, file := range zr.File { + slc := strings.Split(file.Name, slashSeparator) + if len(slc) != 2 { // expecting bucket/configfile in the zipfile + rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/")) + continue + } + bucket := slc[0] + meta, err := readBucketMetadata(ctx, objectAPI, bucket) + if err == nil { + bucketMap[bucket] = &meta + } else if err != errConfigNotFound { + rpt.SetStatus(bucket, "", err) + } + } + + // import object lock config if any - order of import matters here. + for _, file := range zr.File { + slc := strings.Split(file.Name, slashSeparator) + if len(slc) != 2 { // expecting bucket/configfile in the zipfile + rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/")) + continue + } + bucket, fileName := slc[0], slc[1] + if fileName == objectLockConfig { + reader, err := file.Open() + if err != nil { + rpt.SetStatus(bucket, fileName, err) + continue + } + config, err := objectlock.ParseObjectLockConfig(reader) + if err != nil { + rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err)) + continue + } + + configData, err := xml.Marshal(config) + if err != nil { + rpt.SetStatus(bucket, fileName, err) + continue + } + if _, ok := bucketMap[bucket]; !ok { + opts := MakeBucketOptions{ + LockEnabled: config.Enabled(), + ForceCreate: true, // ignore if it already exists + } + err = objectAPI.MakeBucket(ctx, bucket, opts) + if err != nil { + rpt.SetStatus(bucket, fileName, err) + continue + } + v, _ := globalBucketMetadataSys.Get(bucket) + bucketMap[bucket] = &v + } + + bucketMap[bucket].ObjectLockConfigXML = configData + bucketMap[bucket].ObjectLockConfigUpdatedAt = updatedAt + rpt.SetStatus(bucket, fileName, nil) + } + } + + // import versioning metadata + for _, file := range zr.File { + slc := strings.Split(file.Name, slashSeparator) + if len(slc) != 2 { // expecting bucket/configfile in the zipfile + rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/")) + continue + } + bucket, fileName := slc[0], slc[1] + if fileName == bucketVersioningConfig { + reader, err := file.Open() + if err != nil { + rpt.SetStatus(bucket, fileName, err) + continue + } + v, err := versioning.ParseConfig(io.LimitReader(reader, maxBucketVersioningConfigSize)) + if err != nil { + rpt.SetStatus(bucket, fileName, err) + continue + } + if _, ok := bucketMap[bucket]; !ok { + if err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{ + ForceCreate: true, // ignore if it already exists + }); err != nil { + rpt.SetStatus(bucket, fileName, err) + continue + } + v, _ := globalBucketMetadataSys.Get(bucket) + bucketMap[bucket] = &v + } + + if globalSiteReplicationSys.isEnabled() && v.Suspended() { + rpt.SetStatus(bucket, fileName, fmt.Errorf("Cluster replication is enabled for this site, so the versioning state cannot be suspended.")) + continue + } + + if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && v.Suspended() { + rpt.SetStatus(bucket, fileName, fmt.Errorf("An Object Lock configuration is present on this bucket, so the versioning state cannot be suspended.")) + continue + } + if rcfg, _ := getReplicationConfig(ctx, bucket); rcfg != nil && v.Suspended() { + rpt.SetStatus(bucket, fileName, fmt.Errorf("A replication configuration is present on this bucket, so the versioning state cannot be suspended.")) + continue + } + + configData, err := xml.Marshal(v) + if err != nil { + rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err)) + continue + } + + bucketMap[bucket].VersioningConfigXML = configData + bucketMap[bucket].VersioningConfigUpdatedAt = updatedAt + rpt.SetStatus(bucket, fileName, nil) + } + } + + for _, file := range zr.File { + reader, err := file.Open() + if err != nil { + rpt.SetStatus(file.Name, "", err) + continue + } + sz := file.FileInfo().Size() + slc := strings.Split(file.Name, slashSeparator) + if len(slc) != 2 { // expecting bucket/configfile in the zipfile + rpt.SetStatus(file.Name, "", fmt.Errorf("malformed zip - expecting format bucket/")) + continue + } + bucket, fileName := slc[0], slc[1] + + // create bucket if it does not exist yet. + if _, ok := bucketMap[bucket]; !ok { + err = objectAPI.MakeBucket(ctx, bucket, MakeBucketOptions{ + ForceCreate: true, // ignore if it already exists + }) + if err != nil { + rpt.SetStatus(bucket, "", err) + continue + } + v, _ := globalBucketMetadataSys.Get(bucket) + bucketMap[bucket] = &v + } + if _, ok := bucketMap[bucket]; !ok { + continue + } + switch fileName { + case bucketNotificationConfig: + config, err := event.ParseConfig(io.LimitReader(reader, sz), globalSite.Region(), globalEventNotifier.targetList) + if err != nil { + rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err)) + continue + } + + configData, err := xml.Marshal(config) + if err != nil { + rpt.SetStatus(bucket, fileName, err) + continue + } + + bucketMap[bucket].NotificationConfigXML = configData + bucketMap[bucket].NotificationConfigUpdatedAt = updatedAt + rpt.SetStatus(bucket, fileName, nil) + case bucketPolicyConfig: + // Error out if Content-Length is beyond allowed size. + if sz > maxBucketPolicySize { + rpt.SetStatus(bucket, fileName, errors.New(ErrPolicyTooLarge.String())) + continue + } + + bucketPolicyBytes, err := io.ReadAll(io.LimitReader(reader, sz)) + if err != nil { + rpt.SetStatus(bucket, fileName, err) + continue + } + + bucketPolicy, err := policy.ParseBucketPolicyConfig(bytes.NewReader(bucketPolicyBytes), bucket) + if err != nil { + rpt.SetStatus(bucket, fileName, err) + continue + } + + // Version in policy must not be empty + if bucketPolicy.Version == "" { + rpt.SetStatus(bucket, fileName, errors.New(ErrPolicyInvalidVersion.String())) + continue + } + + configData, err := json.Marshal(bucketPolicy) + if err != nil { + rpt.SetStatus(bucket, fileName, err) + continue + } + + bucketMap[bucket].PolicyConfigJSON = configData + bucketMap[bucket].PolicyConfigUpdatedAt = updatedAt + rpt.SetStatus(bucket, fileName, nil) + case bucketLifecycleConfig: + bucketLifecycle, err := lifecycle.ParseLifecycleConfig(io.LimitReader(reader, sz)) + if err != nil { + rpt.SetStatus(bucket, fileName, err) + continue + } + rcfg, err := globalBucketObjectLockSys.Get(bucket) + if err != nil { + rpt.SetStatus(bucket, fileName, err) + continue + } + // Validate the received bucket policy document + if err = bucketLifecycle.Validate(rcfg); err != nil { + rpt.SetStatus(bucket, fileName, err) + continue + } + + // Validate the transition storage ARNs + if err = validateTransitionTier(bucketLifecycle); err != nil { + rpt.SetStatus(bucket, fileName, err) + continue + } + + configData, err := xml.Marshal(bucketLifecycle) + if err != nil { + rpt.SetStatus(bucket, fileName, err) + continue + } + + bucketMap[bucket].LifecycleConfigXML = configData + bucketMap[bucket].LifecycleConfigUpdatedAt = updatedAt + rpt.SetStatus(bucket, fileName, nil) + case bucketSSEConfig: + // Parse bucket encryption xml + encConfig, err := validateBucketSSEConfig(io.LimitReader(reader, maxBucketSSEConfigSize)) + if err != nil { + rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err)) + continue + } + + // Return error if KMS is not initialized + if GlobalKMS == nil { + rpt.SetStatus(bucket, fileName, fmt.Errorf("%s", errorCodes[ErrKMSNotConfigured].Description)) + continue + } + kmsKey := encConfig.KeyID() + if kmsKey != "" { + _, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{ + Name: kmsKey, + AssociatedData: kms.Context{"MinIO admin API": "ServerInfoHandler"}, // Context for a test key operation + }) + if err != nil { + if errors.Is(err, kes.ErrKeyNotFound) { + rpt.SetStatus(bucket, fileName, errKMSKeyNotFound) + continue + } + rpt.SetStatus(bucket, fileName, err) + continue + } + } + + configData, err := xml.Marshal(encConfig) + if err != nil { + rpt.SetStatus(bucket, fileName, err) + continue + } + + bucketMap[bucket].EncryptionConfigXML = configData + bucketMap[bucket].EncryptionConfigUpdatedAt = updatedAt + rpt.SetStatus(bucket, fileName, nil) + case bucketTaggingConfig: + tags, err := tags.ParseBucketXML(io.LimitReader(reader, sz)) + if err != nil { + rpt.SetStatus(bucket, fileName, fmt.Errorf("%s (%s)", errorCodes[ErrMalformedXML].Description, err)) + continue + } + + configData, err := xml.Marshal(tags) + if err != nil { + rpt.SetStatus(bucket, fileName, err) + continue + } + + bucketMap[bucket].TaggingConfigXML = configData + bucketMap[bucket].TaggingConfigUpdatedAt = updatedAt + rpt.SetStatus(bucket, fileName, nil) + case bucketQuotaConfigFile: + data, err := io.ReadAll(reader) + if err != nil { + rpt.SetStatus(bucket, fileName, err) + continue + } + + _, err = parseBucketQuota(bucket, data) + if err != nil { + rpt.SetStatus(bucket, fileName, err) + continue + } + + bucketMap[bucket].QuotaConfigJSON = data + bucketMap[bucket].QuotaConfigUpdatedAt = updatedAt + rpt.SetStatus(bucket, fileName, nil) + } + } + + enc := func(b []byte) *string { + if b == nil { + return nil + } + v := base64.StdEncoding.EncodeToString(b) + return &v + } + + for bucket, meta := range bucketMap { + err := globalBucketMetadataSys.save(ctx, *meta) + if err != nil { + rpt.SetStatus(bucket, "", err) + continue + } + // Call site replication hook. + if err = globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{ + Bucket: bucket, + Quota: meta.QuotaConfigJSON, + Policy: meta.PolicyConfigJSON, + Versioning: enc(meta.VersioningConfigXML), + Tags: enc(meta.TaggingConfigXML), + ObjectLockConfig: enc(meta.ObjectLockConfigXML), + SSEConfig: enc(meta.EncryptionConfigXML), + UpdatedAt: updatedAt, + }); err != nil { + rpt.SetStatus(bucket, "", err) + continue + } + } + + rptData, err := json.Marshal(rpt.BucketMetaImportErrs) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, rptData) +} + +// ReplicationDiffHandler - POST returns info on unreplicated versions for a remote target ARN +// to the connected HTTP client. +func (a adminAPIHandlers) ReplicationDiffHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.ReplicationDiff) + if objectAPI == nil { + return + } + + // Check if bucket exists. + if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + opts := extractReplicateDiffOpts(r.Form) + if opts.ARN != "" { + tgt := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, bucket, opts.ARN) + if tgt.Empty() { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrInvalidRequest, fmt.Errorf("invalid arn : '%s'", opts.ARN)), r.URL) + return + } + } + + keepAliveTicker := time.NewTicker(500 * time.Millisecond) + defer keepAliveTicker.Stop() + + diffCh, err := getReplicationDiff(ctx, objectAPI, bucket, opts) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + enc := json.NewEncoder(w) + for { + select { + case entry, ok := <-diffCh: + if !ok { + return + } + if err := enc.Encode(entry); err != nil { + return + } + if len(diffCh) == 0 { + // Flush if nothing is queued + xhttp.Flush(w) + } + case <-keepAliveTicker.C: + if len(diffCh) > 0 { + continue + } + if _, err := w.Write([]byte(" ")); err != nil { + return + } + xhttp.Flush(w) + case <-ctx.Done(): + return + } + } +} + +// ReplicationMRFHandler - POST returns info on entries in the MRF backlog for a node or all nodes +func (a adminAPIHandlers) ReplicationMRFHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.ReplicationDiff) + if objectAPI == nil { + return + } + + // Check if bucket exists. + if bucket != "" { + if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + + q := r.Form + node := q.Get("node") + + keepAliveTicker := time.NewTicker(500 * time.Millisecond) + defer keepAliveTicker.Stop() + + mrfCh, err := globalNotificationSys.GetReplicationMRF(ctx, bucket, node) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + enc := json.NewEncoder(w) + for { + select { + case entry, ok := <-mrfCh: + if !ok { + return + } + if err := enc.Encode(entry); err != nil { + return + } + if len(mrfCh) == 0 { + // Flush if nothing is queued + xhttp.Flush(w) + } + case <-keepAliveTicker.C: + if len(mrfCh) > 0 { + continue + } + if _, err := w.Write([]byte(" ")); err != nil { + return + } + xhttp.Flush(w) + case <-ctx.Done(): + return + } + } +} diff --git a/cmd/admin-handler-utils.go b/cmd/admin-handler-utils.go new file mode 100644 index 00000000..cdfb7987 --- /dev/null +++ b/cmd/admin-handler-utils.go @@ -0,0 +1,263 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "net/http" + + "github.com/minio/kms-go/kes" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/config" + "github.com/minio/pkg/v3/policy" +) + +// validateAdminReq will validate request against and return whether it is allowed. +// If any of the supplied actions are allowed it will be successful. +// If nil ObjectLayer is returned, the operation is not permitted. +// When nil ObjectLayer has been returned an error has always been sent to w. +func validateAdminReq(ctx context.Context, w http.ResponseWriter, r *http.Request, actions ...policy.AdminAction) (ObjectLayer, auth.Credentials) { + // Get current object layer instance. + objectAPI := newObjectLayerFn() + if objectAPI == nil || globalNotificationSys == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return nil, auth.Credentials{} + } + + for _, action := range actions { + // Validate request signature. + cred, adminAPIErr := checkAdminRequestAuth(ctx, r, action, "") + switch adminAPIErr { + case ErrNone: + return objectAPI, cred + case ErrAccessDenied: + // Try another + continue + default: + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL) + return nil, cred + } + } + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return nil, auth.Credentials{} +} + +// AdminError - is a generic error for all admin APIs. +type AdminError struct { + Code string + Message string + StatusCode int +} + +func (ae AdminError) Error() string { + return ae.Message +} + +func toAdminAPIErr(ctx context.Context, err error) APIError { + if err == nil { + return noError + } + + var apiErr APIError + switch e := err.(type) { + case policy.Error: + apiErr = APIError{ + Code: "XMinioMalformedIAMPolicy", + Description: e.Error(), + HTTPStatusCode: http.StatusBadRequest, + } + case config.ErrConfigNotFound: + apiErr = APIError{ + Code: "XMinioConfigNotFoundError", + Description: e.Error(), + HTTPStatusCode: http.StatusNotFound, + } + case config.ErrConfigGeneric: + apiErr = APIError{ + Code: "XMinioConfigError", + Description: e.Error(), + HTTPStatusCode: http.StatusBadRequest, + } + case AdminError: + apiErr = APIError{ + Code: e.Code, + Description: e.Message, + HTTPStatusCode: e.StatusCode, + } + case SRError: + apiErr = errorCodes.ToAPIErrWithErr(e.Code, e.Cause) + case decomError: + apiErr = APIError{ + Code: "XMinioDecommissionNotAllowed", + Description: e.Err, + HTTPStatusCode: http.StatusBadRequest, + } + default: + switch { + case errors.Is(err, errTooManyPolicies): + apiErr = APIError{ + Code: "XMinioAdminInvalidRequest", + Description: err.Error(), + HTTPStatusCode: http.StatusBadRequest, + } + case errors.Is(err, errDecommissionAlreadyRunning): + apiErr = APIError{ + Code: "XMinioDecommissionNotAllowed", + Description: err.Error(), + HTTPStatusCode: http.StatusBadRequest, + } + case errors.Is(err, errDecommissionComplete): + apiErr = APIError{ + Code: "XMinioDecommissionNotAllowed", + Description: err.Error(), + HTTPStatusCode: http.StatusBadRequest, + } + case errors.Is(err, errDecommissionRebalanceAlreadyRunning): + apiErr = APIError{ + Code: "XMinioDecommissionNotAllowed", + Description: err.Error(), + HTTPStatusCode: http.StatusBadRequest, + } + case errors.Is(err, errRebalanceDecommissionAlreadyRunning): + apiErr = APIError{ + Code: "XMinioRebalanceNotAllowed", + Description: err.Error(), + HTTPStatusCode: http.StatusBadRequest, + } + case errors.Is(err, errConfigNotFound): + apiErr = APIError{ + Code: "XMinioConfigError", + Description: err.Error(), + HTTPStatusCode: http.StatusNotFound, + } + case errors.Is(err, errIAMActionNotAllowed): + apiErr = APIError{ + Code: "XMinioIAMActionNotAllowed", + Description: err.Error(), + HTTPStatusCode: http.StatusForbidden, + } + case errors.Is(err, errIAMServiceAccountNotAllowed): + apiErr = APIError{ + Code: "XMinioIAMServiceAccountNotAllowed", + Description: err.Error(), + HTTPStatusCode: http.StatusBadRequest, + } + case errors.Is(err, errIAMNotInitialized): + apiErr = APIError{ + Code: "XMinioIAMNotInitialized", + Description: err.Error(), + HTTPStatusCode: http.StatusServiceUnavailable, + } + case errors.Is(err, errPolicyInUse): + apiErr = APIError{ + Code: "XMinioIAMPolicyInUse", + Description: "The policy cannot be removed, as it is in use", + HTTPStatusCode: http.StatusBadRequest, + } + case errors.Is(err, errSessionPolicyTooLarge): + apiErr = APIError{ + Code: "XMinioIAMServiceAccountSessionPolicyTooLarge", + Description: err.Error(), + HTTPStatusCode: http.StatusBadRequest, + } + case errors.Is(err, kes.ErrKeyExists): + apiErr = APIError{ + Code: "XMinioKMSKeyExists", + Description: err.Error(), + HTTPStatusCode: http.StatusConflict, + } + + // Tier admin API errors + case errors.Is(err, madmin.ErrTierNameEmpty): + apiErr = APIError{ + Code: "XMinioAdminTierNameEmpty", + Description: err.Error(), + HTTPStatusCode: http.StatusBadRequest, + } + case errors.Is(err, madmin.ErrTierInvalidConfig): + apiErr = APIError{ + Code: "XMinioAdminTierInvalidConfig", + Description: err.Error(), + HTTPStatusCode: http.StatusBadRequest, + } + case errors.Is(err, madmin.ErrTierInvalidConfigVersion): + apiErr = APIError{ + Code: "XMinioAdminTierInvalidConfigVersion", + Description: err.Error(), + HTTPStatusCode: http.StatusBadRequest, + } + case errors.Is(err, madmin.ErrTierTypeUnsupported): + apiErr = APIError{ + Code: "XMinioAdminTierTypeUnsupported", + Description: err.Error(), + HTTPStatusCode: http.StatusBadRequest, + } + case errIsTierPermError(err): + apiErr = APIError{ + Code: "XMinioAdminTierInsufficientPermissions", + Description: err.Error(), + HTTPStatusCode: http.StatusBadRequest, + } + case errors.Is(err, errTierInvalidConfig): + apiErr = APIError{ + Code: "XMinioAdminTierInvalidConfig", + Description: err.Error(), + HTTPStatusCode: http.StatusBadRequest, + } + default: + apiErr = errorCodes.ToAPIErrWithErr(toAdminAPIErrCode(ctx, err), err) + } + } + return apiErr +} + +// toAdminAPIErrCode - converts errErasureWriteQuorum error to admin API +// specific error. +func toAdminAPIErrCode(ctx context.Context, err error) APIErrorCode { + if errors.Is(err, errErasureWriteQuorum) { + return ErrAdminConfigNoQuorum + } + return toAPIErrorCode(ctx, err) +} + +// wraps export error for more context +func exportError(ctx context.Context, err error, fname, entity string) APIError { + if entity == "" { + return toAPIError(ctx, fmt.Errorf("error exporting %s with: %w", fname, err)) + } + return toAPIError(ctx, fmt.Errorf("error exporting %s from %s with: %w", entity, fname, err)) +} + +// wraps import error for more context +func importError(ctx context.Context, err error, fname, entity string) APIError { + if entity == "" { + return toAPIError(ctx, fmt.Errorf("error importing %s with: %w", fname, err)) + } + return toAPIError(ctx, fmt.Errorf("error importing %s from %s with: %w", entity, fname, err)) +} + +// wraps import error for more context +func importErrorWithAPIErr(ctx context.Context, apiErr APIErrorCode, err error, fname, entity string) APIError { + if entity == "" { + return errorCodes.ToAPIErrWithErr(apiErr, fmt.Errorf("error importing %s with: %w", fname, err)) + } + return errorCodes.ToAPIErrWithErr(apiErr, fmt.Errorf("error importing %s from %s with: %w", entity, fname, err)) +} diff --git a/cmd/admin-handlers-config-kv.go b/cmd/admin-handlers-config-kv.go new file mode 100644 index 00000000..2169103d --- /dev/null +++ b/cmd/admin-handlers-config-kv.go @@ -0,0 +1,542 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "io" + "net/http" + "strconv" + "strings" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/config/etcd" + xldap "github.com/minio/minio/internal/config/identity/ldap" + "github.com/minio/minio/internal/config/identity/openid" + idplugin "github.com/minio/minio/internal/config/identity/plugin" + polplugin "github.com/minio/minio/internal/config/policy/plugin" + "github.com/minio/minio/internal/config/storageclass" + "github.com/minio/minio/internal/config/subnet" + "github.com/minio/minio/internal/logger" + "github.com/minio/mux" + "github.com/minio/pkg/v3/policy" +) + +// DelConfigKVHandler - DELETE /minio/admin/v3/del-config-kv +func (a adminAPIHandlers) DelConfigKVHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction) + if objectAPI == nil { + return + } + + if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 { + // More than maxConfigSize bytes were available + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL) + return + } + + password := cred.SecretKey + kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength)) + if err != nil { + adminLogIf(ctx, err) + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL) + return + } + + subSys, _, _, err := config.GetSubSys(string(kvBytes)) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + cfg, err := readServerConfig(ctx, objectAPI, nil) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + if err = cfg.DelFrom(bytes.NewReader(kvBytes)); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + if err = validateConfig(ctx, cfg, subSys); err != nil { + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL) + return + } + + // Check if subnet proxy being deleted and if so the value of proxy of subnet + // target of logger webhook configuration also should be deleted + loggerWebhookProxyDeleted := setLoggerWebhookSubnetProxy(subSys, cfg) + + if err = saveServerConfig(ctx, objectAPI, cfg); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // freshly retrieve the config so that default values are loaded for reset config + if cfg, err = getValidConfig(objectAPI); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + dynamic := config.SubSystemsDynamic.Contains(subSys) + if dynamic { + applyDynamic(ctx, objectAPI, cfg, subSys, r, w) + if subSys == config.SubnetSubSys && loggerWebhookProxyDeleted { + // Logger webhook proxy deleted, apply the dynamic changes + applyDynamic(ctx, objectAPI, cfg, config.LoggerWebhookSubSys, r, w) + } + } +} + +func applyDynamic(ctx context.Context, objectAPI ObjectLayer, cfg config.Config, subSys string, + r *http.Request, w http.ResponseWriter, +) { + // Apply dynamic values. + if err := applyDynamicConfigForSubSys(GlobalContext, objectAPI, cfg, subSys); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + globalNotificationSys.SignalConfigReload(subSys) + // Tell the client that dynamic config was applied. + w.Header().Set(madmin.ConfigAppliedHeader, madmin.ConfigAppliedTrue) +} + +type badConfigErr struct { + Err error +} + +// Error - return the error message +func (bce badConfigErr) Error() string { + return bce.Err.Error() +} + +// Unwrap the error to its underlying error. +func (bce badConfigErr) Unwrap() error { + return bce.Err +} + +type setConfigResult struct { + Cfg config.Config + SubSys string + Dynamic bool + LoggerWebhookCfgUpdated bool +} + +// SetConfigKVHandler - PUT /minio/admin/v3/set-config-kv +func (a adminAPIHandlers) SetConfigKVHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction) + if objectAPI == nil { + return + } + + if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 { + // More than maxConfigSize bytes were available + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL) + return + } + + password := cred.SecretKey + kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength)) + if err != nil { + adminLogIf(ctx, err) + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL) + return + } + + result, err := setConfigKV(ctx, objectAPI, kvBytes) + if err != nil { + switch err.(type) { + case badConfigErr: + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL) + default: + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + } + return + } + + if result.Dynamic { + applyDynamic(ctx, objectAPI, result.Cfg, result.SubSys, r, w) + // If logger webhook config updated (proxy due to callhome), explicitly dynamically + // apply the config + if result.LoggerWebhookCfgUpdated { + applyDynamic(ctx, objectAPI, result.Cfg, config.LoggerWebhookSubSys, r, w) + } + } + + writeSuccessResponseHeadersOnly(w) +} + +func setConfigKV(ctx context.Context, objectAPI ObjectLayer, kvBytes []byte) (result setConfigResult, err error) { + result.Cfg, err = readServerConfig(ctx, objectAPI, nil) + if err != nil { + return result, err + } + + result.Dynamic, err = result.Cfg.ReadConfig(bytes.NewReader(kvBytes)) + if err != nil { + return result, err + } + + result.SubSys, _, _, err = config.GetSubSys(string(kvBytes)) + if err != nil { + return result, err + } + + tgts, err := config.ParseConfigTargetID(bytes.NewReader(kvBytes)) + if err != nil { + return result, err + } + ctx = context.WithValue(ctx, config.ContextKeyForTargetFromConfig, tgts) + if verr := validateConfig(ctx, result.Cfg, result.SubSys); verr != nil { + err = badConfigErr{Err: verr} + return result, err + } + + // Check if subnet proxy being set and if so set the same value to proxy of subnet + // target of logger webhook configuration + result.LoggerWebhookCfgUpdated = setLoggerWebhookSubnetProxy(result.SubSys, result.Cfg) + + // Update the actual server config on disk. + if err = saveServerConfig(ctx, objectAPI, result.Cfg); err != nil { + return result, err + } + + // Write the config input KV to history. + err = saveServerConfigHistory(ctx, objectAPI, kvBytes) + return result, err +} + +// GetConfigKVHandler - GET /minio/admin/v3/get-config-kv?key={key} +// +// `key` can be one of three forms: +// 1. `subsys:target` -> request for config of a single subsystem and target pair. +// 2. `subsys:` -> request for config of a single subsystem and the default target. +// 3. `subsys` -> request for config of all targets for the given subsystem. +// +// This is a reporting API and config secrets are redacted in the response. +func (a adminAPIHandlers) GetConfigKVHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction) + if objectAPI == nil { + return + } + + cfg := globalServerConfig.Clone() + vars := mux.Vars(r) + key := vars["key"] + + var subSys, target string + { + ws := strings.SplitN(key, madmin.SubSystemSeparator, 2) + subSys = ws[0] + if len(ws) == 2 { + if ws[1] == "" { + target = madmin.Default + } else { + target = ws[1] + } + } + } + + subSysConfigs, err := cfg.GetSubsysInfo(subSys, target, true) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + var s strings.Builder + for _, subSysConfig := range subSysConfigs { + subSysConfig.WriteTo(&s, false) + } + + password := cred.SecretKey + econfigData, err := madmin.EncryptData(password, []byte(s.String())) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, econfigData) +} + +func (a adminAPIHandlers) ClearConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction) + if objectAPI == nil { + return + } + + vars := mux.Vars(r) + restoreID := vars["restoreId"] + if restoreID == "" { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + if restoreID == "all" { + chEntries, err := listServerConfigHistory(ctx, objectAPI, false, -1) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + for _, chEntry := range chEntries { + if err = delServerConfigHistory(ctx, objectAPI, chEntry.RestoreID); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + } + } else if err := delServerConfigHistory(ctx, objectAPI, restoreID); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } +} + +// RestoreConfigHistoryKVHandler - restores a config with KV settings for the given KV id. +func (a adminAPIHandlers) RestoreConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction) + if objectAPI == nil { + return + } + + vars := mux.Vars(r) + restoreID := vars["restoreId"] + if restoreID == "" { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + + kvBytes, err := readServerConfigHistory(ctx, objectAPI, restoreID) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + cfg, err := readServerConfig(ctx, objectAPI, nil) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + if _, err = cfg.ReadConfig(bytes.NewReader(kvBytes)); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + if err = validateConfig(ctx, cfg, ""); err != nil { + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL) + return + } + + if err = saveServerConfig(ctx, objectAPI, cfg); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + delServerConfigHistory(ctx, objectAPI, restoreID) +} + +// ListConfigHistoryKVHandler - lists all the KV ids. +func (a adminAPIHandlers) ListConfigHistoryKVHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction) + if objectAPI == nil { + return + } + + vars := mux.Vars(r) + count, err := strconv.Atoi(vars["count"]) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + chEntries, err := listServerConfigHistory(ctx, objectAPI, true, count) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + data, err := json.Marshal(chEntries) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + password := cred.SecretKey + econfigData, err := madmin.EncryptData(password, data) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, econfigData) +} + +// HelpConfigKVHandler - GET /minio/admin/v3/help-config-kv?subSys={subSys}&key={key} +func (a adminAPIHandlers) HelpConfigKVHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction) + if objectAPI == nil { + return + } + + vars := mux.Vars(r) + + subSys := vars["subSys"] + key := vars["key"] + + _, envOnly := r.Form["env"] + + rd, err := GetHelp(subSys, key, envOnly) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + json.NewEncoder(w).Encode(rd) +} + +// SetConfigHandler - PUT /minio/admin/v3/config +func (a adminAPIHandlers) SetConfigHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction) + if objectAPI == nil { + return + } + + if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 { + // More than maxConfigSize bytes were available + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL) + return + } + + password := cred.SecretKey + kvBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength)) + if err != nil { + adminLogIf(ctx, err) + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL) + return + } + + cfg := newServerConfig() + if _, err = cfg.ReadConfig(bytes.NewReader(kvBytes)); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + if err = validateConfig(ctx, cfg, ""); err != nil { + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL) + return + } + + // Update the actual server config on disk. + if err = saveServerConfig(ctx, objectAPI, cfg); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Write to the config input KV to history. + if err = saveServerConfigHistory(ctx, objectAPI, kvBytes); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseHeadersOnly(w) +} + +// GetConfigHandler - GET /minio/admin/v3/config +// +// This endpoint is mainly for exporting and backing up the configuration. +// Secrets are not redacted. +func (a adminAPIHandlers) GetConfigHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction) + if objectAPI == nil { + return + } + + cfg := globalServerConfig.Clone() + + var s strings.Builder + hkvs := config.HelpSubSysMap[""] + for _, hkv := range hkvs { + // We ignore the error below, as we cannot get one. + cfgSubsysItems, _ := cfg.GetSubsysInfo(hkv.Key, "", false) + + for _, item := range cfgSubsysItems { + off := item.Config.Get(config.Enable) == config.EnableOff + switch hkv.Key { + case config.EtcdSubSys: + off = !etcd.Enabled(item.Config) + case config.StorageClassSubSys: + off = !storageclass.Enabled(item.Config) + case config.PolicyPluginSubSys: + off = !polplugin.Enabled(item.Config) + case config.IdentityOpenIDSubSys: + off = !openid.Enabled(item.Config) + case config.IdentityLDAPSubSys: + off = !xldap.Enabled(item.Config) + case config.IdentityTLSSubSys: + off = !globalIAMSys.STSTLSConfig.Enabled + case config.IdentityPluginSubSys: + off = !idplugin.Enabled(item.Config) + } + item.WriteTo(&s, off) + } + } + + password := cred.SecretKey + econfigData, err := madmin.EncryptData(password, []byte(s.String())) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, econfigData) +} + +// setLoggerWebhookSubnetProxy - Sets the logger webhook's subnet proxy value to +// one being set for subnet proxy +func setLoggerWebhookSubnetProxy(subSys string, cfg config.Config) bool { + if subSys == config.SubnetSubSys || subSys == config.LoggerWebhookSubSys { + subnetWebhookCfg := cfg[config.LoggerWebhookSubSys][subnet.LoggerWebhookName] + loggerWebhookSubnetProxy := subnetWebhookCfg.Get(logger.Proxy) + subnetProxy := cfg[config.SubnetSubSys][config.Default].Get(logger.Proxy) + if loggerWebhookSubnetProxy != subnetProxy { + subnetWebhookCfg.Set(logger.Proxy, subnetProxy) + return true + } + } + return false +} diff --git a/cmd/admin-handlers-idp-config.go b/cmd/admin-handlers-idp-config.go new file mode 100644 index 00000000..7b5792f0 --- /dev/null +++ b/cmd/admin-handlers-idp-config.go @@ -0,0 +1,439 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/config" + cfgldap "github.com/minio/minio/internal/config/identity/ldap" + "github.com/minio/minio/internal/config/identity/openid" + "github.com/minio/mux" + "github.com/minio/pkg/v3/ldap" + "github.com/minio/pkg/v3/policy" +) + +func addOrUpdateIDPHandler(ctx context.Context, w http.ResponseWriter, r *http.Request, isUpdate bool) { + objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction) + if objectAPI == nil { + return + } + + if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 { + // More than maxConfigSize bytes were available + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL) + return + } + + // Ensure body content type is opaque to ensure that request body has not + // been interpreted as form data. + contentType := r.Header.Get("Content-Type") + if contentType != "application/octet-stream" { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL) + return + } + + password := cred.SecretKey + reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength)) + if err != nil { + adminLogIf(ctx, err) + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL) + return + } + + idpCfgType := mux.Vars(r)["type"] + if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL) + return + } + + var subSys string + switch idpCfgType { + case madmin.OpenidIDPCfg: + subSys = madmin.IdentityOpenIDSubSys + case madmin.LDAPIDPCfg: + subSys = madmin.IdentityLDAPSubSys + } + + cfgName := mux.Vars(r)["name"] + cfgTarget := madmin.Default + if cfgName != "" { + cfgTarget = cfgName + if idpCfgType == madmin.LDAPIDPCfg && cfgName != madmin.Default { + // LDAP does not support multiple configurations. So cfgName must be + // empty or `madmin.Default`. + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigLDAPNonDefaultConfigName), r.URL) + return + } + } + + // Check that this is a valid Create vs Update API call. + s := globalServerConfig.Clone() + if apiErrCode := handleCreateUpdateValidation(s, subSys, cfgTarget, isUpdate); apiErrCode != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(apiErrCode), r.URL) + return + } + + cfgData := "" + { + tgtSuffix := "" + if cfgTarget != madmin.Default { + tgtSuffix = config.SubSystemSeparator + cfgTarget + } + cfgData = subSys + tgtSuffix + config.KvSpaceSeparator + string(reqBytes) + } + + cfg, err := readServerConfig(ctx, objectAPI, nil) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + dynamic, err := cfg.ReadConfig(strings.NewReader(cfgData)) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // IDP config is not dynamic. Sanity check. + if dynamic { + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), "", r.URL) + return + } + + if err = validateConfig(ctx, cfg, subSys); err != nil { + var validationErr ldap.Validation + if errors.As(err, &validationErr) { + // If we got an LDAP validation error, we need to send appropriate + // error message back to client (likely mc). + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigLDAPValidation), + validationErr.FormatError(), r.URL) + return + } + + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL) + return + } + + // Update the actual server config on disk. + if err = saveServerConfig(ctx, objectAPI, cfg); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Write to the config input KV to history. + if err = saveServerConfigHistory(ctx, objectAPI, []byte(cfgData)); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseHeadersOnly(w) +} + +func handleCreateUpdateValidation(s config.Config, subSys, cfgTarget string, isUpdate bool) APIErrorCode { + if cfgTarget != madmin.Default { + // This cannot give an error at this point. + subSysTargets, _ := s.GetAvailableTargets(subSys) + subSysTargetsSet := set.CreateStringSet(subSysTargets...) + if isUpdate && !subSysTargetsSet.Contains(cfgTarget) { + return ErrAdminConfigIDPCfgNameDoesNotExist + } + if !isUpdate && subSysTargetsSet.Contains(cfgTarget) { + return ErrAdminConfigIDPCfgNameAlreadyExists + } + + return ErrNone + } + + // For the default configuration name, since it will always be an available + // target, we need to check if a configuration value has been set previously + // to figure out if this is a valid create or update API call. + + // This cannot really error (FIXME: improve the type for GetConfigInfo) + var cfgInfos []madmin.IDPCfgInfo + switch subSys { + case madmin.IdentityOpenIDSubSys: + cfgInfos, _ = globalIAMSys.OpenIDConfig.GetConfigInfo(s, cfgTarget) + case madmin.IdentityLDAPSubSys: + cfgInfos, _ = globalIAMSys.LDAPConfig.GetConfigInfo(s, cfgTarget) + } + + if len(cfgInfos) > 0 && !isUpdate { + return ErrAdminConfigIDPCfgNameAlreadyExists + } + if len(cfgInfos) == 0 && isUpdate { + return ErrAdminConfigIDPCfgNameDoesNotExist + } + return ErrNone +} + +// AddIdentityProviderCfg: adds a new IDP config for openid/ldap. +// +// PUT /idp-cfg/openid/dex1 -> create named config `dex1` +// +// PUT /idp-cfg/openid/_ -> create (default) named config `_` +func (a adminAPIHandlers) AddIdentityProviderCfg(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + addOrUpdateIDPHandler(ctx, w, r, false) +} + +// UpdateIdentityProviderCfg: updates an existing IDP config for openid/ldap. +// +// POST /idp-cfg/openid/dex1 -> update named config `dex1` +// +// POST /idp-cfg/openid/_ -> update (default) named config `_` +func (a adminAPIHandlers) UpdateIdentityProviderCfg(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + addOrUpdateIDPHandler(ctx, w, r, true) +} + +// ListIdentityProviderCfg: +// +// GET /idp-cfg/openid -> lists openid provider configs. +func (a adminAPIHandlers) ListIdentityProviderCfg(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction) + if objectAPI == nil { + return + } + password := cred.SecretKey + + idpCfgType := mux.Vars(r)["type"] + if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL) + return + } + + var cfgList []madmin.IDPListItem + var err error + switch idpCfgType { + case madmin.OpenidIDPCfg: + cfg := globalServerConfig.Clone() + cfgList, err = globalIAMSys.OpenIDConfig.GetConfigList(cfg) + case madmin.LDAPIDPCfg: + cfg := globalServerConfig.Clone() + cfgList, err = globalIAMSys.LDAPConfig.GetConfigList(cfg) + + default: + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + data, err := json.Marshal(cfgList) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + econfigData, err := madmin.EncryptData(password, data) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, econfigData) +} + +// GetIdentityProviderCfg: +// +// GET /idp-cfg/openid/dex_test +func (a adminAPIHandlers) GetIdentityProviderCfg(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, cred := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction) + if objectAPI == nil { + return + } + + idpCfgType := mux.Vars(r)["type"] + cfgName := mux.Vars(r)["name"] + password := cred.SecretKey + + if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL) + return + } + + cfg := globalServerConfig.Clone() + var cfgInfos []madmin.IDPCfgInfo + var err error + switch idpCfgType { + case madmin.OpenidIDPCfg: + cfgInfos, err = globalIAMSys.OpenIDConfig.GetConfigInfo(cfg, cfgName) + case madmin.LDAPIDPCfg: + cfgInfos, err = globalIAMSys.LDAPConfig.GetConfigInfo(cfg, cfgName) + } + if err != nil { + if errors.Is(err, openid.ErrProviderConfigNotFound) || errors.Is(err, cfgldap.ErrProviderConfigNotFound) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL) + return + } + + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + res := madmin.IDPConfig{ + Type: idpCfgType, + Name: cfgName, + Info: cfgInfos, + } + data, err := json.Marshal(res) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + econfigData, err := madmin.EncryptData(password, data) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, econfigData) +} + +// DeleteIdentityProviderCfg: +// +// DELETE /idp-cfg/openid/dex_test +func (a adminAPIHandlers) DeleteIdentityProviderCfg(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConfigUpdateAdminAction) + if objectAPI == nil { + return + } + + idpCfgType := mux.Vars(r)["type"] + cfgName := mux.Vars(r)["name"] + if !madmin.ValidIDPConfigTypes.Contains(idpCfgType) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigInvalidIDPType), r.URL) + return + } + + cfgCopy := globalServerConfig.Clone() + var subSys string + switch idpCfgType { + case madmin.OpenidIDPCfg: + subSys = config.IdentityOpenIDSubSys + cfgInfos, err := globalIAMSys.OpenIDConfig.GetConfigInfo(cfgCopy, cfgName) + if err != nil { + if errors.Is(err, openid.ErrProviderConfigNotFound) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL) + return + } + + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + hasEnv := false + for _, ci := range cfgInfos { + if ci.IsCfg && ci.IsEnv { + hasEnv = true + break + } + } + + if hasEnv { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigEnvOverridden), r.URL) + return + } + case madmin.LDAPIDPCfg: + subSys = config.IdentityLDAPSubSys + cfgInfos, err := globalIAMSys.LDAPConfig.GetConfigInfo(cfgCopy, cfgName) + if err != nil { + if errors.Is(err, openid.ErrProviderConfigNotFound) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL) + return + } + + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + hasEnv := false + for _, ci := range cfgInfos { + if ci.IsCfg && ci.IsEnv { + hasEnv = true + break + } + } + + if hasEnv { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigEnvOverridden), r.URL) + return + } + default: + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + + cfg, err := readServerConfig(ctx, objectAPI, nil) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + cfgKey := fmt.Sprintf("%s:%s", subSys, cfgName) + if cfgName == madmin.Default { + cfgKey = subSys + } + if err = cfg.DelKVS(cfgKey); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + if err = validateConfig(ctx, cfg, subSys); err != nil { + var validationErr ldap.Validation + if errors.As(err, &validationErr) { + // If we got an LDAP validation error, we need to send appropriate + // error message back to client (likely mc). + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigLDAPValidation), + validationErr.FormatError(), r.URL) + return + } + + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), err.Error(), r.URL) + return + } + if err = saveServerConfig(ctx, objectAPI, cfg); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + dynamic := config.SubSystemsDynamic.Contains(subSys) + if dynamic { + applyDynamic(ctx, objectAPI, cfg, subSys, r, w) + } +} diff --git a/cmd/admin-handlers-idp-ldap.go b/cmd/admin-handlers-idp-ldap.go new file mode 100644 index 00000000..6807de18 --- /dev/null +++ b/cmd/admin-handlers-idp-ldap.go @@ -0,0 +1,657 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "strings" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/auth" + "github.com/minio/mux" + xldap "github.com/minio/pkg/v3/ldap" + "github.com/minio/pkg/v3/policy" +) + +// ListLDAPPolicyMappingEntities lists users/groups mapped to given/all policies. +// +// GET /idp/ldap/policy-entities?[query-params] +// +// Query params: +// +// user=... -> repeatable query parameter, specifying users to query for +// policy mapping +// +// group=... -> repeatable query parameter, specifying groups to query for +// policy mapping +// +// policy=... -> repeatable query parameter, specifying policy to query for +// user/group mapping +// +// When all query parameters are omitted, returns mappings for all policies. +func (a adminAPIHandlers) ListLDAPPolicyMappingEntities(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Check authorization. + + objectAPI, cred := validateAdminReq(ctx, w, r, + policy.ListGroupsAdminAction, policy.ListUsersAdminAction, policy.ListUserPoliciesAdminAction) + if objectAPI == nil { + return + } + + // Validate API arguments. + + q := madmin.PolicyEntitiesQuery{ + Users: r.Form["user"], + Groups: r.Form["group"], + Policy: r.Form["policy"], + } + + // Query IAM + + res, err := globalIAMSys.QueryLDAPPolicyEntities(r.Context(), q) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Encode result and send response. + + data, err := json.Marshal(res) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + password := cred.SecretKey + econfigData, err := madmin.EncryptData(password, data) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + writeSuccessResponseJSON(w, econfigData) +} + +// AttachDetachPolicyLDAP attaches or detaches policies from an LDAP entity +// (user or group). +// +// POST /idp/ldap/policy/{operation} +func (a adminAPIHandlers) AttachDetachPolicyLDAP(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Check authorization. + + objectAPI, cred := validateAdminReq(ctx, w, r, policy.UpdatePolicyAssociationAction) + if objectAPI == nil { + return + } + + // fail if ldap is not enabled + if !globalIAMSys.LDAPConfig.Enabled() { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminLDAPNotEnabled), r.URL) + return + } + + if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 { + // More than maxConfigSize bytes were available + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL) + return + } + + // Ensure body content type is opaque to ensure that request body has not + // been interpreted as form data. + contentType := r.Header.Get("Content-Type") + if contentType != "application/octet-stream" { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL) + return + } + + // Validate operation + operation := mux.Vars(r)["operation"] + if operation != "attach" && operation != "detach" { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL) + return + } + + isAttach := operation == "attach" + + // Validate API arguments in body. + password := cred.SecretKey + reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength)) + if err != nil { + adminLogIf(ctx, err) + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL) + return + } + + var par madmin.PolicyAssociationReq + err = json.Unmarshal(reqBytes, &par) + if err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + + if err := par.IsValid(); err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL) + return + } + + // Call IAM subsystem + updatedAt, addedOrRemoved, _, err := globalIAMSys.PolicyDBUpdateLDAP(ctx, isAttach, par) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + respBody := madmin.PolicyAssociationResp{ + UpdatedAt: updatedAt, + } + if isAttach { + respBody.PoliciesAttached = addedOrRemoved + } else { + respBody.PoliciesDetached = addedOrRemoved + } + + data, err := json.Marshal(respBody) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + encryptedData, err := madmin.EncryptData(password, data) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, encryptedData) +} + +// AddServiceAccountLDAP adds a new service account for provided LDAP username or DN +// +// PUT /minio/admin/v3/idp/ldap/add-service-account +func (a adminAPIHandlers) AddServiceAccountLDAP(w http.ResponseWriter, r *http.Request) { + ctx, cred, opts, createReq, targetUser, APIError := commonAddServiceAccount(r, true) + if APIError.Code != "" { + writeErrorResponseJSON(ctx, w, APIError, r.URL) + return + } + + // fail if ldap is not enabled + if !globalIAMSys.LDAPConfig.Enabled() { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminLDAPNotEnabled), r.URL) + return + } + + // Find the user for the request sender (as it may be sent via a service + // account or STS account): + requestorUser := cred.AccessKey + requestorParentUser := cred.AccessKey + requestorGroups := cred.Groups + requestorIsDerivedCredential := false + if cred.IsServiceAccount() || cred.IsTemp() { + requestorParentUser = cred.ParentUser + requestorIsDerivedCredential = true + } + + // Check if we are creating svc account for request sender. + isSvcAccForRequestor := targetUser == requestorUser || targetUser == requestorParentUser + + var ( + targetGroups []string + err error + ) + + // If we are creating svc account for request sender, ensure that targetUser + // is a real user (i.e. not derived credentials). + if isSvcAccForRequestor { + if requestorIsDerivedCredential { + if requestorParentUser == "" { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, + errors.New("service accounts cannot be generated for temporary credentials without parent")), r.URL) + return + } + targetUser = requestorParentUser + } + targetGroups = requestorGroups + + // Deny if the target user is not LDAP + foundResult, err := globalIAMSys.LDAPConfig.GetValidatedDNForUsername(targetUser) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + if foundResult == nil { + err := errors.New("Specified user does not exist on LDAP server") + APIErr := errorCodes.ToAPIErrWithErr(ErrAdminNoSuchUser, err) + writeErrorResponseJSON(ctx, w, APIErr, r.URL) + return + } + + // In case of LDAP/OIDC we need to set `opts.claims` to ensure + // it is associated with the LDAP/OIDC user properly. + for k, v := range cred.Claims { + if k == expClaim { + continue + } + opts.claims[k] = v + } + } else { + // We still need to ensure that the target user is a valid LDAP user. + // + // The target user may be supplied as a (short) username or a DN. + // However, for now, we only support using the short username. + + isDN := globalIAMSys.LDAPConfig.ParsesAsDN(targetUser) + opts.claims[ldapUserN] = targetUser // simple username + var lookupResult *xldap.DNSearchResult + lookupResult, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(targetUser) + if err != nil { + // if not found, check if DN + if strings.Contains(err.Error(), "User DN not found for:") { + if isDN { + // warn user that DNs are not allowed + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminLDAPExpectedLoginName, err), r.URL) + } else { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminNoSuchUser, err), r.URL) + } + } + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + targetUser = lookupResult.NormDN + opts.claims[ldapUser] = targetUser // DN + opts.claims[ldapActualUser] = lookupResult.ActualDN + + // Check if this user or their groups have a policy applied. + ldapPolicies, err := globalIAMSys.PolicyDBGet(targetUser, targetGroups...) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + if len(ldapPolicies) == 0 { + err = fmt.Errorf("No policy set for user `%s` or any of their groups: `%s`", opts.claims[ldapActualUser], strings.Join(targetGroups, "`,`")) + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminNoSuchUser, err), r.URL) + return + } + + // Add LDAP attributes that were looked up into the claims. + for attribKey, attribValue := range lookupResult.Attributes { + opts.claims[ldapAttribPrefix+attribKey] = attribValue + } + } + + newCred, updatedAt, err := globalIAMSys.NewServiceAccount(ctx, targetUser, targetGroups, opts) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + createResp := madmin.AddServiceAccountResp{ + Credentials: madmin.Credentials{ + AccessKey: newCred.AccessKey, + SecretKey: newCred.SecretKey, + Expiration: newCred.Expiration, + }, + } + + data, err := json.Marshal(createResp) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + encryptedData, err := madmin.EncryptData(cred.SecretKey, data) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, encryptedData) + + // Call hook for cluster-replication if the service account is not for a + // root user. + if newCred.ParentUser != globalActiveCred.AccessKey { + replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemSvcAcc, + SvcAccChange: &madmin.SRSvcAccChange{ + Create: &madmin.SRSvcAccCreate{ + Parent: newCred.ParentUser, + AccessKey: newCred.AccessKey, + SecretKey: newCred.SecretKey, + Groups: newCred.Groups, + Name: newCred.Name, + Description: newCred.Description, + Claims: opts.claims, + SessionPolicy: madmin.SRSessionPolicy(createReq.Policy), + Status: auth.AccountOn, + Expiration: createReq.Expiration, + }, + }, + UpdatedAt: updatedAt, + })) + } +} + +// ListAccessKeysLDAP - GET /minio/admin/v3/idp/ldap/list-access-keys +func (a adminAPIHandlers) ListAccessKeysLDAP(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Get current object layer instance. + objectAPI := newObjectLayerFn() + if objectAPI == nil || globalNotificationSys == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + cred, owner, s3Err := validateAdminSignature(ctx, r, "") + if s3Err != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + + userDN := r.Form.Get("userDN") + + // If listing is requested for a specific user (who is not the request + // sender), check that the user has permissions. + if userDN != "" && userDN != cred.ParentUser { + if !globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.ListServiceAccountsAdminAction, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + Claims: cred.Claims, + }) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + } else { + if !globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.ListServiceAccountsAdminAction, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + Claims: cred.Claims, + DenyOnly: true, + }) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + userDN = cred.AccessKey + if cred.ParentUser != "" { + userDN = cred.ParentUser + } + } + + dnResult, err := globalIAMSys.LDAPConfig.GetValidatedDNForUsername(userDN) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + if dnResult == nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL) + return + } + targetAccount := dnResult.NormDN + + listType := r.Form.Get("listType") + if listType != "sts-only" && listType != "svcacc-only" && listType != "" { + // default to both + listType = "" + } + + var serviceAccounts []auth.Credentials + var stsKeys []auth.Credentials + + if listType == "" || listType == "sts-only" { + stsKeys, err = globalIAMSys.ListSTSAccounts(ctx, targetAccount) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + } + if listType == "" || listType == "svcacc-only" { + serviceAccounts, err = globalIAMSys.ListServiceAccounts(ctx, targetAccount) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + } + + var serviceAccountList []madmin.ServiceAccountInfo + var stsKeyList []madmin.ServiceAccountInfo + + for _, svc := range serviceAccounts { + expiryTime := svc.Expiration + serviceAccountList = append(serviceAccountList, madmin.ServiceAccountInfo{ + AccessKey: svc.AccessKey, + Expiration: &expiryTime, + Name: svc.Name, + Description: svc.Description, + }) + } + for _, sts := range stsKeys { + expiryTime := sts.Expiration + stsKeyList = append(stsKeyList, madmin.ServiceAccountInfo{ + AccessKey: sts.AccessKey, + Expiration: &expiryTime, + }) + } + + listResp := madmin.ListAccessKeysLDAPResp{ + ServiceAccounts: serviceAccountList, + STSKeys: stsKeyList, + } + + data, err := json.Marshal(listResp) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + encryptedData, err := madmin.EncryptData(cred.SecretKey, data) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, encryptedData) +} + +// ListAccessKeysLDAPBulk - GET /minio/admin/v3/idp/ldap/list-access-keys-bulk +func (a adminAPIHandlers) ListAccessKeysLDAPBulk(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Get current object layer instance. + objectAPI := newObjectLayerFn() + if objectAPI == nil || globalNotificationSys == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + cred, owner, s3Err := validateAdminSignature(ctx, r, "") + if s3Err != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + + dnList := r.Form["userDNs"] + isAll := r.Form.Get("all") == "true" + selfOnly := !isAll && len(dnList) == 0 + + if isAll && len(dnList) > 0 { + // This should be checked on client side, so return generic error + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + + // Empty DN list and not self, list access keys for all users + if isAll { + if !globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.ListUsersAdminAction, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + Claims: cred.Claims, + }) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + } else if len(dnList) == 1 { + var dn string + foundResult, err := globalIAMSys.LDAPConfig.GetValidatedDNForUsername(dnList[0]) + if err == nil { + dn = foundResult.NormDN + } + if dn == cred.ParentUser || dnList[0] == cred.ParentUser { + selfOnly = true + } + } + + if !globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.ListServiceAccountsAdminAction, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + Claims: cred.Claims, + DenyOnly: selfOnly, + }) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + + if selfOnly && len(dnList) == 0 { + selfDN := cred.AccessKey + if cred.ParentUser != "" { + selfDN = cred.ParentUser + } + dnList = append(dnList, selfDN) + } + + var ldapUserList []string + if isAll { + ldapUsers, err := globalIAMSys.ListLDAPUsers(ctx) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + for user := range ldapUsers { + ldapUserList = append(ldapUserList, user) + } + } else { + for _, userDN := range dnList { + // Validate the userDN + foundResult, err := globalIAMSys.LDAPConfig.GetValidatedDNForUsername(userDN) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + if foundResult == nil { + continue + } + ldapUserList = append(ldapUserList, foundResult.NormDN) + } + } + + listType := r.Form.Get("listType") + var listSTSKeys, listServiceAccounts bool + switch listType { + case madmin.AccessKeyListUsersOnly: + listSTSKeys = false + listServiceAccounts = false + case madmin.AccessKeyListSTSOnly: + listSTSKeys = true + listServiceAccounts = false + case madmin.AccessKeyListSvcaccOnly: + listSTSKeys = false + listServiceAccounts = true + case madmin.AccessKeyListAll: + listSTSKeys = true + listServiceAccounts = true + default: + err := errors.New("invalid list type") + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrInvalidRequest, err), r.URL) + return + } + + accessKeyMap := make(map[string]madmin.ListAccessKeysLDAPResp) + for _, internalDN := range ldapUserList { + externalDN := globalIAMSys.LDAPConfig.DecodeDN(internalDN) + accessKeys := madmin.ListAccessKeysLDAPResp{} + if listSTSKeys { + stsKeys, err := globalIAMSys.ListSTSAccounts(ctx, internalDN) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + for _, sts := range stsKeys { + accessKeys.STSKeys = append(accessKeys.STSKeys, madmin.ServiceAccountInfo{ + AccessKey: sts.AccessKey, + Expiration: &sts.Expiration, + }) + } + // if only STS keys, skip if user has no STS keys + if !listServiceAccounts && len(stsKeys) == 0 { + continue + } + } + + if listServiceAccounts { + serviceAccounts, err := globalIAMSys.ListServiceAccounts(ctx, internalDN) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + for _, svc := range serviceAccounts { + accessKeys.ServiceAccounts = append(accessKeys.ServiceAccounts, madmin.ServiceAccountInfo{ + AccessKey: svc.AccessKey, + Expiration: &svc.Expiration, + Name: svc.Name, + Description: svc.Description, + }) + } + // if only service accounts, skip if user has no service accounts + if !listSTSKeys && len(serviceAccounts) == 0 { + continue + } + } + accessKeyMap[externalDN] = accessKeys + } + + data, err := json.Marshal(accessKeyMap) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + encryptedData, err := madmin.EncryptData(cred.SecretKey, data) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, encryptedData) +} diff --git a/cmd/admin-handlers-idp-openid.go b/cmd/admin-handlers-idp-openid.go new file mode 100644 index 00000000..7e238783 --- /dev/null +++ b/cmd/admin-handlers-idp-openid.go @@ -0,0 +1,248 @@ +// Copyright (c) 2015-2025 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "encoding/json" + "errors" + "net/http" + "sort" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/pkg/v3/policy" +) + +const dummyRoleARN = "dummy-internal" + +// ListAccessKeysOpenIDBulk - GET /minio/admin/v3/idp/openid/list-access-keys-bulk +func (a adminAPIHandlers) ListAccessKeysOpenIDBulk(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Get current object layer instance. + objectAPI := newObjectLayerFn() + if objectAPI == nil || globalNotificationSys == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + cred, owner, s3Err := validateAdminSignature(ctx, r, "") + if s3Err != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + + if !globalIAMSys.OpenIDConfig.Enabled { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminOpenIDNotEnabled), r.URL) + return + } + + userList := r.Form["users"] + isAll := r.Form.Get("all") == "true" + selfOnly := !isAll && len(userList) == 0 + cfgName := r.Form.Get("configName") + allConfigs := r.Form.Get("allConfigs") == "true" + if cfgName == "" && !allConfigs { + cfgName = madmin.Default + } + + if isAll && len(userList) > 0 { + // This should be checked on client side, so return generic error + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + + // Empty DN list and not self, list access keys for all users + if isAll { + if !globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.ListUsersAdminAction, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + Claims: cred.Claims, + }) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + } else if len(userList) == 1 && userList[0] == cred.ParentUser { + selfOnly = true + } + + if !globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.ListServiceAccountsAdminAction, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + Claims: cred.Claims, + DenyOnly: selfOnly, + }) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + + if selfOnly && len(userList) == 0 { + selfDN := cred.AccessKey + if cred.ParentUser != "" { + selfDN = cred.ParentUser + } + userList = append(userList, selfDN) + } + + listType := r.Form.Get("listType") + var listSTSKeys, listServiceAccounts bool + switch listType { + case madmin.AccessKeyListUsersOnly: + listSTSKeys = false + listServiceAccounts = false + case madmin.AccessKeyListSTSOnly: + listSTSKeys = true + listServiceAccounts = false + case madmin.AccessKeyListSvcaccOnly: + listSTSKeys = false + listServiceAccounts = true + case madmin.AccessKeyListAll: + listSTSKeys = true + listServiceAccounts = true + default: + err := errors.New("invalid list type") + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrInvalidRequest, err), r.URL) + return + } + + s := globalServerConfig.Clone() + roleArnMap := make(map[string]string) + // Map of configs to a map of users to their access keys + cfgToUsersMap := make(map[string]map[string]madmin.OpenIDUserAccessKeys) + configs, err := globalIAMSys.OpenIDConfig.GetConfigList(s) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + for _, config := range configs { + if !allConfigs && cfgName != config.Name { + continue + } + arn := dummyRoleARN + if config.RoleARN != "" { + arn = config.RoleARN + } + roleArnMap[arn] = config.Name + newResp := make(map[string]madmin.OpenIDUserAccessKeys) + cfgToUsersMap[config.Name] = newResp + } + if len(roleArnMap) == 0 { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchConfigTarget), r.URL) + return + } + + userSet := set.CreateStringSet(userList...) + accessKeys, err := globalIAMSys.ListAllAccessKeys(ctx) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + for _, accessKey := range accessKeys { + // Filter out any disqualifying access keys + _, ok := accessKey.Claims[subClaim] + if !ok { + continue // OpenID access keys must have a sub claim + } + if (!listSTSKeys && !accessKey.IsServiceAccount()) || (!listServiceAccounts && accessKey.IsServiceAccount()) { + continue // skip if not the type we want + } + arn, ok := accessKey.Claims[roleArnClaim].(string) + if !ok { + if _, ok := accessKey.Claims[iamPolicyClaimNameOpenID()]; !ok { + continue // skip if no roleArn and no policy claim + } + // claim-based provider is in the roleArnMap under dummy ARN + arn = dummyRoleARN + } + matchingCfgName, ok := roleArnMap[arn] + if !ok { + continue // skip if not part of the target config + } + var id string + if idClaim := globalIAMSys.OpenIDConfig.GetUserIDClaim(matchingCfgName); idClaim != "" { + id, _ = accessKey.Claims[idClaim].(string) + } + if !userSet.IsEmpty() && !userSet.Contains(accessKey.ParentUser) && !userSet.Contains(id) { + continue // skip if not in the user list + } + openIDUserAccessKeys, ok := cfgToUsersMap[matchingCfgName][accessKey.ParentUser] + + // Add new user to map if not already present + if !ok { + var readableClaim string + if rc := globalIAMSys.OpenIDConfig.GetUserReadableClaim(matchingCfgName); rc != "" { + readableClaim, _ = accessKey.Claims[rc].(string) + } + openIDUserAccessKeys = madmin.OpenIDUserAccessKeys{ + MinioAccessKey: accessKey.ParentUser, + ID: id, + ReadableName: readableClaim, + } + } + svcAccInfo := madmin.ServiceAccountInfo{ + AccessKey: accessKey.AccessKey, + Expiration: &accessKey.Expiration, + } + if accessKey.IsServiceAccount() { + openIDUserAccessKeys.ServiceAccounts = append(openIDUserAccessKeys.ServiceAccounts, svcAccInfo) + } else { + openIDUserAccessKeys.STSKeys = append(openIDUserAccessKeys.STSKeys, svcAccInfo) + } + cfgToUsersMap[matchingCfgName][accessKey.ParentUser] = openIDUserAccessKeys + } + + // Convert map to slice and sort + resp := make([]madmin.ListAccessKeysOpenIDResp, 0, len(cfgToUsersMap)) + for cfgName, usersMap := range cfgToUsersMap { + users := make([]madmin.OpenIDUserAccessKeys, 0, len(usersMap)) + for _, user := range usersMap { + users = append(users, user) + } + sort.Slice(users, func(i, j int) bool { + return users[i].MinioAccessKey < users[j].MinioAccessKey + }) + resp = append(resp, madmin.ListAccessKeysOpenIDResp{ + ConfigName: cfgName, + Users: users, + }) + } + sort.Slice(resp, func(i, j int) bool { + return resp[i].ConfigName < resp[j].ConfigName + }) + + data, err := json.Marshal(resp) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + encryptedData, err := madmin.EncryptData(cred.SecretKey, data) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, encryptedData) +} diff --git a/cmd/admin-handlers-pools.go b/cmd/admin-handlers-pools.go new file mode 100644 index 00000000..c4f98c45 --- /dev/null +++ b/cmd/admin-handlers-pools.go @@ -0,0 +1,393 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/minio/mux" + "github.com/minio/pkg/v3/env" + "github.com/minio/pkg/v3/policy" +) + +var ( + errRebalanceDecommissionAlreadyRunning = errors.New("Rebalance cannot be started, decommission is already in progress") + errDecommissionRebalanceAlreadyRunning = errors.New("Decommission cannot be started, rebalance is already in progress") +) + +func (a adminAPIHandlers) StartDecommission(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.DecommissionAdminAction) + if objectAPI == nil { + return + } + + // Legacy args style such as non-ellipses style is not supported with this API. + if globalEndpoints.Legacy() { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + + z, ok := objectAPI.(*erasureServerPools) + if !ok || len(z.serverPools) == 1 { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + + if z.IsDecommissionRunning() { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errDecommissionAlreadyRunning), r.URL) + return + } + + if z.IsRebalanceStarted(ctx) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminRebalanceAlreadyStarted), r.URL) + return + } + + vars := mux.Vars(r) + v := vars["pool"] + byID := vars["by-id"] == "true" + + pools := strings.Split(v, ",") + poolIndices := make([]int, 0, len(pools)) + + for _, pool := range pools { + var idx int + if byID { + var err error + idx, err = strconv.Atoi(pool) + if err != nil { + // We didn't find any matching pools, invalid input + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL) + return + } + } else { + idx = globalEndpoints.GetPoolIdx(pool) + if idx == -1 { + // We didn't find any matching pools, invalid input + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL) + return + } + } + var pool *erasureSets + for pidx := range z.serverPools { + if pidx == idx { + pool = z.serverPools[idx] + break + } + } + if pool == nil { + // We didn't find any matching pools, invalid input + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL) + return + } + + poolIndices = append(poolIndices, idx) + } + + if len(poolIndices) == 0 || !proxyDecommissionRequest(ctx, globalEndpoints[poolIndices[0]].Endpoints[0], w, r) { + if err := z.Decommission(r.Context(), poolIndices...); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + } +} + +func (a adminAPIHandlers) CancelDecommission(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.DecommissionAdminAction) + if objectAPI == nil { + return + } + + // Legacy args style such as non-ellipses style is not supported with this API. + if globalEndpoints.Legacy() { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + + pools, ok := objectAPI.(*erasureServerPools) + if !ok { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + + vars := mux.Vars(r) + v := vars["pool"] + byID := vars["by-id"] == "true" + idx := -1 + + if byID { + if i, err := strconv.Atoi(v); err == nil && i >= 0 && i < len(globalEndpoints) { + idx = i + } + } else { + idx = globalEndpoints.GetPoolIdx(v) + } + + if idx == -1 { + // We didn't find any matching pools, invalid input + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL) + return + } + + if !proxyDecommissionRequest(ctx, globalEndpoints[idx].Endpoints[0], w, r) { + if err := pools.DecommissionCancel(ctx, idx); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + } +} + +func (a adminAPIHandlers) StatusPool(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerInfoAdminAction, policy.DecommissionAdminAction) + if objectAPI == nil { + return + } + + // Legacy args style such as non-ellipses style is not supported with this API. + if globalEndpoints.Legacy() { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + + pools, ok := objectAPI.(*erasureServerPools) + if !ok { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + + vars := mux.Vars(r) + v := vars["pool"] + byID := vars["by-id"] == "true" + idx := -1 + + if byID { + if i, err := strconv.Atoi(v); err == nil && i >= 0 && i < len(globalEndpoints) { + idx = i + } + } else { + idx = globalEndpoints.GetPoolIdx(v) + } + + if idx == -1 { + apiErr := toAdminAPIErr(ctx, errInvalidArgument) + apiErr.Description = fmt.Sprintf("specified pool '%s' not found, please specify a valid pool", v) + // We didn't find any matching pools, invalid input + writeErrorResponseJSON(ctx, w, apiErr, r.URL) + return + } + + status, err := pools.Status(r.Context(), idx) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + adminLogIf(r.Context(), json.NewEncoder(w).Encode(&status)) +} + +func (a adminAPIHandlers) ListPools(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerInfoAdminAction, policy.DecommissionAdminAction) + if objectAPI == nil { + return + } + + // Legacy args style such as non-ellipses style is not supported with this API. + if globalEndpoints.Legacy() { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + + pools, ok := objectAPI.(*erasureServerPools) + if !ok { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + + poolsStatus := make([]PoolStatus, len(globalEndpoints)) + for idx := range globalEndpoints { + status, err := pools.Status(r.Context(), idx) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + poolsStatus[idx] = status + } + + adminLogIf(r.Context(), json.NewEncoder(w).Encode(poolsStatus)) +} + +func (a adminAPIHandlers) RebalanceStart(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.RebalanceAdminAction) + if objectAPI == nil { + return + } + + // NB rebalance-start admin API is always coordinated from first pool's + // first node. The following is required to serialize (the effects of) + // concurrent rebalance-start commands. + if ep := globalEndpoints[0].Endpoints[0]; !ep.IsLocal { + for nodeIdx, proxyEp := range globalProxyEndpoints { + if proxyEp.Host == ep.Host { + if proxied, success := proxyRequestByNodeIndex(ctx, w, r, nodeIdx, false); proxied && success { + return + } + } + } + } + + pools, ok := objectAPI.(*erasureServerPools) + if !ok || len(pools.serverPools) == 1 { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + + if pools.IsDecommissionRunning() { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errRebalanceDecommissionAlreadyRunning), r.URL) + return + } + + if pools.IsRebalanceStarted(ctx) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminRebalanceAlreadyStarted), r.URL) + return + } + + bucketInfos, err := objectAPI.ListBuckets(ctx, BucketOptions{}) + if err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + buckets := make([]string, 0, len(bucketInfos)) + for _, bInfo := range bucketInfos { + buckets = append(buckets, bInfo.Name) + } + + var id string + if id, err = pools.initRebalanceMeta(ctx, buckets); err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Rebalance routine is run on the first node of any pool participating in rebalance. + pools.StartRebalance() + + b, err := json.Marshal(struct { + ID string `json:"id"` + }{ID: id}) + if err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, b) + // Notify peers to load rebalance.bin and start rebalance routine if they happen to be + // participating pool's leader node + globalNotificationSys.LoadRebalanceMeta(ctx, true) +} + +func (a adminAPIHandlers) RebalanceStatus(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.RebalanceAdminAction) + if objectAPI == nil { + return + } + + // Proxy rebalance-status to first pool first node, so that users see a + // consistent view of rebalance progress even though different rebalancing + // pools may temporarily have out of date info on the others. + if ep := globalEndpoints[0].Endpoints[0]; !ep.IsLocal { + for nodeIdx, proxyEp := range globalProxyEndpoints { + if proxyEp.Host == ep.Host { + if proxied, success := proxyRequestByNodeIndex(ctx, w, r, nodeIdx, false); proxied && success { + return + } + } + } + } + + pools, ok := objectAPI.(*erasureServerPools) + if !ok { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + + rs, err := rebalanceStatus(ctx, pools) + if err != nil { + if errors.Is(err, errRebalanceNotStarted) || errors.Is(err, errConfigNotFound) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminRebalanceNotStarted), r.URL) + return + } + adminLogIf(ctx, fmt.Errorf("failed to fetch rebalance status: %w", err)) + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + adminLogIf(r.Context(), json.NewEncoder(w).Encode(rs)) +} + +func (a adminAPIHandlers) RebalanceStop(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.RebalanceAdminAction) + if objectAPI == nil { + return + } + + pools, ok := objectAPI.(*erasureServerPools) + if !ok { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + + // Cancel any ongoing rebalance operation + globalNotificationSys.StopRebalance(r.Context()) + writeSuccessResponseHeadersOnly(w) + adminLogIf(ctx, pools.saveRebalanceStats(GlobalContext, 0, rebalSaveStoppedAt)) + globalNotificationSys.LoadRebalanceMeta(ctx, false) +} + +func proxyDecommissionRequest(ctx context.Context, defaultEndPoint Endpoint, w http.ResponseWriter, r *http.Request) (proxy bool) { + host := env.Get("_MINIO_DECOM_ENDPOINT_HOST", defaultEndPoint.Host) + if host == "" { + return proxy + } + for nodeIdx, proxyEp := range globalProxyEndpoints { + if proxyEp.Host == host && !proxyEp.IsLocal { + if proxied, success := proxyRequestByNodeIndex(ctx, w, r, nodeIdx, false); proxied && success { + return true + } + } + } + return proxy +} diff --git a/cmd/admin-handlers-site-replication.go b/cmd/admin-handlers-site-replication.go new file mode 100644 index 00000000..bda09395 --- /dev/null +++ b/cmd/admin-handlers-site-replication.go @@ -0,0 +1,623 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/gob" + "encoding/json" + "errors" + "io" + "net/http" + "strings" + "sync/atomic" + "time" + + "github.com/dustin/go-humanize" + "github.com/minio/madmin-go/v3" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/mux" + "github.com/minio/pkg/v3/policy" +) + +// SiteReplicationAdd - PUT /minio/admin/v3/site-replication/add +func (a adminAPIHandlers) SiteReplicationAdd(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, cred := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction) + if objectAPI == nil { + return + } + + var sites []madmin.PeerSite + if err := parseJSONBody(ctx, r.Body, &sites, cred.SecretKey); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + opts := getSRAddOptions(r) + status, err := globalSiteReplicationSys.AddPeerClusters(ctx, sites, opts) + if err != nil { + adminLogIf(ctx, err) + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + body, err := json.Marshal(status) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, body) +} + +func getSRAddOptions(r *http.Request) (opts madmin.SRAddOptions) { + opts.ReplicateILMExpiry = r.Form.Get("replicateILMExpiry") == "true" + return opts +} + +// SRPeerJoin - PUT /minio/admin/v3/site-replication/join +// +// used internally to tell current cluster to enable SR with +// the provided peer clusters and service account. +func (a adminAPIHandlers) SRPeerJoin(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, cred := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction) + if objectAPI == nil { + return + } + + var joinArg madmin.SRPeerJoinReq + if err := parseJSONBody(ctx, r.Body, &joinArg, cred.SecretKey); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + if err := globalSiteReplicationSys.PeerJoinReq(ctx, joinArg); err != nil { + adminLogIf(ctx, err) + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } +} + +// SRPeerBucketOps - PUT /minio/admin/v3/site-replication/bucket-ops?bucket=x&operation=y +func (a adminAPIHandlers) SRPeerBucketOps(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction) + if objectAPI == nil { + return + } + + vars := mux.Vars(r) + bucket := vars["bucket"] + operation := madmin.BktOp(vars["operation"]) + + var err error + switch operation { + default: + err = errSRInvalidRequest(errInvalidArgument) + case madmin.MakeWithVersioningBktOp: + createdAt, cerr := time.Parse(time.RFC3339Nano, strings.TrimSpace(r.Form.Get("createdAt"))) + if cerr != nil { + createdAt = timeSentinel + } + + opts := MakeBucketOptions{ + LockEnabled: r.Form.Get("lockEnabled") == "true", + VersioningEnabled: r.Form.Get("versioningEnabled") == "true", + ForceCreate: r.Form.Get("forceCreate") == "true", + CreatedAt: createdAt, + } + err = globalSiteReplicationSys.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts) + case madmin.ConfigureReplBktOp: + err = globalSiteReplicationSys.PeerBucketConfigureReplHandler(ctx, bucket) + case madmin.DeleteBucketBktOp, madmin.ForceDeleteBucketBktOp: + err = globalSiteReplicationSys.PeerBucketDeleteHandler(ctx, bucket, DeleteBucketOptions{ + Force: operation == madmin.ForceDeleteBucketBktOp, + SRDeleteOp: getSRBucketDeleteOp(true), + }) + case madmin.PurgeDeletedBucketOp: + globalSiteReplicationSys.purgeDeletedBucket(ctx, objectAPI, bucket) + } + if err != nil { + adminLogIf(ctx, err) + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } +} + +// SRPeerReplicateIAMItem - PUT /minio/admin/v3/site-replication/iam-item +func (a adminAPIHandlers) SRPeerReplicateIAMItem(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction) + if objectAPI == nil { + return + } + + var item madmin.SRIAMItem + if err := parseJSONBody(ctx, r.Body, &item, ""); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + var err error + switch item.Type { + default: + err = errSRInvalidRequest(errInvalidArgument) + case madmin.SRIAMItemPolicy: + if item.Policy == nil { + err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt) + } else { + policy, perr := policy.ParseConfig(bytes.NewReader(item.Policy)) + if perr != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, perr), r.URL) + return + } + if policy.IsEmpty() { + err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, nil, item.UpdatedAt) + } else { + err = globalSiteReplicationSys.PeerAddPolicyHandler(ctx, item.Name, policy, item.UpdatedAt) + } + } + case madmin.SRIAMItemSvcAcc: + err = globalSiteReplicationSys.PeerSvcAccChangeHandler(ctx, item.SvcAccChange, item.UpdatedAt) + case madmin.SRIAMItemPolicyMapping: + err = globalSiteReplicationSys.PeerPolicyMappingHandler(ctx, item.PolicyMapping, item.UpdatedAt) + case madmin.SRIAMItemSTSAcc: + err = globalSiteReplicationSys.PeerSTSAccHandler(ctx, item.STSCredential, item.UpdatedAt) + case madmin.SRIAMItemIAMUser: + err = globalSiteReplicationSys.PeerIAMUserChangeHandler(ctx, item.IAMUser, item.UpdatedAt) + case madmin.SRIAMItemGroupInfo: + err = globalSiteReplicationSys.PeerGroupInfoChangeHandler(ctx, item.GroupInfo, item.UpdatedAt) + } + if err != nil { + adminLogIf(ctx, err) + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } +} + +// SRPeerReplicateBucketItem - PUT /minio/admin/v3/site-replication/peer/bucket-meta +func (a adminAPIHandlers) SRPeerReplicateBucketItem(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction) + if objectAPI == nil { + return + } + + var item madmin.SRBucketMeta + if err := parseJSONBody(ctx, r.Body, &item, ""); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + if item.Bucket == "" { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errSRInvalidRequest(errInvalidArgument)), r.URL) + return + } + + var err error + switch item.Type { + default: + err = globalSiteReplicationSys.PeerBucketMetadataUpdateHandler(ctx, item) + case madmin.SRBucketMetaTypePolicy: + if item.Policy == nil { + err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil, item.UpdatedAt) + } else { + bktPolicy, berr := policy.ParseBucketPolicyConfig(bytes.NewReader(item.Policy), item.Bucket) + if berr != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, berr), r.URL) + return + } + if bktPolicy.IsEmpty() { + err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, nil, item.UpdatedAt) + } else { + err = globalSiteReplicationSys.PeerBucketPolicyHandler(ctx, item.Bucket, bktPolicy, item.UpdatedAt) + } + } + case madmin.SRBucketMetaTypeQuotaConfig: + if item.Quota == nil { + err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, nil, item.UpdatedAt) + } else { + quotaConfig, err := parseBucketQuota(item.Bucket, item.Quota) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + if err = globalSiteReplicationSys.PeerBucketQuotaConfigHandler(ctx, item.Bucket, quotaConfig, item.UpdatedAt); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + } + case madmin.SRBucketMetaTypeVersionConfig: + err = globalSiteReplicationSys.PeerBucketVersioningHandler(ctx, item.Bucket, item.Versioning, item.UpdatedAt) + case madmin.SRBucketMetaTypeTags: + err = globalSiteReplicationSys.PeerBucketTaggingHandler(ctx, item.Bucket, item.Tags, item.UpdatedAt) + case madmin.SRBucketMetaTypeObjectLockConfig: + err = globalSiteReplicationSys.PeerBucketObjectLockConfigHandler(ctx, item.Bucket, item.ObjectLockConfig, item.UpdatedAt) + case madmin.SRBucketMetaTypeSSEConfig: + err = globalSiteReplicationSys.PeerBucketSSEConfigHandler(ctx, item.Bucket, item.SSEConfig, item.UpdatedAt) + case madmin.SRBucketMetaLCConfig: + err = globalSiteReplicationSys.PeerBucketLCConfigHandler(ctx, item.Bucket, item.ExpiryLCConfig, item.UpdatedAt) + } + if err != nil { + adminLogIf(ctx, err) + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } +} + +// SiteReplicationInfo - GET /minio/admin/v3/site-replication/info +func (a adminAPIHandlers) SiteReplicationInfo(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationInfoAction) + if objectAPI == nil { + return + } + + info, err := globalSiteReplicationSys.GetClusterInfo(ctx) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + if err = json.NewEncoder(w).Encode(info); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } +} + +func (a adminAPIHandlers) SRPeerGetIDPSettings(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction) + if objectAPI == nil { + return + } + + idpSettings := globalSiteReplicationSys.GetIDPSettings(ctx) + if err := json.NewEncoder(w).Encode(idpSettings); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } +} + +func parseJSONBody(ctx context.Context, body io.Reader, v any, encryptionKey string) error { + data, err := io.ReadAll(body) + if err != nil { + return SRError{ + Cause: err, + Code: ErrSiteReplicationInvalidRequest, + } + } + if encryptionKey != "" { + data, err = madmin.DecryptData(encryptionKey, bytes.NewReader(data)) + if err != nil { + return SRError{ + Cause: err, + Code: ErrSiteReplicationInvalidRequest, + } + } + } + return json.Unmarshal(data, v) +} + +// SiteReplicationStatus - GET /minio/admin/v3/site-replication/status +func (a adminAPIHandlers) SiteReplicationStatus(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationInfoAction) + if objectAPI == nil { + return + } + opts := getSRStatusOptions(r) + // default options to all if status options are unset for backward compatibility + var dfltOpts madmin.SRStatusOptions + if opts == dfltOpts { + opts.Buckets = true + opts.Users = true + opts.Policies = true + opts.Groups = true + opts.ILMExpiryRules = true + } + info, err := globalSiteReplicationSys.SiteReplicationStatus(ctx, objectAPI, opts) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + // Report the ILMExpiryStats only if at least one site has replication of ILM expiry enabled + var replicateILMExpiry bool + for _, site := range info.Sites { + if site.ReplicateILMExpiry { + replicateILMExpiry = true + break + } + } + if !replicateILMExpiry { + // explicitly send nil for ILMExpiryStats + info.ILMExpiryStats = nil + } + + if err = json.NewEncoder(w).Encode(info); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } +} + +// SiteReplicationMetaInfo - GET /minio/admin/v3/site-replication/metainfo +func (a adminAPIHandlers) SiteReplicationMetaInfo(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationInfoAction) + if objectAPI == nil { + return + } + + opts := getSRStatusOptions(r) + info, err := globalSiteReplicationSys.SiteReplicationMetaInfo(ctx, objectAPI, opts) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + if err = json.NewEncoder(w).Encode(info); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } +} + +// SiteReplicationEdit - PUT /minio/admin/v3/site-replication/edit +func (a adminAPIHandlers) SiteReplicationEdit(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, cred := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction) + if objectAPI == nil { + return + } + var site madmin.PeerInfo + err := parseJSONBody(ctx, r.Body, &site, cred.SecretKey) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + opts := getSREditOptions(r) + status, err := globalSiteReplicationSys.EditPeerCluster(ctx, site, opts) + if err != nil { + adminLogIf(ctx, err) + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + body, err := json.Marshal(status) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, body) +} + +func getSREditOptions(r *http.Request) (opts madmin.SREditOptions) { + opts.DisableILMExpiryReplication = r.Form.Get("disableILMExpiryReplication") == "true" + opts.EnableILMExpiryReplication = r.Form.Get("enableILMExpiryReplication") == "true" + return opts +} + +// SRPeerEdit - PUT /minio/admin/v3/site-replication/peer/edit +// +// used internally to tell current cluster to update endpoint for peer +func (a adminAPIHandlers) SRPeerEdit(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationAddAction) + if objectAPI == nil { + return + } + + var pi madmin.PeerInfo + if err := parseJSONBody(ctx, r.Body, &pi, ""); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + if err := globalSiteReplicationSys.PeerEditReq(ctx, pi); err != nil { + adminLogIf(ctx, err) + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } +} + +// SRStateEdit - PUT /minio/admin/v3/site-replication/state/edit +// +// used internally to tell current cluster to update site replication state +func (a adminAPIHandlers) SRStateEdit(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationOperationAction) + if objectAPI == nil { + return + } + + var state madmin.SRStateEditReq + if err := parseJSONBody(ctx, r.Body, &state, ""); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + if err := globalSiteReplicationSys.PeerStateEditReq(ctx, state); err != nil { + adminLogIf(ctx, err) + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } +} + +func getSRStatusOptions(r *http.Request) (opts madmin.SRStatusOptions) { + q := r.Form + opts.Buckets = q.Get("buckets") == "true" + opts.Policies = q.Get("policies") == "true" + opts.Groups = q.Get("groups") == "true" + opts.Users = q.Get("users") == "true" + opts.ILMExpiryRules = q.Get("ilm-expiry-rules") == "true" + opts.PeerState = q.Get("peer-state") == "true" + opts.Entity = madmin.GetSREntityType(q.Get("entity")) + opts.EntityValue = q.Get("entityvalue") + opts.ShowDeleted = q.Get("showDeleted") == "true" + opts.Metrics = q.Get("metrics") == "true" + return opts +} + +// SiteReplicationRemove - PUT /minio/admin/v3/site-replication/remove +func (a adminAPIHandlers) SiteReplicationRemove(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationRemoveAction) + if objectAPI == nil { + return + } + var rreq madmin.SRRemoveReq + err := parseJSONBody(ctx, r.Body, &rreq, "") + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + status, err := globalSiteReplicationSys.RemovePeerCluster(ctx, objectAPI, rreq) + if err != nil { + adminLogIf(ctx, err) + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + body, err := json.Marshal(status) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + writeSuccessResponseJSON(w, body) +} + +// SRPeerRemove - PUT /minio/admin/v3/site-replication/peer/remove +// +// used internally to tell current cluster to update endpoint for peer +func (a adminAPIHandlers) SRPeerRemove(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationRemoveAction) + if objectAPI == nil { + return + } + + var req madmin.SRRemoveReq + if err := parseJSONBody(ctx, r.Body, &req, ""); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + if err := globalSiteReplicationSys.InternalRemoveReq(ctx, objectAPI, req); err != nil { + adminLogIf(ctx, err) + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } +} + +// SiteReplicationResyncOp - PUT /minio/admin/v3/site-replication/resync/op +func (a adminAPIHandlers) SiteReplicationResyncOp(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.SiteReplicationResyncAction) + if objectAPI == nil { + return + } + + var peerSite madmin.PeerInfo + if err := parseJSONBody(ctx, r.Body, &peerSite, ""); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + vars := mux.Vars(r) + op := madmin.SiteResyncOp(vars["operation"]) + var ( + status madmin.SRResyncOpStatus + err error + ) + switch op { + case madmin.SiteResyncStart: + status, err = globalSiteReplicationSys.startResync(ctx, objectAPI, peerSite) + case madmin.SiteResyncCancel: + status, err = globalSiteReplicationSys.cancelResync(ctx, objectAPI, peerSite) + default: + err = errSRInvalidRequest(errInvalidArgument) + } + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + body, err := json.Marshal(status) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + writeSuccessResponseJSON(w, body) +} + +// SiteReplicationDevNull - everything goes to io.Discard +// [POST] /minio/admin/v3/site-replication/devnull +func (a adminAPIHandlers) SiteReplicationDevNull(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + globalSiteNetPerfRX.Connect() + defer globalSiteNetPerfRX.Disconnect() + + connectTime := time.Now() + for { + n, err := io.CopyN(xioutil.Discard, r.Body, 128*humanize.KiByte) + atomic.AddUint64(&globalSiteNetPerfRX.RX, uint64(n)) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + // If there is a disconnection before globalNetPerfMinDuration (we give a margin of error of 1 sec) + // would mean the network is not stable. Logging here will help in debugging network issues. + if time.Since(connectTime) < (globalNetPerfMinDuration - time.Second) { + adminLogIf(ctx, err) + } + } + if err != nil { + if errors.Is(err, io.EOF) { + w.WriteHeader(http.StatusNoContent) + } else { + w.WriteHeader(http.StatusBadRequest) + } + break + } + } +} + +// SiteReplicationNetPerf - everything goes to io.Discard +// [POST] /minio/admin/v3/site-replication/netperf +func (a adminAPIHandlers) SiteReplicationNetPerf(w http.ResponseWriter, r *http.Request) { + durationStr := r.Form.Get(peerRESTDuration) + duration, _ := time.ParseDuration(durationStr) + if duration < globalNetPerfMinDuration { + duration = globalNetPerfMinDuration + } + result := siteNetperf(r.Context(), duration) + adminLogIf(r.Context(), gob.NewEncoder(w).Encode(result)) +} diff --git a/cmd/admin-handlers-users-race_test.go b/cmd/admin-handlers-users-race_test.go new file mode 100644 index 00000000..79474a01 --- /dev/null +++ b/cmd/admin-handlers-users-race_test.go @@ -0,0 +1,152 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//go:build !race +// +build !race + +// Tests in this file are not run under the `-race` flag as they are too slow +// and cause context deadline errors. + +package cmd + +import ( + "context" + "fmt" + "runtime" + "testing" + "time" + + "github.com/minio/madmin-go/v3" + minio "github.com/minio/minio-go/v7" + "github.com/minio/pkg/v3/sync/errgroup" +) + +func runAllIAMConcurrencyTests(suite *TestSuiteIAM, c *check) { + suite.SetUpSuite(c) + suite.TestDeleteUserRace(c) + suite.TearDownSuite(c) +} + +func TestIAMInternalIDPConcurrencyServerSuite(t *testing.T) { + if runtime.GOOS == globalWindowsOSName { + t.Skip("windows is clunky") + } + + baseTestCases := []TestSuiteCommon{ + // Init and run test on ErasureSD backend with signature v4. + {serverType: "ErasureSD", signer: signerV4}, + // Init and run test on ErasureSD backend, with tls enabled. + {serverType: "ErasureSD", signer: signerV4, secure: true}, + // Init and run test on Erasure backend. + {serverType: "Erasure", signer: signerV4}, + // Init and run test on ErasureSet backend. + {serverType: "ErasureSet", signer: signerV4}, + } + testCases := []*TestSuiteIAM{} + for _, bt := range baseTestCases { + testCases = append(testCases, + newTestSuiteIAM(bt, false), + newTestSuiteIAM(bt, true), + ) + } + for i, testCase := range testCases { + etcdStr := "" + if testCase.withEtcdBackend { + etcdStr = " (with etcd backend)" + } + t.Run( + fmt.Sprintf("Test: %d, ServerType: %s%s", i+1, testCase.serverType, etcdStr), + func(t *testing.T) { + runAllIAMConcurrencyTests(testCase, &check{t, testCase.serverType}) + }, + ) + } +} + +func (s *TestSuiteIAM) TestDeleteUserRace(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket creat error: %v", err) + } + + // Create a policy policy + policy := "mypolicy" + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + userCount := 50 + accessKeys := make([]string, userCount) + secretKeys := make([]string, userCount) + for i := range userCount { + accessKey, secretKey := mustGenerateCredentials(c) + err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled) + if err != nil { + c.Fatalf("Unable to set user: %v", err) + } + + userReq := madmin.PolicyAssociationReq{ + Policies: []string{policy}, + User: accessKey, + } + if _, err := s.adm.AttachPolicy(ctx, userReq); err != nil { + c.Fatalf("Unable to attach policy: %v", err) + } + + accessKeys[i] = accessKey + secretKeys[i] = secretKey + } + + g := errgroup.Group{} + for i := range userCount { + g.Go(func(i int) func() error { + return func() error { + uClient := s.getUserClient(c, accessKeys[i], secretKeys[i], "") + err := s.adm.RemoveUser(ctx, accessKeys[i]) + if err != nil { + return err + } + c.mustNotListObjects(ctx, uClient, bucket) + return nil + } + }(i), i) + } + if errs := g.Wait(); len(errs) > 0 { + c.Fatalf("unable to remove users: %v", errs) + } +} diff --git a/cmd/admin-handlers-users.go b/cmd/admin-handlers-users.go new file mode 100644 index 00000000..8530046b --- /dev/null +++ b/cmd/admin-handlers-users.go @@ -0,0 +1,3000 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "maps" + "net/http" + "os" + "slices" + "sort" + "strconv" + "strings" + "time" + "unicode/utf8" + + "github.com/klauspost/compress/zip" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/config/dns" + "github.com/minio/minio/internal/logger" + "github.com/minio/mux" + xldap "github.com/minio/pkg/v3/ldap" + "github.com/minio/pkg/v3/policy" + "github.com/puzpuzpuz/xsync/v3" +) + +// RemoveUser - DELETE /minio/admin/v3/remove-user?accessKey= +func (a adminAPIHandlers) RemoveUser(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, cred := validateAdminReq(ctx, w, r, policy.DeleteUserAdminAction) + if objectAPI == nil { + return + } + + vars := mux.Vars(r) + accessKey := vars["accessKey"] + + ok, _, err := globalIAMSys.IsTempUser(accessKey) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + if ok { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL) + return + } + + // This API only supports removal of internal users not service accounts. + ok, _, err = globalIAMSys.IsServiceAccount(accessKey) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + if ok { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL) + return + } + + // When the user is root credential you are not allowed to + // remove the root user. Also you cannot delete yourself. + if accessKey == globalActiveCred.AccessKey || accessKey == cred.AccessKey { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL) + return + } + + if err := globalIAMSys.DeleteUser(ctx, accessKey, true); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemIAMUser, + IAMUser: &madmin.SRIAMUser{ + AccessKey: accessKey, + IsDeleteReq: true, + }, + UpdatedAt: UTCNow(), + })) +} + +// ListBucketUsers - GET /minio/admin/v3/list-users?bucket={bucket} +func (a adminAPIHandlers) ListBucketUsers(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, cred := validateAdminReq(ctx, w, r, policy.ListUsersAdminAction) + if objectAPI == nil { + return + } + + bucket := mux.Vars(r)["bucket"] + + password := cred.SecretKey + + allCredentials, err := globalIAMSys.ListBucketUsers(ctx, bucket) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + data, err := json.Marshal(allCredentials) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + econfigData, err := madmin.EncryptData(password, data) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, econfigData) +} + +// ListUsers - GET /minio/admin/v3/list-users +func (a adminAPIHandlers) ListUsers(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, cred := validateAdminReq(ctx, w, r, policy.ListUsersAdminAction) + if objectAPI == nil { + return + } + + password := cred.SecretKey + + allCredentials, err := globalIAMSys.ListUsers(ctx) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Add ldap users which have mapped policies if in LDAP mode + // FIXME(vadmeste): move this to policy info in the future + ldapUsers, err := globalIAMSys.ListLDAPUsers(ctx) + if err != nil && err != errIAMActionNotAllowed { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + maps.Copy(allCredentials, ldapUsers) + + // Marshal the response + data, err := json.Marshal(allCredentials) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + econfigData, err := madmin.EncryptData(password, data) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, econfigData) +} + +// GetUserInfo - GET /minio/admin/v3/user-info +func (a adminAPIHandlers) GetUserInfo(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + vars := mux.Vars(r) + name := vars["accessKey"] + + // Get current object layer instance. + objectAPI := newObjectLayerFn() + if objectAPI == nil || globalNotificationSys == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + cred, owner, s3Err := validateAdminSignature(ctx, r, "") + if s3Err != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + + checkDenyOnly := name == cred.AccessKey + + if !globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.GetUserAdminAction, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + Claims: cred.Claims, + DenyOnly: checkDenyOnly, + }) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + + userInfo, err := globalIAMSys.GetUserInfo(ctx, name) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + data, err := json.Marshal(userInfo) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, data) +} + +// UpdateGroupMembers - PUT /minio/admin/v3/update-group-members +func (a adminAPIHandlers) UpdateGroupMembers(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.AddUserToGroupAdminAction) + if objectAPI == nil { + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + + var updReq madmin.GroupAddRemove + err = json.Unmarshal(data, &updReq) + if err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + + // Reject if the group add and remove are temporary credentials, or root credential. + for _, member := range updReq.Members { + ok, _, err := globalIAMSys.IsTempUser(member) + if err != nil && err != errNoSuchUser { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + if ok { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL) + return + } + // When the user is root credential you are not allowed to + // add policies for root user. + if member == globalActiveCred.AccessKey { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL) + return + } + } + + var updatedAt time.Time + if updReq.IsRemove { + updatedAt, err = globalIAMSys.RemoveUsersFromGroup(ctx, updReq.Group, updReq.Members) + } else { + // Check if group already exists + if _, gerr := globalIAMSys.GetGroupDescription(updReq.Group); gerr != nil { + // If group does not exist, then check if the group has beginning and end space characters + // we will reject such group names. + if errors.Is(gerr, errNoSuchGroup) && hasSpaceBE(updReq.Group) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL) + return + } + } + + if globalIAMSys.LDAPConfig.Enabled() { + // We don't allow internal group manipulation in this API when LDAP + // is enabled for now. + err = errIAMActionNotAllowed + } else { + updatedAt, err = globalIAMSys.AddUsersToGroup(ctx, updReq.Group, updReq.Members) + } + } + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemGroupInfo, + GroupInfo: &madmin.SRGroupInfo{ + UpdateReq: updReq, + }, + UpdatedAt: updatedAt, + })) +} + +// GetGroup - /minio/admin/v3/group?group=mygroup1 +func (a adminAPIHandlers) GetGroup(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.GetGroupAdminAction) + if objectAPI == nil { + return + } + + vars := mux.Vars(r) + group := vars["group"] + + gdesc, err := globalIAMSys.GetGroupDescription(group) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + body, err := json.Marshal(gdesc) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, body) +} + +// ListGroups - GET /minio/admin/v3/groups +func (a adminAPIHandlers) ListGroups(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.ListGroupsAdminAction) + if objectAPI == nil { + return + } + + groups, err := globalIAMSys.ListGroups(ctx) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + body, err := json.Marshal(groups) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, body) +} + +// SetGroupStatus - PUT /minio/admin/v3/set-group-status?group=mygroup1&status=enabled +func (a adminAPIHandlers) SetGroupStatus(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.EnableGroupAdminAction) + if objectAPI == nil { + return + } + + vars := mux.Vars(r) + group := vars["group"] + status := vars["status"] + + var ( + err error + updatedAt time.Time + ) + switch status { + case statusEnabled: + updatedAt, err = globalIAMSys.SetGroupStatus(ctx, group, true) + case statusDisabled: + updatedAt, err = globalIAMSys.SetGroupStatus(ctx, group, false) + default: + err = errInvalidArgument + } + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemGroupInfo, + GroupInfo: &madmin.SRGroupInfo{ + UpdateReq: madmin.GroupAddRemove{ + Group: group, + Status: madmin.GroupStatus(status), + IsRemove: false, + }, + }, + UpdatedAt: updatedAt, + })) +} + +// SetUserStatus - PUT /minio/admin/v3/set-user-status?accessKey=&status=[enabled|disabled] +func (a adminAPIHandlers) SetUserStatus(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, creds := validateAdminReq(ctx, w, r, policy.EnableUserAdminAction) + if objectAPI == nil { + return + } + + vars := mux.Vars(r) + accessKey := vars["accessKey"] + status := vars["status"] + + // you cannot enable or disable yourself. + if accessKey == creds.AccessKey { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errInvalidArgument), r.URL) + return + } + + updatedAt, err := globalIAMSys.SetUserStatus(ctx, accessKey, madmin.AccountStatus(status)) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemIAMUser, + IAMUser: &madmin.SRIAMUser{ + AccessKey: accessKey, + IsDeleteReq: false, + UserReq: &madmin.AddOrUpdateUserReq{ + Status: madmin.AccountStatus(status), + }, + }, + UpdatedAt: updatedAt, + })) +} + +// AddUser - PUT /minio/admin/v3/add-user?accessKey= +func (a adminAPIHandlers) AddUser(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + vars := mux.Vars(r) + accessKey := vars["accessKey"] + + // Get current object layer instance. + objectAPI := newObjectLayerFn() + if objectAPI == nil || globalNotificationSys == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + cred, owner, s3Err := validateAdminSignature(ctx, r, "") + if s3Err != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + + // Not allowed to add a user with same access key as root credential + if accessKey == globalActiveCred.AccessKey { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL) + return + } + + user, exists := globalIAMSys.GetUser(ctx, accessKey) + if exists && (user.Credentials.IsTemp() || user.Credentials.IsServiceAccount()) { + // Updating STS credential is not allowed, and this API does not + // support updating service accounts. + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL) + return + } + + if (cred.IsTemp() || cred.IsServiceAccount()) && cred.ParentUser == accessKey { + // Incoming access key matches parent user then we should + // reject password change requests. + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL) + return + } + + // Check if accessKey has beginning and end space characters, this only applies to new users. + if !exists && hasSpaceBE(accessKey) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL) + return + } + + if !utf8.ValidString(accessKey) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserValidUTF), r.URL) + return + } + + checkDenyOnly := accessKey == cred.AccessKey + + if !globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.CreateUserAdminAction, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + Claims: cred.Claims, + DenyOnly: checkDenyOnly, + }) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + + if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 { + // More than maxConfigSize bytes were available + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL) + return + } + + password := cred.SecretKey + configBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength)) + if err != nil { + adminLogIf(ctx, err) + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL) + return + } + + var ureq madmin.AddOrUpdateUserReq + if err = json.Unmarshal(configBytes, &ureq); err != nil { + adminLogIf(ctx, err) + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigBadJSON), r.URL) + return + } + + // We don't allow internal user creation with LDAP enabled for now. + if globalIAMSys.LDAPConfig.Enabled() { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL) + return + } + + updatedAt, err := globalIAMSys.CreateUser(ctx, accessKey, ureq) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemIAMUser, + IAMUser: &madmin.SRIAMUser{ + AccessKey: accessKey, + IsDeleteReq: false, + UserReq: &ureq, + }, + UpdatedAt: updatedAt, + })) +} + +// TemporaryAccountInfo - GET /minio/admin/v3/temporary-account-info +func (a adminAPIHandlers) TemporaryAccountInfo(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Get current object layer instance. + objectAPI := newObjectLayerFn() + if objectAPI == nil || globalNotificationSys == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + cred, owner, s3Err := validateAdminSignature(ctx, r, "") + if s3Err != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + + accessKey := mux.Vars(r)["accessKey"] + if accessKey == "" { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + + args := policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.ListTemporaryAccountsAdminAction, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + Claims: cred.Claims, + } + + if !globalIAMSys.IsAllowed(args) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + + stsAccount, sessionPolicy, err := globalIAMSys.GetTemporaryAccount(ctx, accessKey) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + var stsAccountPolicy policy.Policy + + if sessionPolicy != nil { + stsAccountPolicy = *sessionPolicy + } else { + policiesNames, err := globalIAMSys.PolicyDBGet(stsAccount.ParentUser, stsAccount.Groups...) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + if len(policiesNames) == 0 { + policySet, _ := args.GetPolicies(iamPolicyClaimNameOpenID()) + policiesNames = policySet.ToSlice() + } + + stsAccountPolicy = globalIAMSys.GetCombinedPolicy(policiesNames...) + } + + policyJSON, err := json.MarshalIndent(stsAccountPolicy, "", " ") + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + infoResp := madmin.TemporaryAccountInfoResp{ + ParentUser: stsAccount.ParentUser, + AccountStatus: stsAccount.Status, + ImpliedPolicy: sessionPolicy == nil, + Policy: string(policyJSON), + Expiration: &stsAccount.Expiration, + } + + data, err := json.Marshal(infoResp) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + encryptedData, err := madmin.EncryptData(cred.SecretKey, data) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, encryptedData) +} + +// AddServiceAccount - PUT /minio/admin/v3/add-service-account +func (a adminAPIHandlers) AddServiceAccount(w http.ResponseWriter, r *http.Request) { + ctx, cred, opts, createReq, targetUser, APIError := commonAddServiceAccount(r, false) + if APIError.Code != "" { + writeErrorResponseJSON(ctx, w, APIError, r.URL) + return + } + + if createReq.AccessKey == globalActiveCred.AccessKey { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAddUserInvalidArgument), r.URL) + return + } + + var ( + targetGroups []string + err error + ) + + // Find the user for the request sender (as it may be sent via a service + // account or STS account): + requestorUser := cred.AccessKey + requestorParentUser := cred.AccessKey + requestorGroups := cred.Groups + requestorIsDerivedCredential := false + if cred.IsServiceAccount() || cred.IsTemp() { + requestorParentUser = cred.ParentUser + requestorIsDerivedCredential = true + } + + if globalIAMSys.GetUsersSysType() == MinIOUsersSysType && targetUser != cred.AccessKey { + // For internal IDP, ensure that the targetUser's parent account exists. + // It could be a regular user account or the root account. + _, isRegularUser := globalIAMSys.GetUser(ctx, targetUser) + if !isRegularUser && targetUser != globalActiveCred.AccessKey { + apiErr := toAdminAPIErr(ctx, errNoSuchUser) + apiErr.Description = fmt.Sprintf("Specified target user %s does not exist", targetUser) + writeErrorResponseJSON(ctx, w, apiErr, r.URL) + return + } + } + + // Check if we are creating svc account for request sender. + isSvcAccForRequestor := targetUser == requestorUser || targetUser == requestorParentUser + + // If we are creating svc account for request sender, ensure + // that targetUser is a real user (i.e. not derived + // credentials). + if isSvcAccForRequestor { + if requestorIsDerivedCredential { + if requestorParentUser == "" { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, + errors.New("service accounts cannot be generated for temporary credentials without parent")), r.URL) + return + } + targetUser = requestorParentUser + } + targetGroups = requestorGroups + + // In case of LDAP/OIDC we need to set `opts.claims` to ensure + // it is associated with the LDAP/OIDC user properly. + for k, v := range cred.Claims { + if k == expClaim { + continue + } + opts.claims[k] = v + } + } else if globalIAMSys.LDAPConfig.Enabled() { + // In case of LDAP we need to resolve the targetUser to a DN and + // query their groups: + opts.claims[ldapUserN] = targetUser // simple username + var lookupResult *xldap.DNSearchResult + lookupResult, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(targetUser) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + targetUser = lookupResult.NormDN + opts.claims[ldapUser] = targetUser // username DN + opts.claims[ldapActualUser] = lookupResult.ActualDN + + // Add LDAP attributes that were looked up into the claims. + for attribKey, attribValue := range lookupResult.Attributes { + opts.claims[ldapAttribPrefix+attribKey] = attribValue + } + + // NOTE: if not using LDAP, then internal IDP or open ID is + // being used - in the former, group info is enforced when + // generated credentials are used to make requests, and in the + // latter, a group notion is not supported. + } + + newCred, updatedAt, err := globalIAMSys.NewServiceAccount(ctx, targetUser, targetGroups, opts) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + createResp := madmin.AddServiceAccountResp{ + Credentials: madmin.Credentials{ + AccessKey: newCred.AccessKey, + SecretKey: newCred.SecretKey, + Expiration: newCred.Expiration, + }, + } + + data, err := json.Marshal(createResp) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + encryptedData, err := madmin.EncryptData(cred.SecretKey, data) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, encryptedData) + + // Call hook for cluster-replication if the service account is not for a + // root user. + if newCred.ParentUser != globalActiveCred.AccessKey { + replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemSvcAcc, + SvcAccChange: &madmin.SRSvcAccChange{ + Create: &madmin.SRSvcAccCreate{ + Parent: newCred.ParentUser, + AccessKey: newCred.AccessKey, + SecretKey: newCred.SecretKey, + Groups: newCred.Groups, + Name: newCred.Name, + Description: newCred.Description, + Claims: opts.claims, + SessionPolicy: madmin.SRSessionPolicy(createReq.Policy), + Status: auth.AccountOn, + Expiration: createReq.Expiration, + }, + }, + UpdatedAt: updatedAt, + })) + } +} + +// UpdateServiceAccount - POST /minio/admin/v3/update-service-account +func (a adminAPIHandlers) UpdateServiceAccount(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Get current object layer instance. + objectAPI := newObjectLayerFn() + if objectAPI == nil || globalNotificationSys == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + cred, owner, s3Err := validateAdminSignature(ctx, r, "") + if s3Err != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + + accessKey := mux.Vars(r)["accessKey"] + if accessKey == "" { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + + svcAccount, _, err := globalIAMSys.GetServiceAccount(ctx, accessKey) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + password := cred.SecretKey + reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength)) + if err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL) + return + } + + var updateReq madmin.UpdateServiceAccountReq + if err = json.Unmarshal(reqBytes, &updateReq); err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL) + return + } + + if err := updateReq.Validate(); err != nil { + // Since this validation would happen client side as well, we only send + // a generic error message here. + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL) + return + } + + condValues := getConditionValues(r, "", cred) + err = addExpirationToCondValues(updateReq.NewExpiration, condValues) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Permission checks: + // + // 1. Any type of account (i.e. access keys (previously/still called service + // accounts), STS accounts, internal IDP accounts, etc) with the + // policy.UpdateServiceAccountAdminAction permission can update any service + // account. + // + // 2. We would like to let a user update their own access keys, however it + // is currently blocked pending a re-design. Users are still able to delete + // and re-create them. + if !globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.UpdateServiceAccountAdminAction, + ConditionValues: condValues, + IsOwner: owner, + Claims: cred.Claims, + }) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + + var sp *policy.Policy + if len(updateReq.NewPolicy) > 0 { + sp, err = policy.ParseConfig(bytes.NewReader(updateReq.NewPolicy)) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + if sp.Version == "" && len(sp.Statements) == 0 { + sp = nil + } + } + opts := updateServiceAccountOpts{ + secretKey: updateReq.NewSecretKey, + status: updateReq.NewStatus, + name: updateReq.NewName, + description: updateReq.NewDescription, + expiration: updateReq.NewExpiration, + sessionPolicy: sp, + } + updatedAt, err := globalIAMSys.UpdateServiceAccount(ctx, accessKey, opts) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Call site replication hook - non-root user accounts are replicated. + if svcAccount.ParentUser != globalActiveCred.AccessKey { + replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemSvcAcc, + SvcAccChange: &madmin.SRSvcAccChange{ + Update: &madmin.SRSvcAccUpdate{ + AccessKey: accessKey, + SecretKey: opts.secretKey, + Status: opts.status, + Name: opts.name, + Description: opts.description, + SessionPolicy: madmin.SRSessionPolicy(updateReq.NewPolicy), + Expiration: updateReq.NewExpiration, + }, + }, + UpdatedAt: updatedAt, + })) + } + + writeSuccessNoContent(w) +} + +// InfoServiceAccount - GET /minio/admin/v3/info-service-account +func (a adminAPIHandlers) InfoServiceAccount(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Get current object layer instance. + objectAPI := newObjectLayerFn() + if objectAPI == nil || globalNotificationSys == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + cred, owner, s3Err := validateAdminSignature(ctx, r, "") + if s3Err != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + + accessKey := mux.Vars(r)["accessKey"] + if accessKey == "" { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + + svcAccount, sessionPolicy, err := globalIAMSys.GetServiceAccount(ctx, accessKey) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + if !globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.ListServiceAccountsAdminAction, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + Claims: cred.Claims, + }) { + requestUser := cred.AccessKey + if cred.ParentUser != "" { + requestUser = cred.ParentUser + } + + if requestUser != svcAccount.ParentUser { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + } + + // if session policy is nil or empty, then it is implied policy + impliedPolicy := sessionPolicy == nil || (sessionPolicy.Version == "" && len(sessionPolicy.Statements) == 0) + + var svcAccountPolicy policy.Policy + + if !impliedPolicy { + svcAccountPolicy = *sessionPolicy + } else { + policiesNames, err := globalIAMSys.PolicyDBGet(svcAccount.ParentUser, svcAccount.Groups...) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + svcAccountPolicy = globalIAMSys.GetCombinedPolicy(policiesNames...) + } + + policyJSON, err := json.MarshalIndent(svcAccountPolicy, "", " ") + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + var expiration *time.Time + if !svcAccount.Expiration.IsZero() && !svcAccount.Expiration.Equal(timeSentinel) { + expiration = &svcAccount.Expiration + } + + infoResp := madmin.InfoServiceAccountResp{ + ParentUser: svcAccount.ParentUser, + Name: svcAccount.Name, + Description: svcAccount.Description, + AccountStatus: svcAccount.Status, + ImpliedPolicy: impliedPolicy, + Policy: string(policyJSON), + Expiration: expiration, + } + + data, err := json.Marshal(infoResp) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + encryptedData, err := madmin.EncryptData(cred.SecretKey, data) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, encryptedData) +} + +// ListServiceAccounts - GET /minio/admin/v3/list-service-accounts +func (a adminAPIHandlers) ListServiceAccounts(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Get current object layer instance. + objectAPI := newObjectLayerFn() + if objectAPI == nil || globalNotificationSys == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + cred, owner, s3Err := validateAdminSignature(ctx, r, "") + if s3Err != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + + var targetAccount string + + // If listing is requested for a specific user (who is not the request + // sender), check that the user has permissions. + user := r.Form.Get("user") + if user != "" && user != cred.AccessKey { + if !globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.ListServiceAccountsAdminAction, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + Claims: cred.Claims, + }) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + targetAccount = user + } else { + targetAccount = cred.AccessKey + if cred.ParentUser != "" { + targetAccount = cred.ParentUser + } + } + + serviceAccounts, err := globalIAMSys.ListServiceAccounts(ctx, targetAccount) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + var serviceAccountList []madmin.ServiceAccountInfo + + for _, svc := range serviceAccounts { + expiryTime := svc.Expiration + serviceAccountList = append(serviceAccountList, madmin.ServiceAccountInfo{ + Description: svc.Description, + ParentUser: svc.ParentUser, + Name: svc.Name, + AccountStatus: svc.Status, + AccessKey: svc.AccessKey, + ImpliedPolicy: svc.IsImpliedPolicy(), + Expiration: &expiryTime, + }) + } + + listResp := madmin.ListServiceAccountsResp{ + Accounts: serviceAccountList, + } + + data, err := json.Marshal(listResp) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + encryptedData, err := madmin.EncryptData(cred.SecretKey, data) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, encryptedData) +} + +// DeleteServiceAccount - DELETE /minio/admin/v3/delete-service-account +func (a adminAPIHandlers) DeleteServiceAccount(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Get current object layer instance. + objectAPI := newObjectLayerFn() + if objectAPI == nil || globalNotificationSys == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + cred, owner, s3Err := validateAdminSignature(ctx, r, "") + if s3Err != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + + serviceAccount := mux.Vars(r)["accessKey"] + if serviceAccount == "" { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL) + return + } + + if serviceAccount == siteReplicatorSvcAcc && globalSiteReplicationSys.isEnabled() { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidArgument), r.URL) + return + } + // We do not care if service account is readable or not at this point, + // since this is a delete call we shall allow it to be deleted if possible. + svcAccount, _, err := globalIAMSys.GetServiceAccount(ctx, serviceAccount) + if errors.Is(err, errNoSuchServiceAccount) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound), r.URL) + return + } + + adminPrivilege := globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.RemoveServiceAccountAdminAction, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + Claims: cred.Claims, + }) + + if !adminPrivilege { + parentUser := cred.AccessKey + if cred.ParentUser != "" { + parentUser = cred.ParentUser + } + if svcAccount.ParentUser != "" && parentUser != svcAccount.ParentUser { + // The service account belongs to another user but return not + // found error to mitigate brute force attacks. or the + // serviceAccount doesn't exist. + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminServiceAccountNotFound), r.URL) + return + } + } + + if err := globalIAMSys.DeleteServiceAccount(ctx, serviceAccount, true); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Call site replication hook - non-root user accounts are replicated. + if svcAccount.ParentUser != "" && svcAccount.ParentUser != globalActiveCred.AccessKey { + replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemSvcAcc, + SvcAccChange: &madmin.SRSvcAccChange{ + Delete: &madmin.SRSvcAccDelete{ + AccessKey: serviceAccount, + }, + }, + UpdatedAt: UTCNow(), + })) + } + + writeSuccessNoContent(w) +} + +// ListAccessKeysBulk - GET /minio/admin/v3/list-access-keys-bulk +func (a adminAPIHandlers) ListAccessKeysBulk(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Get current object layer instance. + objectAPI := newObjectLayerFn() + if objectAPI == nil || globalNotificationSys == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + cred, owner, s3Err := validateAdminSignature(ctx, r, "") + if s3Err != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + + users := r.Form["users"] + isAll := r.Form.Get("all") == "true" + selfOnly := !isAll && len(users) == 0 + + if isAll && len(users) > 0 { + // This should be checked on client side, so return generic error + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + + // Empty user list and not self, list access keys for all users + if isAll { + if !globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.ListUsersAdminAction, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + Claims: cred.Claims, + }) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + } else if len(users) == 1 { + if users[0] == cred.AccessKey || users[0] == cred.ParentUser { + selfOnly = true + } + } + + if !globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.ListServiceAccountsAdminAction, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + Claims: cred.Claims, + DenyOnly: selfOnly, + }) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + + if selfOnly && len(users) == 0 { + selfUser := cred.AccessKey + if cred.ParentUser != "" { + selfUser = cred.ParentUser + } + users = append(users, selfUser) + } + + var checkedUserList []string + if isAll { + users, err := globalIAMSys.ListUsers(ctx) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + for user := range users { + checkedUserList = append(checkedUserList, user) + } + checkedUserList = append(checkedUserList, globalActiveCred.AccessKey) + } else { + for _, user := range users { + // Validate the user + _, ok := globalIAMSys.GetUser(ctx, user) + if !ok { + continue + } + checkedUserList = append(checkedUserList, user) + } + } + + listType := r.Form.Get("listType") + var listSTSKeys, listServiceAccounts bool + switch listType { + case madmin.AccessKeyListUsersOnly: + listSTSKeys = false + listServiceAccounts = false + case madmin.AccessKeyListSTSOnly: + listSTSKeys = true + listServiceAccounts = false + case madmin.AccessKeyListSvcaccOnly: + listSTSKeys = false + listServiceAccounts = true + case madmin.AccessKeyListAll: + listSTSKeys = true + listServiceAccounts = true + default: + err := errors.New("invalid list type") + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrInvalidRequest, err), r.URL) + return + } + + accessKeyMap := make(map[string]madmin.ListAccessKeysResp) + for _, user := range checkedUserList { + accessKeys := madmin.ListAccessKeysResp{} + if listSTSKeys { + stsKeys, err := globalIAMSys.ListSTSAccounts(ctx, user) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + for _, sts := range stsKeys { + accessKeys.STSKeys = append(accessKeys.STSKeys, madmin.ServiceAccountInfo{ + AccessKey: sts.AccessKey, + Expiration: &sts.Expiration, + }) + } + // if only STS keys, skip if user has no STS keys + if !listServiceAccounts && len(stsKeys) == 0 { + continue + } + } + + if listServiceAccounts { + serviceAccounts, err := globalIAMSys.ListServiceAccounts(ctx, user) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + for _, svc := range serviceAccounts { + accessKeys.ServiceAccounts = append(accessKeys.ServiceAccounts, madmin.ServiceAccountInfo{ + AccessKey: svc.AccessKey, + Expiration: &svc.Expiration, + }) + } + // if only service accounts, skip if user has no service accounts + if !listSTSKeys && len(serviceAccounts) == 0 { + continue + } + } + accessKeyMap[user] = accessKeys + } + + data, err := json.Marshal(accessKeyMap) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + encryptedData, err := madmin.EncryptData(cred.SecretKey, data) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, encryptedData) +} + +// AccountInfoHandler returns usage, permissions and other bucket metadata for incoming us +func (a adminAPIHandlers) AccountInfoHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Get current object layer instance. + objectAPI := newObjectLayerFn() + if objectAPI == nil || globalNotificationSys == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + cred, owner, s3Err := validateAdminSignature(ctx, r, "") + if s3Err != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + + // Set prefix value for "s3:prefix" policy conditionals. + r.Header.Set("prefix", "") + + // Set delimiter value for "s3:delimiter" policy conditionals. + r.Header.Set("delimiter", SlashSeparator) + + // Check if we are asked to return prefix usage + enablePrefixUsage := r.Form.Get("prefix-usage") == "true" + + isAllowedAccess := func(bucketName string) (rd, wr bool) { + if globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.ListBucketAction, + BucketName: bucketName, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + ObjectName: "", + Claims: cred.Claims, + }) { + rd = true + } + + if globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.GetBucketLocationAction, + BucketName: bucketName, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + ObjectName: "", + Claims: cred.Claims, + }) { + rd = true + } + + if globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.PutObjectAction, + BucketName: bucketName, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + ObjectName: "", + Claims: cred.Claims, + }) { + wr = true + } + + return rd, wr + } + + // If etcd, dns federation configured list buckets from etcd. + var err error + var buckets []BucketInfo + if globalDNSConfig != nil && globalBucketFederation { + dnsBuckets, err := globalDNSConfig.List() + if err != nil && !IsErrIgnored(err, + dns.ErrNoEntriesFound, + dns.ErrDomainMissing) { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + for _, dnsRecords := range dnsBuckets { + buckets = append(buckets, BucketInfo{ + Name: dnsRecords[0].Key, + Created: dnsRecords[0].CreationDate, + }) + } + sort.Slice(buckets, func(i, j int) bool { + return buckets[i].Name < buckets[j].Name + }) + } else { + buckets, err = objectAPI.ListBuckets(ctx, BucketOptions{Cached: true}) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + } + + accountName := cred.AccessKey + if cred.IsTemp() || cred.IsServiceAccount() { + // For derived credentials, check the parent user's permissions. + accountName = cred.ParentUser + } + + roleArn := policy.Args{Claims: cred.Claims}.GetRoleArn() + policySetFromClaims, hasPolicyClaim := policy.GetPoliciesFromClaims(cred.Claims, iamPolicyClaimNameOpenID()) + var effectivePolicy policy.Policy + + var buf []byte + switch { + case accountName == globalActiveCred.AccessKey || newGlobalAuthZPluginFn() != nil: + // For owner account and when plugin authZ is configured always set + // effective policy as `consoleAdmin`. + // + // In the latter case, we let the UI render everything, but individual + // actions would fail if not permitted by the external authZ service. + for _, policy := range policy.DefaultPolicies { + if policy.Name == "consoleAdmin" { + effectivePolicy = policy.Definition + break + } + } + + case roleArn != "": + _, policy, err := globalIAMSys.GetRolePolicy(roleArn) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + policySlice := newMappedPolicy(policy).toSlice() + effectivePolicy = globalIAMSys.GetCombinedPolicy(policySlice...) + + case hasPolicyClaim: + effectivePolicy = globalIAMSys.GetCombinedPolicy(policySetFromClaims.ToSlice()...) + + default: + policies, err := globalIAMSys.PolicyDBGet(accountName, cred.Groups...) + if err != nil { + adminLogIf(ctx, err) + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + effectivePolicy = globalIAMSys.GetCombinedPolicy(policies...) + } + + buf, err = json.MarshalIndent(effectivePolicy, "", " ") + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + acctInfo := madmin.AccountInfo{ + AccountName: accountName, + Server: objectAPI.BackendInfo(), + Policy: buf, + } + + for _, bucket := range buckets { + rd, wr := isAllowedAccess(bucket.Name) + if rd || wr { + // Fetch the data usage of the current bucket + bui := globalBucketQuotaSys.GetBucketUsageInfo(ctx, bucket.Name) + size := bui.Size + objectsCount := bui.ObjectsCount + objectsHist := bui.ObjectSizesHistogram + versionsHist := bui.ObjectVersionsHistogram + + // Fetch the prefix usage of the current bucket + var prefixUsage map[string]uint64 + if enablePrefixUsage { + prefixUsage, _ = loadPrefixUsageFromBackend(ctx, objectAPI, bucket.Name) + } + + lcfg, _ := globalBucketObjectLockSys.Get(bucket.Name) + quota, _ := globalBucketQuotaSys.Get(ctx, bucket.Name) + rcfg, _, _ := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket.Name) + tcfg, _, _ := globalBucketMetadataSys.GetTaggingConfig(bucket.Name) + + acctInfo.Buckets = append(acctInfo.Buckets, madmin.BucketAccessInfo{ + Name: bucket.Name, + Created: bucket.Created, + Size: size, + Objects: objectsCount, + ObjectSizesHistogram: objectsHist, + ObjectVersionsHistogram: versionsHist, + PrefixUsage: prefixUsage, + Details: &madmin.BucketDetails{ + Versioning: globalBucketVersioningSys.Enabled(bucket.Name), + VersioningSuspended: globalBucketVersioningSys.Suspended(bucket.Name), + Replication: rcfg != nil, + Locking: lcfg.LockEnabled, + Quota: quota, + Tagging: tcfg, + }, + Access: madmin.AccountAccess{ + Read: rd, + Write: wr, + }, + }) + } + } + + usageInfoJSON, err := json.Marshal(acctInfo) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, usageInfoJSON) +} + +// InfoCannedPolicy - GET /minio/admin/v3/info-canned-policy?name={policyName} +// +// Newer API response with policy timestamps is returned with query parameter +// `v=2` like: +// +// GET /minio/admin/v3/info-canned-policy?name={policyName}&v=2 +// +// The newer API will eventually become the default (and only) one. The older +// response is to return only the policy JSON. The newer response returns +// timestamps along with the policy JSON. Both versions are supported for now, +// for smooth transition to new API. +func (a adminAPIHandlers) InfoCannedPolicy(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.GetPolicyAdminAction) + if objectAPI == nil { + return + } + + name := mux.Vars(r)["name"] + policies := newMappedPolicy(name).toSlice() + if len(policies) != 1 { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errTooManyPolicies), r.URL) + return + } + setReqInfoPolicyName(ctx, name) + + policyDoc, err := globalIAMSys.InfoPolicy(name) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Is the new API version being requested? + infoPolicyAPIVersion := r.Form.Get("v") + if infoPolicyAPIVersion == "2" { + buf, err := json.MarshalIndent(policyDoc, "", " ") + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + w.Write(buf) + return + } else if infoPolicyAPIVersion != "" { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errors.New("invalid version parameter 'v' supplied")), r.URL) + return + } + + // Return the older API response value of just the policy json. + buf, err := json.MarshalIndent(policyDoc.Policy, "", " ") + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + w.Write(buf) +} + +// ListBucketPolicies - GET /minio/admin/v3/list-canned-policies?bucket={bucket} +func (a adminAPIHandlers) ListBucketPolicies(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.ListUserPoliciesAdminAction) + if objectAPI == nil { + return + } + + bucket := mux.Vars(r)["bucket"] + policies, err := globalIAMSys.ListPolicies(ctx, bucket) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + newPolicies := make(map[string]policy.Policy) + for name, p := range policies { + _, err = json.Marshal(p) + if err != nil { + adminLogIf(ctx, err) + continue + } + newPolicies[name] = p + } + if err = json.NewEncoder(w).Encode(newPolicies); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } +} + +// ListCannedPolicies - GET /minio/admin/v3/list-canned-policies +func (a adminAPIHandlers) ListCannedPolicies(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.ListUserPoliciesAdminAction) + if objectAPI == nil { + return + } + + policies, err := globalIAMSys.ListPolicies(ctx, "") + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + newPolicies := make(map[string]policy.Policy) + for name, p := range policies { + _, err = json.Marshal(p) + if err != nil { + adminLogIf(ctx, err) + continue + } + newPolicies[name] = p + } + if err = json.NewEncoder(w).Encode(newPolicies); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } +} + +// RemoveCannedPolicy - DELETE /minio/admin/v3/remove-canned-policy?name= +func (a adminAPIHandlers) RemoveCannedPolicy(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.DeletePolicyAdminAction) + if objectAPI == nil { + return + } + + vars := mux.Vars(r) + policyName := vars["name"] + setReqInfoPolicyName(ctx, policyName) + + if err := globalIAMSys.DeletePolicy(ctx, policyName, true); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Call cluster-replication policy creation hook to replicate policy deletion to + // other minio clusters. + replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemPolicy, + Name: policyName, + UpdatedAt: UTCNow(), + })) +} + +// AddCannedPolicy - PUT /minio/admin/v3/add-canned-policy?name= +func (a adminAPIHandlers) AddCannedPolicy(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.CreatePolicyAdminAction) + if objectAPI == nil { + return + } + + vars := mux.Vars(r) + policyName := vars["name"] + + // Policy has space characters in begin and end reject such inputs. + if hasSpaceBE(policyName) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL) + return + } + setReqInfoPolicyName(ctx, policyName) + + // Reject policy names with commas. + if strings.Contains(policyName, ",") { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrPolicyInvalidName), r.URL) + return + } + + // Error out if Content-Length is missing. + if r.ContentLength <= 0 { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL) + return + } + + // Error out if Content-Length is beyond allowed size. + if r.ContentLength > maxBucketPolicySize { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL) + return + } + + iamPolicyBytes, err := io.ReadAll(io.LimitReader(r.Body, r.ContentLength)) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + iamPolicy, err := policy.ParseConfig(bytes.NewReader(iamPolicyBytes)) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Version in policy must not be empty + if iamPolicy.Version == "" { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrPolicyInvalidVersion), r.URL) + return + } + + updatedAt, err := globalIAMSys.SetPolicy(ctx, policyName, *iamPolicy) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Call cluster-replication policy creation hook to replicate policy to + // other minio clusters. + replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemPolicy, + Name: policyName, + Policy: iamPolicyBytes, + UpdatedAt: updatedAt, + })) +} + +// SetPolicyForUserOrGroup - sets a policy on a user or a group. +// +// PUT /minio/admin/v3/set-policy?policy=xxx&user-or-group=?[&is-group] +// +// Deprecated: This API is replaced by attach/detach policy APIs for specific +// type of users (builtin or LDAP). +func (a adminAPIHandlers) SetPolicyForUserOrGroup(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.AttachPolicyAdminAction) + if objectAPI == nil { + return + } + + vars := mux.Vars(r) + policyName := vars["policyName"] + entityName := vars["userOrGroup"] + isGroup := vars["isGroup"] == "true" + setReqInfoPolicyName(ctx, policyName) + + if !isGroup { + ok, _, err := globalIAMSys.IsTempUser(entityName) + if err != nil && err != errNoSuchUser { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + if ok { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL) + return + } + // When the user is root credential you are not allowed to + // add policies for root user. + if entityName == globalActiveCred.AccessKey { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errIAMActionNotAllowed), r.URL) + return + } + } + + // Validate that user or group exists. + if !isGroup { + if globalIAMSys.GetUsersSysType() == MinIOUsersSysType { + _, ok := globalIAMSys.GetUser(ctx, entityName) + if !ok { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUser), r.URL) + return + } + } + } else { + _, err := globalIAMSys.GetGroupDescription(entityName) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + } + + userType := regUser + if globalIAMSys.GetUsersSysType() == LDAPUsersSysType { + userType = stsUser + + // Validate that the user or group exists in LDAP and use the normalized + // form of the entityName (which will be an LDAP DN). + var err error + if isGroup { + var foundGroupDN *xldap.DNSearchResult + var underBaseDN bool + if foundGroupDN, underBaseDN, err = globalIAMSys.LDAPConfig.GetValidatedGroupDN(nil, entityName); err != nil { + iamLogIf(ctx, err) + } else if foundGroupDN == nil || !underBaseDN { + err = errNoSuchGroup + } else { + entityName = foundGroupDN.NormDN + } + } else { + var foundUserDN *xldap.DNSearchResult + if foundUserDN, err = globalIAMSys.LDAPConfig.GetValidatedDNForUsername(entityName); err != nil { + iamLogIf(ctx, err) + } else if foundUserDN == nil { + err = errNoSuchUser + } else { + entityName = foundUserDN.NormDN + } + } + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + } + + updatedAt, err := globalIAMSys.PolicyDBSet(ctx, entityName, policyName, userType, isGroup) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemPolicyMapping, + PolicyMapping: &madmin.SRPolicyMapping{ + UserOrGroup: entityName, + UserType: int(userType), + IsGroup: isGroup, + Policy: policyName, + }, + UpdatedAt: updatedAt, + })) +} + +// ListPolicyMappingEntities - GET /minio/admin/v3/idp/builtin/policy-entities?policy=xxx&user=xxx&group=xxx +func (a adminAPIHandlers) ListPolicyMappingEntities(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Check authorization. + objectAPI, cred := validateAdminReq(ctx, w, r, + policy.ListGroupsAdminAction, policy.ListUsersAdminAction, policy.ListUserPoliciesAdminAction) + if objectAPI == nil { + return + } + + // Validate API arguments. + q := madmin.PolicyEntitiesQuery{ + Users: r.Form["user"], + Groups: r.Form["group"], + Policy: r.Form["policy"], + } + + // Query IAM + res, err := globalIAMSys.QueryPolicyEntities(r.Context(), q) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Encode result and send response. + data, err := json.Marshal(res) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + password := cred.SecretKey + econfigData, err := madmin.EncryptData(password, data) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + writeSuccessResponseJSON(w, econfigData) +} + +// AttachDetachPolicyBuiltin - POST /minio/admin/v3/idp/builtin/policy/{operation} +func (a adminAPIHandlers) AttachDetachPolicyBuiltin(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, cred := validateAdminReq(ctx, w, r, policy.UpdatePolicyAssociationAction, + policy.AttachPolicyAdminAction) + if objectAPI == nil { + return + } + + if r.ContentLength > maxEConfigJSONSize || r.ContentLength == -1 { + // More than maxConfigSize bytes were available + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminConfigTooLarge), r.URL) + return + } + + // Ensure body content type is opaque to ensure that request body has not + // been interpreted as form data. + contentType := r.Header.Get("Content-Type") + if contentType != "application/octet-stream" { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL) + return + } + + operation := mux.Vars(r)["operation"] + if operation != "attach" && operation != "detach" { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminInvalidArgument), r.URL) + return + } + isAttach := operation == "attach" + + password := cred.SecretKey + reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength)) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + var par madmin.PolicyAssociationReq + if err = json.Unmarshal(reqBytes, &par); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + if err = par.IsValid(); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + updatedAt, addedOrRemoved, _, err := globalIAMSys.PolicyDBUpdateBuiltin(ctx, isAttach, par) + if err != nil { + if err == errNoSuchUser || err == errNoSuchGroup { + if globalIAMSys.LDAPConfig.Enabled() { + // When LDAP is enabled, warn user that they are using the wrong + // API. FIXME: error can be no such group as well - fix errNoSuchUserLDAPWarn + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errNoSuchUserLDAPWarn), r.URL) + return + } + } + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + setReqInfoPolicyName(ctx, strings.Join(addedOrRemoved, ",")) + + respBody := madmin.PolicyAssociationResp{ + UpdatedAt: updatedAt, + } + if isAttach { + respBody.PoliciesAttached = addedOrRemoved + } else { + respBody.PoliciesDetached = addedOrRemoved + } + + data, err := json.Marshal(respBody) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + encryptedData, err := madmin.EncryptData(password, data) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, encryptedData) +} + +// RevokeTokens - POST /minio/admin/v3/revoke-tokens/{userProvider} +func (a adminAPIHandlers) RevokeTokens(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Get current object layer instance. + objectAPI := newObjectLayerFn() + if objectAPI == nil || globalNotificationSys == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + cred, owner, s3Err := validateAdminSignature(ctx, r, "") + if s3Err != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + + userProvider := mux.Vars(r)["userProvider"] + + user := r.Form.Get("user") + tokenRevokeType := r.Form.Get("tokenRevokeType") + fullRevoke := r.Form.Get("fullRevoke") == "true" + isTokenSelfRevoke := user == "" + if !isTokenSelfRevoke { + var err error + user, err = getUserWithProvider(ctx, userProvider, user, false) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + } + + if (user != "" && tokenRevokeType == "" && !fullRevoke) || (tokenRevokeType != "" && fullRevoke) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + + adminPrivilege := globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.RemoveServiceAccountAdminAction, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + Claims: cred.Claims, + }) + + if !adminPrivilege || isTokenSelfRevoke { + parentUser := cred.AccessKey + if cred.ParentUser != "" { + parentUser = cred.ParentUser + } + if !isTokenSelfRevoke && user != parentUser { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + user = parentUser + } + + // Infer token revoke type from the request if requestor is STS. + if isTokenSelfRevoke && tokenRevokeType == "" && !fullRevoke { + if cred.IsTemp() { + tokenRevokeType, _ = cred.Claims[tokenRevokeTypeClaim].(string) + } + if tokenRevokeType == "" { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNoTokenRevokeType), r.URL) + return + } + } + + err := globalIAMSys.RevokeTokens(ctx, user, tokenRevokeType) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessNoContent(w) +} + +// InfoAccessKey - GET /minio/admin/v3/info-access-key?access-key= +func (a adminAPIHandlers) InfoAccessKey(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Get current object layer instance. + objectAPI := newObjectLayerFn() + if objectAPI == nil || globalNotificationSys == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + cred, owner, s3Err := validateAdminSignature(ctx, r, "") + if s3Err != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + + accessKey := mux.Vars(r)["accessKey"] + if accessKey == "" { + accessKey = cred.AccessKey + } + + u, ok := globalIAMSys.GetUser(ctx, accessKey) + targetCred := u.Credentials + + if !globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.ListServiceAccountsAdminAction, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + Claims: cred.Claims, + }) { + // If requested user does not exist and requestor is not allowed to list service accounts, return access denied. + if !ok { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + + requestUser := cred.AccessKey + if cred.ParentUser != "" { + requestUser = cred.ParentUser + } + + if requestUser != targetCred.ParentUser { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + } + + if !ok { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminNoSuchAccessKey), r.URL) + return + } + + var ( + sessionPolicy *policy.Policy + err error + userType string + ) + switch { + case targetCred.IsTemp(): + userType = "STS" + _, sessionPolicy, err = globalIAMSys.GetTemporaryAccount(ctx, accessKey) + if err == errNoSuchTempAccount { + err = errNoSuchAccessKey + } + case targetCred.IsServiceAccount(): + userType = "Service Account" + _, sessionPolicy, err = globalIAMSys.GetServiceAccount(ctx, accessKey) + if err == errNoSuchServiceAccount { + err = errNoSuchAccessKey + } + default: + err = errNoSuchAccessKey + } + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // if session policy is nil or empty, then it is implied policy + impliedPolicy := sessionPolicy == nil || (sessionPolicy.Version == "" && len(sessionPolicy.Statements) == 0) + + var svcAccountPolicy policy.Policy + + if !impliedPolicy { + svcAccountPolicy = *sessionPolicy + } else { + policiesNames, err := globalIAMSys.PolicyDBGet(targetCred.ParentUser, targetCred.Groups...) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + svcAccountPolicy = globalIAMSys.GetCombinedPolicy(policiesNames...) + } + + policyJSON, err := json.MarshalIndent(svcAccountPolicy, "", " ") + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + var expiration *time.Time + if !targetCred.Expiration.IsZero() && !targetCred.Expiration.Equal(timeSentinel) { + expiration = &targetCred.Expiration + } + + userProvider := guessUserProvider(targetCred) + + infoResp := madmin.InfoAccessKeyResp{ + AccessKey: accessKey, + InfoServiceAccountResp: madmin.InfoServiceAccountResp{ + ParentUser: targetCred.ParentUser, + Name: targetCred.Name, + Description: targetCred.Description, + AccountStatus: targetCred.Status, + ImpliedPolicy: impliedPolicy, + Policy: string(policyJSON), + Expiration: expiration, + }, + + UserType: userType, + UserProvider: userProvider, + } + + populateProviderInfoFromClaims(targetCred.Claims, userProvider, &infoResp) + + data, err := json.Marshal(infoResp) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + encryptedData, err := madmin.EncryptData(cred.SecretKey, data) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, encryptedData) +} + +const ( + allPoliciesFile = "policies.json" + allUsersFile = "users.json" + allGroupsFile = "groups.json" + allSvcAcctsFile = "svcaccts.json" + userPolicyMappingsFile = "user_mappings.json" + groupPolicyMappingsFile = "group_mappings.json" + stsUserPolicyMappingsFile = "stsuser_mappings.json" + + iamAssetsDir = "iam-assets" +) + +var iamExportFiles = []string{ + allPoliciesFile, + allUsersFile, + allGroupsFile, + allSvcAcctsFile, + userPolicyMappingsFile, + groupPolicyMappingsFile, + stsUserPolicyMappingsFile, +} + +// ExportIAMHandler - exports all iam info as a zipped file +func (a adminAPIHandlers) ExportIAM(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Get current object layer instance. + objectAPI, _ := validateAdminReq(ctx, w, r, policy.ExportIAMAction) + if objectAPI == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + // Initialize a zip writer which will provide a zipped content + // of bucket metadata + zipWriter := zip.NewWriter(w) + defer zipWriter.Close() + rawDataFn := func(r io.Reader, filename string, sz int) error { + header, zerr := zip.FileInfoHeader(dummyFileInfo{ + name: filename, + size: int64(sz), + mode: 0o600, + modTime: time.Now(), + isDir: false, + sys: nil, + }) + if zerr != nil { + adminLogIf(ctx, zerr) + return nil + } + header.Method = zip.Deflate + zwriter, zerr := zipWriter.CreateHeader(header) + if zerr != nil { + adminLogIf(ctx, zerr) + return nil + } + if _, err := io.Copy(zwriter, r); err != nil { + adminLogIf(ctx, err) + } + return nil + } + + for _, f := range iamExportFiles { + iamFile := pathJoin(iamAssetsDir, f) + switch f { + case allPoliciesFile: + allPolicies, err := globalIAMSys.ListPolicies(ctx, "") + if err != nil { + adminLogIf(ctx, err) + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + + policiesData, err := json.Marshal(allPolicies) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + if err = rawDataFn(bytes.NewReader(policiesData), iamFile, len(policiesData)); err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + case allUsersFile: + userIdentities := make(map[string]UserIdentity) + err := globalIAMSys.store.loadUsers(ctx, regUser, userIdentities) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + userAccounts := make(map[string]madmin.AddOrUpdateUserReq) + for u, uid := range userIdentities { + userAccounts[u] = madmin.AddOrUpdateUserReq{ + SecretKey: uid.Credentials.SecretKey, + Status: func() madmin.AccountStatus { + // Export current credential status + if uid.Credentials.Status == auth.AccountOff { + return madmin.AccountDisabled + } + return madmin.AccountEnabled + }(), + } + } + userData, err := json.Marshal(userAccounts) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + + if err = rawDataFn(bytes.NewReader(userData), iamFile, len(userData)); err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + case allGroupsFile: + groups := make(map[string]GroupInfo) + err := globalIAMSys.store.loadGroups(ctx, groups) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + groupData, err := json.Marshal(groups) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + + if err = rawDataFn(bytes.NewReader(groupData), iamFile, len(groupData)); err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + case allSvcAcctsFile: + serviceAccounts := make(map[string]UserIdentity) + err := globalIAMSys.store.loadUsers(ctx, svcUser, serviceAccounts) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + svcAccts := make(map[string]madmin.SRSvcAccCreate) + for user, acc := range serviceAccounts { + if user == siteReplicatorSvcAcc { + // skip site-replication service account. + continue + } + claims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, acc.Credentials.AccessKey) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + sa, policy, err := globalIAMSys.GetServiceAccount(ctx, acc.Credentials.AccessKey) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + + var policyJSON []byte + if policy != nil { + policyJSON, err = json.Marshal(policy) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + } + svcAccts[user] = madmin.SRSvcAccCreate{ + Parent: acc.Credentials.ParentUser, + AccessKey: user, + SecretKey: acc.Credentials.SecretKey, + Groups: acc.Credentials.Groups, + Claims: claims, + SessionPolicy: policyJSON, + Status: acc.Credentials.Status, + Name: sa.Name, + Description: sa.Description, + Expiration: &sa.Expiration, + } + } + + svcAccData, err := json.Marshal(svcAccts) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + + if err = rawDataFn(bytes.NewReader(svcAccData), iamFile, len(svcAccData)); err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + case userPolicyMappingsFile: + userPolicyMap := xsync.NewMapOf[string, MappedPolicy]() + err := globalIAMSys.store.loadMappedPolicies(ctx, regUser, false, userPolicyMap) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + userPolData, err := json.Marshal(mappedPoliciesToMap(userPolicyMap)) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + + if err = rawDataFn(bytes.NewReader(userPolData), iamFile, len(userPolData)); err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + case groupPolicyMappingsFile: + groupPolicyMap := xsync.NewMapOf[string, MappedPolicy]() + err := globalIAMSys.store.loadMappedPolicies(ctx, regUser, true, groupPolicyMap) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + grpPolData, err := json.Marshal(mappedPoliciesToMap(groupPolicyMap)) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + + if err = rawDataFn(bytes.NewReader(grpPolData), iamFile, len(grpPolData)); err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + case stsUserPolicyMappingsFile: + userPolicyMap := xsync.NewMapOf[string, MappedPolicy]() + err := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, false, userPolicyMap) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + userPolData, err := json.Marshal(mappedPoliciesToMap(userPolicyMap)) + if err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + if err = rawDataFn(bytes.NewReader(userPolData), iamFile, len(userPolData)); err != nil { + writeErrorResponse(ctx, w, exportError(ctx, err, iamFile, ""), r.URL) + return + } + } + } +} + +// ImportIAM - imports all IAM info into MinIO +func (a adminAPIHandlers) ImportIAM(w http.ResponseWriter, r *http.Request) { + a.importIAM(w, r, "") +} + +// ImportIAMV2 - imports all IAM info into MinIO +func (a adminAPIHandlers) ImportIAMV2(w http.ResponseWriter, r *http.Request) { + a.importIAM(w, r, "v2") +} + +// ImportIAM - imports all IAM info into MinIO +func (a adminAPIHandlers) importIAM(w http.ResponseWriter, r *http.Request, apiVer string) { + ctx := r.Context() + + // Validate signature, permissions and get current object layer instance. + objectAPI, _ := validateAdminReq(ctx, w, r, policy.ImportIAMAction) + if objectAPI == nil || globalNotificationSys == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + data, err := io.ReadAll(r.Body) + if err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + reader := bytes.NewReader(data) + zr, err := zip.NewReader(reader, int64(len(data))) + if err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + + var skipped, removed, added madmin.IAMEntities + var failed madmin.IAMErrEntities + + // import policies first + { + f, err := zr.Open(pathJoin(iamAssetsDir, allPoliciesFile)) + switch { + case errors.Is(err, os.ErrNotExist): + case err != nil: + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allPoliciesFile, ""), r.URL) + return + default: + defer f.Close() + var allPolicies map[string]policy.Policy + data, err = io.ReadAll(f) + if err != nil { + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allPoliciesFile, ""), r.URL) + return + } + err = json.Unmarshal(data, &allPolicies) + if err != nil { + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, allPoliciesFile, ""), r.URL) + return + } + for policyName, policy := range allPolicies { + if policy.IsEmpty() { + err = globalIAMSys.DeletePolicy(ctx, policyName, true) + removed.Policies = append(removed.Policies, policyName) + } else { + _, err = globalIAMSys.SetPolicy(ctx, policyName, policy) + added.Policies = append(added.Policies, policyName) + } + if err != nil { + writeErrorResponseJSON(ctx, w, importError(ctx, err, allPoliciesFile, policyName), r.URL) + return + } + } + } + } + + // import users + { + f, err := zr.Open(pathJoin(iamAssetsDir, allUsersFile)) + switch { + case errors.Is(err, os.ErrNotExist): + case err != nil: + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allUsersFile, ""), r.URL) + return + default: + defer f.Close() + var userAccts map[string]madmin.AddOrUpdateUserReq + data, err := io.ReadAll(f) + if err != nil { + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allUsersFile, ""), r.URL) + return + } + err = json.Unmarshal(data, &userAccts) + if err != nil { + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, allUsersFile, ""), r.URL) + return + } + for accessKey, ureq := range userAccts { + // Not allowed to add a user with same access key as root credential + if accessKey == globalActiveCred.AccessKey { + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAddUserInvalidArgument, err, allUsersFile, accessKey), r.URL) + return + } + + user, exists := globalIAMSys.GetUser(ctx, accessKey) + if exists && (user.Credentials.IsTemp() || user.Credentials.IsServiceAccount()) { + // Updating STS credential is not allowed, and this API does not + // support updating service accounts. + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAddUserInvalidArgument, err, allUsersFile, accessKey), r.URL) + return + } + + // Check if accessKey has beginning and end space characters, this only applies to new users. + if !exists && hasSpaceBE(accessKey) { + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminResourceInvalidArgument, err, allUsersFile, accessKey), r.URL) + return + } + + if _, err = globalIAMSys.CreateUser(ctx, accessKey, ureq); err != nil { + failed.Users = append(failed.Users, madmin.IAMErrEntity{Name: accessKey, Error: err}) + } else { + added.Users = append(added.Users, accessKey) + } + } + } + } + + // import groups + { + f, err := zr.Open(pathJoin(iamAssetsDir, allGroupsFile)) + switch { + case errors.Is(err, os.ErrNotExist): + case err != nil: + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allGroupsFile, ""), r.URL) + return + default: + defer f.Close() + var grpInfos map[string]GroupInfo + data, err := io.ReadAll(f) + if err != nil { + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allGroupsFile, ""), r.URL) + return + } + if err = json.Unmarshal(data, &grpInfos); err != nil { + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, allGroupsFile, ""), r.URL) + return + } + for group, grpInfo := range grpInfos { + // Check if group already exists + if _, gerr := globalIAMSys.GetGroupDescription(group); gerr != nil { + // If group does not exist, then check if the group has beginning and end space characters + // we will reject such group names. + if errors.Is(gerr, errNoSuchGroup) && hasSpaceBE(group) { + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminResourceInvalidArgument, gerr, allGroupsFile, group), r.URL) + return + } + } + if _, gerr := globalIAMSys.AddUsersToGroup(ctx, group, grpInfo.Members); gerr != nil { + failed.Groups = append(failed.Groups, madmin.IAMErrEntity{Name: group, Error: err}) + } else { + added.Groups = append(added.Groups, group) + } + } + } + } + + // import service accounts + { + f, err := zr.Open(pathJoin(iamAssetsDir, allSvcAcctsFile)) + switch { + case errors.Is(err, os.ErrNotExist): + case err != nil: + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allSvcAcctsFile, ""), r.URL) + return + default: + defer f.Close() + var serviceAcctReqs map[string]madmin.SRSvcAccCreate + data, err := io.ReadAll(f) + if err != nil { + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, allSvcAcctsFile, ""), r.URL) + return + } + if err = json.Unmarshal(data, &serviceAcctReqs); err != nil { + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, allSvcAcctsFile, ""), r.URL) + return + } + + // Validations for LDAP enabled deployments. + if globalIAMSys.LDAPConfig.Enabled() { + skippedAccessKeys, err := globalIAMSys.NormalizeLDAPAccessKeypairs(ctx, serviceAcctReqs) + skipped.ServiceAccounts = append(skipped.ServiceAccounts, skippedAccessKeys...) + if err != nil { + writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, ""), r.URL) + return + } + } + + for user, svcAcctReq := range serviceAcctReqs { + if slices.Contains(skipped.ServiceAccounts, user) { + continue + } + var sp *policy.Policy + var err error + if len(svcAcctReq.SessionPolicy) > 0 { + sp, err = policy.ParseConfig(bytes.NewReader(svcAcctReq.SessionPolicy)) + if err != nil { + writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL) + return + } + } + // service account access key cannot have space characters + // beginning and end of the string. + if hasSpaceBE(svcAcctReq.AccessKey) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument), r.URL) + return + } + updateReq := true + _, _, err = globalIAMSys.GetServiceAccount(ctx, svcAcctReq.AccessKey) + if err != nil { + if !errors.Is(err, errNoSuchServiceAccount) { + writeErrorResponseJSON(ctx, w, importError(ctx, err, allSvcAcctsFile, user), r.URL) + return + } + updateReq = false + } + if updateReq { + // If the service account exists, we remove it to ensure a + // clean import. + err := globalIAMSys.DeleteServiceAccount(ctx, svcAcctReq.AccessKey, true) + if err != nil { + delErr := fmt.Errorf("failed to delete existing service account (%s) before importing it: %w", svcAcctReq.AccessKey, err) + writeErrorResponseJSON(ctx, w, importError(ctx, delErr, allSvcAcctsFile, user), r.URL) + return + } + } + opts := newServiceAccountOpts{ + accessKey: user, + secretKey: svcAcctReq.SecretKey, + sessionPolicy: sp, + claims: svcAcctReq.Claims, + name: svcAcctReq.Name, + description: svcAcctReq.Description, + expiration: svcAcctReq.Expiration, + allowSiteReplicatorAccount: false, + } + + if _, _, err = globalIAMSys.NewServiceAccount(ctx, svcAcctReq.Parent, svcAcctReq.Groups, opts); err != nil { + failed.ServiceAccounts = append(failed.ServiceAccounts, madmin.IAMErrEntity{Name: user, Error: err}) + } else { + added.ServiceAccounts = append(added.ServiceAccounts, user) + } + } + } + } + + // import user policy mappings + { + f, err := zr.Open(pathJoin(iamAssetsDir, userPolicyMappingsFile)) + switch { + case errors.Is(err, os.ErrNotExist): + case err != nil: + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, userPolicyMappingsFile, ""), r.URL) + return + default: + defer f.Close() + var userPolicyMap map[string]MappedPolicy + data, err := io.ReadAll(f) + if err != nil { + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, userPolicyMappingsFile, ""), r.URL) + return + } + if err = json.Unmarshal(data, &userPolicyMap); err != nil { + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, userPolicyMappingsFile, ""), r.URL) + return + } + for u, pm := range userPolicyMap { + // disallow setting policy mapping if user is a temporary user + ok, _, err := globalIAMSys.IsTempUser(u) + if err != nil && err != errNoSuchUser { + writeErrorResponseJSON(ctx, w, importError(ctx, err, userPolicyMappingsFile, u), r.URL) + return + } + if ok { + writeErrorResponseJSON(ctx, w, importError(ctx, errIAMActionNotAllowed, userPolicyMappingsFile, u), r.URL) + return + } + if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, regUser, false); err != nil { + failed.UserPolicies = append( + failed.UserPolicies, + madmin.IAMErrPolicyEntity{ + Name: u, + Policies: strings.Split(pm.Policies, ","), + Error: err, + }) + } else { + added.UserPolicies = append(added.UserPolicies, map[string][]string{u: strings.Split(pm.Policies, ",")}) + } + } + } + } + + // import group policy mappings + { + f, err := zr.Open(pathJoin(iamAssetsDir, groupPolicyMappingsFile)) + switch { + case errors.Is(err, os.ErrNotExist): + case err != nil: + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, groupPolicyMappingsFile, ""), r.URL) + return + default: + defer f.Close() + var grpPolicyMap map[string]MappedPolicy + data, err := io.ReadAll(f) + if err != nil { + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, groupPolicyMappingsFile, ""), r.URL) + return + } + if err = json.Unmarshal(data, &grpPolicyMap); err != nil { + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, groupPolicyMappingsFile, ""), r.URL) + return + } + + // Validations for LDAP enabled deployments. + if globalIAMSys.LDAPConfig.Enabled() { + isGroup := true + skippedDN, err := globalIAMSys.NormalizeLDAPMappingImport(ctx, isGroup, grpPolicyMap) + skipped.Groups = append(skipped.Groups, skippedDN...) + if err != nil { + writeErrorResponseJSON(ctx, w, importError(ctx, err, groupPolicyMappingsFile, ""), r.URL) + return + } + } + + for g, pm := range grpPolicyMap { + if slices.Contains(skipped.Groups, g) { + continue + } + if _, err := globalIAMSys.PolicyDBSet(ctx, g, pm.Policies, unknownIAMUserType, true); err != nil { + failed.GroupPolicies = append( + failed.GroupPolicies, + madmin.IAMErrPolicyEntity{ + Name: g, + Policies: strings.Split(pm.Policies, ","), + Error: err, + }) + } else { + added.GroupPolicies = append(added.GroupPolicies, map[string][]string{g: strings.Split(pm.Policies, ",")}) + } + } + } + } + + // import sts user policy mappings + { + f, err := zr.Open(pathJoin(iamAssetsDir, stsUserPolicyMappingsFile)) + switch { + case errors.Is(err, os.ErrNotExist): + case err != nil: + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, stsUserPolicyMappingsFile, ""), r.URL) + return + default: + defer f.Close() + var userPolicyMap map[string]MappedPolicy + data, err := io.ReadAll(f) + if err != nil { + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrInvalidRequest, err, stsUserPolicyMappingsFile, ""), r.URL) + return + } + if err = json.Unmarshal(data, &userPolicyMap); err != nil { + writeErrorResponseJSON(ctx, w, importErrorWithAPIErr(ctx, ErrAdminConfigBadJSON, err, stsUserPolicyMappingsFile, ""), r.URL) + return + } + + // Validations for LDAP enabled deployments. + if globalIAMSys.LDAPConfig.Enabled() { + isGroup := true + skippedDN, err := globalIAMSys.NormalizeLDAPMappingImport(ctx, !isGroup, userPolicyMap) + skipped.Users = append(skipped.Users, skippedDN...) + if err != nil { + writeErrorResponseJSON(ctx, w, importError(ctx, err, stsUserPolicyMappingsFile, ""), r.URL) + return + } + } + for u, pm := range userPolicyMap { + if slices.Contains(skipped.Users, u) { + continue + } + // disallow setting policy mapping if user is a temporary user + ok, _, err := globalIAMSys.IsTempUser(u) + if err != nil && err != errNoSuchUser { + writeErrorResponseJSON(ctx, w, importError(ctx, err, stsUserPolicyMappingsFile, u), r.URL) + return + } + if ok { + writeErrorResponseJSON(ctx, w, importError(ctx, errIAMActionNotAllowed, stsUserPolicyMappingsFile, u), r.URL) + return + } + + if _, err := globalIAMSys.PolicyDBSet(ctx, u, pm.Policies, stsUser, false); err != nil { + failed.STSPolicies = append( + failed.STSPolicies, + madmin.IAMErrPolicyEntity{ + Name: u, + Policies: strings.Split(pm.Policies, ","), + Error: err, + }) + } else { + added.STSPolicies = append(added.STSPolicies, map[string][]string{u: strings.Split(pm.Policies, ",")}) + } + } + } + } + + if apiVer == "v2" { + iamr := madmin.ImportIAMResult{ + Skipped: skipped, + Removed: removed, + Added: added, + Failed: failed, + } + + b, err := json.Marshal(iamr) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, b) + } +} + +func addExpirationToCondValues(exp *time.Time, condValues map[string][]string) error { + if exp == nil || exp.IsZero() || exp.Equal(timeSentinel) { + return nil + } + dur := time.Until(*exp) + if dur <= 0 { + return errors.New("unsupported expiration time") + } + condValues["DurationSeconds"] = []string{strconv.FormatInt(int64(dur.Seconds()), 10)} + return nil +} + +func commonAddServiceAccount(r *http.Request, ldap bool) (context.Context, auth.Credentials, newServiceAccountOpts, madmin.AddServiceAccountReq, string, APIError) { + ctx := r.Context() + + // Get current object layer instance. + objectAPI := newObjectLayerFn() + if objectAPI == nil || globalNotificationSys == nil { + return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrServerNotInitialized) + } + + cred, owner, s3Err := validateAdminSignature(ctx, r, "") + if s3Err != ErrNone { + return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(s3Err) + } + + password := cred.SecretKey + reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength)) + if err != nil { + return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err) + } + + var createReq madmin.AddServiceAccountReq + if err = json.Unmarshal(reqBytes, &createReq); err != nil { + return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err) + } + + if createReq.Expiration != nil && !createReq.Expiration.IsZero() { + // truncate expiration at the second. + truncateTime := createReq.Expiration.Truncate(time.Second) + createReq.Expiration = &truncateTime + } + + // service account access key cannot have space characters beginning and end of the string. + if hasSpaceBE(createReq.AccessKey) { + return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument) + } + + if err := createReq.Validate(); err != nil { + // Since this validation would happen client side as well, we only send + // a generic error message here. + return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAdminResourceInvalidArgument) + } + // If the request did not set a TargetUser, the service account is + // created for the request sender. + targetUser := createReq.TargetUser + if targetUser == "" { + targetUser = cred.AccessKey + } + + description := createReq.Description + if description == "" { + description = createReq.Comment + } + opts := newServiceAccountOpts{ + accessKey: createReq.AccessKey, + secretKey: createReq.SecretKey, + name: createReq.Name, + description: description, + expiration: createReq.Expiration, + claims: make(map[string]any), + } + + condValues := getConditionValues(r, "", cred) + err = addExpirationToCondValues(createReq.Expiration, condValues) + if err != nil { + return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err) + } + + denyOnly := (targetUser == cred.AccessKey || targetUser == cred.ParentUser) + if ldap && !denyOnly { + res, _ := globalIAMSys.LDAPConfig.GetValidatedDNForUsername(targetUser) + if res != nil && res.NormDN == cred.ParentUser { + denyOnly = true + } + } + + // Check if action is allowed if creating access key for another user + // Check if action is explicitly denied if for self + if !globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.CreateServiceAccountAdminAction, + ConditionValues: condValues, + IsOwner: owner, + Claims: cred.Claims, + DenyOnly: denyOnly, + }) { + return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", errorCodes.ToAPIErr(ErrAccessDenied) + } + + var sp *policy.Policy + if len(createReq.Policy) > 0 { + sp, err = policy.ParseConfig(bytes.NewReader(createReq.Policy)) + if err != nil { + return ctx, auth.Credentials{}, newServiceAccountOpts{}, madmin.AddServiceAccountReq{}, "", toAdminAPIErr(ctx, err) + } + } + + opts.sessionPolicy = sp + + return ctx, cred, opts, createReq, targetUser, APIError{} +} + +// setReqInfoPolicyName will set the given policyName as a tag on the context's request info, +// so that it appears in audit logs. +func setReqInfoPolicyName(ctx context.Context, policyName string) { + reqInfo := logger.GetReqInfo(ctx) + reqInfo.SetTags("policyName", policyName) +} diff --git a/cmd/admin-handlers-users_test.go b/cmd/admin-handlers-users_test.go new file mode 100644 index 00000000..82826458 --- /dev/null +++ b/cmd/admin-handlers-users_test.go @@ -0,0 +1,1850 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "runtime" + "slices" + "strings" + "testing" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio-go/v7/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio-go/v7/pkg/signer" + "github.com/minio/minio/internal/auth" + "github.com/minio/pkg/v3/env" +) + +const ( + testDefaultTimeout = 30 * time.Second +) + +// API suite container for IAM +type TestSuiteIAM struct { + TestSuiteCommon + + ServerTypeDescription string + + // Flag to turn on tests for etcd backend IAM + withEtcdBackend bool + + endpoint string + adm *madmin.AdminClient + client *minio.Client +} + +func newTestSuiteIAM(c TestSuiteCommon, withEtcdBackend bool) *TestSuiteIAM { + etcdStr := "" + if withEtcdBackend { + etcdStr = " (with etcd backend)" + } + return &TestSuiteIAM{ + TestSuiteCommon: c, + ServerTypeDescription: fmt.Sprintf("%s%s", c.serverType, etcdStr), + withEtcdBackend: withEtcdBackend, + } +} + +func (s *TestSuiteIAM) iamSetup(c *check) { + var err error + // strip url scheme from endpoint + s.endpoint = strings.TrimPrefix(s.endPoint, "http://") + if s.secure { + s.endpoint = strings.TrimPrefix(s.endPoint, "https://") + } + + s.adm, err = madmin.New(s.endpoint, s.accessKey, s.secretKey, s.secure) + if err != nil { + c.Fatalf("error creating admin client: %v", err) + } + // Set transport, so that TLS is handled correctly. + s.adm.SetCustomTransport(s.TestSuiteCommon.client.Transport) + + s.client, err = minio.New(s.endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(s.accessKey, s.secretKey, ""), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("error creating minio client: %v", err) + } +} + +// List of all IAM test suites (i.e. test server configuration combinations) +// common to tests. +var iamTestSuites = func() []*TestSuiteIAM { + baseTestCases := []TestSuiteCommon{ + // Init and run test on ErasureSD backend with signature v4. + {serverType: "ErasureSD", signer: signerV4}, + // Init and run test on ErasureSD backend, with tls enabled. + {serverType: "ErasureSD", signer: signerV4, secure: true}, + // Init and run test on Erasure backend. + {serverType: "Erasure", signer: signerV4}, + // Init and run test on ErasureSet backend. + {serverType: "ErasureSet", signer: signerV4}, + } + testCases := []*TestSuiteIAM{} + for _, bt := range baseTestCases { + testCases = append(testCases, + newTestSuiteIAM(bt, false), + newTestSuiteIAM(bt, true), + ) + } + return testCases +}() + +const ( + EnvTestEtcdBackend = "_MINIO_ETCD_TEST_SERVER" +) + +func (s *TestSuiteIAM) setUpEtcd(c *check, etcdServer string) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + configCmds := []string{ + "etcd", + "endpoints=" + etcdServer, + "path_prefix=" + mustGetUUID(), + } + _, err := s.adm.SetConfigKV(ctx, strings.Join(configCmds, " ")) + if err != nil { + c.Fatalf("unable to setup Etcd for tests: %v", err) + } + + s.RestartIAMSuite(c) +} + +func (s *TestSuiteIAM) SetUpSuite(c *check) { + // If etcd backend is specified and etcd server is not present, the test + // is skipped. + etcdServer := env.Get(EnvTestEtcdBackend, "") + if s.withEtcdBackend && etcdServer == "" { + c.Skip("Skipping etcd backend IAM test as no etcd server is configured.") + } + + s.TestSuiteCommon.SetUpSuite(c) + + s.iamSetup(c) + + if s.withEtcdBackend { + s.setUpEtcd(c, etcdServer) + } +} + +func (s *TestSuiteIAM) RestartIAMSuite(c *check) { + s.RestartTestServer(c) + + s.iamSetup(c) +} + +func (s *TestSuiteIAM) getAdminClient(c *check, accessKey, secretKey, sessionToken string) *madmin.AdminClient { + madmClnt, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, sessionToken), + Secure: s.secure, + }) + if err != nil { + c.Fatalf("error creating user admin client: %s", err) + } + madmClnt.SetCustomTransport(s.TestSuiteCommon.client.Transport) + return madmClnt +} + +func (s *TestSuiteIAM) getUserClient(c *check, accessKey, secretKey, sessionToken string) *minio.Client { + client, err := minio.New(s.endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, sessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("error creating user minio client: %s", err) + } + return client +} + +func TestIAMInternalIDPServerSuite(t *testing.T) { + if runtime.GOOS == globalWindowsOSName { + t.Skip("windows is clunky disable these tests") + } + for i, testCase := range iamTestSuites { + t.Run( + fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription), + func(t *testing.T) { + suite := testCase + c := &check{t, testCase.serverType} + + suite.SetUpSuite(c) + suite.TestUserCreate(c) + suite.TestUserPolicyEscalationBug(c) + suite.TestPolicyCreate(c) + suite.TestCannedPolicies(c) + suite.TestGroupAddRemove(c) + suite.TestServiceAccountOpsByAdmin(c) + suite.TestServiceAccountPrivilegeEscalationBug(c) + suite.TestServiceAccountPrivilegeEscalationBug2_2025_10_15(c, true) + suite.TestServiceAccountPrivilegeEscalationBug2_2025_10_15(c, false) + suite.TestServiceAccountOpsByUser(c) + suite.TestServiceAccountDurationSecondsCondition(c) + suite.TestAddServiceAccountPerms(c) + suite.TearDownSuite(c) + }, + ) + } +} + +func (s *TestSuiteIAM) TestUserCreate(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + // 1. Create a user. + accessKey, secretKey := mustGenerateCredentials(c) + err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled) + if err != nil { + c.Fatalf("Unable to set user: %v", err) + } + + // 2. Check new user appears in listing + usersMap, err := s.adm.ListUsers(ctx) + if err != nil { + c.Fatalf("error listing: %v", err) + } + v, ok := usersMap[accessKey] + if !ok { + c.Fatalf("user not listed: %s", accessKey) + } + c.Assert(v.Status, madmin.AccountEnabled) + + // 3. Associate policy and check that user can access + _, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{ + Policies: []string{"readwrite"}, + User: accessKey, + }) + if err != nil { + c.Fatalf("unable to attach policy: %v", err) + } + + client := s.getUserClient(c, accessKey, secretKey, "") + err = client.MakeBucket(ctx, getRandomBucketName(), minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("user could not create bucket: %v", err) + } + + // 3.10. Check that user's password can be updated. + _, newSecretKey := mustGenerateCredentials(c) + err = s.adm.SetUser(ctx, accessKey, newSecretKey, madmin.AccountEnabled) + if err != nil { + c.Fatalf("Unable to update user's secret key: %v", err) + } + // 3.10.1 Check that old password no longer works. + err = client.MakeBucket(ctx, getRandomBucketName(), minio.MakeBucketOptions{}) + if err == nil { + c.Fatalf("user was unexpectedly able to create bucket with bad password!") + } + // 3.10.2 Check that new password works. + client = s.getUserClient(c, accessKey, newSecretKey, "") + err = client.MakeBucket(ctx, getRandomBucketName(), minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("user could not create bucket: %v", err) + } + + // 4. Check that user can be disabled and verify it. + err = s.adm.SetUserStatus(ctx, accessKey, madmin.AccountDisabled) + if err != nil { + c.Fatalf("could not set user account to disabled") + } + usersMap, err = s.adm.ListUsers(ctx) + if err != nil { + c.Fatalf("error listing: %v", err) + } + v, ok = usersMap[accessKey] + if !ok { + c.Fatalf("user was not listed after disabling: %s", accessKey) + } + c.Assert(v.Status, madmin.AccountDisabled) + err = client.MakeBucket(ctx, getRandomBucketName(), minio.MakeBucketOptions{}) + if err == nil { + c.Fatalf("user account was not disabled!") + } + + // 5. Check that user can be deleted and verify it. + err = s.adm.RemoveUser(ctx, accessKey) + if err != nil { + c.Fatalf("user could not be deleted: %v", err) + } + usersMap, err = s.adm.ListUsers(ctx) + if err != nil { + c.Fatalf("error listing: %v", err) + } + _, ok = usersMap[accessKey] + if ok { + c.Fatalf("user not deleted: %s", accessKey) + } + err = client.MakeBucket(ctx, getRandomBucketName(), minio.MakeBucketOptions{}) + if err == nil { + c.Fatalf("user account was not deleted!") + } +} + +func (s *TestSuiteIAM) TestUserPolicyEscalationBug(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket creat error: %v", err) + } + + // 2. Create a user, associate policy and verify access + accessKey, secretKey := mustGenerateCredentials(c) + err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled) + if err != nil { + c.Fatalf("Unable to set user: %v", err) + } + // 2.1 check that user does not have any access to the bucket + uClient := s.getUserClient(c, accessKey, secretKey, "") + c.mustNotListObjects(ctx, uClient, bucket) + + // 2.2 create and associate policy to user + policy := "mypolicy-test-user-update" + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::%s" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket, bucket) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + _, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{ + Policies: []string{policy}, + User: accessKey, + }) + if err != nil { + c.Fatalf("unable to attach policy: %v", err) + } + // 2.3 check user has access to bucket + c.mustListObjects(ctx, uClient, bucket) + // 2.3 check that user cannot delete the bucket + err = uClient.RemoveBucket(ctx, bucket) + if err == nil || err.Error() != "Access Denied." { + c.Fatalf("bucket was deleted unexpectedly or got unexpected err: %v", err) + } + + // 3. Craft a request to update the user's permissions + ep := s.adm.GetEndpointURL() + urlValue := url.Values{} + urlValue.Add("accessKey", accessKey) + u, err := url.Parse(fmt.Sprintf("%s://%s/minio/admin/v3/add-user?%s", ep.Scheme, ep.Host, s3utils.QueryEncode(urlValue))) + if err != nil { + c.Fatalf("unexpected url parse err: %v", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), nil) + if err != nil { + c.Fatalf("unexpected new request err: %v", err) + } + reqBodyArg := madmin.UserInfo{ + SecretKey: secretKey, + PolicyName: "consoleAdmin", + Status: madmin.AccountEnabled, + } + buf, err := json.Marshal(reqBodyArg) + if err != nil { + c.Fatalf("unexpected json encode err: %v", err) + } + buf, err = madmin.EncryptData(secretKey, buf) + if err != nil { + c.Fatalf("unexpected encryption err: %v", err) + } + + req.ContentLength = int64(len(buf)) + sum := sha256.Sum256(buf) + req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum[:])) + req.Body = io.NopCloser(bytes.NewReader(buf)) + req = signer.SignV4(*req, accessKey, secretKey, "", "") + + // 3.1 Execute the request. + resp, err := s.TestSuiteCommon.client.Do(req) + if err != nil { + c.Fatalf("unexpected request err: %v", err) + } + if resp.StatusCode != 200 { + c.Fatalf("got unexpected response: %#v\n", resp) + } + + // 3.2 check that user cannot delete the bucket + err = uClient.RemoveBucket(ctx, bucket) + if err == nil || err.Error() != "Access Denied." { + c.Fatalf("User was able to escalate privileges (Err=%v)!", err) + } +} + +func (s *TestSuiteIAM) TestAddServiceAccountPerms(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + // 1. Create a policy + policy1 := "deny-svc" + policy2 := "allow-svc" + policyBytes := []byte(`{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Action": [ + "admin:CreateServiceAccount" + ] + } + ] +}`) + + newPolicyBytes := []byte(`{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::testbucket" + ] + } + ] +}`) + + err := s.adm.AddCannedPolicy(ctx, policy1, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + err = s.adm.AddCannedPolicy(ctx, policy2, newPolicyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + // 2. Verify that policy json is validated by server + invalidPolicyBytes := policyBytes[:len(policyBytes)-1] + err = s.adm.AddCannedPolicy(ctx, policy1+"invalid", invalidPolicyBytes) + if err == nil { + c.Fatalf("invalid policy creation success") + } + + // 3. Create a user, associate policy and verify access + accessKey, secretKey := mustGenerateCredentials(c) + err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled) + if err != nil { + c.Fatalf("Unable to set user: %v", err) + } + // 3.1 check that user does not have any access to the bucket + uClient := s.getUserClient(c, accessKey, secretKey, "") + c.mustNotListObjects(ctx, uClient, "testbucket") + + // 3.2 associate policy to user + _, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{ + Policies: []string{policy1}, + User: accessKey, + }) + if err != nil { + c.Fatalf("unable to attach policy: %v", err) + } + + admClnt := s.getAdminClient(c, accessKey, secretKey, "") + + // 3.3 check user does not have explicit permissions to create service account. + c.mustNotCreateSvcAccount(ctx, accessKey, admClnt) + + // 4. Verify the policy appears in listing + ps, err := s.adm.ListCannedPolicies(ctx) + if err != nil { + c.Fatalf("policy list err: %v", err) + } + _, ok := ps[policy1] + if !ok { + c.Fatalf("policy was missing!") + } + + // Detach policy1 to set up for policy2 + _, err = s.adm.DetachPolicy(ctx, madmin.PolicyAssociationReq{ + Policies: []string{policy1}, + User: accessKey, + }) + if err != nil { + c.Fatalf("unable to detach policy: %v", err) + } + + // 3.2 associate policy to user + _, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{ + Policies: []string{policy2}, + User: accessKey, + }) + if err != nil { + c.Fatalf("unable to attach policy: %v", err) + } + + // 3.3 check user can create service account implicitly. + c.mustCreateSvcAccount(ctx, accessKey, admClnt) + + _, ok = ps[policy2] + if !ok { + c.Fatalf("policy was missing!") + } + + err = s.adm.RemoveUser(ctx, accessKey) + if err != nil { + c.Fatalf("user could not be deleted: %v", err) + } + + err = s.adm.RemoveCannedPolicy(ctx, policy1) + if err != nil { + c.Fatalf("policy del err: %v", err) + } + + err = s.adm.RemoveCannedPolicy(ctx, policy2) + if err != nil { + c.Fatalf("policy del err: %v", err) + } +} + +func (s *TestSuiteIAM) TestPolicyCreate(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket creat error: %v", err) + } + + // 1. Create a policy + policy := "mypolicy" + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::%s" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket, bucket) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + // 2. Verify that policy json is validated by server + invalidPolicyBytes := policyBytes[:len(policyBytes)-1] + err = s.adm.AddCannedPolicy(ctx, policy+"invalid", invalidPolicyBytes) + if err == nil { + c.Fatalf("invalid policy creation success") + } + + // 3. Create a user, associate policy and verify access + accessKey, secretKey := mustGenerateCredentials(c) + err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled) + if err != nil { + c.Fatalf("Unable to set user: %v", err) + } + // 3.1 check that user does not have any access to the bucket + uClient := s.getUserClient(c, accessKey, secretKey, "") + c.mustNotListObjects(ctx, uClient, bucket) + + // 3.2 associate policy to user + _, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{ + Policies: []string{policy}, + User: accessKey, + }) + if err != nil { + c.Fatalf("unable to attach policy: %v", err) + } + // 3.3 check user has access to bucket + c.mustListObjects(ctx, uClient, bucket) + // 3.4 Check that user cannot exceed their permissions + err = uClient.RemoveBucket(ctx, bucket) + if err == nil { + c.Fatalf("bucket was deleted!") + } + + // 4. Verify the policy appears in listing + ps, err := s.adm.ListCannedPolicies(ctx) + if err != nil { + c.Fatalf("policy list err: %v", err) + } + _, ok := ps[policy] + if !ok { + c.Fatalf("policy was missing!") + } + + // 5. Check that policy cannot be deleted when attached to a user. + err = s.adm.RemoveCannedPolicy(ctx, policy) + if err == nil { + c.Fatalf("policy could be unexpectedly deleted!") + } + + // 6. Delete the user and then delete the policy. + err = s.adm.RemoveUser(ctx, accessKey) + if err != nil { + c.Fatalf("user could not be deleted: %v", err) + } + err = s.adm.RemoveCannedPolicy(ctx, policy) + if err != nil { + c.Fatalf("policy del err: %v", err) + } +} + +func (s *TestSuiteIAM) TestCannedPolicies(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + policies, err := s.adm.ListCannedPolicies(ctx) + if err != nil { + c.Fatalf("unable to list policies: %v", err) + } + + defaultPolicies := []string{ + "readwrite", + "readonly", + "writeonly", + "diagnostics", + "consoleAdmin", + } + + for _, v := range defaultPolicies { + if _, ok := policies[v]; !ok { + c.Fatalf("Failed to find %s in policies list", v) + } + } + + bucket := getRandomBucketName() + err = s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket creat error: %v", err) + } + + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::%s" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket, bucket) + + // Check that default policies can be overwritten. + err = s.adm.AddCannedPolicy(ctx, "readwrite", policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + info, err := s.adm.InfoCannedPolicy(ctx, "readwrite") + if err != nil { + c.Fatalf("policy info err: %v", err) + } + + // Check that policy with comma is rejected. + err = s.adm.AddCannedPolicy(ctx, "invalid,policy", policyBytes) + if err == nil { + c.Fatalf("invalid policy created successfully") + } + + infoStr := string(info) + if !strings.Contains(infoStr, `"s3:PutObject"`) || !strings.Contains(infoStr, ":"+bucket+"/") { + c.Fatalf("policy contains unexpected content!") + } +} + +func (s *TestSuiteIAM) TestGroupAddRemove(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket creat error: %v", err) + } + + policy := "mypolicy" + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::%s" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket, bucket) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + accessKey, secretKey := mustGenerateCredentials(c) + err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled) + if err != nil { + c.Fatalf("Unable to set user: %v", err) + } + + // 1. Add user to a new group + group := "mygroup" + err = s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{ + Group: group, + Members: []string{accessKey}, + }) + if err != nil { + c.Fatalf("Unable to add user to group: %v", err) + } + + // 2. Check that user has no access + uClient := s.getUserClient(c, accessKey, secretKey, "") + c.mustNotListObjects(ctx, uClient, bucket) + + // 3. Associate policy to group and check user got access. + _, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{ + Policies: []string{policy}, + Group: group, + }) + if err != nil { + c.Fatalf("unable to attach policy: %v", err) + } + // 3.1 check user has access to bucket + c.mustListObjects(ctx, uClient, bucket) + // 3.2 Check that user cannot exceed their permissions + err = uClient.RemoveBucket(ctx, bucket) + if err == nil { + c.Fatalf("bucket was deleted!") + } + + // 4. List groups and members and verify + groups, err := s.adm.ListGroups(ctx) + if err != nil { + c.Fatalf("group list err: %v", err) + } + expected := []string{group} + if !slices.Equal(groups, expected) { + c.Fatalf("expected group listing: %v, got: %v", expected, groups) + } + groupInfo, err := s.adm.GetGroupDescription(ctx, group) + if err != nil { + c.Fatalf("group desc err: %v", err) + } + c.Assert(groupInfo.Name, group) + c.Assert(set.CreateStringSet(groupInfo.Members...), set.CreateStringSet(accessKey)) + c.Assert(groupInfo.Policy, policy) + c.Assert(groupInfo.Status, string(madmin.GroupEnabled)) + + // 5. Disable/enable the group and verify that user access is revoked/restored. + err = s.adm.SetGroupStatus(ctx, group, madmin.GroupDisabled) + if err != nil { + c.Fatalf("group set status err: %v", err) + } + groupInfo, err = s.adm.GetGroupDescription(ctx, group) + if err != nil { + c.Fatalf("group desc err: %v", err) + } + c.Assert(groupInfo.Status, string(madmin.GroupDisabled)) + c.mustNotListObjects(ctx, uClient, bucket) + + err = s.adm.SetGroupStatus(ctx, group, madmin.GroupEnabled) + if err != nil { + c.Fatalf("group set status err: %v", err) + } + groupInfo, err = s.adm.GetGroupDescription(ctx, group) + if err != nil { + c.Fatalf("group desc err: %v", err) + } + c.Assert(groupInfo.Status, string(madmin.GroupEnabled)) + c.mustListObjects(ctx, uClient, bucket) + + // 6. Verify that group cannot be deleted with users. + err = s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{ + Group: group, + IsRemove: true, + }) + if err == nil { + c.Fatalf("group was removed!") + } + groupInfo, err = s.adm.GetGroupDescription(ctx, group) + if err != nil { + c.Fatalf("group desc err: %v", err) + } + c.Assert(groupInfo.Name, group) + + // 7. Remove user from group and verify access is revoked. + err = s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{ + Group: group, + Members: []string{accessKey}, + IsRemove: true, + }) + if err != nil { + c.Fatalf("group update err: %v", err) + } + c.mustNotListObjects(ctx, uClient, bucket) + + // 7.1 verify group still exists + groupInfo, err = s.adm.GetGroupDescription(ctx, group) + if err != nil { + c.Fatalf("group desc err: %v", err) + } + c.Assert(groupInfo.Name, group) + c.Assert(len(groupInfo.Members), 0) + + // 8. Delete group and verify + err = s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{ + Group: group, + IsRemove: true, + }) + if err != nil { + c.Fatalf("group update err: %v", err) + } + groups, err = s.adm.ListGroups(ctx) + if err != nil { + c.Fatalf("group list err: %v", err) + } + if set.CreateStringSet(groups...).Contains(group) { + c.Fatalf("created group still present!") + } + _, err = s.adm.GetGroupDescription(ctx, group) + if err == nil { + c.Fatalf("group appears to exist") + } +} + +func (s *TestSuiteIAM) TestServiceAccountOpsByUser(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket creat error: %v", err) + } + + // Create policy, user and associate policy + policy := "mypolicy" + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::%s" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket, bucket) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + accessKey, secretKey := mustGenerateCredentials(c) + err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled) + if err != nil { + c.Fatalf("Unable to set user: %v", err) + } + + _, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{ + Policies: []string{policy}, + User: accessKey, + }) + if err != nil { + c.Fatalf("unable to attach policy: %v", err) + } + + // Create an madmin client with user creds + userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, ""), + Secure: s.secure, + }) + if err != nil { + c.Fatalf("Err creating user admin client: %v", err) + } + userAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport) + + // Create svc acc + cr := c.mustCreateSvcAccount(ctx, accessKey, userAdmClient) + + // 1. Check that svc account appears in listing + c.assertSvcAccAppearsInListing(ctx, userAdmClient, accessKey, cr.AccessKey) + + // 2. Check that svc account info can be queried + c.assertSvcAccInfoQueryable(ctx, userAdmClient, accessKey, cr.AccessKey, false) + + // 3. Check S3 access + c.assertSvcAccS3Access(ctx, s, cr, bucket) + + // 5. Check that service account can be deleted. + c.assertSvcAccDeletion(ctx, s, userAdmClient, accessKey, bucket) + + // 6. Check that service account cannot be created for some other user. + c.mustNotCreateSvcAccount(ctx, globalActiveCred.AccessKey, userAdmClient) +} + +func (s *TestSuiteIAM) TestServiceAccountDurationSecondsCondition(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket creat error: %v", err) + } + + // Create policy, user and associate policy + policy := "mypolicy" + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Action": [ + "admin:CreateServiceAccount", + "admin:UpdateServiceAccount" + ], + "Condition": {"NumericGreaterThan": {"svc:DurationSeconds": "3600"}} + }, + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::%s" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket, bucket) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + accessKey, secretKey := mustGenerateCredentials(c) + err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled) + if err != nil { + c.Fatalf("Unable to set user: %v", err) + } + + _, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{ + Policies: []string{policy}, + User: accessKey, + }) + if err != nil { + c.Fatalf("unable to attach policy: %v", err) + } + + // Create an madmin client with user creds + userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, ""), + Secure: s.secure, + }) + if err != nil { + c.Fatalf("Err creating user admin client: %v", err) + } + userAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport) + + distantExpiration := time.Now().Add(30 * time.Minute) + cr, err := userAdmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{ + TargetUser: accessKey, + AccessKey: "svc-accesskey", + SecretKey: "svc-secretkey", + Expiration: &distantExpiration, + }) + if err != nil { + c.Fatalf("Unable to create svc acc: %v", err) + } + + c.assertSvcAccS3Access(ctx, s, cr, bucket) + + closeExpiration := time.Now().Add(2 * time.Hour) + _, err = userAdmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{ + TargetUser: accessKey, + AccessKey: "svc-accesskey", + SecretKey: "svc-secretkey", + Expiration: &closeExpiration, + }) + if err == nil { + c.Fatalf("Creating a svc acc with distant expiration should fail") + } +} + +func (s *TestSuiteIAM) TestServiceAccountOpsByAdmin(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket creat error: %v", err) + } + + // Create policy, user and associate policy + policy := "mypolicy" + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::%s" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket, bucket) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + accessKey, secretKey := mustGenerateCredentials(c) + err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled) + if err != nil { + c.Fatalf("Unable to set user: %v", err) + } + + _, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{ + Policies: []string{policy}, + User: accessKey, + }) + if err != nil { + c.Fatalf("unable to attach policy: %v", err) + } + + // 1. Create a service account for the user + cr := c.mustCreateSvcAccount(ctx, accessKey, s.adm) + + // 1.2 Check that svc account appears in listing + c.assertSvcAccAppearsInListing(ctx, s.adm, accessKey, cr.AccessKey) + + // 1.3 Check that svc account info can be queried + c.assertSvcAccInfoQueryable(ctx, s.adm, accessKey, cr.AccessKey, false) + + // 2. Check that svc account can access the bucket + c.assertSvcAccS3Access(ctx, s, cr, bucket) + + // 3. Check that svc account can restrict the policy, and that the + // session policy can be updated. + c.assertSvcAccSessionPolicyUpdate(ctx, s, s.adm, accessKey, bucket) + + // 4. Check that service account's secret key and account status can be + // updated. + c.assertSvcAccSecretKeyAndStatusUpdate(ctx, s, s.adm, accessKey, bucket) + + // 5. Check that service account can be deleted. + c.assertSvcAccDeletion(ctx, s, s.adm, accessKey, bucket) +} + +func (s *TestSuiteIAM) TestServiceAccountPrivilegeEscalationBug(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + err := s.client.MakeBucket(ctx, "public", minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket creat error: %v", err) + } + + err = s.client.MakeBucket(ctx, "private", minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket creat error: %v", err) + } + + pubPolicyBytes := []byte(`{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:*" + ], + "Resource": [ + "arn:aws:s3:::public", + "arn:aws:s3:::public/*" + ] + } + ] +}`) + + fullS3PolicyBytes := []byte(`{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:*" + ], + "Resource": [ + "arn:aws:s3:::*" + ] + } + ] +} +`) + + // Create a service account for the root user. + cr, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{ + TargetUser: globalActiveCred.AccessKey, + Policy: pubPolicyBytes, + }) + if err != nil { + c.Fatalf("admin should be able to create service account for themselves %s", err) + } + + svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "") + + // Check that the service account can access the public bucket. + buckets, err := svcClient.ListBuckets(ctx) + if err != nil { + c.Fatalf("err fetching buckets %s", err) + } + if len(buckets) != 1 || buckets[0].Name != "public" { + c.Fatalf("service account should only have access to public bucket") + } + + // Create an madmin client with the service account creds. + svcAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{ + Creds: credentials.NewStaticV4(cr.AccessKey, cr.SecretKey, ""), + Secure: s.secure, + }) + if err != nil { + c.Fatalf("Err creating svcacct admin client: %v", err) + } + svcAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport) + + // Attempt to update the policy on the service account. + err = svcAdmClient.UpdateServiceAccount(ctx, cr.AccessKey, + madmin.UpdateServiceAccountReq{ + NewPolicy: fullS3PolicyBytes, + }) + + if err == nil { + c.Fatalf("service account should not be able to update policy on itself") + } else if !strings.Contains(err.Error(), "Access Denied") { + c.Fatalf("unexpected error: %v", err) + } +} + +func (s *TestSuiteIAM) TestServiceAccountPrivilegeEscalationBug2_2025_10_15(c *check, forRoot bool) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + for i := range 3 { + err := s.client.MakeBucket(ctx, fmt.Sprintf("bucket%d", i+1), minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket create error: %v", err) + } + defer func(i int) { + _ = s.client.RemoveBucket(ctx, fmt.Sprintf("bucket%d", i+1)) + }(i) + } + + allow2BucketsPolicyBytes := []byte(`{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ListBucket1AndBucket2", + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": ["arn:aws:s3:::bucket1", "arn:aws:s3:::bucket2"] + }, + { + "Sid": "ReadWriteBucket1AndBucket2Objects", + "Effect": "Allow", + "Action": [ + "s3:DeleteObject", + "s3:DeleteObjectVersion", + "s3:GetObject", + "s3:GetObjectVersion", + "s3:PutObject" + ], + "Resource": ["arn:aws:s3:::bucket1/*", "arn:aws:s3:::bucket2/*"] + } + ] +}`) + + if forRoot { + // Create a service account for the root user. + _, err := s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{ + Policy: allow2BucketsPolicyBytes, + AccessKey: "restricted", + SecretKey: "restricted123", + }) + if err != nil { + c.Fatalf("could not create service account") + } + defer func() { + _ = s.adm.DeleteServiceAccount(ctx, "restricted") + }() + } else { + // Create a regular user and attach consoleAdmin policy + err := s.adm.AddUser(ctx, "foobar", "foobar123") + if err != nil { + c.Fatalf("could not create user") + } + + _, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{ + Policies: []string{"consoleAdmin"}, + User: "foobar", + }) + if err != nil { + c.Fatalf("could not attach policy") + } + + // Create a service account for the regular user. + _, err = s.adm.AddServiceAccount(ctx, madmin.AddServiceAccountReq{ + Policy: allow2BucketsPolicyBytes, + TargetUser: "foobar", + AccessKey: "restricted", + SecretKey: "restricted123", + }) + if err != nil { + c.Fatalf("could not create service account: %v", err) + } + defer func() { + _ = s.adm.DeleteServiceAccount(ctx, "restricted") + _ = s.adm.RemoveUser(ctx, "foobar") + }() + } + restrictedClient := s.getUserClient(c, "restricted", "restricted123", "") + + buckets, err := restrictedClient.ListBuckets(ctx) + if err != nil { + c.Fatalf("err fetching buckets %s", err) + } + if len(buckets) != 2 || buckets[0].Name != "bucket1" || buckets[1].Name != "bucket2" { + c.Fatalf("restricted service account should only have access to bucket1 and bucket2") + } + + // Try to escalate privileges + restrictedAdmClient := s.getAdminClient(c, "restricted", "restricted123", "") + _, err = restrictedAdmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{ + AccessKey: "newroot", + SecretKey: "newroot123", + }) + if err == nil { + c.Fatalf("restricted service account was able to create service account bypassing sub-policy!") + } +} + +func (s *TestSuiteIAM) SetUpAccMgmtPlugin(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + pluginEndpoint := env.Get("_MINIO_POLICY_PLUGIN_ENDPOINT", "") + if pluginEndpoint == "" { + c.Skip("_MINIO_POLICY_PLUGIN_ENDPOINT not given - skipping.") + } + + configCmds := []string{ + "policy_plugin", + "url=" + pluginEndpoint, + } + + _, err := s.adm.SetConfigKV(ctx, strings.Join(configCmds, " ")) + if err != nil { + c.Fatalf("unable to setup access management plugin for tests: %v", err) + } + + s.RestartIAMSuite(c) +} + +// TestIAM_AMPInternalIDPServerSuite - tests for access management plugin +func TestIAM_AMPInternalIDPServerSuite(t *testing.T) { + for i, testCase := range iamTestSuites { + t.Run( + fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription), + func(t *testing.T) { + suite := testCase + c := &check{t, testCase.serverType} + + suite.SetUpSuite(c) + defer suite.TearDownSuite(c) + + suite.SetUpAccMgmtPlugin(c) + + suite.TestAccMgmtPlugin(c) + }, + ) + } +} + +// TestAccMgmtPlugin - this test assumes that the access-management-plugin is +// the same as the example in `docs/iam/access-manager-plugin.go` - +// specifically, it denies only `s3:Put*` operations on non-root accounts. +func (s *TestSuiteIAM) TestAccMgmtPlugin(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + // 0. Check that owner is able to make-bucket. + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket creat error: %v", err) + } + + // 1. Create a user. + accessKey, secretKey := mustGenerateCredentials(c) + err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled) + if err != nil { + c.Fatalf("Unable to set user: %v", err) + } + + // 2. Check new user appears in listing + usersMap, err := s.adm.ListUsers(ctx) + if err != nil { + c.Fatalf("error listing: %v", err) + } + v, ok := usersMap[accessKey] + if !ok { + c.Fatalf("user not listed: %s", accessKey) + } + c.Assert(v.Status, madmin.AccountEnabled) + + // 3. Check that user is able to make a bucket. + client := s.getUserClient(c, accessKey, secretKey, "") + err = client.MakeBucket(ctx, getRandomBucketName(), minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("user not create bucket: %v", err) + } + + // 3.1 check user has access to bucket + c.mustListObjects(ctx, client, bucket) + + // 3.2 check that user cannot upload an object. + _, err = client.PutObject(ctx, bucket, "objectName", bytes.NewBuffer([]byte("some content")), 12, minio.PutObjectOptions{}) + if err == nil { + c.Fatalf("user was able to upload unexpectedly") + } + + // Create an madmin client with user creds + userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, ""), + Secure: s.secure, + }) + if err != nil { + c.Fatalf("Err creating user admin client: %v", err) + } + userAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport) + + // Create svc acc + cr := c.mustCreateSvcAccount(ctx, accessKey, userAdmClient) + + // 1. Check that svc account appears in listing + c.assertSvcAccAppearsInListing(ctx, userAdmClient, accessKey, cr.AccessKey) + + // 2. Check that svc account info can be queried + c.assertSvcAccInfoQueryable(ctx, userAdmClient, accessKey, cr.AccessKey, false) + + // 3. Check S3 access + c.assertSvcAccS3Access(ctx, s, cr, bucket) + + // Check that session policies do not apply - as policy enforcement is + // delegated to plugin. + { + svcAK, svcSK := mustGenerateCredentials(c) + + // This policy does not allow listing objects. + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket) + cr, err := userAdmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{ + Policy: policyBytes, + TargetUser: accessKey, + AccessKey: svcAK, + SecretKey: svcSK, + }) + if err != nil { + c.Fatalf("Unable to create svc acc: %v", err) + } + svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "") + // Though the attached policy does not allow listing, it will be + // ignored because the plugin allows it. + c.mustListObjects(ctx, svcClient, bucket) + } + + // 4. Check that service account's secret key and account status can be + // updated. + c.assertSvcAccSecretKeyAndStatusUpdate(ctx, s, userAdmClient, accessKey, bucket) + + // 5. Check that service account can be deleted. + c.assertSvcAccDeletion(ctx, s, userAdmClient, accessKey, bucket) + + // 6. Check that service account **can** be created for some other user. + // This is possible because the policy enforced in the plugin. + c.mustCreateSvcAccount(ctx, globalActiveCred.AccessKey, userAdmClient) +} + +func (c *check) mustCreateIAMUser(ctx context.Context, admClnt *madmin.AdminClient) madmin.Credentials { + c.Helper() + randUser := mustGetUUID() + randPass := mustGetUUID() + err := admClnt.AddUser(ctx, randUser, randPass) + if err != nil { + c.Fatalf("should be able to create a user: %v", err) + } + return madmin.Credentials{ + AccessKey: randUser, + SecretKey: randPass, + } +} + +func (c *check) mustGetIAMUserInfo(ctx context.Context, admClnt *madmin.AdminClient, accessKey string) madmin.UserInfo { + c.Helper() + ui, err := admClnt.GetUserInfo(ctx, accessKey) + if err != nil { + c.Fatalf("should be able to get user info: %v", err) + } + return ui +} + +func (c *check) mustNotCreateIAMUser(ctx context.Context, admClnt *madmin.AdminClient) { + c.Helper() + randUser := mustGetUUID() + randPass := mustGetUUID() + err := admClnt.AddUser(ctx, randUser, randPass) + if err == nil { + c.Fatalf("should not be able to create a user") + } +} + +func (c *check) mustCreateSvcAccount(ctx context.Context, tgtUser string, admClnt *madmin.AdminClient) madmin.Credentials { + c.Helper() + cr, err := admClnt.AddServiceAccount(ctx, madmin.AddServiceAccountReq{ + TargetUser: tgtUser, + }) + if err != nil { + c.Fatalf("user should be able to create service accounts %s", err) + } + return cr +} + +func (c *check) mustNotCreateSvcAccount(ctx context.Context, tgtUser string, admClnt *madmin.AdminClient) { + c.Helper() + _, err := admClnt.AddServiceAccount(ctx, madmin.AddServiceAccountReq{ + TargetUser: tgtUser, + }) + if err == nil { + c.Fatalf("user was able to add service accounts unexpectedly!") + } +} + +func (c *check) mustNotListObjects(ctx context.Context, client *minio.Client, bucket string) { + c.Helper() + res := client.ListObjects(ctx, bucket, minio.ListObjectsOptions{}) + v, ok := <-res + if !ok || v.Err == nil { + c.Fatalf("user was able to list unexpectedly! on %s", bucket) + } +} + +func (c *check) mustPutObjectWithTags(ctx context.Context, client *minio.Client, bucket, object string) { + c.Helper() + _, err := client.PutObject(ctx, bucket, object, bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{ + UserTags: map[string]string{ + "security": "public", + "virus": "true", + }, + }) + if err != nil { + c.Fatalf("user was unable to upload the object: %v", err) + } +} + +func (c *check) mustGetObject(ctx context.Context, client *minio.Client, bucket, object string) { + c.Helper() + + r, err := client.GetObject(ctx, bucket, object, minio.GetObjectOptions{}) + if err != nil { + c.Fatalf("user was unable to download the object: %v", err) + } + defer r.Close() + + _, err = io.Copy(io.Discard, r) + if err != nil { + c.Fatalf("user was unable to download the object: %v", err) + } +} + +func (c *check) mustHeadObject(ctx context.Context, client *minio.Client, bucket, object string, tagCount int) { + c.Helper() + + oinfo, err := client.StatObject(ctx, bucket, object, minio.StatObjectOptions{}) + if err != nil { + c.Fatalf("user was unable to download the object: %v", err) + } + + if oinfo.UserTagCount != tagCount { + c.Fatalf("expected tagCount: %d, got %d", tagCount, oinfo.UserTagCount) + } +} + +func (c *check) mustListObjects(ctx context.Context, client *minio.Client, bucket string) { + c.Helper() + res := client.ListObjects(ctx, bucket, minio.ListObjectsOptions{}) + v, ok := <-res + if ok && v.Err != nil { + c.Fatalf("user was unable to list: %v", v.Err) + } +} + +func (c *check) mustListBuckets(ctx context.Context, client *minio.Client) { + c.Helper() + _, err := client.ListBuckets(ctx) + if err != nil { + c.Fatalf("user was unable to list buckets: %v", err) + } +} + +func (c *check) mustNotDelete(ctx context.Context, client *minio.Client, bucket string, vid string) { + c.Helper() + + err := client.RemoveObject(ctx, bucket, "some-object", minio.RemoveObjectOptions{VersionID: vid}) + if err == nil { + c.Fatalf("user must not be allowed to delete") + } + + err = client.RemoveObject(ctx, bucket, "some-object", minio.RemoveObjectOptions{}) + if err != nil { + c.Fatal("user must be able to create delete marker") + } +} + +func (c *check) mustDownload(ctx context.Context, client *minio.Client, bucket string) { + c.Helper() + rd, err := client.GetObject(ctx, bucket, "some-object", minio.GetObjectOptions{}) + if err != nil { + c.Fatalf("download did not succeed got %#v", err) + } + if _, err = io.Copy(io.Discard, rd); err != nil { + c.Fatalf("download did not succeed got %#v", err) + } +} + +func (c *check) mustUploadReturnVersions(ctx context.Context, client *minio.Client, bucket string) []string { + c.Helper() + versions := []string{} + for range 5 { + ui, err := client.PutObject(ctx, bucket, "some-object", bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{}) + if err != nil { + c.Fatalf("upload did not succeed got %#v", err) + } + versions = append(versions, ui.VersionID) + } + return versions +} + +func (c *check) mustUpload(ctx context.Context, client *minio.Client, bucket string) { + c.Helper() + _, err := client.PutObject(ctx, bucket, "some-object", bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{}) + if err != nil { + c.Fatalf("upload did not succeed got %#v", err) + } +} + +func (c *check) mustNotUpload(ctx context.Context, client *minio.Client, bucket string) { + c.Helper() + _, err := client.PutObject(ctx, bucket, "some-object", bytes.NewBuffer([]byte("stuff")), 5, minio.PutObjectOptions{}) + if e, ok := err.(minio.ErrorResponse); ok { + if e.Code == "AccessDenied" { + return + } + } + c.Fatalf("upload did not get an AccessDenied error - got %#v instead", err) +} + +func (c *check) assertSvcAccS3Access(ctx context.Context, s *TestSuiteIAM, cr madmin.Credentials, bucket string) { + svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "") + c.mustListObjects(ctx, svcClient, bucket) +} + +func (c *check) assertSvcAccAppearsInListing(ctx context.Context, madmClient *madmin.AdminClient, parentAK, svcAK string) { + c.Helper() + listResp, err := madmClient.ListServiceAccounts(ctx, parentAK) + if err != nil { + c.Fatalf("unable to list svc accounts: %v", err) + } + var accessKeys []string + for _, item := range listResp.Accounts { + accessKeys = append(accessKeys, item.AccessKey) + } + if !set.CreateStringSet(accessKeys...).Contains(svcAK) { + c.Fatalf("service account did not appear in listing!") + } +} + +func (c *check) assertSvcAccInfoQueryable(ctx context.Context, madmClient *madmin.AdminClient, parentAK, svcAK string, skipParentUserCheck bool) { + infoResp, err := madmClient.InfoServiceAccount(ctx, svcAK) + if err != nil { + c.Fatalf("unable to get svc acc info: %v", err) + } + if !skipParentUserCheck { + c.Assert(infoResp.ParentUser, parentAK) + } + c.Assert(infoResp.AccountStatus, "on") + c.Assert(infoResp.ImpliedPolicy, true) +} + +// This test assumes that the policy for `accessKey` allows listing on the given +// bucket. It creates a session policy that restricts listing on the bucket and +// then enables it again in a session policy update call. +func (c *check) assertSvcAccSessionPolicyUpdate(ctx context.Context, s *TestSuiteIAM, madmClient *madmin.AdminClient, accessKey, bucket string) { + c.Helper() + svcAK, svcSK := mustGenerateCredentials(c) + + // This policy does not allow listing objects. + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket) + cr, err := madmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{ + Policy: policyBytes, + TargetUser: accessKey, + AccessKey: svcAK, + SecretKey: svcSK, + }) + if err != nil { + c.Fatalf("Unable to create svc acc: %v", err) + } + svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "") + c.mustNotListObjects(ctx, svcClient, bucket) + + // This policy allows listing objects. + newPolicyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::%s" + ] + } + ] +}`, bucket) + err = madmClient.UpdateServiceAccount(ctx, svcAK, madmin.UpdateServiceAccountReq{ + NewPolicy: newPolicyBytes, + }) + if err != nil { + c.Fatalf("unable to update session policy for svc acc: %v", err) + } + c.mustListObjects(ctx, svcClient, bucket) +} + +func (c *check) assertSvcAccSecretKeyAndStatusUpdate(ctx context.Context, s *TestSuiteIAM, madmClient *madmin.AdminClient, accessKey, bucket string) { + c.Helper() + svcAK, svcSK := mustGenerateCredentials(c) + cr, err := madmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{ + TargetUser: accessKey, + AccessKey: svcAK, + SecretKey: svcSK, + }) + if err != nil { + c.Fatalf("Unable to create svc acc: %v", err) + } + svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "") + c.mustListObjects(ctx, svcClient, bucket) + + _, svcSK2 := mustGenerateCredentials(c) + err = madmClient.UpdateServiceAccount(ctx, svcAK, madmin.UpdateServiceAccountReq{ + NewSecretKey: svcSK2, + }) + if err != nil { + c.Fatalf("unable to update secret key for svc acc: %v", err) + } + // old creds should not work: + c.mustNotListObjects(ctx, svcClient, bucket) + // new creds work: + svcClient2 := s.getUserClient(c, cr.AccessKey, svcSK2, "") + c.mustListObjects(ctx, svcClient2, bucket) + + // update status to disabled + err = madmClient.UpdateServiceAccount(ctx, svcAK, madmin.UpdateServiceAccountReq{ + NewStatus: "off", + }) + if err != nil { + c.Fatalf("unable to update secret key for svc acc: %v", err) + } + c.mustNotListObjects(ctx, svcClient2, bucket) +} + +func (c *check) assertSvcAccDeletion(ctx context.Context, s *TestSuiteIAM, madmClient *madmin.AdminClient, accessKey, bucket string) { + c.Helper() + svcAK, svcSK := mustGenerateCredentials(c) + cr, err := madmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{ + TargetUser: accessKey, + AccessKey: svcAK, + SecretKey: svcSK, + }) + if err != nil { + c.Fatalf("Unable to create svc acc: %v", err) + } + svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "") + c.mustListObjects(ctx, svcClient, bucket) + + err = madmClient.DeleteServiceAccount(ctx, svcAK) + if err != nil { + c.Fatalf("unable to delete svc acc: %v", err) + } + c.mustNotListObjects(ctx, svcClient, bucket) +} + +func mustGenerateCredentials(c *check) (string, string) { + c.Helper() + ak, sk, err := auth.GenerateCredentials() + if err != nil { + c.Fatalf("unable to generate credentials: %v", err) + } + return ak, sk +} diff --git a/cmd/admin-handlers.go b/cmd/admin-handlers.go new file mode 100644 index 00000000..0dcad4cc --- /dev/null +++ b/cmd/admin-handlers.go @@ -0,0 +1,3562 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + crand "crypto/rand" + "crypto/rsa" + "crypto/subtle" + "crypto/x509" + "encoding/base64" + "encoding/json" + "encoding/pem" + "errors" + "fmt" + "hash/crc32" + "io" + "math" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "regexp" + "runtime" + "sort" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/dustin/go-humanize" + "github.com/klauspost/compress/zip" + "github.com/minio/madmin-go/v3" + "github.com/minio/madmin-go/v3/estream" + "github.com/minio/madmin-go/v3/logger/log" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/dsync" + "github.com/minio/minio/internal/grid" + "github.com/minio/minio/internal/handlers" + xhttp "github.com/minio/minio/internal/http" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/kms" + "github.com/minio/minio/internal/logger" + "github.com/minio/mux" + xnet "github.com/minio/pkg/v3/net" + "github.com/minio/pkg/v3/policy" + "github.com/secure-io/sio-go" + "github.com/zeebo/xxh3" +) + +const ( + maxEConfigJSONSize = 262272 + kubernetesVersionEndpoint = "https://kubernetes.default.svc/version" + anonymizeParam = "anonymize" + anonymizeStrict = "strict" +) + +// Only valid query params for mgmt admin APIs. +const ( + mgmtBucket = "bucket" + mgmtPrefix = "prefix" + mgmtClientToken = "clientToken" + mgmtForceStart = "forceStart" + mgmtForceStop = "forceStop" +) + +// ServerUpdateV2Handler - POST /minio/admin/v3/update?updateURL={updateURL}&type=2 +// ---------- +// updates all minio servers and restarts them gracefully. +func (a adminAPIHandlers) ServerUpdateV2Handler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerUpdateAdminAction) + if objectAPI == nil { + return + } + + if globalInplaceUpdateDisabled { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL) + return + } + + if currentReleaseTime.IsZero() || currentReleaseTime.Equal(timeSentinel) { + apiErr := errorCodes.ToAPIErr(ErrMethodNotAllowed) + apiErr.Description = fmt.Sprintf("unable to perform in-place update, release time is unrecognized: %s", currentReleaseTime) + writeErrorResponseJSON(ctx, w, apiErr, r.URL) + return + } + + vars := mux.Vars(r) + updateURL := vars["updateURL"] + dryRun := r.Form.Get("dry-run") == "true" + + mode := getMinioMode() + if updateURL == "" { + updateURL = minioReleaseInfoURL + if runtime.GOOS == globalWindowsOSName { + updateURL = minioReleaseWindowsInfoURL + } + } + + local := globalLocalNodeName + if local == "" { + local = "127.0.0.1" + } + + u, err := url.Parse(updateURL) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + content, err := downloadReleaseURL(u, updateTimeout, mode) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + sha256Sum, lrTime, releaseInfo, err := parseReleaseData(content) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + updateStatus := madmin.ServerUpdateStatusV2{ + DryRun: dryRun, + Results: make([]madmin.ServerPeerUpdateStatus, 0, len(globalNotificationSys.allPeerClients)), + } + peerResults := make(map[string]madmin.ServerPeerUpdateStatus, len(globalNotificationSys.allPeerClients)) + failedClients := make(map[int]bool, len(globalNotificationSys.allPeerClients)) + + if lrTime.Sub(currentReleaseTime) <= 0 { + updateStatus.Results = append(updateStatus.Results, madmin.ServerPeerUpdateStatus{ + Host: local, + Err: fmt.Sprintf("server is running the latest version: %s", Version), + CurrentVersion: Version, + }) + + for _, client := range globalNotificationSys.peerClients { + updateStatus.Results = append(updateStatus.Results, madmin.ServerPeerUpdateStatus{ + Host: client.String(), + Err: fmt.Sprintf("server is running the latest version: %s", Version), + CurrentVersion: Version, + }) + } + + // Marshal API response + jsonBytes, err := json.Marshal(updateStatus) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, jsonBytes) + return + } + + u.Path = path.Dir(u.Path) + SlashSeparator + releaseInfo + // Download Binary Once + binC, bin, err := downloadBinary(u, mode) + if err != nil { + adminLogIf(ctx, fmt.Errorf("server update failed with %w", err)) + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + if globalIsDistErasure { + // Push binary to other servers + for idx, nerr := range globalNotificationSys.VerifyBinary(ctx, u, sha256Sum, releaseInfo, binC) { + if nerr.Err != nil { + peerResults[nerr.Host.String()] = madmin.ServerPeerUpdateStatus{ + Host: nerr.Host.String(), + Err: nerr.Err.Error(), + CurrentVersion: Version, + } + failedClients[idx] = true + } else { + peerResults[nerr.Host.String()] = madmin.ServerPeerUpdateStatus{ + Host: nerr.Host.String(), + CurrentVersion: Version, + UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout), + } + } + } + } + + if err = verifyBinary(u, sha256Sum, releaseInfo, mode, bytes.NewReader(bin)); err != nil { + peerResults[local] = madmin.ServerPeerUpdateStatus{ + Host: local, + Err: err.Error(), + CurrentVersion: Version, + } + } else { + peerResults[local] = madmin.ServerPeerUpdateStatus{ + Host: local, + CurrentVersion: Version, + UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout), + } + } + + if !dryRun { + if globalIsDistErasure { + ng := WithNPeers(len(globalNotificationSys.peerClients)) + for idx, client := range globalNotificationSys.peerClients { + if failedClients[idx] { + continue + } + client := client + ng.Go(ctx, func() error { + return client.CommitBinary(ctx) + }, idx, *client.host) + } + + for _, nerr := range ng.Wait() { + if nerr.Err != nil { + prs, ok := peerResults[nerr.Host.String()] + if ok { + prs.Err = nerr.Err.Error() + peerResults[nerr.Host.String()] = prs + } else { + peerResults[nerr.Host.String()] = madmin.ServerPeerUpdateStatus{ + Host: nerr.Host.String(), + Err: nerr.Err.Error(), + CurrentVersion: Version, + UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout), + } + } + } + } + } + prs := peerResults[local] + if prs.Err == "" { + if err = commitBinary(); err != nil { + prs.Err = err.Error() + } + peerResults[local] = prs + } + } + + prs, ok := peerResults[local] + if ok { + prs.WaitingDrives = waitingDrivesNode() + peerResults[local] = prs + } + + if globalIsDistErasure { + // Notify all other MinIO peers signal service. + startTime := time.Now().Add(restartUpdateDelay) + ng := WithNPeers(len(globalNotificationSys.peerClients)) + for idx, client := range globalNotificationSys.peerClients { + if failedClients[idx] { + continue + } + client := client + ng.Go(ctx, func() error { + prs, ok := peerResults[client.String()] + // We restart only on success, not for any failures. + if ok && prs.Err == "" { + return client.SignalService(serviceRestart, "", dryRun, &startTime) + } + return nil + }, idx, *client.host) + } + + for _, nerr := range ng.Wait() { + if nerr.Err != nil { + waitingDrives := map[string]madmin.DiskMetrics{} + jerr := json.Unmarshal([]byte(nerr.Err.Error()), &waitingDrives) + if jerr == nil { + prs, ok := peerResults[nerr.Host.String()] + if ok { + prs.WaitingDrives = waitingDrives + peerResults[nerr.Host.String()] = prs + } + continue + } + } + } + } + + for _, pr := range peerResults { + updateStatus.Results = append(updateStatus.Results, pr) + } + + // Marshal API response + jsonBytes, err := json.Marshal(updateStatus) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, jsonBytes) + + if !dryRun { + prs, ok := peerResults[local] + // We restart only on success, not for any failures. + if ok && prs.Err == "" { + globalServiceSignalCh <- serviceRestart + } + } +} + +// ServerUpdateHandler - POST /minio/admin/v3/update?updateURL={updateURL} +// ---------- +// updates all minio servers and restarts them gracefully. +func (a adminAPIHandlers) ServerUpdateHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerUpdateAdminAction) + if objectAPI == nil { + return + } + + if globalInplaceUpdateDisabled || currentReleaseTime.IsZero() { + // if MINIO_UPDATE=off - inplace update is disabled, mostly in containers. + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL) + return + } + + vars := mux.Vars(r) + updateURL := vars["updateURL"] + mode := getMinioMode() + if updateURL == "" { + updateURL = minioReleaseInfoURL + if runtime.GOOS == globalWindowsOSName { + updateURL = minioReleaseWindowsInfoURL + } + } + + u, err := url.Parse(updateURL) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + content, err := downloadReleaseURL(u, updateTimeout, mode) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + sha256Sum, lrTime, releaseInfo, err := parseReleaseData(content) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + if lrTime.Sub(currentReleaseTime) <= 0 { + updateStatus := madmin.ServerUpdateStatus{ + CurrentVersion: Version, + UpdatedVersion: Version, + } + + // Marshal API response + jsonBytes, err := json.Marshal(updateStatus) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, jsonBytes) + return + } + + u.Path = path.Dir(u.Path) + SlashSeparator + releaseInfo + + // Download Binary Once + binC, bin, err := downloadBinary(u, mode) + if err != nil { + adminLogIf(ctx, fmt.Errorf("server update failed with %w", err)) + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Push binary to other servers + for _, nerr := range globalNotificationSys.VerifyBinary(ctx, u, sha256Sum, releaseInfo, binC) { + if nerr.Err != nil { + err := AdminError{ + Code: AdminUpdateApplyFailure, + Message: nerr.Err.Error(), + StatusCode: http.StatusInternalServerError, + } + logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String()) + adminLogIf(ctx, fmt.Errorf("server update failed with %w", err)) + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + } + + err = verifyBinary(u, sha256Sum, releaseInfo, mode, bytes.NewReader(bin)) + if err != nil { + adminLogIf(ctx, fmt.Errorf("server update failed with %w", err)) + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + for _, nerr := range globalNotificationSys.CommitBinary(ctx) { + if nerr.Err != nil { + err := AdminError{ + Code: AdminUpdateApplyFailure, + Message: nerr.Err.Error(), + StatusCode: http.StatusInternalServerError, + } + logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String()) + adminLogIf(ctx, fmt.Errorf("server update failed with %w", err)) + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + } + + err = commitBinary() + if err != nil { + adminLogIf(ctx, fmt.Errorf("server update failed with %w", err)) + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + updateStatus := madmin.ServerUpdateStatus{ + CurrentVersion: Version, + UpdatedVersion: lrTime.Format(MinioReleaseTagTimeLayout), + } + + // Marshal API response + jsonBytes, err := json.Marshal(updateStatus) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, jsonBytes) + + // Notify all other MinIO peers signal service. + for _, nerr := range globalNotificationSys.SignalService(serviceRestart) { + if nerr.Err != nil { + logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String()) + adminLogIf(ctx, nerr.Err) + } + } + + globalServiceSignalCh <- serviceRestart +} + +// ServiceHandler - POST /minio/admin/v3/service?action={action} +// ---------- +// Supports following actions: +// - restart (restarts all the MinIO instances in a setup) +// - stop (stops all the MinIO instances in a setup) +// - freeze (freezes all incoming S3 API calls) +// - unfreeze (unfreezes previously frozen S3 API calls) +func (a adminAPIHandlers) ServiceHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + vars := mux.Vars(r) + action := vars["action"] + + var serviceSig serviceSignal + switch madmin.ServiceAction(action) { + case madmin.ServiceActionRestart: + serviceSig = serviceRestart + case madmin.ServiceActionStop: + serviceSig = serviceStop + case madmin.ServiceActionFreeze: + serviceSig = serviceFreeze + case madmin.ServiceActionUnfreeze: + serviceSig = serviceUnFreeze + default: + adminLogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action), logger.ErrorKind) + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL) + return + } + + var objectAPI ObjectLayer + switch serviceSig { + case serviceRestart: + objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceRestartAdminAction) + case serviceStop: + objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceStopAdminAction) + case serviceFreeze, serviceUnFreeze: + objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceFreezeAdminAction) + } + if objectAPI == nil { + return + } + + // Notify all other MinIO peers signal service. + for _, nerr := range globalNotificationSys.SignalService(serviceSig) { + if nerr.Err != nil { + logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String()) + adminLogIf(ctx, nerr.Err) + } + } + + // Reply to the client before restarting, stopping MinIO server. + writeSuccessResponseHeadersOnly(w) + + switch serviceSig { + case serviceFreeze: + freezeServices() + case serviceUnFreeze: + unfreezeServices() + case serviceRestart, serviceStop: + globalServiceSignalCh <- serviceSig + } +} + +type servicePeerResult struct { + Host string `json:"host"` + Err string `json:"err,omitempty"` + WaitingDrives map[string]madmin.DiskMetrics `json:"waitingDrives,omitempty"` +} + +type serviceResult struct { + Action madmin.ServiceAction `json:"action"` + DryRun bool `json:"dryRun"` + Results []servicePeerResult `json:"results,omitempty"` +} + +// ServiceV2Handler - POST /minio/admin/v3/service?action={action}&type=2 +// ---------- +// Supports following actions: +// - restart (restarts all the MinIO instances in a setup) +// - stop (stops all the MinIO instances in a setup) +// - freeze (freezes all incoming S3 API calls) +// - unfreeze (unfreezes previously frozen S3 API calls) +// +// This newer API now returns back status per remote peer and local regarding +// if a "restart/stop" was successful or not. Service signal now supports +// a dry-run that helps skip the nodes that may have hung drives. By default +// restart/stop will ignore the servers that are hung on drives. You can use +// 'force' param to force restart even with hung drives if needed. +func (a adminAPIHandlers) ServiceV2Handler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + vars := mux.Vars(r) + action := vars["action"] + dryRun := r.Form.Get("dry-run") == "true" + + var serviceSig serviceSignal + act := madmin.ServiceAction(action) + switch act { + case madmin.ServiceActionRestart: + serviceSig = serviceRestart + case madmin.ServiceActionStop: + serviceSig = serviceStop + case madmin.ServiceActionFreeze: + serviceSig = serviceFreeze + case madmin.ServiceActionUnfreeze: + serviceSig = serviceUnFreeze + default: + adminLogIf(ctx, fmt.Errorf("Unrecognized service action %s requested", action), logger.ErrorKind) + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL) + return + } + + var objectAPI ObjectLayer + var execAt *time.Time + switch serviceSig { + case serviceRestart: + objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceRestartAdminAction) + t := time.Now().Add(restartUpdateDelay) + execAt = &t + case serviceStop: + objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceStopAdminAction) + case serviceFreeze, serviceUnFreeze: + objectAPI, _ = validateAdminReq(ctx, w, r, policy.ServiceFreezeAdminAction) + } + if objectAPI == nil { + return + } + + // Notify all other MinIO peers signal service. + srvResult := serviceResult{Action: act, Results: []servicePeerResult{}} + + process := act == madmin.ServiceActionRestart || act == madmin.ServiceActionStop + if process { + localhost := globalLocalNodeName + if globalLocalNodeName == "" { + localhost = "127.0.0.1" + } + waitingDrives := waitingDrivesNode() + srvResult.Results = append(srvResult.Results, servicePeerResult{ + Host: localhost, + WaitingDrives: waitingDrives, + }) + } + + if globalIsDistErasure { + for _, nerr := range globalNotificationSys.SignalServiceV2(serviceSig, dryRun, execAt) { + if nerr.Err != nil && process { + waitingDrives := map[string]madmin.DiskMetrics{} + jerr := json.Unmarshal([]byte(nerr.Err.Error()), &waitingDrives) + if jerr == nil { + srvResult.Results = append(srvResult.Results, servicePeerResult{ + Host: nerr.Host.String(), + WaitingDrives: waitingDrives, + }) + continue + } + } + errStr := "" + if nerr.Err != nil { + errStr = nerr.Err.Error() + } + srvResult.Results = append(srvResult.Results, servicePeerResult{ + Host: nerr.Host.String(), + Err: errStr, + }) + } + } + + srvResult.DryRun = dryRun + + buf, err := json.Marshal(srvResult) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Reply to the client before restarting, stopping MinIO server. + writeSuccessResponseJSON(w, buf) + + switch serviceSig { + case serviceFreeze: + freezeServices() + case serviceUnFreeze: + unfreezeServices() + case serviceRestart, serviceStop: + if !dryRun { + globalServiceSignalCh <- serviceSig + } + } +} + +// ServerProperties holds some server information such as, version, region +// uptime, etc.. +type ServerProperties struct { + Uptime int64 `json:"uptime"` + Version string `json:"version"` + CommitID string `json:"commitID"` + DeploymentID string `json:"deploymentID"` + Region string `json:"region"` + SQSARN []string `json:"sqsARN"` +} + +// serverConnStats holds transferred bytes from/to the server +type serverConnStats struct { + internodeInputBytes uint64 + internodeOutputBytes uint64 + s3InputBytes uint64 + s3OutputBytes uint64 +} + +// ServerHTTPAPIStats holds total number of HTTP operations from/to the server, +// including the average duration the call was spent. +type ServerHTTPAPIStats struct { + APIStats map[string]int `json:"apiStats"` +} + +// ServerHTTPStats holds all type of http operations performed to/from the server +// including their average execution time. +type ServerHTTPStats struct { + S3RequestsInQueue int32 `json:"s3RequestsInQueue"` + S3RequestsIncoming uint64 `json:"s3RequestsIncoming"` + CurrentS3Requests ServerHTTPAPIStats `json:"currentS3Requests"` + TotalS3Requests ServerHTTPAPIStats `json:"totalS3Requests"` + TotalS3Errors ServerHTTPAPIStats `json:"totalS3Errors"` + TotalS35xxErrors ServerHTTPAPIStats `json:"totalS35xxErrors"` + TotalS34xxErrors ServerHTTPAPIStats `json:"totalS34xxErrors"` + TotalS3Canceled ServerHTTPAPIStats `json:"totalS3Canceled"` + TotalS3RejectedAuth uint64 `json:"totalS3RejectedAuth"` + TotalS3RejectedTime uint64 `json:"totalS3RejectedTime"` + TotalS3RejectedHeader uint64 `json:"totalS3RejectedHeader"` + TotalS3RejectedInvalid uint64 `json:"totalS3RejectedInvalid"` +} + +// StorageInfoHandler - GET /minio/admin/v3/storageinfo +// ---------- +// Get server information +func (a adminAPIHandlers) StorageInfoHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.StorageInfoAdminAction) + if objectAPI == nil { + return + } + + storageInfo := objectAPI.StorageInfo(ctx, true) + + // Collect any disk healing. + healing, _ := getAggregatedBackgroundHealState(ctx, nil) + healDisks := make(map[string]struct{}, len(healing.HealDisks)) + for _, disk := range healing.HealDisks { + healDisks[disk] = struct{}{} + } + + // find all disks which belong to each respective endpoints + for i, disk := range storageInfo.Disks { + if _, ok := healDisks[disk.Endpoint]; ok { + storageInfo.Disks[i].Healing = true + } + } + + // Marshal API response + jsonBytes, err := json.Marshal(storageInfo) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Reply with storage information (across nodes in a + // distributed setup) as json. + writeSuccessResponseJSON(w, jsonBytes) +} + +// MetricsHandler - GET /minio/admin/v3/metrics +// ---------- +// Get realtime server metrics +func (a adminAPIHandlers) MetricsHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.ServerInfoAdminAction) + if objectAPI == nil { + return + } + const defaultMetricsInterval = time.Second + + interval, err := time.ParseDuration(r.Form.Get("interval")) + if err != nil || interval < time.Second { + interval = defaultMetricsInterval + } + + n, err := strconv.Atoi(r.Form.Get("n")) + if err != nil || n <= 0 { + n = math.MaxInt32 + } + + var types madmin.MetricType + if t, _ := strconv.ParseUint(r.Form.Get("types"), 10, 64); t != 0 { + types = madmin.MetricType(t) + } else { + types = madmin.MetricsAll + } + + disks := strings.Split(r.Form.Get("disks"), ",") + byDisk := strings.EqualFold(r.Form.Get("by-disk"), "true") + var diskMap map[string]struct{} + if len(disks) > 0 && disks[0] != "" { + diskMap = make(map[string]struct{}, len(disks)) + for _, k := range disks { + if k != "" { + diskMap[k] = struct{}{} + } + } + } + jobID := r.Form.Get("by-jobID") + + hosts := strings.Split(r.Form.Get("hosts"), ",") + byHost := strings.EqualFold(r.Form.Get("by-host"), "true") + var hostMap map[string]struct{} + if len(hosts) > 0 && hosts[0] != "" { + hostMap = make(map[string]struct{}, len(hosts)) + for _, k := range hosts { + if k != "" { + hostMap[k] = struct{}{} + } + } + } + dID := r.Form.Get("by-depID") + done := ctx.Done() + ticker := time.NewTicker(interval) + defer ticker.Stop() + w.Header().Set(xhttp.ContentType, string(mimeJSON)) + + enc := json.NewEncoder(w) + for n > 0 { + var m madmin.RealtimeMetrics + mLocal := collectLocalMetrics(types, collectMetricsOpts{ + hosts: hostMap, + disks: diskMap, + jobID: jobID, + depID: dID, + }) + m.Merge(&mLocal) + // Allow half the interval for collecting remote... + cctx, cancel := context.WithTimeout(ctx, interval/2) + mRemote := collectRemoteMetrics(cctx, types, collectMetricsOpts{ + hosts: hostMap, + disks: diskMap, + jobID: jobID, + depID: dID, + }) + cancel() + m.Merge(&mRemote) + if !byHost { + m.ByHost = nil + } + if !byDisk { + m.ByDisk = nil + } + + m.Final = n <= 1 + + // Marshal API reesponse + if err := enc.Encode(&m); err != nil { + n = 0 + } + + n-- + if n <= 0 { + break + } + + // Flush before waiting for next... + xhttp.Flush(w) + + select { + case <-ticker.C: + case <-done: + return + } + } +} + +// DataUsageInfoHandler - GET /minio/admin/v3/datausage?capacity={true} +// ---------- +// Get server/cluster data usage info +func (a adminAPIHandlers) DataUsageInfoHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.DataUsageInfoAdminAction) + if objectAPI == nil { + return + } + + dataUsageInfo, err := loadDataUsageFromBackend(ctx, objectAPI) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + dataUsageInfoJSON, err := json.Marshal(dataUsageInfo) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Get capacity info when asked. + if r.Form.Get("capacity") == "true" { + sinfo := objectAPI.StorageInfo(ctx, false) + dataUsageInfo.TotalCapacity = GetTotalUsableCapacity(sinfo.Disks, sinfo) + dataUsageInfo.TotalFreeCapacity = GetTotalUsableCapacityFree(sinfo.Disks, sinfo) + if dataUsageInfo.TotalCapacity > dataUsageInfo.TotalFreeCapacity { + dataUsageInfo.TotalUsedCapacity = dataUsageInfo.TotalCapacity - dataUsageInfo.TotalFreeCapacity + } + } + + writeSuccessResponseJSON(w, dataUsageInfoJSON) +} + +func lriToLockEntry(l lockRequesterInfo, now time.Time, resource, server string) *madmin.LockEntry { + t := time.Unix(0, l.Timestamp) + entry := &madmin.LockEntry{ + Timestamp: t, + Elapsed: now.Sub(t), + Resource: resource, + ServerList: []string{server}, + Source: l.Source, + Owner: l.Owner, + ID: l.UID, + Quorum: l.Quorum, + } + if l.Writer { + entry.Type = "WRITE" + } else { + entry.Type = "READ" + } + return entry +} + +func topLockEntries(peerLocks []*PeerLocks, stale bool) madmin.LockEntries { + now := time.Now().UTC() + entryMap := make(map[string]*madmin.LockEntry) + toEntry := func(lri lockRequesterInfo) string { + return fmt.Sprintf("%s/%s", lri.Name, lri.UID) + } + for _, peerLock := range peerLocks { + if peerLock == nil { + continue + } + for k, v := range peerLock.Locks { + for _, lockReqInfo := range v { + if val, ok := entryMap[toEntry(lockReqInfo)]; ok { + val.ServerList = append(val.ServerList, peerLock.Addr) + } else { + entryMap[toEntry(lockReqInfo)] = lriToLockEntry(lockReqInfo, now, k, peerLock.Addr) + } + } + } + } + var lockEntries madmin.LockEntries + for _, v := range entryMap { + if stale { + lockEntries = append(lockEntries, *v) + continue + } + if len(v.ServerList) >= v.Quorum { + lockEntries = append(lockEntries, *v) + } + } + sort.Sort(lockEntries) + return lockEntries +} + +// PeerLocks holds server information result of one node +type PeerLocks struct { + Addr string + Locks map[string][]lockRequesterInfo +} + +// ForceUnlockHandler force unlocks requested resource +func (a adminAPIHandlers) ForceUnlockHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.ForceUnlockAdminAction) + if objectAPI == nil { + return + } + + z, ok := objectAPI.(*erasureServerPools) + if !ok { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + + vars := mux.Vars(r) + + var args dsync.LockArgs + var lockers []dsync.NetLocker + for path := range strings.SplitSeq(vars["paths"], ",") { + if path == "" { + continue + } + args.Resources = append(args.Resources, path) + } + + for _, lks := range z.serverPools[0].erasureLockers { + lockers = append(lockers, lks...) + } + + for _, locker := range lockers { + locker.ForceUnlock(ctx, args) + } +} + +// TopLocksHandler Get list of locks in use +func (a adminAPIHandlers) TopLocksHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.TopLocksAdminAction) + if objectAPI == nil { + return + } + + count := 10 // by default list only top 10 entries + if countStr := r.Form.Get("count"); countStr != "" { + var err error + count, err = strconv.Atoi(countStr) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + } + stale := r.Form.Get("stale") == "true" // list also stale locks + + peerLocks := globalNotificationSys.GetLocks(ctx, r) + + topLocks := topLockEntries(peerLocks, stale) + + // Marshal API response upto requested count. + if len(topLocks) > count && count > 0 { + topLocks = topLocks[:count] + } + + jsonBytes, err := json.Marshal(topLocks) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Reply with storage information (across nodes in a + // distributed setup) as json. + writeSuccessResponseJSON(w, jsonBytes) +} + +// StartProfilingResult contains the status of the starting +// profiling action in a given server - deprecated API +type StartProfilingResult struct { + NodeName string `json:"nodeName"` + Success bool `json:"success"` + Error string `json:"error"` +} + +// StartProfilingHandler - POST /minio/admin/v3/profiling/start?profilerType={profilerType} +// ---------- +// Enable server profiling +func (a adminAPIHandlers) StartProfilingHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Validate request signature. + _, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.ProfilingAdminAction, "") + if adminAPIErr != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL) + return + } + + if globalNotificationSys == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + vars := mux.Vars(r) + profiles := strings.Split(vars["profilerType"], ",") + thisAddr, err := xnet.ParseHost(globalLocalNodeName) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + globalProfilerMu.Lock() + defer globalProfilerMu.Unlock() + + if globalProfiler == nil { + globalProfiler = make(map[string]minioProfiler, 10) + } + + // Stop profiler of all types if already running + for k, v := range globalProfiler { + for _, p := range profiles { + if p == k { + v.Stop() + delete(globalProfiler, k) + } + } + } + + // Start profiling on remote servers. + var hostErrs []NotificationPeerErr + for _, profiler := range profiles { + hostErrs = append(hostErrs, globalNotificationSys.StartProfiling(ctx, profiler)...) + + // Start profiling locally as well. + prof, err := startProfiler(profiler) + if err != nil { + hostErrs = append(hostErrs, NotificationPeerErr{ + Host: *thisAddr, + Err: err, + }) + } else { + globalProfiler[profiler] = prof + hostErrs = append(hostErrs, NotificationPeerErr{ + Host: *thisAddr, + }) + } + } + + var startProfilingResult []StartProfilingResult + + for _, nerr := range hostErrs { + result := StartProfilingResult{NodeName: nerr.Host.String()} + if nerr.Err != nil { + result.Error = nerr.Err.Error() + } else { + result.Success = true + } + startProfilingResult = append(startProfilingResult, result) + } + + // Create JSON result and send it to the client + startProfilingResultInBytes, err := json.Marshal(startProfilingResult) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, startProfilingResultInBytes) +} + +// ProfileHandler - POST /minio/admin/v3/profile/?profilerType={profilerType} +// ---------- +// Enable server profiling +func (a adminAPIHandlers) ProfileHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Validate request signature. + _, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.ProfilingAdminAction, "") + if adminAPIErr != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL) + return + } + + if globalNotificationSys == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + profileStr := r.Form.Get("profilerType") + profiles := strings.Split(profileStr, ",") + duration := time.Minute + if dstr := r.Form.Get("duration"); dstr != "" { + var err error + duration, err = time.ParseDuration(dstr) + if err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + } + + globalProfilerMu.Lock() + if globalProfiler == nil { + globalProfiler = make(map[string]minioProfiler, 10) + } + + // Stop profiler of all types if already running + for k, v := range globalProfiler { + v.Stop() + delete(globalProfiler, k) + } + + // Start profiling on remote servers. + for _, profiler := range profiles { + // Limit start time to max 10s. + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + globalNotificationSys.StartProfiling(ctx, profiler) + // StartProfiling blocks, so we can cancel now. + cancel() + + // Start profiling locally as well. + prof, err := startProfiler(profiler) + if err == nil { + globalProfiler[profiler] = prof + } + } + globalProfilerMu.Unlock() + + timer := time.NewTimer(duration) + defer timer.Stop() + for { + select { + case <-ctx.Done(): + // Stop remote profiles + go globalNotificationSys.DownloadProfilingData(GlobalContext, io.Discard) + + // Stop local + globalProfilerMu.Lock() + defer globalProfilerMu.Unlock() + for k, v := range globalProfiler { + v.Stop() + delete(globalProfiler, k) + } + return + case <-timer.C: + if !globalNotificationSys.DownloadProfilingData(ctx, w) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminProfilerNotEnabled), r.URL) + return + } + return + } + } +} + +// dummyFileInfo represents a dummy representation of a profile data file +// present only in memory, it helps to generate the zip stream. +type dummyFileInfo struct { + name string + size int64 + mode os.FileMode + modTime time.Time + isDir bool + sys any +} + +func (f dummyFileInfo) Name() string { return f.name } +func (f dummyFileInfo) Size() int64 { return f.size } +func (f dummyFileInfo) Mode() os.FileMode { return f.mode } +func (f dummyFileInfo) ModTime() time.Time { return f.modTime } +func (f dummyFileInfo) IsDir() bool { return f.isDir } +func (f dummyFileInfo) Sys() any { return f.sys } + +// DownloadProfilingHandler - POST /minio/admin/v3/profiling/download +// ---------- +// Download profiling information of all nodes in a zip format - deprecated API +func (a adminAPIHandlers) DownloadProfilingHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Validate request signature. + _, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.ProfilingAdminAction, "") + if adminAPIErr != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL) + return + } + + if globalNotificationSys == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if !globalNotificationSys.DownloadProfilingData(ctx, w) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAdminProfilerNotEnabled), r.URL) + return + } +} + +type healInitParams struct { + bucket, objPrefix string + hs madmin.HealOpts + clientToken string + forceStart, forceStop bool +} + +// extractHealInitParams - Validates params for heal init API. +func extractHealInitParams(vars map[string]string, qParams url.Values, r io.Reader) (hip healInitParams, err APIErrorCode) { + hip.bucket = vars[mgmtBucket] + hip.objPrefix = vars[mgmtPrefix] + + if hip.bucket == "" { + if hip.objPrefix != "" { + // Bucket is required if object-prefix is given + err = ErrHealMissingBucket + return hip, err + } + } else if isReservedOrInvalidBucket(hip.bucket, false) { + err = ErrInvalidBucketName + return hip, err + } + + // empty prefix is valid. + if !IsValidObjectPrefix(hip.objPrefix) { + err = ErrInvalidObjectName + return hip, err + } + + if len(qParams[mgmtClientToken]) > 0 { + hip.clientToken = qParams[mgmtClientToken][0] + } + if _, ok := qParams[mgmtForceStart]; ok { + hip.forceStart = true + } + if _, ok := qParams[mgmtForceStop]; ok { + hip.forceStop = true + } + + // Invalid request conditions: + // + // Cannot have both forceStart and forceStop in the same + // request; If clientToken is provided, request can only be + // to continue receiving logs, so it cannot be start or + // stop; + if (hip.forceStart && hip.forceStop) || + (hip.clientToken != "" && (hip.forceStart || hip.forceStop)) { + err = ErrInvalidRequest + return hip, err + } + + // ignore body if clientToken is provided + if hip.clientToken == "" { + jerr := json.NewDecoder(r).Decode(&hip.hs) + if jerr != nil { + adminLogIf(GlobalContext, jerr, logger.ErrorKind) + err = ErrRequestBodyParse + return hip, err + } + } + + err = ErrNone + return hip, err +} + +// HealHandler - POST /minio/admin/v3/heal/ +// ----------- +// Start heal processing and return heal status items. +// +// On a successful heal sequence start, a unique client token is +// returned. Subsequent requests to this endpoint providing the client +// token will receive heal status records from the running heal +// sequence. +// +// If no client token is provided, and a heal sequence is in progress +// an error is returned with information about the running heal +// sequence. However, if the force-start flag is provided, the server +// aborts the running heal sequence and starts a new one. +func (a adminAPIHandlers) HealHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealAdminAction) + if objectAPI == nil { + return + } + + hip, errCode := extractHealInitParams(mux.Vars(r), r.Form, r.Body) + if errCode != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL) + return + } + + // Analyze the heal token and route the request accordingly + token, _, success := proxyRequestByToken(ctx, w, r, hip.clientToken, false) + if success { + return + } + hip.clientToken = token + // if request was not successful, try this server locally if token + // is not found the call will fail anyways. if token is empty + // try this server to generate a new token. + + type healResp struct { + respBytes []byte + apiErr APIError + errBody string + } + + // Define a closure to start sending whitespace to client + // after 10s unless a response item comes in + keepConnLive := func(w http.ResponseWriter, r *http.Request, respCh chan healResp) { + ticker := time.NewTicker(time.Second * 10) + defer ticker.Stop() + started := false + forLoop: + for { + select { + case <-r.Context().Done(): + return + case <-ticker.C: + if !started { + // Start writing response to client + started = true + setCommonHeaders(w) + setEventStreamHeaders(w) + // Set 200 OK status + w.WriteHeader(200) + } + // Send whitespace and keep connection open + if _, err := w.Write([]byte(" ")); err != nil { + return + } + xhttp.Flush(w) + case hr := <-respCh: + switch hr.apiErr { + case noError: + if started { + if _, err := w.Write(hr.respBytes); err != nil { + return + } + xhttp.Flush(w) + } else { + writeSuccessResponseJSON(w, hr.respBytes) + } + default: + var errorRespJSON []byte + if hr.errBody == "" { + errorRespJSON = encodeResponseJSON(getAPIErrorResponse(ctx, hr.apiErr, + r.URL.Path, w.Header().Get(xhttp.AmzRequestID), + w.Header().Get(xhttp.AmzRequestHostID))) + } else { + errorRespJSON = encodeResponseJSON(APIErrorResponse{ + Code: hr.apiErr.Code, + Message: hr.errBody, + Resource: r.URL.Path, + RequestID: w.Header().Get(xhttp.AmzRequestID), + HostID: globalDeploymentID(), + }) + } + if !started { + setCommonHeaders(w) + w.Header().Set(xhttp.ContentType, string(mimeJSON)) + w.WriteHeader(hr.apiErr.HTTPStatusCode) + } + if _, err := w.Write(errorRespJSON); err != nil { + return + } + xhttp.Flush(w) + } + break forLoop + } + } + } + + healPath := pathJoin(hip.bucket, hip.objPrefix) + if hip.clientToken == "" && !hip.forceStart && !hip.forceStop { + nh, exists := globalAllHealState.getHealSequence(healPath) + if exists && !nh.hasEnded() && len(nh.currentStatus.Items) > 0 { + clientToken := nh.clientToken + if globalIsDistErasure { + clientToken = fmt.Sprintf("%s%s%d", nh.clientToken, getKeySeparator(), GetProxyEndpointLocalIndex(globalProxyEndpoints)) + } + b, err := json.Marshal(madmin.HealStartSuccess{ + ClientToken: clientToken, + ClientAddress: nh.clientAddress, + StartTime: nh.startTime, + }) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + // Client token not specified but a heal sequence exists on a path, + // Send the token back to client. + writeSuccessResponseJSON(w, b) + return + } + } + + if hip.clientToken != "" && !hip.forceStart && !hip.forceStop { + // Since clientToken is given, fetch heal status from running + // heal sequence. + respBytes, errCode := globalAllHealState.PopHealStatusJSON( + healPath, hip.clientToken) + if errCode != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(errCode), r.URL) + } else { + writeSuccessResponseJSON(w, respBytes) + } + return + } + + respCh := make(chan healResp, 1) + switch { + case hip.forceStop: + go func() { + respBytes, apiErr := globalAllHealState.stopHealSequence(healPath) + hr := healResp{respBytes: respBytes, apiErr: apiErr} + respCh <- hr + }() + case hip.clientToken == "": + nh := newHealSequence(GlobalContext, hip.bucket, hip.objPrefix, handlers.GetSourceIP(r), hip.hs, hip.forceStart) + go func() { + respBytes, apiErr, errMsg := globalAllHealState.LaunchNewHealSequence(nh, objectAPI) + hr := healResp{respBytes, apiErr, errMsg} + respCh <- hr + }() + } + + // Due to the force-starting functionality, the Launch + // call above can take a long time - to keep the + // connection alive, we start sending whitespace + keepConnLive(w, r, respCh) +} + +// getAggregatedBackgroundHealState returns the heal state of disks. +// If no ObjectLayer is provided no set status is returned. +func getAggregatedBackgroundHealState(ctx context.Context, o ObjectLayer) (madmin.BgHealState, error) { + // Get local heal status first + bgHealStates, ok := getLocalBackgroundHealStatus(ctx, o) + if !ok { + return bgHealStates, errServerNotInitialized + } + + if globalIsDistErasure { + // Get heal status from other peers + peersHealStates, nerrs := globalNotificationSys.BackgroundHealStatus(ctx) + var errCount int + for _, nerr := range nerrs { + if nerr.Err != nil { + adminLogIf(ctx, nerr.Err) + errCount++ + } + } + if errCount == len(nerrs) { + return madmin.BgHealState{}, fmt.Errorf("all remote servers failed to report heal status, cluster is unhealthy") + } + bgHealStates.Merge(peersHealStates...) + } + + return bgHealStates, nil +} + +func (a adminAPIHandlers) BackgroundHealStatusHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealAdminAction) + if objectAPI == nil { + return + } + + aggregateHealStateResult, err := getAggregatedBackgroundHealState(r.Context(), objectAPI) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + if err := json.NewEncoder(w).Encode(aggregateHealStateResult); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } +} + +// SitePerfHandler - measures network throughput between site replicated setups +func (a adminAPIHandlers) SitePerfHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction) + if objectAPI == nil { + return + } + + if !globalSiteReplicationSys.isEnabled() { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + + nsLock := objectAPI.NewNSLock(minioMetaBucket, "site-net-perf") + lkctx, err := nsLock.GetLock(ctx, globalOperationTimeout) + if err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(toAPIErrorCode(ctx, err)), r.URL) + return + } + ctx = lkctx.Context() + defer nsLock.Unlock(lkctx) + + durationStr := r.Form.Get(peerRESTDuration) + duration, err := time.ParseDuration(durationStr) + if err != nil { + duration = globalNetPerfMinDuration + } + + if duration < globalNetPerfMinDuration { + // We need sample size of minimum 10 secs. + duration = globalNetPerfMinDuration + } + + duration = duration.Round(time.Second) + + results, err := globalSiteReplicationSys.Netperf(ctx, duration) + if err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(toAPIErrorCode(ctx, err)), r.URL) + return + } + enc := json.NewEncoder(w) + if err := enc.Encode(results); err != nil { + return + } +} + +// ClientDevNullExtraTime - return extratime for last devnull +// [POST] /minio/admin/v3/speedtest/client/devnull/extratime +func (a adminAPIHandlers) ClientDevNullExtraTime(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.BandwidthMonitorAction) + if objectAPI == nil { + return + } + + enc := json.NewEncoder(w) + if err := enc.Encode(madmin.ClientPerfExtraTime{TimeSpent: atomic.LoadInt64(&globalLastClientPerfExtraTime)}); err != nil { + return + } +} + +// ClientDevNull - everything goes to io.Discard +// [POST] /minio/admin/v3/speedtest/client/devnull +func (a adminAPIHandlers) ClientDevNull(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + timeStart := time.Now() + objectAPI, _ := validateAdminReq(ctx, w, r, policy.BandwidthMonitorAction) + if objectAPI == nil { + return + } + + nsLock := objectAPI.NewNSLock(minioMetaBucket, "client-perf") + lkctx, err := nsLock.GetLock(ctx, globalOperationTimeout) + if err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(toAPIErrorCode(ctx, err)), r.URL) + return + } + ctx = lkctx.Context() + defer nsLock.Unlock(lkctx) + timeEnd := time.Now() + + atomic.SwapInt64(&globalLastClientPerfExtraTime, timeEnd.Sub(timeStart).Nanoseconds()) + + ctx, cancel := context.WithTimeout(ctx, madmin.MaxClientPerfTimeout) + defer cancel() + totalRx := int64(0) + connectTime := time.Now() + for { + n, err := io.CopyN(xioutil.Discard, r.Body, 128*humanize.KiByte) + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { + // would mean the network is not stable. Logging here will help in debugging network issues. + if time.Since(connectTime) < (globalNetPerfMinDuration - time.Second) { + adminLogIf(ctx, err) + } + } + totalRx += n + if err != nil || ctx.Err() != nil || totalRx > 100*humanize.GiByte { + break + } + } + w.WriteHeader(http.StatusOK) +} + +// NetperfHandler - perform mesh style network throughput test +func (a adminAPIHandlers) NetperfHandler(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction) + if objectAPI == nil { + return + } + + if !globalIsDistErasure { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + + nsLock := objectAPI.NewNSLock(minioMetaBucket, "netperf") + lkctx, err := nsLock.GetLock(ctx, globalOperationTimeout) + if err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(toAPIErrorCode(ctx, err)), r.URL) + return + } + ctx = lkctx.Context() + defer nsLock.Unlock(lkctx) + + // Freeze all incoming S3 API calls before running speedtest. + globalNotificationSys.ServiceFreeze(ctx, true) + + // Unfreeze as soon as request context is canceled or when the function returns. + go func() { + <-ctx.Done() + globalNotificationSys.ServiceFreeze(ctx, false) + }() + + durationStr := r.Form.Get(peerRESTDuration) + duration, err := time.ParseDuration(durationStr) + if err != nil { + duration = globalNetPerfMinDuration + } + + if duration < globalNetPerfMinDuration { + // We need sample size of minimum 10 secs. + duration = globalNetPerfMinDuration + } + + duration = duration.Round(time.Second) + + results := globalNotificationSys.Netperf(ctx, duration) + enc := json.NewEncoder(w) + if err := enc.Encode(madmin.NetperfResult{NodeResults: results}); err != nil { + return + } +} + +func isAllowedRWAccess(r *http.Request, cred auth.Credentials, bucketName string) (rd, wr bool) { + owner := cred.AccessKey == globalActiveCred.AccessKey + + // Set prefix value for "s3:prefix" policy conditionals. + r.Header.Set("prefix", "") + + // Set delimiter value for "s3:delimiter" policy conditionals. + r.Header.Set("delimiter", SlashSeparator) + + isAllowedAccess := func(bucketName string) (rd, wr bool) { + if globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.GetObjectAction, + BucketName: bucketName, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + ObjectName: "", + Claims: cred.Claims, + }) { + rd = true + } + + if globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.PutObjectAction, + BucketName: bucketName, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + ObjectName: "", + Claims: cred.Claims, + }) { + wr = true + } + + return rd, wr + } + return isAllowedAccess(bucketName) +} + +// ObjectSpeedTestHandler - reports maximum speed of a cluster by performing PUT and +// GET operations on the server, supports auto tuning by default by automatically +// increasing concurrency and stopping when we have reached the limits on the +// system. +func (a adminAPIHandlers) ObjectSpeedTestHandler(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + + objectAPI, creds := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction) + if objectAPI == nil { + return + } + + if !globalAPIConfig.permitRootAccess() { + rd, wr := isAllowedRWAccess(r, creds, globalObjectPerfBucket) + if !rd || !wr { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, AdminError{ + Code: "XMinioSpeedtestInsufficientPermissions", + Message: fmt.Sprintf("%s does not have read and write access to '%s' bucket", creds.AccessKey, + globalObjectPerfBucket), + StatusCode: http.StatusForbidden, + }), r.URL) + return + } + } + + sizeStr := r.Form.Get(peerRESTSize) + durationStr := r.Form.Get(peerRESTDuration) + concurrentStr := r.Form.Get(peerRESTConcurrent) + storageClass := strings.TrimSpace(r.Form.Get(peerRESTStorageClass)) + customBucket := strings.TrimSpace(r.Form.Get(peerRESTBucket)) + autotune := r.Form.Get("autotune") == "true" + noClear := r.Form.Get("noclear") == "true" + enableSha256 := r.Form.Get("enableSha256") == "true" + enableMultipart := r.Form.Get("enableMultipart") == "true" + + size, err := strconv.Atoi(sizeStr) + if err != nil { + size = 64 * humanize.MiByte + } + + concurrent, err := strconv.Atoi(concurrentStr) + if err != nil { + concurrent = 32 + } + + duration, err := time.ParseDuration(durationStr) + if err != nil { + duration = time.Second * 10 + } + + storageInfo := objectAPI.StorageInfo(ctx, false) + + sufficientCapacity, canAutotune, capacityErrMsg := validateObjPerfOptions(ctx, storageInfo, concurrent, size, autotune) + if !sufficientCapacity { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, AdminError{ + Code: "XMinioSpeedtestInsufficientCapacity", + Message: capacityErrMsg, + StatusCode: http.StatusInsufficientStorage, + }), r.URL) + return + } + + if autotune && !canAutotune { + autotune = false + } + + if customBucket == "" { + customBucket = globalObjectPerfBucket + + bucketExists, err := makeObjectPerfBucket(ctx, objectAPI, customBucket) + if err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if !noClear && !bucketExists { + defer deleteObjectPerfBucket(objectAPI) + } + } + + if !noClear { + defer objectAPI.DeleteObject(ctx, customBucket, speedTest+SlashSeparator, ObjectOptions{ + DeletePrefix: true, + }) + } + + // Freeze all incoming S3 API calls before running speedtest. + globalNotificationSys.ServiceFreeze(ctx, true) + + // Unfreeze as soon as request context is canceled or when the function returns. + go func() { + <-ctx.Done() + globalNotificationSys.ServiceFreeze(ctx, false) + }() + + keepAliveTicker := time.NewTicker(500 * time.Millisecond) + defer keepAliveTicker.Stop() + + enc := json.NewEncoder(w) + ch := objectSpeedTest(ctx, speedTestOpts{ + objectSize: size, + concurrencyStart: concurrent, + duration: duration, + autotune: autotune, + storageClass: storageClass, + bucketName: customBucket, + enableSha256: enableSha256, + enableMultipart: enableMultipart, + creds: creds, + }) + var prevResult madmin.SpeedTestResult + for { + select { + case <-ctx.Done(): + return + case <-keepAliveTicker.C: + // if previous result is set keep writing the + // previous result back to the client + if prevResult.Version != "" { + if err := enc.Encode(prevResult); err != nil { + return + } + } else { + // first result is not yet obtained, keep writing + // empty entry to prevent client from disconnecting. + if err := enc.Encode(madmin.SpeedTestResult{}); err != nil { + return + } + } + xhttp.Flush(w) + case result, ok := <-ch: + if !ok { + return + } + if err := enc.Encode(result); err != nil { + return + } + prevResult = result + xhttp.Flush(w) + } + } +} + +func makeObjectPerfBucket(ctx context.Context, objectAPI ObjectLayer, bucketName string) (bucketExists bool, err error) { + if err = objectAPI.MakeBucket(ctx, bucketName, MakeBucketOptions{VersioningEnabled: globalSiteReplicationSys.isEnabled()}); err != nil { + if _, ok := err.(BucketExists); !ok { + // Only BucketExists error can be ignored. + return false, err + } + bucketExists = true + } + + if globalSiteReplicationSys.isEnabled() { + configData := []byte(`Enabledspeedtest/*`) + if _, err = globalBucketMetadataSys.Update(ctx, bucketName, bucketVersioningConfig, configData); err != nil { + return false, err + } + } + + return bucketExists, nil +} + +func deleteObjectPerfBucket(objectAPI ObjectLayer) { + objectAPI.DeleteBucket(context.Background(), globalObjectPerfBucket, DeleteBucketOptions{ + Force: true, + SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()), + }) +} + +func validateObjPerfOptions(ctx context.Context, storageInfo madmin.StorageInfo, concurrent int, size int, autotune bool) (bool, bool, string) { + capacityNeeded := uint64(concurrent * size) + capacity := GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo) + + if capacity < capacityNeeded { + return false, false, fmt.Sprintf("not enough usable space available to perform speedtest - expected %s, got %s", + humanize.IBytes(capacityNeeded), humanize.IBytes(capacity)) + } + + // Verify if we can employ autotune without running out of capacity, + // if we do run out of capacity, make sure to turn-off autotuning + // in such situations. + if autotune { + newConcurrent := concurrent + (concurrent+1)/2 + autoTunedCapacityNeeded := uint64(newConcurrent * size) + if capacity < autoTunedCapacityNeeded { + // Turn-off auto-tuning if next possible concurrency would reach beyond disk capacity. + return true, false, "" + } + } + + return true, autotune, "" +} + +// DriveSpeedtestHandler - reports throughput of drives available in the cluster +func (a adminAPIHandlers) DriveSpeedtestHandler(w http.ResponseWriter, r *http.Request) { + ctx, cancel := context.WithCancel(r.Context()) + defer cancel() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction) + if objectAPI == nil { + return + } + + // Freeze all incoming S3 API calls before running speedtest. + globalNotificationSys.ServiceFreeze(ctx, true) + + // Unfreeze as soon as request context is canceled or when the function returns. + go func() { + <-ctx.Done() + globalNotificationSys.ServiceFreeze(ctx, false) + }() + + serial := r.Form.Get("serial") == "true" + blockSizeStr := r.Form.Get("blocksize") + fileSizeStr := r.Form.Get("filesize") + + blockSize, err := strconv.ParseUint(blockSizeStr, 10, 64) + if err != nil { + blockSize = 4 * humanize.MiByte // default value + } + + fileSize, err := strconv.ParseUint(fileSizeStr, 10, 64) + if err != nil { + fileSize = 1 * humanize.GiByte // default value + } + + opts := madmin.DriveSpeedTestOpts{ + Serial: serial, + BlockSize: blockSize, + FileSize: fileSize, + } + + keepAliveTicker := time.NewTicker(500 * time.Millisecond) + defer keepAliveTicker.Stop() + + ch := globalNotificationSys.DriveSpeedTest(ctx, opts) + + enc := json.NewEncoder(w) + for { + select { + case <-ctx.Done(): + return + case <-keepAliveTicker.C: + // Write a blank entry to prevent client from disconnecting + if err := enc.Encode(madmin.DriveSpeedTestResult{}); err != nil { + return + } + xhttp.Flush(w) + case result, ok := <-ch: + if !ok { + return + } + if err := enc.Encode(result); err != nil { + return + } + xhttp.Flush(w) + } + } +} + +// Admin API errors +const ( + AdminUpdateUnexpectedFailure = "XMinioAdminUpdateUnexpectedFailure" + AdminUpdateURLNotReachable = "XMinioAdminUpdateURLNotReachable" + AdminUpdateApplyFailure = "XMinioAdminUpdateApplyFailure" +) + +// Returns true if the madmin.TraceInfo should be traced, +// false if certain conditions are not met. +// - input entry is not of the type *madmin.TraceInfo* +// - errOnly entries are to be traced, not status code 2xx, 3xx. +// - madmin.TraceInfo type is asked by opts +func shouldTrace(trcInfo madmin.TraceInfo, opts madmin.ServiceTraceOpts) (shouldTrace bool) { + // Reject all unwanted types. + want := opts.TraceTypes() + if !want.Contains(trcInfo.TraceType) { + return false + } + + isHTTP := trcInfo.TraceType.Overlaps(madmin.TraceInternal|madmin.TraceS3) && trcInfo.HTTP != nil + + // Check latency... + if opts.Threshold > 0 && trcInfo.Duration < opts.Threshold { + return false + } + + // Check internal path + isInternal := isHTTP && HasPrefix(trcInfo.HTTP.ReqInfo.Path, minioReservedBucketPath+SlashSeparator) + if isInternal && !opts.Internal { + return false + } + + // Filter non-errors. + if isHTTP && opts.OnlyErrors && trcInfo.HTTP.RespInfo.StatusCode < http.StatusBadRequest { + return false + } + + return true +} + +func extractTraceOptions(r *http.Request) (opts madmin.ServiceTraceOpts, err error) { + if err := opts.ParseParams(r); err != nil { + return opts, err + } + // Support deprecated 'all' query + if r.Form.Get("all") == "true" { + opts.S3 = true + opts.Internal = true + opts.Storage = true + opts.OS = true + // Older mc - cannot deal with more types... + } + return opts, err +} + +// TraceHandler - POST /minio/admin/v3/trace +// ---------- +// The handler sends http trace to the connected HTTP client. +func (a adminAPIHandlers) TraceHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Validate request signature. + _, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.TraceAdminAction, "") + if adminAPIErr != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL) + return + } + + traceOpts, err := extractTraceOptions(r) + if err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + setEventStreamHeaders(w) + + // Trace Publisher and peer-trace-client uses nonblocking send and hence does not wait for slow receivers. + // Keep 100k buffered channel. + // If receiver cannot keep up with that we drop events. + traceCh := make(chan []byte, 100000) + peers, _ := newPeerRestClients(globalEndpoints) + err = globalTrace.SubscribeJSON(traceOpts.TraceTypes(), traceCh, ctx.Done(), func(entry madmin.TraceInfo) bool { + return shouldTrace(entry, traceOpts) + }, nil) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Publish bootstrap events that have already occurred before client could subscribe. + if traceOpts.TraceTypes().Contains(madmin.TraceBootstrap) { + go globalBootstrapTracer.Publish(ctx, globalTrace) + } + + for _, peer := range peers { + if peer == nil { + continue + } + peer.Trace(ctx, traceCh, traceOpts) + } + + keepAliveTicker := time.NewTicker(time.Second) + defer keepAliveTicker.Stop() + + for { + select { + case entry := <-traceCh: + if _, err := w.Write(entry); err != nil { + return + } + grid.PutByteBuffer(entry) + if len(traceCh) == 0 { + // Flush if nothing is queued + xhttp.Flush(w) + } + case <-keepAliveTicker.C: + if len(traceCh) > 0 { + continue + } + if _, err := w.Write([]byte(" ")); err != nil { + return + } + xhttp.Flush(w) + case <-ctx.Done(): + return + } + } +} + +// The ConsoleLogHandler handler sends console logs to the connected HTTP client. +func (a adminAPIHandlers) ConsoleLogHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + objectAPI, _ := validateAdminReq(ctx, w, r, policy.ConsoleLogAdminAction) + if objectAPI == nil { + return + } + node := r.Form.Get("node") + // limit buffered console entries if client requested it. + limitStr := r.Form.Get("limit") + limitLines, err := strconv.Atoi(limitStr) + if err != nil { + limitLines = 10 + } + + logKind := madmin.LogKind(strings.ToUpper(r.Form.Get("logType"))).LogMask() + if logKind == 0 { + logKind = madmin.LogMaskAll + } + + // Avoid reusing tcp connection if read timeout is hit + // This is needed to make r.Context().Done() work as + // expected in case of read timeout + w.Header().Set("Connection", "close") + + setEventStreamHeaders(w) + + logCh := make(chan log.Info, 1000) + peers, _ := newPeerRestClients(globalEndpoints) + encodedCh := make(chan []byte, 1000+len(peers)*1000) + err = globalConsoleSys.Subscribe(logCh, ctx.Done(), node, limitLines, logKind, nil) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + // Convert local entries to JSON + go func() { + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + for { + select { + case <-ctx.Done(): + return + case li := <-logCh: + if !li.SendLog(node, logKind) { + continue + } + buf.Reset() + if err := enc.Encode(li); err != nil { + continue + } + select { + case <-ctx.Done(): + return + case encodedCh <- append(grid.GetByteBuffer()[:0], buf.Bytes()...): + } + } + } + }() + + // Collect from matching peers + for _, peer := range peers { + if peer == nil { + continue + } + if node == "" || strings.EqualFold(peer.host.Name, node) { + peer.ConsoleLog(ctx, logKind, encodedCh) + } + } + + keepAliveTicker := time.NewTicker(500 * time.Millisecond) + defer keepAliveTicker.Stop() + for { + select { + case log, ok := <-encodedCh: + if !ok { + return + } + _, err = w.Write(log) + if err != nil { + return + } + grid.PutByteBuffer(log) + if len(logCh) == 0 { + // Flush if nothing is queued + xhttp.Flush(w) + } + case <-keepAliveTicker.C: + if len(logCh) > 0 { + continue + } + if _, err := w.Write([]byte(" ")); err != nil { + return + } + xhttp.Flush(w) + case <-ctx.Done(): + return + } + } +} + +// KMSCreateKeyHandler - POST /minio/admin/v3/kms/key/create?key-id= +func (a adminAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSCreateKeyAdminAction) + if objectAPI == nil { + return + } + + if GlobalKMS == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL) + return + } + + if err := GlobalKMS.CreateKey(ctx, &kms.CreateKeyRequest{ + Name: r.Form.Get("key-id"), + }); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + writeSuccessResponseHeadersOnly(w) +} + +// KMSStatusHandler - GET /minio/admin/v3/kms/status +func (a adminAPIHandlers) KMSStatusHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSKeyStatusAdminAction) + if objectAPI == nil { + return + } + + if GlobalKMS == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL) + return + } + + stat, err := GlobalKMS.Status(ctx) + if err != nil { + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL) + return + } + resp, err := json.Marshal(stat) + if err != nil { + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL) + return + } + writeSuccessResponseJSON(w, resp) +} + +// KMSKeyStatusHandler - GET /minio/admin/v3/kms/key/status?key-id= +func (a adminAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSKeyStatusAdminAction) + if objectAPI == nil { + return + } + + if GlobalKMS == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL) + return + } + + keyID := r.Form.Get("key-id") + if keyID == "" { + keyID = GlobalKMS.DefaultKey + } + response := madmin.KMSKeyStatus{ + KeyID: keyID, + } + + kmsContext := kms.Context{"MinIO admin API": "KMSKeyStatusHandler"} // Context for a test key operation + // 1. Generate a new key using the KMS. + key, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{ + Name: keyID, + AssociatedData: kmsContext, + }) + if err != nil { + response.EncryptionErr = err.Error() + resp, err := json.Marshal(response) + if err != nil { + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL) + return + } + writeSuccessResponseJSON(w, resp) + return + } + + // 2. Verify that we can indeed decrypt the (encrypted) key + decryptedKey, err := GlobalKMS.Decrypt(ctx, &kms.DecryptRequest{ + Name: key.KeyID, + Ciphertext: key.Ciphertext, + AssociatedData: kmsContext, + }) + if err != nil { + response.DecryptionErr = err.Error() + resp, err := json.Marshal(response) + if err != nil { + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL) + return + } + writeSuccessResponseJSON(w, resp) + return + } + + // 3. Compare generated key with decrypted key + if subtle.ConstantTimeCompare(key.Plaintext, decryptedKey) != 1 { + response.DecryptionErr = "The generated and the decrypted data key do not match" + resp, err := json.Marshal(response) + if err != nil { + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL) + return + } + writeSuccessResponseJSON(w, resp) + return + } + + resp, err := json.Marshal(response) + if err != nil { + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL) + return + } + writeSuccessResponseJSON(w, resp) +} + +func getPoolsInfo(ctx context.Context, allDisks []madmin.Disk) (map[int]map[int]madmin.ErasureSetInfo, error) { + objectAPI := newObjectLayerFn() + if objectAPI == nil { + return nil, errServerNotInitialized + } + + z, ok := objectAPI.(*erasureServerPools) + if !ok { + return nil, errServerNotInitialized + } + + poolsInfo := make(map[int]map[int]madmin.ErasureSetInfo) + for _, d := range allDisks { + poolInfo, ok := poolsInfo[d.PoolIndex] + if !ok { + poolInfo = make(map[int]madmin.ErasureSetInfo) + } + erasureSet, ok := poolInfo[d.SetIndex] + if !ok { + erasureSet.ID = d.SetIndex + cache := dataUsageCache{} + if err := cache.load(ctx, z.serverPools[d.PoolIndex].sets[d.SetIndex], dataUsageCacheName); err == nil { + dataUsageInfo := cache.dui(dataUsageRoot, nil) + erasureSet.ObjectsCount = dataUsageInfo.ObjectsTotalCount + erasureSet.VersionsCount = dataUsageInfo.VersionsTotalCount + erasureSet.DeleteMarkersCount = dataUsageInfo.DeleteMarkersTotalCount + erasureSet.Usage = dataUsageInfo.ObjectsTotalSize + } + } + erasureSet.RawCapacity += d.TotalSpace + erasureSet.RawUsage += d.UsedSpace + if d.Healing { + erasureSet.HealDisks = 1 + } + poolInfo[d.SetIndex] = erasureSet + poolsInfo[d.PoolIndex] = poolInfo + } + return poolsInfo, nil +} + +func getServerInfo(ctx context.Context, pools, metrics bool, r *http.Request) madmin.InfoMessage { + const operationTimeout = 10 * time.Second + ldap := madmin.LDAP{} + if globalIAMSys.LDAPConfig.Enabled() { + ldapConn, err := globalIAMSys.LDAPConfig.LDAP.Connect() + //nolint:gocritic + if err != nil { + ldap.Status = string(madmin.ItemOffline) + } else if ldapConn == nil { + ldap.Status = "Not Configured" + } else { + // Close ldap connection to avoid leaks. + ldapConn.Close() + ldap.Status = string(madmin.ItemOnline) + } + } + + log, audit := fetchLoggerInfo(ctx) + + // Get the notification target info + notifyTarget := fetchLambdaInfo() + + local := getLocalServerProperty(globalEndpoints, r, metrics) + servers := globalNotificationSys.ServerInfo(ctx, metrics) + servers = append(servers, local) + + var poolsInfo map[int]map[int]madmin.ErasureSetInfo + var backend madmin.ErasureBackend + + mode := madmin.ItemInitializing + + buckets := madmin.Buckets{} + objects := madmin.Objects{} + versions := madmin.Versions{} + deleteMarkers := madmin.DeleteMarkers{} + usage := madmin.Usage{} + + objectAPI := newObjectLayerFn() + if objectAPI != nil { + mode = madmin.ItemOnline + + // Load data usage + ctx2, cancel := context.WithTimeout(ctx, operationTimeout) + dataUsageInfo, err := loadDataUsageFromBackend(ctx2, objectAPI) + cancel() + if err == nil { + buckets = madmin.Buckets{Count: dataUsageInfo.BucketsCount} + objects = madmin.Objects{Count: dataUsageInfo.ObjectsTotalCount} + versions = madmin.Versions{Count: dataUsageInfo.VersionsTotalCount} + deleteMarkers = madmin.DeleteMarkers{Count: dataUsageInfo.DeleteMarkersTotalCount} + usage = madmin.Usage{Size: dataUsageInfo.ObjectsTotalSize} + } else { + buckets = madmin.Buckets{Error: err.Error()} + objects = madmin.Objects{Error: err.Error()} + deleteMarkers = madmin.DeleteMarkers{Error: err.Error()} + usage = madmin.Usage{Error: err.Error()} + } + + // Fetching the backend information + backendInfo := objectAPI.BackendInfo() + // Calculate the number of online/offline disks of all nodes + var allDisks []madmin.Disk + for _, s := range servers { + allDisks = append(allDisks, s.Disks...) + } + onlineDisks, offlineDisks := getOnlineOfflineDisksStats(allDisks) + + backend = madmin.ErasureBackend{ + Type: madmin.ErasureType, + OnlineDisks: onlineDisks.Sum(), + OfflineDisks: offlineDisks.Sum(), + StandardSCParity: backendInfo.StandardSCParity, + RRSCParity: backendInfo.RRSCParity, + TotalSets: backendInfo.TotalSets, + DrivesPerSet: backendInfo.DrivesPerSet, + } + + if pools { + ctx2, cancel := context.WithTimeout(ctx, operationTimeout) + poolsInfo, _ = getPoolsInfo(ctx2, allDisks) + cancel() + } + } + + domain := globalDomainNames + services := madmin.Services{ + LDAP: ldap, + Logger: log, + Audit: audit, + Notifications: notifyTarget, + } + { + ctx2, cancel := context.WithTimeout(ctx, operationTimeout) + services.KMSStatus = fetchKMSStatus(ctx2) + cancel() + } + + return madmin.InfoMessage{ + Mode: string(mode), + Domain: domain, + Region: globalSite.Region(), + SQSARN: globalEventNotifier.GetARNList(), + DeploymentID: globalDeploymentID(), + Buckets: buckets, + Objects: objects, + Versions: versions, + DeleteMarkers: deleteMarkers, + Usage: usage, + Services: services, + Backend: backend, + Servers: servers, + Pools: poolsInfo, + } +} + +func getKubernetesInfo(dctx context.Context) madmin.KubernetesInfo { + ctx, cancel := context.WithCancel(dctx) + defer cancel() + + ki := madmin.KubernetesInfo{} + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, kubernetesVersionEndpoint, nil) + if err != nil { + ki.Error = err.Error() + return ki + } + + client := &http.Client{ + Transport: globalRemoteTargetTransport, + Timeout: 10 * time.Second, + } + + resp, err := client.Do(req) + if err != nil { + ki.Error = err.Error() + return ki + } + defer resp.Body.Close() + decoder := json.NewDecoder(resp.Body) + if err := decoder.Decode(&ki); err != nil { + ki.Error = err.Error() + } + return ki +} + +func fetchHealthInfo(healthCtx context.Context, objectAPI ObjectLayer, query *url.Values, healthInfoCh chan madmin.HealthInfo, healthInfo madmin.HealthInfo) { + hostAnonymizer := createHostAnonymizer() + + anonParam := query.Get(anonymizeParam) + // anonAddr - Anonymizes hosts in given input string + // (only if the anonymize param is set to srict). + anonAddr := func(addr string) string { + if anonParam != anonymizeStrict { + return addr + } + newAddr, found := hostAnonymizer[addr] + if found { + return newAddr + } + + // If we reach here, it means that the given addr doesn't contain any of the hosts. + // Return it as is. Can happen for drive paths in non-distributed mode + return addr + } + + // anonymizedAddr - Updated the addr of the node info with anonymized one + anonymizeAddr := func(info madmin.NodeInfo) { + info.SetAddr(anonAddr(info.GetAddr())) + } + + partialWrite := func(oinfo madmin.HealthInfo) { + select { + case healthInfoCh <- oinfo: + case <-healthCtx.Done(): + } + } + + getAndWritePlatformInfo := func() { + if IsKubernetes() { + healthInfo.Sys.KubernetesInfo = getKubernetesInfo(healthCtx) + partialWrite(healthInfo) + } + } + + getAndWriteCPUs := func() { + if query.Get("syscpu") == "true" { + localCPUInfo := madmin.GetCPUs(healthCtx, globalLocalNodeName) + anonymizeAddr(&localCPUInfo) + healthInfo.Sys.CPUInfo = append(healthInfo.Sys.CPUInfo, localCPUInfo) + + peerCPUInfo := globalNotificationSys.GetCPUs(healthCtx) + for _, cpuInfo := range peerCPUInfo { + anonymizeAddr(&cpuInfo) + healthInfo.Sys.CPUInfo = append(healthInfo.Sys.CPUInfo, cpuInfo) + } + + partialWrite(healthInfo) + } + } + + getAndWritePartitions := func() { + if query.Get("sysdrivehw") == "true" { + localPartitions := madmin.GetPartitions(healthCtx, globalLocalNodeName) + anonymizeAddr(&localPartitions) + healthInfo.Sys.Partitions = append(healthInfo.Sys.Partitions, localPartitions) + + peerPartitions := globalNotificationSys.GetPartitions(healthCtx) + for _, p := range peerPartitions { + anonymizeAddr(&p) + healthInfo.Sys.Partitions = append(healthInfo.Sys.Partitions, p) + } + partialWrite(healthInfo) + } + } + + getAndWriteNetInfo := func() { + if query.Get(string(madmin.HealthDataTypeSysNet)) == "true" { + localNetInfo := madmin.GetNetInfo(globalLocalNodeName, globalInternodeInterface) + healthInfo.Sys.NetInfo = append(healthInfo.Sys.NetInfo, localNetInfo) + + peerNetInfos := globalNotificationSys.GetNetInfo(healthCtx) + for _, n := range peerNetInfos { + anonymizeAddr(&n) + healthInfo.Sys.NetInfo = append(healthInfo.Sys.NetInfo, n) + } + partialWrite(healthInfo) + } + } + + getAndWriteOSInfo := func() { + if query.Get("sysosinfo") == "true" { + localOSInfo := madmin.GetOSInfo(healthCtx, globalLocalNodeName) + anonymizeAddr(&localOSInfo) + healthInfo.Sys.OSInfo = append(healthInfo.Sys.OSInfo, localOSInfo) + + peerOSInfos := globalNotificationSys.GetOSInfo(healthCtx) + for _, o := range peerOSInfos { + anonymizeAddr(&o) + healthInfo.Sys.OSInfo = append(healthInfo.Sys.OSInfo, o) + } + partialWrite(healthInfo) + } + } + + getAndWriteMemInfo := func() { + if query.Get("sysmem") == "true" { + localMemInfo := madmin.GetMemInfo(healthCtx, globalLocalNodeName) + anonymizeAddr(&localMemInfo) + healthInfo.Sys.MemInfo = append(healthInfo.Sys.MemInfo, localMemInfo) + + peerMemInfos := globalNotificationSys.GetMemInfo(healthCtx) + for _, m := range peerMemInfos { + anonymizeAddr(&m) + healthInfo.Sys.MemInfo = append(healthInfo.Sys.MemInfo, m) + } + partialWrite(healthInfo) + } + } + + getAndWriteSysErrors := func() { + if query.Get(string(madmin.HealthDataTypeSysErrors)) == "true" { + localSysErrors := madmin.GetSysErrors(healthCtx, globalLocalNodeName) + anonymizeAddr(&localSysErrors) + healthInfo.Sys.SysErrs = append(healthInfo.Sys.SysErrs, localSysErrors) + partialWrite(healthInfo) + + peerSysErrs := globalNotificationSys.GetSysErrors(healthCtx) + for _, se := range peerSysErrs { + anonymizeAddr(&se) + healthInfo.Sys.SysErrs = append(healthInfo.Sys.SysErrs, se) + } + partialWrite(healthInfo) + } + } + + getAndWriteSysConfig := func() { + if query.Get(string(madmin.HealthDataTypeSysConfig)) == "true" { + localSysConfig := madmin.GetSysConfig(healthCtx, globalLocalNodeName) + anonymizeAddr(&localSysConfig) + healthInfo.Sys.SysConfig = append(healthInfo.Sys.SysConfig, localSysConfig) + partialWrite(healthInfo) + + peerSysConfig := globalNotificationSys.GetSysConfig(healthCtx) + for _, sc := range peerSysConfig { + anonymizeAddr(&sc) + healthInfo.Sys.SysConfig = append(healthInfo.Sys.SysConfig, sc) + } + partialWrite(healthInfo) + } + } + + getAndWriteSysServices := func() { + if query.Get(string(madmin.HealthDataTypeSysServices)) == "true" { + localSysServices := madmin.GetSysServices(healthCtx, globalLocalNodeName) + anonymizeAddr(&localSysServices) + healthInfo.Sys.SysServices = append(healthInfo.Sys.SysServices, localSysServices) + partialWrite(healthInfo) + + peerSysServices := globalNotificationSys.GetSysServices(healthCtx) + for _, ss := range peerSysServices { + anonymizeAddr(&ss) + healthInfo.Sys.SysServices = append(healthInfo.Sys.SysServices, ss) + } + partialWrite(healthInfo) + } + } + + // collect all realtime metrics except disk + // disk metrics are already included under drive info of each server + getRealtimeMetrics := func() *madmin.RealtimeMetrics { + var m madmin.RealtimeMetrics + types := madmin.MetricsAll &^ madmin.MetricsDisk + mLocal := collectLocalMetrics(types, collectMetricsOpts{}) + m.Merge(&mLocal) + cctx, cancel := context.WithTimeout(healthCtx, time.Second/2) + mRemote := collectRemoteMetrics(cctx, types, collectMetricsOpts{}) + cancel() + m.Merge(&mRemote) + for idx, host := range m.Hosts { + m.Hosts[idx] = anonAddr(host) + } + for host, metrics := range m.ByHost { + m.ByHost[anonAddr(host)] = metrics + delete(m.ByHost, host) + } + return &m + } + + anonymizeCmdLine := func(cmdLine string) string { + if anonParam != anonymizeStrict { + return cmdLine + } + + if !globalIsDistErasure { + // FS mode - single server - hard code to `server1` + anonCmdLine := strings.ReplaceAll(cmdLine, globalLocalNodeName, "server1") + if len(globalMinioConsoleHost) > 0 { + anonCmdLine = strings.ReplaceAll(anonCmdLine, globalMinioConsoleHost, "server1") + } + return anonCmdLine + } + + // Server start command regex groups: + // 1 - minio server + // 2 - flags e.g. `--address :9000 --certs-dir /etc/minio/certs` + // 3 - pool args e.g. `https://node{01...16}.domain/data/disk{001...204} https://node{17...32}.domain/data/disk{001...204}` + re := regexp.MustCompile(`^(.*minio\s+server\s+)(--[^\s]+\s+[^\s]+\s+)*(.*)`) + + // stays unchanged in the anonymized version + cmdLineWithoutPools := re.ReplaceAllString(cmdLine, `$1$2`) + + // to be anonymized + poolsArgs := re.ReplaceAllString(cmdLine, `$3`) + var anonPools []string + + if !strings.Contains(poolsArgs, "{") || !strings.Contains(poolsArgs, "}") { + // No ellipses pattern. Anonymize host name from every pool arg + pools := strings.Fields(poolsArgs) + anonPools = make([]string, len(pools)) + for index, arg := range pools { + anonPools[index] = anonAddr(arg) + } + return cmdLineWithoutPools + strings.Join(anonPools, " ") + } + + // Ellipses pattern in pool args. Regex groups: + // 1 - server prefix + // 2 - number sequence for servers + // 3 - server suffix + // 4 - drive prefix (starting with /) + // 5 - number sequence for drives + // 6 - drive suffix + re = regexp.MustCompile(`([^\s^{]*)({\d+...\d+})?([^\s^{^/]*)(/[^\s^{]*)({\d+...\d+})?([^\s]*)`) + poolsMatches := re.FindAllStringSubmatch(poolsArgs, -1) + + anonPools = make([]string, len(poolsMatches)) + idxMap := map[int]string{ + 1: "spfx", + 3: "ssfx", + } + for pi, poolsMatch := range poolsMatches { + // Replace the server prefix/suffix with anonymized ones + for idx, lbl := range idxMap { + if len(poolsMatch[idx]) > 0 { + poolsMatch[idx] = fmt.Sprintf("%s%d", lbl, crc32.ChecksumIEEE([]byte(poolsMatch[idx]))) + } + } + + // Remove the original pools args present at index 0 + anonPools[pi] = strings.Join(poolsMatch[1:], "") + } + return cmdLineWithoutPools + strings.Join(anonPools, " ") + } + + anonymizeProcInfo := func(p *madmin.ProcInfo) { + p.CmdLine = anonymizeCmdLine(p.CmdLine) + anonymizeAddr(p) + } + + getAndWriteProcInfo := func() { + if query.Get("sysprocess") == "true" { + localProcInfo := madmin.GetProcInfo(healthCtx, globalLocalNodeName) + anonymizeProcInfo(&localProcInfo) + healthInfo.Sys.ProcInfo = append(healthInfo.Sys.ProcInfo, localProcInfo) + peerProcInfos := globalNotificationSys.GetProcInfo(healthCtx) + for _, p := range peerProcInfos { + anonymizeProcInfo(&p) + healthInfo.Sys.ProcInfo = append(healthInfo.Sys.ProcInfo, p) + } + partialWrite(healthInfo) + } + } + + getAndWriteMinioConfig := func() { + if query.Get("minioconfig") == "true" { + config, err := readServerConfig(healthCtx, objectAPI, nil) + if err != nil { + healthInfo.Minio.Config = madmin.MinioConfig{ + Error: err.Error(), + } + } else { + healthInfo.Minio.Config = madmin.MinioConfig{ + Config: config.RedactSensitiveInfo(), + } + } + partialWrite(healthInfo) + } + } + + anonymizeNetwork := func(network map[string]string) map[string]string { + anonNetwork := map[string]string{} + for endpoint, status := range network { + anonEndpoint := anonAddr(endpoint) + anonNetwork[anonEndpoint] = status + } + return anonNetwork + } + + anonymizeDrives := func(drives []madmin.Disk) []madmin.Disk { + anonDrives := []madmin.Disk{} + for _, drive := range drives { + drive.Endpoint = anonAddr(drive.Endpoint) + anonDrives = append(anonDrives, drive) + } + return anonDrives + } + + go func() { + defer xioutil.SafeClose(healthInfoCh) + + partialWrite(healthInfo) // Write first message with only version and deployment id populated + getAndWritePlatformInfo() + getAndWriteCPUs() + getAndWritePartitions() + getAndWriteNetInfo() + getAndWriteOSInfo() + getAndWriteMemInfo() + getAndWriteProcInfo() + getAndWriteMinioConfig() + getAndWriteSysErrors() + getAndWriteSysServices() + getAndWriteSysConfig() + + if query.Get("minioinfo") == "true" { + infoMessage := getServerInfo(healthCtx, false, true, nil) + servers := make([]madmin.ServerInfo, 0, len(infoMessage.Servers)) + for _, server := range infoMessage.Servers { + anonEndpoint := anonAddr(server.Endpoint) + servers = append(servers, madmin.ServerInfo{ + State: server.State, + Endpoint: anonEndpoint, + Uptime: server.Uptime, + Version: server.Version, + CommitID: server.CommitID, + Network: anonymizeNetwork(server.Network), + Drives: anonymizeDrives(server.Disks), + PoolNumber: func() int { + if len(server.PoolNumbers) == 1 { + return server.PoolNumbers[0] + } + return math.MaxInt // this indicates that its unset. + }(), + PoolNumbers: server.PoolNumbers, + MemStats: madmin.MemStats{ + Alloc: server.MemStats.Alloc, + TotalAlloc: server.MemStats.TotalAlloc, + Mallocs: server.MemStats.Mallocs, + Frees: server.MemStats.Frees, + HeapAlloc: server.MemStats.HeapAlloc, + }, + GoMaxProcs: server.GoMaxProcs, + NumCPU: server.NumCPU, + RuntimeVersion: server.RuntimeVersion, + GCStats: server.GCStats, + MinioEnvVars: server.MinioEnvVars, + }) + } + + tls := getTLSInfo() + isK8s := IsKubernetes() + isDocker := IsDocker() + healthInfo.Minio.Info = madmin.MinioInfo{ + Mode: infoMessage.Mode, + Domain: infoMessage.Domain, + Region: infoMessage.Region, + SQSARN: infoMessage.SQSARN, + DeploymentID: infoMessage.DeploymentID, + Buckets: infoMessage.Buckets, + Objects: infoMessage.Objects, + Usage: infoMessage.Usage, + Services: infoMessage.Services, + Backend: infoMessage.Backend, + Servers: servers, + TLS: &tls, + IsKubernetes: &isK8s, + IsDocker: &isDocker, + Metrics: getRealtimeMetrics(), + } + partialWrite(healthInfo) + } + }() +} + +// HealthInfoHandler - GET /minio/admin/v3/healthinfo +// ---------- +// Get server health info +func (a adminAPIHandlers) HealthInfoHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.HealthInfoAdminAction) + if objectAPI == nil { + return + } + + query := r.Form + healthInfoCh := make(chan madmin.HealthInfo) + enc := json.NewEncoder(w) + + healthInfo := madmin.HealthInfo{ + TimeStamp: time.Now().UTC(), + Version: madmin.HealthInfoVersion, + Minio: madmin.MinioHealthInfo{ + Info: madmin.MinioInfo{ + DeploymentID: globalDeploymentID(), + }, + }, + } + + errResp := func(err error) { + errorResponse := getAPIErrorResponse(ctx, toAdminAPIErr(ctx, err), r.URL.String(), + w.Header().Get(xhttp.AmzRequestID), w.Header().Get(xhttp.AmzRequestHostID)) + encodedErrorResponse := encodeResponse(errorResponse) + healthInfo.Error = string(encodedErrorResponse) + adminLogIf(ctx, enc.Encode(healthInfo)) + } + + deadline := 10 * time.Second // Default deadline is 10secs for health diagnostics. + if dstr := query.Get("deadline"); dstr != "" { + var err error + deadline, err = time.ParseDuration(dstr) + if err != nil { + errResp(err) + return + } + } + + nsLock := objectAPI.NewNSLock(minioMetaBucket, "health-check-in-progress") + lkctx, err := nsLock.GetLock(ctx, newDynamicTimeout(deadline, deadline)) + if err != nil { // returns a locked lock + errResp(err) + return + } + + defer nsLock.Unlock(lkctx) + healthCtx, healthCancel := context.WithTimeout(lkctx.Context(), deadline) + defer healthCancel() + + go fetchHealthInfo(healthCtx, objectAPI, &query, healthInfoCh, healthInfo) + + setCommonHeaders(w) + setEventStreamHeaders(w) + w.WriteHeader(http.StatusOK) + + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + + for { + select { + case oinfo, ok := <-healthInfoCh: + if !ok { + return + } + if err := enc.Encode(oinfo); err != nil { + return + } + if len(healthInfoCh) == 0 { + // Flush if nothing is queued + xhttp.Flush(w) + } + case <-ticker.C: + if _, err := w.Write([]byte(" ")); err != nil { + return + } + xhttp.Flush(w) + case <-healthCtx.Done(): + return + } + } +} + +func getTLSInfo() madmin.TLSInfo { + tlsInfo := madmin.TLSInfo{ + TLSEnabled: globalIsTLS, + Certs: []madmin.TLSCert{}, + } + + if globalIsTLS { + for _, c := range globalPublicCerts { + check := xxh3.Hash(c.RawIssuer) + check ^= xxh3.Hash(c.RawSubjectPublicKeyInfo) + // We XOR, so order doesn't matter. + for _, v := range c.DNSNames { + check ^= xxh3.HashString(v) + } + for _, v := range c.EmailAddresses { + check ^= xxh3.HashString(v) + } + for _, v := range c.IPAddresses { + check ^= xxh3.HashString(v.String()) + } + for _, v := range c.URIs { + check ^= xxh3.HashString(v.String()) + } + tlsInfo.Certs = append(tlsInfo.Certs, madmin.TLSCert{ + PubKeyAlgo: c.PublicKeyAlgorithm.String(), + SignatureAlgo: c.SignatureAlgorithm.String(), + NotBefore: c.NotBefore, + NotAfter: c.NotAfter, + Checksum: strconv.FormatUint(check, 16), + }) + } + } + return tlsInfo +} + +// ServerInfoHandler - GET /minio/admin/v3/info +// ---------- +// Get server information +func (a adminAPIHandlers) ServerInfoHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Validate request signature. + _, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.ServerInfoAdminAction, "") + if adminAPIErr != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL) + return + } + + metrics := r.Form.Get(peerRESTMetrics) == "true" + + // Marshal API response + jsonBytes, err := json.Marshal(getServerInfo(ctx, true, metrics, r)) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Reply with storage information (across nodes in a + // distributed setup) as json. + writeSuccessResponseJSON(w, jsonBytes) +} + +func fetchLambdaInfo() []map[string][]madmin.TargetIDStatus { + lambdaMap := make(map[string][]madmin.TargetIDStatus) + + for _, tgt := range globalEventNotifier.Targets() { + targetIDStatus := make(map[string]madmin.Status) + active, _ := tgt.IsActive() + targetID := tgt.ID() + if active { + targetIDStatus[targetID.ID] = madmin.Status{Status: string(madmin.ItemOnline)} + } else { + targetIDStatus[targetID.ID] = madmin.Status{Status: string(madmin.ItemOffline)} + } + list := lambdaMap[targetID.Name] + list = append(list, targetIDStatus) + lambdaMap[targetID.Name] = list + } + + notify := make([]map[string][]madmin.TargetIDStatus, len(lambdaMap)) + counter := 0 + for key, value := range lambdaMap { + v := make(map[string][]madmin.TargetIDStatus) + v[key] = value + notify[counter] = v + counter++ + } + return notify +} + +// fetchKMSStatus fetches KMS-related status information for all instances +func fetchKMSStatus(ctx context.Context) []madmin.KMS { + if GlobalKMS == nil { + return []madmin.KMS{} + } + + stat, err := GlobalKMS.Status(ctx) + if err != nil { + kmsLogIf(ctx, err, "failed to fetch KMS status information") + return []madmin.KMS{} + } + + stats := make([]madmin.KMS, 0, len(stat.Endpoints)) + for endpoint, state := range stat.Endpoints { + stats = append(stats, madmin.KMS{ + Status: string(state), + Endpoint: endpoint, + }) + } + return stats +} + +func targetStatus(ctx context.Context, h logger.Target) madmin.Status { + if h.IsOnline(ctx) { + return madmin.Status{Status: string(madmin.ItemOnline)} + } + return madmin.Status{Status: string(madmin.ItemOffline)} +} + +// fetchLoggerInfo return log info +func fetchLoggerInfo(ctx context.Context) ([]madmin.Logger, []madmin.Audit) { + var loggerInfo []madmin.Logger + var auditloggerInfo []madmin.Audit + for _, tgt := range logger.SystemTargets() { + if tgt.Endpoint() != "" { + loggerInfo = append(loggerInfo, madmin.Logger{tgt.String(): targetStatus(ctx, tgt)}) + } + } + + for _, tgt := range logger.AuditTargets() { + if tgt.Endpoint() != "" { + auditloggerInfo = append(auditloggerInfo, madmin.Audit{tgt.String(): targetStatus(ctx, tgt)}) + } + } + + return loggerInfo, auditloggerInfo +} + +func embedFileInZip(zipWriter *zip.Writer, name string, data []byte, fileMode os.FileMode) error { + // Send profiling data to zip as file + header, zerr := zip.FileInfoHeader(dummyFileInfo{ + name: name, + size: int64(len(data)), + mode: fileMode, + modTime: UTCNow(), + isDir: false, + sys: nil, + }) + if zerr != nil { + return zerr + } + header.Method = zip.Deflate + zwriter, zerr := zipWriter.CreateHeader(header) + if zerr != nil { + return zerr + } + _, err := io.Copy(zwriter, bytes.NewReader(data)) + return err +} + +// getClusterMetaInfo gets information of the current cluster and +// returns it. +// This is not a critical function, and it is allowed +// to fail with a ten seconds timeout, returning nil. +func getClusterMetaInfo(ctx context.Context) []byte { + objectAPI := newObjectLayerFn() + if objectAPI == nil { + return nil + } + + // Add a ten seconds timeout because getting profiling data + // is critical for debugging, in contrary to getting cluster info + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + resultCh := make(chan madmin.ClusterRegistrationInfo) + + go func() { + defer xioutil.SafeClose(resultCh) + + ci := madmin.ClusterRegistrationInfo{} + ci.Info.NoOfServerPools = len(globalEndpoints) + ci.Info.NoOfServers = totalNodeCount() + ci.Info.MinioVersion = Version + + si := objectAPI.StorageInfo(ctx, false) + + ci.Info.NoOfDrives = len(si.Disks) + for _, disk := range si.Disks { + ci.Info.TotalDriveSpace += disk.TotalSpace + ci.Info.UsedDriveSpace += disk.UsedSpace + } + + dataUsageInfo, _ := loadDataUsageFromBackend(ctx, objectAPI) + + ci.UsedCapacity = dataUsageInfo.ObjectsTotalSize + ci.Info.NoOfBuckets = dataUsageInfo.BucketsCount + ci.Info.NoOfObjects = dataUsageInfo.ObjectsTotalCount + + ci.DeploymentID = globalDeploymentID() + ci.ClusterName = fmt.Sprintf("%d-servers-%d-disks-%s", ci.Info.NoOfServers, ci.Info.NoOfDrives, ci.Info.MinioVersion) + + select { + case resultCh <- ci: + case <-ctx.Done(): + return + } + }() + + select { + case <-ctx.Done(): + return nil + case ci := <-resultCh: + out, err := json.MarshalIndent(ci, "", " ") + if err != nil { + bugLogIf(ctx, err) + return nil + } + return out + } +} + +func bytesToPublicKey(pub []byte) (*rsa.PublicKey, error) { + block, _ := pem.Decode(pub) + if block != nil { + pub = block.Bytes + } + key, err := x509.ParsePKCS1PublicKey(pub) + if err != nil { + return nil, err + } + return key, nil +} + +// getRawDataer provides an interface for getting raw FS files. +type getRawDataer interface { + GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, info StatInfo) error) error +} + +// InspectDataHandler - GET /minio/admin/v3/inspect-data +// ---------- +// Download file from all nodes in a zip format +func (a adminAPIHandlers) InspectDataHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + // Validate request signature. + _, adminAPIErr := checkAdminRequestAuth(ctx, r, policy.InspectDataAction, "") + if adminAPIErr != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(adminAPIErr), r.URL) + return + } + + objLayer := newObjectLayerFn() + o, ok := objLayer.(getRawDataer) + if !ok { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + + if err := parseForm(r); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + volume := r.Form.Get("volume") + if len(volume) == 0 { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketName), r.URL) + return + } + file := r.Form.Get("file") + if len(file) == 0 { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + + file = filepath.ToSlash(file) + // Reject attempts to traverse parent or absolute paths. + if hasBadPathComponent(volume) || hasBadPathComponent(file) { + writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidResourceName), r.URL) + return + } + + var publicKey *rsa.PublicKey + + publicKeyB64 := r.Form.Get("public-key") + if publicKeyB64 != "" { + publicKeyBytes, err := base64.StdEncoding.DecodeString(publicKeyB64) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + publicKey, err = bytesToPublicKey(publicKeyBytes) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + } + addErr := func(msg string) {} + + // Write a version for making *incompatible* changes. + // The AdminClient will reject any version it does not know. + var inspectZipW *zip.Writer + if publicKey != nil { + w.WriteHeader(200) + stream := estream.NewWriter(w) + defer stream.Close() + + clusterKey, err := bytesToPublicKey(getSubnetAdminPublicKey()) + if err != nil { + bugLogIf(ctx, stream.AddError(err.Error())) + return + } + err = stream.AddKeyEncrypted(clusterKey) + if err != nil { + bugLogIf(ctx, stream.AddError(err.Error())) + return + } + if b := getClusterMetaInfo(ctx); len(b) > 0 { + w, err := stream.AddEncryptedStream("cluster.info", nil) + if err != nil { + bugLogIf(ctx, err) + return + } + w.Write(b) + w.Close() + } + + // Add new key for inspect data. + if err := stream.AddKeyEncrypted(publicKey); err != nil { + bugLogIf(ctx, stream.AddError(err.Error())) + return + } + encStream, err := stream.AddEncryptedStream("inspect.zip", nil) + if err != nil { + bugLogIf(ctx, stream.AddError(err.Error())) + return + } + addErr = func(msg string) { + inspectZipW.Close() + encStream.Close() + stream.AddError(msg) + } + defer encStream.Close() + + inspectZipW = zip.NewWriter(encStream) + defer inspectZipW.Close() + } else { + // Legacy: Remove if we stop supporting inspection without public key. + var key [32]byte + // MUST use crypto/rand + n, err := crand.Read(key[:]) + if err != nil || n != len(key) { + bugLogIf(ctx, err) + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Write a version for making *incompatible* changes. + // The AdminClient will reject any version it does not know. + if publicKey == nil { + w.Write([]byte{1}) + w.Write(key[:]) + } + + stream, err := sio.AES_256_GCM.Stream(key[:]) + if err != nil { + bugLogIf(ctx, err) + return + } + // Zero nonce, we only use each key once, and 32 bytes is plenty. + nonce := make([]byte, stream.NonceSize()) + encw := stream.EncryptWriter(w, nonce, nil) + defer encw.Close() + + // Initialize a zip writer which will provide a zipped content + // of profiling data of all nodes + inspectZipW = zip.NewWriter(encw) + defer inspectZipW.Close() + + if b := getClusterMetaInfo(ctx); len(b) > 0 { + adminLogIf(ctx, embedFileInZip(inspectZipW, "cluster.info", b, 0o600)) + } + } + + rawDataFn := func(r io.Reader, host, disk, filename string, si StatInfo) error { + // Prefix host+disk + filename = path.Join(host, disk, filename) + if si.Dir { + filename += "/" + si.Size = 0 + } + if si.Mode == 0 { + // Not, set it to default. + si.Mode = 0o600 + } + if si.ModTime.IsZero() { + // Set time to now. + si.ModTime = time.Now() + } + header, zerr := zip.FileInfoHeader(dummyFileInfo{ + name: filename, + size: si.Size, + mode: os.FileMode(si.Mode), + modTime: si.ModTime, + isDir: si.Dir, + sys: nil, + }) + if zerr != nil { + bugLogIf(ctx, zerr) + return nil + } + header.Method = zip.Deflate + zwriter, zerr := inspectZipW.CreateHeader(header) + if zerr != nil { + bugLogIf(ctx, zerr) + return nil + } + if _, err := io.Copy(zwriter, r); err != nil { + adminLogIf(ctx, err) + } + return nil + } + + // save args passed to inspect command + var sb bytes.Buffer + fmt.Fprintf(&sb, "Inspect path: %s%s%s\n", volume, slashSeparator, file) + sb.WriteString("Server command line args:") + for _, pool := range globalEndpoints { + sb.WriteString(" ") + sb.WriteString(pool.CmdLine) + } + sb.WriteString("\n") + adminLogIf(ctx, embedFileInZip(inspectZipW, "inspect-input.txt", sb.Bytes(), 0o600)) + + err := o.GetRawData(ctx, volume, file, rawDataFn) + if err != nil { + if errors.Is(err, errFileNotFound) { + addErr("GetRawData: No files matched the given pattern") + return + } + embedFileInZip(inspectZipW, "GetRawData-err.txt", []byte(err.Error()), 0o600) + adminLogIf(ctx, err) + } + + // save the format.json as part of inspect by default + if volume != minioMetaBucket || file != formatConfigFile { + err = o.GetRawData(ctx, minioMetaBucket, formatConfigFile, rawDataFn) + } + if !errors.Is(err, errFileNotFound) { + adminLogIf(ctx, err) + } + + scheme := "https" + if !globalIsTLS { + scheme = "http" + } + + // save MinIO start script to inspect command + var scrb bytes.Buffer + fmt.Fprintf(&scrb, `#!/usr/bin/env bash + +function main() { + for file in $(ls -1); do + dest_file=$(echo "$file" | cut -d ":" -f1) + mv "$file" "$dest_file" + done + + # Read content of inspect-input.txt + MINIO_OPTS=$(grep "Server command line args" <./inspect-input.txt | sed "s/Server command line args: //g" | sed -r "s#%s:\/\/#\.\/#g") + + # Start MinIO instance using the options + START_CMD="CI=on _MINIO_AUTO_DRIVE_HEALING=off minio server ${MINIO_OPTS} &" + echo + echo "Starting MinIO instance: ${START_CMD}" + echo + eval "$START_CMD" + MINIO_SRVR_PID="$!" + echo "MinIO Server PID: ${MINIO_SRVR_PID}" + echo + echo "Waiting for MinIO instance to get ready!" + sleep 10 +} + +main "$@"`, scheme) + adminLogIf(ctx, embedFileInZip(inspectZipW, "start-minio.sh", scrb.Bytes(), 0o755)) +} + +func getSubnetAdminPublicKey() []byte { + if globalIsCICD { + return subnetAdminPublicKeyDev + } + return subnetAdminPublicKey +} + +func createHostAnonymizerForFSMode() map[string]string { + hostAnonymizer := map[string]string{ + globalLocalNodeName: "server1", + } + + apiEndpoints := getAPIEndpoints() + for _, ep := range apiEndpoints { + if len(ep) == 0 { + continue + } + if url, err := xnet.ParseHTTPURL(ep); err == nil { + // In FS mode the drive names don't include the host. + // So mapping just the host should be sufficient. + hostAnonymizer[url.Host] = "server1" + } + } + return hostAnonymizer +} + +// anonymizeHost - Add entries related to given endpoint in the host anonymizer map +// The health report data can contain the hostname in various forms e.g. host, host:port, +// host:port/drivepath, full url (http://host:port/drivepath) +// The anonymizer map will have mappings for all these variants for efficiently replacing +// any of these strings to the anonymized versions at the time of health report generation. +func anonymizeHost(hostAnonymizer map[string]string, endpoint Endpoint, poolNum int, srvrNum int) { + if len(endpoint.Host) == 0 { + return + } + + currentURL := endpoint.String() + + // mapIfNotPresent - Maps the given key to the value only if the key is not present in the map + mapIfNotPresent := func(m map[string]string, key string, val string) { + _, found := m[key] + if !found { + m[key] = val + } + } + + _, found := hostAnonymizer[currentURL] + if !found { + // In distributed setup, anonymized addr = 'poolNum.serverNum' + newHost := fmt.Sprintf("pool%d.server%d", poolNum, srvrNum) + schemePfx := endpoint.Scheme + "://" + + // Hostname + mapIfNotPresent(hostAnonymizer, endpoint.Hostname(), newHost) + + newHostPort := newHost + if len(endpoint.Port()) > 0 { + // Host + port + newHostPort = newHost + ":" + endpoint.Port() + mapIfNotPresent(hostAnonymizer, endpoint.Host, newHostPort) + mapIfNotPresent(hostAnonymizer, schemePfx+endpoint.Host, newHostPort) + } + + newHostPortPath := newHostPort + if len(endpoint.Path) > 0 { + // Host + port + path + currentHostPortPath := endpoint.Host + endpoint.Path + newHostPortPath = newHostPort + endpoint.Path + mapIfNotPresent(hostAnonymizer, currentHostPortPath, newHostPortPath) + mapIfNotPresent(hostAnonymizer, schemePfx+currentHostPortPath, newHostPortPath) + } + + // Full url + hostAnonymizer[currentURL] = schemePfx + newHostPortPath + } +} + +// createHostAnonymizer - Creates a map of various strings to corresponding anonymized names +func createHostAnonymizer() map[string]string { + if !globalIsDistErasure { + return createHostAnonymizerForFSMode() + } + + hostAnonymizer := map[string]string{} + hosts := set.NewStringSet() + srvrIdx := 0 + + for poolIdx, pool := range globalEndpoints { + for _, endpoint := range pool.Endpoints { + if !hosts.Contains(endpoint.Host) { + hosts.Add(endpoint.Host) + srvrIdx++ + } + anonymizeHost(hostAnonymizer, endpoint, poolIdx+1, srvrIdx) + } + } + return hostAnonymizer +} diff --git a/cmd/admin-handlers_test.go b/cmd/admin-handlers_test.go new file mode 100644 index 00000000..3f8b3482 --- /dev/null +++ b/cmd/admin-handlers_test.go @@ -0,0 +1,535 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "sort" + "sync" + "testing" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/auth" + "github.com/minio/mux" +) + +// adminErasureTestBed - encapsulates subsystems that need to be setup for +// admin-handler unit tests. +type adminErasureTestBed struct { + erasureDirs []string + objLayer ObjectLayer + router *mux.Router + done context.CancelFunc +} + +// prepareAdminErasureTestBed - helper function that setups a single-node +// Erasure backend for admin-handler tests. +func prepareAdminErasureTestBed(ctx context.Context) (*adminErasureTestBed, error) { + ctx, cancel := context.WithCancel(ctx) + + // reset global variables to start afresh. + resetTestGlobals() + + // Set globalIsErasure to indicate that the setup uses an erasure + // code backend. + globalIsErasure = true + + // Initializing objectLayer for HealFormatHandler. + objLayer, erasureDirs, xlErr := initTestErasureObjLayer(ctx) + if xlErr != nil { + cancel() + return nil, xlErr + } + + // Initialize minio server config. + if err := newTestConfig(globalMinioDefaultRegion, objLayer); err != nil { + cancel() + return nil, err + } + + // Initialize boot time + globalBootTime = UTCNow() + + globalEndpoints = mustGetPoolEndpoints(0, erasureDirs...) + + initAllSubsystems(ctx) + + initConfigSubsystem(ctx, objLayer) + + globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second) + + // Setup admin mgmt REST API handlers. + adminRouter := mux.NewRouter() + registerAdminRouter(adminRouter, true) + + return &adminErasureTestBed{ + erasureDirs: erasureDirs, + objLayer: objLayer, + router: adminRouter, + done: cancel, + }, nil +} + +// TearDown - method that resets the test bed for subsequent unit +// tests to start afresh. +func (atb *adminErasureTestBed) TearDown() { + atb.done() + removeRoots(atb.erasureDirs) + resetTestGlobals() +} + +// initTestObjLayer - Helper function to initialize an Erasure-based object +// layer and set globalObjectAPI. +func initTestErasureObjLayer(ctx context.Context) (ObjectLayer, []string, error) { + erasureDirs, err := getRandomDisks(16) + if err != nil { + return nil, nil, err + } + endpoints := mustGetPoolEndpoints(0, erasureDirs...) + globalPolicySys = NewPolicySys() + objLayer, err := newErasureServerPools(ctx, endpoints) + if err != nil { + return nil, nil, err + } + + // Make objLayer available to all internal services via globalObjectAPI. + globalObjLayerMutex.Lock() + globalObjectAPI = objLayer + globalObjLayerMutex.Unlock() + return objLayer, erasureDirs, nil +} + +// cmdType - Represents different service subcomands like status, stop +// and restart. +type cmdType int + +const ( + restartCmd cmdType = iota + stopCmd +) + +// toServiceSignal - Helper function that translates a given cmdType +// value to its corresponding serviceSignal value. +func (c cmdType) toServiceSignal() serviceSignal { + switch c { + case restartCmd: + return serviceRestart + case stopCmd: + return serviceStop + } + return serviceRestart +} + +func (c cmdType) toServiceAction() madmin.ServiceAction { + switch c { + case restartCmd: + return madmin.ServiceActionRestart + case stopCmd: + return madmin.ServiceActionStop + } + return madmin.ServiceActionRestart +} + +// testServiceSignalReceiver - Helper function that simulates a +// go-routine waiting on service signal. +func testServiceSignalReceiver(cmd cmdType, t *testing.T) { + expectedCmd := cmd.toServiceSignal() + serviceCmd := <-globalServiceSignalCh + if serviceCmd != expectedCmd { + t.Errorf("Expected service command %v but received %v", expectedCmd, serviceCmd) + } +} + +// getServiceCmdRequest - Constructs a management REST API request for service +// subcommands for a given cmdType value. +func getServiceCmdRequest(cmd cmdType, cred auth.Credentials) (*http.Request, error) { + queryVal := url.Values{} + queryVal.Set("action", string(cmd.toServiceAction())) + queryVal.Set("type", "2") + resource := adminPathPrefix + adminAPIVersionPrefix + "/service?" + queryVal.Encode() + req, err := newTestRequest(http.MethodPost, resource, 0, nil) + if err != nil { + return nil, err + } + + // management REST API uses signature V4 for authentication. + err = signRequestV4(req, cred.AccessKey, cred.SecretKey) + if err != nil { + return nil, err + } + return req, nil +} + +// testServicesCmdHandler - parametrizes service subcommand tests on +// cmdType value. +func testServicesCmdHandler(cmd cmdType, t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + adminTestBed, err := prepareAdminErasureTestBed(ctx) + if err != nil { + t.Fatal("Failed to initialize a single node Erasure backend for admin handler tests.", err) + } + defer adminTestBed.TearDown() + + // Initialize admin peers to make admin RPC calls. Note: In a + // single node setup, this degenerates to a simple function + // call under the hood. + globalMinioAddr = "127.0.0.1:9000" + + var wg sync.WaitGroup + + // Setting up a go routine to simulate ServerRouter's + // handleServiceSignals for stop and restart commands. + if cmd == restartCmd { + wg.Add(1) + go func() { + defer wg.Done() + testServiceSignalReceiver(cmd, t) + }() + } + credentials := globalActiveCred + + req, err := getServiceCmdRequest(cmd, credentials) + if err != nil { + t.Fatalf("Failed to build service status request %v", err) + } + + rec := httptest.NewRecorder() + adminTestBed.router.ServeHTTP(rec, req) + + resp, _ := io.ReadAll(rec.Body) + if rec.Code != http.StatusOK { + t.Errorf("Expected to receive %d status code but received %d. Body (%s)", + http.StatusOK, rec.Code, string(resp)) + } + + result := &serviceResult{} + if err := json.Unmarshal(resp, result); err != nil { + t.Error(err) + } + _ = result + + // Wait until testServiceSignalReceiver() called in a goroutine quits. + wg.Wait() +} + +// Test for service restart management REST API. +func TestServiceRestartHandler(t *testing.T) { + testServicesCmdHandler(restartCmd, t) +} + +// buildAdminRequest - helper function to build an admin API request. +func buildAdminRequest(queryVal url.Values, method, path string, + contentLength int64, bodySeeker io.ReadSeeker) (*http.Request, error, +) { + req, err := newTestRequest(method, + adminPathPrefix+adminAPIVersionPrefix+path+"?"+queryVal.Encode(), + contentLength, bodySeeker) + if err != nil { + return nil, err + } + + cred := globalActiveCred + err = signRequestV4(req, cred.AccessKey, cred.SecretKey) + if err != nil { + return nil, err + } + + return req, nil +} + +func TestAdminServerInfo(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + adminTestBed, err := prepareAdminErasureTestBed(ctx) + if err != nil { + t.Fatal("Failed to initialize a single node Erasure backend for admin handler tests.", err) + } + + defer adminTestBed.TearDown() + + // Initialize admin peers to make admin RPC calls. + globalMinioAddr = "127.0.0.1:9000" + + // Prepare query params for set-config mgmt REST API. + queryVal := url.Values{} + queryVal.Set("info", "") + + req, err := buildAdminRequest(queryVal, http.MethodGet, "/info", 0, nil) + if err != nil { + t.Fatalf("Failed to construct get-config object request - %v", err) + } + + rec := httptest.NewRecorder() + adminTestBed.router.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Errorf("Expected to succeed but failed with %d", rec.Code) + } + + results := madmin.InfoMessage{} + err = json.NewDecoder(rec.Body).Decode(&results) + if err != nil { + t.Fatalf("Failed to decode set config result json %v", err) + } + + if results.Region != globalMinioDefaultRegion { + t.Errorf("Expected %s, got %s", globalMinioDefaultRegion, results.Region) + } +} + +// TestToAdminAPIErrCode - test for toAdminAPIErrCode helper function. +func TestToAdminAPIErrCode(t *testing.T) { + testCases := []struct { + err error + expectedAPIErr APIErrorCode + }{ + // 1. Server not in quorum. + { + err: errErasureWriteQuorum, + expectedAPIErr: ErrAdminConfigNoQuorum, + }, + // 2. No error. + { + err: nil, + expectedAPIErr: ErrNone, + }, + // 3. Non-admin API specific error. + { + err: errDiskNotFound, + expectedAPIErr: toAPIErrorCode(GlobalContext, errDiskNotFound), + }, + } + + for i, test := range testCases { + actualErr := toAdminAPIErrCode(GlobalContext, test.err) + if actualErr != test.expectedAPIErr { + t.Errorf("Test %d: Expected %v but received %v", + i+1, test.expectedAPIErr, actualErr) + } + } +} + +func TestExtractHealInitParams(t *testing.T) { + mkParams := func(clientToken string, forceStart, forceStop bool) url.Values { + v := url.Values{} + if clientToken != "" { + v.Add(mgmtClientToken, clientToken) + } + if forceStart { + v.Add(mgmtForceStart, "") + } + if forceStop { + v.Add(mgmtForceStop, "") + } + return v + } + qParamsArr := []url.Values{ + // Invalid cases + mkParams("", true, true), + mkParams("111", true, true), + mkParams("111", true, false), + mkParams("111", false, true), + // Valid cases follow + mkParams("", true, false), + mkParams("", false, true), + mkParams("", false, false), + mkParams("111", false, false), + } + varsArr := []map[string]string{ + // Invalid cases + {mgmtPrefix: "objprefix"}, + // Valid cases + {}, + {mgmtBucket: "bucket"}, + {mgmtBucket: "bucket", mgmtPrefix: "objprefix"}, + } + + // Body is always valid - we do not test JSON decoding. + body := `{"recursive": false, "dryRun": true, "remove": false, "scanMode": 0}` + + // Test all combinations! + for pIdx, params := range qParamsArr { + for vIdx, vars := range varsArr { + _, err := extractHealInitParams(vars, params, bytes.NewReader([]byte(body))) + isErrCase := false + if pIdx < 4 || vIdx < 1 { + isErrCase = true + } + + if err != ErrNone && !isErrCase { + t.Errorf("Got unexpected error: %v %v %v", pIdx, vIdx, err) + } else if err == ErrNone && isErrCase { + t.Errorf("Got no error but expected one: %v %v", pIdx, vIdx) + } + } + } +} + +type byResourceUID struct{ madmin.LockEntries } + +func (b byResourceUID) Less(i, j int) bool { + toUniqLock := func(entry madmin.LockEntry) string { + return fmt.Sprintf("%s/%s", entry.Resource, entry.ID) + } + return toUniqLock(b.LockEntries[i]) < toUniqLock(b.LockEntries[j]) +} + +func TestTopLockEntries(t *testing.T) { + locksHeld := make(map[string][]lockRequesterInfo) + var owners []string + for i := range 4 { + owners = append(owners, fmt.Sprintf("node-%d", i)) + } + + // Simulate DeleteObjects of 10 objects in a single request. i.e same lock + // request UID, but 10 different resource names associated with it. + var lris []lockRequesterInfo + uuid := mustGetUUID() + for i := range 10 { + resource := fmt.Sprintf("bucket/delete-object-%d", i) + lri := lockRequesterInfo{ + Name: resource, + Writer: true, + UID: uuid, + Owner: owners[i%len(owners)], + Group: true, + Quorum: 3, + } + lris = append(lris, lri) + locksHeld[resource] = []lockRequesterInfo{lri} + } + + // Add a few concurrent read locks to the mix + for i := range 50 { + resource := fmt.Sprintf("bucket/get-object-%d", i) + lri := lockRequesterInfo{ + Name: resource, + UID: mustGetUUID(), + Owner: owners[i%len(owners)], + Quorum: 2, + } + lris = append(lris, lri) + locksHeld[resource] = append(locksHeld[resource], lri) + // concurrent read lock, same resource different uid + lri.UID = mustGetUUID() + lris = append(lris, lri) + locksHeld[resource] = append(locksHeld[resource], lri) + } + + var peerLocks []*PeerLocks + for _, owner := range owners { + peerLocks = append(peerLocks, &PeerLocks{ + Addr: owner, + Locks: locksHeld, + }) + } + var exp madmin.LockEntries + for _, lri := range lris { + lockType := func(lri lockRequesterInfo) string { + if lri.Writer { + return "WRITE" + } + return "READ" + } + exp = append(exp, madmin.LockEntry{ + Resource: lri.Name, + Type: lockType(lri), + ServerList: owners, + Owner: lri.Owner, + ID: lri.UID, + Quorum: lri.Quorum, + Timestamp: time.Unix(0, lri.Timestamp), + }) + } + + testCases := []struct { + peerLocks []*PeerLocks + expected madmin.LockEntries + }{ + { + peerLocks: peerLocks, + expected: exp, + }, + } + + // printEntries := func(entries madmin.LockEntries) { + // for i, entry := range entries { + // fmt.Printf("%d: %s %s %s %s %v %d\n", i, entry.Resource, entry.ID, entry.Owner, entry.Type, entry.ServerList, entry.Elapsed) + // } + // } + + check := func(exp, got madmin.LockEntries) (int, bool) { + if len(exp) != len(got) { + return 0, false + } + sort.Slice(exp, byResourceUID{exp}.Less) + sort.Slice(got, byResourceUID{got}.Less) + // printEntries(exp) + // printEntries(got) + for i, e := range exp { + if !e.Timestamp.Equal(got[i].Timestamp) { + return i, false + } + // Skip checking elapsed since it's time sensitive. + // if e.Elapsed != got[i].Elapsed { + // return false + // } + if e.Resource != got[i].Resource { + return i, false + } + if e.Type != got[i].Type { + return i, false + } + if e.Source != got[i].Source { + return i, false + } + if e.Owner != got[i].Owner { + return i, false + } + if e.ID != got[i].ID { + return i, false + } + if len(e.ServerList) != len(got[i].ServerList) { + return i, false + } + for j := range e.ServerList { + if e.ServerList[j] != got[i].ServerList[j] { + return i, false + } + } + } + return 0, true + } + + for i, tc := range testCases { + got := topLockEntries(tc.peerLocks, false) + if idx, ok := check(tc.expected, got); !ok { + t.Fatalf("%d: mismatch at %d \n expected %#v but got %#v", i, idx, tc.expected[idx], got[idx]) + } + } +} diff --git a/cmd/admin-heal-ops.go b/cmd/admin-heal-ops.go new file mode 100644 index 00000000..0b297634 --- /dev/null +++ b/cmd/admin-heal-ops.go @@ -0,0 +1,918 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "maps" + "net/http" + "sort" + "sync" + "time" + + "github.com/minio/madmin-go/v3" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/logger" +) + +// healStatusSummary - overall short summary of a healing sequence +type healStatusSummary string + +// healStatusSummary constants +const ( + healNotStartedStatus healStatusSummary = "not started" + healRunningStatus = "running" + healStoppedStatus = "stopped" + healFinishedStatus = "finished" +) + +const ( + // a heal sequence with this many un-consumed heal result + // items blocks until heal-status consumption resumes or is + // aborted due to timeout. + maxUnconsumedHealResultItems = 1000 + + // if no heal-results are consumed (via the heal-status API) + // for this timeout duration, the heal sequence is aborted. + healUnconsumedTimeout = 24 * time.Hour + + // time-duration to keep heal sequence state after it + // completes. + keepHealSeqStateDuration = time.Minute * 10 + + // nopHeal is a no operating healing action to + // wait for the current healing operation to finish + nopHeal = "" +) + +var ( + errHealIdleTimeout = errors.New("healing results were not consumed for too long") + errHealStopSignalled = errors.New("heal stop signaled") + + errFnHealFromAPIErr = func(ctx context.Context, err error) error { + apiErr := toAdminAPIErr(ctx, err) + return fmt.Errorf("Heal internal error: %s: %s", + apiErr.Code, apiErr.Description) + } +) + +// healSequenceStatus - accumulated status of the heal sequence +type healSequenceStatus struct { + // summary and detail for failures + Summary healStatusSummary `json:"Summary"` + FailureDetail string `json:"Detail,omitempty"` + StartTime time.Time `json:"StartTime"` + + // settings for the heal sequence + HealSettings madmin.HealOpts `json:"Settings"` + + // slice of available heal result records + Items []madmin.HealResultItem `json:"Items"` +} + +// structure to hold state of all heal sequences in server memory +type allHealState struct { + sync.RWMutex + + // map of heal path to heal sequence + healSeqMap map[string]*healSequence // Indexed by endpoint + // keep track of the healing status of disks in the memory + // false: the disk needs to be healed but no healing routine is started + // true: the disk is currently healing + healLocalDisks map[Endpoint]bool + healStatus map[string]healingTracker // Indexed by disk ID +} + +// newHealState - initialize global heal state management +func newHealState(ctx context.Context, cleanup bool) *allHealState { + hstate := &allHealState{ + healSeqMap: make(map[string]*healSequence), + healLocalDisks: make(map[Endpoint]bool), + healStatus: make(map[string]healingTracker), + } + if cleanup { + go hstate.periodicHealSeqsClean(ctx) + } + return hstate +} + +func (ahs *allHealState) popHealLocalDisks(healLocalDisks ...Endpoint) { + ahs.Lock() + defer ahs.Unlock() + + for _, ep := range healLocalDisks { + delete(ahs.healLocalDisks, ep) + } + for id, disk := range ahs.healStatus { + for _, ep := range healLocalDisks { + if disk.Endpoint == ep.String() { + delete(ahs.healStatus, id) + } + } + } +} + +// updateHealStatus will update the heal status. +func (ahs *allHealState) updateHealStatus(tracker *healingTracker) { + ahs.Lock() + defer ahs.Unlock() + + tracker.mu.RLock() + t := *tracker + t.QueuedBuckets = append(make([]string, 0, len(tracker.QueuedBuckets)), tracker.QueuedBuckets...) + t.HealedBuckets = append(make([]string, 0, len(tracker.HealedBuckets)), tracker.HealedBuckets...) + ahs.healStatus[tracker.ID] = t + tracker.mu.RUnlock() +} + +// Sort by zone, set and disk index +func sortDisks(disks []madmin.Disk) { + sort.Slice(disks, func(i, j int) bool { + a, b := &disks[i], &disks[j] + if a.PoolIndex != b.PoolIndex { + return a.PoolIndex < b.PoolIndex + } + if a.SetIndex != b.SetIndex { + return a.SetIndex < b.SetIndex + } + return a.DiskIndex < b.DiskIndex + }) +} + +// getLocalHealingDisks returns local healing disks indexed by endpoint. +func (ahs *allHealState) getLocalHealingDisks() map[string]madmin.HealingDisk { + ahs.RLock() + defer ahs.RUnlock() + dst := make(map[string]madmin.HealingDisk, len(ahs.healStatus)) + for _, v := range ahs.healStatus { + dst[v.Endpoint] = v.toHealingDisk() + } + + return dst +} + +// getHealLocalDiskEndpoints() returns the list of disks that need +// to be healed but there is no healing routine in progress on them. +func (ahs *allHealState) getHealLocalDiskEndpoints() Endpoints { + ahs.RLock() + defer ahs.RUnlock() + + var endpoints Endpoints + for ep, healing := range ahs.healLocalDisks { + if !healing { + endpoints = append(endpoints, ep) + } + } + return endpoints +} + +// Set, in the memory, the state of the disk as currently healing or not +func (ahs *allHealState) setDiskHealingStatus(ep Endpoint, healing bool) { + ahs.Lock() + defer ahs.Unlock() + + ahs.healLocalDisks[ep] = healing +} + +func (ahs *allHealState) pushHealLocalDisks(healLocalDisks ...Endpoint) { + ahs.Lock() + defer ahs.Unlock() + + for _, ep := range healLocalDisks { + ahs.healLocalDisks[ep] = false + } +} + +func (ahs *allHealState) periodicHealSeqsClean(ctx context.Context) { + // Launch clean-up routine to remove this heal sequence (after + // it ends) from the global state after timeout has elapsed. + periodicTimer := time.NewTimer(time.Minute * 5) + defer periodicTimer.Stop() + + for { + select { + case <-periodicTimer.C: + now := UTCNow() + ahs.Lock() + for path, h := range ahs.healSeqMap { + if h.hasEnded() && h.endTime.Add(keepHealSeqStateDuration).Before(now) { + delete(ahs.healSeqMap, path) + } + } + ahs.Unlock() + + periodicTimer.Reset(time.Minute * 5) + case <-ctx.Done(): + // server could be restarting - need + // to exit immediately + return + } + } +} + +// getHealSequenceByToken - Retrieve a heal sequence by token. The second +// argument returns if a heal sequence actually exists. +func (ahs *allHealState) getHealSequenceByToken(token string) (h *healSequence, exists bool) { + ahs.RLock() + defer ahs.RUnlock() + for _, healSeq := range ahs.healSeqMap { + if healSeq.clientToken == token { + return healSeq, true + } + } + return nil, false +} + +// getHealSequence - Retrieve a heal sequence by path. The second +// argument returns if a heal sequence actually exists. +func (ahs *allHealState) getHealSequence(path string) (h *healSequence, exists bool) { + ahs.RLock() + defer ahs.RUnlock() + h, exists = ahs.healSeqMap[path] + return h, exists +} + +func (ahs *allHealState) stopHealSequence(path string) ([]byte, APIError) { + var hsp madmin.HealStopSuccess + he, exists := ahs.getHealSequence(path) + if !exists { + hsp = madmin.HealStopSuccess{ + ClientToken: "unknown", + StartTime: UTCNow(), + } + } else { + clientToken := he.clientToken + if globalIsDistErasure { + clientToken = fmt.Sprintf("%s%s%d", he.clientToken, getKeySeparator(), GetProxyEndpointLocalIndex(globalProxyEndpoints)) + } + + hsp = madmin.HealStopSuccess{ + ClientToken: clientToken, + ClientAddress: he.clientAddress, + StartTime: he.startTime, + } + + he.stop() + for !he.hasEnded() { + time.Sleep(1 * time.Second) + } + ahs.Lock() + defer ahs.Unlock() + // Heal sequence explicitly stopped, remove it. + delete(ahs.healSeqMap, path) + } + + b, err := json.Marshal(&hsp) + return b, toAdminAPIErr(GlobalContext, err) +} + +// LaunchNewHealSequence - launches a background routine that performs +// healing according to the healSequence argument. For each heal +// sequence, state is stored in the `globalAllHealState`, which is a +// map of the heal path to `healSequence` which holds state about the +// heal sequence. +// +// Heal results are persisted in server memory for +// `keepHealSeqStateDuration`. This function also launches a +// background routine to clean up heal results after the +// aforementioned duration. +func (ahs *allHealState) LaunchNewHealSequence(h *healSequence, objAPI ObjectLayer) ( + respBytes []byte, apiErr APIError, errMsg string, +) { + if h.forceStarted { + _, apiErr = ahs.stopHealSequence(pathJoin(h.bucket, h.object)) + if apiErr.Code != "" { + return respBytes, apiErr, "" + } + } else { + oh, exists := ahs.getHealSequence(pathJoin(h.bucket, h.object)) + if exists && !oh.hasEnded() { + errMsg = "Heal is already running on the given path " + + "(use force-start option to stop and start afresh). " + + fmt.Sprintf("The heal was started by IP %s at %s, token is %s", + h.clientAddress, h.startTime.Format(http.TimeFormat), h.clientToken) + return nil, errorCodes.ToAPIErr(ErrHealAlreadyRunning), errMsg + } + } + + ahs.Lock() + defer ahs.Unlock() + + // Check if new heal sequence to be started overlaps with any + // existing, running sequence + hpath := pathJoin(h.bucket, h.object) + for k, hSeq := range ahs.healSeqMap { + if !hSeq.hasEnded() && (HasPrefix(k, hpath) || HasPrefix(hpath, k)) { + errMsg = "The provided heal sequence path overlaps with an existing " + + fmt.Sprintf("heal path: %s", k) + return nil, errorCodes.ToAPIErr(ErrHealOverlappingPaths), errMsg + } + } + + // Add heal state and start sequence + ahs.healSeqMap[hpath] = h + + clientToken := h.clientToken + if globalIsDistErasure { + clientToken = fmt.Sprintf("%s%s%d", h.clientToken, getKeySeparator(), GetProxyEndpointLocalIndex(globalProxyEndpoints)) + } + + if h.clientToken == bgHealingUUID { + // For background heal do nothing, do not spawn an unnecessary goroutine. + } else { + // Launch top-level background heal go-routine + go h.healSequenceStart(objAPI) + } + + b, err := json.Marshal(madmin.HealStartSuccess{ + ClientToken: clientToken, + ClientAddress: h.clientAddress, + StartTime: h.startTime, + }) + if err != nil { + bugLogIf(h.ctx, err) + return nil, toAdminAPIErr(h.ctx, err), "" + } + return b, noError, "" +} + +// PopHealStatusJSON - Called by heal-status API. It fetches the heal +// status results from global state and returns its JSON +// representation. The clientToken helps ensure there aren't +// conflicting clients fetching status. +func (ahs *allHealState) PopHealStatusJSON(hpath string, + clientToken string) ([]byte, APIErrorCode, +) { + // fetch heal state for given path + h, exists := ahs.getHealSequence(hpath) + if !exists { + // heal sequence doesn't exist, must have finished. + jbytes, err := json.Marshal(healSequenceStatus{ + Summary: healFinishedStatus, + }) + return jbytes, toAdminAPIErrCode(GlobalContext, err) + } + + // Check if client-token is valid + if clientToken != h.clientToken { + return nil, ErrHealInvalidClientToken + } + + // Take lock to access and update the heal-sequence + h.mutex.Lock() + defer h.mutex.Unlock() + + numItems := len(h.currentStatus.Items) + + // calculate index of most recently available heal result + // record. + lastResultIndex := h.lastSentResultIndex + if numItems > 0 { + lastResultIndex = h.currentStatus.Items[numItems-1].ResultIndex + } + + h.lastSentResultIndex = lastResultIndex + + jbytes, err := json.Marshal(h.currentStatus) + if err != nil { + h.currentStatus.Items = nil + + bugLogIf(h.ctx, err) + return nil, ErrInternalError + } + + h.currentStatus.Items = nil + + return jbytes, ErrNone +} + +// healSource denotes single entity and heal option. +type healSource struct { + bucket string + object string + versionID string + noWait bool // a non blocking call, if task queue is full return right away. + opts *madmin.HealOpts // optional heal option overrides default setting +} + +// healSequence - state for each heal sequence initiated on the +// server. +type healSequence struct { + // bucket, and object on which heal seq. was initiated + bucket, object string + + // Report healing progress + reportProgress bool + + // time at which heal sequence was started + startTime time.Time + + // time at which heal sequence has ended + endTime time.Time + + // Heal client info + clientToken, clientAddress string + + // was this heal sequence force started? + forceStarted bool + + // heal settings applied to this heal sequence + settings madmin.HealOpts + + // current accumulated status of the heal sequence + currentStatus healSequenceStatus + + // channel signaled by background routine when traversal has + // completed + traverseAndHealDoneCh chan error + + // canceler to cancel heal sequence. + cancelCtx context.CancelFunc + + // the last result index sent to client + lastSentResultIndex int64 + + // Number of total items scanned against item type + scannedItemsMap map[madmin.HealItemType]int64 + + // Number of total items healed against item type + healedItemsMap map[madmin.HealItemType]int64 + + // Number of total items where healing failed against item type + healFailedItemsMap map[madmin.HealItemType]int64 + + // The time of the last scan/heal activity + lastHealActivity time.Time + + // Holds the request-info for logging + ctx context.Context + + // used to lock this structure as it is concurrently accessed + mutex sync.RWMutex +} + +// NewHealSequence - creates healSettings, assumes bucket and +// objPrefix are already validated. +func newHealSequence(ctx context.Context, bucket, objPrefix, clientAddr string, + hs madmin.HealOpts, forceStart bool, +) *healSequence { + reqInfo := &logger.ReqInfo{RemoteHost: clientAddr, API: "Heal", BucketName: bucket} + reqInfo.AppendTags("prefix", objPrefix) + ctx, cancel := context.WithCancel(logger.SetReqInfo(ctx, reqInfo)) + + clientToken := mustGetUUID() + + return &healSequence{ + bucket: bucket, + object: objPrefix, + reportProgress: true, + startTime: UTCNow(), + clientToken: clientToken, + clientAddress: clientAddr, + forceStarted: forceStart, + settings: hs, + currentStatus: healSequenceStatus{ + Summary: healNotStartedStatus, + HealSettings: hs, + }, + traverseAndHealDoneCh: make(chan error), + cancelCtx: cancel, + ctx: ctx, + scannedItemsMap: make(map[madmin.HealItemType]int64), + healedItemsMap: make(map[madmin.HealItemType]int64), + healFailedItemsMap: make(map[madmin.HealItemType]int64), + } +} + +// getScannedItemsCount - returns a count of all scanned items +func (h *healSequence) getScannedItemsCount() int64 { + var count int64 + h.mutex.RLock() + defer h.mutex.RUnlock() + + for _, v := range h.scannedItemsMap { + count += v + } + return count +} + +// getScannedItemsMap - returns map of all scanned items against type +func (h *healSequence) getScannedItemsMap() map[madmin.HealItemType]int64 { + h.mutex.RLock() + defer h.mutex.RUnlock() + + // Make a copy before returning the value + retMap := make(map[madmin.HealItemType]int64, len(h.scannedItemsMap)) + maps.Copy(retMap, h.scannedItemsMap) + + return retMap +} + +// getHealedItemsMap - returns the map of all healed items against type +func (h *healSequence) getHealedItemsMap() map[madmin.HealItemType]int64 { + h.mutex.RLock() + defer h.mutex.RUnlock() + + // Make a copy before returning the value + retMap := make(map[madmin.HealItemType]int64, len(h.healedItemsMap)) + maps.Copy(retMap, h.healedItemsMap) + + return retMap +} + +// getHealFailedItemsMap - returns map of all items where heal failed against +// drive endpoint and status +func (h *healSequence) getHealFailedItemsMap() map[madmin.HealItemType]int64 { + h.mutex.RLock() + defer h.mutex.RUnlock() + + // Make a copy before returning the value + retMap := make(map[madmin.HealItemType]int64, len(h.healFailedItemsMap)) + maps.Copy(retMap, h.healFailedItemsMap) + + return retMap +} + +func (h *healSequence) countFailed(healType madmin.HealItemType) { + h.mutex.Lock() + defer h.mutex.Unlock() + + h.healFailedItemsMap[healType]++ + h.lastHealActivity = UTCNow() +} + +func (h *healSequence) countScanned(healType madmin.HealItemType) { + h.mutex.Lock() + defer h.mutex.Unlock() + + h.scannedItemsMap[healType]++ + h.lastHealActivity = UTCNow() +} + +func (h *healSequence) countHealed(healType madmin.HealItemType) { + h.mutex.Lock() + defer h.mutex.Unlock() + + h.healedItemsMap[healType]++ + h.lastHealActivity = UTCNow() +} + +// isQuitting - determines if the heal sequence is quitting (due to an +// external signal) +func (h *healSequence) isQuitting() bool { + select { + case <-h.ctx.Done(): + return true + default: + return false + } +} + +// check if the heal sequence has ended +func (h *healSequence) hasEnded() bool { + h.mutex.RLock() + defer h.mutex.RUnlock() + // background heal never ends + if h.clientToken == bgHealingUUID { + return false + } + return !h.endTime.IsZero() +} + +// stops the heal sequence - safe to call multiple times. +func (h *healSequence) stop() { + h.cancelCtx() +} + +// pushHealResultItem - pushes a heal result item for consumption in +// the heal-status API. It blocks if there are +// maxUnconsumedHealResultItems. When it blocks, the heal sequence +// routine is effectively paused - this happens when the server has +// accumulated the maximum number of heal records per heal +// sequence. When the client consumes further records, the heal +// sequence automatically resumes. The return value indicates if the +// operation succeeded. +func (h *healSequence) pushHealResultItem(r madmin.HealResultItem) error { + // start a timer to keep an upper time limit to find an empty + // slot to add the given heal result - if no slot is found it + // means that the server is holding the maximum amount of + // heal-results in memory and the client has not consumed it + // for too long. + unconsumedTimer := time.NewTimer(healUnconsumedTimeout) + defer unconsumedTimer.Stop() + + var itemsLen int + for { + h.mutex.Lock() + itemsLen = len(h.currentStatus.Items) + if itemsLen == maxUnconsumedHealResultItems { + // wait for a second, or quit if an external + // stop signal is received or the + // unconsumedTimer fires. + select { + // Check after a second + case <-time.After(time.Second): + h.mutex.Unlock() + continue + + case <-h.ctx.Done(): + h.mutex.Unlock() + // discard result and return. + return errHealStopSignalled + + // Timeout if no results consumed for too long. + case <-unconsumedTimer.C: + h.mutex.Unlock() + return errHealIdleTimeout + } + } + break + } + + // Set the correct result index for the new result item + if itemsLen > 0 { + r.ResultIndex = 1 + h.currentStatus.Items[itemsLen-1].ResultIndex + } else { + r.ResultIndex = 1 + h.lastSentResultIndex + } + + // append to results + h.currentStatus.Items = append(h.currentStatus.Items, r) + + // release lock + h.mutex.Unlock() + + return nil +} + +// healSequenceStart - this is the top-level background heal +// routine. It launches another go-routine that actually traverses +// on-disk data, checks and heals according to the selected +// settings. This go-routine itself, (1) monitors the traversal +// routine for completion, and (2) listens for external stop +// signals. When either event happens, it sets the finish status for +// the heal-sequence. +func (h *healSequence) healSequenceStart(objAPI ObjectLayer) { + // Set status as running + h.mutex.Lock() + h.currentStatus.Summary = healRunningStatus + h.currentStatus.StartTime = UTCNow() + h.mutex.Unlock() + + go h.traverseAndHeal(objAPI) + + select { + case err, ok := <-h.traverseAndHealDoneCh: + if !ok { + return + } + h.mutex.Lock() + h.endTime = UTCNow() + // Heal traversal is complete. + if err == nil { + // heal traversal succeeded. + h.currentStatus.Summary = healFinishedStatus + } else { + // heal traversal had an error. + h.currentStatus.Summary = healStoppedStatus + h.currentStatus.FailureDetail = err.Error() + } + h.mutex.Unlock() + case <-h.ctx.Done(): + h.mutex.Lock() + h.endTime = UTCNow() + h.currentStatus.Summary = healFinishedStatus + h.mutex.Unlock() + + // drain traverse channel so the traversal + // go-routine does not leak. + go func() { + // Eventually the traversal go-routine closes + // the channel and returns, so this go-routine + // itself will not leak. + <-h.traverseAndHealDoneCh + }() + } +} + +func (h *healSequence) queueHealTask(source healSource, healType madmin.HealItemType) error { + // Send heal request + task := healTask{ + bucket: source.bucket, + object: source.object, + versionID: source.versionID, + opts: h.settings, + } + if source.opts != nil { + task.opts = *source.opts + } else { + task.opts.ScanMode = madmin.HealNormalScan + } + + h.countScanned(healType) + + if source.noWait { + select { + case globalBackgroundHealRoutine.tasks <- task: + if serverDebugLog { + fmt.Printf("Task in the queue: %#v\n", task) + } + default: + // task queue is full, no more workers, we shall move on and heal later. + return nil + } + // Don't wait for result + return nil + } + + // respCh must be set to wait for result. + // We make it size 1, so a result can always be written + // even if we aren't listening. + task.respCh = make(chan healResult, 1) + select { + case globalBackgroundHealRoutine.tasks <- task: + if serverDebugLog { + fmt.Printf("Task in the queue: %#v\n", task) + } + case <-h.ctx.Done(): + return nil + } + + countOKDrives := func(drives []madmin.HealDriveInfo) (count int) { + for _, drive := range drives { + if drive.State == madmin.DriveStateOk { + count++ + } + } + return count + } + + // task queued, now wait for the response. + select { + case res := <-task.respCh: + if res.err == nil { + h.countHealed(healType) + } else { + h.countFailed(healType) + } + if !h.reportProgress { + if errors.Is(res.err, errSkipFile) { // this is only sent usually by nopHeal + return nil + } + + // Report caller of any failure + return res.err + } + res.result.Type = healType + if res.err != nil { + res.result.Detail = res.err.Error() + } + if res.result.ParityBlocks > 0 && res.result.DataBlocks > 0 && res.result.DataBlocks > res.result.ParityBlocks { + if got := countOKDrives(res.result.After.Drives); got < res.result.ParityBlocks { + res.result.Detail = fmt.Sprintf("quorum loss - expected %d minimum, got drive states in OK %d", res.result.ParityBlocks, got) + } + } + return h.pushHealResultItem(res.result) + case <-h.ctx.Done(): + return nil + } +} + +func (h *healSequence) healDiskMeta(objAPI ObjectLayer) error { + // Start healing the config prefix. + return h.healMinioSysMeta(objAPI, minioConfigPrefix)() +} + +func (h *healSequence) healItems(objAPI ObjectLayer) error { + if h.clientToken == bgHealingUUID { + // For background heal do nothing. + return nil + } + + if h.bucket == "" { // heal internal meta only during a site-wide heal + if err := h.healDiskMeta(objAPI); err != nil { + return err + } + } + + // Heal buckets and objects + return h.healBuckets(objAPI) +} + +// traverseAndHeal - traverses on-disk data and performs healing +// according to settings. At each "safe" point it also checks if an +// external quit signal has been received and quits if so. Since the +// healing traversal may be mutating on-disk data when an external +// quit signal is received, this routine cannot quit immediately and +// has to wait until a safe point is reached, such as between scanning +// two objects. +func (h *healSequence) traverseAndHeal(objAPI ObjectLayer) { + h.traverseAndHealDoneCh <- h.healItems(objAPI) + xioutil.SafeClose(h.traverseAndHealDoneCh) +} + +// healMinioSysMeta - heals all files under a given meta prefix, returns a function +// which in-turn heals the respective meta directory path and any files in int. +func (h *healSequence) healMinioSysMeta(objAPI ObjectLayer, metaPrefix string) func() error { + return func() error { + // NOTE: Healing on meta is run regardless + // of any bucket being selected, this is to ensure that + // meta are always upto date and correct. + h.settings.Recursive = true + return objAPI.HealObjects(h.ctx, minioMetaBucket, metaPrefix, h.settings, func(bucket, object, versionID string, scanMode madmin.HealScanMode) error { + if h.isQuitting() { + return errHealStopSignalled + } + + err := h.queueHealTask(healSource{ + bucket: bucket, + object: object, + versionID: versionID, + }, madmin.HealItemBucketMetadata) + return err + }) + } +} + +// healBuckets - check for all buckets heal or just particular bucket. +func (h *healSequence) healBuckets(objAPI ObjectLayer) error { + if h.isQuitting() { + return errHealStopSignalled + } + + // 1. If a bucket was specified, heal only the bucket. + if h.bucket != "" { + return h.healBucket(objAPI, h.bucket, false) + } + + buckets, err := objAPI.ListBuckets(h.ctx, BucketOptions{}) + if err != nil { + return errFnHealFromAPIErr(h.ctx, err) + } + + // Heal latest buckets first. + sort.Slice(buckets, func(i, j int) bool { + return buckets[i].Created.After(buckets[j].Created) + }) + + for _, bucket := range buckets { + if err = h.healBucket(objAPI, bucket.Name, false); err != nil { + return err + } + } + + return nil +} + +// healBucket - traverses and heals given bucket +func (h *healSequence) healBucket(objAPI ObjectLayer, bucket string, bucketsOnly bool) error { + if err := h.queueHealTask(healSource{bucket: bucket}, madmin.HealItemBucket); err != nil { + return err + } + + if bucketsOnly { + return nil + } + + if err := objAPI.HealObjects(h.ctx, bucket, h.object, h.settings, h.healObject); err != nil { + return errFnHealFromAPIErr(h.ctx, err) + } + return nil +} + +// healObject - heal the given object and record result +func (h *healSequence) healObject(bucket, object, versionID string, scanMode madmin.HealScanMode) error { + if h.isQuitting() { + return errHealStopSignalled + } + + err := h.queueHealTask(healSource{ + bucket: bucket, + object: object, + versionID: versionID, + opts: &h.settings, + }, madmin.HealItemObject) + + // Wait and proceed if there are active requests + waitForLowHTTPReq() + + return err +} diff --git a/cmd/admin-router.go b/cmd/admin-router.go new file mode 100644 index 00000000..d218e475 --- /dev/null +++ b/cmd/admin-router.go @@ -0,0 +1,441 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "net/http" + + "github.com/klauspost/compress/gzhttp" + "github.com/klauspost/compress/gzip" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/logger" + "github.com/minio/mux" +) + +const ( + adminPathPrefix = minioReservedBucketPath + "/admin" + adminAPIVersion = madmin.AdminAPIVersion + adminAPIVersionPrefix = SlashSeparator + adminAPIVersion + adminAPISiteReplicationDevNull = "/site-replication/devnull" + adminAPISiteReplicationNetPerf = "/site-replication/netperf" + adminAPIClientDevNull = "/speedtest/client/devnull" + adminAPIClientDevExtraTime = "/speedtest/client/devnull/extratime" +) + +var gzipHandler = func() func(http.Handler) http.HandlerFunc { + gz, err := gzhttp.NewWrapper(gzhttp.MinSize(1000), gzhttp.CompressionLevel(gzip.BestSpeed)) + if err != nil { + // Static params, so this is very unlikely. + logger.Fatal(err, "Unable to initialize server") + } + return gz +}() + +// Set of handler options as bit flags +type hFlag uint8 + +const ( + // this flag disables gzip compression of responses + noGZFlag = 1 << iota + + // this flag enables tracing body and headers instead of just headers + traceAllFlag + + // pass this flag to skip checking if object layer is available + noObjLayerFlag +) + +// Has checks if the given flag is enabled in `h`. +func (h hFlag) Has(flag hFlag) bool { + // Use bitwise-AND and check if the result is non-zero. + return h&flag != 0 +} + +// adminMiddleware performs some common admin handler functionality for all +// handlers: +// +// - updates request context with `logger.ReqInfo` and api name based on the +// name of the function handler passed (this handler must be a method of +// `adminAPIHandlers`). +// +// - sets up call to send AuditLog +// +// While this is a middleware function (i.e. it takes a handler function and +// returns one), due to flags being passed based on required conditions, it is +// done per-"handler function registration" in the router. +// +// The passed in handler function must be a method of `adminAPIHandlers` for the +// name displayed in logs and trace to be accurate. The name is extracted via +// reflection. +// +// When no flags are passed, gzip compression, http tracing of headers and +// checking of object layer availability are all enabled. Use flags to modify +// this behavior. +func adminMiddleware(f http.HandlerFunc, flags ...hFlag) http.HandlerFunc { + // Collect all flags with bitwise-OR and assign operator + var handlerFlags hFlag + for _, flag := range flags { + handlerFlags |= flag + } + + // Get name of the handler using reflection. + handlerName := getHandlerName(f, "adminAPIHandlers") + + var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { + // Update request context with `logger.ReqInfo`. + r = r.WithContext(newContext(r, w, handlerName)) + + defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r)) + + // Check if object layer is available, if not return error early. + if !handlerFlags.Has(noObjLayerFlag) { + objectAPI := newObjectLayerFn() + if objectAPI == nil || globalNotificationSys == nil { + writeErrorResponseJSON(r.Context(), w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + } + + // Apply http tracing "middleware" based on presence of flag. + var f2 http.HandlerFunc + if handlerFlags.Has(traceAllFlag) { + f2 = httpTraceAll(f) + } else { + f2 = httpTraceHdrs(f) + } + + // call the final handler + f2(w, r) + } + + // Enable compression of responses based on presence of flag. + if !handlerFlags.Has(noGZFlag) { + handler = gzipHandler(handler) + } + + return handler +} + +// adminAPIHandlers provides HTTP handlers for MinIO admin API. +type adminAPIHandlers struct{} + +// registerAdminRouter - Add handler functions for each service REST API routes. +func registerAdminRouter(router *mux.Router, enableConfigOps bool) { + adminAPI := adminAPIHandlers{} + // Admin router + adminRouter := router.PathPrefix(adminPathPrefix).Subrouter() + + adminVersions := []string{ + adminAPIVersionPrefix, + } + + for _, adminVersion := range adminVersions { + // Restart and stop MinIO service type=2 + adminRouter.Methods(http.MethodPost).Path(adminVersion+"/service").HandlerFunc(adminMiddleware(adminAPI.ServiceV2Handler, traceAllFlag)).Queries("action", "{action:.*}", "type", "2") + + // Deprecated: Restart and stop MinIO service. + adminRouter.Methods(http.MethodPost).Path(adminVersion+"/service").HandlerFunc(adminMiddleware(adminAPI.ServiceHandler, traceAllFlag)).Queries("action", "{action:.*}") + + // Update all MinIO servers type=2 + adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update").HandlerFunc(adminMiddleware(adminAPI.ServerUpdateV2Handler, traceAllFlag)).Queries("updateURL", "{updateURL:.*}", "type", "2") + + // Deprecated: Update MinIO servers. + adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update").HandlerFunc(adminMiddleware(adminAPI.ServerUpdateHandler, traceAllFlag)).Queries("updateURL", "{updateURL:.*}") + + // Info operations + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/info").HandlerFunc(adminMiddleware(adminAPI.ServerInfoHandler, traceAllFlag, noObjLayerFlag)) + adminRouter.Methods(http.MethodGet, http.MethodPost).Path(adminVersion + "/inspect-data").HandlerFunc(adminMiddleware(adminAPI.InspectDataHandler, noGZFlag, traceHdrsS3HFlag)) + + // StorageInfo operations + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/storageinfo").HandlerFunc(adminMiddleware(adminAPI.StorageInfoHandler, traceAllFlag)) + // DataUsageInfo operations + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/datausageinfo").HandlerFunc(adminMiddleware(adminAPI.DataUsageInfoHandler, traceAllFlag)) + // Metrics operation + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/metrics").HandlerFunc(adminMiddleware(adminAPI.MetricsHandler, traceHdrsS3HFlag)) + + if globalIsDistErasure || globalIsErasure { + // Heal operations + + // Heal processing endpoint. + adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/").HandlerFunc(adminMiddleware(adminAPI.HealHandler, traceAllFlag)) + adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}").HandlerFunc(adminMiddleware(adminAPI.HealHandler, traceAllFlag)) + adminRouter.Methods(http.MethodPost).Path(adminVersion + "/heal/{bucket}/{prefix:.*}").HandlerFunc(adminMiddleware(adminAPI.HealHandler, traceAllFlag)) + adminRouter.Methods(http.MethodPost).Path(adminVersion + "/background-heal/status").HandlerFunc(adminMiddleware(adminAPI.BackgroundHealStatusHandler, traceAllFlag)) + + // Pool operations + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/pools/list").HandlerFunc(adminMiddleware(adminAPI.ListPools, traceAllFlag)) + adminRouter.Methods(http.MethodGet).Path(adminVersion+"/pools/status").HandlerFunc(adminMiddleware(adminAPI.StatusPool, traceAllFlag)).Queries("pool", "{pool:.*}") + + adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/decommission").HandlerFunc(adminMiddleware(adminAPI.StartDecommission, traceAllFlag)).Queries("pool", "{pool:.*}") + adminRouter.Methods(http.MethodPost).Path(adminVersion+"/pools/cancel").HandlerFunc(adminMiddleware(adminAPI.CancelDecommission, traceAllFlag)).Queries("pool", "{pool:.*}") + + // Rebalance operations + adminRouter.Methods(http.MethodPost).Path(adminVersion + "/rebalance/start").HandlerFunc(adminMiddleware(adminAPI.RebalanceStart, traceAllFlag)) + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/rebalance/status").HandlerFunc(adminMiddleware(adminAPI.RebalanceStatus, traceAllFlag)) + adminRouter.Methods(http.MethodPost).Path(adminVersion + "/rebalance/stop").HandlerFunc(adminMiddleware(adminAPI.RebalanceStop, traceAllFlag)) + } + + // Profiling operations - deprecated API + adminRouter.Methods(http.MethodPost).Path(adminVersion+"/profiling/start").HandlerFunc(adminMiddleware(adminAPI.StartProfilingHandler, traceAllFlag, noObjLayerFlag)). + Queries("profilerType", "{profilerType:.*}") + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/profiling/download").HandlerFunc(adminMiddleware(adminAPI.DownloadProfilingHandler, traceHdrsS3HFlag, noObjLayerFlag)) + // Profiling operations + adminRouter.Methods(http.MethodPost).Path(adminVersion + "/profile").HandlerFunc(adminMiddleware(adminAPI.ProfileHandler, traceHdrsS3HFlag, noObjLayerFlag)) + + // Config KV operations. + if enableConfigOps { + adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-config-kv").HandlerFunc(adminMiddleware(adminAPI.GetConfigKVHandler)).Queries("key", "{key:.*}") + adminRouter.Methods(http.MethodPut).Path(adminVersion + "/set-config-kv").HandlerFunc(adminMiddleware(adminAPI.SetConfigKVHandler)) + adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/del-config-kv").HandlerFunc(adminMiddleware(adminAPI.DelConfigKVHandler)) + } + + // Enable config help in all modes. + adminRouter.Methods(http.MethodGet).Path(adminVersion+"/help-config-kv").HandlerFunc(adminMiddleware(adminAPI.HelpConfigKVHandler, traceAllFlag)).Queries("subSys", "{subSys:.*}", "key", "{key:.*}") + + // Config KV history operations. + if enableConfigOps { + adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-config-history-kv").HandlerFunc(adminMiddleware(adminAPI.ListConfigHistoryKVHandler, traceAllFlag)).Queries("count", "{count:[0-9]+}") + adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/clear-config-history-kv").HandlerFunc(adminMiddleware(adminAPI.ClearConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}") + adminRouter.Methods(http.MethodPut).Path(adminVersion+"/restore-config-history-kv").HandlerFunc(adminMiddleware(adminAPI.RestoreConfigHistoryKVHandler)).Queries("restoreId", "{restoreId:.*}") + } + + // Config import/export bulk operations + if enableConfigOps { + // Get config + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/config").HandlerFunc(adminMiddleware(adminAPI.GetConfigHandler)) + // Set config + adminRouter.Methods(http.MethodPut).Path(adminVersion + "/config").HandlerFunc(adminMiddleware(adminAPI.SetConfigHandler)) + } + + // -- IAM APIs -- + + // Add policy IAM + adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-canned-policy").HandlerFunc(adminMiddleware(adminAPI.AddCannedPolicy, traceAllFlag)).Queries("name", "{name:.*}") + + // Add user IAM + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/accountinfo").HandlerFunc(adminMiddleware(adminAPI.AccountInfoHandler, traceAllFlag)) + + adminRouter.Methods(http.MethodPut).Path(adminVersion+"/add-user").HandlerFunc(adminMiddleware(adminAPI.AddUser)).Queries("accessKey", "{accessKey:.*}") + + adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-status").HandlerFunc(adminMiddleware(adminAPI.SetUserStatus)).Queries("accessKey", "{accessKey:.*}").Queries("status", "{status:.*}") + + // Service accounts ops + adminRouter.Methods(http.MethodPut).Path(adminVersion + "/add-service-account").HandlerFunc(adminMiddleware(adminAPI.AddServiceAccount)) + adminRouter.Methods(http.MethodPost).Path(adminVersion+"/update-service-account").HandlerFunc(adminMiddleware(adminAPI.UpdateServiceAccount)).Queries("accessKey", "{accessKey:.*}") + adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-service-account").HandlerFunc(adminMiddleware(adminAPI.InfoServiceAccount)).Queries("accessKey", "{accessKey:.*}") + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-service-accounts").HandlerFunc(adminMiddleware(adminAPI.ListServiceAccounts)) + adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/delete-service-account").HandlerFunc(adminMiddleware(adminAPI.DeleteServiceAccount)).Queries("accessKey", "{accessKey:.*}") + + // STS accounts ops + adminRouter.Methods(http.MethodGet).Path(adminVersion+"/temporary-account-info").HandlerFunc(adminMiddleware(adminAPI.TemporaryAccountInfo)).Queries("accessKey", "{accessKey:.*}") + + // Access key (service account/STS) operations + adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-access-keys-bulk").HandlerFunc(adminMiddleware(adminAPI.ListAccessKeysBulk)).Queries("listType", "{listType:.*}") + adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-access-key").HandlerFunc(adminMiddleware(adminAPI.InfoAccessKey)).Queries("accessKey", "{accessKey:.*}") + + // Info policy IAM latest + adminRouter.Methods(http.MethodGet).Path(adminVersion+"/info-canned-policy").HandlerFunc(adminMiddleware(adminAPI.InfoCannedPolicy)).Queries("name", "{name:.*}") + // List policies latest + adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-canned-policies").HandlerFunc(adminMiddleware(adminAPI.ListBucketPolicies)).Queries("bucket", "{bucket:.*}") + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-canned-policies").HandlerFunc(adminMiddleware(adminAPI.ListCannedPolicies)) + + // Builtin IAM policy associations + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/builtin/policy-entities").HandlerFunc(adminMiddleware(adminAPI.ListPolicyMappingEntities)) + + // Remove policy IAM + adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-canned-policy").HandlerFunc(adminMiddleware(adminAPI.RemoveCannedPolicy)).Queries("name", "{name:.*}") + + // Set user or group policy + adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-user-or-group-policy"). + HandlerFunc(adminMiddleware(adminAPI.SetPolicyForUserOrGroup)). + Queries("policyName", "{policyName:.*}", "userOrGroup", "{userOrGroup:.*}", "isGroup", "{isGroup:true|false}") + + // Attach/Detach policies to/from user or group + adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/builtin/policy/{operation}").HandlerFunc(adminMiddleware(adminAPI.AttachDetachPolicyBuiltin)) + + // Remove user IAM + adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-user").HandlerFunc(adminMiddleware(adminAPI.RemoveUser)).Queries("accessKey", "{accessKey:.*}") + + // List users + adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-users").HandlerFunc(adminMiddleware(adminAPI.ListBucketUsers)).Queries("bucket", "{bucket:.*}") + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-users").HandlerFunc(adminMiddleware(adminAPI.ListUsers)) + + // User info + adminRouter.Methods(http.MethodGet).Path(adminVersion+"/user-info").HandlerFunc(adminMiddleware(adminAPI.GetUserInfo)).Queries("accessKey", "{accessKey:.*}") + // Add/Remove members from group + adminRouter.Methods(http.MethodPut).Path(adminVersion + "/update-group-members").HandlerFunc(adminMiddleware(adminAPI.UpdateGroupMembers)) + + // Get Group + adminRouter.Methods(http.MethodGet).Path(adminVersion+"/group").HandlerFunc(adminMiddleware(adminAPI.GetGroup)).Queries("group", "{group:.*}") + + // List Groups + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/groups").HandlerFunc(adminMiddleware(adminAPI.ListGroups)) + + // Set Group Status + adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-group-status").HandlerFunc(adminMiddleware(adminAPI.SetGroupStatus)).Queries("group", "{group:.*}").Queries("status", "{status:.*}") + + // Export IAM info to zipped file + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/export-iam").HandlerFunc(adminMiddleware(adminAPI.ExportIAM, noGZFlag)) + + // Import IAM info + adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-iam").HandlerFunc(adminMiddleware(adminAPI.ImportIAM, noGZFlag)) + adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-iam-v2").HandlerFunc(adminMiddleware(adminAPI.ImportIAMV2, noGZFlag)) + + // Identity Provider configuration APIs + adminRouter.Methods(http.MethodPut).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.AddIdentityProviderCfg)) + adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.UpdateIdentityProviderCfg)) + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp-config/{type}").HandlerFunc(adminMiddleware(adminAPI.ListIdentityProviderCfg)) + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.GetIdentityProviderCfg)) + adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/idp-config/{type}/{name}").HandlerFunc(adminMiddleware(adminAPI.DeleteIdentityProviderCfg)) + + // LDAP specific service accounts ops + adminRouter.Methods(http.MethodPut).Path(adminVersion + "/idp/ldap/add-service-account").HandlerFunc(adminMiddleware(adminAPI.AddServiceAccountLDAP)) + adminRouter.Methods(http.MethodGet).Path(adminVersion+"/idp/ldap/list-access-keys"). + HandlerFunc(adminMiddleware(adminAPI.ListAccessKeysLDAP)).Queries("userDN", "{userDN:.*}", "listType", "{listType:.*}") + adminRouter.Methods(http.MethodGet).Path(adminVersion+"/idp/ldap/list-access-keys-bulk"). + HandlerFunc(adminMiddleware(adminAPI.ListAccessKeysLDAPBulk)).Queries("listType", "{listType:.*}") + + // LDAP IAM operations + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/idp/ldap/policy-entities").HandlerFunc(adminMiddleware(adminAPI.ListLDAPPolicyMappingEntities)) + adminRouter.Methods(http.MethodPost).Path(adminVersion + "/idp/ldap/policy/{operation}").HandlerFunc(adminMiddleware(adminAPI.AttachDetachPolicyLDAP)) + + // OpenID specific service accounts ops + adminRouter.Methods(http.MethodGet).Path(adminVersion+"/idp/openid/list-access-keys-bulk"). + HandlerFunc(adminMiddleware(adminAPI.ListAccessKeysOpenIDBulk)).Queries("listType", "{listType:.*}") + + // -- END IAM APIs -- + + // GetBucketQuotaConfig + adminRouter.Methods(http.MethodGet).Path(adminVersion+"/get-bucket-quota").HandlerFunc( + adminMiddleware(adminAPI.GetBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}") + // PutBucketQuotaConfig + adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-bucket-quota").HandlerFunc( + adminMiddleware(adminAPI.PutBucketQuotaConfigHandler)).Queries("bucket", "{bucket:.*}") + + // Bucket replication operations + // GetBucketTargetHandler + adminRouter.Methods(http.MethodGet).Path(adminVersion+"/list-remote-targets").HandlerFunc( + adminMiddleware(adminAPI.ListRemoteTargetsHandler)).Queries("bucket", "{bucket:.*}", "type", "{type:.*}") + // SetRemoteTargetHandler + adminRouter.Methods(http.MethodPut).Path(adminVersion+"/set-remote-target").HandlerFunc( + adminMiddleware(adminAPI.SetRemoteTargetHandler)).Queries("bucket", "{bucket:.*}") + // RemoveRemoteTargetHandler + adminRouter.Methods(http.MethodDelete).Path(adminVersion+"/remove-remote-target").HandlerFunc( + adminMiddleware(adminAPI.RemoveRemoteTargetHandler)).Queries("bucket", "{bucket:.*}", "arn", "{arn:.*}") + // ReplicationDiff - MinIO extension API + adminRouter.Methods(http.MethodPost).Path(adminVersion+"/replication/diff").HandlerFunc( + adminMiddleware(adminAPI.ReplicationDiffHandler)).Queries("bucket", "{bucket:.*}") + // ReplicationMRFHandler - MinIO extension API + adminRouter.Methods(http.MethodGet).Path(adminVersion+"/replication/mrf").HandlerFunc( + adminMiddleware(adminAPI.ReplicationMRFHandler)).Queries("bucket", "{bucket:.*}") + + // Batch job operations + adminRouter.Methods(http.MethodPost).Path(adminVersion + "/start-job").HandlerFunc( + adminMiddleware(adminAPI.StartBatchJob)) + + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/list-jobs").HandlerFunc( + adminMiddleware(adminAPI.ListBatchJobs)) + + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/status-job").HandlerFunc( + adminMiddleware(adminAPI.BatchJobStatus)) + + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/describe-job").HandlerFunc( + adminMiddleware(adminAPI.DescribeBatchJob)) + adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/cancel-job").HandlerFunc( + adminMiddleware(adminAPI.CancelBatchJob)) + + // Bucket migration operations + // ExportBucketMetaHandler + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/export-bucket-metadata").HandlerFunc( + adminMiddleware(adminAPI.ExportBucketMetadataHandler)) + // ImportBucketMetaHandler + adminRouter.Methods(http.MethodPut).Path(adminVersion + "/import-bucket-metadata").HandlerFunc( + adminMiddleware(adminAPI.ImportBucketMetadataHandler)) + + // Remote Tier management operations + adminRouter.Methods(http.MethodPut).Path(adminVersion + "/tier").HandlerFunc(adminMiddleware(adminAPI.AddTierHandler)) + adminRouter.Methods(http.MethodPost).Path(adminVersion + "/tier/{tier}").HandlerFunc(adminMiddleware(adminAPI.EditTierHandler)) + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier").HandlerFunc(adminMiddleware(adminAPI.ListTierHandler)) + adminRouter.Methods(http.MethodDelete).Path(adminVersion + "/tier/{tier}").HandlerFunc(adminMiddleware(adminAPI.RemoveTierHandler)) + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier/{tier}").HandlerFunc(adminMiddleware(adminAPI.VerifyTierHandler)) + // Tier stats + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/tier-stats").HandlerFunc(adminMiddleware(adminAPI.TierStatsHandler)) + + // Cluster Replication APIs + adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/add").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationAdd)) + adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/remove").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationRemove)) + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/info").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationInfo)) + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/metainfo").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationMetaInfo)) + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/status").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationStatus)) + adminRouter.Methods(http.MethodPost).Path(adminVersion + adminAPISiteReplicationDevNull).HandlerFunc(adminMiddleware(adminAPI.SiteReplicationDevNull, noObjLayerFlag)) + adminRouter.Methods(http.MethodPost).Path(adminVersion + adminAPISiteReplicationNetPerf).HandlerFunc(adminMiddleware(adminAPI.SiteReplicationNetPerf, noObjLayerFlag)) + + adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/join").HandlerFunc(adminMiddleware(adminAPI.SRPeerJoin)) + adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/peer/bucket-ops").HandlerFunc(adminMiddleware(adminAPI.SRPeerBucketOps)).Queries("bucket", "{bucket:.*}").Queries("operation", "{operation:.*}") + adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/iam-item").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateIAMItem)) + adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/bucket-meta").HandlerFunc(adminMiddleware(adminAPI.SRPeerReplicateBucketItem)) + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/site-replication/peer/idp-settings").HandlerFunc(adminMiddleware(adminAPI.SRPeerGetIDPSettings)) + adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/edit").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationEdit)) + adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/edit").HandlerFunc(adminMiddleware(adminAPI.SRPeerEdit)) + adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/peer/remove").HandlerFunc(adminMiddleware(adminAPI.SRPeerRemove)) + adminRouter.Methods(http.MethodPut).Path(adminVersion+"/site-replication/resync/op").HandlerFunc(adminMiddleware(adminAPI.SiteReplicationResyncOp)).Queries("operation", "{operation:.*}") + adminRouter.Methods(http.MethodPut).Path(adminVersion + "/site-replication/state/edit").HandlerFunc(adminMiddleware(adminAPI.SRStateEdit)) + + if globalIsDistErasure { + // Top locks + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/top/locks").HandlerFunc(adminMiddleware(adminAPI.TopLocksHandler)) + // Force unlocks paths + adminRouter.Methods(http.MethodPost).Path(adminVersion+"/force-unlock"). + Queries("paths", "{paths:.*}").HandlerFunc(adminMiddleware(adminAPI.ForceUnlockHandler)) + } + + adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest").HandlerFunc(adminMiddleware(adminAPI.ObjectSpeedTestHandler, noGZFlag)) + adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/object").HandlerFunc(adminMiddleware(adminAPI.ObjectSpeedTestHandler, noGZFlag)) + adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/drive").HandlerFunc(adminMiddleware(adminAPI.DriveSpeedtestHandler, noGZFlag)) + adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/net").HandlerFunc(adminMiddleware(adminAPI.NetperfHandler, noGZFlag)) + adminRouter.Methods(http.MethodPost).Path(adminVersion + "/speedtest/site").HandlerFunc(adminMiddleware(adminAPI.SitePerfHandler, noGZFlag)) + adminRouter.Methods(http.MethodPost).Path(adminVersion + adminAPIClientDevNull).HandlerFunc(adminMiddleware(adminAPI.ClientDevNull, noGZFlag)) + adminRouter.Methods(http.MethodPost).Path(adminVersion + adminAPIClientDevExtraTime).HandlerFunc(adminMiddleware(adminAPI.ClientDevNullExtraTime, noGZFlag)) + + // HTTP Trace + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/trace").HandlerFunc(adminMiddleware(adminAPI.TraceHandler, noObjLayerFlag)) + + // Console Logs + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/log").HandlerFunc(adminMiddleware(adminAPI.ConsoleLogHandler, traceAllFlag)) + + // -- KMS APIs -- + // + adminRouter.Methods(http.MethodPost).Path(adminVersion + "/kms/status").HandlerFunc(adminMiddleware(adminAPI.KMSStatusHandler, traceAllFlag)) + adminRouter.Methods(http.MethodPost).Path(adminVersion+"/kms/key/create").HandlerFunc(adminMiddleware(adminAPI.KMSCreateKeyHandler, traceAllFlag)).Queries("key-id", "{key-id:.*}") + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/kms/key/status").HandlerFunc(adminMiddleware(adminAPI.KMSKeyStatusHandler, traceAllFlag)) + + // Keep obdinfo for backward compatibility with mc + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/obdinfo"). + HandlerFunc(adminMiddleware(adminAPI.HealthInfoHandler)) + // -- Health API -- + adminRouter.Methods(http.MethodGet).Path(adminVersion + "/healthinfo"). + HandlerFunc(adminMiddleware(adminAPI.HealthInfoHandler)) + + // STS Revocation + adminRouter.Methods(http.MethodPost).Path(adminVersion + "/revoke-tokens/{userProvider}").HandlerFunc(adminMiddleware(adminAPI.RevokeTokens)) + } + + // If none of the routes match add default error handler routes + adminRouter.NotFoundHandler = httpTraceAll(errorResponseHandler) + adminRouter.MethodNotAllowedHandler = httpTraceAll(methodNotAllowedHandler("Admin")) +} diff --git a/cmd/admin-server-info.go b/cmd/admin-server-info.go new file mode 100644 index 00000000..4a98f9ba --- /dev/null +++ b/cmd/admin-server-info.go @@ -0,0 +1,171 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "math" + "net/http" + "os" + "runtime" + "runtime/debug" + "sort" + "strings" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/kms" + xnet "github.com/minio/pkg/v3/net" +) + +// getLocalServerProperty - returns madmin.ServerProperties for only the +// local endpoints from given list of endpoints +func getLocalServerProperty(endpointServerPools EndpointServerPools, r *http.Request, metrics bool) madmin.ServerProperties { + addr := globalLocalNodeName + if r != nil { + addr = r.Host + } + if globalIsDistErasure { + addr = globalLocalNodeName + } + poolNumbers := make(map[int]struct{}) + network := make(map[string]string) + for _, ep := range endpointServerPools { + for _, endpoint := range ep.Endpoints { + if endpoint.IsLocal { + poolNumbers[endpoint.PoolIdx+1] = struct{}{} + } + nodeName := endpoint.Host + if nodeName == "" { + nodeName = addr + } + if endpoint.IsLocal { + // Only proceed for local endpoints + network[nodeName] = string(madmin.ItemOnline) + continue + } + _, present := network[nodeName] + if !present { + if err := isServerResolvable(endpoint, 5*time.Second); err == nil { + network[nodeName] = string(madmin.ItemOnline) + } else { + if xnet.IsNetworkOrHostDown(err, false) { + network[nodeName] = string(madmin.ItemOffline) + } else if xnet.IsNetworkOrHostDown(err, true) { + network[nodeName] = "connection attempt timedout" + } + } + } + } + } + + var memstats runtime.MemStats + runtime.ReadMemStats(&memstats) + + gcStats := debug.GCStats{ + // If stats.PauseQuantiles is non-empty, ReadGCStats fills + // it with quantiles summarizing the distribution of pause time. + // For example, if len(stats.PauseQuantiles) is 5, it will be + // filled with the minimum, 25%, 50%, 75%, and maximum pause times. + PauseQuantiles: make([]time.Duration, 5), + } + debug.ReadGCStats(&gcStats) + // Truncate GC stats to max 5 entries. + if len(gcStats.PauseEnd) > 5 { + gcStats.PauseEnd = gcStats.PauseEnd[len(gcStats.PauseEnd)-5:] + } + if len(gcStats.Pause) > 5 { + gcStats.Pause = gcStats.Pause[len(gcStats.Pause)-5:] + } + + props := madmin.ServerProperties{ + Endpoint: addr, + Uptime: UTCNow().Unix() - globalBootTime.Unix(), + Version: Version, + CommitID: CommitID, + Network: network, + MemStats: madmin.MemStats{ + Alloc: memstats.Alloc, + TotalAlloc: memstats.TotalAlloc, + Mallocs: memstats.Mallocs, + Frees: memstats.Frees, + HeapAlloc: memstats.HeapAlloc, + }, + GoMaxProcs: runtime.GOMAXPROCS(0), + NumCPU: runtime.NumCPU(), + RuntimeVersion: runtime.Version(), + GCStats: &madmin.GCStats{ + LastGC: gcStats.LastGC, + NumGC: gcStats.NumGC, + PauseTotal: gcStats.PauseTotal, + Pause: gcStats.Pause, + PauseEnd: gcStats.PauseEnd, + }, + MinioEnvVars: make(map[string]string, 10), + } + + for poolNumber := range poolNumbers { + props.PoolNumbers = append(props.PoolNumbers, poolNumber) + } + sort.Ints(props.PoolNumbers) + props.PoolNumber = func() int { + if len(props.PoolNumbers) == 1 { + return props.PoolNumbers[0] + } + return math.MaxInt // this indicates that its unset. + }() + + sensitive := map[string]struct{}{ + config.EnvAccessKey: {}, + config.EnvSecretKey: {}, + config.EnvRootUser: {}, + config.EnvRootPassword: {}, + config.EnvMinIOSubnetAPIKey: {}, + kms.EnvKMSSecretKey: {}, + } + for _, v := range os.Environ() { + if !strings.HasPrefix(v, "MINIO") && !strings.HasPrefix(v, "_MINIO") { + continue + } + split := strings.SplitN(v, "=", 2) + key := split[0] + value := "" + if len(split) > 1 { + value = split[1] + } + + // Do not send sensitive creds. + if _, ok := sensitive[key]; ok || strings.Contains(strings.ToLower(key), "password") || strings.HasSuffix(strings.ToLower(key), "key") { + props.MinioEnvVars[key] = "*** EXISTS, REDACTED ***" + continue + } + props.MinioEnvVars[key] = value + } + + objLayer := newObjectLayerFn() + if objLayer != nil { + storageInfo := objLayer.LocalStorageInfo(GlobalContext, metrics) + props.State = string(madmin.ItemOnline) + props.Disks = storageInfo.Disks + } else { + props.State = string(madmin.ItemInitializing) + props.Disks = getOfflineDisks("", globalEndpoints) + } + + return props +} diff --git a/cmd/api-datatypes.go b/cmd/api-datatypes.go new file mode 100644 index 00000000..cc3bcb1c --- /dev/null +++ b/cmd/api-datatypes.go @@ -0,0 +1,85 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "encoding/xml" + "time" +) + +// DeletedObject objects deleted +type DeletedObject struct { + DeleteMarker bool `xml:"DeleteMarker,omitempty"` + DeleteMarkerVersionID string `xml:"DeleteMarkerVersionId,omitempty"` + ObjectName string `xml:"Key,omitempty"` + VersionID string `xml:"VersionId,omitempty"` + // MTime of DeleteMarker on source that needs to be propagated to replica + DeleteMarkerMTime DeleteMarkerMTime `xml:"-"` + // MinIO extensions to support delete marker replication + ReplicationState ReplicationState `xml:"-"` + + found bool // the object was found during deletion +} + +// DeleteMarkerMTime is an embedded type containing time.Time for XML marshal +type DeleteMarkerMTime struct { + time.Time +} + +// MarshalXML encodes expiration date if it is non-zero and encodes +// empty string otherwise +func (t DeleteMarkerMTime) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { + if t.IsZero() { + return nil + } + return e.EncodeElement(t.Format(time.RFC3339), startElement) +} + +// ObjectV object version key/versionId +type ObjectV struct { + ObjectName string `xml:"Key"` + VersionID string `xml:"VersionId"` +} + +// ObjectToDelete carries key name for the object to delete. +type ObjectToDelete struct { + ObjectV + // Replication status of DeleteMarker + DeleteMarkerReplicationStatus string `xml:"DeleteMarkerReplicationStatus"` + // Status of versioned delete (of object or DeleteMarker) + VersionPurgeStatus VersionPurgeStatusType `xml:"VersionPurgeStatus"` + // VersionPurgeStatuses holds the internal + VersionPurgeStatuses string `xml:"VersionPurgeStatuses"` + // ReplicateDecisionStr stringified representation of replication decision + ReplicateDecisionStr string `xml:"-"` +} + +// createBucketLocationConfiguration container for bucket configuration request from client. +// Used for parsing the location from the request body for Makebucket. +type createBucketLocationConfiguration struct { + XMLName xml.Name `xml:"CreateBucketConfiguration" json:"-"` + Location string `xml:"LocationConstraint"` +} + +// DeleteObjectsRequest - xml carrying the object key names which needs to be deleted. +type DeleteObjectsRequest struct { + // Element to enable quiet mode for the request + Quiet bool + // List of objects to be deleted + Objects []ObjectToDelete `xml:"Object"` +} diff --git a/cmd/api-errors.go b/cmd/api-errors.go new file mode 100644 index 00000000..6ccd5fab --- /dev/null +++ b/cmd/api-errors.go @@ -0,0 +1,2639 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/xml" + "errors" + "fmt" + "net/http" + "net/url" + "os" + "strconv" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/minio/minio/internal/ioutil" + "google.golang.org/api/googleapi" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/tags" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/config/dns" + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/kms" + "github.com/minio/minio/internal/logger" + + objectlock "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/versioning" + levent "github.com/minio/minio/internal/config/lambda/event" + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/hash" + "github.com/minio/pkg/v3/policy" +) + +// APIError structure +type APIError struct { + Code string + Description string + HTTPStatusCode int + ObjectSize string + RangeRequested string +} + +// APIErrorResponse - error response format +type APIErrorResponse struct { + XMLName xml.Name `xml:"Error" json:"-"` + Code string + Message string + Key string `xml:"Key,omitempty" json:"Key,omitempty"` + BucketName string `xml:"BucketName,omitempty" json:"BucketName,omitempty"` + Resource string + Region string `xml:"Region,omitempty" json:"Region,omitempty"` + RequestID string `xml:"RequestId" json:"RequestId"` + HostID string `xml:"HostId" json:"HostId"` + ActualObjectSize string `xml:"ActualObjectSize,omitempty" json:"ActualObjectSize,omitempty"` + RangeRequested string `xml:"RangeRequested,omitempty" json:"RangeRequested,omitempty"` +} + +// APIErrorCode type of error status. +type APIErrorCode int + +//go:generate stringer -type=APIErrorCode -trimprefix=Err $GOFILE + +// Error codes, non exhaustive list - http://docs.aws.amazon.com/AmazonS3/latest/API/ErrorResponses.html +const ( + ErrNone APIErrorCode = iota + ErrAccessDenied + ErrBadDigest + ErrEntityTooSmall + ErrEntityTooLarge + ErrPolicyTooLarge + ErrIncompleteBody + ErrInternalError + ErrInvalidAccessKeyID + ErrAccessKeyDisabled + ErrInvalidArgument + ErrInvalidBucketName + ErrInvalidDigest + ErrInvalidRange + ErrInvalidRangePartNumber + ErrInvalidCopyPartRange + ErrInvalidCopyPartRangeSource + ErrInvalidMaxKeys + ErrInvalidEncodingMethod + ErrInvalidMaxUploads + ErrInvalidMaxParts + ErrInvalidPartNumberMarker + ErrInvalidPartNumber + ErrInvalidRequestBody + ErrInvalidCopySource + ErrInvalidMetadataDirective + ErrInvalidCopyDest + ErrInvalidPolicyDocument + ErrInvalidObjectState + ErrMalformedXML + ErrMissingContentLength + ErrMissingContentMD5 + ErrMissingRequestBodyError + ErrMissingSecurityHeader + ErrNoSuchBucket + ErrNoSuchBucketPolicy + ErrNoSuchBucketLifecycle + ErrNoSuchLifecycleConfiguration + ErrInvalidLifecycleWithObjectLock + ErrNoSuchBucketSSEConfig + ErrNoSuchCORSConfiguration + ErrNoSuchWebsiteConfiguration + ErrReplicationConfigurationNotFoundError + ErrRemoteDestinationNotFoundError + ErrReplicationDestinationMissingLock + ErrRemoteTargetNotFoundError + ErrReplicationRemoteConnectionError + ErrReplicationBandwidthLimitError + ErrBucketRemoteIdenticalToSource + ErrBucketRemoteAlreadyExists + ErrBucketRemoteLabelInUse + ErrBucketRemoteArnTypeInvalid + ErrBucketRemoteArnInvalid + ErrBucketRemoteRemoveDisallowed + ErrRemoteTargetNotVersionedError + ErrReplicationSourceNotVersionedError + ErrReplicationNeedsVersioningError + ErrReplicationBucketNeedsVersioningError + ErrReplicationDenyEditError + ErrRemoteTargetDenyAddError + ErrReplicationNoExistingObjects + ErrReplicationValidationError + ErrReplicationPermissionCheckError + ErrObjectRestoreAlreadyInProgress + ErrNoSuchKey + ErrNoSuchUpload + ErrInvalidVersionID + ErrNoSuchVersion + ErrNotImplemented + ErrPreconditionFailed + ErrRequestTimeTooSkewed + ErrSignatureDoesNotMatch + ErrMethodNotAllowed + ErrInvalidPart + ErrInvalidPartOrder + ErrMissingPart + ErrAuthorizationHeaderMalformed + ErrMalformedPOSTRequest + ErrPOSTFileRequired + ErrSignatureVersionNotSupported + ErrBucketNotEmpty + ErrAllAccessDisabled + ErrPolicyInvalidVersion + ErrMissingFields + ErrMissingCredTag + ErrCredMalformed + ErrInvalidRegion + ErrInvalidServiceS3 + ErrInvalidServiceSTS + ErrInvalidRequestVersion + ErrMissingSignTag + ErrMissingSignHeadersTag + ErrMalformedDate + ErrMalformedPresignedDate + ErrMalformedCredentialDate + ErrMalformedExpires + ErrNegativeExpires + ErrAuthHeaderEmpty + ErrExpiredPresignRequest + ErrRequestNotReadyYet + ErrUnsignedHeaders + ErrMissingDateHeader + ErrInvalidQuerySignatureAlgo + ErrInvalidQueryParams + ErrBucketAlreadyOwnedByYou + ErrInvalidDuration + ErrBucketAlreadyExists + ErrMetadataTooLarge + ErrUnsupportedMetadata + ErrUnsupportedHostHeader + ErrMaximumExpires + ErrSlowDownRead + ErrSlowDownWrite + ErrMaxVersionsExceeded + ErrInvalidPrefixMarker + ErrBadRequest + ErrKeyTooLongError + ErrInvalidBucketObjectLockConfiguration + ErrObjectLockConfigurationNotFound + ErrObjectLockConfigurationNotAllowed + ErrNoSuchObjectLockConfiguration + ErrObjectLocked + ErrInvalidRetentionDate + ErrPastObjectLockRetainDate + ErrUnknownWORMModeDirective + ErrBucketTaggingNotFound + ErrObjectLockInvalidHeaders + ErrInvalidTagDirective + ErrPolicyAlreadyAttached + ErrPolicyNotAttached + ErrExcessData + ErrPolicyInvalidName + ErrNoTokenRevokeType + ErrAdminOpenIDNotEnabled + ErrAdminNoSuchAccessKey + // Add new error codes here. + + // SSE-S3/SSE-KMS related API errors + ErrInvalidEncryptionMethod + ErrInvalidEncryptionKeyID + + // Server-Side-Encryption (with Customer provided key) related API errors. + ErrInsecureSSECustomerRequest + ErrSSEMultipartEncrypted + ErrSSEEncryptedObject + ErrInvalidEncryptionParameters + ErrInvalidEncryptionParametersSSEC + + ErrInvalidSSECustomerAlgorithm + ErrInvalidSSECustomerKey + ErrMissingSSECustomerKey + ErrMissingSSECustomerKeyMD5 + ErrSSECustomerKeyMD5Mismatch + ErrInvalidSSECustomerParameters + ErrIncompatibleEncryptionMethod + ErrKMSNotConfigured + ErrKMSKeyNotFoundException + ErrKMSDefaultKeyAlreadyConfigured + + ErrNoAccessKey + ErrInvalidToken + + // Bucket notification related errors. + ErrEventNotification + ErrARNNotification + ErrRegionNotification + ErrOverlappingFilterNotification + ErrFilterNameInvalid + ErrFilterNamePrefix + ErrFilterNameSuffix + ErrFilterValueInvalid + ErrOverlappingConfigs + ErrUnsupportedNotification + + // S3 extended errors. + ErrContentSHA256Mismatch + ErrContentChecksumMismatch + + // Add new extended error codes here. + + // MinIO extended errors. + ErrStorageFull + ErrRequestBodyParse + ErrObjectExistsAsDirectory + ErrInvalidObjectName + ErrInvalidObjectNamePrefixSlash + ErrInvalidResourceName + ErrInvalidLifecycleQueryParameter + ErrServerNotInitialized + ErrBucketMetadataNotInitialized + ErrRequestTimedout + ErrClientDisconnected + ErrTooManyRequests + ErrInvalidRequest + ErrTransitionStorageClassNotFoundError + // MinIO storage class error codes + ErrInvalidStorageClass + ErrBackendDown + // Add new extended error codes here. + // Please open a https://github.com/minio/minio/issues before adding + // new error codes here. + + ErrMalformedJSON + ErrAdminNoSuchUser + ErrAdminNoSuchUserLDAPWarn + ErrAdminLDAPExpectedLoginName + ErrAdminNoSuchGroup + ErrAdminGroupNotEmpty + ErrAdminGroupDisabled + ErrAdminInvalidGroupName + ErrAdminNoSuchJob + ErrAdminNoSuchPolicy + ErrAdminPolicyChangeAlreadyApplied + ErrAdminInvalidArgument + ErrAdminInvalidAccessKey + ErrAdminInvalidSecretKey + ErrAdminConfigNoQuorum + ErrAdminConfigTooLarge + ErrAdminConfigBadJSON + ErrAdminNoSuchConfigTarget + ErrAdminConfigEnvOverridden + ErrAdminConfigDuplicateKeys + ErrAdminConfigInvalidIDPType + ErrAdminConfigLDAPNonDefaultConfigName + ErrAdminConfigLDAPValidation + ErrAdminConfigIDPCfgNameAlreadyExists + ErrAdminConfigIDPCfgNameDoesNotExist + ErrInsecureClientRequest + ErrObjectTampered + ErrAdminLDAPNotEnabled + + // Site-Replication errors + ErrSiteReplicationInvalidRequest + ErrSiteReplicationPeerResp + ErrSiteReplicationBackendIssue + ErrSiteReplicationServiceAccountError + ErrSiteReplicationBucketConfigError + ErrSiteReplicationBucketMetaError + ErrSiteReplicationIAMError + ErrSiteReplicationConfigMissing + ErrSiteReplicationIAMConfigMismatch + + // Pool rebalance errors + ErrAdminRebalanceAlreadyStarted + ErrAdminRebalanceNotStarted + + // Bucket Quota error codes + ErrAdminBucketQuotaExceeded + ErrAdminNoSuchQuotaConfiguration + + ErrHealNotImplemented + ErrHealNoSuchProcess + ErrHealInvalidClientToken + ErrHealMissingBucket + ErrHealAlreadyRunning + ErrHealOverlappingPaths + ErrIncorrectContinuationToken + + // S3 Select Errors + ErrEmptyRequestBody + ErrUnsupportedFunction + ErrInvalidExpressionType + ErrBusy + ErrUnauthorizedAccess + ErrExpressionTooLong + ErrIllegalSQLFunctionArgument + ErrInvalidKeyPath + ErrInvalidCompressionFormat + ErrInvalidFileHeaderInfo + ErrInvalidJSONType + ErrInvalidQuoteFields + ErrInvalidRequestParameter + ErrInvalidDataType + ErrInvalidTextEncoding + ErrInvalidDataSource + ErrInvalidTableAlias + ErrMissingRequiredParameter + ErrObjectSerializationConflict + ErrUnsupportedSQLOperation + ErrUnsupportedSQLStructure + ErrUnsupportedSyntax + ErrUnsupportedRangeHeader + ErrLexerInvalidChar + ErrLexerInvalidOperator + ErrLexerInvalidLiteral + ErrLexerInvalidIONLiteral + ErrParseExpectedDatePart + ErrParseExpectedKeyword + ErrParseExpectedTokenType + ErrParseExpected2TokenTypes + ErrParseExpectedNumber + ErrParseExpectedRightParenBuiltinFunctionCall + ErrParseExpectedTypeName + ErrParseExpectedWhenClause + ErrParseUnsupportedToken + ErrParseUnsupportedLiteralsGroupBy + ErrParseExpectedMember + ErrParseUnsupportedSelect + ErrParseUnsupportedCase + ErrParseUnsupportedCaseClause + ErrParseUnsupportedAlias + ErrParseUnsupportedSyntax + ErrParseUnknownOperator + ErrParseMissingIdentAfterAt + ErrParseUnexpectedOperator + ErrParseUnexpectedTerm + ErrParseUnexpectedToken + ErrParseUnexpectedKeyword + ErrParseExpectedExpression + ErrParseExpectedLeftParenAfterCast + ErrParseExpectedLeftParenValueConstructor + ErrParseExpectedLeftParenBuiltinFunctionCall + ErrParseExpectedArgumentDelimiter + ErrParseCastArity + ErrParseInvalidTypeParam + ErrParseEmptySelect + ErrParseSelectMissingFrom + ErrParseExpectedIdentForGroupName + ErrParseExpectedIdentForAlias + ErrParseUnsupportedCallWithStar + ErrParseNonUnaryAggregateFunctionCall + ErrParseMalformedJoin + ErrParseExpectedIdentForAt + ErrParseAsteriskIsNotAloneInSelectList + ErrParseCannotMixSqbAndWildcardInSelectList + ErrParseInvalidContextForWildcardInSelectList + ErrIncorrectSQLFunctionArgumentType + ErrValueParseFailure + ErrEvaluatorInvalidArguments + ErrIntegerOverflow + ErrLikeInvalidInputs + ErrCastFailed + ErrInvalidCast + ErrEvaluatorInvalidTimestampFormatPattern + ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing + ErrEvaluatorTimestampFormatPatternDuplicateFields + ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch + ErrEvaluatorUnterminatedTimestampFormatPatternToken + ErrEvaluatorInvalidTimestampFormatPatternToken + ErrEvaluatorInvalidTimestampFormatPatternSymbol + ErrEvaluatorBindingDoesNotExist + ErrMissingHeaders + ErrInvalidColumnIndex + + ErrAdminConfigNotificationTargetsFailed + ErrAdminProfilerNotEnabled + ErrInvalidDecompressedSize + ErrAddUserInvalidArgument + ErrAddUserValidUTF + ErrAdminResourceInvalidArgument + ErrAdminAccountNotEligible + ErrAccountNotEligible + ErrAdminServiceAccountNotFound + ErrPostPolicyConditionInvalidFormat + + ErrInvalidChecksum + + // Lambda functions + ErrLambdaARNInvalid + ErrLambdaARNNotFound + + // New Codes for GetObjectAttributes and GetObjectVersionAttributes + ErrInvalidAttributeName + + ErrAdminNoAccessKey + ErrAdminNoSecretKey + + ErrIAMNotInitialized + + apiErrCodeEnd // This is used only for the testing code +) + +type errorCodeMap map[APIErrorCode]APIError + +func (e errorCodeMap) ToAPIErrWithErr(errCode APIErrorCode, err error) APIError { + apiErr, ok := e[errCode] + if !ok { + apiErr = e[ErrInternalError] + } + if err != nil { + apiErr.Description = fmt.Sprintf("%s (%s)", apiErr.Description, err) + } + if region := globalSite.Region(); region != "" { + if errCode == ErrAuthorizationHeaderMalformed { + apiErr.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", region) + return apiErr + } + } + return apiErr +} + +func (e errorCodeMap) ToAPIErr(errCode APIErrorCode) APIError { + return e.ToAPIErrWithErr(errCode, nil) +} + +// error code to APIError structure, these fields carry respective +// descriptions for all the error responses. +var errorCodes = errorCodeMap{ + ErrInvalidCopyDest: { + Code: "InvalidRequest", + Description: "This copy request is illegal because it is trying to copy an object to itself without changing the object's metadata, storage class, website redirect location or encryption attributes.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidCopySource: { + Code: "InvalidArgument", + Description: "Copy Source must mention the source bucket and key: sourcebucket/sourcekey.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidMetadataDirective: { + Code: "InvalidArgument", + Description: "Unknown metadata directive.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidStorageClass: { + Code: "InvalidStorageClass", + Description: "Invalid storage class.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidRequestBody: { + Code: "InvalidArgument", + Description: "Body shouldn't be set for this request.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidMaxUploads: { + Code: "InvalidArgument", + Description: "Argument max-uploads must be an integer between 0 and 2147483647", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidMaxKeys: { + Code: "InvalidArgument", + Description: "Argument maxKeys must be an integer between 0 and 2147483647", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidEncodingMethod: { + Code: "InvalidArgument", + Description: "Invalid Encoding Method specified in Request", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidMaxParts: { + Code: "InvalidArgument", + Description: "Part number must be an integer between 1 and 10000, inclusive", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidPartNumberMarker: { + Code: "InvalidArgument", + Description: "Argument partNumberMarker must be an integer.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidPartNumber: { + Code: "InvalidPartNumber", + Description: "The requested partnumber is not satisfiable", + HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable, + }, + ErrInvalidPolicyDocument: { + Code: "InvalidPolicyDocument", + Description: "The content of the form does not meet the conditions specified in the policy document.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAccessDenied: { + Code: "AccessDenied", + Description: "Access Denied.", + HTTPStatusCode: http.StatusForbidden, + }, + ErrBadDigest: { + Code: "BadDigest", + Description: "The Content-Md5 you specified did not match what we received.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrEntityTooSmall: { + Code: "EntityTooSmall", + Description: "Your proposed upload is smaller than the minimum allowed object size.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrEntityTooLarge: { + Code: "EntityTooLarge", + Description: "Your proposed upload exceeds the maximum allowed object size.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrExcessData: { + Code: "ExcessData", + Description: "More data provided than indicated content length", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrPolicyInvalidName: { + Code: "PolicyInvalidName", + Description: "Policy name may not contain comma", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminOpenIDNotEnabled: { + Code: "OpenIDNotEnabled", + Description: "No enabled OpenID Connect identity providers", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrPolicyTooLarge: { + Code: "PolicyTooLarge", + Description: "Policy exceeds the maximum allowed document size.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrIncompleteBody: { + Code: "IncompleteBody", + Description: "You did not provide the number of bytes specified by the Content-Length HTTP header.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInternalError: { + Code: "InternalError", + Description: "We encountered an internal error, please try again.", + HTTPStatusCode: http.StatusInternalServerError, + }, + ErrInvalidAccessKeyID: { + Code: "InvalidAccessKeyId", + Description: "The Access Key Id you provided does not exist in our records.", + HTTPStatusCode: http.StatusForbidden, + }, + ErrAccessKeyDisabled: { + Code: "InvalidAccessKeyId", + Description: "Your account is disabled; please contact your administrator.", + HTTPStatusCode: http.StatusForbidden, + }, + ErrInvalidArgument: { + Code: "InvalidArgument", + Description: "Invalid argument", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidBucketName: { + Code: "InvalidBucketName", + Description: "The specified bucket is not valid.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidDigest: { + Code: "InvalidDigest", + Description: "The Content-Md5 you specified is not valid.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidRange: { + Code: "InvalidRange", + Description: "The requested range is not satisfiable", + HTTPStatusCode: http.StatusRequestedRangeNotSatisfiable, + }, + ErrInvalidRangePartNumber: { + Code: "InvalidRequest", + Description: "Cannot specify both Range header and partNumber query parameter", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrMalformedXML: { + Code: "MalformedXML", + Description: "The XML you provided was not well-formed or did not validate against our published schema.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrMissingContentLength: { + Code: "MissingContentLength", + Description: "You must provide the Content-Length HTTP header.", + HTTPStatusCode: http.StatusLengthRequired, + }, + ErrMissingContentMD5: { + Code: "MissingContentMD5", + Description: "Missing or invalid required header for this request: Content-Md5 or Amz-Content-Checksum", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrMissingSecurityHeader: { + Code: "MissingSecurityHeader", + Description: "Your request was missing a required header", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrMissingRequestBodyError: { + Code: "MissingRequestBodyError", + Description: "Request body is empty.", + HTTPStatusCode: http.StatusLengthRequired, + }, + ErrNoSuchBucket: { + Code: "NoSuchBucket", + Description: "The specified bucket does not exist", + HTTPStatusCode: http.StatusNotFound, + }, + ErrNoSuchBucketPolicy: { + Code: "NoSuchBucketPolicy", + Description: "The bucket policy does not exist", + HTTPStatusCode: http.StatusNotFound, + }, + ErrNoSuchBucketLifecycle: { + Code: "NoSuchBucketLifecycle", + Description: "The bucket lifecycle configuration does not exist", + HTTPStatusCode: http.StatusNotFound, + }, + ErrNoSuchLifecycleConfiguration: { + Code: "NoSuchLifecycleConfiguration", + Description: "The lifecycle configuration does not exist", + HTTPStatusCode: http.StatusNotFound, + }, + ErrInvalidLifecycleWithObjectLock: { + Code: "InvalidLifecycleWithObjectLock", + Description: "The lifecycle configuration containing MaxNoncurrentVersions is not supported with object locking", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrNoSuchBucketSSEConfig: { + Code: "ServerSideEncryptionConfigurationNotFoundError", + Description: "The server side encryption configuration was not found", + HTTPStatusCode: http.StatusNotFound, + }, + ErrNoSuchKey: { + Code: "NoSuchKey", + Description: "The specified key does not exist.", + HTTPStatusCode: http.StatusNotFound, + }, + ErrNoSuchUpload: { + Code: "NoSuchUpload", + Description: "The specified multipart upload does not exist. The upload ID may be invalid, or the upload may have been aborted or completed.", + HTTPStatusCode: http.StatusNotFound, + }, + ErrInvalidVersionID: { + Code: "InvalidArgument", + Description: "Invalid version id specified", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrNoSuchVersion: { + Code: "NoSuchVersion", + Description: "The specified version does not exist.", + HTTPStatusCode: http.StatusNotFound, + }, + ErrNotImplemented: { + Code: "NotImplemented", + Description: "A header you provided implies functionality that is not implemented", + HTTPStatusCode: http.StatusNotImplemented, + }, + ErrPreconditionFailed: { + Code: "PreconditionFailed", + Description: "At least one of the pre-conditions you specified did not hold", + HTTPStatusCode: http.StatusPreconditionFailed, + }, + ErrRequestTimeTooSkewed: { + Code: "RequestTimeTooSkewed", + Description: "The difference between the request time and the server's time is too large.", + HTTPStatusCode: http.StatusForbidden, + }, + ErrSignatureDoesNotMatch: { + Code: "SignatureDoesNotMatch", + Description: "The request signature we calculated does not match the signature you provided. Check your key and signing method.", + HTTPStatusCode: http.StatusForbidden, + }, + ErrMethodNotAllowed: { + Code: "MethodNotAllowed", + Description: "The specified method is not allowed against this resource.", + HTTPStatusCode: http.StatusMethodNotAllowed, + }, + ErrInvalidPart: { + Code: "InvalidPart", + Description: "One or more of the specified parts could not be found. The part may not have been uploaded, or the specified entity tag may not match the part's entity tag.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrMissingPart: { + Code: "InvalidRequest", + Description: "You must specify at least one part", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidPartOrder: { + Code: "InvalidPartOrder", + Description: "The list of parts was not in ascending order. The parts list must be specified in order by part number.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidObjectState: { + Code: "InvalidObjectState", + Description: "The operation is not valid for the current state of the object.", + HTTPStatusCode: http.StatusForbidden, + }, + ErrAuthorizationHeaderMalformed: { + Code: "AuthorizationHeaderMalformed", + Description: "The authorization header is malformed; the region is wrong; expecting 'us-east-1'.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrMalformedPOSTRequest: { + Code: "MalformedPOSTRequest", + Description: "The body of your POST request is not well-formed multipart/form-data.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrPOSTFileRequired: { + Code: "InvalidArgument", + Description: "POST requires exactly one file upload per request.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrSignatureVersionNotSupported: { + Code: "InvalidRequest", + Description: "The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrBucketNotEmpty: { + Code: "BucketNotEmpty", + Description: "The bucket you tried to delete is not empty", + HTTPStatusCode: http.StatusConflict, + }, + ErrBucketAlreadyExists: { + Code: "BucketAlreadyExists", + Description: "The requested bucket name is not available. The bucket namespace is shared by all users of the system. Please select a different name and try again.", + HTTPStatusCode: http.StatusConflict, + }, + ErrAllAccessDisabled: { + Code: "AllAccessDisabled", + Description: "All access to this resource has been disabled.", + HTTPStatusCode: http.StatusForbidden, + }, + ErrPolicyInvalidVersion: { + Code: "MalformedPolicy", + Description: "The policy must contain a valid version string", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrMissingFields: { + Code: "MissingFields", + Description: "Missing fields in request.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrMissingCredTag: { + Code: "InvalidRequest", + Description: "Missing Credential field for this request.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrCredMalformed: { + Code: "AuthorizationQueryParametersError", + Description: "Error parsing the X-Amz-Credential parameter; the Credential is mal-formed; expecting \"/YYYYMMDD/REGION/SERVICE/aws4_request\".", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrMalformedDate: { + Code: "MalformedDate", + Description: "Invalid date format header, expected to be in ISO8601, RFC1123 or RFC1123Z time format.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrMalformedPresignedDate: { + Code: "AuthorizationQueryParametersError", + Description: "X-Amz-Date must be in the ISO8601 Long Format \"yyyyMMdd'T'HHmmss'Z'\"", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrMalformedCredentialDate: { + Code: "AuthorizationQueryParametersError", + Description: "Error parsing the X-Amz-Credential parameter; incorrect date format. This date in the credential must be in the format \"yyyyMMdd\".", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidRegion: { + Code: "InvalidRegion", + Description: "Region does not match.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidServiceS3: { + Code: "AuthorizationParametersError", + Description: "Error parsing the Credential/X-Amz-Credential parameter; incorrect service. This endpoint belongs to \"s3\".", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidServiceSTS: { + Code: "AuthorizationParametersError", + Description: "Error parsing the Credential parameter; incorrect service. This endpoint belongs to \"sts\".", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidRequestVersion: { + Code: "AuthorizationQueryParametersError", + Description: "Error parsing the X-Amz-Credential parameter; incorrect terminal. This endpoint uses \"aws4_request\".", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrMissingSignTag: { + Code: "AccessDenied", + Description: "Signature header missing Signature field.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrMissingSignHeadersTag: { + Code: "InvalidArgument", + Description: "Signature header missing SignedHeaders field.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrMalformedExpires: { + Code: "AuthorizationQueryParametersError", + Description: "X-Amz-Expires should be a number", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrNegativeExpires: { + Code: "AuthorizationQueryParametersError", + Description: "X-Amz-Expires must be non-negative", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAuthHeaderEmpty: { + Code: "InvalidArgument", + Description: "Authorization header is invalid -- one and only one ' ' (space) required.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrMissingDateHeader: { + Code: "AccessDenied", + Description: "AWS authentication requires a valid Date or x-amz-date header", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidQuerySignatureAlgo: { + Code: "AuthorizationQueryParametersError", + Description: "X-Amz-Algorithm only supports \"AWS4-HMAC-SHA256\".", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrExpiredPresignRequest: { + Code: "AccessDenied", + Description: "Request has expired", + HTTPStatusCode: http.StatusForbidden, + }, + ErrRequestNotReadyYet: { + Code: "AccessDenied", + Description: "Request is not valid yet", + HTTPStatusCode: http.StatusForbidden, + }, + ErrSlowDownRead: { + Code: "SlowDownRead", + Description: "Resource requested is unreadable, please reduce your request rate", + HTTPStatusCode: http.StatusServiceUnavailable, + }, + ErrSlowDownWrite: { + Code: "SlowDownWrite", + Description: "Resource requested is unwritable, please reduce your request rate", + HTTPStatusCode: http.StatusServiceUnavailable, + }, + ErrMaxVersionsExceeded: { + Code: "MaxVersionsExceeded", + Description: "You've exceeded the limit on the number of versions you can create on this object", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidPrefixMarker: { + Code: "InvalidPrefixMarker", + Description: "Invalid marker prefix combination", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrBadRequest: { + Code: "BadRequest", + Description: "400 BadRequest", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrKeyTooLongError: { + Code: "KeyTooLongError", + Description: "Your key is too long", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrUnsignedHeaders: { + Code: "AccessDenied", + Description: "There were headers present in the request which were not signed", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidQueryParams: { + Code: "AuthorizationQueryParametersError", + Description: "Query-string authentication version 4 requires the X-Amz-Algorithm, X-Amz-Credential, X-Amz-Signature, X-Amz-Date, X-Amz-SignedHeaders, and X-Amz-Expires parameters.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrBucketAlreadyOwnedByYou: { + Code: "BucketAlreadyOwnedByYou", + Description: "Your previous request to create the named bucket succeeded and you already own it.", + HTTPStatusCode: http.StatusConflict, + }, + ErrInvalidDuration: { + Code: "InvalidDuration", + Description: "Duration provided in the request is invalid.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidBucketObjectLockConfiguration: { + Code: "InvalidRequest", + Description: "Bucket is missing ObjectLockConfiguration", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrBucketTaggingNotFound: { + Code: "NoSuchTagSet", + Description: "The TagSet does not exist", + HTTPStatusCode: http.StatusNotFound, + }, + ErrObjectLockConfigurationNotFound: { + Code: "ObjectLockConfigurationNotFoundError", + Description: "Object Lock configuration does not exist for this bucket", + HTTPStatusCode: http.StatusNotFound, + }, + ErrObjectLockConfigurationNotAllowed: { + Code: "InvalidBucketState", + Description: "Object Lock configuration cannot be enabled on existing buckets", + HTTPStatusCode: http.StatusConflict, + }, + ErrNoSuchCORSConfiguration: { + Code: "NoSuchCORSConfiguration", + Description: "The CORS configuration does not exist", + HTTPStatusCode: http.StatusNotFound, + }, + ErrNoSuchWebsiteConfiguration: { + Code: "NoSuchWebsiteConfiguration", + Description: "The specified bucket does not have a website configuration", + HTTPStatusCode: http.StatusNotFound, + }, + ErrReplicationConfigurationNotFoundError: { + Code: "ReplicationConfigurationNotFoundError", + Description: "The replication configuration was not found", + HTTPStatusCode: http.StatusNotFound, + }, + ErrRemoteDestinationNotFoundError: { + Code: "RemoteDestinationNotFoundError", + Description: "The remote destination bucket does not exist", + HTTPStatusCode: http.StatusNotFound, + }, + ErrReplicationDestinationMissingLock: { + Code: "ReplicationDestinationMissingLockError", + Description: "The replication destination bucket does not have object locking enabled", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrRemoteTargetNotFoundError: { + Code: "XMinioAdminRemoteTargetNotFoundError", + Description: "The remote target does not exist", + HTTPStatusCode: http.StatusNotFound, + }, + ErrReplicationRemoteConnectionError: { + Code: "XMinioAdminReplicationRemoteConnectionError", + Description: "Remote service connection error", + HTTPStatusCode: http.StatusServiceUnavailable, + }, + ErrReplicationBandwidthLimitError: { + Code: "XMinioAdminReplicationBandwidthLimitError", + Description: "Bandwidth limit for remote target must be at least 100MBps", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrReplicationNoExistingObjects: { + Code: "XMinioReplicationNoExistingObjects", + Description: "No matching ExistingObjects rule enabled", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrRemoteTargetDenyAddError: { + Code: "XMinioAdminRemoteTargetDenyAdd", + Description: "Cannot add remote target endpoint since this server is in a cluster replication setup", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrReplicationDenyEditError: { + Code: "XMinioReplicationDenyEdit", + Description: "Cannot alter local replication config since this server is in a cluster replication setup", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrBucketRemoteIdenticalToSource: { + Code: "XMinioAdminRemoteIdenticalToSource", + Description: "The remote target cannot be identical to source", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrBucketRemoteAlreadyExists: { + Code: "XMinioAdminBucketRemoteAlreadyExists", + Description: "The remote target already exists", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrBucketRemoteLabelInUse: { + Code: "XMinioAdminBucketRemoteLabelInUse", + Description: "The remote target with this label already exists", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrBucketRemoteRemoveDisallowed: { + Code: "XMinioAdminRemoteRemoveDisallowed", + Description: "This ARN is in use by an existing configuration", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrBucketRemoteArnTypeInvalid: { + Code: "XMinioAdminRemoteARNTypeInvalid", + Description: "The bucket remote ARN type is not valid", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrBucketRemoteArnInvalid: { + Code: "XMinioAdminRemoteArnInvalid", + Description: "The bucket remote ARN does not have correct format", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrRemoteTargetNotVersionedError: { + Code: "RemoteTargetNotVersionedError", + Description: "The remote target does not have versioning enabled", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrReplicationSourceNotVersionedError: { + Code: "ReplicationSourceNotVersionedError", + Description: "The replication source does not have versioning enabled", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrReplicationNeedsVersioningError: { + Code: "InvalidRequest", + Description: "Versioning must be 'Enabled' on the bucket to apply a replication configuration", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrReplicationBucketNeedsVersioningError: { + Code: "InvalidRequest", + Description: "Versioning must be 'Enabled' on the bucket to add a replication target", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrReplicationValidationError: { + Code: "InvalidRequest", + Description: "Replication validation failed on target", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrReplicationPermissionCheckError: { + Code: "ReplicationPermissionCheck", + Description: "X-Minio-Source-Replication-Check cannot be specified in request. Request cannot be completed", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrNoSuchObjectLockConfiguration: { + Code: "NoSuchObjectLockConfiguration", + Description: "The specified object does not have a ObjectLock configuration", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrObjectLocked: { + Code: "InvalidRequest", + Description: "Object is WORM protected and cannot be overwritten", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidRetentionDate: { + Code: "InvalidRequest", + Description: "Date must be provided in ISO 8601 format", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrPastObjectLockRetainDate: { + Code: "InvalidRequest", + Description: "the retain until date must be in the future", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrUnknownWORMModeDirective: { + Code: "InvalidRequest", + Description: "unknown wormMode directive", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrObjectLockInvalidHeaders: { + Code: "InvalidRequest", + Description: "x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrObjectRestoreAlreadyInProgress: { + Code: "RestoreAlreadyInProgress", + Description: "Object restore is already in progress", + HTTPStatusCode: http.StatusConflict, + }, + ErrTransitionStorageClassNotFoundError: { + Code: "TransitionStorageClassNotFoundError", + Description: "The transition storage class was not found", + HTTPStatusCode: http.StatusNotFound, + }, + + // Bucket notification related errors. + ErrEventNotification: { + Code: "InvalidArgument", + Description: "A specified event is not supported for notifications.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrARNNotification: { + Code: "InvalidArgument", + Description: "A specified destination ARN does not exist or is not well-formed. Verify the destination ARN.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrRegionNotification: { + Code: "InvalidArgument", + Description: "A specified destination is in a different region than the bucket. You must use a destination that resides in the same region as the bucket.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrOverlappingFilterNotification: { + Code: "InvalidArgument", + Description: "An object key name filtering rule defined with overlapping prefixes, overlapping suffixes, or overlapping combinations of prefixes and suffixes for the same event types.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrFilterNameInvalid: { + Code: "InvalidArgument", + Description: "filter rule name must be either prefix or suffix", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrFilterNamePrefix: { + Code: "InvalidArgument", + Description: "Cannot specify more than one prefix rule in a filter.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrFilterNameSuffix: { + Code: "InvalidArgument", + Description: "Cannot specify more than one suffix rule in a filter.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrFilterValueInvalid: { + Code: "InvalidArgument", + Description: "Size of filter rule value cannot exceed 1024 bytes in UTF-8 representation", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrOverlappingConfigs: { + Code: "InvalidArgument", + Description: "Configurations overlap. Configurations on the same bucket cannot share a common event type.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrUnsupportedNotification: { + Code: "UnsupportedNotification", + Description: "MinIO server does not support Topic or Cloud Function based notifications.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidCopyPartRange: { + Code: "InvalidArgument", + Description: "The x-amz-copy-source-range value must be of the form bytes=first-last where first and last are the zero-based offsets of the first and last bytes to copy", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidCopyPartRangeSource: { + Code: "InvalidArgument", + Description: "Range specified is not valid for source object", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrMetadataTooLarge: { + Code: "MetadataTooLarge", + Description: "Your metadata headers exceed the maximum allowed metadata size.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidTagDirective: { + Code: "InvalidArgument", + Description: "Unknown tag directive.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidEncryptionMethod: { + Code: "InvalidArgument", + Description: "Server Side Encryption with AWS KMS managed key requires HTTP header x-amz-server-side-encryption : aws:kms", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrIncompatibleEncryptionMethod: { + Code: "InvalidArgument", + Description: "Server Side Encryption with Customer provided key is incompatible with the encryption method specified", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidEncryptionKeyID: { + Code: "InvalidRequest", + Description: "The specified KMS KeyID contains unsupported characters", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInsecureSSECustomerRequest: { + Code: "InvalidRequest", + Description: "Requests specifying Server Side Encryption with Customer provided keys must be made over a secure connection.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrSSEMultipartEncrypted: { + Code: "InvalidRequest", + Description: "The multipart upload initiate requested encryption. Subsequent part requests must include the appropriate encryption parameters.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrSSEEncryptedObject: { + Code: "InvalidRequest", + Description: "The object was stored using a form of Server Side Encryption. The correct parameters must be provided to retrieve the object.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidEncryptionParameters: { + Code: "InvalidRequest", + Description: "The encryption parameters are not applicable to this object.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidEncryptionParametersSSEC: { + Code: "InvalidRequest", + Description: "SSE-C encryption parameters are not supported on replicated bucket.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidSSECustomerAlgorithm: { + Code: "InvalidArgument", + Description: "Requests specifying Server Side Encryption with Customer provided keys must provide a valid encryption algorithm.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidSSECustomerKey: { + Code: "InvalidArgument", + Description: "The secret key was invalid for the specified algorithm.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrMissingSSECustomerKey: { + Code: "InvalidArgument", + Description: "Requests specifying Server Side Encryption with Customer provided keys must provide an appropriate secret key.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrMissingSSECustomerKeyMD5: { + Code: "InvalidArgument", + Description: "Requests specifying Server Side Encryption with Customer provided keys must provide the client calculated MD5 of the secret key.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrSSECustomerKeyMD5Mismatch: { + Code: "InvalidArgument", + Description: "The calculated MD5 hash of the key did not match the hash that was provided.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidSSECustomerParameters: { + Code: "InvalidArgument", + Description: "The provided encryption parameters did not match the ones used originally.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrKMSNotConfigured: { + Code: "NotImplemented", + Description: "Server side encryption specified but KMS is not configured", + HTTPStatusCode: http.StatusNotImplemented, + }, + ErrKMSKeyNotFoundException: { + Code: "KMS.NotFoundException", + Description: "Invalid keyId", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrKMSDefaultKeyAlreadyConfigured: { + Code: "KMS.DefaultKeyAlreadyConfiguredException", + Description: "A default encryption already exists and cannot be changed on KMS", + HTTPStatusCode: http.StatusConflict, + }, + ErrNoAccessKey: { + Code: "AccessDenied", + Description: "No AWSAccessKey was presented", + HTTPStatusCode: http.StatusForbidden, + }, + ErrInvalidToken: { + Code: "InvalidTokenId", + Description: "The security token included in the request is invalid", + HTTPStatusCode: http.StatusForbidden, + }, + ErrNoTokenRevokeType: { + Code: "InvalidArgument", + Description: "No token revoke type specified and one could not be inferred from the request", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminNoSuchAccessKey: { + Code: "XMinioAdminNoSuchAccessKey", + Description: "The specified access key does not exist.", + HTTPStatusCode: http.StatusNotFound, + }, + + // S3 extensions. + ErrContentSHA256Mismatch: { + Code: "XAmzContentSHA256Mismatch", + Description: "The provided 'x-amz-content-sha256' header does not match what was computed.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrContentChecksumMismatch: { + Code: "XAmzContentChecksumMismatch", + Description: "The provided 'x-amz-checksum' header does not match what was computed.", + HTTPStatusCode: http.StatusBadRequest, + }, + + // MinIO extensions. + ErrStorageFull: { + Code: "XMinioStorageFull", + Description: "Storage backend has reached its minimum free drive threshold. Please delete a few objects to proceed.", + HTTPStatusCode: http.StatusInsufficientStorage, + }, + ErrRequestBodyParse: { + Code: "XMinioRequestBodyParse", + Description: "The request body failed to parse.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrObjectExistsAsDirectory: { + Code: "XMinioObjectExistsAsDirectory", + Description: "Object name already exists as a directory.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidObjectName: { + Code: "XMinioInvalidObjectName", + Description: "Object name contains unsupported characters.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidObjectNamePrefixSlash: { + Code: "XMinioInvalidObjectName", + Description: "Object name contains a leading slash.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidResourceName: { + Code: "XMinioInvalidResourceName", + Description: "Resource name contains bad components such as \"..\" or \".\".", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrServerNotInitialized: { + Code: "XMinioServerNotInitialized", + Description: "Server not initialized yet, please try again.", + HTTPStatusCode: http.StatusServiceUnavailable, + }, + ErrIAMNotInitialized: { + Code: "XMinioIAMNotInitialized", + Description: "IAM sub-system not initialized yet, please try again.", + HTTPStatusCode: http.StatusServiceUnavailable, + }, + ErrBucketMetadataNotInitialized: { + Code: "XMinioBucketMetadataNotInitialized", + Description: "Bucket metadata not initialized yet, please try again.", + HTTPStatusCode: http.StatusServiceUnavailable, + }, + ErrMalformedJSON: { + Code: "XMinioMalformedJSON", + Description: "The JSON you provided was not well-formed or did not validate against our published format.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidLifecycleQueryParameter: { + Code: "XMinioInvalidLifecycleParameter", + Description: "The boolean value provided for withUpdatedAt query parameter was invalid.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminNoSuchUser: { + Code: "XMinioAdminNoSuchUser", + Description: "The specified user does not exist.", + HTTPStatusCode: http.StatusNotFound, + }, + ErrAdminNoSuchUserLDAPWarn: { + Code: "XMinioAdminNoSuchUser", + Description: "The specified user does not exist. If you meant a user in LDAP, use `mc idp ldap`", + HTTPStatusCode: http.StatusNotFound, + }, + ErrAdminNoSuchGroup: { + Code: "XMinioAdminNoSuchGroup", + Description: "The specified group does not exist.", + HTTPStatusCode: http.StatusNotFound, + }, + ErrAdminNoSuchJob: { + Code: "XMinioAdminNoSuchJob", + Description: "The specified job does not exist.", + HTTPStatusCode: http.StatusNotFound, + }, + ErrAdminGroupNotEmpty: { + Code: "XMinioAdminGroupNotEmpty", + Description: "The specified group is not empty - cannot remove it.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminGroupDisabled: { + Code: "XMinioAdminGroupDisabled", + Description: "The specified group is disabled.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminNoSuchPolicy: { + Code: "XMinioAdminNoSuchPolicy", + Description: "The canned policy does not exist.", + HTTPStatusCode: http.StatusNotFound, + }, + ErrAdminPolicyChangeAlreadyApplied: { + Code: "XMinioAdminPolicyChangeAlreadyApplied", + Description: "The specified policy change is already in effect.", + HTTPStatusCode: http.StatusBadRequest, + }, + + ErrAdminInvalidArgument: { + Code: "XMinioAdminInvalidArgument", + Description: "Invalid arguments specified.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminInvalidAccessKey: { + Code: "XMinioAdminInvalidAccessKey", + Description: "The access key is invalid.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminInvalidSecretKey: { + Code: "XMinioAdminInvalidSecretKey", + Description: "The secret key is invalid.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminNoAccessKey: { + Code: "XMinioAdminNoAccessKey", + Description: "No access key was provided.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminNoSecretKey: { + Code: "XMinioAdminNoSecretKey", + Description: "No secret key was provided.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminConfigNoQuorum: { + Code: "XMinioAdminConfigNoQuorum", + Description: "Configuration update failed because server quorum was not met", + HTTPStatusCode: http.StatusServiceUnavailable, + }, + ErrAdminConfigTooLarge: { + Code: "XMinioAdminConfigTooLarge", + Description: fmt.Sprintf("Configuration data provided exceeds the allowed maximum of %d bytes", + maxEConfigJSONSize), + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminNoSuchConfigTarget: { + Code: "XMinioAdminNoSuchConfigTarget", + Description: "No such named configuration target exists", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminConfigBadJSON: { + Code: "XMinioAdminConfigBadJSON", + Description: "JSON configuration provided is of incorrect format", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminConfigEnvOverridden: { + Code: "XMinioAdminConfigEnvOverridden", + Description: "Unable to update config via Admin API due to environment variable override", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminConfigDuplicateKeys: { + Code: "XMinioAdminConfigDuplicateKeys", + Description: "JSON configuration provided has objects with duplicate keys", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminConfigInvalidIDPType: { + Code: "XMinioAdminConfigInvalidIDPType", + Description: fmt.Sprintf("Invalid IDP configuration type - must be one of %v", madmin.ValidIDPConfigTypes), + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminConfigLDAPNonDefaultConfigName: { + Code: "XMinioAdminConfigLDAPNonDefaultConfigName", + Description: "Only a single LDAP configuration is supported - config name must be empty or `_`", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminConfigLDAPValidation: { + Code: "XMinioAdminConfigLDAPValidation", + Description: "LDAP Configuration validation failed", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminConfigIDPCfgNameAlreadyExists: { + Code: "XMinioAdminConfigIDPCfgNameAlreadyExists", + Description: "An IDP configuration with the given name already exists", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminConfigIDPCfgNameDoesNotExist: { + Code: "XMinioAdminConfigIDPCfgNameDoesNotExist", + Description: "No such IDP configuration exists", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminConfigNotificationTargetsFailed: { + Code: "XMinioAdminNotificationTargetsTestFailed", + Description: "Configuration update failed due an unsuccessful attempt to connect to one or more notification servers", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminProfilerNotEnabled: { + Code: "XMinioAdminProfilerNotEnabled", + Description: "Unable to perform the requested operation because profiling is not enabled", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminBucketQuotaExceeded: { + Code: "XMinioAdminBucketQuotaExceeded", + Description: "Bucket quota exceeded", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminNoSuchQuotaConfiguration: { + Code: "XMinioAdminNoSuchQuotaConfiguration", + Description: "The quota configuration does not exist", + HTTPStatusCode: http.StatusNotFound, + }, + ErrInsecureClientRequest: { + Code: "XMinioInsecureClientRequest", + Description: "Cannot respond to plain-text request from TLS-encrypted server", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrRequestTimedout: { + Code: "RequestTimeout", + Description: "A timeout occurred while trying to lock a resource, please reduce your request rate", + HTTPStatusCode: http.StatusServiceUnavailable, + }, + ErrClientDisconnected: { + Code: "ClientDisconnected", + Description: "Client disconnected before response was ready", + HTTPStatusCode: 499, // No official code, use nginx value. + }, + ErrTooManyRequests: { + Code: "TooManyRequests", + Description: "Please reduce your request rate", + HTTPStatusCode: http.StatusTooManyRequests, + }, + ErrUnsupportedMetadata: { + Code: "InvalidArgument", + Description: "Your metadata headers are not supported.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrUnsupportedHostHeader: { + Code: "InvalidArgument", + Description: "Your Host header is malformed.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrObjectTampered: { + Code: "XMinioObjectTampered", + Description: errObjectTampered.Error(), + HTTPStatusCode: http.StatusPartialContent, + }, + + ErrSiteReplicationInvalidRequest: { + Code: "XMinioSiteReplicationInvalidRequest", + Description: "Invalid site-replication request", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrSiteReplicationPeerResp: { + Code: "XMinioSiteReplicationPeerResp", + Description: "Error received when contacting a peer site", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrSiteReplicationBackendIssue: { + Code: "XMinioSiteReplicationBackendIssue", + Description: "Error when requesting object layer backend", + HTTPStatusCode: http.StatusServiceUnavailable, + }, + ErrSiteReplicationServiceAccountError: { + Code: "XMinioSiteReplicationServiceAccountError", + Description: "Site replication related service account error", + HTTPStatusCode: http.StatusServiceUnavailable, + }, + ErrSiteReplicationBucketConfigError: { + Code: "XMinioSiteReplicationBucketConfigError", + Description: "Error while configuring replication on a bucket", + HTTPStatusCode: http.StatusServiceUnavailable, + }, + ErrSiteReplicationBucketMetaError: { + Code: "XMinioSiteReplicationBucketMetaError", + Description: "Error while replicating bucket metadata", + HTTPStatusCode: http.StatusServiceUnavailable, + }, + ErrSiteReplicationIAMError: { + Code: "XMinioSiteReplicationIAMError", + Description: "Error while replicating an IAM item", + HTTPStatusCode: http.StatusServiceUnavailable, + }, + ErrSiteReplicationConfigMissing: { + Code: "XMinioSiteReplicationConfigMissingError", + Description: "Site not found in site replication configuration", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrSiteReplicationIAMConfigMismatch: { + Code: "XMinioSiteReplicationIAMConfigMismatch", + Description: "IAM configuration mismatch between sites", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminRebalanceAlreadyStarted: { + Code: "XMinioAdminRebalanceAlreadyStarted", + Description: "Pool rebalance is already started", + HTTPStatusCode: http.StatusConflict, + }, + ErrAdminRebalanceNotStarted: { + Code: "XMinioAdminRebalanceNotStarted", + Description: "Pool rebalance is not started", + HTTPStatusCode: http.StatusNotFound, + }, + ErrMaximumExpires: { + Code: "AuthorizationQueryParametersError", + Description: "X-Amz-Expires must be less than a week (in seconds); that is, the given X-Amz-Expires must be less than 604800 seconds", + HTTPStatusCode: http.StatusBadRequest, + }, + + // Generic Invalid-Request error. Should be used for response errors only for unlikely + // corner case errors for which introducing new APIErrorCode is not worth it. LogIf() + // should be used to log the error at the source of the error for debugging purposes. + ErrInvalidRequest: { + Code: "InvalidRequest", + Description: "Invalid Request", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrHealNotImplemented: { + Code: "XMinioHealNotImplemented", + Description: "This server does not implement heal functionality.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrHealNoSuchProcess: { + Code: "XMinioHealNoSuchProcess", + Description: "No such heal process is running on the server", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrHealInvalidClientToken: { + Code: "XMinioHealInvalidClientToken", + Description: "Client token mismatch", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrHealMissingBucket: { + Code: "XMinioHealMissingBucket", + Description: "A heal start request with a non-empty object-prefix parameter requires a bucket to be specified.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrHealAlreadyRunning: { + Code: "XMinioHealAlreadyRunning", + Description: "", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrHealOverlappingPaths: { + Code: "XMinioHealOverlappingPaths", + Description: "", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrBackendDown: { + Code: "XMinioBackendDown", + Description: "Remote backend is unreachable", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrIncorrectContinuationToken: { + Code: "InvalidArgument", + Description: "The continuation token provided is incorrect", + HTTPStatusCode: http.StatusBadRequest, + }, + // S3 Select API Errors + ErrEmptyRequestBody: { + Code: "EmptyRequestBody", + Description: "Request body cannot be empty.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrUnsupportedFunction: { + Code: "UnsupportedFunction", + Description: "Encountered an unsupported SQL function.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidDataSource: { + Code: "InvalidDataSource", + Description: "Invalid data source type. Only CSV and JSON are supported at this time.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidExpressionType: { + Code: "InvalidExpressionType", + Description: "The ExpressionType is invalid. Only SQL expressions are supported at this time.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrBusy: { + Code: "ServerBusy", + Description: "The service is unavailable. Please retry.", + HTTPStatusCode: http.StatusServiceUnavailable, + }, + ErrUnauthorizedAccess: { + Code: "UnauthorizedAccess", + Description: "You are not authorized to perform this operation", + HTTPStatusCode: http.StatusUnauthorized, + }, + ErrExpressionTooLong: { + Code: "ExpressionTooLong", + Description: "The SQL expression is too long: The maximum byte-length for the SQL expression is 256 KB.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrIllegalSQLFunctionArgument: { + Code: "IllegalSqlFunctionArgument", + Description: "Illegal argument was used in the SQL function.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidKeyPath: { + Code: "InvalidKeyPath", + Description: "Key path in the SQL expression is invalid.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidCompressionFormat: { + Code: "InvalidCompressionFormat", + Description: "The file is not in a supported compression format. Only GZIP is supported at this time.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidFileHeaderInfo: { + Code: "InvalidFileHeaderInfo", + Description: "The FileHeaderInfo is invalid. Only NONE, USE, and IGNORE are supported.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidJSONType: { + Code: "InvalidJsonType", + Description: "The JsonType is invalid. Only DOCUMENT and LINES are supported at this time.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidQuoteFields: { + Code: "InvalidQuoteFields", + Description: "The QuoteFields is invalid. Only ALWAYS and ASNEEDED are supported.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidRequestParameter: { + Code: "InvalidRequestParameter", + Description: "The value of a parameter in SelectRequest element is invalid. Check the service API documentation and try again.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidDataType: { + Code: "InvalidDataType", + Description: "The SQL expression contains an invalid data type.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidTextEncoding: { + Code: "InvalidTextEncoding", + Description: "Invalid encoding type. Only UTF-8 encoding is supported at this time.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidTableAlias: { + Code: "InvalidTableAlias", + Description: "The SQL expression contains an invalid table alias.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrMissingRequiredParameter: { + Code: "MissingRequiredParameter", + Description: "The SelectRequest entity is missing a required parameter. Check the service documentation and try again.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrObjectSerializationConflict: { + Code: "ObjectSerializationConflict", + Description: "The SelectRequest entity can only contain one of CSV or JSON. Check the service documentation and try again.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrUnsupportedSQLOperation: { + Code: "UnsupportedSqlOperation", + Description: "Encountered an unsupported SQL operation.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrUnsupportedSQLStructure: { + Code: "UnsupportedSqlStructure", + Description: "Encountered an unsupported SQL structure. Check the SQL Reference.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrUnsupportedSyntax: { + Code: "UnsupportedSyntax", + Description: "Encountered invalid syntax.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrUnsupportedRangeHeader: { + Code: "UnsupportedRangeHeader", + Description: "Range header is not supported for this operation.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrLexerInvalidChar: { + Code: "LexerInvalidChar", + Description: "The SQL expression contains an invalid character.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrLexerInvalidOperator: { + Code: "LexerInvalidOperator", + Description: "The SQL expression contains an invalid literal.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrLexerInvalidLiteral: { + Code: "LexerInvalidLiteral", + Description: "The SQL expression contains an invalid operator.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrLexerInvalidIONLiteral: { + Code: "LexerInvalidIONLiteral", + Description: "The SQL expression contains an invalid operator.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseExpectedDatePart: { + Code: "ParseExpectedDatePart", + Description: "Did not find the expected date part in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseExpectedKeyword: { + Code: "ParseExpectedKeyword", + Description: "Did not find the expected keyword in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseExpectedTokenType: { + Code: "ParseExpectedTokenType", + Description: "Did not find the expected token in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseExpected2TokenTypes: { + Code: "ParseExpected2TokenTypes", + Description: "Did not find the expected token in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseExpectedNumber: { + Code: "ParseExpectedNumber", + Description: "Did not find the expected number in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseExpectedRightParenBuiltinFunctionCall: { + Code: "ParseExpectedRightParenBuiltinFunctionCall", + Description: "Did not find the expected right parenthesis character in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseExpectedTypeName: { + Code: "ParseExpectedTypeName", + Description: "Did not find the expected type name in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseExpectedWhenClause: { + Code: "ParseExpectedWhenClause", + Description: "Did not find the expected WHEN clause in the SQL expression. CASE is not supported.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseUnsupportedToken: { + Code: "ParseUnsupportedToken", + Description: "The SQL expression contains an unsupported token.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseUnsupportedLiteralsGroupBy: { + Code: "ParseUnsupportedLiteralsGroupBy", + Description: "The SQL expression contains an unsupported use of GROUP BY.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseExpectedMember: { + Code: "ParseExpectedMember", + Description: "The SQL expression contains an unsupported use of MEMBER.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseUnsupportedSelect: { + Code: "ParseUnsupportedSelect", + Description: "The SQL expression contains an unsupported use of SELECT.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseUnsupportedCase: { + Code: "ParseUnsupportedCase", + Description: "The SQL expression contains an unsupported use of CASE.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseUnsupportedCaseClause: { + Code: "ParseUnsupportedCaseClause", + Description: "The SQL expression contains an unsupported use of CASE.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseUnsupportedAlias: { + Code: "ParseUnsupportedAlias", + Description: "The SQL expression contains an unsupported use of ALIAS.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseUnsupportedSyntax: { + Code: "ParseUnsupportedSyntax", + Description: "The SQL expression contains unsupported syntax.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseUnknownOperator: { + Code: "ParseUnknownOperator", + Description: "The SQL expression contains an invalid operator.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseMissingIdentAfterAt: { + Code: "ParseMissingIdentAfterAt", + Description: "Did not find the expected identifier after the @ symbol in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseUnexpectedOperator: { + Code: "ParseUnexpectedOperator", + Description: "The SQL expression contains an unexpected operator.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseUnexpectedTerm: { + Code: "ParseUnexpectedTerm", + Description: "The SQL expression contains an unexpected term.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseUnexpectedToken: { + Code: "ParseUnexpectedToken", + Description: "The SQL expression contains an unexpected token.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseUnexpectedKeyword: { + Code: "ParseUnexpectedKeyword", + Description: "The SQL expression contains an unexpected keyword.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseExpectedExpression: { + Code: "ParseExpectedExpression", + Description: "Did not find the expected SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseExpectedLeftParenAfterCast: { + Code: "ParseExpectedLeftParenAfterCast", + Description: "Did not find expected the left parenthesis in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseExpectedLeftParenValueConstructor: { + Code: "ParseExpectedLeftParenValueConstructor", + Description: "Did not find expected the left parenthesis in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseExpectedLeftParenBuiltinFunctionCall: { + Code: "ParseExpectedLeftParenBuiltinFunctionCall", + Description: "Did not find the expected left parenthesis in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseExpectedArgumentDelimiter: { + Code: "ParseExpectedArgumentDelimiter", + Description: "Did not find the expected argument delimiter in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseCastArity: { + Code: "ParseCastArity", + Description: "The SQL expression CAST has incorrect arity.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseInvalidTypeParam: { + Code: "ParseInvalidTypeParam", + Description: "The SQL expression contains an invalid parameter value.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseEmptySelect: { + Code: "ParseEmptySelect", + Description: "The SQL expression contains an empty SELECT.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseSelectMissingFrom: { + Code: "ParseSelectMissingFrom", + Description: "GROUP is not supported in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseExpectedIdentForGroupName: { + Code: "ParseExpectedIdentForGroupName", + Description: "GROUP is not supported in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseExpectedIdentForAlias: { + Code: "ParseExpectedIdentForAlias", + Description: "Did not find the expected identifier for the alias in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseUnsupportedCallWithStar: { + Code: "ParseUnsupportedCallWithStar", + Description: "Only COUNT with (*) as a parameter is supported in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseNonUnaryAggregateFunctionCall: { + Code: "ParseNonUnaryAggregateFunctionCall", + Description: "Only one argument is supported for aggregate functions in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseMalformedJoin: { + Code: "ParseMalformedJoin", + Description: "JOIN is not supported in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseExpectedIdentForAt: { + Code: "ParseExpectedIdentForAt", + Description: "Did not find the expected identifier for AT name in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseAsteriskIsNotAloneInSelectList: { + Code: "ParseAsteriskIsNotAloneInSelectList", + Description: "Other expressions are not allowed in the SELECT list when '*' is used without dot notation in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseCannotMixSqbAndWildcardInSelectList: { + Code: "ParseCannotMixSqbAndWildcardInSelectList", + Description: "Cannot mix [] and * in the same expression in a SELECT list in SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrParseInvalidContextForWildcardInSelectList: { + Code: "ParseInvalidContextForWildcardInSelectList", + Description: "Invalid use of * in SELECT list in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrIncorrectSQLFunctionArgumentType: { + Code: "IncorrectSqlFunctionArgumentType", + Description: "Incorrect type of arguments in function call in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrValueParseFailure: { + Code: "ValueParseFailure", + Description: "Time stamp parse failure in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrEvaluatorInvalidArguments: { + Code: "EvaluatorInvalidArguments", + Description: "Incorrect number of arguments in the function call in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrIntegerOverflow: { + Code: "IntegerOverflow", + Description: "Int overflow or underflow in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrLikeInvalidInputs: { + Code: "LikeInvalidInputs", + Description: "Invalid argument given to the LIKE clause in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrCastFailed: { + Code: "CastFailed", + Description: "Attempt to convert from one data type to another using CAST failed in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidCast: { + Code: "InvalidCast", + Description: "Attempt to convert from one data type to another using CAST failed in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrEvaluatorInvalidTimestampFormatPattern: { + Code: "EvaluatorInvalidTimestampFormatPattern", + Description: "Time stamp format pattern requires additional fields in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing: { + Code: "EvaluatorInvalidTimestampFormatPatternSymbolForParsing", + Description: "Time stamp format pattern contains a valid format symbol that cannot be applied to time stamp parsing in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrEvaluatorTimestampFormatPatternDuplicateFields: { + Code: "EvaluatorTimestampFormatPatternDuplicateFields", + Description: "Time stamp format pattern contains multiple format specifiers representing the time stamp field in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch: { + Code: "EvaluatorUnterminatedTimestampFormatPatternToken", + Description: "Time stamp format pattern contains unterminated token in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrEvaluatorUnterminatedTimestampFormatPatternToken: { + Code: "EvaluatorInvalidTimestampFormatPatternToken", + Description: "Time stamp format pattern contains an invalid token in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrEvaluatorInvalidTimestampFormatPatternToken: { + Code: "EvaluatorInvalidTimestampFormatPatternToken", + Description: "Time stamp format pattern contains an invalid token in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrEvaluatorInvalidTimestampFormatPatternSymbol: { + Code: "EvaluatorInvalidTimestampFormatPatternSymbol", + Description: "Time stamp format pattern contains an invalid symbol in the SQL expression.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrEvaluatorBindingDoesNotExist: { + Code: "ErrEvaluatorBindingDoesNotExist", + Description: "A column name or a path provided does not exist in the SQL expression", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrMissingHeaders: { + Code: "MissingHeaders", + Description: "Some headers in the query are missing from the file. Check the file and try again.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidColumnIndex: { + Code: "InvalidColumnIndex", + Description: "The column index is invalid. Please check the service documentation and try again.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrInvalidDecompressedSize: { + Code: "XMinioInvalidDecompressedSize", + Description: "The data provided is unfit for decompression", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAddUserInvalidArgument: { + Code: "XMinioInvalidIAMCredentials", + Description: "Credential is not allowed to be same as admin access key", + HTTPStatusCode: http.StatusForbidden, + }, + ErrAdminResourceInvalidArgument: { + Code: "XMinioInvalidResource", + Description: "Policy, user or group names are not allowed to begin or end with space characters", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminAccountNotEligible: { + Code: "XMinioInvalidIAMCredentials", + Description: "The administrator key is not eligible for this operation", + HTTPStatusCode: http.StatusForbidden, + }, + ErrAccountNotEligible: { + Code: "XMinioInvalidIAMCredentials", + Description: "The account key is not eligible for this operation", + HTTPStatusCode: http.StatusForbidden, + }, + ErrAdminServiceAccountNotFound: { + Code: "XMinioInvalidIAMCredentials", + Description: "The specified service account is not found", + HTTPStatusCode: http.StatusNotFound, + }, + ErrPostPolicyConditionInvalidFormat: { + Code: "PostPolicyInvalidKeyName", + Description: "Invalid according to Policy: Policy Condition failed", + HTTPStatusCode: http.StatusForbidden, + }, + ErrInvalidChecksum: { + Code: "InvalidArgument", + Description: "Invalid checksum provided.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrLambdaARNInvalid: { + Code: "LambdaARNInvalid", + Description: "The specified lambda ARN is invalid", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrLambdaARNNotFound: { + Code: "LambdaARNNotFound", + Description: "The specified lambda ARN does not exist", + HTTPStatusCode: http.StatusNotFound, + }, + ErrPolicyAlreadyAttached: { + Code: "XMinioPolicyAlreadyAttached", + Description: "The specified policy is already attached.", + HTTPStatusCode: http.StatusConflict, + }, + ErrPolicyNotAttached: { + Code: "XMinioPolicyNotAttached", + Description: "The specified policy is not found.", + HTTPStatusCode: http.StatusNotFound, + }, + ErrInvalidAttributeName: { + Code: "InvalidArgument", + Description: "Invalid attribute name specified.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminLDAPNotEnabled: { + Code: "XMinioLDAPNotEnabled", + Description: "LDAP is not enabled. LDAP must be enabled to make LDAP requests.", + HTTPStatusCode: http.StatusNotImplemented, + }, + ErrAdminLDAPExpectedLoginName: { + Code: "XMinioLDAPExpectedLoginName", + Description: "Expected LDAP short username but was given full DN.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAdminInvalidGroupName: { + Code: "XMinioInvalidGroupName", + Description: "The group name is invalid.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrAddUserValidUTF: { + Code: "XMinioInvalidUTF", + Description: "Invalid UTF-8 character detected.", + HTTPStatusCode: http.StatusBadRequest, + }, +} + +// toAPIErrorCode - Converts embedded errors. Convenience +// function written to handle all cases where we have known types of +// errors returned by underlying layers. +func toAPIErrorCode(ctx context.Context, err error) (apiErr APIErrorCode) { + if err == nil { + return ErrNone + } + + // Errors that are generated by net.Conn and any context errors must be handled here. + if errors.Is(err, os.ErrDeadlineExceeded) || errors.Is(err, context.DeadlineExceeded) { + return ErrRequestTimedout + } + + // Only return ErrClientDisconnected if the provided context is actually canceled. + // This way downstream context.Canceled will still report ErrRequestTimedout + if contextCanceled(ctx) && errors.Is(ctx.Err(), context.Canceled) { + return ErrClientDisconnected + } + + // Unwrap the error first + err = unwrapAll(err) + + switch err { + case errInvalidArgument: + apiErr = ErrAdminInvalidArgument + case errNoSuchPolicy: + apiErr = ErrAdminNoSuchPolicy + case errNoSuchUser: + apiErr = ErrAdminNoSuchUser + case errNoSuchUserLDAPWarn: + apiErr = ErrAdminNoSuchUserLDAPWarn + case errNoSuchServiceAccount: + apiErr = ErrAdminServiceAccountNotFound + case errNoSuchAccessKey: + apiErr = ErrAdminNoSuchAccessKey + case errNoSuchGroup: + apiErr = ErrAdminNoSuchGroup + case errGroupNotEmpty: + apiErr = ErrAdminGroupNotEmpty + case errGroupNameContainsReservedChars: + apiErr = ErrAdminInvalidGroupName + case errNoSuchJob: + apiErr = ErrAdminNoSuchJob + case errNoPolicyToAttachOrDetach: + apiErr = ErrAdminPolicyChangeAlreadyApplied + case errSignatureMismatch: + apiErr = ErrSignatureDoesNotMatch + case errInvalidRange: + apiErr = ErrInvalidRange + case errDataTooLarge: + apiErr = ErrEntityTooLarge + case errDataTooSmall: + apiErr = ErrEntityTooSmall + case errAuthentication: + apiErr = ErrAccessDenied + case auth.ErrContainsReservedChars: + apiErr = ErrAdminInvalidAccessKey + case auth.ErrInvalidAccessKeyLength: + apiErr = ErrAdminInvalidAccessKey + case auth.ErrInvalidSecretKeyLength: + apiErr = ErrAdminInvalidSecretKey + case auth.ErrNoAccessKeyWithSecretKey: + apiErr = ErrAdminNoAccessKey + case auth.ErrNoSecretKeyWithAccessKey: + apiErr = ErrAdminNoSecretKey + case errInvalidStorageClass: + apiErr = ErrInvalidStorageClass + case errErasureReadQuorum: + apiErr = ErrSlowDownRead + case errErasureWriteQuorum: + apiErr = ErrSlowDownWrite + case errMaxVersionsExceeded: + apiErr = ErrMaxVersionsExceeded + // SSE errors + case errInvalidEncryptionParameters: + apiErr = ErrInvalidEncryptionParameters + case errInvalidEncryptionParametersSSEC: + apiErr = ErrInvalidEncryptionParametersSSEC + case crypto.ErrInvalidEncryptionMethod: + apiErr = ErrInvalidEncryptionMethod + case crypto.ErrInvalidEncryptionKeyID: + apiErr = ErrInvalidEncryptionKeyID + case crypto.ErrInvalidCustomerAlgorithm: + apiErr = ErrInvalidSSECustomerAlgorithm + case crypto.ErrMissingCustomerKey: + apiErr = ErrMissingSSECustomerKey + case crypto.ErrMissingCustomerKeyMD5: + apiErr = ErrMissingSSECustomerKeyMD5 + case crypto.ErrCustomerKeyMD5Mismatch: + apiErr = ErrSSECustomerKeyMD5Mismatch + case errObjectTampered: + apiErr = ErrObjectTampered + case errEncryptedObject: + apiErr = ErrSSEEncryptedObject + case errInvalidSSEParameters: + apiErr = ErrInvalidSSECustomerParameters + case crypto.ErrInvalidCustomerKey, crypto.ErrSecretKeyMismatch: + apiErr = ErrAccessDenied // no access without correct key + case crypto.ErrIncompatibleEncryptionMethod: + apiErr = ErrIncompatibleEncryptionMethod + case errKMSNotConfigured: + apiErr = ErrKMSNotConfigured + case errKMSKeyNotFound: + apiErr = ErrKMSKeyNotFoundException + case errKMSDefaultKeyAlreadyConfigured: + apiErr = ErrKMSDefaultKeyAlreadyConfigured + case context.Canceled: + apiErr = ErrClientDisconnected + case context.DeadlineExceeded: + apiErr = ErrRequestTimedout + case objectlock.ErrInvalidRetentionDate: + apiErr = ErrInvalidRetentionDate + case objectlock.ErrPastObjectLockRetainDate: + apiErr = ErrPastObjectLockRetainDate + case objectlock.ErrUnknownWORMModeDirective: + apiErr = ErrUnknownWORMModeDirective + case objectlock.ErrObjectLockInvalidHeaders: + apiErr = ErrObjectLockInvalidHeaders + case objectlock.ErrMalformedXML: + apiErr = ErrMalformedXML + case errInvalidMaxParts: + apiErr = ErrInvalidMaxParts + case ioutil.ErrOverread: + apiErr = ErrExcessData + case errServerNotInitialized: + apiErr = ErrServerNotInitialized + case errBucketMetadataNotInitialized: + apiErr = ErrBucketMetadataNotInitialized + case hash.ErrInvalidChecksum: + apiErr = ErrInvalidChecksum + } + + // Compression errors + if err == errInvalidDecompressedSize { + apiErr = ErrInvalidDecompressedSize + } + + if apiErr != ErrNone { + // If there was a match in the above switch case. + return apiErr + } + + // etcd specific errors, a key is always a bucket for us return + // ErrNoSuchBucket in such a case. + if errors.Is(err, dns.ErrNoEntriesFound) { + return ErrNoSuchBucket + } + + switch err.(type) { + case StorageFull: + apiErr = ErrStorageFull + case hash.BadDigest: + apiErr = ErrBadDigest + case AllAccessDisabled: + apiErr = ErrAllAccessDisabled + case IncompleteBody: + apiErr = ErrIncompleteBody + case ObjectExistsAsDirectory: + apiErr = ErrObjectExistsAsDirectory + case PrefixAccessDenied: + apiErr = ErrAccessDenied + case BucketNameInvalid: + apiErr = ErrInvalidBucketName + case BucketNotFound: + apiErr = ErrNoSuchBucket + case BucketAlreadyOwnedByYou: + apiErr = ErrBucketAlreadyOwnedByYou + case BucketNotEmpty: + apiErr = ErrBucketNotEmpty + case BucketAlreadyExists: + apiErr = ErrBucketAlreadyExists + case BucketExists: + apiErr = ErrBucketAlreadyOwnedByYou + case ObjectNotFound: + apiErr = ErrNoSuchKey + case MethodNotAllowed: + apiErr = ErrMethodNotAllowed + case ObjectLocked: + apiErr = ErrObjectLocked + case InvalidVersionID: + apiErr = ErrInvalidVersionID + case VersionNotFound: + apiErr = ErrNoSuchVersion + case ObjectAlreadyExists: + apiErr = ErrMethodNotAllowed + case ObjectNameInvalid: + apiErr = ErrInvalidObjectName + case ObjectNamePrefixAsSlash: + apiErr = ErrInvalidObjectNamePrefixSlash + case InvalidUploadID: + apiErr = ErrNoSuchUpload + case InvalidPart: + apiErr = ErrInvalidPart + case InsufficientWriteQuorum: + apiErr = ErrSlowDownWrite + case InsufficientReadQuorum: + apiErr = ErrSlowDownRead + case InvalidUploadIDKeyCombination: + apiErr = ErrNotImplemented + case MalformedUploadID: + apiErr = ErrNoSuchUpload + case PartTooSmall: + apiErr = ErrEntityTooSmall + case SignatureDoesNotMatch: + apiErr = ErrSignatureDoesNotMatch + case hash.SHA256Mismatch: + apiErr = ErrContentSHA256Mismatch + case hash.ChecksumMismatch: + apiErr = ErrContentChecksumMismatch + case hash.SizeTooSmall: + apiErr = ErrEntityTooSmall + case hash.SizeTooLarge: + apiErr = ErrEntityTooLarge + case NotImplemented: + apiErr = ErrNotImplemented + case PartTooBig: + apiErr = ErrEntityTooLarge + case UnsupportedMetadata: + apiErr = ErrUnsupportedMetadata + case BucketPolicyNotFound: + apiErr = ErrNoSuchBucketPolicy + case BucketLifecycleNotFound: + apiErr = ErrNoSuchLifecycleConfiguration + case BucketSSEConfigNotFound: + apiErr = ErrNoSuchBucketSSEConfig + case BucketTaggingNotFound: + apiErr = ErrBucketTaggingNotFound + case BucketObjectLockConfigNotFound: + apiErr = ErrObjectLockConfigurationNotFound + case BucketQuotaConfigNotFound: + apiErr = ErrAdminNoSuchQuotaConfiguration + case BucketReplicationConfigNotFound: + apiErr = ErrReplicationConfigurationNotFoundError + case BucketRemoteDestinationNotFound: + apiErr = ErrRemoteDestinationNotFoundError + case BucketRemoteTargetNotFound: + apiErr = ErrRemoteTargetNotFoundError + case RemoteTargetConnectionErr: + apiErr = ErrReplicationRemoteConnectionError + case BucketRemoteAlreadyExists: + apiErr = ErrBucketRemoteAlreadyExists + case BucketRemoteLabelInUse: + apiErr = ErrBucketRemoteLabelInUse + case BucketRemoteArnTypeInvalid: + apiErr = ErrBucketRemoteArnTypeInvalid + case BucketRemoteArnInvalid: + apiErr = ErrBucketRemoteArnInvalid + case BucketRemoteRemoveDisallowed: + apiErr = ErrBucketRemoteRemoveDisallowed + case BucketRemoteTargetNotVersioned: + apiErr = ErrRemoteTargetNotVersionedError + case BucketReplicationSourceNotVersioned: + apiErr = ErrReplicationSourceNotVersionedError + case TransitionStorageClassNotFound: + apiErr = ErrTransitionStorageClassNotFoundError + case InvalidObjectState: + apiErr = ErrInvalidObjectState + case PreConditionFailed: + apiErr = ErrPreconditionFailed + case BucketQuotaExceeded: + apiErr = ErrAdminBucketQuotaExceeded + case *event.ErrInvalidEventName: + apiErr = ErrEventNotification + case *event.ErrInvalidARN: + apiErr = ErrARNNotification + case *event.ErrARNNotFound: + apiErr = ErrARNNotification + case *levent.ErrInvalidARN: + apiErr = ErrLambdaARNInvalid + case *levent.ErrARNNotFound: + apiErr = ErrLambdaARNNotFound + case *event.ErrUnknownRegion: + apiErr = ErrRegionNotification + case *event.ErrInvalidFilterName: + apiErr = ErrFilterNameInvalid + case *event.ErrFilterNamePrefix: + apiErr = ErrFilterNamePrefix + case *event.ErrFilterNameSuffix: + apiErr = ErrFilterNameSuffix + case *event.ErrInvalidFilterValue: + apiErr = ErrFilterValueInvalid + case *event.ErrDuplicateEventName: + apiErr = ErrOverlappingConfigs + case *event.ErrDuplicateQueueConfiguration: + apiErr = ErrOverlappingFilterNotification + case *event.ErrUnsupportedConfiguration: + apiErr = ErrUnsupportedNotification + case OperationTimedOut: + apiErr = ErrRequestTimedout + case BackendDown: + apiErr = ErrBackendDown + case ObjectNameTooLong: + apiErr = ErrKeyTooLongError + case dns.ErrInvalidBucketName: + apiErr = ErrInvalidBucketName + case dns.ErrBucketConflict: + apiErr = ErrBucketAlreadyExists + default: + if strings.Contains(err.Error(), "request declared a Content-Length") { + apiErr = ErrIncompleteBody + } else { + apiErr = ErrInternalError + } + } + + return apiErr +} + +var noError = APIError{} + +// toAPIError - Converts embedded errors. Convenience +// function written to handle all cases where we have known types of +// errors returned by underlying layers. +func toAPIError(ctx context.Context, err error) APIError { + if err == nil { + return noError + } + + apiErr := errorCodes.ToAPIErr(toAPIErrorCode(ctx, err)) + switch apiErr.Code { + case "NotImplemented": + apiErr = APIError{ + Code: apiErr.Code, + Description: fmt.Sprintf("%s (%v)", apiErr.Description, err), + HTTPStatusCode: apiErr.HTTPStatusCode, + } + case "XMinioBackendDown": + apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err) + case "InternalError": + // If we see an internal error try to interpret + // any underlying errors if possible depending on + // their internal error types. + switch e := err.(type) { + case kms.Error: + apiErr = APIError{ + Code: e.APICode, + Description: e.Err, + HTTPStatusCode: e.Code, + } + case batchReplicationJobError: + apiErr = APIError{ + Description: e.Description, + Code: e.Code, + HTTPStatusCode: e.HTTPStatusCode, + } + case InvalidRange: + apiErr = APIError{ + Code: "InvalidRange", + Description: e.Error(), + HTTPStatusCode: errorCodes[ErrInvalidRange].HTTPStatusCode, + ObjectSize: strconv.FormatInt(e.ResourceSize, 10), + RangeRequested: fmt.Sprintf("%d-%d", e.OffsetBegin, e.OffsetEnd), + } + case InvalidArgument: + apiErr = APIError{ + Code: "InvalidArgument", + Description: e.Error(), + HTTPStatusCode: errorCodes[ErrInvalidRequest].HTTPStatusCode, + } + case *xml.SyntaxError: + apiErr = APIError{ + Code: "MalformedXML", + Description: fmt.Sprintf("%s (%s)", errorCodes[ErrMalformedXML].Description, e), + HTTPStatusCode: errorCodes[ErrMalformedXML].HTTPStatusCode, + } + case url.EscapeError: + apiErr = APIError{ + Code: "XMinioInvalidObjectName", + Description: fmt.Sprintf("%s (%s)", errorCodes[ErrInvalidObjectName].Description, e), + HTTPStatusCode: http.StatusBadRequest, + } + case versioning.Error: + apiErr = APIError{ + Code: "IllegalVersioningConfigurationException", + Description: fmt.Sprintf("Versioning configuration specified in the request is invalid. (%s)", e), + HTTPStatusCode: http.StatusBadRequest, + } + case lifecycle.Error: + apiErr = APIError{ + Code: "InvalidArgument", + Description: e.Error(), + HTTPStatusCode: http.StatusBadRequest, + } + case replication.Error: + apiErr = APIError{ + Code: "MalformedXML", + Description: e.Error(), + HTTPStatusCode: http.StatusBadRequest, + } + case tags.Error: + apiErr = APIError{ + Code: e.Code(), + Description: e.Error(), + HTTPStatusCode: http.StatusBadRequest, + } + case policy.Error: + apiErr = APIError{ + Code: "MalformedPolicy", + Description: e.Error(), + HTTPStatusCode: http.StatusBadRequest, + } + case crypto.Error: + apiErr = APIError{ + Code: "XMinioEncryptionError", + Description: e.Error(), + HTTPStatusCode: http.StatusBadRequest, + } + case minio.ErrorResponse: + apiErr = APIError{ + Code: e.Code, + Description: e.Message, + HTTPStatusCode: e.StatusCode, + } + if strings.Contains(e.Message, "KMS is not configured") { + apiErr = APIError{ + Code: "NotImplemented", + Description: e.Message, + HTTPStatusCode: http.StatusNotImplemented, + } + } + case *googleapi.Error: + apiErr = APIError{ + Code: "XGCSInternalError", + Description: e.Message, + HTTPStatusCode: e.Code, + } + // GCS may send multiple errors, just pick the first one + // since S3 only sends one Error XML response. + if len(e.Errors) >= 1 { + apiErr.Code = e.Errors[0].Reason + } + case *azcore.ResponseError: + apiErr = APIError{ + Code: e.ErrorCode, + Description: e.Error(), + HTTPStatusCode: e.StatusCode, + } + // Add more other SDK related errors here if any in future. + default: + //nolint:gocritic + if errors.Is(err, errMalformedEncoding) || errors.Is(err, errChunkTooBig) || errors.Is(err, strconv.ErrRange) { + apiErr = APIError{ + Code: "BadRequest", + Description: err.Error(), + HTTPStatusCode: http.StatusBadRequest, + } + } else { + apiErr = APIError{ + Code: apiErr.Code, + Description: fmt.Sprintf("%s: cause(%v)", apiErr.Description, err), + HTTPStatusCode: apiErr.HTTPStatusCode, + } + } + } + } + + if apiErr.Code == "InternalError" { + // Make sure to log the errors which we cannot translate + // to a meaningful S3 API errors. This is added to aid in + // debugging unexpected/unhandled errors. + internalLogIf(ctx, err) + } + + return apiErr +} + +// getAPIError provides API Error for input API error code. +func getAPIError(code APIErrorCode) APIError { + if apiErr, ok := errorCodes[code]; ok { + return apiErr + } + return errorCodes.ToAPIErr(ErrInternalError) +} + +// getAPIErrorResponse gets in standard error and resource value and +// provides a encodable populated response values +func getAPIErrorResponse(ctx context.Context, err APIError, resource, requestID, hostID string) APIErrorResponse { + reqInfo := logger.GetReqInfo(ctx) + return APIErrorResponse{ + Code: err.Code, + Message: err.Description, + BucketName: reqInfo.BucketName, + Key: reqInfo.ObjectName, + Resource: resource, + Region: globalSite.Region(), + RequestID: requestID, + HostID: hostID, + ActualObjectSize: err.ObjectSize, + RangeRequested: err.RangeRequested, + } +} diff --git a/cmd/api-errors_test.go b/cmd/api-errors_test.go new file mode 100644 index 00000000..fda913b6 --- /dev/null +++ b/cmd/api-errors_test.go @@ -0,0 +1,89 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "errors" + "testing" + + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/hash" +) + +var toAPIErrorTests = []struct { + err error + errCode APIErrorCode +}{ + {err: hash.BadDigest{}, errCode: ErrBadDigest}, + {err: hash.SHA256Mismatch{}, errCode: ErrContentSHA256Mismatch}, + {err: IncompleteBody{}, errCode: ErrIncompleteBody}, + {err: ObjectExistsAsDirectory{}, errCode: ErrObjectExistsAsDirectory}, + {err: BucketNameInvalid{}, errCode: ErrInvalidBucketName}, + {err: BucketExists{}, errCode: ErrBucketAlreadyOwnedByYou}, + {err: ObjectNotFound{}, errCode: ErrNoSuchKey}, + {err: ObjectNameInvalid{}, errCode: ErrInvalidObjectName}, + {err: InvalidUploadID{}, errCode: ErrNoSuchUpload}, + {err: InvalidPart{}, errCode: ErrInvalidPart}, + {err: InsufficientReadQuorum{}, errCode: ErrSlowDownRead}, + {err: InsufficientWriteQuorum{}, errCode: ErrSlowDownWrite}, + {err: InvalidUploadIDKeyCombination{}, errCode: ErrNotImplemented}, + {err: MalformedUploadID{}, errCode: ErrNoSuchUpload}, + {err: PartTooSmall{}, errCode: ErrEntityTooSmall}, + {err: BucketNotEmpty{}, errCode: ErrBucketNotEmpty}, + {err: BucketNotFound{}, errCode: ErrNoSuchBucket}, + {err: StorageFull{}, errCode: ErrStorageFull}, + {err: NotImplemented{}, errCode: ErrNotImplemented}, + {err: errSignatureMismatch, errCode: ErrSignatureDoesNotMatch}, + + // SSE-C errors + {err: crypto.ErrInvalidCustomerAlgorithm, errCode: ErrInvalidSSECustomerAlgorithm}, + {err: crypto.ErrMissingCustomerKey, errCode: ErrMissingSSECustomerKey}, + {err: crypto.ErrInvalidCustomerKey, errCode: ErrAccessDenied}, + {err: crypto.ErrMissingCustomerKeyMD5, errCode: ErrMissingSSECustomerKeyMD5}, + {err: crypto.ErrCustomerKeyMD5Mismatch, errCode: ErrSSECustomerKeyMD5Mismatch}, + {err: errObjectTampered, errCode: ErrObjectTampered}, + + {err: nil, errCode: ErrNone}, + {err: errors.New("Custom error"), errCode: ErrInternalError}, // Case where err type is unknown. +} + +func TestAPIErrCode(t *testing.T) { + ctx := t.Context() + for i, testCase := range toAPIErrorTests { + errCode := toAPIErrorCode(ctx, testCase.err) + if errCode != testCase.errCode { + t.Errorf("Test %d: Expected error code %d, got %d", i+1, testCase.errCode, errCode) + } + } +} + +// Check if an API error is properly defined +func TestAPIErrCodeDefinition(t *testing.T) { + for errAPI := ErrNone + 1; errAPI < apiErrCodeEnd; errAPI++ { + errCode, ok := errorCodes[errAPI] + if !ok { + t.Fatal(errAPI, "error code is not defined in the API error code table") + } + if errCode.Code == "" { + t.Fatal(errAPI, "error code has an empty XML code") + } + if errCode.HTTPStatusCode == 0 { + t.Fatal(errAPI, "error code has a zero HTTP status code") + } + } +} diff --git a/cmd/api-headers.go b/cmd/api-headers.go new file mode 100644 index 00000000..c2ca23fb --- /dev/null +++ b/cmd/api-headers.go @@ -0,0 +1,269 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "encoding/xml" + "fmt" + "mime" + "net/http" + "strconv" + "strings" + "time" + + "github.com/minio/minio-go/v7/pkg/tags" + "github.com/minio/minio/internal/crypto" + xhttp "github.com/minio/minio/internal/http" + xxml "github.com/minio/xxml" +) + +// Returns a hexadecimal representation of time at the +// time response is sent to the client. +func mustGetRequestID(t time.Time) string { + return fmt.Sprintf("%X", t.UnixNano()) +} + +// setEventStreamHeaders to allow proxies to avoid buffering proxy responses +func setEventStreamHeaders(w http.ResponseWriter) { + w.Header().Set(xhttp.ContentType, "text/event-stream") + w.Header().Set(xhttp.CacheControl, "no-cache") // nginx to turn off buffering + w.Header().Set("X-Accel-Buffering", "no") // nginx to turn off buffering +} + +// Write http common headers +func setCommonHeaders(w http.ResponseWriter) { + // Set the "Server" http header. + w.Header().Set(xhttp.ServerInfo, MinioStoreName) + + // Set `x-amz-bucket-region` only if region is set on the server + // by default minio uses an empty region. + if region := globalSite.Region(); region != "" { + w.Header().Set(xhttp.AmzBucketRegion, region) + } + w.Header().Set(xhttp.AcceptRanges, "bytes") + + // Remove sensitive information + crypto.RemoveSensitiveHeaders(w.Header()) +} + +// Encodes the response headers into XML format. +func encodeResponse(response any) []byte { + var buf bytes.Buffer + buf.WriteString(xml.Header) + if err := xml.NewEncoder(&buf).Encode(response); err != nil { + bugLogIf(GlobalContext, err) + return nil + } + return buf.Bytes() +} + +// Use this encodeResponseList() to support control characters +// this function must be used by only ListObjects() for objects +// with control characters, this is a specialized extension +// to support AWS S3 compatible behavior. +// +// Do not use this function for anything other than ListObjects() +// variants, please open a github discussion if you wish to use +// this in other places. +func encodeResponseList(response any) []byte { + var buf bytes.Buffer + buf.WriteString(xxml.Header) + if err := xxml.NewEncoder(&buf).Encode(response); err != nil { + bugLogIf(GlobalContext, err) + return nil + } + return buf.Bytes() +} + +// Encodes the response headers into JSON format. +func encodeResponseJSON(response any) []byte { + var bytesBuffer bytes.Buffer + e := json.NewEncoder(&bytesBuffer) + e.Encode(response) + return bytesBuffer.Bytes() +} + +// Write parts count +func setPartsCountHeaders(w http.ResponseWriter, objInfo ObjectInfo) { + if strings.Contains(objInfo.ETag, "-") && len(objInfo.Parts) > 0 { + w.Header()[xhttp.AmzMpPartsCount] = []string{strconv.Itoa(len(objInfo.Parts))} + } +} + +// Write object header +func setObjectHeaders(ctx context.Context, w http.ResponseWriter, objInfo ObjectInfo, rs *HTTPRangeSpec, opts ObjectOptions) (err error) { + // set common headers + setCommonHeaders(w) + + // Set last modified time. + lastModified := objInfo.ModTime.UTC().Format(http.TimeFormat) + w.Header().Set(xhttp.LastModified, lastModified) + + // Set Etag if available. + if objInfo.ETag != "" { + w.Header()[xhttp.ETag] = []string{"\"" + objInfo.ETag + "\""} + } + + if objInfo.ContentType != "" { + w.Header().Set(xhttp.ContentType, objInfo.ContentType) + } + + if objInfo.ContentEncoding != "" { + w.Header().Set(xhttp.ContentEncoding, objInfo.ContentEncoding) + } + + if !objInfo.Expires.IsZero() { + w.Header().Set(xhttp.Expires, objInfo.Expires.UTC().Format(http.TimeFormat)) + } + + // Set tag count if object has tags + if len(objInfo.UserTags) > 0 { + tags, _ := tags.ParseObjectTags(objInfo.UserTags) + if tags != nil && tags.Count() > 0 { + w.Header()[xhttp.AmzTagCount] = []string{strconv.Itoa(tags.Count())} + if opts.Tagging { + // This is MinIO only extension to return back tags along with the count. + w.Header()[xhttp.AmzObjectTagging] = []string{objInfo.UserTags} + } + } + } + + // Set all other user defined metadata. + for k, v := range objInfo.UserDefined { + // Empty values for object lock and retention can be skipped. + if v == "" && equals(k, xhttp.AmzObjectLockMode, xhttp.AmzObjectLockRetainUntilDate) { + continue + } + + if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) { + // Do not need to send any internal metadata + // values to client. + continue + } + + // https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w + if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) { + continue + } + + var isSet bool + for _, userMetadataPrefix := range userMetadataKeyPrefixes { + if !stringsHasPrefixFold(k, userMetadataPrefix) { + continue + } + // check the doc https://docs.aws.amazon.com/AmazonS3/latest/userguide/UsingMetadata.html + // For metadata values like "ö", "ÄMÄZÕÑ S3", and "öha, das sollte eigentlich + // funktionieren", tested against a real AWS S3 bucket, S3 may encode incorrectly. For + // example, "ö" was encoded as =?UTF-8?B?w4PCtg==?=, producing invalid UTF-8 instead + // of =?UTF-8?B?w7Y=?=. This mirrors errors like the ä½ in another string. + // + // S3 uses B-encoding (Base64) for non-ASCII-heavy metadata and Q-encoding + // (quoted-printable) for mostly ASCII strings. Long strings are split at word + // boundaries to fit RFC 2047’s 75-character limit, ensuring HTTP parser + // compatibility. + // + // However, this splitting increases header size and can introduce errors, unlike Go’s + // mime package in MinIO, which correctly encodes strings with fixed B/Q encodings, + // avoiding S3’s heuristic-driven issues. + // + // For MinIO developers, decode S3 metadata with mime.WordDecoder, validate outputs, + // report encoding bugs to AWS, and use ASCII-only metadata to ensure reliable S3 API + // compatibility. + if needsMimeEncoding(v) { + // see https://github.com/golang/go/blob/release-branch.go1.24/src/net/mail/message.go#L325 + if strings.ContainsAny(v, "\"#$%&'(),.:;<>@[]^`{|}~") { + v = mime.BEncoding.Encode("UTF-8", v) + } else { + v = mime.QEncoding.Encode("UTF-8", v) + } + } + w.Header()[strings.ToLower(k)] = []string{v} + isSet = true + break + } + + if !isSet { + w.Header().Set(k, v) + } + } + + var start, rangeLen int64 + totalObjectSize, err := objInfo.GetActualSize() + if err != nil { + return err + } + + if rs == nil && opts.PartNumber > 0 { + rs = partNumberToRangeSpec(objInfo, opts.PartNumber) + } + + // For providing ranged content + start, rangeLen, err = rs.GetOffsetLength(totalObjectSize) + if err != nil { + return err + } + + // Set content length. + w.Header().Set(xhttp.ContentLength, strconv.FormatInt(rangeLen, 10)) + if rs != nil { + contentRange := fmt.Sprintf("bytes %d-%d/%d", start, start+rangeLen-1, totalObjectSize) + w.Header().Set(xhttp.ContentRange, contentRange) + } + + // Set the relevant version ID as part of the response header. + if objInfo.VersionID != "" && objInfo.VersionID != nullVersionID { + w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID} + } + + if objInfo.ReplicationStatus.String() != "" { + w.Header()[xhttp.AmzBucketReplicationStatus] = []string{objInfo.ReplicationStatus.String()} + } + + if objInfo.IsRemote() { + // Check if object is being restored. For more information on x-amz-restore header see + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_HeadObject.html#API_HeadObject_ResponseSyntax + w.Header()[xhttp.AmzStorageClass] = []string{filterStorageClass(ctx, objInfo.TransitionedObject.Tier)} + } + + if lc, err := globalLifecycleSys.Get(objInfo.Bucket); err == nil { + lc.SetPredictionHeaders(w, objInfo.ToLifecycleOpts()) + } + + if v, ok := objInfo.UserDefined[ReservedMetadataPrefix+"compression"]; ok { + if i := strings.LastIndexByte(v, '/'); i >= 0 { + v = v[i+1:] + } + w.Header()[xhttp.MinIOCompressed] = []string{v} + } + + return nil +} + +// needsEncoding reports whether s contains any bytes that need to be encoded. +// see mime.needsEncoding +func needsMimeEncoding(s string) bool { + for _, b := range s { + if (b < ' ' || b > '~') && b != '\t' { + return true + } + } + return false +} diff --git a/cmd/api-headers_test.go b/cmd/api-headers_test.go new file mode 100644 index 00000000..9db65460 --- /dev/null +++ b/cmd/api-headers_test.go @@ -0,0 +1,42 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "testing" +) + +func TestNewRequestID(t *testing.T) { + // Ensure that it returns an alphanumeric result of length 16. + id := mustGetRequestID(UTCNow()) + + if len(id) != 16 { + t.Fail() + } + + var e rune + for _, char := range id { + e = char + + // Ensure that it is alphanumeric, in this case, between 0-9 and A-Z. + isAlnum := ('0' <= e && e <= '9') || ('A' <= e && e <= 'Z') + if !isAlnum { + t.Fail() + } + } +} diff --git a/cmd/api-resources.go b/cmd/api-resources.go new file mode 100644 index 00000000..11cf53ad --- /dev/null +++ b/cmd/api-resources.go @@ -0,0 +1,153 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "encoding/base64" + "net/url" + "strconv" +) + +// Parse bucket url queries +func getListObjectsV1Args(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType string, errCode APIErrorCode) { + errCode = ErrNone + + if values.Get("max-keys") != "" { + var err error + if maxkeys, err = strconv.Atoi(values.Get("max-keys")); err != nil { + errCode = ErrInvalidMaxKeys + return prefix, marker, delimiter, maxkeys, encodingType, errCode + } + } else { + maxkeys = maxObjectList + } + + prefix = values.Get("prefix") + marker = values.Get("marker") + delimiter = values.Get("delimiter") + encodingType = values.Get("encoding-type") + return prefix, marker, delimiter, maxkeys, encodingType, errCode +} + +func getListBucketObjectVersionsArgs(values url.Values) (prefix, marker, delimiter string, maxkeys int, encodingType, versionIDMarker string, errCode APIErrorCode) { + errCode = ErrNone + + if values.Get("max-keys") != "" { + var err error + if maxkeys, err = strconv.Atoi(values.Get("max-keys")); err != nil { + errCode = ErrInvalidMaxKeys + return prefix, marker, delimiter, maxkeys, encodingType, versionIDMarker, errCode + } + } else { + maxkeys = maxObjectList + } + + prefix = values.Get("prefix") + marker = values.Get("key-marker") + delimiter = values.Get("delimiter") + encodingType = values.Get("encoding-type") + versionIDMarker = values.Get("version-id-marker") + return prefix, marker, delimiter, maxkeys, encodingType, versionIDMarker, errCode +} + +// Parse bucket url queries for ListObjects V2. +func getListObjectsV2Args(values url.Values) (prefix, token, startAfter, delimiter string, fetchOwner bool, maxkeys int, encodingType string, errCode APIErrorCode) { + errCode = ErrNone + + // The continuation-token cannot be empty. + if val, ok := values["continuation-token"]; ok { + if len(val[0]) == 0 { + errCode = ErrIncorrectContinuationToken + return prefix, token, startAfter, delimiter, fetchOwner, maxkeys, encodingType, errCode + } + } + + if values.Get("max-keys") != "" { + var err error + if maxkeys, err = strconv.Atoi(values.Get("max-keys")); err != nil { + errCode = ErrInvalidMaxKeys + return prefix, token, startAfter, delimiter, fetchOwner, maxkeys, encodingType, errCode + } + } else { + maxkeys = maxObjectList + } + + prefix = values.Get("prefix") + startAfter = values.Get("start-after") + delimiter = values.Get("delimiter") + fetchOwner = values.Get("fetch-owner") == "true" + encodingType = values.Get("encoding-type") + + if token = values.Get("continuation-token"); token != "" { + decodedToken, err := base64.StdEncoding.DecodeString(token) + if err != nil { + errCode = ErrIncorrectContinuationToken + return prefix, token, startAfter, delimiter, fetchOwner, maxkeys, encodingType, errCode + } + token = string(decodedToken) + } + return prefix, token, startAfter, delimiter, fetchOwner, maxkeys, encodingType, errCode +} + +// Parse bucket url queries for ?uploads +func getBucketMultipartResources(values url.Values) (prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int, encodingType string, errCode APIErrorCode) { + errCode = ErrNone + + if values.Get("max-uploads") != "" { + var err error + if maxUploads, err = strconv.Atoi(values.Get("max-uploads")); err != nil { + errCode = ErrInvalidMaxUploads + return prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, encodingType, errCode + } + } else { + maxUploads = maxUploadsList + } + + prefix = values.Get("prefix") + keyMarker = values.Get("key-marker") + uploadIDMarker = values.Get("upload-id-marker") + delimiter = values.Get("delimiter") + encodingType = values.Get("encoding-type") + return prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, encodingType, errCode +} + +// Parse object url queries +func getObjectResources(values url.Values) (uploadID string, partNumberMarker, maxParts int, encodingType string, errCode APIErrorCode) { + var err error + errCode = ErrNone + + if values.Get("max-parts") != "" { + if maxParts, err = strconv.Atoi(values.Get("max-parts")); err != nil { + errCode = ErrInvalidMaxParts + return uploadID, partNumberMarker, maxParts, encodingType, errCode + } + } else { + maxParts = maxPartsList + } + + if values.Get("part-number-marker") != "" { + if partNumberMarker, err = strconv.Atoi(values.Get("part-number-marker")); err != nil { + errCode = ErrInvalidPartNumberMarker + return uploadID, partNumberMarker, maxParts, encodingType, errCode + } + } + + uploadID = values.Get("uploadId") + encodingType = values.Get("encoding-type") + return uploadID, partNumberMarker, maxParts, encodingType, errCode +} diff --git a/cmd/api-resources_test.go b/cmd/api-resources_test.go new file mode 100644 index 00000000..a9aac0b6 --- /dev/null +++ b/cmd/api-resources_test.go @@ -0,0 +1,222 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "net/url" + "testing" +) + +// Test list objects resources V2. +func TestListObjectsV2Resources(t *testing.T) { + testCases := []struct { + values url.Values + prefix, token, startAfter, delimiter string + fetchOwner bool + maxKeys int + encodingType string + errCode APIErrorCode + }{ + { + values: url.Values{ + "prefix": []string{"photos/"}, + "continuation-token": []string{"dG9rZW4="}, + "start-after": []string{"start-after"}, + "delimiter": []string{SlashSeparator}, + "fetch-owner": []string{"true"}, + "max-keys": []string{"100"}, + "encoding-type": []string{"gzip"}, + }, + prefix: "photos/", + token: "token", + startAfter: "start-after", + delimiter: SlashSeparator, + fetchOwner: true, + maxKeys: 100, + encodingType: "gzip", + errCode: ErrNone, + }, + { + values: url.Values{ + "prefix": []string{"photos/"}, + "continuation-token": []string{"dG9rZW4="}, + "start-after": []string{"start-after"}, + "delimiter": []string{SlashSeparator}, + "fetch-owner": []string{"true"}, + "encoding-type": []string{"gzip"}, + }, + prefix: "photos/", + token: "token", + startAfter: "start-after", + delimiter: SlashSeparator, + fetchOwner: true, + maxKeys: maxObjectList, + encodingType: "gzip", + errCode: ErrNone, + }, + { + values: url.Values{ + "prefix": []string{"photos/"}, + "continuation-token": []string{""}, + "start-after": []string{"start-after"}, + "delimiter": []string{SlashSeparator}, + "fetch-owner": []string{"true"}, + "encoding-type": []string{"gzip"}, + }, + prefix: "", + token: "", + startAfter: "", + delimiter: "", + fetchOwner: false, + maxKeys: 0, + encodingType: "", + errCode: ErrIncorrectContinuationToken, + }, + } + + for i, testCase := range testCases { + prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(testCase.values) + + if errCode != testCase.errCode { + t.Errorf("Test %d: Expected error code:%d, got %d", i+1, testCase.errCode, errCode) + } + if prefix != testCase.prefix { + t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.prefix, prefix) + } + if token != testCase.token { + t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.token, token) + } + if startAfter != testCase.startAfter { + t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.startAfter, startAfter) + } + if delimiter != testCase.delimiter { + t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.delimiter, delimiter) + } + if fetchOwner != testCase.fetchOwner { + t.Errorf("Test %d: Expected %t, got %t", i+1, testCase.fetchOwner, fetchOwner) + } + if maxKeys != testCase.maxKeys { + t.Errorf("Test %d: Expected %d, got %d", i+1, testCase.maxKeys, maxKeys) + } + if encodingType != testCase.encodingType { + t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.encodingType, encodingType) + } + } +} + +// Test list objects resources V1. +func TestListObjectsV1Resources(t *testing.T) { + testCases := []struct { + values url.Values + prefix, marker, delimiter string + maxKeys int + encodingType string + }{ + { + values: url.Values{ + "prefix": []string{"photos/"}, + "marker": []string{"test"}, + "delimiter": []string{SlashSeparator}, + "max-keys": []string{"100"}, + "encoding-type": []string{"gzip"}, + }, + prefix: "photos/", + marker: "test", + delimiter: SlashSeparator, + maxKeys: 100, + encodingType: "gzip", + }, + { + values: url.Values{ + "prefix": []string{"photos/"}, + "marker": []string{"test"}, + "delimiter": []string{SlashSeparator}, + "encoding-type": []string{"gzip"}, + }, + prefix: "photos/", + marker: "test", + delimiter: SlashSeparator, + maxKeys: maxObjectList, + encodingType: "gzip", + }, + } + + for i, testCase := range testCases { + prefix, marker, delimiter, maxKeys, encodingType, argsErr := getListObjectsV1Args(testCase.values) + if argsErr != ErrNone { + t.Errorf("Test %d: argument parsing failed, got %v", i+1, argsErr) + } + if prefix != testCase.prefix { + t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.prefix, prefix) + } + if marker != testCase.marker { + t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.marker, marker) + } + if delimiter != testCase.delimiter { + t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.delimiter, delimiter) + } + if maxKeys != testCase.maxKeys { + t.Errorf("Test %d: Expected %d, got %d", i+1, testCase.maxKeys, maxKeys) + } + if encodingType != testCase.encodingType { + t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.encodingType, encodingType) + } + } +} + +// Validates extracting information for object resources. +func TestGetObjectsResources(t *testing.T) { + testCases := []struct { + values url.Values + uploadID string + partNumberMarker, maxParts int + encodingType string + }{ + { + values: url.Values{ + "uploadId": []string{"11123-11312312311231-12313"}, + "part-number-marker": []string{"1"}, + "max-parts": []string{"1000"}, + "encoding-type": []string{"gzip"}, + }, + uploadID: "11123-11312312311231-12313", + partNumberMarker: 1, + maxParts: 1000, + encodingType: "gzip", + }, + } + + for i, testCase := range testCases { + uploadID, partNumberMarker, maxParts, encodingType, argsErr := getObjectResources(testCase.values) + if argsErr != ErrNone { + t.Errorf("Test %d: argument parsing failed, got %v", i+1, argsErr) + } + if uploadID != testCase.uploadID { + t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.uploadID, uploadID) + } + if partNumberMarker != testCase.partNumberMarker { + t.Errorf("Test %d: Expected %d, got %d", i+1, testCase.partNumberMarker, partNumberMarker) + } + if maxParts != testCase.maxParts { + t.Errorf("Test %d: Expected %d, got %d", i+1, testCase.maxParts, maxParts) + } + if encodingType != testCase.encodingType { + t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.encodingType, encodingType) + } + } +} diff --git a/cmd/api-response.go b/cmd/api-response.go new file mode 100644 index 00000000..28501b5b --- /dev/null +++ b/cmd/api-response.go @@ -0,0 +1,1017 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/base64" + "encoding/xml" + "fmt" + "net/http" + "net/url" + "path" + "strconv" + "strings" + "time" + + "github.com/minio/minio/internal/amztime" + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/handlers" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/policy" + xxml "github.com/minio/xxml" +) + +const ( + maxObjectList = 1000 // Limit number of objects in a listObjectsResponse/listObjectsVersionsResponse. + maxDeleteList = 1000 // Limit number of objects deleted in a delete call. + maxUploadsList = 10000 // Limit number of uploads in a listUploadsResponse. + maxPartsList = 10000 // Limit number of parts in a listPartsResponse. +) + +// LocationResponse - format for location response. +type LocationResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ LocationConstraint" json:"-"` + Location string `xml:",chardata"` +} + +// PolicyStatus captures information returned by GetBucketPolicyStatusHandler +type PolicyStatus struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ PolicyStatus" json:"-"` + IsPublic string +} + +// ListVersionsResponse - format for list bucket versions response. +type ListVersionsResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListVersionsResult" json:"-"` + + Name string + Prefix string + KeyMarker string + + // When response is truncated (the IsTruncated element value in the response + // is true), you can use the key name in this field as marker in the subsequent + // request to get next set of objects. Server lists objects in alphabetical + // order Note: This element is returned only if you have delimiter request parameter + // specified. If response does not include the NextMaker and it is truncated, + // you can use the value of the last Key in the response as the marker in the + // subsequent request to get the next set of object keys. + NextKeyMarker string `xml:"NextKeyMarker,omitempty"` + + // When the number of responses exceeds the value of MaxKeys, + // NextVersionIdMarker specifies the first object version not + // returned that satisfies the search criteria. Use this value + // for the version-id-marker request parameter in a subsequent request. + NextVersionIDMarker string `xml:"NextVersionIdMarker"` + + // Marks the last version of the Key returned in a truncated response. + VersionIDMarker string `xml:"VersionIdMarker"` + + MaxKeys int + Delimiter string `xml:"Delimiter,omitempty"` + // A flag that indicates whether or not ListObjects returned all of the results + // that satisfied the search criteria. + IsTruncated bool + + CommonPrefixes []CommonPrefix + Versions []ObjectVersion + + // Encoding type used to encode object keys in the response. + EncodingType string `xml:"EncodingType,omitempty"` +} + +// ListObjectsResponse - format for list objects response. +type ListObjectsResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"` + + Name string + Prefix string + Marker string + + // When response is truncated (the IsTruncated element value in the response + // is true), you can use the key name in this field as marker in the subsequent + // request to get next set of objects. Server lists objects in alphabetical + // order Note: This element is returned only if you have delimiter request parameter + // specified. If response does not include the NextMaker and it is truncated, + // you can use the value of the last Key in the response as the marker in the + // subsequent request to get the next set of object keys. + NextMarker string `xml:"NextMarker,omitempty"` + + MaxKeys int + Delimiter string `xml:"Delimiter,omitempty"` + // A flag that indicates whether or not ListObjects returned all of the results + // that satisfied the search criteria. + IsTruncated bool + + Contents []Object + CommonPrefixes []CommonPrefix + + // Encoding type used to encode object keys in the response. + EncodingType string `xml:"EncodingType,omitempty"` +} + +// ListObjectsV2Response - format for list objects response. +type ListObjectsV2Response struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListBucketResult" json:"-"` + + Name string + Prefix string + StartAfter string `xml:"StartAfter,omitempty"` + // When response is truncated (the IsTruncated element value in the response + // is true), you can use the key name in this field as marker in the subsequent + // request to get next set of objects. Server lists objects in alphabetical + // order Note: This element is returned only if you have delimiter request parameter + // specified. If response does not include the NextMaker and it is truncated, + // you can use the value of the last Key in the response as the marker in the + // subsequent request to get the next set of object keys. + ContinuationToken string `xml:"ContinuationToken,omitempty"` + NextContinuationToken string `xml:"NextContinuationToken,omitempty"` + + KeyCount int + MaxKeys int + Delimiter string `xml:"Delimiter,omitempty"` + // A flag that indicates whether or not ListObjects returned all of the results + // that satisfied the search criteria. + IsTruncated bool + + Contents []Object + CommonPrefixes []CommonPrefix + + // Encoding type used to encode object keys in the response. + EncodingType string `xml:"EncodingType,omitempty"` +} + +// Part container for part metadata. +type Part struct { + PartNumber int + LastModified string + ETag string + Size int64 + + // Checksum values + ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"` + ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"` + ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"` + ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"` + ChecksumCRC64NVME string `xml:",omitempty"` +} + +// ListPartsResponse - format for list parts response. +type ListPartsResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListPartsResult" json:"-"` + + Bucket string + Key string + UploadID string `xml:"UploadId"` + + Initiator Initiator + Owner Owner + + // The class of storage used to store the object. + StorageClass string + + PartNumberMarker int + NextPartNumberMarker int + MaxParts int + IsTruncated bool + + ChecksumAlgorithm string + ChecksumType string + + // List of parts. + Parts []Part `xml:"Part"` +} + +// ListMultipartUploadsResponse - format for list multipart uploads response. +type ListMultipartUploadsResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult" json:"-"` + + Bucket string + KeyMarker string + UploadIDMarker string `xml:"UploadIdMarker"` + NextKeyMarker string + NextUploadIDMarker string `xml:"NextUploadIdMarker"` + Delimiter string `xml:"Delimiter,omitempty"` + Prefix string + EncodingType string `xml:"EncodingType,omitempty"` + MaxUploads int + IsTruncated bool + + // List of pending uploads. + Uploads []Upload `xml:"Upload"` + + // Delimed common prefixes. + CommonPrefixes []CommonPrefix +} + +// ListBucketsResponse - format for list buckets response +type ListBucketsResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListAllMyBucketsResult" json:"-"` + + Owner Owner + + // Container for one or more buckets. + Buckets struct { + Buckets []Bucket `xml:"Bucket"` + } // Buckets are nested +} + +// Upload container for in progress multipart upload +type Upload struct { + Key string + UploadID string `xml:"UploadId"` + Initiator Initiator + Owner Owner + StorageClass string + Initiated string +} + +// CommonPrefix container for prefix response in ListObjectsResponse +type CommonPrefix struct { + Prefix string +} + +// Bucket container for bucket metadata +type Bucket struct { + Name string + CreationDate string // time string of format "2006-01-02T15:04:05.000Z" +} + +// ObjectVersion container for object version metadata +type ObjectVersion struct { + Object + IsLatest bool + VersionID string `xml:"VersionId"` + + isDeleteMarker bool +} + +// MarshalXML - marshal ObjectVersion +func (o ObjectVersion) MarshalXML(e *xxml.Encoder, start xxml.StartElement) error { + if o.isDeleteMarker { + start.Name.Local = "DeleteMarker" + } else { + start.Name.Local = "Version" + } + type objectVersionWrapper ObjectVersion + return e.EncodeElement(objectVersionWrapper(o), start) +} + +// DeleteMarkerVersion container for delete marker metadata +type DeleteMarkerVersion struct { + Key string + LastModified string // time string of format "2006-01-02T15:04:05.000Z" + + // Owner of the object. + Owner Owner + + IsLatest bool + VersionID string `xml:"VersionId"` +} + +// Metadata metadata items implemented to ensure XML marshaling works. +type Metadata struct { + Items []struct { + Key string + Value string + } +} + +// Set add items, duplicate items get replaced. +func (s *Metadata) Set(k, v string) { + for i, item := range s.Items { + if item.Key == k { + s.Items[i] = struct { + Key string + Value string + }{ + Key: k, + Value: v, + } + return + } + } + s.Items = append(s.Items, struct { + Key string + Value string + }{ + Key: k, + Value: v, + }) +} + +type xmlKeyEntry struct { + XMLName xxml.Name + Value string `xml:",chardata"` +} + +// MarshalXML - StringMap marshals into XML. +func (s *Metadata) MarshalXML(e *xxml.Encoder, start xxml.StartElement) error { + if s == nil { + return nil + } + + if len(s.Items) == 0 { + return nil + } + + if err := e.EncodeToken(start); err != nil { + return err + } + + for _, item := range s.Items { + if err := e.Encode(xmlKeyEntry{ + XMLName: xxml.Name{Local: item.Key}, + Value: item.Value, + }); err != nil { + return err + } + } + + return e.EncodeToken(start.End()) +} + +// ObjectInternalInfo contains some internal information about a given +// object, it will printed in listing calls with enabled metadata. +type ObjectInternalInfo struct { + K int // Data blocks + M int // Parity blocks +} + +// Object container for object metadata +type Object struct { + Key string + LastModified string // time string of format "2006-01-02T15:04:05.000Z" + ETag string + Size int64 + + // Owner of the object. + Owner *Owner `xml:"Owner,omitempty"` + + // The class of storage used to store the object. + StorageClass string + + // UserMetadata user-defined metadata + UserMetadata *Metadata `xml:"UserMetadata,omitempty"` + UserTags string `xml:"UserTags,omitempty"` + + Internal *ObjectInternalInfo `xml:"Internal,omitempty"` +} + +// CopyObjectResponse container returns ETag and LastModified of the successfully copied object +type CopyObjectResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyObjectResult" json:"-"` + LastModified string // time string of format "2006-01-02T15:04:05.000Z" + ETag string // md5sum of the copied object. +} + +// CopyObjectPartResponse container returns ETag and LastModified of the successfully copied object +type CopyObjectPartResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CopyPartResult" json:"-"` + LastModified string // time string of format "2006-01-02T15:04:05.000Z" + ETag string // md5sum of the copied object part. +} + +// Initiator inherit from Owner struct, fields are same +type Initiator Owner + +// Owner - bucket owner/principal +type Owner struct { + ID string + DisplayName string +} + +// InitiateMultipartUploadResponse container for InitiateMultiPartUpload response, provides uploadID to start MultiPart upload +type InitiateMultipartUploadResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ InitiateMultipartUploadResult" json:"-"` + + Bucket string + Key string + UploadID string `xml:"UploadId"` +} + +// CompleteMultipartUploadResponse container for completed multipart upload response +type CompleteMultipartUploadResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ CompleteMultipartUploadResult" json:"-"` + + Location string + Bucket string + Key string + ETag string + + ChecksumCRC32 string `xml:"ChecksumCRC32,omitempty"` + ChecksumCRC32C string `xml:"ChecksumCRC32C,omitempty"` + ChecksumSHA1 string `xml:"ChecksumSHA1,omitempty"` + ChecksumSHA256 string `xml:"ChecksumSHA256,omitempty"` + ChecksumCRC64NVME string `xml:",omitempty"` +} + +// DeleteError structure. +type DeleteError struct { + Code string + Message string + Key string + VersionID string `xml:"VersionId"` +} + +// DeleteObjectsResponse container for multiple object deletes. +type DeleteObjectsResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ DeleteResult" json:"-"` + + // Collection of all deleted objects + DeletedObjects []DeletedObject `xml:"Deleted,omitempty"` + + // Collection of errors deleting certain objects. + Errors []DeleteError `xml:"Error,omitempty"` +} + +// PostResponse container for POST object request when success_action_status is set to 201 +type PostResponse struct { + Bucket string + Key string + ETag string + Location string +} + +// returns "https" if the tls boolean is true, "http" otherwise. +func getURLScheme(tls bool) string { + if tls { + return httpsScheme + } + return httpScheme +} + +// getObjectLocation gets the fully qualified URL of an object. +func getObjectLocation(r *http.Request, domains []string, bucket, object string) string { + // unit tests do not have host set. + if r.Host == "" { + return path.Clean(r.URL.Path) + } + proto := handlers.GetSourceScheme(r) + if proto == "" { + proto = getURLScheme(globalIsTLS) + } + u := &url.URL{ + Host: r.Host, + Path: path.Join(SlashSeparator, bucket, object), + Scheme: proto, + } + // If domain is set then we need to use bucket DNS style. + for _, domain := range domains { + if strings.HasPrefix(r.Host, bucket+"."+domain) { + u.Path = path.Join(SlashSeparator, object) + break + } + } + return u.String() +} + +// generates ListBucketsResponse from array of BucketInfo which can be +// serialized to match XML and JSON API spec output. +func generateListBucketsResponse(buckets []BucketInfo) ListBucketsResponse { + listbuckets := make([]Bucket, 0, len(buckets)) + data := ListBucketsResponse{} + owner := Owner{ + ID: globalMinioDefaultOwnerID, + DisplayName: "minio", + } + + for _, bucket := range buckets { + listbuckets = append(listbuckets, Bucket{ + Name: bucket.Name, + CreationDate: amztime.ISO8601Format(bucket.Created.UTC()), + }) + } + + data.Owner = owner + data.Buckets.Buckets = listbuckets + + return data +} + +func cleanReservedKeys(metadata map[string]string) map[string]string { + m := cloneMSS(metadata) + + switch kind, _ := crypto.IsEncrypted(metadata); kind { + case crypto.S3: + m[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES + case crypto.S3KMS: + m[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionKMS + m[xhttp.AmzServerSideEncryptionKmsID] = kmsKeyIDFromMetadata(metadata) + if kmsCtx, ok := metadata[crypto.MetaContext]; ok { + m[xhttp.AmzServerSideEncryptionKmsContext] = kmsCtx + } + case crypto.SSEC: + m[xhttp.AmzServerSideEncryptionCustomerAlgorithm] = xhttp.AmzEncryptionAES + } + + var toRemove []string + for k := range cleanMinioInternalMetadataKeys(m) { + if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) { + // Do not need to send any internal metadata + // values to client. + toRemove = append(toRemove, k) + continue + } + + // https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w + if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) { + toRemove = append(toRemove, k) + continue + } + } + + for _, k := range toRemove { + delete(m, k) + delete(m, strings.ToLower(k)) + } + + return m +} + +// generates an ListBucketVersions response for the said bucket with other enumerated options. +func generateListVersionsResponse(ctx context.Context, bucket, prefix, marker, versionIDMarker, delimiter, encodingType string, maxKeys int, resp ListObjectVersionsInfo, metadata metaCheckFn) ListVersionsResponse { + versions := make([]ObjectVersion, 0, len(resp.Objects)) + + owner := &Owner{ + ID: globalMinioDefaultOwnerID, + DisplayName: "minio", + } + data := ListVersionsResponse{} + var lastObjMetaName string + var tagErr, metaErr APIErrorCode = -1, -1 + + for _, object := range resp.Objects { + if object.Name == "" { + continue + } + // Cache checks for the same object + if metadata != nil && lastObjMetaName != object.Name { + tagErr = metadata(object.Name, policy.GetObjectTaggingAction) + metaErr = metadata(object.Name, policy.GetObjectAction) + lastObjMetaName = object.Name + } + content := ObjectVersion{} + content.Key = s3EncodeName(object.Name, encodingType) + content.LastModified = amztime.ISO8601Format(object.ModTime.UTC()) + if object.ETag != "" { + content.ETag = "\"" + object.ETag + "\"" + } + content.Size = object.Size + if object.StorageClass != "" { + content.StorageClass = filterStorageClass(ctx, object.StorageClass) + } else { + content.StorageClass = globalMinioDefaultStorageClass + } + if tagErr == ErrNone { + content.UserTags = object.UserTags + } + if metaErr == ErrNone { + content.UserMetadata = &Metadata{} + switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind { + case crypto.S3: + content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES) + case crypto.S3KMS: + content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS) + case crypto.SSEC: + content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES) + } + for k, v := range cleanReservedKeys(object.UserDefined) { + content.UserMetadata.Set(k, v) + } + content.Internal = &ObjectInternalInfo{ + K: object.DataBlocks, + M: object.ParityBlocks, + } + } + content.Owner = owner + content.VersionID = object.VersionID + if content.VersionID == "" { + content.VersionID = nullVersionID + } + content.IsLatest = object.IsLatest + content.isDeleteMarker = object.DeleteMarker + versions = append(versions, content) + } + + data.Name = bucket + data.Versions = versions + data.EncodingType = encodingType + data.Prefix = s3EncodeName(prefix, encodingType) + data.KeyMarker = s3EncodeName(marker, encodingType) + data.Delimiter = s3EncodeName(delimiter, encodingType) + data.MaxKeys = maxKeys + + data.NextKeyMarker = s3EncodeName(resp.NextMarker, encodingType) + data.NextVersionIDMarker = resp.NextVersionIDMarker + data.VersionIDMarker = versionIDMarker + data.IsTruncated = resp.IsTruncated + + prefixes := make([]CommonPrefix, 0, len(resp.Prefixes)) + for _, prefix := range resp.Prefixes { + prefixItem := CommonPrefix{} + prefixItem.Prefix = s3EncodeName(prefix, encodingType) + prefixes = append(prefixes, prefixItem) + } + data.CommonPrefixes = prefixes + return data +} + +// generates an ListObjectsV1 response for the said bucket with other enumerated options. +func generateListObjectsV1Response(ctx context.Context, bucket, prefix, marker, delimiter, encodingType string, maxKeys int, resp ListObjectsInfo) ListObjectsResponse { + contents := make([]Object, 0, len(resp.Objects)) + owner := &Owner{ + ID: globalMinioDefaultOwnerID, + DisplayName: "minio", + } + data := ListObjectsResponse{} + + for _, object := range resp.Objects { + content := Object{} + if object.Name == "" { + continue + } + content.Key = s3EncodeName(object.Name, encodingType) + content.LastModified = amztime.ISO8601Format(object.ModTime.UTC()) + if object.ETag != "" { + content.ETag = "\"" + object.ETag + "\"" + } + content.Size = object.Size + if object.StorageClass != "" { + content.StorageClass = filterStorageClass(ctx, object.StorageClass) + } else { + content.StorageClass = globalMinioDefaultStorageClass + } + content.Owner = owner + contents = append(contents, content) + } + data.Name = bucket + data.Contents = contents + + data.EncodingType = encodingType + data.Prefix = s3EncodeName(prefix, encodingType) + data.Marker = s3EncodeName(marker, encodingType) + data.Delimiter = s3EncodeName(delimiter, encodingType) + data.MaxKeys = maxKeys + data.NextMarker = s3EncodeName(resp.NextMarker, encodingType) + data.IsTruncated = resp.IsTruncated + + prefixes := make([]CommonPrefix, 0, len(resp.Prefixes)) + for _, prefix := range resp.Prefixes { + prefixItem := CommonPrefix{} + prefixItem.Prefix = s3EncodeName(prefix, encodingType) + prefixes = append(prefixes, prefixItem) + } + data.CommonPrefixes = prefixes + return data +} + +// generates an ListObjectsV2 response for the said bucket with other enumerated options. +func generateListObjectsV2Response(ctx context.Context, bucket, prefix, token, nextToken, startAfter, delimiter, encodingType string, fetchOwner, isTruncated bool, maxKeys int, objects []ObjectInfo, prefixes []string, metadata metaCheckFn) ListObjectsV2Response { + contents := make([]Object, 0, len(objects)) + var owner *Owner + if fetchOwner { + owner = &Owner{ + ID: globalMinioDefaultOwnerID, + DisplayName: "minio", + } + } + + data := ListObjectsV2Response{} + + for _, object := range objects { + content := Object{} + if object.Name == "" { + continue + } + content.Key = s3EncodeName(object.Name, encodingType) + content.LastModified = amztime.ISO8601Format(object.ModTime.UTC()) + if object.ETag != "" { + content.ETag = "\"" + object.ETag + "\"" + } + content.Size = object.Size + if object.StorageClass != "" { + content.StorageClass = filterStorageClass(ctx, object.StorageClass) + } else { + content.StorageClass = globalMinioDefaultStorageClass + } + content.Owner = owner + if metadata != nil { + if metadata(object.Name, policy.GetObjectTaggingAction) == ErrNone { + content.UserTags = object.UserTags + } + if metadata(object.Name, policy.GetObjectAction) == ErrNone { + content.UserMetadata = &Metadata{} + switch kind, _ := crypto.IsEncrypted(object.UserDefined); kind { + case crypto.S3: + content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES) + case crypto.S3KMS: + content.UserMetadata.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS) + case crypto.SSEC: + content.UserMetadata.Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, xhttp.AmzEncryptionAES) + } + for k, v := range cleanReservedKeys(object.UserDefined) { + content.UserMetadata.Set(k, v) + } + content.Internal = &ObjectInternalInfo{ + K: object.DataBlocks, + M: object.ParityBlocks, + } + } + } + contents = append(contents, content) + } + data.Name = bucket + data.Contents = contents + + data.EncodingType = encodingType + data.StartAfter = s3EncodeName(startAfter, encodingType) + data.Delimiter = s3EncodeName(delimiter, encodingType) + data.Prefix = s3EncodeName(prefix, encodingType) + data.MaxKeys = maxKeys + data.ContinuationToken = base64.StdEncoding.EncodeToString([]byte(token)) + data.NextContinuationToken = base64.StdEncoding.EncodeToString([]byte(nextToken)) + data.IsTruncated = isTruncated + + commonPrefixes := make([]CommonPrefix, 0, len(prefixes)) + for _, prefix := range prefixes { + prefixItem := CommonPrefix{} + prefixItem.Prefix = s3EncodeName(prefix, encodingType) + commonPrefixes = append(commonPrefixes, prefixItem) + } + data.CommonPrefixes = commonPrefixes + data.KeyCount = len(data.Contents) + len(data.CommonPrefixes) + return data +} + +type metaCheckFn = func(name string, action policy.Action) (s3Err APIErrorCode) + +// generates CopyObjectResponse from etag and lastModified time. +func generateCopyObjectResponse(etag string, lastModified time.Time) CopyObjectResponse { + return CopyObjectResponse{ + ETag: "\"" + etag + "\"", + LastModified: amztime.ISO8601Format(lastModified.UTC()), + } +} + +// generates CopyObjectPartResponse from etag and lastModified time. +func generateCopyObjectPartResponse(etag string, lastModified time.Time) CopyObjectPartResponse { + return CopyObjectPartResponse{ + ETag: "\"" + etag + "\"", + LastModified: amztime.ISO8601Format(lastModified.UTC()), + } +} + +// generates InitiateMultipartUploadResponse for given bucket, key and uploadID. +func generateInitiateMultipartUploadResponse(bucket, key, uploadID string) InitiateMultipartUploadResponse { + return InitiateMultipartUploadResponse{ + Bucket: bucket, + Key: key, + UploadID: uploadID, + } +} + +// generates CompleteMultipartUploadResponse for given bucket, key, location and ETag. +func generateCompleteMultipartUploadResponse(bucket, key, location string, oi ObjectInfo, h http.Header) CompleteMultipartUploadResponse { + cs, _ := oi.decryptChecksums(0, h) + c := CompleteMultipartUploadResponse{ + Location: location, + Bucket: bucket, + Key: key, + // AWS S3 quotes the ETag in XML, make sure we are compatible here. + ETag: "\"" + oi.ETag + "\"", + ChecksumSHA1: cs[hash.ChecksumSHA1.String()], + ChecksumSHA256: cs[hash.ChecksumSHA256.String()], + ChecksumCRC32: cs[hash.ChecksumCRC32.String()], + ChecksumCRC32C: cs[hash.ChecksumCRC32C.String()], + ChecksumCRC64NVME: cs[hash.ChecksumCRC64NVME.String()], + } + return c +} + +// generates ListPartsResponse from ListPartsInfo. +func generateListPartsResponse(partsInfo ListPartsInfo, encodingType string) ListPartsResponse { + listPartsResponse := ListPartsResponse{} + listPartsResponse.Bucket = partsInfo.Bucket + listPartsResponse.Key = s3EncodeName(partsInfo.Object, encodingType) + listPartsResponse.UploadID = partsInfo.UploadID + listPartsResponse.StorageClass = globalMinioDefaultStorageClass + + // Dumb values not meaningful + listPartsResponse.Initiator = Initiator{ + ID: globalMinioDefaultOwnerID, + DisplayName: globalMinioDefaultOwnerID, + } + listPartsResponse.Owner = Owner{ + ID: globalMinioDefaultOwnerID, + DisplayName: globalMinioDefaultOwnerID, + } + + listPartsResponse.MaxParts = partsInfo.MaxParts + listPartsResponse.PartNumberMarker = partsInfo.PartNumberMarker + listPartsResponse.IsTruncated = partsInfo.IsTruncated + listPartsResponse.NextPartNumberMarker = partsInfo.NextPartNumberMarker + listPartsResponse.ChecksumAlgorithm = partsInfo.ChecksumAlgorithm + listPartsResponse.ChecksumType = partsInfo.ChecksumType + + listPartsResponse.Parts = make([]Part, len(partsInfo.Parts)) + for index, part := range partsInfo.Parts { + newPart := Part{} + newPart.PartNumber = part.PartNumber + newPart.ETag = "\"" + part.ETag + "\"" + newPart.Size = part.Size + newPart.LastModified = amztime.ISO8601Format(part.LastModified.UTC()) + newPart.ChecksumCRC32 = part.ChecksumCRC32 + newPart.ChecksumCRC32C = part.ChecksumCRC32C + newPart.ChecksumSHA1 = part.ChecksumSHA1 + newPart.ChecksumSHA256 = part.ChecksumSHA256 + newPart.ChecksumCRC64NVME = part.ChecksumCRC64NVME + listPartsResponse.Parts[index] = newPart + } + return listPartsResponse +} + +// generates ListMultipartUploadsResponse for given bucket and ListMultipartsInfo. +func generateListMultipartUploadsResponse(bucket string, multipartsInfo ListMultipartsInfo, encodingType string) ListMultipartUploadsResponse { + listMultipartUploadsResponse := ListMultipartUploadsResponse{} + listMultipartUploadsResponse.Bucket = bucket + listMultipartUploadsResponse.Delimiter = s3EncodeName(multipartsInfo.Delimiter, encodingType) + listMultipartUploadsResponse.IsTruncated = multipartsInfo.IsTruncated + listMultipartUploadsResponse.EncodingType = encodingType + listMultipartUploadsResponse.Prefix = s3EncodeName(multipartsInfo.Prefix, encodingType) + listMultipartUploadsResponse.KeyMarker = s3EncodeName(multipartsInfo.KeyMarker, encodingType) + listMultipartUploadsResponse.NextKeyMarker = s3EncodeName(multipartsInfo.NextKeyMarker, encodingType) + listMultipartUploadsResponse.MaxUploads = multipartsInfo.MaxUploads + listMultipartUploadsResponse.NextUploadIDMarker = multipartsInfo.NextUploadIDMarker + listMultipartUploadsResponse.UploadIDMarker = multipartsInfo.UploadIDMarker + listMultipartUploadsResponse.CommonPrefixes = make([]CommonPrefix, len(multipartsInfo.CommonPrefixes)) + for index, commonPrefix := range multipartsInfo.CommonPrefixes { + listMultipartUploadsResponse.CommonPrefixes[index] = CommonPrefix{ + Prefix: s3EncodeName(commonPrefix, encodingType), + } + } + listMultipartUploadsResponse.Uploads = make([]Upload, len(multipartsInfo.Uploads)) + for index, upload := range multipartsInfo.Uploads { + newUpload := Upload{} + newUpload.UploadID = upload.UploadID + newUpload.Key = s3EncodeName(upload.Object, encodingType) + newUpload.Initiated = amztime.ISO8601Format(upload.Initiated.UTC()) + listMultipartUploadsResponse.Uploads[index] = newUpload + } + return listMultipartUploadsResponse +} + +// generate multi objects delete response. +func generateMultiDeleteResponse(quiet bool, deletedObjects []DeletedObject, errs []DeleteError) DeleteObjectsResponse { + deleteResp := DeleteObjectsResponse{} + if !quiet { + deleteResp.DeletedObjects = deletedObjects + } + deleteResp.Errors = errs + return deleteResp +} + +func writeResponse(w http.ResponseWriter, statusCode int, response []byte, mType mimeType) { + if statusCode == 0 { + statusCode = 200 + } + // Similar check to http.checkWriteHeaderCode + if statusCode < 100 || statusCode > 999 { + bugLogIf(context.Background(), fmt.Errorf("invalid WriteHeader code %v", statusCode)) + statusCode = http.StatusInternalServerError + } + setCommonHeaders(w) + if mType != mimeNone { + w.Header().Set(xhttp.ContentType, string(mType)) + } + w.Header().Set(xhttp.ContentLength, strconv.Itoa(len(response))) + w.WriteHeader(statusCode) + if response != nil { + w.Write(response) + } +} + +// mimeType represents various MIME type used API responses. +type mimeType string + +const ( + // Means no response type. + mimeNone mimeType = "" + // Means response type is JSON. + mimeJSON mimeType = "application/json" + // Means response type is XML. + mimeXML mimeType = "application/xml" +) + +// writeSuccessResponseJSON writes success headers and response if any, +// with content-type set to `application/json`. +func writeSuccessResponseJSON(w http.ResponseWriter, response []byte) { + writeResponse(w, http.StatusOK, response, mimeJSON) +} + +// writeSuccessResponseXML writes success headers and response if any, +// with content-type set to `application/xml`. +func writeSuccessResponseXML(w http.ResponseWriter, response []byte) { + writeResponse(w, http.StatusOK, response, mimeXML) +} + +// writeSuccessNoContent writes success headers with http status 204 +func writeSuccessNoContent(w http.ResponseWriter) { + writeResponse(w, http.StatusNoContent, nil, mimeNone) +} + +// writeRedirectSeeOther writes Location header with http status 303 +func writeRedirectSeeOther(w http.ResponseWriter, location string) { + w.Header().Set(xhttp.Location, location) + writeResponse(w, http.StatusSeeOther, nil, mimeNone) +} + +func writeSuccessResponseHeadersOnly(w http.ResponseWriter) { + writeResponse(w, http.StatusOK, nil, mimeNone) +} + +// writeErrorResponse writes error headers +func writeErrorResponse(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) { + switch err.HTTPStatusCode { + case http.StatusServiceUnavailable, http.StatusTooManyRequests: + // Set retry-after header to indicate user-agents to retry request after 60 seconds. + // https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Retry-After + w.Header().Set(xhttp.RetryAfter, "60") + } + + switch err.Code { + case "InvalidRegion": + err.Description = fmt.Sprintf("Region does not match; expecting '%s'.", globalSite.Region()) + case "AuthorizationHeaderMalformed": + err.Description = fmt.Sprintf("The authorization header is malformed; the region is wrong; expecting '%s'.", globalSite.Region()) + } + + // Similar check to http.checkWriteHeaderCode + if err.HTTPStatusCode < 100 || err.HTTPStatusCode > 999 { + bugLogIf(ctx, fmt.Errorf("invalid WriteHeader code %v from %v", err.HTTPStatusCode, err.Code)) + err.HTTPStatusCode = http.StatusInternalServerError + } + + // Generate error response. + errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path, + w.Header().Get(xhttp.AmzRequestID), w.Header().Get(xhttp.AmzRequestHostID)) + encodedErrorResponse := encodeResponse(errorResponse) + writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeXML) +} + +func writeErrorResponseHeadersOnly(w http.ResponseWriter, err APIError) { + w.Header().Set(xMinIOErrCodeHeader, err.Code) + w.Header().Set(xMinIOErrDescHeader, "\""+err.Description+"\"") + writeResponse(w, err.HTTPStatusCode, nil, mimeNone) +} + +func writeErrorResponseString(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) { + // Generate string error response. + writeResponse(w, err.HTTPStatusCode, []byte(err.Description), mimeNone) +} + +// writeErrorResponseJSON - writes error response in JSON format; +// useful for admin APIs. +func writeErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIError, reqURL *url.URL) { + // Generate error response. + errorResponse := getAPIErrorResponse(ctx, err, reqURL.Path, w.Header().Get(xhttp.AmzRequestID), w.Header().Get(xhttp.AmzRequestHostID)) + encodedErrorResponse := encodeResponseJSON(errorResponse) + writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON) +} + +// writeCustomErrorResponseJSON - similar to writeErrorResponseJSON, +// but accepts the error message directly (this allows messages to be +// dynamically generated.) +func writeCustomErrorResponseJSON(ctx context.Context, w http.ResponseWriter, err APIError, + errBody string, reqURL *url.URL, +) { + reqInfo := logger.GetReqInfo(ctx) + errorResponse := APIErrorResponse{ + Code: err.Code, + Message: errBody, + Resource: reqURL.Path, + BucketName: reqInfo.BucketName, + Key: reqInfo.ObjectName, + RequestID: w.Header().Get(xhttp.AmzRequestID), + HostID: globalDeploymentID(), + } + encodedErrorResponse := encodeResponseJSON(errorResponse) + writeResponse(w, err.HTTPStatusCode, encodedErrorResponse, mimeJSON) +} diff --git a/cmd/api-response_test.go b/cmd/api-response_test.go new file mode 100644 index 00000000..8f89b340 --- /dev/null +++ b/cmd/api-response_test.go @@ -0,0 +1,124 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "net/http" + "testing" +) + +// Tests object location. +func TestObjectLocation(t *testing.T) { + testCases := []struct { + request *http.Request + bucket, object string + domains []string + expectedLocation string + }{ + // Server binding to localhost IP with https. + { + request: &http.Request{ + Host: "127.0.0.1:9000", + Header: map[string][]string{ + "X-Forwarded-Scheme": {httpScheme}, + }, + }, + bucket: "testbucket1", + object: "test/1.txt", + expectedLocation: "http://127.0.0.1:9000/testbucket1/test/1.txt", + }, + { + request: &http.Request{ + Host: "127.0.0.1:9000", + Header: map[string][]string{ + "X-Forwarded-Scheme": {httpsScheme}, + }, + }, + bucket: "testbucket1", + object: "test/1.txt", + expectedLocation: "https://127.0.0.1:9000/testbucket1/test/1.txt", + }, + // Server binding to fqdn. + { + request: &http.Request{ + Host: "s3.mybucket.org", + Header: map[string][]string{ + "X-Forwarded-Scheme": {httpScheme}, + }, + }, + bucket: "mybucket", + object: "test/1.txt", + expectedLocation: "http://s3.mybucket.org/mybucket/test/1.txt", + }, + // Server binding to fqdn. + { + request: &http.Request{ + Host: "mys3.mybucket.org", + Header: map[string][]string{}, + }, + bucket: "mybucket", + object: "test/1.txt", + expectedLocation: "http://mys3.mybucket.org/mybucket/test/1.txt", + }, + // Server with virtual domain name. + { + request: &http.Request{ + Host: "mybucket.mys3.bucket.org", + Header: map[string][]string{}, + }, + domains: []string{"mys3.bucket.org"}, + bucket: "mybucket", + object: "test/1.txt", + expectedLocation: "http://mybucket.mys3.bucket.org/test/1.txt", + }, + { + request: &http.Request{ + Host: "mybucket.mys3.bucket.org", + Header: map[string][]string{ + "X-Forwarded-Scheme": {httpsScheme}, + }, + }, + domains: []string{"mys3.bucket.org"}, + bucket: "mybucket", + object: "test/1.txt", + expectedLocation: "https://mybucket.mys3.bucket.org/test/1.txt", + }, + } + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + gotLocation := getObjectLocation(testCase.request, testCase.domains, testCase.bucket, testCase.object) + if testCase.expectedLocation != gotLocation { + t.Errorf("expected %s, got %s", testCase.expectedLocation, gotLocation) + } + }) + } +} + +// Tests getURLScheme function behavior. +func TestGetURLScheme(t *testing.T) { + tls := false + gotScheme := getURLScheme(tls) + if gotScheme != httpScheme { + t.Errorf("Expected %s, got %s", httpScheme, gotScheme) + } + tls = true + gotScheme = getURLScheme(tls) + if gotScheme != httpsScheme { + t.Errorf("Expected %s, got %s", httpsScheme, gotScheme) + } +} diff --git a/cmd/api-router.go b/cmd/api-router.go new file mode 100644 index 00000000..896ca48b --- /dev/null +++ b/cmd/api-router.go @@ -0,0 +1,695 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "net" + "net/http" + + consoleapi "github.com/minio/console/api" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/mux" + "github.com/minio/pkg/v3/wildcard" + "github.com/rs/cors" +) + +func newHTTPServerFn() *xhttp.Server { + globalObjLayerMutex.RLock() + defer globalObjLayerMutex.RUnlock() + return globalHTTPServer +} + +func setHTTPServer(h *xhttp.Server) { + globalObjLayerMutex.Lock() + globalHTTPServer = h + globalObjLayerMutex.Unlock() +} + +func newConsoleServerFn() *consoleapi.Server { + globalObjLayerMutex.RLock() + defer globalObjLayerMutex.RUnlock() + return globalConsoleSrv +} + +func setConsoleSrv(srv *consoleapi.Server) { + globalObjLayerMutex.Lock() + globalConsoleSrv = srv + globalObjLayerMutex.Unlock() +} + +func newObjectLayerFn() ObjectLayer { + globalObjLayerMutex.RLock() + defer globalObjLayerMutex.RUnlock() + return globalObjectAPI +} + +func setObjectLayer(o ObjectLayer) { + globalObjLayerMutex.Lock() + globalObjectAPI = o + globalObjLayerMutex.Unlock() +} + +// objectAPIHandlers implements and provides http handlers for S3 API. +type objectAPIHandlers struct { + ObjectAPI func() ObjectLayer +} + +// getHost tries its best to return the request host. +// According to section 14.23 of RFC 2616 the Host header +// can include the port number if the default value of 80 is not used. +func getHost(r *http.Request) string { + if r.URL.IsAbs() { + return r.URL.Host + } + return r.Host +} + +func notImplementedHandler(w http.ResponseWriter, r *http.Request) { + writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) +} + +type rejectedAPI struct { + api string + methods []string + queries []string + path string +} + +var rejectedObjAPIs = []rejectedAPI{ + { + api: "torrent", + methods: []string{http.MethodPut, http.MethodDelete, http.MethodGet}, + queries: []string{"torrent", ""}, + path: "/{object:.+}", + }, + { + api: "acl", + methods: []string{http.MethodDelete}, + queries: []string{"acl", ""}, + path: "/{object:.+}", + }, +} + +var rejectedBucketAPIs = []rejectedAPI{ + { + api: "inventory", + methods: []string{http.MethodGet, http.MethodPut, http.MethodDelete}, + queries: []string{"inventory", ""}, + }, + { + api: "cors", + methods: []string{http.MethodPut, http.MethodDelete}, + queries: []string{"cors", ""}, + }, + { + api: "metrics", + methods: []string{http.MethodGet, http.MethodPut, http.MethodDelete}, + queries: []string{"metrics", ""}, + }, + { + api: "website", + methods: []string{http.MethodPut}, + queries: []string{"website", ""}, + }, + { + api: "logging", + methods: []string{http.MethodPut, http.MethodDelete}, + queries: []string{"logging", ""}, + }, + { + api: "accelerate", + methods: []string{http.MethodPut, http.MethodDelete}, + queries: []string{"accelerate", ""}, + }, + { + api: "requestPayment", + methods: []string{http.MethodPut, http.MethodDelete}, + queries: []string{"requestPayment", ""}, + }, + { + api: "acl", + methods: []string{http.MethodDelete, http.MethodPut, http.MethodHead}, + queries: []string{"acl", ""}, + }, + { + api: "publicAccessBlock", + methods: []string{http.MethodDelete, http.MethodPut, http.MethodGet}, + queries: []string{"publicAccessBlock", ""}, + }, + { + api: "ownershipControls", + methods: []string{http.MethodDelete, http.MethodPut, http.MethodGet}, + queries: []string{"ownershipControls", ""}, + }, + { + api: "intelligent-tiering", + methods: []string{http.MethodDelete, http.MethodPut, http.MethodGet}, + queries: []string{"intelligent-tiering", ""}, + }, + { + api: "analytics", + methods: []string{http.MethodDelete, http.MethodPut, http.MethodGet}, + queries: []string{"analytics", ""}, + }, +} + +// Set of s3 handler options as bit flags. +type s3HFlag uint8 + +const ( + // when provided, disables Gzip compression. + noGZS3HFlag = 1 << iota + + // when provided, enables only tracing of headers. Otherwise, both headers + // and body are traced. + traceHdrsS3HFlag + + // when provided, disables throttling via the `maxClients` middleware. + noThrottleS3HFlag +) + +func (h s3HFlag) has(flag s3HFlag) bool { + // Use bitwise-AND and check if the result is non-zero. + return h&flag != 0 +} + +// s3APIMiddleware - performs some common handler functionality for S3 API +// handlers. +// +// It is set per-"handler function registration" in the router to allow for +// behavior modification via flags. +// +// This middleware always calls `collectAPIStats` to collect API stats. +// +// The passed in handler function must be a method of `objectAPIHandlers` for +// the name displayed in logs and trace to be accurate. The name is extracted +// via reflection. +// +// When **no** flags are passed, the behavior is to trace both headers and body, +// gzip the response and throttle the handler via `maxClients`. Each of these +// can be disabled via the corresponding `s3HFlag`. +// +// CAUTION: for requests involving large req/resp bodies ensure to pass the +// `traceHdrsS3HFlag`, otherwise both headers and body will be traced, causing +// high memory usage! +func s3APIMiddleware(f http.HandlerFunc, flags ...s3HFlag) http.HandlerFunc { + // Collect all flags with bitwise-OR and assign operator + var handlerFlags s3HFlag + for _, flag := range flags { + handlerFlags |= flag + } + + // Get name of the handler using reflection. + handlerName := getHandlerName(f, "objectAPIHandlers") + + var handler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { + // Wrap the actual handler with the appropriate tracing middleware. + var tracedHandler http.HandlerFunc + if handlerFlags.has(traceHdrsS3HFlag) { + tracedHandler = httpTraceHdrs(f) + } else { + tracedHandler = httpTraceAll(f) + } + + // Skip wrapping with the gzip middleware if specified. + gzippedHandler := tracedHandler + if !handlerFlags.has(noGZS3HFlag) { + gzippedHandler = gzipHandler(gzippedHandler) + } + + // Skip wrapping with throttling middleware if specified. + throttledHandler := gzippedHandler + if !handlerFlags.has(noThrottleS3HFlag) { + throttledHandler = maxClients(throttledHandler) + } + + // Collect API stats using the API name got from reflection in + // `getHandlerName`. + statsCollectedHandler := collectAPIStats(handlerName, throttledHandler) + + // Call the final handler. + statsCollectedHandler(w, r) + } + + return handler +} + +// registerAPIRouter - registers S3 compatible APIs. +func registerAPIRouter(router *mux.Router) { + // Initialize API. + api := objectAPIHandlers{ + ObjectAPI: newObjectLayerFn, + } + + // API Router + apiRouter := router.PathPrefix(SlashSeparator).Subrouter() + + var routers []*mux.Router + for _, domainName := range globalDomainNames { + if IsKubernetes() { + routers = append(routers, apiRouter.MatcherFunc(func(r *http.Request, match *mux.RouteMatch) bool { + host, _, err := net.SplitHostPort(getHost(r)) + if err != nil { + host = r.Host + } + // Make sure to skip matching minio.` this is + // specifically meant for operator/k8s deployment + // The reason we need to skip this is for a special + // usecase where we need to make sure that + // minio..svc. is ignored + // by the bucketDNS style to ensure that path style + // is available and honored at this domain. + // + // All other `..svc.` + // makes sure that buckets are routed through this matcher + // to match for `` + return host != minioReservedBucket+"."+domainName + }).Host("{bucket:.+}."+domainName).Subrouter()) + } else { + routers = append(routers, apiRouter.Host("{bucket:.+}."+domainName).Subrouter()) + } + } + routers = append(routers, apiRouter.PathPrefix("/{bucket}").Subrouter()) + + for _, router := range routers { + // Register all rejected object APIs + for _, r := range rejectedObjAPIs { + t := router.Methods(r.methods...). + HandlerFunc(collectAPIStats(r.api, httpTraceAll(notImplementedHandler))). + Queries(r.queries...) + t.Path(r.path) + } + + // Object operations + // HeadObject + router.Methods(http.MethodHead).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.HeadObjectHandler)) + + // GetObjectAttributes + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.GetObjectAttributesHandler, traceHdrsS3HFlag)). + Queries("attributes", "") + + // CopyObjectPart + router.Methods(http.MethodPut).Path("/{object:.+}"). + HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?"). + HandlerFunc(s3APIMiddleware(api.CopyObjectPartHandler)). + Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}") + // PutObjectPart + router.Methods(http.MethodPut).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.PutObjectPartHandler, traceHdrsS3HFlag)). + Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}") + // ListObjectParts + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.ListObjectPartsHandler)). + Queries("uploadId", "{uploadId:.*}") + // CompleteMultipartUpload + router.Methods(http.MethodPost).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.CompleteMultipartUploadHandler)). + Queries("uploadId", "{uploadId:.*}") + // NewMultipartUpload + router.Methods(http.MethodPost).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.NewMultipartUploadHandler)). + Queries("uploads", "") + // AbortMultipartUpload + router.Methods(http.MethodDelete).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.AbortMultipartUploadHandler)). + Queries("uploadId", "{uploadId:.*}") + // GetObjectACL - this is a dummy call. + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.GetObjectACLHandler, traceHdrsS3HFlag)). + Queries("acl", "") + // PutObjectACL - this is a dummy call. + router.Methods(http.MethodPut).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.PutObjectACLHandler, traceHdrsS3HFlag)). + Queries("acl", "") + // GetObjectTagging + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.GetObjectTaggingHandler, traceHdrsS3HFlag)). + Queries("tagging", "") + // PutObjectTagging + router.Methods(http.MethodPut).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.PutObjectTaggingHandler, traceHdrsS3HFlag)). + Queries("tagging", "") + // DeleteObjectTagging + router.Methods(http.MethodDelete).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.DeleteObjectTaggingHandler, traceHdrsS3HFlag)). + Queries("tagging", "") + // SelectObjectContent + router.Methods(http.MethodPost).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.SelectObjectContentHandler, traceHdrsS3HFlag)). + Queries("select", "").Queries("select-type", "2") + // GetObjectRetention + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.GetObjectRetentionHandler)). + Queries("retention", "") + // GetObjectLegalHold + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.GetObjectLegalHoldHandler)). + Queries("legal-hold", "") + // GetObject with lambda ARNs + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.GetObjectLambdaHandler, traceHdrsS3HFlag)). + Queries("lambdaArn", "{lambdaArn:.*}") + // GetObject + router.Methods(http.MethodGet).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.GetObjectHandler, traceHdrsS3HFlag)) + // CopyObject + router.Methods(http.MethodPut).Path("/{object:.+}"). + HeadersRegexp(xhttp.AmzCopySource, ".*?(\\/|%2F).*?"). + HandlerFunc(s3APIMiddleware(api.CopyObjectHandler)) + // PutObjectRetention + router.Methods(http.MethodPut).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.PutObjectRetentionHandler)). + Queries("retention", "") + // PutObjectLegalHold + router.Methods(http.MethodPut).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.PutObjectLegalHoldHandler)). + Queries("legal-hold", "") + + // PutObject with auto-extract support for zip + router.Methods(http.MethodPut).Path("/{object:.+}"). + HeadersRegexp(xhttp.AmzSnowballExtract, "true"). + HandlerFunc(s3APIMiddleware(api.PutObjectExtractHandler, traceHdrsS3HFlag)) + + // AppendObject to be rejected + router.Methods(http.MethodPut).Path("/{object:.+}"). + HeadersRegexp(xhttp.AmzWriteOffsetBytes, ""). + HandlerFunc(s3APIMiddleware(errorResponseHandler)) + + // PutObject + router.Methods(http.MethodPut).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.PutObjectHandler, traceHdrsS3HFlag)) + + // DeleteObject + router.Methods(http.MethodDelete).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.DeleteObjectHandler)) + + // PostRestoreObject + router.Methods(http.MethodPost).Path("/{object:.+}"). + HandlerFunc(s3APIMiddleware(api.PostRestoreObjectHandler)). + Queries("restore", "") + + // Bucket operations + + // GetBucketLocation + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketLocationHandler)). + Queries("location", "") + // GetBucketPolicy + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketPolicyHandler)). + Queries("policy", "") + // GetBucketLifecycle + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketLifecycleHandler)). + Queries("lifecycle", "") + // GetBucketEncryption + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketEncryptionHandler)). + Queries("encryption", "") + // GetBucketObjectLockConfig + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketObjectLockConfigHandler)). + Queries("object-lock", "") + // GetBucketReplicationConfig + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketReplicationConfigHandler)). + Queries("replication", "") + // GetBucketVersioning + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketVersioningHandler)). + Queries("versioning", "") + // GetBucketNotification + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketNotificationHandler)). + Queries("notification", "") + // ListenNotification + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ListenNotificationHandler, noThrottleS3HFlag, traceHdrsS3HFlag)). + Queries("events", "{events:.*}") + // ResetBucketReplicationStatus - MinIO extension API + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ResetBucketReplicationStatusHandler)). + Queries("replication-reset-status", "") + + // Dummy Bucket Calls + // GetBucketACL -- this is a dummy call. + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketACLHandler)). + Queries("acl", "") + // PutBucketACL -- this is a dummy call. + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketACLHandler)). + Queries("acl", "") + // GetBucketCors - this is a dummy call. + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketCorsHandler)). + Queries("cors", "") + // PutBucketCors - this is a dummy call. + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketCorsHandler)). + Queries("cors", "") + // DeleteBucketCors - this is a dummy call. + router.Methods(http.MethodDelete). + HandlerFunc(s3APIMiddleware(api.DeleteBucketCorsHandler)). + Queries("cors", "") + // GetBucketWebsiteHandler - this is a dummy call. + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketWebsiteHandler)). + Queries("website", "") + // GetBucketAccelerateHandler - this is a dummy call. + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketAccelerateHandler)). + Queries("accelerate", "") + // GetBucketRequestPaymentHandler - this is a dummy call. + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketRequestPaymentHandler)). + Queries("requestPayment", "") + // GetBucketLoggingHandler - this is a dummy call. + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketLoggingHandler)). + Queries("logging", "") + + // GetBucketTaggingHandler + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketTaggingHandler)). + Queries("tagging", "") + // DeleteBucketWebsiteHandler + router.Methods(http.MethodDelete). + HandlerFunc(s3APIMiddleware(api.DeleteBucketWebsiteHandler)). + Queries("website", "") + // DeleteBucketTaggingHandler + router.Methods(http.MethodDelete). + HandlerFunc(s3APIMiddleware(api.DeleteBucketTaggingHandler)). + Queries("tagging", "") + + // ListMultipartUploads + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ListMultipartUploadsHandler)). + Queries("uploads", "") + // ListObjectsV2M + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ListObjectsV2MHandler)). + Queries("list-type", "2", "metadata", "true") + // ListObjectsV2 + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ListObjectsV2Handler)). + Queries("list-type", "2") + // ListObjectVersions + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ListObjectVersionsMHandler)). + Queries("versions", "", "metadata", "true") + // ListObjectVersions + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ListObjectVersionsHandler)). + Queries("versions", "") + // GetBucketPolicyStatus + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketPolicyStatusHandler)). + Queries("policyStatus", "") + // PutBucketLifecycle + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketLifecycleHandler)). + Queries("lifecycle", "") + // PutBucketReplicationConfig + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketReplicationConfigHandler)). + Queries("replication", "") + // PutBucketEncryption + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketEncryptionHandler)). + Queries("encryption", "") + + // PutBucketPolicy + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketPolicyHandler)). + Queries("policy", "") + + // PutBucketObjectLockConfig + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketObjectLockConfigHandler)). + Queries("object-lock", "") + // PutBucketTaggingHandler + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketTaggingHandler)). + Queries("tagging", "") + // PutBucketVersioning + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketVersioningHandler)). + Queries("versioning", "") + // PutBucketNotification + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketNotificationHandler)). + Queries("notification", "") + // ResetBucketReplicationStart - MinIO extension API + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.ResetBucketReplicationStartHandler)). + Queries("replication-reset", "") + + // PutBucket + router.Methods(http.MethodPut). + HandlerFunc(s3APIMiddleware(api.PutBucketHandler)) + // HeadBucket + router.Methods(http.MethodHead). + HandlerFunc(s3APIMiddleware(api.HeadBucketHandler)) + // PostPolicy + router.Methods(http.MethodPost). + MatcherFunc(func(r *http.Request, _ *mux.RouteMatch) bool { + return isRequestPostPolicySignatureV4(r) + }). + HandlerFunc(s3APIMiddleware(api.PostPolicyBucketHandler, traceHdrsS3HFlag)) + // DeleteMultipleObjects + router.Methods(http.MethodPost). + HandlerFunc(s3APIMiddleware(api.DeleteMultipleObjectsHandler)). + Queries("delete", "") + // DeleteBucketPolicy + router.Methods(http.MethodDelete). + HandlerFunc(s3APIMiddleware(api.DeleteBucketPolicyHandler)). + Queries("policy", "") + // DeleteBucketReplication + router.Methods(http.MethodDelete). + HandlerFunc(s3APIMiddleware(api.DeleteBucketReplicationConfigHandler)). + Queries("replication", "") + // DeleteBucketLifecycle + router.Methods(http.MethodDelete). + HandlerFunc(s3APIMiddleware(api.DeleteBucketLifecycleHandler)). + Queries("lifecycle", "") + // DeleteBucketEncryption + router.Methods(http.MethodDelete). + HandlerFunc(s3APIMiddleware(api.DeleteBucketEncryptionHandler)). + Queries("encryption", "") + // DeleteBucket + router.Methods(http.MethodDelete). + HandlerFunc(s3APIMiddleware(api.DeleteBucketHandler)) + + // MinIO extension API for replication. + // + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketReplicationMetricsV2Handler)). + Queries("replication-metrics", "2") + // deprecated handler + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.GetBucketReplicationMetricsHandler)). + Queries("replication-metrics", "") + + // ValidateBucketReplicationCreds + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ValidateBucketReplicationCredsHandler)). + Queries("replication-check", "") + + // Register rejected bucket APIs + for _, r := range rejectedBucketAPIs { + router.Methods(r.methods...). + HandlerFunc(collectAPIStats(r.api, httpTraceAll(notImplementedHandler))). + Queries(r.queries...) + } + + // S3 ListObjectsV1 (Legacy) + router.Methods(http.MethodGet). + HandlerFunc(s3APIMiddleware(api.ListObjectsV1Handler)) + } + + // Root operation + + // ListenNotification + apiRouter.Methods(http.MethodGet).Path(SlashSeparator). + HandlerFunc(s3APIMiddleware(api.ListenNotificationHandler, noThrottleS3HFlag, traceHdrsS3HFlag)). + Queries("events", "{events:.*}") + + // ListBuckets + apiRouter.Methods(http.MethodGet).Path(SlashSeparator). + HandlerFunc(s3APIMiddleware(api.ListBucketsHandler)) + + // S3 browser with signature v4 adds '//' for ListBuckets request, so rather + // than failing with UnknownAPIRequest we simply handle it for now. + apiRouter.Methods(http.MethodGet).Path(SlashSeparator + SlashSeparator). + HandlerFunc(s3APIMiddleware(api.ListBucketsHandler)) + + // If none of the routes match add default error handler routes + apiRouter.NotFoundHandler = collectAPIStats("notfound", httpTraceAll(errorResponseHandler)) + apiRouter.MethodNotAllowedHandler = collectAPIStats("methodnotallowed", httpTraceAll(methodNotAllowedHandler("S3"))) +} + +// corsHandler handler for CORS (Cross Origin Resource Sharing) +func corsHandler(handler http.Handler) http.Handler { + commonS3Headers := []string{ + xhttp.Date, + xhttp.ETag, + xhttp.ServerInfo, + xhttp.Connection, + xhttp.AcceptRanges, + xhttp.ContentRange, + xhttp.ContentEncoding, + xhttp.ContentLength, + xhttp.ContentType, + xhttp.ContentDisposition, + xhttp.LastModified, + xhttp.ContentLanguage, + xhttp.CacheControl, + xhttp.RetryAfter, + xhttp.AmzBucketRegion, + xhttp.Expires, + "X-Amz*", + "x-amz*", + "*", + } + opts := cors.Options{ + AllowOriginFunc: func(origin string) bool { + for _, allowedOrigin := range globalAPIConfig.getCorsAllowOrigins() { + if wildcard.MatchSimple(allowedOrigin, origin) { + return true + } + } + return false + }, + AllowedMethods: []string{ + http.MethodGet, + http.MethodPut, + http.MethodHead, + http.MethodPost, + http.MethodDelete, + http.MethodOptions, + http.MethodPatch, + }, + AllowedHeaders: commonS3Headers, + ExposedHeaders: commonS3Headers, + AllowCredentials: true, + } + return cors.New(opts).Handler(handler) +} diff --git a/cmd/api-utils.go b/cmd/api-utils.go new file mode 100644 index 00000000..ee8fa533 --- /dev/null +++ b/cmd/api-utils.go @@ -0,0 +1,119 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "net/http" + "reflect" + "runtime" + "strings" +) + +func shouldEscape(c byte) bool { + if 'A' <= c && c <= 'Z' || 'a' <= c && c <= 'z' || '0' <= c && c <= '9' { + return false + } + + switch c { + case '-', '_', '.', '/', '*': + return false + } + return true +} + +// s3URLEncode is based on Golang's url.QueryEscape() code, +// while considering some S3 exceptions: +// - Avoid encoding '/' and '*' +// - Force encoding of '~' +func s3URLEncode(s string) string { + spaceCount, hexCount := 0, 0 + for i := range len(s) { + c := s[i] + if shouldEscape(c) { + if c == ' ' { + spaceCount++ + } else { + hexCount++ + } + } + } + + if spaceCount == 0 && hexCount == 0 { + return s + } + + var buf [64]byte + var t []byte + + required := len(s) + 2*hexCount + if required <= len(buf) { + t = buf[:required] + } else { + t = make([]byte, required) + } + + if hexCount == 0 { + copy(t, s) + for i := range len(s) { + if s[i] == ' ' { + t[i] = '+' + } + } + return string(t) + } + + j := 0 + for i := range len(s) { + switch c := s[i]; { + case c == ' ': + t[j] = '+' + j++ + case shouldEscape(c): + t[j] = '%' + t[j+1] = "0123456789ABCDEF"[c>>4] + t[j+2] = "0123456789ABCDEF"[c&15] + j += 3 + default: + t[j] = s[i] + j++ + } + } + return string(t) +} + +// s3EncodeName encodes string in response when encodingType is specified in AWS S3 requests. +func s3EncodeName(name, encodingType string) string { + if strings.ToLower(encodingType) == "url" { + return s3URLEncode(name) + } + return name +} + +// getHandlerName returns the name of the handler function. It takes the type +// name as a string to clean up the name retrieved via reflection. This function +// only works correctly when the type is present in the cmd package. +func getHandlerName(f http.HandlerFunc, cmdType string) string { + name := runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name() + + packageName := fmt.Sprintf("github.com/minio/minio/cmd.%s.", cmdType) + name = strings.TrimPrefix(name, packageName) + name = strings.TrimSuffix(name, "Handler-fm") + name = strings.TrimSuffix(name, "-fm") + return name +} diff --git a/cmd/api-utils_test.go b/cmd/api-utils_test.go new file mode 100644 index 00000000..b8bc050f --- /dev/null +++ b/cmd/api-utils_test.go @@ -0,0 +1,49 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "testing" +) + +func TestS3EncodeName(t *testing.T) { + testCases := []struct { + inputText, encodingType, expectedOutput string + }{ + {"a b", "", "a b"}, + {"a b", "url", "a+b"}, + {"p- ", "url", "p-+"}, + {"p-%", "url", "p-%25"}, + {"p/", "url", "p/"}, + {"p/", "url", "p/"}, + {"~user", "url", "%7Euser"}, + {"*user", "url", "*user"}, + {"user+password", "url", "user%2Bpassword"}, + {"_user", "url", "_user"}, + {"firstname.lastname", "url", "firstname.lastname"}, + } + for i, testCase := range testCases { + t.Run(fmt.Sprintf("Test%d", i+1), func(t *testing.T) { + outputText := s3EncodeName(testCase.inputText, testCase.encodingType) + if testCase.expectedOutput != outputText { + t.Errorf("Expected `%s`, got `%s`", testCase.expectedOutput, outputText) + } + }) + } +} diff --git a/cmd/apierrorcode_string.go b/cmd/apierrorcode_string.go new file mode 100644 index 00000000..d9973d88 --- /dev/null +++ b/cmd/apierrorcode_string.go @@ -0,0 +1,354 @@ +// Code generated by "stringer -type=APIErrorCode -trimprefix=Err api-errors.go"; DO NOT EDIT. + +package cmd + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[ErrNone-0] + _ = x[ErrAccessDenied-1] + _ = x[ErrBadDigest-2] + _ = x[ErrEntityTooSmall-3] + _ = x[ErrEntityTooLarge-4] + _ = x[ErrPolicyTooLarge-5] + _ = x[ErrIncompleteBody-6] + _ = x[ErrInternalError-7] + _ = x[ErrInvalidAccessKeyID-8] + _ = x[ErrAccessKeyDisabled-9] + _ = x[ErrInvalidArgument-10] + _ = x[ErrInvalidBucketName-11] + _ = x[ErrInvalidDigest-12] + _ = x[ErrInvalidRange-13] + _ = x[ErrInvalidRangePartNumber-14] + _ = x[ErrInvalidCopyPartRange-15] + _ = x[ErrInvalidCopyPartRangeSource-16] + _ = x[ErrInvalidMaxKeys-17] + _ = x[ErrInvalidEncodingMethod-18] + _ = x[ErrInvalidMaxUploads-19] + _ = x[ErrInvalidMaxParts-20] + _ = x[ErrInvalidPartNumberMarker-21] + _ = x[ErrInvalidPartNumber-22] + _ = x[ErrInvalidRequestBody-23] + _ = x[ErrInvalidCopySource-24] + _ = x[ErrInvalidMetadataDirective-25] + _ = x[ErrInvalidCopyDest-26] + _ = x[ErrInvalidPolicyDocument-27] + _ = x[ErrInvalidObjectState-28] + _ = x[ErrMalformedXML-29] + _ = x[ErrMissingContentLength-30] + _ = x[ErrMissingContentMD5-31] + _ = x[ErrMissingRequestBodyError-32] + _ = x[ErrMissingSecurityHeader-33] + _ = x[ErrNoSuchBucket-34] + _ = x[ErrNoSuchBucketPolicy-35] + _ = x[ErrNoSuchBucketLifecycle-36] + _ = x[ErrNoSuchLifecycleConfiguration-37] + _ = x[ErrInvalidLifecycleWithObjectLock-38] + _ = x[ErrNoSuchBucketSSEConfig-39] + _ = x[ErrNoSuchCORSConfiguration-40] + _ = x[ErrNoSuchWebsiteConfiguration-41] + _ = x[ErrReplicationConfigurationNotFoundError-42] + _ = x[ErrRemoteDestinationNotFoundError-43] + _ = x[ErrReplicationDestinationMissingLock-44] + _ = x[ErrRemoteTargetNotFoundError-45] + _ = x[ErrReplicationRemoteConnectionError-46] + _ = x[ErrReplicationBandwidthLimitError-47] + _ = x[ErrBucketRemoteIdenticalToSource-48] + _ = x[ErrBucketRemoteAlreadyExists-49] + _ = x[ErrBucketRemoteLabelInUse-50] + _ = x[ErrBucketRemoteArnTypeInvalid-51] + _ = x[ErrBucketRemoteArnInvalid-52] + _ = x[ErrBucketRemoteRemoveDisallowed-53] + _ = x[ErrRemoteTargetNotVersionedError-54] + _ = x[ErrReplicationSourceNotVersionedError-55] + _ = x[ErrReplicationNeedsVersioningError-56] + _ = x[ErrReplicationBucketNeedsVersioningError-57] + _ = x[ErrReplicationDenyEditError-58] + _ = x[ErrRemoteTargetDenyAddError-59] + _ = x[ErrReplicationNoExistingObjects-60] + _ = x[ErrReplicationValidationError-61] + _ = x[ErrReplicationPermissionCheckError-62] + _ = x[ErrObjectRestoreAlreadyInProgress-63] + _ = x[ErrNoSuchKey-64] + _ = x[ErrNoSuchUpload-65] + _ = x[ErrInvalidVersionID-66] + _ = x[ErrNoSuchVersion-67] + _ = x[ErrNotImplemented-68] + _ = x[ErrPreconditionFailed-69] + _ = x[ErrRequestTimeTooSkewed-70] + _ = x[ErrSignatureDoesNotMatch-71] + _ = x[ErrMethodNotAllowed-72] + _ = x[ErrInvalidPart-73] + _ = x[ErrInvalidPartOrder-74] + _ = x[ErrMissingPart-75] + _ = x[ErrAuthorizationHeaderMalformed-76] + _ = x[ErrMalformedPOSTRequest-77] + _ = x[ErrPOSTFileRequired-78] + _ = x[ErrSignatureVersionNotSupported-79] + _ = x[ErrBucketNotEmpty-80] + _ = x[ErrAllAccessDisabled-81] + _ = x[ErrPolicyInvalidVersion-82] + _ = x[ErrMissingFields-83] + _ = x[ErrMissingCredTag-84] + _ = x[ErrCredMalformed-85] + _ = x[ErrInvalidRegion-86] + _ = x[ErrInvalidServiceS3-87] + _ = x[ErrInvalidServiceSTS-88] + _ = x[ErrInvalidRequestVersion-89] + _ = x[ErrMissingSignTag-90] + _ = x[ErrMissingSignHeadersTag-91] + _ = x[ErrMalformedDate-92] + _ = x[ErrMalformedPresignedDate-93] + _ = x[ErrMalformedCredentialDate-94] + _ = x[ErrMalformedExpires-95] + _ = x[ErrNegativeExpires-96] + _ = x[ErrAuthHeaderEmpty-97] + _ = x[ErrExpiredPresignRequest-98] + _ = x[ErrRequestNotReadyYet-99] + _ = x[ErrUnsignedHeaders-100] + _ = x[ErrMissingDateHeader-101] + _ = x[ErrInvalidQuerySignatureAlgo-102] + _ = x[ErrInvalidQueryParams-103] + _ = x[ErrBucketAlreadyOwnedByYou-104] + _ = x[ErrInvalidDuration-105] + _ = x[ErrBucketAlreadyExists-106] + _ = x[ErrMetadataTooLarge-107] + _ = x[ErrUnsupportedMetadata-108] + _ = x[ErrUnsupportedHostHeader-109] + _ = x[ErrMaximumExpires-110] + _ = x[ErrSlowDownRead-111] + _ = x[ErrSlowDownWrite-112] + _ = x[ErrMaxVersionsExceeded-113] + _ = x[ErrInvalidPrefixMarker-114] + _ = x[ErrBadRequest-115] + _ = x[ErrKeyTooLongError-116] + _ = x[ErrInvalidBucketObjectLockConfiguration-117] + _ = x[ErrObjectLockConfigurationNotFound-118] + _ = x[ErrObjectLockConfigurationNotAllowed-119] + _ = x[ErrNoSuchObjectLockConfiguration-120] + _ = x[ErrObjectLocked-121] + _ = x[ErrInvalidRetentionDate-122] + _ = x[ErrPastObjectLockRetainDate-123] + _ = x[ErrUnknownWORMModeDirective-124] + _ = x[ErrBucketTaggingNotFound-125] + _ = x[ErrObjectLockInvalidHeaders-126] + _ = x[ErrInvalidTagDirective-127] + _ = x[ErrPolicyAlreadyAttached-128] + _ = x[ErrPolicyNotAttached-129] + _ = x[ErrExcessData-130] + _ = x[ErrPolicyInvalidName-131] + _ = x[ErrNoTokenRevokeType-132] + _ = x[ErrAdminOpenIDNotEnabled-133] + _ = x[ErrAdminNoSuchAccessKey-134] + _ = x[ErrInvalidEncryptionMethod-135] + _ = x[ErrInvalidEncryptionKeyID-136] + _ = x[ErrInsecureSSECustomerRequest-137] + _ = x[ErrSSEMultipartEncrypted-138] + _ = x[ErrSSEEncryptedObject-139] + _ = x[ErrInvalidEncryptionParameters-140] + _ = x[ErrInvalidEncryptionParametersSSEC-141] + _ = x[ErrInvalidSSECustomerAlgorithm-142] + _ = x[ErrInvalidSSECustomerKey-143] + _ = x[ErrMissingSSECustomerKey-144] + _ = x[ErrMissingSSECustomerKeyMD5-145] + _ = x[ErrSSECustomerKeyMD5Mismatch-146] + _ = x[ErrInvalidSSECustomerParameters-147] + _ = x[ErrIncompatibleEncryptionMethod-148] + _ = x[ErrKMSNotConfigured-149] + _ = x[ErrKMSKeyNotFoundException-150] + _ = x[ErrKMSDefaultKeyAlreadyConfigured-151] + _ = x[ErrNoAccessKey-152] + _ = x[ErrInvalidToken-153] + _ = x[ErrEventNotification-154] + _ = x[ErrARNNotification-155] + _ = x[ErrRegionNotification-156] + _ = x[ErrOverlappingFilterNotification-157] + _ = x[ErrFilterNameInvalid-158] + _ = x[ErrFilterNamePrefix-159] + _ = x[ErrFilterNameSuffix-160] + _ = x[ErrFilterValueInvalid-161] + _ = x[ErrOverlappingConfigs-162] + _ = x[ErrUnsupportedNotification-163] + _ = x[ErrContentSHA256Mismatch-164] + _ = x[ErrContentChecksumMismatch-165] + _ = x[ErrStorageFull-166] + _ = x[ErrRequestBodyParse-167] + _ = x[ErrObjectExistsAsDirectory-168] + _ = x[ErrInvalidObjectName-169] + _ = x[ErrInvalidObjectNamePrefixSlash-170] + _ = x[ErrInvalidResourceName-171] + _ = x[ErrInvalidLifecycleQueryParameter-172] + _ = x[ErrServerNotInitialized-173] + _ = x[ErrBucketMetadataNotInitialized-174] + _ = x[ErrRequestTimedout-175] + _ = x[ErrClientDisconnected-176] + _ = x[ErrTooManyRequests-177] + _ = x[ErrInvalidRequest-178] + _ = x[ErrTransitionStorageClassNotFoundError-179] + _ = x[ErrInvalidStorageClass-180] + _ = x[ErrBackendDown-181] + _ = x[ErrMalformedJSON-182] + _ = x[ErrAdminNoSuchUser-183] + _ = x[ErrAdminNoSuchUserLDAPWarn-184] + _ = x[ErrAdminLDAPExpectedLoginName-185] + _ = x[ErrAdminNoSuchGroup-186] + _ = x[ErrAdminGroupNotEmpty-187] + _ = x[ErrAdminGroupDisabled-188] + _ = x[ErrAdminInvalidGroupName-189] + _ = x[ErrAdminNoSuchJob-190] + _ = x[ErrAdminNoSuchPolicy-191] + _ = x[ErrAdminPolicyChangeAlreadyApplied-192] + _ = x[ErrAdminInvalidArgument-193] + _ = x[ErrAdminInvalidAccessKey-194] + _ = x[ErrAdminInvalidSecretKey-195] + _ = x[ErrAdminConfigNoQuorum-196] + _ = x[ErrAdminConfigTooLarge-197] + _ = x[ErrAdminConfigBadJSON-198] + _ = x[ErrAdminNoSuchConfigTarget-199] + _ = x[ErrAdminConfigEnvOverridden-200] + _ = x[ErrAdminConfigDuplicateKeys-201] + _ = x[ErrAdminConfigInvalidIDPType-202] + _ = x[ErrAdminConfigLDAPNonDefaultConfigName-203] + _ = x[ErrAdminConfigLDAPValidation-204] + _ = x[ErrAdminConfigIDPCfgNameAlreadyExists-205] + _ = x[ErrAdminConfigIDPCfgNameDoesNotExist-206] + _ = x[ErrInsecureClientRequest-207] + _ = x[ErrObjectTampered-208] + _ = x[ErrAdminLDAPNotEnabled-209] + _ = x[ErrSiteReplicationInvalidRequest-210] + _ = x[ErrSiteReplicationPeerResp-211] + _ = x[ErrSiteReplicationBackendIssue-212] + _ = x[ErrSiteReplicationServiceAccountError-213] + _ = x[ErrSiteReplicationBucketConfigError-214] + _ = x[ErrSiteReplicationBucketMetaError-215] + _ = x[ErrSiteReplicationIAMError-216] + _ = x[ErrSiteReplicationConfigMissing-217] + _ = x[ErrSiteReplicationIAMConfigMismatch-218] + _ = x[ErrAdminRebalanceAlreadyStarted-219] + _ = x[ErrAdminRebalanceNotStarted-220] + _ = x[ErrAdminBucketQuotaExceeded-221] + _ = x[ErrAdminNoSuchQuotaConfiguration-222] + _ = x[ErrHealNotImplemented-223] + _ = x[ErrHealNoSuchProcess-224] + _ = x[ErrHealInvalidClientToken-225] + _ = x[ErrHealMissingBucket-226] + _ = x[ErrHealAlreadyRunning-227] + _ = x[ErrHealOverlappingPaths-228] + _ = x[ErrIncorrectContinuationToken-229] + _ = x[ErrEmptyRequestBody-230] + _ = x[ErrUnsupportedFunction-231] + _ = x[ErrInvalidExpressionType-232] + _ = x[ErrBusy-233] + _ = x[ErrUnauthorizedAccess-234] + _ = x[ErrExpressionTooLong-235] + _ = x[ErrIllegalSQLFunctionArgument-236] + _ = x[ErrInvalidKeyPath-237] + _ = x[ErrInvalidCompressionFormat-238] + _ = x[ErrInvalidFileHeaderInfo-239] + _ = x[ErrInvalidJSONType-240] + _ = x[ErrInvalidQuoteFields-241] + _ = x[ErrInvalidRequestParameter-242] + _ = x[ErrInvalidDataType-243] + _ = x[ErrInvalidTextEncoding-244] + _ = x[ErrInvalidDataSource-245] + _ = x[ErrInvalidTableAlias-246] + _ = x[ErrMissingRequiredParameter-247] + _ = x[ErrObjectSerializationConflict-248] + _ = x[ErrUnsupportedSQLOperation-249] + _ = x[ErrUnsupportedSQLStructure-250] + _ = x[ErrUnsupportedSyntax-251] + _ = x[ErrUnsupportedRangeHeader-252] + _ = x[ErrLexerInvalidChar-253] + _ = x[ErrLexerInvalidOperator-254] + _ = x[ErrLexerInvalidLiteral-255] + _ = x[ErrLexerInvalidIONLiteral-256] + _ = x[ErrParseExpectedDatePart-257] + _ = x[ErrParseExpectedKeyword-258] + _ = x[ErrParseExpectedTokenType-259] + _ = x[ErrParseExpected2TokenTypes-260] + _ = x[ErrParseExpectedNumber-261] + _ = x[ErrParseExpectedRightParenBuiltinFunctionCall-262] + _ = x[ErrParseExpectedTypeName-263] + _ = x[ErrParseExpectedWhenClause-264] + _ = x[ErrParseUnsupportedToken-265] + _ = x[ErrParseUnsupportedLiteralsGroupBy-266] + _ = x[ErrParseExpectedMember-267] + _ = x[ErrParseUnsupportedSelect-268] + _ = x[ErrParseUnsupportedCase-269] + _ = x[ErrParseUnsupportedCaseClause-270] + _ = x[ErrParseUnsupportedAlias-271] + _ = x[ErrParseUnsupportedSyntax-272] + _ = x[ErrParseUnknownOperator-273] + _ = x[ErrParseMissingIdentAfterAt-274] + _ = x[ErrParseUnexpectedOperator-275] + _ = x[ErrParseUnexpectedTerm-276] + _ = x[ErrParseUnexpectedToken-277] + _ = x[ErrParseUnexpectedKeyword-278] + _ = x[ErrParseExpectedExpression-279] + _ = x[ErrParseExpectedLeftParenAfterCast-280] + _ = x[ErrParseExpectedLeftParenValueConstructor-281] + _ = x[ErrParseExpectedLeftParenBuiltinFunctionCall-282] + _ = x[ErrParseExpectedArgumentDelimiter-283] + _ = x[ErrParseCastArity-284] + _ = x[ErrParseInvalidTypeParam-285] + _ = x[ErrParseEmptySelect-286] + _ = x[ErrParseSelectMissingFrom-287] + _ = x[ErrParseExpectedIdentForGroupName-288] + _ = x[ErrParseExpectedIdentForAlias-289] + _ = x[ErrParseUnsupportedCallWithStar-290] + _ = x[ErrParseNonUnaryAggregateFunctionCall-291] + _ = x[ErrParseMalformedJoin-292] + _ = x[ErrParseExpectedIdentForAt-293] + _ = x[ErrParseAsteriskIsNotAloneInSelectList-294] + _ = x[ErrParseCannotMixSqbAndWildcardInSelectList-295] + _ = x[ErrParseInvalidContextForWildcardInSelectList-296] + _ = x[ErrIncorrectSQLFunctionArgumentType-297] + _ = x[ErrValueParseFailure-298] + _ = x[ErrEvaluatorInvalidArguments-299] + _ = x[ErrIntegerOverflow-300] + _ = x[ErrLikeInvalidInputs-301] + _ = x[ErrCastFailed-302] + _ = x[ErrInvalidCast-303] + _ = x[ErrEvaluatorInvalidTimestampFormatPattern-304] + _ = x[ErrEvaluatorInvalidTimestampFormatPatternSymbolForParsing-305] + _ = x[ErrEvaluatorTimestampFormatPatternDuplicateFields-306] + _ = x[ErrEvaluatorTimestampFormatPatternHourClockAmPmMismatch-307] + _ = x[ErrEvaluatorUnterminatedTimestampFormatPatternToken-308] + _ = x[ErrEvaluatorInvalidTimestampFormatPatternToken-309] + _ = x[ErrEvaluatorInvalidTimestampFormatPatternSymbol-310] + _ = x[ErrEvaluatorBindingDoesNotExist-311] + _ = x[ErrMissingHeaders-312] + _ = x[ErrInvalidColumnIndex-313] + _ = x[ErrAdminConfigNotificationTargetsFailed-314] + _ = x[ErrAdminProfilerNotEnabled-315] + _ = x[ErrInvalidDecompressedSize-316] + _ = x[ErrAddUserInvalidArgument-317] + _ = x[ErrAddUserValidUTF-318] + _ = x[ErrAdminResourceInvalidArgument-319] + _ = x[ErrAdminAccountNotEligible-320] + _ = x[ErrAccountNotEligible-321] + _ = x[ErrAdminServiceAccountNotFound-322] + _ = x[ErrPostPolicyConditionInvalidFormat-323] + _ = x[ErrInvalidChecksum-324] + _ = x[ErrLambdaARNInvalid-325] + _ = x[ErrLambdaARNNotFound-326] + _ = x[ErrInvalidAttributeName-327] + _ = x[ErrAdminNoAccessKey-328] + _ = x[ErrAdminNoSecretKey-329] + _ = x[ErrIAMNotInitialized-330] + _ = x[apiErrCodeEnd-331] +} + +const _APIErrorCode_name = "NoneAccessDeniedBadDigestEntityTooSmallEntityTooLargePolicyTooLargeIncompleteBodyInternalErrorInvalidAccessKeyIDAccessKeyDisabledInvalidArgumentInvalidBucketNameInvalidDigestInvalidRangeInvalidRangePartNumberInvalidCopyPartRangeInvalidCopyPartRangeSourceInvalidMaxKeysInvalidEncodingMethodInvalidMaxUploadsInvalidMaxPartsInvalidPartNumberMarkerInvalidPartNumberInvalidRequestBodyInvalidCopySourceInvalidMetadataDirectiveInvalidCopyDestInvalidPolicyDocumentInvalidObjectStateMalformedXMLMissingContentLengthMissingContentMD5MissingRequestBodyErrorMissingSecurityHeaderNoSuchBucketNoSuchBucketPolicyNoSuchBucketLifecycleNoSuchLifecycleConfigurationInvalidLifecycleWithObjectLockNoSuchBucketSSEConfigNoSuchCORSConfigurationNoSuchWebsiteConfigurationReplicationConfigurationNotFoundErrorRemoteDestinationNotFoundErrorReplicationDestinationMissingLockRemoteTargetNotFoundErrorReplicationRemoteConnectionErrorReplicationBandwidthLimitErrorBucketRemoteIdenticalToSourceBucketRemoteAlreadyExistsBucketRemoteLabelInUseBucketRemoteArnTypeInvalidBucketRemoteArnInvalidBucketRemoteRemoveDisallowedRemoteTargetNotVersionedErrorReplicationSourceNotVersionedErrorReplicationNeedsVersioningErrorReplicationBucketNeedsVersioningErrorReplicationDenyEditErrorRemoteTargetDenyAddErrorReplicationNoExistingObjectsReplicationValidationErrorReplicationPermissionCheckErrorObjectRestoreAlreadyInProgressNoSuchKeyNoSuchUploadInvalidVersionIDNoSuchVersionNotImplementedPreconditionFailedRequestTimeTooSkewedSignatureDoesNotMatchMethodNotAllowedInvalidPartInvalidPartOrderMissingPartAuthorizationHeaderMalformedMalformedPOSTRequestPOSTFileRequiredSignatureVersionNotSupportedBucketNotEmptyAllAccessDisabledPolicyInvalidVersionMissingFieldsMissingCredTagCredMalformedInvalidRegionInvalidServiceS3InvalidServiceSTSInvalidRequestVersionMissingSignTagMissingSignHeadersTagMalformedDateMalformedPresignedDateMalformedCredentialDateMalformedExpiresNegativeExpiresAuthHeaderEmptyExpiredPresignRequestRequestNotReadyYetUnsignedHeadersMissingDateHeaderInvalidQuerySignatureAlgoInvalidQueryParamsBucketAlreadyOwnedByYouInvalidDurationBucketAlreadyExistsMetadataTooLargeUnsupportedMetadataUnsupportedHostHeaderMaximumExpiresSlowDownReadSlowDownWriteMaxVersionsExceededInvalidPrefixMarkerBadRequestKeyTooLongErrorInvalidBucketObjectLockConfigurationObjectLockConfigurationNotFoundObjectLockConfigurationNotAllowedNoSuchObjectLockConfigurationObjectLockedInvalidRetentionDatePastObjectLockRetainDateUnknownWORMModeDirectiveBucketTaggingNotFoundObjectLockInvalidHeadersInvalidTagDirectivePolicyAlreadyAttachedPolicyNotAttachedExcessDataPolicyInvalidNameNoTokenRevokeTypeAdminOpenIDNotEnabledAdminNoSuchAccessKeyInvalidEncryptionMethodInvalidEncryptionKeyIDInsecureSSECustomerRequestSSEMultipartEncryptedSSEEncryptedObjectInvalidEncryptionParametersInvalidEncryptionParametersSSECInvalidSSECustomerAlgorithmInvalidSSECustomerKeyMissingSSECustomerKeyMissingSSECustomerKeyMD5SSECustomerKeyMD5MismatchInvalidSSECustomerParametersIncompatibleEncryptionMethodKMSNotConfiguredKMSKeyNotFoundExceptionKMSDefaultKeyAlreadyConfiguredNoAccessKeyInvalidTokenEventNotificationARNNotificationRegionNotificationOverlappingFilterNotificationFilterNameInvalidFilterNamePrefixFilterNameSuffixFilterValueInvalidOverlappingConfigsUnsupportedNotificationContentSHA256MismatchContentChecksumMismatchStorageFullRequestBodyParseObjectExistsAsDirectoryInvalidObjectNameInvalidObjectNamePrefixSlashInvalidResourceNameInvalidLifecycleQueryParameterServerNotInitializedBucketMetadataNotInitializedRequestTimedoutClientDisconnectedTooManyRequestsInvalidRequestTransitionStorageClassNotFoundErrorInvalidStorageClassBackendDownMalformedJSONAdminNoSuchUserAdminNoSuchUserLDAPWarnAdminLDAPExpectedLoginNameAdminNoSuchGroupAdminGroupNotEmptyAdminGroupDisabledAdminInvalidGroupNameAdminNoSuchJobAdminNoSuchPolicyAdminPolicyChangeAlreadyAppliedAdminInvalidArgumentAdminInvalidAccessKeyAdminInvalidSecretKeyAdminConfigNoQuorumAdminConfigTooLargeAdminConfigBadJSONAdminNoSuchConfigTargetAdminConfigEnvOverriddenAdminConfigDuplicateKeysAdminConfigInvalidIDPTypeAdminConfigLDAPNonDefaultConfigNameAdminConfigLDAPValidationAdminConfigIDPCfgNameAlreadyExistsAdminConfigIDPCfgNameDoesNotExistInsecureClientRequestObjectTamperedAdminLDAPNotEnabledSiteReplicationInvalidRequestSiteReplicationPeerRespSiteReplicationBackendIssueSiteReplicationServiceAccountErrorSiteReplicationBucketConfigErrorSiteReplicationBucketMetaErrorSiteReplicationIAMErrorSiteReplicationConfigMissingSiteReplicationIAMConfigMismatchAdminRebalanceAlreadyStartedAdminRebalanceNotStartedAdminBucketQuotaExceededAdminNoSuchQuotaConfigurationHealNotImplementedHealNoSuchProcessHealInvalidClientTokenHealMissingBucketHealAlreadyRunningHealOverlappingPathsIncorrectContinuationTokenEmptyRequestBodyUnsupportedFunctionInvalidExpressionTypeBusyUnauthorizedAccessExpressionTooLongIllegalSQLFunctionArgumentInvalidKeyPathInvalidCompressionFormatInvalidFileHeaderInfoInvalidJSONTypeInvalidQuoteFieldsInvalidRequestParameterInvalidDataTypeInvalidTextEncodingInvalidDataSourceInvalidTableAliasMissingRequiredParameterObjectSerializationConflictUnsupportedSQLOperationUnsupportedSQLStructureUnsupportedSyntaxUnsupportedRangeHeaderLexerInvalidCharLexerInvalidOperatorLexerInvalidLiteralLexerInvalidIONLiteralParseExpectedDatePartParseExpectedKeywordParseExpectedTokenTypeParseExpected2TokenTypesParseExpectedNumberParseExpectedRightParenBuiltinFunctionCallParseExpectedTypeNameParseExpectedWhenClauseParseUnsupportedTokenParseUnsupportedLiteralsGroupByParseExpectedMemberParseUnsupportedSelectParseUnsupportedCaseParseUnsupportedCaseClauseParseUnsupportedAliasParseUnsupportedSyntaxParseUnknownOperatorParseMissingIdentAfterAtParseUnexpectedOperatorParseUnexpectedTermParseUnexpectedTokenParseUnexpectedKeywordParseExpectedExpressionParseExpectedLeftParenAfterCastParseExpectedLeftParenValueConstructorParseExpectedLeftParenBuiltinFunctionCallParseExpectedArgumentDelimiterParseCastArityParseInvalidTypeParamParseEmptySelectParseSelectMissingFromParseExpectedIdentForGroupNameParseExpectedIdentForAliasParseUnsupportedCallWithStarParseNonUnaryAggregateFunctionCallParseMalformedJoinParseExpectedIdentForAtParseAsteriskIsNotAloneInSelectListParseCannotMixSqbAndWildcardInSelectListParseInvalidContextForWildcardInSelectListIncorrectSQLFunctionArgumentTypeValueParseFailureEvaluatorInvalidArgumentsIntegerOverflowLikeInvalidInputsCastFailedInvalidCastEvaluatorInvalidTimestampFormatPatternEvaluatorInvalidTimestampFormatPatternSymbolForParsingEvaluatorTimestampFormatPatternDuplicateFieldsEvaluatorTimestampFormatPatternHourClockAmPmMismatchEvaluatorUnterminatedTimestampFormatPatternTokenEvaluatorInvalidTimestampFormatPatternTokenEvaluatorInvalidTimestampFormatPatternSymbolEvaluatorBindingDoesNotExistMissingHeadersInvalidColumnIndexAdminConfigNotificationTargetsFailedAdminProfilerNotEnabledInvalidDecompressedSizeAddUserInvalidArgumentAddUserValidUTFAdminResourceInvalidArgumentAdminAccountNotEligibleAccountNotEligibleAdminServiceAccountNotFoundPostPolicyConditionInvalidFormatInvalidChecksumLambdaARNInvalidLambdaARNNotFoundInvalidAttributeNameAdminNoAccessKeyAdminNoSecretKeyIAMNotInitializedapiErrCodeEnd" + +var _APIErrorCode_index = [...]uint16{0, 4, 16, 25, 39, 53, 67, 81, 94, 112, 129, 144, 161, 174, 186, 208, 228, 254, 268, 289, 306, 321, 344, 361, 379, 396, 420, 435, 456, 474, 486, 506, 523, 546, 567, 579, 597, 618, 646, 676, 697, 720, 746, 783, 813, 846, 871, 903, 933, 962, 987, 1009, 1035, 1057, 1085, 1114, 1148, 1179, 1216, 1240, 1264, 1292, 1318, 1349, 1379, 1388, 1400, 1416, 1429, 1443, 1461, 1481, 1502, 1518, 1529, 1545, 1556, 1584, 1604, 1620, 1648, 1662, 1679, 1699, 1712, 1726, 1739, 1752, 1768, 1785, 1806, 1820, 1841, 1854, 1876, 1899, 1915, 1930, 1945, 1966, 1984, 1999, 2016, 2041, 2059, 2082, 2097, 2116, 2132, 2151, 2172, 2186, 2198, 2211, 2230, 2249, 2259, 2274, 2310, 2341, 2374, 2403, 2415, 2435, 2459, 2483, 2504, 2528, 2547, 2568, 2585, 2595, 2612, 2629, 2650, 2670, 2693, 2715, 2741, 2762, 2780, 2807, 2838, 2865, 2886, 2907, 2931, 2956, 2984, 3012, 3028, 3051, 3081, 3092, 3104, 3121, 3136, 3154, 3183, 3200, 3216, 3232, 3250, 3268, 3291, 3312, 3335, 3346, 3362, 3385, 3402, 3430, 3449, 3479, 3499, 3527, 3542, 3560, 3575, 3589, 3624, 3643, 3654, 3667, 3682, 3705, 3731, 3747, 3765, 3783, 3804, 3818, 3835, 3866, 3886, 3907, 3928, 3947, 3966, 3984, 4007, 4031, 4055, 4080, 4115, 4140, 4174, 4207, 4228, 4242, 4261, 4290, 4313, 4340, 4374, 4406, 4436, 4459, 4487, 4519, 4547, 4571, 4595, 4624, 4642, 4659, 4681, 4698, 4716, 4736, 4762, 4778, 4797, 4818, 4822, 4840, 4857, 4883, 4897, 4921, 4942, 4957, 4975, 4998, 5013, 5032, 5049, 5066, 5090, 5117, 5140, 5163, 5180, 5202, 5218, 5238, 5257, 5279, 5300, 5320, 5342, 5366, 5385, 5427, 5448, 5471, 5492, 5523, 5542, 5564, 5584, 5610, 5631, 5653, 5673, 5697, 5720, 5739, 5759, 5781, 5804, 5835, 5873, 5914, 5944, 5958, 5979, 5995, 6017, 6047, 6073, 6101, 6135, 6153, 6176, 6211, 6251, 6293, 6325, 6342, 6367, 6382, 6399, 6409, 6420, 6458, 6512, 6558, 6610, 6658, 6701, 6745, 6773, 6787, 6805, 6841, 6864, 6887, 6909, 6924, 6952, 6975, 6993, 7020, 7052, 7067, 7083, 7100, 7120, 7136, 7152, 7169, 7182} + +func (i APIErrorCode) String() string { + if i < 0 || i >= APIErrorCode(len(_APIErrorCode_index)-1) { + return "APIErrorCode(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _APIErrorCode_name[_APIErrorCode_index[i]:_APIErrorCode_index[i+1]] +} diff --git a/cmd/auth-handler.go b/cmd/auth-handler.go new file mode 100644 index 00000000..6e824e31 --- /dev/null +++ b/cmd/auth-handler.go @@ -0,0 +1,785 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/subtle" + "encoding/base64" + "encoding/hex" + "errors" + "io" + "mime" + "net/http" + "net/url" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/minio/minio/internal/auth" + objectlock "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/etag" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" + xjwt "github.com/minio/minio/internal/jwt" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/mcontext" + "github.com/minio/pkg/v3/policy" +) + +// Verify if request has JWT. +func isRequestJWT(r *http.Request) bool { + return strings.HasPrefix(r.Header.Get(xhttp.Authorization), jwtAlgorithm) +} + +// Verify if request has AWS Signature Version '4'. +func isRequestSignatureV4(r *http.Request) bool { + return strings.HasPrefix(r.Header.Get(xhttp.Authorization), signV4Algorithm) +} + +// Verify if request has AWS Signature Version '2'. +func isRequestSignatureV2(r *http.Request) bool { + return (!strings.HasPrefix(r.Header.Get(xhttp.Authorization), signV4Algorithm) && + strings.HasPrefix(r.Header.Get(xhttp.Authorization), signV2Algorithm)) +} + +// Verify if request has AWS PreSign Version '4'. +func isRequestPresignedSignatureV4(r *http.Request) bool { + _, ok := r.Form[xhttp.AmzCredential] + return ok +} + +// Verify request has AWS PreSign Version '2'. +func isRequestPresignedSignatureV2(r *http.Request) bool { + _, ok := r.Form[xhttp.AmzAccessKeyID] + return ok +} + +// Verify if request has AWS Post policy Signature Version '4'. +func isRequestPostPolicySignatureV4(r *http.Request) bool { + mediaType, _, err := mime.ParseMediaType(r.Header.Get(xhttp.ContentType)) + if err != nil { + return false + } + return mediaType == "multipart/form-data" && r.Method == http.MethodPost +} + +// Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation. +func isRequestSignStreamingV4(r *http.Request) bool { + return r.Header.Get(xhttp.AmzContentSha256) == streamingContentSHA256 && + r.Method == http.MethodPut +} + +// Verify if the request has AWS Streaming Signature Version '4'. This is only valid for 'PUT' operation. +func isRequestSignStreamingTrailerV4(r *http.Request) bool { + return r.Header.Get(xhttp.AmzContentSha256) == streamingContentSHA256Trailer && + r.Method == http.MethodPut +} + +// Verify if the request has AWS Streaming Signature Version '4', with unsigned content and trailer. +func isRequestUnsignedTrailerV4(r *http.Request) bool { + return r.Header.Get(xhttp.AmzContentSha256) == unsignedPayloadTrailer && + r.Method == http.MethodPut +} + +// Authorization type. +// +//go:generate stringer -type=authType -trimprefix=authType $GOFILE +type authType int + +// List of all supported auth types. +const ( + authTypeUnknown authType = iota + authTypeAnonymous + authTypePresigned + authTypePresignedV2 + authTypePostPolicy + authTypeStreamingSigned + authTypeSigned + authTypeSignedV2 + authTypeJWT + authTypeSTS + authTypeStreamingSignedTrailer + authTypeStreamingUnsignedTrailer +) + +// Get request authentication type. +func getRequestAuthType(r *http.Request) (at authType) { + if r.URL != nil { + var err error + r.Form, err = url.ParseQuery(r.URL.RawQuery) + if err != nil { + authNLogIf(r.Context(), err) + return authTypeUnknown + } + } + if isRequestSignatureV2(r) { + return authTypeSignedV2 + } else if isRequestPresignedSignatureV2(r) { + return authTypePresignedV2 + } else if isRequestSignStreamingV4(r) { + return authTypeStreamingSigned + } else if isRequestSignStreamingTrailerV4(r) { + return authTypeStreamingSignedTrailer + } else if isRequestUnsignedTrailerV4(r) { + return authTypeStreamingUnsignedTrailer + } else if isRequestSignatureV4(r) { + return authTypeSigned + } else if isRequestPresignedSignatureV4(r) { + return authTypePresigned + } else if isRequestJWT(r) { + return authTypeJWT + } else if isRequestPostPolicySignatureV4(r) { + return authTypePostPolicy + } else if _, ok := r.Form[xhttp.Action]; ok { + return authTypeSTS + } else if _, ok := r.Header[xhttp.Authorization]; !ok { + return authTypeAnonymous + } + return authTypeUnknown +} + +func validateAdminSignature(ctx context.Context, r *http.Request, region string) (auth.Credentials, bool, APIErrorCode) { + var cred auth.Credentials + var owner bool + s3Err := ErrAccessDenied + if _, ok := r.Header[xhttp.AmzContentSha256]; ok && + getRequestAuthType(r) == authTypeSigned { + // Get credential information from the request. + cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3) + if s3Err != ErrNone { + return cred, owner, s3Err + } + + // we only support V4 (no presign) with auth body + s3Err = isReqAuthenticated(ctx, r, region, serviceS3) + } + if s3Err != ErrNone { + return cred, owner, s3Err + } + + logger.GetReqInfo(ctx).Cred = cred + logger.GetReqInfo(ctx).Owner = owner + logger.GetReqInfo(ctx).Region = globalSite.Region() + + return cred, owner, ErrNone +} + +// checkAdminRequestAuth checks for authentication and authorization for the incoming +// request. It only accepts V2 and V4 requests. Presigned, JWT and anonymous requests +// are automatically rejected. +func checkAdminRequestAuth(ctx context.Context, r *http.Request, action policy.AdminAction, region string) (auth.Credentials, APIErrorCode) { + cred, owner, s3Err := validateAdminSignature(ctx, r, region) + if s3Err != ErrNone { + return cred, s3Err + } + if globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.Action(action), + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + Claims: cred.Claims, + }) { + // Request is allowed return the appropriate access key. + return cred, ErrNone + } + + return cred, ErrAccessDenied +} + +// Fetch the security token set by the client. +func getSessionToken(r *http.Request) (token string) { + token = r.Header.Get(xhttp.AmzSecurityToken) + if token != "" { + return token + } + return r.Form.Get(xhttp.AmzSecurityToken) +} + +// Fetch claims in the security token returned by the client, doesn't return +// errors - upon errors the returned claims map will be empty. +func mustGetClaimsFromToken(r *http.Request) map[string]any { + claims, _ := getClaimsFromToken(getSessionToken(r)) + return claims +} + +func getClaimsFromTokenWithSecret(token, secret string) (*xjwt.MapClaims, error) { + // JWT token for x-amz-security-token is signed with admin + // secret key, temporary credentials become invalid if + // server admin credentials change. This is done to ensure + // that clients cannot decode the token using the temp + // secret keys and generate an entirely new claim by essentially + // hijacking the policies. We need to make sure that this is + // based on admin credential such that token cannot be decoded + // on the client side and is treated like an opaque value. + claims, err := auth.ExtractClaims(token, secret) + if err != nil { + if subtle.ConstantTimeCompare([]byte(secret), []byte(globalActiveCred.SecretKey)) == 1 { + return nil, errAuthentication + } + claims, err = auth.ExtractClaims(token, globalActiveCred.SecretKey) + if err != nil { + return nil, errAuthentication + } + } + + // If AuthZPlugin is set, return without any further checks. + if newGlobalAuthZPluginFn() != nil { + return claims, nil + } + + // Check if a session policy is set. If so, decode it here. + sp, spok := claims.Lookup(policy.SessionPolicyName) + if spok { + // Looks like subpolicy is set and is a string, if set then its + // base64 encoded, decode it. Decoding fails reject such + // requests. + spBytes, err := base64.StdEncoding.DecodeString(sp) + if err != nil { + // Base64 decoding fails, we should log to indicate + // something is malforming the request sent by client. + authNLogIf(GlobalContext, err, logger.ErrorKind) + return nil, errAuthentication + } + claims.MapClaims[sessionPolicyNameExtracted] = string(spBytes) + } + + return claims, nil +} + +// Fetch claims in the security token returned by the client. +func getClaimsFromToken(token string) (map[string]any, error) { + jwtClaims, err := getClaimsFromTokenWithSecret(token, globalActiveCred.SecretKey) + if err != nil { + return nil, err + } + return jwtClaims.Map(), nil +} + +// Fetch claims in the security token returned by the client and validate the token. +func checkClaimsFromToken(r *http.Request, cred auth.Credentials) (map[string]any, APIErrorCode) { + token := getSessionToken(r) + if token != "" && cred.AccessKey == "" { + // x-amz-security-token is not allowed for anonymous access. + return nil, ErrNoAccessKey + } + + if token == "" && cred.IsTemp() && !cred.IsServiceAccount() { + // Temporary credentials should always have x-amz-security-token + return nil, ErrInvalidToken + } + + if token != "" && !cred.IsTemp() { + // x-amz-security-token should not present for static credentials. + return nil, ErrInvalidToken + } + + if !cred.IsServiceAccount() && cred.IsTemp() && subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 { + // validate token for temporary credentials only. + return nil, ErrInvalidToken + } + + // Expired credentials must return error right away. + if cred.IsTemp() && cred.IsExpired() { + return nil, toAPIErrorCode(r.Context(), errInvalidAccessKeyID) + } + secret := globalActiveCred.SecretKey + if globalSiteReplicationSys.isEnabled() && cred.AccessKey != siteReplicatorSvcAcc { + nsecret, err := getTokenSigningKey() + if err != nil { + return nil, toAPIErrorCode(r.Context(), err) + } + // sign root's temporary accounts also with site replicator creds + if cred.ParentUser != globalActiveCred.AccessKey || cred.IsTemp() { + secret = nsecret + } + } + if cred.IsServiceAccount() { + token = cred.SessionToken + secret = cred.SecretKey + } + + if token != "" { + claims, err := getClaimsFromTokenWithSecret(token, secret) + if err != nil { + return nil, toAPIErrorCode(r.Context(), err) + } + return claims.Map(), ErrNone + } + + claims := xjwt.NewMapClaims() + return claims.Map(), ErrNone +} + +// Check request auth type verifies the incoming http request +// - validates the request signature +// - validates the policy action if anonymous tests bucket policies if any, +// for authenticated requests validates IAM policies. +// +// returns APIErrorCode if any to be replied to the client. +func checkRequestAuthType(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName string) (s3Err APIErrorCode) { + logger.GetReqInfo(ctx).BucketName = bucketName + logger.GetReqInfo(ctx).ObjectName = objectName + + _, _, s3Err = checkRequestAuthTypeCredential(ctx, r, action) + return s3Err +} + +// checkRequestAuthTypeWithVID is similar to checkRequestAuthType +// passes versionID additionally. +func checkRequestAuthTypeWithVID(ctx context.Context, r *http.Request, action policy.Action, bucketName, objectName, versionID string) (s3Err APIErrorCode) { + logger.GetReqInfo(ctx).BucketName = bucketName + logger.GetReqInfo(ctx).ObjectName = objectName + logger.GetReqInfo(ctx).VersionID = versionID + + _, _, s3Err = checkRequestAuthTypeCredential(ctx, r, action) + return s3Err +} + +func authenticateRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) { + if logger.GetReqInfo(ctx) == nil { + bugLogIf(ctx, errors.New("unexpected context.Context does not have a logger.ReqInfo"), logger.ErrorKind) + return ErrAccessDenied + } + + var cred auth.Credentials + var owner bool + switch getRequestAuthType(r) { + case authTypeUnknown, authTypeStreamingSigned, authTypeStreamingSignedTrailer, authTypeStreamingUnsignedTrailer: + return ErrSignatureVersionNotSupported + case authTypePresignedV2, authTypeSignedV2: + if s3Err = isReqAuthenticatedV2(r); s3Err != ErrNone { + return s3Err + } + cred, owner, s3Err = getReqAccessKeyV2(r) + case authTypeSigned, authTypePresigned: + region := globalSite.Region() + switch action { + case policy.GetBucketLocationAction, policy.ListAllMyBucketsAction: + region = "" + } + if s3Err = isReqAuthenticated(ctx, r, region, serviceS3); s3Err != ErrNone { + return s3Err + } + cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3) + } + if s3Err != ErrNone { + return s3Err + } + + logger.GetReqInfo(ctx).Cred = cred + logger.GetReqInfo(ctx).Owner = owner + logger.GetReqInfo(ctx).Region = globalSite.Region() + + // region is valid only for CreateBucketAction. + var region string + if action == policy.CreateBucketAction { + // To extract region from XML in request body, get copy of request body. + payload, err := io.ReadAll(io.LimitReader(r.Body, maxLocationConstraintSize)) + if err != nil { + authZLogIf(ctx, err, logger.ErrorKind) + return ErrMalformedXML + } + + // Populate payload to extract location constraint. + r.Body = io.NopCloser(bytes.NewReader(payload)) + region, s3Err = parseLocationConstraint(r) + if s3Err != ErrNone { + return s3Err + } + + // Populate payload again to handle it in HTTP handler. + r.Body = io.NopCloser(bytes.NewReader(payload)) + } + + logger.GetReqInfo(ctx).Region = region + + return s3Err +} + +func authorizeRequest(ctx context.Context, r *http.Request, action policy.Action) (s3Err APIErrorCode) { + reqInfo := logger.GetReqInfo(ctx) + if reqInfo == nil { + return ErrAccessDenied + } + + cred := reqInfo.Cred + owner := reqInfo.Owner + region := reqInfo.Region + bucket := reqInfo.BucketName + object := reqInfo.ObjectName + versionID := reqInfo.VersionID + + if action != policy.ListAllMyBucketsAction && cred.AccessKey == "" { + // Anonymous checks are not meant for ListAllBuckets action + if globalPolicySys.IsAllowed(policy.BucketPolicyArgs{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: action, + BucketName: bucket, + ConditionValues: getConditionValues(r, region, auth.AnonymousCredentials), + IsOwner: false, + ObjectName: object, + }) { + // Request is allowed return the appropriate access key. + return ErrNone + } + + if action == policy.ListBucketVersionsAction { + // In AWS S3 s3:ListBucket permission is same as s3:ListBucketVersions permission + // verify as a fallback. + if globalPolicySys.IsAllowed(policy.BucketPolicyArgs{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.ListBucketAction, + BucketName: bucket, + ConditionValues: getConditionValues(r, region, auth.AnonymousCredentials), + IsOwner: false, + ObjectName: object, + }) { + // Request is allowed return the appropriate access key. + return ErrNone + } + } + + return ErrAccessDenied + } + if action == policy.DeleteObjectAction && versionID != "" { + if !globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.Action(policy.DeleteObjectVersionAction), + BucketName: bucket, + ConditionValues: getConditionValues(r, "", cred), + ObjectName: object, + IsOwner: owner, + Claims: cred.Claims, + DenyOnly: true, + }) { // Request is not allowed if Deny action on DeleteObjectVersionAction + return ErrAccessDenied + } + } + if globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: action, + BucketName: bucket, + ConditionValues: getConditionValues(r, "", cred), + ObjectName: object, + IsOwner: owner, + Claims: cred.Claims, + }) { + // Request is allowed return the appropriate access key. + return ErrNone + } + + if action == policy.ListBucketVersionsAction { + // In AWS S3 s3:ListBucket permission is same as s3:ListBucketVersions permission + // verify as a fallback. + if globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.ListBucketAction, + BucketName: bucket, + ConditionValues: getConditionValues(r, "", cred), + ObjectName: object, + IsOwner: owner, + Claims: cred.Claims, + }) { + // Request is allowed return the appropriate access key. + return ErrNone + } + } + + return ErrAccessDenied +} + +// Check request auth type verifies the incoming http request +// - validates the request signature +// - validates the policy action if anonymous tests bucket policies if any, +// for authenticated requests validates IAM policies. +// +// returns APIErrorCode if any to be replied to the client. +// Additionally returns the accessKey used in the request, and if this request is by an admin. +func checkRequestAuthTypeCredential(ctx context.Context, r *http.Request, action policy.Action) (cred auth.Credentials, owner bool, s3Err APIErrorCode) { + s3Err = authenticateRequest(ctx, r, action) + reqInfo := logger.GetReqInfo(ctx) + if reqInfo == nil { + return cred, owner, ErrAccessDenied + } + + cred = reqInfo.Cred + owner = reqInfo.Owner + if s3Err != ErrNone { + return cred, owner, s3Err + } + + return cred, owner, authorizeRequest(ctx, r, action) +} + +// Verify if request has valid AWS Signature Version '2'. +func isReqAuthenticatedV2(r *http.Request) (s3Error APIErrorCode) { + if isRequestSignatureV2(r) { + return doesSignV2Match(r) + } + return doesPresignV2SignatureMatch(r) +} + +func reqSignatureV4Verify(r *http.Request, region string, stype serviceType) (s3Error APIErrorCode) { + sha256sum := getContentSha256Cksum(r, stype) + switch { + case isRequestSignatureV4(r): + return doesSignatureMatch(sha256sum, r, region, stype) + case isRequestPresignedSignatureV4(r): + return doesPresignedSignatureMatch(sha256sum, r, region, stype) + default: + return ErrAccessDenied + } +} + +// Verify if request has valid AWS Signature Version '4'. +func isReqAuthenticated(ctx context.Context, r *http.Request, region string, stype serviceType) (s3Error APIErrorCode) { + if errCode := reqSignatureV4Verify(r, region, stype); errCode != ErrNone { + return errCode + } + + clientETag, err := etag.FromContentMD5(r.Header) + if err != nil { + return ErrInvalidDigest + } + + // Extract either 'X-Amz-Content-Sha256' header or 'X-Amz-Content-Sha256' query parameter (if V4 presigned) + // Do not verify 'X-Amz-Content-Sha256' if skipSHA256. + var contentSHA256 []byte + if skipSHA256 := skipContentSha256Cksum(r); !skipSHA256 && isRequestPresignedSignatureV4(r) { + if sha256Sum, ok := r.Form[xhttp.AmzContentSha256]; ok && len(sha256Sum) > 0 { + contentSHA256, err = hex.DecodeString(sha256Sum[0]) + if err != nil { + return ErrContentSHA256Mismatch + } + } + } else if _, ok := r.Header[xhttp.AmzContentSha256]; !skipSHA256 && ok { + contentSHA256, err = hex.DecodeString(r.Header.Get(xhttp.AmzContentSha256)) + if err != nil || len(contentSHA256) == 0 { + return ErrContentSHA256Mismatch + } + } + + // Verify 'Content-Md5' and/or 'X-Amz-Content-Sha256' if present. + // The verification happens implicit during reading. + reader, err := hash.NewReader(ctx, r.Body, -1, clientETag.String(), hex.EncodeToString(contentSHA256), -1) + if err != nil { + return toAPIErrorCode(ctx, err) + } + r.Body = reader + return ErrNone +} + +// List of all support S3 auth types. +var supportedS3AuthTypes = map[authType]struct{}{ + authTypeAnonymous: {}, + authTypePresigned: {}, + authTypePresignedV2: {}, + authTypeSigned: {}, + authTypeSignedV2: {}, + authTypePostPolicy: {}, + authTypeStreamingSigned: {}, + authTypeStreamingSignedTrailer: {}, + authTypeStreamingUnsignedTrailer: {}, +} + +// Validate if the authType is valid and supported. +func isSupportedS3AuthType(aType authType) bool { + _, ok := supportedS3AuthTypes[aType] + return ok +} + +// setAuthMiddleware to validate authorization header for the incoming request. +func setAuthMiddleware(h http.Handler) http.Handler { + // handler for validating incoming authorization headers. + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt) + + aType := getRequestAuthType(r) + switch aType { + case authTypeSigned, authTypeSignedV2, authTypeStreamingSigned, authTypeStreamingSignedTrailer: + // Verify if date headers are set, if not reject the request + amzDate, errCode := parseAmzDateHeader(r) + if errCode != ErrNone { + if ok { + tc.FuncName = "handler.Auth" + tc.ResponseRecorder.LogErrBody = true + } + + // All our internal APIs are sensitive towards Date + // header, for all requests where Date header is not + // present we will reject such clients. + defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r)) + writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(errCode), r.URL) + atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1) + return + } + // Verify if the request date header is shifted by less than globalMaxSkewTime parameter in the past + // or in the future, reject request otherwise. + curTime := UTCNow() + if curTime.Sub(amzDate) > globalMaxSkewTime || amzDate.Sub(curTime) > globalMaxSkewTime { + if ok { + tc.FuncName = "handler.Auth" + tc.ResponseRecorder.LogErrBody = true + } + + defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r)) + writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrRequestTimeTooSkewed), r.URL) + atomic.AddUint64(&globalHTTPStats.rejectedRequestsTime, 1) + return + } + h.ServeHTTP(w, r) + return + case authTypeJWT, authTypeSTS: + h.ServeHTTP(w, r) + return + default: + if isSupportedS3AuthType(aType) { + h.ServeHTTP(w, r) + return + } + } + + if ok { + tc.FuncName = "handler.Auth" + tc.ResponseRecorder.LogErrBody = true + } + + defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r)) + writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrSignatureVersionNotSupported), r.URL) + atomic.AddUint64(&globalHTTPStats.rejectedRequestsAuth, 1) + }) +} + +func isPutRetentionAllowed(bucketName, objectName string, retDays int, retDate time.Time, retMode objectlock.RetMode, byPassSet bool, r *http.Request, cred auth.Credentials, owner bool) (s3Err APIErrorCode) { + var retSet bool + if cred.AccessKey == "" { + return ErrAccessDenied + } + + conditions := getConditionValues(r, "", cred) + conditions["object-lock-mode"] = []string{string(retMode)} + conditions["object-lock-retain-until-date"] = []string{retDate.UTC().Format(time.RFC3339)} + if retDays > 0 { + conditions["object-lock-remaining-retention-days"] = []string{strconv.Itoa(retDays)} + } + if retMode == objectlock.RetGovernance && byPassSet { + byPassSet = globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.BypassGovernanceRetentionAction, + BucketName: bucketName, + ObjectName: objectName, + ConditionValues: conditions, + IsOwner: owner, + Claims: cred.Claims, + }) + } + if globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.PutObjectRetentionAction, + BucketName: bucketName, + ConditionValues: conditions, + ObjectName: objectName, + IsOwner: owner, + Claims: cred.Claims, + }) { + retSet = true + } + if byPassSet || retSet { + return ErrNone + } + return ErrAccessDenied +} + +// isPutActionAllowed - check if PUT operation is allowed on the resource, this +// call verifies bucket policies and IAM policies, supports multi user +// checks etc. +func isPutActionAllowed(ctx context.Context, atype authType, bucketName, objectName string, r *http.Request, action policy.Action) (s3Err APIErrorCode) { + var cred auth.Credentials + var owner bool + region := globalSite.Region() + switch atype { + case authTypeUnknown: + return ErrSignatureVersionNotSupported + case authTypeSignedV2, authTypePresignedV2: + cred, owner, s3Err = getReqAccessKeyV2(r) + case authTypeStreamingSigned, authTypePresigned, authTypeSigned, authTypeStreamingSignedTrailer: + cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3) + case authTypeStreamingUnsignedTrailer: + cred, owner, s3Err = getReqAccessKeyV4(r, region, serviceS3) + if s3Err == ErrMissingFields { + // Could be anonymous. cred + owner is zero value. + s3Err = ErrNone + } + } + if s3Err != ErrNone { + return s3Err + } + + logger.GetReqInfo(ctx).Cred = cred + logger.GetReqInfo(ctx).Owner = owner + logger.GetReqInfo(ctx).Region = region + + // Do not check for PutObjectRetentionAction permission, + // if mode and retain until date are not set. + // Can happen when bucket has default lock config set + if action == policy.PutObjectRetentionAction && + r.Header.Get(xhttp.AmzObjectLockMode) == "" && + r.Header.Get(xhttp.AmzObjectLockRetainUntilDate) == "" { + return ErrNone + } + + if cred.AccessKey == "" { + if globalPolicySys.IsAllowed(policy.BucketPolicyArgs{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: action, + BucketName: bucketName, + ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials), + IsOwner: false, + ObjectName: objectName, + }) { + return ErrNone + } + return ErrAccessDenied + } + + if globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: action, + BucketName: bucketName, + ConditionValues: getConditionValues(r, "", cred), + ObjectName: objectName, + IsOwner: owner, + Claims: cred.Claims, + }) { + return ErrNone + } + return ErrAccessDenied +} diff --git a/cmd/auth-handler_test.go b/cmd/auth-handler_test.go new file mode 100644 index 00000000..47f12add --- /dev/null +++ b/cmd/auth-handler_test.go @@ -0,0 +1,501 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "io" + "net/http" + "net/url" + "os" + "testing" + "time" + + "github.com/minio/minio/internal/auth" + "github.com/minio/pkg/v3/policy" +) + +type nullReader struct{} + +func (r *nullReader) Read(b []byte) (int, error) { + return len(b), nil +} + +// Test get request auth type. +func TestGetRequestAuthType(t *testing.T) { + type testCase struct { + req *http.Request + authT authType + } + nopCloser := io.NopCloser(io.LimitReader(&nullReader{}, 1024)) + testCases := []testCase{ + // Test case - 1 + // Check for generic signature v4 header. + { + req: &http.Request{ + URL: &url.URL{ + Host: "127.0.0.1:9000", + Scheme: httpScheme, + Path: SlashSeparator, + }, + Header: http.Header{ + "Authorization": []string{"AWS4-HMAC-SHA256 "}, + "X-Amz-Content-Sha256": []string{streamingContentSHA256}, + "Content-Encoding": []string{streamingContentEncoding}, + }, + Method: http.MethodPut, + Body: nopCloser, + }, + authT: authTypeStreamingSigned, + }, + // Test case - 2 + // Check for JWT header. + { + req: &http.Request{ + URL: &url.URL{ + Host: "127.0.0.1:9000", + Scheme: httpScheme, + Path: SlashSeparator, + }, + Header: http.Header{ + "Authorization": []string{"Bearer 12313123"}, + }, + }, + authT: authTypeJWT, + }, + // Test case - 3 + // Empty authorization header. + { + req: &http.Request{ + URL: &url.URL{ + Host: "127.0.0.1:9000", + Scheme: httpScheme, + Path: SlashSeparator, + }, + Header: http.Header{ + "Authorization": []string{""}, + }, + }, + authT: authTypeUnknown, + }, + // Test case - 4 + // Check for presigned. + { + req: &http.Request{ + URL: &url.URL{ + Host: "127.0.0.1:9000", + Scheme: httpScheme, + Path: SlashSeparator, + RawQuery: "X-Amz-Credential=EXAMPLEINVALIDEXAMPL%2Fs3%2F20160314%2Fus-east-1", + }, + }, + authT: authTypePresigned, + }, + // Test case - 5 + // Check for post policy. + { + req: &http.Request{ + URL: &url.URL{ + Host: "127.0.0.1:9000", + Scheme: httpScheme, + Path: SlashSeparator, + }, + Header: http.Header{ + "Content-Type": []string{"multipart/form-data"}, + }, + Method: http.MethodPost, + Body: nopCloser, + }, + authT: authTypePostPolicy, + }, + } + + // .. Tests all request auth type. + for i, testc := range testCases { + authT := getRequestAuthType(testc.req) + if authT != testc.authT { + t.Errorf("Test %d: Expected %d, got %d", i+1, testc.authT, authT) + } + } +} + +// Test all s3 supported auth types. +func TestS3SupportedAuthType(t *testing.T) { + type testCase struct { + authT authType + pass bool + } + // List of all valid and invalid test cases. + testCases := []testCase{ + // Test 1 - supported s3 type anonymous. + { + authT: authTypeAnonymous, + pass: true, + }, + // Test 2 - supported s3 type presigned. + { + authT: authTypePresigned, + pass: true, + }, + // Test 3 - supported s3 type signed. + { + authT: authTypeSigned, + pass: true, + }, + // Test 4 - supported s3 type with post policy. + { + authT: authTypePostPolicy, + pass: true, + }, + // Test 5 - supported s3 type with streaming signed. + { + authT: authTypeStreamingSigned, + pass: true, + }, + // Test 6 - supported s3 type with signature v2. + { + authT: authTypeSignedV2, + pass: true, + }, + // Test 7 - supported s3 type with presign v2. + { + authT: authTypePresignedV2, + pass: true, + }, + // Test 8 - JWT is not supported s3 type. + { + authT: authTypeJWT, + pass: false, + }, + // Test 9 - unknown auth header is not supported s3 type. + { + authT: authTypeUnknown, + pass: false, + }, + // Test 10 - some new auth type is not supported s3 type. + { + authT: authType(9), + pass: false, + }, + } + // Validate all the test cases. + for i, tt := range testCases { + ok := isSupportedS3AuthType(tt.authT) + if ok != tt.pass { + t.Errorf("Test %d:, Expected %t, got %t", i+1, tt.pass, ok) + } + } +} + +func TestIsRequestPresignedSignatureV2(t *testing.T) { + testCases := []struct { + inputQueryKey string + inputQueryValue string + expectedResult bool + }{ + // Test case - 1. + // Test case with query key "AWSAccessKeyId" set. + {"", "", false}, + // Test case - 2. + {"AWSAccessKeyId", "", true}, + // Test case - 3. + {"X-Amz-Content-Sha256", "", false}, + } + + for i, testCase := range testCases { + // creating an input HTTP request. + // Only the query parameters are relevant for this particular test. + inputReq, err := http.NewRequest(http.MethodGet, "http://example.com", nil) + if err != nil { + t.Fatalf("Error initializing input HTTP request: %v", err) + } + q := inputReq.URL.Query() + q.Add(testCase.inputQueryKey, testCase.inputQueryValue) + inputReq.URL.RawQuery = q.Encode() + inputReq.ParseForm() + + actualResult := isRequestPresignedSignatureV2(inputReq) + if testCase.expectedResult != actualResult { + t.Errorf("Test %d: Expected the result to `%v`, but instead got `%v`", i+1, testCase.expectedResult, actualResult) + } + } +} + +// TestIsRequestPresignedSignatureV4 - Test validates the logic for presign signature version v4 detection. +func TestIsRequestPresignedSignatureV4(t *testing.T) { + testCases := []struct { + inputQueryKey string + inputQueryValue string + expectedResult bool + }{ + // Test case - 1. + // Test case with query key ""X-Amz-Credential" set. + {"", "", false}, + // Test case - 2. + {"X-Amz-Credential", "", true}, + // Test case - 3. + {"X-Amz-Content-Sha256", "", false}, + } + + for i, testCase := range testCases { + // creating an input HTTP request. + // Only the query parameters are relevant for this particular test. + inputReq, err := http.NewRequest(http.MethodGet, "http://example.com", nil) + if err != nil { + t.Fatalf("Error initializing input HTTP request: %v", err) + } + q := inputReq.URL.Query() + q.Add(testCase.inputQueryKey, testCase.inputQueryValue) + inputReq.URL.RawQuery = q.Encode() + inputReq.ParseForm() + + actualResult := isRequestPresignedSignatureV4(inputReq) + if testCase.expectedResult != actualResult { + t.Errorf("Test %d: Expected the result to `%v`, but instead got `%v`", i+1, testCase.expectedResult, actualResult) + } + } +} + +// Provides a fully populated http request instance, fails otherwise. +func mustNewRequest(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request { + req, err := newTestRequest(method, urlStr, contentLength, body) + if err != nil { + t.Fatalf("Unable to initialize new http request %s", err) + } + return req +} + +// This is similar to mustNewRequest but additionally the request +// is signed with AWS Signature V4, fails if not able to do so. +func mustNewSignedRequest(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request { + req := mustNewRequest(method, urlStr, contentLength, body, t) + cred := globalActiveCred + if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil { + t.Fatalf("Unable to initialized new signed http request %s", err) + } + return req +} + +// This is similar to mustNewRequest but additionally the request +// is signed with AWS Signature V2, fails if not able to do so. +func mustNewSignedV2Request(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request { + req := mustNewRequest(method, urlStr, contentLength, body, t) + cred := globalActiveCred + if err := signRequestV2(req, cred.AccessKey, cred.SecretKey); err != nil { + t.Fatalf("Unable to initialized new signed http request %s", err) + } + return req +} + +// This is similar to mustNewRequest but additionally the request +// is presigned with AWS Signature V2, fails if not able to do so. +func mustNewPresignedV2Request(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request { + req := mustNewRequest(method, urlStr, contentLength, body, t) + cred := globalActiveCred + if err := preSignV2(req, cred.AccessKey, cred.SecretKey, time.Now().Add(10*time.Minute).Unix()); err != nil { + t.Fatalf("Unable to initialized new signed http request %s", err) + } + return req +} + +// This is similar to mustNewRequest but additionally the request +// is presigned with AWS Signature V4, fails if not able to do so. +func mustNewPresignedRequest(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request { + req := mustNewRequest(method, urlStr, contentLength, body, t) + cred := globalActiveCred + if err := preSignV4(req, cred.AccessKey, cred.SecretKey, time.Now().Add(10*time.Minute).Unix()); err != nil { + t.Fatalf("Unable to initialized new signed http request %s", err) + } + return req +} + +func mustNewSignedShortMD5Request(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request { + req := mustNewRequest(method, urlStr, contentLength, body, t) + req.Header.Set("Content-Md5", "invalid-digest") + cred := globalActiveCred + if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil { + t.Fatalf("Unable to initialized new signed http request %s", err) + } + return req +} + +func mustNewSignedEmptyMD5Request(method string, urlStr string, contentLength int64, body io.ReadSeeker, t *testing.T) *http.Request { + req := mustNewRequest(method, urlStr, contentLength, body, t) + req.Header.Set("Content-Md5", "") + cred := globalActiveCred + if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil { + t.Fatalf("Unable to initialized new signed http request %s", err) + } + return req +} + +func mustNewSignedBadMD5Request(method string, urlStr string, contentLength int64, + body io.ReadSeeker, t *testing.T, +) *http.Request { + req := mustNewRequest(method, urlStr, contentLength, body, t) + req.Header.Set("Content-Md5", "YWFhYWFhYWFhYWFhYWFhCg==") + cred := globalActiveCred + if err := signRequestV4(req, cred.AccessKey, cred.SecretKey); err != nil { + t.Fatalf("Unable to initialized new signed http request %s", err) + } + return req +} + +// Tests is requested authenticated function, tests replies for s3 errors. +func TestIsReqAuthenticated(t *testing.T) { + ctx, cancel := context.WithCancel(GlobalContext) + defer cancel() + + objLayer, fsDir, err := prepareFS(ctx) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(fsDir) + if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil { + t.Fatalf("unable initialize config file, %s", err) + } + + initAllSubsystems(ctx) + + initConfigSubsystem(ctx, objLayer) + + creds, err := auth.CreateCredentials("myuser", "mypassword") + if err != nil { + t.Fatalf("unable create credential, %s", err) + } + + globalActiveCred = creds + + globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second) + + // List of test cases for validating http request authentication. + testCases := []struct { + req *http.Request + s3Error APIErrorCode + }{ + // When request is unsigned, access denied is returned. + {mustNewRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrAccessDenied}, + // Empty Content-Md5 header. + {mustNewSignedEmptyMD5Request(http.MethodPut, "http://127.0.0.1:9000/", 5, bytes.NewReader([]byte("hello")), t), ErrInvalidDigest}, + // Short Content-Md5 header. + {mustNewSignedShortMD5Request(http.MethodPut, "http://127.0.0.1:9000/", 5, bytes.NewReader([]byte("hello")), t), ErrInvalidDigest}, + // When request is properly signed, but has bad Content-MD5 header. + {mustNewSignedBadMD5Request(http.MethodPut, "http://127.0.0.1:9000/", 5, bytes.NewReader([]byte("hello")), t), ErrBadDigest}, + // When request is properly signed, error is none. + {mustNewSignedRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrNone}, + } + + // Validates all testcases. + for i, testCase := range testCases { + s3Error := isReqAuthenticated(ctx, testCase.req, globalSite.Region(), serviceS3) + if s3Error != testCase.s3Error { + if _, err := io.ReadAll(testCase.req.Body); toAPIErrorCode(ctx, err) != testCase.s3Error { + t.Fatalf("Test %d: Unexpected S3 error: want %d - got %d (got after reading request %s)", i, testCase.s3Error, s3Error, toAPIError(ctx, err).Code) + } + } + } +} + +func TestCheckAdminRequestAuthType(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + objLayer, fsDir, err := prepareFS(ctx) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(fsDir) + + if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil { + t.Fatalf("unable initialize config file, %s", err) + } + + creds, err := auth.CreateCredentials("myuser", "mypassword") + if err != nil { + t.Fatalf("unable create credential, %s", err) + } + + globalActiveCred = creds + testCases := []struct { + Request *http.Request + ErrCode APIErrorCode + }{ + {Request: mustNewRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied}, + {Request: mustNewSignedRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrNone}, + {Request: mustNewSignedV2Request(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied}, + {Request: mustNewPresignedV2Request(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied}, + {Request: mustNewPresignedRequest(http.MethodGet, "http://127.0.0.1:9000", 0, nil, t), ErrCode: ErrAccessDenied}, + } + for i, testCase := range testCases { + if _, s3Error := checkAdminRequestAuth(ctx, testCase.Request, policy.AllAdminActions, globalSite.Region()); s3Error != testCase.ErrCode { + t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i, testCase.ErrCode, s3Error) + } + } +} + +func TestValidateAdminSignature(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + objLayer, fsDir, err := prepareFS(ctx) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(fsDir) + + if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil { + t.Fatalf("unable initialize config file, %s", err) + } + + initAllSubsystems(ctx) + + initConfigSubsystem(ctx, objLayer) + + creds, err := auth.CreateCredentials("admin", "mypassword") + if err != nil { + t.Fatalf("unable create credential, %s", err) + } + globalActiveCred = creds + + globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second) + + testCases := []struct { + AccessKey string + SecretKey string + ErrCode APIErrorCode + }{ + {"", "", ErrInvalidAccessKeyID}, + {"admin", "", ErrSignatureDoesNotMatch}, + {"admin", "wrongpassword", ErrSignatureDoesNotMatch}, + {"wronguser", "mypassword", ErrInvalidAccessKeyID}, + {"", "mypassword", ErrInvalidAccessKeyID}, + {"admin", "mypassword", ErrNone}, + } + + for i, testCase := range testCases { + req := mustNewRequest(http.MethodGet, "http://localhost:9000/", 0, nil, t) + if err := signRequestV4(req, testCase.AccessKey, testCase.SecretKey); err != nil { + t.Fatalf("Unable to initialized new signed http request %s", err) + } + _, _, s3Error := validateAdminSignature(ctx, req, globalMinioDefaultRegion) + if s3Error != testCase.ErrCode { + t.Errorf("Test %d: Unexpected s3error returned wanted %d, got %d", i+1, testCase.ErrCode, s3Error) + } + } +} diff --git a/cmd/authtype_string.go b/cmd/authtype_string.go new file mode 100644 index 00000000..0fea5481 --- /dev/null +++ b/cmd/authtype_string.go @@ -0,0 +1,34 @@ +// Code generated by "stringer -type=authType -trimprefix=authType auth-handler.go"; DO NOT EDIT. + +package cmd + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[authTypeUnknown-0] + _ = x[authTypeAnonymous-1] + _ = x[authTypePresigned-2] + _ = x[authTypePresignedV2-3] + _ = x[authTypePostPolicy-4] + _ = x[authTypeStreamingSigned-5] + _ = x[authTypeSigned-6] + _ = x[authTypeSignedV2-7] + _ = x[authTypeJWT-8] + _ = x[authTypeSTS-9] + _ = x[authTypeStreamingSignedTrailer-10] + _ = x[authTypeStreamingUnsignedTrailer-11] +} + +const _authType_name = "UnknownAnonymousPresignedPresignedV2PostPolicyStreamingSignedSignedSignedV2JWTSTSStreamingSignedTrailerStreamingUnsignedTrailer" + +var _authType_index = [...]uint8{0, 7, 16, 25, 36, 46, 61, 67, 75, 78, 81, 103, 127} + +func (i authType) String() string { + if i < 0 || i >= authType(len(_authType_index)-1) { + return "authType(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _authType_name[_authType_index[i]:_authType_index[i+1]] +} diff --git a/cmd/background-heal-ops.go b/cmd/background-heal-ops.go new file mode 100644 index 00000000..3eeff509 --- /dev/null +++ b/cmd/background-heal-ops.go @@ -0,0 +1,189 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "fmt" + "runtime" + "strconv" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/pkg/v3/env" +) + +// healTask represents what to heal along with options +// +// path: '/' => Heal disk formats along with metadata +// path: 'bucket/' or '/bucket/' => Heal bucket +// path: 'bucket/object' => Heal object +type healTask struct { + bucket string + object string + versionID string + opts madmin.HealOpts + // Healing response will be sent here + respCh chan healResult +} + +// healResult represents a healing result with a possible error +type healResult struct { + result madmin.HealResultItem + err error +} + +// healRoutine receives heal tasks, to heal buckets, objects and format.json +type healRoutine struct { + tasks chan healTask + workers int +} + +func activeListeners() int { + // Bucket notification and http trace are not costly, it is okay to ignore them + // while counting the number of concurrent connections + return int(globalHTTPListen.Subscribers()) + int(globalTrace.Subscribers()) +} + +func waitForLowIO(maxIO int, maxWait time.Duration, currentIO func() int) { + // No need to wait run at full speed. + if maxIO <= 0 { + return + } + + const waitTick = 100 * time.Millisecond + + tmpMaxWait := maxWait + + for currentIO() >= maxIO { + if tmpMaxWait > 0 { + if tmpMaxWait < waitTick { + time.Sleep(tmpMaxWait) + return + } + time.Sleep(waitTick) + tmpMaxWait -= waitTick + } + if tmpMaxWait <= 0 { + return + } + } +} + +func currentHTTPIO() int { + httpServer := newHTTPServerFn() + if httpServer == nil { + return 0 + } + + return httpServer.GetRequestCount() - activeListeners() +} + +func waitForLowHTTPReq() { + maxIO, maxWait, _ := globalHealConfig.Clone() + waitForLowIO(maxIO, maxWait, currentHTTPIO) +} + +func initBackgroundHealing(ctx context.Context, objAPI ObjectLayer) { + bgSeq := newBgHealSequence() + // Run the background healer + for range globalBackgroundHealRoutine.workers { + go globalBackgroundHealRoutine.AddWorker(ctx, objAPI, bgSeq) + } + + globalBackgroundHealState.LaunchNewHealSequence(bgSeq, objAPI) +} + +// Wait for heal requests and process them +func (h *healRoutine) AddWorker(ctx context.Context, objAPI ObjectLayer, bgSeq *healSequence) { + for { + select { + case task, ok := <-h.tasks: + if !ok { + return + } + + var res madmin.HealResultItem + var err error + switch task.bucket { + case nopHeal: + err = errSkipFile + case SlashSeparator: + res, err = healDiskFormat(ctx, objAPI, task.opts) + default: + if task.object == "" { + res, err = objAPI.HealBucket(ctx, task.bucket, task.opts) + } else { + res, err = objAPI.HealObject(ctx, task.bucket, task.object, task.versionID, task.opts) + } + } + + if task.respCh != nil { + task.respCh <- healResult{result: res, err: err} + continue + } + + // when respCh is not set caller is not waiting but we + // update the relevant metrics for them + if bgSeq != nil { + if err == nil { + bgSeq.countHealed(res.Type) + } else { + bgSeq.countFailed(res.Type) + } + } + case <-ctx.Done(): + return + } + } +} + +func newHealRoutine() *healRoutine { + workers := runtime.GOMAXPROCS(0) / 2 + + if envHealWorkers := env.Get("_MINIO_HEAL_WORKERS", ""); envHealWorkers != "" { + if numHealers, err := strconv.Atoi(envHealWorkers); err != nil { + bugLogIf(context.Background(), fmt.Errorf("invalid _MINIO_HEAL_WORKERS value: %w", err)) + } else { + workers = numHealers + } + } + + if workers == 0 { + workers = 4 + } + + return &healRoutine{ + tasks: make(chan healTask), + workers: workers, + } +} + +// healDiskFormat - heals format.json, return value indicates if a +// failure error occurred. +func healDiskFormat(ctx context.Context, objAPI ObjectLayer, opts madmin.HealOpts) (madmin.HealResultItem, error) { + res, err := objAPI.HealFormat(ctx, opts.DryRun) + + // return any error, ignore error returned when disks have + // already healed. + if err != nil && err != errNoHealRequired { + return madmin.HealResultItem{}, err + } + + return res, nil +} diff --git a/cmd/background-newdisks-heal-ops.go b/cmd/background-newdisks-heal-ops.go new file mode 100644 index 00000000..330bace4 --- /dev/null +++ b/cmd/background-newdisks-heal-ops.go @@ -0,0 +1,605 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "slices" + "sort" + "strings" + "sync" + "time" + + "github.com/dustin/go-humanize" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/config" + "github.com/minio/pkg/v3/env" +) + +const ( + defaultMonitorNewDiskInterval = time.Second * 10 + healingTrackerFilename = ".healing.bin" +) + +//go:generate msgp -file $GOFILE -unexported + +// healingTracker is used to persist healing information during a heal. +type healingTracker struct { + disk StorageAPI `msg:"-"` + mu *sync.RWMutex `msg:"-"` + + ID string + PoolIndex int + SetIndex int + DiskIndex int + Path string + Endpoint string + Started time.Time + LastUpdate time.Time + + ObjectsTotalCount uint64 + ObjectsTotalSize uint64 + + ItemsHealed uint64 + ItemsFailed uint64 + + BytesDone uint64 + BytesFailed uint64 + + // Last object scanned. + Bucket string `json:"-"` + Object string `json:"-"` + + // Numbers when current bucket started healing, + // for resuming with correct numbers. + ResumeItemsHealed uint64 `json:"-"` + ResumeItemsFailed uint64 `json:"-"` + ResumeItemsSkipped uint64 `json:"-"` + ResumeBytesDone uint64 `json:"-"` + ResumeBytesFailed uint64 `json:"-"` + ResumeBytesSkipped uint64 `json:"-"` + + // Filled on startup/restarts. + QueuedBuckets []string + + // Filled during heal. + HealedBuckets []string + + // ID of the current healing operation + HealID string + + ItemsSkipped uint64 + BytesSkipped uint64 + + RetryAttempts uint64 + + Finished bool // finished healing, whether with errors or not + + // Add future tracking capabilities + // Be sure that they are included in toHealingDisk +} + +// loadHealingTracker will load the healing tracker from the supplied disk. +// The disk ID will be validated against the loaded one. +func loadHealingTracker(ctx context.Context, disk StorageAPI) (*healingTracker, error) { + if disk == nil { + return nil, errors.New("loadHealingTracker: nil drive given") + } + diskID, err := disk.GetDiskID() + if err != nil { + return nil, err + } + b, err := disk.ReadAll(ctx, minioMetaBucket, + pathJoin(bucketMetaPrefix, healingTrackerFilename)) + if err != nil { + return nil, err + } + var h healingTracker + _, err = h.UnmarshalMsg(b) + if err != nil { + return nil, err + } + if h.ID != diskID && h.ID != "" { + return nil, fmt.Errorf("loadHealingTracker: drive id mismatch expected %s, got %s", h.ID, diskID) + } + h.disk = disk + h.ID = diskID + h.mu = &sync.RWMutex{} + return &h, nil +} + +// newHealingTracker will create a new healing tracker for the disk. +func newHealingTracker() *healingTracker { + return &healingTracker{ + mu: &sync.RWMutex{}, + } +} + +func initHealingTracker(disk StorageAPI, healID string) *healingTracker { + h := newHealingTracker() + diskID, _ := disk.GetDiskID() + h.disk = disk + h.ID = diskID + h.HealID = healID + h.Path = disk.String() + h.Endpoint = disk.Endpoint().String() + h.Started = time.Now().UTC() + h.PoolIndex, h.SetIndex, h.DiskIndex = disk.GetDiskLoc() + return h +} + +func (h *healingTracker) resetHealing() { + h.mu.Lock() + defer h.mu.Unlock() + + h.ItemsHealed = 0 + h.ItemsFailed = 0 + h.BytesDone = 0 + h.BytesFailed = 0 + h.ResumeItemsHealed = 0 + h.ResumeItemsFailed = 0 + h.ResumeBytesDone = 0 + h.ResumeBytesFailed = 0 + h.ItemsSkipped = 0 + h.BytesSkipped = 0 + + h.HealedBuckets = nil + h.Object = "" + h.Bucket = "" +} + +func (h *healingTracker) getLastUpdate() time.Time { + h.mu.RLock() + defer h.mu.RUnlock() + + return h.LastUpdate +} + +func (h *healingTracker) getBucket() string { + h.mu.RLock() + defer h.mu.RUnlock() + + return h.Bucket +} + +func (h *healingTracker) setBucket(bucket string) { + h.mu.Lock() + defer h.mu.Unlock() + + h.Bucket = bucket +} + +func (h *healingTracker) getObject() string { + h.mu.RLock() + defer h.mu.RUnlock() + + return h.Object +} + +func (h *healingTracker) setObject(object string) { + h.mu.Lock() + defer h.mu.Unlock() + + h.Object = object +} + +func (h *healingTracker) updateProgress(success, skipped bool, bytes uint64) { + h.mu.Lock() + defer h.mu.Unlock() + + switch { + case success: + h.ItemsHealed++ + h.BytesDone += bytes + case skipped: + h.ItemsSkipped++ + h.BytesSkipped += bytes + default: + h.ItemsFailed++ + h.BytesFailed += bytes + } +} + +// update will update the tracker on the disk. +// If the tracker has been deleted an error is returned. +func (h *healingTracker) update(ctx context.Context) error { + h.mu.Lock() + if h.ID == "" || h.PoolIndex < 0 || h.SetIndex < 0 || h.DiskIndex < 0 { + h.ID, _ = h.disk.GetDiskID() + h.PoolIndex, h.SetIndex, h.DiskIndex = h.disk.GetDiskLoc() + } + h.mu.Unlock() + return h.save(ctx) +} + +// save will unconditionally save the tracker and will be created if not existing. +func (h *healingTracker) save(ctx context.Context) error { + h.mu.Lock() + if h.PoolIndex < 0 || h.SetIndex < 0 || h.DiskIndex < 0 { + // Attempt to get location. + if api := newObjectLayerFn(); api != nil { + if ep, ok := api.(*erasureServerPools); ok { + h.PoolIndex, h.SetIndex, h.DiskIndex, _ = ep.getPoolAndSet(h.ID) + } + } + } + h.LastUpdate = time.Now().UTC() + htrackerBytes, err := h.MarshalMsg(nil) + h.mu.Unlock() + if err != nil { + return err + } + globalBackgroundHealState.updateHealStatus(h) + return h.disk.WriteAll(ctx, minioMetaBucket, + pathJoin(bucketMetaPrefix, healingTrackerFilename), + htrackerBytes) +} + +// delete the tracker on disk. +func (h *healingTracker) delete(ctx context.Context) error { + return h.disk.Delete(ctx, minioMetaBucket, + pathJoin(bucketMetaPrefix, healingTrackerFilename), + DeleteOptions{ + Recursive: false, + Immediate: false, + }, + ) +} + +func (h *healingTracker) isHealed(bucket string) bool { + h.mu.RLock() + defer h.mu.RUnlock() + return slices.Contains(h.HealedBuckets, bucket) +} + +// resume will reset progress to the numbers at the start of the bucket. +func (h *healingTracker) resume() { + h.mu.Lock() + defer h.mu.Unlock() + + h.ItemsHealed = h.ResumeItemsHealed + h.ItemsFailed = h.ResumeItemsFailed + h.ItemsSkipped = h.ResumeItemsSkipped + h.BytesDone = h.ResumeBytesDone + h.BytesFailed = h.ResumeBytesFailed + h.BytesSkipped = h.ResumeBytesSkipped +} + +// bucketDone should be called when a bucket is done healing. +// Adds the bucket to the list of healed buckets and updates resume numbers. +func (h *healingTracker) bucketDone(bucket string) { + h.mu.Lock() + defer h.mu.Unlock() + + h.ResumeItemsHealed = h.ItemsHealed + h.ResumeItemsFailed = h.ItemsFailed + h.ResumeItemsSkipped = h.ItemsSkipped + h.ResumeBytesDone = h.BytesDone + h.ResumeBytesFailed = h.BytesFailed + h.ResumeBytesSkipped = h.BytesSkipped + h.HealedBuckets = append(h.HealedBuckets, bucket) + for i, b := range h.QueuedBuckets { + if b == bucket { + // Delete... + h.QueuedBuckets = append(h.QueuedBuckets[:i], h.QueuedBuckets[i+1:]...) + } + } +} + +// setQueuedBuckets will add buckets, but exclude any that is already in h.HealedBuckets. +// Order is preserved. +func (h *healingTracker) setQueuedBuckets(buckets []BucketInfo) { + h.mu.Lock() + defer h.mu.Unlock() + + s := set.CreateStringSet(h.HealedBuckets...) + h.QueuedBuckets = make([]string, 0, len(buckets)) + for _, b := range buckets { + if !s.Contains(b.Name) { + h.QueuedBuckets = append(h.QueuedBuckets, b.Name) + } + } +} + +func (h *healingTracker) printTo(writer io.Writer) { + h.mu.RLock() + defer h.mu.RUnlock() + + b, err := json.MarshalIndent(h, "", " ") + if err != nil { + writer.Write([]byte(err.Error())) + return + } + writer.Write(b) +} + +// toHealingDisk converts the information to madmin.HealingDisk +func (h *healingTracker) toHealingDisk() madmin.HealingDisk { + h.mu.RLock() + defer h.mu.RUnlock() + + return madmin.HealingDisk{ + ID: h.ID, + HealID: h.HealID, + Endpoint: h.Endpoint, + PoolIndex: h.PoolIndex, + SetIndex: h.SetIndex, + DiskIndex: h.DiskIndex, + Finished: h.Finished, + Path: h.Path, + Started: h.Started.UTC(), + LastUpdate: h.LastUpdate.UTC(), + ObjectsTotalCount: h.ObjectsTotalCount, + ObjectsTotalSize: h.ObjectsTotalSize, + ItemsHealed: h.ItemsHealed, + ItemsSkipped: h.ItemsSkipped, + ItemsFailed: h.ItemsFailed, + BytesDone: h.BytesDone, + BytesSkipped: h.BytesSkipped, + BytesFailed: h.BytesFailed, + Bucket: h.Bucket, + Object: h.Object, + QueuedBuckets: h.QueuedBuckets, + HealedBuckets: h.HealedBuckets, + RetryAttempts: h.RetryAttempts, + + ObjectsHealed: h.ItemsHealed, // Deprecated July 2021 + ObjectsFailed: h.ItemsFailed, // Deprecated July 2021 + + } +} + +func initAutoHeal(ctx context.Context, objAPI ObjectLayer) { + z, ok := objAPI.(*erasureServerPools) + if !ok { + return + } + + initBackgroundHealing(ctx, objAPI) // start quick background healing + if env.Get("_MINIO_AUTO_DRIVE_HEALING", config.EnableOn) == config.EnableOn { + globalBackgroundHealState.pushHealLocalDisks(getLocalDisksToHeal()...) + go monitorLocalDisksAndHeal(ctx, z) + } + + go globalMRFState.startMRFPersistence() + go globalMRFState.healRoutine(z) +} + +func getLocalDisksToHeal() (disksToHeal Endpoints) { + globalLocalDrivesMu.RLock() + localDrives := cloneDrives(globalLocalDrivesMap) + globalLocalDrivesMu.RUnlock() + for _, disk := range localDrives { + _, err := disk.DiskInfo(context.Background(), DiskInfoOptions{}) + if errors.Is(err, errUnformattedDisk) { + disksToHeal = append(disksToHeal, disk.Endpoint()) + continue + } + if h := disk.Healing(); h != nil && !h.Finished { + disksToHeal = append(disksToHeal, disk.Endpoint()) + } + } + if len(disksToHeal) == globalEndpoints.NEndpoints() { + // When all disks == all command line endpoints + // this is a fresh setup, no need to trigger healing. + return Endpoints{} + } + return disksToHeal +} + +var newDiskHealingTimeout = newDynamicTimeout(30*time.Second, 10*time.Second) + +var errRetryHealing = errors.New("some items failed to heal, we will retry healing this drive again") + +func healFreshDisk(ctx context.Context, z *erasureServerPools, endpoint Endpoint) error { + poolIdx, setIdx := endpoint.PoolIdx, endpoint.SetIdx + disk := getStorageViaEndpoint(endpoint) + if disk == nil { + return fmt.Errorf("Unexpected error disk must be initialized by now after formatting: %s", endpoint) + } + + _, err := disk.DiskInfo(ctx, DiskInfoOptions{}) + if err != nil { + if errors.Is(err, errDriveIsRoot) { + // This is a root drive, ignore and move on + return nil + } + if !errors.Is(err, errUnformattedDisk) { + return err + } + } + + // Prevent parallel erasure set healing + locker := z.NewNSLock(minioMetaBucket, fmt.Sprintf("new-drive-healing/%d/%d", poolIdx, setIdx)) + lkctx, err := locker.GetLock(ctx, newDiskHealingTimeout) + if err != nil { + return fmt.Errorf("Healing of drive '%v' on %s pool, belonging to %s erasure set already in progress: %w", + disk, humanize.Ordinal(poolIdx+1), humanize.Ordinal(setIdx+1), err) + } + ctx = lkctx.Context() + defer locker.Unlock(lkctx) + + // Load healing tracker in this disk + tracker, err := loadHealingTracker(ctx, disk) + if err != nil { + // A healing tracker may be deleted if another disk in the + // same erasure set with same healing-id successfully finished + // healing. + if errors.Is(err, errFileNotFound) { + return nil + } + healingLogIf(ctx, fmt.Errorf("Unable to load healing tracker on '%s': %w, re-initializing..", disk, err)) + tracker = initHealingTracker(disk, mustGetUUID()) + } + + healingLogEvent(ctx, "Healing drive '%s' - 'mc admin heal alias/ --verbose' to check the current status.", endpoint) + + buckets, _ := z.ListBuckets(ctx, BucketOptions{}) + // Buckets data are dispersed in multiple pools/sets, make + // sure to heal all bucket metadata configuration. + buckets = append(buckets, BucketInfo{ + Name: pathJoin(minioMetaBucket, minioConfigPrefix), + }, BucketInfo{ + Name: pathJoin(minioMetaBucket, bucketMetaPrefix), + }) + + // Heal latest buckets first. + sort.Slice(buckets, func(i, j int) bool { + a, b := strings.HasPrefix(buckets[i].Name, minioMetaBucket), strings.HasPrefix(buckets[j].Name, minioMetaBucket) + if a != b { + return a + } + return buckets[i].Created.After(buckets[j].Created) + }) + + // Load bucket totals + cache := dataUsageCache{} + if err := cache.load(ctx, z.serverPools[poolIdx].sets[setIdx], dataUsageCacheName); err == nil { + dataUsageInfo := cache.dui(dataUsageRoot, nil) + tracker.ObjectsTotalCount = dataUsageInfo.ObjectsTotalCount + tracker.ObjectsTotalSize = dataUsageInfo.ObjectsTotalSize + } + + tracker.PoolIndex, tracker.SetIndex, tracker.DiskIndex = disk.GetDiskLoc() + tracker.setQueuedBuckets(buckets) + if err := tracker.save(ctx); err != nil { + return err + } + + // Start or resume healing of this erasure set + if err = z.serverPools[poolIdx].sets[setIdx].healErasureSet(ctx, tracker.QueuedBuckets, tracker); err != nil { + return err + } + + // if objects have failed healing, we attempt a retry to heal the drive upto 3 times before giving up. + if tracker.ItemsFailed > 0 && tracker.RetryAttempts < 4 { + tracker.RetryAttempts++ + + healingLogEvent(ctx, "Healing of drive '%s' is incomplete, retrying %s time (healed: %d, skipped: %d, failed: %d).", disk, + humanize.Ordinal(int(tracker.RetryAttempts)), tracker.ItemsHealed, tracker.ItemsSkipped, tracker.ItemsFailed) + + tracker.resetHealing() + bugLogIf(ctx, tracker.update(ctx)) + + return errRetryHealing + } + + if tracker.ItemsFailed > 0 { + healingLogEvent(ctx, "Healing of drive '%s' is incomplete, retried %d times (healed: %d, skipped: %d, failed: %d).", disk, + tracker.RetryAttempts, tracker.ItemsHealed, tracker.ItemsSkipped, tracker.ItemsFailed) + } else { + if tracker.RetryAttempts > 0 { + healingLogEvent(ctx, "Healing of drive '%s' is complete, retried %d times (healed: %d, skipped: %d).", disk, + tracker.RetryAttempts-1, tracker.ItemsHealed, tracker.ItemsSkipped) + } else { + healingLogEvent(ctx, "Healing of drive '%s' is finished (healed: %d, skipped: %d).", disk, tracker.ItemsHealed, tracker.ItemsSkipped) + } + } + if serverDebugLog { + tracker.printTo(os.Stdout) + fmt.Printf("\n") + } + + if tracker.HealID == "" { // HealID was empty only before Feb 2023 + bugLogIf(ctx, tracker.delete(ctx)) + return nil + } + + // Remove .healing.bin from all disks with similar heal-id + disks, err := z.GetDisks(poolIdx, setIdx) + if err != nil { + return err + } + + for _, disk := range disks { + if disk == nil { + continue + } + + t, err := loadHealingTracker(ctx, disk) + if err != nil { + if !errors.Is(err, errFileNotFound) { + healingLogIf(ctx, err) + } + continue + } + if t.HealID == tracker.HealID { + t.Finished = true + t.update(ctx) + } + } + + return nil +} + +// monitorLocalDisksAndHeal - ensures that detected new disks are healed +// 1. Only the concerned erasure set will be listed and healed +// 2. Only the node hosting the disk is responsible to perform the heal +func monitorLocalDisksAndHeal(ctx context.Context, z *erasureServerPools) { + // Perform automatic disk healing when a disk is replaced locally. + diskCheckTimer := time.NewTimer(defaultMonitorNewDiskInterval) + defer diskCheckTimer.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-diskCheckTimer.C: + healDisks := globalBackgroundHealState.getHealLocalDiskEndpoints() + if len(healDisks) == 0 { + // Reset for next interval. + diskCheckTimer.Reset(defaultMonitorNewDiskInterval) + continue + } + + // Reformat disks immediately + _, err := z.HealFormat(context.Background(), false) + if err != nil && !errors.Is(err, errNoHealRequired) { + healingLogIf(ctx, err) + // Reset for next interval. + diskCheckTimer.Reset(defaultMonitorNewDiskInterval) + continue + } + + for _, disk := range healDisks { + go func(disk Endpoint) { + globalBackgroundHealState.setDiskHealingStatus(disk, true) + if err := healFreshDisk(ctx, z, disk); err != nil { + globalBackgroundHealState.setDiskHealingStatus(disk, false) + timedout := OperationTimedOut{} + if !errors.Is(err, context.Canceled) && !errors.As(err, &timedout) && !errors.Is(err, errRetryHealing) { + printEndpointError(disk, err, false) + } + return + } + // Only upon success pop the healed disk. + globalBackgroundHealState.popHealLocalDisks(disk) + }(disk) + } + + // Reset for next interval. + diskCheckTimer.Reset(defaultMonitorNewDiskInterval) + } + } +} diff --git a/cmd/background-newdisks-heal-ops_gen.go b/cmd/background-newdisks-heal-ops_gen.go new file mode 100644 index 00000000..7c3eb6b8 --- /dev/null +++ b/cmd/background-newdisks-heal-ops_gen.go @@ -0,0 +1,890 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *healingTracker) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ID": + z.ID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + case "PoolIndex": + z.PoolIndex, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "PoolIndex") + return + } + case "SetIndex": + z.SetIndex, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "SetIndex") + return + } + case "DiskIndex": + z.DiskIndex, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "DiskIndex") + return + } + case "Path": + z.Path, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Path") + return + } + case "Endpoint": + z.Endpoint, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + case "Started": + z.Started, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "Started") + return + } + case "LastUpdate": + z.LastUpdate, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "LastUpdate") + return + } + case "ObjectsTotalCount": + z.ObjectsTotalCount, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ObjectsTotalCount") + return + } + case "ObjectsTotalSize": + z.ObjectsTotalSize, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ObjectsTotalSize") + return + } + case "ItemsHealed": + z.ItemsHealed, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ItemsHealed") + return + } + case "ItemsFailed": + z.ItemsFailed, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ItemsFailed") + return + } + case "BytesDone": + z.BytesDone, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "BytesDone") + return + } + case "BytesFailed": + z.BytesFailed, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "BytesFailed") + return + } + case "Bucket": + z.Bucket, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "Object": + z.Object, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + case "ResumeItemsHealed": + z.ResumeItemsHealed, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ResumeItemsHealed") + return + } + case "ResumeItemsFailed": + z.ResumeItemsFailed, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ResumeItemsFailed") + return + } + case "ResumeItemsSkipped": + z.ResumeItemsSkipped, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ResumeItemsSkipped") + return + } + case "ResumeBytesDone": + z.ResumeBytesDone, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ResumeBytesDone") + return + } + case "ResumeBytesFailed": + z.ResumeBytesFailed, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ResumeBytesFailed") + return + } + case "ResumeBytesSkipped": + z.ResumeBytesSkipped, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ResumeBytesSkipped") + return + } + case "QueuedBuckets": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "QueuedBuckets") + return + } + if cap(z.QueuedBuckets) >= int(zb0002) { + z.QueuedBuckets = (z.QueuedBuckets)[:zb0002] + } else { + z.QueuedBuckets = make([]string, zb0002) + } + for za0001 := range z.QueuedBuckets { + z.QueuedBuckets[za0001], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "QueuedBuckets", za0001) + return + } + } + case "HealedBuckets": + var zb0003 uint32 + zb0003, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "HealedBuckets") + return + } + if cap(z.HealedBuckets) >= int(zb0003) { + z.HealedBuckets = (z.HealedBuckets)[:zb0003] + } else { + z.HealedBuckets = make([]string, zb0003) + } + for za0002 := range z.HealedBuckets { + z.HealedBuckets[za0002], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "HealedBuckets", za0002) + return + } + } + case "HealID": + z.HealID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "HealID") + return + } + case "ItemsSkipped": + z.ItemsSkipped, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ItemsSkipped") + return + } + case "BytesSkipped": + z.BytesSkipped, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "BytesSkipped") + return + } + case "RetryAttempts": + z.RetryAttempts, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "RetryAttempts") + return + } + case "Finished": + z.Finished, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Finished") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *healingTracker) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 29 + // write "ID" + err = en.Append(0xde, 0x0, 0x1d, 0xa2, 0x49, 0x44) + if err != nil { + return + } + err = en.WriteString(z.ID) + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + // write "PoolIndex" + err = en.Append(0xa9, 0x50, 0x6f, 0x6f, 0x6c, 0x49, 0x6e, 0x64, 0x65, 0x78) + if err != nil { + return + } + err = en.WriteInt(z.PoolIndex) + if err != nil { + err = msgp.WrapError(err, "PoolIndex") + return + } + // write "SetIndex" + err = en.Append(0xa8, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78) + if err != nil { + return + } + err = en.WriteInt(z.SetIndex) + if err != nil { + err = msgp.WrapError(err, "SetIndex") + return + } + // write "DiskIndex" + err = en.Append(0xa9, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x64, 0x65, 0x78) + if err != nil { + return + } + err = en.WriteInt(z.DiskIndex) + if err != nil { + err = msgp.WrapError(err, "DiskIndex") + return + } + // write "Path" + err = en.Append(0xa4, 0x50, 0x61, 0x74, 0x68) + if err != nil { + return + } + err = en.WriteString(z.Path) + if err != nil { + err = msgp.WrapError(err, "Path") + return + } + // write "Endpoint" + err = en.Append(0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Endpoint) + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + // write "Started" + err = en.Append(0xa7, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteTime(z.Started) + if err != nil { + err = msgp.WrapError(err, "Started") + return + } + // write "LastUpdate" + err = en.Append(0xaa, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65) + if err != nil { + return + } + err = en.WriteTime(z.LastUpdate) + if err != nil { + err = msgp.WrapError(err, "LastUpdate") + return + } + // write "ObjectsTotalCount" + err = en.Append(0xb1, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x43, 0x6f, 0x75, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteUint64(z.ObjectsTotalCount) + if err != nil { + err = msgp.WrapError(err, "ObjectsTotalCount") + return + } + // write "ObjectsTotalSize" + err = en.Append(0xb0, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteUint64(z.ObjectsTotalSize) + if err != nil { + err = msgp.WrapError(err, "ObjectsTotalSize") + return + } + // write "ItemsHealed" + err = en.Append(0xab, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteUint64(z.ItemsHealed) + if err != nil { + err = msgp.WrapError(err, "ItemsHealed") + return + } + // write "ItemsFailed" + err = en.Append(0xab, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteUint64(z.ItemsFailed) + if err != nil { + err = msgp.WrapError(err, "ItemsFailed") + return + } + // write "BytesDone" + err = en.Append(0xa9, 0x42, 0x79, 0x74, 0x65, 0x73, 0x44, 0x6f, 0x6e, 0x65) + if err != nil { + return + } + err = en.WriteUint64(z.BytesDone) + if err != nil { + err = msgp.WrapError(err, "BytesDone") + return + } + // write "BytesFailed" + err = en.Append(0xab, 0x42, 0x79, 0x74, 0x65, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteUint64(z.BytesFailed) + if err != nil { + err = msgp.WrapError(err, "BytesFailed") + return + } + // write "Bucket" + err = en.Append(0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Bucket) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + // write "Object" + err = en.Append(0xa6, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Object) + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + // write "ResumeItemsHealed" + err = en.Append(0xb1, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteUint64(z.ResumeItemsHealed) + if err != nil { + err = msgp.WrapError(err, "ResumeItemsHealed") + return + } + // write "ResumeItemsFailed" + err = en.Append(0xb1, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteUint64(z.ResumeItemsFailed) + if err != nil { + err = msgp.WrapError(err, "ResumeItemsFailed") + return + } + // write "ResumeItemsSkipped" + err = en.Append(0xb2, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteUint64(z.ResumeItemsSkipped) + if err != nil { + err = msgp.WrapError(err, "ResumeItemsSkipped") + return + } + // write "ResumeBytesDone" + err = en.Append(0xaf, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x44, 0x6f, 0x6e, 0x65) + if err != nil { + return + } + err = en.WriteUint64(z.ResumeBytesDone) + if err != nil { + err = msgp.WrapError(err, "ResumeBytesDone") + return + } + // write "ResumeBytesFailed" + err = en.Append(0xb1, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteUint64(z.ResumeBytesFailed) + if err != nil { + err = msgp.WrapError(err, "ResumeBytesFailed") + return + } + // write "ResumeBytesSkipped" + err = en.Append(0xb2, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteUint64(z.ResumeBytesSkipped) + if err != nil { + err = msgp.WrapError(err, "ResumeBytesSkipped") + return + } + // write "QueuedBuckets" + err = en.Append(0xad, 0x51, 0x75, 0x65, 0x75, 0x65, 0x64, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.QueuedBuckets))) + if err != nil { + err = msgp.WrapError(err, "QueuedBuckets") + return + } + for za0001 := range z.QueuedBuckets { + err = en.WriteString(z.QueuedBuckets[za0001]) + if err != nil { + err = msgp.WrapError(err, "QueuedBuckets", za0001) + return + } + } + // write "HealedBuckets" + err = en.Append(0xad, 0x48, 0x65, 0x61, 0x6c, 0x65, 0x64, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.HealedBuckets))) + if err != nil { + err = msgp.WrapError(err, "HealedBuckets") + return + } + for za0002 := range z.HealedBuckets { + err = en.WriteString(z.HealedBuckets[za0002]) + if err != nil { + err = msgp.WrapError(err, "HealedBuckets", za0002) + return + } + } + // write "HealID" + err = en.Append(0xa6, 0x48, 0x65, 0x61, 0x6c, 0x49, 0x44) + if err != nil { + return + } + err = en.WriteString(z.HealID) + if err != nil { + err = msgp.WrapError(err, "HealID") + return + } + // write "ItemsSkipped" + err = en.Append(0xac, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteUint64(z.ItemsSkipped) + if err != nil { + err = msgp.WrapError(err, "ItemsSkipped") + return + } + // write "BytesSkipped" + err = en.Append(0xac, 0x42, 0x79, 0x74, 0x65, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteUint64(z.BytesSkipped) + if err != nil { + err = msgp.WrapError(err, "BytesSkipped") + return + } + // write "RetryAttempts" + err = en.Append(0xad, 0x52, 0x65, 0x74, 0x72, 0x79, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteUint64(z.RetryAttempts) + if err != nil { + err = msgp.WrapError(err, "RetryAttempts") + return + } + // write "Finished" + err = en.Append(0xa8, 0x46, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteBool(z.Finished) + if err != nil { + err = msgp.WrapError(err, "Finished") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *healingTracker) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 29 + // string "ID" + o = append(o, 0xde, 0x0, 0x1d, 0xa2, 0x49, 0x44) + o = msgp.AppendString(o, z.ID) + // string "PoolIndex" + o = append(o, 0xa9, 0x50, 0x6f, 0x6f, 0x6c, 0x49, 0x6e, 0x64, 0x65, 0x78) + o = msgp.AppendInt(o, z.PoolIndex) + // string "SetIndex" + o = append(o, 0xa8, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78) + o = msgp.AppendInt(o, z.SetIndex) + // string "DiskIndex" + o = append(o, 0xa9, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x6e, 0x64, 0x65, 0x78) + o = msgp.AppendInt(o, z.DiskIndex) + // string "Path" + o = append(o, 0xa4, 0x50, 0x61, 0x74, 0x68) + o = msgp.AppendString(o, z.Path) + // string "Endpoint" + o = append(o, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74) + o = msgp.AppendString(o, z.Endpoint) + // string "Started" + o = append(o, 0xa7, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64) + o = msgp.AppendTime(o, z.Started) + // string "LastUpdate" + o = append(o, 0xaa, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65) + o = msgp.AppendTime(o, z.LastUpdate) + // string "ObjectsTotalCount" + o = append(o, 0xb1, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendUint64(o, z.ObjectsTotalCount) + // string "ObjectsTotalSize" + o = append(o, 0xb0, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendUint64(o, z.ObjectsTotalSize) + // string "ItemsHealed" + o = append(o, 0xab, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x65, 0x64) + o = msgp.AppendUint64(o, z.ItemsHealed) + // string "ItemsFailed" + o = append(o, 0xab, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64) + o = msgp.AppendUint64(o, z.ItemsFailed) + // string "BytesDone" + o = append(o, 0xa9, 0x42, 0x79, 0x74, 0x65, 0x73, 0x44, 0x6f, 0x6e, 0x65) + o = msgp.AppendUint64(o, z.BytesDone) + // string "BytesFailed" + o = append(o, 0xab, 0x42, 0x79, 0x74, 0x65, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64) + o = msgp.AppendUint64(o, z.BytesFailed) + // string "Bucket" + o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + o = msgp.AppendString(o, z.Bucket) + // string "Object" + o = append(o, 0xa6, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74) + o = msgp.AppendString(o, z.Object) + // string "ResumeItemsHealed" + o = append(o, 0xb1, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x48, 0x65, 0x61, 0x6c, 0x65, 0x64) + o = msgp.AppendUint64(o, z.ResumeItemsHealed) + // string "ResumeItemsFailed" + o = append(o, 0xb1, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64) + o = msgp.AppendUint64(o, z.ResumeItemsFailed) + // string "ResumeItemsSkipped" + o = append(o, 0xb2, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64) + o = msgp.AppendUint64(o, z.ResumeItemsSkipped) + // string "ResumeBytesDone" + o = append(o, 0xaf, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x44, 0x6f, 0x6e, 0x65) + o = msgp.AppendUint64(o, z.ResumeBytesDone) + // string "ResumeBytesFailed" + o = append(o, 0xb1, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64) + o = msgp.AppendUint64(o, z.ResumeBytesFailed) + // string "ResumeBytesSkipped" + o = append(o, 0xb2, 0x52, 0x65, 0x73, 0x75, 0x6d, 0x65, 0x42, 0x79, 0x74, 0x65, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64) + o = msgp.AppendUint64(o, z.ResumeBytesSkipped) + // string "QueuedBuckets" + o = append(o, 0xad, 0x51, 0x75, 0x65, 0x75, 0x65, 0x64, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.QueuedBuckets))) + for za0001 := range z.QueuedBuckets { + o = msgp.AppendString(o, z.QueuedBuckets[za0001]) + } + // string "HealedBuckets" + o = append(o, 0xad, 0x48, 0x65, 0x61, 0x6c, 0x65, 0x64, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.HealedBuckets))) + for za0002 := range z.HealedBuckets { + o = msgp.AppendString(o, z.HealedBuckets[za0002]) + } + // string "HealID" + o = append(o, 0xa6, 0x48, 0x65, 0x61, 0x6c, 0x49, 0x44) + o = msgp.AppendString(o, z.HealID) + // string "ItemsSkipped" + o = append(o, 0xac, 0x49, 0x74, 0x65, 0x6d, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64) + o = msgp.AppendUint64(o, z.ItemsSkipped) + // string "BytesSkipped" + o = append(o, 0xac, 0x42, 0x79, 0x74, 0x65, 0x73, 0x53, 0x6b, 0x69, 0x70, 0x70, 0x65, 0x64) + o = msgp.AppendUint64(o, z.BytesSkipped) + // string "RetryAttempts" + o = append(o, 0xad, 0x52, 0x65, 0x74, 0x72, 0x79, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73) + o = msgp.AppendUint64(o, z.RetryAttempts) + // string "Finished" + o = append(o, 0xa8, 0x46, 0x69, 0x6e, 0x69, 0x73, 0x68, 0x65, 0x64) + o = msgp.AppendBool(o, z.Finished) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *healingTracker) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ID": + z.ID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + case "PoolIndex": + z.PoolIndex, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "PoolIndex") + return + } + case "SetIndex": + z.SetIndex, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SetIndex") + return + } + case "DiskIndex": + z.DiskIndex, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DiskIndex") + return + } + case "Path": + z.Path, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Path") + return + } + case "Endpoint": + z.Endpoint, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + case "Started": + z.Started, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Started") + return + } + case "LastUpdate": + z.LastUpdate, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "LastUpdate") + return + } + case "ObjectsTotalCount": + z.ObjectsTotalCount, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjectsTotalCount") + return + } + case "ObjectsTotalSize": + z.ObjectsTotalSize, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjectsTotalSize") + return + } + case "ItemsHealed": + z.ItemsHealed, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ItemsHealed") + return + } + case "ItemsFailed": + z.ItemsFailed, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ItemsFailed") + return + } + case "BytesDone": + z.BytesDone, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "BytesDone") + return + } + case "BytesFailed": + z.BytesFailed, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "BytesFailed") + return + } + case "Bucket": + z.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "Object": + z.Object, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + case "ResumeItemsHealed": + z.ResumeItemsHealed, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResumeItemsHealed") + return + } + case "ResumeItemsFailed": + z.ResumeItemsFailed, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResumeItemsFailed") + return + } + case "ResumeItemsSkipped": + z.ResumeItemsSkipped, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResumeItemsSkipped") + return + } + case "ResumeBytesDone": + z.ResumeBytesDone, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResumeBytesDone") + return + } + case "ResumeBytesFailed": + z.ResumeBytesFailed, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResumeBytesFailed") + return + } + case "ResumeBytesSkipped": + z.ResumeBytesSkipped, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResumeBytesSkipped") + return + } + case "QueuedBuckets": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "QueuedBuckets") + return + } + if cap(z.QueuedBuckets) >= int(zb0002) { + z.QueuedBuckets = (z.QueuedBuckets)[:zb0002] + } else { + z.QueuedBuckets = make([]string, zb0002) + } + for za0001 := range z.QueuedBuckets { + z.QueuedBuckets[za0001], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "QueuedBuckets", za0001) + return + } + } + case "HealedBuckets": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "HealedBuckets") + return + } + if cap(z.HealedBuckets) >= int(zb0003) { + z.HealedBuckets = (z.HealedBuckets)[:zb0003] + } else { + z.HealedBuckets = make([]string, zb0003) + } + for za0002 := range z.HealedBuckets { + z.HealedBuckets[za0002], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "HealedBuckets", za0002) + return + } + } + case "HealID": + z.HealID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "HealID") + return + } + case "ItemsSkipped": + z.ItemsSkipped, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ItemsSkipped") + return + } + case "BytesSkipped": + z.BytesSkipped, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "BytesSkipped") + return + } + case "RetryAttempts": + z.RetryAttempts, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "RetryAttempts") + return + } + case "Finished": + z.Finished, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Finished") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *healingTracker) Msgsize() (s int) { + s = 3 + 3 + msgp.StringPrefixSize + len(z.ID) + 10 + msgp.IntSize + 9 + msgp.IntSize + 10 + msgp.IntSize + 5 + msgp.StringPrefixSize + len(z.Path) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 8 + msgp.TimeSize + 11 + msgp.TimeSize + 18 + msgp.Uint64Size + 17 + msgp.Uint64Size + 12 + msgp.Uint64Size + 12 + msgp.Uint64Size + 10 + msgp.Uint64Size + 12 + msgp.Uint64Size + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Object) + 18 + msgp.Uint64Size + 18 + msgp.Uint64Size + 19 + msgp.Uint64Size + 16 + msgp.Uint64Size + 18 + msgp.Uint64Size + 19 + msgp.Uint64Size + 14 + msgp.ArrayHeaderSize + for za0001 := range z.QueuedBuckets { + s += msgp.StringPrefixSize + len(z.QueuedBuckets[za0001]) + } + s += 14 + msgp.ArrayHeaderSize + for za0002 := range z.HealedBuckets { + s += msgp.StringPrefixSize + len(z.HealedBuckets[za0002]) + } + s += 7 + msgp.StringPrefixSize + len(z.HealID) + 13 + msgp.Uint64Size + 13 + msgp.Uint64Size + 14 + msgp.Uint64Size + 9 + msgp.BoolSize + return +} diff --git a/cmd/background-newdisks-heal-ops_gen_test.go b/cmd/background-newdisks-heal-ops_gen_test.go new file mode 100644 index 00000000..36e9eb71 --- /dev/null +++ b/cmd/background-newdisks-heal-ops_gen_test.go @@ -0,0 +1,123 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalhealingTracker(t *testing.T) { + v := healingTracker{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsghealingTracker(b *testing.B) { + v := healingTracker{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsghealingTracker(b *testing.B) { + v := healingTracker{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalhealingTracker(b *testing.B) { + v := healingTracker{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodehealingTracker(t *testing.T) { + v := healingTracker{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodehealingTracker Msgsize() is inaccurate") + } + + vn := healingTracker{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodehealingTracker(b *testing.B) { + v := healingTracker{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodehealingTracker(b *testing.B) { + v := healingTracker{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/batch-expire.go b/cmd/batch-expire.go new file mode 100644 index 00000000..fab59266 --- /dev/null +++ b/cmd/batch-expire.go @@ -0,0 +1,839 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "runtime" + "strconv" + "time" + + "github.com/minio/minio-go/v7/pkg/tags" + "github.com/minio/minio/internal/bucket/versioning" + xhttp "github.com/minio/minio/internal/http" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/pkg/v3/env" + "github.com/minio/pkg/v3/wildcard" + "github.com/minio/pkg/v3/workers" + "github.com/minio/pkg/v3/xtime" + "gopkg.in/yaml.v3" +) + +// expire: # Expire objects that match a condition +// apiVersion: v1 +// bucket: mybucket # Bucket where this batch job will expire matching objects from +// prefix: myprefix # (Optional) Prefix under which this job will expire objects matching the rules below. +// rules: +// - type: object # regular objects with zero or more older versions +// name: NAME # match object names that satisfy the wildcard expression. +// olderThan: 70h # match objects older than this value +// createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date" +// tags: +// - key: name +// value: pick* # match objects with tag 'name', all values starting with 'pick' +// metadata: +// - key: content-type +// value: image/* # match objects with 'content-type', all values starting with 'image/' +// size: +// lessThan: "10MiB" # match objects with size less than this value (e.g. 10MiB) +// greaterThan: 1MiB # match objects with size greater than this value (e.g. 1MiB) +// purge: +// # retainVersions: 0 # (default) delete all versions of the object. This option is the fastest. +// # retainVersions: 5 # keep the latest 5 versions of the object. +// +// - type: deleted # objects with delete marker as their latest version +// name: NAME # match object names that satisfy the wildcard expression. +// olderThan: 10h # match objects older than this value (e.g. 7d10h31s) +// createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date" +// purge: +// # retainVersions: 0 # (default) delete all versions of the object. This option is the fastest. +// # retainVersions: 5 # keep the latest 5 versions of the object including delete markers. +// +// notify: +// endpoint: https://notify.endpoint # notification endpoint to receive job completion status +// token: Bearer xxxxx # optional authentication token for the notification endpoint +// +// retry: +// attempts: 10 # number of retries for the job before giving up +// delay: 500ms # least amount of delay between each retry + +//go:generate msgp -file $GOFILE + +// BatchJobExpirePurge type accepts non-negative versions to be retained +type BatchJobExpirePurge struct { + line, col int + RetainVersions int `yaml:"retainVersions" json:"retainVersions"` +} + +var _ yaml.Unmarshaler = &BatchJobExpirePurge{} + +// UnmarshalYAML - BatchJobExpirePurge extends unmarshal to extract line, col +func (p *BatchJobExpirePurge) UnmarshalYAML(val *yaml.Node) error { + type purge BatchJobExpirePurge + var tmp purge + err := val.Decode(&tmp) + if err != nil { + return err + } + + *p = BatchJobExpirePurge(tmp) + p.line, p.col = val.Line, val.Column + return nil +} + +// Validate returns nil if value is valid, ie > 0. +func (p BatchJobExpirePurge) Validate() error { + if p.RetainVersions < 0 { + return BatchJobYamlErr{ + line: p.line, + col: p.col, + msg: "retainVersions must be >= 0", + } + } + return nil +} + +// BatchJobExpireFilter holds all the filters currently supported for batch replication +type BatchJobExpireFilter struct { + line, col int + OlderThan xtime.Duration `yaml:"olderThan,omitempty" json:"olderThan"` + CreatedBefore *time.Time `yaml:"createdBefore,omitempty" json:"createdBefore"` + Tags []BatchJobKV `yaml:"tags,omitempty" json:"tags"` + Metadata []BatchJobKV `yaml:"metadata,omitempty" json:"metadata"` + Size BatchJobSizeFilter `yaml:"size" json:"size"` + Type string `yaml:"type" json:"type"` + Name string `yaml:"name" json:"name"` + Purge BatchJobExpirePurge `yaml:"purge" json:"purge"` +} + +var _ yaml.Unmarshaler = &BatchJobExpireFilter{} + +// UnmarshalYAML - BatchJobExpireFilter extends unmarshal to extract line, col +// information +func (ef *BatchJobExpireFilter) UnmarshalYAML(value *yaml.Node) error { + type expFilter BatchJobExpireFilter + var tmp expFilter + err := value.Decode(&tmp) + if err != nil { + return err + } + *ef = BatchJobExpireFilter(tmp) + ef.line, ef.col = value.Line, value.Column + return err +} + +// Matches returns true if obj matches the filter conditions specified in ef. +func (ef BatchJobExpireFilter) Matches(obj ObjectInfo, now time.Time) bool { + switch ef.Type { + case BatchJobExpireObject: + if obj.DeleteMarker { + return false + } + case BatchJobExpireDeleted: + if !obj.DeleteMarker { + return false + } + default: + // we should never come here, Validate should have caught this. + batchLogOnceIf(context.Background(), fmt.Errorf("invalid filter type: %s", ef.Type), ef.Type) + return false + } + + if len(ef.Name) > 0 && !wildcard.Match(ef.Name, obj.Name) { + return false + } + if ef.OlderThan > 0 && now.Sub(obj.ModTime) <= ef.OlderThan.D() { + return false + } + + if ef.CreatedBefore != nil && !obj.ModTime.Before(*ef.CreatedBefore) { + return false + } + + if len(ef.Tags) > 0 && !obj.DeleteMarker { + // Only parse object tags if tags filter is specified. + var tagMap map[string]string + if len(obj.UserTags) != 0 { + t, err := tags.ParseObjectTags(obj.UserTags) + if err != nil { + return false + } + tagMap = t.ToMap() + } + + for _, kv := range ef.Tags { + // Object (version) must match all tags specified in + // the filter + var match bool + for t, v := range tagMap { + if kv.Match(BatchJobKV{Key: t, Value: v}) { + match = true + } + } + if !match { + return false + } + } + } + + if len(ef.Metadata) > 0 && !obj.DeleteMarker { + for _, kv := range ef.Metadata { + // Object (version) must match all x-amz-meta and + // standard metadata headers + // specified in the filter + var match bool + for k, v := range obj.UserDefined { + if !stringsHasPrefixFold(k, "x-amz-meta-") && !isStandardHeader(k) { + continue + } + // We only need to match x-amz-meta or standardHeaders + if kv.Match(BatchJobKV{Key: k, Value: v}) { + match = true + } + } + if !match { + return false + } + } + } + + return ef.Size.InRange(obj.Size) +} + +const ( + // BatchJobExpireObject - object type + BatchJobExpireObject string = "object" + // BatchJobExpireDeleted - delete marker type + BatchJobExpireDeleted string = "deleted" +) + +// Validate returns nil if ef has valid fields, validation error otherwise. +func (ef BatchJobExpireFilter) Validate() error { + switch ef.Type { + case BatchJobExpireObject: + case BatchJobExpireDeleted: + if len(ef.Tags) > 0 || len(ef.Metadata) > 0 { + return BatchJobYamlErr{ + line: ef.line, + col: ef.col, + msg: "delete type filter can't have tags or metadata", + } + } + default: + return BatchJobYamlErr{ + line: ef.line, + col: ef.col, + msg: "invalid batch-expire type", + } + } + + for _, tag := range ef.Tags { + if err := tag.Validate(); err != nil { + return err + } + } + + for _, meta := range ef.Metadata { + if err := meta.Validate(); err != nil { + return err + } + } + if err := ef.Purge.Validate(); err != nil { + return err + } + if err := ef.Size.Validate(); err != nil { + return err + } + if ef.CreatedBefore != nil && !ef.CreatedBefore.Before(time.Now()) { + return BatchJobYamlErr{ + line: ef.line, + col: ef.col, + msg: "CreatedBefore is in the future", + } + } + return nil +} + +// BatchJobExpire represents configuration parameters for a batch expiration +// job typically supplied in yaml form +type BatchJobExpire struct { + line, col int + APIVersion string `yaml:"apiVersion" json:"apiVersion"` + Bucket string `yaml:"bucket" json:"bucket"` + Prefix BatchJobPrefix `yaml:"prefix" json:"prefix"` + NotificationCfg BatchJobNotification `yaml:"notify" json:"notify"` + Retry BatchJobRetry `yaml:"retry" json:"retry"` + Rules []BatchJobExpireFilter `yaml:"rules" json:"rules"` +} + +var _ yaml.Unmarshaler = &BatchJobExpire{} + +// RedactSensitive will redact any sensitive information in b. +func (r *BatchJobExpire) RedactSensitive() { + if r == nil { + return + } + if r.NotificationCfg.Token != "" { + r.NotificationCfg.Token = redactedText + } +} + +// UnmarshalYAML - BatchJobExpire extends default unmarshal to extract line, col information. +func (r *BatchJobExpire) UnmarshalYAML(val *yaml.Node) error { + type expireJob BatchJobExpire + var tmp expireJob + err := val.Decode(&tmp) + if err != nil { + return err + } + + *r = BatchJobExpire(tmp) + r.line, r.col = val.Line, val.Column + return nil +} + +// Notify notifies notification endpoint if configured regarding job failure or success. +func (r BatchJobExpire) Notify(ctx context.Context, body io.Reader) error { + if r.NotificationCfg.Endpoint == "" { + return nil + } + + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, r.NotificationCfg.Endpoint, body) + if err != nil { + return err + } + + if r.NotificationCfg.Token != "" { + req.Header.Set("Authorization", r.NotificationCfg.Token) + } + + clnt := http.Client{Transport: getRemoteInstanceTransport()} + resp, err := clnt.Do(req) + if err != nil { + return err + } + + xhttp.DrainBody(resp.Body) + if resp.StatusCode != http.StatusOK { + return errors.New(resp.Status) + } + + return nil +} + +// Expire expires object versions which have already matched supplied filter conditions +func (r *BatchJobExpire) Expire(ctx context.Context, api ObjectLayer, vc *versioning.Versioning, objsToDel []ObjectToDelete) []error { + opts := ObjectOptions{ + PrefixEnabledFn: vc.PrefixEnabled, + VersionSuspended: vc.Suspended(), + } + + allErrs := make([]error, 0, len(objsToDel)) + + for { + count := len(objsToDel) + if count == 0 { + break + } + if count > maxDeleteList { + count = maxDeleteList + } + _, errs := api.DeleteObjects(ctx, r.Bucket, objsToDel[:count], opts) + allErrs = append(allErrs, errs...) + // Next batch of deletion + objsToDel = objsToDel[count:] + } + + return allErrs +} + +const ( + batchExpireName = "batch-expire.bin" + batchExpireFormat = 1 + batchExpireVersionV1 = 1 + batchExpireVersion = batchExpireVersionV1 + batchExpireAPIVersion = "v1" + batchExpireJobDefaultRetries = 3 + batchExpireJobDefaultRetryDelay = 250 * time.Millisecond +) + +type objInfoCache map[string]*ObjectInfo + +func newObjInfoCache() objInfoCache { + return objInfoCache(make(map[string]*ObjectInfo)) +} + +func (oiCache objInfoCache) Add(toDel ObjectToDelete, oi *ObjectInfo) { + oiCache[fmt.Sprintf("%s-%s", toDel.ObjectName, toDel.VersionID)] = oi +} + +func (oiCache objInfoCache) Get(toDel ObjectToDelete) (*ObjectInfo, bool) { + oi, ok := oiCache[fmt.Sprintf("%s-%s", toDel.ObjectName, toDel.VersionID)] + return oi, ok +} + +func batchObjsForDelete(ctx context.Context, r *BatchJobExpire, ri *batchJobInfo, job BatchJobRequest, api ObjectLayer, wk *workers.Workers, expireCh <-chan []expireObjInfo) { + vc, _ := globalBucketVersioningSys.Get(r.Bucket) + retryAttempts := job.Expire.Retry.Attempts + if retryAttempts <= 0 { + retryAttempts = batchExpireJobDefaultRetries + } + delay := job.Expire.Retry.Delay + if delay <= 0 { + delay = batchExpireJobDefaultRetryDelay + } + + var i int + for toExpire := range expireCh { + select { + case <-ctx.Done(): + return + default: + } + if i > 0 { + if wait := globalBatchConfig.ExpirationWait(); wait > 0 { + time.Sleep(wait) + } + } + i++ + wk.Take() + go func(toExpire []expireObjInfo) { + defer wk.Give() + + toExpireAll := make([]expireObjInfo, 0, len(toExpire)) + toDel := make([]ObjectToDelete, 0, len(toExpire)) + oiCache := newObjInfoCache() + for _, exp := range toExpire { + if exp.ExpireAll { + toExpireAll = append(toExpireAll, exp) + continue + } + // Cache ObjectInfo value via pointers for + // subsequent use to track objects which + // couldn't be deleted. + od := ObjectToDelete{ + ObjectV: ObjectV{ + ObjectName: exp.Name, + VersionID: exp.VersionID, + }, + } + toDel = append(toDel, od) + oiCache.Add(od, &exp.ObjectInfo) + } + + // DeleteObject(deletePrefix: true) to expire all versions of an object + for _, exp := range toExpireAll { + var success bool + for attempts := 1; attempts <= retryAttempts; attempts++ { + select { + case <-ctx.Done(): + ri.trackMultipleObjectVersions(exp, success) + return + default: + } + stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts) + _, err := api.DeleteObject(ctx, exp.Bucket, encodeDirObject(exp.Name), ObjectOptions{ + DeletePrefix: true, + DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls) + }) + if err != nil { + stopFn(exp, err) + batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s due to %v (attempts=%d)", exp.Bucket, exp.Name, err, attempts)) + } else { + stopFn(exp, err) + success = true + break + } + } + ri.trackMultipleObjectVersions(exp, success) + } + + // DeleteMultiple objects + toDelCopy := make([]ObjectToDelete, len(toDel)) + for attempts := 1; attempts <= retryAttempts; attempts++ { + select { + case <-ctx.Done(): + return + default: + } + + stopFn := globalBatchJobsMetrics.trace(batchJobMetricExpire, ri.JobID, attempts) + // Copying toDel to select from objects whose + // deletion failed + copy(toDelCopy, toDel) + var failed int + errs := r.Expire(ctx, api, vc, toDel) + // reslice toDel in preparation for next retry attempt + toDel = toDel[:0] + for i, err := range errs { + if err != nil { + stopFn(toDelCopy[i], err) + batchLogIf(ctx, fmt.Errorf("Failed to expire %s/%s versionID=%s due to %v (attempts=%d)", ri.Bucket, toDelCopy[i].ObjectName, toDelCopy[i].VersionID, + err, attempts)) + failed++ + if oi, ok := oiCache.Get(toDelCopy[i]); ok { + ri.trackCurrentBucketObject(r.Bucket, *oi, false, attempts) + } + if attempts != retryAttempts { + // retry + toDel = append(toDel, toDelCopy[i]) + } + } else { + stopFn(toDelCopy[i], nil) + if oi, ok := oiCache.Get(toDelCopy[i]); ok { + ri.trackCurrentBucketObject(r.Bucket, *oi, true, attempts) + } + } + } + + globalBatchJobsMetrics.save(ri.JobID, ri) + + if failed == 0 { + break + } + + // Add a delay between retry attempts + if attempts < retryAttempts { + time.Sleep(delay) + } + } + }(toExpire) + } +} + +type expireObjInfo struct { + ObjectInfo + ExpireAll bool + DeleteMarkerCount int64 +} + +// Start the batch expiration job, resumes if there was a pending job via "job.ID" +func (r *BatchJobExpire) Start(ctx context.Context, api ObjectLayer, job BatchJobRequest) error { + ri := &batchJobInfo{ + JobID: job.ID, + JobType: string(job.Type()), + StartTime: job.Started, + } + if err := ri.loadOrInit(ctx, api, job); err != nil { + return err + } + + globalBatchJobsMetrics.save(job.ID, ri) + lastObject := ri.Object + + now := time.Now().UTC() + + workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_EXPIRATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2))) + if err != nil { + return err + } + + wk, err := workers.New(workerSize) + if err != nil { + // invalid worker size. + return err + } + + ctx, cancelCause := context.WithCancelCause(ctx) + defer cancelCause(nil) + + results := make(chan itemOrErr[ObjectInfo], workerSize) + go func() { + prefixes := r.Prefix.F() + if len(prefixes) == 0 { + prefixes = []string{""} + } + for _, prefix := range prefixes { + prefixResultCh := make(chan itemOrErr[ObjectInfo], workerSize) + err := api.Walk(ctx, r.Bucket, prefix, prefixResultCh, WalkOptions{ + Marker: lastObject, + LatestOnly: false, // we need to visit all versions of the object to implement purge: retainVersions + VersionsSort: WalkVersionsSortDesc, + }) + if err != nil { + cancelCause(err) + xioutil.SafeClose(results) + return + } + for result := range prefixResultCh { + results <- result + } + } + xioutil.SafeClose(results) + }() + + // Goroutine to periodically save batch-expire job's in-memory state + saverQuitCh := make(chan struct{}) + go func() { + saveTicker := time.NewTicker(10 * time.Second) + defer saveTicker.Stop() + quit := false + after := time.Minute + for !quit { + select { + case <-saveTicker.C: + case <-ctx.Done(): + quit = true + case <-saverQuitCh: + quit = true + } + + if quit { + // save immediately if we are quitting + after = 0 + } + + ctx, cancel := context.WithTimeout(GlobalContext, 30*time.Second) // independent context + batchLogIf(ctx, ri.updateAfter(ctx, api, after, job)) + cancel() + } + }() + + expireCh := make(chan []expireObjInfo, workerSize) + expireDoneCh := make(chan struct{}) + go func() { + defer close(expireDoneCh) + batchObjsForDelete(ctx, r, ri, job, api, wk, expireCh) + }() + + var ( + prevObj ObjectInfo + matchedFilter BatchJobExpireFilter + versionsCount int + toDel []expireObjInfo + failed bool + done bool + ) + deleteMarkerCountMap := map[string]int64{} + pushToExpire := func() { + // set preObject deleteMarkerCount + if len(toDel) > 0 { + lastDelIndex := len(toDel) - 1 + lastDel := toDel[lastDelIndex] + if lastDel.ExpireAll { + toDel[lastDelIndex].DeleteMarkerCount = deleteMarkerCountMap[lastDel.Name] + // delete the key + delete(deleteMarkerCountMap, lastDel.Name) + } + } + // send down filtered entries to be deleted using + // DeleteObjects method + if len(toDel) > 10 { // batch up to 10 objects/versions to be expired simultaneously. + xfer := make([]expireObjInfo, len(toDel)) + copy(xfer, toDel) + select { + case expireCh <- xfer: + toDel = toDel[:0] // resetting toDel + case <-ctx.Done(): + done = true + } + } + } + for { + select { + case result, ok := <-results: + if !ok { + done = true + break + } + if result.Err != nil { + failed = true + batchLogIf(ctx, result.Err) + continue + } + if result.Item.DeleteMarker { + deleteMarkerCountMap[result.Item.Name]++ + } + // Apply filter to find the matching rule to apply expiry + // actions accordingly. + // nolint:gocritic + if result.Item.IsLatest { + var match BatchJobExpireFilter + var found bool + for _, rule := range r.Rules { + if rule.Matches(result.Item, now) { + match = rule + found = true + break + } + } + if !found { + continue + } + + if prevObj.Name != result.Item.Name { + // switch the object + pushToExpire() + } + + prevObj = result.Item + matchedFilter = match + versionsCount = 1 + // Include the latest version + if matchedFilter.Purge.RetainVersions == 0 { + toDel = append(toDel, expireObjInfo{ + ObjectInfo: result.Item, + ExpireAll: true, + }) + continue + } + } else if prevObj.Name == result.Item.Name { + if matchedFilter.Purge.RetainVersions == 0 { + continue // including latest version in toDel suffices, skipping other versions + } + versionsCount++ + } else { + // switch the object + pushToExpire() + // a file switched with no LatestVersion, logging it + batchLogIf(ctx, fmt.Errorf("skipping object %s, no latest version found", result.Item.Name)) + continue + } + + if versionsCount <= matchedFilter.Purge.RetainVersions { + continue // retain versions + } + toDel = append(toDel, expireObjInfo{ + ObjectInfo: result.Item, + }) + pushToExpire() + case <-ctx.Done(): + done = true + } + if done { + break + } + } + + if context.Cause(ctx) != nil { + xioutil.SafeClose(expireCh) + return context.Cause(ctx) + } + pushToExpire() + // Send any remaining objects downstream + if len(toDel) > 0 { + select { + case <-ctx.Done(): + case expireCh <- toDel: + } + } + xioutil.SafeClose(expireCh) + + <-expireDoneCh // waits for the expire goroutine to complete + wk.Wait() // waits for all expire workers to retire + + ri.Complete = !failed && ri.ObjectsFailed == 0 + ri.Failed = failed || ri.ObjectsFailed > 0 + globalBatchJobsMetrics.save(job.ID, ri) + + // Close the saverQuitCh - this also triggers saving in-memory state + // immediately one last time before we exit this method. + xioutil.SafeClose(saverQuitCh) + + // Notify expire jobs final status to the configured endpoint + buf, _ := json.Marshal(ri) + if err := r.Notify(context.Background(), bytes.NewReader(buf)); err != nil { + batchLogIf(context.Background(), fmt.Errorf("unable to notify %v", err)) + } + + return nil +} + +//msgp:ignore batchExpireJobError +type batchExpireJobError struct { + Code string + Description string + HTTPStatusCode int +} + +func (e batchExpireJobError) Error() string { + return e.Description +} + +// maxBatchRules maximum number of rules a batch-expiry job supports +const maxBatchRules = 50 + +// Validate validates the job definition input +func (r *BatchJobExpire) Validate(ctx context.Context, job BatchJobRequest, o ObjectLayer) error { + if r == nil { + return nil + } + + if r.APIVersion != batchExpireAPIVersion { + return batchExpireJobError{ + Code: "InvalidArgument", + Description: "Unsupported batch expire API version", + HTTPStatusCode: http.StatusBadRequest, + } + } + + if r.Bucket == "" { + return batchExpireJobError{ + Code: "InvalidArgument", + Description: "Bucket argument missing", + HTTPStatusCode: http.StatusBadRequest, + } + } + + if _, err := o.GetBucketInfo(ctx, r.Bucket, BucketOptions{}); err != nil { + if isErrBucketNotFound(err) { + return batchExpireJobError{ + Code: "NoSuchSourceBucket", + Description: "The specified source bucket does not exist", + HTTPStatusCode: http.StatusNotFound, + } + } + return err + } + + if len(r.Rules) > maxBatchRules { + return batchExpireJobError{ + Code: "InvalidArgument", + Description: "Too many rules. Batch expire job can't have more than 100 rules", + HTTPStatusCode: http.StatusBadRequest, + } + } + + for _, rule := range r.Rules { + if err := rule.Validate(); err != nil { + return batchExpireJobError{ + Code: "InvalidArgument", + Description: fmt.Sprintf("Invalid batch expire rule: %s", err), + HTTPStatusCode: http.StatusBadRequest, + } + } + } + + if err := r.Retry.Validate(); err != nil { + return batchExpireJobError{ + Code: "InvalidArgument", + Description: fmt.Sprintf("Invalid batch expire retry configuration: %s", err), + HTTPStatusCode: http.StatusBadRequest, + } + } + return nil +} diff --git a/cmd/batch-expire_gen.go b/cmd/batch-expire_gen.go new file mode 100644 index 00000000..bb2021fc --- /dev/null +++ b/cmd/batch-expire_gen.go @@ -0,0 +1,864 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "time" + + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *BatchJobExpire) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "APIVersion": + z.APIVersion, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "APIVersion") + return + } + case "Bucket": + z.Bucket, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "Prefix": + err = z.Prefix.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + case "NotificationCfg": + err = z.NotificationCfg.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "NotificationCfg") + return + } + case "Retry": + err = z.Retry.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Retry") + return + } + case "Rules": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Rules") + return + } + if cap(z.Rules) >= int(zb0002) { + z.Rules = (z.Rules)[:zb0002] + } else { + z.Rules = make([]BatchJobExpireFilter, zb0002) + } + for za0001 := range z.Rules { + err = z.Rules[za0001].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Rules", za0001) + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *BatchJobExpire) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 6 + // write "APIVersion" + err = en.Append(0x86, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e) + if err != nil { + return + } + err = en.WriteString(z.APIVersion) + if err != nil { + err = msgp.WrapError(err, "APIVersion") + return + } + // write "Bucket" + err = en.Append(0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Bucket) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + // write "Prefix" + err = en.Append(0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78) + if err != nil { + return + } + err = z.Prefix.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + // write "NotificationCfg" + err = en.Append(0xaf, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x66, 0x67) + if err != nil { + return + } + err = z.NotificationCfg.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "NotificationCfg") + return + } + // write "Retry" + err = en.Append(0xa5, 0x52, 0x65, 0x74, 0x72, 0x79) + if err != nil { + return + } + err = z.Retry.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Retry") + return + } + // write "Rules" + err = en.Append(0xa5, 0x52, 0x75, 0x6c, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Rules))) + if err != nil { + err = msgp.WrapError(err, "Rules") + return + } + for za0001 := range z.Rules { + err = z.Rules[za0001].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Rules", za0001) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *BatchJobExpire) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 6 + // string "APIVersion" + o = append(o, 0x86, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e) + o = msgp.AppendString(o, z.APIVersion) + // string "Bucket" + o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + o = msgp.AppendString(o, z.Bucket) + // string "Prefix" + o = append(o, 0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78) + o, err = z.Prefix.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + // string "NotificationCfg" + o = append(o, 0xaf, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x66, 0x67) + o, err = z.NotificationCfg.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "NotificationCfg") + return + } + // string "Retry" + o = append(o, 0xa5, 0x52, 0x65, 0x74, 0x72, 0x79) + o, err = z.Retry.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Retry") + return + } + // string "Rules" + o = append(o, 0xa5, 0x52, 0x75, 0x6c, 0x65, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Rules))) + for za0001 := range z.Rules { + o, err = z.Rules[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Rules", za0001) + return + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchJobExpire) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "APIVersion": + z.APIVersion, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "APIVersion") + return + } + case "Bucket": + z.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "Prefix": + bts, err = z.Prefix.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + case "NotificationCfg": + bts, err = z.NotificationCfg.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "NotificationCfg") + return + } + case "Retry": + bts, err = z.Retry.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Retry") + return + } + case "Rules": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Rules") + return + } + if cap(z.Rules) >= int(zb0002) { + z.Rules = (z.Rules)[:zb0002] + } else { + z.Rules = make([]BatchJobExpireFilter, zb0002) + } + for za0001 := range z.Rules { + bts, err = z.Rules[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Rules", za0001) + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *BatchJobExpire) Msgsize() (s int) { + s = 1 + 11 + msgp.StringPrefixSize + len(z.APIVersion) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + z.Prefix.Msgsize() + 16 + z.NotificationCfg.Msgsize() + 6 + z.Retry.Msgsize() + 6 + msgp.ArrayHeaderSize + for za0001 := range z.Rules { + s += z.Rules[za0001].Msgsize() + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *BatchJobExpireFilter) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "OlderThan": + err = z.OlderThan.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "OlderThan") + return + } + case "CreatedBefore": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "CreatedBefore") + return + } + z.CreatedBefore = nil + } else { + if z.CreatedBefore == nil { + z.CreatedBefore = new(time.Time) + } + *z.CreatedBefore, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "CreatedBefore") + return + } + } + case "Tags": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Tags") + return + } + if cap(z.Tags) >= int(zb0002) { + z.Tags = (z.Tags)[:zb0002] + } else { + z.Tags = make([]BatchJobKV, zb0002) + } + for za0001 := range z.Tags { + err = z.Tags[za0001].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Tags", za0001) + return + } + } + case "Metadata": + var zb0003 uint32 + zb0003, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Metadata") + return + } + if cap(z.Metadata) >= int(zb0003) { + z.Metadata = (z.Metadata)[:zb0003] + } else { + z.Metadata = make([]BatchJobKV, zb0003) + } + for za0002 := range z.Metadata { + err = z.Metadata[za0002].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Metadata", za0002) + return + } + } + case "Size": + err = z.Size.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + case "Type": + z.Type, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + case "Name": + z.Name, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "Purge": + var zb0004 uint32 + zb0004, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Purge") + return + } + for zb0004 > 0 { + zb0004-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "Purge") + return + } + switch msgp.UnsafeString(field) { + case "RetainVersions": + z.Purge.RetainVersions, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Purge", "RetainVersions") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "Purge") + return + } + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *BatchJobExpireFilter) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 8 + // write "OlderThan" + err = en.Append(0x88, 0xa9, 0x4f, 0x6c, 0x64, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e) + if err != nil { + return + } + err = z.OlderThan.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "OlderThan") + return + } + // write "CreatedBefore" + err = en.Append(0xad, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65) + if err != nil { + return + } + if z.CreatedBefore == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = en.WriteTime(*z.CreatedBefore) + if err != nil { + err = msgp.WrapError(err, "CreatedBefore") + return + } + } + // write "Tags" + err = en.Append(0xa4, 0x54, 0x61, 0x67, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Tags))) + if err != nil { + err = msgp.WrapError(err, "Tags") + return + } + for za0001 := range z.Tags { + err = z.Tags[za0001].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Tags", za0001) + return + } + } + // write "Metadata" + err = en.Append(0xa8, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Metadata))) + if err != nil { + err = msgp.WrapError(err, "Metadata") + return + } + for za0002 := range z.Metadata { + err = z.Metadata[za0002].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Metadata", za0002) + return + } + } + // write "Size" + err = en.Append(0xa4, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = z.Size.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + // write "Type" + err = en.Append(0xa4, 0x54, 0x79, 0x70, 0x65) + if err != nil { + return + } + err = en.WriteString(z.Type) + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + // write "Name" + err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteString(z.Name) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + // write "Purge" + err = en.Append(0xa5, 0x50, 0x75, 0x72, 0x67, 0x65) + if err != nil { + return + } + // map header, size 1 + // write "RetainVersions" + err = en.Append(0x81, 0xae, 0x52, 0x65, 0x74, 0x61, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73) + if err != nil { + return + } + err = en.WriteInt(z.Purge.RetainVersions) + if err != nil { + err = msgp.WrapError(err, "Purge", "RetainVersions") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *BatchJobExpireFilter) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 8 + // string "OlderThan" + o = append(o, 0x88, 0xa9, 0x4f, 0x6c, 0x64, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e) + o, err = z.OlderThan.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "OlderThan") + return + } + // string "CreatedBefore" + o = append(o, 0xad, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65) + if z.CreatedBefore == nil { + o = msgp.AppendNil(o) + } else { + o = msgp.AppendTime(o, *z.CreatedBefore) + } + // string "Tags" + o = append(o, 0xa4, 0x54, 0x61, 0x67, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Tags))) + for za0001 := range z.Tags { + o, err = z.Tags[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Tags", za0001) + return + } + } + // string "Metadata" + o = append(o, 0xa8, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61) + o = msgp.AppendArrayHeader(o, uint32(len(z.Metadata))) + for za0002 := range z.Metadata { + o, err = z.Metadata[za0002].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Metadata", za0002) + return + } + } + // string "Size" + o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65) + o, err = z.Size.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + // string "Type" + o = append(o, 0xa4, 0x54, 0x79, 0x70, 0x65) + o = msgp.AppendString(o, z.Type) + // string "Name" + o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65) + o = msgp.AppendString(o, z.Name) + // string "Purge" + o = append(o, 0xa5, 0x50, 0x75, 0x72, 0x67, 0x65) + // map header, size 1 + // string "RetainVersions" + o = append(o, 0x81, 0xae, 0x52, 0x65, 0x74, 0x61, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73) + o = msgp.AppendInt(o, z.Purge.RetainVersions) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchJobExpireFilter) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "OlderThan": + bts, err = z.OlderThan.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "OlderThan") + return + } + case "CreatedBefore": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.CreatedBefore = nil + } else { + if z.CreatedBefore == nil { + z.CreatedBefore = new(time.Time) + } + *z.CreatedBefore, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "CreatedBefore") + return + } + } + case "Tags": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tags") + return + } + if cap(z.Tags) >= int(zb0002) { + z.Tags = (z.Tags)[:zb0002] + } else { + z.Tags = make([]BatchJobKV, zb0002) + } + for za0001 := range z.Tags { + bts, err = z.Tags[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Tags", za0001) + return + } + } + case "Metadata": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Metadata") + return + } + if cap(z.Metadata) >= int(zb0003) { + z.Metadata = (z.Metadata)[:zb0003] + } else { + z.Metadata = make([]BatchJobKV, zb0003) + } + for za0002 := range z.Metadata { + bts, err = z.Metadata[za0002].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Metadata", za0002) + return + } + } + case "Size": + bts, err = z.Size.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + case "Type": + z.Type, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + case "Name": + z.Name, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "Purge": + var zb0004 uint32 + zb0004, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Purge") + return + } + for zb0004 > 0 { + zb0004-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "Purge") + return + } + switch msgp.UnsafeString(field) { + case "RetainVersions": + z.Purge.RetainVersions, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Purge", "RetainVersions") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "Purge") + return + } + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *BatchJobExpireFilter) Msgsize() (s int) { + s = 1 + 10 + z.OlderThan.Msgsize() + 14 + if z.CreatedBefore == nil { + s += msgp.NilSize + } else { + s += msgp.TimeSize + } + s += 5 + msgp.ArrayHeaderSize + for za0001 := range z.Tags { + s += z.Tags[za0001].Msgsize() + } + s += 9 + msgp.ArrayHeaderSize + for za0002 := range z.Metadata { + s += z.Metadata[za0002].Msgsize() + } + s += 5 + z.Size.Msgsize() + 5 + msgp.StringPrefixSize + len(z.Type) + 5 + msgp.StringPrefixSize + len(z.Name) + 6 + 1 + 15 + msgp.IntSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *BatchJobExpirePurge) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "RetainVersions": + z.RetainVersions, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "RetainVersions") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z BatchJobExpirePurge) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "RetainVersions" + err = en.Append(0x81, 0xae, 0x52, 0x65, 0x74, 0x61, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73) + if err != nil { + return + } + err = en.WriteInt(z.RetainVersions) + if err != nil { + err = msgp.WrapError(err, "RetainVersions") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z BatchJobExpirePurge) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "RetainVersions" + o = append(o, 0x81, 0xae, 0x52, 0x65, 0x74, 0x61, 0x69, 0x6e, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73) + o = msgp.AppendInt(o, z.RetainVersions) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchJobExpirePurge) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "RetainVersions": + z.RetainVersions, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "RetainVersions") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z BatchJobExpirePurge) Msgsize() (s int) { + s = 1 + 15 + msgp.IntSize + return +} diff --git a/cmd/batch-expire_gen_test.go b/cmd/batch-expire_gen_test.go new file mode 100644 index 00000000..d30cc337 --- /dev/null +++ b/cmd/batch-expire_gen_test.go @@ -0,0 +1,349 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalBatchJobExpire(t *testing.T) { + v := BatchJobExpire{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBatchJobExpire(b *testing.B) { + v := BatchJobExpire{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBatchJobExpire(b *testing.B) { + v := BatchJobExpire{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBatchJobExpire(b *testing.B) { + v := BatchJobExpire{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBatchJobExpire(t *testing.T) { + v := BatchJobExpire{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBatchJobExpire Msgsize() is inaccurate") + } + + vn := BatchJobExpire{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBatchJobExpire(b *testing.B) { + v := BatchJobExpire{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBatchJobExpire(b *testing.B) { + v := BatchJobExpire{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBatchJobExpireFilter(t *testing.T) { + v := BatchJobExpireFilter{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBatchJobExpireFilter(b *testing.B) { + v := BatchJobExpireFilter{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBatchJobExpireFilter(b *testing.B) { + v := BatchJobExpireFilter{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBatchJobExpireFilter(b *testing.B) { + v := BatchJobExpireFilter{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBatchJobExpireFilter(t *testing.T) { + v := BatchJobExpireFilter{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBatchJobExpireFilter Msgsize() is inaccurate") + } + + vn := BatchJobExpireFilter{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBatchJobExpireFilter(b *testing.B) { + v := BatchJobExpireFilter{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBatchJobExpireFilter(b *testing.B) { + v := BatchJobExpireFilter{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBatchJobExpirePurge(t *testing.T) { + v := BatchJobExpirePurge{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBatchJobExpirePurge(b *testing.B) { + v := BatchJobExpirePurge{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBatchJobExpirePurge(b *testing.B) { + v := BatchJobExpirePurge{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBatchJobExpirePurge(b *testing.B) { + v := BatchJobExpirePurge{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBatchJobExpirePurge(t *testing.T) { + v := BatchJobExpirePurge{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBatchJobExpirePurge Msgsize() is inaccurate") + } + + vn := BatchJobExpirePurge{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBatchJobExpirePurge(b *testing.B) { + v := BatchJobExpirePurge{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBatchJobExpirePurge(b *testing.B) { + v := BatchJobExpirePurge{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/batch-expire_test.go b/cmd/batch-expire_test.go new file mode 100644 index 00000000..18f7150d --- /dev/null +++ b/cmd/batch-expire_test.go @@ -0,0 +1,125 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "slices" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestParseBatchJobExpire(t *testing.T) { + expireYaml := ` +expire: # Expire objects that match a condition + apiVersion: v1 + bucket: mybucket # Bucket where this batch job will expire matching objects from + prefix: myprefix # (Optional) Prefix under which this job will expire objects matching the rules below. + rules: + - type: object # regular objects with zero or more older versions + name: NAME # match object names that satisfy the wildcard expression. + olderThan: 7d10h # match objects older than this value + createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date" + tags: + - key: name + value: pick* # match objects with tag 'name', all values starting with 'pick' + metadata: + - key: content-type + value: image/* # match objects with 'content-type', all values starting with 'image/' + size: + lessThan: "10MiB" # match objects with size less than this value (e.g. 10MiB) + greaterThan: 1MiB # match objects with size greater than this value (e.g. 1MiB) + purge: + # retainVersions: 0 # (default) delete all versions of the object. This option is the fastest. + # retainVersions: 5 # keep the latest 5 versions of the object. + + - type: deleted # objects with delete marker as their latest version + name: NAME # match object names that satisfy the wildcard expression. + olderThan: 10h # match objects older than this value (e.g. 7d10h31s) + createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date" + purge: + # retainVersions: 0 # (default) delete all versions of the object. This option is the fastest. + # retainVersions: 5 # keep the latest 5 versions of the object including delete markers. + + notify: + endpoint: https://notify.endpoint # notification endpoint to receive job completion status + token: Bearer xxxxx # optional authentication token for the notification endpoint + + retry: + attempts: 10 # number of retries for the job before giving up + delay: 500ms # least amount of delay between each retry +` + var job BatchJobRequest + err := yaml.Unmarshal([]byte(expireYaml), &job) + if err != nil { + t.Fatal("Failed to parse batch-job-expire yaml", err) + } + if !slices.Equal(job.Expire.Prefix.F(), []string{"myprefix"}) { + t.Fatal("Failed to parse batch-job-expire yaml") + } + + multiPrefixExpireYaml := ` +expire: # Expire objects that match a condition + apiVersion: v1 + bucket: mybucket # Bucket where this batch job will expire matching objects from + prefix: # (Optional) Prefix under which this job will expire objects matching the rules below. + - myprefix + - myprefix1 + rules: + - type: object # regular objects with zero or more older versions + name: NAME # match object names that satisfy the wildcard expression. + olderThan: 7d10h # match objects older than this value + createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date" + tags: + - key: name + value: pick* # match objects with tag 'name', all values starting with 'pick' + metadata: + - key: content-type + value: image/* # match objects with 'content-type', all values starting with 'image/' + size: + lessThan: "10MiB" # match objects with size less than this value (e.g. 10MiB) + greaterThan: 1MiB # match objects with size greater than this value (e.g. 1MiB) + purge: + # retainVersions: 0 # (default) delete all versions of the object. This option is the fastest. + # retainVersions: 5 # keep the latest 5 versions of the object. + + - type: deleted # objects with delete marker as their latest version + name: NAME # match object names that satisfy the wildcard expression. + olderThan: 10h # match objects older than this value (e.g. 7d10h31s) + createdBefore: "2006-01-02T15:04:05.00Z" # match objects created before "date" + purge: + # retainVersions: 0 # (default) delete all versions of the object. This option is the fastest. + # retainVersions: 5 # keep the latest 5 versions of the object including delete markers. + + notify: + endpoint: https://notify.endpoint # notification endpoint to receive job completion status + token: Bearer xxxxx # optional authentication token for the notification endpoint + + retry: + attempts: 10 # number of retries for the job before giving up + delay: 500ms # least amount of delay between each retry +` + var multiPrefixJob BatchJobRequest + err = yaml.Unmarshal([]byte(multiPrefixExpireYaml), &multiPrefixJob) + if err != nil { + t.Fatal("Failed to parse batch-job-expire yaml", err) + } + if !slices.Equal(multiPrefixJob.Expire.Prefix.F(), []string{"myprefix", "myprefix1"}) { + t.Fatal("Failed to parse batch-job-expire yaml") + } +} diff --git a/cmd/batch-handlers.go b/cmd/batch-handlers.go new file mode 100644 index 00000000..484882e9 --- /dev/null +++ b/cmd/batch-handlers.go @@ -0,0 +1,2356 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io" + "maps" + "math/rand" + "net/http" + "net/url" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "time" + + "github.com/dustin/go-humanize" + "github.com/lithammer/shortuuid/v4" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio-go/v7/pkg/encrypt" + "github.com/minio/minio-go/v7/pkg/tags" + "github.com/minio/minio/internal/config/batch" + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/pkg/v3/console" + "github.com/minio/pkg/v3/env" + "github.com/minio/pkg/v3/policy" + "github.com/minio/pkg/v3/workers" + "gopkg.in/yaml.v3" +) + +var globalBatchConfig batch.Config + +const ( + // Keep the completed/failed job stats 3 days before removing it + oldJobsExpiration = 3 * 24 * time.Hour + + redactedText = "**REDACTED**" +) + +// BatchJobRequest this is an internal data structure not for external consumption. +type BatchJobRequest struct { + ID string `yaml:"-" json:"name"` + User string `yaml:"-" json:"user"` + Started time.Time `yaml:"-" json:"started"` + Replicate *BatchJobReplicateV1 `yaml:"replicate" json:"replicate"` + KeyRotate *BatchJobKeyRotateV1 `yaml:"keyrotate" json:"keyrotate"` + Expire *BatchJobExpire `yaml:"expire" json:"expire"` + ctx context.Context `msg:"-"` +} + +// RedactSensitive will redact any sensitive information in b. +func (j *BatchJobRequest) RedactSensitive() { + j.Replicate.RedactSensitive() + j.Expire.RedactSensitive() + j.KeyRotate.RedactSensitive() +} + +// RedactSensitive will redact any sensitive information in b. +func (r *BatchJobReplicateV1) RedactSensitive() { + if r == nil { + return + } + if r.Target.Creds.SecretKey != "" { + r.Target.Creds.SecretKey = redactedText + } + if r.Target.Creds.SessionToken != "" { + r.Target.Creds.SessionToken = redactedText + } +} + +// RedactSensitive will redact any sensitive information in b. +func (r *BatchJobKeyRotateV1) RedactSensitive() {} + +func notifyEndpoint(ctx context.Context, ri *batchJobInfo, endpoint, token string) error { + if endpoint == "" { + return nil + } + + buf, err := json.Marshal(ri) + if err != nil { + return err + } + + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpoint, bytes.NewReader(buf)) + if err != nil { + return err + } + + if token != "" { + req.Header.Set("Authorization", token) + } + req.Header.Set("Content-Type", "application/json") + + clnt := http.Client{Transport: getRemoteInstanceTransport()} + resp, err := clnt.Do(req) + if err != nil { + return err + } + + xhttp.DrainBody(resp.Body) + if resp.StatusCode != http.StatusOK { + return errors.New(resp.Status) + } + + return nil +} + +// Notify notifies notification endpoint if configured regarding job failure or success. +func (r BatchJobReplicateV1) Notify(ctx context.Context, ri *batchJobInfo) error { + return notifyEndpoint(ctx, ri, r.Flags.Notify.Endpoint, r.Flags.Notify.Token) +} + +// ReplicateFromSource - this is not implemented yet where source is 'remote' and target is local. +func (r *BatchJobReplicateV1) ReplicateFromSource(ctx context.Context, api ObjectLayer, core *minio.Core, srcObjInfo ObjectInfo, retry bool) error { + srcBucket := r.Source.Bucket + tgtBucket := r.Target.Bucket + srcObject := srcObjInfo.Name + tgtObject := srcObjInfo.Name + if r.Target.Prefix != "" { + tgtObject = pathJoin(r.Target.Prefix, srcObjInfo.Name) + } + + versionID := srcObjInfo.VersionID + if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 { + versionID = "" + } + if srcObjInfo.DeleteMarker { + _, err := api.DeleteObject(ctx, tgtBucket, tgtObject, ObjectOptions{ + VersionID: versionID, + // Since we are preserving a delete marker, we have to make sure this is always true. + // regardless of the current configuration of the bucket we must preserve all versions + // on the pool being batch replicated from source. + Versioned: true, + MTime: srcObjInfo.ModTime, + DeleteMarker: srcObjInfo.DeleteMarker, + ReplicationRequest: true, + }) + return err + } + + opts := ObjectOptions{ + VersionID: srcObjInfo.VersionID, + MTime: srcObjInfo.ModTime, + PreserveETag: srcObjInfo.ETag, + UserDefined: srcObjInfo.UserDefined, + } + if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 { + opts.VersionID = "" + } + if crypto.S3.IsEncrypted(srcObjInfo.UserDefined) { + opts.ServerSideEncryption = encrypt.NewSSE() + } + slc := strings.Split(srcObjInfo.ETag, "-") + if len(slc) == 2 { + partsCount, err := strconv.Atoi(slc[1]) + if err != nil { + return err + } + return r.copyWithMultipartfromSource(ctx, api, core, srcObjInfo, opts, partsCount) + } + gopts := minio.GetObjectOptions{ + VersionID: srcObjInfo.VersionID, + } + if err := gopts.SetMatchETag(srcObjInfo.ETag); err != nil { + return err + } + rd, objInfo, _, err := core.GetObject(ctx, srcBucket, srcObject, gopts) + if err != nil { + return ErrorRespToObjectError(err, srcBucket, srcObject, srcObjInfo.VersionID) + } + defer rd.Close() + + hr, err := hash.NewReader(ctx, rd, objInfo.Size, "", "", objInfo.Size) + if err != nil { + return err + } + pReader := NewPutObjReader(hr) + _, err = api.PutObject(ctx, tgtBucket, tgtObject, pReader, opts) + return err +} + +func (r *BatchJobReplicateV1) copyWithMultipartfromSource(ctx context.Context, api ObjectLayer, c *minio.Core, srcObjInfo ObjectInfo, opts ObjectOptions, partsCount int) (err error) { + srcBucket := r.Source.Bucket + tgtBucket := r.Target.Bucket + srcObject := srcObjInfo.Name + tgtObject := srcObjInfo.Name + if r.Target.Prefix != "" { + tgtObject = pathJoin(r.Target.Prefix, srcObjInfo.Name) + } + if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 { + opts.VersionID = "" + } + var uploadedParts []CompletePart + res, err := api.NewMultipartUpload(context.Background(), tgtBucket, tgtObject, opts) + if err != nil { + return err + } + + defer func() { + if err != nil { + // block and abort remote upload upon failure. + attempts := 1 + for attempts <= 3 { + aerr := api.AbortMultipartUpload(ctx, tgtBucket, tgtObject, res.UploadID, ObjectOptions{}) + if aerr == nil { + return + } + batchLogIf(ctx, + fmt.Errorf("trying %s: Unable to cleanup failed multipart replication %s on remote %s/%s: %w - this may consume space on remote cluster", + humanize.Ordinal(attempts), res.UploadID, tgtBucket, tgtObject, aerr)) + attempts++ + time.Sleep(time.Second) + } + } + }() + + var ( + hr *hash.Reader + pInfo PartInfo + ) + + for i := range partsCount { + gopts := minio.GetObjectOptions{ + VersionID: srcObjInfo.VersionID, + PartNumber: i + 1, + } + if err := gopts.SetMatchETag(srcObjInfo.ETag); err != nil { + return err + } + rd, objInfo, _, err := c.GetObject(ctx, srcBucket, srcObject, gopts) + if err != nil { + return ErrorRespToObjectError(err, srcBucket, srcObject, srcObjInfo.VersionID) + } + defer rd.Close() + + hr, err = hash.NewReader(ctx, io.LimitReader(rd, objInfo.Size), objInfo.Size, "", "", objInfo.Size) + if err != nil { + return err + } + pReader := NewPutObjReader(hr) + opts.PreserveETag = "" + pInfo, err = api.PutObjectPart(ctx, tgtBucket, tgtObject, res.UploadID, i+1, pReader, opts) + if err != nil { + return err + } + if pInfo.Size != objInfo.Size { + return fmt.Errorf("Part size mismatch: got %d, want %d", pInfo.Size, objInfo.Size) + } + uploadedParts = append(uploadedParts, CompletePart{ + PartNumber: pInfo.PartNumber, + ETag: pInfo.ETag, + }) + } + _, err = api.CompleteMultipartUpload(ctx, tgtBucket, tgtObject, res.UploadID, uploadedParts, opts) + return err +} + +// StartFromSource starts the batch replication job from remote source, resumes if there was a pending job via "job.ID" +func (r *BatchJobReplicateV1) StartFromSource(ctx context.Context, api ObjectLayer, job BatchJobRequest) error { + ri := &batchJobInfo{ + JobID: job.ID, + JobType: string(job.Type()), + StartTime: job.Started, + } + if err := ri.loadOrInit(ctx, api, job); err != nil { + return err + } + if ri.Complete { + return nil + } + globalBatchJobsMetrics.save(job.ID, ri) + + retryAttempts := job.Replicate.Flags.Retry.Attempts + if retryAttempts <= 0 { + retryAttempts = batchReplJobDefaultRetries + } + delay := job.Replicate.Flags.Retry.Delay + if delay <= 0 { + delay = batchReplJobDefaultRetryDelay + } + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) + + hasTags := len(r.Flags.Filter.Tags) != 0 + isMetadata := len(r.Flags.Filter.Metadata) != 0 + isStorageClassOnly := len(r.Flags.Filter.Metadata) == 1 && strings.EqualFold(r.Flags.Filter.Metadata[0].Key, xhttp.AmzStorageClass) + + skip := func(oi ObjectInfo) (ok bool) { + if r.Flags.Filter.OlderThan > 0 && time.Since(oi.ModTime) < r.Flags.Filter.OlderThan.D() { + // skip all objects that are newer than specified older duration + return true + } + + if r.Flags.Filter.NewerThan > 0 && time.Since(oi.ModTime) >= r.Flags.Filter.NewerThan.D() { + // skip all objects that are older than specified newer duration + return true + } + + if !r.Flags.Filter.CreatedAfter.IsZero() && r.Flags.Filter.CreatedAfter.After(oi.ModTime) { + // skip all objects that are created before the specified time. + return true + } + + if !r.Flags.Filter.CreatedBefore.IsZero() && r.Flags.Filter.CreatedBefore.Before(oi.ModTime) { + // skip all objects that are created after the specified time. + return true + } + + if hasTags { + // Only parse object tags if tags filter is specified. + tagMap := map[string]string{} + tagStr := oi.UserTags + if len(tagStr) != 0 { + t, err := tags.ParseObjectTags(tagStr) + if err != nil { + return false + } + tagMap = t.ToMap() + } + for _, kv := range r.Flags.Filter.Tags { + for t, v := range tagMap { + if kv.Match(BatchJobKV{Key: t, Value: v}) { + return true + } + } + } + + // None of the provided tags filter match skip the object + return false + } + + for _, kv := range r.Flags.Filter.Metadata { + for k, v := range oi.UserDefined { + if !stringsHasPrefixFold(k, "x-amz-meta-") && !isStandardHeader(k) { + continue + } + // We only need to match x-amz-meta or standardHeaders + if kv.Match(BatchJobKV{Key: k, Value: v}) { + return true + } + } + } + + // None of the provided filters match + return false + } + + u, err := url.Parse(r.Source.Endpoint) + if err != nil { + return err + } + + cred := r.Source.Creds + + c, err := minio.New(u.Host, &minio.Options{ + Creds: credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken), + Secure: u.Scheme == "https", + Transport: getRemoteInstanceTransport(), + BucketLookup: lookupStyle(r.Source.Path), + }) + if err != nil { + return err + } + + c.SetAppInfo("minio-"+batchJobPrefix, r.APIVersion+" "+job.ID) + core := &minio.Core{Client: c} + + workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_REPLICATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2))) + if err != nil { + return err + } + + wk, err := workers.New(workerSize) + if err != nil { + // invalid worker size. + return err + } + + retry := false + for attempts := 1; attempts <= retryAttempts; attempts++ { + attempts := attempts + // one of source/target is s3, skip delete marker and all versions under the same object name. + s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 + minioSrc := r.Source.Type == BatchJobReplicateResourceMinIO + ctx, cancel := context.WithCancel(ctx) + + objInfoCh := make(chan minio.ObjectInfo, 1) + go func() { + prefixes := r.Source.Prefix.F() + if len(prefixes) == 0 { + prefixes = []string{""} + } + for _, prefix := range prefixes { + prefixObjInfoCh := c.ListObjects(ctx, r.Source.Bucket, minio.ListObjectsOptions{ + Prefix: prefix, + WithVersions: minioSrc, + Recursive: true, + WithMetadata: true, + }) + for obj := range prefixObjInfoCh { + objInfoCh <- obj + } + } + xioutil.SafeClose(objInfoCh) + }() + + prevObj := "" + skipReplicate := false + + for obj := range objInfoCh { + oi := toObjectInfo(r.Source.Bucket, obj.Key, obj) + if !minioSrc { + // Check if metadata filter was requested and it is expected to have + // all user metadata or just storageClass. If its only storageClass + // List() already returns relevant information for filter to be applied. + if isMetadata && !isStorageClassOnly { + oi2, err := c.StatObject(ctx, r.Source.Bucket, obj.Key, minio.StatObjectOptions{}) + if err == nil { + oi = toObjectInfo(r.Source.Bucket, obj.Key, oi2) + } else { + if !isErrMethodNotAllowed(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) && + !isErrObjectNotFound(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) { + batchLogIf(ctx, err) + } + continue + } + } + if hasTags { + tags, err := c.GetObjectTagging(ctx, r.Source.Bucket, obj.Key, minio.GetObjectTaggingOptions{}) + if err == nil { + oi.UserTags = tags.String() + } else { + if !isErrMethodNotAllowed(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) && + !isErrObjectNotFound(ErrorRespToObjectError(err, r.Source.Bucket, obj.Key)) { + batchLogIf(ctx, err) + } + continue + } + } + } + if skip(oi) { + continue + } + if obj.Key != prevObj { + prevObj = obj.Key + // skip replication of delete marker and all versions under the same object name if one of source or target is s3. + skipReplicate = obj.IsDeleteMarker && s3Type + } + if skipReplicate { + continue + } + + wk.Take() + go func() { + defer wk.Give() + stopFn := globalBatchJobsMetrics.trace(batchJobMetricReplication, job.ID, attempts) + success := true + if err := r.ReplicateFromSource(ctx, api, core, oi, retry); err != nil { + // object must be deleted concurrently, allow these failures but do not count them + if isErrVersionNotFound(err) || isErrObjectNotFound(err) { + return + } + stopFn(oi, err) + batchLogIf(ctx, err) + success = false + } else { + stopFn(oi, nil) + } + ri.trackCurrentBucketObject(r.Target.Bucket, oi, success, attempts) + globalBatchJobsMetrics.save(job.ID, ri) + // persist in-memory state to disk after every 10secs. + batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job)) + + if wait := globalBatchConfig.ReplicationWait(); wait > 0 { + time.Sleep(wait) + } + }() + } + wk.Wait() + + ri.RetryAttempts = attempts + ri.Complete = ri.ObjectsFailed == 0 + ri.Failed = ri.ObjectsFailed > 0 + + globalBatchJobsMetrics.save(job.ID, ri) + // persist in-memory state to disk. + batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job)) + + if err := r.Notify(ctx, ri); err != nil { + batchLogIf(ctx, fmt.Errorf("unable to notify %v", err)) + } + + cancel() + if ri.Failed { + ri.ObjectsFailed = 0 + ri.Bucket = "" + ri.Object = "" + ri.Objects = 0 + ri.BytesFailed = 0 + ri.BytesTransferred = 0 + retry = true // indicate we are retrying.. + time.Sleep(delay + time.Duration(rnd.Float64()*float64(delay))) + continue + } + + break + } + + return nil +} + +// toObjectInfo converts minio.ObjectInfo to ObjectInfo +func toObjectInfo(bucket, object string, objInfo minio.ObjectInfo) ObjectInfo { + tags, _ := tags.MapToObjectTags(objInfo.UserTags) + oi := ObjectInfo{ + Bucket: bucket, + Name: object, + ModTime: objInfo.LastModified, + Size: objInfo.Size, + ETag: objInfo.ETag, + VersionID: objInfo.VersionID, + IsLatest: objInfo.IsLatest, + DeleteMarker: objInfo.IsDeleteMarker, + ContentType: objInfo.ContentType, + Expires: objInfo.Expires, + StorageClass: objInfo.StorageClass, + ReplicationStatusInternal: objInfo.ReplicationStatus, + UserTags: tags.String(), + } + + oi.UserDefined = make(map[string]string, len(objInfo.Metadata)) + for k, v := range objInfo.Metadata { + oi.UserDefined[k] = v[0] + } + + ce, ok := oi.UserDefined[xhttp.ContentEncoding] + if !ok { + ce, ok = oi.UserDefined[strings.ToLower(xhttp.ContentEncoding)] + } + if ok { + oi.ContentEncoding = ce + } + + _, ok = oi.UserDefined[xhttp.AmzStorageClass] + if !ok { + oi.UserDefined[xhttp.AmzStorageClass] = objInfo.StorageClass + } + + maps.Copy(oi.UserDefined, objInfo.UserMetadata) + + return oi +} + +func (r BatchJobReplicateV1) writeAsArchive(ctx context.Context, objAPI ObjectLayer, remoteClnt *minio.Client, entries []ObjectInfo, prefix string) error { + input := make(chan minio.SnowballObject, 1) + opts := minio.SnowballOptions{ + Opts: minio.PutObjectOptions{}, + InMemory: *r.Source.Snowball.InMemory, + Compress: *r.Source.Snowball.Compress, + SkipErrs: *r.Source.Snowball.SkipErrs, + } + + go func() { + defer xioutil.SafeClose(input) + + for _, entry := range entries { + gr, err := objAPI.GetObjectNInfo(ctx, r.Source.Bucket, + entry.Name, nil, nil, ObjectOptions{ + VersionID: entry.VersionID, + }) + if err != nil { + batchLogIf(ctx, err) + continue + } + + if prefix != "" { + entry.Name = pathJoin(prefix, entry.Name) + } + + snowballObj := minio.SnowballObject{ + // Create path to store objects within the bucket. + Key: entry.Name, + Size: entry.Size, + ModTime: entry.ModTime, + VersionID: entry.VersionID, + Content: gr, + Headers: make(http.Header), + Close: func() { + gr.Close() + }, + } + + opts, _, err := batchReplicationOpts(ctx, "", gr.ObjInfo) + if err != nil { + batchLogIf(ctx, err) + continue + } + // TODO: I am not sure we read it back, but we aren't sending whether checksums are single/multipart. + for k, vals := range opts.Header() { + for _, v := range vals { + snowballObj.Headers.Add(k, v) + } + } + + input <- snowballObj + } + }() + + // Collect and upload all entries. + return remoteClnt.PutObjectsSnowball(ctx, r.Target.Bucket, opts, input) +} + +// ReplicateToTarget read from source and replicate to configured target +func (r *BatchJobReplicateV1) ReplicateToTarget(ctx context.Context, api ObjectLayer, c *minio.Core, srcObjInfo ObjectInfo, retry bool) error { + srcBucket := r.Source.Bucket + tgtBucket := r.Target.Bucket + tgtPrefix := r.Target.Prefix + srcObject := srcObjInfo.Name + s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 + + if srcObjInfo.DeleteMarker || !srcObjInfo.VersionPurgeStatus.Empty() { + if retry && !s3Type { + if _, err := c.StatObject(ctx, tgtBucket, pathJoin(tgtPrefix, srcObject), minio.StatObjectOptions{ + VersionID: srcObjInfo.VersionID, + Internal: minio.AdvancedGetOptions{ + ReplicationProxyRequest: "false", + }, + }); isErrMethodNotAllowed(ErrorRespToObjectError(err, tgtBucket, pathJoin(tgtPrefix, srcObject))) { + return nil + } + } + + versionID := srcObjInfo.VersionID + dmVersionID := "" + if srcObjInfo.VersionPurgeStatus.Empty() { + dmVersionID = srcObjInfo.VersionID + } + if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 { + dmVersionID = "" + versionID = "" + } + return c.RemoveObject(ctx, tgtBucket, pathJoin(tgtPrefix, srcObject), minio.RemoveObjectOptions{ + VersionID: versionID, + Internal: minio.AdvancedRemoveOptions{ + ReplicationDeleteMarker: dmVersionID != "", + ReplicationMTime: srcObjInfo.ModTime, + ReplicationStatus: minio.ReplicationStatusReplica, + ReplicationRequest: true, // always set this to distinguish between `mc mirror` replication and serverside + }, + }) + } + + if retry && !s3Type { // when we are retrying avoid copying if necessary. + gopts := minio.GetObjectOptions{} + if err := gopts.SetMatchETag(srcObjInfo.ETag); err != nil { + return err + } + if _, err := c.StatObject(ctx, tgtBucket, pathJoin(tgtPrefix, srcObject), gopts); err == nil { + return nil + } + } + + versioned := globalBucketVersioningSys.PrefixEnabled(srcBucket, srcObject) + versionSuspended := globalBucketVersioningSys.PrefixSuspended(srcBucket, srcObject) + + opts := ObjectOptions{ + VersionID: srcObjInfo.VersionID, + Versioned: versioned, + VersionSuspended: versionSuspended, + } + rd, err := api.GetObjectNInfo(ctx, srcBucket, srcObject, nil, http.Header{}, opts) + if err != nil { + return err + } + defer rd.Close() + objInfo := rd.ObjInfo + + size, err := objInfo.GetActualSize() + if err != nil { + return err + } + + putOpts, isMP, err := batchReplicationOpts(ctx, "", objInfo) + if err != nil { + return err + } + if r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 { + putOpts.Internal = minio.AdvancedPutOptions{} + } + if isMP { + if err := replicateObjectWithMultipart(ctx, c, tgtBucket, pathJoin(tgtPrefix, objInfo.Name), rd, objInfo, putOpts); err != nil { + return err + } + } else { + if _, err = c.PutObject(ctx, tgtBucket, pathJoin(tgtPrefix, objInfo.Name), rd, size, "", "", putOpts); err != nil { + return err + } + } + return nil +} + +//go:generate msgp -file $GOFILE -unexported + +// batchJobInfo current batch replication information +type batchJobInfo struct { + mu sync.RWMutex `json:"-" msg:"-"` + + Version int `json:"-" msg:"v"` + JobID string `json:"jobID" msg:"jid"` + JobType string `json:"jobType" msg:"jt"` + StartTime time.Time `json:"startTime" msg:"st"` + LastUpdate time.Time `json:"lastUpdate" msg:"lu"` + RetryAttempts int `json:"retryAttempts" msg:"ra"` + Attempts int `json:"attempts" msg:"at"` + + Complete bool `json:"complete" msg:"cmp"` + Failed bool `json:"failed" msg:"fld"` + + // Last bucket/object batch replicated + Bucket string `json:"-" msg:"lbkt"` + Object string `json:"-" msg:"lobj"` + + // Verbose information + Objects int64 `json:"objects" msg:"ob"` + DeleteMarkers int64 `json:"deleteMarkers" msg:"dm"` + ObjectsFailed int64 `json:"objectsFailed" msg:"obf"` + DeleteMarkersFailed int64 `json:"deleteMarkersFailed" msg:"dmf"` + BytesTransferred int64 `json:"bytesTransferred" msg:"bt"` + BytesFailed int64 `json:"bytesFailed" msg:"bf"` +} + +const ( + batchReplName = "batch-replicate.bin" + batchReplFormat = 1 + batchReplVersionV1 = 1 + batchReplVersion = batchReplVersionV1 + batchJobName = "job.bin" + batchJobPrefix = "batch-jobs" + batchJobReportsPrefix = batchJobPrefix + "/reports" + + batchReplJobAPIVersion = "v1" + batchReplJobDefaultRetries = 3 + batchReplJobDefaultRetryDelay = time.Second +) + +func getJobPath(job BatchJobRequest) string { + return pathJoin(batchJobPrefix, job.ID) +} + +func (ri *batchJobInfo) getJobReportPath() (string, error) { + var fileName string + switch madmin.BatchJobType(ri.JobType) { + case madmin.BatchJobReplicate: + fileName = batchReplName + case madmin.BatchJobKeyRotate: + fileName = batchKeyRotationName + case madmin.BatchJobExpire: + fileName = batchExpireName + default: + return "", fmt.Errorf("unknown job type: %v", ri.JobType) + } + return pathJoin(batchJobReportsPrefix, ri.JobID, fileName), nil +} + +func (ri *batchJobInfo) loadOrInit(ctx context.Context, api ObjectLayer, job BatchJobRequest) error { + err := ri.load(ctx, api, job) + if errors.Is(err, errNoSuchJob) { + switch { + case job.Replicate != nil: + ri.Version = batchReplVersionV1 + case job.KeyRotate != nil: + ri.Version = batchKeyRotateVersionV1 + case job.Expire != nil: + ri.Version = batchExpireVersionV1 + } + return nil + } + return err +} + +func (ri *batchJobInfo) load(ctx context.Context, api ObjectLayer, job BatchJobRequest) error { + path, err := job.getJobReportPath() + if err != nil { + batchLogIf(ctx, err) + return err + } + return ri.loadByPath(ctx, api, path) +} + +func (ri *batchJobInfo) loadByPath(ctx context.Context, api ObjectLayer, path string) error { + var format, version uint16 + switch filepath.Base(path) { + case batchReplName: + version = batchReplVersionV1 + format = batchReplFormat + case batchKeyRotationName: + version = batchKeyRotateVersionV1 + format = batchKeyRotationFormat + case batchExpireName: + version = batchExpireVersionV1 + format = batchExpireFormat + default: + return errors.New("no supported batch job request specified") + } + + data, err := readConfig(ctx, api, path) + if err != nil { + if errors.Is(err, errConfigNotFound) || isErrObjectNotFound(err) { + return errNoSuchJob + } + return err + } + if len(data) == 0 { + // Seems to be empty create a new batchRepl object. + return nil + } + if len(data) <= 4 { + return fmt.Errorf("%s: no data", ri.JobType) + } + // Read header + switch binary.LittleEndian.Uint16(data[0:2]) { + case format: + default: + return fmt.Errorf("%s: unknown format: %d", ri.JobType, binary.LittleEndian.Uint16(data[0:2])) + } + switch binary.LittleEndian.Uint16(data[2:4]) { + case version: + default: + return fmt.Errorf("%s: unknown version: %d", ri.JobType, binary.LittleEndian.Uint16(data[2:4])) + } + + ri.mu.Lock() + defer ri.mu.Unlock() + + // OK, parse data. + if _, err = ri.UnmarshalMsg(data[4:]); err != nil { + return err + } + + switch ri.Version { + case batchReplVersionV1: + default: + return fmt.Errorf("unexpected batch %s meta version: %d", ri.JobType, ri.Version) + } + + return nil +} + +func (ri *batchJobInfo) clone() *batchJobInfo { + ri.mu.RLock() + defer ri.mu.RUnlock() + + return &batchJobInfo{ + Version: ri.Version, + JobID: ri.JobID, + JobType: ri.JobType, + RetryAttempts: ri.RetryAttempts, + Complete: ri.Complete, + Failed: ri.Failed, + StartTime: ri.StartTime, + LastUpdate: ri.LastUpdate, + Bucket: ri.Bucket, + Object: ri.Object, + Objects: ri.Objects, + ObjectsFailed: ri.ObjectsFailed, + DeleteMarkers: ri.DeleteMarkers, + DeleteMarkersFailed: ri.DeleteMarkersFailed, + BytesTransferred: ri.BytesTransferred, + BytesFailed: ri.BytesFailed, + Attempts: ri.Attempts, + } +} + +func (ri *batchJobInfo) countItem(size int64, dmarker, success bool, attempt int) { + if ri == nil { + return + } + ri.Attempts++ + if success { + if dmarker { + ri.DeleteMarkers++ + } else { + ri.Objects++ + ri.BytesTransferred += size + } + if attempt > 1 { + if dmarker { + ri.DeleteMarkersFailed-- + } else { + ri.ObjectsFailed-- + ri.BytesFailed += size + } + } + } else { + if attempt > 1 { + // Only count first attempt + return + } + if dmarker { + ri.DeleteMarkersFailed++ + } else { + ri.ObjectsFailed++ + ri.BytesFailed += size + } + } +} + +func (ri *batchJobInfo) updateAfter(ctx context.Context, api ObjectLayer, duration time.Duration, job BatchJobRequest) error { + if ri == nil { + return errInvalidArgument + } + now := UTCNow() + ri.mu.Lock() + var ( + format, version uint16 + jobTyp string + ) + + if now.Sub(ri.LastUpdate) >= duration { + switch job.Type() { + case madmin.BatchJobReplicate: + format = batchReplFormat + version = batchReplVersion + jobTyp = string(job.Type()) + ri.Version = batchReplVersionV1 + case madmin.BatchJobKeyRotate: + format = batchKeyRotationFormat + version = batchKeyRotateVersion + jobTyp = string(job.Type()) + ri.Version = batchKeyRotateVersionV1 + case madmin.BatchJobExpire: + format = batchExpireFormat + version = batchExpireVersion + jobTyp = string(job.Type()) + ri.Version = batchExpireVersionV1 + default: + return errInvalidArgument + } + if serverDebugLog { + console.Debugf("%s: persisting info on drive: threshold:%s, %s:%#v\n", jobTyp, now.Sub(ri.LastUpdate), jobTyp, ri) + } + ri.LastUpdate = now + + data := make([]byte, 4, ri.Msgsize()+4) + + // Initialize the header. + binary.LittleEndian.PutUint16(data[0:2], format) + binary.LittleEndian.PutUint16(data[2:4], version) + + buf, err := ri.MarshalMsg(data) + ri.mu.Unlock() + if err != nil { + return err + } + path, err := ri.getJobReportPath() + if err != nil { + batchLogIf(ctx, err) + return err + } + return saveConfig(ctx, api, path, buf) + } + ri.mu.Unlock() + return nil +} + +// Note: to be used only with batch jobs that affect multiple versions through +// a single action. e.g batch-expire has an option to expire all versions of an +// object which matches the given filters. +func (ri *batchJobInfo) trackMultipleObjectVersions(info expireObjInfo, success bool) { + if ri == nil { + return + } + + ri.mu.Lock() + defer ri.mu.Unlock() + + if success { + ri.Bucket = info.Bucket + ri.Object = info.Name + ri.Objects += int64(info.NumVersions) - info.DeleteMarkerCount + ri.DeleteMarkers += info.DeleteMarkerCount + } else { + ri.ObjectsFailed += int64(info.NumVersions) - info.DeleteMarkerCount + ri.DeleteMarkersFailed += info.DeleteMarkerCount + } +} + +func (ri *batchJobInfo) trackCurrentBucketObject(bucket string, info ObjectInfo, success bool, attempt int) { + if ri == nil { + return + } + + ri.mu.Lock() + defer ri.mu.Unlock() + + if success { + ri.Bucket = bucket + ri.Object = info.Name + } + ri.countItem(info.Size, info.DeleteMarker, success, attempt) +} + +func (ri *batchJobInfo) trackCurrentBucketBatch(bucket string, batch []ObjectInfo) { + if ri == nil { + return + } + + ri.mu.Lock() + defer ri.mu.Unlock() + + ri.Bucket = bucket + for i := range batch { + ri.Object = batch[i].Name + ri.countItem(batch[i].Size, batch[i].DeleteMarker, true, 1) + } +} + +// Start start the batch replication job, resumes if there was a pending job via "job.ID" +func (r *BatchJobReplicateV1) Start(ctx context.Context, api ObjectLayer, job BatchJobRequest) error { + ri := &batchJobInfo{ + JobID: job.ID, + JobType: string(job.Type()), + StartTime: job.Started, + } + if err := ri.loadOrInit(ctx, api, job); err != nil { + return err + } + if ri.Complete { + return nil + } + globalBatchJobsMetrics.save(job.ID, ri) + lastObject := ri.Object + + retryAttempts := job.Replicate.Flags.Retry.Attempts + if retryAttempts <= 0 { + retryAttempts = batchReplJobDefaultRetries + } + delay := job.Replicate.Flags.Retry.Delay + if delay <= 0 { + delay = batchReplJobDefaultRetryDelay + } + + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) + + selectObj := func(info FileInfo) (ok bool) { + if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan.D() { + // skip all objects that are newer than specified older duration + return false + } + + if r.Flags.Filter.NewerThan > 0 && time.Since(info.ModTime) >= r.Flags.Filter.NewerThan.D() { + // skip all objects that are older than specified newer duration + return false + } + + if !r.Flags.Filter.CreatedAfter.IsZero() && r.Flags.Filter.CreatedAfter.After(info.ModTime) { + // skip all objects that are created before the specified time. + return false + } + + if !r.Flags.Filter.CreatedBefore.IsZero() && r.Flags.Filter.CreatedBefore.Before(info.ModTime) { + // skip all objects that are created after the specified time. + return false + } + + if len(r.Flags.Filter.Tags) > 0 { + // Only parse object tags if tags filter is specified. + tagMap := map[string]string{} + tagStr := info.Metadata[xhttp.AmzObjectTagging] + if len(tagStr) != 0 { + t, err := tags.ParseObjectTags(tagStr) + if err != nil { + return false + } + tagMap = t.ToMap() + } + + for _, kv := range r.Flags.Filter.Tags { + for t, v := range tagMap { + if kv.Match(BatchJobKV{Key: t, Value: v}) { + return true + } + } + } + + // None of the provided tags filter match skip the object + return false + } + + if len(r.Flags.Filter.Metadata) > 0 { + for _, kv := range r.Flags.Filter.Metadata { + for k, v := range info.Metadata { + if !stringsHasPrefixFold(k, "x-amz-meta-") && !isStandardHeader(k) { + continue + } + // We only need to match x-amz-meta or standardHeaders + if kv.Match(BatchJobKV{Key: k, Value: v}) { + return true + } + } + } + + // None of the provided metadata filters match skip the object. + return false + } + + // if one of source or target is non MinIO, just replicate the top most version like `mc mirror` + isSourceOrTargetS3 := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 + return !isSourceOrTargetS3 || info.IsLatest + } + + u, err := url.Parse(r.Target.Endpoint) + if err != nil { + return err + } + + cred := r.Target.Creds + + c, err := minio.NewCore(u.Host, &minio.Options{ + Creds: credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken), + Secure: u.Scheme == "https", + Transport: getRemoteInstanceTransport(), + BucketLookup: lookupStyle(r.Target.Path), + }) + if err != nil { + return err + } + + c.SetAppInfo("minio-"+batchJobPrefix, r.APIVersion+" "+job.ID) + + retry := false + for attempts := 1; attempts <= retryAttempts; attempts++ { + attempts := attempts + var ( + walkCh = make(chan itemOrErr[ObjectInfo], 100) + slowCh = make(chan itemOrErr[ObjectInfo], 100) + ) + + if r.Source.Snowball.Disable != nil && !*r.Source.Snowball.Disable && r.Source.Type.isMinio() && r.Target.Type.isMinio() { + go func() { + // Snowball currently needs the high level minio-go Client, not the Core one + cl, err := minio.New(u.Host, &minio.Options{ + Creds: credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken), + Secure: u.Scheme == "https", + Transport: getRemoteInstanceTransport(), + BucketLookup: lookupStyle(r.Target.Path), + }) + if err != nil { + batchLogOnceIf(ctx, err, job.ID+"minio.New") + return + } + + // Already validated before arriving here + smallerThan, _ := humanize.ParseBytes(*r.Source.Snowball.SmallerThan) + + batch := make([]ObjectInfo, 0, *r.Source.Snowball.Batch) + writeFn := func(batch []ObjectInfo) { + if len(batch) > 0 { + if err := r.writeAsArchive(ctx, api, cl, batch, r.Target.Prefix); err != nil { + batchLogOnceIf(ctx, err, job.ID+"writeAsArchive") + for _, b := range batch { + slowCh <- itemOrErr[ObjectInfo]{Item: b} + } + } else { + ri.trackCurrentBucketBatch(r.Source.Bucket, batch) + globalBatchJobsMetrics.save(job.ID, ri) + // persist in-memory state to disk after every 10secs. + batchLogOnceIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job), job.ID+"updateAfter") + } + } + } + for obj := range walkCh { + if obj.Item.DeleteMarker || !obj.Item.VersionPurgeStatus.Empty() || obj.Item.Size >= int64(smallerThan) { + slowCh <- obj + continue + } + + batch = append(batch, obj.Item) + + if len(batch) < *r.Source.Snowball.Batch { + continue + } + writeFn(batch) + batch = batch[:0] + } + writeFn(batch) + xioutil.SafeClose(slowCh) + }() + } else { + slowCh = walkCh + } + + workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_REPLICATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2))) + if err != nil { + return err + } + + wk, err := workers.New(workerSize) + if err != nil { + // invalid worker size. + return err + } + + walkQuorum := env.Get("_MINIO_BATCH_REPLICATION_WALK_QUORUM", "strict") + if walkQuorum == "" { + walkQuorum = "strict" + } + ctx, cancelCause := context.WithCancelCause(ctx) + // one of source/target is s3, skip delete marker and all versions under the same object name. + s3Type := r.Target.Type == BatchJobReplicateResourceS3 || r.Source.Type == BatchJobReplicateResourceS3 + + go func() { + prefixes := r.Source.Prefix.F() + if len(prefixes) == 0 { + prefixes = []string{""} + } + for _, prefix := range prefixes { + prefixWalkCh := make(chan itemOrErr[ObjectInfo], 100) + if err := api.Walk(ctx, r.Source.Bucket, prefix, prefixWalkCh, WalkOptions{ + Marker: lastObject, + Filter: selectObj, + AskDisks: walkQuorum, + }); err != nil { + cancelCause(err) + xioutil.SafeClose(walkCh) + return + } + for obj := range prefixWalkCh { + walkCh <- obj + } + } + xioutil.SafeClose(walkCh) + }() + + prevObj := "" + + skipReplicate := false + for res := range slowCh { + if res.Err != nil { + ri.Failed = true + batchLogOnceIf(ctx, res.Err, job.ID+"res.Err") + continue + } + result := res.Item + if result.Name != prevObj { + prevObj = result.Name + skipReplicate = result.DeleteMarker && s3Type + } + if skipReplicate { + continue + } + wk.Take() + go func() { + defer wk.Give() + + stopFn := globalBatchJobsMetrics.trace(batchJobMetricReplication, job.ID, attempts) + success := true + if err := r.ReplicateToTarget(ctx, api, c, result, retry); err != nil { + if minio.ToErrorResponse(err).Code == "PreconditionFailed" { + // pre-condition failed means we already have the object copied over. + return + } + // object must be deleted concurrently, allow these failures but do not count them + if isErrVersionNotFound(err) || isErrObjectNotFound(err) { + return + } + stopFn(result, err) + batchLogOnceIf(ctx, err, job.ID+"ReplicateToTarget") + success = false + } else { + stopFn(result, nil) + } + ri.trackCurrentBucketObject(r.Source.Bucket, result, success, attempts) + globalBatchJobsMetrics.save(job.ID, ri) + // persist in-memory state to disk after every 10secs. + batchLogOnceIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job), job.ID+"updateAfter2") + + if wait := globalBatchConfig.ReplicationWait(); wait > 0 { + time.Sleep(wait) + } + }() + } + wk.Wait() + // Do not need to retry if we can't list objects on source. + if context.Cause(ctx) != nil { + return context.Cause(ctx) + } + ri.RetryAttempts = attempts + ri.Complete = ri.ObjectsFailed == 0 + ri.Failed = ri.ObjectsFailed > 0 + + globalBatchJobsMetrics.save(job.ID, ri) + // persist in-memory state to disk. + batchLogOnceIf(ctx, ri.updateAfter(ctx, api, 0, job), job.ID+"updateAfter3") + + if err := r.Notify(ctx, ri); err != nil { + batchLogOnceIf(ctx, fmt.Errorf("unable to notify %v", err), job.ID+"notify") + } + + cancelCause(nil) + if ri.Failed { + ri.ObjectsFailed = 0 + ri.Bucket = "" + ri.Object = "" + ri.Objects = 0 + ri.BytesFailed = 0 + ri.BytesTransferred = 0 + retry = true // indicate we are retrying.. + time.Sleep(delay + time.Duration(rnd.Float64()*float64(delay))) + continue + } + + break + } + + return nil +} + +//msgp:ignore batchReplicationJobError +type batchReplicationJobError struct { + Code string + Description string + HTTPStatusCode int + ObjectSize int64 +} + +func (e batchReplicationJobError) Error() string { + return e.Description +} + +// Validate validates the job definition input +func (r *BatchJobReplicateV1) Validate(ctx context.Context, job BatchJobRequest, o ObjectLayer) error { + if r == nil { + return nil + } + + if r.APIVersion != batchReplJobAPIVersion { + return errInvalidArgument + } + + if r.Source.Endpoint != "" && r.Target.Endpoint != "" { + return errInvalidArgument + } + + if r.Source.Creds.Empty() && r.Target.Creds.Empty() { + return errInvalidArgument + } + + if r.Source.Bucket == "" || r.Target.Bucket == "" { + return errInvalidArgument + } + + var isRemoteToLocal bool + localBkt := r.Source.Bucket + if r.Source.Endpoint != "" { + localBkt = r.Target.Bucket + isRemoteToLocal = true + } + info, err := o.GetBucketInfo(ctx, localBkt, BucketOptions{}) + if err != nil { + if isErrBucketNotFound(err) { + return batchReplicationJobError{ + Code: "NoSuchSourceBucket", + Description: fmt.Sprintf("The specified bucket %s does not exist", localBkt), + HTTPStatusCode: http.StatusNotFound, + } + } + return err + } + + if err := r.Source.Type.Validate(); err != nil { + return err + } + if err := r.Source.Snowball.Validate(); err != nil { + return err + } + + if !r.Source.Creds.Empty() { + if err := r.Source.Creds.Validate(); err != nil { + return err + } + } + if r.Target.Endpoint == "" && !r.Target.Creds.Empty() { + return errInvalidArgument + } + + if r.Source.Endpoint == "" && !r.Source.Creds.Empty() { + return errInvalidArgument + } + + if r.Source.Endpoint != "" && !r.Source.Type.isMinio() && !r.Source.ValidPath() { + return errInvalidArgument + } + + if r.Target.Endpoint != "" && !r.Target.Type.isMinio() && !r.Target.ValidPath() { + return errInvalidArgument + } + + if !r.Target.Creds.Empty() { + if err := r.Target.Creds.Validate(); err != nil { + return err + } + } + + if err := r.Target.Type.Validate(); err != nil { + return err + } + + for _, tag := range r.Flags.Filter.Tags { + if err := tag.Validate(); err != nil { + return err + } + } + + for _, meta := range r.Flags.Filter.Metadata { + if err := meta.Validate(); err != nil { + return err + } + } + + if err := r.Flags.Retry.Validate(); err != nil { + return err + } + + remoteEp := r.Target.Endpoint + remoteBkt := r.Target.Bucket + cred := r.Target.Creds + pathStyle := r.Target.Path + + if r.Source.Endpoint != "" { + remoteEp = r.Source.Endpoint + cred = r.Source.Creds + remoteBkt = r.Source.Bucket + pathStyle = r.Source.Path + } + + u, err := url.Parse(remoteEp) + if err != nil { + return err + } + + c, err := minio.NewCore(u.Host, &minio.Options{ + Creds: credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken), + Secure: u.Scheme == "https", + Transport: getRemoteInstanceTransport(), + BucketLookup: lookupStyle(pathStyle), + }) + if err != nil { + return err + } + c.SetAppInfo("minio-"+batchJobPrefix, r.APIVersion+" "+job.ID) + + vcfg, err := c.GetBucketVersioning(ctx, remoteBkt) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchBucket" { + return batchReplicationJobError{ + Code: "NoSuchTargetBucket", + Description: "The specified target bucket does not exist", + HTTPStatusCode: http.StatusNotFound, + } + } + return err + } + // if both source and target are minio instances + minioType := r.Target.Type == BatchJobReplicateResourceMinIO && r.Source.Type == BatchJobReplicateResourceMinIO + // If source has versioning enabled, target must have versioning enabled + if minioType && ((info.Versioning && !vcfg.Enabled() && !isRemoteToLocal) || (!info.Versioning && vcfg.Enabled() && isRemoteToLocal)) { + return batchReplicationJobError{ + Code: "InvalidBucketState", + Description: fmt.Sprintf("The source '%s' has versioning enabled, target '%s' must have versioning enabled", + r.Source.Bucket, r.Target.Bucket), + HTTPStatusCode: http.StatusBadRequest, + } + } + + r.clnt = c + return nil +} + +// Type returns type of batch job, currently only supports 'replicate' +func (j BatchJobRequest) Type() madmin.BatchJobType { + switch { + case j.Replicate != nil: + return madmin.BatchJobReplicate + case j.KeyRotate != nil: + return madmin.BatchJobKeyRotate + case j.Expire != nil: + return madmin.BatchJobExpire + } + return madmin.BatchJobType("unknown") +} + +// Validate validates the current job, used by 'save()' before +// persisting the job request +func (j BatchJobRequest) Validate(ctx context.Context, o ObjectLayer) error { + switch { + case j.Replicate != nil: + return j.Replicate.Validate(ctx, j, o) + case j.KeyRotate != nil: + return j.KeyRotate.Validate(ctx, j, o) + case j.Expire != nil: + return j.Expire.Validate(ctx, j, o) + } + return errInvalidArgument +} + +func (j BatchJobRequest) delete(ctx context.Context, api ObjectLayer) { + deleteConfig(ctx, api, getJobPath(j)) +} + +func (j BatchJobRequest) getJobReportPath() (string, error) { + var fileName string + switch { + case j.Replicate != nil: + fileName = batchReplName + case j.KeyRotate != nil: + fileName = batchKeyRotationName + case j.Expire != nil: + fileName = batchExpireName + default: + return "", errors.New("unknown job type") + } + return pathJoin(batchJobReportsPrefix, j.ID, fileName), nil +} + +func (j *BatchJobRequest) save(ctx context.Context, api ObjectLayer) error { + if j.Replicate == nil && j.KeyRotate == nil && j.Expire == nil { + return errInvalidArgument + } + + if err := j.Validate(ctx, api); err != nil { + return err + } + + job, err := j.MarshalMsg(nil) + if err != nil { + return err + } + + return saveConfig(ctx, api, getJobPath(*j), job) +} + +func (j *BatchJobRequest) load(ctx context.Context, api ObjectLayer, name string) error { + if j == nil { + return nil + } + + job, err := readConfig(ctx, api, name) + if err != nil { + if errors.Is(err, errConfigNotFound) || isErrObjectNotFound(err) { + err = errNoSuchJob + } + return err + } + + _, err = j.UnmarshalMsg(job) + return err +} + +func batchReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (putOpts minio.PutObjectOptions, isMP bool, err error) { + // TODO: support custom storage class for remote replication + putOpts, isMP, err = putReplicationOpts(ctx, "", objInfo) + if err != nil { + return putOpts, isMP, err + } + putOpts.Internal = minio.AdvancedPutOptions{ + SourceVersionID: objInfo.VersionID, + SourceMTime: objInfo.ModTime, + SourceETag: objInfo.ETag, + ReplicationRequest: true, + } + return putOpts, isMP, nil +} + +// ListBatchJobs - lists all currently active batch jobs, optionally takes {jobType} +// input to list only active batch jobs of 'jobType' +func (a adminAPIHandlers) ListBatchJobs(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.ListBatchJobsAction) + if objectAPI == nil { + return + } + + jobType := r.Form.Get("jobType") + + resultCh := make(chan itemOrErr[ObjectInfo]) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + if err := objectAPI.Walk(ctx, minioMetaBucket, batchJobPrefix, resultCh, WalkOptions{}); err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + listResult := madmin.ListBatchJobsResult{} + for result := range resultCh { + if result.Err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, result.Err), r.URL) + return + } + if strings.HasPrefix(result.Item.Name, batchJobReportsPrefix+slashSeparator) { + continue + } + req := &BatchJobRequest{} + if err := req.load(ctx, objectAPI, result.Item.Name); err != nil { + if !errors.Is(err, errNoSuchJob) { + batchLogIf(ctx, err) + } + continue + } + + if jobType == string(req.Type()) || jobType == "" { + listResult.Jobs = append(listResult.Jobs, madmin.BatchJobResult{ + ID: req.ID, + Type: req.Type(), + Started: req.Started, + User: req.User, + Elapsed: time.Since(req.Started), + }) + } + } + + batchLogIf(ctx, json.NewEncoder(w).Encode(&listResult)) +} + +// BatchJobStatus - returns the status of a batch job saved in the disk +func (a adminAPIHandlers) BatchJobStatus(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.ListBatchJobsAction) + if objectAPI == nil { + return + } + + jobID := r.Form.Get("jobId") + if jobID == "" { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL) + return + } + + req := BatchJobRequest{ID: jobID} + if i := strings.Index(jobID, "-"); i > 0 { + switch madmin.BatchJobType(jobID[:i]) { + case madmin.BatchJobReplicate: + req.Replicate = &BatchJobReplicateV1{} + case madmin.BatchJobKeyRotate: + req.KeyRotate = &BatchJobKeyRotateV1{} + case madmin.BatchJobExpire: + req.Expire = &BatchJobExpire{} + default: + writeErrorResponseJSON(ctx, w, toAPIError(ctx, errors.New("job ID format unrecognized")), r.URL) + return + } + } + + ri := &batchJobInfo{} + if err := ri.load(ctx, objectAPI, req); err != nil { + if !errors.Is(err, errNoSuchJob) { + batchLogIf(ctx, err) + } + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + buf, err := json.Marshal(madmin.BatchJobStatus{LastMetric: ri.metric()}) + if err != nil { + batchLogIf(ctx, err) + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + w.Write(buf) +} + +var errNoSuchJob = errors.New("no such job") + +// DescribeBatchJob returns the currently active batch job definition +func (a adminAPIHandlers) DescribeBatchJob(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.DescribeBatchJobAction) + if objectAPI == nil { + return + } + + jobID := r.Form.Get("jobId") + if jobID == "" { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL) + return + } + + req := &BatchJobRequest{} + if err := req.load(ctx, objectAPI, pathJoin(batchJobPrefix, jobID)); err != nil { + if !errors.Is(err, errNoSuchJob) { + batchLogIf(ctx, err) + } + + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Remove sensitive fields. + req.RedactSensitive() + buf, err := yaml.Marshal(req) + if err != nil { + batchLogIf(ctx, err) + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + w.Write(buf) +} + +// StartBatchJob queue a new job for execution +func (a adminAPIHandlers) StartBatchJob(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, creds := validateAdminReq(ctx, w, r, policy.StartBatchJobAction) + if objectAPI == nil { + return + } + + buf, err := io.ReadAll(xioutil.HardLimitReader(r.Body, humanize.MiByte*4)) + if err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + user := creds.AccessKey + if creds.ParentUser != "" { + user = creds.ParentUser + } + + job := &BatchJobRequest{} + if err = yaml.Unmarshal(buf, job); err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Fill with default values + if job.Replicate != nil { + if job.Replicate.Source.Snowball.Disable == nil { + job.Replicate.Source.Snowball.Disable = ptr(false) + } + if job.Replicate.Source.Snowball.Batch == nil { + job.Replicate.Source.Snowball.Batch = ptr(100) + } + if job.Replicate.Source.Snowball.InMemory == nil { + job.Replicate.Source.Snowball.InMemory = ptr(true) + } + if job.Replicate.Source.Snowball.Compress == nil { + job.Replicate.Source.Snowball.Compress = ptr(false) + } + if job.Replicate.Source.Snowball.SmallerThan == nil { + job.Replicate.Source.Snowball.SmallerThan = ptr("5MiB") + } + if job.Replicate.Source.Snowball.SkipErrs == nil { + job.Replicate.Source.Snowball.SkipErrs = ptr(true) + } + } + + // Validate the incoming job request + if err := job.Validate(ctx, objectAPI); err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + job.ID = fmt.Sprintf("%s-%s%s%d", job.Type(), shortuuid.New(), getKeySeparator(), GetProxyEndpointLocalIndex(globalProxyEndpoints)) + job.User = user + job.Started = time.Now() + + if err := job.save(ctx, objectAPI); err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if err = globalBatchJobPool.queueJob(job); err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + buf, err = json.Marshal(&madmin.BatchJobResult{ + ID: job.ID, + Type: job.Type(), + Started: job.Started, + User: job.User, + }) + if err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + writeSuccessResponseJSON(w, buf) +} + +// CancelBatchJob cancels a job in progress +func (a adminAPIHandlers) CancelBatchJob(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.CancelBatchJobAction) + if objectAPI == nil { + return + } + + jobID := r.Form.Get("id") + if jobID == "" { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, errInvalidArgument), r.URL) + return + } + + if _, proxied, _ := proxyRequestByToken(ctx, w, r, jobID, true); proxied { + return + } + + if err := globalBatchJobPool.canceler(jobID, true); err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrInvalidRequest, err), r.URL) + return + } + + j := BatchJobRequest{ + ID: jobID, + } + + j.delete(ctx, objectAPI) + + writeSuccessNoContent(w) +} + +//msgp:ignore BatchJobPool + +// BatchJobPool batch job pool +type BatchJobPool struct { + ctx context.Context + objLayer ObjectLayer + once sync.Once + mu sync.Mutex + jobCh chan *BatchJobRequest + jmu sync.Mutex // protects jobCancelers + jobCancelers map[string]context.CancelFunc + workerKillCh chan struct{} + workerSize int +} + +var globalBatchJobPool *BatchJobPool + +// newBatchJobPool creates a pool of job manifest workers of specified size +func newBatchJobPool(ctx context.Context, o ObjectLayer, workers int) *BatchJobPool { + jpool := &BatchJobPool{ + ctx: ctx, + objLayer: o, + jobCh: make(chan *BatchJobRequest, 10000), + workerKillCh: make(chan struct{}, workers), + jobCancelers: make(map[string]context.CancelFunc), + } + jpool.ResizeWorkers(workers) + + randomWait := func() time.Duration { + // randomWait depends on the number of nodes to avoid triggering resume and cleanups at the same time. + return time.Duration(rand.Float64() * float64(time.Duration(globalEndpoints.NEndpoints())*time.Hour)) + } + + go func() { + jpool.resume(randomWait) + jpool.cleanupReports(randomWait) + }() + + return jpool +} + +func (j *BatchJobPool) cleanupReports(randomWait func() time.Duration) { + t := time.NewTimer(randomWait()) + defer t.Stop() + + for { + select { + case <-GlobalContext.Done(): + return + case <-t.C: + results := make(chan itemOrErr[ObjectInfo], 100) + ctx, cancel := context.WithCancel(j.ctx) + defer cancel() + if err := j.objLayer.Walk(ctx, minioMetaBucket, batchJobReportsPrefix, results, WalkOptions{}); err != nil { + batchLogIf(j.ctx, err) + t.Reset(randomWait()) + continue + } + for result := range results { + if result.Err != nil { + batchLogIf(j.ctx, result.Err) + continue + } + ri := &batchJobInfo{} + if err := ri.loadByPath(ctx, j.objLayer, result.Item.Name); err != nil { + batchLogIf(ctx, err) + continue + } + if (ri.Complete || ri.Failed) && time.Since(ri.LastUpdate) > oldJobsExpiration { + deleteConfig(ctx, j.objLayer, result.Item.Name) + } + } + + t.Reset(randomWait()) + } + } +} + +func (j *BatchJobPool) resume(randomWait func() time.Duration) { + time.Sleep(randomWait()) + + results := make(chan itemOrErr[ObjectInfo], 100) + ctx, cancel := context.WithCancel(j.ctx) + defer cancel() + if err := j.objLayer.Walk(ctx, minioMetaBucket, batchJobPrefix, results, WalkOptions{}); err != nil { + batchLogIf(j.ctx, err) + return + } + for result := range results { + if result.Err != nil { + batchLogIf(j.ctx, result.Err) + continue + } + if strings.HasPrefix(result.Item.Name, batchJobReportsPrefix+slashSeparator) { + continue + } + // ignore batch-replicate.bin and batch-rotate.bin entries + if strings.HasSuffix(result.Item.Name, slashSeparator) { + continue + } + req := &BatchJobRequest{} + if err := req.load(ctx, j.objLayer, result.Item.Name); err != nil { + batchLogIf(ctx, err) + continue + } + _, nodeIdx := parseRequestToken(req.ID) + if nodeIdx > -1 && GetProxyEndpointLocalIndex(globalProxyEndpoints) != nodeIdx { + // This job doesn't belong on this node. + continue + } + if err := j.queueJob(req); err != nil { + batchLogIf(ctx, err) + continue + } + } +} + +// AddWorker adds a replication worker to the pool +func (j *BatchJobPool) AddWorker() { + if j == nil { + return + } + for { + select { + case <-j.ctx.Done(): + return + case job, ok := <-j.jobCh: + if !ok { + return + } + switch { + case job.Replicate != nil: + if job.Replicate.RemoteToLocal() { + if err := job.Replicate.StartFromSource(job.ctx, j.objLayer, *job); err != nil { + if !isErrBucketNotFound(err) { + batchLogIf(j.ctx, err) + j.canceler(job.ID, false) + continue + } + // Bucket not found proceed to delete such a job. + } + } else { + if err := job.Replicate.Start(job.ctx, j.objLayer, *job); err != nil { + if !isErrBucketNotFound(err) { + batchLogIf(j.ctx, err) + j.canceler(job.ID, false) + continue + } + // Bucket not found proceed to delete such a job. + } + } + case job.KeyRotate != nil: + if err := job.KeyRotate.Start(job.ctx, j.objLayer, *job); err != nil { + if !isErrBucketNotFound(err) { + batchLogIf(j.ctx, err) + continue + } + } + case job.Expire != nil: + if err := job.Expire.Start(job.ctx, j.objLayer, *job); err != nil { + if !isErrBucketNotFound(err) { + batchLogIf(j.ctx, err) + continue + } + } + } + j.canceler(job.ID, false) + case <-j.workerKillCh: + return + } + } +} + +// ResizeWorkers sets replication workers pool to new size +func (j *BatchJobPool) ResizeWorkers(n int) { + if j == nil { + return + } + + j.mu.Lock() + defer j.mu.Unlock() + + for j.workerSize < n { + j.workerSize++ + go j.AddWorker() + } + for j.workerSize > n { + j.workerSize-- + go func() { j.workerKillCh <- struct{}{} }() + } +} + +func (j *BatchJobPool) queueJob(req *BatchJobRequest) error { + if j == nil { + return errInvalidArgument + } + jctx, jcancel := context.WithCancel(j.ctx) + j.jmu.Lock() + j.jobCancelers[req.ID] = jcancel + j.jmu.Unlock() + req.ctx = jctx + + select { + case <-j.ctx.Done(): + j.once.Do(func() { + xioutil.SafeClose(j.jobCh) + }) + case j.jobCh <- req: + default: + return fmt.Errorf("batch job queue is currently full please try again later %#v", req) + } + return nil +} + +// delete canceler from the map, cancel job if requested +func (j *BatchJobPool) canceler(jobID string, cancel bool) error { + if j == nil { + return errInvalidArgument + } + j.jmu.Lock() + defer j.jmu.Unlock() + if canceler, ok := j.jobCancelers[jobID]; ok { + if cancel { + canceler() + } + } + if cancel { + delete(j.jobCancelers, jobID) + } + return nil +} + +//msgp:ignore batchJobMetrics +type batchJobMetrics struct { + sync.RWMutex + metrics map[string]*batchJobInfo +} + +//msgp:ignore batchJobMetric +//go:generate stringer -type=batchJobMetric -trimprefix=batchJobMetric $GOFILE +type batchJobMetric uint8 + +const ( + batchJobMetricReplication batchJobMetric = iota + batchJobMetricKeyRotation + batchJobMetricExpire +) + +func batchJobTrace(d batchJobMetric, job string, startTime time.Time, duration time.Duration, info objTraceInfoer, attempts int, err error) madmin.TraceInfo { + var errStr string + if err != nil { + errStr = err.Error() + } + traceType := madmin.TraceBatchReplication + switch d { + case batchJobMetricKeyRotation: + traceType = madmin.TraceBatchKeyRotation + case batchJobMetricExpire: + traceType = madmin.TraceBatchExpire + } + funcName := fmt.Sprintf("%s() (job-name=%s)", d.String(), job) + if attempts > 0 { + funcName = fmt.Sprintf("%s() (job-name=%s,attempts=%s)", d.String(), job, humanize.Ordinal(attempts)) + } + return madmin.TraceInfo{ + TraceType: traceType, + Time: startTime, + NodeName: globalLocalNodeName, + FuncName: funcName, + Duration: duration, + Path: fmt.Sprintf("%s (versionID=%s)", info.TraceObjName(), info.TraceVersionID()), + Error: errStr, + } +} + +func (ri *batchJobInfo) metric() madmin.JobMetric { + m := madmin.JobMetric{ + JobID: ri.JobID, + JobType: ri.JobType, + StartTime: ri.StartTime, + LastUpdate: ri.LastUpdate, + RetryAttempts: ri.RetryAttempts, + Complete: ri.Complete, + Failed: ri.Failed, + } + + switch ri.JobType { + case string(madmin.BatchJobReplicate): + m.Replicate = &madmin.ReplicateInfo{ + Bucket: ri.Bucket, + Object: ri.Object, + Objects: ri.Objects, + DeleteMarkers: ri.DeleteMarkers, + ObjectsFailed: ri.ObjectsFailed, + DeleteMarkersFailed: ri.DeleteMarkersFailed, + BytesTransferred: ri.BytesTransferred, + BytesFailed: ri.BytesFailed, + } + case string(madmin.BatchJobKeyRotate): + m.KeyRotate = &madmin.KeyRotationInfo{ + Bucket: ri.Bucket, + Object: ri.Object, + Objects: ri.Objects, + ObjectsFailed: ri.ObjectsFailed, + } + case string(madmin.BatchJobExpire): + m.Expired = &madmin.ExpirationInfo{ + Bucket: ri.Bucket, + Object: ri.Object, + Objects: ri.Objects, + DeleteMarkers: ri.DeleteMarkers, + ObjectsFailed: ri.ObjectsFailed, + DeleteMarkersFailed: ri.DeleteMarkersFailed, + } + } + + return m +} + +func (m *batchJobMetrics) report(jobID string) (metrics *madmin.BatchJobMetrics) { + metrics = &madmin.BatchJobMetrics{CollectedAt: time.Now(), Jobs: make(map[string]madmin.JobMetric)} + m.RLock() + defer m.RUnlock() + + if jobID != "" { + if job, ok := m.metrics[jobID]; ok { + metrics.Jobs[jobID] = job.metric() + } + return metrics + } + + for id, job := range m.metrics { + metrics.Jobs[id] = job.metric() + } + return metrics +} + +// keep job metrics for some time after the job is completed +// in-case some one wants to look at the older results. +func (m *batchJobMetrics) purgeJobMetrics() { + t := time.NewTicker(6 * time.Hour) + defer t.Stop() + + for { + select { + case <-GlobalContext.Done(): + return + case <-t.C: + var toDeleteJobMetrics []string + m.RLock() + for id, metrics := range m.metrics { + if time.Since(metrics.LastUpdate) > oldJobsExpiration && (metrics.Complete || metrics.Failed) { + toDeleteJobMetrics = append(toDeleteJobMetrics, id) + } + } + m.RUnlock() + for _, jobID := range toDeleteJobMetrics { + m.delete(jobID) + j := BatchJobRequest{ + ID: jobID, + } + j.delete(GlobalContext, newObjectLayerFn()) + } + } + } +} + +// load metrics from disk on startup +func (m *batchJobMetrics) init(ctx context.Context, objectAPI ObjectLayer) error { + resultCh := make(chan itemOrErr[ObjectInfo]) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + if err := objectAPI.Walk(ctx, minioMetaBucket, batchJobReportsPrefix, resultCh, WalkOptions{}); err != nil { + return err + } + + for result := range resultCh { + if result.Err != nil { + return result.Err + } + ri := &batchJobInfo{} + if err := ri.loadByPath(ctx, objectAPI, result.Item.Name); err != nil { + if !errors.Is(err, errNoSuchJob) { + batchLogIf(ctx, err) + } + continue + } + m.metrics[ri.JobID] = ri + } + return nil +} + +func (m *batchJobMetrics) delete(jobID string) { + m.Lock() + defer m.Unlock() + + delete(m.metrics, jobID) +} + +func (m *batchJobMetrics) save(jobID string, ri *batchJobInfo) { + m.Lock() + defer m.Unlock() + + m.metrics[jobID] = ri.clone() +} + +type objTraceInfoer interface { + TraceObjName() string + TraceVersionID() string +} + +// TraceObjName returns name of object being traced +func (td ObjectToDelete) TraceObjName() string { + return td.ObjectName +} + +// TraceVersionID returns version-id of object being traced +func (td ObjectToDelete) TraceVersionID() string { + return td.VersionID +} + +// TraceObjName returns name of object being traced +func (oi ObjectInfo) TraceObjName() string { + return oi.Name +} + +// TraceVersionID returns version-id of object being traced +func (oi ObjectInfo) TraceVersionID() string { + return oi.VersionID +} + +func (m *batchJobMetrics) trace(d batchJobMetric, job string, attempts int) func(info objTraceInfoer, err error) { + startTime := time.Now() + return func(info objTraceInfoer, err error) { + duration := time.Since(startTime) + if globalTrace.NumSubscribers(madmin.TraceBatch) > 0 { + globalTrace.Publish(batchJobTrace(d, job, startTime, duration, info, attempts, err)) + return + } + switch d { + case batchJobMetricReplication: + if globalTrace.NumSubscribers(madmin.TraceBatchReplication) > 0 { + globalTrace.Publish(batchJobTrace(d, job, startTime, duration, info, attempts, err)) + } + case batchJobMetricKeyRotation: + if globalTrace.NumSubscribers(madmin.TraceBatchKeyRotation) > 0 { + globalTrace.Publish(batchJobTrace(d, job, startTime, duration, info, attempts, err)) + } + case batchJobMetricExpire: + if globalTrace.NumSubscribers(madmin.TraceBatchExpire) > 0 { + globalTrace.Publish(batchJobTrace(d, job, startTime, duration, info, attempts, err)) + } + } + } +} + +func lookupStyle(s string) minio.BucketLookupType { + var lookup minio.BucketLookupType + switch s { + case "on": + lookup = minio.BucketLookupPath + case "off": + lookup = minio.BucketLookupDNS + default: + lookup = minio.BucketLookupAuto + } + return lookup +} + +// BatchJobPrefix - to support prefix field yaml unmarshalling with string or slice of strings +type BatchJobPrefix []string + +var _ yaml.Unmarshaler = &BatchJobPrefix{} + +// UnmarshalYAML - to support prefix field yaml unmarshalling with string or slice of strings +func (b *BatchJobPrefix) UnmarshalYAML(value *yaml.Node) error { + // try slice first + tmpSlice := []string{} + if err := value.Decode(&tmpSlice); err == nil { + *b = tmpSlice + return nil + } + // try string + tmpStr := "" + if err := value.Decode(&tmpStr); err == nil { + *b = []string{tmpStr} + return nil + } + return fmt.Errorf("unable to decode %s", value.Value) +} + +// F - return prefix(es) as slice +func (b *BatchJobPrefix) F() []string { + return *b +} diff --git a/cmd/batch-handlers_gen.go b/cmd/batch-handlers_gen.go new file mode 100644 index 00000000..ad29932b --- /dev/null +++ b/cmd/batch-handlers_gen.go @@ -0,0 +1,952 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *BatchJobPrefix) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if cap((*z)) >= int(zb0002) { + (*z) = (*z)[:zb0002] + } else { + (*z) = make(BatchJobPrefix, zb0002) + } + for zb0001 := range *z { + (*z)[zb0001], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, zb0001) + return + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z BatchJobPrefix) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteArrayHeader(uint32(len(z))) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0003 := range z { + err = en.WriteString(z[zb0003]) + if err != nil { + err = msgp.WrapError(err, zb0003) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z BatchJobPrefix) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendArrayHeader(o, uint32(len(z))) + for zb0003 := range z { + o = msgp.AppendString(o, z[zb0003]) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchJobPrefix) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if cap((*z)) >= int(zb0002) { + (*z) = (*z)[:zb0002] + } else { + (*z) = make(BatchJobPrefix, zb0002) + } + for zb0001 := range *z { + (*z)[zb0001], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, zb0001) + return + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z BatchJobPrefix) Msgsize() (s int) { + s = msgp.ArrayHeaderSize + for zb0003 := range z { + s += msgp.StringPrefixSize + len(z[zb0003]) + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *BatchJobRequest) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ID": + z.ID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + case "User": + z.User, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "User") + return + } + case "Started": + z.Started, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "Started") + return + } + case "Replicate": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "Replicate") + return + } + z.Replicate = nil + } else { + if z.Replicate == nil { + z.Replicate = new(BatchJobReplicateV1) + } + err = z.Replicate.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Replicate") + return + } + } + case "KeyRotate": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "KeyRotate") + return + } + z.KeyRotate = nil + } else { + if z.KeyRotate == nil { + z.KeyRotate = new(BatchJobKeyRotateV1) + } + err = z.KeyRotate.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "KeyRotate") + return + } + } + case "Expire": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "Expire") + return + } + z.Expire = nil + } else { + if z.Expire == nil { + z.Expire = new(BatchJobExpire) + } + err = z.Expire.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Expire") + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *BatchJobRequest) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 6 + // write "ID" + err = en.Append(0x86, 0xa2, 0x49, 0x44) + if err != nil { + return + } + err = en.WriteString(z.ID) + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + // write "User" + err = en.Append(0xa4, 0x55, 0x73, 0x65, 0x72) + if err != nil { + return + } + err = en.WriteString(z.User) + if err != nil { + err = msgp.WrapError(err, "User") + return + } + // write "Started" + err = en.Append(0xa7, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteTime(z.Started) + if err != nil { + err = msgp.WrapError(err, "Started") + return + } + // write "Replicate" + err = en.Append(0xa9, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65) + if err != nil { + return + } + if z.Replicate == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.Replicate.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Replicate") + return + } + } + // write "KeyRotate" + err = en.Append(0xa9, 0x4b, 0x65, 0x79, 0x52, 0x6f, 0x74, 0x61, 0x74, 0x65) + if err != nil { + return + } + if z.KeyRotate == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.KeyRotate.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "KeyRotate") + return + } + } + // write "Expire" + err = en.Append(0xa6, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65) + if err != nil { + return + } + if z.Expire == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.Expire.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Expire") + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *BatchJobRequest) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 6 + // string "ID" + o = append(o, 0x86, 0xa2, 0x49, 0x44) + o = msgp.AppendString(o, z.ID) + // string "User" + o = append(o, 0xa4, 0x55, 0x73, 0x65, 0x72) + o = msgp.AppendString(o, z.User) + // string "Started" + o = append(o, 0xa7, 0x53, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64) + o = msgp.AppendTime(o, z.Started) + // string "Replicate" + o = append(o, 0xa9, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65) + if z.Replicate == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.Replicate.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Replicate") + return + } + } + // string "KeyRotate" + o = append(o, 0xa9, 0x4b, 0x65, 0x79, 0x52, 0x6f, 0x74, 0x61, 0x74, 0x65) + if z.KeyRotate == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.KeyRotate.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "KeyRotate") + return + } + } + // string "Expire" + o = append(o, 0xa6, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65) + if z.Expire == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.Expire.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Expire") + return + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchJobRequest) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ID": + z.ID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + case "User": + z.User, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "User") + return + } + case "Started": + z.Started, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Started") + return + } + case "Replicate": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.Replicate = nil + } else { + if z.Replicate == nil { + z.Replicate = new(BatchJobReplicateV1) + } + bts, err = z.Replicate.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Replicate") + return + } + } + case "KeyRotate": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.KeyRotate = nil + } else { + if z.KeyRotate == nil { + z.KeyRotate = new(BatchJobKeyRotateV1) + } + bts, err = z.KeyRotate.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "KeyRotate") + return + } + } + case "Expire": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.Expire = nil + } else { + if z.Expire == nil { + z.Expire = new(BatchJobExpire) + } + bts, err = z.Expire.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Expire") + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *BatchJobRequest) Msgsize() (s int) { + s = 1 + 3 + msgp.StringPrefixSize + len(z.ID) + 5 + msgp.StringPrefixSize + len(z.User) + 8 + msgp.TimeSize + 10 + if z.Replicate == nil { + s += msgp.NilSize + } else { + s += z.Replicate.Msgsize() + } + s += 10 + if z.KeyRotate == nil { + s += msgp.NilSize + } else { + s += z.KeyRotate.Msgsize() + } + s += 7 + if z.Expire == nil { + s += msgp.NilSize + } else { + s += z.Expire.Msgsize() + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *batchJobInfo) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "v": + z.Version, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Version") + return + } + case "jid": + z.JobID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "JobID") + return + } + case "jt": + z.JobType, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "JobType") + return + } + case "st": + z.StartTime, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "StartTime") + return + } + case "lu": + z.LastUpdate, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "LastUpdate") + return + } + case "ra": + z.RetryAttempts, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "RetryAttempts") + return + } + case "at": + z.Attempts, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Attempts") + return + } + case "cmp": + z.Complete, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Complete") + return + } + case "fld": + z.Failed, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Failed") + return + } + case "lbkt": + z.Bucket, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "lobj": + z.Object, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + case "ob": + z.Objects, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + case "dm": + z.DeleteMarkers, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "DeleteMarkers") + return + } + case "obf": + z.ObjectsFailed, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ObjectsFailed") + return + } + case "dmf": + z.DeleteMarkersFailed, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "DeleteMarkersFailed") + return + } + case "bt": + z.BytesTransferred, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "BytesTransferred") + return + } + case "bf": + z.BytesFailed, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "BytesFailed") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *batchJobInfo) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 17 + // write "v" + err = en.Append(0xde, 0x0, 0x11, 0xa1, 0x76) + if err != nil { + return + } + err = en.WriteInt(z.Version) + if err != nil { + err = msgp.WrapError(err, "Version") + return + } + // write "jid" + err = en.Append(0xa3, 0x6a, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteString(z.JobID) + if err != nil { + err = msgp.WrapError(err, "JobID") + return + } + // write "jt" + err = en.Append(0xa2, 0x6a, 0x74) + if err != nil { + return + } + err = en.WriteString(z.JobType) + if err != nil { + err = msgp.WrapError(err, "JobType") + return + } + // write "st" + err = en.Append(0xa2, 0x73, 0x74) + if err != nil { + return + } + err = en.WriteTime(z.StartTime) + if err != nil { + err = msgp.WrapError(err, "StartTime") + return + } + // write "lu" + err = en.Append(0xa2, 0x6c, 0x75) + if err != nil { + return + } + err = en.WriteTime(z.LastUpdate) + if err != nil { + err = msgp.WrapError(err, "LastUpdate") + return + } + // write "ra" + err = en.Append(0xa2, 0x72, 0x61) + if err != nil { + return + } + err = en.WriteInt(z.RetryAttempts) + if err != nil { + err = msgp.WrapError(err, "RetryAttempts") + return + } + // write "at" + err = en.Append(0xa2, 0x61, 0x74) + if err != nil { + return + } + err = en.WriteInt(z.Attempts) + if err != nil { + err = msgp.WrapError(err, "Attempts") + return + } + // write "cmp" + err = en.Append(0xa3, 0x63, 0x6d, 0x70) + if err != nil { + return + } + err = en.WriteBool(z.Complete) + if err != nil { + err = msgp.WrapError(err, "Complete") + return + } + // write "fld" + err = en.Append(0xa3, 0x66, 0x6c, 0x64) + if err != nil { + return + } + err = en.WriteBool(z.Failed) + if err != nil { + err = msgp.WrapError(err, "Failed") + return + } + // write "lbkt" + err = en.Append(0xa4, 0x6c, 0x62, 0x6b, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Bucket) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + // write "lobj" + err = en.Append(0xa4, 0x6c, 0x6f, 0x62, 0x6a) + if err != nil { + return + } + err = en.WriteString(z.Object) + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + // write "ob" + err = en.Append(0xa2, 0x6f, 0x62) + if err != nil { + return + } + err = en.WriteInt64(z.Objects) + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + // write "dm" + err = en.Append(0xa2, 0x64, 0x6d) + if err != nil { + return + } + err = en.WriteInt64(z.DeleteMarkers) + if err != nil { + err = msgp.WrapError(err, "DeleteMarkers") + return + } + // write "obf" + err = en.Append(0xa3, 0x6f, 0x62, 0x66) + if err != nil { + return + } + err = en.WriteInt64(z.ObjectsFailed) + if err != nil { + err = msgp.WrapError(err, "ObjectsFailed") + return + } + // write "dmf" + err = en.Append(0xa3, 0x64, 0x6d, 0x66) + if err != nil { + return + } + err = en.WriteInt64(z.DeleteMarkersFailed) + if err != nil { + err = msgp.WrapError(err, "DeleteMarkersFailed") + return + } + // write "bt" + err = en.Append(0xa2, 0x62, 0x74) + if err != nil { + return + } + err = en.WriteInt64(z.BytesTransferred) + if err != nil { + err = msgp.WrapError(err, "BytesTransferred") + return + } + // write "bf" + err = en.Append(0xa2, 0x62, 0x66) + if err != nil { + return + } + err = en.WriteInt64(z.BytesFailed) + if err != nil { + err = msgp.WrapError(err, "BytesFailed") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *batchJobInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 17 + // string "v" + o = append(o, 0xde, 0x0, 0x11, 0xa1, 0x76) + o = msgp.AppendInt(o, z.Version) + // string "jid" + o = append(o, 0xa3, 0x6a, 0x69, 0x64) + o = msgp.AppendString(o, z.JobID) + // string "jt" + o = append(o, 0xa2, 0x6a, 0x74) + o = msgp.AppendString(o, z.JobType) + // string "st" + o = append(o, 0xa2, 0x73, 0x74) + o = msgp.AppendTime(o, z.StartTime) + // string "lu" + o = append(o, 0xa2, 0x6c, 0x75) + o = msgp.AppendTime(o, z.LastUpdate) + // string "ra" + o = append(o, 0xa2, 0x72, 0x61) + o = msgp.AppendInt(o, z.RetryAttempts) + // string "at" + o = append(o, 0xa2, 0x61, 0x74) + o = msgp.AppendInt(o, z.Attempts) + // string "cmp" + o = append(o, 0xa3, 0x63, 0x6d, 0x70) + o = msgp.AppendBool(o, z.Complete) + // string "fld" + o = append(o, 0xa3, 0x66, 0x6c, 0x64) + o = msgp.AppendBool(o, z.Failed) + // string "lbkt" + o = append(o, 0xa4, 0x6c, 0x62, 0x6b, 0x74) + o = msgp.AppendString(o, z.Bucket) + // string "lobj" + o = append(o, 0xa4, 0x6c, 0x6f, 0x62, 0x6a) + o = msgp.AppendString(o, z.Object) + // string "ob" + o = append(o, 0xa2, 0x6f, 0x62) + o = msgp.AppendInt64(o, z.Objects) + // string "dm" + o = append(o, 0xa2, 0x64, 0x6d) + o = msgp.AppendInt64(o, z.DeleteMarkers) + // string "obf" + o = append(o, 0xa3, 0x6f, 0x62, 0x66) + o = msgp.AppendInt64(o, z.ObjectsFailed) + // string "dmf" + o = append(o, 0xa3, 0x64, 0x6d, 0x66) + o = msgp.AppendInt64(o, z.DeleteMarkersFailed) + // string "bt" + o = append(o, 0xa2, 0x62, 0x74) + o = msgp.AppendInt64(o, z.BytesTransferred) + // string "bf" + o = append(o, 0xa2, 0x62, 0x66) + o = msgp.AppendInt64(o, z.BytesFailed) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *batchJobInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "v": + z.Version, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Version") + return + } + case "jid": + z.JobID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "JobID") + return + } + case "jt": + z.JobType, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "JobType") + return + } + case "st": + z.StartTime, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "StartTime") + return + } + case "lu": + z.LastUpdate, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "LastUpdate") + return + } + case "ra": + z.RetryAttempts, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "RetryAttempts") + return + } + case "at": + z.Attempts, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Attempts") + return + } + case "cmp": + z.Complete, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Complete") + return + } + case "fld": + z.Failed, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Failed") + return + } + case "lbkt": + z.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "lobj": + z.Object, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + case "ob": + z.Objects, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + case "dm": + z.DeleteMarkers, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "DeleteMarkers") + return + } + case "obf": + z.ObjectsFailed, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjectsFailed") + return + } + case "dmf": + z.DeleteMarkersFailed, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "DeleteMarkersFailed") + return + } + case "bt": + z.BytesTransferred, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "BytesTransferred") + return + } + case "bf": + z.BytesFailed, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "BytesFailed") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *batchJobInfo) Msgsize() (s int) { + s = 3 + 2 + msgp.IntSize + 4 + msgp.StringPrefixSize + len(z.JobID) + 3 + msgp.StringPrefixSize + len(z.JobType) + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.IntSize + 3 + msgp.IntSize + 4 + msgp.BoolSize + 4 + msgp.BoolSize + 5 + msgp.StringPrefixSize + len(z.Bucket) + 5 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size + return +} diff --git a/cmd/batch-handlers_gen_test.go b/cmd/batch-handlers_gen_test.go new file mode 100644 index 00000000..be48e477 --- /dev/null +++ b/cmd/batch-handlers_gen_test.go @@ -0,0 +1,349 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalBatchJobPrefix(t *testing.T) { + v := BatchJobPrefix{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBatchJobPrefix(b *testing.B) { + v := BatchJobPrefix{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBatchJobPrefix(b *testing.B) { + v := BatchJobPrefix{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBatchJobPrefix(b *testing.B) { + v := BatchJobPrefix{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBatchJobPrefix(t *testing.T) { + v := BatchJobPrefix{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBatchJobPrefix Msgsize() is inaccurate") + } + + vn := BatchJobPrefix{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBatchJobPrefix(b *testing.B) { + v := BatchJobPrefix{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBatchJobPrefix(b *testing.B) { + v := BatchJobPrefix{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBatchJobRequest(t *testing.T) { + v := BatchJobRequest{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBatchJobRequest(b *testing.B) { + v := BatchJobRequest{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBatchJobRequest(b *testing.B) { + v := BatchJobRequest{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBatchJobRequest(b *testing.B) { + v := BatchJobRequest{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBatchJobRequest(t *testing.T) { + v := BatchJobRequest{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBatchJobRequest Msgsize() is inaccurate") + } + + vn := BatchJobRequest{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBatchJobRequest(b *testing.B) { + v := BatchJobRequest{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBatchJobRequest(b *testing.B) { + v := BatchJobRequest{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalbatchJobInfo(t *testing.T) { + v := batchJobInfo{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgbatchJobInfo(b *testing.B) { + v := batchJobInfo{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgbatchJobInfo(b *testing.B) { + v := batchJobInfo{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalbatchJobInfo(b *testing.B) { + v := batchJobInfo{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodebatchJobInfo(t *testing.T) { + v := batchJobInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodebatchJobInfo Msgsize() is inaccurate") + } + + vn := batchJobInfo{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodebatchJobInfo(b *testing.B) { + v := batchJobInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodebatchJobInfo(b *testing.B) { + v := batchJobInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/batch-handlers_test.go b/cmd/batch-handlers_test.go new file mode 100644 index 00000000..213e7670 --- /dev/null +++ b/cmd/batch-handlers_test.go @@ -0,0 +1,75 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "slices" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestBatchJobPrefix_UnmarshalYAML(t *testing.T) { + type args struct { + yamlStr string + } + type PrefixTemp struct { + Prefix BatchJobPrefix `yaml:"prefix"` + } + tests := []struct { + name string + b PrefixTemp + args args + want []string + wantErr bool + }{ + { + name: "test1", + b: PrefixTemp{}, + args: args{ + yamlStr: ` +prefix: "foo" +`, + }, + want: []string{"foo"}, + wantErr: false, + }, + { + name: "test2", + b: PrefixTemp{}, + args: args{ + yamlStr: ` +prefix: + - "foo" + - "bar" +`, + }, + want: []string{"foo", "bar"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if err := yaml.Unmarshal([]byte(tt.args.yamlStr), &tt.b); (err != nil) != tt.wantErr { + t.Errorf("UnmarshalYAML() error = %v, wantErr %v", err, tt.wantErr) + } + if !slices.Equal(tt.b.Prefix.F(), tt.want) { + t.Errorf("UnmarshalYAML() = %v, want %v", tt.b.Prefix.F(), tt.want) + } + }) + } +} diff --git a/cmd/batch-job-common-types.go b/cmd/batch-job-common-types.go new file mode 100644 index 00000000..d02a71dd --- /dev/null +++ b/cmd/batch-job-common-types.go @@ -0,0 +1,290 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "strings" + "time" + + "github.com/dustin/go-humanize" + "github.com/minio/pkg/v3/wildcard" + "gopkg.in/yaml.v3" +) + +//go:generate msgp -file $GOFILE +//msgp:ignore BatchJobYamlErr + +// BatchJobYamlErr can be used to return yaml validation errors with line, +// column information guiding user to fix syntax errors +type BatchJobYamlErr struct { + line, col int + msg string +} + +// message returns the error message excluding line, col information. +// Intended to be used in unit tests. +func (b BatchJobYamlErr) message() string { + return b.msg +} + +// Error implements Error interface +func (b BatchJobYamlErr) Error() string { + return fmt.Sprintf("%s\n Hint: error near line: %d, col: %d", b.msg, b.line, b.col) +} + +// BatchJobKV is a key-value data type which supports wildcard matching +type BatchJobKV struct { + line, col int + Key string `yaml:"key" json:"key"` + Value string `yaml:"value" json:"value"` +} + +var _ yaml.Unmarshaler = &BatchJobKV{} + +// UnmarshalYAML - BatchJobKV extends default unmarshal to extract line, col information. +func (kv *BatchJobKV) UnmarshalYAML(val *yaml.Node) error { + type jobKV BatchJobKV + var tmp jobKV + err := val.Decode(&tmp) + if err != nil { + return err + } + *kv = BatchJobKV(tmp) + kv.line, kv.col = val.Line, val.Column + return nil +} + +// Validate returns an error if key is empty +func (kv BatchJobKV) Validate() error { + if kv.Key == "" { + return BatchJobYamlErr{ + line: kv.line, + col: kv.col, + msg: "key can't be empty", + } + } + return nil +} + +// Empty indicates if kv is not set +func (kv BatchJobKV) Empty() bool { + return kv.Key == "" && kv.Value == "" +} + +// Match matches input kv with kv, value will be wildcard matched depending on the user input +func (kv BatchJobKV) Match(ikv BatchJobKV) bool { + if kv.Empty() { + return true + } + if strings.EqualFold(kv.Key, ikv.Key) { + return wildcard.Match(kv.Value, ikv.Value) + } + return false +} + +// BatchJobNotification stores notification endpoint and token information. +// Used by batch jobs to notify of their status. +type BatchJobNotification struct { + line, col int + Endpoint string `yaml:"endpoint" json:"endpoint"` + Token string `yaml:"token" json:"token"` +} + +var _ yaml.Unmarshaler = &BatchJobNotification{} + +// UnmarshalYAML - BatchJobNotification extends unmarshal to extract line, column information +func (b *BatchJobNotification) UnmarshalYAML(val *yaml.Node) error { + type notification BatchJobNotification + var tmp notification + err := val.Decode(&tmp) + if err != nil { + return err + } + + *b = BatchJobNotification(tmp) + b.line, b.col = val.Line, val.Column + return nil +} + +// BatchJobRetry stores retry configuration used in the event of failures. +type BatchJobRetry struct { + line, col int + Attempts int `yaml:"attempts" json:"attempts"` // number of retry attempts + Delay time.Duration `yaml:"delay" json:"delay"` // delay between each retries +} + +var _ yaml.Unmarshaler = &BatchJobRetry{} + +// UnmarshalYAML - BatchJobRetry extends unmarshal to extract line, column information +func (r *BatchJobRetry) UnmarshalYAML(val *yaml.Node) error { + type retry BatchJobRetry + var tmp retry + err := val.Decode(&tmp) + if err != nil { + return err + } + + *r = BatchJobRetry(tmp) + r.line, r.col = val.Line, val.Column + return nil +} + +// Validate validates input replicate retries. +func (r BatchJobRetry) Validate() error { + if r.Attempts < 0 { + return BatchJobYamlErr{ + line: r.line, + col: r.col, + msg: "Invalid arguments specified", + } + } + + if r.Delay < 0 { + return BatchJobYamlErr{ + line: r.line, + col: r.col, + msg: "Invalid arguments specified", + } + } + + return nil +} + +// # snowball based archive transfer is by default enabled when source +// # is local and target is remote which is also minio. +// snowball: +// disable: false # optionally turn-off snowball archive transfer +// batch: 100 # upto this many objects per archive +// inmemory: true # indicates if the archive must be staged locally or in-memory +// compress: true # S2/Snappy compressed archive +// smallerThan: 5MiB # create archive for all objects smaller than 5MiB +// skipErrs: false # skips any source side read() errors + +// BatchJobSnowball describes the snowball feature when replicating objects from a local source to a remote target +type BatchJobSnowball struct { + line, col int + Disable *bool `yaml:"disable" json:"disable"` + Batch *int `yaml:"batch" json:"batch"` + InMemory *bool `yaml:"inmemory" json:"inmemory"` + Compress *bool `yaml:"compress" json:"compress"` + SmallerThan *string `yaml:"smallerThan" json:"smallerThan"` + SkipErrs *bool `yaml:"skipErrs" json:"skipErrs"` +} + +var _ yaml.Unmarshaler = &BatchJobSnowball{} + +// UnmarshalYAML - BatchJobSnowball extends unmarshal to extract line, column information +func (b *BatchJobSnowball) UnmarshalYAML(val *yaml.Node) error { + type snowball BatchJobSnowball + var tmp snowball + err := val.Decode(&tmp) + if err != nil { + return err + } + + *b = BatchJobSnowball(tmp) + b.line, b.col = val.Line, val.Column + return nil +} + +// Validate the snowball parameters in the job description +func (b BatchJobSnowball) Validate() error { + if *b.Batch <= 0 { + return BatchJobYamlErr{ + line: b.line, + col: b.col, + msg: "batch number should be non positive zero", + } + } + _, err := humanize.ParseBytes(*b.SmallerThan) + if err != nil { + return BatchJobYamlErr{ + line: b.line, + col: b.col, + msg: err.Error(), + } + } + return nil +} + +// BatchJobSizeFilter supports size based filters - LesserThan and GreaterThan +type BatchJobSizeFilter struct { + line, col int + UpperBound BatchJobSize `yaml:"lessThan" json:"lessThan"` + LowerBound BatchJobSize `yaml:"greaterThan" json:"greaterThan"` +} + +// UnmarshalYAML - BatchJobSizeFilter extends unmarshal to extract line, column information +func (sf *BatchJobSizeFilter) UnmarshalYAML(val *yaml.Node) error { + type sizeFilter BatchJobSizeFilter + var tmp sizeFilter + err := val.Decode(&tmp) + if err != nil { + return err + } + + *sf = BatchJobSizeFilter(tmp) + sf.line, sf.col = val.Line, val.Column + return nil +} + +// InRange returns true in the following cases and false otherwise, +// - sf.LowerBound < sz, when sf.LowerBound alone is specified +// - sz < sf.UpperBound, when sf.UpperBound alone is specified +// - sf.LowerBound < sz < sf.UpperBound when both are specified, +func (sf BatchJobSizeFilter) InRange(sz int64) bool { + if sf.UpperBound > 0 && sz > int64(sf.UpperBound) { + return false + } + + if sf.LowerBound > 0 && sz < int64(sf.LowerBound) { + return false + } + return true +} + +// Validate checks if sf is a valid batch-job size filter +func (sf BatchJobSizeFilter) Validate() error { + if sf.LowerBound > 0 && sf.UpperBound > 0 && sf.LowerBound >= sf.UpperBound { + return BatchJobYamlErr{ + line: sf.line, + col: sf.col, + msg: "invalid batch-job size filter", + } + } + return nil +} + +// BatchJobSize supports humanized byte values in yaml files type BatchJobSize uint64 +type BatchJobSize int64 + +// UnmarshalYAML to parse humanized byte values +func (s *BatchJobSize) UnmarshalYAML(unmarshal func(any) error) error { + var batchExpireSz string + err := unmarshal(&batchExpireSz) + if err != nil { + return err + } + sz, err := humanize.ParseBytes(batchExpireSz) + if err != nil { + return err + } + *s = BatchJobSize(sz) + return nil +} diff --git a/cmd/batch-job-common-types_gen.go b/cmd/batch-job-common-types_gen.go new file mode 100644 index 00000000..12edddf4 --- /dev/null +++ b/cmd/batch-job-common-types_gen.go @@ -0,0 +1,1054 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *BatchJobKV) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Key": + z.Key, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Key") + return + } + case "Value": + z.Value, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Value") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z BatchJobKV) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "Key" + err = en.Append(0x82, 0xa3, 0x4b, 0x65, 0x79) + if err != nil { + return + } + err = en.WriteString(z.Key) + if err != nil { + err = msgp.WrapError(err, "Key") + return + } + // write "Value" + err = en.Append(0xa5, 0x56, 0x61, 0x6c, 0x75, 0x65) + if err != nil { + return + } + err = en.WriteString(z.Value) + if err != nil { + err = msgp.WrapError(err, "Value") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z BatchJobKV) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "Key" + o = append(o, 0x82, 0xa3, 0x4b, 0x65, 0x79) + o = msgp.AppendString(o, z.Key) + // string "Value" + o = append(o, 0xa5, 0x56, 0x61, 0x6c, 0x75, 0x65) + o = msgp.AppendString(o, z.Value) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchJobKV) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Key": + z.Key, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Key") + return + } + case "Value": + z.Value, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Value") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z BatchJobKV) Msgsize() (s int) { + s = 1 + 4 + msgp.StringPrefixSize + len(z.Key) + 6 + msgp.StringPrefixSize + len(z.Value) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *BatchJobNotification) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Endpoint": + z.Endpoint, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + case "Token": + z.Token, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Token") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z BatchJobNotification) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "Endpoint" + err = en.Append(0x82, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Endpoint) + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + // write "Token" + err = en.Append(0xa5, 0x54, 0x6f, 0x6b, 0x65, 0x6e) + if err != nil { + return + } + err = en.WriteString(z.Token) + if err != nil { + err = msgp.WrapError(err, "Token") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z BatchJobNotification) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "Endpoint" + o = append(o, 0x82, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74) + o = msgp.AppendString(o, z.Endpoint) + // string "Token" + o = append(o, 0xa5, 0x54, 0x6f, 0x6b, 0x65, 0x6e) + o = msgp.AppendString(o, z.Token) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchJobNotification) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Endpoint": + z.Endpoint, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + case "Token": + z.Token, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Token") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z BatchJobNotification) Msgsize() (s int) { + s = 1 + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 6 + msgp.StringPrefixSize + len(z.Token) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *BatchJobRetry) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Attempts": + z.Attempts, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Attempts") + return + } + case "Delay": + z.Delay, err = dc.ReadDuration() + if err != nil { + err = msgp.WrapError(err, "Delay") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z BatchJobRetry) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "Attempts" + err = en.Append(0x82, 0xa8, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteInt(z.Attempts) + if err != nil { + err = msgp.WrapError(err, "Attempts") + return + } + // write "Delay" + err = en.Append(0xa5, 0x44, 0x65, 0x6c, 0x61, 0x79) + if err != nil { + return + } + err = en.WriteDuration(z.Delay) + if err != nil { + err = msgp.WrapError(err, "Delay") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z BatchJobRetry) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "Attempts" + o = append(o, 0x82, 0xa8, 0x41, 0x74, 0x74, 0x65, 0x6d, 0x70, 0x74, 0x73) + o = msgp.AppendInt(o, z.Attempts) + // string "Delay" + o = append(o, 0xa5, 0x44, 0x65, 0x6c, 0x61, 0x79) + o = msgp.AppendDuration(o, z.Delay) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchJobRetry) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Attempts": + z.Attempts, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Attempts") + return + } + case "Delay": + z.Delay, bts, err = msgp.ReadDurationBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Delay") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z BatchJobRetry) Msgsize() (s int) { + s = 1 + 9 + msgp.IntSize + 6 + msgp.DurationSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *BatchJobSize) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 int64 + zb0001, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = BatchJobSize(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z BatchJobSize) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteInt64(int64(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z BatchJobSize) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendInt64(o, int64(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchJobSize) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 int64 + zb0001, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = BatchJobSize(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z BatchJobSize) Msgsize() (s int) { + s = msgp.Int64Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *BatchJobSizeFilter) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "UpperBound": + { + var zb0002 int64 + zb0002, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "UpperBound") + return + } + z.UpperBound = BatchJobSize(zb0002) + } + case "LowerBound": + { + var zb0003 int64 + zb0003, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "LowerBound") + return + } + z.LowerBound = BatchJobSize(zb0003) + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z BatchJobSizeFilter) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "UpperBound" + err = en.Append(0x82, 0xaa, 0x55, 0x70, 0x70, 0x65, 0x72, 0x42, 0x6f, 0x75, 0x6e, 0x64) + if err != nil { + return + } + err = en.WriteInt64(int64(z.UpperBound)) + if err != nil { + err = msgp.WrapError(err, "UpperBound") + return + } + // write "LowerBound" + err = en.Append(0xaa, 0x4c, 0x6f, 0x77, 0x65, 0x72, 0x42, 0x6f, 0x75, 0x6e, 0x64) + if err != nil { + return + } + err = en.WriteInt64(int64(z.LowerBound)) + if err != nil { + err = msgp.WrapError(err, "LowerBound") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z BatchJobSizeFilter) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "UpperBound" + o = append(o, 0x82, 0xaa, 0x55, 0x70, 0x70, 0x65, 0x72, 0x42, 0x6f, 0x75, 0x6e, 0x64) + o = msgp.AppendInt64(o, int64(z.UpperBound)) + // string "LowerBound" + o = append(o, 0xaa, 0x4c, 0x6f, 0x77, 0x65, 0x72, 0x42, 0x6f, 0x75, 0x6e, 0x64) + o = msgp.AppendInt64(o, int64(z.LowerBound)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchJobSizeFilter) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "UpperBound": + { + var zb0002 int64 + zb0002, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "UpperBound") + return + } + z.UpperBound = BatchJobSize(zb0002) + } + case "LowerBound": + { + var zb0003 int64 + zb0003, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "LowerBound") + return + } + z.LowerBound = BatchJobSize(zb0003) + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z BatchJobSizeFilter) Msgsize() (s int) { + s = 1 + 11 + msgp.Int64Size + 11 + msgp.Int64Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *BatchJobSnowball) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Disable": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "Disable") + return + } + z.Disable = nil + } else { + if z.Disable == nil { + z.Disable = new(bool) + } + *z.Disable, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Disable") + return + } + } + case "Batch": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "Batch") + return + } + z.Batch = nil + } else { + if z.Batch == nil { + z.Batch = new(int) + } + *z.Batch, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Batch") + return + } + } + case "InMemory": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "InMemory") + return + } + z.InMemory = nil + } else { + if z.InMemory == nil { + z.InMemory = new(bool) + } + *z.InMemory, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "InMemory") + return + } + } + case "Compress": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "Compress") + return + } + z.Compress = nil + } else { + if z.Compress == nil { + z.Compress = new(bool) + } + *z.Compress, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Compress") + return + } + } + case "SmallerThan": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "SmallerThan") + return + } + z.SmallerThan = nil + } else { + if z.SmallerThan == nil { + z.SmallerThan = new(string) + } + *z.SmallerThan, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "SmallerThan") + return + } + } + case "SkipErrs": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "SkipErrs") + return + } + z.SkipErrs = nil + } else { + if z.SkipErrs == nil { + z.SkipErrs = new(bool) + } + *z.SkipErrs, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "SkipErrs") + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *BatchJobSnowball) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 6 + // write "Disable" + err = en.Append(0x86, 0xa7, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65) + if err != nil { + return + } + if z.Disable == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = en.WriteBool(*z.Disable) + if err != nil { + err = msgp.WrapError(err, "Disable") + return + } + } + // write "Batch" + err = en.Append(0xa5, 0x42, 0x61, 0x74, 0x63, 0x68) + if err != nil { + return + } + if z.Batch == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = en.WriteInt(*z.Batch) + if err != nil { + err = msgp.WrapError(err, "Batch") + return + } + } + // write "InMemory" + err = en.Append(0xa8, 0x49, 0x6e, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79) + if err != nil { + return + } + if z.InMemory == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = en.WriteBool(*z.InMemory) + if err != nil { + err = msgp.WrapError(err, "InMemory") + return + } + } + // write "Compress" + err = en.Append(0xa8, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73) + if err != nil { + return + } + if z.Compress == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = en.WriteBool(*z.Compress) + if err != nil { + err = msgp.WrapError(err, "Compress") + return + } + } + // write "SmallerThan" + err = en.Append(0xab, 0x53, 0x6d, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e) + if err != nil { + return + } + if z.SmallerThan == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = en.WriteString(*z.SmallerThan) + if err != nil { + err = msgp.WrapError(err, "SmallerThan") + return + } + } + // write "SkipErrs" + err = en.Append(0xa8, 0x53, 0x6b, 0x69, 0x70, 0x45, 0x72, 0x72, 0x73) + if err != nil { + return + } + if z.SkipErrs == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = en.WriteBool(*z.SkipErrs) + if err != nil { + err = msgp.WrapError(err, "SkipErrs") + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *BatchJobSnowball) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 6 + // string "Disable" + o = append(o, 0x86, 0xa7, 0x44, 0x69, 0x73, 0x61, 0x62, 0x6c, 0x65) + if z.Disable == nil { + o = msgp.AppendNil(o) + } else { + o = msgp.AppendBool(o, *z.Disable) + } + // string "Batch" + o = append(o, 0xa5, 0x42, 0x61, 0x74, 0x63, 0x68) + if z.Batch == nil { + o = msgp.AppendNil(o) + } else { + o = msgp.AppendInt(o, *z.Batch) + } + // string "InMemory" + o = append(o, 0xa8, 0x49, 0x6e, 0x4d, 0x65, 0x6d, 0x6f, 0x72, 0x79) + if z.InMemory == nil { + o = msgp.AppendNil(o) + } else { + o = msgp.AppendBool(o, *z.InMemory) + } + // string "Compress" + o = append(o, 0xa8, 0x43, 0x6f, 0x6d, 0x70, 0x72, 0x65, 0x73, 0x73) + if z.Compress == nil { + o = msgp.AppendNil(o) + } else { + o = msgp.AppendBool(o, *z.Compress) + } + // string "SmallerThan" + o = append(o, 0xab, 0x53, 0x6d, 0x61, 0x6c, 0x6c, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e) + if z.SmallerThan == nil { + o = msgp.AppendNil(o) + } else { + o = msgp.AppendString(o, *z.SmallerThan) + } + // string "SkipErrs" + o = append(o, 0xa8, 0x53, 0x6b, 0x69, 0x70, 0x45, 0x72, 0x72, 0x73) + if z.SkipErrs == nil { + o = msgp.AppendNil(o) + } else { + o = msgp.AppendBool(o, *z.SkipErrs) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchJobSnowball) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Disable": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.Disable = nil + } else { + if z.Disable == nil { + z.Disable = new(bool) + } + *z.Disable, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Disable") + return + } + } + case "Batch": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.Batch = nil + } else { + if z.Batch == nil { + z.Batch = new(int) + } + *z.Batch, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Batch") + return + } + } + case "InMemory": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.InMemory = nil + } else { + if z.InMemory == nil { + z.InMemory = new(bool) + } + *z.InMemory, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "InMemory") + return + } + } + case "Compress": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.Compress = nil + } else { + if z.Compress == nil { + z.Compress = new(bool) + } + *z.Compress, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Compress") + return + } + } + case "SmallerThan": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.SmallerThan = nil + } else { + if z.SmallerThan == nil { + z.SmallerThan = new(string) + } + *z.SmallerThan, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SmallerThan") + return + } + } + case "SkipErrs": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.SkipErrs = nil + } else { + if z.SkipErrs == nil { + z.SkipErrs = new(bool) + } + *z.SkipErrs, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SkipErrs") + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *BatchJobSnowball) Msgsize() (s int) { + s = 1 + 8 + if z.Disable == nil { + s += msgp.NilSize + } else { + s += msgp.BoolSize + } + s += 6 + if z.Batch == nil { + s += msgp.NilSize + } else { + s += msgp.IntSize + } + s += 9 + if z.InMemory == nil { + s += msgp.NilSize + } else { + s += msgp.BoolSize + } + s += 9 + if z.Compress == nil { + s += msgp.NilSize + } else { + s += msgp.BoolSize + } + s += 12 + if z.SmallerThan == nil { + s += msgp.NilSize + } else { + s += msgp.StringPrefixSize + len(*z.SmallerThan) + } + s += 9 + if z.SkipErrs == nil { + s += msgp.NilSize + } else { + s += msgp.BoolSize + } + return +} diff --git a/cmd/batch-job-common-types_gen_test.go b/cmd/batch-job-common-types_gen_test.go new file mode 100644 index 00000000..3e0684a8 --- /dev/null +++ b/cmd/batch-job-common-types_gen_test.go @@ -0,0 +1,575 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalBatchJobKV(t *testing.T) { + v := BatchJobKV{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBatchJobKV(b *testing.B) { + v := BatchJobKV{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBatchJobKV(b *testing.B) { + v := BatchJobKV{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBatchJobKV(b *testing.B) { + v := BatchJobKV{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBatchJobKV(t *testing.T) { + v := BatchJobKV{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBatchJobKV Msgsize() is inaccurate") + } + + vn := BatchJobKV{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBatchJobKV(b *testing.B) { + v := BatchJobKV{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBatchJobKV(b *testing.B) { + v := BatchJobKV{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBatchJobNotification(t *testing.T) { + v := BatchJobNotification{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBatchJobNotification(b *testing.B) { + v := BatchJobNotification{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBatchJobNotification(b *testing.B) { + v := BatchJobNotification{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBatchJobNotification(b *testing.B) { + v := BatchJobNotification{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBatchJobNotification(t *testing.T) { + v := BatchJobNotification{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBatchJobNotification Msgsize() is inaccurate") + } + + vn := BatchJobNotification{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBatchJobNotification(b *testing.B) { + v := BatchJobNotification{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBatchJobNotification(b *testing.B) { + v := BatchJobNotification{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBatchJobRetry(t *testing.T) { + v := BatchJobRetry{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBatchJobRetry(b *testing.B) { + v := BatchJobRetry{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBatchJobRetry(b *testing.B) { + v := BatchJobRetry{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBatchJobRetry(b *testing.B) { + v := BatchJobRetry{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBatchJobRetry(t *testing.T) { + v := BatchJobRetry{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBatchJobRetry Msgsize() is inaccurate") + } + + vn := BatchJobRetry{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBatchJobRetry(b *testing.B) { + v := BatchJobRetry{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBatchJobRetry(b *testing.B) { + v := BatchJobRetry{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBatchJobSizeFilter(t *testing.T) { + v := BatchJobSizeFilter{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBatchJobSizeFilter(b *testing.B) { + v := BatchJobSizeFilter{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBatchJobSizeFilter(b *testing.B) { + v := BatchJobSizeFilter{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBatchJobSizeFilter(b *testing.B) { + v := BatchJobSizeFilter{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBatchJobSizeFilter(t *testing.T) { + v := BatchJobSizeFilter{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBatchJobSizeFilter Msgsize() is inaccurate") + } + + vn := BatchJobSizeFilter{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBatchJobSizeFilter(b *testing.B) { + v := BatchJobSizeFilter{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBatchJobSizeFilter(b *testing.B) { + v := BatchJobSizeFilter{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBatchJobSnowball(t *testing.T) { + v := BatchJobSnowball{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBatchJobSnowball(b *testing.B) { + v := BatchJobSnowball{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBatchJobSnowball(b *testing.B) { + v := BatchJobSnowball{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBatchJobSnowball(b *testing.B) { + v := BatchJobSnowball{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBatchJobSnowball(t *testing.T) { + v := BatchJobSnowball{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBatchJobSnowball Msgsize() is inaccurate") + } + + vn := BatchJobSnowball{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBatchJobSnowball(b *testing.B) { + v := BatchJobSnowball{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBatchJobSnowball(b *testing.B) { + v := BatchJobSnowball{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/batch-job-common-types_test.go b/cmd/batch-job-common-types_test.go new file mode 100644 index 00000000..5281c121 --- /dev/null +++ b/cmd/batch-job-common-types_test.go @@ -0,0 +1,148 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "testing" +) + +func TestBatchJobSizeInRange(t *testing.T) { + tests := []struct { + objSize int64 + sizeFilter BatchJobSizeFilter + want bool + }{ + { + // 1Mib < 2Mib < 10MiB -> in range + objSize: 2 << 20, + sizeFilter: BatchJobSizeFilter{ + UpperBound: 10 << 20, + LowerBound: 1 << 20, + }, + want: true, + }, + { + // 2KiB < 1 MiB -> out of range from left + objSize: 2 << 10, + sizeFilter: BatchJobSizeFilter{ + UpperBound: 10 << 20, + LowerBound: 1 << 20, + }, + want: false, + }, + { + // 11MiB > 10 MiB -> out of range from right + objSize: 11 << 20, + sizeFilter: BatchJobSizeFilter{ + UpperBound: 10 << 20, + LowerBound: 1 << 20, + }, + want: false, + }, + { + // 2MiB < 10MiB -> in range + objSize: 2 << 20, + sizeFilter: BatchJobSizeFilter{ + UpperBound: 10 << 20, + }, + want: true, + }, + { + // 2MiB > 1MiB -> in range + objSize: 2 << 20, + sizeFilter: BatchJobSizeFilter{ + LowerBound: 1 << 20, + }, + want: true, + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("test-%d", i+1), func(t *testing.T) { + if got := test.sizeFilter.InRange(test.objSize); got != test.want { + t.Fatalf("Expected %v but got %v", test.want, got) + } + }) + } +} + +func TestBatchJobSizeValidate(t *testing.T) { + errInvalidBatchJobSizeFilter := BatchJobYamlErr{ + msg: "invalid batch-job size filter", + } + + tests := []struct { + sizeFilter BatchJobSizeFilter + err error + }{ + { + // Unspecified size filter is a valid filter + sizeFilter: BatchJobSizeFilter{ + UpperBound: 0, + LowerBound: 0, + }, + err: nil, + }, + { + sizeFilter: BatchJobSizeFilter{ + UpperBound: 0, + LowerBound: 1 << 20, + }, + err: nil, + }, + { + sizeFilter: BatchJobSizeFilter{ + UpperBound: 10 << 20, + LowerBound: 0, + }, + err: nil, + }, + { + // LowerBound > UpperBound -> empty range + sizeFilter: BatchJobSizeFilter{ + UpperBound: 1 << 20, + LowerBound: 10 << 20, + }, + err: errInvalidBatchJobSizeFilter, + }, + { + // LowerBound == UpperBound -> empty range + sizeFilter: BatchJobSizeFilter{ + UpperBound: 1 << 20, + LowerBound: 1 << 20, + }, + err: errInvalidBatchJobSizeFilter, + }, + } + for i, test := range tests { + t.Run(fmt.Sprintf("test-%d", i+1), func(t *testing.T) { + err := test.sizeFilter.Validate() + if err != nil { + gotErr := err.(BatchJobYamlErr) + testErr := test.err.(BatchJobYamlErr) + if gotErr.message() != testErr.message() { + t.Fatalf("Expected %v but got %v", test.err, err) + } + } + if err == nil && test.err != nil { + t.Fatalf("Expected %v but got nil", test.err) + } + }) + } +} diff --git a/cmd/batch-replicate.go b/cmd/batch-replicate.go new file mode 100644 index 00000000..37a1834d --- /dev/null +++ b/cmd/batch-replicate.go @@ -0,0 +1,184 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "time" + + miniogo "github.com/minio/minio-go/v7" + "github.com/minio/minio/internal/auth" + "github.com/minio/pkg/v3/xtime" +) + +//go:generate msgp -file $GOFILE + +// replicate: +// # source of the objects to be replicated +// source: +// type: "minio" +// bucket: "testbucket" +// prefix: "spark/" +// +// # optional flags based filtering criteria +// # for source objects +// flags: +// filter: +// newerThan: "7d" +// olderThan: "7d" +// createdAfter: "date" +// createdBefore: "date" +// tags: +// - key: "name" +// value: "value*" +// metadata: +// - key: "content-type" +// value: "image/*" +// notify: +// endpoint: "https://splunk-hec.dev.com" +// token: "Splunk ..." # e.g. "Bearer token" +// +// # target where the objects must be replicated +// target: +// type: "minio" +// bucket: "testbucket1" +// endpoint: "https://play.min.io" +// path: "on" +// credentials: +// accessKey: "minioadmin" +// secretKey: "minioadmin" +// sessionToken: "" + +// BatchReplicateFilter holds all the filters currently supported for batch replication +type BatchReplicateFilter struct { + NewerThan xtime.Duration `yaml:"newerThan,omitempty" json:"newerThan"` + OlderThan xtime.Duration `yaml:"olderThan,omitempty" json:"olderThan"` + CreatedAfter time.Time `yaml:"createdAfter,omitempty" json:"createdAfter"` + CreatedBefore time.Time `yaml:"createdBefore,omitempty" json:"createdBefore"` + Tags []BatchJobKV `yaml:"tags,omitempty" json:"tags"` + Metadata []BatchJobKV `yaml:"metadata,omitempty" json:"metadata"` +} + +// BatchJobReplicateFlags various configurations for replication job definition currently includes +// - filter +// - notify +// - retry +type BatchJobReplicateFlags struct { + Filter BatchReplicateFilter `yaml:"filter" json:"filter"` + Notify BatchJobNotification `yaml:"notify" json:"notify"` + Retry BatchJobRetry `yaml:"retry" json:"retry"` +} + +// BatchJobReplicateResourceType defines the type of batch jobs +type BatchJobReplicateResourceType string + +// Validate validates if the replicate resource type is recognized and supported +func (t BatchJobReplicateResourceType) Validate() error { + switch t { + case BatchJobReplicateResourceMinIO: + case BatchJobReplicateResourceS3: + default: + return errInvalidArgument + } + return nil +} + +func (t BatchJobReplicateResourceType) isMinio() bool { + return t == BatchJobReplicateResourceMinIO +} + +// Different types of batch jobs.. +const ( + BatchJobReplicateResourceMinIO BatchJobReplicateResourceType = "minio" + BatchJobReplicateResourceS3 BatchJobReplicateResourceType = "s3" + + // add future targets +) + +// BatchJobReplicateCredentials access credentials for batch replication it may +// be either for target or source. +type BatchJobReplicateCredentials struct { + AccessKey string `xml:"AccessKeyId" json:"accessKey,omitempty" yaml:"accessKey"` + SecretKey string `xml:"SecretAccessKey" json:"secretKey,omitempty" yaml:"secretKey"` + SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty" yaml:"sessionToken"` +} + +// Empty indicates if credentials are not set +func (c BatchJobReplicateCredentials) Empty() bool { + return c.AccessKey == "" && c.SecretKey == "" && c.SessionToken == "" +} + +// Validate validates if credentials are valid +func (c BatchJobReplicateCredentials) Validate() error { + if !auth.IsAccessKeyValid(c.AccessKey) || !auth.IsSecretKeyValid(c.SecretKey) { + return errInvalidArgument + } + return nil +} + +// BatchJobReplicateTarget describes target element of the replication job that receives +// the filtered data from source +type BatchJobReplicateTarget struct { + Type BatchJobReplicateResourceType `yaml:"type" json:"type"` + Bucket string `yaml:"bucket" json:"bucket"` + Prefix string `yaml:"prefix" json:"prefix"` + Endpoint string `yaml:"endpoint" json:"endpoint"` + Path string `yaml:"path" json:"path"` + Creds BatchJobReplicateCredentials `yaml:"credentials" json:"credentials"` +} + +// ValidPath returns true if path is valid +func (t BatchJobReplicateTarget) ValidPath() bool { + return t.Path == "on" || t.Path == "off" || t.Path == "auto" || t.Path == "" +} + +// BatchJobReplicateSource describes source element of the replication job that is +// the source of the data for the target +type BatchJobReplicateSource struct { + Type BatchJobReplicateResourceType `yaml:"type" json:"type"` + Bucket string `yaml:"bucket" json:"bucket"` + Prefix BatchJobPrefix `yaml:"prefix" json:"prefix"` + Endpoint string `yaml:"endpoint" json:"endpoint"` + Path string `yaml:"path" json:"path"` + Creds BatchJobReplicateCredentials `yaml:"credentials" json:"credentials"` + Snowball BatchJobSnowball `yaml:"snowball" json:"snowball"` +} + +// ValidPath returns true if path is valid +func (s BatchJobReplicateSource) ValidPath() bool { + switch s.Path { + case "on", "off", "auto", "": + return true + default: + return false + } +} + +// BatchJobReplicateV1 v1 of batch job replication +type BatchJobReplicateV1 struct { + APIVersion string `yaml:"apiVersion" json:"apiVersion"` + Flags BatchJobReplicateFlags `yaml:"flags" json:"flags"` + Target BatchJobReplicateTarget `yaml:"target" json:"target"` + Source BatchJobReplicateSource `yaml:"source" json:"source"` + + clnt *miniogo.Core `msg:"-"` +} + +// RemoteToLocal returns true if source is remote and target is local +func (r BatchJobReplicateV1) RemoteToLocal() bool { + return !r.Source.Creds.Empty() +} diff --git a/cmd/batch-replicate_gen.go b/cmd/batch-replicate_gen.go new file mode 100644 index 00000000..16f3eed4 --- /dev/null +++ b/cmd/batch-replicate_gen.go @@ -0,0 +1,1718 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *BatchJobReplicateCredentials) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "AccessKey": + z.AccessKey, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "AccessKey") + return + } + case "SecretKey": + z.SecretKey, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "SecretKey") + return + } + case "SessionToken": + z.SessionToken, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "SessionToken") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z BatchJobReplicateCredentials) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "AccessKey" + err = en.Append(0x83, 0xa9, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79) + if err != nil { + return + } + err = en.WriteString(z.AccessKey) + if err != nil { + err = msgp.WrapError(err, "AccessKey") + return + } + // write "SecretKey" + err = en.Append(0xa9, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79) + if err != nil { + return + } + err = en.WriteString(z.SecretKey) + if err != nil { + err = msgp.WrapError(err, "SecretKey") + return + } + // write "SessionToken" + err = en.Append(0xac, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e) + if err != nil { + return + } + err = en.WriteString(z.SessionToken) + if err != nil { + err = msgp.WrapError(err, "SessionToken") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z BatchJobReplicateCredentials) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "AccessKey" + o = append(o, 0x83, 0xa9, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79) + o = msgp.AppendString(o, z.AccessKey) + // string "SecretKey" + o = append(o, 0xa9, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79) + o = msgp.AppendString(o, z.SecretKey) + // string "SessionToken" + o = append(o, 0xac, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e) + o = msgp.AppendString(o, z.SessionToken) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchJobReplicateCredentials) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "AccessKey": + z.AccessKey, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AccessKey") + return + } + case "SecretKey": + z.SecretKey, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SecretKey") + return + } + case "SessionToken": + z.SessionToken, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SessionToken") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z BatchJobReplicateCredentials) Msgsize() (s int) { + s = 1 + 10 + msgp.StringPrefixSize + len(z.AccessKey) + 10 + msgp.StringPrefixSize + len(z.SecretKey) + 13 + msgp.StringPrefixSize + len(z.SessionToken) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *BatchJobReplicateFlags) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Filter": + err = z.Filter.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Filter") + return + } + case "Notify": + err = z.Notify.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Notify") + return + } + case "Retry": + err = z.Retry.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Retry") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *BatchJobReplicateFlags) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "Filter" + err = en.Append(0x83, 0xa6, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72) + if err != nil { + return + } + err = z.Filter.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Filter") + return + } + // write "Notify" + err = en.Append(0xa6, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79) + if err != nil { + return + } + err = z.Notify.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Notify") + return + } + // write "Retry" + err = en.Append(0xa5, 0x52, 0x65, 0x74, 0x72, 0x79) + if err != nil { + return + } + err = z.Retry.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Retry") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *BatchJobReplicateFlags) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "Filter" + o = append(o, 0x83, 0xa6, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72) + o, err = z.Filter.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Filter") + return + } + // string "Notify" + o = append(o, 0xa6, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79) + o, err = z.Notify.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Notify") + return + } + // string "Retry" + o = append(o, 0xa5, 0x52, 0x65, 0x74, 0x72, 0x79) + o, err = z.Retry.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Retry") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchJobReplicateFlags) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Filter": + bts, err = z.Filter.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Filter") + return + } + case "Notify": + bts, err = z.Notify.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Notify") + return + } + case "Retry": + bts, err = z.Retry.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Retry") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *BatchJobReplicateFlags) Msgsize() (s int) { + s = 1 + 7 + z.Filter.Msgsize() + 7 + z.Notify.Msgsize() + 6 + z.Retry.Msgsize() + return +} + +// DecodeMsg implements msgp.Decodable +func (z *BatchJobReplicateResourceType) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 string + zb0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = BatchJobReplicateResourceType(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z BatchJobReplicateResourceType) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteString(string(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z BatchJobReplicateResourceType) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendString(o, string(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchJobReplicateResourceType) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 string + zb0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = BatchJobReplicateResourceType(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z BatchJobReplicateResourceType) Msgsize() (s int) { + s = msgp.StringPrefixSize + len(string(z)) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *BatchJobReplicateSource) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Type": + { + var zb0002 string + zb0002, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + z.Type = BatchJobReplicateResourceType(zb0002) + } + case "Bucket": + z.Bucket, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "Prefix": + err = z.Prefix.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + case "Endpoint": + z.Endpoint, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + case "Path": + z.Path, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Path") + return + } + case "Creds": + var zb0003 uint32 + zb0003, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Creds") + return + } + for zb0003 > 0 { + zb0003-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "Creds") + return + } + switch msgp.UnsafeString(field) { + case "AccessKey": + z.Creds.AccessKey, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Creds", "AccessKey") + return + } + case "SecretKey": + z.Creds.SecretKey, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Creds", "SecretKey") + return + } + case "SessionToken": + z.Creds.SessionToken, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Creds", "SessionToken") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "Creds") + return + } + } + } + case "Snowball": + err = z.Snowball.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Snowball") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *BatchJobReplicateSource) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 7 + // write "Type" + err = en.Append(0x87, 0xa4, 0x54, 0x79, 0x70, 0x65) + if err != nil { + return + } + err = en.WriteString(string(z.Type)) + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + // write "Bucket" + err = en.Append(0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Bucket) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + // write "Prefix" + err = en.Append(0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78) + if err != nil { + return + } + err = z.Prefix.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + // write "Endpoint" + err = en.Append(0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Endpoint) + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + // write "Path" + err = en.Append(0xa4, 0x50, 0x61, 0x74, 0x68) + if err != nil { + return + } + err = en.WriteString(z.Path) + if err != nil { + err = msgp.WrapError(err, "Path") + return + } + // write "Creds" + err = en.Append(0xa5, 0x43, 0x72, 0x65, 0x64, 0x73) + if err != nil { + return + } + // map header, size 3 + // write "AccessKey" + err = en.Append(0x83, 0xa9, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79) + if err != nil { + return + } + err = en.WriteString(z.Creds.AccessKey) + if err != nil { + err = msgp.WrapError(err, "Creds", "AccessKey") + return + } + // write "SecretKey" + err = en.Append(0xa9, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79) + if err != nil { + return + } + err = en.WriteString(z.Creds.SecretKey) + if err != nil { + err = msgp.WrapError(err, "Creds", "SecretKey") + return + } + // write "SessionToken" + err = en.Append(0xac, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e) + if err != nil { + return + } + err = en.WriteString(z.Creds.SessionToken) + if err != nil { + err = msgp.WrapError(err, "Creds", "SessionToken") + return + } + // write "Snowball" + err = en.Append(0xa8, 0x53, 0x6e, 0x6f, 0x77, 0x62, 0x61, 0x6c, 0x6c) + if err != nil { + return + } + err = z.Snowball.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Snowball") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *BatchJobReplicateSource) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 7 + // string "Type" + o = append(o, 0x87, 0xa4, 0x54, 0x79, 0x70, 0x65) + o = msgp.AppendString(o, string(z.Type)) + // string "Bucket" + o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + o = msgp.AppendString(o, z.Bucket) + // string "Prefix" + o = append(o, 0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78) + o, err = z.Prefix.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + // string "Endpoint" + o = append(o, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74) + o = msgp.AppendString(o, z.Endpoint) + // string "Path" + o = append(o, 0xa4, 0x50, 0x61, 0x74, 0x68) + o = msgp.AppendString(o, z.Path) + // string "Creds" + o = append(o, 0xa5, 0x43, 0x72, 0x65, 0x64, 0x73) + // map header, size 3 + // string "AccessKey" + o = append(o, 0x83, 0xa9, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79) + o = msgp.AppendString(o, z.Creds.AccessKey) + // string "SecretKey" + o = append(o, 0xa9, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79) + o = msgp.AppendString(o, z.Creds.SecretKey) + // string "SessionToken" + o = append(o, 0xac, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e) + o = msgp.AppendString(o, z.Creds.SessionToken) + // string "Snowball" + o = append(o, 0xa8, 0x53, 0x6e, 0x6f, 0x77, 0x62, 0x61, 0x6c, 0x6c) + o, err = z.Snowball.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Snowball") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchJobReplicateSource) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Type": + { + var zb0002 string + zb0002, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + z.Type = BatchJobReplicateResourceType(zb0002) + } + case "Bucket": + z.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "Prefix": + bts, err = z.Prefix.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + case "Endpoint": + z.Endpoint, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + case "Path": + z.Path, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Path") + return + } + case "Creds": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Creds") + return + } + for zb0003 > 0 { + zb0003-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "Creds") + return + } + switch msgp.UnsafeString(field) { + case "AccessKey": + z.Creds.AccessKey, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Creds", "AccessKey") + return + } + case "SecretKey": + z.Creds.SecretKey, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Creds", "SecretKey") + return + } + case "SessionToken": + z.Creds.SessionToken, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Creds", "SessionToken") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "Creds") + return + } + } + } + case "Snowball": + bts, err = z.Snowball.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Snowball") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *BatchJobReplicateSource) Msgsize() (s int) { + s = 1 + 5 + msgp.StringPrefixSize + len(string(z.Type)) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + z.Prefix.Msgsize() + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 5 + msgp.StringPrefixSize + len(z.Path) + 6 + 1 + 10 + msgp.StringPrefixSize + len(z.Creds.AccessKey) + 10 + msgp.StringPrefixSize + len(z.Creds.SecretKey) + 13 + msgp.StringPrefixSize + len(z.Creds.SessionToken) + 9 + z.Snowball.Msgsize() + return +} + +// DecodeMsg implements msgp.Decodable +func (z *BatchJobReplicateTarget) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Type": + { + var zb0002 string + zb0002, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + z.Type = BatchJobReplicateResourceType(zb0002) + } + case "Bucket": + z.Bucket, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "Prefix": + z.Prefix, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + case "Endpoint": + z.Endpoint, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + case "Path": + z.Path, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Path") + return + } + case "Creds": + var zb0003 uint32 + zb0003, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Creds") + return + } + for zb0003 > 0 { + zb0003-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "Creds") + return + } + switch msgp.UnsafeString(field) { + case "AccessKey": + z.Creds.AccessKey, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Creds", "AccessKey") + return + } + case "SecretKey": + z.Creds.SecretKey, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Creds", "SecretKey") + return + } + case "SessionToken": + z.Creds.SessionToken, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Creds", "SessionToken") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "Creds") + return + } + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *BatchJobReplicateTarget) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 6 + // write "Type" + err = en.Append(0x86, 0xa4, 0x54, 0x79, 0x70, 0x65) + if err != nil { + return + } + err = en.WriteString(string(z.Type)) + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + // write "Bucket" + err = en.Append(0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Bucket) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + // write "Prefix" + err = en.Append(0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78) + if err != nil { + return + } + err = en.WriteString(z.Prefix) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + // write "Endpoint" + err = en.Append(0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Endpoint) + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + // write "Path" + err = en.Append(0xa4, 0x50, 0x61, 0x74, 0x68) + if err != nil { + return + } + err = en.WriteString(z.Path) + if err != nil { + err = msgp.WrapError(err, "Path") + return + } + // write "Creds" + err = en.Append(0xa5, 0x43, 0x72, 0x65, 0x64, 0x73) + if err != nil { + return + } + // map header, size 3 + // write "AccessKey" + err = en.Append(0x83, 0xa9, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79) + if err != nil { + return + } + err = en.WriteString(z.Creds.AccessKey) + if err != nil { + err = msgp.WrapError(err, "Creds", "AccessKey") + return + } + // write "SecretKey" + err = en.Append(0xa9, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79) + if err != nil { + return + } + err = en.WriteString(z.Creds.SecretKey) + if err != nil { + err = msgp.WrapError(err, "Creds", "SecretKey") + return + } + // write "SessionToken" + err = en.Append(0xac, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e) + if err != nil { + return + } + err = en.WriteString(z.Creds.SessionToken) + if err != nil { + err = msgp.WrapError(err, "Creds", "SessionToken") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *BatchJobReplicateTarget) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 6 + // string "Type" + o = append(o, 0x86, 0xa4, 0x54, 0x79, 0x70, 0x65) + o = msgp.AppendString(o, string(z.Type)) + // string "Bucket" + o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + o = msgp.AppendString(o, z.Bucket) + // string "Prefix" + o = append(o, 0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78) + o = msgp.AppendString(o, z.Prefix) + // string "Endpoint" + o = append(o, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74) + o = msgp.AppendString(o, z.Endpoint) + // string "Path" + o = append(o, 0xa4, 0x50, 0x61, 0x74, 0x68) + o = msgp.AppendString(o, z.Path) + // string "Creds" + o = append(o, 0xa5, 0x43, 0x72, 0x65, 0x64, 0x73) + // map header, size 3 + // string "AccessKey" + o = append(o, 0x83, 0xa9, 0x41, 0x63, 0x63, 0x65, 0x73, 0x73, 0x4b, 0x65, 0x79) + o = msgp.AppendString(o, z.Creds.AccessKey) + // string "SecretKey" + o = append(o, 0xa9, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x4b, 0x65, 0x79) + o = msgp.AppendString(o, z.Creds.SecretKey) + // string "SessionToken" + o = append(o, 0xac, 0x53, 0x65, 0x73, 0x73, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e) + o = msgp.AppendString(o, z.Creds.SessionToken) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchJobReplicateTarget) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Type": + { + var zb0002 string + zb0002, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + z.Type = BatchJobReplicateResourceType(zb0002) + } + case "Bucket": + z.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "Prefix": + z.Prefix, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + case "Endpoint": + z.Endpoint, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + case "Path": + z.Path, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Path") + return + } + case "Creds": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Creds") + return + } + for zb0003 > 0 { + zb0003-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "Creds") + return + } + switch msgp.UnsafeString(field) { + case "AccessKey": + z.Creds.AccessKey, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Creds", "AccessKey") + return + } + case "SecretKey": + z.Creds.SecretKey, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Creds", "SecretKey") + return + } + case "SessionToken": + z.Creds.SessionToken, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Creds", "SessionToken") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "Creds") + return + } + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *BatchJobReplicateTarget) Msgsize() (s int) { + s = 1 + 5 + msgp.StringPrefixSize + len(string(z.Type)) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 5 + msgp.StringPrefixSize + len(z.Path) + 6 + 1 + 10 + msgp.StringPrefixSize + len(z.Creds.AccessKey) + 10 + msgp.StringPrefixSize + len(z.Creds.SecretKey) + 13 + msgp.StringPrefixSize + len(z.Creds.SessionToken) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *BatchJobReplicateV1) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "APIVersion": + z.APIVersion, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "APIVersion") + return + } + case "Flags": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + for zb0002 > 0 { + zb0002-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + switch msgp.UnsafeString(field) { + case "Filter": + err = z.Flags.Filter.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Flags", "Filter") + return + } + case "Notify": + err = z.Flags.Notify.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Flags", "Notify") + return + } + case "Retry": + err = z.Flags.Retry.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Flags", "Retry") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + } + } + case "Target": + err = z.Target.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Target") + return + } + case "Source": + err = z.Source.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Source") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *BatchJobReplicateV1) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 4 + // write "APIVersion" + err = en.Append(0x84, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e) + if err != nil { + return + } + err = en.WriteString(z.APIVersion) + if err != nil { + err = msgp.WrapError(err, "APIVersion") + return + } + // write "Flags" + err = en.Append(0xa5, 0x46, 0x6c, 0x61, 0x67, 0x73) + if err != nil { + return + } + // map header, size 3 + // write "Filter" + err = en.Append(0x83, 0xa6, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72) + if err != nil { + return + } + err = z.Flags.Filter.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Flags", "Filter") + return + } + // write "Notify" + err = en.Append(0xa6, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79) + if err != nil { + return + } + err = z.Flags.Notify.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Flags", "Notify") + return + } + // write "Retry" + err = en.Append(0xa5, 0x52, 0x65, 0x74, 0x72, 0x79) + if err != nil { + return + } + err = z.Flags.Retry.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Flags", "Retry") + return + } + // write "Target" + err = en.Append(0xa6, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74) + if err != nil { + return + } + err = z.Target.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Target") + return + } + // write "Source" + err = en.Append(0xa6, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65) + if err != nil { + return + } + err = z.Source.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Source") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *BatchJobReplicateV1) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 4 + // string "APIVersion" + o = append(o, 0x84, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e) + o = msgp.AppendString(o, z.APIVersion) + // string "Flags" + o = append(o, 0xa5, 0x46, 0x6c, 0x61, 0x67, 0x73) + // map header, size 3 + // string "Filter" + o = append(o, 0x83, 0xa6, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72) + o, err = z.Flags.Filter.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Flags", "Filter") + return + } + // string "Notify" + o = append(o, 0xa6, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79) + o, err = z.Flags.Notify.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Flags", "Notify") + return + } + // string "Retry" + o = append(o, 0xa5, 0x52, 0x65, 0x74, 0x72, 0x79) + o, err = z.Flags.Retry.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Flags", "Retry") + return + } + // string "Target" + o = append(o, 0xa6, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74) + o, err = z.Target.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Target") + return + } + // string "Source" + o = append(o, 0xa6, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65) + o, err = z.Source.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Source") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchJobReplicateV1) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "APIVersion": + z.APIVersion, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "APIVersion") + return + } + case "Flags": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + for zb0002 > 0 { + zb0002-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + switch msgp.UnsafeString(field) { + case "Filter": + bts, err = z.Flags.Filter.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Flags", "Filter") + return + } + case "Notify": + bts, err = z.Flags.Notify.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Flags", "Notify") + return + } + case "Retry": + bts, err = z.Flags.Retry.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Flags", "Retry") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + } + } + case "Target": + bts, err = z.Target.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Target") + return + } + case "Source": + bts, err = z.Source.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Source") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *BatchJobReplicateV1) Msgsize() (s int) { + s = 1 + 11 + msgp.StringPrefixSize + len(z.APIVersion) + 6 + 1 + 7 + z.Flags.Filter.Msgsize() + 7 + z.Flags.Notify.Msgsize() + 6 + z.Flags.Retry.Msgsize() + 7 + z.Target.Msgsize() + 7 + z.Source.Msgsize() + return +} + +// DecodeMsg implements msgp.Decodable +func (z *BatchReplicateFilter) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "NewerThan": + err = z.NewerThan.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "NewerThan") + return + } + case "OlderThan": + err = z.OlderThan.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "OlderThan") + return + } + case "CreatedAfter": + z.CreatedAfter, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "CreatedAfter") + return + } + case "CreatedBefore": + z.CreatedBefore, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "CreatedBefore") + return + } + case "Tags": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Tags") + return + } + if cap(z.Tags) >= int(zb0002) { + z.Tags = (z.Tags)[:zb0002] + } else { + z.Tags = make([]BatchJobKV, zb0002) + } + for za0001 := range z.Tags { + err = z.Tags[za0001].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Tags", za0001) + return + } + } + case "Metadata": + var zb0003 uint32 + zb0003, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Metadata") + return + } + if cap(z.Metadata) >= int(zb0003) { + z.Metadata = (z.Metadata)[:zb0003] + } else { + z.Metadata = make([]BatchJobKV, zb0003) + } + for za0002 := range z.Metadata { + err = z.Metadata[za0002].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Metadata", za0002) + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *BatchReplicateFilter) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 6 + // write "NewerThan" + err = en.Append(0x86, 0xa9, 0x4e, 0x65, 0x77, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e) + if err != nil { + return + } + err = z.NewerThan.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "NewerThan") + return + } + // write "OlderThan" + err = en.Append(0xa9, 0x4f, 0x6c, 0x64, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e) + if err != nil { + return + } + err = z.OlderThan.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "OlderThan") + return + } + // write "CreatedAfter" + err = en.Append(0xac, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x66, 0x74, 0x65, 0x72) + if err != nil { + return + } + err = en.WriteTime(z.CreatedAfter) + if err != nil { + err = msgp.WrapError(err, "CreatedAfter") + return + } + // write "CreatedBefore" + err = en.Append(0xad, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65) + if err != nil { + return + } + err = en.WriteTime(z.CreatedBefore) + if err != nil { + err = msgp.WrapError(err, "CreatedBefore") + return + } + // write "Tags" + err = en.Append(0xa4, 0x54, 0x61, 0x67, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Tags))) + if err != nil { + err = msgp.WrapError(err, "Tags") + return + } + for za0001 := range z.Tags { + err = z.Tags[za0001].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Tags", za0001) + return + } + } + // write "Metadata" + err = en.Append(0xa8, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Metadata))) + if err != nil { + err = msgp.WrapError(err, "Metadata") + return + } + for za0002 := range z.Metadata { + err = z.Metadata[za0002].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Metadata", za0002) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *BatchReplicateFilter) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 6 + // string "NewerThan" + o = append(o, 0x86, 0xa9, 0x4e, 0x65, 0x77, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e) + o, err = z.NewerThan.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "NewerThan") + return + } + // string "OlderThan" + o = append(o, 0xa9, 0x4f, 0x6c, 0x64, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e) + o, err = z.OlderThan.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "OlderThan") + return + } + // string "CreatedAfter" + o = append(o, 0xac, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x66, 0x74, 0x65, 0x72) + o = msgp.AppendTime(o, z.CreatedAfter) + // string "CreatedBefore" + o = append(o, 0xad, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65) + o = msgp.AppendTime(o, z.CreatedBefore) + // string "Tags" + o = append(o, 0xa4, 0x54, 0x61, 0x67, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Tags))) + for za0001 := range z.Tags { + o, err = z.Tags[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Tags", za0001) + return + } + } + // string "Metadata" + o = append(o, 0xa8, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61) + o = msgp.AppendArrayHeader(o, uint32(len(z.Metadata))) + for za0002 := range z.Metadata { + o, err = z.Metadata[za0002].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Metadata", za0002) + return + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchReplicateFilter) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "NewerThan": + bts, err = z.NewerThan.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "NewerThan") + return + } + case "OlderThan": + bts, err = z.OlderThan.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "OlderThan") + return + } + case "CreatedAfter": + z.CreatedAfter, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "CreatedAfter") + return + } + case "CreatedBefore": + z.CreatedBefore, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "CreatedBefore") + return + } + case "Tags": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tags") + return + } + if cap(z.Tags) >= int(zb0002) { + z.Tags = (z.Tags)[:zb0002] + } else { + z.Tags = make([]BatchJobKV, zb0002) + } + for za0001 := range z.Tags { + bts, err = z.Tags[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Tags", za0001) + return + } + } + case "Metadata": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Metadata") + return + } + if cap(z.Metadata) >= int(zb0003) { + z.Metadata = (z.Metadata)[:zb0003] + } else { + z.Metadata = make([]BatchJobKV, zb0003) + } + for za0002 := range z.Metadata { + bts, err = z.Metadata[za0002].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Metadata", za0002) + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *BatchReplicateFilter) Msgsize() (s int) { + s = 1 + 10 + z.NewerThan.Msgsize() + 10 + z.OlderThan.Msgsize() + 13 + msgp.TimeSize + 14 + msgp.TimeSize + 5 + msgp.ArrayHeaderSize + for za0001 := range z.Tags { + s += z.Tags[za0001].Msgsize() + } + s += 9 + msgp.ArrayHeaderSize + for za0002 := range z.Metadata { + s += z.Metadata[za0002].Msgsize() + } + return +} diff --git a/cmd/batch-replicate_gen_test.go b/cmd/batch-replicate_gen_test.go new file mode 100644 index 00000000..68ab1865 --- /dev/null +++ b/cmd/batch-replicate_gen_test.go @@ -0,0 +1,688 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalBatchJobReplicateCredentials(t *testing.T) { + v := BatchJobReplicateCredentials{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBatchJobReplicateCredentials(b *testing.B) { + v := BatchJobReplicateCredentials{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBatchJobReplicateCredentials(b *testing.B) { + v := BatchJobReplicateCredentials{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBatchJobReplicateCredentials(b *testing.B) { + v := BatchJobReplicateCredentials{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBatchJobReplicateCredentials(t *testing.T) { + v := BatchJobReplicateCredentials{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBatchJobReplicateCredentials Msgsize() is inaccurate") + } + + vn := BatchJobReplicateCredentials{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBatchJobReplicateCredentials(b *testing.B) { + v := BatchJobReplicateCredentials{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBatchJobReplicateCredentials(b *testing.B) { + v := BatchJobReplicateCredentials{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBatchJobReplicateFlags(t *testing.T) { + v := BatchJobReplicateFlags{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBatchJobReplicateFlags(b *testing.B) { + v := BatchJobReplicateFlags{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBatchJobReplicateFlags(b *testing.B) { + v := BatchJobReplicateFlags{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBatchJobReplicateFlags(b *testing.B) { + v := BatchJobReplicateFlags{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBatchJobReplicateFlags(t *testing.T) { + v := BatchJobReplicateFlags{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBatchJobReplicateFlags Msgsize() is inaccurate") + } + + vn := BatchJobReplicateFlags{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBatchJobReplicateFlags(b *testing.B) { + v := BatchJobReplicateFlags{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBatchJobReplicateFlags(b *testing.B) { + v := BatchJobReplicateFlags{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBatchJobReplicateSource(t *testing.T) { + v := BatchJobReplicateSource{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBatchJobReplicateSource(b *testing.B) { + v := BatchJobReplicateSource{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBatchJobReplicateSource(b *testing.B) { + v := BatchJobReplicateSource{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBatchJobReplicateSource(b *testing.B) { + v := BatchJobReplicateSource{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBatchJobReplicateSource(t *testing.T) { + v := BatchJobReplicateSource{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBatchJobReplicateSource Msgsize() is inaccurate") + } + + vn := BatchJobReplicateSource{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBatchJobReplicateSource(b *testing.B) { + v := BatchJobReplicateSource{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBatchJobReplicateSource(b *testing.B) { + v := BatchJobReplicateSource{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBatchJobReplicateTarget(t *testing.T) { + v := BatchJobReplicateTarget{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBatchJobReplicateTarget(b *testing.B) { + v := BatchJobReplicateTarget{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBatchJobReplicateTarget(b *testing.B) { + v := BatchJobReplicateTarget{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBatchJobReplicateTarget(b *testing.B) { + v := BatchJobReplicateTarget{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBatchJobReplicateTarget(t *testing.T) { + v := BatchJobReplicateTarget{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBatchJobReplicateTarget Msgsize() is inaccurate") + } + + vn := BatchJobReplicateTarget{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBatchJobReplicateTarget(b *testing.B) { + v := BatchJobReplicateTarget{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBatchJobReplicateTarget(b *testing.B) { + v := BatchJobReplicateTarget{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBatchJobReplicateV1(t *testing.T) { + v := BatchJobReplicateV1{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBatchJobReplicateV1(b *testing.B) { + v := BatchJobReplicateV1{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBatchJobReplicateV1(b *testing.B) { + v := BatchJobReplicateV1{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBatchJobReplicateV1(b *testing.B) { + v := BatchJobReplicateV1{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBatchJobReplicateV1(t *testing.T) { + v := BatchJobReplicateV1{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBatchJobReplicateV1 Msgsize() is inaccurate") + } + + vn := BatchJobReplicateV1{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBatchJobReplicateV1(b *testing.B) { + v := BatchJobReplicateV1{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBatchJobReplicateV1(b *testing.B) { + v := BatchJobReplicateV1{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBatchReplicateFilter(t *testing.T) { + v := BatchReplicateFilter{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBatchReplicateFilter(b *testing.B) { + v := BatchReplicateFilter{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBatchReplicateFilter(b *testing.B) { + v := BatchReplicateFilter{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBatchReplicateFilter(b *testing.B) { + v := BatchReplicateFilter{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBatchReplicateFilter(t *testing.T) { + v := BatchReplicateFilter{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBatchReplicateFilter Msgsize() is inaccurate") + } + + vn := BatchReplicateFilter{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBatchReplicateFilter(b *testing.B) { + v := BatchReplicateFilter{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBatchReplicateFilter(b *testing.B) { + v := BatchReplicateFilter{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/batch-replicate_test.go b/cmd/batch-replicate_test.go new file mode 100644 index 00000000..e84c59cf --- /dev/null +++ b/cmd/batch-replicate_test.go @@ -0,0 +1,182 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "slices" + "testing" + + "gopkg.in/yaml.v3" +) + +func TestParseBatchJobReplicate(t *testing.T) { + replicateYaml := ` +replicate: + apiVersion: v1 + # source of the objects to be replicated + source: + type: minio # valid values are "s3" or "minio" + bucket: mytest + prefix: object-prefix1 # 'PREFIX' is optional + # If your source is the 'local' alias specified to 'mc batch start', then the 'endpoint' and 'credentials' fields are optional and can be omitted + # Either the 'source' or 'remote' *must* be the "local" deployment +# endpoint: "http://127.0.0.1:9000" +# # path: "on|off|auto" # "on" enables path-style bucket lookup. "off" enables virtual host (DNS)-style bucket lookup. Defaults to "auto" +# credentials: +# accessKey: minioadmin # Required +# secretKey: minioadmin # Required +# # sessionToken: SESSION-TOKEN # Optional only available when rotating credentials are used + snowball: # automatically activated if the source is local + disable: true # optionally turn-off snowball archive transfer +# batch: 100 # upto this many objects per archive +# inmemory: true # indicates if the archive must be staged locally or in-memory +# compress: false # S2/Snappy compressed archive +# smallerThan: 5MiB # create archive for all objects smaller than 5MiB +# skipErrs: false # skips any source side read() errors + + # target where the objects must be replicated + target: + type: minio # valid values are "s3" or "minio" + bucket: mytest + prefix: stage # 'PREFIX' is optional + # If your source is the 'local' alias specified to 'mc batch start', then the 'endpoint' and 'credentials' fields are optional and can be omitted + + # Either the 'source' or 'remote' *must* be the "local" deployment + endpoint: "http://127.0.0.1:9001" + # path: "on|off|auto" # "on" enables path-style bucket lookup. "off" enables virtual host (DNS)-style bucket lookup. Defaults to "auto" + credentials: + accessKey: minioadmin + secretKey: minioadmin + # sessionToken: SESSION-TOKEN # Optional only available when rotating credentials are used + + # NOTE: All flags are optional + # - filtering criteria only applies for all source objects match the criteria + # - configurable notification endpoints + # - configurable retries for the job (each retry skips successfully previously replaced objects) + flags: + filter: + newerThan: "7d10h31s" # match objects newer than this value (e.g. 7d10h31s) + olderThan: "7d" # match objects older than this value (e.g. 7d10h31s) +# createdAfter: "date" # match objects created after "date" +# createdBefore: "date" # match objects created before "date" + + ## NOTE: tags are not supported when "source" is remote. + tags: + - key: "name" + value: "pick*" # match objects with tag 'name', with all values starting with 'pick' + + metadata: + - key: "content-type" + value: "image/*" # match objects with 'content-type', with all values starting with 'image/' + +# notify: +# endpoint: "https://notify.endpoint" # notification endpoint to receive job status events +# token: "Bearer xxxxx" # optional authentication token for the notification endpoint +# +# retry: +# attempts: 10 # number of retries for the job before giving up +# delay: "500ms" # least amount of delay between each retry + +` + var job BatchJobRequest + err := yaml.Unmarshal([]byte(replicateYaml), &job) + if err != nil { + t.Fatal("Failed to parse batch-job-replicate yaml", err) + } + if !slices.Equal(job.Replicate.Source.Prefix.F(), []string{"object-prefix1"}) { + t.Fatal("Failed to parse batch-job-replicate yaml", err) + } + multiPrefixReplicateYaml := ` +replicate: + apiVersion: v1 + # source of the objects to be replicated + source: + type: minio # valid values are "s3" or "minio" + bucket: mytest + prefix: # 'PREFIX' is optional + - object-prefix1 + - object-prefix2 + # If your source is the 'local' alias specified to 'mc batch start', then the 'endpoint' and 'credentials' fields are optional and can be omitted + # Either the 'source' or 'remote' *must* be the "local" deployment +# endpoint: "http://127.0.0.1:9000" +# # path: "on|off|auto" # "on" enables path-style bucket lookup. "off" enables virtual host (DNS)-style bucket lookup. Defaults to "auto" +# credentials: +# accessKey: minioadmin # Required +# secretKey: minioadmin # Required +# # sessionToken: SESSION-TOKEN # Optional only available when rotating credentials are used + snowball: # automatically activated if the source is local + disable: true # optionally turn-off snowball archive transfer +# batch: 100 # upto this many objects per archive +# inmemory: true # indicates if the archive must be staged locally or in-memory +# compress: false # S2/Snappy compressed archive +# smallerThan: 5MiB # create archive for all objects smaller than 5MiB +# skipErrs: false # skips any source side read() errors + + # target where the objects must be replicated + target: + type: minio # valid values are "s3" or "minio" + bucket: mytest + prefix: stage # 'PREFIX' is optional + # If your source is the 'local' alias specified to 'mc batch start', then the 'endpoint' and 'credentials' fields are optional and can be omitted + + # Either the 'source' or 'remote' *must* be the "local" deployment + endpoint: "http://127.0.0.1:9001" + # path: "on|off|auto" # "on" enables path-style bucket lookup. "off" enables virtual host (DNS)-style bucket lookup. Defaults to "auto" + credentials: + accessKey: minioadmin + secretKey: minioadmin + # sessionToken: SESSION-TOKEN # Optional only available when rotating credentials are used + + # NOTE: All flags are optional + # - filtering criteria only applies for all source objects match the criteria + # - configurable notification endpoints + # - configurable retries for the job (each retry skips successfully previously replaced objects) + flags: + filter: + newerThan: "7d10h31s" # match objects newer than this value (e.g. 7d10h31s) + olderThan: "7d" # match objects older than this value (e.g. 7d10h31s) +# createdAfter: "date" # match objects created after "date" +# createdBefore: "date" # match objects created before "date" + + ## NOTE: tags are not supported when "source" is remote. + tags: + - key: "name" + value: "pick*" # match objects with tag 'name', with all values starting with 'pick' + + metadata: + - key: "content-type" + value: "image/*" # match objects with 'content-type', with all values starting with 'image/' + +# notify: +# endpoint: "https://notify.endpoint" # notification endpoint to receive job status events +# token: "Bearer xxxxx" # optional authentication token for the notification endpoint +# +# retry: +# attempts: 10 # number of retries for the job before giving up +# delay: "500ms" # least amount of delay between each retry + +` + var multiPrefixJob BatchJobRequest + err = yaml.Unmarshal([]byte(multiPrefixReplicateYaml), &multiPrefixJob) + if err != nil { + t.Fatal("Failed to parse batch-job-replicate yaml", err) + } + if !slices.Equal(multiPrefixJob.Replicate.Source.Prefix.F(), []string{"object-prefix1", "object-prefix2"}) { + t.Fatal("Failed to parse batch-job-replicate yaml") + } +} diff --git a/cmd/batch-rotate.go b/cmd/batch-rotate.go new file mode 100644 index 00000000..3e8f18fa --- /dev/null +++ b/cmd/batch-rotate.go @@ -0,0 +1,494 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/base64" + "fmt" + "maps" + "math/rand" + "net/http" + "runtime" + "strconv" + "strings" + "time" + + jsoniter "github.com/json-iterator/go" + "github.com/minio/minio-go/v7/pkg/tags" + "github.com/minio/minio/internal/crypto" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/kms" + "github.com/minio/pkg/v3/env" + "github.com/minio/pkg/v3/workers" +) + +// keyrotate: +// apiVersion: v1 +// bucket: BUCKET +// prefix: PREFIX +// encryption: +// type: sse-s3 # valid values are sse-s3 and sse-kms +// key: # valid only for sse-kms +// context: # valid only for sse-kms +// # optional flags based filtering criteria +// # for all objects +// flags: +// filter: +// newerThan: "7d" # match objects newer than this value (e.g. 7d10h31s) +// olderThan: "7d" # match objects older than this value (e.g. 7d10h31s) +// createdAfter: "date" # match objects created after "date" +// createdBefore: "date" # match objects created before "date" +// tags: +// - key: "name" +// value: "pick*" # match objects with tag 'name', with all values starting with 'pick' +// metadata: +// - key: "content-type" +// value: "image/*" # match objects with 'content-type', with all values starting with 'image/' +// kmskey: "key-id" # match objects with KMS key-id (applicable only for sse-kms) +// notify: +// endpoint: "https://notify.endpoint" # notification endpoint to receive job status events +// token: "Bearer xxxxx" # optional authentication token for the notification endpoint + +// retry: +// attempts: 10 # number of retries for the job before giving up +// delay: "500ms" # least amount of delay between each retry + +//go:generate msgp -file $GOFILE -unexported + +// BatchKeyRotationType defines key rotation type +type BatchKeyRotationType string + +const ( + sses3 BatchKeyRotationType = "sse-s3" + ssekms BatchKeyRotationType = "sse-kms" +) + +// BatchJobKeyRotateEncryption defines key rotation encryption options passed +type BatchJobKeyRotateEncryption struct { + Type BatchKeyRotationType `yaml:"type" json:"type"` + Key string `yaml:"key" json:"key"` + Context string `yaml:"context" json:"context"` + kmsContext kms.Context `msg:"-"` +} + +// Validate validates input key rotation encryption options. +func (e BatchJobKeyRotateEncryption) Validate() error { + if e.Type != sses3 && e.Type != ssekms { + return errInvalidArgument + } + spaces := strings.HasPrefix(e.Key, " ") || strings.HasSuffix(e.Key, " ") + if e.Type == ssekms && spaces { + return crypto.ErrInvalidEncryptionKeyID + } + + if e.Type == ssekms && GlobalKMS != nil { + ctx := kms.Context{} + if e.Context != "" { + b, err := base64.StdEncoding.DecodeString(e.Context) + if err != nil { + return err + } + + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err := json.Unmarshal(b, &ctx); err != nil { + return err + } + } + e.kmsContext = kms.Context{} + maps.Copy(e.kmsContext, ctx) + ctx["MinIO batch API"] = "batchrotate" // Context for a test key operation + if _, err := GlobalKMS.GenerateKey(GlobalContext, &kms.GenerateKeyRequest{Name: e.Key, AssociatedData: ctx}); err != nil { + return err + } + } + return nil +} + +// BatchKeyRotateFilter holds all the filters currently supported for batch replication +type BatchKeyRotateFilter struct { + NewerThan time.Duration `yaml:"newerThan,omitempty" json:"newerThan"` + OlderThan time.Duration `yaml:"olderThan,omitempty" json:"olderThan"` + CreatedAfter time.Time `yaml:"createdAfter,omitempty" json:"createdAfter"` + CreatedBefore time.Time `yaml:"createdBefore,omitempty" json:"createdBefore"` + Tags []BatchJobKV `yaml:"tags,omitempty" json:"tags"` + Metadata []BatchJobKV `yaml:"metadata,omitempty" json:"metadata"` + KMSKeyID string `yaml:"kmskeyid" json:"kmskey"` +} + +// BatchKeyRotateNotification success or failure notification endpoint for each job attempts +type BatchKeyRotateNotification struct { + Endpoint string `yaml:"endpoint" json:"endpoint"` + Token string `yaml:"token" json:"token"` +} + +// BatchJobKeyRotateFlags various configurations for replication job definition currently includes +// - filter +// - notify +// - retry +type BatchJobKeyRotateFlags struct { + Filter BatchKeyRotateFilter `yaml:"filter" json:"filter"` + Notify BatchJobNotification `yaml:"notify" json:"notify"` + Retry BatchJobRetry `yaml:"retry" json:"retry"` +} + +// BatchJobKeyRotateV1 v1 of batch key rotation job +type BatchJobKeyRotateV1 struct { + APIVersion string `yaml:"apiVersion" json:"apiVersion"` + Flags BatchJobKeyRotateFlags `yaml:"flags" json:"flags"` + Bucket string `yaml:"bucket" json:"bucket"` + Prefix string `yaml:"prefix" json:"prefix"` + Encryption BatchJobKeyRotateEncryption `yaml:"encryption" json:"encryption"` +} + +// Notify notifies notification endpoint if configured regarding job failure or success. +func (r BatchJobKeyRotateV1) Notify(ctx context.Context, ri *batchJobInfo) error { + return notifyEndpoint(ctx, ri, r.Flags.Notify.Endpoint, r.Flags.Notify.Token) +} + +// KeyRotate rotates encryption key of an object +func (r *BatchJobKeyRotateV1) KeyRotate(ctx context.Context, api ObjectLayer, objInfo ObjectInfo) error { + srcBucket := r.Bucket + srcObject := objInfo.Name + + if objInfo.DeleteMarker || !objInfo.VersionPurgeStatus.Empty() { + return nil + } + sseKMS := crypto.S3KMS.IsEncrypted(objInfo.UserDefined) + sseS3 := crypto.S3.IsEncrypted(objInfo.UserDefined) + if !sseKMS && !sseS3 { // neither sse-s3 nor sse-kms disallowed + return errInvalidEncryptionParameters + } + if sseKMS && r.Encryption.Type == sses3 { // previously encrypted with sse-kms, now sse-s3 disallowed + return errInvalidEncryptionParameters + } + versioned := globalBucketVersioningSys.PrefixEnabled(srcBucket, srcObject) + versionSuspended := globalBucketVersioningSys.PrefixSuspended(srcBucket, srcObject) + + lock := api.NewNSLock(r.Bucket, objInfo.Name) + lkctx, err := lock.GetLock(ctx, globalOperationTimeout) + if err != nil { + return err + } + ctx = lkctx.Context() + defer lock.Unlock(lkctx) + + opts := ObjectOptions{ + VersionID: objInfo.VersionID, + Versioned: versioned, + VersionSuspended: versionSuspended, + NoLock: true, + } + obj, err := api.GetObjectInfo(ctx, r.Bucket, objInfo.Name, opts) + if err != nil { + return err + } + oi := obj.Clone() + var ( + newKeyID string + newKeyContext kms.Context + ) + encMetadata := make(map[string]string) + for k, v := range oi.UserDefined { + if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) { + encMetadata[k] = v + } + } + + if (sseKMS || sseS3) && r.Encryption.Type == ssekms { + if err = r.Encryption.Validate(); err != nil { + return err + } + newKeyID = strings.TrimPrefix(r.Encryption.Key, crypto.ARNPrefix) + newKeyContext = r.Encryption.kmsContext + } + if err = rotateKey(ctx, []byte{}, newKeyID, []byte{}, r.Bucket, oi.Name, encMetadata, newKeyContext); err != nil { + return err + } + + // Since we are rotating the keys, make sure to update the metadata. + oi.metadataOnly = true + oi.keyRotation = true + maps.Copy(oi.UserDefined, encMetadata) + if _, err := api.CopyObject(ctx, r.Bucket, oi.Name, r.Bucket, oi.Name, oi, ObjectOptions{ + VersionID: oi.VersionID, + }, ObjectOptions{ + VersionID: oi.VersionID, + NoLock: true, + }); err != nil { + return err + } + + return nil +} + +const ( + batchKeyRotationName = "batch-rotate.bin" + batchKeyRotationFormat = 1 + batchKeyRotateVersionV1 = 1 + batchKeyRotateVersion = batchKeyRotateVersionV1 + batchKeyRotateAPIVersion = "v1" + batchKeyRotateJobDefaultRetries = 3 + batchKeyRotateJobDefaultRetryDelay = 25 * time.Millisecond +) + +// Start the batch key rottion job, resumes if there was a pending job via "job.ID" +func (r *BatchJobKeyRotateV1) Start(ctx context.Context, api ObjectLayer, job BatchJobRequest) error { + ri := &batchJobInfo{ + JobID: job.ID, + JobType: string(job.Type()), + StartTime: job.Started, + } + if err := ri.loadOrInit(ctx, api, job); err != nil { + return err + } + if ri.Complete { + return nil + } + + globalBatchJobsMetrics.save(job.ID, ri) + lastObject := ri.Object + + retryAttempts := job.KeyRotate.Flags.Retry.Attempts + if retryAttempts <= 0 { + retryAttempts = batchKeyRotateJobDefaultRetries + } + delay := job.KeyRotate.Flags.Retry.Delay + if delay <= 0 { + delay = batchKeyRotateJobDefaultRetryDelay + } + + rnd := rand.New(rand.NewSource(time.Now().UnixNano())) + + selectObj := func(info FileInfo) (ok bool) { + if r.Flags.Filter.OlderThan > 0 && time.Since(info.ModTime) < r.Flags.Filter.OlderThan { + // skip all objects that are newer than specified older duration + return false + } + + if r.Flags.Filter.NewerThan > 0 && time.Since(info.ModTime) >= r.Flags.Filter.NewerThan { + // skip all objects that are older than specified newer duration + return false + } + + if !r.Flags.Filter.CreatedAfter.IsZero() && r.Flags.Filter.CreatedAfter.Before(info.ModTime) { + // skip all objects that are created before the specified time. + return false + } + + if !r.Flags.Filter.CreatedBefore.IsZero() && r.Flags.Filter.CreatedBefore.After(info.ModTime) { + // skip all objects that are created after the specified time. + return false + } + + if len(r.Flags.Filter.Tags) > 0 { + // Only parse object tags if tags filter is specified. + tagMap := map[string]string{} + tagStr := info.Metadata[xhttp.AmzObjectTagging] + if len(tagStr) != 0 { + t, err := tags.ParseObjectTags(tagStr) + if err != nil { + return false + } + tagMap = t.ToMap() + } + + for _, kv := range r.Flags.Filter.Tags { + for t, v := range tagMap { + if kv.Match(BatchJobKV{Key: t, Value: v}) { + return true + } + } + } + + // None of the provided tags filter match skip the object + return false + } + + if len(r.Flags.Filter.Metadata) > 0 { + for _, kv := range r.Flags.Filter.Metadata { + for k, v := range info.Metadata { + if !stringsHasPrefixFold(k, "x-amz-meta-") && !isStandardHeader(k) { + continue + } + // We only need to match x-amz-meta or standardHeaders + if kv.Match(BatchJobKV{Key: k, Value: v}) { + return true + } + } + } + + // None of the provided metadata filters match skip the object. + return false + } + if r.Flags.Filter.KMSKeyID != "" { + if v, ok := info.Metadata[xhttp.AmzServerSideEncryptionKmsID]; ok && strings.TrimPrefix(v, crypto.ARNPrefix) != r.Flags.Filter.KMSKeyID { + return false + } + } + return true + } + + workerSize, err := strconv.Atoi(env.Get("_MINIO_BATCH_KEYROTATION_WORKERS", strconv.Itoa(runtime.GOMAXPROCS(0)/2))) + if err != nil { + return err + } + + wk, err := workers.New(workerSize) + if err != nil { + // invalid worker size. + return err + } + + ctx, cancel := context.WithCancel(ctx) + + results := make(chan itemOrErr[ObjectInfo], 100) + if err := api.Walk(ctx, r.Bucket, r.Prefix, results, WalkOptions{ + Marker: lastObject, + Filter: selectObj, + }); err != nil { + cancel() + // Do not need to retry if we can't list objects on source. + return err + } + failed := false + for res := range results { + if res.Err != nil { + failed = true + batchLogIf(ctx, res.Err) + break + } + result := res.Item + sseKMS := crypto.S3KMS.IsEncrypted(result.UserDefined) + sseS3 := crypto.S3.IsEncrypted(result.UserDefined) + if !sseKMS && !sseS3 { // neither sse-s3 nor sse-kms disallowed + continue + } + wk.Take() + go func() { + defer wk.Give() + for attempts := 1; attempts <= retryAttempts; attempts++ { + stopFn := globalBatchJobsMetrics.trace(batchJobMetricKeyRotation, job.ID, attempts) + success := true + if err := r.KeyRotate(ctx, api, result); err != nil { + stopFn(result, err) + batchLogIf(ctx, err) + success = false + if attempts >= retryAttempts { + auditOptions := AuditLogOptions{ + Event: "KeyRotate", + APIName: "StartBatchJob", + Bucket: result.Bucket, + Object: result.Name, + VersionID: result.VersionID, + Error: err.Error(), + } + auditLogInternal(ctx, auditOptions) + } + } else { + stopFn(result, nil) + } + ri.trackCurrentBucketObject(r.Bucket, result, success, attempts) + globalBatchJobsMetrics.save(job.ID, ri) + // persist in-memory state to disk after every 10secs. + batchLogIf(ctx, ri.updateAfter(ctx, api, 10*time.Second, job)) + if success { + break + } + if delay > 0 { + time.Sleep(delay + time.Duration(rnd.Float64()*float64(delay))) + } + } + + if wait := globalBatchConfig.KeyRotationWait(); wait > 0 { + time.Sleep(wait) + } + }() + } + wk.Wait() + + ri.Complete = !failed && ri.ObjectsFailed == 0 + ri.Failed = failed || ri.ObjectsFailed > 0 + globalBatchJobsMetrics.save(job.ID, ri) + // persist in-memory state to disk. + batchLogIf(ctx, ri.updateAfter(ctx, api, 0, job)) + + if err := r.Notify(ctx, ri); err != nil { + batchLogIf(ctx, fmt.Errorf("unable to notify %v", err)) + } + + cancel() + return nil +} + +//msgp:ignore batchKeyRotationJobError +type batchKeyRotationJobError struct { + Code string + Description string + HTTPStatusCode int +} + +func (e batchKeyRotationJobError) Error() string { + return e.Description +} + +// Validate validates the job definition input +func (r *BatchJobKeyRotateV1) Validate(ctx context.Context, job BatchJobRequest, o ObjectLayer) error { + if r == nil { + return nil + } + + if r.APIVersion != batchKeyRotateAPIVersion { + return errInvalidArgument + } + + if r.Bucket == "" { + return errInvalidArgument + } + + if _, err := o.GetBucketInfo(ctx, r.Bucket, BucketOptions{}); err != nil { + if isErrBucketNotFound(err) { + return batchKeyRotationJobError{ + Code: "NoSuchSourceBucket", + Description: "The specified source bucket does not exist", + HTTPStatusCode: http.StatusNotFound, + } + } + return err + } + if GlobalKMS == nil { + return errKMSNotConfigured + } + if err := r.Encryption.Validate(); err != nil { + return err + } + + for _, tag := range r.Flags.Filter.Tags { + if err := tag.Validate(); err != nil { + return err + } + } + + for _, meta := range r.Flags.Filter.Metadata { + if err := meta.Validate(); err != nil { + return err + } + } + + return r.Flags.Retry.Validate() +} diff --git a/cmd/batch-rotate_gen.go b/cmd/batch-rotate_gen.go new file mode 100644 index 00000000..e714e1c8 --- /dev/null +++ b/cmd/batch-rotate_gen.go @@ -0,0 +1,1178 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *BatchJobKeyRotateEncryption) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Type": + { + var zb0002 string + zb0002, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + z.Type = BatchKeyRotationType(zb0002) + } + case "Key": + z.Key, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Key") + return + } + case "Context": + z.Context, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Context") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z BatchJobKeyRotateEncryption) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "Type" + err = en.Append(0x83, 0xa4, 0x54, 0x79, 0x70, 0x65) + if err != nil { + return + } + err = en.WriteString(string(z.Type)) + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + // write "Key" + err = en.Append(0xa3, 0x4b, 0x65, 0x79) + if err != nil { + return + } + err = en.WriteString(z.Key) + if err != nil { + err = msgp.WrapError(err, "Key") + return + } + // write "Context" + err = en.Append(0xa7, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Context) + if err != nil { + err = msgp.WrapError(err, "Context") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z BatchJobKeyRotateEncryption) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "Type" + o = append(o, 0x83, 0xa4, 0x54, 0x79, 0x70, 0x65) + o = msgp.AppendString(o, string(z.Type)) + // string "Key" + o = append(o, 0xa3, 0x4b, 0x65, 0x79) + o = msgp.AppendString(o, z.Key) + // string "Context" + o = append(o, 0xa7, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x78, 0x74) + o = msgp.AppendString(o, z.Context) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchJobKeyRotateEncryption) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Type": + { + var zb0002 string + zb0002, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + z.Type = BatchKeyRotationType(zb0002) + } + case "Key": + z.Key, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Key") + return + } + case "Context": + z.Context, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Context") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z BatchJobKeyRotateEncryption) Msgsize() (s int) { + s = 1 + 5 + msgp.StringPrefixSize + len(string(z.Type)) + 4 + msgp.StringPrefixSize + len(z.Key) + 8 + msgp.StringPrefixSize + len(z.Context) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *BatchJobKeyRotateFlags) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Filter": + err = z.Filter.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Filter") + return + } + case "Notify": + err = z.Notify.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Notify") + return + } + case "Retry": + err = z.Retry.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Retry") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *BatchJobKeyRotateFlags) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "Filter" + err = en.Append(0x83, 0xa6, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72) + if err != nil { + return + } + err = z.Filter.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Filter") + return + } + // write "Notify" + err = en.Append(0xa6, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79) + if err != nil { + return + } + err = z.Notify.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Notify") + return + } + // write "Retry" + err = en.Append(0xa5, 0x52, 0x65, 0x74, 0x72, 0x79) + if err != nil { + return + } + err = z.Retry.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Retry") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *BatchJobKeyRotateFlags) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "Filter" + o = append(o, 0x83, 0xa6, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72) + o, err = z.Filter.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Filter") + return + } + // string "Notify" + o = append(o, 0xa6, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79) + o, err = z.Notify.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Notify") + return + } + // string "Retry" + o = append(o, 0xa5, 0x52, 0x65, 0x74, 0x72, 0x79) + o, err = z.Retry.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Retry") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchJobKeyRotateFlags) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Filter": + bts, err = z.Filter.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Filter") + return + } + case "Notify": + bts, err = z.Notify.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Notify") + return + } + case "Retry": + bts, err = z.Retry.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Retry") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *BatchJobKeyRotateFlags) Msgsize() (s int) { + s = 1 + 7 + z.Filter.Msgsize() + 7 + z.Notify.Msgsize() + 6 + z.Retry.Msgsize() + return +} + +// DecodeMsg implements msgp.Decodable +func (z *BatchJobKeyRotateV1) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "APIVersion": + z.APIVersion, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "APIVersion") + return + } + case "Flags": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + for zb0002 > 0 { + zb0002-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + switch msgp.UnsafeString(field) { + case "Filter": + err = z.Flags.Filter.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Flags", "Filter") + return + } + case "Notify": + err = z.Flags.Notify.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Flags", "Notify") + return + } + case "Retry": + err = z.Flags.Retry.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Flags", "Retry") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + } + } + case "Bucket": + z.Bucket, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "Prefix": + z.Prefix, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + case "Encryption": + err = z.Encryption.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Encryption") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *BatchJobKeyRotateV1) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 5 + // write "APIVersion" + err = en.Append(0x85, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e) + if err != nil { + return + } + err = en.WriteString(z.APIVersion) + if err != nil { + err = msgp.WrapError(err, "APIVersion") + return + } + // write "Flags" + err = en.Append(0xa5, 0x46, 0x6c, 0x61, 0x67, 0x73) + if err != nil { + return + } + // map header, size 3 + // write "Filter" + err = en.Append(0x83, 0xa6, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72) + if err != nil { + return + } + err = z.Flags.Filter.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Flags", "Filter") + return + } + // write "Notify" + err = en.Append(0xa6, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79) + if err != nil { + return + } + err = z.Flags.Notify.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Flags", "Notify") + return + } + // write "Retry" + err = en.Append(0xa5, 0x52, 0x65, 0x74, 0x72, 0x79) + if err != nil { + return + } + err = z.Flags.Retry.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Flags", "Retry") + return + } + // write "Bucket" + err = en.Append(0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Bucket) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + // write "Prefix" + err = en.Append(0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78) + if err != nil { + return + } + err = en.WriteString(z.Prefix) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + // write "Encryption" + err = en.Append(0xaa, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e) + if err != nil { + return + } + err = z.Encryption.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Encryption") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *BatchJobKeyRotateV1) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 5 + // string "APIVersion" + o = append(o, 0x85, 0xaa, 0x41, 0x50, 0x49, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e) + o = msgp.AppendString(o, z.APIVersion) + // string "Flags" + o = append(o, 0xa5, 0x46, 0x6c, 0x61, 0x67, 0x73) + // map header, size 3 + // string "Filter" + o = append(o, 0x83, 0xa6, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72) + o, err = z.Flags.Filter.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Flags", "Filter") + return + } + // string "Notify" + o = append(o, 0xa6, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x79) + o, err = z.Flags.Notify.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Flags", "Notify") + return + } + // string "Retry" + o = append(o, 0xa5, 0x52, 0x65, 0x74, 0x72, 0x79) + o, err = z.Flags.Retry.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Flags", "Retry") + return + } + // string "Bucket" + o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + o = msgp.AppendString(o, z.Bucket) + // string "Prefix" + o = append(o, 0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78) + o = msgp.AppendString(o, z.Prefix) + // string "Encryption" + o = append(o, 0xaa, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e) + o, err = z.Encryption.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Encryption") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchJobKeyRotateV1) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "APIVersion": + z.APIVersion, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "APIVersion") + return + } + case "Flags": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + for zb0002 > 0 { + zb0002-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + switch msgp.UnsafeString(field) { + case "Filter": + bts, err = z.Flags.Filter.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Flags", "Filter") + return + } + case "Notify": + bts, err = z.Flags.Notify.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Flags", "Notify") + return + } + case "Retry": + bts, err = z.Flags.Retry.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Flags", "Retry") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + } + } + case "Bucket": + z.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "Prefix": + z.Prefix, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + case "Encryption": + bts, err = z.Encryption.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Encryption") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *BatchJobKeyRotateV1) Msgsize() (s int) { + s = 1 + 11 + msgp.StringPrefixSize + len(z.APIVersion) + 6 + 1 + 7 + z.Flags.Filter.Msgsize() + 7 + z.Flags.Notify.Msgsize() + 6 + z.Flags.Retry.Msgsize() + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 11 + z.Encryption.Msgsize() + return +} + +// DecodeMsg implements msgp.Decodable +func (z *BatchKeyRotateFilter) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "NewerThan": + z.NewerThan, err = dc.ReadDuration() + if err != nil { + err = msgp.WrapError(err, "NewerThan") + return + } + case "OlderThan": + z.OlderThan, err = dc.ReadDuration() + if err != nil { + err = msgp.WrapError(err, "OlderThan") + return + } + case "CreatedAfter": + z.CreatedAfter, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "CreatedAfter") + return + } + case "CreatedBefore": + z.CreatedBefore, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "CreatedBefore") + return + } + case "Tags": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Tags") + return + } + if cap(z.Tags) >= int(zb0002) { + z.Tags = (z.Tags)[:zb0002] + } else { + z.Tags = make([]BatchJobKV, zb0002) + } + for za0001 := range z.Tags { + err = z.Tags[za0001].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Tags", za0001) + return + } + } + case "Metadata": + var zb0003 uint32 + zb0003, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Metadata") + return + } + if cap(z.Metadata) >= int(zb0003) { + z.Metadata = (z.Metadata)[:zb0003] + } else { + z.Metadata = make([]BatchJobKV, zb0003) + } + for za0002 := range z.Metadata { + err = z.Metadata[za0002].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Metadata", za0002) + return + } + } + case "KMSKeyID": + z.KMSKeyID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "KMSKeyID") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *BatchKeyRotateFilter) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 7 + // write "NewerThan" + err = en.Append(0x87, 0xa9, 0x4e, 0x65, 0x77, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e) + if err != nil { + return + } + err = en.WriteDuration(z.NewerThan) + if err != nil { + err = msgp.WrapError(err, "NewerThan") + return + } + // write "OlderThan" + err = en.Append(0xa9, 0x4f, 0x6c, 0x64, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e) + if err != nil { + return + } + err = en.WriteDuration(z.OlderThan) + if err != nil { + err = msgp.WrapError(err, "OlderThan") + return + } + // write "CreatedAfter" + err = en.Append(0xac, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x66, 0x74, 0x65, 0x72) + if err != nil { + return + } + err = en.WriteTime(z.CreatedAfter) + if err != nil { + err = msgp.WrapError(err, "CreatedAfter") + return + } + // write "CreatedBefore" + err = en.Append(0xad, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65) + if err != nil { + return + } + err = en.WriteTime(z.CreatedBefore) + if err != nil { + err = msgp.WrapError(err, "CreatedBefore") + return + } + // write "Tags" + err = en.Append(0xa4, 0x54, 0x61, 0x67, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Tags))) + if err != nil { + err = msgp.WrapError(err, "Tags") + return + } + for za0001 := range z.Tags { + err = z.Tags[za0001].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Tags", za0001) + return + } + } + // write "Metadata" + err = en.Append(0xa8, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Metadata))) + if err != nil { + err = msgp.WrapError(err, "Metadata") + return + } + for za0002 := range z.Metadata { + err = z.Metadata[za0002].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Metadata", za0002) + return + } + } + // write "KMSKeyID" + err = en.Append(0xa8, 0x4b, 0x4d, 0x53, 0x4b, 0x65, 0x79, 0x49, 0x44) + if err != nil { + return + } + err = en.WriteString(z.KMSKeyID) + if err != nil { + err = msgp.WrapError(err, "KMSKeyID") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *BatchKeyRotateFilter) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 7 + // string "NewerThan" + o = append(o, 0x87, 0xa9, 0x4e, 0x65, 0x77, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e) + o = msgp.AppendDuration(o, z.NewerThan) + // string "OlderThan" + o = append(o, 0xa9, 0x4f, 0x6c, 0x64, 0x65, 0x72, 0x54, 0x68, 0x61, 0x6e) + o = msgp.AppendDuration(o, z.OlderThan) + // string "CreatedAfter" + o = append(o, 0xac, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x66, 0x74, 0x65, 0x72) + o = msgp.AppendTime(o, z.CreatedAfter) + // string "CreatedBefore" + o = append(o, 0xad, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65) + o = msgp.AppendTime(o, z.CreatedBefore) + // string "Tags" + o = append(o, 0xa4, 0x54, 0x61, 0x67, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Tags))) + for za0001 := range z.Tags { + o, err = z.Tags[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Tags", za0001) + return + } + } + // string "Metadata" + o = append(o, 0xa8, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61) + o = msgp.AppendArrayHeader(o, uint32(len(z.Metadata))) + for za0002 := range z.Metadata { + o, err = z.Metadata[za0002].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Metadata", za0002) + return + } + } + // string "KMSKeyID" + o = append(o, 0xa8, 0x4b, 0x4d, 0x53, 0x4b, 0x65, 0x79, 0x49, 0x44) + o = msgp.AppendString(o, z.KMSKeyID) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchKeyRotateFilter) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "NewerThan": + z.NewerThan, bts, err = msgp.ReadDurationBytes(bts) + if err != nil { + err = msgp.WrapError(err, "NewerThan") + return + } + case "OlderThan": + z.OlderThan, bts, err = msgp.ReadDurationBytes(bts) + if err != nil { + err = msgp.WrapError(err, "OlderThan") + return + } + case "CreatedAfter": + z.CreatedAfter, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "CreatedAfter") + return + } + case "CreatedBefore": + z.CreatedBefore, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "CreatedBefore") + return + } + case "Tags": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tags") + return + } + if cap(z.Tags) >= int(zb0002) { + z.Tags = (z.Tags)[:zb0002] + } else { + z.Tags = make([]BatchJobKV, zb0002) + } + for za0001 := range z.Tags { + bts, err = z.Tags[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Tags", za0001) + return + } + } + case "Metadata": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Metadata") + return + } + if cap(z.Metadata) >= int(zb0003) { + z.Metadata = (z.Metadata)[:zb0003] + } else { + z.Metadata = make([]BatchJobKV, zb0003) + } + for za0002 := range z.Metadata { + bts, err = z.Metadata[za0002].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Metadata", za0002) + return + } + } + case "KMSKeyID": + z.KMSKeyID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "KMSKeyID") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *BatchKeyRotateFilter) Msgsize() (s int) { + s = 1 + 10 + msgp.DurationSize + 10 + msgp.DurationSize + 13 + msgp.TimeSize + 14 + msgp.TimeSize + 5 + msgp.ArrayHeaderSize + for za0001 := range z.Tags { + s += z.Tags[za0001].Msgsize() + } + s += 9 + msgp.ArrayHeaderSize + for za0002 := range z.Metadata { + s += z.Metadata[za0002].Msgsize() + } + s += 9 + msgp.StringPrefixSize + len(z.KMSKeyID) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *BatchKeyRotateNotification) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Endpoint": + z.Endpoint, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + case "Token": + z.Token, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Token") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z BatchKeyRotateNotification) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "Endpoint" + err = en.Append(0x82, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Endpoint) + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + // write "Token" + err = en.Append(0xa5, 0x54, 0x6f, 0x6b, 0x65, 0x6e) + if err != nil { + return + } + err = en.WriteString(z.Token) + if err != nil { + err = msgp.WrapError(err, "Token") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z BatchKeyRotateNotification) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "Endpoint" + o = append(o, 0x82, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74) + o = msgp.AppendString(o, z.Endpoint) + // string "Token" + o = append(o, 0xa5, 0x54, 0x6f, 0x6b, 0x65, 0x6e) + o = msgp.AppendString(o, z.Token) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchKeyRotateNotification) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Endpoint": + z.Endpoint, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + case "Token": + z.Token, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Token") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z BatchKeyRotateNotification) Msgsize() (s int) { + s = 1 + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 6 + msgp.StringPrefixSize + len(z.Token) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *BatchKeyRotationType) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 string + zb0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = BatchKeyRotationType(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z BatchKeyRotationType) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteString(string(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z BatchKeyRotationType) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendString(o, string(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BatchKeyRotationType) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 string + zb0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = BatchKeyRotationType(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z BatchKeyRotationType) Msgsize() (s int) { + s = msgp.StringPrefixSize + len(string(z)) + return +} diff --git a/cmd/batch-rotate_gen_test.go b/cmd/batch-rotate_gen_test.go new file mode 100644 index 00000000..906b7938 --- /dev/null +++ b/cmd/batch-rotate_gen_test.go @@ -0,0 +1,575 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalBatchJobKeyRotateEncryption(t *testing.T) { + v := BatchJobKeyRotateEncryption{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBatchJobKeyRotateEncryption(b *testing.B) { + v := BatchJobKeyRotateEncryption{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBatchJobKeyRotateEncryption(b *testing.B) { + v := BatchJobKeyRotateEncryption{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBatchJobKeyRotateEncryption(b *testing.B) { + v := BatchJobKeyRotateEncryption{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBatchJobKeyRotateEncryption(t *testing.T) { + v := BatchJobKeyRotateEncryption{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBatchJobKeyRotateEncryption Msgsize() is inaccurate") + } + + vn := BatchJobKeyRotateEncryption{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBatchJobKeyRotateEncryption(b *testing.B) { + v := BatchJobKeyRotateEncryption{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBatchJobKeyRotateEncryption(b *testing.B) { + v := BatchJobKeyRotateEncryption{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBatchJobKeyRotateFlags(t *testing.T) { + v := BatchJobKeyRotateFlags{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBatchJobKeyRotateFlags(b *testing.B) { + v := BatchJobKeyRotateFlags{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBatchJobKeyRotateFlags(b *testing.B) { + v := BatchJobKeyRotateFlags{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBatchJobKeyRotateFlags(b *testing.B) { + v := BatchJobKeyRotateFlags{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBatchJobKeyRotateFlags(t *testing.T) { + v := BatchJobKeyRotateFlags{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBatchJobKeyRotateFlags Msgsize() is inaccurate") + } + + vn := BatchJobKeyRotateFlags{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBatchJobKeyRotateFlags(b *testing.B) { + v := BatchJobKeyRotateFlags{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBatchJobKeyRotateFlags(b *testing.B) { + v := BatchJobKeyRotateFlags{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBatchJobKeyRotateV1(t *testing.T) { + v := BatchJobKeyRotateV1{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBatchJobKeyRotateV1(b *testing.B) { + v := BatchJobKeyRotateV1{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBatchJobKeyRotateV1(b *testing.B) { + v := BatchJobKeyRotateV1{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBatchJobKeyRotateV1(b *testing.B) { + v := BatchJobKeyRotateV1{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBatchJobKeyRotateV1(t *testing.T) { + v := BatchJobKeyRotateV1{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBatchJobKeyRotateV1 Msgsize() is inaccurate") + } + + vn := BatchJobKeyRotateV1{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBatchJobKeyRotateV1(b *testing.B) { + v := BatchJobKeyRotateV1{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBatchJobKeyRotateV1(b *testing.B) { + v := BatchJobKeyRotateV1{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBatchKeyRotateFilter(t *testing.T) { + v := BatchKeyRotateFilter{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBatchKeyRotateFilter(b *testing.B) { + v := BatchKeyRotateFilter{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBatchKeyRotateFilter(b *testing.B) { + v := BatchKeyRotateFilter{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBatchKeyRotateFilter(b *testing.B) { + v := BatchKeyRotateFilter{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBatchKeyRotateFilter(t *testing.T) { + v := BatchKeyRotateFilter{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBatchKeyRotateFilter Msgsize() is inaccurate") + } + + vn := BatchKeyRotateFilter{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBatchKeyRotateFilter(b *testing.B) { + v := BatchKeyRotateFilter{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBatchKeyRotateFilter(b *testing.B) { + v := BatchKeyRotateFilter{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBatchKeyRotateNotification(t *testing.T) { + v := BatchKeyRotateNotification{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBatchKeyRotateNotification(b *testing.B) { + v := BatchKeyRotateNotification{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBatchKeyRotateNotification(b *testing.B) { + v := BatchKeyRotateNotification{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBatchKeyRotateNotification(b *testing.B) { + v := BatchKeyRotateNotification{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBatchKeyRotateNotification(t *testing.T) { + v := BatchKeyRotateNotification{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBatchKeyRotateNotification Msgsize() is inaccurate") + } + + vn := BatchKeyRotateNotification{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBatchKeyRotateNotification(b *testing.B) { + v := BatchKeyRotateNotification{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBatchKeyRotateNotification(b *testing.B) { + v := BatchKeyRotateNotification{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/batchjobmetric_string.go b/cmd/batchjobmetric_string.go new file mode 100644 index 00000000..a1697a19 --- /dev/null +++ b/cmd/batchjobmetric_string.go @@ -0,0 +1,25 @@ +// Code generated by "stringer -type=batchJobMetric -trimprefix=batchJobMetric batch-handlers.go"; DO NOT EDIT. + +package cmd + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[batchJobMetricReplication-0] + _ = x[batchJobMetricKeyRotation-1] + _ = x[batchJobMetricExpire-2] +} + +const _batchJobMetric_name = "ReplicationKeyRotationExpire" + +var _batchJobMetric_index = [...]uint8{0, 11, 22, 28} + +func (i batchJobMetric) String() string { + if i >= batchJobMetric(len(_batchJobMetric_index)-1) { + return "batchJobMetric(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _batchJobMetric_name[_batchJobMetric_index[i]:_batchJobMetric_index[i+1]] +} diff --git a/cmd/benchmark-utils_test.go b/cmd/benchmark-utils_test.go new file mode 100644 index 00000000..0f7025a9 --- /dev/null +++ b/cmd/benchmark-utils_test.go @@ -0,0 +1,235 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "math" + "math/rand" + "strconv" + "testing" + + "github.com/dustin/go-humanize" +) + +// Benchmark utility functions for ObjectLayer.PutObject(). +// Creates Object layer setup ( MakeBucket ) and then runs the PutObject benchmark. +func runPutObjectBenchmark(b *testing.B, obj ObjectLayer, objSize int) { + var err error + // obtains random bucket name. + bucket := getRandomBucketName() + // create bucket. + err = obj.MakeBucket(b.Context(), bucket, MakeBucketOptions{}) + if err != nil { + b.Fatal(err) + } + + // get text data generated for number of bytes equal to object size. + textData := generateBytesData(objSize) + // generate md5sum for the generated data. + // md5sum of the data to written is required as input for PutObject. + + md5hex := getMD5Hash(textData) + sha256hex := "" + + // benchmark utility which helps obtain number of allocations and bytes allocated per ops. + b.ReportAllocs() + // the actual benchmark for PutObject starts here. Reset the benchmark timer. + + for i := 0; b.Loop(); i++ { + // insert the object. + objInfo, err := obj.PutObject(b.Context(), bucket, "object"+strconv.Itoa(i), + mustGetPutObjReader(b, bytes.NewReader(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{}) + if err != nil { + b.Fatal(err) + } + if objInfo.ETag != md5hex { + b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, objInfo.ETag, md5hex) + } + } + // Benchmark ends here. Stop timer. + b.StopTimer() +} + +// Benchmark utility functions for ObjectLayer.PutObjectPart(). +// Creates Object layer setup ( MakeBucket ) and then runs the PutObjectPart benchmark. +func runPutObjectPartBenchmark(b *testing.B, obj ObjectLayer, partSize int) { + var err error + // obtains random bucket name. + bucket := getRandomBucketName() + object := getRandomObjectName() + + // create bucket. + err = obj.MakeBucket(b.Context(), bucket, MakeBucketOptions{}) + if err != nil { + b.Fatal(err) + } + + objSize := 128 * humanize.MiByte + + // PutObjectPart returns etag of the object inserted. + // etag variable is assigned with that value. + var etag string + // get text data generated for number of bytes equal to object size. + textData := generateBytesData(objSize) + // generate md5sum for the generated data. + // md5sum of the data to written is required as input for NewMultipartUpload. + res, err := obj.NewMultipartUpload(b.Context(), bucket, object, ObjectOptions{}) + if err != nil { + b.Fatal(err) + } + + sha256hex := "" + + var textPartData []byte + // benchmark utility which helps obtain number of allocations and bytes allocated per ops. + b.ReportAllocs() + // the actual benchmark for PutObjectPart starts here. Reset the benchmark timer. + + for i := 0; b.Loop(); i++ { + // insert the object. + totalPartsNR := int(math.Ceil(float64(objSize) / float64(partSize))) + for j := range totalPartsNR { + if j < totalPartsNR-1 { + textPartData = textData[j*partSize : (j+1)*partSize-1] + } else { + textPartData = textData[j*partSize:] + } + md5hex := getMD5Hash(textPartData) + var partInfo PartInfo + partInfo, err = obj.PutObjectPart(b.Context(), bucket, object, res.UploadID, j, + mustGetPutObjReader(b, bytes.NewReader(textPartData), int64(len(textPartData)), md5hex, sha256hex), ObjectOptions{}) + if err != nil { + b.Fatal(err) + } + if partInfo.ETag != md5hex { + b.Fatalf("Write no: %d: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", i+1, etag, md5hex) + } + } + } + // Benchmark ends here. Stop timer. + b.StopTimer() +} + +// creates Erasure/FS backend setup, obtains the object layer and calls the runPutObjectPartBenchmark function. +func benchmarkPutObjectPart(b *testing.B, instanceType string, objSize int) { + // create a temp Erasure/FS backend. + ctx, cancel := context.WithCancel(b.Context()) + defer cancel() + objLayer, disks, err := prepareTestBackend(ctx, instanceType) + if err != nil { + b.Fatalf("Failed obtaining Temp Backend: %s", err) + } + // cleaning up the backend by removing all the directories and files created on function return. + defer removeRoots(disks) + + // uses *testing.B and the object Layer to run the benchmark. + runPutObjectPartBenchmark(b, objLayer, objSize) +} + +// creates Erasure/FS backend setup, obtains the object layer and calls the runPutObjectBenchmark function. +func benchmarkPutObject(b *testing.B, instanceType string, objSize int) { + // create a temp Erasure/FS backend. + ctx, cancel := context.WithCancel(b.Context()) + defer cancel() + objLayer, disks, err := prepareTestBackend(ctx, instanceType) + if err != nil { + b.Fatalf("Failed obtaining Temp Backend: %s", err) + } + // cleaning up the backend by removing all the directories and files created on function return. + defer removeRoots(disks) + + // uses *testing.B and the object Layer to run the benchmark. + runPutObjectBenchmark(b, objLayer, objSize) +} + +// creates Erasure/FS backend setup, obtains the object layer and runs parallel benchmark for put object. +func benchmarkPutObjectParallel(b *testing.B, instanceType string, objSize int) { + // create a temp Erasure/FS backend. + ctx, cancel := context.WithCancel(b.Context()) + defer cancel() + objLayer, disks, err := prepareTestBackend(ctx, instanceType) + if err != nil { + b.Fatalf("Failed obtaining Temp Backend: %s", err) + } + // cleaning up the backend by removing all the directories and files created on function return. + defer removeRoots(disks) + + // uses *testing.B and the object Layer to run the benchmark. + runPutObjectBenchmarkParallel(b, objLayer, objSize) +} + +// randomly picks a character and returns its equivalent byte array. +func getRandomByte() []byte { + const letterBytes = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" + // seeding the random number generator. + rand.Seed(UTCNow().UnixNano()) + // pick a character randomly. + return []byte{letterBytes[rand.Intn(len(letterBytes))]} +} + +// picks a random byte and repeats it to size bytes. +func generateBytesData(size int) []byte { + // repeat the random character chosen size + return bytes.Repeat(getRandomByte(), size) +} + +// Parallel benchmark utility functions for ObjectLayer.PutObject(). +// Creates Object layer setup ( MakeBucket ) and then runs the PutObject benchmark. +func runPutObjectBenchmarkParallel(b *testing.B, obj ObjectLayer, objSize int) { + // obtains random bucket name. + bucket := getRandomBucketName() + // create bucket. + err := obj.MakeBucket(b.Context(), bucket, MakeBucketOptions{}) + if err != nil { + b.Fatal(err) + } + + // get text data generated for number of bytes equal to object size. + textData := generateBytesData(objSize) + // generate md5sum for the generated data. + // md5sum of the data to written is required as input for PutObject. + + md5hex := getMD5Hash(textData) + sha256hex := "" + + // benchmark utility which helps obtain number of allocations and bytes allocated per ops. + b.ReportAllocs() + // the actual benchmark for PutObject starts here. Reset the benchmark timer. + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + i := 0 + for pb.Next() { + // insert the object. + objInfo, err := obj.PutObject(b.Context(), bucket, "object"+strconv.Itoa(i), + mustGetPutObjReader(b, bytes.NewReader(textData), int64(len(textData)), md5hex, sha256hex), ObjectOptions{}) + if err != nil { + b.Fatal(err) + } + if objInfo.ETag != md5hex { + b.Fatalf("Write no: Md5Sum mismatch during object write into the bucket: Expected %s, got %s", objInfo.ETag, md5hex) + } + i++ + } + }) + + // Benchmark ends here. Stop timer. + b.StopTimer() +} diff --git a/cmd/bitrot-streaming.go b/cmd/bitrot-streaming.go new file mode 100644 index 00000000..7c1a313b --- /dev/null +++ b/cmd/bitrot-streaming.go @@ -0,0 +1,215 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "errors" + "hash" + "io" + "sync" + + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/ringbuffer" +) + +// Calculates bitrot in chunks and writes the hash into the stream. +type streamingBitrotWriter struct { + iow io.WriteCloser + closeWithErr func(err error) + h hash.Hash + shardSize int64 + canClose *sync.WaitGroup + byteBuf []byte + finished bool +} + +func (b *streamingBitrotWriter) Write(p []byte) (int, error) { + if len(p) == 0 { + return 0, nil + } + if b.finished { + return 0, errors.New("bitrot write not allowed") + } + if int64(len(p)) > b.shardSize { + return 0, errors.New("unexpected bitrot buffer size") + } + if int64(len(p)) < b.shardSize { + b.finished = true + } + b.h.Reset() + b.h.Write(p) + hashBytes := b.h.Sum(nil) + _, err := b.iow.Write(hashBytes) + if err != nil { + b.closeWithErr(err) + return 0, err + } + n, err := b.iow.Write(p) + if err != nil { + b.closeWithErr(err) + return n, err + } + if n != len(p) { + err = io.ErrShortWrite + b.closeWithErr(err) + } + return n, err +} + +func (b *streamingBitrotWriter) Close() error { + // Close the underlying writer. + // This will also flush the ring buffer if used. + err := b.iow.Close() + + // Wait for all data to be written before returning else it causes race conditions. + // Race condition is because of io.PipeWriter implementation. i.e consider the following + // sequent of operations: + // 1) pipe.Write() + // 2) pipe.Close() + // Now pipe.Close() can return before the data is read on the other end of the pipe and written to the disk + // Hence an immediate Read() on the file can return incorrect data. + if b.canClose != nil { + b.canClose.Wait() + } + + // Recycle the buffer. + if b.byteBuf != nil { + globalBytePoolCap.Load().Put(b.byteBuf) + b.byteBuf = nil + } + return err +} + +// newStreamingBitrotWriterBuffer returns streaming bitrot writer implementation. +// The output is written to the supplied writer w. +func newStreamingBitrotWriterBuffer(w io.Writer, algo BitrotAlgorithm, shardSize int64) io.Writer { + return &streamingBitrotWriter{iow: ioutil.NopCloser(w), h: algo.New(), shardSize: shardSize, canClose: nil, closeWithErr: func(err error) {}} +} + +// Returns streaming bitrot writer implementation. +func newStreamingBitrotWriter(disk StorageAPI, origvolume, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.Writer { + h := algo.New() + buf := globalBytePoolCap.Load().Get() + rb := ringbuffer.NewBuffer(buf[:cap(buf)]).SetBlocking(true) + + bw := &streamingBitrotWriter{ + iow: ioutil.NewDeadlineWriter(rb.WriteCloser(), globalDriveConfig.GetMaxTimeout()), + closeWithErr: rb.CloseWithError, + h: h, + shardSize: shardSize, + canClose: &sync.WaitGroup{}, + byteBuf: buf, + } + bw.canClose.Add(1) + go func() { + defer bw.canClose.Done() + + totalFileSize := int64(-1) // For compressed objects length will be unknown (represented by length=-1) + if length != -1 { + bitrotSumsTotalSize := ceilFrac(length, shardSize) * int64(h.Size()) // Size used for storing bitrot checksums. + totalFileSize = bitrotSumsTotalSize + length + } + rb.CloseWithError(disk.CreateFile(context.TODO(), origvolume, volume, filePath, totalFileSize, rb)) + }() + return bw +} + +// ReadAt() implementation which verifies the bitrot hash available as part of the stream. +type streamingBitrotReader struct { + disk StorageAPI + data []byte + rc io.Reader + volume string + filePath string + tillOffset int64 + currOffset int64 + h hash.Hash + shardSize int64 + hashBytes []byte +} + +func (b *streamingBitrotReader) Close() error { + if b.rc == nil { + return nil + } + if closer, ok := b.rc.(io.Closer); ok { + // drain the body for connection reuse at network layer. + xhttp.DrainBody(io.NopCloser(b.rc)) + return closer.Close() + } + return nil +} + +func (b *streamingBitrotReader) ReadAt(buf []byte, offset int64) (int, error) { + var err error + if offset%b.shardSize != 0 { + // Offset should always be aligned to b.shardSize + // Can never happen unless there are programmer bugs + return 0, errUnexpected + } + if b.rc == nil { + // For the first ReadAt() call we need to open the stream for reading. + b.currOffset = offset + streamOffset := (offset/b.shardSize)*int64(b.h.Size()) + offset + if len(b.data) == 0 && b.tillOffset != streamOffset { + b.rc, err = b.disk.ReadFileStream(context.TODO(), b.volume, b.filePath, streamOffset, b.tillOffset-streamOffset) + } else { + b.rc = io.NewSectionReader(bytes.NewReader(b.data), streamOffset, b.tillOffset-streamOffset) + } + if err != nil { + return 0, err + } + } + if offset != b.currOffset { + // Can never happen unless there are programmer bugs + return 0, errUnexpected + } + b.h.Reset() + _, err = io.ReadFull(b.rc, b.hashBytes) + if err != nil { + return 0, err + } + _, err = io.ReadFull(b.rc, buf) + if err != nil { + return 0, err + } + b.h.Write(buf) + if !bytes.Equal(b.h.Sum(nil), b.hashBytes) { + return 0, errFileCorrupt + } + b.currOffset += int64(len(buf)) + return len(buf), nil +} + +// Returns streaming bitrot reader implementation. +func newStreamingBitrotReader(disk StorageAPI, data []byte, volume, filePath string, tillOffset int64, algo BitrotAlgorithm, shardSize int64) *streamingBitrotReader { + h := algo.New() + return &streamingBitrotReader{ + disk: disk, + data: data, + volume: volume, + filePath: filePath, + tillOffset: ceilFrac(tillOffset, shardSize)*int64(h.Size()) + tillOffset, + h: h, + shardSize: shardSize, + hashBytes: make([]byte, h.Size()), + } +} diff --git a/cmd/bitrot-whole.go b/cmd/bitrot-whole.go new file mode 100644 index 00000000..3278b1c7 --- /dev/null +++ b/cmd/bitrot-whole.go @@ -0,0 +1,91 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "hash" + "io" +) + +// Implementation to calculate bitrot for the whole file. +type wholeBitrotWriter struct { + disk StorageAPI + volume string + filePath string + shardSize int64 // This is the shard size of the erasure logic + hash.Hash // For bitrot hash +} + +func (b *wholeBitrotWriter) Write(p []byte) (int, error) { + err := b.disk.AppendFile(context.TODO(), b.volume, b.filePath, p) + if err != nil { + return 0, err + } + _, err = b.Hash.Write(p) + if err != nil { + return 0, err + } + return len(p), nil +} + +func (b *wholeBitrotWriter) Close() error { + return nil +} + +// Returns whole-file bitrot writer. +func newWholeBitrotWriter(disk StorageAPI, volume, filePath string, algo BitrotAlgorithm, shardSize int64) io.WriteCloser { + return &wholeBitrotWriter{disk, volume, filePath, shardSize, algo.New()} +} + +// Implementation to verify bitrot for the whole file. +type wholeBitrotReader struct { + disk StorageAPI + volume string + filePath string + verifier *BitrotVerifier // Holds the bit-rot info + tillOffset int64 // Affects the length of data requested in disk.ReadFile depending on Read()'s offset + buf []byte // Holds bit-rot verified data +} + +func (b *wholeBitrotReader) ReadAt(buf []byte, offset int64) (n int, err error) { + if b.buf == nil { + b.buf = make([]byte, b.tillOffset-offset) + if _, err := b.disk.ReadFile(context.TODO(), b.volume, b.filePath, offset, b.buf, b.verifier); err != nil { + return 0, err + } + } + if len(b.buf) < len(buf) { + return 0, errLessData + } + n = copy(buf, b.buf) + b.buf = b.buf[n:] + return n, nil +} + +// Returns whole-file bitrot reader. +func newWholeBitrotReader(disk StorageAPI, volume, filePath string, algo BitrotAlgorithm, tillOffset int64, sum []byte) *wholeBitrotReader { + return &wholeBitrotReader{ + disk: disk, + volume: volume, + filePath: filePath, + verifier: &BitrotVerifier{algo, sum}, + tillOffset: tillOffset, + buf: nil, + } +} diff --git a/cmd/bitrot.go b/cmd/bitrot.go new file mode 100644 index 00000000..f1b2a0e4 --- /dev/null +++ b/cmd/bitrot.go @@ -0,0 +1,255 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "encoding/hex" + "errors" + "fmt" + "hash" + "io" + + "github.com/minio/highwayhash" + "github.com/minio/minio/internal/hash/sha256" + "golang.org/x/crypto/blake2b" + + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/logger" +) + +// magic HH-256 key as HH-256 hash of the first 100 decimals of π as utf-8 string with a zero key. +var magicHighwayHash256Key = []byte("\x4b\xe7\x34\xfa\x8e\x23\x8a\xcd\x26\x3e\x83\xe6\xbb\x96\x85\x52\x04\x0f\x93\x5d\xa3\x9f\x44\x14\x97\xe0\x9d\x13\x22\xde\x36\xa0") + +var bitrotAlgorithms = map[BitrotAlgorithm]string{ + SHA256: "sha256", + BLAKE2b512: "blake2b", + HighwayHash256: "highwayhash256", + HighwayHash256S: "highwayhash256S", +} + +// New returns a new hash.Hash calculating the given bitrot algorithm. +func (a BitrotAlgorithm) New() hash.Hash { + switch a { + case SHA256: + return sha256.New() + case BLAKE2b512: + b2, _ := blake2b.New512(nil) // New512 never returns an error if the key is nil + return b2 + case HighwayHash256: + hh, _ := highwayhash.New(magicHighwayHash256Key) // New will never return error since key is 256 bit + return hh + case HighwayHash256S: + hh, _ := highwayhash.New(magicHighwayHash256Key) // New will never return error since key is 256 bit + return hh + default: + logger.CriticalIf(GlobalContext, errors.New("Unsupported bitrot algorithm")) + return nil + } +} + +// Available reports whether the given algorithm is available. +func (a BitrotAlgorithm) Available() bool { + _, ok := bitrotAlgorithms[a] + return ok +} + +// String returns the string identifier for a given bitrot algorithm. +// If the algorithm is not supported String panics. +func (a BitrotAlgorithm) String() string { + name, ok := bitrotAlgorithms[a] + if !ok { + logger.CriticalIf(GlobalContext, errors.New("Unsupported bitrot algorithm")) + } + return name +} + +// NewBitrotVerifier returns a new BitrotVerifier implementing the given algorithm. +func NewBitrotVerifier(algorithm BitrotAlgorithm, checksum []byte) *BitrotVerifier { + return &BitrotVerifier{algorithm, checksum} +} + +// BitrotVerifier can be used to verify protected data. +type BitrotVerifier struct { + algorithm BitrotAlgorithm + sum []byte +} + +// BitrotAlgorithmFromString returns a bitrot algorithm from the given string representation. +// It returns 0 if the string representation does not match any supported algorithm. +// The zero value of a bitrot algorithm is never supported. +func BitrotAlgorithmFromString(s string) (a BitrotAlgorithm) { + for alg, name := range bitrotAlgorithms { + if name == s { + return alg + } + } + return a +} + +func newBitrotWriter(disk StorageAPI, origvolume, volume, filePath string, length int64, algo BitrotAlgorithm, shardSize int64) io.Writer { + if algo == HighwayHash256S { + return newStreamingBitrotWriter(disk, origvolume, volume, filePath, length, algo, shardSize) + } + return newWholeBitrotWriter(disk, volume, filePath, algo, shardSize) +} + +func newBitrotReader(disk StorageAPI, data []byte, bucket string, filePath string, tillOffset int64, algo BitrotAlgorithm, sum []byte, shardSize int64) io.ReaderAt { + if algo == HighwayHash256S { + return newStreamingBitrotReader(disk, data, bucket, filePath, tillOffset, algo, shardSize) + } + return newWholeBitrotReader(disk, bucket, filePath, algo, tillOffset, sum) +} + +// Close all the readers. +func closeBitrotReaders(rs []io.ReaderAt) { + for _, r := range rs { + if r != nil { + if br, ok := r.(io.Closer); ok { + br.Close() + } + } + } +} + +// Close all the writers. +func closeBitrotWriters(ws []io.Writer) []error { + errs := make([]error, len(ws)) + for i, w := range ws { + if w == nil { + errs[i] = errDiskNotFound + continue + } + if bw, ok := w.(io.Closer); ok { + errs[i] = bw.Close() + } else { + errs[i] = nil + } + } + return errs +} + +// Returns hash sum for whole-bitrot, nil for streaming-bitrot. +func bitrotWriterSum(w io.Writer) []byte { + if bw, ok := w.(*wholeBitrotWriter); ok { + return bw.Sum(nil) + } + return nil +} + +// Returns the size of the file with bitrot protection +func bitrotShardFileSize(size int64, shardSize int64, algo BitrotAlgorithm) int64 { + if algo != HighwayHash256S { + return size + } + return ceilFrac(size, shardSize)*int64(algo.New().Size()) + size +} + +// bitrotVerify a single stream of data. +func bitrotVerify(r io.Reader, wantSize, partSize int64, algo BitrotAlgorithm, want []byte, shardSize int64) error { + if algo != HighwayHash256S { + h := algo.New() + if n, err := io.Copy(h, r); err != nil || n != wantSize { + // Premature failure in reading the object, file is corrupt. + return errFileCorrupt + } + if !bytes.Equal(h.Sum(nil), want) { + return errFileCorrupt + } + return nil + } + + h := algo.New() + hashBuf := make([]byte, h.Size()) + left := wantSize + + // Calculate the size of the bitrot file and compare + // it with the actual file size. + if left != bitrotShardFileSize(partSize, shardSize, algo) { + return errFileCorrupt + } + + bufp := xioutil.ODirectPoolSmall.Get() + defer xioutil.ODirectPoolSmall.Put(bufp) + + for left > 0 { + // Read expected hash... + h.Reset() + n, err := io.ReadFull(r, hashBuf) + if err != nil { + // Read's failed for object with right size, file is corrupt. + return err + } + // Subtract hash length.. + left -= int64(n) + if left < shardSize { + shardSize = left + } + + read, err := io.CopyBuffer(h, io.LimitReader(r, shardSize), *bufp) + if err != nil { + // Read's failed for object with right size, at different offsets. + return errFileCorrupt + } + + left -= read + if !bytes.Equal(h.Sum(nil), hashBuf[:n]) { + return errFileCorrupt + } + } + return nil +} + +// bitrotSelfTest performs a self-test to ensure that bitrot +// algorithms compute correct checksums. If any algorithm +// produces an incorrect checksum it fails with a hard error. +// +// bitrotSelfTest tries to catch any issue in the bitrot implementation +// early instead of silently corrupting data. +func bitrotSelfTest() { + checksums := map[BitrotAlgorithm]string{ + SHA256: "a7677ff19e0182e4d52e3a3db727804abc82a5818749336369552e54b838b004", + BLAKE2b512: "e519b7d84b1c3c917985f544773a35cf265dcab10948be3550320d156bab612124a5ae2ae5a8c73c0eea360f68b0e28136f26e858756dbfe7375a7389f26c669", + HighwayHash256: "39c0407ed3f01b18d22c85db4aeff11e060ca5f43131b0126731ca197cd42313", + HighwayHash256S: "39c0407ed3f01b18d22c85db4aeff11e060ca5f43131b0126731ca197cd42313", + } + for algorithm := range bitrotAlgorithms { + if !algorithm.Available() { + continue + } + + checksum, err := hex.DecodeString(checksums[algorithm]) + if err != nil { + logger.Fatal(errSelfTestFailure, fmt.Sprintf("bitrot: failed to decode %v checksum %s for selftest: %v", algorithm, checksums[algorithm], err)) + } + var ( + hash = algorithm.New() + msg = make([]byte, 0, hash.Size()*hash.BlockSize()) + sum = make([]byte, 0, hash.Size()) + ) + for i := 0; i < hash.Size()*hash.BlockSize(); i += hash.Size() { + hash.Write(msg) + sum = hash.Sum(sum[:0]) + msg = append(msg, sum...) + hash.Reset() + } + if !bytes.Equal(sum, checksum) { + logger.Fatal(errSelfTestFailure, fmt.Sprintf("bitrot: %v selftest checksum mismatch: got %x - want %x", algorithm, sum, checksum)) + } + } +} diff --git a/cmd/bitrot_test.go b/cmd/bitrot_test.go new file mode 100644 index 00000000..636d1873 --- /dev/null +++ b/cmd/bitrot_test.go @@ -0,0 +1,83 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "io" + "testing" +) + +func testBitrotReaderWriterAlgo(t *testing.T, bitrotAlgo BitrotAlgorithm) { + tmpDir := t.TempDir() + + volume := "testvol" + filePath := "testfile" + + disk, err := newLocalXLStorage(tmpDir) + if err != nil { + t.Fatal(err) + } + + disk.MakeVol(t.Context(), volume) + + writer := newBitrotWriter(disk, "", volume, filePath, 35, bitrotAlgo, 10) + + _, err = writer.Write([]byte("aaaaaaaaaa")) + if err != nil { + t.Fatal(err) + } + _, err = writer.Write([]byte("aaaaaaaaaa")) + if err != nil { + t.Fatal(err) + } + _, err = writer.Write([]byte("aaaaaaaaaa")) + if err != nil { + t.Fatal(err) + } + _, err = writer.Write([]byte("aaaaa")) + if err != nil { + t.Fatal(err) + } + if bw, ok := writer.(io.Closer); ok { + bw.Close() + } + + reader := newBitrotReader(disk, nil, volume, filePath, 35, bitrotAlgo, bitrotWriterSum(writer), 10) + b := make([]byte, 10) + if _, err = reader.ReadAt(b, 0); err != nil { + t.Fatal(err) + } + if _, err = reader.ReadAt(b, 10); err != nil { + t.Fatal(err) + } + if _, err = reader.ReadAt(b, 20); err != nil { + t.Fatal(err) + } + if _, err = reader.ReadAt(b[:5], 30); err != nil { + t.Fatal(err) + } + if br, ok := reader.(io.Closer); ok { + br.Close() + } +} + +func TestAllBitrotAlgorithms(t *testing.T) { + for bitrotAlgo := range bitrotAlgorithms { + testBitrotReaderWriterAlgo(t, bitrotAlgo) + } +} diff --git a/cmd/bootstrap-messages.go b/cmd/bootstrap-messages.go new file mode 100644 index 00000000..c01b4529 --- /dev/null +++ b/cmd/bootstrap-messages.go @@ -0,0 +1,67 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "sync" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/pubsub" +) + +const bootstrapTraceLimit = 4 << 10 + +type bootstrapTracer struct { + mu sync.RWMutex + info []madmin.TraceInfo +} + +var globalBootstrapTracer = &bootstrapTracer{} + +func (bs *bootstrapTracer) Record(info madmin.TraceInfo) { + bs.mu.Lock() + defer bs.mu.Unlock() + + if len(bs.info) > bootstrapTraceLimit { + return + } + bs.info = append(bs.info, info) +} + +func (bs *bootstrapTracer) Events() []madmin.TraceInfo { + traceInfo := make([]madmin.TraceInfo, 0, bootstrapTraceLimit) + + bs.mu.RLock() + traceInfo = append(traceInfo, bs.info...) + bs.mu.RUnlock() + + return traceInfo +} + +func (bs *bootstrapTracer) Publish(ctx context.Context, trace *pubsub.PubSub[madmin.TraceInfo, madmin.TraceType]) { + for _, bsEvent := range bs.Events() { + if bsEvent.Message != "" { + select { + case <-ctx.Done(): + default: + trace.Publish(bsEvent) + } + } + } +} diff --git a/cmd/bootstrap-peer-server.go b/cmd/bootstrap-peer-server.go new file mode 100644 index 00000000..4fb179bb --- /dev/null +++ b/cmd/bootstrap-peer-server.go @@ -0,0 +1,288 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "crypto/md5" + "encoding/hex" + "errors" + "fmt" + "io" + "math/rand" + "os" + "reflect" + "strings" + "sync" + "time" + + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/grid" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/env" +) + +// To abstract a node over network. +type bootstrapRESTServer struct{} + +//go:generate msgp -file=$GOFILE + +// ServerSystemConfig - captures information about server configuration. +type ServerSystemConfig struct { + NEndpoints int + CmdLines []string + MinioEnv map[string]string + Checksum string +} + +// Diff - returns error on first difference found in two configs. +func (s1 *ServerSystemConfig) Diff(s2 *ServerSystemConfig) error { + if s1.Checksum != s2.Checksum { + return fmt.Errorf("Expected MinIO binary checksum: %s, seen: %s", s1.Checksum, s2.Checksum) + } + + ns1 := s1.NEndpoints + ns2 := s2.NEndpoints + if ns1 != ns2 { + return fmt.Errorf("Expected number of endpoints %d, seen %d", ns1, ns2) + } + + for i, cmdLine := range s1.CmdLines { + if cmdLine != s2.CmdLines[i] { + return fmt.Errorf("Expected command line argument %s, seen %s", cmdLine, + s2.CmdLines[i]) + } + } + + if reflect.DeepEqual(s1.MinioEnv, s2.MinioEnv) { + return nil + } + + // Report differences in environment variables. + var missing []string + var mismatching []string + for k, v := range s1.MinioEnv { + ev, ok := s2.MinioEnv[k] + if !ok { + missing = append(missing, k) + } else if v != ev { + mismatching = append(mismatching, k) + } + } + var extra []string + for k := range s2.MinioEnv { + _, ok := s1.MinioEnv[k] + if !ok { + extra = append(extra, k) + } + } + msg := "Expected MINIO_* environment name and values across all servers to be same: " + if len(missing) > 0 { + msg += fmt.Sprintf(`Missing environment values: %v. `, missing) + } + if len(mismatching) > 0 { + msg += fmt.Sprintf(`Mismatching environment values: %v. `, mismatching) + } + if len(extra) > 0 { + msg += fmt.Sprintf(`Extra environment values: %v. `, extra) + } + + return errors.New(strings.TrimSpace(msg)) +} + +var skipEnvs = map[string]struct{}{ + "MINIO_OPTS": {}, + "MINIO_CERT_PASSWD": {}, + "MINIO_SERVER_DEBUG": {}, + "MINIO_DSYNC_TRACE": {}, + "MINIO_ROOT_USER": {}, + "MINIO_ROOT_PASSWORD": {}, + "MINIO_ACCESS_KEY": {}, + "MINIO_SECRET_KEY": {}, + "MINIO_OPERATOR_VERSION": {}, + "MINIO_VSPHERE_PLUGIN_VERSION": {}, + "MINIO_CI_CD": {}, +} + +func getServerSystemCfg() *ServerSystemConfig { + envs := env.List("MINIO_") + envValues := make(map[string]string, len(envs)) + for _, envK := range envs { + // skip certain environment variables as part + // of the whitelist and could be configured + // differently on each nodes, update skipEnvs() + // map if there are such environment values + if _, ok := skipEnvs[envK]; ok { + continue + } + envValues[envK] = logger.HashString(env.Get(envK, "")) + } + scfg := &ServerSystemConfig{NEndpoints: globalEndpoints.NEndpoints(), MinioEnv: envValues, Checksum: binaryChecksum} + var cmdLines []string + for _, ep := range globalEndpoints { + cmdLines = append(cmdLines, ep.CmdLine) + } + scfg.CmdLines = cmdLines + return scfg +} + +func (s *bootstrapRESTServer) VerifyHandler(params *grid.MSS) (*ServerSystemConfig, *grid.RemoteErr) { + return getServerSystemCfg(), nil +} + +var serverVerifyHandler = grid.NewSingleHandler[*grid.MSS, *ServerSystemConfig](grid.HandlerServerVerify, grid.NewMSS, func() *ServerSystemConfig { return &ServerSystemConfig{} }) + +// registerBootstrapRESTHandlers - register bootstrap rest router. +func registerBootstrapRESTHandlers(gm *grid.Manager) { + server := &bootstrapRESTServer{} + logger.FatalIf(serverVerifyHandler.Register(gm, server.VerifyHandler), "unable to register handler") +} + +// client to talk to bootstrap NEndpoints. +type bootstrapRESTClient struct { + gridConn *grid.Connection +} + +// Verify function verifies the server config. +func (client *bootstrapRESTClient) Verify(ctx context.Context, srcCfg *ServerSystemConfig) (err error) { + if newObjectLayerFn() != nil { + return nil + } + + recvCfg, err := serverVerifyHandler.Call(ctx, client.gridConn, grid.NewMSS()) + if err != nil { + return err + } + // We do not need the response after returning. + defer serverVerifyHandler.PutResponse(recvCfg) + + return srcCfg.Diff(recvCfg) +} + +// Stringer provides a canonicalized representation of node. +func (client *bootstrapRESTClient) String() string { + return client.gridConn.String() +} + +var binaryChecksum = getBinaryChecksum() + +func getBinaryChecksum() string { + mw := md5.New() + binPath, err := os.Executable() + if err != nil { + logger.Error("Calculating checksum failed: %s", err) + return "00000000000000000000000000000000" + } + b, err := os.Open(binPath) + if err != nil { + logger.Error("Calculating checksum failed: %s", err) + return "00000000000000000000000000000000" + } + + defer b.Close() + io.Copy(mw, b) + return hex.EncodeToString(mw.Sum(nil)) +} + +func verifyServerSystemConfig(ctx context.Context, endpointServerPools EndpointServerPools, gm *grid.Manager) error { + srcCfg := getServerSystemCfg() + clnts := newBootstrapRESTClients(endpointServerPools, gm) + var onlineServers int + var offlineEndpoints []error + var incorrectConfigs []error + var retries int + var mu sync.Mutex + for onlineServers < len(clnts)/2 { + var wg sync.WaitGroup + wg.Add(len(clnts)) + onlineServers = 0 + for _, clnt := range clnts { + go func(clnt *bootstrapRESTClient) { + defer wg.Done() + + if clnt.gridConn.State() != grid.StateConnected { + mu.Lock() + offlineEndpoints = append(offlineEndpoints, fmt.Errorf("%s is unreachable: %w", clnt, grid.ErrDisconnected)) + mu.Unlock() + return + } + + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + + err := clnt.Verify(ctx, srcCfg) + mu.Lock() + if err != nil { + bootstrapTraceMsg(fmt.Sprintf("bootstrapVerify: %v, endpoint: %s", err, clnt)) + if !isNetworkError(err) { + bootLogOnceIf(context.Background(), fmt.Errorf("%s has incorrect configuration: %w", clnt, err), "incorrect_"+clnt.String()) + incorrectConfigs = append(incorrectConfigs, fmt.Errorf("%s has incorrect configuration: %w", clnt, err)) + } else { + offlineEndpoints = append(offlineEndpoints, fmt.Errorf("%s is unreachable: %w", clnt, err)) + } + } else { + onlineServers++ + } + mu.Unlock() + }(clnt) + } + wg.Wait() + + select { + case <-ctx.Done(): + return ctx.Err() + default: + // Sleep and stagger to avoid blocked CPU and thundering + // herd upon start up sequence. + time.Sleep(25*time.Millisecond + time.Duration(rand.Int63n(int64(100*time.Millisecond)))) + retries++ + // after 20 retries start logging that servers are not reachable yet + if retries >= 20 { + logger.Info(fmt.Sprintf("Waiting for at least %d remote servers with valid configuration to be online", len(clnts)/2)) + if len(offlineEndpoints) > 0 { + logger.Info(fmt.Sprintf("Following servers are currently offline or unreachable %s", offlineEndpoints)) + } + if len(incorrectConfigs) > 0 { + logger.Info(fmt.Sprintf("Following servers have mismatching configuration %s", incorrectConfigs)) + } + retries = 0 // reset to log again after 20 retries. + } + offlineEndpoints = nil + incorrectConfigs = nil + } + } + return nil +} + +func newBootstrapRESTClients(endpointServerPools EndpointServerPools, gm *grid.Manager) []*bootstrapRESTClient { + seenClient := set.NewStringSet() + var clnts []*bootstrapRESTClient + for _, ep := range endpointServerPools { + for _, endpoint := range ep.Endpoints { + if endpoint.IsLocal { + continue + } + if seenClient.Contains(endpoint.Host) { + continue + } + seenClient.Add(endpoint.Host) + clnts = append(clnts, &bootstrapRESTClient{gm.Connection(endpoint.GridHost())}) + } + } + return clnts +} diff --git a/cmd/bootstrap-peer-server_gen.go b/cmd/bootstrap-peer-server_gen.go new file mode 100644 index 00000000..79b35bbb --- /dev/null +++ b/cmd/bootstrap-peer-server_gen.go @@ -0,0 +1,292 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *ServerSystemConfig) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "NEndpoints": + z.NEndpoints, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "NEndpoints") + return + } + case "CmdLines": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "CmdLines") + return + } + if cap(z.CmdLines) >= int(zb0002) { + z.CmdLines = (z.CmdLines)[:zb0002] + } else { + z.CmdLines = make([]string, zb0002) + } + for za0001 := range z.CmdLines { + z.CmdLines[za0001], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "CmdLines", za0001) + return + } + } + case "MinioEnv": + var zb0003 uint32 + zb0003, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "MinioEnv") + return + } + if z.MinioEnv == nil { + z.MinioEnv = make(map[string]string, zb0003) + } else if len(z.MinioEnv) > 0 { + clear(z.MinioEnv) + } + for zb0003 > 0 { + zb0003-- + var za0002 string + za0002, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "MinioEnv") + return + } + var za0003 string + za0003, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "MinioEnv", za0002) + return + } + z.MinioEnv[za0002] = za0003 + } + case "Checksum": + z.Checksum, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Checksum") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *ServerSystemConfig) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 4 + // write "NEndpoints" + err = en.Append(0x84, 0xaa, 0x4e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteInt(z.NEndpoints) + if err != nil { + err = msgp.WrapError(err, "NEndpoints") + return + } + // write "CmdLines" + err = en.Append(0xa8, 0x43, 0x6d, 0x64, 0x4c, 0x69, 0x6e, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.CmdLines))) + if err != nil { + err = msgp.WrapError(err, "CmdLines") + return + } + for za0001 := range z.CmdLines { + err = en.WriteString(z.CmdLines[za0001]) + if err != nil { + err = msgp.WrapError(err, "CmdLines", za0001) + return + } + } + // write "MinioEnv" + err = en.Append(0xa8, 0x4d, 0x69, 0x6e, 0x69, 0x6f, 0x45, 0x6e, 0x76) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.MinioEnv))) + if err != nil { + err = msgp.WrapError(err, "MinioEnv") + return + } + for za0002, za0003 := range z.MinioEnv { + err = en.WriteString(za0002) + if err != nil { + err = msgp.WrapError(err, "MinioEnv") + return + } + err = en.WriteString(za0003) + if err != nil { + err = msgp.WrapError(err, "MinioEnv", za0002) + return + } + } + // write "Checksum" + err = en.Append(0xa8, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d) + if err != nil { + return + } + err = en.WriteString(z.Checksum) + if err != nil { + err = msgp.WrapError(err, "Checksum") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ServerSystemConfig) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 4 + // string "NEndpoints" + o = append(o, 0x84, 0xaa, 0x4e, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74, 0x73) + o = msgp.AppendInt(o, z.NEndpoints) + // string "CmdLines" + o = append(o, 0xa8, 0x43, 0x6d, 0x64, 0x4c, 0x69, 0x6e, 0x65, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.CmdLines))) + for za0001 := range z.CmdLines { + o = msgp.AppendString(o, z.CmdLines[za0001]) + } + // string "MinioEnv" + o = append(o, 0xa8, 0x4d, 0x69, 0x6e, 0x69, 0x6f, 0x45, 0x6e, 0x76) + o = msgp.AppendMapHeader(o, uint32(len(z.MinioEnv))) + for za0002, za0003 := range z.MinioEnv { + o = msgp.AppendString(o, za0002) + o = msgp.AppendString(o, za0003) + } + // string "Checksum" + o = append(o, 0xa8, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d) + o = msgp.AppendString(o, z.Checksum) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ServerSystemConfig) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "NEndpoints": + z.NEndpoints, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "NEndpoints") + return + } + case "CmdLines": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "CmdLines") + return + } + if cap(z.CmdLines) >= int(zb0002) { + z.CmdLines = (z.CmdLines)[:zb0002] + } else { + z.CmdLines = make([]string, zb0002) + } + for za0001 := range z.CmdLines { + z.CmdLines[za0001], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "CmdLines", za0001) + return + } + } + case "MinioEnv": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "MinioEnv") + return + } + if z.MinioEnv == nil { + z.MinioEnv = make(map[string]string, zb0003) + } else if len(z.MinioEnv) > 0 { + clear(z.MinioEnv) + } + for zb0003 > 0 { + var za0003 string + zb0003-- + var za0002 string + za0002, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "MinioEnv") + return + } + za0003, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "MinioEnv", za0002) + return + } + z.MinioEnv[za0002] = za0003 + } + case "Checksum": + z.Checksum, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Checksum") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ServerSystemConfig) Msgsize() (s int) { + s = 1 + 11 + msgp.IntSize + 9 + msgp.ArrayHeaderSize + for za0001 := range z.CmdLines { + s += msgp.StringPrefixSize + len(z.CmdLines[za0001]) + } + s += 9 + msgp.MapHeaderSize + if z.MinioEnv != nil { + for za0002, za0003 := range z.MinioEnv { + _ = za0003 + s += msgp.StringPrefixSize + len(za0002) + msgp.StringPrefixSize + len(za0003) + } + } + s += 9 + msgp.StringPrefixSize + len(z.Checksum) + return +} diff --git a/cmd/bootstrap-peer-server_gen_test.go b/cmd/bootstrap-peer-server_gen_test.go new file mode 100644 index 00000000..efb0ee0c --- /dev/null +++ b/cmd/bootstrap-peer-server_gen_test.go @@ -0,0 +1,123 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalServerSystemConfig(t *testing.T) { + v := ServerSystemConfig{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgServerSystemConfig(b *testing.B) { + v := ServerSystemConfig{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgServerSystemConfig(b *testing.B) { + v := ServerSystemConfig{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalServerSystemConfig(b *testing.B) { + v := ServerSystemConfig{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeServerSystemConfig(t *testing.T) { + v := ServerSystemConfig{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeServerSystemConfig Msgsize() is inaccurate") + } + + vn := ServerSystemConfig{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeServerSystemConfig(b *testing.B) { + v := ServerSystemConfig{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeServerSystemConfig(b *testing.B) { + v := ServerSystemConfig{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/bucket-encryption-handlers.go b/cmd/bucket-encryption-handlers.go new file mode 100644 index 00000000..1fe7631d --- /dev/null +++ b/cmd/bucket-encryption-handlers.go @@ -0,0 +1,214 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "encoding/base64" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + + "github.com/minio/kms-go/kes" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/kms" + "github.com/minio/minio/internal/logger" + "github.com/minio/mux" + "github.com/minio/pkg/v3/policy" +) + +const ( + // Bucket Encryption configuration file name. + bucketSSEConfig = "bucket-encryption.xml" +) + +// PutBucketEncryptionHandler - Stores given bucket encryption configuration +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketEncryption.html +func (api objectAPIHandlers) PutBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PutBucketEncryption") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + vars := mux.Vars(r) + bucket := vars["bucket"] + + if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketEncryptionAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Check if bucket exists. + if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Parse bucket encryption xml + encConfig, err := validateBucketSSEConfig(io.LimitReader(r.Body, maxBucketSSEConfigSize)) + if err != nil { + apiErr := APIError{ + Code: "MalformedXML", + Description: fmt.Sprintf("%s (%s)", errorCodes[ErrMalformedXML].Description, err), + HTTPStatusCode: errorCodes[ErrMalformedXML].HTTPStatusCode, + } + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + + // Return error if KMS is not initialized + if GlobalKMS == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL) + return + } + kmsKey := encConfig.KeyID() + if kmsKey != "" { + kmsContext := kms.Context{"MinIO admin API": "ServerInfoHandler"} // Context for a test key operation + _, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{Name: kmsKey, AssociatedData: kmsContext}) + if err != nil { + if errors.Is(err, kes.ErrKeyNotFound) { + writeErrorResponse(ctx, w, toAPIError(ctx, errKMSKeyNotFound), r.URL) + return + } + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + + configData, err := xml.Marshal(encConfig) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Store the bucket encryption configuration in the object layer + updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Call site replication hook. + // + // We encode the xml bytes as base64 to ensure there are no encoding + // errors. + cfgStr := base64.StdEncoding.EncodeToString(configData) + replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaTypeSSEConfig, + Bucket: bucket, + SSEConfig: &cfgStr, + UpdatedAt: updatedAt, + })) + + writeSuccessResponseHeadersOnly(w) +} + +// GetBucketEncryptionHandler - Returns bucket policy configuration +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketEncryption.html +func (api objectAPIHandlers) GetBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetBucketEncryption") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + vars := mux.Vars(r) + bucket := vars["bucket"] + + if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketEncryptionAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Check if bucket exists + var err error + if _, err = objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + config, _, err := globalBucketMetadataSys.GetSSEConfig(bucket) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + configData, err := xml.Marshal(config) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Write bucket encryption configuration to client + writeSuccessResponseXML(w, configData) +} + +// DeleteBucketEncryptionHandler - Removes bucket encryption configuration +func (api objectAPIHandlers) DeleteBucketEncryptionHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "DeleteBucketEncryption") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + vars := mux.Vars(r) + bucket := vars["bucket"] + + if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketEncryptionAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Check if bucket exists + var err error + if _, err = objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Delete bucket encryption config from object layer + updatedAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketSSEConfig) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Call site replication hook. + replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaTypeSSEConfig, + Bucket: bucket, + SSEConfig: nil, + UpdatedAt: updatedAt, + })) + + writeSuccessNoContent(w) +} diff --git a/cmd/bucket-encryption.go b/cmd/bucket-encryption.go new file mode 100644 index 00000000..c74fbae3 --- /dev/null +++ b/cmd/bucket-encryption.go @@ -0,0 +1,53 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "errors" + "io" + + sse "github.com/minio/minio/internal/bucket/encryption" +) + +// BucketSSEConfigSys - in-memory cache of bucket encryption config +type BucketSSEConfigSys struct{} + +// NewBucketSSEConfigSys - Creates an empty in-memory bucket encryption configuration cache +func NewBucketSSEConfigSys() *BucketSSEConfigSys { + return &BucketSSEConfigSys{} +} + +// Get - gets bucket encryption config for the given bucket. +func (sys *BucketSSEConfigSys) Get(bucket string) (*sse.BucketSSEConfig, error) { + sseCfg, _, err := globalBucketMetadataSys.GetSSEConfig(bucket) + return sseCfg, err +} + +// validateBucketSSEConfig parses bucket encryption configuration and validates if it is supported by MinIO. +func validateBucketSSEConfig(r io.Reader) (*sse.BucketSSEConfig, error) { + encConfig, err := sse.ParseBucketSSEConfig(r) + if err != nil { + return nil, err + } + + if len(encConfig.Rules) == 1 { + return encConfig, nil + } + + return nil, errors.New("Unsupported bucket encryption configuration") +} diff --git a/cmd/bucket-encryption_test.go b/cmd/bucket-encryption_test.go new file mode 100644 index 00000000..3d5ff48d --- /dev/null +++ b/cmd/bucket-encryption_test.go @@ -0,0 +1,70 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "testing" +) + +func TestValidateBucketSSEConfig(t *testing.T) { + testCases := []struct { + inputXML string + expectedErr error + shouldPass bool + }{ + // MinIO supported XML + { + inputXML: ` + + + AES256 + + + `, + expectedErr: nil, + shouldPass: true, + }, + // Unsupported XML + { + inputXML: ` + + + aws:kms + my-key + + + `, + expectedErr: nil, + shouldPass: true, + }, + } + + for i, tc := range testCases { + _, err := validateBucketSSEConfig(bytes.NewReader([]byte(tc.inputXML))) + if tc.shouldPass && err != nil { + t.Fatalf("Test case %d: Expected to succeed but got %s", i+1, err) + } + + if !tc.shouldPass { + if err == nil || err != nil && err.Error() != tc.expectedErr.Error() { + t.Fatalf("Test case %d: Expected %s but got %s", i+1, tc.expectedErr, err) + } + } + } +} diff --git a/cmd/bucket-handlers.go b/cmd/bucket-handlers.go new file mode 100644 index 00000000..564572f2 --- /dev/null +++ b/cmd/bucket-handlers.go @@ -0,0 +1,2035 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/base64" + "encoding/hex" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "io" + "mime" + "mime/multipart" + "net/http" + "net/textproto" + "net/url" + "path" + "runtime" + "sort" + "strconv" + "strings" + "sync" + + "github.com/google/uuid" + "github.com/minio/mux" + "github.com/valyala/bytebufferpool" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio-go/v7/pkg/tags" + "github.com/minio/minio/internal/auth" + sse "github.com/minio/minio/internal/bucket/encryption" + objectlock "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/config/dns" + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/etag" + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/handlers" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/kms" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/policy" + "github.com/minio/pkg/v3/sync/errgroup" +) + +const ( + objectLockConfig = "object-lock.xml" + bucketTaggingConfig = "tagging.xml" + bucketReplicationConfig = "replication.xml" + + xMinIOErrCodeHeader = "x-minio-error-code" + xMinIOErrDescHeader = "x-minio-error-desc" + + postPolicyBucketTagging = "tagging" +) + +// Check if there are buckets on server without corresponding entry in etcd backend and +// make entries. Here is the general flow +// - Range over all the available buckets +// - Check if a bucket has an entry in etcd backend +// -- If no, make an entry +// -- If yes, check if the entry matches local IP check if we +// +// need to update the entry then proceed to update +// +// -- If yes, check if the IP of entry matches local IP. +// +// This means entry is for this instance. +// +// -- If IP of the entry doesn't match, this means entry is +// +// for another instance. Log an error to console. +func initFederatorBackend(buckets []string, objLayer ObjectLayer) { + if len(buckets) == 0 { + return + } + + // Get buckets in the DNS + dnsBuckets, err := globalDNSConfig.List() + if err != nil && !IsErrIgnored(err, dns.ErrNoEntriesFound, dns.ErrNotImplemented, dns.ErrDomainMissing) { + dnsLogIf(GlobalContext, err) + return + } + + bucketsSet := set.NewStringSet() + bucketsToBeUpdated := set.NewStringSet() + bucketsInConflict := set.NewStringSet() + + // This means that domain is updated, we should update + // all bucket entries with new domain name. + domainMissing := err == dns.ErrDomainMissing + if dnsBuckets != nil { + for _, bucket := range buckets { + bucketsSet.Add(bucket) + r, ok := dnsBuckets[bucket] + if !ok { + bucketsToBeUpdated.Add(bucket) + continue + } + if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() { + if globalDomainIPs.Difference(set.CreateStringSet(getHostsSlice(r)...)).IsEmpty() && !domainMissing { + // No difference in terms of domainIPs and nothing + // has changed so we don't change anything on the etcd. + // + // Additionally also check if domain is updated/missing with more + // entries, if that is the case we should update the + // new domain entries as well. + continue + } + + // if domain IPs intersect then it won't be an empty set. + // such an intersection means that bucket exists on etcd. + // but if we do see a difference with local domain IPs with + // hostSlice from etcd then we should update with newer + // domainIPs, we proceed to do that here. + bucketsToBeUpdated.Add(bucket) + continue + } + + // No IPs seem to intersect, this means that bucket exists but has + // different IP addresses perhaps from a different deployment. + // bucket names are globally unique in federation at a given + // path prefix, name collision is not allowed. We simply log + // an error and continue. + bucketsInConflict.Add(bucket) + } + } + + // Add/update buckets that are not registered with the DNS + bucketsToBeUpdatedSlice := bucketsToBeUpdated.ToSlice() + g := errgroup.WithNErrs(len(bucketsToBeUpdatedSlice)).WithConcurrency(50) + + for index := range bucketsToBeUpdatedSlice { + g.Go(func() error { + return globalDNSConfig.Put(bucketsToBeUpdatedSlice[index]) + }, index) + } + + ctx := GlobalContext + for _, err := range g.Wait() { + if err != nil { + dnsLogIf(ctx, err) + return + } + } + + for _, bucket := range bucketsInConflict.ToSlice() { + dnsLogIf(ctx, fmt.Errorf("Unable to add bucket DNS entry for bucket %s, an entry exists for the same bucket by a different tenant. This local bucket will be ignored. Bucket names are globally unique in federated deployments. Use path style requests on following addresses '%v' to access this bucket", bucket, globalDomainIPs.ToSlice())) + } + + var wg sync.WaitGroup + // Remove buckets that are in DNS for this server, but aren't local + for bucket, records := range dnsBuckets { + if bucketsSet.Contains(bucket) { + continue + } + + if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(records)...)).IsEmpty() { + // This is not for our server, so we can continue + continue + } + + wg.Add(1) + go func(bucket string) { + defer wg.Done() + // We go to here, so we know the bucket no longer exists, + // but is registered in DNS to this server + if err := globalDNSConfig.Delete(bucket); err != nil { + dnsLogIf(GlobalContext, fmt.Errorf("Failed to remove DNS entry for %s due to %w", + bucket, err)) + } + }(bucket) + } + wg.Wait() +} + +// GetBucketLocationHandler - GET Bucket location. +// ------------------------- +// This operation returns bucket location. +func (api objectAPIHandlers) GetBucketLocationHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetBucketLocation") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketLocationAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + getBucketInfo := objectAPI.GetBucketInfo + + if _, err := getBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Generate response. + encodedSuccessResponse := encodeResponse(LocationResponse{}) + // Get current region. + region := globalSite.Region() + if region != globalMinioDefaultRegion { + encodedSuccessResponse = encodeResponse(LocationResponse{ + Location: region, + }) + } + + // Write success response. + writeSuccessResponseXML(w, encodedSuccessResponse) +} + +// ListMultipartUploadsHandler - GET Bucket (List Multipart uploads) +// ------------------------- +// This operation lists in-progress multipart uploads. An in-progress +// multipart upload is a multipart upload that has been initiated, +// using the Initiate Multipart Upload request, but has not yet been +// completed or aborted. This operation returns at most 1,000 multipart +// uploads in the response. +func (api objectAPIHandlers) ListMultipartUploadsHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "ListMultipartUploads") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketMultipartUploadsAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + prefix, keyMarker, uploadIDMarker, delimiter, maxUploads, encodingType, errCode := getBucketMultipartResources(r.Form) + if errCode != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL) + return + } + + if maxUploads < 0 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidMaxUploads), r.URL) + return + } + + if keyMarker != "" { + // Marker not common with prefix is not implemented. + if !HasPrefix(keyMarker, prefix) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + } + + listMultipartsInfo, err := objectAPI.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + // generate response + response := generateListMultipartUploadsResponse(bucket, listMultipartsInfo, encodingType) + encodedSuccessResponse := encodeResponse(response) + + // write success response. + writeSuccessResponseXML(w, encodedSuccessResponse) +} + +// ListBucketsHandler - GET Service. +// ----------- +// This implementation of the GET operation returns a list of all buckets +// owned by the authenticated sender of the request. +func (api objectAPIHandlers) ListBucketsHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "ListBuckets") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + listBuckets := objectAPI.ListBuckets + + cred, owner, s3Error := checkRequestAuthTypeCredential(ctx, r, policy.ListAllMyBucketsAction) + if s3Error != ErrNone && s3Error != ErrAccessDenied { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Anonymous users, should be rejected. + if cred.AccessKey == "" { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + + // If etcd, dns federation configured list buckets from etcd. + var bucketsInfo []BucketInfo + if globalDNSConfig != nil && globalBucketFederation { + dnsBuckets, err := globalDNSConfig.List() + if err != nil && !IsErrIgnored(err, + dns.ErrNoEntriesFound, + dns.ErrDomainMissing) { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + for _, dnsRecords := range dnsBuckets { + bucketsInfo = append(bucketsInfo, BucketInfo{ + Name: dnsRecords[0].Key, + Created: dnsRecords[0].CreationDate, + }) + } + sort.Slice(bucketsInfo, func(i, j int) bool { + return bucketsInfo[i].Name < bucketsInfo[j].Name + }) + } else { + // Invoke the list buckets. + var err error + bucketsInfo, err = listBuckets(ctx, BucketOptions{}) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + + if s3Error == ErrAccessDenied { + // Set prefix value for "s3:prefix" policy conditionals. + r.Header.Set("prefix", "") + + // Set delimiter value for "s3:delimiter" policy conditionals. + r.Header.Set("delimiter", SlashSeparator) + + n := 0 + // Use the following trick to filter in place + // https://github.com/golang/go/wiki/SliceTricks#filter-in-place + for _, bucketInfo := range bucketsInfo { + if globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.ListBucketAction, + BucketName: bucketInfo.Name, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + ObjectName: "", + Claims: cred.Claims, + }) { + bucketsInfo[n] = bucketInfo + n++ + } else if globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.GetBucketLocationAction, + BucketName: bucketInfo.Name, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + ObjectName: "", + Claims: cred.Claims, + }) { + bucketsInfo[n] = bucketInfo + n++ + } + } + bucketsInfo = bucketsInfo[:n] + // No buckets can be filtered return access denied error. + if len(bucketsInfo) == 0 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + } + + // Generate response. + response := generateListBucketsResponse(bucketsInfo) + encodedSuccessResponse := encodeResponse(response) + + // Write response. + writeSuccessResponseXML(w, encodedSuccessResponse) +} + +// DeleteMultipleObjectsHandler - deletes multiple objects. +func (api objectAPIHandlers) DeleteMultipleObjectsHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "DeleteMultipleObjects") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + // Content-Md5 is required should be set + // http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html + if !validateLengthAndChecksum(r) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL) + return + } + + // Content-Length is required and should be non-zero + // http://docs.aws.amazon.com/AmazonS3/latest/API/multiobjectdeleteapi.html + if r.ContentLength <= 0 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL) + return + } + + // The max. XML contains 100000 object names (each at most 1024 bytes long) + XML overhead + const maxBodySize = 2 * 100000 * 1024 + + if r.ContentLength > maxBodySize { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL) + return + } + + // Unmarshal list of keys to be deleted. + deleteObjectsReq := &DeleteObjectsRequest{} + if err := xmlDecoder(r.Body, deleteObjectsReq, maxBodySize); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + objects := make([]ObjectV, len(deleteObjectsReq.Objects)) + // Convert object name delete objects if it has `/` in the beginning. + for i := range deleteObjectsReq.Objects { + deleteObjectsReq.Objects[i].ObjectName = trimLeadingSlash(deleteObjectsReq.Objects[i].ObjectName) + objects[i] = deleteObjectsReq.Objects[i].ObjectV + } + + // Make sure to update context to print ObjectNames for multi objects. + ctx = updateReqContext(ctx, objects...) + + // Call checkRequestAuthType to populate ReqInfo.AccessKey before GetBucketInfo() + // Ignore errors here to preserve the S3 error behavior of GetBucketInfo() + checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, "") + + deleteObjectsFn := objectAPI.DeleteObjects + + // Return Malformed XML as S3 spec if the number of objects is empty + if len(deleteObjectsReq.Objects) == 0 || len(deleteObjectsReq.Objects) > maxDeleteList { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedXML), r.URL) + return + } + + objectsToDelete := map[ObjectToDelete]int{} + getObjectInfoFn := objectAPI.GetObjectInfo + + var ( + hasLockEnabled bool + dsc ReplicateDecision + goi ObjectInfo + gerr error + ) + replicateDeletes := hasReplicationRules(ctx, bucket, deleteObjectsReq.Objects) + if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled { + hasLockEnabled = true + } + + type deleteResult struct { + delInfo DeletedObject + errInfo DeleteError + } + + deleteResults := make([]deleteResult, len(deleteObjectsReq.Objects)) + + vc, _ := globalBucketVersioningSys.Get(bucket) + oss := make([]*objSweeper, len(deleteObjectsReq.Objects)) + for index, object := range deleteObjectsReq.Objects { + if apiErrCode := checkRequestAuthTypeWithVID(ctx, r, policy.DeleteObjectAction, bucket, object.ObjectName, object.VersionID); apiErrCode != ErrNone { + if apiErrCode == ErrSignatureDoesNotMatch || apiErrCode == ErrInvalidAccessKeyID { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErrCode), r.URL) + return + } + apiErr := errorCodes.ToAPIErr(apiErrCode) + deleteResults[index].errInfo = DeleteError{ + Code: apiErr.Code, + Message: apiErr.Description, + Key: object.ObjectName, + VersionID: object.VersionID, + } + continue + } + if object.VersionID != "" && object.VersionID != nullVersionID { + if _, err := uuid.Parse(object.VersionID); err != nil { + apiErr := errorCodes.ToAPIErr(ErrNoSuchVersion) + deleteResults[index].errInfo = DeleteError{ + Code: apiErr.Code, + Message: fmt.Sprintf("%s (%s)", apiErr.Description, err), + Key: object.ObjectName, + VersionID: object.VersionID, + } + continue + } + } + + opts := ObjectOptions{ + VersionID: object.VersionID, + Versioned: vc.PrefixEnabled(object.ObjectName), + VersionSuspended: vc.Suspended(), + } + + if replicateDeletes || object.VersionID != "" && hasLockEnabled || !globalTierConfigMgr.Empty() { + if !globalTierConfigMgr.Empty() && object.VersionID == "" && opts.VersionSuspended { + opts.VersionID = nullVersionID + } + goi, gerr = getObjectInfoFn(ctx, bucket, object.ObjectName, opts) + } + + if !globalTierConfigMgr.Empty() { + oss[index] = newObjSweeper(bucket, object.ObjectName).WithVersion(opts.VersionID).WithVersioning(opts.Versioned, opts.VersionSuspended) + oss[index].SetTransitionState(goi.TransitionedObject) + } + + // All deletes on directory objects needs to be for `nullVersionID` + if isDirObject(object.ObjectName) && object.VersionID == "" { + object.VersionID = nullVersionID + } + + if replicateDeletes { + dsc = checkReplicateDelete(ctx, bucket, ObjectToDelete{ + ObjectV: ObjectV{ + ObjectName: object.ObjectName, + VersionID: object.VersionID, + }, + }, goi, opts, gerr) + if dsc.ReplicateAny() { + if object.VersionID != "" { + object.VersionPurgeStatus = replication.VersionPurgePending + object.VersionPurgeStatuses = dsc.PendingStatus() + } else { + object.DeleteMarkerReplicationStatus = dsc.PendingStatus() + } + object.ReplicateDecisionStr = dsc.String() + } + } + if object.VersionID != "" && hasLockEnabled { + if err := enforceRetentionBypassForDelete(ctx, r, bucket, object, goi, gerr); err != nil { + apiErr := toAPIError(ctx, err) + deleteResults[index].errInfo = DeleteError{ + Code: apiErr.Code, + Message: apiErr.Description, + Key: object.ObjectName, + VersionID: object.VersionID, + } + continue + } + } + + // Avoid duplicate objects, we use map to filter them out. + if _, ok := objectsToDelete[object]; !ok { + objectsToDelete[object] = index + } + } + + toNames := func(input map[ObjectToDelete]int) (output []ObjectToDelete) { + output = make([]ObjectToDelete, len(input)) + idx := 0 + for obj := range input { + output[idx] = obj + idx++ + } + return output + } + + // Disable timeouts and cancellation + ctx = bgContext(ctx) + + deleteList := toNames(objectsToDelete) + dObjects, errs := deleteObjectsFn(ctx, bucket, deleteList, ObjectOptions{ + PrefixEnabledFn: vc.PrefixEnabled, + VersionSuspended: vc.Suspended(), + }) + + // Are all objects saying bucket not found? + if isAllBucketsNotFound(errs) { + writeErrorResponse(ctx, w, toAPIError(ctx, errs[0]), r.URL) + return + } + + for i := range errs { + // DeleteMarkerVersionID is not used specifically to avoid + // lookup errors, since DeleteMarkerVersionID is only + // created during DeleteMarker creation when client didn't + // specify a versionID. + objToDel := ObjectToDelete{ + ObjectV: ObjectV{ + ObjectName: decodeDirObject(dObjects[i].ObjectName), + VersionID: dObjects[i].VersionID, + }, + VersionPurgeStatus: dObjects[i].VersionPurgeStatus(), + VersionPurgeStatuses: dObjects[i].ReplicationState.VersionPurgeStatusInternal, + DeleteMarkerReplicationStatus: dObjects[i].ReplicationState.ReplicationStatusInternal, + ReplicateDecisionStr: dObjects[i].ReplicationState.ReplicateDecisionStr, + } + dindex := objectsToDelete[objToDel] + if errs[i] == nil || isErrObjectNotFound(errs[i]) || isErrVersionNotFound(errs[i]) { + if replicateDeletes { + dObjects[i].ReplicationState = deleteList[i].ReplicationState() + } + deleteResults[dindex].delInfo = dObjects[i] + continue + } + apiErr := toAPIError(ctx, errs[i]) + deleteResults[dindex].errInfo = DeleteError{ + Code: apiErr.Code, + Message: apiErr.Description, + Key: deleteList[i].ObjectName, + VersionID: deleteList[i].VersionID, + } + } + + // Generate response + deleteErrors := make([]DeleteError, 0, len(deleteObjectsReq.Objects)) + deletedObjects := make([]DeletedObject, 0, len(deleteObjectsReq.Objects)) + for _, deleteResult := range deleteResults { + if deleteResult.errInfo.Code != "" { + deleteErrors = append(deleteErrors, deleteResult.errInfo) + } else { + // All deletes on directory objects was with `nullVersionID`. + // Remove it from response. + if isDirObject(deleteResult.delInfo.ObjectName) && deleteResult.delInfo.VersionID == nullVersionID { + deleteResult.delInfo.VersionID = "" + } + deletedObjects = append(deletedObjects, deleteResult.delInfo) + } + } + + response := generateMultiDeleteResponse(deleteObjectsReq.Quiet, deletedObjects, deleteErrors) + encodedSuccessResponse := encodeResponse(response) + + // Write success response. + writeSuccessResponseXML(w, encodedSuccessResponse) + for _, dobj := range deletedObjects { + if dobj.ObjectName == "" { + continue + } + + if replicateDeletes && (dobj.DeleteMarkerReplicationStatus() == replication.Pending || dobj.VersionPurgeStatus() == replication.VersionPurgePending) { + // copy so we can re-add null ID. + dobj := dobj + if isDirObject(dobj.ObjectName) && dobj.VersionID == "" { + dobj.VersionID = nullVersionID + } + dv := DeletedObjectReplicationInfo{ + DeletedObject: dobj, + Bucket: bucket, + EventType: ReplicateIncomingDelete, + } + scheduleReplicationDelete(ctx, dv, objectAPI) + } + + eventName := event.ObjectRemovedDelete + objInfo := ObjectInfo{ + Name: dobj.ObjectName, + VersionID: dobj.VersionID, + DeleteMarker: dobj.DeleteMarker, + } + + if objInfo.DeleteMarker { + objInfo.VersionID = dobj.DeleteMarkerVersionID + eventName = event.ObjectRemovedDeleteMarkerCreated + } + + sendEvent(eventArgs{ + EventName: eventName, + BucketName: bucket, + Object: objInfo, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) + } + + // Clean up transitioned objects from remote tier + for _, os := range oss { + if os == nil { // skip objects that weren't deleted due to invalid versionID etc. + continue + } + os.Sweep() + } +} + +// PutBucketHandler - PUT Bucket +// ---------- +// This implementation of the PUT operation creates a new bucket for authenticated request +func (api objectAPIHandlers) PutBucketHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PutBucket") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectLockEnabled := false + if vs := r.Header.Get(xhttp.AmzObjectLockEnabled); len(vs) > 0 { + v := strings.ToLower(vs) + switch v { + case "true", "false": + objectLockEnabled = v == "true" + default: + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + } + + forceCreate := false + if vs := r.Header.Get(xhttp.MinIOForceCreate); len(vs) > 0 { + v := strings.ToLower(vs) + switch v { + case "true", "false": + forceCreate = v == "true" + default: + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + } + + cred, owner, s3Error := checkRequestAuthTypeCredential(ctx, r, policy.CreateBucketAction) + if s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + if objectLockEnabled { + // Creating a bucket with locking requires the user having more permissions + for _, action := range []policy.Action{policy.PutBucketObjectLockConfigurationAction, policy.PutBucketVersioningAction} { + if !globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: action, + ConditionValues: getConditionValues(r, "", cred), + BucketName: bucket, + IsOwner: owner, + Claims: cred.Claims, + }) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + } + } + + // Parse incoming location constraint. + _, s3Error = parseLocationConstraint(r) + if s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // check if client is attempting to create more buckets, complain about it. + if currBuckets := globalBucketMetadataSys.Count(); currBuckets+1 > maxBuckets { + internalLogIf(ctx, fmt.Errorf("Please avoid creating more buckets %d beyond recommended %d", currBuckets+1, maxBuckets), logger.WarningKind) + } + + opts := MakeBucketOptions{ + LockEnabled: objectLockEnabled, + ForceCreate: forceCreate, + } + + if globalDNSConfig != nil { + sr, err := globalDNSConfig.Get(bucket) + if err != nil { + // ErrNotImplemented indicates a DNS backend that doesn't need to check if bucket already + // exists elsewhere + if err == dns.ErrNoEntriesFound || err == dns.ErrNotImplemented { + // Proceed to creating a bucket. + if err = objectAPI.MakeBucket(ctx, bucket, opts); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if err = globalDNSConfig.Put(bucket); err != nil { + objectAPI.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{ + Force: true, + SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()), + }) + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Load updated bucket metadata into memory. + globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket) + + // Make sure to add Location information here only for bucket + w.Header().Set(xhttp.Location, pathJoin(SlashSeparator, bucket)) + + writeSuccessResponseHeadersOnly(w) + + sendEvent(eventArgs{ + EventName: event.BucketCreated, + BucketName: bucket, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) + + return + } + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + apiErr := ErrBucketAlreadyExists + if !globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(sr)...)).IsEmpty() { + apiErr = ErrBucketAlreadyOwnedByYou + } + // No IPs seem to intersect, this means that bucket exists but has + // different IP addresses perhaps from a different deployment. + // bucket names are globally unique in federation at a given + // path prefix, name collision is not allowed. Return appropriate error. + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(apiErr), r.URL) + return + } + + // Proceed to creating a bucket. + if err := objectAPI.MakeBucket(ctx, bucket, opts); err != nil { + if _, ok := err.(BucketExists); ok { + // Though bucket exists locally, we send the site-replication + // hook to ensure all sites have this bucket. If the hook + // succeeds, the client will still receive a bucket exists + // message. + globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts) + } + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Load updated bucket metadata into memory. + globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket) + + // Call site replication hook + replLogIf(ctx, globalSiteReplicationSys.MakeBucketHook(ctx, bucket, opts)) + + // Make sure to add Location information here only for bucket + w.Header().Set(xhttp.Location, pathJoin(SlashSeparator, bucket)) + + writeSuccessResponseHeadersOnly(w) + + sendEvent(eventArgs{ + EventName: event.BucketCreated, + BucketName: bucket, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) +} + +// multipartReader is just like https://pkg.go.dev/net/http#Request.MultipartReader but +// rejects multipart/mixed as its not supported in S3 API. +func multipartReader(r *http.Request) (*multipart.Reader, error) { + v := r.Header.Get("Content-Type") + if v == "" { + return nil, http.ErrNotMultipart + } + if r.Body == nil { + return nil, errors.New("missing form body") + } + d, params, err := mime.ParseMediaType(v) + if err != nil { + return nil, http.ErrNotMultipart + } + if d != "multipart/form-data" { + return nil, http.ErrNotMultipart + } + boundary, ok := params["boundary"] + if !ok { + return nil, http.ErrMissingBoundary + } + return multipart.NewReader(r.Body, boundary), nil +} + +// PostPolicyBucketHandler - POST policy +// ---------- +// This implementation of the POST operation handles object creation with a specified +// signature policy in multipart/form-data +func (api objectAPIHandlers) PostPolicyBucketHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PostPolicyBucket") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if crypto.S3KMS.IsRequested(r.Header) { // SSE-KMS is not supported + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + + bucket := mux.Vars(r)["bucket"] + resource, err := getResource(r.URL.Path, r.Host, globalDomainNames) + if err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + + // Make sure that the URL does not contain object name. + if bucket != path.Clean(resource[1:]) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL) + return + } + + if r.ContentLength <= 0 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEmptyRequestBody), r.URL) + return + } + + // Here the parameter is the size of the form data that should + // be loaded in memory, the remaining being put in temporary files. + mp, err := multipartReader(r) + if err != nil { + apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest) + apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err) + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + + const mapEntryOverhead = 200 + + var ( + reader io.Reader + actualSize int64 = -1 + fileName string + fanOutEntries = make([]minio.PutObjectFanOutEntry, 0, 100) + ) + + maxParts := 1000 + // Canonicalize the form values into http.Header. + formValues := make(http.Header) + var headerLen int64 + for { + part, err := mp.NextRawPart() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest) + apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err) + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + if maxParts <= 0 { + apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest) + apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, multipart.ErrMessageTooLarge) + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + maxParts-- + + name := part.FormName() + if name == "" { + continue + } + + fileName = part.FileName() + + // Multiple values for the same key (one map entry, longer slice) are cheaper + // than the same number of values for different keys (many map entries), but + // using a consistent per-value cost for overhead is simpler. + maxMemoryBytes := 2 * int64(10<<20) + maxMemoryBytes -= int64(len(name)) + maxMemoryBytes -= mapEntryOverhead + if maxMemoryBytes < 0 { + // We can't actually take this path, since nextPart would already have + // rejected the MIME headers for being too large. Check anyway. + apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest) + apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, multipart.ErrMessageTooLarge) + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + + headerLen += int64(len(name)) + int64(len(fileName)) + if name != "file" { + if http.CanonicalHeaderKey(name) == http.CanonicalHeaderKey("x-minio-fanout-list") { + dec := json.NewDecoder(part) + + // while the array contains values + for dec.More() { + var m minio.PutObjectFanOutEntry + if err := dec.Decode(&m); err != nil { + part.Close() + apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest) + apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err) + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + fanOutEntries = append(fanOutEntries, m) + } + part.Close() + continue + } + + buf := bytebufferpool.Get() + // value, store as string in memory + n, err := io.CopyN(buf, part, maxMemoryBytes+1) + value := buf.String() + buf.Reset() + bytebufferpool.Put(buf) + part.Close() + + if err != nil && err != io.EOF { + apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest) + apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err) + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + maxMemoryBytes -= n + if maxMemoryBytes < 0 { + apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest) + apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, multipart.ErrMessageTooLarge) + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + if n > maxFormFieldSize { + apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest) + apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, multipart.ErrMessageTooLarge) + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + headerLen += n + formValues[http.CanonicalHeaderKey(name)] = append(formValues[http.CanonicalHeaderKey(name)], value) + continue + } + + // In accordance with https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPOST.html + // The file or text content. + // The file or text content must be the last field in the form. + // You cannot upload more than one file at a time. + reader = part + + possibleShardSize := (r.ContentLength - headerLen) + if globalStorageClass.ShouldInline(possibleShardSize, false) { // keep versioned false for this check + var b bytes.Buffer + n, err := io.Copy(&b, reader) + if err != nil { + apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest) + apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, err) + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + reader = &b + actualSize = n + } + + // we have found the File part of the request we are done processing multipart-form + break + } + + // check if have a file + if reader == nil { + apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest) + apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, errors.New("The file or text content is missing")) + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + + if keyName, ok := formValues["Key"]; !ok { + apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest) + apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, errors.New("The name of the uploaded key is missing")) + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } else if fileName == "" && len(keyName) >= 1 { + // if we can't get fileName. We use keyName[0] to fileName + fileName = keyName[0] + } + + if fileName == "" { + apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest) + apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, errors.New("The file or text content is missing")) + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + checksum, err := hash.GetContentChecksum(formValues) + if err != nil { + apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest) + apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, fmt.Errorf("Invalid checksum: %w", err)) + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + if checksum != nil && checksum.Type.Trailing() { + // Not officially supported in POST requests. + apiErr := errorCodes.ToAPIErr(ErrMalformedPOSTRequest) + apiErr.Description = fmt.Sprintf("%s (%v)", apiErr.Description, errors.New("Trailing checksums not available for POST operations")) + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + + formValues.Set("Bucket", bucket) + if fileName != "" && strings.Contains(formValues.Get("Key"), "${filename}") { + // S3 feature to replace ${filename} found in Key form field + // by the filename attribute passed in multipart + formValues.Set("Key", strings.ReplaceAll(formValues.Get("Key"), "${filename}", fileName)) + } + object := trimLeadingSlash(formValues.Get("Key")) + + successRedirect := formValues.Get("success_action_redirect") + successStatus := formValues.Get("success_action_status") + var redirectURL *url.URL + if successRedirect != "" { + redirectURL, err = url.Parse(successRedirect) + if err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL) + return + } + } + + // Verify policy signature. + cred, errCode := doesPolicySignatureMatch(formValues) + if errCode != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL) + return + } + + if len(fanOutEntries) > 0 { + // Once signature is validated, check if the user has + // explicit permissions for the user. + if !globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.PutObjectFanOutAction, + ConditionValues: getConditionValues(r, "", cred), + BucketName: bucket, + ObjectName: object, + IsOwner: globalActiveCred.AccessKey == cred.AccessKey, + Claims: cred.Claims, + }) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + } else { + // Once signature is validated, check if the user has + // explicit permissions for the user. + if !globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.PutObjectAction, + ConditionValues: getConditionValues(r, "", cred), + BucketName: bucket, + ObjectName: object, + IsOwner: globalActiveCred.AccessKey == cred.AccessKey, + Claims: cred.Claims, + }) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + } + + policyBytes, err := base64.StdEncoding.DecodeString(formValues.Get("Policy")) + if err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL) + return + } + + clientETag, err := etag.FromContentMD5(formValues) + if err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDigest), r.URL) + return + } + + var forceMD5 []byte + // Optimization: If SSE-KMS and SSE-C did not request Content-Md5. Use uuid as etag. Optionally enable this also + // for server that is started with `--no-compat`. + kind, _ := crypto.IsRequested(formValues) + if !etag.ContentMD5Requested(formValues) && (kind == crypto.SSEC || kind == crypto.S3KMS || !globalServerCtxt.StrictS3Compat) { + forceMD5 = mustGetUUIDBytes() + } + + hashReader, err := hash.NewReaderWithOpts(ctx, reader, hash.Options{ + Size: actualSize, + MD5Hex: clientETag.String(), + SHA256Hex: "", + ActualSize: actualSize, + DisableMD5: false, + ForceMD5: forceMD5, + }) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if checksum != nil && checksum.Valid() { + if err = hashReader.AddChecksumNoTrailer(formValues, false); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + + // Handle policy if it is set. + if len(policyBytes) > 0 { + postPolicyForm, err := parsePostPolicyForm(bytes.NewReader(policyBytes)) + if err != nil { + errAPI := errorCodes.ToAPIErr(ErrPostPolicyConditionInvalidFormat) + errAPI.Description = fmt.Sprintf("%s '(%s)'", errAPI.Description, err) + writeErrorResponse(ctx, w, errAPI, r.URL) + return + } + + // Make sure formValues adhere to policy restrictions. + if err = checkPostPolicy(formValues, postPolicyForm); err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrAccessDenied, err), r.URL) + return + } + + // Ensure that the object size is within expected range, also the file size + // should not exceed the maximum single Put size (5 GiB) + lengthRange := postPolicyForm.Conditions.ContentLengthRange + if lengthRange.Valid { + hashReader.SetExpectedMin(lengthRange.Min) + hashReader.SetExpectedMax(lengthRange.Max) + } + } + + // Extract metadata to be saved from received Form. + metadata := make(map[string]string) + err = extractMetadataFromMime(ctx, textproto.MIMEHeader(formValues), metadata) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + rawReader := hashReader + pReader := NewPutObjReader(rawReader) + var objectEncryptionKey crypto.ObjectKey + + // Check if bucket encryption is enabled + sseConfig, _ := globalBucketSSEConfigSys.Get(bucket) + sseConfig.Apply(formValues, sse.ApplyOptions{ + AutoEncrypt: globalAutoEncryption, + }) + + var opts ObjectOptions + opts, err = putOptsFromReq(ctx, r, bucket, object, metadata) + if err != nil { + writeErrorResponseHeadersOnly(w, toAPIError(ctx, err)) + return + } + opts.WantChecksum = checksum + + fanOutOpts := fanOutOptions{Checksum: checksum} + if crypto.Requested(formValues) { + if crypto.SSECopy.IsRequested(r.Header) { + writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL) + return + } + + if crypto.SSEC.IsRequested(r.Header) && crypto.S3.IsRequested(r.Header) { + writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionMethod), r.URL) + return + } + + if crypto.SSEC.IsRequested(r.Header) && crypto.S3KMS.IsRequested(r.Header) { + writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionMethod), r.URL) + return + } + + var ( + reader io.Reader + keyID string + key []byte + kmsCtx kms.Context + ) + kind, _ := crypto.IsRequested(formValues) + switch kind { + case crypto.SSEC: + key, err = ParseSSECustomerHeader(formValues) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + case crypto.S3KMS: + keyID, kmsCtx, err = crypto.S3KMS.ParseHTTP(formValues) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + + if len(fanOutEntries) == 0 { + reader, objectEncryptionKey, err = newEncryptReader(ctx, hashReader, kind, keyID, key, bucket, object, metadata, kmsCtx) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + wantSize := int64(-1) + if actualSize >= 0 { + info := ObjectInfo{Size: actualSize} + wantSize = info.EncryptedSize() + } + + // do not try to verify encrypted content/ + hashReader, err = hash.NewReader(ctx, reader, wantSize, "", "", actualSize) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if checksum != nil && checksum.Valid() { + if err = hashReader.AddChecksumNoTrailer(formValues, true); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + opts.EncryptFn = metadataEncrypter(objectEncryptionKey) + pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } else { + fanOutOpts = fanOutOptions{ + Key: key, + Kind: kind, + KeyID: keyID, + KmsCtx: kmsCtx, + Checksum: checksum, + } + } + } + + if len(fanOutEntries) > 0 { + // Fan-out requires no copying, and must be carried from original source + // https://en.wikipedia.org/wiki/Copy_protection so the incoming stream + // is always going to be in-memory as we cannot re-read from what we + // wrote to disk - since that amounts to "copying" from a "copy" + // instead of "copying" from source, we need the stream to be seekable + // to ensure that we can make fan-out calls concurrently. + buf := bytebufferpool.Get() + defer func() { + buf.Reset() + bytebufferpool.Put(buf) + }() + + md5w := md5.New() + + // Maximum allowed fan-out object size. + const maxFanOutSize = 16 << 20 + + n, err := io.Copy(io.MultiWriter(buf, md5w), ioutil.HardLimitReader(pReader, maxFanOutSize)) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Set the correct hex md5sum for the fan-out stream. + fanOutOpts.MD5Hex = hex.EncodeToString(md5w.Sum(nil)) + + concurrentSize := min(runtime.GOMAXPROCS(0), 100) + + fanOutResp := make([]minio.PutObjectFanOutResponse, 0, len(fanOutEntries)) + eventArgsList := make([]eventArgs, 0, len(fanOutEntries)) + for { + var objInfos []ObjectInfo + var errs []error + + var done bool + if len(fanOutEntries) < concurrentSize { + objInfos, errs = fanOutPutObject(ctx, bucket, objectAPI, fanOutEntries, buf.Bytes()[:n], fanOutOpts) + done = true + } else { + objInfos, errs = fanOutPutObject(ctx, bucket, objectAPI, fanOutEntries[:concurrentSize], buf.Bytes()[:n], fanOutOpts) + fanOutEntries = fanOutEntries[concurrentSize:] + } + + for i, objInfo := range objInfos { + if errs[i] != nil { + fanOutResp = append(fanOutResp, minio.PutObjectFanOutResponse{ + Key: objInfo.Name, + Error: errs[i].Error(), + }) + eventArgsList = append(eventArgsList, eventArgs{ + EventName: event.ObjectCreatedPost, + BucketName: objInfo.Bucket, + Object: ObjectInfo{Name: objInfo.Name}, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: fmt.Sprintf("%s MinIO-Fan-Out (failed: %v)", r.UserAgent(), errs[i]), + Host: handlers.GetSourceIP(r), + }) + continue + } + + fanOutResp = append(fanOutResp, minio.PutObjectFanOutResponse{ + Key: objInfo.Name, + ETag: getDecryptedETag(formValues, objInfo, false), + VersionID: objInfo.VersionID, + LastModified: &objInfo.ModTime, + }) + + eventArgsList = append(eventArgsList, eventArgs{ + EventName: event.ObjectCreatedPost, + BucketName: objInfo.Bucket, + Object: objInfo, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent() + " " + "MinIO-Fan-Out", + Host: handlers.GetSourceIP(r), + }) + } + + if done { + break + } + } + + enc := json.NewEncoder(w) + for i, fanOutResp := range fanOutResp { + if err = enc.Encode(&fanOutResp); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Notify object created events. + sendEvent(eventArgsList[i]) + + if eventArgsList[i].Object.NumVersions > int(scannerExcessObjectVersions.Load()) { + // Send events for excessive versions. + sendEvent(eventArgs{ + EventName: event.ObjectManyVersions, + BucketName: eventArgsList[i].Object.Bucket, + Object: eventArgsList[i].Object, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent() + " " + "MinIO-Fan-Out", + Host: handlers.GetSourceIP(r), + }) + + auditLogInternal(context.Background(), AuditLogOptions{ + Event: "scanner:manyversions", + APIName: "PostPolicyBucket", + Bucket: eventArgsList[i].Object.Bucket, + Object: eventArgsList[i].Object.Name, + VersionID: eventArgsList[i].Object.VersionID, + Status: http.StatusText(http.StatusOK), + }) + } + } + + return + } + + if formValues.Get(postPolicyBucketTagging) != "" { + tags, err := tags.ParseObjectXML(strings.NewReader(formValues.Get(postPolicyBucketTagging))) + if err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMalformedPOSTRequest), r.URL) + return + } + tagsStr := tags.String() + opts.UserDefined[xhttp.AmzObjectTagging] = tagsStr + } else { + // avoid user set an invalid tag using `X-Amz-Tagging` + delete(opts.UserDefined, xhttp.AmzObjectTagging) + } + + objInfo, err := objectAPI.PutObject(ctx, bucket, object, pReader, opts) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + etag := getDecryptedETag(formValues, objInfo, false) + + // We must not use the http.Header().Set method here because some (broken) + // clients expect the ETag header key to be literally "ETag" - not "Etag" (case-sensitive). + // Therefore, we have to set the ETag directly as map entry. + w.Header()[xhttp.ETag] = []string{`"` + etag + `"`} + + // Set the relevant version ID as part of the response header. + if objInfo.VersionID != "" && objInfo.VersionID != nullVersionID { + w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID} + } + + if obj := getObjectLocation(r, globalDomainNames, bucket, object); obj != "" { + w.Header().Set(xhttp.Location, obj) + } + + // Notify object created event. + defer sendEvent(eventArgs{ + EventName: event.ObjectCreatedPost, + BucketName: objInfo.Bucket, + Object: objInfo, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) + + if objInfo.NumVersions > int(scannerExcessObjectVersions.Load()) { + defer sendEvent(eventArgs{ + EventName: event.ObjectManyVersions, + BucketName: objInfo.Bucket, + Object: objInfo, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) + + auditLogInternal(context.Background(), AuditLogOptions{ + Event: "scanner:manyversions", + APIName: "PostPolicyBucket", + Bucket: objInfo.Bucket, + Object: objInfo.Name, + VersionID: objInfo.VersionID, + Status: http.StatusText(http.StatusOK), + }) + } + + if redirectURL != nil { // success_action_redirect is valid and set. + v := redirectURL.Query() + v.Add("bucket", objInfo.Bucket) + v.Add("key", objInfo.Name) + v.Add("etag", "\""+objInfo.ETag+"\"") + redirectURL.RawQuery = v.Encode() + writeRedirectSeeOther(w, redirectURL.String()) + return + } + + // Add checksum header. + if checksum != nil && checksum.Valid() { + hash.AddChecksumHeader(w, checksum.AsMap()) + } + + // Decide what http response to send depending on success_action_status parameter + switch successStatus { + case "201": + resp := encodeResponse(PostResponse{ + Bucket: objInfo.Bucket, + Key: objInfo.Name, + ETag: `"` + objInfo.ETag + `"`, + Location: w.Header().Get(xhttp.Location), + }) + writeResponse(w, http.StatusCreated, resp, mimeXML) + case "200": + writeSuccessResponseHeadersOnly(w) + default: + writeSuccessNoContent(w) + } +} + +// GetBucketPolicyStatusHandler - Retrieves the policy status +// for an MinIO bucket, indicating whether the bucket is public. +func (api objectAPIHandlers) GetBucketPolicyStatusHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetBucketPolicyStatus") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrServerNotInitialized)) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyStatusAction, bucket, ""); s3Error != ErrNone { + writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error)) + return + } + + // Check if bucket exists. + if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Check if anonymous (non-owner) has access to list objects. + readable := globalPolicySys.IsAllowed(policy.BucketPolicyArgs{ + Action: policy.ListBucketAction, + BucketName: bucket, + ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials), + IsOwner: false, + }) + + // Check if anonymous (non-owner) has access to upload objects. + writable := globalPolicySys.IsAllowed(policy.BucketPolicyArgs{ + Action: policy.PutObjectAction, + BucketName: bucket, + ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials), + IsOwner: false, + }) + + encodedSuccessResponse := encodeResponse(PolicyStatus{ + IsPublic: func() string { + // Silly to have special 'boolean' values yes + // but complying with silly implementation + // https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetBucketPolicyStatus.html + if readable && writable { + return "TRUE" + } + return "FALSE" + }(), + }) + + writeSuccessResponseXML(w, encodedSuccessResponse) +} + +// HeadBucketHandler - HEAD Bucket +// ---------- +// This operation is useful to determine if a bucket exists. +// The operation returns a 200 OK if the bucket exists and you +// have permission to access it. Otherwise, the operation might +// return responses such as 404 Not Found and 403 Forbidden. +func (api objectAPIHandlers) HeadBucketHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "HeadBucket") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrServerNotInitialized)) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.HeadBucketAction, bucket, ""); s3Error != ErrNone { + if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone { + writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error)) + return + } + } + + getBucketInfo := objectAPI.GetBucketInfo + + if _, err := getBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponseHeadersOnly(w, toAPIError(ctx, err)) + return + } + + writeResponse(w, http.StatusOK, nil, mimeXML) +} + +// DeleteBucketHandler - Delete bucket +func (api objectAPIHandlers) DeleteBucketHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "DeleteBucket") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + // Verify if the caller has sufficient permissions. + if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + forceDelete := false + if value := r.Header.Get(xhttp.MinIOForceDelete); value != "" { + var err error + forceDelete, err = strconv.ParseBool(value) + if err != nil { + apiErr := errorCodes.ToAPIErr(ErrInvalidRequest) + apiErr.Description = err.Error() + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + + // if force delete header is set, we need to evaluate the policy anyways + // regardless of it being true or not. + if s3Error := checkRequestAuthType(ctx, r, policy.ForceDeleteBucketAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + if forceDelete { + if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL) + return + } + rcfg, err := getReplicationConfig(ctx, bucket) + switch { + case err != nil: + if _, ok := err.(BucketReplicationConfigNotFound); !ok { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL) + return + } + case rcfg != nil && rcfg.HasActiveRules("", true): + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMethodNotAllowed), r.URL) + return + } + } + } + + // Return an error if the bucket does not exist + if !forceDelete { + if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + + // Attempt to delete bucket. + if err := objectAPI.DeleteBucket(ctx, bucket, DeleteBucketOptions{ + Force: forceDelete, + SRDeleteOp: getSRBucketDeleteOp(globalSiteReplicationSys.isEnabled()), + }); err != nil { + apiErr := toAPIError(ctx, err) + if _, ok := err.(BucketNotEmpty); ok { + if globalBucketVersioningSys.Enabled(bucket) || globalBucketVersioningSys.Suspended(bucket) { + apiErr.Description = "The bucket you tried to delete is not empty. You must delete all versions in the bucket." + } + } + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + + if globalDNSConfig != nil { + if err := globalDNSConfig.Delete(bucket); err != nil { + dnsLogIf(ctx, fmt.Errorf("Unable to delete bucket DNS entry %w, please delete it manually, bucket on MinIO no longer exists", err)) + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + + globalNotificationSys.DeleteBucketMetadata(ctx, bucket) + globalReplicationPool.Get().deleteResyncMetadata(ctx, bucket) + + // Call site replication hook. + replLogIf(ctx, globalSiteReplicationSys.DeleteBucketHook(ctx, bucket, forceDelete)) + + // Write success response. + writeSuccessNoContent(w) + + sendEvent(eventArgs{ + EventName: event.BucketRemoved, + BucketName: bucket, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) +} + +// PutBucketObjectLockConfigHandler - PUT Bucket object lock configuration. +// ---------- +// Places an Object Lock configuration on the specified bucket. The rule +// specified in the Object Lock configuration will be applied by default +// to every new object placed in the specified bucket. +func (api objectAPIHandlers) PutBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PutBucketObjectLockConfig") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketObjectLockConfigurationAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + config, err := objectlock.ParseObjectLockConfig(r.Body) + if err != nil { + apiErr := errorCodes.ToAPIErr(ErrInvalidArgument) + apiErr.Description = err.Error() + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + + // Audit log tags. + reqInfo := logger.GetReqInfo(ctx) + reqInfo.SetTags("retention", config.String()) + + configData, err := xml.Marshal(config) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Deny object locking configuration settings on existing buckets without object lock enabled. + if _, _, err = globalBucketMetadataSys.GetObjectLockConfig(bucket); err != nil { + if _, ok := err.(BucketObjectLockConfigNotFound); ok { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrObjectLockConfigurationNotAllowed), r.URL) + } else { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + } + return + } + + updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Call site replication hook. + // + // We encode the xml bytes as base64 to ensure there are no encoding + // errors. + cfgStr := base64.StdEncoding.EncodeToString(configData) + replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaTypeObjectLockConfig, + Bucket: bucket, + ObjectLockConfig: &cfgStr, + UpdatedAt: updatedAt, + })) + + // Write success response. + writeSuccessResponseHeadersOnly(w) +} + +// GetBucketObjectLockConfigHandler - GET Bucket object lock configuration. +// ---------- +// Gets the Object Lock configuration for a bucket. The rule specified in +// the Object Lock configuration will be applied by default to every new +// object placed in the specified bucket. +func (api objectAPIHandlers) GetBucketObjectLockConfigHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetBucketObjectLockConfig") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + // check if user has permissions to perform this operation + if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketObjectLockConfigurationAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + configData, err := xml.Marshal(config) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Write success response. + writeSuccessResponseXML(w, configData) +} + +// PutBucketTaggingHandler - PUT Bucket tagging. +// ---------- +func (api objectAPIHandlers) PutBucketTaggingHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PutBucketTagging") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + // Check if bucket exists. + if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketTaggingAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + tags, err := tags.ParseBucketXML(io.LimitReader(r.Body, r.ContentLength)) + if err != nil { + apiErr := errorCodes.ToAPIErr(ErrMalformedXML) + apiErr.Description = err.Error() + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + + configData, err := xml.Marshal(tags) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Call site replication hook. + // + // We encode the xml bytes as base64 to ensure there are no encoding + // errors. + cfgStr := base64.StdEncoding.EncodeToString(configData) + replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaTypeTags, + Bucket: bucket, + Tags: &cfgStr, + UpdatedAt: updatedAt, + })) + + // Write success response. + writeSuccessResponseHeadersOnly(w) +} + +// GetBucketTaggingHandler - GET Bucket tagging. +// ---------- +func (api objectAPIHandlers) GetBucketTaggingHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetBucketTagging") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + // check if user has permissions to perform this operation + if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketTaggingAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + config, _, err := globalBucketMetadataSys.GetTaggingConfig(bucket) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + configData, err := xml.Marshal(config) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Write success response. + writeSuccessResponseXML(w, configData) +} + +// DeleteBucketTaggingHandler - DELETE Bucket tagging. +// ---------- +func (api objectAPIHandlers) DeleteBucketTaggingHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "DeleteBucketTagging") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketTaggingAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + updatedAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketTaggingConfig) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaTypeTags, + Bucket: bucket, + UpdatedAt: updatedAt, + })) + + // Write success response. + writeSuccessNoContent(w) +} diff --git a/cmd/bucket-handlers_test.go b/cmd/bucket-handlers_test.go new file mode 100644 index 00000000..0adb9b8f --- /dev/null +++ b/cmd/bucket-handlers_test.go @@ -0,0 +1,946 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "encoding/xml" + "fmt" + "io" + "net/http" + "net/http/httptest" + "strconv" + "testing" + + "github.com/minio/minio/internal/auth" +) + +// Wrapper for calling RemoveBucket HTTP handler tests for both Erasure multiple disks and single node setup. +func TestRemoveBucketHandler(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testRemoveBucketHandler, endpoints: []string{"RemoveBucket"}}) +} + +func testRemoveBucketHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + _, err := obj.PutObject(GlobalContext, bucketName, "test-object", mustGetPutObjReader(t, bytes.NewReader([]byte{}), int64(0), "", "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"), ObjectOptions{}) + // if object upload fails stop the test. + if err != nil { + t.Fatalf("Error uploading object: %v", err) + } + + // initialize httptest Recorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + // construct HTTP request for DELETE bucket. + req, err := newTestSignedRequestV4(http.MethodDelete, getBucketLocationURL("", bucketName), 0, nil, credentials.AccessKey, credentials.SecretKey, nil) + if err != nil { + t.Fatalf("Test %s: Failed to create HTTP request for RemoveBucketHandler: %v", instanceType, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + switch rec.Code { + case http.StatusOK, http.StatusCreated, http.StatusAccepted, http.StatusNoContent: + t.Fatalf("Test %v: expected failure, but succeeded with %v", instanceType, rec.Code) + } + + // Verify response of the V2 signed HTTP request. + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + recV2 := httptest.NewRecorder() + // construct HTTP request for DELETE bucket. + reqV2, err := newTestSignedRequestV2(http.MethodDelete, getBucketLocationURL("", bucketName), 0, nil, credentials.AccessKey, credentials.SecretKey, nil) + if err != nil { + t.Fatalf("Test %s: Failed to create HTTP request for RemoveBucketHandler: %v", instanceType, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(recV2, reqV2) + switch recV2.Code { + case http.StatusOK, http.StatusCreated, http.StatusAccepted, http.StatusNoContent: + t.Fatalf("Test %v: expected failure, but succeeded with %v", instanceType, recV2.Code) + } +} + +// Wrapper for calling GetBucketPolicy HTTP handler tests for both Erasure multiple disks and single node setup. +func TestGetBucketLocationHandler(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testGetBucketLocationHandler, endpoints: []string{"GetBucketLocation"}}) +} + +func testGetBucketLocationHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + // test cases with sample input and expected output. + testCases := []struct { + bucketName string + accessKey string + secretKey string + // expected Response. + expectedRespStatus int + locationResponse []byte + errorResponse APIErrorResponse + shouldPass bool + }{ + // Test case - 1. + // Tests for authenticated request and proper response. + { + bucketName: bucketName, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusOK, + locationResponse: []byte(` +`), + errorResponse: APIErrorResponse{}, + shouldPass: true, + }, + // Test case - 2. + // Tests for signature mismatch error. + { + bucketName: bucketName, + accessKey: "abcd", + secretKey: "abcd", + expectedRespStatus: http.StatusForbidden, + locationResponse: []byte(""), + errorResponse: APIErrorResponse{ + Resource: SlashSeparator + bucketName + SlashSeparator, + Code: "InvalidAccessKeyId", + Message: "The Access Key Id you provided does not exist in our records.", + }, + shouldPass: false, + }, + } + + for i, testCase := range testCases { + if i != 1 { + continue + } + // initialize httptest Recorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + // construct HTTP request for Get bucket location. + req, err := newTestSignedRequestV4(http.MethodGet, getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for GetBucketLocationHandler: %v", i+1, instanceType, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + if rec.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) + } + if !bytes.Equal(testCase.locationResponse, rec.Body.Bytes()) && testCase.shouldPass { + t.Errorf("Test %d: %s: Expected the response to be `%s`, but instead found `%s`", i+1, instanceType, string(testCase.locationResponse), rec.Body.String()) + } + errorResponse := APIErrorResponse{} + err = xml.Unmarshal(rec.Body.Bytes(), &errorResponse) + if err != nil && !testCase.shouldPass { + t.Fatalf("Test %d: %s: Unable to marshal response body %s", i+1, instanceType, rec.Body.String()) + } + if errorResponse.Resource != testCase.errorResponse.Resource { + t.Errorf("Test %d: %s: Expected the error resource to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Resource, errorResponse.Resource) + } + if errorResponse.Message != testCase.errorResponse.Message { + t.Errorf("Test %d: %s: Expected the error message to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Message, errorResponse.Message) + } + if errorResponse.Code != testCase.errorResponse.Code { + t.Errorf("Test %d: %s: Expected the error code to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Code, errorResponse.Code) + } + + // Verify response of the V2 signed HTTP request. + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + recV2 := httptest.NewRecorder() + // construct HTTP request for PUT bucket policy endpoint. + reqV2, err := newTestSignedRequestV2(http.MethodGet, getBucketLocationURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: %v", i+1, instanceType, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(recV2, reqV2) + if recV2.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code) + } + + errorResponse = APIErrorResponse{} + err = xml.Unmarshal(recV2.Body.Bytes(), &errorResponse) + if err != nil && !testCase.shouldPass { + t.Fatalf("Test %d: %s: Unable to marshal response body %s", i+1, instanceType, recV2.Body.String()) + } + if errorResponse.Resource != testCase.errorResponse.Resource { + t.Errorf("Test %d: %s: Expected the error resource to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Resource, errorResponse.Resource) + } + if errorResponse.Message != testCase.errorResponse.Message { + t.Errorf("Test %d: %s: Expected the error message to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Message, errorResponse.Message) + } + if errorResponse.Code != testCase.errorResponse.Code { + t.Errorf("Test %d: %s: Expected the error code to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Code, errorResponse.Code) + } + } + + // Test for Anonymous/unsigned http request. + // ListBucketsHandler doesn't support bucket policies, setting the policies shouldn't make any difference. + anonReq, err := newTestRequest(http.MethodGet, getBucketLocationURL("", bucketName), 0, nil) + if err != nil { + t.Fatalf("MinIO %s: Failed to create an anonymous request.", instanceType) + } + + // ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse, + // sets the bucket policy using the policy statement generated from `getReadOnlyBucketStatement` so that the + // unsigned request goes through and its validated again. + ExecObjectLayerAPIAnonTest(t, obj, "TestGetBucketLocationHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonReadOnlyBucketPolicy(bucketName)) + + // HTTP request for testing when `objectLayer` is set to `nil`. + // There is no need to use an existing bucket and valid input for creating the request + // since the `objectLayer==nil` check is performed before any other checks inside the handlers. + // The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called. + + nilBucket := "dummy-bucket" + nilReq, err := newTestRequest(http.MethodGet, getBucketLocationURL("", nilBucket), 0, nil) + if err != nil { + t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType) + } + // Executes the object layer set to `nil` test. + // `ExecObjectLayerAPINilTest` manages the operation. + ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq) +} + +// Wrapper for calling HeadBucket HTTP handler tests for both Erasure multiple disks and single node setup. +func TestHeadBucketHandler(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testHeadBucketHandler, endpoints: []string{"HeadBucket"}}) +} + +func testHeadBucketHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + // test cases with sample input and expected output. + testCases := []struct { + bucketName string + accessKey string + secretKey string + // expected Response. + expectedRespStatus int + }{ + // Test case - 1. + // Bucket exists. + { + bucketName: bucketName, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusOK, + }, + // Test case - 2. + // Non-existent bucket name. + { + bucketName: "2333", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusNotFound, + }, + // Test case - 3. + // Testing for signature mismatch error. + // setting invalid access and secret key. + { + bucketName: bucketName, + accessKey: "abcd", + secretKey: "abcd", + expectedRespStatus: http.StatusForbidden, + }, + } + + for i, testCase := range testCases { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + // construct HTTP request for HEAD bucket. + req, err := newTestSignedRequestV4(http.MethodHead, getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for HeadBucketHandler: %v", i+1, instanceType, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + if rec.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) + } + + // Verify response the V2 signed HTTP request. + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + recV2 := httptest.NewRecorder() + // construct HTTP request for PUT bucket policy endpoint. + reqV2, err := newTestSignedRequestV2(http.MethodHead, getHEADBucketURL("", testCase.bucketName), 0, nil, testCase.accessKey, testCase.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: %v", i+1, instanceType, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(recV2, reqV2) + if recV2.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code) + } + } + + // Test for Anonymous/unsigned http request. + anonReq, err := newTestRequest(http.MethodHead, getHEADBucketURL("", bucketName), 0, nil) + if err != nil { + t.Fatalf("MinIO %s: Failed to create an anonymous request for bucket \"%s\": %v", + instanceType, bucketName, err) + } + + // ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse, + // sets the bucket policy using the policy statement generated from `getReadOnlyBucketStatement` so that the + // unsigned request goes through and its validated again. + ExecObjectLayerAPIAnonTest(t, obj, "TestHeadBucketHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonReadOnlyBucketPolicy(bucketName)) + + // HTTP request for testing when `objectLayer` is set to `nil`. + // There is no need to use an existing bucket and valid input for creating the request + // since the `objectLayer==nil` check is performed before any other checks inside the handlers. + // The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called. + + nilBucket := "dummy-bucket" + nilReq, err := newTestRequest(http.MethodHead, getHEADBucketURL("", nilBucket), 0, nil) + if err != nil { + t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType) + } + // execute the object layer set to `nil` test. + // `ExecObjectLayerAPINilTest` manages the operation. + ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq) +} + +// Wrapper for calling TestListMultipartUploadsHandler tests for both Erasure multiple disks and single node setup. +func TestListMultipartUploadsHandler(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testListMultipartUploadsHandler, endpoints: []string{"ListMultipartUploads"}}) +} + +// testListMultipartUploadsHandler - Tests validate listing of multipart uploads. +func testListMultipartUploadsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + // Collection of non-exhaustive ListMultipartUploads test cases, valid errors + // and success responses. + testCases := []struct { + // Inputs to ListMultipartUploads. + bucket string + prefix string + keyMarker string + uploadIDMarker string + delimiter string + maxUploads string + accessKey string + secretKey string + expectedRespStatus int + shouldPass bool + }{ + // Test case - 1. + // Setting invalid bucket name. + { + bucket: ".test", + prefix: "", + keyMarker: "", + uploadIDMarker: "", + delimiter: "", + maxUploads: "0", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusBadRequest, + shouldPass: false, + }, + // Test case - 2. + // Setting a non-existent bucket. + { + bucket: "volatile-bucket-1", + prefix: "", + keyMarker: "", + uploadIDMarker: "", + delimiter: "", + maxUploads: "0", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusNotFound, + shouldPass: false, + }, + // Test case -3. + // Delimiter unsupported, but response is empty. + { + bucket: bucketName, + prefix: "", + keyMarker: "", + uploadIDMarker: "", + delimiter: "-", + maxUploads: "0", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusOK, + shouldPass: true, + }, + // Test case - 4. + // Setting Invalid prefix and marker combination. + { + bucket: bucketName, + prefix: "asia", + keyMarker: "europe-object", + uploadIDMarker: "", + delimiter: "", + maxUploads: "0", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusNotImplemented, + shouldPass: false, + }, + // Test case - 5. + // Invalid upload id and marker combination. + { + bucket: bucketName, + prefix: "asia", + keyMarker: "asia/europe/", + uploadIDMarker: "abc", + delimiter: "", + maxUploads: "0", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusNotImplemented, + shouldPass: false, + }, + // Test case - 6. + // Setting a negative value to max-uploads parameter, should result in http.StatusBadRequest. + { + bucket: bucketName, + prefix: "", + keyMarker: "", + uploadIDMarker: "", + delimiter: "", + maxUploads: "-1", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusBadRequest, + shouldPass: false, + }, + // Test case - 7. + // Case with right set of parameters, + // should result in success 200OK. + { + bucket: bucketName, + prefix: "", + keyMarker: "", + uploadIDMarker: "", + delimiter: SlashSeparator, + maxUploads: "100", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusOK, + shouldPass: true, + }, + // Test case - 8. + // Good case without delimiter. + { + bucket: bucketName, + prefix: "", + keyMarker: "", + uploadIDMarker: "", + delimiter: "", + maxUploads: "100", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusOK, + shouldPass: true, + }, + // Test case - 9. + // Setting Invalid AccessKey and SecretKey to induce and verify Signature Mismatch error. + { + bucket: bucketName, + prefix: "", + keyMarker: "", + uploadIDMarker: "", + delimiter: "", + maxUploads: "100", + accessKey: "abcd", + secretKey: "abcd", + expectedRespStatus: http.StatusForbidden, + shouldPass: true, + }, + } + + for i, testCase := range testCases { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + + // construct HTTP request for List multipart uploads endpoint. + u := getListMultipartUploadsURLWithParams("", testCase.bucket, testCase.prefix, testCase.keyMarker, testCase.uploadIDMarker, testCase.delimiter, testCase.maxUploads) + req, gerr := newTestSignedRequestV4(http.MethodGet, u, 0, nil, testCase.accessKey, testCase.secretKey, nil) + if gerr != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for ListMultipartUploadsHandler: %v", i+1, instanceType, gerr) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + if rec.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) + } + + // Verify response the V2 signed HTTP request. + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + recV2 := httptest.NewRecorder() + // construct HTTP request for PUT bucket policy endpoint. + + // verify response for V2 signed HTTP request. + reqV2, err := newTestSignedRequestV2(http.MethodGet, u, 0, nil, testCase.accessKey, testCase.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: %v", i+1, instanceType, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(recV2, reqV2) + if recV2.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code) + } + } + + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + + // construct HTTP request for List multipart uploads endpoint. + u := getListMultipartUploadsURLWithParams("", bucketName, "", "", "", "", "") + req, err := newTestSignedRequestV4(http.MethodGet, u, 0, nil, "", "", nil) // Generate an anonymous request. + if err != nil { + t.Fatalf("Test %s: Failed to create HTTP request for ListMultipartUploadsHandler: %v", instanceType, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusForbidden { + t.Errorf("Test %s: Expected the response status to be `http.StatusForbidden`, but instead found `%d`", instanceType, rec.Code) + } + + url := getListMultipartUploadsURLWithParams("", testCases[6].bucket, testCases[6].prefix, testCases[6].keyMarker, + testCases[6].uploadIDMarker, testCases[6].delimiter, testCases[6].maxUploads) + // Test for Anonymous/unsigned http request. + anonReq, err := newTestRequest(http.MethodGet, url, 0, nil) + if err != nil { + t.Fatalf("MinIO %s: Failed to create an anonymous request for bucket \"%s\": %v", + instanceType, bucketName, err) + } + + // ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse, + // sets the bucket policy using the policy statement generated from `getWriteOnlyBucketStatement` so that the + // unsigned request goes through and its validated again. + ExecObjectLayerAPIAnonTest(t, obj, "TestListMultipartUploadsHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonWriteOnlyBucketPolicy(bucketName)) + + // HTTP request for testing when `objectLayer` is set to `nil`. + // There is no need to use an existing bucket and valid input for creating the request + // since the `objectLayer==nil` check is performed before any other checks inside the handlers. + // The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called. + + nilBucket := "dummy-bucket" + url = getListMultipartUploadsURLWithParams("", nilBucket, "dummy-prefix", testCases[6].keyMarker, + testCases[6].uploadIDMarker, testCases[6].delimiter, testCases[6].maxUploads) + + nilReq, err := newTestRequest(http.MethodGet, url, 0, nil) + if err != nil { + t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType) + } + // execute the object layer set to `nil` test. + // `ExecObjectLayerAPINilTest` manages the operation. + ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq) +} + +// Wrapper for calling TestListBucketsHandler tests for both Erasure multiple disks and single node setup. +func TestListBucketsHandler(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testListBucketsHandler, endpoints: []string{"ListBuckets"}}) +} + +// testListBucketsHandler - Tests validate listing of buckets. +func testListBucketsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + testCases := []struct { + bucketName string + accessKey string + secretKey string + expectedRespStatus int + }{ + // Test case - 1. + // Validate a good case request succeeds. + { + bucketName: bucketName, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusOK, + }, + // Test case - 2. + // Test case with invalid accessKey to produce and validate Signature Mismatch error. + { + bucketName: bucketName, + accessKey: "abcd", + secretKey: "abcd", + expectedRespStatus: http.StatusForbidden, + }, + } + + for i, testCase := range testCases { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + req, lerr := newTestSignedRequestV4(http.MethodGet, getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey, nil) + if lerr != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for ListBucketsHandler: %v", i+1, instanceType, lerr) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + if rec.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) + } + + // Verify response of the V2 signed HTTP request. + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + recV2 := httptest.NewRecorder() + // construct HTTP request for PUT bucket policy endpoint. + + // verify response for V2 signed HTTP request. + reqV2, err := newTestSignedRequestV2(http.MethodGet, getListBucketURL(""), 0, nil, testCase.accessKey, testCase.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: %v", i+1, instanceType, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(recV2, reqV2) + if recV2.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code) + } + } + + // Test for Anonymous/unsigned http request. + // ListBucketsHandler doesn't support bucket policies, setting the policies shouldn't make a difference. + anonReq, err := newTestRequest(http.MethodGet, getListBucketURL(""), 0, nil) + if err != nil { + t.Fatalf("MinIO %s: Failed to create an anonymous request.", instanceType) + } + + // ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse, + // sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the + // unsigned request goes through and its validated again. + ExecObjectLayerAPIAnonTest(t, obj, "ListBucketsHandler", "", "", instanceType, apiRouter, anonReq, getAnonWriteOnlyBucketPolicy("*")) + + // HTTP request for testing when `objectLayer` is set to `nil`. + // There is no need to use an existing bucket and valid input for creating the request + // since the `objectLayer==nil` check is performed before any other checks inside the handlers. + // The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called. + + nilReq, err := newTestRequest(http.MethodGet, getListBucketURL(""), 0, nil) + if err != nil { + t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType) + } + // execute the object layer set to `nil` test. + // `ExecObjectLayerAPINilTest` manages the operation. + ExecObjectLayerAPINilTest(t, "", "", instanceType, apiRouter, nilReq) +} + +// Wrapper for calling DeleteMultipleObjects HTTP handler tests for both Erasure multiple disks and single node setup. +func TestAPIDeleteMultipleObjectsHandler(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPIDeleteMultipleObjectsHandler, endpoints: []string{"DeleteMultipleObjects", "PutBucketPolicy"}}) +} + +func testAPIDeleteMultipleObjectsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + var err error + + sha256sum := "" + var objectNames []string + for i := range 10 { + contentBytes := []byte("hello") + objectName := "test-object-" + strconv.Itoa(i) + if i == 0 { + objectName += "/" + contentBytes = []byte{} + } + // uploading the object. + _, err = obj.PutObject(GlobalContext, bucketName, objectName, mustGetPutObjReader(t, bytes.NewReader(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{}) + // if object upload fails stop the test. + if err != nil { + t.Fatalf("Put Object %d: Error uploading object: %v", i, err) + } + + // object used for the test. + objectNames = append(objectNames, objectName) + } + + contentBytes := []byte("hello") + for _, name := range []string{"private/object", "public/object"} { + // Uploading the object with retention enabled + _, err = obj.PutObject(GlobalContext, bucketName, name, mustGetPutObjReader(t, bytes.NewReader(contentBytes), int64(len(contentBytes)), "", sha256sum), ObjectOptions{}) + // if object upload fails stop the test. + if err != nil { + t.Fatalf("Put Object %s: Error uploading object: %v", name, err) + } + } + + // The following block will create a bucket policy with delete object to 'public/*'. This is + // to test a mixed response of a successful & failure while deleting objects in a single request + policyBytes := fmt.Appendf(nil, `{"Id": "Policy1637752602639", "Version": "2012-10-17", "Statement": [{"Sid": "Stmt1637752600730", "Action": "s3:DeleteObject", "Effect": "Allow", "Resource": "arn:aws:s3:::%s/public/*", "Principal": "*"}]}`, bucketName) + rec := httptest.NewRecorder() + req, err := newTestSignedRequestV4(http.MethodPut, getPutPolicyURL("", bucketName), int64(len(policyBytes)), bytes.NewReader(policyBytes), + credentials.AccessKey, credentials.SecretKey, nil) + if err != nil { + t.Fatalf("Failed to create HTTP request for PutBucketPolicyHandler: %v", err) + } + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusNoContent { + t.Errorf("Expected the response status to be `%d`, but instead found `%d`", 200, rec.Code) + } + + getObjectToDeleteList := func(objectNames []string) (objectList []ObjectToDelete) { + for _, objectName := range objectNames { + objectList = append(objectList, ObjectToDelete{ + ObjectV: ObjectV{ + ObjectName: objectName, + }, + }) + } + + return objectList + } + + getDeleteErrorList := func(objects []ObjectToDelete) (deleteErrorList []DeleteError) { + for _, obj := range objects { + deleteErrorList = append(deleteErrorList, DeleteError{ + Code: errorCodes[ErrAccessDenied].Code, + Message: errorCodes[ErrAccessDenied].Description, + Key: obj.ObjectName, + }) + } + + return deleteErrorList + } + + objects := []ObjectToDelete{} + objects = append(objects, ObjectToDelete{ + ObjectV: ObjectV{ + ObjectName: "private/object", + }, + }) + objects = append(objects, ObjectToDelete{ + ObjectV: ObjectV{ + ObjectName: "public/object", + }, + }) + requestList := []DeleteObjectsRequest{ + {Quiet: false, Objects: getObjectToDeleteList(objectNames[:5])}, + {Quiet: true, Objects: getObjectToDeleteList(objectNames[5:])}, + {Quiet: false, Objects: objects}, + } + + // generate multi objects delete response. + successRequest0 := encodeResponse(requestList[0]) + + deletedObjects := make([]DeletedObject, len(requestList[0].Objects)) + for i := range requestList[0].Objects { + var vid string + if isDirObject(requestList[0].Objects[i].ObjectName) { + vid = "" + } + deletedObjects[i] = DeletedObject{ + ObjectName: requestList[0].Objects[i].ObjectName, + VersionID: vid, + } + } + + successResponse0 := generateMultiDeleteResponse(requestList[0].Quiet, deletedObjects, nil) + encodedSuccessResponse0 := encodeResponse(successResponse0) + + successRequest1 := encodeResponse(requestList[1]) + + deletedObjects = make([]DeletedObject, len(requestList[1].Objects)) + for i := range requestList[1].Objects { + var vid string + if isDirObject(requestList[0].Objects[i].ObjectName) { + vid = "" + } + deletedObjects[i] = DeletedObject{ + ObjectName: requestList[1].Objects[i].ObjectName, + VersionID: vid, + } + } + + successResponse1 := generateMultiDeleteResponse(requestList[1].Quiet, deletedObjects, nil) + encodedSuccessResponse1 := encodeResponse(successResponse1) + + // generate multi objects delete response for errors. + // errorRequest := encodeResponse(requestList[1]) + errorResponse := generateMultiDeleteResponse(requestList[1].Quiet, deletedObjects, nil) + encodedErrorResponse := encodeResponse(errorResponse) + + anonRequest := encodeResponse(requestList[0]) + anonResponse := generateMultiDeleteResponse(requestList[0].Quiet, nil, getDeleteErrorList(requestList[0].Objects)) + encodedAnonResponse := encodeResponse(anonResponse) + + anonRequestWithPartialPublicAccess := encodeResponse(requestList[2]) + anonResponseWithPartialPublicAccess := generateMultiDeleteResponse(requestList[2].Quiet, + []DeletedObject{ + {ObjectName: "public/object"}, + }, + []DeleteError{ + { + Code: errorCodes[ErrAccessDenied].Code, + Message: errorCodes[ErrAccessDenied].Description, + Key: "private/object", + }, + }) + encodedAnonResponseWithPartialPublicAccess := encodeResponse(anonResponseWithPartialPublicAccess) + + testCases := []struct { + bucket string + objects []byte + accessKey string + secretKey string + expectedContent []byte + expectedRespStatus int + }{ + // Test case - 0. + // Delete objects with invalid access key. + 0: { + bucket: bucketName, + objects: successRequest0, + accessKey: "Invalid-AccessID", + secretKey: credentials.SecretKey, + expectedContent: nil, + expectedRespStatus: http.StatusForbidden, + }, + // Test case - 1. + // Delete valid objects with quiet flag off. + 1: { + bucket: bucketName, + objects: successRequest0, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedContent: encodedSuccessResponse0, + expectedRespStatus: http.StatusOK, + }, + // Test case - 2. + // Delete deleted objects with quiet flag off. + 2: { + bucket: bucketName, + objects: successRequest0, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedContent: encodedSuccessResponse0, + expectedRespStatus: http.StatusOK, + }, + // Test case - 3. + // Delete valid objects with quiet flag on. + 3: { + bucket: bucketName, + objects: successRequest1, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedContent: encodedSuccessResponse1, + expectedRespStatus: http.StatusOK, + }, + // Test case - 4. + // Delete previously deleted objects. + 4: { + bucket: bucketName, + objects: successRequest1, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedContent: encodedErrorResponse, + expectedRespStatus: http.StatusOK, + }, + // Test case - 5. + // Anonymous user access denied response + // Currently anonymous users cannot delete multiple objects in MinIO server + 5: { + bucket: bucketName, + objects: anonRequest, + accessKey: "", + secretKey: "", + expectedContent: encodedAnonResponse, + expectedRespStatus: http.StatusOK, + }, + // Test case - 6. + // Anonymous user has access to some public folder, issue removing with + // another private object as well + 6: { + bucket: bucketName, + objects: anonRequestWithPartialPublicAccess, + accessKey: "", + secretKey: "", + expectedContent: encodedAnonResponseWithPartialPublicAccess, + expectedRespStatus: http.StatusOK, + }, + // Test case - 7. + // Bucket does not exist. + 7: { + bucket: "unknown-bucket-name", + objects: successRequest0, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusNotFound, + }, + } + + for i, testCase := range testCases { + var req *http.Request + var actualContent []byte + + // Generate a signed or anonymous request based on the testCase + if testCase.accessKey != "" { + req, err = newTestSignedRequestV4(http.MethodPost, getDeleteMultipleObjectsURL("", testCase.bucket), + int64(len(testCase.objects)), bytes.NewReader(testCase.objects), testCase.accessKey, testCase.secretKey, nil) + } else { + req, err = newTestRequest(http.MethodPost, getDeleteMultipleObjectsURL("", testCase.bucket), + int64(len(testCase.objects)), bytes.NewReader(testCase.objects)) + } + if err != nil { + t.Fatalf("Failed to create HTTP request for DeleteMultipleObjects: %v", err) + } + + rec := httptest.NewRecorder() + + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to executes the registered handler. + apiRouter.ServeHTTP(rec, req) + // Assert the response code with the expected status. + if rec.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: MinIO %s: Expected the response status to be `%d`, but instead found `%d`", i, instanceType, testCase.expectedRespStatus, rec.Code) + } + + // read the response body. + actualContent, err = io.ReadAll(rec.Body) + if err != nil { + t.Fatalf("Test %d : MinIO %s: Failed parsing response body: %v", i, instanceType, err) + } + + // Verify whether the bucket obtained object is same as the one created. + if testCase.expectedContent != nil && !bytes.Equal(testCase.expectedContent, actualContent) { + t.Log(string(testCase.expectedContent), string(actualContent)) + t.Errorf("Test %d : MinIO %s: Object content differs from expected value.", i, instanceType) + } + } + + // HTTP request to test the case of `objectLayer` being set to `nil`. + // There is no need to use an existing bucket or valid input for creating the request, + // since the `objectLayer==nil` check is performed before any other checks inside the handlers. + // The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called. + // Indicating that all parts are uploaded and initiating completeMultipartUpload. + nilBucket := "dummy-bucket" + nilObject := "" + + nilReq, err := newTestSignedRequestV4(http.MethodPost, getDeleteMultipleObjectsURL("", nilBucket), 0, nil, "", "", nil) + if err != nil { + t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType) + } + // execute the object layer set to `nil` test. + // `ExecObjectLayerAPINilTest` manages the operation. + ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq) +} diff --git a/cmd/bucket-lifecycle-audit.go b/cmd/bucket-lifecycle-audit.go new file mode 100644 index 00000000..1fa76e09 --- /dev/null +++ b/cmd/bucket-lifecycle-audit.go @@ -0,0 +1,93 @@ +// Copyright (c) 2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "strconv" + + "github.com/minio/minio/internal/bucket/lifecycle" +) + +//go:generate stringer -type lcEventSrc -trimprefix lcEventSrc_ $GOFILE +type lcEventSrc uint8 + +//nolint:staticcheck,revive // Underscores are used here to indicate where common prefix ends and the enumeration name begins +const ( + lcEventSrc_None lcEventSrc = iota + lcEventSrc_Heal + lcEventSrc_Scanner + lcEventSrc_Decom + lcEventSrc_Rebal + lcEventSrc_s3HeadObject + lcEventSrc_s3GetObject + lcEventSrc_s3ListObjects + lcEventSrc_s3PutObject + lcEventSrc_s3CopyObject + lcEventSrc_s3CompleteMultipartUpload +) + +//revive:enable:var-naming +type lcAuditEvent struct { + lifecycle.Event + source lcEventSrc +} + +func (lae lcAuditEvent) Tags() map[string]string { + event := lae.Event + src := lae.source + const ( + ilmSrc = "ilm-src" + ilmAction = "ilm-action" + ilmDue = "ilm-due" + ilmRuleID = "ilm-rule-id" + ilmTier = "ilm-tier" + ilmNewerNoncurrentVersions = "ilm-newer-noncurrent-versions" + ilmNoncurrentDays = "ilm-noncurrent-days" + ) + tags := make(map[string]string, 5) + if src > lcEventSrc_None { + tags[ilmSrc] = src.String() + } + tags[ilmAction] = event.Action.String() + tags[ilmRuleID] = event.RuleID + + if !event.Due.IsZero() { + tags[ilmDue] = event.Due.Format(iso8601Format) + } + + // rule with Transition/NoncurrentVersionTransition in effect + if event.StorageClass != "" { + tags[ilmTier] = event.StorageClass + } + + // rule with NewernoncurrentVersions in effect + if event.NewerNoncurrentVersions > 0 { + tags[ilmNewerNoncurrentVersions] = strconv.Itoa(event.NewerNoncurrentVersions) + } + if event.NoncurrentDays > 0 { + tags[ilmNoncurrentDays] = strconv.Itoa(event.NoncurrentDays) + } + return tags +} + +func newLifecycleAuditEvent(src lcEventSrc, event lifecycle.Event) lcAuditEvent { + return lcAuditEvent{ + Event: event, + source: src, + } +} diff --git a/cmd/bucket-lifecycle-handlers.go b/cmd/bucket-lifecycle-handlers.go new file mode 100644 index 00000000..e917c9ad --- /dev/null +++ b/cmd/bucket-lifecycle-handlers.go @@ -0,0 +1,230 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "encoding/xml" + "net/http" + "strconv" + "time" + + "github.com/minio/minio/internal/bucket/lifecycle" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" + "github.com/minio/mux" + "github.com/minio/pkg/v3/policy" +) + +const ( + // Lifecycle configuration file. + bucketLifecycleConfig = "lifecycle.xml" +) + +// PutBucketLifecycleHandler - This HTTP handler stores given bucket lifecycle configuration as per +// https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html +func (api objectAPIHandlers) PutBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PutBucketLifecycle") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + vars := mux.Vars(r) + bucket := vars["bucket"] + + // PutBucketLifecycle always needs a Content-Md5 + if !validateLengthAndChecksum(r) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketLifecycleAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Check if bucket exists. + rcfg, err := globalBucketObjectLockSys.Get(bucket) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + bucketLifecycle, err := lifecycle.ParseLifecycleConfigWithID(r.Body) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Validate the received bucket policy document + if err = bucketLifecycle.Validate(rcfg); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Validate the transition storage ARNs + if err = validateTransitionTier(bucketLifecycle); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Create a map of updated set of rules in request + updatedRules := make(map[string]lifecycle.Rule, len(bucketLifecycle.Rules)) + for _, rule := range bucketLifecycle.Rules { + updatedRules[rule.ID] = rule + } + + // Get list of rules for the bucket from disk + meta, err := globalBucketMetadataSys.GetConfigFromDisk(ctx, bucket) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + expiryRuleRemoved := false + if len(meta.LifecycleConfigXML) > 0 { + var lcCfg lifecycle.Lifecycle + if err := xml.Unmarshal(meta.LifecycleConfigXML, &lcCfg); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + for _, rl := range lcCfg.Rules { + updRule, ok := updatedRules[rl.ID] + // original rule had expiry that is no longer in the new config, + // or rule is present but missing expiration flags + if (!rl.Expiration.IsNull() || !rl.NoncurrentVersionExpiration.IsNull()) && + (!ok || (updRule.Expiration.IsNull() && updRule.NoncurrentVersionExpiration.IsNull())) { + expiryRuleRemoved = true + } + } + } + + if bucketLifecycle.HasExpiry() || expiryRuleRemoved { + currtime := time.Now() + bucketLifecycle.ExpiryUpdatedAt = &currtime + } + + configData, err := xml.Marshal(bucketLifecycle) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Success. + writeSuccessResponseHeadersOnly(w) +} + +// GetBucketLifecycleHandler - This HTTP handler returns bucket policy configuration. +func (api objectAPIHandlers) GetBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetBucketLifecycle") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + vars := mux.Vars(r) + bucket := vars["bucket"] + + var withUpdatedAt bool + if updatedAtStr := r.Form.Get("withUpdatedAt"); updatedAtStr != "" { + var err error + withUpdatedAt, err = strconv.ParseBool(updatedAtStr) + if err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidLifecycleQueryParameter), r.URL) + return + } + } + + if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketLifecycleAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Check if bucket exists. + if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + config, updatedAt, err := globalBucketMetadataSys.GetLifecycleConfig(bucket) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + // explicitly set ExpiryUpdatedAt nil as its meant for internal consumption only + config.ExpiryUpdatedAt = nil + + configData, err := xml.Marshal(config) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if withUpdatedAt { + w.Header().Set(xhttp.MinIOLifecycleCfgUpdatedAt, updatedAt.Format(iso8601Format)) + } + // Write lifecycle configuration to client. + writeSuccessResponseXML(w, configData) +} + +// DeleteBucketLifecycleHandler - This HTTP handler removes bucket lifecycle configuration. +func (api objectAPIHandlers) DeleteBucketLifecycleHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "DeleteBucketLifecycle") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + vars := mux.Vars(r) + bucket := vars["bucket"] + + if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketLifecycleAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Check if bucket exists. + if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if _, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketLifecycleConfig); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Success. + writeSuccessNoContent(w) +} diff --git a/cmd/bucket-lifecycle-handlers_test.go b/cmd/bucket-lifecycle-handlers_test.go new file mode 100644 index 00000000..7dc2cf02 --- /dev/null +++ b/cmd/bucket-lifecycle-handlers_test.go @@ -0,0 +1,305 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "encoding/xml" + "net/http" + "net/http/httptest" + "testing" + + "github.com/minio/minio/internal/auth" +) + +// Test S3 Bucket lifecycle APIs with wrong credentials +func TestBucketLifecycleWrongCredentials(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testBucketLifecycleHandlersWrongCredentials, endpoints: []string{"GetBucketLifecycle", "PutBucketLifecycle", "DeleteBucketLifecycle"}}) +} + +// Test for authentication +func testBucketLifecycleHandlersWrongCredentials(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + // test cases with sample input and expected output. + testCases := []struct { + method string + bucketName string + accessKey string + secretKey string + // Sent body + body []byte + // Expected response + expectedRespStatus int + lifecycleResponse []byte + errorResponse APIErrorResponse + shouldPass bool + }{ + // GET empty credentials + { + method: http.MethodGet, bucketName: bucketName, + accessKey: "", + secretKey: "", + expectedRespStatus: http.StatusForbidden, + lifecycleResponse: []byte(""), + errorResponse: APIErrorResponse{ + Resource: SlashSeparator + bucketName + SlashSeparator, + Code: "AccessDenied", + Message: "Access Denied.", + }, + shouldPass: false, + }, + // GET wrong credentials + { + method: http.MethodGet, bucketName: bucketName, + accessKey: "abcd", + secretKey: "abcd", + expectedRespStatus: http.StatusForbidden, + lifecycleResponse: []byte(""), + errorResponse: APIErrorResponse{ + Resource: SlashSeparator + bucketName + SlashSeparator, + Code: "InvalidAccessKeyId", + Message: "The Access Key Id you provided does not exist in our records.", + }, + shouldPass: false, + }, + // PUT empty credentials + { + method: http.MethodPut, + bucketName: bucketName, + accessKey: "", + secretKey: "", + expectedRespStatus: http.StatusForbidden, + lifecycleResponse: []byte(""), + errorResponse: APIErrorResponse{ + Resource: SlashSeparator + bucketName + SlashSeparator, + Code: "AccessDenied", + Message: "Access Denied.", + }, + shouldPass: false, + }, + // PUT wrong credentials + { + method: http.MethodPut, + bucketName: bucketName, + accessKey: "abcd", + secretKey: "abcd", + expectedRespStatus: http.StatusForbidden, + lifecycleResponse: []byte(""), + errorResponse: APIErrorResponse{ + Resource: SlashSeparator + bucketName + SlashSeparator, + Code: "InvalidAccessKeyId", + Message: "The Access Key Id you provided does not exist in our records.", + }, + shouldPass: false, + }, + // DELETE empty credentials + { + method: http.MethodDelete, + bucketName: bucketName, + accessKey: "", + secretKey: "", + expectedRespStatus: http.StatusForbidden, + lifecycleResponse: []byte(""), + errorResponse: APIErrorResponse{ + Resource: SlashSeparator + bucketName + SlashSeparator, + Code: "AccessDenied", + Message: "Access Denied.", + }, + shouldPass: false, + }, + // DELETE wrong credentials + { + method: http.MethodDelete, + bucketName: bucketName, + accessKey: "abcd", + secretKey: "abcd", + expectedRespStatus: http.StatusForbidden, + lifecycleResponse: []byte(""), + errorResponse: APIErrorResponse{ + Resource: SlashSeparator + bucketName + SlashSeparator, + Code: "InvalidAccessKeyId", + Message: "The Access Key Id you provided does not exist in our records.", + }, + shouldPass: false, + }, + } + + testBucketLifecycle(obj, instanceType, bucketName, apiRouter, t, testCases) +} + +// Test S3 Bucket lifecycle APIs +func TestBucketLifecycle(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testBucketLifecycleHandlers, endpoints: []string{"GetBucketLifecycle", "PutBucketLifecycle", "DeleteBucketLifecycle"}}) +} + +// Simple tests of bucket lifecycle: PUT, GET, DELETE. +// Tests are related and the order is important. +func testBucketLifecycleHandlers(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + creds auth.Credentials, t *testing.T, +) { + // test cases with sample input and expected output. + testCases := []struct { + method string + bucketName string + accessKey string + secretKey string + // Sent body + body []byte + // Expected response + expectedRespStatus int + lifecycleResponse []byte + errorResponse APIErrorResponse + shouldPass bool + }{ + // Test case - 1. + // Filter contains more than (Prefix,Tag,And) rule + { + method: http.MethodPut, + bucketName: bucketName, + accessKey: creds.AccessKey, + secretKey: creds.SecretKey, + body: []byte(`idlogs/Key1Value1Enabled365`), + expectedRespStatus: http.StatusBadRequest, + lifecycleResponse: []byte(``), + errorResponse: APIErrorResponse{ + Resource: SlashSeparator + bucketName + SlashSeparator, + Code: "InvalidArgument", + Message: "Filter must have exactly one of Prefix, Tag, or And specified", + }, + + shouldPass: false, + }, + // Date contains wrong format + { + method: http.MethodPut, + bucketName: bucketName, + accessKey: creds.AccessKey, + secretKey: creds.SecretKey, + body: []byte(`idlogs/Key1Value1Enabled365`), + expectedRespStatus: http.StatusBadRequest, + lifecycleResponse: []byte(``), + errorResponse: APIErrorResponse{ + Resource: SlashSeparator + bucketName + SlashSeparator, + Code: "InvalidArgument", + Message: "Date must be provided in ISO 8601 format", + }, + + shouldPass: false, + }, + { + method: http.MethodPut, + bucketName: bucketName, + accessKey: creds.AccessKey, + secretKey: creds.SecretKey, + body: []byte(`idlogs/Enabled365`), + expectedRespStatus: http.StatusOK, + lifecycleResponse: []byte(``), + errorResponse: APIErrorResponse{}, + shouldPass: true, + }, + { + method: http.MethodGet, + accessKey: creds.AccessKey, + secretKey: creds.SecretKey, + bucketName: bucketName, + body: []byte(``), + expectedRespStatus: http.StatusOK, + lifecycleResponse: []byte(`idEnabledlogs/365`), + errorResponse: APIErrorResponse{}, + shouldPass: true, + }, + { + method: http.MethodDelete, + accessKey: creds.AccessKey, + secretKey: creds.SecretKey, + bucketName: bucketName, + body: []byte(``), + expectedRespStatus: http.StatusNoContent, + lifecycleResponse: []byte(``), + errorResponse: APIErrorResponse{}, + shouldPass: true, + }, + { + method: http.MethodGet, + accessKey: creds.AccessKey, + secretKey: creds.SecretKey, + bucketName: bucketName, + body: []byte(``), + expectedRespStatus: http.StatusNotFound, + lifecycleResponse: []byte(``), + errorResponse: APIErrorResponse{ + Resource: SlashSeparator + bucketName + SlashSeparator, + Code: "NoSuchLifecycleConfiguration", + Message: "The lifecycle configuration does not exist", + }, + shouldPass: false, + }, + } + + testBucketLifecycle(obj, instanceType, bucketName, apiRouter, t, testCases) +} + +// testBucketLifecycle is a generic testing of lifecycle requests +func testBucketLifecycle(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + t *testing.T, testCases []struct { + method string + bucketName string + accessKey string + secretKey string + body []byte + expectedRespStatus int + lifecycleResponse []byte + errorResponse APIErrorResponse + shouldPass bool + }, +) { + for i, testCase := range testCases { + // initialize httptest Recorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + // construct HTTP request + req, err := newTestSignedRequestV4(testCase.method, getBucketLifecycleURL("", testCase.bucketName), + int64(len(testCase.body)), bytes.NewReader(testCase.body), testCase.accessKey, testCase.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for GetBucketLocationHandler: %v", i+1, instanceType, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + if rec.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) + } + if testCase.shouldPass && !bytes.Equal(testCase.lifecycleResponse, rec.Body.Bytes()) { + t.Errorf("Test %d: %s: Expected the response to be `%s`, but instead found `%s`", i+1, instanceType, string(testCase.lifecycleResponse), rec.Body.String()) + } + errorResponse := APIErrorResponse{} + err = xml.Unmarshal(rec.Body.Bytes(), &errorResponse) + if err != nil && !testCase.shouldPass { + t.Fatalf("Test %d: %s: Unable to marshal response body %s", i+1, instanceType, rec.Body.String()) + } + if errorResponse.Resource != testCase.errorResponse.Resource { + t.Errorf("Test %d: %s: Expected the error resource to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Resource, errorResponse.Resource) + } + if errorResponse.Message != testCase.errorResponse.Message { + t.Errorf("Test %d: %s: Expected the error message to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Message, errorResponse.Message) + } + if errorResponse.Code != testCase.errorResponse.Code { + t.Errorf("Test %d: %s: Expected the error code to be `%s`, but instead found `%s`", i+1, instanceType, testCase.errorResponse.Code, errorResponse.Code) + } + } +} diff --git a/cmd/bucket-lifecycle.go b/cmd/bucket-lifecycle.go new file mode 100644 index 00000000..24fdc67d --- /dev/null +++ b/cmd/bucket-lifecycle.go @@ -0,0 +1,1126 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/xml" + "errors" + "fmt" + "io" + "maps" + "net/http" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/google/uuid" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7/pkg/tags" + "github.com/minio/minio/internal/amztime" + sse "github.com/minio/minio/internal/bucket/encryption" + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/event" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/s3select" + xnet "github.com/minio/pkg/v3/net" + "github.com/zeebo/xxh3" +) + +const ( + // Disabled means the lifecycle rule is inactive + Disabled = "Disabled" + // TransitionStatus status of transition + TransitionStatus = "transition-status" + // TransitionedObjectName name of transitioned object + TransitionedObjectName = "transitioned-object" + // TransitionedVersionID is version of remote object + TransitionedVersionID = "transitioned-versionID" + // TransitionTier name of transition storage class + TransitionTier = "transition-tier" +) + +// LifecycleSys - Bucket lifecycle subsystem. +type LifecycleSys struct{} + +// Get - gets lifecycle config associated to a given bucket name. +func (sys *LifecycleSys) Get(bucketName string) (lc *lifecycle.Lifecycle, err error) { + lc, _, err = globalBucketMetadataSys.GetLifecycleConfig(bucketName) + return lc, err +} + +// NewLifecycleSys - creates new lifecycle system. +func NewLifecycleSys() *LifecycleSys { + return &LifecycleSys{} +} + +func ilmTrace(startTime time.Time, duration time.Duration, oi ObjectInfo, event string, metadata map[string]string, err string) madmin.TraceInfo { + sz, _ := oi.GetActualSize() + if metadata == nil { + metadata = make(map[string]string) + } + metadata["version-id"] = oi.VersionID + return madmin.TraceInfo{ + TraceType: madmin.TraceILM, + Time: startTime, + NodeName: globalLocalNodeName, + FuncName: event, + Duration: duration, + Path: pathJoin(oi.Bucket, oi.Name), + Bytes: sz, + Error: err, + Message: getSource(4), + Custom: metadata, + } +} + +func (sys *LifecycleSys) trace(oi ObjectInfo) func(event string, metadata map[string]string, err error) { + startTime := time.Now() + return func(event string, metadata map[string]string, err error) { + duration := time.Since(startTime) + if globalTrace.NumSubscribers(madmin.TraceILM) > 0 { + e := "" + if err != nil { + e = err.Error() + } + globalTrace.Publish(ilmTrace(startTime, duration, oi, event, metadata, e)) + } + } +} + +type expiryTask struct { + objInfo ObjectInfo + event lifecycle.Event + src lcEventSrc +} + +// expiryStats records metrics related to ILM expiry activities +type expiryStats struct { + missedExpiryTasks atomic.Int64 + missedFreeVersTasks atomic.Int64 + missedTierJournalTasks atomic.Int64 + workers atomic.Int32 +} + +// MissedTasks returns the number of ILM expiry tasks that were missed since +// there were no available workers. +func (e *expiryStats) MissedTasks() int64 { + return e.missedExpiryTasks.Load() +} + +// MissedFreeVersTasks returns the number of free version collection tasks that +// were missed since there were no available workers. +func (e *expiryStats) MissedFreeVersTasks() int64 { + return e.missedFreeVersTasks.Load() +} + +// MissedTierJournalTasks returns the number of tasks to remove tiered objects +// that were missed since there were no available workers. +func (e *expiryStats) MissedTierJournalTasks() int64 { + return e.missedTierJournalTasks.Load() +} + +// NumWorkers returns the number of active workers executing one of ILM expiry +// tasks or free version collection tasks. +func (e *expiryStats) NumWorkers() int32 { + return e.workers.Load() +} + +type expiryOp interface { + OpHash() uint64 +} + +type freeVersionTask struct { + ObjectInfo +} + +func (f freeVersionTask) OpHash() uint64 { + return xxh3.HashString(f.TransitionedObject.Tier + f.TransitionedObject.Name) +} + +func (n noncurrentVersionsTask) OpHash() uint64 { + return xxh3.HashString(n.bucket + n.versions[0].ObjectName) +} + +func (j jentry) OpHash() uint64 { + return xxh3.HashString(j.TierName + j.ObjName) +} + +func (e expiryTask) OpHash() uint64 { + return xxh3.HashString(e.objInfo.Bucket + e.objInfo.Name) +} + +// expiryState manages all ILM related expiration activities. +type expiryState struct { + mu sync.RWMutex + workers atomic.Pointer[[]chan expiryOp] + + ctx context.Context + objAPI ObjectLayer + + stats expiryStats +} + +// PendingTasks returns the number of pending ILM expiry tasks. +func (es *expiryState) PendingTasks() int { + w := es.workers.Load() + if w == nil || len(*w) == 0 { + return 0 + } + var tasks int + for _, wrkr := range *w { + tasks += len(wrkr) + } + return tasks +} + +// enqueueTierJournalEntry enqueues a tier journal entry referring to a remote +// object corresponding to a 'replaced' object versions. This applies only to +// non-versioned or version suspended buckets. +func (es *expiryState) enqueueTierJournalEntry(je jentry) { + wrkr := es.getWorkerCh(je.OpHash()) + if wrkr == nil { + es.stats.missedTierJournalTasks.Add(1) + return + } + select { + case <-GlobalContext.Done(): + case wrkr <- je: + default: + es.stats.missedTierJournalTasks.Add(1) + } +} + +// enqueueFreeVersion enqueues a free version to be deleted +func (es *expiryState) enqueueFreeVersion(oi ObjectInfo) { + task := freeVersionTask{ObjectInfo: oi} + wrkr := es.getWorkerCh(task.OpHash()) + if wrkr == nil { + es.stats.missedFreeVersTasks.Add(1) + return + } + select { + case <-GlobalContext.Done(): + case wrkr <- task: + default: + es.stats.missedFreeVersTasks.Add(1) + } +} + +// enqueueByDays enqueues object versions expired by days for expiry. +func (es *expiryState) enqueueByDays(oi ObjectInfo, event lifecycle.Event, src lcEventSrc) { + task := expiryTask{objInfo: oi, event: event, src: src} + wrkr := es.getWorkerCh(task.OpHash()) + if wrkr == nil { + es.stats.missedExpiryTasks.Add(1) + return + } + select { + case <-GlobalContext.Done(): + case wrkr <- task: + default: + es.stats.missedExpiryTasks.Add(1) + } +} + +func (es *expiryState) enqueueNoncurrentVersions(bucket string, versions []ObjectToDelete, events []lifecycle.Event) { + if len(versions) == 0 { + return + } + + task := noncurrentVersionsTask{ + bucket: bucket, + versions: versions, + events: events, + } + wrkr := es.getWorkerCh(task.OpHash()) + if wrkr == nil { + es.stats.missedExpiryTasks.Add(1) + return + } + select { + case <-GlobalContext.Done(): + case wrkr <- task: + default: + es.stats.missedExpiryTasks.Add(1) + } +} + +// globalExpiryState is the per-node instance which manages all ILM expiry tasks. +var globalExpiryState *expiryState + +// newExpiryState creates an expiryState with buffered channels allocated for +// each ILM expiry task type. +func newExpiryState(ctx context.Context, objAPI ObjectLayer, n int) *expiryState { + es := &expiryState{ + ctx: ctx, + objAPI: objAPI, + } + workers := make([]chan expiryOp, 0, n) + es.workers.Store(&workers) + es.ResizeWorkers(n) + return es +} + +func (es *expiryState) getWorkerCh(h uint64) chan<- expiryOp { + w := es.workers.Load() + if w == nil || len(*w) == 0 { + return nil + } + workers := *w + return workers[h%uint64(len(workers))] +} + +func (es *expiryState) ResizeWorkers(n int) { + if n == 0 { + n = 100 + } + + // Lock to avoid multiple resizes to happen at the same time. + es.mu.Lock() + defer es.mu.Unlock() + var workers []chan expiryOp + if v := es.workers.Load(); v != nil { + // Copy to new array. + workers = append(workers, *v...) + } + + if n == len(workers) || n < 1 { + return + } + + for len(workers) < n { + input := make(chan expiryOp, 10000) + workers = append(workers, input) + go es.Worker(input) + es.stats.workers.Add(1) + } + + for len(workers) > n { + worker := workers[len(workers)-1] + workers = workers[:len(workers)-1] + worker <- expiryOp(nil) + es.stats.workers.Add(-1) + } + // Atomically replace workers. + es.workers.Store(&workers) +} + +// Worker handles 4 types of expiration tasks. +// 1. Expiry of objects, includes regular and transitioned objects +// 2. Expiry of noncurrent versions due to NewerNoncurrentVersions +// 3. Expiry of free-versions, for remote objects of transitioned object which have been expired since. +// 4. Expiry of remote objects corresponding to objects in a +// non-versioned/version suspended buckets +func (es *expiryState) Worker(input <-chan expiryOp) { + for { + select { + case <-es.ctx.Done(): + return + case v, ok := <-input: + if !ok { + return + } + if v == nil { + // ResizeWorkers signaling worker to quit + return + } + switch v := v.(type) { + case expiryTask: + if v.objInfo.TransitionedObject.Status != "" { + applyExpiryOnTransitionedObject(es.ctx, es.objAPI, v.objInfo, v.event, v.src) + } else { + applyExpiryOnNonTransitionedObjects(es.ctx, es.objAPI, v.objInfo, v.event, v.src) + } + case noncurrentVersionsTask: + deleteObjectVersions(es.ctx, es.objAPI, v.bucket, v.versions, v.events) + case jentry: + transitionLogIf(es.ctx, deleteObjectFromRemoteTier(es.ctx, v.ObjName, v.VersionID, v.TierName)) + case freeVersionTask: + oi := v.ObjectInfo + traceFn := globalLifecycleSys.trace(oi) + if !oi.TransitionedObject.FreeVersion { + // nothing to be done + continue + } + + ignoreNotFoundErr := func(err error) error { + switch { + case isErrVersionNotFound(err), isErrObjectNotFound(err): + return nil + } + return err + } + // Remove the remote object + err := deleteObjectFromRemoteTier(es.ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier) + if ignoreNotFoundErr(err) != nil { + transitionLogIf(es.ctx, err) + traceFn(ILMFreeVersionDelete, nil, err) + continue + } + + // Remove this free version + _, err = es.objAPI.DeleteObject(es.ctx, oi.Bucket, oi.Name, ObjectOptions{ + VersionID: oi.VersionID, + InclFreeVersions: true, + }) + if err == nil { + auditLogLifecycle(es.ctx, oi, ILMFreeVersionDelete, nil, traceFn) + } + if ignoreNotFoundErr(err) != nil { + transitionLogIf(es.ctx, err) + } + default: + bugLogIf(es.ctx, fmt.Errorf("Invalid work type - %v", v)) + } + } + } +} + +func initBackgroundExpiry(ctx context.Context, objectAPI ObjectLayer) { + globalExpiryState = newExpiryState(ctx, objectAPI, globalILMConfig.getExpirationWorkers()) +} + +type noncurrentVersionsTask struct { + bucket string + versions []ObjectToDelete + events []lifecycle.Event +} + +type transitionTask struct { + objInfo ObjectInfo + src lcEventSrc + event lifecycle.Event +} + +type transitionState struct { + transitionCh chan transitionTask + + ctx context.Context + objAPI ObjectLayer + mu sync.Mutex + numWorkers int + killCh chan struct{} + + activeTasks atomic.Int64 + missedImmediateTasks atomic.Int64 + + lastDayMu sync.RWMutex + lastDayStats map[string]*lastDayTierStats +} + +func (t *transitionState) queueTransitionTask(oi ObjectInfo, event lifecycle.Event, src lcEventSrc) { + task := transitionTask{objInfo: oi, event: event, src: src} + select { + case <-t.ctx.Done(): + case t.transitionCh <- task: + default: + switch src { + case lcEventSrc_s3PutObject, lcEventSrc_s3CopyObject, lcEventSrc_s3CompleteMultipartUpload: + // Update missed immediate tasks only for incoming requests. + t.missedImmediateTasks.Add(1) + } + } +} + +var globalTransitionState *transitionState + +// newTransitionState returns a transitionState object ready to be initialized +// via its Init method. +func newTransitionState(ctx context.Context) *transitionState { + return &transitionState{ + transitionCh: make(chan transitionTask, 100000), + ctx: ctx, + killCh: make(chan struct{}), + lastDayStats: make(map[string]*lastDayTierStats), + } +} + +// Init initializes t with given objAPI and instantiates the configured number +// of transition workers. +func (t *transitionState) Init(objAPI ObjectLayer) { + n := globalAPIConfig.getTransitionWorkers() + // Prefer ilm.transition_workers over now deprecated api.transition_workers + if tw := globalILMConfig.getTransitionWorkers(); tw > 0 { + n = tw + } + t.mu.Lock() + defer t.mu.Unlock() + + t.objAPI = objAPI + t.updateWorkers(n) +} + +// PendingTasks returns the number of ILM transition tasks waiting for a worker +// goroutine. +func (t *transitionState) PendingTasks() int { + return len(t.transitionCh) +} + +// ActiveTasks returns the number of active (ongoing) ILM transition tasks. +func (t *transitionState) ActiveTasks() int64 { + return t.activeTasks.Load() +} + +// MissedImmediateTasks returns the number of tasks - deferred to scanner due +// to tasks channel being backlogged. +func (t *transitionState) MissedImmediateTasks() int64 { + return t.missedImmediateTasks.Load() +} + +// worker waits for transition tasks +func (t *transitionState) worker(objectAPI ObjectLayer) { + for { + select { + case <-t.killCh: + return + case <-t.ctx.Done(): + return + case task, ok := <-t.transitionCh: + if !ok { + return + } + t.activeTasks.Add(1) + if err := transitionObject(t.ctx, objectAPI, task.objInfo, newLifecycleAuditEvent(task.src, task.event)); err != nil { + if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) && !xnet.IsNetworkOrHostDown(err, false) { + if !strings.Contains(err.Error(), "use of closed network connection") { + transitionLogIf(t.ctx, fmt.Errorf("Transition to %s failed for %s/%s version:%s with %w", + task.event.StorageClass, task.objInfo.Bucket, task.objInfo.Name, task.objInfo.VersionID, err)) + } + } + } else { + ts := tierStats{ + TotalSize: uint64(task.objInfo.Size), + NumVersions: 1, + } + if task.objInfo.IsLatest { + ts.NumObjects = 1 + } + t.addLastDayStats(task.event.StorageClass, ts) + } + t.activeTasks.Add(-1) + } + } +} + +func (t *transitionState) addLastDayStats(tier string, ts tierStats) { + t.lastDayMu.Lock() + defer t.lastDayMu.Unlock() + + if _, ok := t.lastDayStats[tier]; !ok { + t.lastDayStats[tier] = &lastDayTierStats{} + } + t.lastDayStats[tier].addStats(ts) +} + +func (t *transitionState) getDailyAllTierStats() DailyAllTierStats { + t.lastDayMu.RLock() + defer t.lastDayMu.RUnlock() + + res := make(DailyAllTierStats, len(t.lastDayStats)) + for tier, st := range t.lastDayStats { + res[tier] = st.clone() + } + return res +} + +// UpdateWorkers at the end of this function leaves n goroutines waiting for +// transition tasks +func (t *transitionState) UpdateWorkers(n int) { + t.mu.Lock() + defer t.mu.Unlock() + if t.objAPI == nil { // Init hasn't been called yet. + return + } + t.updateWorkers(n) +} + +func (t *transitionState) updateWorkers(n int) { + if n == 0 { + n = 100 + } + + for t.numWorkers < n { + go t.worker(t.objAPI) + t.numWorkers++ + } + + for t.numWorkers > n { + go func() { t.killCh <- struct{}{} }() + t.numWorkers-- + } +} + +var errInvalidStorageClass = errors.New("invalid storage class") + +func validateTransitionTier(lc *lifecycle.Lifecycle) error { + for _, rule := range lc.Rules { + if rule.Transition.StorageClass != "" { + if valid := globalTierConfigMgr.IsTierValid(rule.Transition.StorageClass); !valid { + return errInvalidStorageClass + } + } + if rule.NoncurrentVersionTransition.StorageClass != "" { + if valid := globalTierConfigMgr.IsTierValid(rule.NoncurrentVersionTransition.StorageClass); !valid { + return errInvalidStorageClass + } + } + } + return nil +} + +// enqueueTransitionImmediate enqueues obj for transition if eligible. +// This is to be called after a successful upload of an object (version). +func enqueueTransitionImmediate(obj ObjectInfo, src lcEventSrc) { + if lc, err := globalLifecycleSys.Get(obj.Bucket); err == nil { + switch event := lc.Eval(obj.ToLifecycleOpts()); event.Action { + case lifecycle.TransitionAction, lifecycle.TransitionVersionAction: + if obj.DeleteMarker || obj.IsDir { + // nothing to transition + return + } + globalTransitionState.queueTransitionTask(obj, event, src) + } + } +} + +// expireTransitionedObject handles expiry of transitioned/restored objects +// (versions) in one of the following situations: +// +// 1. when a restored (via PostRestoreObject API) object expires. +// 2. when a transitioned object expires (based on an ILM rule). +func expireTransitionedObject(ctx context.Context, objectAPI ObjectLayer, oi *ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc) error { + traceFn := globalLifecycleSys.trace(*oi) + opts := ObjectOptions{ + Versioned: globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name), + Expiration: ExpirationOptions{Expire: true}, + } + if lcEvent.Action.DeleteVersioned() { + opts.VersionID = oi.VersionID + } + tags := newLifecycleAuditEvent(src, lcEvent).Tags() + if lcEvent.Action.DeleteRestored() { + // delete locally restored copy of object or object version + // from the source, while leaving metadata behind. The data on + // transitioned tier lies untouched and still accessible + opts.Transition.ExpireRestored = true + _, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts) + if err == nil { + // TODO consider including expiry of restored object to events we + // notify. + auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn) + } + return err + } + + // Delete remote object from warm-tier + err := deleteObjectFromRemoteTier(ctx, oi.TransitionedObject.Name, oi.TransitionedObject.VersionID, oi.TransitionedObject.Tier) + if err == nil { + // Skip adding free version since we successfully deleted the + // remote object + opts.SkipFreeVersion = true + } else { + transitionLogIf(ctx, err) + } + + // Now, delete object from hot-tier namespace + if _, err := objectAPI.DeleteObject(ctx, oi.Bucket, oi.Name, opts); err != nil { + return err + } + + // Send audit for the lifecycle delete operation + defer auditLogLifecycle(ctx, *oi, ILMExpiry, tags, traceFn) + + eventName := event.ObjectRemovedDelete + if oi.DeleteMarker { + eventName = event.ObjectRemovedDeleteMarkerCreated + } + objInfo := ObjectInfo{ + Name: oi.Name, + VersionID: oi.VersionID, + DeleteMarker: oi.DeleteMarker, + } + // Notify object deleted event. + sendEvent(eventArgs{ + EventName: eventName, + BucketName: oi.Bucket, + Object: objInfo, + UserAgent: "Internal: [ILM-Expiry]", + Host: globalLocalNodeName, + }) + + return nil +} + +// generate an object name for transitioned object +func genTransitionObjName(bucket string) (string, error) { + u, err := uuid.NewRandom() + if err != nil { + return "", err + } + us := u.String() + hash := xxh3.HashString(pathJoin(globalDeploymentID(), bucket)) + obj := fmt.Sprintf("%s/%s/%s/%s", strconv.FormatUint(hash, 16), us[0:2], us[2:4], us) + return obj, nil +} + +// transition object to target specified by the transition ARN. When an object is transitioned to another +// storage specified by the transition ARN, the metadata is left behind on source cluster and original content +// is moved to the transition tier. Note that in the case of encrypted objects, entire encrypted stream is moved +// to the transition tier without decrypting or re-encrypting. +func transitionObject(ctx context.Context, objectAPI ObjectLayer, oi ObjectInfo, lae lcAuditEvent) (err error) { + timeILM := globalScannerMetrics.timeILM(lae.Action) + defer func() { + if err != nil { + return + } + timeILM(1) + }() + + opts := ObjectOptions{ + Transition: TransitionOptions{ + Status: lifecycle.TransitionPending, + Tier: lae.StorageClass, + ETag: oi.ETag, + }, + LifecycleAuditEvent: lae, + VersionID: oi.VersionID, + Versioned: globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name), + VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name), + MTime: oi.ModTime, + } + return objectAPI.TransitionObject(ctx, oi.Bucket, oi.Name, opts) +} + +type auditTierOp struct { + Tier string `json:"tier"` + TimeToResponseNS int64 `json:"timeToResponseNS"` + OutputBytes int64 `json:"tx,omitempty"` + Error string `json:"error,omitempty"` +} + +func (op auditTierOp) String() string { + // flattening the auditTierOp{} for audit + return fmt.Sprintf("tier:%s,respNS:%d,tx:%d,err:%s", op.Tier, op.TimeToResponseNS, op.OutputBytes, op.Error) +} + +func auditTierActions(ctx context.Context, tier string, bytes int64) func(err error) { + startTime := time.Now() + return func(err error) { + // Record only when audit targets configured. + if len(logger.AuditTargets()) == 0 { + return + } + + op := auditTierOp{ + Tier: tier, + OutputBytes: bytes, + } + + if err == nil { + since := time.Since(startTime) + op.TimeToResponseNS = since.Nanoseconds() + globalTierMetrics.Observe(tier, since) + globalTierMetrics.logSuccess(tier) + } else { + op.Error = err.Error() + globalTierMetrics.logFailure(tier) + } + + logger.GetReqInfo(ctx).AppendTags("tierStats", op.String()) + } +} + +// getTransitionedObjectReader returns a reader from the transitioned tier. +func getTransitionedObjectReader(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, oi ObjectInfo, opts ObjectOptions) (gr *GetObjectReader, err error) { + tgtClient, err := globalTierConfigMgr.getDriver(ctx, oi.TransitionedObject.Tier) + if err != nil { + return nil, fmt.Errorf("transition storage class not configured: %w", err) + } + + fn, off, length, err := NewGetObjectReader(rs, oi, opts, h) + if err != nil { + return nil, ErrorRespToObjectError(err, bucket, object) + } + gopts := WarmBackendGetOpts{} + + // get correct offsets for object + if off >= 0 && length >= 0 { + gopts.startOffset = off + gopts.length = length + } + + timeTierAction := auditTierActions(ctx, oi.TransitionedObject.Tier, length) + reader, err := tgtClient.Get(ctx, oi.TransitionedObject.Name, remoteVersionID(oi.TransitionedObject.VersionID), gopts) + if err != nil { + return nil, err + } + closer := func() { + timeTierAction(reader.Close()) + } + return fn(reader, h, closer) +} + +// RestoreRequestType represents type of restore. +type RestoreRequestType string + +const ( + // SelectRestoreRequest specifies select request. This is the only valid value + SelectRestoreRequest RestoreRequestType = "SELECT" +) + +// Encryption specifies encryption setting on restored bucket +type Encryption struct { + EncryptionType sse.Algorithm `xml:"EncryptionType"` + KMSContext string `xml:"KMSContext,omitempty"` + KMSKeyID string `xml:"KMSKeyId,omitempty"` +} + +// MetadataEntry denotes name and value. +type MetadataEntry struct { + Name string `xml:"Name"` + Value string `xml:"Value"` +} + +// S3Location specifies s3 location that receives result of a restore object request +type S3Location struct { + BucketName string `xml:"BucketName,omitempty"` + Encryption Encryption `xml:"Encryption,omitempty"` + Prefix string `xml:"Prefix,omitempty"` + StorageClass string `xml:"StorageClass,omitempty"` + Tagging *tags.Tags `xml:"Tagging,omitempty"` + UserMetadata []MetadataEntry `xml:"UserMetadata"` +} + +// OutputLocation specifies bucket where object needs to be restored +type OutputLocation struct { + S3 S3Location `xml:"S3,omitempty"` +} + +// IsEmpty returns true if output location not specified. +func (o *OutputLocation) IsEmpty() bool { + return o.S3.BucketName == "" +} + +// SelectParameters specifies sql select parameters +type SelectParameters struct { + s3select.S3Select +} + +// IsEmpty returns true if no select parameters set +func (sp *SelectParameters) IsEmpty() bool { + return sp == nil +} + +var selectParamsXMLName = "SelectParameters" + +// UnmarshalXML - decodes XML data. +func (sp *SelectParameters) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + // Essentially the same as S3Select barring the xml name. + if start.Name.Local == selectParamsXMLName { + start.Name = xml.Name{Space: "", Local: "SelectRequest"} + } + return sp.S3Select.UnmarshalXML(d, start) +} + +// RestoreObjectRequest - xml to restore a transitioned object +type RestoreObjectRequest struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ RestoreRequest" json:"-"` + Days int `xml:"Days,omitempty"` + Type RestoreRequestType `xml:"Type,omitempty"` + Tier string `xml:"Tier"` + Description string `xml:"Description,omitempty"` + SelectParameters *SelectParameters `xml:"SelectParameters,omitempty"` + OutputLocation OutputLocation `xml:"OutputLocation,omitempty"` +} + +// Maximum 2MiB size per restore object request. +const maxRestoreObjectRequestSize = 2 << 20 + +// parseRestoreRequest parses RestoreObjectRequest from xml +func parseRestoreRequest(reader io.Reader) (*RestoreObjectRequest, error) { + req := RestoreObjectRequest{} + if err := xml.NewDecoder(io.LimitReader(reader, maxRestoreObjectRequestSize)).Decode(&req); err != nil { + return nil, err + } + return &req, nil +} + +// validate a RestoreObjectRequest as per AWS S3 spec https://docs.aws.amazon.com/AmazonS3/latest/API/API_RestoreObject.html +func (r *RestoreObjectRequest) validate(ctx context.Context, objAPI ObjectLayer) error { + if r.Type != SelectRestoreRequest && !r.SelectParameters.IsEmpty() { + return fmt.Errorf("Select parameters can only be specified with SELECT request type") + } + if r.Type == SelectRestoreRequest && r.SelectParameters.IsEmpty() { + return fmt.Errorf("SELECT restore request requires select parameters to be specified") + } + + if r.Type != SelectRestoreRequest && !r.OutputLocation.IsEmpty() { + return fmt.Errorf("OutputLocation required only for SELECT request type") + } + if r.Type == SelectRestoreRequest && r.OutputLocation.IsEmpty() { + return fmt.Errorf("OutputLocation required for SELECT requests") + } + + if r.Days != 0 && r.Type == SelectRestoreRequest { + return fmt.Errorf("Days cannot be specified with SELECT restore request") + } + if r.Days == 0 && r.Type != SelectRestoreRequest { + return fmt.Errorf("restoration days should be at least 1") + } + // Check if bucket exists. + if !r.OutputLocation.IsEmpty() { + if _, err := objAPI.GetBucketInfo(ctx, r.OutputLocation.S3.BucketName, BucketOptions{}); err != nil { + return err + } + if r.OutputLocation.S3.Prefix == "" { + return fmt.Errorf("Prefix is a required parameter in OutputLocation") + } + if r.OutputLocation.S3.Encryption.EncryptionType != xhttp.AmzEncryptionAES { + return NotImplemented{} + } + } + return nil +} + +// postRestoreOpts returns ObjectOptions with version-id from the POST restore object request for a given bucket and object. +func postRestoreOpts(ctx context.Context, r *http.Request, bucket, object string) (opts ObjectOptions, err error) { + versioned := globalBucketVersioningSys.PrefixEnabled(bucket, object) + versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object) + vid := strings.TrimSpace(r.Form.Get(xhttp.VersionID)) + if vid != "" && vid != nullVersionID { + _, err := uuid.Parse(vid) + if err != nil { + s3LogIf(ctx, err) + return opts, InvalidVersionID{ + Bucket: bucket, + Object: object, + VersionID: vid, + } + } + if !versioned && !versionSuspended { + return opts, InvalidArgument{ + Bucket: bucket, + Object: object, + Err: fmt.Errorf("version-id specified %s but versioning is not enabled on %s", opts.VersionID, bucket), + } + } + } + return ObjectOptions{ + Versioned: versioned, + VersionSuspended: versionSuspended, + VersionID: vid, + }, nil +} + +// set ObjectOptions for PUT call to restore temporary copy of transitioned data +func putRestoreOpts(bucket, object string, rreq *RestoreObjectRequest, objInfo ObjectInfo) (putOpts ObjectOptions) { + meta := make(map[string]string) + sc := rreq.OutputLocation.S3.StorageClass + if sc == "" { + sc = objInfo.StorageClass + } + meta[strings.ToLower(xhttp.AmzStorageClass)] = sc + + if rreq.Type == SelectRestoreRequest { + for _, v := range rreq.OutputLocation.S3.UserMetadata { + if !stringsHasPrefixFold(v.Name, "x-amz-meta") { + meta["x-amz-meta-"+v.Name] = v.Value + continue + } + meta[v.Name] = v.Value + } + if tags := rreq.OutputLocation.S3.Tagging.String(); tags != "" { + meta[xhttp.AmzObjectTagging] = tags + } + if rreq.OutputLocation.S3.Encryption.EncryptionType != "" { + meta[xhttp.AmzServerSideEncryption] = xhttp.AmzEncryptionAES + } + return ObjectOptions{ + Versioned: globalBucketVersioningSys.PrefixEnabled(bucket, object), + VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object), + UserDefined: meta, + } + } + maps.Copy(meta, objInfo.UserDefined) + if len(objInfo.UserTags) != 0 { + meta[xhttp.AmzObjectTagging] = objInfo.UserTags + } + // Set restore object status + restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days) + meta[xhttp.AmzRestore] = completedRestoreObj(restoreExpiry).String() + return ObjectOptions{ + Versioned: globalBucketVersioningSys.PrefixEnabled(bucket, object), + VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, object), + UserDefined: meta, + VersionID: objInfo.VersionID, + MTime: objInfo.ModTime, + Expires: objInfo.Expires, + } +} + +var errRestoreHDRMalformed = fmt.Errorf("x-amz-restore header malformed") + +// IsRemote returns true if this object version's contents are in its remote +// tier. +func (fi FileInfo) IsRemote() bool { + if fi.TransitionStatus != lifecycle.TransitionComplete { + return false + } + return !isRestoredObjectOnDisk(fi.Metadata) +} + +// IsRemote returns true if this object version's contents are in its remote +// tier. +func (oi ObjectInfo) IsRemote() bool { + if oi.TransitionedObject.Status != lifecycle.TransitionComplete { + return false + } + return !isRestoredObjectOnDisk(oi.UserDefined) +} + +// restoreObjStatus represents a restore-object's status. It can be either +// ongoing or completed. +type restoreObjStatus struct { + ongoing bool + expiry time.Time +} + +// ongoingRestoreObj constructs restoreObjStatus for an ongoing restore-object. +func ongoingRestoreObj() restoreObjStatus { + return restoreObjStatus{ + ongoing: true, + } +} + +// completedRestoreObj constructs restoreObjStatus for a completed restore-object with given expiry. +func completedRestoreObj(expiry time.Time) restoreObjStatus { + return restoreObjStatus{ + ongoing: false, + expiry: expiry.UTC(), + } +} + +// String returns x-amz-restore compatible representation of r. +func (r restoreObjStatus) String() string { + if r.Ongoing() { + return `ongoing-request="true"` + } + return fmt.Sprintf(`ongoing-request="false", expiry-date="%s"`, r.expiry.Format(http.TimeFormat)) +} + +// Expiry returns expiry of restored object and true if restore-object has completed. +// Otherwise returns zero value of time.Time and false. +func (r restoreObjStatus) Expiry() (time.Time, bool) { + if r.Ongoing() { + return time.Time{}, false + } + return r.expiry, true +} + +// Ongoing returns true if restore-object is ongoing. +func (r restoreObjStatus) Ongoing() bool { + return r.ongoing +} + +// OnDisk returns true if restored object contents exist in MinIO. Otherwise returns false. +// The restore operation could be in one of the following states, +// - in progress (no content on MinIO's disks yet) +// - completed +// - completed but expired (again, no content on MinIO's disks) +func (r restoreObjStatus) OnDisk() bool { + if expiry, ok := r.Expiry(); ok && time.Now().UTC().Before(expiry) { + // completed + return true + } + return false // in progress or completed but expired +} + +// parseRestoreObjStatus parses restoreHdr from AmzRestore header. If the value is valid it returns a +// restoreObjStatus value with the status and expiry (if any). Otherwise returns +// the empty value and an error indicating the parse failure. +func parseRestoreObjStatus(restoreHdr string) (restoreObjStatus, error) { + tokens := strings.SplitN(restoreHdr, ",", 2) + progressTokens := strings.SplitN(tokens[0], "=", 2) + if len(progressTokens) != 2 { + return restoreObjStatus{}, errRestoreHDRMalformed + } + if strings.TrimSpace(progressTokens[0]) != "ongoing-request" { + return restoreObjStatus{}, errRestoreHDRMalformed + } + + switch progressTokens[1] { + case "true", `"true"`: // true without double quotes is deprecated in Feb 2022 + if len(tokens) == 1 { + return ongoingRestoreObj(), nil + } + case "false", `"false"`: // false without double quotes is deprecated in Feb 2022 + if len(tokens) != 2 { + return restoreObjStatus{}, errRestoreHDRMalformed + } + expiryTokens := strings.SplitN(tokens[1], "=", 2) + if len(expiryTokens) != 2 { + return restoreObjStatus{}, errRestoreHDRMalformed + } + if strings.TrimSpace(expiryTokens[0]) != "expiry-date" { + return restoreObjStatus{}, errRestoreHDRMalformed + } + expiry, err := amztime.ParseHeader(strings.Trim(expiryTokens[1], `"`)) + if err != nil { + return restoreObjStatus{}, errRestoreHDRMalformed + } + return completedRestoreObj(expiry), nil + } + return restoreObjStatus{}, errRestoreHDRMalformed +} + +// isRestoredObjectOnDisk returns true if the restored object is on disk. Note +// this function must be called only if object version's transition status is +// complete. +func isRestoredObjectOnDisk(meta map[string]string) (onDisk bool) { + if restoreHdr, ok := meta[xhttp.AmzRestore]; ok { + if restoreStatus, err := parseRestoreObjStatus(restoreHdr); err == nil { + return restoreStatus.OnDisk() + } + } + return onDisk +} + +// ToLifecycleOpts returns lifecycle.ObjectOpts value for oi. +func (oi ObjectInfo) ToLifecycleOpts() lifecycle.ObjectOpts { + return lifecycle.ObjectOpts{ + Name: oi.Name, + UserTags: oi.UserTags, + VersionID: oi.VersionID, + ModTime: oi.ModTime, + Size: oi.Size, + IsLatest: oi.IsLatest, + NumVersions: oi.NumVersions, + DeleteMarker: oi.DeleteMarker, + SuccessorModTime: oi.SuccessorModTime, + RestoreOngoing: oi.RestoreOngoing, + RestoreExpires: oi.RestoreExpires, + TransitionStatus: oi.TransitionedObject.Status, + UserDefined: oi.UserDefined, + VersionPurgeStatus: oi.VersionPurgeStatus, + ReplicationStatus: oi.ReplicationStatus, + } +} diff --git a/cmd/bucket-lifecycle_test.go b/cmd/bucket-lifecycle_test.go new file mode 100644 index 00000000..a37e3492 --- /dev/null +++ b/cmd/bucket-lifecycle_test.go @@ -0,0 +1,247 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "net/http" + "testing" + "time" + + "github.com/minio/minio/internal/bucket/lifecycle" + xhttp "github.com/minio/minio/internal/http" +) + +// TestParseRestoreObjStatus tests parseRestoreObjStatus +func TestParseRestoreObjStatus(t *testing.T) { + testCases := []struct { + restoreHdr string + expectedStatus restoreObjStatus + expectedErr error + }{ + { + // valid: represents a restored object, 'pending' expiry. + restoreHdr: `ongoing-request="false", expiry-date="Fri, 21 Dec 2012 00:00:00 GMT"`, + expectedStatus: restoreObjStatus{ + ongoing: false, + expiry: time.Date(2012, 12, 21, 0, 0, 0, 0, time.UTC), + }, + expectedErr: nil, + }, + { + // valid: represents an ongoing restore object request. + restoreHdr: `ongoing-request="true"`, + expectedStatus: restoreObjStatus{ + ongoing: true, + }, + expectedErr: nil, + }, + { + // invalid; ongoing restore object request can't have expiry set on it. + restoreHdr: `ongoing-request="true", expiry-date="Fri, 21 Dec 2012 00:00:00 GMT"`, + expectedStatus: restoreObjStatus{}, + expectedErr: errRestoreHDRMalformed, + }, + { + // invalid; completed restore object request must have expiry set on it. + restoreHdr: `ongoing-request="false"`, + expectedStatus: restoreObjStatus{}, + expectedErr: errRestoreHDRMalformed, + }, + } + for i, tc := range testCases { + actual, err := parseRestoreObjStatus(tc.restoreHdr) + if err != tc.expectedErr { + t.Fatalf("Test %d: got %v expected %v", i+1, err, tc.expectedErr) + } + if actual != tc.expectedStatus { + t.Fatalf("Test %d: got %v expected %v", i+1, actual, tc.expectedStatus) + } + } +} + +// TestRestoreObjStatusRoundTrip restoreObjStatus roundtrip +func TestRestoreObjStatusRoundTrip(t *testing.T) { + testCases := []restoreObjStatus{ + ongoingRestoreObj(), + completedRestoreObj(time.Now().UTC()), + } + for i, tc := range testCases { + actual, err := parseRestoreObjStatus(tc.String()) + if err != nil { + t.Fatalf("Test %d: parse restore object failed: %v", i+1, err) + } + if actual.ongoing != tc.ongoing || actual.expiry.Format(http.TimeFormat) != tc.expiry.Format(http.TimeFormat) { + t.Fatalf("Test %d: got %v expected %v", i+1, actual, tc) + } + } +} + +// TestRestoreObjOnDisk tests restoreObjStatus' OnDisk method +func TestRestoreObjOnDisk(t *testing.T) { + testCases := []struct { + restoreStatus restoreObjStatus + ondisk bool + }{ + { + // restore in progress + restoreStatus: ongoingRestoreObj(), + ondisk: false, + }, + { + // restore completed but expired + restoreStatus: completedRestoreObj(time.Now().Add(-time.Hour)), + ondisk: false, + }, + { + // restore completed + restoreStatus: completedRestoreObj(time.Now().Add(time.Hour)), + ondisk: true, + }, + } + + for i, tc := range testCases { + if actual := tc.restoreStatus.OnDisk(); actual != tc.ondisk { + t.Fatalf("Test %d: expected %v but got %v", i+1, tc.ondisk, actual) + } + } +} + +// TestIsRestoredObjectOnDisk tests isRestoredObjectOnDisk helper function +func TestIsRestoredObjectOnDisk(t *testing.T) { + testCases := []struct { + meta map[string]string + ondisk bool + }{ + { + // restore in progress + meta: map[string]string{ + xhttp.AmzRestore: ongoingRestoreObj().String(), + }, + ondisk: false, + }, + { + // restore completed + meta: map[string]string{ + xhttp.AmzRestore: completedRestoreObj(time.Now().Add(time.Hour)).String(), + }, + ondisk: true, + }, + { + // restore completed but expired + meta: map[string]string{ + xhttp.AmzRestore: completedRestoreObj(time.Now().Add(-time.Hour)).String(), + }, + ondisk: false, + }, + } + + for i, tc := range testCases { + if actual := isRestoredObjectOnDisk(tc.meta); actual != tc.ondisk { + t.Fatalf("Test %d: expected %v but got %v for %v", i+1, tc.ondisk, actual, tc.meta) + } + } +} + +func TestObjectIsRemote(t *testing.T) { + fi := newFileInfo("object", 8, 8) + fi.Erasure.Index = 1 + if !fi.IsValid() { + t.Fatalf("unable to get xl meta") + } + + testCases := []struct { + meta map[string]string + remote bool + }{ + { + // restore in progress + meta: map[string]string{ + xhttp.AmzRestore: ongoingRestoreObj().String(), + }, + remote: true, + }, + { + // restore completed + meta: map[string]string{ + xhttp.AmzRestore: completedRestoreObj(time.Now().Add(time.Hour)).String(), + }, + remote: false, + }, + { + // restore completed but expired + meta: map[string]string{ + xhttp.AmzRestore: completedRestoreObj(time.Now().Add(-time.Hour)).String(), + }, + remote: true, + }, + { + // restore never initiated + meta: map[string]string{}, + remote: true, + }, + } + for i, tc := range testCases { + // Set transition status to complete + fi.TransitionStatus = lifecycle.TransitionComplete + fi.Metadata = tc.meta + if got := fi.IsRemote(); got != tc.remote { + t.Fatalf("Test %d.a: expected %v got %v", i+1, tc.remote, got) + } + oi := fi.ToObjectInfo("bucket", "object", false) + if got := oi.IsRemote(); got != tc.remote { + t.Fatalf("Test %d.b: expected %v got %v", i+1, tc.remote, got) + } + } + // Reset transition status; An object that's not transitioned is not remote. + fi.TransitionStatus = "" + fi.Metadata = nil + if got := fi.IsRemote(); got != false { + t.Fatalf("Expected object not to be remote but got %v", got) + } +} + +func TestValidateTransitionTier(t *testing.T) { + globalTierConfigMgr = NewTierConfigMgr() + testCases := []struct { + xml []byte + expectedErr error + }{ + { + // non-existent storage-class + xml: []byte(`ruleEnabled1"NONEXISTENT"`), + expectedErr: errInvalidStorageClass, + }, + { + // no transition rule + xml: []byte(`ruleEnabled1`), + expectedErr: nil, + }, + } + for i, tc := range testCases { + lc, err := lifecycle.ParseLifecycleConfig(bytes.NewReader(tc.xml)) + if err != nil { + t.Fatalf("Test %d: Failed to parse lifecycle config %v", i+1, err) + } + + err = validateTransitionTier(lc) + if err != tc.expectedErr { + t.Fatalf("Test %d: Expected %v but got %v", i+1, tc.expectedErr, err) + } + } +} diff --git a/cmd/bucket-listobjects-handlers.go b/cmd/bucket-listobjects-handlers.go new file mode 100644 index 00000000..0afd76c4 --- /dev/null +++ b/cmd/bucket-listobjects-handlers.go @@ -0,0 +1,325 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "net/http" + "strconv" + "strings" + + "github.com/minio/minio/internal/logger" + "github.com/minio/mux" + + "github.com/minio/pkg/v3/policy" +) + +// Validate all the ListObjects query arguments, returns an APIErrorCode +// if one of the args do not meet the required conditions. +// Special conditions required by MinIO server are as below +// - delimiter if set should be equal to '/', otherwise the request is rejected. +// - marker if set should have a common prefix with 'prefix' param, otherwise +// the request is rejected. +func validateListObjectsArgs(prefix, marker, delimiter, encodingType string, maxKeys int) APIErrorCode { + // Max keys cannot be negative. + if maxKeys < 0 { + return ErrInvalidMaxKeys + } + + if encodingType != "" { + // AWS S3 spec only supports 'url' encoding type + if !strings.EqualFold(encodingType, "url") { + return ErrInvalidEncodingMethod + } + } + + if !IsValidObjectPrefix(prefix) { + return ErrInvalidObjectName + } + + if marker != "" && !HasPrefix(marker, prefix) { + return ErrNotImplemented + } + + return ErrNone +} + +func (api objectAPIHandlers) ListObjectVersionsHandler(w http.ResponseWriter, r *http.Request) { + api.listObjectVersionsHandler(w, r, false) +} + +func (api objectAPIHandlers) ListObjectVersionsMHandler(w http.ResponseWriter, r *http.Request) { + api.listObjectVersionsHandler(w, r, true) +} + +// ListObjectVersionsHandler - GET Bucket Object versions +// You can use the versions subresource to list metadata about all +// of the versions of objects in a bucket. +func (api objectAPIHandlers) listObjectVersionsHandler(w http.ResponseWriter, r *http.Request, metadata bool) { + ctx := newContext(r, w, "ListObjectVersions") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketVersionsAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + var checkObjMeta metaCheckFn + if metadata { + checkObjMeta = func(name string, action policy.Action) (s3Err APIErrorCode) { + return checkRequestAuthType(ctx, r, action, bucket, name) + } + } + urlValues := r.Form + + // Extract all the listBucketVersions query params to their native values. + prefix, marker, delimiter, maxkeys, encodingType, versionIDMarker, errCode := getListBucketObjectVersionsArgs(urlValues) + if errCode != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL) + return + } + + // Validate the query params before beginning to serve the request. + if s3Error := validateListObjectsArgs(prefix, marker, delimiter, encodingType, maxkeys); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + listObjectVersions := objectAPI.ListObjectVersions + + // Initiate a list object versions operation based on the input params. + // On success would return back ListObjectsInfo object to be + // marshaled into S3 compatible XML header. + listObjectVersionsInfo, err := listObjectVersions(ctx, bucket, prefix, marker, versionIDMarker, delimiter, maxkeys) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if err = DecryptETags(ctx, GlobalKMS, listObjectVersionsInfo.Objects); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + response := generateListVersionsResponse(ctx, bucket, prefix, marker, versionIDMarker, delimiter, encodingType, maxkeys, listObjectVersionsInfo, checkObjMeta) + + // Write success response. + writeSuccessResponseXML(w, encodeResponseList(response)) +} + +// ListObjectsV2MHandler - GET Bucket (List Objects) Version 2 with metadata. +// -------------------------- +// This implementation of the GET operation returns some or all (up to 1000) +// of the objects in a bucket. You can use the request parameters as selection +// criteria to return a subset of the objects in a bucket. +// +// NOTE: It is recommended that this API to be used for application development. +// MinIO continues to support ListObjectsV1 and V2 for supporting legacy tools. +func (api objectAPIHandlers) ListObjectsV2MHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "ListObjectsV2M") + api.listObjectsV2Handler(ctx, w, r, true) +} + +// ListObjectsV2Handler - GET Bucket (List Objects) Version 2. +// -------------------------- +// This implementation of the GET operation returns some or all (up to 1000) +// of the objects in a bucket. You can use the request parameters as selection +// criteria to return a subset of the objects in a bucket. +// +// NOTE: It is recommended that this API to be used for application development. +// MinIO continues to support ListObjectsV1 for supporting legacy tools. +func (api objectAPIHandlers) ListObjectsV2Handler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "ListObjectsV2") + api.listObjectsV2Handler(ctx, w, r, false) +} + +// listObjectsV2Handler performs listing either with or without extra metadata. +func (api objectAPIHandlers) listObjectsV2Handler(ctx context.Context, w http.ResponseWriter, r *http.Request, metadata bool) { + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + var checkObjMeta metaCheckFn + if metadata { + checkObjMeta = func(name string, action policy.Action) (s3Err APIErrorCode) { + return checkRequestAuthType(ctx, r, action, bucket, name) + } + } + urlValues := r.Form + + // Extract all the listObjectsV2 query params to their native values. + prefix, token, startAfter, delimiter, fetchOwner, maxKeys, encodingType, errCode := getListObjectsV2Args(urlValues) + if errCode != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(errCode), r.URL) + return + } + + // Validate the query params before beginning to serve the request. + if s3Error := validateListObjectsArgs(prefix, token, delimiter, encodingType, maxKeys); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + var ( + listObjectsV2Info ListObjectsV2Info + err error + ) + + if r.Header.Get(xMinIOExtract) == "true" && strings.Contains(prefix, archivePattern) { + // Initiate a list objects operation inside a zip file based in the input params + listObjectsV2Info, err = listObjectsV2InArchive(ctx, objectAPI, bucket, prefix, token, delimiter, maxKeys, startAfter, r.Header) + } else { + // Initiate a list objects operation based on the input params. + // On success would return back ListObjectsInfo object to be + // marshaled into S3 compatible XML header. + listObjectsV2Info, err = objectAPI.ListObjectsV2(ctx, bucket, prefix, token, delimiter, maxKeys, fetchOwner, startAfter) + } + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if err = DecryptETags(ctx, GlobalKMS, listObjectsV2Info.Objects); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + response := generateListObjectsV2Response(ctx, bucket, prefix, token, listObjectsV2Info.NextContinuationToken, startAfter, + delimiter, encodingType, fetchOwner, listObjectsV2Info.IsTruncated, + maxKeys, listObjectsV2Info.Objects, listObjectsV2Info.Prefixes, checkObjMeta) + + // Write success response. + writeSuccessResponseXML(w, encodeResponseList(response)) +} + +func parseRequestToken(token string) (subToken string, nodeIndex int) { + if token == "" { + return token, -1 + } + i := strings.Index(token, getKeySeparator()) + if i < 0 { + return token, -1 + } + nodeIndex, err := strconv.Atoi(token[i+1:]) + if err != nil { + return token, -1 + } + subToken = token[:i] + return subToken, nodeIndex +} + +func proxyRequestByToken(ctx context.Context, w http.ResponseWriter, r *http.Request, token string, returnErr bool) (subToken string, proxied bool, success bool) { + var nodeIndex int + if subToken, nodeIndex = parseRequestToken(token); nodeIndex >= 0 { + proxied, success = proxyRequestByNodeIndex(ctx, w, r, nodeIndex, returnErr) + } + return subToken, proxied, success +} + +func proxyRequestByNodeIndex(ctx context.Context, w http.ResponseWriter, r *http.Request, index int, returnErr bool) (proxied, success bool) { + if len(globalProxyEndpoints) == 0 { + return proxied, success + } + if index < 0 || index >= len(globalProxyEndpoints) { + return proxied, success + } + ep := globalProxyEndpoints[index] + if ep.IsLocal { + return proxied, success + } + return true, proxyRequest(ctx, w, r, ep, returnErr) +} + +// ListObjectsV1Handler - GET Bucket (List Objects) Version 1. +// -------------------------- +// This implementation of the GET operation returns some or all (up to 1000) +// of the objects in a bucket. You can use the request parameters as selection +// criteria to return a subset of the objects in a bucket. +func (api objectAPIHandlers) ListObjectsV1Handler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "ListObjectsV1") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Extract all the listObjectsV1 query params to their native values. + prefix, marker, delimiter, maxKeys, encodingType, s3Error := getListObjectsV1Args(r.Form) + if s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Validate all the query params before beginning to serve the request. + if s3Error := validateListObjectsArgs(prefix, marker, delimiter, encodingType, maxKeys); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + listObjects := objectAPI.ListObjects + + // Initiate a list objects operation based on the input params. + // On success would return back ListObjectsInfo object to be + // marshaled into S3 compatible XML header. + listObjectsInfo, err := listObjects(ctx, bucket, prefix, marker, delimiter, maxKeys) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if err = DecryptETags(ctx, GlobalKMS, listObjectsInfo.Objects); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + response := generateListObjectsV1Response(ctx, bucket, prefix, marker, delimiter, encodingType, maxKeys, listObjectsInfo) + + // Write success response. + writeSuccessResponseXML(w, encodeResponseList(response)) +} diff --git a/cmd/bucket-metadata-sys.go b/cmd/bucket-metadata-sys.go new file mode 100644 index 00000000..20be4ffd --- /dev/null +++ b/cmd/bucket-metadata-sys.go @@ -0,0 +1,659 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/xml" + "errors" + "fmt" + "math/rand" + "sync" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio-go/v7/pkg/tags" + bucketsse "github.com/minio/minio/internal/bucket/encryption" + "github.com/minio/minio/internal/bucket/lifecycle" + objectlock "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/bucket/versioning" + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/kms" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/policy" + "github.com/minio/pkg/v3/sync/errgroup" + "golang.org/x/sync/singleflight" +) + +// BucketMetadataSys captures all bucket metadata for a given cluster. +type BucketMetadataSys struct { + objAPI ObjectLayer + + sync.RWMutex + initialized bool + group *singleflight.Group + metadataMap map[string]BucketMetadata +} + +// Count returns number of bucket metadata map entries. +func (sys *BucketMetadataSys) Count() int { + sys.RLock() + defer sys.RUnlock() + + return len(sys.metadataMap) +} + +// Remove bucket metadata from memory. +func (sys *BucketMetadataSys) Remove(buckets ...string) { + sys.Lock() + for _, bucket := range buckets { + sys.group.Forget(bucket) + delete(sys.metadataMap, bucket) + globalBucketMonitor.DeleteBucket(bucket) + } + sys.Unlock() +} + +// RemoveStaleBuckets removes all stale buckets in memory that are not on disk. +func (sys *BucketMetadataSys) RemoveStaleBuckets(diskBuckets set.StringSet) { + sys.Lock() + defer sys.Unlock() + + for bucket := range sys.metadataMap { + if diskBuckets.Contains(bucket) { + continue + } // doesn't exist on disk remove from memory. + delete(sys.metadataMap, bucket) + globalBucketMonitor.DeleteBucket(bucket) + } +} + +// Set - sets a new metadata in-memory. +// Only a shallow copy is saved and fields with references +// cannot be modified without causing a race condition, +// so they should be replaced atomically and not appended to, etc. +// Data is not persisted to disk. +func (sys *BucketMetadataSys) Set(bucket string, meta BucketMetadata) { + if !isMinioMetaBucketName(bucket) { + sys.Lock() + sys.metadataMap[bucket] = meta + sys.Unlock() + } +} + +func (sys *BucketMetadataSys) updateAndParse(ctx context.Context, bucket string, configFile string, configData []byte, parse bool) (updatedAt time.Time, err error) { + objAPI := newObjectLayerFn() + if objAPI == nil { + return updatedAt, errServerNotInitialized + } + + if isMinioMetaBucketName(bucket) { + return updatedAt, errInvalidArgument + } + + meta, err := loadBucketMetadataParse(ctx, objAPI, bucket, parse) + if err != nil { + if !globalIsErasure && !globalIsDistErasure && errors.Is(err, errVolumeNotFound) { + // Only single drive mode needs this fallback. + meta = newBucketMetadata(bucket) + } else { + return updatedAt, err + } + } + updatedAt = UTCNow() + switch configFile { + case bucketPolicyConfig: + meta.PolicyConfigJSON = configData + meta.PolicyConfigUpdatedAt = updatedAt + case bucketNotificationConfig: + meta.NotificationConfigXML = configData + meta.NotificationConfigUpdatedAt = updatedAt + case bucketLifecycleConfig: + meta.LifecycleConfigXML = configData + meta.LifecycleConfigUpdatedAt = updatedAt + case bucketSSEConfig: + meta.EncryptionConfigXML = configData + meta.EncryptionConfigUpdatedAt = updatedAt + case bucketTaggingConfig: + meta.TaggingConfigXML = configData + meta.TaggingConfigUpdatedAt = updatedAt + case bucketQuotaConfigFile: + meta.QuotaConfigJSON = configData + meta.QuotaConfigUpdatedAt = updatedAt + case objectLockConfig: + meta.ObjectLockConfigXML = configData + meta.ObjectLockConfigUpdatedAt = updatedAt + case bucketVersioningConfig: + meta.VersioningConfigXML = configData + meta.VersioningConfigUpdatedAt = updatedAt + case bucketReplicationConfig: + meta.ReplicationConfigXML = configData + meta.ReplicationConfigUpdatedAt = updatedAt + case bucketTargetsFile: + meta.BucketTargetsConfigJSON, meta.BucketTargetsConfigMetaJSON, err = encryptBucketMetadata(ctx, meta.Name, configData, kms.Context{ + bucket: meta.Name, + bucketTargetsFile: bucketTargetsFile, + }) + if err != nil { + return updatedAt, fmt.Errorf("Error encrypting bucket target metadata %w", err) + } + meta.BucketTargetsConfigUpdatedAt = updatedAt + meta.BucketTargetsConfigMetaUpdatedAt = updatedAt + default: + return updatedAt, fmt.Errorf("Unknown bucket %s metadata update requested %s", bucket, configFile) + } + + return updatedAt, sys.save(ctx, meta) +} + +func (sys *BucketMetadataSys) save(ctx context.Context, meta BucketMetadata) error { + objAPI := newObjectLayerFn() + if objAPI == nil { + return errServerNotInitialized + } + + if isMinioMetaBucketName(meta.Name) { + return errInvalidArgument + } + + if err := meta.Save(ctx, objAPI); err != nil { + return err + } + + sys.Set(meta.Name, meta) + globalNotificationSys.LoadBucketMetadata(bgContext(ctx), meta.Name) // Do not use caller context here + return nil +} + +// Delete delete the bucket metadata for the specified bucket. +// must be used by all callers instead of using Update() with nil configData. +func (sys *BucketMetadataSys) Delete(ctx context.Context, bucket string, configFile string) (updatedAt time.Time, err error) { + if configFile == bucketLifecycleConfig { + // Get bucket config from current site + meta, e := globalBucketMetadataSys.GetConfigFromDisk(ctx, bucket) + if e != nil && !errors.Is(e, errConfigNotFound) { + return updatedAt, e + } + var expiryRuleRemoved bool + if len(meta.LifecycleConfigXML) > 0 { + var lcCfg lifecycle.Lifecycle + if err := xml.Unmarshal(meta.LifecycleConfigXML, &lcCfg); err != nil { + return updatedAt, err + } + // find a single expiry rule set the flag + for _, rl := range lcCfg.Rules { + if !rl.Expiration.IsNull() || !rl.NoncurrentVersionExpiration.IsNull() { + expiryRuleRemoved = true + break + } + } + } + + // Form empty ILM details with `ExpiryUpdatedAt` field and save + var cfgData []byte + if expiryRuleRemoved { + var lcCfg lifecycle.Lifecycle + currtime := time.Now() + lcCfg.ExpiryUpdatedAt = &currtime + cfgData, err = xml.Marshal(lcCfg) + if err != nil { + return updatedAt, err + } + } + return sys.updateAndParse(ctx, bucket, configFile, cfgData, false) + } + return sys.updateAndParse(ctx, bucket, configFile, nil, false) +} + +// Update update bucket metadata for the specified bucket. +// The configData data should not be modified after being sent here. +func (sys *BucketMetadataSys) Update(ctx context.Context, bucket string, configFile string, configData []byte) (updatedAt time.Time, err error) { + return sys.updateAndParse(ctx, bucket, configFile, configData, true) +} + +// Get metadata for a bucket. +// If no metadata exists errConfigNotFound is returned and a new metadata is returned. +// Only a shallow copy is returned, so referenced data should not be modified, +// but can be replaced atomically. +// +// This function should only be used with +// - GetBucketInfo +// - ListBuckets +// For all other bucket specific metadata, use the relevant +// calls implemented specifically for each of those features. +func (sys *BucketMetadataSys) Get(bucket string) (BucketMetadata, error) { + if isMinioMetaBucketName(bucket) { + return newBucketMetadata(bucket), errConfigNotFound + } + + sys.RLock() + defer sys.RUnlock() + + meta, ok := sys.metadataMap[bucket] + if !ok { + return newBucketMetadata(bucket), errConfigNotFound + } + + return meta, nil +} + +// GetVersioningConfig returns configured versioning config +// The returned object may not be modified. +func (sys *BucketMetadataSys) GetVersioningConfig(bucket string) (*versioning.Versioning, time.Time, error) { + meta, _, err := sys.GetConfig(GlobalContext, bucket) + if err != nil { + if errors.Is(err, errConfigNotFound) { + return &versioning.Versioning{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/"}, meta.Created, nil + } + return &versioning.Versioning{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/"}, time.Time{}, err + } + return meta.versioningConfig, meta.VersioningConfigUpdatedAt, nil +} + +// GetBucketPolicy returns configured bucket policy +func (sys *BucketMetadataSys) GetBucketPolicy(bucket string) (*policy.BucketPolicy, time.Time, error) { + meta, _, err := sys.GetConfig(GlobalContext, bucket) + if err != nil { + if errors.Is(err, errConfigNotFound) { + return nil, time.Time{}, BucketPolicyNotFound{Bucket: bucket} + } + return nil, time.Time{}, err + } + if meta.policyConfig == nil { + return nil, time.Time{}, BucketPolicyNotFound{Bucket: bucket} + } + return meta.policyConfig, meta.PolicyConfigUpdatedAt, nil +} + +// GetTaggingConfig returns configured tagging config +// The returned object may not be modified. +func (sys *BucketMetadataSys) GetTaggingConfig(bucket string) (*tags.Tags, time.Time, error) { + meta, _, err := sys.GetConfig(GlobalContext, bucket) + if err != nil { + if errors.Is(err, errConfigNotFound) { + return nil, time.Time{}, BucketTaggingNotFound{Bucket: bucket} + } + return nil, time.Time{}, err + } + if meta.taggingConfig == nil { + return nil, time.Time{}, BucketTaggingNotFound{Bucket: bucket} + } + return meta.taggingConfig, meta.TaggingConfigUpdatedAt, nil +} + +// GetObjectLockConfig returns configured object lock config +// The returned object may not be modified. +func (sys *BucketMetadataSys) GetObjectLockConfig(bucket string) (*objectlock.Config, time.Time, error) { + meta, _, err := sys.GetConfig(GlobalContext, bucket) + if err != nil { + if errors.Is(err, errConfigNotFound) { + return nil, time.Time{}, BucketObjectLockConfigNotFound{Bucket: bucket} + } + return nil, time.Time{}, err + } + if meta.objectLockConfig == nil { + return nil, time.Time{}, BucketObjectLockConfigNotFound{Bucket: bucket} + } + return meta.objectLockConfig, meta.ObjectLockConfigUpdatedAt, nil +} + +// GetLifecycleConfig returns configured lifecycle config +// The returned object may not be modified. +func (sys *BucketMetadataSys) GetLifecycleConfig(bucket string) (*lifecycle.Lifecycle, time.Time, error) { + meta, _, err := sys.GetConfig(GlobalContext, bucket) + if err != nil { + if errors.Is(err, errConfigNotFound) { + return nil, time.Time{}, BucketLifecycleNotFound{Bucket: bucket} + } + return nil, time.Time{}, err + } + // there could be just `ExpiryUpdatedAt` field populated as part + // of last delete all. Treat this situation as not lifecycle configuration + // available + if meta.lifecycleConfig == nil || len(meta.lifecycleConfig.Rules) == 0 { + return nil, time.Time{}, BucketLifecycleNotFound{Bucket: bucket} + } + return meta.lifecycleConfig, meta.LifecycleConfigUpdatedAt, nil +} + +// GetNotificationConfig returns configured notification config +// The returned object may not be modified. +func (sys *BucketMetadataSys) GetNotificationConfig(bucket string) (*event.Config, error) { + meta, _, err := sys.GetConfig(GlobalContext, bucket) + if err != nil { + return nil, err + } + return meta.notificationConfig, nil +} + +// GetSSEConfig returns configured SSE config +// The returned object may not be modified. +func (sys *BucketMetadataSys) GetSSEConfig(bucket string) (*bucketsse.BucketSSEConfig, time.Time, error) { + meta, _, err := sys.GetConfig(GlobalContext, bucket) + if err != nil { + if errors.Is(err, errConfigNotFound) { + return nil, time.Time{}, BucketSSEConfigNotFound{Bucket: bucket} + } + return nil, time.Time{}, err + } + if meta.sseConfig == nil { + return nil, time.Time{}, BucketSSEConfigNotFound{Bucket: bucket} + } + return meta.sseConfig, meta.EncryptionConfigUpdatedAt, nil +} + +// CreatedAt returns the time of creation of bucket +func (sys *BucketMetadataSys) CreatedAt(bucket string) (time.Time, error) { + meta, _, err := sys.GetConfig(GlobalContext, bucket) + if err != nil { + return time.Time{}, err + } + return meta.Created.UTC(), nil +} + +// GetPolicyConfig returns configured bucket policy +// The returned object may not be modified. +func (sys *BucketMetadataSys) GetPolicyConfig(bucket string) (*policy.BucketPolicy, time.Time, error) { + meta, _, err := sys.GetConfig(GlobalContext, bucket) + if err != nil { + if errors.Is(err, errConfigNotFound) { + return nil, time.Time{}, BucketPolicyNotFound{Bucket: bucket} + } + return nil, time.Time{}, err + } + if meta.policyConfig == nil { + return nil, time.Time{}, BucketPolicyNotFound{Bucket: bucket} + } + return meta.policyConfig, meta.PolicyConfigUpdatedAt, nil +} + +// GetQuotaConfig returns configured bucket quota +// The returned object may not be modified. +func (sys *BucketMetadataSys) GetQuotaConfig(ctx context.Context, bucket string) (*madmin.BucketQuota, time.Time, error) { + meta, _, err := sys.GetConfig(ctx, bucket) + if err != nil { + if errors.Is(err, errConfigNotFound) { + return nil, time.Time{}, BucketQuotaConfigNotFound{Bucket: bucket} + } + return nil, time.Time{}, err + } + return meta.quotaConfig, meta.QuotaConfigUpdatedAt, nil +} + +// GetReplicationConfig returns configured bucket replication config +// The returned object may not be modified. +func (sys *BucketMetadataSys) GetReplicationConfig(ctx context.Context, bucket string) (*replication.Config, time.Time, error) { + meta, reloaded, err := sys.GetConfig(ctx, bucket) + if err != nil { + if errors.Is(err, errConfigNotFound) { + return nil, time.Time{}, BucketReplicationConfigNotFound{Bucket: bucket} + } + return nil, time.Time{}, err + } + + if meta.replicationConfig == nil { + return nil, time.Time{}, BucketReplicationConfigNotFound{Bucket: bucket} + } + if reloaded { + globalBucketTargetSys.set(bucket, meta) + } + return meta.replicationConfig, meta.ReplicationConfigUpdatedAt, nil +} + +// GetBucketTargetsConfig returns configured bucket targets for this bucket +// The returned object may not be modified. +func (sys *BucketMetadataSys) GetBucketTargetsConfig(bucket string) (*madmin.BucketTargets, error) { + meta, reloaded, err := sys.GetConfig(GlobalContext, bucket) + if err != nil { + if errors.Is(err, errConfigNotFound) { + return nil, BucketRemoteTargetNotFound{Bucket: bucket} + } + return nil, err + } + if meta.bucketTargetConfig == nil { + return nil, BucketRemoteTargetNotFound{Bucket: bucket} + } + if reloaded { + globalBucketTargetSys.set(bucket, meta) + } + return meta.bucketTargetConfig, nil +} + +// GetConfigFromDisk read bucket metadata config from disk. +func (sys *BucketMetadataSys) GetConfigFromDisk(ctx context.Context, bucket string) (BucketMetadata, error) { + objAPI := newObjectLayerFn() + if objAPI == nil { + return newBucketMetadata(bucket), errServerNotInitialized + } + + if isMinioMetaBucketName(bucket) { + return newBucketMetadata(bucket), errInvalidArgument + } + + return loadBucketMetadata(ctx, objAPI, bucket) +} + +var errBucketMetadataNotInitialized = errors.New("bucket metadata not initialized yet") + +// GetConfig returns a specific configuration from the bucket metadata. +// The returned object may not be modified. +// reloaded will be true if metadata refreshed from disk +func (sys *BucketMetadataSys) GetConfig(ctx context.Context, bucket string) (meta BucketMetadata, reloaded bool, err error) { + objAPI := newObjectLayerFn() + if objAPI == nil { + return newBucketMetadata(bucket), reloaded, errServerNotInitialized + } + + if isMinioMetaBucketName(bucket) { + return newBucketMetadata(bucket), reloaded, errInvalidArgument + } + + sys.RLock() + meta, ok := sys.metadataMap[bucket] + sys.RUnlock() + if ok { + return meta, reloaded, nil + } + + val, err, _ := sys.group.Do(bucket, func() (val any, err error) { + meta, err = loadBucketMetadata(ctx, objAPI, bucket) + if err != nil { + if !sys.Initialized() { + // bucket metadata not yet initialized + return newBucketMetadata(bucket), errBucketMetadataNotInitialized + } + } + return meta, err + }) + meta, _ = val.(BucketMetadata) + if err != nil { + return meta, false, err + } + sys.Lock() + sys.metadataMap[bucket] = meta + sys.Unlock() + + return meta, true, nil +} + +// Init - initializes bucket metadata system for all buckets. +func (sys *BucketMetadataSys) Init(ctx context.Context, buckets []string, objAPI ObjectLayer) error { + if objAPI == nil { + return errServerNotInitialized + } + + sys.objAPI = objAPI + + // Load bucket metadata sys. + sys.init(ctx, buckets) + return nil +} + +// concurrently load bucket metadata to speed up loading bucket metadata. +func (sys *BucketMetadataSys) concurrentLoad(ctx context.Context, buckets []string) { + g := errgroup.WithNErrs(len(buckets)) + bucketMetas := make([]BucketMetadata, len(buckets)) + for index := range buckets { + g.Go(func() error { + // Sleep and stagger to avoid blocked CPU and thundering + // herd upon start up sequence. + time.Sleep(25*time.Millisecond + time.Duration(rand.Int63n(int64(100*time.Millisecond)))) + + _, _ = sys.objAPI.HealBucket(ctx, buckets[index], madmin.HealOpts{Recreate: true}) + meta, err := loadBucketMetadata(ctx, sys.objAPI, buckets[index]) + if err != nil { + return err + } + bucketMetas[index] = meta + return nil + }, index) + } + + errs := g.Wait() + for index, err := range errs { + if err != nil { + internalLogOnceIf(ctx, fmt.Errorf("Unable to load bucket metadata, will be retried: %w", err), + "load-bucket-metadata-"+buckets[index], logger.WarningKind) + } + } + + // Hold lock here to update in-memory map at once, + // instead of serializing the Go routines. + sys.Lock() + for i, meta := range bucketMetas { + if errs[i] != nil { + continue + } + sys.metadataMap[buckets[i]] = meta + } + sys.Unlock() + + for i, meta := range bucketMetas { + if errs[i] != nil { + continue + } + globalEventNotifier.set(buckets[i], meta) // set notification targets + globalBucketTargetSys.set(buckets[i], meta) // set remote replication targets + } +} + +func (sys *BucketMetadataSys) refreshBucketsMetadataLoop(ctx context.Context) { + const bucketMetadataRefresh = 15 * time.Minute + + sleeper := newDynamicSleeper(2, 150*time.Millisecond, false) + + t := time.NewTimer(bucketMetadataRefresh) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + buckets, err := sys.objAPI.ListBuckets(ctx, BucketOptions{NoMetadata: true}) + if err != nil { + internalLogIf(ctx, err, logger.WarningKind) + break + } + + // Handle if we have some buckets in-memory those are stale. + // first delete them and then replace the newer state() + // from disk. + diskBuckets := set.CreateStringSet() + for _, bucket := range buckets { + diskBuckets.Add(bucket.Name) + } + sys.RemoveStaleBuckets(diskBuckets) + + for i := range buckets { + wait := sleeper.Timer(ctx) + + bucket := buckets[i].Name + updated := false + + meta, err := loadBucketMetadata(ctx, sys.objAPI, bucket) + if err != nil { + internalLogIf(ctx, err, logger.WarningKind) + wait() // wait to proceed to next entry. + continue + } + + sys.Lock() + // Update if the bucket metadata in the memory is older than on-disk one + if lu := sys.metadataMap[bucket].lastUpdate(); lu.Before(meta.lastUpdate()) { + updated = true + sys.metadataMap[bucket] = meta + } + sys.Unlock() + + if updated { + globalEventNotifier.set(bucket, meta) + globalBucketTargetSys.set(bucket, meta) + } + + wait() // wait to proceed to next entry. + } + } + t.Reset(bucketMetadataRefresh) + } +} + +// Initialized indicates if bucket metadata sys is initialized atleast once. +func (sys *BucketMetadataSys) Initialized() bool { + sys.RLock() + defer sys.RUnlock() + + return sys.initialized +} + +// Loads bucket metadata for all buckets into BucketMetadataSys. +func (sys *BucketMetadataSys) init(ctx context.Context, buckets []string) { + count := globalEndpoints.ESCount() * 10 + for { + if len(buckets) < count { + sys.concurrentLoad(ctx, buckets) + break + } + sys.concurrentLoad(ctx, buckets[:count]) + buckets = buckets[count:] + } + + sys.Lock() + sys.initialized = true + sys.Unlock() + + if globalIsDistErasure { + go sys.refreshBucketsMetadataLoop(ctx) + } +} + +// Reset the state of the BucketMetadataSys. +func (sys *BucketMetadataSys) Reset() { + sys.Lock() + clear(sys.metadataMap) + sys.Unlock() +} + +// NewBucketMetadataSys - creates new policy system. +func NewBucketMetadataSys() *BucketMetadataSys { + return &BucketMetadataSys{ + metadataMap: make(map[string]BucketMetadata), + group: &singleflight.Group{}, + } +} diff --git a/cmd/bucket-metadata.go b/cmd/bucket-metadata.go new file mode 100644 index 00000000..e7811817 --- /dev/null +++ b/cmd/bucket-metadata.go @@ -0,0 +1,594 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/binary" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "path" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7/pkg/tags" + bucketsse "github.com/minio/minio/internal/bucket/encryption" + "github.com/minio/minio/internal/bucket/lifecycle" + objectlock "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/bucket/versioning" + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/kms" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/policy" + "github.com/minio/sio" +) + +const ( + legacyBucketObjectLockEnabledConfigFile = "object-lock-enabled.json" + legacyBucketObjectLockEnabledConfig = `{"x-amz-bucket-object-lock-enabled":true}` + + bucketMetadataFile = ".metadata.bin" + bucketMetadataFormat = 1 + bucketMetadataVersion = 1 +) + +var ( + enabledBucketObjectLockConfig = []byte(`Enabled`) + enabledBucketVersioningConfig = []byte(`Enabled`) +) + +//go:generate msgp -file $GOFILE + +// BucketMetadata contains bucket metadata. +// When adding/removing fields, regenerate the marshal code using the go generate above. +// Only changing meaning of fields requires a version bump. +// bucketMetadataFormat refers to the format. +// bucketMetadataVersion can be used to track a rolling upgrade of a field. +type BucketMetadata struct { + Name string + Created time.Time + LockEnabled bool // legacy not used anymore. + PolicyConfigJSON []byte + NotificationConfigXML []byte + LifecycleConfigXML []byte + ObjectLockConfigXML []byte + VersioningConfigXML []byte + EncryptionConfigXML []byte + TaggingConfigXML []byte + QuotaConfigJSON []byte + ReplicationConfigXML []byte + BucketTargetsConfigJSON []byte + BucketTargetsConfigMetaJSON []byte + + PolicyConfigUpdatedAt time.Time + ObjectLockConfigUpdatedAt time.Time + EncryptionConfigUpdatedAt time.Time + TaggingConfigUpdatedAt time.Time + QuotaConfigUpdatedAt time.Time + ReplicationConfigUpdatedAt time.Time + VersioningConfigUpdatedAt time.Time + LifecycleConfigUpdatedAt time.Time + NotificationConfigUpdatedAt time.Time + BucketTargetsConfigUpdatedAt time.Time + BucketTargetsConfigMetaUpdatedAt time.Time + // Add a new UpdatedAt field and update lastUpdate function + + // Unexported fields. Must be updated atomically. + policyConfig *policy.BucketPolicy + notificationConfig *event.Config + lifecycleConfig *lifecycle.Lifecycle + objectLockConfig *objectlock.Config + versioningConfig *versioning.Versioning + sseConfig *bucketsse.BucketSSEConfig + taggingConfig *tags.Tags + quotaConfig *madmin.BucketQuota + replicationConfig *replication.Config + bucketTargetConfig *madmin.BucketTargets + bucketTargetConfigMeta map[string]string +} + +// newBucketMetadata creates BucketMetadata with the supplied name and Created to Now. +func newBucketMetadata(name string) BucketMetadata { + return BucketMetadata{ + Name: name, + notificationConfig: &event.Config{ + XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/", + }, + quotaConfig: &madmin.BucketQuota{}, + versioningConfig: &versioning.Versioning{ + XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/", + }, + bucketTargetConfig: &madmin.BucketTargets{}, + bucketTargetConfigMeta: make(map[string]string), + } +} + +// Return the last update of this bucket metadata, which +// means, the last update of any policy document. +func (b BucketMetadata) lastUpdate() (t time.Time) { + if b.PolicyConfigUpdatedAt.After(t) { + t = b.PolicyConfigUpdatedAt + } + if b.ObjectLockConfigUpdatedAt.After(t) { + t = b.ObjectLockConfigUpdatedAt + } + if b.EncryptionConfigUpdatedAt.After(t) { + t = b.EncryptionConfigUpdatedAt + } + if b.TaggingConfigUpdatedAt.After(t) { + t = b.TaggingConfigUpdatedAt + } + if b.QuotaConfigUpdatedAt.After(t) { + t = b.QuotaConfigUpdatedAt + } + if b.ReplicationConfigUpdatedAt.After(t) { + t = b.ReplicationConfigUpdatedAt + } + if b.VersioningConfigUpdatedAt.After(t) { + t = b.VersioningConfigUpdatedAt + } + if b.LifecycleConfigUpdatedAt.After(t) { + t = b.LifecycleConfigUpdatedAt + } + if b.NotificationConfigUpdatedAt.After(t) { + t = b.NotificationConfigUpdatedAt + } + if b.BucketTargetsConfigUpdatedAt.After(t) { + t = b.BucketTargetsConfigUpdatedAt + } + if b.BucketTargetsConfigMetaUpdatedAt.After(t) { + t = b.BucketTargetsConfigMetaUpdatedAt + } + + return t +} + +// Versioning returns true if versioning is enabled +func (b BucketMetadata) Versioning() bool { + return b.LockEnabled || (b.versioningConfig != nil && b.versioningConfig.Enabled()) || (b.objectLockConfig != nil && b.objectLockConfig.Enabled()) +} + +// ObjectLocking returns true if object locking is enabled +func (b BucketMetadata) ObjectLocking() bool { + return b.LockEnabled || (b.objectLockConfig != nil && b.objectLockConfig.Enabled()) +} + +// SetCreatedAt preserves the CreatedAt time for bucket across sites in site replication. It defaults to +// creation time of bucket on this cluster in all other cases. +func (b *BucketMetadata) SetCreatedAt(createdAt time.Time) { + if b.Created.IsZero() { + b.Created = UTCNow() + } + if !createdAt.IsZero() { + b.Created = createdAt.UTC() + } +} + +// Load - loads the metadata of bucket by name from ObjectLayer api. +// If an error is returned the returned metadata will be default initialized. +func readBucketMetadata(ctx context.Context, api ObjectLayer, name string) (BucketMetadata, error) { + if name == "" { + internalLogIf(ctx, errors.New("bucket name cannot be empty"), logger.WarningKind) + return BucketMetadata{}, errInvalidArgument + } + b := newBucketMetadata(name) + configFile := path.Join(bucketMetaPrefix, name, bucketMetadataFile) + data, err := readConfig(ctx, api, configFile) + if err != nil { + return b, err + } + if len(data) <= 4 { + return b, fmt.Errorf("loadBucketMetadata: no data") + } + // Read header + switch binary.LittleEndian.Uint16(data[0:2]) { + case bucketMetadataFormat: + default: + return b, fmt.Errorf("loadBucketMetadata: unknown format: %d", binary.LittleEndian.Uint16(data[0:2])) + } + switch binary.LittleEndian.Uint16(data[2:4]) { + case bucketMetadataVersion: + default: + return b, fmt.Errorf("loadBucketMetadata: unknown version: %d", binary.LittleEndian.Uint16(data[2:4])) + } + _, err = b.UnmarshalMsg(data[4:]) + return b, err +} + +func loadBucketMetadataParse(ctx context.Context, objectAPI ObjectLayer, bucket string, parse bool) (BucketMetadata, error) { + b, err := readBucketMetadata(ctx, objectAPI, bucket) + b.Name = bucket // in-case parsing failed for some reason, make sure bucket name is not empty. + if err != nil && !errors.Is(err, errConfigNotFound) { + return b, err + } + if err == nil { + b.defaultTimestamps() + } + + // If bucket metadata is missing look for legacy files, + // since we only ever had b.Created as non-zero when + // migration was complete in 2020-May release. So this + // a check to avoid migrating for buckets that already + // have this field set. + if b.Created.IsZero() { + configs, err := b.getAllLegacyConfigs(ctx, objectAPI) + if err != nil { + return b, err + } + + if len(configs) > 0 { + // Old bucket without bucket metadata. Hence we migrate existing settings. + if err = b.convertLegacyConfigs(ctx, objectAPI, configs); err != nil { + return b, err + } + } + } + + if parse { + // nothing to update, parse and proceed. + if err = b.parseAllConfigs(ctx, objectAPI); err != nil { + return b, err + } + } + + // migrate unencrypted remote targets + if err = b.migrateTargetConfig(ctx, objectAPI); err != nil { + return b, err + } + + return b, nil +} + +// loadBucketMetadata loads and migrates to bucket metadata. +func loadBucketMetadata(ctx context.Context, objectAPI ObjectLayer, bucket string) (BucketMetadata, error) { + return loadBucketMetadataParse(ctx, objectAPI, bucket, true) +} + +// parseAllConfigs will parse all configs and populate the private fields. +// The first error encountered is returned. +func (b *BucketMetadata) parseAllConfigs(ctx context.Context, objectAPI ObjectLayer) (err error) { + if len(b.PolicyConfigJSON) != 0 { + b.policyConfig, err = policy.ParseBucketPolicyConfig(bytes.NewReader(b.PolicyConfigJSON), b.Name) + if err != nil { + return err + } + } else { + b.policyConfig = nil + } + + if len(b.NotificationConfigXML) != 0 { + if err = xml.Unmarshal(b.NotificationConfigXML, b.notificationConfig); err != nil { + return err + } + } + + if len(b.LifecycleConfigXML) != 0 { + b.lifecycleConfig, err = lifecycle.ParseLifecycleConfig(bytes.NewReader(b.LifecycleConfigXML)) + if err != nil { + return err + } + } else { + b.lifecycleConfig = nil + } + + if len(b.EncryptionConfigXML) != 0 { + b.sseConfig, err = bucketsse.ParseBucketSSEConfig(bytes.NewReader(b.EncryptionConfigXML)) + if err != nil { + return err + } + } else { + b.sseConfig = nil + } + + if len(b.TaggingConfigXML) != 0 { + b.taggingConfig, err = tags.ParseBucketXML(bytes.NewReader(b.TaggingConfigXML)) + if err != nil { + return err + } + } else { + b.taggingConfig = nil + } + + if bytes.Equal(b.ObjectLockConfigXML, enabledBucketObjectLockConfig) { + b.VersioningConfigXML = enabledBucketVersioningConfig + } + + if len(b.ObjectLockConfigXML) != 0 { + b.objectLockConfig, err = objectlock.ParseObjectLockConfig(bytes.NewReader(b.ObjectLockConfigXML)) + if err != nil { + return err + } + } else { + b.objectLockConfig = nil + } + + if len(b.VersioningConfigXML) != 0 { + b.versioningConfig, err = versioning.ParseConfig(bytes.NewReader(b.VersioningConfigXML)) + if err != nil { + return err + } + } + + if len(b.QuotaConfigJSON) != 0 { + b.quotaConfig, err = parseBucketQuota(b.Name, b.QuotaConfigJSON) + if err != nil { + return err + } + } + + if len(b.ReplicationConfigXML) != 0 { + b.replicationConfig, err = replication.ParseConfig(bytes.NewReader(b.ReplicationConfigXML)) + if err != nil { + return err + } + } else { + b.replicationConfig = nil + } + + if len(b.BucketTargetsConfigJSON) != 0 { + b.bucketTargetConfig, err = parseBucketTargetConfig(b.Name, b.BucketTargetsConfigJSON, b.BucketTargetsConfigMetaJSON) + if err != nil { + return err + } + } else { + b.bucketTargetConfig = &madmin.BucketTargets{} + } + return nil +} + +func (b *BucketMetadata) getAllLegacyConfigs(ctx context.Context, objectAPI ObjectLayer) (map[string][]byte, error) { + legacyConfigs := []string{ + legacyBucketObjectLockEnabledConfigFile, + bucketPolicyConfig, + bucketNotificationConfig, + bucketLifecycleConfig, + bucketQuotaConfigFile, + bucketSSEConfig, + bucketTaggingConfig, + bucketReplicationConfig, + bucketTargetsFile, + objectLockConfig, + } + + configs := make(map[string][]byte, len(legacyConfigs)) + + // Handle migration from lockEnabled to newer format. + if b.LockEnabled { + configs[objectLockConfig] = enabledBucketObjectLockConfig + b.LockEnabled = false // legacy value unset it + // we are only interested in b.ObjectLockConfigXML or objectLockConfig value + } + + for _, legacyFile := range legacyConfigs { + configFile := path.Join(bucketMetaPrefix, b.Name, legacyFile) + + configData, info, err := readConfigWithMetadata(ctx, objectAPI, configFile, ObjectOptions{}) + if err != nil { + if _, ok := err.(ObjectExistsAsDirectory); ok { + // in FS mode it possible that we have actual + // files in this folder with `.minio.sys/buckets/bucket/configFile` + continue + } + if errors.Is(err, errConfigNotFound) { + // legacy file config not found, proceed to look for new metadata. + continue + } + + return nil, err + } + configs[legacyFile] = configData + b.Created = info.ModTime + } + + return configs, nil +} + +func (b *BucketMetadata) convertLegacyConfigs(ctx context.Context, objectAPI ObjectLayer, configs map[string][]byte) error { + for legacyFile, configData := range configs { + switch legacyFile { + case legacyBucketObjectLockEnabledConfigFile: + if string(configData) == legacyBucketObjectLockEnabledConfig { + b.ObjectLockConfigXML = enabledBucketObjectLockConfig + b.VersioningConfigXML = enabledBucketVersioningConfig + b.LockEnabled = false // legacy value unset it + // we are only interested in b.ObjectLockConfigXML + } + case bucketPolicyConfig: + b.PolicyConfigJSON = configData + case bucketNotificationConfig: + b.NotificationConfigXML = configData + case bucketLifecycleConfig: + b.LifecycleConfigXML = configData + case bucketSSEConfig: + b.EncryptionConfigXML = configData + case bucketTaggingConfig: + b.TaggingConfigXML = configData + case objectLockConfig: + b.ObjectLockConfigXML = configData + b.VersioningConfigXML = enabledBucketVersioningConfig + case bucketQuotaConfigFile: + b.QuotaConfigJSON = configData + case bucketReplicationConfig: + b.ReplicationConfigXML = configData + case bucketTargetsFile: + b.BucketTargetsConfigJSON = configData + } + } + b.defaultTimestamps() + + if err := b.Save(ctx, objectAPI); err != nil { + return err + } + + for legacyFile := range configs { + configFile := path.Join(bucketMetaPrefix, b.Name, legacyFile) + if err := deleteConfig(ctx, objectAPI, configFile); err != nil && !errors.Is(err, errConfigNotFound) { + internalLogIf(ctx, err, logger.WarningKind) + } + } + + return nil +} + +// default timestamps to metadata Created timestamp if unset. +func (b *BucketMetadata) defaultTimestamps() { + if b.PolicyConfigUpdatedAt.IsZero() { + b.PolicyConfigUpdatedAt = b.Created + } + + if b.EncryptionConfigUpdatedAt.IsZero() { + b.EncryptionConfigUpdatedAt = b.Created + } + + if b.TaggingConfigUpdatedAt.IsZero() { + b.TaggingConfigUpdatedAt = b.Created + } + + if b.ObjectLockConfigUpdatedAt.IsZero() { + b.ObjectLockConfigUpdatedAt = b.Created + } + + if b.QuotaConfigUpdatedAt.IsZero() { + b.QuotaConfigUpdatedAt = b.Created + } + + if b.ReplicationConfigUpdatedAt.IsZero() { + b.ReplicationConfigUpdatedAt = b.Created + } + + if b.VersioningConfigUpdatedAt.IsZero() { + b.VersioningConfigUpdatedAt = b.Created + } + + if b.LifecycleConfigUpdatedAt.IsZero() { + b.LifecycleConfigUpdatedAt = b.Created + } + + if b.NotificationConfigUpdatedAt.IsZero() { + b.NotificationConfigUpdatedAt = b.Created + } + + if b.BucketTargetsConfigUpdatedAt.IsZero() { + b.BucketTargetsConfigUpdatedAt = b.Created + } + + if b.BucketTargetsConfigMetaUpdatedAt.IsZero() { + b.BucketTargetsConfigMetaUpdatedAt = b.Created + } +} + +// Save config to supplied ObjectLayer api. +func (b *BucketMetadata) Save(ctx context.Context, api ObjectLayer) error { + if err := b.parseAllConfigs(ctx, api); err != nil { + return err + } + + data := make([]byte, 4, b.Msgsize()+4) + + // Initialize the header. + binary.LittleEndian.PutUint16(data[0:2], bucketMetadataFormat) + binary.LittleEndian.PutUint16(data[2:4], bucketMetadataVersion) + + // Marshal the bucket metadata + data, err := b.MarshalMsg(data) + if err != nil { + return err + } + + configFile := path.Join(bucketMetaPrefix, b.Name, bucketMetadataFile) + return saveConfig(ctx, api, configFile, data) +} + +// migrate config for remote targets by encrypting data if currently unencrypted and kms is configured. +func (b *BucketMetadata) migrateTargetConfig(ctx context.Context, objectAPI ObjectLayer) error { + var err error + // early return if no targets or already encrypted + if len(b.BucketTargetsConfigJSON) == 0 || GlobalKMS == nil || len(b.BucketTargetsConfigMetaJSON) != 0 { + return nil + } + + encBytes, metaBytes, err := encryptBucketMetadata(ctx, b.Name, b.BucketTargetsConfigJSON, kms.Context{b.Name: b.Name, bucketTargetsFile: bucketTargetsFile}) + if err != nil { + return err + } + + b.BucketTargetsConfigJSON = encBytes + b.BucketTargetsConfigMetaJSON = metaBytes + return b.Save(ctx, objectAPI) +} + +// encrypt bucket metadata if kms is configured. +func encryptBucketMetadata(ctx context.Context, bucket string, input []byte, kmsContext kms.Context) (output, metabytes []byte, err error) { + if GlobalKMS == nil { + output = input + return output, metabytes, err + } + + metadata := make(map[string]string) + key, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{AssociatedData: kmsContext}) + if err != nil { + return output, metabytes, err + } + + outbuf := bytes.NewBuffer(nil) + objectKey := crypto.GenerateKey(key.Plaintext, rand.Reader) + sealedKey := objectKey.Seal(key.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, "") + crypto.S3.CreateMetadata(metadata, key.KeyID, key.Ciphertext, sealedKey) + _, err = sio.Encrypt(outbuf, bytes.NewBuffer(input), sio.Config{Key: objectKey[:], MinVersion: sio.Version20}) + if err != nil { + return output, metabytes, err + } + metabytes, err = json.Marshal(metadata) + if err != nil { + return output, metabytes, err + } + return outbuf.Bytes(), metabytes, nil +} + +// decrypt bucket metadata if kms is configured. +func decryptBucketMetadata(input []byte, bucket string, meta map[string]string, kmsContext kms.Context) ([]byte, error) { + if GlobalKMS == nil { + return nil, errKMSNotConfigured + } + keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(meta) + if err != nil { + return nil, err + } + extKey, err := GlobalKMS.Decrypt(context.TODO(), &kms.DecryptRequest{ + Name: keyID, + Ciphertext: kmsKey, + AssociatedData: kmsContext, + }) + if err != nil { + return nil, err + } + var objectKey crypto.ObjectKey + if err = objectKey.Unseal(extKey, sealedKey, crypto.S3.String(), bucket, ""); err != nil { + return nil, err + } + + outbuf := bytes.NewBuffer(nil) + _, err = sio.Decrypt(outbuf, bytes.NewBuffer(input), sio.Config{Key: objectKey[:], MinVersion: sio.Version20}) + return outbuf.Bytes(), err +} diff --git a/cmd/bucket-metadata_gen.go b/cmd/bucket-metadata_gen.go new file mode 100644 index 00000000..0407b66e --- /dev/null +++ b/cmd/bucket-metadata_gen.go @@ -0,0 +1,710 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *BucketMetadata) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Name": + z.Name, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "Created": + z.Created, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "Created") + return + } + case "LockEnabled": + z.LockEnabled, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "LockEnabled") + return + } + case "PolicyConfigJSON": + z.PolicyConfigJSON, err = dc.ReadBytes(z.PolicyConfigJSON) + if err != nil { + err = msgp.WrapError(err, "PolicyConfigJSON") + return + } + case "NotificationConfigXML": + z.NotificationConfigXML, err = dc.ReadBytes(z.NotificationConfigXML) + if err != nil { + err = msgp.WrapError(err, "NotificationConfigXML") + return + } + case "LifecycleConfigXML": + z.LifecycleConfigXML, err = dc.ReadBytes(z.LifecycleConfigXML) + if err != nil { + err = msgp.WrapError(err, "LifecycleConfigXML") + return + } + case "ObjectLockConfigXML": + z.ObjectLockConfigXML, err = dc.ReadBytes(z.ObjectLockConfigXML) + if err != nil { + err = msgp.WrapError(err, "ObjectLockConfigXML") + return + } + case "VersioningConfigXML": + z.VersioningConfigXML, err = dc.ReadBytes(z.VersioningConfigXML) + if err != nil { + err = msgp.WrapError(err, "VersioningConfigXML") + return + } + case "EncryptionConfigXML": + z.EncryptionConfigXML, err = dc.ReadBytes(z.EncryptionConfigXML) + if err != nil { + err = msgp.WrapError(err, "EncryptionConfigXML") + return + } + case "TaggingConfigXML": + z.TaggingConfigXML, err = dc.ReadBytes(z.TaggingConfigXML) + if err != nil { + err = msgp.WrapError(err, "TaggingConfigXML") + return + } + case "QuotaConfigJSON": + z.QuotaConfigJSON, err = dc.ReadBytes(z.QuotaConfigJSON) + if err != nil { + err = msgp.WrapError(err, "QuotaConfigJSON") + return + } + case "ReplicationConfigXML": + z.ReplicationConfigXML, err = dc.ReadBytes(z.ReplicationConfigXML) + if err != nil { + err = msgp.WrapError(err, "ReplicationConfigXML") + return + } + case "BucketTargetsConfigJSON": + z.BucketTargetsConfigJSON, err = dc.ReadBytes(z.BucketTargetsConfigJSON) + if err != nil { + err = msgp.WrapError(err, "BucketTargetsConfigJSON") + return + } + case "BucketTargetsConfigMetaJSON": + z.BucketTargetsConfigMetaJSON, err = dc.ReadBytes(z.BucketTargetsConfigMetaJSON) + if err != nil { + err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON") + return + } + case "PolicyConfigUpdatedAt": + z.PolicyConfigUpdatedAt, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "PolicyConfigUpdatedAt") + return + } + case "ObjectLockConfigUpdatedAt": + z.ObjectLockConfigUpdatedAt, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "ObjectLockConfigUpdatedAt") + return + } + case "EncryptionConfigUpdatedAt": + z.EncryptionConfigUpdatedAt, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "EncryptionConfigUpdatedAt") + return + } + case "TaggingConfigUpdatedAt": + z.TaggingConfigUpdatedAt, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "TaggingConfigUpdatedAt") + return + } + case "QuotaConfigUpdatedAt": + z.QuotaConfigUpdatedAt, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "QuotaConfigUpdatedAt") + return + } + case "ReplicationConfigUpdatedAt": + z.ReplicationConfigUpdatedAt, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "ReplicationConfigUpdatedAt") + return + } + case "VersioningConfigUpdatedAt": + z.VersioningConfigUpdatedAt, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "VersioningConfigUpdatedAt") + return + } + case "LifecycleConfigUpdatedAt": + z.LifecycleConfigUpdatedAt, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "LifecycleConfigUpdatedAt") + return + } + case "NotificationConfigUpdatedAt": + z.NotificationConfigUpdatedAt, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "NotificationConfigUpdatedAt") + return + } + case "BucketTargetsConfigUpdatedAt": + z.BucketTargetsConfigUpdatedAt, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "BucketTargetsConfigUpdatedAt") + return + } + case "BucketTargetsConfigMetaUpdatedAt": + z.BucketTargetsConfigMetaUpdatedAt, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "BucketTargetsConfigMetaUpdatedAt") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *BucketMetadata) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 25 + // write "Name" + err = en.Append(0xde, 0x0, 0x19, 0xa4, 0x4e, 0x61, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteString(z.Name) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + // write "Created" + err = en.Append(0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteTime(z.Created) + if err != nil { + err = msgp.WrapError(err, "Created") + return + } + // write "LockEnabled" + err = en.Append(0xab, 0x4c, 0x6f, 0x63, 0x6b, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteBool(z.LockEnabled) + if err != nil { + err = msgp.WrapError(err, "LockEnabled") + return + } + // write "PolicyConfigJSON" + err = en.Append(0xb0, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e) + if err != nil { + return + } + err = en.WriteBytes(z.PolicyConfigJSON) + if err != nil { + err = msgp.WrapError(err, "PolicyConfigJSON") + return + } + // write "NotificationConfigXML" + err = en.Append(0xb5, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c) + if err != nil { + return + } + err = en.WriteBytes(z.NotificationConfigXML) + if err != nil { + err = msgp.WrapError(err, "NotificationConfigXML") + return + } + // write "LifecycleConfigXML" + err = en.Append(0xb2, 0x4c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c) + if err != nil { + return + } + err = en.WriteBytes(z.LifecycleConfigXML) + if err != nil { + err = msgp.WrapError(err, "LifecycleConfigXML") + return + } + // write "ObjectLockConfigXML" + err = en.Append(0xb3, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c) + if err != nil { + return + } + err = en.WriteBytes(z.ObjectLockConfigXML) + if err != nil { + err = msgp.WrapError(err, "ObjectLockConfigXML") + return + } + // write "VersioningConfigXML" + err = en.Append(0xb3, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c) + if err != nil { + return + } + err = en.WriteBytes(z.VersioningConfigXML) + if err != nil { + err = msgp.WrapError(err, "VersioningConfigXML") + return + } + // write "EncryptionConfigXML" + err = en.Append(0xb3, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c) + if err != nil { + return + } + err = en.WriteBytes(z.EncryptionConfigXML) + if err != nil { + err = msgp.WrapError(err, "EncryptionConfigXML") + return + } + // write "TaggingConfigXML" + err = en.Append(0xb0, 0x54, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c) + if err != nil { + return + } + err = en.WriteBytes(z.TaggingConfigXML) + if err != nil { + err = msgp.WrapError(err, "TaggingConfigXML") + return + } + // write "QuotaConfigJSON" + err = en.Append(0xaf, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e) + if err != nil { + return + } + err = en.WriteBytes(z.QuotaConfigJSON) + if err != nil { + err = msgp.WrapError(err, "QuotaConfigJSON") + return + } + // write "ReplicationConfigXML" + err = en.Append(0xb4, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c) + if err != nil { + return + } + err = en.WriteBytes(z.ReplicationConfigXML) + if err != nil { + err = msgp.WrapError(err, "ReplicationConfigXML") + return + } + // write "BucketTargetsConfigJSON" + err = en.Append(0xb7, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e) + if err != nil { + return + } + err = en.WriteBytes(z.BucketTargetsConfigJSON) + if err != nil { + err = msgp.WrapError(err, "BucketTargetsConfigJSON") + return + } + // write "BucketTargetsConfigMetaJSON" + err = en.Append(0xbb, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x4a, 0x53, 0x4f, 0x4e) + if err != nil { + return + } + err = en.WriteBytes(z.BucketTargetsConfigMetaJSON) + if err != nil { + err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON") + return + } + // write "PolicyConfigUpdatedAt" + err = en.Append(0xb5, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + if err != nil { + return + } + err = en.WriteTime(z.PolicyConfigUpdatedAt) + if err != nil { + err = msgp.WrapError(err, "PolicyConfigUpdatedAt") + return + } + // write "ObjectLockConfigUpdatedAt" + err = en.Append(0xb9, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + if err != nil { + return + } + err = en.WriteTime(z.ObjectLockConfigUpdatedAt) + if err != nil { + err = msgp.WrapError(err, "ObjectLockConfigUpdatedAt") + return + } + // write "EncryptionConfigUpdatedAt" + err = en.Append(0xb9, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + if err != nil { + return + } + err = en.WriteTime(z.EncryptionConfigUpdatedAt) + if err != nil { + err = msgp.WrapError(err, "EncryptionConfigUpdatedAt") + return + } + // write "TaggingConfigUpdatedAt" + err = en.Append(0xb6, 0x54, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + if err != nil { + return + } + err = en.WriteTime(z.TaggingConfigUpdatedAt) + if err != nil { + err = msgp.WrapError(err, "TaggingConfigUpdatedAt") + return + } + // write "QuotaConfigUpdatedAt" + err = en.Append(0xb4, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + if err != nil { + return + } + err = en.WriteTime(z.QuotaConfigUpdatedAt) + if err != nil { + err = msgp.WrapError(err, "QuotaConfigUpdatedAt") + return + } + // write "ReplicationConfigUpdatedAt" + err = en.Append(0xba, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + if err != nil { + return + } + err = en.WriteTime(z.ReplicationConfigUpdatedAt) + if err != nil { + err = msgp.WrapError(err, "ReplicationConfigUpdatedAt") + return + } + // write "VersioningConfigUpdatedAt" + err = en.Append(0xb9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + if err != nil { + return + } + err = en.WriteTime(z.VersioningConfigUpdatedAt) + if err != nil { + err = msgp.WrapError(err, "VersioningConfigUpdatedAt") + return + } + // write "LifecycleConfigUpdatedAt" + err = en.Append(0xb8, 0x4c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + if err != nil { + return + } + err = en.WriteTime(z.LifecycleConfigUpdatedAt) + if err != nil { + err = msgp.WrapError(err, "LifecycleConfigUpdatedAt") + return + } + // write "NotificationConfigUpdatedAt" + err = en.Append(0xbb, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + if err != nil { + return + } + err = en.WriteTime(z.NotificationConfigUpdatedAt) + if err != nil { + err = msgp.WrapError(err, "NotificationConfigUpdatedAt") + return + } + // write "BucketTargetsConfigUpdatedAt" + err = en.Append(0xbc, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + if err != nil { + return + } + err = en.WriteTime(z.BucketTargetsConfigUpdatedAt) + if err != nil { + err = msgp.WrapError(err, "BucketTargetsConfigUpdatedAt") + return + } + // write "BucketTargetsConfigMetaUpdatedAt" + err = en.Append(0xd9, 0x20, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + if err != nil { + return + } + err = en.WriteTime(z.BucketTargetsConfigMetaUpdatedAt) + if err != nil { + err = msgp.WrapError(err, "BucketTargetsConfigMetaUpdatedAt") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *BucketMetadata) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 25 + // string "Name" + o = append(o, 0xde, 0x0, 0x19, 0xa4, 0x4e, 0x61, 0x6d, 0x65) + o = msgp.AppendString(o, z.Name) + // string "Created" + o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64) + o = msgp.AppendTime(o, z.Created) + // string "LockEnabled" + o = append(o, 0xab, 0x4c, 0x6f, 0x63, 0x6b, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64) + o = msgp.AppendBool(o, z.LockEnabled) + // string "PolicyConfigJSON" + o = append(o, 0xb0, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e) + o = msgp.AppendBytes(o, z.PolicyConfigJSON) + // string "NotificationConfigXML" + o = append(o, 0xb5, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c) + o = msgp.AppendBytes(o, z.NotificationConfigXML) + // string "LifecycleConfigXML" + o = append(o, 0xb2, 0x4c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c) + o = msgp.AppendBytes(o, z.LifecycleConfigXML) + // string "ObjectLockConfigXML" + o = append(o, 0xb3, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c) + o = msgp.AppendBytes(o, z.ObjectLockConfigXML) + // string "VersioningConfigXML" + o = append(o, 0xb3, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c) + o = msgp.AppendBytes(o, z.VersioningConfigXML) + // string "EncryptionConfigXML" + o = append(o, 0xb3, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c) + o = msgp.AppendBytes(o, z.EncryptionConfigXML) + // string "TaggingConfigXML" + o = append(o, 0xb0, 0x54, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c) + o = msgp.AppendBytes(o, z.TaggingConfigXML) + // string "QuotaConfigJSON" + o = append(o, 0xaf, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e) + o = msgp.AppendBytes(o, z.QuotaConfigJSON) + // string "ReplicationConfigXML" + o = append(o, 0xb4, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x58, 0x4d, 0x4c) + o = msgp.AppendBytes(o, z.ReplicationConfigXML) + // string "BucketTargetsConfigJSON" + o = append(o, 0xb7, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4a, 0x53, 0x4f, 0x4e) + o = msgp.AppendBytes(o, z.BucketTargetsConfigJSON) + // string "BucketTargetsConfigMetaJSON" + o = append(o, 0xbb, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x4a, 0x53, 0x4f, 0x4e) + o = msgp.AppendBytes(o, z.BucketTargetsConfigMetaJSON) + // string "PolicyConfigUpdatedAt" + o = append(o, 0xb5, 0x50, 0x6f, 0x6c, 0x69, 0x63, 0x79, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + o = msgp.AppendTime(o, z.PolicyConfigUpdatedAt) + // string "ObjectLockConfigUpdatedAt" + o = append(o, 0xb9, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x6b, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + o = msgp.AppendTime(o, z.ObjectLockConfigUpdatedAt) + // string "EncryptionConfigUpdatedAt" + o = append(o, 0xb9, 0x45, 0x6e, 0x63, 0x72, 0x79, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + o = msgp.AppendTime(o, z.EncryptionConfigUpdatedAt) + // string "TaggingConfigUpdatedAt" + o = append(o, 0xb6, 0x54, 0x61, 0x67, 0x67, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + o = msgp.AppendTime(o, z.TaggingConfigUpdatedAt) + // string "QuotaConfigUpdatedAt" + o = append(o, 0xb4, 0x51, 0x75, 0x6f, 0x74, 0x61, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + o = msgp.AppendTime(o, z.QuotaConfigUpdatedAt) + // string "ReplicationConfigUpdatedAt" + o = append(o, 0xba, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + o = msgp.AppendTime(o, z.ReplicationConfigUpdatedAt) + // string "VersioningConfigUpdatedAt" + o = append(o, 0xb9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + o = msgp.AppendTime(o, z.VersioningConfigUpdatedAt) + // string "LifecycleConfigUpdatedAt" + o = append(o, 0xb8, 0x4c, 0x69, 0x66, 0x65, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + o = msgp.AppendTime(o, z.LifecycleConfigUpdatedAt) + // string "NotificationConfigUpdatedAt" + o = append(o, 0xbb, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + o = msgp.AppendTime(o, z.NotificationConfigUpdatedAt) + // string "BucketTargetsConfigUpdatedAt" + o = append(o, 0xbc, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + o = msgp.AppendTime(o, z.BucketTargetsConfigUpdatedAt) + // string "BucketTargetsConfigMetaUpdatedAt" + o = append(o, 0xd9, 0x20, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73, 0x43, 0x6f, 0x6e, 0x66, 0x69, 0x67, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + o = msgp.AppendTime(o, z.BucketTargetsConfigMetaUpdatedAt) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BucketMetadata) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Name": + z.Name, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "Created": + z.Created, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Created") + return + } + case "LockEnabled": + z.LockEnabled, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "LockEnabled") + return + } + case "PolicyConfigJSON": + z.PolicyConfigJSON, bts, err = msgp.ReadBytesBytes(bts, z.PolicyConfigJSON) + if err != nil { + err = msgp.WrapError(err, "PolicyConfigJSON") + return + } + case "NotificationConfigXML": + z.NotificationConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.NotificationConfigXML) + if err != nil { + err = msgp.WrapError(err, "NotificationConfigXML") + return + } + case "LifecycleConfigXML": + z.LifecycleConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.LifecycleConfigXML) + if err != nil { + err = msgp.WrapError(err, "LifecycleConfigXML") + return + } + case "ObjectLockConfigXML": + z.ObjectLockConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.ObjectLockConfigXML) + if err != nil { + err = msgp.WrapError(err, "ObjectLockConfigXML") + return + } + case "VersioningConfigXML": + z.VersioningConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.VersioningConfigXML) + if err != nil { + err = msgp.WrapError(err, "VersioningConfigXML") + return + } + case "EncryptionConfigXML": + z.EncryptionConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.EncryptionConfigXML) + if err != nil { + err = msgp.WrapError(err, "EncryptionConfigXML") + return + } + case "TaggingConfigXML": + z.TaggingConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.TaggingConfigXML) + if err != nil { + err = msgp.WrapError(err, "TaggingConfigXML") + return + } + case "QuotaConfigJSON": + z.QuotaConfigJSON, bts, err = msgp.ReadBytesBytes(bts, z.QuotaConfigJSON) + if err != nil { + err = msgp.WrapError(err, "QuotaConfigJSON") + return + } + case "ReplicationConfigXML": + z.ReplicationConfigXML, bts, err = msgp.ReadBytesBytes(bts, z.ReplicationConfigXML) + if err != nil { + err = msgp.WrapError(err, "ReplicationConfigXML") + return + } + case "BucketTargetsConfigJSON": + z.BucketTargetsConfigJSON, bts, err = msgp.ReadBytesBytes(bts, z.BucketTargetsConfigJSON) + if err != nil { + err = msgp.WrapError(err, "BucketTargetsConfigJSON") + return + } + case "BucketTargetsConfigMetaJSON": + z.BucketTargetsConfigMetaJSON, bts, err = msgp.ReadBytesBytes(bts, z.BucketTargetsConfigMetaJSON) + if err != nil { + err = msgp.WrapError(err, "BucketTargetsConfigMetaJSON") + return + } + case "PolicyConfigUpdatedAt": + z.PolicyConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "PolicyConfigUpdatedAt") + return + } + case "ObjectLockConfigUpdatedAt": + z.ObjectLockConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjectLockConfigUpdatedAt") + return + } + case "EncryptionConfigUpdatedAt": + z.EncryptionConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "EncryptionConfigUpdatedAt") + return + } + case "TaggingConfigUpdatedAt": + z.TaggingConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "TaggingConfigUpdatedAt") + return + } + case "QuotaConfigUpdatedAt": + z.QuotaConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "QuotaConfigUpdatedAt") + return + } + case "ReplicationConfigUpdatedAt": + z.ReplicationConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicationConfigUpdatedAt") + return + } + case "VersioningConfigUpdatedAt": + z.VersioningConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "VersioningConfigUpdatedAt") + return + } + case "LifecycleConfigUpdatedAt": + z.LifecycleConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "LifecycleConfigUpdatedAt") + return + } + case "NotificationConfigUpdatedAt": + z.NotificationConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "NotificationConfigUpdatedAt") + return + } + case "BucketTargetsConfigUpdatedAt": + z.BucketTargetsConfigUpdatedAt, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "BucketTargetsConfigUpdatedAt") + return + } + case "BucketTargetsConfigMetaUpdatedAt": + z.BucketTargetsConfigMetaUpdatedAt, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "BucketTargetsConfigMetaUpdatedAt") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *BucketMetadata) Msgsize() (s int) { + s = 3 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 12 + msgp.BoolSize + 17 + msgp.BytesPrefixSize + len(z.PolicyConfigJSON) + 22 + msgp.BytesPrefixSize + len(z.NotificationConfigXML) + 19 + msgp.BytesPrefixSize + len(z.LifecycleConfigXML) + 20 + msgp.BytesPrefixSize + len(z.ObjectLockConfigXML) + 20 + msgp.BytesPrefixSize + len(z.VersioningConfigXML) + 20 + msgp.BytesPrefixSize + len(z.EncryptionConfigXML) + 17 + msgp.BytesPrefixSize + len(z.TaggingConfigXML) + 16 + msgp.BytesPrefixSize + len(z.QuotaConfigJSON) + 21 + msgp.BytesPrefixSize + len(z.ReplicationConfigXML) + 24 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigJSON) + 28 + msgp.BytesPrefixSize + len(z.BucketTargetsConfigMetaJSON) + 22 + msgp.TimeSize + 26 + msgp.TimeSize + 26 + msgp.TimeSize + 23 + msgp.TimeSize + 21 + msgp.TimeSize + 27 + msgp.TimeSize + 26 + msgp.TimeSize + 25 + msgp.TimeSize + 28 + msgp.TimeSize + 29 + msgp.TimeSize + 34 + msgp.TimeSize + return +} diff --git a/cmd/bucket-metadata_gen_test.go b/cmd/bucket-metadata_gen_test.go new file mode 100644 index 00000000..4c2d25c7 --- /dev/null +++ b/cmd/bucket-metadata_gen_test.go @@ -0,0 +1,123 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalBucketMetadata(t *testing.T) { + v := BucketMetadata{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBucketMetadata(b *testing.B) { + v := BucketMetadata{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBucketMetadata(b *testing.B) { + v := BucketMetadata{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBucketMetadata(b *testing.B) { + v := BucketMetadata{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBucketMetadata(t *testing.T) { + v := BucketMetadata{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBucketMetadata Msgsize() is inaccurate") + } + + vn := BucketMetadata{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBucketMetadata(b *testing.B) { + v := BucketMetadata{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBucketMetadata(b *testing.B) { + v := BucketMetadata{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/bucket-notification-handlers.go b/cmd/bucket-notification-handlers.go new file mode 100644 index 00000000..c41823b4 --- /dev/null +++ b/cmd/bucket-notification-handlers.go @@ -0,0 +1,163 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "encoding/xml" + "io" + "net/http" + "reflect" + + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/logger" + "github.com/minio/mux" + "github.com/minio/pkg/v3/policy" +) + +const ( + bucketNotificationConfig = "notification.xml" +) + +// GetBucketNotificationHandler - This HTTP handler returns event notification configuration +// as per http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html. +// It returns empty configuration if its not set. +func (api objectAPIHandlers) GetBucketNotificationHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetBucketNotification") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucketName := vars["bucket"] + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketNotificationAction, bucketName, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + _, err := objAPI.GetBucketInfo(ctx, bucketName, BucketOptions{}) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + config, err := globalBucketMetadataSys.GetNotificationConfig(bucketName) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + region := globalSite.Region() + config.SetRegion(region) + if err = config.Validate(region, globalEventNotifier.targetList); err != nil { + arnErr, ok := err.(*event.ErrARNNotFound) + if ok { + for i, queue := range config.QueueList { + // Remove ARN not found queues, because we previously allowed + // adding unexpected entries into the config. + // + // With newer config disallowing changing / turning off + // notification targets without removing ARN in notification + // configuration we won't see this problem anymore. + if reflect.DeepEqual(queue.ARN, arnErr.ARN) && i < len(config.QueueList) { + config.QueueList = append(config.QueueList[:i], + config.QueueList[i+1:]...) + } + // This is a one time activity we shall do this + // here and allow stale ARN to be removed. We shall + // never reach a stage where we will have stale + // notification configs. + } + } else { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + + configData, err := xml.Marshal(config) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + writeSuccessResponseXML(w, configData) +} + +// PutBucketNotificationHandler - This HTTP handler stores given notification configuration as per +// http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html. +func (api objectAPIHandlers) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PutBucketNotification") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + vars := mux.Vars(r) + bucketName := vars["bucket"] + + if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketNotificationAction, bucketName, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + _, err := objectAPI.GetBucketInfo(ctx, bucketName, BucketOptions{}) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // PutBucketNotification always needs a Content-Length. + if r.ContentLength <= 0 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL) + return + } + + config, err := event.ParseConfig(io.LimitReader(r.Body, r.ContentLength), globalSite.Region(), globalEventNotifier.targetList) + if err != nil { + apiErr := errorCodes.ToAPIErr(ErrMalformedXML) + if event.IsEventError(err) { + apiErr = toAPIError(ctx, err) + } + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + + configData, err := xml.Marshal(config) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if _, err = globalBucketMetadataSys.Update(ctx, bucketName, bucketNotificationConfig, configData); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + rulesMap := config.ToRulesMap() + globalEventNotifier.AddRulesMap(bucketName, rulesMap) + + writeSuccessResponseHeadersOnly(w) +} diff --git a/cmd/bucket-object-lock.go b/cmd/bucket-object-lock.go new file mode 100644 index 00000000..d0ad8514 --- /dev/null +++ b/cmd/bucket-object-lock.go @@ -0,0 +1,345 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "math" + "net/http" + + "github.com/minio/minio/internal/auth" + objectlock "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/replication" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/policy" +) + +// BucketObjectLockSys - map of bucket and retention configuration. +type BucketObjectLockSys struct{} + +// Get - Get retention configuration. +func (sys *BucketObjectLockSys) Get(bucketName string) (r objectlock.Retention, err error) { + config, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucketName) + if err != nil { + if errors.Is(err, BucketObjectLockConfigNotFound{Bucket: bucketName}) { + return r, nil + } + if errors.Is(err, errInvalidArgument) { + return r, err + } + return r, err + } + return config.ToRetention(), nil +} + +// enforceRetentionForDeletion checks if it is appropriate to remove an +// object according to locking configuration when this is lifecycle/ bucket quota asking. +func enforceRetentionForDeletion(ctx context.Context, objInfo ObjectInfo) (locked bool) { + if objInfo.DeleteMarker { + return false + } + + lhold := objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined) + if lhold.Status.Valid() && lhold.Status == objectlock.LegalHoldOn { + return true + } + + ret := objectlock.GetObjectRetentionMeta(objInfo.UserDefined) + if ret.Mode.Valid() && (ret.Mode == objectlock.RetCompliance || ret.Mode == objectlock.RetGovernance) { + t, err := objectlock.UTCNowNTP() + if err != nil { + internalLogIf(ctx, err, logger.WarningKind) + return true + } + if ret.RetainUntilDate.After(t) { + return true + } + } + return false +} + +// enforceRetentionBypassForDelete enforces whether an existing object under governance can be deleted +// with governance bypass headers set in the request. +// Objects under site wide WORM can never be overwritten. +// For objects in "Governance" mode, overwrite is allowed if a) object retention date is past OR +// governance bypass headers are set and user has governance bypass permissions. +// Objects in "Compliance" mode can be overwritten only if retention date is past. +func enforceRetentionBypassForDelete(ctx context.Context, r *http.Request, bucket string, object ObjectToDelete, oi ObjectInfo, gerr error) error { + if gerr != nil { // error from GetObjectInfo + if _, ok := gerr.(MethodNotAllowed); ok { + // This happens usually for a delete marker + if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() { + // Delete marker should be present and valid. + return nil + } + } + if isErrObjectNotFound(gerr) || isErrVersionNotFound(gerr) { + return nil + } + return gerr + } + + lhold := objectlock.GetObjectLegalHoldMeta(oi.UserDefined) + if lhold.Status.Valid() && lhold.Status == objectlock.LegalHoldOn { + return ObjectLocked{} + } + + ret := objectlock.GetObjectRetentionMeta(oi.UserDefined) + if ret.Mode.Valid() { + switch ret.Mode { + case objectlock.RetCompliance: + // In compliance mode, a protected object version can't be overwritten + // or deleted by any user, including the root user in your AWS account. + // When an object is locked in compliance mode, its retention mode can't + // be changed, and its retention period can't be shortened. Compliance mode + // ensures that an object version can't be overwritten or deleted for the + // duration of the retention period. + t, err := objectlock.UTCNowNTP() + if err != nil { + internalLogIf(ctx, err, logger.WarningKind) + return ObjectLocked{} + } + + if !ret.RetainUntilDate.Before(t) { + return ObjectLocked{} + } + return nil + case objectlock.RetGovernance: + // In governance mode, users can't overwrite or delete an object + // version or alter its lock settings unless they have special + // permissions. With governance mode, you protect objects against + // being deleted by most users, but you can still grant some users + // permission to alter the retention settings or delete the object + // if necessary. You can also use governance mode to test retention-period + // settings before creating a compliance-mode retention period. + // To override or remove governance-mode retention settings, a + // user must have the s3:BypassGovernanceRetention permission + // and must explicitly include x-amz-bypass-governance-retention:true + // as a request header with any request that requires overriding + // governance mode. + // + byPassSet := objectlock.IsObjectLockGovernanceBypassSet(r.Header) + if !byPassSet { + t, err := objectlock.UTCNowNTP() + if err != nil { + internalLogIf(ctx, err, logger.WarningKind) + return ObjectLocked{} + } + + if !ret.RetainUntilDate.Before(t) { + return ObjectLocked{} + } + return nil + } + // https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes + // If you try to delete objects protected by governance mode and have s3:BypassGovernanceRetention, the operation will succeed. + if checkRequestAuthType(ctx, r, policy.BypassGovernanceRetentionAction, bucket, object.ObjectName) != ErrNone { + return errAuthentication + } + } + } + return nil +} + +// enforceRetentionBypassForPut enforces whether an existing object under governance can be overwritten +// with governance bypass headers set in the request. +// Objects under site wide WORM cannot be overwritten. +// For objects in "Governance" mode, overwrite is allowed if a) object retention date is past OR +// governance bypass headers are set and user has governance bypass permissions. +// Objects in compliance mode can be overwritten only if retention date is being extended. No mode change is permitted. +func enforceRetentionBypassForPut(ctx context.Context, r *http.Request, oi ObjectInfo, objRetention *objectlock.ObjectRetention, cred auth.Credentials, owner bool) error { + byPassSet := objectlock.IsObjectLockGovernanceBypassSet(r.Header) + + t, err := objectlock.UTCNowNTP() + if err != nil { + internalLogIf(ctx, err, logger.WarningKind) + return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID} + } + + // Pass in relative days from current time, to additionally + // to verify "object-lock-remaining-retention-days" policy if any. + days := int(math.Ceil(math.Abs(objRetention.RetainUntilDate.Sub(t).Hours()) / 24)) + + ret := objectlock.GetObjectRetentionMeta(oi.UserDefined) + if ret.Mode.Valid() { + // Retention has expired you may change whatever you like. + if ret.RetainUntilDate.Before(t) { + apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name, + days, objRetention.RetainUntilDate.Time, + objRetention.Mode, byPassSet, r, cred, + owner) + if apiErr == ErrAccessDenied { + return errAuthentication + } + return nil + } + + switch ret.Mode { + case objectlock.RetGovernance: + govPerm := isPutRetentionAllowed(oi.Bucket, oi.Name, days, + objRetention.RetainUntilDate.Time, objRetention.Mode, + byPassSet, r, cred, owner) + // Governance mode retention period cannot be shortened, if x-amz-bypass-governance is not set. + if !byPassSet { + if objRetention.Mode != objectlock.RetGovernance || objRetention.RetainUntilDate.Before((ret.RetainUntilDate.Time)) { + return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID} + } + } + if govPerm == ErrAccessDenied { + return errAuthentication + } + return nil + case objectlock.RetCompliance: + // Compliance retention mode cannot be changed or shortened. + // https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-overview.html#object-lock-retention-modes + if objRetention.Mode != objectlock.RetCompliance || objRetention.RetainUntilDate.Before((ret.RetainUntilDate.Time)) { + return ObjectLocked{Bucket: oi.Bucket, Object: oi.Name, VersionID: oi.VersionID} + } + apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name, + days, objRetention.RetainUntilDate.Time, objRetention.Mode, + false, r, cred, owner) + if apiErr == ErrAccessDenied { + return errAuthentication + } + return nil + } + return nil + } // No pre-existing retention metadata present. + + apiErr := isPutRetentionAllowed(oi.Bucket, oi.Name, + days, objRetention.RetainUntilDate.Time, + objRetention.Mode, byPassSet, r, cred, owner) + if apiErr == ErrAccessDenied { + return errAuthentication + } + return nil +} + +// checkPutObjectLockAllowed enforces object retention policy and legal hold policy +// for requests with WORM headers +// See https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lock-managing.html for the spec. +// For non-existing objects with object retention headers set, this method returns ErrNone if bucket has +// locking enabled and user has requisite permissions (s3:PutObjectRetention) +// If object exists on object store and site wide WORM enabled - this method +// returns an error. For objects in "Governance" mode, overwrite is allowed if the retention date has expired. +// For objects in "Compliance" mode, retention date cannot be shortened, and mode cannot be altered. +// For objects with legal hold header set, the s3:PutObjectLegalHold permission is expected to be set +// Both legal hold and retention can be applied independently on an object +func checkPutObjectLockAllowed(ctx context.Context, rq *http.Request, bucket, object string, getObjectInfoFn GetObjectInfoFn, retentionPermErr, legalHoldPermErr APIErrorCode) (objectlock.RetMode, objectlock.RetentionDate, objectlock.ObjectLegalHold, APIErrorCode) { + var mode objectlock.RetMode + var retainDate objectlock.RetentionDate + var legalHold objectlock.ObjectLegalHold + + retentionRequested := objectlock.IsObjectLockRetentionRequested(rq.Header) + legalHoldRequested := objectlock.IsObjectLockLegalHoldRequested(rq.Header) + + retentionCfg, err := globalBucketObjectLockSys.Get(bucket) + if err != nil { + return mode, retainDate, legalHold, ErrInvalidBucketObjectLockConfiguration + } + + if !retentionCfg.LockEnabled { + if legalHoldRequested || retentionRequested { + return mode, retainDate, legalHold, ErrInvalidBucketObjectLockConfiguration + } + + // If this not a WORM enabled bucket, we should return right here. + return mode, retainDate, legalHold, ErrNone + } + + opts, err := getOpts(ctx, rq, bucket, object) + if err != nil { + return mode, retainDate, legalHold, toAPIErrorCode(ctx, err) + } + + replica := rq.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String() + + if opts.VersionID != "" && !replica { + if objInfo, err := getObjectInfoFn(ctx, bucket, object, opts); err == nil { + r := objectlock.GetObjectRetentionMeta(objInfo.UserDefined) + t, err := objectlock.UTCNowNTP() + if err != nil { + internalLogIf(ctx, err, logger.WarningKind) + return mode, retainDate, legalHold, ErrObjectLocked + } + if r.Mode == objectlock.RetCompliance && r.RetainUntilDate.After(t) { + return mode, retainDate, legalHold, ErrObjectLocked + } + mode = r.Mode + retainDate = r.RetainUntilDate + legalHold = objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined) + // Disallow overwriting an object on legal hold + if legalHold.Status == objectlock.LegalHoldOn { + return mode, retainDate, legalHold, ErrObjectLocked + } + } + } + + if legalHoldRequested { + var lerr error + if legalHold, lerr = objectlock.ParseObjectLockLegalHoldHeaders(rq.Header); lerr != nil { + return mode, retainDate, legalHold, toAPIErrorCode(ctx, lerr) + } + if legalHoldPermErr != ErrNone { + return mode, retainDate, legalHold, legalHoldPermErr + } + } + + if retentionRequested { + legalHold, err := objectlock.ParseObjectLockLegalHoldHeaders(rq.Header) + if err != nil { + return mode, retainDate, legalHold, toAPIErrorCode(ctx, err) + } + rMode, rDate, err := objectlock.ParseObjectLockRetentionHeaders(rq.Header) + if err != nil && (!replica || rMode != "" || !rDate.IsZero()) { + return mode, retainDate, legalHold, toAPIErrorCode(ctx, err) + } + if retentionPermErr != ErrNone { + return mode, retainDate, legalHold, retentionPermErr + } + return rMode, rDate, legalHold, ErrNone + } + if replica { // replica inherits retention metadata only from source + return "", objectlock.RetentionDate{}, legalHold, ErrNone + } + if !retentionRequested && retentionCfg.Validity > 0 { + if retentionPermErr != ErrNone { + return mode, retainDate, legalHold, retentionPermErr + } + + t, err := objectlock.UTCNowNTP() + if err != nil { + internalLogIf(ctx, err, logger.WarningKind) + return mode, retainDate, legalHold, ErrObjectLocked + } + + if !legalHoldRequested && retentionCfg.LockEnabled { + // inherit retention from bucket configuration + return retentionCfg.Mode, objectlock.RetentionDate{Time: t.Add(retentionCfg.Validity)}, legalHold, ErrNone + } + return "", objectlock.RetentionDate{}, legalHold, ErrNone + } + return mode, retainDate, legalHold, ErrNone +} + +// NewBucketObjectLockSys returns initialized BucketObjectLockSys +func NewBucketObjectLockSys() *BucketObjectLockSys { + return &BucketObjectLockSys{} +} diff --git a/cmd/bucket-policy-handlers.go b/cmd/bucket-policy-handlers.go new file mode 100644 index 00000000..994b0b0d --- /dev/null +++ b/cmd/bucket-policy-handlers.go @@ -0,0 +1,211 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + + humanize "github.com/dustin/go-humanize" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/logger" + "github.com/minio/mux" + "github.com/minio/pkg/v3/policy" +) + +const ( + // As per AWS S3 specification, 20KiB policy JSON data is allowed. + maxBucketPolicySize = 20 * humanize.KiByte + + // Policy configuration file. + bucketPolicyConfig = "policy.json" +) + +// PutBucketPolicyHandler - This HTTP handler stores given bucket policy configuration as per +// https://docs.aws.amazon.com/AmazonS3/latest/dev/access-policy-language-overview.html +func (api objectAPIHandlers) PutBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PutBucketPolicy") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + vars := mux.Vars(r) + bucket := vars["bucket"] + + if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketPolicyAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Check if bucket exists. + if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Error out if Content-Length is missing. + // PutBucketPolicy always needs Content-Length. + if r.ContentLength <= 0 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL) + return + } + + // Error out if Content-Length is beyond allowed size. + if r.ContentLength > maxBucketPolicySize { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPolicyTooLarge), r.URL) + return + } + + bucketPolicyBytes, err := io.ReadAll(io.LimitReader(r.Body, r.ContentLength)) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + bucketPolicy, err := policy.ParseBucketPolicyConfig(bytes.NewReader(bucketPolicyBytes), bucket) + if err != nil { + writeErrorResponse(ctx, w, APIError{ + Code: "MalformedPolicy", + HTTPStatusCode: http.StatusBadRequest, + Description: err.Error(), + }, r.URL) + return + } + + // Version in policy must not be empty + if bucketPolicy.Version == "" { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPolicyInvalidVersion), r.URL) + return + } + + configData, err := json.Marshal(bucketPolicy) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Call site replication hook. + replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaTypePolicy, + Bucket: bucket, + Policy: bucketPolicyBytes, + UpdatedAt: updatedAt, + })) + + // Success. + writeSuccessNoContent(w) +} + +// DeleteBucketPolicyHandler - This HTTP handler removes bucket policy configuration. +func (api objectAPIHandlers) DeleteBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "DeleteBucketPolicy") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + vars := mux.Vars(r) + bucket := vars["bucket"] + + if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketPolicyAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Check if bucket exists. + if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + updatedAt, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketPolicyConfig) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Call site replication hook. + replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaTypePolicy, + Bucket: bucket, + UpdatedAt: updatedAt, + })) + + // Success. + writeSuccessNoContent(w) +} + +// GetBucketPolicyHandler - This HTTP handler returns bucket policy configuration. +func (api objectAPIHandlers) GetBucketPolicyHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetBucketPolicy") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + vars := mux.Vars(r) + bucket := vars["bucket"] + + if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Check if bucket exists. + if _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Read bucket access policy. + config, err := globalPolicySys.Get(bucket) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + configData, err := json.Marshal(config) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Write to client. + writeSuccessResponseJSON(w, configData) +} diff --git a/cmd/bucket-policy-handlers_test.go b/cmd/bucket-policy-handlers_test.go new file mode 100644 index 00000000..e506aceb --- /dev/null +++ b/cmd/bucket-policy-handlers_test.go @@ -0,0 +1,777 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "fmt" + "io" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "sync" + "testing" + + "github.com/minio/minio/internal/auth" + "github.com/minio/pkg/v3/policy" + "github.com/minio/pkg/v3/policy/condition" +) + +func getAnonReadOnlyBucketPolicy(bucketName string) *policy.BucketPolicy { + return &policy.BucketPolicy{ + Version: policy.DefaultVersion, + Statements: []policy.BPStatement{ + policy.NewBPStatement( + "", + policy.Allow, + policy.NewPrincipal("*"), + policy.NewActionSet(policy.GetBucketLocationAction, policy.ListBucketAction), + policy.NewResourceSet(policy.NewResource(bucketName)), + condition.NewFunctions(), + ), + }, + } +} + +func getAnonWriteOnlyBucketPolicy(bucketName string) *policy.BucketPolicy { + return &policy.BucketPolicy{ + Version: policy.DefaultVersion, + Statements: []policy.BPStatement{ + policy.NewBPStatement( + "", + policy.Allow, + policy.NewPrincipal("*"), + policy.NewActionSet( + policy.GetBucketLocationAction, + policy.ListBucketMultipartUploadsAction, + ), + policy.NewResourceSet(policy.NewResource(bucketName)), + condition.NewFunctions(), + ), + }, + } +} + +func getAnonReadOnlyObjectPolicy(bucketName, prefix string) *policy.BucketPolicy { + return &policy.BucketPolicy{ + Version: policy.DefaultVersion, + Statements: []policy.BPStatement{ + policy.NewBPStatement( + "", + policy.Allow, + policy.NewPrincipal("*"), + policy.NewActionSet(policy.GetObjectAction), + policy.NewResourceSet(policy.NewResource(bucketName+"/"+prefix)), + condition.NewFunctions(), + ), + }, + } +} + +func getAnonWriteOnlyObjectPolicy(bucketName, prefix string) *policy.BucketPolicy { + return &policy.BucketPolicy{ + Version: policy.DefaultVersion, + Statements: []policy.BPStatement{ + policy.NewBPStatement( + "", + policy.Allow, + policy.NewPrincipal("*"), + policy.NewActionSet( + policy.AbortMultipartUploadAction, + policy.DeleteObjectAction, + policy.ListMultipartUploadPartsAction, + policy.PutObjectAction, + ), + policy.NewResourceSet(policy.NewResource(bucketName+"/"+prefix)), + condition.NewFunctions(), + ), + }, + } +} + +// Wrapper for calling Create Bucket and ensure we get one and only one success. +func TestCreateBucket(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testCreateBucket, endpoints: []string{"MakeBucket"}}) +} + +// testCreateBucket - Test for calling Create Bucket and ensure we get one and only one success. +func testCreateBucket(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + bucketName1 := fmt.Sprintf("%s-1", bucketName) + + const n = 100 + start := make(chan struct{}) + var ok, errs int + var wg sync.WaitGroup + var mu sync.Mutex + wg.Add(n) + for range n { + go func() { + defer wg.Done() + // Sync start. + <-start + if err := obj.MakeBucket(GlobalContext, bucketName1, MakeBucketOptions{}); err != nil { + if _, ok := err.(BucketExists); !ok { + t.Logf("unexpected error: %T: %v", err, err) + return + } + mu.Lock() + errs++ + mu.Unlock() + return + } + mu.Lock() + ok++ + mu.Unlock() + }() + } + close(start) + wg.Wait() + if ok != 1 { + t.Fatalf("want 1 ok, got %d", ok) + } + if errs != n-1 { + t.Fatalf("want %d errors, got %d", n-1, errs) + } +} + +// Wrapper for calling Put Bucket Policy HTTP handler tests for both Erasure multiple disks and single node setup. +func TestPutBucketPolicyHandler(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testPutBucketPolicyHandler, endpoints: []string{"PutBucketPolicy"}}) +} + +// testPutBucketPolicyHandler - Test for Bucket policy end point. +func testPutBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + bucketName1 := fmt.Sprintf("%s-1", bucketName) + if err := obj.MakeBucket(GlobalContext, bucketName1, MakeBucketOptions{}); err != nil { + t.Fatal(err) + } + + // template for constructing HTTP request body for PUT bucket policy. + bucketPolicyTemplate := `{"Version":"2012-10-17","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]},{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/this*"]}]}` + + bucketPolicyTemplateWithoutVersion := `{"Version":"","Statement":[{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetBucketLocation","s3:ListBucket"],"Resource":["arn:aws:s3:::%s"]},{"Sid":"","Effect":"Allow","Principal":{"AWS":["*"]},"Action":["s3:GetObject"],"Resource":["arn:aws:s3:::%s/this*"]}]}` + + // test cases with sample input and expected output. + testCases := []struct { + bucketName string + // bucket policy to be set, + // set as request body. + bucketPolicyReader io.ReadSeeker + // length in bytes of the bucket policy being set. + policyLen int + accessKey string + secretKey string + // expected Response. + expectedRespStatus int + }{ + // Test case - 1. + { + bucketName: bucketName, + bucketPolicyReader: bytes.NewReader(fmt.Appendf(nil, bucketPolicyTemplate, bucketName, bucketName)), + + policyLen: len(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName)), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusNoContent, + }, + // Test case - 2. + // Setting the content length to be more than max allowed size. + // Expecting StatusBadRequest (400). + { + bucketName: bucketName, + bucketPolicyReader: bytes.NewReader(fmt.Appendf(nil, bucketPolicyTemplate, bucketName, bucketName)), + + policyLen: maxBucketPolicySize + 1, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusBadRequest, + }, + // Test case - 3. + // Case with content-length of the HTTP request set to 0. + // Expecting the HTTP response status to be StatusLengthRequired (411). + { + bucketName: bucketName, + bucketPolicyReader: bytes.NewReader(fmt.Appendf(nil, bucketPolicyTemplate, bucketName, bucketName)), + + policyLen: 0, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusLengthRequired, + }, + // Test case - 4. + // setting the readSeeker to `nil`, bucket policy parser will fail. + { + bucketName: bucketName, + bucketPolicyReader: nil, + + policyLen: 10, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusBadRequest, + }, + // Test case - 5. + // setting the keys to be empty. + // Expecting statusForbidden. + { + bucketName: bucketName, + bucketPolicyReader: nil, + + policyLen: 10, + accessKey: "", + secretKey: "", + expectedRespStatus: http.StatusForbidden, + }, + // Test case - 6. + // setting an invalid bucket policy. + // the bucket policy parser will fail. + { + bucketName: bucketName, + bucketPolicyReader: bytes.NewReader([]byte("dummy-policy")), + + policyLen: len([]byte("dummy-policy")), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusBadRequest, + }, + // Test case - 7. + // Different bucket name used in the HTTP request and the policy string. + // checkBucketPolicyResources should fail. + { + bucketName: bucketName1, + bucketPolicyReader: bytes.NewReader(fmt.Appendf(nil, bucketPolicyTemplate, bucketName, bucketName)), + + policyLen: len(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName)), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusBadRequest, + }, + // Test case - 8. + // non-existent bucket is used. + // writing BucketPolicy should fail. + // should result in 404 StatusNotFound + { + bucketName: "non-existent-bucket", + bucketPolicyReader: bytes.NewReader(fmt.Appendf(nil, bucketPolicyTemplate, "non-existent-bucket", "non-existent-bucket")), + + policyLen: len(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName)), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusNotFound, + }, + // Test case - 9. + // non-existent bucket is used (with invalid bucket name) + // writing BucketPolicy should fail. + // should result in 404 StatusNotFound + { + bucketName: ".invalid-bucket", + bucketPolicyReader: bytes.NewReader(fmt.Appendf(nil, bucketPolicyTemplate, ".invalid-bucket", ".invalid-bucket")), + + policyLen: len(fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName)), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusBadRequest, + }, + // Test case - 10. + // Existent bucket with policy with Version field empty. + // writing BucketPolicy should fail. + // should result in 400 StatusBadRequest. + { + bucketName: bucketName, + bucketPolicyReader: bytes.NewReader(fmt.Appendf(nil, bucketPolicyTemplateWithoutVersion, bucketName, bucketName)), + + policyLen: len(fmt.Sprintf(bucketPolicyTemplateWithoutVersion, bucketName, bucketName)), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusBadRequest, + }, + } + + // Iterating over the test cases, calling the function under test and asserting the response. + for i, testCase := range testCases { + // obtain the put bucket policy request body. + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + recV4 := httptest.NewRecorder() + // construct HTTP request for PUT bucket policy endpoint. + reqV4, err := newTestSignedRequestV4(http.MethodPut, getPutPolicyURL("", testCase.bucketName), + int64(testCase.policyLen), testCase.bucketPolicyReader, testCase.accessKey, testCase.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: %v", i+1, instanceType, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(recV4, reqV4) + if recV4.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV4.Code) + } + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + recV2 := httptest.NewRecorder() + // construct HTTP request for PUT bucket policy endpoint. + reqV2, err := newTestSignedRequestV2(http.MethodPut, getPutPolicyURL("", testCase.bucketName), + int64(testCase.policyLen), testCase.bucketPolicyReader, testCase.accessKey, testCase.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for PutBucketPolicyHandler: %v", i+1, instanceType, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(recV2, reqV2) + if recV2.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code) + } + } + + // Test for Anonymous/unsigned http request. + // Bucket policy related functions doesn't support anonymous requests, setting policies shouldn't make a difference. + bucketPolicyStr := fmt.Sprintf(bucketPolicyTemplate, bucketName, bucketName) + // create unsigned HTTP request for PutBucketPolicyHandler. + anonReq, err := newTestRequest(http.MethodPut, getPutPolicyURL("", bucketName), + int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr))) + if err != nil { + t.Fatalf("MinIO %s: Failed to create an anonymous request for bucket \"%s\": %v", + instanceType, bucketName, err) + } + + // ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse, + // sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the + // unsigned request goes through and its validated again. + ExecObjectLayerAPIAnonTest(t, obj, "PutBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonWriteOnlyBucketPolicy(bucketName)) + + // HTTP request for testing when `objectLayer` is set to `nil`. + // There is no need to use an existing bucket and valid input for creating the request + // since the `objectLayer==nil` check is performed before any other checks inside the handlers. + // The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called. + nilBucket := "dummy-bucket" + + nilReq, err := newTestSignedRequestV4(http.MethodPut, getPutPolicyURL("", nilBucket), + 0, nil, "", "", nil) + if err != nil { + t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType) + } + // execute the object layer set to `nil` test. + // `ExecObjectLayerAPINilTest` manages the operation. + ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq) +} + +// Wrapper for calling Get Bucket Policy HTTP handler tests for both Erasure multiple disks and single node setup. +func TestGetBucketPolicyHandler(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testGetBucketPolicyHandler, endpoints: []string{"PutBucketPolicy", "GetBucketPolicy"}}) +} + +// testGetBucketPolicyHandler - Test for end point which fetches the access policy json of the given bucket. +func testGetBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + // template for constructing HTTP request body for PUT bucket policy. + bucketPolicyTemplate := `{"Version":"2012-10-17","Statement":[{"Action":["s3:GetBucketLocation","s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s"]},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s/this*"]}]}` + + // Writing bucket policy before running test on GetBucketPolicy. + putTestPolicies := []struct { + bucketName string + accessKey string + secretKey string + // expected Response. + expectedRespStatus int + }{ + {bucketName, credentials.AccessKey, credentials.SecretKey, http.StatusNoContent}, + } + + // Iterating over the cases and writing the bucket policy. + // its required to write the policies first before running tests on GetBucketPolicy. + for i, testPolicy := range putTestPolicies { + // obtain the put bucket policy request body. + bucketPolicyStr := fmt.Sprintf(bucketPolicyTemplate, testPolicy.bucketName, testPolicy.bucketName) + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + recV4 := httptest.NewRecorder() + // construct HTTP request for PUT bucket policy endpoint. + reqV4, err := newTestSignedRequestV4(http.MethodPut, getPutPolicyURL("", testPolicy.bucketName), + int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: %v", i+1, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(recV4, reqV4) + if recV4.Code != testPolicy.expectedRespStatus { + t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testPolicy.expectedRespStatus, recV4.Code) + } + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + recV2 := httptest.NewRecorder() + // construct HTTP request for PUT bucket policy endpoint. + reqV2, err := newTestSignedRequestV2(http.MethodPut, getPutPolicyURL("", testPolicy.bucketName), + int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: %v", i+1, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(recV2, reqV2) + if recV2.Code != testPolicy.expectedRespStatus { + t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testPolicy.expectedRespStatus, recV2.Code) + } + } + + // test cases with inputs and expected result for GetBucketPolicyHandler. + testCases := []struct { + bucketName string + accessKey string + secretKey string + // expected output. + expectedBucketPolicy string + expectedRespStatus int + }{ + // Test case - 1. + // Case which valid inputs, expected to return success status of 200OK. + { + bucketName: bucketName, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedBucketPolicy: bucketPolicyTemplate, + expectedRespStatus: http.StatusOK, + }, + // Test case - 2. + // Case with non-existent bucket name. + { + bucketName: "non-existent-bucket", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedBucketPolicy: bucketPolicyTemplate, + expectedRespStatus: http.StatusNotFound, + }, + // Test case - 3. + // Case with non-existent bucket name. + { + bucketName: ".invalid-bucket-name", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedBucketPolicy: "", + expectedRespStatus: http.StatusBadRequest, + }, + } + // Iterating over the cases, fetching the policy and validating the response. + for i, testCase := range testCases { + // expected bucket policy json string. + expectedBucketPolicyStr := fmt.Sprintf(testCase.expectedBucketPolicy, testCase.bucketName, testCase.bucketName) + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + recV4 := httptest.NewRecorder() + // construct HTTP request for PUT bucket policy endpoint. + reqV4, err := newTestSignedRequestV4(http.MethodGet, getGetPolicyURL("", testCase.bucketName), + 0, nil, testCase.accessKey, testCase.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: %v", i+1, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler, GetBucketPolicyHandler handles the request. + apiRouter.ServeHTTP(recV4, reqV4) + // Assert the response code with the expected status. + if recV4.Code != testCase.expectedRespStatus { + t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, recV4.Code) + } + // read the response body. + bucketPolicyReadBuf, err := io.ReadAll(recV4.Body) + if err != nil { + t.Fatalf("Test %d: %s: Failed parsing response body: %v", i+1, instanceType, err) + } + + if recV4.Code != testCase.expectedRespStatus { + // Verify whether the bucket policy fetched is same as the one inserted. + var expectedPolicy *policy.BucketPolicy + expectedPolicy, err = policy.ParseBucketPolicyConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName) + if err != nil { + t.Fatalf("unexpected error. %v", err) + } + var gotPolicy *policy.BucketPolicy + gotPolicy, err = policy.ParseBucketPolicyConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName) + if err != nil { + t.Fatalf("unexpected error. %v", err) + } + + if !reflect.DeepEqual(expectedPolicy, gotPolicy) { + t.Errorf("Test %d: %s: Bucket policy differs from expected value.", i+1, instanceType) + } + } + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + recV2 := httptest.NewRecorder() + // construct HTTP request for PUT bucket policy endpoint. + reqV2, err := newTestSignedRequestV2(http.MethodGet, getGetPolicyURL("", testCase.bucketName), + 0, nil, testCase.accessKey, testCase.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: %v", i+1, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler, GetBucketPolicyHandler handles the request. + apiRouter.ServeHTTP(recV2, reqV2) + // Assert the response code with the expected status. + if recV2.Code != testCase.expectedRespStatus { + t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, recV2.Code) + } + // read the response body. + bucketPolicyReadBuf, err = io.ReadAll(recV2.Body) + if err != nil { + t.Fatalf("Test %d: %s: Failed parsing response body: %v", i+1, instanceType, err) + } + if recV2.Code == http.StatusOK { + // Verify whether the bucket policy fetched is same as the one inserted. + expectedPolicy, err := policy.ParseBucketPolicyConfig(strings.NewReader(expectedBucketPolicyStr), testCase.bucketName) + if err != nil { + t.Fatalf("unexpected error. %v", err) + } + gotPolicy, err := policy.ParseBucketPolicyConfig(bytes.NewReader(bucketPolicyReadBuf), testCase.bucketName) + if err != nil { + t.Fatalf("unexpected error. %v", err) + } + + if !reflect.DeepEqual(expectedPolicy, gotPolicy) { + t.Errorf("Test %d: %s: Bucket policy differs from expected value.", i+1, instanceType) + } + } + } + + // Test for Anonymous/unsigned http request. + // Bucket policy related functions doesn't support anonymous requests, setting policies shouldn't make a difference. + // create unsigned HTTP request for PutBucketPolicyHandler. + anonReq, err := newTestRequest(http.MethodGet, getPutPolicyURL("", bucketName), 0, nil) + if err != nil { + t.Fatalf("MinIO %s: Failed to create an anonymous request for bucket \"%s\": %v", + instanceType, bucketName, err) + } + + // ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse, + // sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the + // unsigned request goes through and its validated again. + ExecObjectLayerAPIAnonTest(t, obj, "GetBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonReadOnlyBucketPolicy(bucketName)) + + // HTTP request for testing when `objectLayer` is set to `nil`. + // There is no need to use an existing bucket and valid input for creating the request + // since the `objectLayer==nil` check is performed before any other checks inside the handlers. + // The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called. + nilBucket := "dummy-bucket" + + nilReq, err := newTestSignedRequestV4(http.MethodGet, getGetPolicyURL("", nilBucket), + 0, nil, "", "", nil) + if err != nil { + t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType) + } + // execute the object layer set to `nil` test. + // `ExecObjectLayerAPINilTest` manages the operation. + ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq) +} + +// Wrapper for calling Delete Bucket Policy HTTP handler tests for both Erasure multiple disks and single node setup. +func TestDeleteBucketPolicyHandler(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testDeleteBucketPolicyHandler, endpoints: []string{"PutBucketPolicy", "DeleteBucketPolicy"}}) +} + +// testDeleteBucketPolicyHandler - Test for Delete bucket policy end point. +func testDeleteBucketPolicyHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + // template for constructing HTTP request body for PUT bucket policy. + bucketPolicyTemplate := `{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "s3:GetBucketLocation", + "s3:ListBucket" + ], + "Effect": "Allow", + "Principal": { + "AWS": [ + "*" + ] + }, + "Resource": [ + "arn:aws:s3:::%s" + ] + }, + { + "Action": [ + "s3:GetObject" + ], + "Effect": "Allow", + "Principal": { + "AWS": [ + "*" + ] + }, + "Resource": [ + "arn:aws:s3:::%s/this*" + ] + } + ] +}` + + // Writing bucket policy before running test on DeleteBucketPolicy. + putTestPolicies := []struct { + bucketName string + accessKey string + secretKey string + // expected Response. + expectedRespStatus int + }{ + { + bucketName: bucketName, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusNoContent, + }, + } + + // Iterating over the cases and writing the bucket policy. + // its required to write the policies first before running tests on GetBucketPolicy. + for i, testPolicy := range putTestPolicies { + // obtain the put bucket policy request body. + bucketPolicyStr := fmt.Sprintf(bucketPolicyTemplate, testPolicy.bucketName, testPolicy.bucketName) + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + recV4 := httptest.NewRecorder() + // construct HTTP request for PUT bucket policy endpoint. + reqV4, err := newTestSignedRequestV4(http.MethodPut, getPutPolicyURL("", testPolicy.bucketName), + int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: %v", i+1, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(recV4, reqV4) + if recV4.Code != testPolicy.expectedRespStatus { + t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testPolicy.expectedRespStatus, recV4.Code) + } + } + + // testcases with input and expected output for DeleteBucketPolicyHandler. + testCases := []struct { + bucketName string + accessKey string + secretKey string + // expected response. + expectedRespStatus int + }{ + // Test case - 1. + { + bucketName: bucketName, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusNoContent, + }, + // Test case - 2. + // Case with non-existent-bucket. + { + bucketName: "non-existent-bucket", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusNotFound, + }, + // Test case - 3. + // Case with non-existent-bucket. + { + bucketName: ".invalid-bucket-name", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusBadRequest, + }, + } + // Iterating over the cases and deleting the bucket policy and then asserting response. + for i, testCase := range testCases { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + recV4 := httptest.NewRecorder() + // construct HTTP request for Delete bucket policy endpoint. + reqV4, err := newTestSignedRequestV4(http.MethodDelete, getDeletePolicyURL("", testCase.bucketName), + 0, nil, testCase.accessKey, testCase.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: %v", i+1, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler, DeleteBucketPolicyHandler handles the request. + apiRouter.ServeHTTP(recV4, reqV4) + // Assert the response code with the expected status. + if recV4.Code != testCase.expectedRespStatus { + t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, recV4.Code) + } + } + + // Iterating over the cases and writing the bucket policy. + // its required to write the policies first before running tests on GetBucketPolicy. + for i, testPolicy := range putTestPolicies { + // obtain the put bucket policy request body. + bucketPolicyStr := fmt.Sprintf(bucketPolicyTemplate, testPolicy.bucketName, testPolicy.bucketName) + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + recV2 := httptest.NewRecorder() + // construct HTTP request for PUT bucket policy endpoint. + reqV2, err := newTestSignedRequestV2(http.MethodPut, getPutPolicyURL("", testPolicy.bucketName), + int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), testPolicy.accessKey, testPolicy.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: Failed to create HTTP request for PutBucketPolicyHandler: %v", i+1, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(recV2, reqV2) + if recV2.Code != testPolicy.expectedRespStatus { + t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testPolicy.expectedRespStatus, recV2.Code) + } + } + + for i, testCase := range testCases { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + recV2 := httptest.NewRecorder() + // construct HTTP request for Delete bucket policy endpoint. + reqV2, err := newTestSignedRequestV2(http.MethodDelete, getDeletePolicyURL("", testCase.bucketName), + 0, nil, testCase.accessKey, testCase.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: Failed to create HTTP request for GetBucketPolicyHandler: %v", i+1, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler, DeleteBucketPolicyHandler handles the request. + apiRouter.ServeHTTP(recV2, reqV2) + // Assert the response code with the expected status. + if recV2.Code != testCase.expectedRespStatus { + t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, recV2.Code) + } + } + // Test for Anonymous/unsigned http request. + // Bucket policy related functions doesn't support anonymous requests, setting policies shouldn't make a difference. + // create unsigned HTTP request for PutBucketPolicyHandler. + anonReq, err := newTestRequest(http.MethodDelete, getPutPolicyURL("", bucketName), 0, nil) + if err != nil { + t.Fatalf("MinIO %s: Failed to create an anonymous request for bucket \"%s\": %v", + instanceType, bucketName, err) + } + + // ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse, + // sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the + // unsigned request goes through and its validated again. + ExecObjectLayerAPIAnonTest(t, obj, "DeleteBucketPolicyHandler", bucketName, "", instanceType, apiRouter, anonReq, getAnonWriteOnlyBucketPolicy(bucketName)) + + // HTTP request for testing when `objectLayer` is set to `nil`. + // There is no need to use an existing bucket and valid input for creating the request + // since the `objectLayer==nil` check is performed before any other checks inside the handlers. + // The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called. + nilBucket := "dummy-bucket" + + nilReq, err := newTestSignedRequestV4(http.MethodDelete, getDeletePolicyURL("", nilBucket), + 0, nil, "", "", nil) + if err != nil { + t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType) + } + // execute the object layer set to `nil` test. + // `ExecObjectLayerAPINilTest` manages the operation. + ExecObjectLayerAPINilTest(t, nilBucket, "", instanceType, apiRouter, nilReq) +} diff --git a/cmd/bucket-policy.go b/cmd/bucket-policy.go new file mode 100644 index 00000000..0bcbe2e6 --- /dev/null +++ b/cmd/bucket-policy.go @@ -0,0 +1,288 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "encoding/json" + "maps" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + jsoniter "github.com/json-iterator/go" + miniogopolicy "github.com/minio/minio-go/v7/pkg/policy" + "github.com/minio/minio-go/v7/pkg/tags" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/handlers" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/policy" +) + +// PolicySys - policy subsystem. +type PolicySys struct{} + +// Get returns stored bucket policy +func (sys *PolicySys) Get(bucket string) (*policy.BucketPolicy, error) { + policy, _, err := globalBucketMetadataSys.GetPolicyConfig(bucket) + return policy, err +} + +// IsAllowed - checks given policy args is allowed to continue the Rest API. +func (sys *PolicySys) IsAllowed(args policy.BucketPolicyArgs) bool { + p, err := sys.Get(args.BucketName) + if err == nil { + return p.IsAllowed(args) + } + + // Log unhandled errors. + if _, ok := err.(BucketPolicyNotFound); !ok { + internalLogIf(GlobalContext, err, logger.WarningKind) + } + + // As policy is not available for given bucket name, returns IsOwner i.e. + // operation is allowed only for owner. + return args.IsOwner +} + +// NewPolicySys - creates new policy system. +func NewPolicySys() *PolicySys { + return &PolicySys{} +} + +func getSTSConditionValues(r *http.Request, lc string, cred auth.Credentials) map[string][]string { + m := make(map[string][]string) + if d := r.Form.Get("DurationSeconds"); d != "" { + m["DurationSeconds"] = []string{d} + } + return m +} + +func getConditionValues(r *http.Request, lc string, cred auth.Credentials) map[string][]string { + currTime := UTCNow() + + var ( + username = cred.AccessKey + claims = cred.Claims + groups = cred.Groups + ) + + if cred.IsTemp() || cred.IsServiceAccount() { + // For derived credentials, check the parent user's permissions. + username = cred.ParentUser + } + + principalType := "Anonymous" + if username != "" { + principalType = "User" + if len(claims) > 0 { + principalType = "AssumedRole" + } + if username == globalActiveCred.AccessKey { + principalType = "Account" + } + } + + vid := r.Form.Get(xhttp.VersionID) + if vid == "" { + if u, err := url.Parse(r.Header.Get(xhttp.AmzCopySource)); err == nil { + vid = u.Query().Get(xhttp.VersionID) + } + } + + authType := getRequestAuthType(r) + var signatureVersion string + switch authType { + case authTypeSignedV2, authTypePresignedV2: + signatureVersion = signV2Algorithm + case authTypeSigned, authTypePresigned, authTypeStreamingSigned, authTypePostPolicy: + signatureVersion = signV4Algorithm + } + + var authtype string + switch authType { + case authTypePresignedV2, authTypePresigned: + authtype = "REST-QUERY-STRING" + case authTypeSignedV2, authTypeSigned, authTypeStreamingSigned: + authtype = "REST-HEADER" + case authTypePostPolicy: + authtype = "POST" + } + + args := map[string][]string{ + "CurrentTime": {currTime.Format(time.RFC3339)}, + "EpochTime": {strconv.FormatInt(currTime.Unix(), 10)}, + "SecureTransport": {strconv.FormatBool(r.TLS != nil)}, + "SourceIp": {handlers.GetSourceIPRaw(r)}, + "UserAgent": {r.UserAgent()}, + "Referer": {r.Referer()}, + "principaltype": {principalType}, + "userid": {username}, + "username": {username}, + "versionid": {vid}, + "signatureversion": {signatureVersion}, + "authType": {authtype}, + } + + if lc != "" { + args["LocationConstraint"] = []string{lc} + } + + cloneHeader := r.Header.Clone() + if v := cloneHeader.Get("x-amz-signature-age"); v != "" { + args["signatureAge"] = []string{v} + cloneHeader.Del("x-amz-signature-age") + } + + if userTags := cloneHeader.Get(xhttp.AmzObjectTagging); userTags != "" { + tag, _ := tags.ParseObjectTags(userTags) + if tag != nil { + tagMap := tag.ToMap() + keys := make([]string, 0, len(tagMap)) + for k, v := range tagMap { + args[pathJoin("ExistingObjectTag", k)] = []string{v} + args[pathJoin("RequestObjectTag", k)] = []string{v} + keys = append(keys, k) + } + args["RequestObjectTagKeys"] = keys + } + } + + for _, objLock := range []string{ + xhttp.AmzObjectLockMode, + xhttp.AmzObjectLockLegalHold, + xhttp.AmzObjectLockRetainUntilDate, + } { + if values, ok := cloneHeader[objLock]; ok { + args[strings.TrimPrefix(objLock, "X-Amz-")] = values + } + cloneHeader.Del(objLock) + } + + for key, values := range cloneHeader { + if strings.EqualFold(key, xhttp.AmzObjectTagging) { + continue + } + if existingValues, found := args[key]; found { + args[key] = append(existingValues, values...) + } else { + args[key] = values + } + } + + cloneURLValues := make(url.Values, len(r.Form)) + maps.Copy(cloneURLValues, r.Form) + + for _, objLock := range []string{ + xhttp.AmzObjectLockMode, + xhttp.AmzObjectLockLegalHold, + xhttp.AmzObjectLockRetainUntilDate, + } { + if values, ok := cloneURLValues[objLock]; ok { + args[strings.TrimPrefix(objLock, "X-Amz-")] = values + } + cloneURLValues.Del(objLock) + } + + for key, values := range cloneURLValues { + if existingValues, found := args[key]; found { + args[key] = append(existingValues, values...) + } else { + args[key] = values + } + } + + // JWT specific values + // + // Add all string claims + for k, v := range claims { + vStr, ok := v.(string) + if ok { + // Trim any LDAP specific prefix + args[strings.ToLower(strings.TrimPrefix(k, "ldap"))] = []string{vStr} + } + } + + // Add groups claim which could be a list. This will ensure that the claim + // `jwt:groups` works. + if grpsVal, ok := claims["groups"]; ok { + if grpsIs, ok := grpsVal.([]any); ok { + grps := []string{} + for _, gI := range grpsIs { + if g, ok := gI.(string); ok { + grps = append(grps, g) + } + } + if len(grps) > 0 { + args["groups"] = grps + } + } + } + + // if not claim groups are available use the one with auth.Credentials + if _, ok := args["groups"]; !ok { + if len(groups) > 0 { + args["groups"] = groups + } + } + + return args +} + +// PolicyToBucketAccessPolicy converts a MinIO policy into a minio-go policy data structure. +func PolicyToBucketAccessPolicy(bucketPolicy *policy.BucketPolicy) (*miniogopolicy.BucketAccessPolicy, error) { + // Return empty BucketAccessPolicy for empty bucket policy. + if bucketPolicy == nil { + return &miniogopolicy.BucketAccessPolicy{Version: policy.DefaultVersion}, nil + } + + data, err := json.Marshal(bucketPolicy) + if err != nil { + // This should not happen because bucketPolicy is valid to convert to JSON data. + return nil, err + } + + var policyInfo miniogopolicy.BucketAccessPolicy + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err = json.Unmarshal(data, &policyInfo); err != nil { + // This should not happen because data is valid to JSON data. + return nil, err + } + + return &policyInfo, nil +} + +// BucketAccessPolicyToPolicy - converts minio-go/policy.BucketAccessPolicy to policy.BucketPolicy. +func BucketAccessPolicyToPolicy(policyInfo *miniogopolicy.BucketAccessPolicy) (*policy.BucketPolicy, error) { + data, err := json.Marshal(policyInfo) + if err != nil { + // This should not happen because policyInfo is valid to convert to JSON data. + return nil, err + } + + var bucketPolicy policy.BucketPolicy + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err = json.Unmarshal(data, &bucketPolicy); err != nil { + // This should not happen because data is valid to JSON data. + return nil, err + } + + return &bucketPolicy, nil +} diff --git a/cmd/bucket-quota.go b/cmd/bucket-quota.go new file mode 100644 index 00000000..d9779c21 --- /dev/null +++ b/cmd/bucket-quota.go @@ -0,0 +1,140 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/cachevalue" + "github.com/minio/minio/internal/logger" +) + +// BucketQuotaSys - map of bucket and quota configuration. +type BucketQuotaSys struct{} + +// Get - Get quota configuration. +func (sys *BucketQuotaSys) Get(ctx context.Context, bucketName string) (*madmin.BucketQuota, error) { + cfg, _, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucketName) + return cfg, err +} + +// NewBucketQuotaSys returns initialized BucketQuotaSys +func NewBucketQuotaSys() *BucketQuotaSys { + return &BucketQuotaSys{} +} + +var bucketStorageCache = cachevalue.New[DataUsageInfo]() + +// Init initialize bucket quota. +func (sys *BucketQuotaSys) Init(objAPI ObjectLayer) { + bucketStorageCache.InitOnce(10*time.Second, + cachevalue.Opts{ReturnLastGood: true, NoWait: true}, + func(ctx context.Context) (DataUsageInfo, error) { + if objAPI == nil { + return DataUsageInfo{}, errServerNotInitialized + } + ctx, done := context.WithTimeout(ctx, 2*time.Second) + defer done() + + return loadDataUsageFromBackend(ctx, objAPI) + }, + ) +} + +// GetBucketUsageInfo return bucket usage info for a given bucket +func (sys *BucketQuotaSys) GetBucketUsageInfo(ctx context.Context, bucket string) BucketUsageInfo { + sys.Init(newObjectLayerFn()) + + dui, err := bucketStorageCache.GetWithCtx(ctx) + timedout := OperationTimedOut{} + if err != nil && !errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &timedout) { + if len(dui.BucketsUsage) > 0 { + internalLogOnceIf(GlobalContext, fmt.Errorf("unable to retrieve usage information for bucket: %s, relying on older value cached in-memory: err(%v)", bucket, err), "bucket-usage-cache-"+bucket) + } else { + internalLogOnceIf(GlobalContext, errors.New("unable to retrieve usage information for bucket: %s, no reliable usage value available - quota will not be enforced"), "bucket-usage-empty-"+bucket) + } + } + + if len(dui.BucketsUsage) > 0 { + bui, ok := dui.BucketsUsage[bucket] + if ok { + return bui + } + } + return BucketUsageInfo{} +} + +// parseBucketQuota parses BucketQuota from json +func parseBucketQuota(bucket string, data []byte) (quotaCfg *madmin.BucketQuota, err error) { + quotaCfg = &madmin.BucketQuota{} + if err = json.Unmarshal(data, quotaCfg); err != nil { + return quotaCfg, err + } + if !quotaCfg.IsValid() { + if quotaCfg.Type == "fifo" { + internalLogIf(GlobalContext, errors.New("Detected older 'fifo' quota config, 'fifo' feature is removed and not supported anymore. Please clear your quota configs using 'mc quota clear alias/bucket' and use 'mc ilm add' for expiration of objects"), logger.WarningKind) + return quotaCfg, fmt.Errorf("invalid quota type 'fifo'") + } + return quotaCfg, fmt.Errorf("Invalid quota config %#v", quotaCfg) + } + return quotaCfg, err +} + +func (sys *BucketQuotaSys) enforceQuotaHard(ctx context.Context, bucket string, size int64) error { + if size < 0 { + return nil + } + + q, err := sys.Get(ctx, bucket) + if err != nil { + return err + } + + var quotaSize uint64 + if q != nil && q.Type == madmin.HardQuota { + if q.Size > 0 { + quotaSize = q.Size + } else if q.Quota > 0 { + quotaSize = q.Quota + } + } + if quotaSize > 0 { + if uint64(size) >= quotaSize { // check if file size already exceeds the quota + return BucketQuotaExceeded{Bucket: bucket} + } + + bui := sys.GetBucketUsageInfo(ctx, bucket) + if bui.Size > 0 && ((bui.Size + uint64(size)) >= quotaSize) { + return BucketQuotaExceeded{Bucket: bucket} + } + } + + return nil +} + +func enforceBucketQuotaHard(ctx context.Context, bucket string, size int64) error { + if globalBucketQuotaSys == nil { + return nil + } + return globalBucketQuotaSys.enforceQuotaHard(ctx, bucket, size) +} diff --git a/cmd/bucket-replication-handlers.go b/cmd/bucket-replication-handlers.go new file mode 100644 index 00000000..05a9d2e8 --- /dev/null +++ b/cmd/bucket-replication-handlers.go @@ -0,0 +1,660 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "io" + "net/http" + "path" + "time" + + "github.com/minio/minio-go/v7" + objectlock "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/replication" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" + "github.com/minio/mux" + "github.com/minio/pkg/v3/policy" +) + +// PutBucketReplicationConfigHandler - PUT Bucket replication configuration. +// ---------- +// Add a replication configuration on the specified bucket as specified in https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketReplication.html +func (api objectAPIHandlers) PutBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PutBucketReplicationConfig") + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + if s3Error := checkRequestAuthType(ctx, r, policy.PutReplicationConfigurationAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + // Check if bucket exists. + if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if globalSiteReplicationSys.isEnabled() && logger.GetReqInfo(ctx).Cred.AccessKey != globalActiveCred.AccessKey { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL) + return + } + if versioned := globalBucketVersioningSys.Enabled(bucket); !versioned { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNeedsVersioningError), r.URL) + return + } + replicationConfig, err := replication.ParseConfig(io.LimitReader(r.Body, r.ContentLength)) + if err != nil { + apiErr := errorCodes.ToAPIErr(ErrMalformedXML) + apiErr.Description = err.Error() + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig, &validateReplicationDestinationOptions{CheckRemoteBucket: true}) + if apiErr != noError { + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + // Validate the received bucket replication config + if err = replicationConfig.Validate(bucket, sameTarget); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + configData, err := xml.Marshal(replicationConfig) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, configData); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Write success response. + writeSuccessResponseHeadersOnly(w) +} + +// GetBucketReplicationConfigHandler - GET Bucket replication configuration. +// ---------- +// Gets the replication configuration for a bucket. +func (api objectAPIHandlers) GetBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetBucketReplicationConfig") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + // check if user has permissions to perform this operation + if s3Error := checkRequestAuthType(ctx, r, policy.GetReplicationConfigurationAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + // Check if bucket exists. + if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + configData, err := xml.Marshal(config) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Write success response. + writeSuccessResponseXML(w, configData) +} + +// DeleteBucketReplicationConfigHandler - DELETE Bucket replication config. +// ---------- +func (api objectAPIHandlers) DeleteBucketReplicationConfigHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "DeleteBucketReplicationConfig") + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.PutReplicationConfigurationAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + // Check if bucket exists. + if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if globalSiteReplicationSys.isEnabled() { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationDenyEditError), r.URL) + return + } + if _, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketReplicationConfig); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + for _, tgt := range targets.Targets { + if err := globalBucketTargetSys.RemoveTarget(ctx, bucket, tgt.Arn); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + if _, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketTargetsFile); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + // Write success response. + writeSuccessResponseHeadersOnly(w) +} + +// GetBucketReplicationMetricsHandler - GET Bucket replication metrics. // Deprecated Aug 2023 +// ---------- +// Gets the replication metrics for a bucket. +func (api objectAPIHandlers) GetBucketReplicationMetricsHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetBucketReplicationMetrics") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + // check if user has permissions to perform this operation + if s3Error := checkRequestAuthType(ctx, r, policy.GetReplicationConfigurationAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Check if bucket exists. + if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if _, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + w.Header().Set(xhttp.ContentType, string(mimeJSON)) + + enc := json.NewEncoder(w) + stats := globalReplicationStats.Load().getLatestReplicationStats(bucket) + bwRpt := globalNotificationSys.GetBandwidthReports(ctx, bucket) + bwMap := bwRpt.BucketStats + for arn, st := range stats.ReplicationStats.Stats { + for opts, bw := range bwMap { + if opts.ReplicationARN != "" && opts.ReplicationARN == arn { + st.BandWidthLimitInBytesPerSecond = bw.LimitInBytesPerSecond + st.CurrentBandwidthInBytesPerSecond = bw.CurrentBandwidthInBytesPerSecond + stats.ReplicationStats.Stats[arn] = st + } + } + } + + if err := enc.Encode(stats.ReplicationStats); err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } +} + +// GetBucketReplicationMetricsV2Handler - GET Bucket replication metrics. +// ---------- +// Gets the replication metrics for a bucket. +func (api objectAPIHandlers) GetBucketReplicationMetricsV2Handler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetBucketReplicationMetricsV2") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + // check if user has permissions to perform this operation + if s3Error := checkRequestAuthType(ctx, r, policy.GetReplicationConfigurationAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Check if bucket exists. + if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if _, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + w.Header().Set(xhttp.ContentType, string(mimeJSON)) + + enc := json.NewEncoder(w) + stats := globalReplicationStats.Load().getLatestReplicationStats(bucket) + bwRpt := globalNotificationSys.GetBandwidthReports(ctx, bucket) + bwMap := bwRpt.BucketStats + for arn, st := range stats.ReplicationStats.Stats { + for opts, bw := range bwMap { + if opts.ReplicationARN != "" && opts.ReplicationARN == arn { + st.BandWidthLimitInBytesPerSecond = bw.LimitInBytesPerSecond + st.CurrentBandwidthInBytesPerSecond = bw.CurrentBandwidthInBytesPerSecond + stats.ReplicationStats.Stats[arn] = st + } + } + } + stats.Uptime = UTCNow().Unix() - globalBootTime.Unix() + + if err := enc.Encode(stats); err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } +} + +// ResetBucketReplicationStartHandler - starts a replication reset for all objects in a bucket which +// qualify for replication and re-sync the object(s) to target, provided ExistingObjectReplication is +// enabled for the qualifying rule. This API is a MinIO only extension provided for situations where +// remote target is entirely lost,and previously replicated objects need to be re-synced. If resync is +// already in progress it returns an error +func (api objectAPIHandlers) ResetBucketReplicationStartHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "ResetBucketReplicationStart") + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + durationStr := r.URL.Query().Get("older-than") + arn := r.URL.Query().Get("arn") + resetID := r.URL.Query().Get("reset-id") + if resetID == "" { + resetID = mustGetUUID() + } + var ( + days time.Duration + err error + ) + if durationStr != "" { + days, err = time.ParseDuration(durationStr) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, InvalidArgument{ + Bucket: bucket, + Err: fmt.Errorf("invalid query parameter older-than %s for %s : %w", durationStr, bucket, err), + }), r.URL) + return + } + } + resetBeforeDate := UTCNow().AddDate(0, 0, -1*int(days/24)) + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.ResetBucketReplicationStateAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Check if bucket exists. + if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + hasARN, hasExistingObjEnabled := config.HasExistingObjectReplication(arn) + if !hasARN { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrRemoteTargetNotFoundError), r.URL) + return + } + + if !hasExistingObjEnabled { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNoExistingObjects), r.URL) + return + } + + tgtArns := config.FilterTargetArns( + replication.ObjectOpts{ + OpType: replication.ResyncReplicationType, + TargetArn: arn, + }) + + if len(tgtArns) == 0 { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{ + Bucket: bucket, + Err: fmt.Errorf("Remote target ARN %s missing or ineligible for replication resync", arn), + }), r.URL) + return + } + + if len(tgtArns) > 1 && arn == "" { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{ + Bucket: bucket, + Err: fmt.Errorf("ARN should be specified for replication reset"), + }), r.URL) + return + } + var rinfo ResyncTargetsInfo + target := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, bucket, tgtArns[0]) + target.ResetBeforeDate = UTCNow().AddDate(0, 0, -1*int(days/24)) + target.ResetID = resetID + rinfo.Targets = append(rinfo.Targets, ResyncTarget{Arn: tgtArns[0], ResetID: target.ResetID}) + if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target, true); err != nil { + switch err.(type) { + case RemoteTargetConnectionErr: + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationRemoteConnectionError, err), r.URL) + default: + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + } + return + } + targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket) + if err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + tgtBytes, err := json.Marshal(&targets) + if err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL) + return + } + if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil { + writeErrorResponseJSON(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if err := globalReplicationPool.Get().resyncer.start(ctx, objectAPI, resyncOpts{ + bucket: bucket, + arn: arn, + resyncID: resetID, + resyncBefore: resetBeforeDate, + }); err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{ + Bucket: bucket, + Err: err, + }), r.URL) + return + } + + data, err := json.Marshal(rinfo) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + // Write success response. + writeSuccessResponseJSON(w, data) +} + +// ResetBucketReplicationStatusHandler - returns the status of replication reset. +// This API is a MinIO only extension +func (api objectAPIHandlers) ResetBucketReplicationStatusHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "ResetBucketReplicationStatus") + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + arn := r.URL.Query().Get("arn") + var err error + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.ResetBucketReplicationStateAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Check if bucket exists. + if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if _, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + brs, err := loadBucketResyncMetadata(ctx, bucket, objectAPI) + if err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrBadRequest, InvalidArgument{ + Bucket: bucket, + Err: fmt.Errorf("replication resync status not available for %s (%s)", arn, err.Error()), + }), r.URL) + return + } + + var rinfo ResyncTargetsInfo + for tarn, st := range brs.TargetsMap { + if arn != "" && tarn != arn { + continue + } + rinfo.Targets = append(rinfo.Targets, ResyncTarget{ + Arn: tarn, + ResetID: st.ResyncID, + StartTime: st.StartTime, + EndTime: st.LastUpdate, + ResyncStatus: st.ResyncStatus.String(), + ReplicatedSize: st.ReplicatedSize, + ReplicatedCount: st.ReplicatedCount, + FailedSize: st.FailedSize, + FailedCount: st.FailedCount, + Bucket: st.Bucket, + Object: st.Object, + }) + } + data, err := json.Marshal(rinfo) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Write success response. + writeSuccessResponseJSON(w, data) +} + +// ValidateBucketReplicationCredsHandler - validate replication credentials for a bucket. +// ---------- +func (api objectAPIHandlers) ValidateBucketReplicationCredsHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "ValidateBucketReplicationCreds") + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + if s3Error := checkRequestAuthType(ctx, r, policy.GetReplicationConfigurationAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + // Check if bucket exists. + if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, err), r.URL) + return + } + + if versioned := globalBucketVersioningSys.Enabled(bucket); !versioned { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationNeedsVersioningError), r.URL) + return + } + replicationConfig, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket) + if err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationConfigurationNotFoundError, err), r.URL) + return + } + + lockEnabled := false + lcfg, _, err := globalBucketMetadataSys.GetObjectLockConfig(bucket) + if err != nil { + if !errors.Is(err, BucketObjectLockConfigNotFound{Bucket: bucket}) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, err), r.URL) + return + } + } + if lcfg != nil { + lockEnabled = lcfg.Enabled() + } + + sameTarget, apiErr := validateReplicationDestination(ctx, bucket, replicationConfig, &validateReplicationDestinationOptions{CheckRemoteBucket: true}) + if apiErr != noError { + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + + // Validate the bucket replication config + if err = replicationConfig.Validate(bucket, sameTarget); err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, err), r.URL) + return + } + buf := bytes.Repeat([]byte("a"), 8) + for _, rule := range replicationConfig.Rules { + if rule.Status == replication.Disabled { + continue + } + clnt := globalBucketTargetSys.GetRemoteTargetClient(bucket, rule.Destination.Bucket) + if clnt == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrRemoteTargetNotFoundError, fmt.Errorf("replication config with rule ID %s has a stale target", rule.ID)), r.URL) + return + } + if lockEnabled { + lock, _, _, _, err := clnt.GetObjectLockConfig(ctx, clnt.Bucket) + if err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, err), r.URL) + return + } + if lock != objectlock.Enabled { + writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationDestinationMissingLock, fmt.Errorf("target bucket %s is not object lock enabled", clnt.Bucket)), r.URL) + return + } + } + vcfg, err := clnt.GetBucketVersioning(ctx, clnt.Bucket) + if err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, err), r.URL) + return + } + if !vcfg.Enabled() { + writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrRemoteTargetNotVersionedError, fmt.Errorf("target bucket %s is not versioned", clnt.Bucket)), r.URL) + return + } + if sameTarget && bucket == clnt.Bucket { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrBucketRemoteIdenticalToSource), r.URL) + return + } + + reader := bytes.NewReader(buf) + // fake a PutObject and RemoveObject call to validate permissions + c := &minio.Core{Client: clnt.Client} + putOpts := minio.PutObjectOptions{ + Internal: minio.AdvancedPutOptions{ + SourceVersionID: mustGetUUID(), + ReplicationStatus: minio.ReplicationStatusReplica, + SourceMTime: time.Now(), + ReplicationRequest: true, // always set this to distinguish between `mc mirror` replication and serverside + ReplicationValidityCheck: true, // set this to validate the replication config + }, + } + obj := path.Join(minioReservedBucket, globalLocalNodeNameHex, "deleteme") + ui, err := c.PutObject(ctx, clnt.Bucket, obj, reader, int64(len(buf)), "", "", putOpts) + if err != nil && !isReplicationPermissionCheck(ErrorRespToObjectError(err, bucket, obj)) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, fmt.Errorf("s3:ReplicateObject permissions missing for replication user: %w", err)), r.URL) + return + } + + err = c.RemoveObject(ctx, clnt.Bucket, obj, minio.RemoveObjectOptions{ + VersionID: ui.VersionID, + Internal: minio.AdvancedRemoveOptions{ + ReplicationDeleteMarker: true, + ReplicationMTime: time.Now(), + ReplicationStatus: minio.ReplicationStatusReplica, + ReplicationRequest: true, // always set this to distinguish between `mc mirror` replication and serverside + ReplicationValidityCheck: true, // set this to validate the replication config + }, + }) + if err != nil && !isReplicationPermissionCheck(ErrorRespToObjectError(err, bucket, obj)) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, fmt.Errorf("s3:ReplicateDelete permissions missing for replication user: %w", err)), r.URL) + return + } + // fake a versioned delete - to ensure deny policies are not in place + err = c.RemoveObject(ctx, clnt.Bucket, obj, minio.RemoveObjectOptions{ + VersionID: ui.VersionID, + Internal: minio.AdvancedRemoveOptions{ + ReplicationDeleteMarker: false, + ReplicationMTime: time.Now(), + ReplicationStatus: minio.ReplicationStatusReplica, + ReplicationRequest: true, // always set this to distinguish between `mc mirror` replication and serverside + ReplicationValidityCheck: true, // set this to validate the replication config + }, + }) + if err != nil && !isReplicationPermissionCheck(ErrorRespToObjectError(err, bucket, obj)) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErrWithErr(ErrReplicationValidationError, fmt.Errorf("s3:ReplicateDelete/s3:DeleteObject permissions missing for replication user: %w", err)), r.URL) + return + } + } + + // Write success response. + writeSuccessResponseHeadersOnly(w) +} diff --git a/cmd/bucket-replication-metrics.go b/cmd/bucket-replication-metrics.go new file mode 100644 index 00000000..aa4cfb96 --- /dev/null +++ b/cmd/bucket-replication-metrics.go @@ -0,0 +1,523 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "sync" + "sync/atomic" + "time" + + "github.com/rcrowley/go-metrics" +) + +//go:generate msgp -file $GOFILE + +const ( + // beta is the weight used to calculate exponential moving average + beta = 0.1 // Number of averages considered = 1/(1-beta) +) + +// rateMeasurement captures the transfer details for one bucket/target +//msgp:ignore rateMeasurement + +type rateMeasurement struct { + lock sync.Mutex + bytesSinceLastWindow uint64 // Total bytes since last window was processed + startTime time.Time // Start time for window + expMovingAvg float64 // Previously calculated exponential moving average +} + +// newRateMeasurement creates a new instance of the measurement with the initial start time. +func newRateMeasurement(initTime time.Time) *rateMeasurement { + return &rateMeasurement{ + startTime: initTime, + } +} + +// incrementBytes add bytes reported for a bucket/target. +func (m *rateMeasurement) incrementBytes(bytes uint64) { + atomic.AddUint64(&m.bytesSinceLastWindow, bytes) +} + +// updateExponentialMovingAverage processes the measurements captured so far. +func (m *rateMeasurement) updateExponentialMovingAverage(endTime time.Time) { + // Calculate aggregate avg bandwidth and exp window avg + m.lock.Lock() + defer func() { + m.startTime = endTime + m.lock.Unlock() + }() + + if m.startTime.IsZero() { + return + } + + if endTime.Before(m.startTime) { + return + } + + duration := endTime.Sub(m.startTime) + + bytesSinceLastWindow := atomic.SwapUint64(&m.bytesSinceLastWindow, 0) + + if m.expMovingAvg == 0 { + // Should address initial calculation and should be fine for resuming from 0 + m.expMovingAvg = float64(bytesSinceLastWindow) / duration.Seconds() + return + } + + increment := float64(bytesSinceLastWindow) / duration.Seconds() + m.expMovingAvg = exponentialMovingAverage(beta, m.expMovingAvg, increment) +} + +// exponentialMovingAverage calculates the exponential moving average +func exponentialMovingAverage(beta, previousAvg, incrementAvg float64) float64 { + return (1-beta)*incrementAvg + beta*previousAvg +} + +// getExpMovingAvgBytesPerSecond returns the exponential moving average for the bucket/target in bytes +func (m *rateMeasurement) getExpMovingAvgBytesPerSecond() float64 { + m.lock.Lock() + defer m.lock.Unlock() + return m.expMovingAvg +} + +// ActiveWorkerStat is stat for active replication workers +type ActiveWorkerStat struct { + Curr int `json:"curr"` + Avg float32 `json:"avg"` + Max int `json:"max"` + hist metrics.Histogram +} + +func newActiveWorkerStat(r metrics.Registry) *ActiveWorkerStat { + h := metrics.NewHistogram(metrics.NewUniformSample(100)) + r.Register("replication.active_workers", h) + return &ActiveWorkerStat{ + hist: h, + } +} + +// update curr and max workers; +func (a *ActiveWorkerStat) update() { + if a == nil { + return + } + a.Curr = globalReplicationPool.Get().ActiveWorkers() + a.hist.Update(int64(a.Curr)) + a.Avg = float32(a.hist.Mean()) + a.Max = int(a.hist.Max()) +} + +func (a *ActiveWorkerStat) get() ActiveWorkerStat { + w := ActiveWorkerStat{ + Curr: a.Curr, + Avg: a.Avg, + Max: a.Max, + } + return w +} + +// QStat holds queue stats for replication +type QStat struct { + Count float64 `json:"count"` + Bytes float64 `json:"bytes"` +} + +func (q *QStat) add(o QStat) QStat { + return QStat{Bytes: q.Bytes + o.Bytes, Count: q.Count + o.Count} +} + +// InQueueMetric holds queue stats for replication +type InQueueMetric struct { + Curr QStat `json:"curr" msg:"cq"` + Avg QStat `json:"avg" msg:"aq"` + Max QStat `json:"max" msg:"pq"` +} + +func (qm InQueueMetric) merge(o InQueueMetric) InQueueMetric { + return InQueueMetric{ + Curr: qm.Curr.add(o.Curr), + Avg: qm.Avg.add(o.Avg), + Max: qm.Max.add(o.Max), + } +} + +type queueCache struct { + srQueueStats InQueueStats + bucketStats map[string]InQueueStats + sync.RWMutex // mutex for queue stats +} + +func newQueueCache(r metrics.Registry) queueCache { + return queueCache{ + bucketStats: make(map[string]InQueueStats), + srQueueStats: newInQueueStats(r, "site"), + } +} + +func (q *queueCache) update() { + q.Lock() + defer q.Unlock() + q.srQueueStats.update() + for _, s := range q.bucketStats { + s.update() + } +} + +func (q *queueCache) getBucketStats(bucket string) InQueueMetric { + q.RLock() + defer q.RUnlock() + v, ok := q.bucketStats[bucket] + if !ok { + return InQueueMetric{} + } + return InQueueMetric{ + Curr: QStat{Bytes: float64(v.nowBytes), Count: float64(v.nowCount)}, + Max: QStat{Bytes: float64(v.histBytes.Max()), Count: float64(v.histCounts.Max())}, + Avg: QStat{Bytes: v.histBytes.Mean(), Count: v.histCounts.Mean()}, + } +} + +func (q *queueCache) getSiteStats() InQueueMetric { + q.RLock() + defer q.RUnlock() + v := q.srQueueStats + return InQueueMetric{ + Curr: QStat{Bytes: float64(v.nowBytes), Count: float64(v.nowCount)}, + Max: QStat{Bytes: float64(v.histBytes.Max()), Count: float64(v.histCounts.Max())}, + Avg: QStat{Bytes: v.histBytes.Mean(), Count: v.histCounts.Mean()}, + } +} + +// InQueueStats holds queue stats for replication +type InQueueStats struct { + nowBytes int64 `json:"-"` + nowCount int64 `json:"-"` + histCounts metrics.Histogram + histBytes metrics.Histogram +} + +func newInQueueStats(r metrics.Registry, lbl string) InQueueStats { + histCounts := metrics.NewHistogram(metrics.NewUniformSample(100)) + histBytes := metrics.NewHistogram(metrics.NewUniformSample(100)) + + r.Register("replication.queue.counts."+lbl, histCounts) + r.Register("replication.queue.bytes."+lbl, histBytes) + + return InQueueStats{ + histCounts: histCounts, + histBytes: histBytes, + } +} + +func (q *InQueueStats) update() { + q.histBytes.Update(atomic.LoadInt64(&q.nowBytes)) + q.histCounts.Update(atomic.LoadInt64(&q.nowCount)) +} + +// XferStats has transfer stats for replication +type XferStats struct { + Curr float64 `json:"currRate" msg:"cr"` + Avg float64 `json:"avgRate" msg:"av"` + Peak float64 `json:"peakRate" msg:"p"` + N int64 `json:"n" msg:"n"` + measure *rateMeasurement `json:"-"` + sma *SMA `json:"-"` +} + +// Clone returns a copy of XferStats +func (rx *XferStats) Clone() *XferStats { + curr := rx.curr() + peak := rx.Peak + if curr > peak { + peak = curr + } + return &XferStats{ + Curr: curr, + Avg: rx.Avg, + Peak: peak, + N: rx.N, + measure: rx.measure, + } +} + +func newXferStats() *XferStats { + return &XferStats{ + measure: newRateMeasurement(time.Now()), + sma: newSMA(50), + } +} + +func (rx *XferStats) String() string { + return fmt.Sprintf("curr=%f avg=%f, peak=%f", rx.curr(), rx.Avg, rx.Peak) +} + +func (rx *XferStats) curr() float64 { + if rx.measure == nil { + return 0.0 + } + return rx.measure.getExpMovingAvgBytesPerSecond() +} + +func (rx *XferStats) merge(o XferStats) XferStats { + curr := calcAvg(rx.curr(), o.curr(), rx.N, o.N) + peak := rx.Peak + if o.Peak > peak { + peak = o.Peak + } + if curr > peak { + peak = curr + } + return XferStats{ + Avg: calcAvg(rx.Avg, o.Avg, rx.N, o.N), + Peak: peak, + Curr: curr, + measure: rx.measure, + N: rx.N + o.N, + } +} + +func calcAvg(x, y float64, n1, n2 int64) float64 { + if n1+n2 == 0 { + return 0 + } + avg := (x*float64(n1) + y*float64(n2)) / float64(n1+n2) + return avg +} + +// Add a new transfer +func (rx *XferStats) addSize(sz int64, t time.Duration) { + if rx.measure == nil { + rx.measure = newRateMeasurement(time.Now()) + } + rx.measure.incrementBytes(uint64(sz)) + rx.Curr = rx.measure.getExpMovingAvgBytesPerSecond() + rx.sma.addSample(rx.Curr) + rx.Avg = rx.sma.simpleMovingAvg() + if rx.Curr > rx.Peak { + rx.Peak = rx.Curr + } + rx.N++ +} + +// ReplicationMRFStats holds stats of MRF backlog saved to disk in the last 5 minutes +// and number of entries that failed replication after 3 retries +type ReplicationMRFStats struct { + LastFailedCount uint64 `json:"failedCount_last5min"` + // Count of unreplicated entries that were dropped after MRF retry limit reached since cluster start. + TotalDroppedCount uint64 `json:"droppedCount_since_uptime"` + // Bytes of unreplicated entries that were dropped after MRF retry limit reached since cluster start. + TotalDroppedBytes uint64 `json:"droppedBytes_since_uptime"` +} + +// SMA struct for calculating simple moving average +type SMA struct { + buf []float64 + window int // len of buf + idx int // current index in buf + CAvg float64 // cumulative average + prevSMA float64 + filledBuf bool +} + +func newSMA(ln int) *SMA { + if ln <= 0 { + ln = defaultWindowSize + } + return &SMA{ + buf: make([]float64, ln), + window: ln, + idx: 0, + } +} + +func (s *SMA) addSample(next float64) { + prev := s.buf[s.idx] + s.buf[s.idx] = next + + if s.filledBuf { + s.prevSMA += (next - prev) / float64(s.window) + s.CAvg += (next - s.CAvg) / float64(s.window) + } else { + s.CAvg = s.simpleMovingAvg() + s.prevSMA = s.CAvg + } + if s.idx == s.window-1 { + s.filledBuf = true + } + s.idx = (s.idx + 1) % s.window +} + +func (s *SMA) simpleMovingAvg() float64 { + if s.filledBuf { + return s.prevSMA + } + var tot float64 + for _, r := range s.buf { + tot += r + } + return tot / float64(s.idx+1) +} + +const ( + defaultWindowSize = 10 +) + +type proxyStatsCache struct { + srProxyStats ProxyMetric + bucketStats map[string]ProxyMetric + sync.RWMutex // mutex for proxy stats +} + +func newProxyStatsCache() proxyStatsCache { + return proxyStatsCache{ + bucketStats: make(map[string]ProxyMetric), + } +} + +func (p *proxyStatsCache) inc(bucket string, api replProxyAPI, isErr bool) { + p.Lock() + defer p.Unlock() + v, ok := p.bucketStats[bucket] + if !ok { + v = ProxyMetric{} + } + switch api { + case putObjectTaggingAPI: + if !isErr { + atomic.AddUint64(&v.PutTagTotal, 1) + atomic.AddUint64(&p.srProxyStats.PutTagTotal, 1) + } else { + atomic.AddUint64(&v.PutTagFailedTotal, 1) + atomic.AddUint64(&p.srProxyStats.PutTagFailedTotal, 1) + } + case getObjectTaggingAPI: + if !isErr { + atomic.AddUint64(&v.GetTagTotal, 1) + atomic.AddUint64(&p.srProxyStats.GetTagTotal, 1) + } else { + atomic.AddUint64(&v.GetTagFailedTotal, 1) + atomic.AddUint64(&p.srProxyStats.GetTagFailedTotal, 1) + } + case removeObjectTaggingAPI: + if !isErr { + atomic.AddUint64(&v.RmvTagTotal, 1) + atomic.AddUint64(&p.srProxyStats.RmvTagTotal, 1) + } else { + atomic.AddUint64(&v.RmvTagFailedTotal, 1) + atomic.AddUint64(&p.srProxyStats.RmvTagFailedTotal, 1) + } + case headObjectAPI: + if !isErr { + atomic.AddUint64(&v.HeadTotal, 1) + atomic.AddUint64(&p.srProxyStats.HeadTotal, 1) + } else { + atomic.AddUint64(&v.HeadFailedTotal, 1) + atomic.AddUint64(&p.srProxyStats.HeadFailedTotal, 1) + } + case getObjectAPI: + if !isErr { + atomic.AddUint64(&v.GetTotal, 1) + atomic.AddUint64(&p.srProxyStats.GetTotal, 1) + } else { + atomic.AddUint64(&v.GetFailedTotal, 1) + atomic.AddUint64(&p.srProxyStats.GetFailedTotal, 1) + } + default: + return + } + p.bucketStats[bucket] = v +} + +func (p *proxyStatsCache) getBucketStats(bucket string) ProxyMetric { + p.RLock() + defer p.RUnlock() + v, ok := p.bucketStats[bucket] + + if !ok { + return ProxyMetric{} + } + return ProxyMetric{ + PutTagTotal: atomic.LoadUint64(&v.PutTagTotal), + GetTagTotal: atomic.LoadUint64(&v.GetTagTotal), + RmvTagTotal: atomic.LoadUint64(&v.RmvTagTotal), + HeadTotal: atomic.LoadUint64(&v.HeadTotal), + GetTotal: atomic.LoadUint64(&v.GetTotal), + + PutTagFailedTotal: atomic.LoadUint64(&v.PutTagFailedTotal), + GetTagFailedTotal: atomic.LoadUint64(&v.GetTagFailedTotal), + RmvTagFailedTotal: atomic.LoadUint64(&v.RmvTagFailedTotal), + HeadFailedTotal: atomic.LoadUint64(&v.HeadFailedTotal), + GetFailedTotal: atomic.LoadUint64(&v.GetFailedTotal), + } +} + +func (p *proxyStatsCache) getSiteStats() ProxyMetric { + v := p.srProxyStats + return ProxyMetric{ + PutTagTotal: atomic.LoadUint64(&v.PutTagTotal), + GetTagTotal: atomic.LoadUint64(&v.GetTagTotal), + RmvTagTotal: atomic.LoadUint64(&v.RmvTagTotal), + HeadTotal: atomic.LoadUint64(&v.HeadTotal), + GetTotal: atomic.LoadUint64(&v.GetTotal), + PutTagFailedTotal: atomic.LoadUint64(&v.PutTagFailedTotal), + GetTagFailedTotal: atomic.LoadUint64(&v.GetTagFailedTotal), + RmvTagFailedTotal: atomic.LoadUint64(&v.RmvTagFailedTotal), + HeadFailedTotal: atomic.LoadUint64(&v.HeadFailedTotal), + GetFailedTotal: atomic.LoadUint64(&v.GetFailedTotal), + } +} + +type replProxyAPI string + +const ( + putObjectTaggingAPI replProxyAPI = "PutObjectTagging" + getObjectTaggingAPI replProxyAPI = "GetObjectTagging" + removeObjectTaggingAPI replProxyAPI = "RemoveObjectTagging" + headObjectAPI replProxyAPI = "HeadObject" + getObjectAPI replProxyAPI = "GetObject" +) + +// ProxyMetric holds stats for replication proxying +type ProxyMetric struct { + PutTagTotal uint64 `json:"putTaggingProxyTotal" msg:"ptc"` + GetTagTotal uint64 `json:"getTaggingProxyTotal" msg:"gtc"` + RmvTagTotal uint64 `json:"removeTaggingProxyTotal" msg:"rtc"` + GetTotal uint64 `json:"getProxyTotal" msg:"gc"` + HeadTotal uint64 `json:"headProxyTotal" msg:"hc"` + PutTagFailedTotal uint64 `json:"putTaggingProxyFailed" msg:"ptf"` + GetTagFailedTotal uint64 `json:"getTaggingProxyFailed" msg:"gtf"` + RmvTagFailedTotal uint64 `json:"removeTaggingProxyFailed" msg:"rtf"` + GetFailedTotal uint64 `json:"getProxyFailed" msg:"gf"` + HeadFailedTotal uint64 `json:"headProxyFailed" msg:"hf"` +} + +func (p *ProxyMetric) add(p2 ProxyMetric) { + atomic.AddUint64(&p.GetTotal, p2.GetTotal) + atomic.AddUint64(&p.HeadTotal, p2.HeadTotal) + atomic.AddUint64(&p.GetTagTotal, p2.GetTagTotal) + atomic.AddUint64(&p.PutTagTotal, p2.PutTagTotal) + atomic.AddUint64(&p.RmvTagTotal, p2.RmvTagTotal) + atomic.AddUint64(&p.GetFailedTotal, p2.GetFailedTotal) + atomic.AddUint64(&p.HeadFailedTotal, p2.HeadFailedTotal) + atomic.AddUint64(&p.GetTagFailedTotal, p2.GetTagFailedTotal) + atomic.AddUint64(&p.PutTagFailedTotal, p2.PutTagFailedTotal) + atomic.AddUint64(&p.RmvTagFailedTotal, p2.RmvTagFailedTotal) +} diff --git a/cmd/bucket-replication-metrics_gen.go b/cmd/bucket-replication-metrics_gen.go new file mode 100644 index 00000000..f8eba540 --- /dev/null +++ b/cmd/bucket-replication-metrics_gen.go @@ -0,0 +1,1528 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *ActiveWorkerStat) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Curr": + z.Curr, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Curr") + return + } + case "Avg": + z.Avg, err = dc.ReadFloat32() + if err != nil { + err = msgp.WrapError(err, "Avg") + return + } + case "Max": + z.Max, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Max") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z ActiveWorkerStat) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "Curr" + err = en.Append(0x83, 0xa4, 0x43, 0x75, 0x72, 0x72) + if err != nil { + return + } + err = en.WriteInt(z.Curr) + if err != nil { + err = msgp.WrapError(err, "Curr") + return + } + // write "Avg" + err = en.Append(0xa3, 0x41, 0x76, 0x67) + if err != nil { + return + } + err = en.WriteFloat32(z.Avg) + if err != nil { + err = msgp.WrapError(err, "Avg") + return + } + // write "Max" + err = en.Append(0xa3, 0x4d, 0x61, 0x78) + if err != nil { + return + } + err = en.WriteInt(z.Max) + if err != nil { + err = msgp.WrapError(err, "Max") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z ActiveWorkerStat) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "Curr" + o = append(o, 0x83, 0xa4, 0x43, 0x75, 0x72, 0x72) + o = msgp.AppendInt(o, z.Curr) + // string "Avg" + o = append(o, 0xa3, 0x41, 0x76, 0x67) + o = msgp.AppendFloat32(o, z.Avg) + // string "Max" + o = append(o, 0xa3, 0x4d, 0x61, 0x78) + o = msgp.AppendInt(o, z.Max) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ActiveWorkerStat) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Curr": + z.Curr, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Curr") + return + } + case "Avg": + z.Avg, bts, err = msgp.ReadFloat32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Avg") + return + } + case "Max": + z.Max, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Max") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z ActiveWorkerStat) Msgsize() (s int) { + s = 1 + 5 + msgp.IntSize + 4 + msgp.Float32Size + 4 + msgp.IntSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *InQueueMetric) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "cq": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Curr") + return + } + for zb0002 > 0 { + zb0002-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "Curr") + return + } + switch msgp.UnsafeString(field) { + case "Count": + z.Curr.Count, err = dc.ReadFloat64() + if err != nil { + err = msgp.WrapError(err, "Curr", "Count") + return + } + case "Bytes": + z.Curr.Bytes, err = dc.ReadFloat64() + if err != nil { + err = msgp.WrapError(err, "Curr", "Bytes") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "Curr") + return + } + } + } + case "aq": + var zb0003 uint32 + zb0003, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Avg") + return + } + for zb0003 > 0 { + zb0003-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "Avg") + return + } + switch msgp.UnsafeString(field) { + case "Count": + z.Avg.Count, err = dc.ReadFloat64() + if err != nil { + err = msgp.WrapError(err, "Avg", "Count") + return + } + case "Bytes": + z.Avg.Bytes, err = dc.ReadFloat64() + if err != nil { + err = msgp.WrapError(err, "Avg", "Bytes") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "Avg") + return + } + } + } + case "pq": + var zb0004 uint32 + zb0004, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Max") + return + } + for zb0004 > 0 { + zb0004-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "Max") + return + } + switch msgp.UnsafeString(field) { + case "Count": + z.Max.Count, err = dc.ReadFloat64() + if err != nil { + err = msgp.WrapError(err, "Max", "Count") + return + } + case "Bytes": + z.Max.Bytes, err = dc.ReadFloat64() + if err != nil { + err = msgp.WrapError(err, "Max", "Bytes") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "Max") + return + } + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *InQueueMetric) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "cq" + err = en.Append(0x83, 0xa2, 0x63, 0x71) + if err != nil { + return + } + // map header, size 2 + // write "Count" + err = en.Append(0x82, 0xa5, 0x43, 0x6f, 0x75, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteFloat64(z.Curr.Count) + if err != nil { + err = msgp.WrapError(err, "Curr", "Count") + return + } + // write "Bytes" + err = en.Append(0xa5, 0x42, 0x79, 0x74, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteFloat64(z.Curr.Bytes) + if err != nil { + err = msgp.WrapError(err, "Curr", "Bytes") + return + } + // write "aq" + err = en.Append(0xa2, 0x61, 0x71) + if err != nil { + return + } + // map header, size 2 + // write "Count" + err = en.Append(0x82, 0xa5, 0x43, 0x6f, 0x75, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteFloat64(z.Avg.Count) + if err != nil { + err = msgp.WrapError(err, "Avg", "Count") + return + } + // write "Bytes" + err = en.Append(0xa5, 0x42, 0x79, 0x74, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteFloat64(z.Avg.Bytes) + if err != nil { + err = msgp.WrapError(err, "Avg", "Bytes") + return + } + // write "pq" + err = en.Append(0xa2, 0x70, 0x71) + if err != nil { + return + } + // map header, size 2 + // write "Count" + err = en.Append(0x82, 0xa5, 0x43, 0x6f, 0x75, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteFloat64(z.Max.Count) + if err != nil { + err = msgp.WrapError(err, "Max", "Count") + return + } + // write "Bytes" + err = en.Append(0xa5, 0x42, 0x79, 0x74, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteFloat64(z.Max.Bytes) + if err != nil { + err = msgp.WrapError(err, "Max", "Bytes") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *InQueueMetric) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "cq" + o = append(o, 0x83, 0xa2, 0x63, 0x71) + // map header, size 2 + // string "Count" + o = append(o, 0x82, 0xa5, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendFloat64(o, z.Curr.Count) + // string "Bytes" + o = append(o, 0xa5, 0x42, 0x79, 0x74, 0x65, 0x73) + o = msgp.AppendFloat64(o, z.Curr.Bytes) + // string "aq" + o = append(o, 0xa2, 0x61, 0x71) + // map header, size 2 + // string "Count" + o = append(o, 0x82, 0xa5, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendFloat64(o, z.Avg.Count) + // string "Bytes" + o = append(o, 0xa5, 0x42, 0x79, 0x74, 0x65, 0x73) + o = msgp.AppendFloat64(o, z.Avg.Bytes) + // string "pq" + o = append(o, 0xa2, 0x70, 0x71) + // map header, size 2 + // string "Count" + o = append(o, 0x82, 0xa5, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendFloat64(o, z.Max.Count) + // string "Bytes" + o = append(o, 0xa5, 0x42, 0x79, 0x74, 0x65, 0x73) + o = msgp.AppendFloat64(o, z.Max.Bytes) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *InQueueMetric) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "cq": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Curr") + return + } + for zb0002 > 0 { + zb0002-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "Curr") + return + } + switch msgp.UnsafeString(field) { + case "Count": + z.Curr.Count, bts, err = msgp.ReadFloat64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Curr", "Count") + return + } + case "Bytes": + z.Curr.Bytes, bts, err = msgp.ReadFloat64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Curr", "Bytes") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "Curr") + return + } + } + } + case "aq": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Avg") + return + } + for zb0003 > 0 { + zb0003-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "Avg") + return + } + switch msgp.UnsafeString(field) { + case "Count": + z.Avg.Count, bts, err = msgp.ReadFloat64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Avg", "Count") + return + } + case "Bytes": + z.Avg.Bytes, bts, err = msgp.ReadFloat64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Avg", "Bytes") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "Avg") + return + } + } + } + case "pq": + var zb0004 uint32 + zb0004, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Max") + return + } + for zb0004 > 0 { + zb0004-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "Max") + return + } + switch msgp.UnsafeString(field) { + case "Count": + z.Max.Count, bts, err = msgp.ReadFloat64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Max", "Count") + return + } + case "Bytes": + z.Max.Bytes, bts, err = msgp.ReadFloat64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Max", "Bytes") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "Max") + return + } + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *InQueueMetric) Msgsize() (s int) { + s = 1 + 3 + 1 + 6 + msgp.Float64Size + 6 + msgp.Float64Size + 3 + 1 + 6 + msgp.Float64Size + 6 + msgp.Float64Size + 3 + 1 + 6 + msgp.Float64Size + 6 + msgp.Float64Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *InQueueStats) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z InQueueStats) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 0 + _ = z + err = en.Append(0x80) + if err != nil { + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z InQueueStats) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 0 + _ = z + o = append(o, 0x80) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *InQueueStats) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z InQueueStats) Msgsize() (s int) { + s = 1 + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ProxyMetric) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ptc": + z.PutTagTotal, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "PutTagTotal") + return + } + case "gtc": + z.GetTagTotal, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "GetTagTotal") + return + } + case "rtc": + z.RmvTagTotal, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "RmvTagTotal") + return + } + case "gc": + z.GetTotal, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "GetTotal") + return + } + case "hc": + z.HeadTotal, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "HeadTotal") + return + } + case "ptf": + z.PutTagFailedTotal, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "PutTagFailedTotal") + return + } + case "gtf": + z.GetTagFailedTotal, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "GetTagFailedTotal") + return + } + case "rtf": + z.RmvTagFailedTotal, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "RmvTagFailedTotal") + return + } + case "gf": + z.GetFailedTotal, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "GetFailedTotal") + return + } + case "hf": + z.HeadFailedTotal, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "HeadFailedTotal") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *ProxyMetric) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 10 + // write "ptc" + err = en.Append(0x8a, 0xa3, 0x70, 0x74, 0x63) + if err != nil { + return + } + err = en.WriteUint64(z.PutTagTotal) + if err != nil { + err = msgp.WrapError(err, "PutTagTotal") + return + } + // write "gtc" + err = en.Append(0xa3, 0x67, 0x74, 0x63) + if err != nil { + return + } + err = en.WriteUint64(z.GetTagTotal) + if err != nil { + err = msgp.WrapError(err, "GetTagTotal") + return + } + // write "rtc" + err = en.Append(0xa3, 0x72, 0x74, 0x63) + if err != nil { + return + } + err = en.WriteUint64(z.RmvTagTotal) + if err != nil { + err = msgp.WrapError(err, "RmvTagTotal") + return + } + // write "gc" + err = en.Append(0xa2, 0x67, 0x63) + if err != nil { + return + } + err = en.WriteUint64(z.GetTotal) + if err != nil { + err = msgp.WrapError(err, "GetTotal") + return + } + // write "hc" + err = en.Append(0xa2, 0x68, 0x63) + if err != nil { + return + } + err = en.WriteUint64(z.HeadTotal) + if err != nil { + err = msgp.WrapError(err, "HeadTotal") + return + } + // write "ptf" + err = en.Append(0xa3, 0x70, 0x74, 0x66) + if err != nil { + return + } + err = en.WriteUint64(z.PutTagFailedTotal) + if err != nil { + err = msgp.WrapError(err, "PutTagFailedTotal") + return + } + // write "gtf" + err = en.Append(0xa3, 0x67, 0x74, 0x66) + if err != nil { + return + } + err = en.WriteUint64(z.GetTagFailedTotal) + if err != nil { + err = msgp.WrapError(err, "GetTagFailedTotal") + return + } + // write "rtf" + err = en.Append(0xa3, 0x72, 0x74, 0x66) + if err != nil { + return + } + err = en.WriteUint64(z.RmvTagFailedTotal) + if err != nil { + err = msgp.WrapError(err, "RmvTagFailedTotal") + return + } + // write "gf" + err = en.Append(0xa2, 0x67, 0x66) + if err != nil { + return + } + err = en.WriteUint64(z.GetFailedTotal) + if err != nil { + err = msgp.WrapError(err, "GetFailedTotal") + return + } + // write "hf" + err = en.Append(0xa2, 0x68, 0x66) + if err != nil { + return + } + err = en.WriteUint64(z.HeadFailedTotal) + if err != nil { + err = msgp.WrapError(err, "HeadFailedTotal") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ProxyMetric) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 10 + // string "ptc" + o = append(o, 0x8a, 0xa3, 0x70, 0x74, 0x63) + o = msgp.AppendUint64(o, z.PutTagTotal) + // string "gtc" + o = append(o, 0xa3, 0x67, 0x74, 0x63) + o = msgp.AppendUint64(o, z.GetTagTotal) + // string "rtc" + o = append(o, 0xa3, 0x72, 0x74, 0x63) + o = msgp.AppendUint64(o, z.RmvTagTotal) + // string "gc" + o = append(o, 0xa2, 0x67, 0x63) + o = msgp.AppendUint64(o, z.GetTotal) + // string "hc" + o = append(o, 0xa2, 0x68, 0x63) + o = msgp.AppendUint64(o, z.HeadTotal) + // string "ptf" + o = append(o, 0xa3, 0x70, 0x74, 0x66) + o = msgp.AppendUint64(o, z.PutTagFailedTotal) + // string "gtf" + o = append(o, 0xa3, 0x67, 0x74, 0x66) + o = msgp.AppendUint64(o, z.GetTagFailedTotal) + // string "rtf" + o = append(o, 0xa3, 0x72, 0x74, 0x66) + o = msgp.AppendUint64(o, z.RmvTagFailedTotal) + // string "gf" + o = append(o, 0xa2, 0x67, 0x66) + o = msgp.AppendUint64(o, z.GetFailedTotal) + // string "hf" + o = append(o, 0xa2, 0x68, 0x66) + o = msgp.AppendUint64(o, z.HeadFailedTotal) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ProxyMetric) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ptc": + z.PutTagTotal, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "PutTagTotal") + return + } + case "gtc": + z.GetTagTotal, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "GetTagTotal") + return + } + case "rtc": + z.RmvTagTotal, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "RmvTagTotal") + return + } + case "gc": + z.GetTotal, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "GetTotal") + return + } + case "hc": + z.HeadTotal, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "HeadTotal") + return + } + case "ptf": + z.PutTagFailedTotal, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "PutTagFailedTotal") + return + } + case "gtf": + z.GetTagFailedTotal, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "GetTagFailedTotal") + return + } + case "rtf": + z.RmvTagFailedTotal, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "RmvTagFailedTotal") + return + } + case "gf": + z.GetFailedTotal, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "GetFailedTotal") + return + } + case "hf": + z.HeadFailedTotal, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "HeadFailedTotal") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ProxyMetric) Msgsize() (s int) { + s = 1 + 4 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.Uint64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.Uint64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *QStat) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Count": + z.Count, err = dc.ReadFloat64() + if err != nil { + err = msgp.WrapError(err, "Count") + return + } + case "Bytes": + z.Bytes, err = dc.ReadFloat64() + if err != nil { + err = msgp.WrapError(err, "Bytes") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z QStat) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "Count" + err = en.Append(0x82, 0xa5, 0x43, 0x6f, 0x75, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteFloat64(z.Count) + if err != nil { + err = msgp.WrapError(err, "Count") + return + } + // write "Bytes" + err = en.Append(0xa5, 0x42, 0x79, 0x74, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteFloat64(z.Bytes) + if err != nil { + err = msgp.WrapError(err, "Bytes") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z QStat) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "Count" + o = append(o, 0x82, 0xa5, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendFloat64(o, z.Count) + // string "Bytes" + o = append(o, 0xa5, 0x42, 0x79, 0x74, 0x65, 0x73) + o = msgp.AppendFloat64(o, z.Bytes) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *QStat) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Count": + z.Count, bts, err = msgp.ReadFloat64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Count") + return + } + case "Bytes": + z.Bytes, bts, err = msgp.ReadFloat64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bytes") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z QStat) Msgsize() (s int) { + s = 1 + 6 + msgp.Float64Size + 6 + msgp.Float64Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ReplicationMRFStats) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "LastFailedCount": + z.LastFailedCount, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "LastFailedCount") + return + } + case "TotalDroppedCount": + z.TotalDroppedCount, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "TotalDroppedCount") + return + } + case "TotalDroppedBytes": + z.TotalDroppedBytes, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "TotalDroppedBytes") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z ReplicationMRFStats) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "LastFailedCount" + err = en.Append(0x83, 0xaf, 0x4c, 0x61, 0x73, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteUint64(z.LastFailedCount) + if err != nil { + err = msgp.WrapError(err, "LastFailedCount") + return + } + // write "TotalDroppedCount" + err = en.Append(0xb1, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x44, 0x72, 0x6f, 0x70, 0x70, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteUint64(z.TotalDroppedCount) + if err != nil { + err = msgp.WrapError(err, "TotalDroppedCount") + return + } + // write "TotalDroppedBytes" + err = en.Append(0xb1, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x44, 0x72, 0x6f, 0x70, 0x70, 0x65, 0x64, 0x42, 0x79, 0x74, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteUint64(z.TotalDroppedBytes) + if err != nil { + err = msgp.WrapError(err, "TotalDroppedBytes") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z ReplicationMRFStats) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "LastFailedCount" + o = append(o, 0x83, 0xaf, 0x4c, 0x61, 0x73, 0x74, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendUint64(o, z.LastFailedCount) + // string "TotalDroppedCount" + o = append(o, 0xb1, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x44, 0x72, 0x6f, 0x70, 0x70, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendUint64(o, z.TotalDroppedCount) + // string "TotalDroppedBytes" + o = append(o, 0xb1, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x44, 0x72, 0x6f, 0x70, 0x70, 0x65, 0x64, 0x42, 0x79, 0x74, 0x65, 0x73) + o = msgp.AppendUint64(o, z.TotalDroppedBytes) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ReplicationMRFStats) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "LastFailedCount": + z.LastFailedCount, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "LastFailedCount") + return + } + case "TotalDroppedCount": + z.TotalDroppedCount, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalDroppedCount") + return + } + case "TotalDroppedBytes": + z.TotalDroppedBytes, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalDroppedBytes") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z ReplicationMRFStats) Msgsize() (s int) { + s = 1 + 16 + msgp.Uint64Size + 18 + msgp.Uint64Size + 18 + msgp.Uint64Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *SMA) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "CAvg": + z.CAvg, err = dc.ReadFloat64() + if err != nil { + err = msgp.WrapError(err, "CAvg") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z SMA) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "CAvg" + err = en.Append(0x81, 0xa4, 0x43, 0x41, 0x76, 0x67) + if err != nil { + return + } + err = en.WriteFloat64(z.CAvg) + if err != nil { + err = msgp.WrapError(err, "CAvg") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z SMA) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "CAvg" + o = append(o, 0x81, 0xa4, 0x43, 0x41, 0x76, 0x67) + o = msgp.AppendFloat64(o, z.CAvg) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *SMA) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "CAvg": + z.CAvg, bts, err = msgp.ReadFloat64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "CAvg") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z SMA) Msgsize() (s int) { + s = 1 + 5 + msgp.Float64Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *XferStats) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "cr": + z.Curr, err = dc.ReadFloat64() + if err != nil { + err = msgp.WrapError(err, "Curr") + return + } + case "av": + z.Avg, err = dc.ReadFloat64() + if err != nil { + err = msgp.WrapError(err, "Avg") + return + } + case "p": + z.Peak, err = dc.ReadFloat64() + if err != nil { + err = msgp.WrapError(err, "Peak") + return + } + case "n": + z.N, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "N") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *XferStats) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 4 + // write "cr" + err = en.Append(0x84, 0xa2, 0x63, 0x72) + if err != nil { + return + } + err = en.WriteFloat64(z.Curr) + if err != nil { + err = msgp.WrapError(err, "Curr") + return + } + // write "av" + err = en.Append(0xa2, 0x61, 0x76) + if err != nil { + return + } + err = en.WriteFloat64(z.Avg) + if err != nil { + err = msgp.WrapError(err, "Avg") + return + } + // write "p" + err = en.Append(0xa1, 0x70) + if err != nil { + return + } + err = en.WriteFloat64(z.Peak) + if err != nil { + err = msgp.WrapError(err, "Peak") + return + } + // write "n" + err = en.Append(0xa1, 0x6e) + if err != nil { + return + } + err = en.WriteInt64(z.N) + if err != nil { + err = msgp.WrapError(err, "N") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *XferStats) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 4 + // string "cr" + o = append(o, 0x84, 0xa2, 0x63, 0x72) + o = msgp.AppendFloat64(o, z.Curr) + // string "av" + o = append(o, 0xa2, 0x61, 0x76) + o = msgp.AppendFloat64(o, z.Avg) + // string "p" + o = append(o, 0xa1, 0x70) + o = msgp.AppendFloat64(o, z.Peak) + // string "n" + o = append(o, 0xa1, 0x6e) + o = msgp.AppendInt64(o, z.N) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *XferStats) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "cr": + z.Curr, bts, err = msgp.ReadFloat64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Curr") + return + } + case "av": + z.Avg, bts, err = msgp.ReadFloat64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Avg") + return + } + case "p": + z.Peak, bts, err = msgp.ReadFloat64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Peak") + return + } + case "n": + z.N, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "N") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *XferStats) Msgsize() (s int) { + s = 1 + 3 + msgp.Float64Size + 3 + msgp.Float64Size + 2 + msgp.Float64Size + 2 + msgp.Int64Size + return +} diff --git a/cmd/bucket-replication-metrics_gen_test.go b/cmd/bucket-replication-metrics_gen_test.go new file mode 100644 index 00000000..7342472f --- /dev/null +++ b/cmd/bucket-replication-metrics_gen_test.go @@ -0,0 +1,914 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalActiveWorkerStat(t *testing.T) { + v := ActiveWorkerStat{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgActiveWorkerStat(b *testing.B) { + v := ActiveWorkerStat{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgActiveWorkerStat(b *testing.B) { + v := ActiveWorkerStat{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalActiveWorkerStat(b *testing.B) { + v := ActiveWorkerStat{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeActiveWorkerStat(t *testing.T) { + v := ActiveWorkerStat{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeActiveWorkerStat Msgsize() is inaccurate") + } + + vn := ActiveWorkerStat{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeActiveWorkerStat(b *testing.B) { + v := ActiveWorkerStat{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeActiveWorkerStat(b *testing.B) { + v := ActiveWorkerStat{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalInQueueMetric(t *testing.T) { + v := InQueueMetric{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgInQueueMetric(b *testing.B) { + v := InQueueMetric{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgInQueueMetric(b *testing.B) { + v := InQueueMetric{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalInQueueMetric(b *testing.B) { + v := InQueueMetric{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeInQueueMetric(t *testing.T) { + v := InQueueMetric{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeInQueueMetric Msgsize() is inaccurate") + } + + vn := InQueueMetric{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeInQueueMetric(b *testing.B) { + v := InQueueMetric{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeInQueueMetric(b *testing.B) { + v := InQueueMetric{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalInQueueStats(t *testing.T) { + v := InQueueStats{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgInQueueStats(b *testing.B) { + v := InQueueStats{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgInQueueStats(b *testing.B) { + v := InQueueStats{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalInQueueStats(b *testing.B) { + v := InQueueStats{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeInQueueStats(t *testing.T) { + v := InQueueStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeInQueueStats Msgsize() is inaccurate") + } + + vn := InQueueStats{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeInQueueStats(b *testing.B) { + v := InQueueStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeInQueueStats(b *testing.B) { + v := InQueueStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalProxyMetric(t *testing.T) { + v := ProxyMetric{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgProxyMetric(b *testing.B) { + v := ProxyMetric{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgProxyMetric(b *testing.B) { + v := ProxyMetric{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalProxyMetric(b *testing.B) { + v := ProxyMetric{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeProxyMetric(t *testing.T) { + v := ProxyMetric{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeProxyMetric Msgsize() is inaccurate") + } + + vn := ProxyMetric{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeProxyMetric(b *testing.B) { + v := ProxyMetric{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeProxyMetric(b *testing.B) { + v := ProxyMetric{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalQStat(t *testing.T) { + v := QStat{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgQStat(b *testing.B) { + v := QStat{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgQStat(b *testing.B) { + v := QStat{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalQStat(b *testing.B) { + v := QStat{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeQStat(t *testing.T) { + v := QStat{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeQStat Msgsize() is inaccurate") + } + + vn := QStat{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeQStat(b *testing.B) { + v := QStat{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeQStat(b *testing.B) { + v := QStat{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalReplicationMRFStats(t *testing.T) { + v := ReplicationMRFStats{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgReplicationMRFStats(b *testing.B) { + v := ReplicationMRFStats{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgReplicationMRFStats(b *testing.B) { + v := ReplicationMRFStats{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalReplicationMRFStats(b *testing.B) { + v := ReplicationMRFStats{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeReplicationMRFStats(t *testing.T) { + v := ReplicationMRFStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeReplicationMRFStats Msgsize() is inaccurate") + } + + vn := ReplicationMRFStats{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeReplicationMRFStats(b *testing.B) { + v := ReplicationMRFStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeReplicationMRFStats(b *testing.B) { + v := ReplicationMRFStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalSMA(t *testing.T) { + v := SMA{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgSMA(b *testing.B) { + v := SMA{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgSMA(b *testing.B) { + v := SMA{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalSMA(b *testing.B) { + v := SMA{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeSMA(t *testing.T) { + v := SMA{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeSMA Msgsize() is inaccurate") + } + + vn := SMA{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeSMA(b *testing.B) { + v := SMA{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeSMA(b *testing.B) { + v := SMA{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalXferStats(t *testing.T) { + v := XferStats{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgXferStats(b *testing.B) { + v := XferStats{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgXferStats(b *testing.B) { + v := XferStats{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalXferStats(b *testing.B) { + v := XferStats{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeXferStats(t *testing.T) { + v := XferStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeXferStats Msgsize() is inaccurate") + } + + vn := XferStats{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeXferStats(b *testing.B) { + v := XferStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeXferStats(b *testing.B) { + v := XferStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/bucket-replication-stats.go b/cmd/bucket-replication-stats.go new file mode 100644 index 00000000..29717928 --- /dev/null +++ b/cmd/bucket-replication-stats.go @@ -0,0 +1,516 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "sync" + "sync/atomic" + "time" + + "github.com/minio/minio/internal/bucket/replication" + "github.com/rcrowley/go-metrics" +) + +func (b *BucketReplicationStats) hasReplicationUsage() bool { + for _, s := range b.Stats { + if s.hasReplicationUsage() { + return true + } + } + return false +} + +// ReplicationStats holds the global in-memory replication stats +type ReplicationStats struct { + // map of site deployment ID to site replication status + // for site replication - maintain stats at global level + srStats *SRStats + // active worker stats + workers *ActiveWorkerStat + // queue stats cache + qCache queueCache + + pCache proxyStatsCache + // mrf backlog stats + mrfStats ReplicationMRFStats + // for bucket replication, continue to use existing cache + Cache map[string]*BucketReplicationStats + mostRecentStats BucketStatsMap + registry metrics.Registry + sync.RWMutex // mutex for Cache + mostRecentStatsMu sync.Mutex // mutex for mostRecentStats + + wlock sync.RWMutex // mutex for active workers + + movingAvgTicker *time.Ticker // Ticker for calculating moving averages + wTimer *time.Ticker // ticker for calculating active workers + qTimer *time.Ticker // ticker for calculating queue stats +} + +func (r *ReplicationStats) trackEWMA() { + for { + select { + case <-r.movingAvgTicker.C: + r.updateMovingAvg() + case <-GlobalContext.Done(): + return + } + } +} + +func (r *ReplicationStats) updateMovingAvg() { + r.RLock() + for _, s := range r.Cache { + for _, st := range s.Stats { + st.XferRateLrg.measure.updateExponentialMovingAverage(time.Now()) + st.XferRateSml.measure.updateExponentialMovingAverage(time.Now()) + } + } + r.RUnlock() +} + +// ActiveWorkers returns worker stats +func (r *ReplicationStats) ActiveWorkers() ActiveWorkerStat { + if r == nil { + return ActiveWorkerStat{} + } + r.wlock.RLock() + defer r.wlock.RUnlock() + w := r.workers.get() + return ActiveWorkerStat{ + Curr: w.Curr, + Max: w.Max, + Avg: w.Avg, + } +} + +func (r *ReplicationStats) collectWorkerMetrics(ctx context.Context) { + if r == nil { + return + } + for { + select { + case <-ctx.Done(): + return + case <-r.wTimer.C: + r.wlock.Lock() + r.workers.update() + r.wlock.Unlock() + } + } +} + +func (r *ReplicationStats) collectQueueMetrics(ctx context.Context) { + if r == nil { + return + } + + for { + select { + case <-ctx.Done(): + return + case <-r.qTimer.C: + r.qCache.update() + } + } +} + +// Delete deletes in-memory replication statistics for a bucket. +func (r *ReplicationStats) Delete(bucket string) { + if r == nil { + return + } + + r.Lock() + defer r.Unlock() + delete(r.Cache, bucket) +} + +// UpdateReplicaStat updates in-memory replica statistics with new values. +func (r *ReplicationStats) UpdateReplicaStat(bucket string, n int64) { + if r == nil { + return + } + + r.Lock() + defer r.Unlock() + bs, ok := r.Cache[bucket] + if !ok { + bs = newBucketReplicationStats() + } + bs.ReplicaSize += n + bs.ReplicaCount++ + r.Cache[bucket] = bs + r.srUpdateReplicaStat(n) +} + +func (r *ReplicationStats) srUpdateReplicaStat(sz int64) { + if r == nil { + return + } + atomic.AddInt64(&r.srStats.ReplicaSize, sz) + atomic.AddInt64(&r.srStats.ReplicaCount, 1) +} + +func (r *ReplicationStats) srUpdate(sr replStat) { + dID, err := globalSiteReplicationSys.getDeplIDForEndpoint(sr.endpoint()) + if err == nil { + r.srStats.update(sr, dID) + } +} + +// Update updates in-memory replication statistics with new values. +func (r *ReplicationStats) Update(bucket string, ri replicatedTargetInfo, status, prevStatus replication.StatusType) { + if r == nil { + return + } + var rs replStat + switch status { + case replication.Pending: + if ri.OpType.IsDataReplication() && prevStatus != status { + rs.set(ri.Arn, ri.Size, 0, status, ri.OpType, ri.endpoint, ri.secure, ri.Err) + } + case replication.Completed: + if ri.OpType.IsDataReplication() { + rs.set(ri.Arn, ri.Size, ri.Duration, status, ri.OpType, ri.endpoint, ri.secure, ri.Err) + } + case replication.Failed: + if ri.OpType.IsDataReplication() && prevStatus == replication.Pending { + rs.set(ri.Arn, ri.Size, ri.Duration, status, ri.OpType, ri.endpoint, ri.secure, ri.Err) + } + case replication.Replica: + if ri.OpType == replication.ObjectReplicationType { + rs.set(ri.Arn, ri.Size, 0, status, ri.OpType, "", false, ri.Err) + } + } + + // update site-replication in-memory stats + if rs.Completed || rs.Failed { + r.srUpdate(rs) + } + + r.Lock() + defer r.Unlock() + + // update bucket replication in-memory stats + bs, ok := r.Cache[bucket] + if !ok { + bs = newBucketReplicationStats() + r.Cache[bucket] = bs + } + b, ok := bs.Stats[ri.Arn] + if !ok { + b = &BucketReplicationStat{ + XferRateLrg: newXferStats(), + XferRateSml: newXferStats(), + } + bs.Stats[ri.Arn] = b + } + + switch { + case rs.Completed: + b.ReplicatedSize += rs.TransferSize + b.ReplicatedCount++ + if rs.TransferDuration > 0 { + b.Latency.update(rs.TransferSize, rs.TransferDuration) + b.updateXferRate(rs.TransferSize, rs.TransferDuration) + } + case rs.Failed: + b.FailStats.addsize(rs.TransferSize, rs.Err) + case rs.Pending: + } +} + +type replStat struct { + Arn string + Completed bool + Pending bool + Failed bool + opType replication.Type + // transfer size + TransferSize int64 + // transfer duration + TransferDuration time.Duration + Endpoint string + Secure bool + Err error +} + +func (rs *replStat) endpoint() string { + scheme := "http" + if rs.Secure { + scheme = "https" + } + return scheme + "://" + rs.Endpoint +} + +func (rs *replStat) set(arn string, n int64, duration time.Duration, status replication.StatusType, opType replication.Type, endpoint string, secure bool, err error) { + rs.Endpoint = endpoint + rs.Secure = secure + rs.TransferSize = n + rs.Arn = arn + rs.TransferDuration = duration + rs.opType = opType + switch status { + case replication.Completed: + rs.Completed = true + case replication.Pending: + rs.Pending = true + case replication.Failed: + rs.Failed = true + rs.Err = err + } +} + +// GetAll returns replication metrics for all buckets at once. +func (r *ReplicationStats) GetAll() map[string]BucketReplicationStats { + if r == nil { + return map[string]BucketReplicationStats{} + } + + r.RLock() + + bucketReplicationStats := make(map[string]BucketReplicationStats, len(r.Cache)) + for k, v := range r.Cache { + bucketReplicationStats[k] = v.Clone() + } + r.RUnlock() + for k, v := range bucketReplicationStats { + v.QStat = r.qCache.getBucketStats(k) + bucketReplicationStats[k] = v + } + + return bucketReplicationStats +} + +func (r *ReplicationStats) getSRMetricsForNode() SRMetricsSummary { + if r == nil { + return SRMetricsSummary{} + } + + m := SRMetricsSummary{ + Uptime: UTCNow().Unix() - globalBootTime.Unix(), + Queued: r.qCache.getSiteStats(), + ActiveWorkers: r.ActiveWorkers(), + Metrics: r.srStats.get(), + Proxied: r.pCache.getSiteStats(), + ReplicaSize: atomic.LoadInt64(&r.srStats.ReplicaSize), + ReplicaCount: atomic.LoadInt64(&r.srStats.ReplicaCount), + } + return m +} + +// Get replication metrics for a bucket from this node since this node came up. +func (r *ReplicationStats) Get(bucket string) BucketReplicationStats { + if r == nil { + return BucketReplicationStats{Stats: make(map[string]*BucketReplicationStat)} + } + + r.RLock() + defer r.RUnlock() + + st, ok := r.Cache[bucket] + if !ok { + return BucketReplicationStats{Stats: make(map[string]*BucketReplicationStat)} + } + return st.Clone() +} + +// NewReplicationStats initialize in-memory replication statistics +func NewReplicationStats(ctx context.Context, objectAPI ObjectLayer) *ReplicationStats { + r := metrics.NewRegistry() + rs := ReplicationStats{ + Cache: make(map[string]*BucketReplicationStats), + qCache: newQueueCache(r), + pCache: newProxyStatsCache(), + srStats: newSRStats(), + movingAvgTicker: time.NewTicker(2 * time.Second), + wTimer: time.NewTicker(2 * time.Second), + qTimer: time.NewTicker(2 * time.Second), + + workers: newActiveWorkerStat(r), + registry: r, + } + go rs.collectWorkerMetrics(ctx) + go rs.collectQueueMetrics(ctx) + return &rs +} + +func (r *ReplicationStats) getAllLatest(bucketsUsage map[string]BucketUsageInfo) (bucketsReplicationStats map[string]BucketStats) { + if r == nil { + return nil + } + peerBucketStatsList := globalNotificationSys.GetClusterAllBucketStats(GlobalContext) + bucketsReplicationStats = make(map[string]BucketStats, len(bucketsUsage)) + + for bucket := range bucketsUsage { + bucketStats := make([]BucketStats, len(peerBucketStatsList)) + for i, peerBucketStats := range peerBucketStatsList { + bucketStat, ok := peerBucketStats.Stats[bucket] + if !ok { + continue + } + bucketStats[i] = bucketStat + } + bucketsReplicationStats[bucket] = r.calculateBucketReplicationStats(bucket, bucketStats) + } + return bucketsReplicationStats +} + +func (r *ReplicationStats) calculateBucketReplicationStats(bucket string, bucketStats []BucketStats) (bs BucketStats) { + if r == nil { + bs = BucketStats{ + ReplicationStats: BucketReplicationStats{ + Stats: make(map[string]*BucketReplicationStat), + }, + QueueStats: ReplicationQueueStats{}, + ProxyStats: ProxyMetric{}, + } + return bs + } + var s BucketReplicationStats + // accumulate cluster bucket stats + stats := make(map[string]*BucketReplicationStat) + var ( + totReplicaSize, totReplicatedSize int64 + totReplicaCount, totReplicatedCount int64 + totFailed RTimedMetrics + tq InQueueMetric + ) + for _, bucketStat := range bucketStats { + totReplicaSize += bucketStat.ReplicationStats.ReplicaSize + totReplicaCount += bucketStat.ReplicationStats.ReplicaCount + for _, q := range bucketStat.QueueStats.Nodes { + tq = tq.merge(q.QStats) + } + + for arn, stat := range bucketStat.ReplicationStats.Stats { + oldst := stats[arn] + if oldst == nil { + oldst = &BucketReplicationStat{ + XferRateLrg: newXferStats(), + XferRateSml: newXferStats(), + } + } + fstats := stat.FailStats.merge(oldst.FailStats) + lrg := oldst.XferRateLrg.merge(*stat.XferRateLrg) + sml := oldst.XferRateSml.merge(*stat.XferRateSml) + stats[arn] = &BucketReplicationStat{ + Failed: fstats.toMetric(), + FailStats: fstats, + ReplicatedSize: stat.ReplicatedSize + oldst.ReplicatedSize, + ReplicatedCount: stat.ReplicatedCount + oldst.ReplicatedCount, + Latency: stat.Latency.merge(oldst.Latency), + XferRateLrg: &lrg, + XferRateSml: &sml, + } + totReplicatedSize += stat.ReplicatedSize + totReplicatedCount += stat.ReplicatedCount + totFailed = totFailed.merge(stat.FailStats) + } + } + + s = BucketReplicationStats{ + Stats: stats, + QStat: tq, + ReplicaSize: totReplicaSize, + ReplicaCount: totReplicaCount, + ReplicatedSize: totReplicatedSize, + ReplicatedCount: totReplicatedCount, + Failed: totFailed.toMetric(), + } + + var qs ReplicationQueueStats + for _, bs := range bucketStats { + qs.Nodes = append(qs.Nodes, bs.QueueStats.Nodes...) + } + qs.Uptime = UTCNow().Unix() - globalBootTime.Unix() + + var ps ProxyMetric + for _, bs := range bucketStats { + ps.add(bs.ProxyStats) + } + bs = BucketStats{ + ReplicationStats: s, + QueueStats: qs, + ProxyStats: ps, + } + r.mostRecentStatsMu.Lock() + if len(r.mostRecentStats.Stats) == 0 { + r.mostRecentStats = BucketStatsMap{Stats: make(map[string]BucketStats, 1), Timestamp: UTCNow()} + } + if len(bs.ReplicationStats.Stats) > 0 { + r.mostRecentStats.Stats[bucket] = bs + } + r.mostRecentStats.Timestamp = UTCNow() + r.mostRecentStatsMu.Unlock() + return bs +} + +// get the most current of in-memory replication stats and data usage info from crawler. +func (r *ReplicationStats) getLatestReplicationStats(bucket string) (s BucketStats) { + if r == nil { + return s + } + bucketStats := globalNotificationSys.GetClusterBucketStats(GlobalContext, bucket) + return r.calculateBucketReplicationStats(bucket, bucketStats) +} + +func (r *ReplicationStats) incQ(bucket string, sz int64, isDeleteRepl bool, opType replication.Type) { + r.qCache.Lock() + defer r.qCache.Unlock() + v, ok := r.qCache.bucketStats[bucket] + if !ok { + v = newInQueueStats(r.registry, bucket) + } + atomic.AddInt64(&v.nowBytes, sz) + atomic.AddInt64(&v.nowCount, 1) + r.qCache.bucketStats[bucket] = v + atomic.AddInt64(&r.qCache.srQueueStats.nowBytes, sz) + atomic.AddInt64(&r.qCache.srQueueStats.nowCount, 1) +} + +func (r *ReplicationStats) decQ(bucket string, sz int64, isDelMarker bool, opType replication.Type) { + r.qCache.Lock() + defer r.qCache.Unlock() + v, ok := r.qCache.bucketStats[bucket] + if !ok { + v = newInQueueStats(r.registry, bucket) + } + atomic.AddInt64(&v.nowBytes, -1*sz) + atomic.AddInt64(&v.nowCount, -1) + r.qCache.bucketStats[bucket] = v + + atomic.AddInt64(&r.qCache.srQueueStats.nowBytes, -1*sz) + atomic.AddInt64(&r.qCache.srQueueStats.nowCount, -1) +} + +// incProxy increments proxy metrics for proxied calls +func (r *ReplicationStats) incProxy(bucket string, api replProxyAPI, isErr bool) { + if r != nil { + r.pCache.inc(bucket, api, isErr) + } +} + +func (r *ReplicationStats) getProxyStats(bucket string) ProxyMetric { + if r == nil { + return ProxyMetric{} + } + return r.pCache.getBucketStats(bucket) +} diff --git a/cmd/bucket-replication-utils.go b/cmd/bucket-replication-utils.go new file mode 100644 index 00000000..41b5dbc6 --- /dev/null +++ b/cmd/bucket-replication-utils.go @@ -0,0 +1,811 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "fmt" + "maps" + "net/http" + "net/url" + "regexp" + "strconv" + "strings" + "sync" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/crypto" + xhttp "github.com/minio/minio/internal/http" +) + +//go:generate msgp -file=$GOFILE + +// replicatedTargetInfo struct represents replication info on a target +type replicatedTargetInfo struct { + Arn string + Size int64 + Duration time.Duration + ReplicationAction replicationAction // full or metadata only + OpType replication.Type // whether incoming replication, existing object, healing etc.. + ReplicationStatus replication.StatusType + PrevReplicationStatus replication.StatusType + VersionPurgeStatus VersionPurgeStatusType + ResyncTimestamp string + ReplicationResynced bool // true only if resync attempted for this target + endpoint string + secure bool + Err error // replication error if any +} + +// Empty returns true for a target if arn is empty +func (rt replicatedTargetInfo) Empty() bool { + return rt.Arn == "" +} + +type replicatedInfos struct { + ReplicationTimeStamp time.Time + Targets []replicatedTargetInfo +} + +func (ri replicatedInfos) CompletedSize() (sz int64) { + for _, t := range ri.Targets { + if t.Empty() { + continue + } + if t.ReplicationStatus == replication.Completed && t.PrevReplicationStatus != replication.Completed { + sz += t.Size + } + } + return sz +} + +// ReplicationAttempted returns true if replication was attempted on any of the targets for the object version +// queued +func (ri replicatedInfos) ReplicationResynced() bool { + for _, t := range ri.Targets { + if t.Empty() || !t.ReplicationResynced { + continue + } + return true + } + return false +} + +func (ri replicatedInfos) ReplicationStatusInternal() string { + b := new(bytes.Buffer) + for _, t := range ri.Targets { + if t.Empty() { + continue + } + fmt.Fprintf(b, "%s=%s;", t.Arn, t.ReplicationStatus.String()) + } + return b.String() +} + +func (ri replicatedInfos) ReplicationStatus() replication.StatusType { + if len(ri.Targets) == 0 { + return replication.StatusType("") + } + completed := 0 + for _, v := range ri.Targets { + switch v.ReplicationStatus { + case replication.Failed: + return replication.Failed + case replication.Completed: + completed++ + } + } + if completed == len(ri.Targets) { + return replication.Completed + } + return replication.Pending +} + +func (ri replicatedInfos) VersionPurgeStatus() VersionPurgeStatusType { + if len(ri.Targets) == 0 { + return VersionPurgeStatusType("") + } + completed := 0 + for _, v := range ri.Targets { + switch v.VersionPurgeStatus { + case replication.VersionPurgeFailed: + return replication.VersionPurgeFailed + case replication.VersionPurgeComplete: + completed++ + } + } + if completed == len(ri.Targets) { + return replication.VersionPurgeComplete + } + return replication.VersionPurgePending +} + +func (ri replicatedInfos) VersionPurgeStatusInternal() string { + b := new(bytes.Buffer) + for _, t := range ri.Targets { + if t.Empty() { + continue + } + if t.VersionPurgeStatus.Empty() { + continue + } + fmt.Fprintf(b, "%s=%s;", t.Arn, t.VersionPurgeStatus) + } + return b.String() +} + +func (ri replicatedInfos) Action() replicationAction { + for _, t := range ri.Targets { + if t.Empty() { + continue + } + // rely on replication action from target that actually performed replication now. + if t.PrevReplicationStatus != replication.Completed { + return t.ReplicationAction + } + } + return replicateNone +} + +var replStatusRegex = regexp.MustCompile(`([^=].*?)=([^,].*?);`) + +// TargetReplicationStatus - returns replication status of a target +func (ri ReplicateObjectInfo) TargetReplicationStatus(arn string) (status replication.StatusType) { + repStatMatches := replStatusRegex.FindAllStringSubmatch(ri.ReplicationStatusInternal, -1) + for _, repStatMatch := range repStatMatches { + if len(repStatMatch) != 3 { + return status + } + if repStatMatch[1] == arn { + return replication.StatusType(repStatMatch[2]) + } + } + return status +} + +// TargetReplicationStatus - returns replication status of a target +func (o ObjectInfo) TargetReplicationStatus(arn string) (status replication.StatusType) { + repStatMatches := replStatusRegex.FindAllStringSubmatch(o.ReplicationStatusInternal, -1) + for _, repStatMatch := range repStatMatches { + if len(repStatMatch) != 3 { + return status + } + if repStatMatch[1] == arn { + return replication.StatusType(repStatMatch[2]) + } + } + return status +} + +type replicateTargetDecision struct { + Replicate bool // Replicate to this target + Synchronous bool // Synchronous replication configured. + Arn string // ARN of replication target + ID string +} + +func (t *replicateTargetDecision) String() string { + return fmt.Sprintf("%t;%t;%s;%s", t.Replicate, t.Synchronous, t.Arn, t.ID) +} + +func newReplicateTargetDecision(arn string, replicate bool, sync bool) replicateTargetDecision { + d := replicateTargetDecision{ + Replicate: replicate, + Synchronous: sync, + Arn: arn, + } + return d +} + +// ReplicateDecision represents replication decision for each target +type ReplicateDecision struct { + targetsMap map[string]replicateTargetDecision +} + +// ReplicateAny returns true if at least one target qualifies for replication +func (d ReplicateDecision) ReplicateAny() bool { + for _, t := range d.targetsMap { + if t.Replicate { + return true + } + } + return false +} + +// Synchronous returns true if at least one target qualifies for synchronous replication +func (d ReplicateDecision) Synchronous() bool { + for _, t := range d.targetsMap { + if t.Synchronous { + return true + } + } + return false +} + +func (d ReplicateDecision) String() string { + b := new(bytes.Buffer) + for key, value := range d.targetsMap { + fmt.Fprintf(b, "%s=%s,", key, value.String()) + } + return strings.TrimSuffix(b.String(), ",") +} + +// Set updates ReplicateDecision with target's replication decision +func (d *ReplicateDecision) Set(t replicateTargetDecision) { + if d.targetsMap == nil { + d.targetsMap = make(map[string]replicateTargetDecision) + } + d.targetsMap[t.Arn] = t +} + +// PendingStatus returns a stringified representation of internal replication status with all targets marked as `PENDING` +func (d ReplicateDecision) PendingStatus() string { + b := new(bytes.Buffer) + for _, k := range d.targetsMap { + if k.Replicate { + fmt.Fprintf(b, "%s=%s;", k.Arn, replication.Pending.String()) + } + } + return b.String() +} + +// ResyncDecision is a struct representing a map with target's individual resync decisions +type ResyncDecision struct { + targets map[string]ResyncTargetDecision +} + +// Empty returns true if no targets with resync decision present +func (r ResyncDecision) Empty() bool { + return r.targets == nil +} + +func (r ResyncDecision) mustResync() bool { + for _, v := range r.targets { + if v.Replicate { + return true + } + } + return false +} + +func (r ResyncDecision) mustResyncTarget(tgtArn string) bool { + if r.targets == nil { + return false + } + v, ok := r.targets[tgtArn] + return ok && v.Replicate +} + +// ResyncTargetDecision is struct that represents resync decision for this target +type ResyncTargetDecision struct { + Replicate bool + ResetID string + ResetBeforeDate time.Time +} + +var errInvalidReplicateDecisionFormat = fmt.Errorf("ReplicateDecision has invalid format") + +// parse k-v pairs of target ARN to stringified ReplicateTargetDecision delimited by ',' into a +// ReplicateDecision struct +func parseReplicateDecision(ctx context.Context, bucket, s string) (r ReplicateDecision, err error) { + r = ReplicateDecision{ + targetsMap: make(map[string]replicateTargetDecision), + } + if len(s) == 0 { + return r, err + } + for p := range strings.SplitSeq(s, ",") { + if p == "" { + continue + } + slc := strings.Split(p, "=") + if len(slc) != 2 { + return r, errInvalidReplicateDecisionFormat + } + tgtStr := strings.TrimSuffix(strings.TrimPrefix(slc[1], `"`), `"`) + tgt := strings.Split(tgtStr, ";") + if len(tgt) != 4 { + return r, errInvalidReplicateDecisionFormat + } + r.targetsMap[slc[0]] = replicateTargetDecision{Replicate: tgt[0] == "true", Synchronous: tgt[1] == "true", Arn: tgt[2], ID: tgt[3]} + } + return r, err +} + +// ReplicationState represents internal replication state +type ReplicationState struct { + ReplicaTimeStamp time.Time // timestamp when last replica update was received + ReplicaStatus replication.StatusType // replica statusstringis + DeleteMarker bool // represents DeleteMarker replication state + ReplicationTimeStamp time.Time // timestamp when last replication activity happened + ReplicationStatusInternal string // stringified representation of all replication activity + // VersionPurgeStatusInternal is internally in the format "arn1=PENDING;arn2=COMPLETED;" + VersionPurgeStatusInternal string // stringified representation of all version purge statuses + ReplicateDecisionStr string // stringified representation of replication decision for each target + Targets map[string]replication.StatusType // map of ARN->replication status for ongoing replication activity + PurgeTargets map[string]VersionPurgeStatusType // map of ARN->VersionPurgeStatus for all the targets + ResetStatusesMap map[string]string // map of ARN-> stringified reset id and timestamp for all the targets +} + +// Equal returns true if replication state is identical for version purge statuses and (replica)tion statuses. +func (rs *ReplicationState) Equal(o ReplicationState) bool { + return rs.ReplicaStatus == o.ReplicaStatus && + rs.ReplicationStatusInternal == o.ReplicationStatusInternal && + rs.VersionPurgeStatusInternal == o.VersionPurgeStatusInternal +} + +// CompositeReplicationStatus returns overall replication status for the object version being replicated. +func (rs *ReplicationState) CompositeReplicationStatus() (st replication.StatusType) { + switch { + case rs.ReplicationStatusInternal != "": + switch replication.StatusType(rs.ReplicationStatusInternal) { + case replication.Pending, replication.Completed, replication.Failed, replication.Replica: // for backward compatibility + return replication.StatusType(rs.ReplicationStatusInternal) + default: + replStatus := getCompositeReplicationStatus(rs.Targets) + // return REPLICA status if replica received timestamp is later than replication timestamp + // provided object replication completed for all targets. + if rs.ReplicaTimeStamp.Equal(timeSentinel) || rs.ReplicaTimeStamp.IsZero() { + return replStatus + } + if replStatus == replication.Completed && rs.ReplicaTimeStamp.After(rs.ReplicationTimeStamp) { + return rs.ReplicaStatus + } + return replStatus + } + case !rs.ReplicaStatus.Empty(): + return rs.ReplicaStatus + default: + return st + } +} + +// CompositeVersionPurgeStatus returns overall replication purge status for the permanent delete being replicated. +func (rs *ReplicationState) CompositeVersionPurgeStatus() VersionPurgeStatusType { + switch VersionPurgeStatusType(rs.VersionPurgeStatusInternal) { + case replication.VersionPurgePending, replication.VersionPurgeComplete, replication.VersionPurgeFailed: // for backward compatibility + return VersionPurgeStatusType(rs.VersionPurgeStatusInternal) + default: + return getCompositeVersionPurgeStatus(rs.PurgeTargets) + } +} + +// TargetState returns replicatedInfos struct initialized with the previous state of replication +func (rs *ReplicationState) targetState(arn string) (r replicatedTargetInfo) { + return replicatedTargetInfo{ + Arn: arn, + PrevReplicationStatus: rs.Targets[arn], + VersionPurgeStatus: rs.PurgeTargets[arn], + ResyncTimestamp: rs.ResetStatusesMap[arn], + } +} + +// getReplicationState returns replication state using target replicated info for the targets +func getReplicationState(rinfos replicatedInfos, prevState ReplicationState, vID string) ReplicationState { + rs := ReplicationState{ + ReplicateDecisionStr: prevState.ReplicateDecisionStr, + ResetStatusesMap: prevState.ResetStatusesMap, + ReplicaTimeStamp: prevState.ReplicaTimeStamp, + ReplicaStatus: prevState.ReplicaStatus, + } + var replStatuses, vpurgeStatuses string + replStatuses = rinfos.ReplicationStatusInternal() + rs.Targets = replicationStatusesMap(replStatuses) + rs.ReplicationStatusInternal = replStatuses + rs.ReplicationTimeStamp = rinfos.ReplicationTimeStamp + + vpurgeStatuses = rinfos.VersionPurgeStatusInternal() + rs.VersionPurgeStatusInternal = vpurgeStatuses + rs.PurgeTargets = versionPurgeStatusesMap(vpurgeStatuses) + + for _, rinfo := range rinfos.Targets { + if rinfo.ResyncTimestamp != "" { + rs.ResetStatusesMap[targetResetHeader(rinfo.Arn)] = rinfo.ResyncTimestamp + } + } + return rs +} + +// constructs a replication status map from string representation +func replicationStatusesMap(s string) map[string]replication.StatusType { + targets := make(map[string]replication.StatusType) + repStatMatches := replStatusRegex.FindAllStringSubmatch(s, -1) + for _, repStatMatch := range repStatMatches { + if len(repStatMatch) != 3 { + continue + } + status := replication.StatusType(repStatMatch[2]) + targets[repStatMatch[1]] = status + } + return targets +} + +// constructs a version purge status map from string representation +func versionPurgeStatusesMap(s string) map[string]VersionPurgeStatusType { + targets := make(map[string]VersionPurgeStatusType) + purgeStatusMatches := replStatusRegex.FindAllStringSubmatch(s, -1) + for _, purgeStatusMatch := range purgeStatusMatches { + if len(purgeStatusMatch) != 3 { + continue + } + targets[purgeStatusMatch[1]] = VersionPurgeStatusType(purgeStatusMatch[2]) + } + return targets +} + +// return the overall replication status for all the targets +func getCompositeReplicationStatus(m map[string]replication.StatusType) replication.StatusType { + if len(m) == 0 { + return replication.StatusType("") + } + completed := 0 + for _, v := range m { + switch v { + case replication.Failed: + return replication.Failed + case replication.Completed: + completed++ + } + } + if completed == len(m) { + return replication.Completed + } + return replication.Pending +} + +// return the overall version purge status for all the targets +func getCompositeVersionPurgeStatus(m map[string]VersionPurgeStatusType) VersionPurgeStatusType { + if len(m) == 0 { + return VersionPurgeStatusType("") + } + completed := 0 + for _, v := range m { + switch v { + case replication.VersionPurgeFailed: + return replication.VersionPurgeFailed + case replication.VersionPurgeComplete: + completed++ + } + } + if completed == len(m) { + return replication.VersionPurgeComplete + } + return replication.VersionPurgePending +} + +// getHealReplicateObjectInfo returns info needed by heal replication in ReplicateObjectInfo +func getHealReplicateObjectInfo(oi ObjectInfo, rcfg replicationConfig) ReplicateObjectInfo { + userDefined := cloneMSS(oi.UserDefined) + if rcfg.Config != nil && rcfg.Config.RoleArn != "" { + // For backward compatibility of objects pending/failed replication. + // Save replication related statuses in the new internal representation for + // compatible behavior. + if !oi.ReplicationStatus.Empty() { + oi.ReplicationStatusInternal = fmt.Sprintf("%s=%s;", rcfg.Config.RoleArn, oi.ReplicationStatus) + } + if !oi.VersionPurgeStatus.Empty() { + oi.VersionPurgeStatusInternal = fmt.Sprintf("%s=%s;", rcfg.Config.RoleArn, oi.VersionPurgeStatus) + } + for k, v := range userDefined { + if strings.EqualFold(k, ReservedMetadataPrefixLower+ReplicationReset) { + delete(userDefined, k) + userDefined[targetResetHeader(rcfg.Config.RoleArn)] = v + } + } + } + + var dsc ReplicateDecision + if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() { + dsc = checkReplicateDelete(GlobalContext, oi.Bucket, ObjectToDelete{ + ObjectV: ObjectV{ + ObjectName: oi.Name, + VersionID: oi.VersionID, + }, + }, oi, ObjectOptions{ + Versioned: globalBucketVersioningSys.PrefixEnabled(oi.Bucket, oi.Name), + VersionSuspended: globalBucketVersioningSys.PrefixSuspended(oi.Bucket, oi.Name), + }, nil) + } else { + dsc = mustReplicate(GlobalContext, oi.Bucket, oi.Name, getMustReplicateOptions(userDefined, oi.UserTags, "", replication.HealReplicationType, ObjectOptions{})) + } + + tgtStatuses := replicationStatusesMap(oi.ReplicationStatusInternal) + purgeStatuses := versionPurgeStatusesMap(oi.VersionPurgeStatusInternal) + existingObjResync := rcfg.Resync(GlobalContext, oi, dsc, tgtStatuses) + tm, _ := time.Parse(time.RFC3339Nano, userDefined[ReservedMetadataPrefixLower+ReplicationTimestamp]) + rstate := oi.ReplicationState() + rstate.ReplicateDecisionStr = dsc.String() + asz, _ := oi.GetActualSize() + + r := ReplicateObjectInfo{ + Name: oi.Name, + Size: oi.Size, + ActualSize: asz, + Bucket: oi.Bucket, + VersionID: oi.VersionID, + ETag: oi.ETag, + ModTime: oi.ModTime, + ReplicationStatus: oi.ReplicationStatus, + ReplicationStatusInternal: oi.ReplicationStatusInternal, + DeleteMarker: oi.DeleteMarker, + VersionPurgeStatusInternal: oi.VersionPurgeStatusInternal, + VersionPurgeStatus: oi.VersionPurgeStatus, + + ReplicationState: rstate, + OpType: replication.HealReplicationType, + Dsc: dsc, + ExistingObjResync: existingObjResync, + TargetStatuses: tgtStatuses, + TargetPurgeStatuses: purgeStatuses, + ReplicationTimestamp: tm, + SSEC: crypto.SSEC.IsEncrypted(oi.UserDefined), + UserTags: oi.UserTags, + } + if r.SSEC { + r.Checksum = oi.Checksum + } + return r +} + +// ReplicationState - returns replication state using other internal replication metadata in ObjectInfo +func (o ObjectInfo) ReplicationState() ReplicationState { + rs := ReplicationState{ + ReplicationStatusInternal: o.ReplicationStatusInternal, + VersionPurgeStatusInternal: o.VersionPurgeStatusInternal, + ReplicateDecisionStr: o.replicationDecision, + Targets: make(map[string]replication.StatusType), + PurgeTargets: make(map[string]VersionPurgeStatusType), + ResetStatusesMap: make(map[string]string), + } + rs.Targets = replicationStatusesMap(o.ReplicationStatusInternal) + rs.PurgeTargets = versionPurgeStatusesMap(o.VersionPurgeStatusInternal) + for k, v := range o.UserDefined { + if strings.HasPrefix(k, ReservedMetadataPrefixLower+ReplicationReset) { + arn := strings.TrimPrefix(k, fmt.Sprintf("%s-", ReservedMetadataPrefixLower+ReplicationReset)) + rs.ResetStatusesMap[arn] = v + } + } + return rs +} + +// ReplicationState returns replication state using other internal replication metadata in ObjectToDelete +func (o ObjectToDelete) ReplicationState() ReplicationState { + r := ReplicationState{ + ReplicationStatusInternal: o.DeleteMarkerReplicationStatus, + VersionPurgeStatusInternal: o.VersionPurgeStatuses, + ReplicateDecisionStr: o.ReplicateDecisionStr, + } + + r.Targets = replicationStatusesMap(o.DeleteMarkerReplicationStatus) + r.PurgeTargets = versionPurgeStatusesMap(o.VersionPurgeStatuses) + return r +} + +// VersionPurgeStatus returns a composite version purge status across targets +func (d *DeletedObject) VersionPurgeStatus() VersionPurgeStatusType { + return d.ReplicationState.CompositeVersionPurgeStatus() +} + +// DeleteMarkerReplicationStatus return composite replication status of delete marker across targets +func (d *DeletedObject) DeleteMarkerReplicationStatus() replication.StatusType { + return d.ReplicationState.CompositeReplicationStatus() +} + +// ResyncTargetsInfo holds a slice of targets with resync info per target +type ResyncTargetsInfo struct { + Targets []ResyncTarget `json:"target,omitempty"` +} + +// ResyncTarget is a struct representing the Target reset ID where target is identified by its Arn +type ResyncTarget struct { + Arn string `json:"arn"` + ResetID string `json:"resetid"` + StartTime time.Time `json:"startTime"` + EndTime time.Time `json:"endTime"` + // Status of resync operation + ResyncStatus string `json:"resyncStatus,omitempty"` + // Completed size in bytes + ReplicatedSize int64 `json:"completedReplicationSize"` + // Failed size in bytes + FailedSize int64 `json:"failedReplicationSize"` + // Total number of failed operations + FailedCount int64 `json:"failedReplicationCount"` + // Total number of failed operations + ReplicatedCount int64 `json:"replicationCount"` + // Last bucket/object replicated. + Bucket string `json:"bucket,omitempty"` + Object string `json:"object,omitempty"` +} + +// VersionPurgeStatusType represents status of a versioned delete or permanent delete w.r.t bucket replication +type VersionPurgeStatusType = replication.VersionPurgeStatusType + +type replicationResyncer struct { + // map of bucket to their resync status + statusMap map[string]BucketReplicationResyncStatus + workerSize int + resyncCancelCh chan struct{} + workerCh chan struct{} + sync.RWMutex +} + +const ( + replicationDir = ".replication" + resyncFileName = "resync.bin" + resyncMetaFormat = 1 + resyncMetaVersionV1 = 1 + resyncMetaVersion = resyncMetaVersionV1 +) + +type resyncOpts struct { + bucket string + arn string + resyncID string + resyncBefore time.Time +} + +// ResyncStatusType status of resync operation +type ResyncStatusType int + +const ( + // NoResync - no resync in progress + NoResync ResyncStatusType = iota + // ResyncPending - resync pending + ResyncPending + // ResyncCanceled - resync canceled + ResyncCanceled + // ResyncStarted - resync in progress + ResyncStarted + // ResyncCompleted - resync finished + ResyncCompleted + // ResyncFailed - resync failed + ResyncFailed +) + +func (rt ResyncStatusType) isValid() bool { + return rt != NoResync +} + +func (rt ResyncStatusType) String() string { + switch rt { + case ResyncStarted: + return "Ongoing" + case ResyncCompleted: + return "Completed" + case ResyncFailed: + return "Failed" + case ResyncPending: + return "Pending" + case ResyncCanceled: + return "Canceled" + default: + return "" + } +} + +// TargetReplicationResyncStatus status of resync of bucket for a specific target +type TargetReplicationResyncStatus struct { + StartTime time.Time `json:"startTime" msg:"st"` + LastUpdate time.Time `json:"lastUpdated" msg:"lst"` + // Resync ID assigned to this reset + ResyncID string `json:"resyncID" msg:"id"` + // ResyncBeforeDate - resync all objects created prior to this date + ResyncBeforeDate time.Time `json:"resyncBeforeDate" msg:"rdt"` + // Status of resync operation + ResyncStatus ResyncStatusType `json:"resyncStatus" msg:"rst"` + // Failed size in bytes + FailedSize int64 `json:"failedReplicationSize" msg:"fs"` + // Total number of failed operations + FailedCount int64 `json:"failedReplicationCount" msg:"frc"` + // Completed size in bytes + ReplicatedSize int64 `json:"completedReplicationSize" msg:"rs"` + // Total number of failed operations + ReplicatedCount int64 `json:"replicationCount" msg:"rrc"` + // Last bucket/object replicated. + Bucket string `json:"-" msg:"bkt"` + Object string `json:"-" msg:"obj"` + Error error `json:"-" msg:"-"` +} + +// BucketReplicationResyncStatus captures current replication resync status +type BucketReplicationResyncStatus struct { + Version int `json:"version" msg:"v"` + // map of remote arn to their resync status for a bucket + TargetsMap map[string]TargetReplicationResyncStatus `json:"resyncMap,omitempty" msg:"brs"` + ID int `json:"id" msg:"id"` + LastUpdate time.Time `json:"lastUpdate" msg:"lu"` +} + +func (rs *BucketReplicationResyncStatus) cloneTgtStats() (m map[string]TargetReplicationResyncStatus) { + m = make(map[string]TargetReplicationResyncStatus) + maps.Copy(m, rs.TargetsMap) + return m +} + +func newBucketResyncStatus(bucket string) BucketReplicationResyncStatus { + return BucketReplicationResyncStatus{ + TargetsMap: make(map[string]TargetReplicationResyncStatus), + Version: resyncMetaVersion, + } +} + +var contentRangeRegexp = regexp.MustCompile(`bytes ([0-9]+)-([0-9]+)/([0-9]+|\\*)`) + +// parse size from content-range header +func parseSizeFromContentRange(h http.Header) (sz int64, err error) { + cr := h.Get(xhttp.ContentRange) + if cr == "" { + return sz, fmt.Errorf("Content-Range not set") + } + parts := contentRangeRegexp.FindStringSubmatch(cr) + if len(parts) != 4 { + return sz, fmt.Errorf("invalid Content-Range header %s", cr) + } + if parts[3] == "*" { + return -1, nil + } + var usz uint64 + usz, err = strconv.ParseUint(parts[3], 10, 64) + if err != nil { + return sz, err + } + return int64(usz), nil +} + +func extractReplicateDiffOpts(q url.Values) (opts madmin.ReplDiffOpts) { + opts.Verbose = q.Get("verbose") == "true" + opts.ARN = q.Get("arn") + opts.Prefix = q.Get("prefix") + return opts +} + +const ( + replicationMRFDir = bucketMetaPrefix + SlashSeparator + replicationDir + SlashSeparator + "mrf" + mrfMetaFormat = 1 + mrfMetaVersionV1 = 1 + mrfMetaVersion = mrfMetaVersionV1 +) + +// MRFReplicateEntry mrf entry to save to disk +type MRFReplicateEntry struct { + Bucket string `json:"bucket" msg:"b"` + Object string `json:"object" msg:"o"` + versionID string `json:"-"` + RetryCount int `json:"retryCount" msg:"rc"` + sz int64 `json:"-"` +} + +// MRFReplicateEntries has the map of MRF entries to save to disk +type MRFReplicateEntries struct { + Entries map[string]MRFReplicateEntry `json:"entries" msg:"e"` + Version int `json:"version" msg:"v"` +} + +// ToMRFEntry returns the relevant info needed by MRF +func (ri ReplicateObjectInfo) ToMRFEntry() MRFReplicateEntry { + return MRFReplicateEntry{ + Bucket: ri.Bucket, + Object: ri.Name, + versionID: ri.VersionID, + sz: ri.Size, + RetryCount: int(ri.RetryCount), + } +} diff --git a/cmd/bucket-replication-utils_gen.go b/cmd/bucket-replication-utils_gen.go new file mode 100644 index 00000000..93386921 --- /dev/null +++ b/cmd/bucket-replication-utils_gen.go @@ -0,0 +1,2485 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/minio/minio/internal/bucket/replication" + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *BucketReplicationResyncStatus) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "v": + z.Version, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Version") + return + } + case "brs": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "TargetsMap") + return + } + if z.TargetsMap == nil { + z.TargetsMap = make(map[string]TargetReplicationResyncStatus, zb0002) + } else if len(z.TargetsMap) > 0 { + clear(z.TargetsMap) + } + for zb0002 > 0 { + zb0002-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "TargetsMap") + return + } + var za0002 TargetReplicationResyncStatus + err = za0002.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "TargetsMap", za0001) + return + } + z.TargetsMap[za0001] = za0002 + } + case "id": + z.ID, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + case "lu": + z.LastUpdate, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "LastUpdate") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *BucketReplicationResyncStatus) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 4 + // write "v" + err = en.Append(0x84, 0xa1, 0x76) + if err != nil { + return + } + err = en.WriteInt(z.Version) + if err != nil { + err = msgp.WrapError(err, "Version") + return + } + // write "brs" + err = en.Append(0xa3, 0x62, 0x72, 0x73) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.TargetsMap))) + if err != nil { + err = msgp.WrapError(err, "TargetsMap") + return + } + for za0001, za0002 := range z.TargetsMap { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "TargetsMap") + return + } + err = za0002.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "TargetsMap", za0001) + return + } + } + // write "id" + err = en.Append(0xa2, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteInt(z.ID) + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + // write "lu" + err = en.Append(0xa2, 0x6c, 0x75) + if err != nil { + return + } + err = en.WriteTime(z.LastUpdate) + if err != nil { + err = msgp.WrapError(err, "LastUpdate") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *BucketReplicationResyncStatus) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 4 + // string "v" + o = append(o, 0x84, 0xa1, 0x76) + o = msgp.AppendInt(o, z.Version) + // string "brs" + o = append(o, 0xa3, 0x62, 0x72, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.TargetsMap))) + for za0001, za0002 := range z.TargetsMap { + o = msgp.AppendString(o, za0001) + o, err = za0002.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "TargetsMap", za0001) + return + } + } + // string "id" + o = append(o, 0xa2, 0x69, 0x64) + o = msgp.AppendInt(o, z.ID) + // string "lu" + o = append(o, 0xa2, 0x6c, 0x75) + o = msgp.AppendTime(o, z.LastUpdate) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BucketReplicationResyncStatus) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "v": + z.Version, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Version") + return + } + case "brs": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "TargetsMap") + return + } + if z.TargetsMap == nil { + z.TargetsMap = make(map[string]TargetReplicationResyncStatus, zb0002) + } else if len(z.TargetsMap) > 0 { + clear(z.TargetsMap) + } + for zb0002 > 0 { + var za0002 TargetReplicationResyncStatus + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "TargetsMap") + return + } + bts, err = za0002.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "TargetsMap", za0001) + return + } + z.TargetsMap[za0001] = za0002 + } + case "id": + z.ID, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + case "lu": + z.LastUpdate, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "LastUpdate") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *BucketReplicationResyncStatus) Msgsize() (s int) { + s = 1 + 2 + msgp.IntSize + 4 + msgp.MapHeaderSize + if z.TargetsMap != nil { + for za0001, za0002 := range z.TargetsMap { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize() + } + } + s += 3 + msgp.IntSize + 3 + msgp.TimeSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *MRFReplicateEntries) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "e": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Entries") + return + } + if z.Entries == nil { + z.Entries = make(map[string]MRFReplicateEntry, zb0002) + } else if len(z.Entries) > 0 { + clear(z.Entries) + } + for zb0002 > 0 { + zb0002-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Entries") + return + } + var za0002 MRFReplicateEntry + var zb0003 uint32 + zb0003, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Entries", za0001) + return + } + for zb0003 > 0 { + zb0003-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "Entries", za0001) + return + } + switch msgp.UnsafeString(field) { + case "b": + za0002.Bucket, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Entries", za0001, "Bucket") + return + } + case "o": + za0002.Object, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Entries", za0001, "Object") + return + } + case "rc": + za0002.RetryCount, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Entries", za0001, "RetryCount") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "Entries", za0001) + return + } + } + } + z.Entries[za0001] = za0002 + } + case "v": + z.Version, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Version") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *MRFReplicateEntries) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "e" + err = en.Append(0x82, 0xa1, 0x65) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.Entries))) + if err != nil { + err = msgp.WrapError(err, "Entries") + return + } + for za0001, za0002 := range z.Entries { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "Entries") + return + } + // map header, size 3 + // write "b" + err = en.Append(0x83, 0xa1, 0x62) + if err != nil { + return + } + err = en.WriteString(za0002.Bucket) + if err != nil { + err = msgp.WrapError(err, "Entries", za0001, "Bucket") + return + } + // write "o" + err = en.Append(0xa1, 0x6f) + if err != nil { + return + } + err = en.WriteString(za0002.Object) + if err != nil { + err = msgp.WrapError(err, "Entries", za0001, "Object") + return + } + // write "rc" + err = en.Append(0xa2, 0x72, 0x63) + if err != nil { + return + } + err = en.WriteInt(za0002.RetryCount) + if err != nil { + err = msgp.WrapError(err, "Entries", za0001, "RetryCount") + return + } + } + // write "v" + err = en.Append(0xa1, 0x76) + if err != nil { + return + } + err = en.WriteInt(z.Version) + if err != nil { + err = msgp.WrapError(err, "Version") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *MRFReplicateEntries) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "e" + o = append(o, 0x82, 0xa1, 0x65) + o = msgp.AppendMapHeader(o, uint32(len(z.Entries))) + for za0001, za0002 := range z.Entries { + o = msgp.AppendString(o, za0001) + // map header, size 3 + // string "b" + o = append(o, 0x83, 0xa1, 0x62) + o = msgp.AppendString(o, za0002.Bucket) + // string "o" + o = append(o, 0xa1, 0x6f) + o = msgp.AppendString(o, za0002.Object) + // string "rc" + o = append(o, 0xa2, 0x72, 0x63) + o = msgp.AppendInt(o, za0002.RetryCount) + } + // string "v" + o = append(o, 0xa1, 0x76) + o = msgp.AppendInt(o, z.Version) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *MRFReplicateEntries) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "e": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Entries") + return + } + if z.Entries == nil { + z.Entries = make(map[string]MRFReplicateEntry, zb0002) + } else if len(z.Entries) > 0 { + clear(z.Entries) + } + for zb0002 > 0 { + var za0002 MRFReplicateEntry + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Entries") + return + } + var zb0003 uint32 + zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Entries", za0001) + return + } + for zb0003 > 0 { + zb0003-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "Entries", za0001) + return + } + switch msgp.UnsafeString(field) { + case "b": + za0002.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Entries", za0001, "Bucket") + return + } + case "o": + za0002.Object, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Entries", za0001, "Object") + return + } + case "rc": + za0002.RetryCount, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Entries", za0001, "RetryCount") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "Entries", za0001) + return + } + } + } + z.Entries[za0001] = za0002 + } + case "v": + z.Version, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Version") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *MRFReplicateEntries) Msgsize() (s int) { + s = 1 + 2 + msgp.MapHeaderSize + if z.Entries != nil { + for za0001, za0002 := range z.Entries { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + 1 + 2 + msgp.StringPrefixSize + len(za0002.Bucket) + 2 + msgp.StringPrefixSize + len(za0002.Object) + 3 + msgp.IntSize + } + } + s += 2 + msgp.IntSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *MRFReplicateEntry) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "b": + z.Bucket, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "o": + z.Object, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + case "rc": + z.RetryCount, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "RetryCount") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z MRFReplicateEntry) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "b" + err = en.Append(0x83, 0xa1, 0x62) + if err != nil { + return + } + err = en.WriteString(z.Bucket) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + // write "o" + err = en.Append(0xa1, 0x6f) + if err != nil { + return + } + err = en.WriteString(z.Object) + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + // write "rc" + err = en.Append(0xa2, 0x72, 0x63) + if err != nil { + return + } + err = en.WriteInt(z.RetryCount) + if err != nil { + err = msgp.WrapError(err, "RetryCount") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z MRFReplicateEntry) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "b" + o = append(o, 0x83, 0xa1, 0x62) + o = msgp.AppendString(o, z.Bucket) + // string "o" + o = append(o, 0xa1, 0x6f) + o = msgp.AppendString(o, z.Object) + // string "rc" + o = append(o, 0xa2, 0x72, 0x63) + o = msgp.AppendInt(o, z.RetryCount) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *MRFReplicateEntry) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "b": + z.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "o": + z.Object, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + case "rc": + z.RetryCount, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "RetryCount") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z MRFReplicateEntry) Msgsize() (s int) { + s = 1 + 2 + msgp.StringPrefixSize + len(z.Bucket) + 2 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.IntSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ReplicateDecision) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z ReplicateDecision) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 0 + _ = z + err = en.Append(0x80) + if err != nil { + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z ReplicateDecision) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 0 + _ = z + o = append(o, 0x80) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ReplicateDecision) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z ReplicateDecision) Msgsize() (s int) { + s = 1 + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ReplicationState) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ReplicaTimeStamp": + z.ReplicaTimeStamp, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "ReplicaTimeStamp") + return + } + case "ReplicaStatus": + err = z.ReplicaStatus.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "ReplicaStatus") + return + } + case "DeleteMarker": + z.DeleteMarker, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "DeleteMarker") + return + } + case "ReplicationTimeStamp": + z.ReplicationTimeStamp, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "ReplicationTimeStamp") + return + } + case "ReplicationStatusInternal": + z.ReplicationStatusInternal, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "ReplicationStatusInternal") + return + } + case "VersionPurgeStatusInternal": + z.VersionPurgeStatusInternal, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "VersionPurgeStatusInternal") + return + } + case "ReplicateDecisionStr": + z.ReplicateDecisionStr, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "ReplicateDecisionStr") + return + } + case "Targets": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Targets") + return + } + if z.Targets == nil { + z.Targets = make(map[string]replication.StatusType, zb0002) + } else if len(z.Targets) > 0 { + clear(z.Targets) + } + for zb0002 > 0 { + zb0002-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Targets") + return + } + var za0002 replication.StatusType + err = za0002.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Targets", za0001) + return + } + z.Targets[za0001] = za0002 + } + case "PurgeTargets": + var zb0003 uint32 + zb0003, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "PurgeTargets") + return + } + if z.PurgeTargets == nil { + z.PurgeTargets = make(map[string]VersionPurgeStatusType, zb0003) + } else if len(z.PurgeTargets) > 0 { + clear(z.PurgeTargets) + } + for zb0003 > 0 { + zb0003-- + var za0003 string + za0003, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "PurgeTargets") + return + } + var za0004 VersionPurgeStatusType + err = za0004.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "PurgeTargets", za0003) + return + } + z.PurgeTargets[za0003] = za0004 + } + case "ResetStatusesMap": + var zb0004 uint32 + zb0004, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "ResetStatusesMap") + return + } + if z.ResetStatusesMap == nil { + z.ResetStatusesMap = make(map[string]string, zb0004) + } else if len(z.ResetStatusesMap) > 0 { + clear(z.ResetStatusesMap) + } + for zb0004 > 0 { + zb0004-- + var za0005 string + za0005, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "ResetStatusesMap") + return + } + var za0006 string + za0006, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "ResetStatusesMap", za0005) + return + } + z.ResetStatusesMap[za0005] = za0006 + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *ReplicationState) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 10 + // write "ReplicaTimeStamp" + err = en.Append(0x8a, 0xb0, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x6d, 0x70) + if err != nil { + return + } + err = en.WriteTime(z.ReplicaTimeStamp) + if err != nil { + err = msgp.WrapError(err, "ReplicaTimeStamp") + return + } + // write "ReplicaStatus" + err = en.Append(0xad, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73) + if err != nil { + return + } + err = z.ReplicaStatus.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "ReplicaStatus") + return + } + // write "DeleteMarker" + err = en.Append(0xac, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72) + if err != nil { + return + } + err = en.WriteBool(z.DeleteMarker) + if err != nil { + err = msgp.WrapError(err, "DeleteMarker") + return + } + // write "ReplicationTimeStamp" + err = en.Append(0xb4, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x6d, 0x70) + if err != nil { + return + } + err = en.WriteTime(z.ReplicationTimeStamp) + if err != nil { + err = msgp.WrapError(err, "ReplicationTimeStamp") + return + } + // write "ReplicationStatusInternal" + err = en.Append(0xb9, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c) + if err != nil { + return + } + err = en.WriteString(z.ReplicationStatusInternal) + if err != nil { + err = msgp.WrapError(err, "ReplicationStatusInternal") + return + } + // write "VersionPurgeStatusInternal" + err = en.Append(0xba, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x50, 0x75, 0x72, 0x67, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c) + if err != nil { + return + } + err = en.WriteString(z.VersionPurgeStatusInternal) + if err != nil { + err = msgp.WrapError(err, "VersionPurgeStatusInternal") + return + } + // write "ReplicateDecisionStr" + err = en.Append(0xb4, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x44, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x72) + if err != nil { + return + } + err = en.WriteString(z.ReplicateDecisionStr) + if err != nil { + err = msgp.WrapError(err, "ReplicateDecisionStr") + return + } + // write "Targets" + err = en.Append(0xa7, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.Targets))) + if err != nil { + err = msgp.WrapError(err, "Targets") + return + } + for za0001, za0002 := range z.Targets { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "Targets") + return + } + err = za0002.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Targets", za0001) + return + } + } + // write "PurgeTargets" + err = en.Append(0xac, 0x50, 0x75, 0x72, 0x67, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.PurgeTargets))) + if err != nil { + err = msgp.WrapError(err, "PurgeTargets") + return + } + for za0003, za0004 := range z.PurgeTargets { + err = en.WriteString(za0003) + if err != nil { + err = msgp.WrapError(err, "PurgeTargets") + return + } + err = za0004.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "PurgeTargets", za0003) + return + } + } + // write "ResetStatusesMap" + err = en.Append(0xb0, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x4d, 0x61, 0x70) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.ResetStatusesMap))) + if err != nil { + err = msgp.WrapError(err, "ResetStatusesMap") + return + } + for za0005, za0006 := range z.ResetStatusesMap { + err = en.WriteString(za0005) + if err != nil { + err = msgp.WrapError(err, "ResetStatusesMap") + return + } + err = en.WriteString(za0006) + if err != nil { + err = msgp.WrapError(err, "ResetStatusesMap", za0005) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ReplicationState) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 10 + // string "ReplicaTimeStamp" + o = append(o, 0x8a, 0xb0, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x6d, 0x70) + o = msgp.AppendTime(o, z.ReplicaTimeStamp) + // string "ReplicaStatus" + o = append(o, 0xad, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73) + o, err = z.ReplicaStatus.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "ReplicaStatus") + return + } + // string "DeleteMarker" + o = append(o, 0xac, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72) + o = msgp.AppendBool(o, z.DeleteMarker) + // string "ReplicationTimeStamp" + o = append(o, 0xb4, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x53, 0x74, 0x61, 0x6d, 0x70) + o = msgp.AppendTime(o, z.ReplicationTimeStamp) + // string "ReplicationStatusInternal" + o = append(o, 0xb9, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c) + o = msgp.AppendString(o, z.ReplicationStatusInternal) + // string "VersionPurgeStatusInternal" + o = append(o, 0xba, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x50, 0x75, 0x72, 0x67, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c) + o = msgp.AppendString(o, z.VersionPurgeStatusInternal) + // string "ReplicateDecisionStr" + o = append(o, 0xb4, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x44, 0x65, 0x63, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x72) + o = msgp.AppendString(o, z.ReplicateDecisionStr) + // string "Targets" + o = append(o, 0xa7, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.Targets))) + for za0001, za0002 := range z.Targets { + o = msgp.AppendString(o, za0001) + o, err = za0002.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Targets", za0001) + return + } + } + // string "PurgeTargets" + o = append(o, 0xac, 0x50, 0x75, 0x72, 0x67, 0x65, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.PurgeTargets))) + for za0003, za0004 := range z.PurgeTargets { + o = msgp.AppendString(o, za0003) + o, err = za0004.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "PurgeTargets", za0003) + return + } + } + // string "ResetStatusesMap" + o = append(o, 0xb0, 0x52, 0x65, 0x73, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73, 0x4d, 0x61, 0x70) + o = msgp.AppendMapHeader(o, uint32(len(z.ResetStatusesMap))) + for za0005, za0006 := range z.ResetStatusesMap { + o = msgp.AppendString(o, za0005) + o = msgp.AppendString(o, za0006) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ReplicationState) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ReplicaTimeStamp": + z.ReplicaTimeStamp, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicaTimeStamp") + return + } + case "ReplicaStatus": + bts, err = z.ReplicaStatus.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicaStatus") + return + } + case "DeleteMarker": + z.DeleteMarker, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DeleteMarker") + return + } + case "ReplicationTimeStamp": + z.ReplicationTimeStamp, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicationTimeStamp") + return + } + case "ReplicationStatusInternal": + z.ReplicationStatusInternal, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicationStatusInternal") + return + } + case "VersionPurgeStatusInternal": + z.VersionPurgeStatusInternal, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "VersionPurgeStatusInternal") + return + } + case "ReplicateDecisionStr": + z.ReplicateDecisionStr, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicateDecisionStr") + return + } + case "Targets": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Targets") + return + } + if z.Targets == nil { + z.Targets = make(map[string]replication.StatusType, zb0002) + } else if len(z.Targets) > 0 { + clear(z.Targets) + } + for zb0002 > 0 { + var za0002 replication.StatusType + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Targets") + return + } + bts, err = za0002.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Targets", za0001) + return + } + z.Targets[za0001] = za0002 + } + case "PurgeTargets": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "PurgeTargets") + return + } + if z.PurgeTargets == nil { + z.PurgeTargets = make(map[string]VersionPurgeStatusType, zb0003) + } else if len(z.PurgeTargets) > 0 { + clear(z.PurgeTargets) + } + for zb0003 > 0 { + var za0004 VersionPurgeStatusType + zb0003-- + var za0003 string + za0003, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "PurgeTargets") + return + } + bts, err = za0004.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "PurgeTargets", za0003) + return + } + z.PurgeTargets[za0003] = za0004 + } + case "ResetStatusesMap": + var zb0004 uint32 + zb0004, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResetStatusesMap") + return + } + if z.ResetStatusesMap == nil { + z.ResetStatusesMap = make(map[string]string, zb0004) + } else if len(z.ResetStatusesMap) > 0 { + clear(z.ResetStatusesMap) + } + for zb0004 > 0 { + var za0006 string + zb0004-- + var za0005 string + za0005, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResetStatusesMap") + return + } + za0006, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResetStatusesMap", za0005) + return + } + z.ResetStatusesMap[za0005] = za0006 + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ReplicationState) Msgsize() (s int) { + s = 1 + 17 + msgp.TimeSize + 14 + z.ReplicaStatus.Msgsize() + 13 + msgp.BoolSize + 21 + msgp.TimeSize + 26 + msgp.StringPrefixSize + len(z.ReplicationStatusInternal) + 27 + msgp.StringPrefixSize + len(z.VersionPurgeStatusInternal) + 21 + msgp.StringPrefixSize + len(z.ReplicateDecisionStr) + 8 + msgp.MapHeaderSize + if z.Targets != nil { + for za0001, za0002 := range z.Targets { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize() + } + } + s += 13 + msgp.MapHeaderSize + if z.PurgeTargets != nil { + for za0003, za0004 := range z.PurgeTargets { + _ = za0004 + s += msgp.StringPrefixSize + len(za0003) + za0004.Msgsize() + } + } + s += 17 + msgp.MapHeaderSize + if z.ResetStatusesMap != nil { + for za0005, za0006 := range z.ResetStatusesMap { + _ = za0006 + s += msgp.StringPrefixSize + len(za0005) + msgp.StringPrefixSize + len(za0006) + } + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ResyncDecision) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z ResyncDecision) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 0 + _ = z + err = en.Append(0x80) + if err != nil { + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z ResyncDecision) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 0 + _ = z + o = append(o, 0x80) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ResyncDecision) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z ResyncDecision) Msgsize() (s int) { + s = 1 + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ResyncStatusType) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 int + zb0001, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = ResyncStatusType(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z ResyncStatusType) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteInt(int(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z ResyncStatusType) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendInt(o, int(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ResyncStatusType) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 int + zb0001, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = ResyncStatusType(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z ResyncStatusType) Msgsize() (s int) { + s = msgp.IntSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ResyncTarget) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Arn": + z.Arn, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Arn") + return + } + case "ResetID": + z.ResetID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "ResetID") + return + } + case "StartTime": + z.StartTime, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "StartTime") + return + } + case "EndTime": + z.EndTime, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "EndTime") + return + } + case "ResyncStatus": + z.ResyncStatus, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "ResyncStatus") + return + } + case "ReplicatedSize": + z.ReplicatedSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ReplicatedSize") + return + } + case "FailedSize": + z.FailedSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "FailedSize") + return + } + case "FailedCount": + z.FailedCount, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "FailedCount") + return + } + case "ReplicatedCount": + z.ReplicatedCount, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ReplicatedCount") + return + } + case "Bucket": + z.Bucket, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "Object": + z.Object, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *ResyncTarget) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 11 + // write "Arn" + err = en.Append(0x8b, 0xa3, 0x41, 0x72, 0x6e) + if err != nil { + return + } + err = en.WriteString(z.Arn) + if err != nil { + err = msgp.WrapError(err, "Arn") + return + } + // write "ResetID" + err = en.Append(0xa7, 0x52, 0x65, 0x73, 0x65, 0x74, 0x49, 0x44) + if err != nil { + return + } + err = en.WriteString(z.ResetID) + if err != nil { + err = msgp.WrapError(err, "ResetID") + return + } + // write "StartTime" + err = en.Append(0xa9, 0x53, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteTime(z.StartTime) + if err != nil { + err = msgp.WrapError(err, "StartTime") + return + } + // write "EndTime" + err = en.Append(0xa7, 0x45, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteTime(z.EndTime) + if err != nil { + err = msgp.WrapError(err, "EndTime") + return + } + // write "ResyncStatus" + err = en.Append(0xac, 0x52, 0x65, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73) + if err != nil { + return + } + err = en.WriteString(z.ResyncStatus) + if err != nil { + err = msgp.WrapError(err, "ResyncStatus") + return + } + // write "ReplicatedSize" + err = en.Append(0xae, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.ReplicatedSize) + if err != nil { + err = msgp.WrapError(err, "ReplicatedSize") + return + } + // write "FailedSize" + err = en.Append(0xaa, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.FailedSize) + if err != nil { + err = msgp.WrapError(err, "FailedSize") + return + } + // write "FailedCount" + err = en.Append(0xab, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteInt64(z.FailedCount) + if err != nil { + err = msgp.WrapError(err, "FailedCount") + return + } + // write "ReplicatedCount" + err = en.Append(0xaf, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteInt64(z.ReplicatedCount) + if err != nil { + err = msgp.WrapError(err, "ReplicatedCount") + return + } + // write "Bucket" + err = en.Append(0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Bucket) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + // write "Object" + err = en.Append(0xa6, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Object) + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ResyncTarget) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 11 + // string "Arn" + o = append(o, 0x8b, 0xa3, 0x41, 0x72, 0x6e) + o = msgp.AppendString(o, z.Arn) + // string "ResetID" + o = append(o, 0xa7, 0x52, 0x65, 0x73, 0x65, 0x74, 0x49, 0x44) + o = msgp.AppendString(o, z.ResetID) + // string "StartTime" + o = append(o, 0xa9, 0x53, 0x74, 0x61, 0x72, 0x74, 0x54, 0x69, 0x6d, 0x65) + o = msgp.AppendTime(o, z.StartTime) + // string "EndTime" + o = append(o, 0xa7, 0x45, 0x6e, 0x64, 0x54, 0x69, 0x6d, 0x65) + o = msgp.AppendTime(o, z.EndTime) + // string "ResyncStatus" + o = append(o, 0xac, 0x52, 0x65, 0x73, 0x79, 0x6e, 0x63, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73) + o = msgp.AppendString(o, z.ResyncStatus) + // string "ReplicatedSize" + o = append(o, 0xae, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.ReplicatedSize) + // string "FailedSize" + o = append(o, 0xaa, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.FailedSize) + // string "FailedCount" + o = append(o, 0xab, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendInt64(o, z.FailedCount) + // string "ReplicatedCount" + o = append(o, 0xaf, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendInt64(o, z.ReplicatedCount) + // string "Bucket" + o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + o = msgp.AppendString(o, z.Bucket) + // string "Object" + o = append(o, 0xa6, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74) + o = msgp.AppendString(o, z.Object) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ResyncTarget) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Arn": + z.Arn, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Arn") + return + } + case "ResetID": + z.ResetID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResetID") + return + } + case "StartTime": + z.StartTime, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "StartTime") + return + } + case "EndTime": + z.EndTime, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "EndTime") + return + } + case "ResyncStatus": + z.ResyncStatus, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResyncStatus") + return + } + case "ReplicatedSize": + z.ReplicatedSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicatedSize") + return + } + case "FailedSize": + z.FailedSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "FailedSize") + return + } + case "FailedCount": + z.FailedCount, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "FailedCount") + return + } + case "ReplicatedCount": + z.ReplicatedCount, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicatedCount") + return + } + case "Bucket": + z.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "Object": + z.Object, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ResyncTarget) Msgsize() (s int) { + s = 1 + 4 + msgp.StringPrefixSize + len(z.Arn) + 8 + msgp.StringPrefixSize + len(z.ResetID) + 10 + msgp.TimeSize + 8 + msgp.TimeSize + 13 + msgp.StringPrefixSize + len(z.ResyncStatus) + 15 + msgp.Int64Size + 11 + msgp.Int64Size + 12 + msgp.Int64Size + 16 + msgp.Int64Size + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Object) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ResyncTargetDecision) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Replicate": + z.Replicate, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Replicate") + return + } + case "ResetID": + z.ResetID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "ResetID") + return + } + case "ResetBeforeDate": + z.ResetBeforeDate, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "ResetBeforeDate") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z ResyncTargetDecision) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "Replicate" + err = en.Append(0x83, 0xa9, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65) + if err != nil { + return + } + err = en.WriteBool(z.Replicate) + if err != nil { + err = msgp.WrapError(err, "Replicate") + return + } + // write "ResetID" + err = en.Append(0xa7, 0x52, 0x65, 0x73, 0x65, 0x74, 0x49, 0x44) + if err != nil { + return + } + err = en.WriteString(z.ResetID) + if err != nil { + err = msgp.WrapError(err, "ResetID") + return + } + // write "ResetBeforeDate" + err = en.Append(0xaf, 0x52, 0x65, 0x73, 0x65, 0x74, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65, 0x44, 0x61, 0x74, 0x65) + if err != nil { + return + } + err = en.WriteTime(z.ResetBeforeDate) + if err != nil { + err = msgp.WrapError(err, "ResetBeforeDate") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z ResyncTargetDecision) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "Replicate" + o = append(o, 0x83, 0xa9, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65) + o = msgp.AppendBool(o, z.Replicate) + // string "ResetID" + o = append(o, 0xa7, 0x52, 0x65, 0x73, 0x65, 0x74, 0x49, 0x44) + o = msgp.AppendString(o, z.ResetID) + // string "ResetBeforeDate" + o = append(o, 0xaf, 0x52, 0x65, 0x73, 0x65, 0x74, 0x42, 0x65, 0x66, 0x6f, 0x72, 0x65, 0x44, 0x61, 0x74, 0x65) + o = msgp.AppendTime(o, z.ResetBeforeDate) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ResyncTargetDecision) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Replicate": + z.Replicate, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Replicate") + return + } + case "ResetID": + z.ResetID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResetID") + return + } + case "ResetBeforeDate": + z.ResetBeforeDate, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResetBeforeDate") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z ResyncTargetDecision) Msgsize() (s int) { + s = 1 + 10 + msgp.BoolSize + 8 + msgp.StringPrefixSize + len(z.ResetID) + 16 + msgp.TimeSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ResyncTargetsInfo) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Targets": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Targets") + return + } + if cap(z.Targets) >= int(zb0002) { + z.Targets = (z.Targets)[:zb0002] + } else { + z.Targets = make([]ResyncTarget, zb0002) + } + for za0001 := range z.Targets { + err = z.Targets[za0001].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Targets", za0001) + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *ResyncTargetsInfo) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "Targets" + err = en.Append(0x81, 0xa7, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Targets))) + if err != nil { + err = msgp.WrapError(err, "Targets") + return + } + for za0001 := range z.Targets { + err = z.Targets[za0001].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Targets", za0001) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ResyncTargetsInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "Targets" + o = append(o, 0x81, 0xa7, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Targets))) + for za0001 := range z.Targets { + o, err = z.Targets[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Targets", za0001) + return + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ResyncTargetsInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Targets": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Targets") + return + } + if cap(z.Targets) >= int(zb0002) { + z.Targets = (z.Targets)[:zb0002] + } else { + z.Targets = make([]ResyncTarget, zb0002) + } + for za0001 := range z.Targets { + bts, err = z.Targets[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Targets", za0001) + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ResyncTargetsInfo) Msgsize() (s int) { + s = 1 + 8 + msgp.ArrayHeaderSize + for za0001 := range z.Targets { + s += z.Targets[za0001].Msgsize() + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *TargetReplicationResyncStatus) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "st": + z.StartTime, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "StartTime") + return + } + case "lst": + z.LastUpdate, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "LastUpdate") + return + } + case "id": + z.ResyncID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "ResyncID") + return + } + case "rdt": + z.ResyncBeforeDate, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "ResyncBeforeDate") + return + } + case "rst": + { + var zb0002 int + zb0002, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "ResyncStatus") + return + } + z.ResyncStatus = ResyncStatusType(zb0002) + } + case "fs": + z.FailedSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "FailedSize") + return + } + case "frc": + z.FailedCount, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "FailedCount") + return + } + case "rs": + z.ReplicatedSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ReplicatedSize") + return + } + case "rrc": + z.ReplicatedCount, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ReplicatedCount") + return + } + case "bkt": + z.Bucket, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "obj": + z.Object, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *TargetReplicationResyncStatus) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 11 + // write "st" + err = en.Append(0x8b, 0xa2, 0x73, 0x74) + if err != nil { + return + } + err = en.WriteTime(z.StartTime) + if err != nil { + err = msgp.WrapError(err, "StartTime") + return + } + // write "lst" + err = en.Append(0xa3, 0x6c, 0x73, 0x74) + if err != nil { + return + } + err = en.WriteTime(z.LastUpdate) + if err != nil { + err = msgp.WrapError(err, "LastUpdate") + return + } + // write "id" + err = en.Append(0xa2, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteString(z.ResyncID) + if err != nil { + err = msgp.WrapError(err, "ResyncID") + return + } + // write "rdt" + err = en.Append(0xa3, 0x72, 0x64, 0x74) + if err != nil { + return + } + err = en.WriteTime(z.ResyncBeforeDate) + if err != nil { + err = msgp.WrapError(err, "ResyncBeforeDate") + return + } + // write "rst" + err = en.Append(0xa3, 0x72, 0x73, 0x74) + if err != nil { + return + } + err = en.WriteInt(int(z.ResyncStatus)) + if err != nil { + err = msgp.WrapError(err, "ResyncStatus") + return + } + // write "fs" + err = en.Append(0xa2, 0x66, 0x73) + if err != nil { + return + } + err = en.WriteInt64(z.FailedSize) + if err != nil { + err = msgp.WrapError(err, "FailedSize") + return + } + // write "frc" + err = en.Append(0xa3, 0x66, 0x72, 0x63) + if err != nil { + return + } + err = en.WriteInt64(z.FailedCount) + if err != nil { + err = msgp.WrapError(err, "FailedCount") + return + } + // write "rs" + err = en.Append(0xa2, 0x72, 0x73) + if err != nil { + return + } + err = en.WriteInt64(z.ReplicatedSize) + if err != nil { + err = msgp.WrapError(err, "ReplicatedSize") + return + } + // write "rrc" + err = en.Append(0xa3, 0x72, 0x72, 0x63) + if err != nil { + return + } + err = en.WriteInt64(z.ReplicatedCount) + if err != nil { + err = msgp.WrapError(err, "ReplicatedCount") + return + } + // write "bkt" + err = en.Append(0xa3, 0x62, 0x6b, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Bucket) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + // write "obj" + err = en.Append(0xa3, 0x6f, 0x62, 0x6a) + if err != nil { + return + } + err = en.WriteString(z.Object) + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *TargetReplicationResyncStatus) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 11 + // string "st" + o = append(o, 0x8b, 0xa2, 0x73, 0x74) + o = msgp.AppendTime(o, z.StartTime) + // string "lst" + o = append(o, 0xa3, 0x6c, 0x73, 0x74) + o = msgp.AppendTime(o, z.LastUpdate) + // string "id" + o = append(o, 0xa2, 0x69, 0x64) + o = msgp.AppendString(o, z.ResyncID) + // string "rdt" + o = append(o, 0xa3, 0x72, 0x64, 0x74) + o = msgp.AppendTime(o, z.ResyncBeforeDate) + // string "rst" + o = append(o, 0xa3, 0x72, 0x73, 0x74) + o = msgp.AppendInt(o, int(z.ResyncStatus)) + // string "fs" + o = append(o, 0xa2, 0x66, 0x73) + o = msgp.AppendInt64(o, z.FailedSize) + // string "frc" + o = append(o, 0xa3, 0x66, 0x72, 0x63) + o = msgp.AppendInt64(o, z.FailedCount) + // string "rs" + o = append(o, 0xa2, 0x72, 0x73) + o = msgp.AppendInt64(o, z.ReplicatedSize) + // string "rrc" + o = append(o, 0xa3, 0x72, 0x72, 0x63) + o = msgp.AppendInt64(o, z.ReplicatedCount) + // string "bkt" + o = append(o, 0xa3, 0x62, 0x6b, 0x74) + o = msgp.AppendString(o, z.Bucket) + // string "obj" + o = append(o, 0xa3, 0x6f, 0x62, 0x6a) + o = msgp.AppendString(o, z.Object) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *TargetReplicationResyncStatus) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "st": + z.StartTime, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "StartTime") + return + } + case "lst": + z.LastUpdate, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "LastUpdate") + return + } + case "id": + z.ResyncID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResyncID") + return + } + case "rdt": + z.ResyncBeforeDate, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResyncBeforeDate") + return + } + case "rst": + { + var zb0002 int + zb0002, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResyncStatus") + return + } + z.ResyncStatus = ResyncStatusType(zb0002) + } + case "fs": + z.FailedSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "FailedSize") + return + } + case "frc": + z.FailedCount, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "FailedCount") + return + } + case "rs": + z.ReplicatedSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicatedSize") + return + } + case "rrc": + z.ReplicatedCount, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicatedCount") + return + } + case "bkt": + z.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "obj": + z.Object, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *TargetReplicationResyncStatus) Msgsize() (s int) { + s = 1 + 3 + msgp.TimeSize + 4 + msgp.TimeSize + 3 + msgp.StringPrefixSize + len(z.ResyncID) + 4 + msgp.TimeSize + 4 + msgp.IntSize + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 4 + msgp.StringPrefixSize + len(z.Bucket) + 4 + msgp.StringPrefixSize + len(z.Object) + return +} diff --git a/cmd/bucket-replication-utils_gen_test.go b/cmd/bucket-replication-utils_gen_test.go new file mode 100644 index 00000000..ec7dd41c --- /dev/null +++ b/cmd/bucket-replication-utils_gen_test.go @@ -0,0 +1,1140 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalBucketReplicationResyncStatus(t *testing.T) { + v := BucketReplicationResyncStatus{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBucketReplicationResyncStatus(b *testing.B) { + v := BucketReplicationResyncStatus{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBucketReplicationResyncStatus(b *testing.B) { + v := BucketReplicationResyncStatus{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBucketReplicationResyncStatus(b *testing.B) { + v := BucketReplicationResyncStatus{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBucketReplicationResyncStatus(t *testing.T) { + v := BucketReplicationResyncStatus{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBucketReplicationResyncStatus Msgsize() is inaccurate") + } + + vn := BucketReplicationResyncStatus{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBucketReplicationResyncStatus(b *testing.B) { + v := BucketReplicationResyncStatus{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBucketReplicationResyncStatus(b *testing.B) { + v := BucketReplicationResyncStatus{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalMRFReplicateEntries(t *testing.T) { + v := MRFReplicateEntries{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgMRFReplicateEntries(b *testing.B) { + v := MRFReplicateEntries{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgMRFReplicateEntries(b *testing.B) { + v := MRFReplicateEntries{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalMRFReplicateEntries(b *testing.B) { + v := MRFReplicateEntries{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeMRFReplicateEntries(t *testing.T) { + v := MRFReplicateEntries{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeMRFReplicateEntries Msgsize() is inaccurate") + } + + vn := MRFReplicateEntries{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeMRFReplicateEntries(b *testing.B) { + v := MRFReplicateEntries{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeMRFReplicateEntries(b *testing.B) { + v := MRFReplicateEntries{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalMRFReplicateEntry(t *testing.T) { + v := MRFReplicateEntry{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgMRFReplicateEntry(b *testing.B) { + v := MRFReplicateEntry{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgMRFReplicateEntry(b *testing.B) { + v := MRFReplicateEntry{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalMRFReplicateEntry(b *testing.B) { + v := MRFReplicateEntry{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeMRFReplicateEntry(t *testing.T) { + v := MRFReplicateEntry{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeMRFReplicateEntry Msgsize() is inaccurate") + } + + vn := MRFReplicateEntry{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeMRFReplicateEntry(b *testing.B) { + v := MRFReplicateEntry{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeMRFReplicateEntry(b *testing.B) { + v := MRFReplicateEntry{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalReplicateDecision(t *testing.T) { + v := ReplicateDecision{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgReplicateDecision(b *testing.B) { + v := ReplicateDecision{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgReplicateDecision(b *testing.B) { + v := ReplicateDecision{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalReplicateDecision(b *testing.B) { + v := ReplicateDecision{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeReplicateDecision(t *testing.T) { + v := ReplicateDecision{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeReplicateDecision Msgsize() is inaccurate") + } + + vn := ReplicateDecision{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeReplicateDecision(b *testing.B) { + v := ReplicateDecision{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeReplicateDecision(b *testing.B) { + v := ReplicateDecision{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalReplicationState(t *testing.T) { + v := ReplicationState{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgReplicationState(b *testing.B) { + v := ReplicationState{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgReplicationState(b *testing.B) { + v := ReplicationState{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalReplicationState(b *testing.B) { + v := ReplicationState{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeReplicationState(t *testing.T) { + v := ReplicationState{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeReplicationState Msgsize() is inaccurate") + } + + vn := ReplicationState{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeReplicationState(b *testing.B) { + v := ReplicationState{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeReplicationState(b *testing.B) { + v := ReplicationState{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalResyncDecision(t *testing.T) { + v := ResyncDecision{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgResyncDecision(b *testing.B) { + v := ResyncDecision{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgResyncDecision(b *testing.B) { + v := ResyncDecision{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalResyncDecision(b *testing.B) { + v := ResyncDecision{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeResyncDecision(t *testing.T) { + v := ResyncDecision{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeResyncDecision Msgsize() is inaccurate") + } + + vn := ResyncDecision{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeResyncDecision(b *testing.B) { + v := ResyncDecision{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeResyncDecision(b *testing.B) { + v := ResyncDecision{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalResyncTarget(t *testing.T) { + v := ResyncTarget{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgResyncTarget(b *testing.B) { + v := ResyncTarget{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgResyncTarget(b *testing.B) { + v := ResyncTarget{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalResyncTarget(b *testing.B) { + v := ResyncTarget{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeResyncTarget(t *testing.T) { + v := ResyncTarget{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeResyncTarget Msgsize() is inaccurate") + } + + vn := ResyncTarget{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeResyncTarget(b *testing.B) { + v := ResyncTarget{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeResyncTarget(b *testing.B) { + v := ResyncTarget{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalResyncTargetDecision(t *testing.T) { + v := ResyncTargetDecision{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgResyncTargetDecision(b *testing.B) { + v := ResyncTargetDecision{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgResyncTargetDecision(b *testing.B) { + v := ResyncTargetDecision{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalResyncTargetDecision(b *testing.B) { + v := ResyncTargetDecision{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeResyncTargetDecision(t *testing.T) { + v := ResyncTargetDecision{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeResyncTargetDecision Msgsize() is inaccurate") + } + + vn := ResyncTargetDecision{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeResyncTargetDecision(b *testing.B) { + v := ResyncTargetDecision{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeResyncTargetDecision(b *testing.B) { + v := ResyncTargetDecision{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalResyncTargetsInfo(t *testing.T) { + v := ResyncTargetsInfo{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgResyncTargetsInfo(b *testing.B) { + v := ResyncTargetsInfo{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgResyncTargetsInfo(b *testing.B) { + v := ResyncTargetsInfo{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalResyncTargetsInfo(b *testing.B) { + v := ResyncTargetsInfo{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeResyncTargetsInfo(t *testing.T) { + v := ResyncTargetsInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeResyncTargetsInfo Msgsize() is inaccurate") + } + + vn := ResyncTargetsInfo{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeResyncTargetsInfo(b *testing.B) { + v := ResyncTargetsInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeResyncTargetsInfo(b *testing.B) { + v := ResyncTargetsInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalTargetReplicationResyncStatus(t *testing.T) { + v := TargetReplicationResyncStatus{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgTargetReplicationResyncStatus(b *testing.B) { + v := TargetReplicationResyncStatus{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgTargetReplicationResyncStatus(b *testing.B) { + v := TargetReplicationResyncStatus{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalTargetReplicationResyncStatus(b *testing.B) { + v := TargetReplicationResyncStatus{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeTargetReplicationResyncStatus(t *testing.T) { + v := TargetReplicationResyncStatus{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeTargetReplicationResyncStatus Msgsize() is inaccurate") + } + + vn := TargetReplicationResyncStatus{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeTargetReplicationResyncStatus(b *testing.B) { + v := TargetReplicationResyncStatus{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeTargetReplicationResyncStatus(b *testing.B) { + v := TargetReplicationResyncStatus{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/bucket-replication-utils_test.go b/cmd/bucket-replication-utils_test.go new file mode 100644 index 00000000..19c1b4b1 --- /dev/null +++ b/cmd/bucket-replication-utils_test.go @@ -0,0 +1,247 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "testing" + + "github.com/minio/minio/internal/bucket/replication" +) + +var replicatedInfosTests = []struct { + name string + tgtInfos []replicatedTargetInfo + expectedCompletedSize int64 + expectedReplicationStatusInternal string + expectedReplicationStatus replication.StatusType + expectedOpType replication.Type + expectedAction replicationAction +}{ + { // 1. empty tgtInfos slice + name: "no replicated targets", + tgtInfos: []replicatedTargetInfo{}, + expectedCompletedSize: 0, + expectedReplicationStatusInternal: "", + expectedReplicationStatus: replication.StatusType(""), + expectedOpType: replication.UnsetReplicationType, + expectedAction: replicateNone, + }, + { // 2. replication completed to single target + name: "replication completed to single target", + tgtInfos: []replicatedTargetInfo{ + { + Arn: "arn1", + Size: 249, + PrevReplicationStatus: replication.Pending, + ReplicationStatus: replication.Completed, + OpType: replication.ObjectReplicationType, + ReplicationAction: replicateAll, + }, + }, + expectedCompletedSize: 249, + expectedReplicationStatusInternal: "arn1=COMPLETED;", + expectedReplicationStatus: replication.Completed, + expectedOpType: replication.ObjectReplicationType, + expectedAction: replicateAll, + }, + { // 3. replication completed to single target; failed to another + name: "replication completed to single target", + tgtInfos: []replicatedTargetInfo{ + { + Arn: "arn1", + Size: 249, + PrevReplicationStatus: replication.Pending, + ReplicationStatus: replication.Completed, + OpType: replication.ObjectReplicationType, + ReplicationAction: replicateAll, + }, + { + Arn: "arn2", + Size: 249, + PrevReplicationStatus: replication.Pending, + ReplicationStatus: replication.Failed, + OpType: replication.ObjectReplicationType, + ReplicationAction: replicateAll, + }, + }, + expectedCompletedSize: 249, + expectedReplicationStatusInternal: "arn1=COMPLETED;arn2=FAILED;", + expectedReplicationStatus: replication.Failed, + expectedOpType: replication.ObjectReplicationType, + expectedAction: replicateAll, + }, + { // 4. replication pending on one target; failed to another + name: "replication completed to single target", + tgtInfos: []replicatedTargetInfo{ + { + Arn: "arn1", + Size: 249, + PrevReplicationStatus: replication.Pending, + ReplicationStatus: replication.Pending, + OpType: replication.ObjectReplicationType, + ReplicationAction: replicateAll, + }, + { + Arn: "arn2", + Size: 249, + PrevReplicationStatus: replication.Pending, + ReplicationStatus: replication.Failed, + OpType: replication.ObjectReplicationType, + ReplicationAction: replicateAll, + }, + }, + expectedCompletedSize: 0, + expectedReplicationStatusInternal: "arn1=PENDING;arn2=FAILED;", + expectedReplicationStatus: replication.Failed, + expectedOpType: replication.ObjectReplicationType, + expectedAction: replicateAll, + }, +} + +func TestReplicatedInfos(t *testing.T) { + for i, test := range replicatedInfosTests { + rinfos := replicatedInfos{ + Targets: test.tgtInfos, + } + if actualSize := rinfos.CompletedSize(); actualSize != test.expectedCompletedSize { + t.Errorf("Test%d (%s): Size got %d , want %d", i+1, test.name, actualSize, test.expectedCompletedSize) + } + if repStatusStr := rinfos.ReplicationStatusInternal(); repStatusStr != test.expectedReplicationStatusInternal { + t.Errorf("Test%d (%s): Internal replication status got %s , want %s", i+1, test.name, repStatusStr, test.expectedReplicationStatusInternal) + } + if repStatus := rinfos.ReplicationStatus(); repStatus != test.expectedReplicationStatus { + t.Errorf("Test%d (%s): ReplicationStatus got %s , want %s", i+1, test.name, repStatus, test.expectedReplicationStatus) + } + if action := rinfos.Action(); action != test.expectedAction { + t.Errorf("Test%d (%s): Action got %s , want %s", i+1, test.name, action, test.expectedAction) + } + } +} + +var parseReplicationDecisionTest = []struct { + name string + dsc string + expDsc ReplicateDecision + expErr error +}{ + { // 1. + name: "empty string", + dsc: "", + expDsc: ReplicateDecision{ + targetsMap: map[string]replicateTargetDecision{}, + }, + expErr: nil, + }, + + { // 2. + name: "replicate decision for one target", + dsc: "arn:minio:replication::id:bucket=true;false;arn:minio:replication::id:bucket;id", + expErr: nil, + expDsc: ReplicateDecision{ + targetsMap: map[string]replicateTargetDecision{ + "arn:minio:replication::id:bucket": newReplicateTargetDecision("arn:minio:replication::id:bucket", true, false), + }, + }, + }, + { // 3. + name: "replicate decision for multiple targets", + dsc: "arn:minio:replication::id:bucket=true;false;arn:minio:replication::id:bucket;id,arn:minio:replication::id2:bucket=false;true;arn:minio:replication::id2:bucket;id2", + expErr: nil, + expDsc: ReplicateDecision{ + targetsMap: map[string]replicateTargetDecision{ + "arn:minio:replication::id:bucket": newReplicateTargetDecision("arn:minio:replication::id:bucket", true, false), + "arn:minio:replication::id2:bucket": newReplicateTargetDecision("arn:minio:replication::id2:bucket", false, true), + }, + }, + }, + { // 4. + name: "invalid format replicate decision for one target", + dsc: "arn:minio:replication::id:bucket:true;false;arn:minio:replication::id:bucket;id", + expErr: errInvalidReplicateDecisionFormat, + expDsc: ReplicateDecision{ + targetsMap: map[string]replicateTargetDecision{ + "arn:minio:replication::id:bucket": newReplicateTargetDecision("arn:minio:replication::id:bucket", true, false), + }, + }, + }, +} + +func TestParseReplicateDecision(t *testing.T) { + for i, test := range parseReplicationDecisionTest { + dsc, err := parseReplicateDecision(t.Context(), "bucket", test.expDsc.String()) + if err != nil { + if test.expErr != err { + t.Errorf("Test%d (%s): Expected parse error got %t , want %t", i+1, test.name, err, test.expErr) + } + continue + } + if len(dsc.targetsMap) != len(test.expDsc.targetsMap) { + t.Errorf("Test%d (%s): Invalid number of entries in targetsMap got %d , want %d", i+1, test.name, len(dsc.targetsMap), len(test.expDsc.targetsMap)) + } + for arn, tdsc := range dsc.targetsMap { + expDsc, ok := test.expDsc.targetsMap[arn] + if !ok || expDsc != tdsc { + t.Errorf("Test%d (%s): Invalid target replicate decision: got %+v, want %+v", i+1, test.name, tdsc, expDsc) + } + } + } +} + +var replicationStateTest = []struct { + name string + rs ReplicationState + arn string + expStatus replication.StatusType +}{ + { // 1. no replication status header + name: "no replicated targets", + rs: ReplicationState{}, + expStatus: replication.StatusType(""), + }, + { // 2. replication status for one target + name: "replication status for one target", + rs: ReplicationState{ReplicationStatusInternal: "arn1=PENDING;", Targets: map[string]replication.StatusType{"arn1": "PENDING"}}, + expStatus: replication.Pending, + }, + { // 3. replication status for one target - incorrect format + name: "replication status for one target", + rs: ReplicationState{ReplicationStatusInternal: "arn1=PENDING"}, + expStatus: replication.StatusType(""), + }, + { // 4. replication status for 3 targets, one of them failed + name: "replication status for 3 targets - one failed", + rs: ReplicationState{ + ReplicationStatusInternal: "arn1=COMPLETED;arn2=COMPLETED;arn3=FAILED;", + Targets: map[string]replication.StatusType{"arn1": "COMPLETED", "arn2": "COMPLETED", "arn3": "FAILED"}, + }, + expStatus: replication.Failed, + }, + { // 5. replication status for replica version + name: "replication status for replica version", + rs: ReplicationState{ReplicationStatusInternal: string(replication.Replica)}, + expStatus: replication.Replica, + }, +} + +func TestCompositeReplicationStatus(t *testing.T) { + for i, test := range replicationStateTest { + if rstatus := test.rs.CompositeReplicationStatus(); rstatus != test.expStatus { + t.Errorf("Test%d (%s): Overall replication status got %s , want %s", i+1, test.name, rstatus, test.expStatus) + } + } +} diff --git a/cmd/bucket-replication.go b/cmd/bucket-replication.go new file mode 100644 index 00000000..5f6268cb --- /dev/null +++ b/cmd/bucket-replication.go @@ -0,0 +1,3803 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/base64" + "encoding/binary" + "errors" + "fmt" + "io" + "maps" + "math/rand" + "net/http" + "net/url" + "path" + "reflect" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/dustin/go-humanize" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/encrypt" + "github.com/minio/minio-go/v7/pkg/tags" + "github.com/minio/minio/internal/amztime" + "github.com/minio/minio/internal/bucket/bandwidth" + objectlock "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/config/storageclass" + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/kms" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/once" + "github.com/tinylib/msgp/msgp" + "github.com/zeebo/xxh3" +) + +const ( + throttleDeadline = 1 * time.Hour + // ReplicationReset has reset id and timestamp of last reset operation + ReplicationReset = "replication-reset" + // ReplicationStatus has internal replication status - stringified representation of target's replication status for all replication + // activity initiated from this cluster + ReplicationStatus = "replication-status" + // ReplicationTimestamp - the last time replication was initiated on this cluster for this object version + ReplicationTimestamp = "replication-timestamp" + // ReplicaStatus - this header is present if a replica was received by this cluster for this object version + ReplicaStatus = "replica-status" + // ReplicaTimestamp - the last time a replica was received by this cluster for this object version + ReplicaTimestamp = "replica-timestamp" + // TaggingTimestamp - the last time a tag metadata modification happened on this cluster for this object version + TaggingTimestamp = "tagging-timestamp" + // ObjectLockRetentionTimestamp - the last time a object lock metadata modification happened on this cluster for this object version + ObjectLockRetentionTimestamp = "objectlock-retention-timestamp" + // ObjectLockLegalHoldTimestamp - the last time a legal hold metadata modification happened on this cluster for this object version + ObjectLockLegalHoldTimestamp = "objectlock-legalhold-timestamp" + + // ReplicationSsecChecksumHeader - the encrypted checksum of the SSE-C encrypted object. + ReplicationSsecChecksumHeader = "X-Minio-Replication-Ssec-Crc" +) + +// gets replication config associated to a given bucket name. +func getReplicationConfig(ctx context.Context, bucketName string) (rc *replication.Config, err error) { + rCfg, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucketName) + if err != nil && !errors.Is(err, BucketReplicationConfigNotFound{Bucket: bucketName}) { + return rCfg, err + } + return rCfg, nil +} + +// validateReplicationDestination returns error if replication destination bucket missing or not configured +// It also returns true if replication destination is same as this server. +func validateReplicationDestination(ctx context.Context, bucket string, rCfg *replication.Config, opts *validateReplicationDestinationOptions) (bool, APIError) { + if opts == nil { + opts = &validateReplicationDestinationOptions{} + } + var arns []string + if rCfg.RoleArn != "" { + arns = append(arns, rCfg.RoleArn) + } else { + for _, rule := range rCfg.Rules { + arns = append(arns, rule.Destination.String()) + } + } + var sameTarget bool + for _, arnStr := range arns { + arn, err := madmin.ParseARN(arnStr) + if err != nil { + return sameTarget, errorCodes.ToAPIErrWithErr(ErrBucketRemoteArnInvalid, err) + } + if arn.Type != madmin.ReplicationService { + return sameTarget, toAPIError(ctx, BucketRemoteArnTypeInvalid{Bucket: bucket}) + } + clnt := globalBucketTargetSys.GetRemoteTargetClient(bucket, arnStr) + if clnt == nil { + return sameTarget, toAPIError(ctx, BucketRemoteTargetNotFound{Bucket: bucket}) + } + if opts.CheckRemoteBucket { // validate remote bucket + found, err := clnt.BucketExists(ctx, arn.Bucket) + if err != nil { + return sameTarget, errorCodes.ToAPIErrWithErr(ErrRemoteDestinationNotFoundError, err) + } + if !found { + return sameTarget, errorCodes.ToAPIErrWithErr(ErrRemoteDestinationNotFoundError, BucketRemoteTargetNotFound{Bucket: arn.Bucket}) + } + if ret, err := globalBucketObjectLockSys.Get(bucket); err == nil { + if ret.LockEnabled { + lock, _, _, _, err := clnt.GetObjectLockConfig(ctx, arn.Bucket) + if err != nil { + return sameTarget, errorCodes.ToAPIErrWithErr(ErrReplicationDestinationMissingLock, err) + } + if lock != objectlock.Enabled { + return sameTarget, errorCodes.ToAPIErrWithErr(ErrReplicationDestinationMissingLock, nil) + } + } + } + } + // if checked bucket, then check the ready is unnecessary + if !opts.CheckRemoteBucket && opts.CheckReady { + endpoint := clnt.EndpointURL().String() + if errInt, ok := opts.checkReadyErr.Load(endpoint); !ok { + err = checkRemoteEndpoint(ctx, clnt.EndpointURL()) + opts.checkReadyErr.Store(endpoint, err) + } else { + if errInt == nil { + err = nil + } else { + err, _ = errInt.(error) + } + } + switch err.(type) { + case BucketRemoteIdenticalToSource: + return true, errorCodes.ToAPIErrWithErr(ErrBucketRemoteIdenticalToSource, fmt.Errorf("remote target endpoint %s is self referential", clnt.EndpointURL().String())) + default: + } + } + // validate replication ARN against target endpoint + selfTarget, _ := isLocalHost(clnt.EndpointURL().Hostname(), clnt.EndpointURL().Port(), globalMinioPort) + if !sameTarget { + sameTarget = selfTarget + } + } + + if len(arns) == 0 { + return false, toAPIError(ctx, BucketRemoteTargetNotFound{Bucket: bucket}) + } + return sameTarget, toAPIError(ctx, nil) +} + +// performs a http request to remote endpoint to check if deployment id of remote endpoint is same as +// local cluster deployment id. This is to prevent replication to self, especially in case of a loadbalancer +// in front of MinIO. +func checkRemoteEndpoint(ctx context.Context, epURL *url.URL) error { + reqURL := &url.URL{ + Scheme: epURL.Scheme, + Host: epURL.Host, + Path: healthCheckPathPrefix + healthCheckReadinessPath, + } + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, reqURL.String(), nil) + if err != nil { + return err + } + + client := &http.Client{ + Transport: globalRemoteTargetTransport, + Timeout: 10 * time.Second, + } + + resp, err := client.Do(req) + if err != nil { + return err + } + if err == nil { + // Drain the connection. + xhttp.DrainBody(resp.Body) + } + if resp != nil { + amzid := resp.Header.Get(xhttp.AmzRequestHostID) + if _, ok := globalNodeNamesHex[amzid]; ok { + return BucketRemoteIdenticalToSource{ + Endpoint: epURL.String(), + } + } + } + return nil +} + +type mustReplicateOptions struct { + meta map[string]string + status replication.StatusType + opType replication.Type + replicationRequest bool // incoming request is a replication request +} + +func (o mustReplicateOptions) ReplicationStatus() (s replication.StatusType) { + if rs, ok := o.meta[xhttp.AmzBucketReplicationStatus]; ok { + return replication.StatusType(rs) + } + return s +} + +func (o mustReplicateOptions) isExistingObjectReplication() bool { + return o.opType == replication.ExistingObjectReplicationType +} + +func (o mustReplicateOptions) isMetadataReplication() bool { + return o.opType == replication.MetadataReplicationType +} + +func (o ObjectInfo) getMustReplicateOptions(op replication.Type, opts ObjectOptions) mustReplicateOptions { + return getMustReplicateOptions(o.UserDefined, o.UserTags, o.ReplicationStatus, op, opts) +} + +func getMustReplicateOptions(userDefined map[string]string, userTags string, status replication.StatusType, op replication.Type, opts ObjectOptions) mustReplicateOptions { + meta := cloneMSS(userDefined) + if userTags != "" { + meta[xhttp.AmzObjectTagging] = userTags + } + + return mustReplicateOptions{ + meta: meta, + status: status, + opType: op, + replicationRequest: opts.ReplicationRequest, + } +} + +// mustReplicate returns 2 booleans - true if object meets replication criteria and true if replication is to be done in +// a synchronous manner. +func mustReplicate(ctx context.Context, bucket, object string, mopts mustReplicateOptions) (dsc ReplicateDecision) { + // object layer not initialized we return with no decision. + if newObjectLayerFn() == nil { + return dsc + } + + // Disable server-side replication on object prefixes which are excluded + // from versioning via the MinIO bucket versioning extension. + if !globalBucketVersioningSys.PrefixEnabled(bucket, object) { + return dsc + } + + replStatus := mopts.ReplicationStatus() + if replStatus == replication.Replica && !mopts.isMetadataReplication() { + return dsc + } + + if mopts.replicationRequest { // incoming replication request on target cluster + return dsc + } + + cfg, err := getReplicationConfig(ctx, bucket) + if err != nil { + replLogOnceIf(ctx, err, bucket) + return dsc + } + if cfg == nil { + return dsc + } + + opts := replication.ObjectOpts{ + Name: object, + SSEC: crypto.SSEC.IsEncrypted(mopts.meta), + Replica: replStatus == replication.Replica, + ExistingObject: mopts.isExistingObjectReplication(), + } + tagStr, ok := mopts.meta[xhttp.AmzObjectTagging] + if ok { + opts.UserTags = tagStr + } + tgtArns := cfg.FilterTargetArns(opts) + for _, tgtArn := range tgtArns { + tgt := globalBucketTargetSys.GetRemoteTargetClient(bucket, tgtArn) + // the target online status should not be used here while deciding + // whether to replicate as the target could be temporarily down + opts.TargetArn = tgtArn + replicate := cfg.Replicate(opts) + var synchronous bool + if tgt != nil { + synchronous = tgt.replicateSync + } + dsc.Set(newReplicateTargetDecision(tgtArn, replicate, synchronous)) + } + return dsc +} + +// Standard headers that needs to be extracted from User metadata. +var standardHeaders = []string{ + xhttp.ContentType, + xhttp.CacheControl, + xhttp.ContentEncoding, + xhttp.ContentLanguage, + xhttp.ContentDisposition, + xhttp.AmzStorageClass, + xhttp.AmzObjectTagging, + xhttp.AmzBucketReplicationStatus, + xhttp.AmzObjectLockMode, + xhttp.AmzObjectLockRetainUntilDate, + xhttp.AmzObjectLockLegalHold, + xhttp.AmzTagCount, + xhttp.AmzServerSideEncryption, +} + +// returns true if any of the objects being deleted qualifies for replication. +func hasReplicationRules(ctx context.Context, bucket string, objects []ObjectToDelete) bool { + c, err := getReplicationConfig(ctx, bucket) + if err != nil || c == nil { + replLogOnceIf(ctx, err, bucket) + return false + } + for _, obj := range objects { + if c.HasActiveRules(obj.ObjectName, true) { + return true + } + } + return false +} + +// isStandardHeader returns true if header is a supported header and not a custom header +func isStandardHeader(matchHeaderKey string) bool { + return equals(matchHeaderKey, standardHeaders...) +} + +// returns whether object version is a deletemarker and if object qualifies for replication +func checkReplicateDelete(ctx context.Context, bucket string, dobj ObjectToDelete, oi ObjectInfo, delOpts ObjectOptions, gerr error) (dsc ReplicateDecision) { + rcfg, err := getReplicationConfig(ctx, bucket) + if err != nil || rcfg == nil { + replLogOnceIf(ctx, err, bucket) + return dsc + } + // If incoming request is a replication request, it does not need to be re-replicated. + if delOpts.ReplicationRequest { + return dsc + } + // Skip replication if this object's prefix is excluded from being + // versioned. + if !delOpts.Versioned { + return dsc + } + opts := replication.ObjectOpts{ + Name: dobj.ObjectName, + SSEC: crypto.SSEC.IsEncrypted(oi.UserDefined), + UserTags: oi.UserTags, + DeleteMarker: oi.DeleteMarker, + VersionID: dobj.VersionID, + OpType: replication.DeleteReplicationType, + } + tgtArns := rcfg.FilterTargetArns(opts) + dsc.targetsMap = make(map[string]replicateTargetDecision, len(tgtArns)) + if len(tgtArns) == 0 { + return dsc + } + var sync, replicate bool + for _, tgtArn := range tgtArns { + opts.TargetArn = tgtArn + replicate = rcfg.Replicate(opts) + // when incoming delete is removal of a delete marker(a.k.a versioned delete), + // GetObjectInfo returns extra information even though it returns errFileNotFound + if gerr != nil { + validReplStatus := false + switch oi.TargetReplicationStatus(tgtArn) { + case replication.Pending, replication.Completed, replication.Failed: + validReplStatus = true + } + if oi.DeleteMarker && (validReplStatus || replicate) { + dsc.Set(newReplicateTargetDecision(tgtArn, replicate, sync)) + continue + } + // can be the case that other cluster is down and duplicate `mc rm --vid` + // is issued - this still needs to be replicated back to the other target + if !oi.VersionPurgeStatus.Empty() { + replicate = oi.VersionPurgeStatus == replication.VersionPurgePending || oi.VersionPurgeStatus == replication.VersionPurgeFailed + dsc.Set(newReplicateTargetDecision(tgtArn, replicate, sync)) + } + continue + } + tgt := globalBucketTargetSys.GetRemoteTargetClient(bucket, tgtArn) + // the target online status should not be used here while deciding + // whether to replicate deletes as the target could be temporarily down + tgtDsc := newReplicateTargetDecision(tgtArn, false, false) + if tgt != nil { + tgtDsc = newReplicateTargetDecision(tgtArn, replicate, tgt.replicateSync) + } + dsc.Set(tgtDsc) + } + return dsc +} + +// replicate deletes to the designated replication target if replication configuration +// has delete marker replication or delete replication (MinIO extension to allow deletes where version id +// is specified) enabled. +// Similar to bucket replication for PUT operation, soft delete (a.k.a setting delete marker) and +// permanent deletes (by specifying a version ID in the delete operation) have three states "Pending", "Complete" +// and "Failed" to mark the status of the replication of "DELETE" operation. All failed operations can +// then be retried by healing. In the case of permanent deletes, until the replication is completed on the +// target cluster, the object version is marked deleted on the source and hidden from listing. It is permanently +// deleted from the source when the VersionPurgeStatus changes to "Complete", i.e after replication succeeds +// on target. +func replicateDelete(ctx context.Context, dobj DeletedObjectReplicationInfo, objectAPI ObjectLayer) { + var replicationStatus replication.StatusType + bucket := dobj.Bucket + versionID := dobj.DeleteMarkerVersionID + if versionID == "" { + versionID = dobj.VersionID + } + + defer func() { + replStatus := string(replicationStatus) + auditLogInternal(context.Background(), AuditLogOptions{ + Event: dobj.EventType, + APIName: ReplicateDeleteAPI, + Bucket: bucket, + Object: dobj.ObjectName, + VersionID: versionID, + Status: replStatus, + }) + }() + + rcfg, err := getReplicationConfig(ctx, bucket) + if err != nil || rcfg == nil { + replLogOnceIf(ctx, fmt.Errorf("unable to obtain replication config for bucket: %s: err: %s", bucket, err), bucket) + sendEvent(eventArgs{ + BucketName: bucket, + Object: ObjectInfo{ + Bucket: bucket, + Name: dobj.ObjectName, + VersionID: versionID, + DeleteMarker: dobj.DeleteMarker, + }, + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + EventName: event.ObjectReplicationNotTracked, + }) + return + } + dsc, err := parseReplicateDecision(ctx, bucket, dobj.ReplicationState.ReplicateDecisionStr) + if err != nil { + replLogOnceIf(ctx, fmt.Errorf("unable to parse replication decision parameters for bucket: %s, err: %s, decision: %s", + bucket, err, dobj.ReplicationState.ReplicateDecisionStr), dobj.ReplicationState.ReplicateDecisionStr) + sendEvent(eventArgs{ + BucketName: bucket, + Object: ObjectInfo{ + Bucket: bucket, + Name: dobj.ObjectName, + VersionID: versionID, + DeleteMarker: dobj.DeleteMarker, + }, + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + EventName: event.ObjectReplicationNotTracked, + }) + return + } + + // Lock the object name before starting replication operation. + // Use separate lock that doesn't collide with regular objects. + lk := objectAPI.NewNSLock(bucket, "/[replicate]/"+dobj.ObjectName) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + globalReplicationPool.Get().queueMRFSave(dobj.ToMRFEntry()) + sendEvent(eventArgs{ + BucketName: bucket, + Object: ObjectInfo{ + Bucket: bucket, + Name: dobj.ObjectName, + VersionID: versionID, + DeleteMarker: dobj.DeleteMarker, + }, + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + EventName: event.ObjectReplicationNotTracked, + }) + return + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + + rinfos := replicatedInfos{Targets: make([]replicatedTargetInfo, 0, len(dsc.targetsMap))} + var wg sync.WaitGroup + var mu sync.Mutex + for _, tgtEntry := range dsc.targetsMap { + if !tgtEntry.Replicate { + continue + } + // if dobj.TargetArn is not empty string, this is a case of specific target being re-synced. + if dobj.TargetArn != "" && dobj.TargetArn != tgtEntry.Arn { + continue + } + tgtClnt := globalBucketTargetSys.GetRemoteTargetClient(bucket, tgtEntry.Arn) + if tgtClnt == nil { + // Skip stale targets if any and log them to be missing at least once. + replLogOnceIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, tgtEntry.Arn), tgtEntry.Arn) + sendEvent(eventArgs{ + EventName: event.ObjectReplicationNotTracked, + BucketName: bucket, + Object: ObjectInfo{ + Bucket: bucket, + Name: dobj.ObjectName, + VersionID: versionID, + DeleteMarker: dobj.DeleteMarker, + }, + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + }) + continue + } + wg.Add(1) + go func(tgt *TargetClient) { + defer wg.Done() + tgtInfo := replicateDeleteToTarget(ctx, dobj, tgt) + + mu.Lock() + rinfos.Targets = append(rinfos.Targets, tgtInfo) + mu.Unlock() + }(tgtClnt) + } + wg.Wait() + + replicationStatus = rinfos.ReplicationStatus() + prevStatus := dobj.DeleteMarkerReplicationStatus() + + if dobj.VersionID != "" { + prevStatus = replication.StatusType(dobj.VersionPurgeStatus()) + replicationStatus = replication.StatusType(rinfos.VersionPurgeStatus()) + } + + // to decrement pending count later. + for _, rinfo := range rinfos.Targets { + if rinfo.ReplicationStatus != rinfo.PrevReplicationStatus { + globalReplicationStats.Load().Update(dobj.Bucket, rinfo, replicationStatus, + prevStatus) + } + } + + eventName := event.ObjectReplicationComplete + if replicationStatus == replication.Failed { + eventName = event.ObjectReplicationFailed + globalReplicationPool.Get().queueMRFSave(dobj.ToMRFEntry()) + } + drs := getReplicationState(rinfos, dobj.ReplicationState, dobj.VersionID) + if replicationStatus != prevStatus { + drs.ReplicationTimeStamp = UTCNow() + } + + dobjInfo, err := objectAPI.DeleteObject(ctx, bucket, dobj.ObjectName, ObjectOptions{ + VersionID: versionID, + MTime: dobj.DeleteMarkerMTime.Time, + DeleteReplication: drs, + Versioned: globalBucketVersioningSys.PrefixEnabled(bucket, dobj.ObjectName), + // Objects matching prefixes should not leave delete markers, + // dramatically reduces namespace pollution while keeping the + // benefits of replication, make sure to apply version suspension + // only at bucket level instead. + VersionSuspended: globalBucketVersioningSys.Suspended(bucket), + }) + if err != nil && !isErrVersionNotFound(err) { // VersionNotFound would be reported by pool that object version is missing on. + sendEvent(eventArgs{ + BucketName: bucket, + Object: ObjectInfo{ + Bucket: bucket, + Name: dobj.ObjectName, + VersionID: versionID, + DeleteMarker: dobj.DeleteMarker, + }, + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + EventName: eventName, + }) + } else { + sendEvent(eventArgs{ + BucketName: bucket, + Object: dobjInfo, + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + EventName: eventName, + }) + } +} + +func replicateDeleteToTarget(ctx context.Context, dobj DeletedObjectReplicationInfo, tgt *TargetClient) (rinfo replicatedTargetInfo) { + versionID := dobj.DeleteMarkerVersionID + if versionID == "" { + versionID = dobj.VersionID + } + + rinfo = dobj.ReplicationState.targetState(tgt.ARN) + rinfo.OpType = dobj.OpType + rinfo.endpoint = tgt.EndpointURL().Host + rinfo.secure = tgt.EndpointURL().Scheme == "https" + defer func() { + if rinfo.ReplicationStatus == replication.Completed && tgt.ResetID != "" && dobj.OpType == replication.ExistingObjectReplicationType { + rinfo.ResyncTimestamp = fmt.Sprintf("%s;%s", UTCNow().Format(http.TimeFormat), tgt.ResetID) + } + }() + + if dobj.VersionID == "" && rinfo.PrevReplicationStatus == replication.Completed && dobj.OpType != replication.ExistingObjectReplicationType { + rinfo.ReplicationStatus = rinfo.PrevReplicationStatus + return rinfo + } + if dobj.VersionID != "" && rinfo.VersionPurgeStatus == replication.VersionPurgeComplete { + return rinfo + } + if globalBucketTargetSys.isOffline(tgt.EndpointURL()) { + replLogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s", dobj.Bucket, tgt.ARN), "replication-target-offline-delete-"+tgt.ARN) + sendEvent(eventArgs{ + BucketName: dobj.Bucket, + Object: ObjectInfo{ + Bucket: dobj.Bucket, + Name: dobj.ObjectName, + VersionID: dobj.VersionID, + DeleteMarker: dobj.DeleteMarker, + }, + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + EventName: event.ObjectReplicationNotTracked, + }) + if dobj.VersionID == "" { + rinfo.ReplicationStatus = replication.Failed + } else { + rinfo.VersionPurgeStatus = replication.VersionPurgeFailed + } + return rinfo + } + // early return if already replicated delete marker for existing object replication/ healing delete markers + if dobj.DeleteMarkerVersionID != "" { + toi, err := tgt.StatObject(ctx, tgt.Bucket, dobj.ObjectName, minio.StatObjectOptions{ + VersionID: versionID, + Internal: minio.AdvancedGetOptions{ + ReplicationProxyRequest: "false", + IsReplicationReadyForDeleteMarker: true, + }, + }) + serr := ErrorRespToObjectError(err, dobj.Bucket, dobj.ObjectName, dobj.VersionID) + switch { + case isErrMethodNotAllowed(serr): + // delete marker already replicated + if dobj.VersionID == "" && rinfo.VersionPurgeStatus.Empty() { + rinfo.ReplicationStatus = replication.Completed + return rinfo + } + case isErrObjectNotFound(serr), isErrVersionNotFound(serr): + // version being purged is already not found on target. + if !rinfo.VersionPurgeStatus.Empty() { + rinfo.VersionPurgeStatus = replication.VersionPurgeComplete + return rinfo + } + case isErrReadQuorum(serr), isErrWriteQuorum(serr): + // destination has some quorum issues, perform removeObject() anyways + // to complete the operation. + default: + if err != nil && minio.IsNetworkOrHostDown(err, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) { + globalBucketTargetSys.markOffline(tgt.EndpointURL()) + } + // mark delete marker replication as failed if target cluster not ready to receive + // this request yet (object version not replicated yet) + if err != nil && !toi.ReplicationReady { + rinfo.ReplicationStatus = replication.Failed + rinfo.Err = err + return rinfo + } + } + } + rmErr := tgt.RemoveObject(ctx, tgt.Bucket, dobj.ObjectName, minio.RemoveObjectOptions{ + VersionID: versionID, + Internal: minio.AdvancedRemoveOptions{ + ReplicationDeleteMarker: dobj.DeleteMarkerVersionID != "", + ReplicationMTime: dobj.DeleteMarkerMTime.Time, + ReplicationStatus: minio.ReplicationStatusReplica, + ReplicationRequest: true, // always set this to distinguish between `mc mirror` replication and serverside + }, + }) + if rmErr != nil { + rinfo.Err = rmErr + if dobj.VersionID == "" { + rinfo.ReplicationStatus = replication.Failed + } else { + rinfo.VersionPurgeStatus = replication.VersionPurgeFailed + } + replLogIf(ctx, fmt.Errorf("unable to replicate delete marker to %s: %s/%s(%s): %w", tgt.EndpointURL(), tgt.Bucket, dobj.ObjectName, versionID, rmErr)) + if rmErr != nil && minio.IsNetworkOrHostDown(rmErr, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) { + globalBucketTargetSys.markOffline(tgt.EndpointURL()) + } + } else { + if dobj.VersionID == "" { + rinfo.ReplicationStatus = replication.Completed + } else { + rinfo.VersionPurgeStatus = replication.VersionPurgeComplete + } + } + return rinfo +} + +func getCopyObjMetadata(oi ObjectInfo, sc string) map[string]string { + meta := make(map[string]string, len(oi.UserDefined)) + for k, v := range oi.UserDefined { + if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) { + continue + } + + if equals(k, xhttp.AmzBucketReplicationStatus) { + continue + } + + // https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w + if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) { + continue + } + meta[k] = v + } + + if oi.ContentEncoding != "" { + meta[xhttp.ContentEncoding] = oi.ContentEncoding + } + + if oi.ContentType != "" { + meta[xhttp.ContentType] = oi.ContentType + } + + meta[xhttp.AmzObjectTagging] = oi.UserTags + meta[xhttp.AmzTagDirective] = "REPLACE" + + if sc == "" { + sc = oi.StorageClass + } + // drop non standard storage classes for tiering from replication + if sc != "" && (sc == storageclass.RRS || sc == storageclass.STANDARD) { + meta[xhttp.AmzStorageClass] = sc + } + + meta[xhttp.MinIOSourceETag] = oi.ETag + meta[xhttp.MinIOSourceMTime] = oi.ModTime.UTC().Format(time.RFC3339Nano) + meta[xhttp.AmzBucketReplicationStatus] = replication.Replica.String() + return meta +} + +type caseInsensitiveMap map[string]string + +// Lookup map entry case insensitively. +func (m caseInsensitiveMap) Lookup(key string) (string, bool) { + if len(m) == 0 { + return "", false + } + for _, k := range []string{ + key, + strings.ToLower(key), + http.CanonicalHeaderKey(key), + } { + v, ok := m[k] + if ok { + return v, ok + } + } + return "", false +} + +func putReplicationOpts(ctx context.Context, sc string, objInfo ObjectInfo) (putOpts minio.PutObjectOptions, isMP bool, err error) { + meta := make(map[string]string) + isSSEC := crypto.SSEC.IsEncrypted(objInfo.UserDefined) + + for k, v := range objInfo.UserDefined { + _, isValidSSEHeader := validSSEReplicationHeaders[k] + // In case of SSE-C objects copy the allowed internal headers as well + if !isSSEC || !isValidSSEHeader { + if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) { + continue + } + if isStandardHeader(k) { + continue + } + } + if isValidSSEHeader { + meta[validSSEReplicationHeaders[k]] = v + } else { + meta[k] = v + } + } + isMP = objInfo.isMultipart() + if len(objInfo.Checksum) > 0 { + // Add encrypted CRC to metadata for SSE-C objects. + if isSSEC { + meta[ReplicationSsecChecksumHeader] = base64.StdEncoding.EncodeToString(objInfo.Checksum) + } else { + cs, mp := getCRCMeta(objInfo, 0, nil) + // Set object checksum. + maps.Copy(meta, cs) + isMP = mp + if !objInfo.isMultipart() && cs[xhttp.AmzChecksumType] == xhttp.AmzChecksumTypeFullObject { + // For objects where checksum is full object, it will be the same. + // Therefore, we use the cheaper PutObject replication. + isMP = false + } + } + } + + if sc == "" && (objInfo.StorageClass == storageclass.STANDARD || objInfo.StorageClass == storageclass.RRS) { + sc = objInfo.StorageClass + } + putOpts = minio.PutObjectOptions{ + UserMetadata: meta, + ContentType: objInfo.ContentType, + ContentEncoding: objInfo.ContentEncoding, + Expires: objInfo.Expires, + StorageClass: sc, + Internal: minio.AdvancedPutOptions{ + SourceVersionID: objInfo.VersionID, + ReplicationStatus: minio.ReplicationStatusReplica, + SourceMTime: objInfo.ModTime, + SourceETag: objInfo.ETag, + ReplicationRequest: true, // always set this to distinguish between `mc mirror` replication and serverside + }, + } + if objInfo.UserTags != "" { + tag, _ := tags.ParseObjectTags(objInfo.UserTags) + if tag != nil { + putOpts.UserTags = tag.ToMap() + // set tag timestamp in opts + tagTimestamp := objInfo.ModTime + if tagTmstampStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp]; ok { + tagTimestamp, err = time.Parse(time.RFC3339Nano, tagTmstampStr) + if err != nil { + return putOpts, false, err + } + } + putOpts.Internal.TaggingTimestamp = tagTimestamp + } + } + + lkMap := caseInsensitiveMap(objInfo.UserDefined) + if lang, ok := lkMap.Lookup(xhttp.ContentLanguage); ok { + putOpts.ContentLanguage = lang + } + if disp, ok := lkMap.Lookup(xhttp.ContentDisposition); ok { + putOpts.ContentDisposition = disp + } + if cc, ok := lkMap.Lookup(xhttp.CacheControl); ok { + putOpts.CacheControl = cc + } + if mode, ok := lkMap.Lookup(xhttp.AmzObjectLockMode); ok { + rmode := minio.RetentionMode(mode) + putOpts.Mode = rmode + } + if retainDateStr, ok := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate); ok { + rdate, err := amztime.ISO8601Parse(retainDateStr) + if err != nil { + return putOpts, false, err + } + putOpts.RetainUntilDate = rdate + // set retention timestamp in opts + retTimestamp := objInfo.ModTime + if retainTmstampStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp]; ok { + retTimestamp, err = time.Parse(time.RFC3339Nano, retainTmstampStr) + if err != nil { + return putOpts, false, err + } + } + putOpts.Internal.RetentionTimestamp = retTimestamp + } + if lhold, ok := lkMap.Lookup(xhttp.AmzObjectLockLegalHold); ok { + putOpts.LegalHold = minio.LegalHoldStatus(lhold) + // set legalhold timestamp in opts + lholdTimestamp := objInfo.ModTime + if lholdTmstampStr, ok := objInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp]; ok { + lholdTimestamp, err = time.Parse(time.RFC3339Nano, lholdTmstampStr) + if err != nil { + return putOpts, false, err + } + } + putOpts.Internal.LegalholdTimestamp = lholdTimestamp + } + if crypto.S3.IsEncrypted(objInfo.UserDefined) { + putOpts.ServerSideEncryption = encrypt.NewSSE() + } + + if crypto.S3KMS.IsEncrypted(objInfo.UserDefined) { + // If KMS key ID replication is enabled (as by default) + // we include the object's KMS key ID. In any case, we + // always set the SSE-KMS header. If no KMS key ID is + // specified, MinIO is supposed to use whatever default + // config applies on the site or bucket. + var keyID string + if kms.ReplicateKeyID() { + keyID = objInfo.KMSKeyID() + } + + sseEnc, err := encrypt.NewSSEKMS(keyID, nil) + if err != nil { + return putOpts, false, err + } + putOpts.ServerSideEncryption = sseEnc + } + return putOpts, isMP, err +} + +type replicationAction string + +const ( + replicateMetadata replicationAction = "metadata" + replicateNone replicationAction = "none" + replicateAll replicationAction = "all" +) + +// matches k1 with all keys, returns 'true' if one of them matches +func equals(k1 string, keys ...string) bool { + for _, k2 := range keys { + if strings.EqualFold(k1, k2) { + return true + } + } + return false +} + +// returns replicationAction by comparing metadata between source and target +func getReplicationAction(oi1 ObjectInfo, oi2 minio.ObjectInfo, opType replication.Type) replicationAction { + // Avoid resyncing null versions created prior to enabling replication if target has a newer copy + if opType == replication.ExistingObjectReplicationType && + oi1.ModTime.Unix() > oi2.LastModified.Unix() && oi1.VersionID == nullVersionID { + return replicateNone + } + sz, _ := oi1.GetActualSize() + + // needs full replication + if oi1.ETag != oi2.ETag || + oi1.VersionID != oi2.VersionID || + sz != oi2.Size || + oi1.DeleteMarker != oi2.IsDeleteMarker || + oi1.ModTime.Unix() != oi2.LastModified.Unix() { + return replicateAll + } + + if oi1.ContentType != oi2.ContentType { + return replicateMetadata + } + + if oi1.ContentEncoding != "" { + enc, ok := oi2.Metadata[xhttp.ContentEncoding] + if !ok { + enc, ok = oi2.Metadata[strings.ToLower(xhttp.ContentEncoding)] + if !ok { + return replicateMetadata + } + } + if strings.Join(enc, ",") != oi1.ContentEncoding { + return replicateMetadata + } + } + + t, _ := tags.ParseObjectTags(oi1.UserTags) + oi2Map := make(map[string]string) + maps.Copy(oi2Map, oi2.UserTags) + if (oi2.UserTagCount > 0 && !reflect.DeepEqual(oi2Map, t.ToMap())) || (oi2.UserTagCount != len(t.ToMap())) { + return replicateMetadata + } + + // Compare only necessary headers + compareKeys := []string{ + "Expires", + "Cache-Control", + "Content-Language", + "Content-Disposition", + "X-Amz-Object-Lock-Mode", + "X-Amz-Object-Lock-Retain-Until-Date", + "X-Amz-Object-Lock-Legal-Hold", + "X-Amz-Website-Redirect-Location", + "X-Amz-Meta-", + } + + // compare metadata on both maps to see if meta is identical + compareMeta1 := make(map[string]string) + for k, v := range oi1.UserDefined { + var found bool + for _, prefix := range compareKeys { + if !stringsHasPrefixFold(k, prefix) { + continue + } + found = true + break + } + if found { + compareMeta1[strings.ToLower(k)] = v + } + } + + compareMeta2 := make(map[string]string) + for k, v := range oi2.Metadata { + var found bool + for _, prefix := range compareKeys { + if !stringsHasPrefixFold(k, prefix) { + continue + } + found = true + break + } + if found { + compareMeta2[strings.ToLower(k)] = strings.Join(v, ",") + } + } + + if !reflect.DeepEqual(compareMeta1, compareMeta2) { + return replicateMetadata + } + + return replicateNone +} + +// replicateObject replicates the specified version of the object to destination bucket +// The source object is then updated to reflect the replication status. +func replicateObject(ctx context.Context, ri ReplicateObjectInfo, objectAPI ObjectLayer) { + var replicationStatus replication.StatusType + defer func() { + if replicationStatus.Empty() { + // replication status is empty means + // replication was not attempted for some + // reason, notify the state of the object + // on disk. + replicationStatus = ri.ReplicationStatus + } + auditLogInternal(ctx, AuditLogOptions{ + Event: ri.EventType, + APIName: ReplicateObjectAPI, + Bucket: ri.Bucket, + Object: ri.Name, + VersionID: ri.VersionID, + Status: replicationStatus.String(), + }) + }() + + bucket := ri.Bucket + object := ri.Name + + cfg, err := getReplicationConfig(ctx, bucket) + if err != nil || cfg == nil { + replLogOnceIf(ctx, err, "get-replication-config-"+bucket) + sendEvent(eventArgs{ + EventName: event.ObjectReplicationNotTracked, + BucketName: bucket, + Object: ri.ToObjectInfo(), + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + }) + return + } + tgtArns := cfg.FilterTargetArns(replication.ObjectOpts{ + Name: object, + SSEC: ri.SSEC, + UserTags: ri.UserTags, + }) + // Lock the object name before starting replication. + // Use separate lock that doesn't collide with regular objects. + lk := objectAPI.NewNSLock(bucket, "/[replicate]/"+object) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + sendEvent(eventArgs{ + EventName: event.ObjectReplicationNotTracked, + BucketName: bucket, + Object: ri.ToObjectInfo(), + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + }) + globalReplicationPool.Get().queueMRFSave(ri.ToMRFEntry()) + return + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + + rinfos := replicatedInfos{Targets: make([]replicatedTargetInfo, 0, len(tgtArns))} + var wg sync.WaitGroup + var mu sync.Mutex + for _, tgtArn := range tgtArns { + tgt := globalBucketTargetSys.GetRemoteTargetClient(bucket, tgtArn) + if tgt == nil { + replLogOnceIf(ctx, fmt.Errorf("failed to get target for bucket:%s arn:%s", bucket, tgtArn), tgtArn) + sendEvent(eventArgs{ + EventName: event.ObjectReplicationNotTracked, + BucketName: bucket, + Object: ri.ToObjectInfo(), + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + }) + continue + } + wg.Add(1) + go func(tgt *TargetClient) { + defer wg.Done() + + var tgtInfo replicatedTargetInfo + if ri.OpType == replication.ObjectReplicationType { + // all incoming calls go through optimized path. + tgtInfo = ri.replicateObject(ctx, objectAPI, tgt) + } else { + tgtInfo = ri.replicateAll(ctx, objectAPI, tgt) + } + + mu.Lock() + rinfos.Targets = append(rinfos.Targets, tgtInfo) + mu.Unlock() + }(tgt) + } + wg.Wait() + + replicationStatus = rinfos.ReplicationStatus() // used in defer function + // FIXME: add support for missing replication events + // - event.ObjectReplicationMissedThreshold + // - event.ObjectReplicationReplicatedAfterThreshold + eventName := event.ObjectReplicationComplete + if replicationStatus == replication.Failed { + eventName = event.ObjectReplicationFailed + } + newReplStatusInternal := rinfos.ReplicationStatusInternal() + // Note that internal replication status(es) may match for previously replicated objects - in such cases + // metadata should be updated with last resync timestamp. + objInfo := ri.ToObjectInfo() + if ri.ReplicationStatusInternal != newReplStatusInternal || rinfos.ReplicationResynced() { + popts := ObjectOptions{ + MTime: ri.ModTime, + VersionID: ri.VersionID, + EvalMetadataFn: func(oi *ObjectInfo, gerr error) (dsc ReplicateDecision, err error) { + oi.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = newReplStatusInternal + oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano) + oi.UserDefined[xhttp.AmzBucketReplicationStatus] = string(rinfos.ReplicationStatus()) + for _, rinfo := range rinfos.Targets { + if rinfo.ResyncTimestamp != "" { + oi.UserDefined[targetResetHeader(rinfo.Arn)] = rinfo.ResyncTimestamp + } + } + if ri.UserTags != "" { + oi.UserDefined[xhttp.AmzObjectTagging] = ri.UserTags + } + return dsc, nil + }, + } + + uobjInfo, _ := objectAPI.PutObjectMetadata(ctx, bucket, object, popts) + if uobjInfo.Name != "" { + objInfo = uobjInfo + } + + opType := replication.MetadataReplicationType + if rinfos.Action() == replicateAll { + opType = replication.ObjectReplicationType + } + for _, rinfo := range rinfos.Targets { + if rinfo.ReplicationStatus != rinfo.PrevReplicationStatus { + rinfo.OpType = opType // update optype to reflect correct operation. + globalReplicationStats.Load().Update(bucket, rinfo, rinfo.ReplicationStatus, rinfo.PrevReplicationStatus) + } + } + } + + sendEvent(eventArgs{ + EventName: eventName, + BucketName: bucket, + Object: objInfo, + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + }) + + // re-queue failures once more - keep a retry count to avoid flooding the queue if + // the target site is down. Leave it to scanner to catch up instead. + if rinfos.ReplicationStatus() != replication.Completed { + ri.OpType = replication.HealReplicationType + ri.EventType = ReplicateMRF + ri.ReplicationStatusInternal = rinfos.ReplicationStatusInternal() + ri.RetryCount++ + globalReplicationPool.Get().queueMRFSave(ri.ToMRFEntry()) + } +} + +// replicateObject replicates object data for specified version of the object to destination bucket +// The source object is then updated to reflect the replication status. +func (ri ReplicateObjectInfo) replicateObject(ctx context.Context, objectAPI ObjectLayer, tgt *TargetClient) (rinfo replicatedTargetInfo) { + startTime := time.Now() + bucket := ri.Bucket + object := ri.Name + + rAction := replicateAll + rinfo = replicatedTargetInfo{ + Size: ri.ActualSize, + Arn: tgt.ARN, + PrevReplicationStatus: ri.TargetReplicationStatus(tgt.ARN), + ReplicationStatus: replication.Failed, + OpType: ri.OpType, + ReplicationAction: rAction, + endpoint: tgt.EndpointURL().Host, + secure: tgt.EndpointURL().Scheme == "https", + } + if ri.TargetReplicationStatus(tgt.ARN) == replication.Completed && !ri.ExistingObjResync.Empty() && !ri.ExistingObjResync.mustResyncTarget(tgt.ARN) { + rinfo.ReplicationStatus = replication.Completed + rinfo.ReplicationResynced = true + return rinfo + } + + if globalBucketTargetSys.isOffline(tgt.EndpointURL()) { + replLogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s retry:%d", bucket, tgt.ARN, ri.RetryCount), "replication-target-offline"+tgt.ARN) + sendEvent(eventArgs{ + EventName: event.ObjectReplicationNotTracked, + BucketName: bucket, + Object: ri.ToObjectInfo(), + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + }) + return rinfo + } + + versioned := globalBucketVersioningSys.PrefixEnabled(bucket, object) + versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object) + + gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, ObjectOptions{ + VersionID: ri.VersionID, + Versioned: versioned, + VersionSuspended: versionSuspended, + ReplicationRequest: true, + }) + if err != nil { + if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) { + objInfo := ri.ToObjectInfo() + sendEvent(eventArgs{ + EventName: event.ObjectReplicationNotTracked, + BucketName: bucket, + Object: objInfo, + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + }) + replLogOnceIf(ctx, fmt.Errorf("unable to read source object %s/%s(%s): %w", bucket, object, objInfo.VersionID, err), object+":"+objInfo.VersionID) + } + return rinfo + } + defer gr.Close() + + objInfo := gr.ObjInfo + + // make sure we have the latest metadata for metrics calculation + rinfo.PrevReplicationStatus = objInfo.TargetReplicationStatus(tgt.ARN) + + // Set the encrypted size for SSE-C objects + var size int64 + if crypto.SSEC.IsEncrypted(objInfo.UserDefined) { + size = objInfo.Size + } else { + size, err = objInfo.GetActualSize() + if err != nil { + replLogIf(ctx, err) + sendEvent(eventArgs{ + EventName: event.ObjectReplicationNotTracked, + BucketName: bucket, + Object: objInfo, + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + }) + return rinfo + } + } + + if tgt.Bucket == "" { + replLogIf(ctx, fmt.Errorf("unable to replicate object %s(%s), bucket is empty for target %s", objInfo.Name, objInfo.VersionID, tgt.EndpointURL())) + sendEvent(eventArgs{ + EventName: event.ObjectReplicationNotTracked, + BucketName: bucket, + Object: objInfo, + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + }) + return rinfo + } + defer func() { + if rinfo.ReplicationStatus == replication.Completed && ri.OpType == replication.ExistingObjectReplicationType && tgt.ResetID != "" { + rinfo.ResyncTimestamp = fmt.Sprintf("%s;%s", UTCNow().Format(http.TimeFormat), tgt.ResetID) + rinfo.ReplicationResynced = true + } + rinfo.Duration = time.Since(startTime) + }() + + rinfo.ReplicationStatus = replication.Completed + rinfo.Size = size + rinfo.ReplicationAction = rAction + // use core client to avoid doing multipart on PUT + c := &minio.Core{Client: tgt.Client} + + putOpts, isMP, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo) + if err != nil { + replLogIf(ctx, fmt.Errorf("failure setting options for replication bucket:%s err:%w", bucket, err)) + sendEvent(eventArgs{ + EventName: event.ObjectReplicationNotTracked, + BucketName: bucket, + Object: objInfo, + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + }) + return rinfo + } + + var headerSize int + for k, v := range putOpts.Header() { + headerSize += len(k) + len(v) + } + + opts := &bandwidth.MonitorReaderOptions{ + BucketOptions: bandwidth.BucketOptions{ + Name: ri.Bucket, + ReplicationARN: tgt.ARN, + }, + HeaderSize: headerSize, + } + newCtx := ctx + if globalBucketMonitor.IsThrottled(bucket, tgt.ARN) && objInfo.Size < minLargeObjSize { + var cancel context.CancelFunc + newCtx, cancel = context.WithTimeout(ctx, throttleDeadline) + defer cancel() + } + r := bandwidth.NewMonitoredReader(newCtx, globalBucketMonitor, gr, opts) + if isMP { + rinfo.Err = replicateObjectWithMultipart(ctx, c, tgt.Bucket, object, r, objInfo, putOpts) + } else { + _, rinfo.Err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts) + } + if rinfo.Err != nil { + if minio.ToErrorResponse(rinfo.Err).Code != "PreconditionFailed" { + rinfo.ReplicationStatus = replication.Failed + replLogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s): to (target: %s): %w", + bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err)) + } + if minio.IsNetworkOrHostDown(rinfo.Err, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) { + globalBucketTargetSys.markOffline(tgt.EndpointURL()) + } + } + return rinfo +} + +// replicateAll replicates metadata for specified version of the object to destination bucket +// if the destination version is missing it automatically does fully copy as well. +// The source object is then updated to reflect the replication status. +func (ri ReplicateObjectInfo) replicateAll(ctx context.Context, objectAPI ObjectLayer, tgt *TargetClient) (rinfo replicatedTargetInfo) { + startTime := time.Now() + bucket := ri.Bucket + object := ri.Name + + // set defaults for replication action based on operation being performed - actual + // replication action can only be determined after stat on remote. This default is + // needed for updating replication metrics correctly when target is offline. + rAction := replicateMetadata + + rinfo = replicatedTargetInfo{ + Size: ri.ActualSize, + Arn: tgt.ARN, + PrevReplicationStatus: ri.TargetReplicationStatus(tgt.ARN), + ReplicationStatus: replication.Failed, + OpType: ri.OpType, + ReplicationAction: rAction, + endpoint: tgt.EndpointURL().Host, + secure: tgt.EndpointURL().Scheme == "https", + } + + if globalBucketTargetSys.isOffline(tgt.EndpointURL()) { + replLogOnceIf(ctx, fmt.Errorf("remote target is offline for bucket:%s arn:%s retry:%d", bucket, tgt.ARN, ri.RetryCount), "replication-target-offline-heal"+tgt.ARN) + sendEvent(eventArgs{ + EventName: event.ObjectReplicationNotTracked, + BucketName: bucket, + Object: ri.ToObjectInfo(), + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + }) + return rinfo + } + + versioned := globalBucketVersioningSys.PrefixEnabled(bucket, object) + versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object) + + gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, nil, http.Header{}, + ObjectOptions{ + VersionID: ri.VersionID, + Versioned: versioned, + VersionSuspended: versionSuspended, + ReplicationRequest: true, + }) + if err != nil { + if !isErrVersionNotFound(err) && !isErrObjectNotFound(err) { + objInfo := ri.ToObjectInfo() + sendEvent(eventArgs{ + EventName: event.ObjectReplicationNotTracked, + BucketName: bucket, + Object: objInfo, + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + }) + replLogIf(ctx, fmt.Errorf("unable to replicate to target %s for %s/%s(%s): %w", tgt.EndpointURL(), bucket, object, objInfo.VersionID, err)) + } + return rinfo + } + defer gr.Close() + + objInfo := gr.ObjInfo + + // make sure we have the latest metadata for metrics calculation + rinfo.PrevReplicationStatus = objInfo.TargetReplicationStatus(tgt.ARN) + + // use latest ObjectInfo to check if previous replication attempt succeeded + if objInfo.TargetReplicationStatus(tgt.ARN) == replication.Completed && !ri.ExistingObjResync.Empty() && !ri.ExistingObjResync.mustResyncTarget(tgt.ARN) { + rinfo.ReplicationStatus = replication.Completed + rinfo.ReplicationResynced = true + return rinfo + } + + size, err := objInfo.GetActualSize() + if err != nil { + replLogIf(ctx, err) + sendEvent(eventArgs{ + EventName: event.ObjectReplicationNotTracked, + BucketName: bucket, + Object: objInfo, + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + }) + return rinfo + } + + // Set the encrypted size for SSE-C objects + isSSEC := crypto.SSEC.IsEncrypted(objInfo.UserDefined) + if isSSEC { + size = objInfo.Size + } + + if tgt.Bucket == "" { + replLogIf(ctx, fmt.Errorf("unable to replicate object %s(%s) to %s, target bucket is missing", objInfo.Name, objInfo.VersionID, tgt.EndpointURL())) + sendEvent(eventArgs{ + EventName: event.ObjectReplicationNotTracked, + BucketName: bucket, + Object: objInfo, + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + }) + return rinfo + } + defer func() { + if rinfo.ReplicationStatus == replication.Completed && ri.OpType == replication.ExistingObjectReplicationType && tgt.ResetID != "" { + rinfo.ResyncTimestamp = fmt.Sprintf("%s;%s", UTCNow().Format(http.TimeFormat), tgt.ResetID) + rinfo.ReplicationResynced = true + } + rinfo.Duration = time.Since(startTime) + }() + sOpts := minio.StatObjectOptions{ + VersionID: objInfo.VersionID, + Internal: minio.AdvancedGetOptions{ + ReplicationProxyRequest: "false", + }, + } + sOpts.Set(xhttp.AmzTagDirective, "ACCESS") + oi, cerr := tgt.StatObject(ctx, tgt.Bucket, object, sOpts) + if cerr == nil { + rAction = getReplicationAction(objInfo, oi, ri.OpType) + rinfo.ReplicationStatus = replication.Completed + if rAction == replicateNone { + if ri.OpType == replication.ExistingObjectReplicationType && + objInfo.ModTime.Unix() > oi.LastModified.Unix() && objInfo.VersionID == nullVersionID { + replLogIf(ctx, fmt.Errorf("unable to replicate %s/%s (null). Newer version exists on target %s", bucket, object, tgt.EndpointURL())) + sendEvent(eventArgs{ + EventName: event.ObjectReplicationNotTracked, + BucketName: bucket, + Object: objInfo, + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + }) + } + // object with same VersionID already exists, replication kicked off by + // PutObject might have completed + if objInfo.TargetReplicationStatus(tgt.ARN) == replication.Pending || + objInfo.TargetReplicationStatus(tgt.ARN) == replication.Failed || + ri.OpType == replication.ExistingObjectReplicationType { + // if metadata is not updated for some reason after replication, such as + // 503 encountered while updating metadata - make sure to set ReplicationStatus + // as Completed. + // + // Note: Replication Stats would have been updated despite metadata update failure. + rinfo.ReplicationAction = rAction + rinfo.ReplicationStatus = replication.Completed + } + return rinfo + } + } else { + // SSEC objects will refuse HeadObject without the decryption key. + // Ignore the error, since we know the object exists and versioning prevents overwriting existing versions. + if isSSEC && strings.Contains(cerr.Error(), errorCodes[ErrSSEEncryptedObject].Description) { + rinfo.ReplicationStatus = replication.Completed + rinfo.ReplicationAction = replicateNone + goto applyAction + } + // if target returns error other than NoSuchKey, defer replication attempt + if minio.IsNetworkOrHostDown(cerr, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) { + globalBucketTargetSys.markOffline(tgt.EndpointURL()) + } + + serr := ErrorRespToObjectError(cerr, bucket, object, objInfo.VersionID) + switch { + case isErrMethodNotAllowed(serr): + rAction = replicateAll + case isErrObjectNotFound(serr), isErrVersionNotFound(serr): + rAction = replicateAll + case isErrReadQuorum(serr), isErrWriteQuorum(serr): + rAction = replicateAll + default: + rinfo.Err = cerr + replLogIf(ctx, fmt.Errorf("unable to replicate %s/%s (%s). Target (%s) returned %s error on HEAD", + bucket, object, objInfo.VersionID, tgt.EndpointURL(), cerr)) + sendEvent(eventArgs{ + EventName: event.ObjectReplicationNotTracked, + BucketName: bucket, + Object: objInfo, + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + }) + return rinfo + } + } +applyAction: + rinfo.ReplicationStatus = replication.Completed + rinfo.Size = size + rinfo.ReplicationAction = rAction + // use core client to avoid doing multipart on PUT + c := &minio.Core{Client: tgt.Client} + if rAction != replicateAll { + // replicate metadata for object tagging/copy with metadata replacement + srcOpts := minio.CopySrcOptions{ + Bucket: tgt.Bucket, + Object: object, + VersionID: objInfo.VersionID, + } + dstOpts := minio.PutObjectOptions{ + Internal: minio.AdvancedPutOptions{ + SourceVersionID: objInfo.VersionID, + ReplicationRequest: true, // always set this to distinguish between `mc mirror` replication and serverside + }, + } + // default timestamps to ModTime unless present in metadata + lkMap := caseInsensitiveMap(objInfo.UserDefined) + if _, ok := lkMap.Lookup(xhttp.AmzObjectLockLegalHold); ok { + dstOpts.Internal.LegalholdTimestamp = objInfo.ModTime + } + if _, ok := lkMap.Lookup(xhttp.AmzObjectLockRetainUntilDate); ok { + dstOpts.Internal.RetentionTimestamp = objInfo.ModTime + } + if objInfo.UserTags != "" { + dstOpts.Internal.TaggingTimestamp = objInfo.ModTime + } + if tagTmStr, ok := lkMap.Lookup(ReservedMetadataPrefixLower + TaggingTimestamp); ok { + ondiskTimestamp, err := time.Parse(time.RFC3339, tagTmStr) + if err == nil { + dstOpts.Internal.TaggingTimestamp = ondiskTimestamp + } + } + if retTmStr, ok := lkMap.Lookup(ReservedMetadataPrefixLower + ObjectLockRetentionTimestamp); ok { + ondiskTimestamp, err := time.Parse(time.RFC3339, retTmStr) + if err == nil { + dstOpts.Internal.RetentionTimestamp = ondiskTimestamp + } + } + if lholdTmStr, ok := lkMap.Lookup(ReservedMetadataPrefixLower + ObjectLockLegalHoldTimestamp); ok { + ondiskTimestamp, err := time.Parse(time.RFC3339, lholdTmStr) + if err == nil { + dstOpts.Internal.LegalholdTimestamp = ondiskTimestamp + } + } + if _, rinfo.Err = c.CopyObject(ctx, tgt.Bucket, object, tgt.Bucket, object, getCopyObjMetadata(objInfo, tgt.StorageClass), srcOpts, dstOpts); rinfo.Err != nil { + rinfo.ReplicationStatus = replication.Failed + replLogIf(ctx, fmt.Errorf("unable to replicate metadata for object %s/%s(%s) to target %s: %w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err)) + } + } else { + putOpts, isMP, err := putReplicationOpts(ctx, tgt.StorageClass, objInfo) + if err != nil { + replLogIf(ctx, fmt.Errorf("failed to set replicate options for object %s/%s(%s) (target %s) err:%w", bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), err)) + sendEvent(eventArgs{ + EventName: event.ObjectReplicationNotTracked, + BucketName: bucket, + Object: objInfo, + UserAgent: "Internal: [Replication]", + Host: globalLocalNodeName, + }) + return rinfo + } + var headerSize int + for k, v := range putOpts.Header() { + headerSize += len(k) + len(v) + } + + opts := &bandwidth.MonitorReaderOptions{ + BucketOptions: bandwidth.BucketOptions{ + Name: objInfo.Bucket, + ReplicationARN: tgt.ARN, + }, + HeaderSize: headerSize, + } + newCtx := ctx + if globalBucketMonitor.IsThrottled(bucket, tgt.ARN) && objInfo.Size < minLargeObjSize { + var cancel context.CancelFunc + newCtx, cancel = context.WithTimeout(ctx, throttleDeadline) + defer cancel() + } + r := bandwidth.NewMonitoredReader(newCtx, globalBucketMonitor, gr, opts) + if isMP { + rinfo.Err = replicateObjectWithMultipart(ctx, c, tgt.Bucket, object, r, objInfo, putOpts) + } else { + _, rinfo.Err = c.PutObject(ctx, tgt.Bucket, object, r, size, "", "", putOpts) + } + if rinfo.Err != nil { + if minio.ToErrorResponse(rinfo.Err).Code != "PreconditionFailed" { + rinfo.ReplicationStatus = replication.Failed + replLogIf(ctx, fmt.Errorf("unable to replicate for object %s/%s(%s) to target %s: %w", + bucket, objInfo.Name, objInfo.VersionID, tgt.EndpointURL(), rinfo.Err)) + } + if minio.IsNetworkOrHostDown(rinfo.Err, true) && !globalBucketTargetSys.isOffline(tgt.EndpointURL()) { + globalBucketTargetSys.markOffline(tgt.EndpointURL()) + } + } + } + return rinfo +} + +func replicateObjectWithMultipart(ctx context.Context, c *minio.Core, bucket, object string, r io.Reader, objInfo ObjectInfo, opts minio.PutObjectOptions) (err error) { + var uploadedParts []minio.CompletePart + // new multipart must not set mtime as it may lead to erroneous cleanups at various intervals. + opts.Internal.SourceMTime = time.Time{} // this value is saved properly in CompleteMultipartUpload() + var uploadID string + attempts := 1 + for attempts <= 3 { + nctx, cancel := context.WithTimeout(ctx, time.Minute) + uploadID, err = c.NewMultipartUpload(nctx, bucket, object, opts) + cancel() + if err == nil { + break + } + if minio.ToErrorResponse(err).Code == "PreconditionFailed" { + return nil + } + attempts++ + time.Sleep(time.Duration(rand.Int63n(int64(time.Second)))) + } + if err != nil { + return err + } + + defer func() { + if err != nil { + // block and abort remote upload upon failure. + attempts := 1 + for attempts <= 3 { + actx, acancel := context.WithTimeout(ctx, time.Minute) + aerr := c.AbortMultipartUpload(actx, bucket, object, uploadID) + acancel() + if aerr == nil { + return + } + attempts++ + time.Sleep(time.Duration(rand.Int63n(int64(time.Second)))) + } + } + }() + + var ( + hr *hash.Reader + isSSEC = crypto.SSEC.IsEncrypted(objInfo.UserDefined) + ) + + var objectSize int64 + for _, partInfo := range objInfo.Parts { + if isSSEC { + hr, err = hash.NewReader(ctx, io.LimitReader(r, partInfo.Size), partInfo.Size, "", "", partInfo.ActualSize) + } else { + hr, err = hash.NewReader(ctx, io.LimitReader(r, partInfo.ActualSize), partInfo.ActualSize, "", "", partInfo.ActualSize) + } + if err != nil { + return err + } + + cHeader := http.Header{} + cHeader.Add(xhttp.MinIOSourceReplicationRequest, "true") + if !isSSEC { + cs, _ := getCRCMeta(objInfo, partInfo.Number, nil) + for k, v := range cs { + cHeader.Add(k, v) + } + } + popts := minio.PutObjectPartOptions{ + SSE: opts.ServerSideEncryption, + CustomHeader: cHeader, + } + + var size int64 + if isSSEC { + size = partInfo.Size + } else { + size = partInfo.ActualSize + } + objectSize += size + pInfo, err := c.PutObjectPart(ctx, bucket, object, uploadID, partInfo.Number, hr, size, popts) + if err != nil { + return err + } + if pInfo.Size != size { + return fmt.Errorf("ssec(%t): Part size mismatch: got %d, want %d", isSSEC, pInfo.Size, size) + } + uploadedParts = append(uploadedParts, minio.CompletePart{ + PartNumber: pInfo.PartNumber, + ETag: pInfo.ETag, + ChecksumCRC32: pInfo.ChecksumCRC32, + ChecksumCRC32C: pInfo.ChecksumCRC32C, + ChecksumSHA1: pInfo.ChecksumSHA1, + ChecksumSHA256: pInfo.ChecksumSHA256, + ChecksumCRC64NVME: pInfo.ChecksumCRC64NVME, + }) + } + userMeta := map[string]string{ + xhttp.MinIOReplicationActualObjectSize: objInfo.UserDefined[ReservedMetadataPrefix+"actual-size"], + } + if isSSEC && objInfo.UserDefined[ReplicationSsecChecksumHeader] != "" { + userMeta[ReplicationSsecChecksumHeader] = objInfo.UserDefined[ReplicationSsecChecksumHeader] + } + + // really big value but its okay on heavily loaded systems. This is just tail end timeout. + cctx, ccancel := context.WithTimeout(ctx, 10*time.Minute) + defer ccancel() + + if len(objInfo.Checksum) > 0 { + cs, _ := getCRCMeta(objInfo, 0, nil) + for k, v := range cs { + userMeta[k] = strings.Split(v, "-")[0] + } + } + _, err = c.CompleteMultipartUpload(cctx, bucket, object, uploadID, uploadedParts, minio.PutObjectOptions{ + UserMetadata: userMeta, + Internal: minio.AdvancedPutOptions{ + SourceMTime: objInfo.ModTime, + SourceETag: objInfo.ETag, + // always set this to distinguish between `mc mirror` replication and serverside + ReplicationRequest: true, + }, + }) + return err +} + +// filterReplicationStatusMetadata filters replication status metadata for COPY +func filterReplicationStatusMetadata(metadata map[string]string) map[string]string { + // Copy on write + dst := metadata + var copied bool + delKey := func(key string) { + if _, ok := metadata[key]; !ok { + return + } + if !copied { + dst = make(map[string]string, len(metadata)) + maps.Copy(dst, metadata) + copied = true + } + delete(dst, key) + } + + delKey(xhttp.AmzBucketReplicationStatus) + return dst +} + +// DeletedObjectReplicationInfo has info on deleted object +type DeletedObjectReplicationInfo struct { + DeletedObject + Bucket string + EventType string + OpType replication.Type + ResetID string + TargetArn string +} + +// ToMRFEntry returns the relevant info needed by MRF +func (di DeletedObjectReplicationInfo) ToMRFEntry() MRFReplicateEntry { + versionID := di.DeleteMarkerVersionID + if versionID == "" { + versionID = di.VersionID + } + return MRFReplicateEntry{ + Bucket: di.Bucket, + Object: di.ObjectName, + versionID: versionID, + } +} + +// Replication specific APIName +const ( + ReplicateObjectAPI = "ReplicateObject" + ReplicateDeleteAPI = "ReplicateDelete" +) + +const ( + // ReplicateQueued - replication being queued trail + ReplicateQueued = "replicate:queue" + + // ReplicateExisting - audit trail for existing objects replication + ReplicateExisting = "replicate:existing" + // ReplicateExistingDelete - audit trail for delete replication triggered for existing delete markers + ReplicateExistingDelete = "replicate:existing:delete" + + // ReplicateMRF - audit trail for replication from Most Recent Failures (MRF) queue + ReplicateMRF = "replicate:mrf" + // ReplicateIncoming - audit trail of inline replication + ReplicateIncoming = "replicate:incoming" + // ReplicateIncomingDelete - audit trail of inline replication of deletes. + ReplicateIncomingDelete = "replicate:incoming:delete" + + // ReplicateHeal - audit trail for healing of failed/pending replications + ReplicateHeal = "replicate:heal" + // ReplicateHealDelete - audit trail of healing of failed/pending delete replications. + ReplicateHealDelete = "replicate:heal:delete" +) + +var ( + globalReplicationPool = once.NewSingleton[ReplicationPool]() + globalReplicationStats atomic.Pointer[ReplicationStats] +) + +// ReplicationPool describes replication pool +type ReplicationPool struct { + // atomic ops: + activeWorkers int32 + activeLrgWorkers int32 + activeMRFWorkers int32 + + objLayer ObjectLayer + ctx context.Context + priority string + maxWorkers int + maxLWorkers int + stats *ReplicationStats + + mu sync.RWMutex + mrfMU sync.Mutex + resyncer *replicationResyncer + + // workers: + workers []chan ReplicationWorkerOperation + lrgworkers []chan ReplicationWorkerOperation + + // mrf: + mrfWorkerKillCh chan struct{} + mrfReplicaCh chan ReplicationWorkerOperation + mrfSaveCh chan MRFReplicateEntry + mrfStopCh chan struct{} + mrfWorkerSize int +} + +// ReplicationWorkerOperation is a shared interface of replication operations. +type ReplicationWorkerOperation interface { + ToMRFEntry() MRFReplicateEntry +} + +const ( + // WorkerMaxLimit max number of workers per node for "fast" mode + WorkerMaxLimit = 500 + + // WorkerMinLimit min number of workers per node for "slow" mode + WorkerMinLimit = 50 + + // WorkerAutoDefault is default number of workers for "auto" mode + WorkerAutoDefault = 100 + + // MRFWorkerMaxLimit max number of mrf workers per node for "fast" mode + MRFWorkerMaxLimit = 8 + + // MRFWorkerMinLimit min number of mrf workers per node for "slow" mode + MRFWorkerMinLimit = 2 + + // MRFWorkerAutoDefault is default number of mrf workers for "auto" mode + MRFWorkerAutoDefault = 4 + + // LargeWorkerCount is default number of workers assigned to large uploads ( >= 128MiB) + LargeWorkerCount = 10 +) + +// NewReplicationPool creates a pool of replication workers of specified size +func NewReplicationPool(ctx context.Context, o ObjectLayer, opts replicationPoolOpts, stats *ReplicationStats) *ReplicationPool { + var workers, failedWorkers int + priority := "auto" + maxWorkers := WorkerMaxLimit + if opts.Priority != "" { + priority = opts.Priority + } + if opts.MaxWorkers > 0 { + maxWorkers = opts.MaxWorkers + } + switch priority { + case "fast": + workers = WorkerMaxLimit + failedWorkers = MRFWorkerMaxLimit + case "slow": + workers = WorkerMinLimit + failedWorkers = MRFWorkerMinLimit + default: + workers = WorkerAutoDefault + failedWorkers = MRFWorkerAutoDefault + } + if maxWorkers > 0 && workers > maxWorkers { + workers = maxWorkers + } + + if maxWorkers > 0 && failedWorkers > maxWorkers { + failedWorkers = maxWorkers + } + maxLWorkers := LargeWorkerCount + if opts.MaxLWorkers > 0 { + maxLWorkers = opts.MaxLWorkers + } + pool := &ReplicationPool{ + workers: make([]chan ReplicationWorkerOperation, 0, workers), + lrgworkers: make([]chan ReplicationWorkerOperation, 0, maxLWorkers), + mrfReplicaCh: make(chan ReplicationWorkerOperation, 100000), + mrfWorkerKillCh: make(chan struct{}, failedWorkers), + resyncer: newresyncer(), + mrfSaveCh: make(chan MRFReplicateEntry, 100000), + mrfStopCh: make(chan struct{}, 1), + ctx: ctx, + objLayer: o, + stats: stats, + priority: priority, + maxWorkers: maxWorkers, + maxLWorkers: maxLWorkers, + } + + pool.ResizeLrgWorkers(maxLWorkers, 0) + pool.ResizeWorkers(workers, 0) + pool.ResizeFailedWorkers(failedWorkers) + go pool.resyncer.PersistToDisk(ctx, o) + go pool.processMRF() + go pool.persistMRF() + return pool +} + +// AddMRFWorker adds a pending/failed replication worker to handle requests that could not be queued +// to the other workers +func (p *ReplicationPool) AddMRFWorker() { + for { + select { + case <-p.ctx.Done(): + return + case oi, ok := <-p.mrfReplicaCh: + if !ok { + return + } + switch v := oi.(type) { + case ReplicateObjectInfo: + p.stats.incQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType) + atomic.AddInt32(&p.activeMRFWorkers, 1) + replicateObject(p.ctx, v, p.objLayer) + atomic.AddInt32(&p.activeMRFWorkers, -1) + p.stats.decQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType) + + default: + bugLogIf(p.ctx, fmt.Errorf("unknown mrf replication type: %T", oi), "unknown-mrf-replicate-type") + } + case <-p.mrfWorkerKillCh: + return + } + } +} + +// AddWorker adds a replication worker to the pool. +// An optional pointer to a tracker that will be atomically +// incremented when operations are running can be provided. +func (p *ReplicationPool) AddWorker(input <-chan ReplicationWorkerOperation, opTracker *int32) { + for { + select { + case <-p.ctx.Done(): + return + case oi, ok := <-input: + if !ok { + return + } + switch v := oi.(type) { + case ReplicateObjectInfo: + if opTracker != nil { + atomic.AddInt32(opTracker, 1) + } + p.stats.incQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType) + replicateObject(p.ctx, v, p.objLayer) + p.stats.decQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType) + if opTracker != nil { + atomic.AddInt32(opTracker, -1) + } + case DeletedObjectReplicationInfo: + if opTracker != nil { + atomic.AddInt32(opTracker, 1) + } + p.stats.incQ(v.Bucket, 0, true, v.OpType) + + replicateDelete(p.ctx, v, p.objLayer) + p.stats.decQ(v.Bucket, 0, true, v.OpType) + + if opTracker != nil { + atomic.AddInt32(opTracker, -1) + } + default: + bugLogIf(p.ctx, fmt.Errorf("unknown replication type: %T", oi), "unknown-replicate-type") + } + } + } +} + +// AddLargeWorker adds a replication worker to the static pool for large uploads. +func (p *ReplicationPool) AddLargeWorker(input <-chan ReplicationWorkerOperation, opTracker *int32) { + for { + select { + case <-p.ctx.Done(): + return + case oi, ok := <-input: + if !ok { + return + } + switch v := oi.(type) { + case ReplicateObjectInfo: + if opTracker != nil { + atomic.AddInt32(opTracker, 1) + } + p.stats.incQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType) + replicateObject(p.ctx, v, p.objLayer) + p.stats.decQ(v.Bucket, v.Size, v.DeleteMarker, v.OpType) + if opTracker != nil { + atomic.AddInt32(opTracker, -1) + } + case DeletedObjectReplicationInfo: + if opTracker != nil { + atomic.AddInt32(opTracker, 1) + } + replicateDelete(p.ctx, v, p.objLayer) + if opTracker != nil { + atomic.AddInt32(opTracker, -1) + } + default: + bugLogIf(p.ctx, fmt.Errorf("unknown replication type: %T", oi), "unknown-replicate-type") + } + } + } +} + +// ResizeLrgWorkers sets replication workers pool for large transfers(>=128MiB) to new size. +// checkOld can be set to an expected value. +// If the worker count changed +func (p *ReplicationPool) ResizeLrgWorkers(n, checkOld int) { + p.mu.Lock() + defer p.mu.Unlock() + + if (checkOld > 0 && len(p.lrgworkers) != checkOld) || n == len(p.lrgworkers) || n < 1 { + // Either already satisfied or worker count changed while we waited for the lock. + return + } + for len(p.lrgworkers) < n { + input := make(chan ReplicationWorkerOperation, 100000) + p.lrgworkers = append(p.lrgworkers, input) + + go p.AddLargeWorker(input, &p.activeLrgWorkers) + } + for len(p.lrgworkers) > n { + worker := p.lrgworkers[len(p.lrgworkers)-1] + p.lrgworkers = p.lrgworkers[:len(p.lrgworkers)-1] + xioutil.SafeClose(worker) + } +} + +// ActiveWorkers returns the number of active workers handling replication traffic. +func (p *ReplicationPool) ActiveWorkers() int { + return int(atomic.LoadInt32(&p.activeWorkers)) +} + +// ActiveMRFWorkers returns the number of active workers handling replication failures. +func (p *ReplicationPool) ActiveMRFWorkers() int { + return int(atomic.LoadInt32(&p.activeMRFWorkers)) +} + +// ActiveLrgWorkers returns the number of active workers handling traffic > 128MiB object size. +func (p *ReplicationPool) ActiveLrgWorkers() int { + return int(atomic.LoadInt32(&p.activeLrgWorkers)) +} + +// ResizeWorkers sets replication workers pool to new size. +// checkOld can be set to an expected value. +// If the worker count changed +func (p *ReplicationPool) ResizeWorkers(n, checkOld int) { + p.mu.Lock() + defer p.mu.Unlock() + + if (checkOld > 0 && len(p.workers) != checkOld) || n == len(p.workers) || n < 1 { + // Either already satisfied or worker count changed while we waited for the lock. + return + } + for len(p.workers) < n { + input := make(chan ReplicationWorkerOperation, 10000) + p.workers = append(p.workers, input) + + go p.AddWorker(input, &p.activeWorkers) + } + for len(p.workers) > n { + worker := p.workers[len(p.workers)-1] + p.workers = p.workers[:len(p.workers)-1] + xioutil.SafeClose(worker) + } +} + +// ResizeWorkerPriority sets replication failed workers pool size +func (p *ReplicationPool) ResizeWorkerPriority(pri string, maxWorkers, maxLWorkers int) { + var workers, mrfWorkers int + p.mu.Lock() + switch pri { + case "fast": + workers = WorkerMaxLimit + mrfWorkers = MRFWorkerMaxLimit + case "slow": + workers = WorkerMinLimit + mrfWorkers = MRFWorkerMinLimit + default: + workers = WorkerAutoDefault + mrfWorkers = MRFWorkerAutoDefault + if len(p.workers) < WorkerAutoDefault { + workers = min(len(p.workers)+1, WorkerAutoDefault) + } + if p.mrfWorkerSize < MRFWorkerAutoDefault { + mrfWorkers = min(p.mrfWorkerSize+1, MRFWorkerAutoDefault) + } + } + if maxWorkers > 0 && workers > maxWorkers { + workers = maxWorkers + } + + if maxWorkers > 0 && mrfWorkers > maxWorkers { + mrfWorkers = maxWorkers + } + if maxLWorkers <= 0 { + maxLWorkers = LargeWorkerCount + } + p.priority = pri + p.maxWorkers = maxWorkers + p.mu.Unlock() + p.ResizeWorkers(workers, 0) + p.ResizeFailedWorkers(mrfWorkers) + p.ResizeLrgWorkers(maxLWorkers, 0) +} + +// ResizeFailedWorkers sets replication failed workers pool size +func (p *ReplicationPool) ResizeFailedWorkers(n int) { + p.mu.Lock() + defer p.mu.Unlock() + + for p.mrfWorkerSize < n { + p.mrfWorkerSize++ + go p.AddMRFWorker() + } + for p.mrfWorkerSize > n { + p.mrfWorkerSize-- + go func() { p.mrfWorkerKillCh <- struct{}{} }() + } +} + +const ( + minLargeObjSize = 128 * humanize.MiByte // 128MiB +) + +// getWorkerCh gets a worker channel deterministically based on bucket and object names. +// Must be able to grab read lock from p. + +func (p *ReplicationPool) getWorkerCh(bucket, object string, sz int64) chan<- ReplicationWorkerOperation { + h := xxh3.HashString(bucket + object) + p.mu.RLock() + defer p.mu.RUnlock() + if len(p.workers) == 0 { + return nil + } + return p.workers[h%uint64(len(p.workers))] +} + +func (p *ReplicationPool) queueReplicaTask(ri ReplicateObjectInfo) { + if p == nil { + return + } + // if object is large, queue it to a static set of large workers + if ri.Size >= int64(minLargeObjSize) { + h := xxh3.HashString(ri.Bucket + ri.Name) + select { + case <-p.ctx.Done(): + case p.lrgworkers[h%uint64(len(p.lrgworkers))] <- ri: + default: + p.queueMRFSave(ri.ToMRFEntry()) + p.mu.RLock() + maxLWorkers := p.maxLWorkers + existing := len(p.lrgworkers) + p.mu.RUnlock() + maxLWorkers = min(maxLWorkers, LargeWorkerCount) + if p.ActiveLrgWorkers() < maxLWorkers { + workers := min(existing+1, maxLWorkers) + p.ResizeLrgWorkers(workers, existing) + } + } + return + } + + var ch, healCh chan<- ReplicationWorkerOperation + switch ri.OpType { + case replication.HealReplicationType, replication.ExistingObjectReplicationType: + ch = p.mrfReplicaCh + healCh = p.getWorkerCh(ri.Name, ri.Bucket, ri.Size) + default: + ch = p.getWorkerCh(ri.Name, ri.Bucket, ri.Size) + } + if ch == nil && healCh == nil { + return + } + + select { + case <-p.ctx.Done(): + case healCh <- ri: + case ch <- ri: + default: + globalReplicationPool.Get().queueMRFSave(ri.ToMRFEntry()) + p.mu.RLock() + prio := p.priority + maxWorkers := p.maxWorkers + p.mu.RUnlock() + switch prio { + case "fast": + replLogOnceIf(GlobalContext, fmt.Errorf("Unable to keep up with incoming traffic"), string(replicationSubsystem), logger.WarningKind) + case "slow": + replLogOnceIf(GlobalContext, fmt.Errorf("Unable to keep up with incoming traffic - we recommend increasing replication priority with `mc admin config set api replication_priority=auto`"), string(replicationSubsystem), logger.WarningKind) + default: + maxWorkers = min(maxWorkers, WorkerMaxLimit) + if p.ActiveWorkers() < maxWorkers { + p.mu.RLock() + workers := min(len(p.workers)+1, maxWorkers) + existing := len(p.workers) + p.mu.RUnlock() + p.ResizeWorkers(workers, existing) + } + maxMRFWorkers := min(maxWorkers, MRFWorkerMaxLimit) + if p.ActiveMRFWorkers() < maxMRFWorkers { + p.mu.RLock() + workers := min(p.mrfWorkerSize+1, maxMRFWorkers) + p.mu.RUnlock() + p.ResizeFailedWorkers(workers) + } + } + } +} + +func queueReplicateDeletesWrapper(doi DeletedObjectReplicationInfo, existingObjectResync ResyncDecision) { + for k, v := range existingObjectResync.targets { + if v.Replicate { + doi.ResetID = v.ResetID + doi.TargetArn = k + + globalReplicationPool.Get().queueReplicaDeleteTask(doi) + } + } +} + +func (p *ReplicationPool) queueReplicaDeleteTask(doi DeletedObjectReplicationInfo) { + if p == nil { + return + } + var ch chan<- ReplicationWorkerOperation + switch doi.OpType { + case replication.HealReplicationType, replication.ExistingObjectReplicationType: + fallthrough + default: + ch = p.getWorkerCh(doi.Bucket, doi.ObjectName, 0) + } + + select { + case <-p.ctx.Done(): + case ch <- doi: + default: + p.queueMRFSave(doi.ToMRFEntry()) + p.mu.RLock() + prio := p.priority + maxWorkers := p.maxWorkers + p.mu.RUnlock() + switch prio { + case "fast": + replLogOnceIf(GlobalContext, fmt.Errorf("Unable to keep up with incoming deletes"), string(replicationSubsystem), logger.WarningKind) + case "slow": + replLogOnceIf(GlobalContext, fmt.Errorf("Unable to keep up with incoming deletes - we recommend increasing replication priority with `mc admin config set api replication_priority=auto`"), string(replicationSubsystem), logger.WarningKind) + default: + maxWorkers = min(maxWorkers, WorkerMaxLimit) + if p.ActiveWorkers() < maxWorkers { + p.mu.RLock() + workers := min(len(p.workers)+1, maxWorkers) + existing := len(p.workers) + p.mu.RUnlock() + p.ResizeWorkers(workers, existing) + } + } + } +} + +type replicationPoolOpts struct { + Priority string + MaxWorkers int + MaxLWorkers int +} + +func initBackgroundReplication(ctx context.Context, objectAPI ObjectLayer) { + stats := NewReplicationStats(ctx, objectAPI) + globalReplicationPool.Set(NewReplicationPool(ctx, objectAPI, globalAPIConfig.getReplicationOpts(), stats)) + globalReplicationStats.Store(stats) + go stats.trackEWMA() +} + +type proxyResult struct { + Proxy bool + Err error +} + +// get Reader from replication target if active-active replication is in place and +// this node returns a 404 +func proxyGetToReplicationTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (gr *GetObjectReader, proxy proxyResult, err error) { + tgt, oi, proxy := proxyHeadToRepTarget(ctx, bucket, object, rs, opts, proxyTargets) + if !proxy.Proxy { + return nil, proxy, nil + } + fn, _, _, err := NewGetObjectReader(nil, oi, opts, h) + if err != nil { + return nil, proxy, err + } + gopts := minio.GetObjectOptions{ + VersionID: opts.VersionID, + ServerSideEncryption: opts.ServerSideEncryption, + Internal: minio.AdvancedGetOptions{ + ReplicationProxyRequest: "true", + }, + PartNumber: opts.PartNumber, + } + // get correct offsets for encrypted object + if rs != nil { + h, err := rs.ToHeader() + if err != nil { + return nil, proxy, err + } + gopts.Set(xhttp.Range, h) + } + // Make sure to match ETag when proxying. + if err = gopts.SetMatchETag(oi.ETag); err != nil { + return nil, proxy, err + } + c := minio.Core{Client: tgt.Client} + obj, _, h, err := c.GetObject(ctx, tgt.Bucket, object, gopts) + if err != nil { + return nil, proxy, err + } + closeReader := func() { obj.Close() } + reader, err := fn(obj, h, closeReader) + if err != nil { + return nil, proxy, err + } + reader.ObjInfo = oi.Clone() + if rs != nil { + contentSize, err := parseSizeFromContentRange(h) + if err != nil { + return nil, proxy, err + } + reader.ObjInfo.Size = contentSize + } + + return reader, proxyResult{Proxy: true}, nil +} + +func getProxyTargets(ctx context.Context, bucket, object string, opts ObjectOptions) (tgts *madmin.BucketTargets) { + if opts.VersionSuspended { + return &madmin.BucketTargets{} + } + if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) { + return &madmin.BucketTargets{} + } + cfg, err := getReplicationConfig(ctx, bucket) + if err != nil || cfg == nil { + replLogOnceIf(ctx, err, bucket) + + return &madmin.BucketTargets{} + } + topts := replication.ObjectOpts{Name: object} + tgtArns := cfg.FilterTargetArns(topts) + tgts = &madmin.BucketTargets{Targets: make([]madmin.BucketTarget, len(tgtArns))} + for i, tgtArn := range tgtArns { + tgt := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, bucket, tgtArn) + tgts.Targets[i] = tgt + } + + return tgts +} + +func proxyHeadToRepTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (tgt *TargetClient, oi ObjectInfo, proxy proxyResult) { + // this option is set when active-active replication is in place between site A -> B, + // and site B does not have the object yet. + if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) { // true only when site B sets MinIOSourceProxyRequest header + return nil, oi, proxy + } + var perr error + for _, t := range proxyTargets.Targets { + tgt = globalBucketTargetSys.GetRemoteTargetClient(bucket, t.Arn) + if tgt == nil || globalBucketTargetSys.isOffline(tgt.EndpointURL()) { + continue + } + // if proxying explicitly disabled on remote target + if tgt.disableProxy { + continue + } + + gopts := minio.GetObjectOptions{ + VersionID: opts.VersionID, + ServerSideEncryption: opts.ServerSideEncryption, + Internal: minio.AdvancedGetOptions{ + ReplicationProxyRequest: "true", + }, + PartNumber: opts.PartNumber, + } + if rs != nil { + h, err := rs.ToHeader() + if err != nil { + replLogIf(ctx, fmt.Errorf("invalid range header for %s/%s(%s) - %w", bucket, object, opts.VersionID, err)) + continue + } + gopts.Set(xhttp.Range, h) + } + + objInfo, err := tgt.StatObject(ctx, t.TargetBucket, object, gopts) + if err != nil { + perr = err + if isErrInvalidRange(ErrorRespToObjectError(err, bucket, object)) { + return nil, oi, proxyResult{Err: err} + } + continue + } + + tags, _ := tags.MapToObjectTags(objInfo.UserTags) + oi = ObjectInfo{ + Bucket: bucket, + Name: object, + ModTime: objInfo.LastModified, + Size: objInfo.Size, + ETag: objInfo.ETag, + VersionID: objInfo.VersionID, + IsLatest: objInfo.IsLatest, + DeleteMarker: objInfo.IsDeleteMarker, + ContentType: objInfo.ContentType, + Expires: objInfo.Expires, + StorageClass: objInfo.StorageClass, + ReplicationStatusInternal: objInfo.ReplicationStatus, + UserTags: tags.String(), + ReplicationStatus: replication.StatusType(objInfo.ReplicationStatus), + } + oi.UserDefined = make(map[string]string, len(objInfo.Metadata)) + for k, v := range objInfo.Metadata { + oi.UserDefined[k] = v[0] + } + ce, ok := oi.UserDefined[xhttp.ContentEncoding] + if !ok { + ce, ok = oi.UserDefined[strings.ToLower(xhttp.ContentEncoding)] + } + if ok { + oi.ContentEncoding = ce + } + return tgt, oi, proxyResult{Proxy: true} + } + proxy.Err = perr + return nil, oi, proxy +} + +// get object info from replication target if active-active replication is in place and +// this node returns a 404 +func proxyHeadToReplicationTarget(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (oi ObjectInfo, proxy proxyResult) { + _, oi, proxy = proxyHeadToRepTarget(ctx, bucket, object, rs, opts, proxyTargets) + return oi, proxy +} + +func scheduleReplication(ctx context.Context, oi ObjectInfo, o ObjectLayer, dsc ReplicateDecision, opType replication.Type) { + tgtStatuses := replicationStatusesMap(oi.ReplicationStatusInternal) + purgeStatuses := versionPurgeStatusesMap(oi.VersionPurgeStatusInternal) + tm, _ := time.Parse(time.RFC3339Nano, oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp]) + rstate := oi.ReplicationState() + rstate.ReplicateDecisionStr = dsc.String() + asz, _ := oi.GetActualSize() + + ri := ReplicateObjectInfo{ + Name: oi.Name, + Size: oi.Size, + ActualSize: asz, + Bucket: oi.Bucket, + VersionID: oi.VersionID, + ETag: oi.ETag, + ModTime: oi.ModTime, + ReplicationStatus: oi.ReplicationStatus, + ReplicationStatusInternal: oi.ReplicationStatusInternal, + DeleteMarker: oi.DeleteMarker, + VersionPurgeStatusInternal: oi.VersionPurgeStatusInternal, + VersionPurgeStatus: oi.VersionPurgeStatus, + + ReplicationState: rstate, + OpType: opType, + Dsc: dsc, + TargetStatuses: tgtStatuses, + TargetPurgeStatuses: purgeStatuses, + ReplicationTimestamp: tm, + SSEC: crypto.SSEC.IsEncrypted(oi.UserDefined), + UserTags: oi.UserTags, + } + if ri.SSEC { + ri.Checksum = oi.Checksum + } + if dsc.Synchronous() { + replicateObject(ctx, ri, o) + } else { + globalReplicationPool.Get().queueReplicaTask(ri) + } +} + +// proxyTaggingToRepTarget proxies tagging requests to remote targets for +// active-active replicated setups +func proxyTaggingToRepTarget(ctx context.Context, bucket, object string, tags *tags.Tags, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (proxy proxyResult) { + // this option is set when active-active replication is in place between site A -> B, + // and request hits site B that does not have the object yet. + if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) { // true only when site B sets MinIOSourceProxyRequest header + return proxy + } + var wg sync.WaitGroup + errs := make([]error, len(proxyTargets.Targets)) + for idx, t := range proxyTargets.Targets { + tgt := globalBucketTargetSys.GetRemoteTargetClient(bucket, t.Arn) + if tgt == nil || globalBucketTargetSys.isOffline(tgt.EndpointURL()) { + continue + } + // if proxying explicitly disabled on remote target + if tgt.disableProxy { + continue + } + idx := idx + wg.Add(1) + go func(idx int, tgt *TargetClient) { + defer wg.Done() + var err error + if tags != nil { + popts := minio.PutObjectTaggingOptions{ + VersionID: opts.VersionID, + Internal: minio.AdvancedObjectTaggingOptions{ + ReplicationProxyRequest: "true", + }, + } + err = tgt.PutObjectTagging(ctx, tgt.Bucket, object, tags, popts) + } else { + dopts := minio.RemoveObjectTaggingOptions{ + VersionID: opts.VersionID, + Internal: minio.AdvancedObjectTaggingOptions{ + ReplicationProxyRequest: "true", + }, + } + err = tgt.RemoveObjectTagging(ctx, tgt.Bucket, object, dopts) + } + if err != nil { + errs[idx] = err + } + }(idx, tgt) + } + wg.Wait() + + var ( + terr error + taggedCount int + ) + for _, err := range errs { + if err == nil { + taggedCount++ + continue + } + if err != nil { + terr = err + } + } + // don't return error if at least one target was tagged successfully + if taggedCount == 0 && terr != nil { + proxy.Err = terr + } + return proxy +} + +// proxyGetTaggingToRepTarget proxies get tagging requests to remote targets for +// active-active replicated setups +func proxyGetTaggingToRepTarget(ctx context.Context, bucket, object string, opts ObjectOptions, proxyTargets *madmin.BucketTargets) (tgs *tags.Tags, proxy proxyResult) { + // this option is set when active-active replication is in place between site A -> B, + // and request hits site B that does not have the object yet. + if opts.ProxyRequest || (opts.ProxyHeaderSet && !opts.ProxyRequest) { // true only when site B sets MinIOSourceProxyRequest header + return nil, proxy + } + var wg sync.WaitGroup + errs := make([]error, len(proxyTargets.Targets)) + tagSlc := make([]map[string]string, len(proxyTargets.Targets)) + for idx, t := range proxyTargets.Targets { + tgt := globalBucketTargetSys.GetRemoteTargetClient(bucket, t.Arn) + if tgt == nil || globalBucketTargetSys.isOffline(tgt.EndpointURL()) { + continue + } + // if proxying explicitly disabled on remote target + if tgt.disableProxy { + continue + } + idx := idx + wg.Add(1) + go func(idx int, tgt *TargetClient) { + defer wg.Done() + var err error + gopts := minio.GetObjectTaggingOptions{ + VersionID: opts.VersionID, + Internal: minio.AdvancedObjectTaggingOptions{ + ReplicationProxyRequest: "true", + }, + } + tgs, err = tgt.GetObjectTagging(ctx, tgt.Bucket, object, gopts) + if err != nil { + errs[idx] = err + } else { + tagSlc[idx] = tgs.ToMap() + } + }(idx, tgt) + } + wg.Wait() + for idx, err := range errs { + errCode := minio.ToErrorResponse(err).Code + if err != nil && errCode != "NoSuchKey" && errCode != "NoSuchVersion" { + return nil, proxyResult{Err: err} + } + if err == nil { + tgs, _ = tags.MapToObjectTags(tagSlc[idx]) + } + } + if len(errs) == 1 { + proxy.Err = errs[0] + } + return tgs, proxy +} + +func scheduleReplicationDelete(ctx context.Context, dv DeletedObjectReplicationInfo, o ObjectLayer) { + globalReplicationPool.Get().queueReplicaDeleteTask(dv) + for arn := range dv.ReplicationState.Targets { + globalReplicationStats.Load().Update(dv.Bucket, replicatedTargetInfo{Arn: arn, Size: 0, Duration: 0, OpType: replication.DeleteReplicationType}, replication.Pending, replication.StatusType("")) + } +} + +type replicationConfig struct { + Config *replication.Config + remotes *madmin.BucketTargets +} + +func (c replicationConfig) Empty() bool { + return c.Config == nil +} + +func (c replicationConfig) Replicate(opts replication.ObjectOpts) bool { + return c.Config.Replicate(opts) +} + +// Resync returns true if replication reset is requested +func (c replicationConfig) Resync(ctx context.Context, oi ObjectInfo, dsc ReplicateDecision, tgtStatuses map[string]replication.StatusType) (r ResyncDecision) { + if c.Empty() { + return r + } + + // Now overlay existing object replication choices for target + if oi.DeleteMarker { + opts := replication.ObjectOpts{ + Name: oi.Name, + DeleteMarker: oi.DeleteMarker, + VersionID: oi.VersionID, + OpType: replication.DeleteReplicationType, + ExistingObject: true, + } + + tgtArns := c.Config.FilterTargetArns(opts) + // indicates no matching target with Existing object replication enabled. + if len(tgtArns) == 0 { + return r + } + for _, t := range tgtArns { + opts.TargetArn = t + // Update replication decision for target based on existing object replciation rule. + dsc.Set(newReplicateTargetDecision(t, c.Replicate(opts), false)) + } + return c.resync(oi, dsc, tgtStatuses) + } + + // Ignore previous replication status when deciding if object can be re-replicated + userDefined := cloneMSS(oi.UserDefined) + delete(userDefined, xhttp.AmzBucketReplicationStatus) + + rdsc := mustReplicate(ctx, oi.Bucket, oi.Name, getMustReplicateOptions(userDefined, oi.UserTags, "", replication.ExistingObjectReplicationType, ObjectOptions{})) + return c.resync(oi, rdsc, tgtStatuses) +} + +// wrapper function for testability. Returns true if a new reset is requested on +// already replicated objects OR object qualifies for existing object replication +// and no reset requested. +func (c replicationConfig) resync(oi ObjectInfo, dsc ReplicateDecision, tgtStatuses map[string]replication.StatusType) (r ResyncDecision) { + r = ResyncDecision{ + targets: make(map[string]ResyncTargetDecision, len(dsc.targetsMap)), + } + if c.remotes == nil { + return r + } + for _, tgt := range c.remotes.Targets { + d, ok := dsc.targetsMap[tgt.Arn] + if !ok { + continue + } + if !d.Replicate { + continue + } + r.targets[d.Arn] = resyncTarget(oi, tgt.Arn, tgt.ResetID, tgt.ResetBeforeDate, tgtStatuses[tgt.Arn]) + } + return r +} + +func targetResetHeader(arn string) string { + return fmt.Sprintf("%s-%s", ReservedMetadataPrefixLower+ReplicationReset, arn) +} + +func resyncTarget(oi ObjectInfo, arn string, resetID string, resetBeforeDate time.Time, tgtStatus replication.StatusType) (rd ResyncTargetDecision) { + rd = ResyncTargetDecision{ + ResetID: resetID, + ResetBeforeDate: resetBeforeDate, + } + rs, ok := oi.UserDefined[targetResetHeader(arn)] + if !ok { + rs, ok = oi.UserDefined[xhttp.MinIOReplicationResetStatus] // for backward compatibility + } + if !ok { // existing object replication is enabled and object version is unreplicated so far. + if resetID != "" && oi.ModTime.Before(resetBeforeDate) { // trigger replication if `mc replicate reset` requested + rd.Replicate = true + return rd + } + // For existing object reset - this condition is needed + rd.Replicate = tgtStatus == "" + return rd + } + if resetID == "" || resetBeforeDate.Equal(timeSentinel) { // no reset in progress + return rd + } + + // if already replicated, return true if a new reset was requested. + splits := strings.SplitN(rs, ";", 2) + if len(splits) != 2 { + return rd + } + newReset := splits[1] != resetID + if !newReset && tgtStatus == replication.Completed { + // already replicated and no reset requested + return rd + } + rd.Replicate = newReset && oi.ModTime.Before(resetBeforeDate) + return rd +} + +const resyncTimeInterval = time.Minute * 1 + +// PersistToDisk persists in-memory resync metadata stats to disk at periodic intervals +func (s *replicationResyncer) PersistToDisk(ctx context.Context, objectAPI ObjectLayer) { + resyncTimer := time.NewTimer(resyncTimeInterval) + defer resyncTimer.Stop() + + // For each bucket name, store the last timestamp of the + // successful save of replication status in the backend disks. + lastResyncStatusSave := make(map[string]time.Time) + + for { + select { + case <-resyncTimer.C: + s.RLock() + for bucket, brs := range s.statusMap { + var updt bool + // Save the replication status if one resync to any bucket target is still not finished + for _, st := range brs.TargetsMap { + if st.LastUpdate.Equal(timeSentinel) { + updt = true + break + } + } + // Save the replication status if a new stats update is found and not saved in the backend yet + if brs.LastUpdate.After(lastResyncStatusSave[bucket]) { + updt = true + } + if updt { + if err := saveResyncStatus(ctx, bucket, brs, objectAPI); err != nil { + replLogIf(ctx, fmt.Errorf("could not save resync metadata to drive for %s - %w", bucket, err)) + } else { + lastResyncStatusSave[bucket] = brs.LastUpdate + } + } + } + s.RUnlock() + + resyncTimer.Reset(resyncTimeInterval) + case <-ctx.Done(): + // server could be restarting - need + // to exit immediately + return + } + } +} + +const ( + resyncWorkerCnt = 10 // limit of number of bucket resyncs is progress at any given time + resyncParallelRoutines = 10 // number of parallel resync ops per bucket +) + +func newresyncer() *replicationResyncer { + rs := replicationResyncer{ + statusMap: make(map[string]BucketReplicationResyncStatus), + workerSize: resyncWorkerCnt, + resyncCancelCh: make(chan struct{}, resyncWorkerCnt), + workerCh: make(chan struct{}, resyncWorkerCnt), + } + for i := 0; i < rs.workerSize; i++ { + rs.workerCh <- struct{}{} + } + return &rs +} + +// mark status of replication resync on remote target for the bucket +func (s *replicationResyncer) markStatus(status ResyncStatusType, opts resyncOpts, objAPI ObjectLayer) { + s.Lock() + defer s.Unlock() + + m := s.statusMap[opts.bucket] + st := m.TargetsMap[opts.arn] + st.LastUpdate = UTCNow() + st.ResyncStatus = status + m.TargetsMap[opts.arn] = st + m.LastUpdate = UTCNow() + s.statusMap[opts.bucket] = m + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + saveResyncStatus(ctx, opts.bucket, m, objAPI) +} + +// update replication resync stats for bucket's remote target +func (s *replicationResyncer) incStats(ts TargetReplicationResyncStatus, opts resyncOpts) { + s.Lock() + defer s.Unlock() + m := s.statusMap[opts.bucket] + st := m.TargetsMap[opts.arn] + st.Object = ts.Object + st.ReplicatedCount += ts.ReplicatedCount + st.FailedCount += ts.FailedCount + st.ReplicatedSize += ts.ReplicatedSize + st.FailedSize += ts.FailedSize + m.TargetsMap[opts.arn] = st + m.LastUpdate = UTCNow() + s.statusMap[opts.bucket] = m +} + +// resyncBucket resyncs all qualifying objects as per replication rules for the target +// ARN +func (s *replicationResyncer) resyncBucket(ctx context.Context, objectAPI ObjectLayer, heal bool, opts resyncOpts) { + select { + case <-s.workerCh: // block till a worker is available + case <-ctx.Done(): + return + } + + resyncStatus := ResyncFailed + defer func() { + s.markStatus(resyncStatus, opts, objectAPI) + globalSiteResyncMetrics.incBucket(opts, resyncStatus) + s.workerCh <- struct{}{} + }() + // Allocate new results channel to receive ObjectInfo. + objInfoCh := make(chan itemOrErr[ObjectInfo]) + cfg, err := getReplicationConfig(ctx, opts.bucket) + if err != nil { + replLogIf(ctx, fmt.Errorf("replication resync of %s for arn %s failed with %w", opts.bucket, opts.arn, err)) + return + } + tgts, err := globalBucketTargetSys.ListBucketTargets(ctx, opts.bucket) + if err != nil { + replLogIf(ctx, fmt.Errorf("replication resync of %s for arn %s failed %w", opts.bucket, opts.arn, err)) + return + } + rcfg := replicationConfig{ + Config: cfg, + remotes: tgts, + } + tgtArns := cfg.FilterTargetArns( + replication.ObjectOpts{ + OpType: replication.ResyncReplicationType, + TargetArn: opts.arn, + }) + if len(tgtArns) != 1 { + replLogIf(ctx, fmt.Errorf("replication resync failed for %s - arn specified %s is missing in the replication config", opts.bucket, opts.arn)) + return + } + tgt := globalBucketTargetSys.GetRemoteTargetClient(opts.bucket, opts.arn) + if tgt == nil { + replLogIf(ctx, fmt.Errorf("replication resync failed for %s - target could not be created for arn %s", opts.bucket, opts.arn)) + return + } + // mark resync status as resync started + if !heal { + s.markStatus(ResyncStarted, opts, objectAPI) + } + + // Walk through all object versions - Walk() is always in ascending order needed to ensure + // delete marker replicated to target after object version is first created. + if err := objectAPI.Walk(ctx, opts.bucket, "", objInfoCh, WalkOptions{}); err != nil { + replLogIf(ctx, err) + return + } + + s.RLock() + m := s.statusMap[opts.bucket] + st := m.TargetsMap[opts.arn] + s.RUnlock() + var lastCheckpoint string + if st.ResyncStatus == ResyncStarted || st.ResyncStatus == ResyncFailed { + lastCheckpoint = st.Object + } + workers := make([]chan ReplicateObjectInfo, resyncParallelRoutines) + resultCh := make(chan TargetReplicationResyncStatus, 1) + defer xioutil.SafeClose(resultCh) + go func() { + for r := range resultCh { + s.incStats(r, opts) + globalSiteResyncMetrics.updateMetric(r, opts.resyncID) + } + }() + + var wg sync.WaitGroup + for i := range resyncParallelRoutines { + wg.Add(1) + workers[i] = make(chan ReplicateObjectInfo, 100) + i := i + go func(ctx context.Context, idx int) { + defer wg.Done() + for roi := range workers[idx] { + select { + case <-ctx.Done(): + return + case <-s.resyncCancelCh: + default: + } + traceFn := s.trace(tgt.ResetID, fmt.Sprintf("%s/%s (%s)", opts.bucket, roi.Name, roi.VersionID)) + if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() { + versionID := "" + dmVersionID := "" + if roi.VersionPurgeStatus.Empty() { + dmVersionID = roi.VersionID + } else { + versionID = roi.VersionID + } + + doi := DeletedObjectReplicationInfo{ + DeletedObject: DeletedObject{ + ObjectName: roi.Name, + DeleteMarkerVersionID: dmVersionID, + VersionID: versionID, + ReplicationState: roi.ReplicationState, + DeleteMarkerMTime: DeleteMarkerMTime{roi.ModTime}, + DeleteMarker: roi.DeleteMarker, + }, + Bucket: roi.Bucket, + OpType: replication.ExistingObjectReplicationType, + EventType: ReplicateExistingDelete, + } + replicateDelete(ctx, doi, objectAPI) + } else { + roi.OpType = replication.ExistingObjectReplicationType + roi.EventType = ReplicateExisting + replicateObject(ctx, roi, objectAPI) + } + + st := TargetReplicationResyncStatus{ + Object: roi.Name, + Bucket: roi.Bucket, + } + + _, err := tgt.StatObject(ctx, tgt.Bucket, roi.Name, minio.StatObjectOptions{ + VersionID: roi.VersionID, + Internal: minio.AdvancedGetOptions{ + ReplicationProxyRequest: "false", + }, + }) + sz := roi.Size + if err != nil { + if roi.DeleteMarker && isErrMethodNotAllowed(ErrorRespToObjectError(err, opts.bucket, roi.Name)) { + st.ReplicatedCount++ + } else { + st.FailedCount++ + } + sz = 0 + } else { + st.ReplicatedCount++ + st.ReplicatedSize += roi.Size + } + traceFn(sz, err) + select { + case <-ctx.Done(): + return + case <-s.resyncCancelCh: + return + case resultCh <- st: + } + } + }(ctx, i) + } + for res := range objInfoCh { + if res.Err != nil { + resyncStatus = ResyncFailed + replLogIf(ctx, res.Err) + return + } + select { + case <-s.resyncCancelCh: + resyncStatus = ResyncCanceled + return + case <-ctx.Done(): + return + default: + } + if heal && lastCheckpoint != "" && lastCheckpoint != res.Item.Name { + continue + } + lastCheckpoint = "" + roi := getHealReplicateObjectInfo(res.Item, rcfg) + if !roi.ExistingObjResync.mustResync() { + continue + } + select { + case <-s.resyncCancelCh: + return + case <-ctx.Done(): + return + default: + h := xxh3.HashString(roi.Bucket + roi.Name) + workers[h%uint64(resyncParallelRoutines)] <- roi + } + } + for i := range resyncParallelRoutines { + xioutil.SafeClose(workers[i]) + } + wg.Wait() + resyncStatus = ResyncCompleted +} + +// start replication resync for the remote target ARN specified +func (s *replicationResyncer) start(ctx context.Context, objAPI ObjectLayer, opts resyncOpts) error { + if opts.bucket == "" { + return fmt.Errorf("bucket name is empty") + } + if opts.arn == "" { + return fmt.Errorf("target ARN specified for resync is empty") + } + // Check if the current bucket has quota restrictions, if not skip it + cfg, err := getReplicationConfig(ctx, opts.bucket) + if err != nil { + return err + } + tgtArns := cfg.FilterTargetArns( + replication.ObjectOpts{ + OpType: replication.ResyncReplicationType, + TargetArn: opts.arn, + }) + + if len(tgtArns) == 0 { + return fmt.Errorf("arn %s specified for resync not found in replication config", opts.arn) + } + globalReplicationPool.Get().resyncer.RLock() + data, ok := globalReplicationPool.Get().resyncer.statusMap[opts.bucket] + globalReplicationPool.Get().resyncer.RUnlock() + if !ok { + data, err = loadBucketResyncMetadata(ctx, opts.bucket, objAPI) + if err != nil { + return err + } + } + // validate if resync is in progress for this arn + for tArn, st := range data.TargetsMap { + if opts.arn == tArn && (st.ResyncStatus == ResyncStarted || st.ResyncStatus == ResyncPending) { + return fmt.Errorf("Resync of bucket %s is already in progress for remote bucket %s", opts.bucket, opts.arn) + } + } + + status := TargetReplicationResyncStatus{ + ResyncID: opts.resyncID, + ResyncBeforeDate: opts.resyncBefore, + StartTime: UTCNow(), + ResyncStatus: ResyncPending, + Bucket: opts.bucket, + } + data.TargetsMap[opts.arn] = status + if err = saveResyncStatus(ctx, opts.bucket, data, objAPI); err != nil { + return err + } + + globalReplicationPool.Get().resyncer.Lock() + defer globalReplicationPool.Get().resyncer.Unlock() + brs, ok := globalReplicationPool.Get().resyncer.statusMap[opts.bucket] + if !ok { + brs = BucketReplicationResyncStatus{ + Version: resyncMetaVersion, + TargetsMap: make(map[string]TargetReplicationResyncStatus), + } + } + brs.TargetsMap[opts.arn] = status + globalReplicationPool.Get().resyncer.statusMap[opts.bucket] = brs + go globalReplicationPool.Get().resyncer.resyncBucket(GlobalContext, objAPI, false, opts) + return nil +} + +func (s *replicationResyncer) trace(resyncID string, path string) func(sz int64, err error) { + startTime := time.Now() + return func(sz int64, err error) { + duration := time.Since(startTime) + if globalTrace.NumSubscribers(madmin.TraceReplicationResync) > 0 { + globalTrace.Publish(replicationResyncTrace(resyncID, startTime, duration, path, err, sz)) + } + } +} + +func replicationResyncTrace(resyncID string, startTime time.Time, duration time.Duration, path string, err error, sz int64) madmin.TraceInfo { + var errStr string + if err != nil { + errStr = err.Error() + } + funcName := fmt.Sprintf("replication.(resyncID=%s)", resyncID) + return madmin.TraceInfo{ + TraceType: madmin.TraceReplicationResync, + Time: startTime, + NodeName: globalLocalNodeName, + FuncName: funcName, + Duration: duration, + Path: path, + Error: errStr, + Bytes: sz, + } +} + +// delete resync metadata from replication resync state in memory +func (p *ReplicationPool) deleteResyncMetadata(ctx context.Context, bucket string) { + if p == nil { + return + } + p.resyncer.Lock() + delete(p.resyncer.statusMap, bucket) + defer p.resyncer.Unlock() + + globalSiteResyncMetrics.deleteBucket(bucket) +} + +// initResync - initializes bucket replication resync for all buckets. +func (p *ReplicationPool) initResync(ctx context.Context, buckets []string, objAPI ObjectLayer) error { + if objAPI == nil { + return errServerNotInitialized + } + // Load bucket metadata sys in background + go p.startResyncRoutine(ctx, buckets, objAPI) + return nil +} + +func (p *ReplicationPool) startResyncRoutine(ctx context.Context, buckets []string, objAPI ObjectLayer) { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + // Run the replication resync in a loop + for { + if err := p.loadResync(ctx, buckets, objAPI); err == nil { + <-ctx.Done() + return + } + duration := max(time.Duration(r.Float64()*float64(time.Minute)), + // Make sure to sleep at least a second to avoid high CPU ticks. + time.Second) + time.Sleep(duration) + } +} + +// Loads bucket replication resync statuses into memory. +func (p *ReplicationPool) loadResync(ctx context.Context, buckets []string, objAPI ObjectLayer) error { + // Make sure only one node running resync on the cluster. + ctx, cancel := globalLeaderLock.GetLock(ctx) + defer cancel() + + for index := range buckets { + bucket := buckets[index] + + meta, err := loadBucketResyncMetadata(ctx, bucket, objAPI) + if err != nil { + if !errors.Is(err, errVolumeNotFound) { + replLogIf(ctx, err) + } + continue + } + + p.resyncer.Lock() + p.resyncer.statusMap[bucket] = meta + p.resyncer.Unlock() + + tgts := meta.cloneTgtStats() + for arn, st := range tgts { + switch st.ResyncStatus { + case ResyncFailed, ResyncStarted, ResyncPending: + go p.resyncer.resyncBucket(ctx, objAPI, true, resyncOpts{ + bucket: bucket, + arn: arn, + resyncID: st.ResyncID, + resyncBefore: st.ResyncBeforeDate, + }) + } + } + } + return nil +} + +// load bucket resync metadata from disk +func loadBucketResyncMetadata(ctx context.Context, bucket string, objAPI ObjectLayer) (brs BucketReplicationResyncStatus, e error) { + brs = newBucketResyncStatus(bucket) + resyncDirPath := path.Join(bucketMetaPrefix, bucket, replicationDir) + data, err := readConfig(GlobalContext, objAPI, pathJoin(resyncDirPath, resyncFileName)) + if err != nil && err != errConfigNotFound { + return brs, err + } + if len(data) == 0 { + // Seems to be empty. + return brs, nil + } + if len(data) <= 4 { + return brs, fmt.Errorf("replication resync: no data") + } + // Read resync meta header + switch binary.LittleEndian.Uint16(data[0:2]) { + case resyncMetaFormat: + default: + return brs, fmt.Errorf("resyncMeta: unknown format: %d", binary.LittleEndian.Uint16(data[0:2])) + } + switch binary.LittleEndian.Uint16(data[2:4]) { + case resyncMetaVersion: + default: + return brs, fmt.Errorf("resyncMeta: unknown version: %d", binary.LittleEndian.Uint16(data[2:4])) + } + // OK, parse data. + if _, err = brs.UnmarshalMsg(data[4:]); err != nil { + return brs, err + } + + switch brs.Version { + case resyncMetaVersionV1: + default: + return brs, fmt.Errorf("unexpected resync meta version: %d", brs.Version) + } + return brs, nil +} + +// save resync status to resync.bin +func saveResyncStatus(ctx context.Context, bucket string, brs BucketReplicationResyncStatus, objectAPI ObjectLayer) error { + data := make([]byte, 4, brs.Msgsize()+4) + + // Initialize the resync meta header. + binary.LittleEndian.PutUint16(data[0:2], resyncMetaFormat) + binary.LittleEndian.PutUint16(data[2:4], resyncMetaVersion) + + buf, err := brs.MarshalMsg(data) + if err != nil { + return err + } + + configFile := path.Join(bucketMetaPrefix, bucket, replicationDir, resyncFileName) + return saveConfig(ctx, objectAPI, configFile, buf) +} + +// getReplicationDiff returns un-replicated objects in a channel. +// If a non-nil channel is returned it must be consumed fully or +// the provided context must be canceled. +func getReplicationDiff(ctx context.Context, objAPI ObjectLayer, bucket string, opts madmin.ReplDiffOpts) (chan madmin.DiffInfo, error) { + cfg, err := getReplicationConfig(ctx, bucket) + if err != nil { + replLogOnceIf(ctx, err, bucket) + return nil, err + } + tgts, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket) + if err != nil { + replLogIf(ctx, err) + return nil, err + } + + objInfoCh := make(chan itemOrErr[ObjectInfo], 10) + if err := objAPI.Walk(ctx, bucket, opts.Prefix, objInfoCh, WalkOptions{}); err != nil { + replLogIf(ctx, err) + return nil, err + } + rcfg := replicationConfig{ + Config: cfg, + remotes: tgts, + } + diffCh := make(chan madmin.DiffInfo, 4000) + go func() { + defer xioutil.SafeClose(diffCh) + for res := range objInfoCh { + if res.Err != nil { + diffCh <- madmin.DiffInfo{Err: res.Err} + return + } + if contextCanceled(ctx) { + // Just consume input... + continue + } + obj := res.Item + + // Ignore object prefixes which are excluded + // from versioning via the MinIO bucket versioning extension. + if globalBucketVersioningSys.PrefixSuspended(bucket, obj.Name) { + continue + } + roi := getHealReplicateObjectInfo(obj, rcfg) + switch roi.ReplicationStatus { + case replication.Completed, replication.Replica: + if !opts.Verbose { + continue + } + fallthrough + default: + // ignore pre-existing objects that don't satisfy replication rule(s) + if roi.ReplicationStatus.Empty() && !roi.ExistingObjResync.mustResync() { + continue + } + tgtsMap := make(map[string]madmin.TgtDiffInfo) + for arn, st := range roi.TargetStatuses { + if opts.ARN == "" || opts.ARN == arn { + if !opts.Verbose && (st == replication.Completed || st == replication.Replica) { + continue + } + tgtsMap[arn] = madmin.TgtDiffInfo{ + ReplicationStatus: st.String(), + } + } + } + for arn, st := range roi.TargetPurgeStatuses { + if opts.ARN == "" || opts.ARN == arn { + if !opts.Verbose && st == replication.VersionPurgeComplete { + continue + } + t, ok := tgtsMap[arn] + if !ok { + t = madmin.TgtDiffInfo{} + } + t.DeleteReplicationStatus = string(st) + tgtsMap[arn] = t + } + } + select { + case diffCh <- madmin.DiffInfo{ + Object: obj.Name, + VersionID: obj.VersionID, + LastModified: obj.ModTime, + IsDeleteMarker: obj.DeleteMarker, + ReplicationStatus: string(roi.ReplicationStatus), + DeleteReplicationStatus: string(roi.VersionPurgeStatus), + ReplicationTimestamp: roi.ReplicationTimestamp, + Targets: tgtsMap, + }: + case <-ctx.Done(): + continue + } + } + } + }() + return diffCh, nil +} + +// QueueReplicationHeal is a wrapper for queueReplicationHeal +func QueueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, retryCount int) { + // ignore modtime zero objects + if oi.ModTime.IsZero() { + return + } + rcfg, err := getReplicationConfig(ctx, bucket) + if err != nil { + replLogOnceIf(ctx, err, bucket) + return + } + tgts, _ := globalBucketTargetSys.ListBucketTargets(ctx, bucket) + queueReplicationHeal(ctx, bucket, oi, replicationConfig{ + Config: rcfg, + remotes: tgts, + }, retryCount) +} + +// queueReplicationHeal enqueues objects that failed replication OR eligible for resyncing through +// an ongoing resync operation or via existing objects replication configuration setting. +func queueReplicationHeal(ctx context.Context, bucket string, oi ObjectInfo, rcfg replicationConfig, retryCount int) (roi ReplicateObjectInfo) { + // ignore modtime zero objects + if oi.ModTime.IsZero() { + return roi + } + + if isVeeamSOSAPIObject(oi.Name) { + return roi + } + if rcfg.Config == nil || rcfg.remotes == nil { + return roi + } + roi = getHealReplicateObjectInfo(oi, rcfg) + roi.RetryCount = uint32(retryCount) + if !roi.Dsc.ReplicateAny() { + return roi + } + // early return if replication already done, otherwise we need to determine if this + // version is an existing object that needs healing. + if oi.ReplicationStatus == replication.Completed && oi.VersionPurgeStatus.Empty() && !roi.ExistingObjResync.mustResync() { + return roi + } + + if roi.DeleteMarker || !roi.VersionPurgeStatus.Empty() { + versionID := "" + dmVersionID := "" + if roi.VersionPurgeStatus.Empty() { + dmVersionID = roi.VersionID + } else { + versionID = roi.VersionID + } + + dv := DeletedObjectReplicationInfo{ + DeletedObject: DeletedObject{ + ObjectName: roi.Name, + DeleteMarkerVersionID: dmVersionID, + VersionID: versionID, + ReplicationState: roi.ReplicationState, + DeleteMarkerMTime: DeleteMarkerMTime{roi.ModTime}, + DeleteMarker: roi.DeleteMarker, + }, + Bucket: roi.Bucket, + OpType: replication.HealReplicationType, + EventType: ReplicateHealDelete, + } + // heal delete marker replication failure or versioned delete replication failure + if roi.ReplicationStatus == replication.Pending || + roi.ReplicationStatus == replication.Failed || + roi.VersionPurgeStatus == replication.VersionPurgeFailed || roi.VersionPurgeStatus == replication.VersionPurgePending { + globalReplicationPool.Get().queueReplicaDeleteTask(dv) + return roi + } + // if replication status is Complete on DeleteMarker and existing object resync required + if roi.ExistingObjResync.mustResync() && (roi.ReplicationStatus == replication.Completed || roi.ReplicationStatus.Empty()) { + queueReplicateDeletesWrapper(dv, roi.ExistingObjResync) + return roi + } + return roi + } + if roi.ExistingObjResync.mustResync() { + roi.OpType = replication.ExistingObjectReplicationType + } + switch roi.ReplicationStatus { + case replication.Pending, replication.Failed: + roi.EventType = ReplicateHeal + globalReplicationPool.Get().queueReplicaTask(roi) + return roi + } + if roi.ExistingObjResync.mustResync() { + roi.EventType = ReplicateExisting + globalReplicationPool.Get().queueReplicaTask(roi) + } + return roi +} + +const ( + mrfSaveInterval = 5 * time.Minute + mrfQueueInterval = mrfSaveInterval + time.Minute // A minute higher than save interval + + mrfRetryLimit = 3 // max number of retries before letting scanner catch up on this object version + mrfMaxEntries = 1000000 +) + +func (p *ReplicationPool) persistMRF() { + if !p.initialized() { + return + } + + entries := make(map[string]MRFReplicateEntry) + mTimer := time.NewTimer(mrfSaveInterval) + defer mTimer.Stop() + + saveMRFToDisk := func() { + if len(entries) == 0 { + return + } + + // queue all entries for healing before overwriting the node mrf file + if !contextCanceled(p.ctx) { + p.queueMRFHeal() + } + + p.saveMRFEntries(p.ctx, entries) + + entries = make(map[string]MRFReplicateEntry) + } + for { + select { + case <-mTimer.C: + saveMRFToDisk() + mTimer.Reset(mrfSaveInterval) + case <-p.ctx.Done(): + p.mrfStopCh <- struct{}{} + xioutil.SafeClose(p.mrfSaveCh) + // We try to save if possible, but we don't care beyond that. + saveMRFToDisk() + return + case e, ok := <-p.mrfSaveCh: + if !ok { + return + } + entries[e.versionID] = e + + if len(entries) >= mrfMaxEntries { + saveMRFToDisk() + } + } + } +} + +func (p *ReplicationPool) queueMRFSave(entry MRFReplicateEntry) { + if !p.initialized() { + return + } + if entry.RetryCount > mrfRetryLimit { // let scanner catch up if retry count exceeded + atomic.AddUint64(&p.stats.mrfStats.TotalDroppedCount, 1) + atomic.AddUint64(&p.stats.mrfStats.TotalDroppedBytes, uint64(entry.sz)) + return + } + + select { + case <-GlobalContext.Done(): + return + case <-p.mrfStopCh: + return + default: + select { + case p.mrfSaveCh <- entry: + default: + atomic.AddUint64(&p.stats.mrfStats.TotalDroppedCount, 1) + atomic.AddUint64(&p.stats.mrfStats.TotalDroppedBytes, uint64(entry.sz)) + } + } +} + +func (p *ReplicationPool) persistToDrive(ctx context.Context, v MRFReplicateEntries) { + newReader := func() io.ReadCloser { + r, w := io.Pipe() + go func() { + // Initialize MRF meta header. + var data [4]byte + binary.LittleEndian.PutUint16(data[0:2], mrfMetaFormat) + binary.LittleEndian.PutUint16(data[2:4], mrfMetaVersion) + mw := msgp.NewWriter(w) + n, err := mw.Write(data[:]) + if err != nil { + w.CloseWithError(err) + return + } + if n != len(data) { + w.CloseWithError(io.ErrShortWrite) + return + } + err = v.EncodeMsg(mw) + mw.Flush() + w.CloseWithError(err) + }() + return r + } + + globalLocalDrivesMu.RLock() + localDrives := cloneDrives(globalLocalDrivesMap) + globalLocalDrivesMu.RUnlock() + + for _, localDrive := range localDrives { + r := newReader() + err := localDrive.CreateFile(ctx, "", minioMetaBucket, pathJoin(replicationMRFDir, globalLocalNodeNameHex+".bin"), -1, r) + r.Close() + if err == nil { + break + } + } +} + +// save mrf entries to nodenamehex.bin +func (p *ReplicationPool) saveMRFEntries(ctx context.Context, entries map[string]MRFReplicateEntry) { + if !p.initialized() { + return + } + atomic.StoreUint64(&p.stats.mrfStats.LastFailedCount, uint64(len(entries))) + if len(entries) == 0 { + return + } + + v := MRFReplicateEntries{ + Entries: entries, + Version: mrfMetaVersion, + } + + p.persistToDrive(ctx, v) +} + +// load mrf entries from disk +func (p *ReplicationPool) loadMRF() (mrfRec MRFReplicateEntries, err error) { + loadMRF := func(rc io.ReadCloser) (re MRFReplicateEntries, err error) { + defer rc.Close() + + if !p.initialized() { + return re, nil + } + var data [4]byte + n, err := rc.Read(data[:]) + if err != nil { + return re, err + } + if n != len(data) { + return re, errors.New("replication mrf: no data") + } + // Read resync meta header + switch binary.LittleEndian.Uint16(data[0:2]) { + case mrfMetaFormat: + default: + return re, fmt.Errorf("replication mrf: unknown format: %d", binary.LittleEndian.Uint16(data[0:2])) + } + switch binary.LittleEndian.Uint16(data[2:4]) { + case mrfMetaVersion: + default: + return re, fmt.Errorf("replication mrf: unknown version: %d", binary.LittleEndian.Uint16(data[2:4])) + } + + // OK, parse data. + // ignore any parsing errors, we do not care this file is generated again anyways. + re.DecodeMsg(msgp.NewReader(rc)) + + return re, nil + } + + globalLocalDrivesMu.RLock() + localDrives := cloneDrives(globalLocalDrivesMap) + globalLocalDrivesMu.RUnlock() + + for _, localDrive := range localDrives { + rc, err := localDrive.ReadFileStream(p.ctx, minioMetaBucket, pathJoin(replicationMRFDir, globalLocalNodeNameHex+".bin"), 0, -1) + if err != nil { + continue + } + + mrfRec, err = loadMRF(rc) + if err != nil { + continue + } + + // finally delete the file after processing mrf entries + localDrive.Delete(p.ctx, minioMetaBucket, pathJoin(replicationMRFDir, globalLocalNodeNameHex+".bin"), DeleteOptions{}) + break + } + + return mrfRec, nil +} + +func (p *ReplicationPool) processMRF() { + if !p.initialized() { + return + } + pTimer := time.NewTimer(mrfQueueInterval) + defer pTimer.Stop() + for { + select { + case <-pTimer.C: + // skip healing if all targets are offline + var offlineCnt int + tgts := globalBucketTargetSys.ListTargets(p.ctx, "", "") + for _, tgt := range tgts { + if globalBucketTargetSys.isOffline(tgt.URL()) { + offlineCnt++ + } + } + if len(tgts) == offlineCnt { + pTimer.Reset(mrfQueueInterval) + continue + } + if err := p.queueMRFHeal(); err != nil && !osIsNotExist(err) { + replLogIf(p.ctx, err) + } + pTimer.Reset(mrfQueueInterval) + case <-p.ctx.Done(): + return + } + } +} + +// process sends error logs to the heal channel for an attempt to heal replication. +func (p *ReplicationPool) queueMRFHeal() error { + p.mrfMU.Lock() + defer p.mrfMU.Unlock() + + if !p.initialized() { + return errServerNotInitialized + } + + mrfRec, err := p.loadMRF() + if err != nil { + return err + } + + // queue replication heal in a goroutine to avoid holding up mrf save routine + go func() { + for vID, e := range mrfRec.Entries { + ctx, cancel := context.WithTimeout(p.ctx, time.Second) // Do not waste more than a second on this. + + oi, err := p.objLayer.GetObjectInfo(ctx, e.Bucket, e.Object, ObjectOptions{ + VersionID: vID, + }) + cancel() + if err != nil { + continue + } + + QueueReplicationHeal(p.ctx, e.Bucket, oi, e.RetryCount) + } + }() + + return nil +} + +func (p *ReplicationPool) initialized() bool { + return p != nil && p.objLayer != nil +} + +// getMRF returns MRF entries for this node. +func (p *ReplicationPool) getMRF(ctx context.Context, bucket string) (ch <-chan madmin.ReplicationMRF, err error) { + mrfRec, err := p.loadMRF() + if err != nil { + return nil, err + } + + mrfCh := make(chan madmin.ReplicationMRF, 100) + go func() { + defer xioutil.SafeClose(mrfCh) + for vID, e := range mrfRec.Entries { + if bucket != "" && e.Bucket != bucket { + continue + } + select { + case mrfCh <- madmin.ReplicationMRF{ + NodeName: globalLocalNodeName, + Object: e.Object, + VersionID: vID, + Bucket: e.Bucket, + RetryCount: e.RetryCount, + }: + case <-ctx.Done(): + return + } + } + }() + + return mrfCh, nil +} + +// validateReplicationDestinationOptions is used to configure the validation of the replication destination. +// validateReplicationDestination uses this to configure the validation. +type validateReplicationDestinationOptions struct { + CheckRemoteBucket bool + CheckReady bool + + checkReadyErr sync.Map +} + +func getCRCMeta(oi ObjectInfo, partNum int, h http.Header) (cs map[string]string, isMP bool) { + meta := make(map[string]string) + cs, isMP = oi.decryptChecksums(partNum, h) + for k, v := range cs { + if k == xhttp.AmzChecksumType { + continue + } + cktype := hash.ChecksumStringToType(k) + if cktype.IsSet() { + meta[cktype.Key()] = v + meta[xhttp.AmzChecksumAlgo] = cktype.String() + } + } + return meta, isMP +} diff --git a/cmd/bucket-replication_test.go b/cmd/bucket-replication_test.go new file mode 100644 index 00000000..ada944d2 --- /dev/null +++ b/cmd/bucket-replication_test.go @@ -0,0 +1,289 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "net/http" + "testing" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/bucket/replication" + xhttp "github.com/minio/minio/internal/http" +) + +var configs = []replication.Config{ + { // Config0 - Replication config has no filters, existing object replication enabled + Rules: []replication.Rule{ + { + Status: replication.Enabled, + Priority: 1, + DeleteMarkerReplication: replication.DeleteMarkerReplication{Status: replication.Enabled}, + DeleteReplication: replication.DeleteReplication{Status: replication.Enabled}, + Filter: replication.Filter{}, + ExistingObjectReplication: replication.ExistingObjectReplication{Status: replication.Enabled}, + SourceSelectionCriteria: replication.SourceSelectionCriteria{ + ReplicaModifications: replication.ReplicaModifications{Status: replication.Enabled}, + }, + }, + }, + }, +} + +var replicationConfigTests = []struct { + info ObjectInfo + name string + rcfg replicationConfig + dsc ReplicateDecision + tgtStatuses map[string]replication.StatusType + expectedSync bool +}{ + { // 1. no replication config + name: "no replication config", + info: ObjectInfo{Size: 100}, + rcfg: replicationConfig{Config: nil}, + expectedSync: false, + }, + { // 2. existing object replication config enabled, no versioning + name: "existing object replication config enabled, no versioning", + info: ObjectInfo{Size: 100}, + rcfg: replicationConfig{Config: &configs[0]}, + expectedSync: false, + }, + { // 3. existing object replication config enabled, versioning suspended + name: "existing object replication config enabled, versioning suspended", + info: ObjectInfo{Size: 100, VersionID: nullVersionID}, + rcfg: replicationConfig{Config: &configs[0]}, + expectedSync: false, + }, + { // 4. existing object replication enabled, versioning enabled; no reset in progress + name: "existing object replication enabled, versioning enabled; no reset in progress", + info: ObjectInfo{ + Size: 100, + ReplicationStatus: replication.Completed, + VersionID: "a3348c34-c352-4498-82f0-1098e8b34df9", + }, + rcfg: replicationConfig{Config: &configs[0]}, + expectedSync: false, + }, +} + +func TestReplicationResync(t *testing.T) { + ctx := t.Context() + for i, test := range replicationConfigTests { + if sync := test.rcfg.Resync(ctx, test.info, test.dsc, test.tgtStatuses); sync.mustResync() != test.expectedSync { + t.Errorf("Test%d (%s): Resync got %t , want %t", i+1, test.name, sync.mustResync(), test.expectedSync) + } + } +} + +var ( + start = UTCNow().AddDate(0, 0, -1) + replicationConfigTests2 = []struct { + info ObjectInfo + name string + rcfg replicationConfig + dsc ReplicateDecision + tgtStatuses map[string]replication.StatusType + expectedSync bool + }{ + { // Cases 1-4: existing object replication enabled, versioning enabled, no reset - replication status varies + // 1: Pending replication + name: "existing object replication on object in Pending replication status", + info: ObjectInfo{ + Size: 100, + ReplicationStatusInternal: "arn1:PENDING;", + ReplicationStatus: replication.Pending, + VersionID: "a3348c34-c352-4498-82f0-1098e8b34df9", + }, + rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{ + Arn: "arn1", + }}}}, + dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}}, + expectedSync: true, + }, + + { // 2. replication status Failed + name: "existing object replication on object in Failed replication status", + info: ObjectInfo{ + Size: 100, + ReplicationStatusInternal: "arn1:FAILED", + ReplicationStatus: replication.Failed, + VersionID: "a3348c34-c352-4498-82f0-1098e8b34df9", + }, + dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}}, + rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{ + Arn: "arn1", + }}}}, + expectedSync: true, + }, + { // 3. replication status unset + name: "existing object replication on pre-existing unreplicated object", + info: ObjectInfo{ + Size: 100, + ReplicationStatus: replication.StatusType(""), + VersionID: "a3348c34-c352-4498-82f0-1098e8b34df9", + }, + rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{ + Arn: "arn1", + }}}}, + dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}}, + expectedSync: true, + }, + { // 4. replication status Complete + name: "existing object replication on object in Completed replication status", + info: ObjectInfo{ + Size: 100, + ReplicationStatusInternal: "arn1:COMPLETED", + ReplicationStatus: replication.Completed, + VersionID: "a3348c34-c352-4498-82f0-1098e8b34df9", + }, + dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", false, false)}}, + rcfg: replicationConfig{remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{ + Arn: "arn1", + }}}}, + expectedSync: false, + }, + { // 5. existing object replication enabled, versioning enabled, replication status Pending & reset ID present + name: "existing object replication with reset in progress and object in Pending status", + info: ObjectInfo{ + Size: 100, + ReplicationStatusInternal: "arn1:PENDING;", + ReplicationStatus: replication.Pending, + VersionID: "a3348c34-c352-4498-82f0-1098e8b34df9", + UserDefined: map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;abc", UTCNow().AddDate(0, -1, 0).String())}, + }, + expectedSync: true, + dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}}, + rcfg: replicationConfig{ + remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{ + Arn: "arn1", + ResetID: "xyz", + ResetBeforeDate: UTCNow(), + }}}, + }, + }, + { // 6. existing object replication enabled, versioning enabled, replication status Failed & reset ID present + name: "existing object replication with reset in progress and object in Failed status", + info: ObjectInfo{ + Size: 100, + ReplicationStatusInternal: "arn1:FAILED;", + ReplicationStatus: replication.Failed, + VersionID: "a3348c34-c352-4498-82f0-1098e8b34df9", + UserDefined: map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;abc", UTCNow().AddDate(0, -1, 0).String())}, + }, + dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}}, + rcfg: replicationConfig{ + remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{ + Arn: "arn1", + ResetID: "xyz", + ResetBeforeDate: UTCNow(), + }}}, + }, + expectedSync: true, + }, + { // 7. existing object replication enabled, versioning enabled, replication status unset & reset ID present + name: "existing object replication with reset in progress and object never replicated before", + info: ObjectInfo{ + Size: 100, + ReplicationStatus: replication.StatusType(""), + VersionID: "a3348c34-c352-4498-82f0-1098e8b34df9", + UserDefined: map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;abc", UTCNow().AddDate(0, -1, 0).String())}, + }, + dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}}, + rcfg: replicationConfig{ + remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{ + Arn: "arn1", + ResetID: "xyz", + ResetBeforeDate: UTCNow(), + }}}, + }, + + expectedSync: true, + }, + + { // 8. existing object replication enabled, versioning enabled, replication status Complete & reset ID present + name: "existing object replication enabled - reset in progress for an object in Completed status", + info: ObjectInfo{ + Size: 100, + ReplicationStatusInternal: "arn1:COMPLETED;", + ReplicationStatus: replication.Completed, + VersionID: "a3348c34-c352-4498-82f0-1098e8b34df8", + UserDefined: map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;abc", UTCNow().AddDate(0, -1, 0).String())}, + }, + expectedSync: true, + dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}}, + rcfg: replicationConfig{ + remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{ + Arn: "arn1", + ResetID: "xyz", + ResetBeforeDate: UTCNow(), + }}}, + }, + }, + { // 9. existing object replication enabled, versioning enabled, replication status Pending & reset ID different + name: "existing object replication enabled, newer reset in progress on object in Pending replication status", + info: ObjectInfo{ + Size: 100, + ReplicationStatusInternal: "arn1:PENDING;", + + ReplicationStatus: replication.Pending, + VersionID: "a3348c34-c352-4498-82f0-1098e8b34df9", + UserDefined: map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;%s", UTCNow().AddDate(0, 0, -1).Format(http.TimeFormat), "abc")}, + ModTime: UTCNow().AddDate(0, 0, -2), + }, + expectedSync: true, + dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}}, + rcfg: replicationConfig{ + remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{ + Arn: "arn1", + ResetID: "xyz", + ResetBeforeDate: UTCNow(), + }}}, + }, + }, + { // 10. existing object replication enabled, versioning enabled, replication status Complete & reset done + name: "reset done on object in Completed Status - ineligbile for re-replication", + info: ObjectInfo{ + Size: 100, + ReplicationStatusInternal: "arn1:COMPLETED;", + ReplicationStatus: replication.Completed, + VersionID: "a3348c34-c352-4498-82f0-1098e8b34df9", + UserDefined: map[string]string{xhttp.MinIOReplicationResetStatus: fmt.Sprintf("%s;%s", start.Format(http.TimeFormat), "xyz")}, + }, + expectedSync: false, + dsc: ReplicateDecision{targetsMap: map[string]replicateTargetDecision{"arn1": newReplicateTargetDecision("arn1", true, false)}}, + rcfg: replicationConfig{ + remotes: &madmin.BucketTargets{Targets: []madmin.BucketTarget{{ + Arn: "arn1", + ResetID: "xyz", + ResetBeforeDate: start, + }}}, + }, + }, + } +) + +func TestReplicationResyncwrapper(t *testing.T) { + for i, test := range replicationConfigTests2 { + if sync := test.rcfg.resync(test.info, test.dsc, test.tgtStatuses); sync.mustResync() != test.expectedSync { + t.Errorf("%s (%s): Replicationresync got %t , want %t", fmt.Sprintf("Test%d - %s", i+1, time.Now().Format(http.TimeFormat)), test.name, sync.mustResync(), test.expectedSync) + } + } +} diff --git a/cmd/bucket-stats.go b/cmd/bucket-stats.go new file mode 100644 index 00000000..e2eb00aa --- /dev/null +++ b/cmd/bucket-stats.go @@ -0,0 +1,432 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "maps" + "math" + "sync/atomic" + "time" + + "github.com/minio/madmin-go/v3" +) + +//go:generate msgp -file $GOFILE + +// ReplicationLatency holds information of bucket operations latency, such us uploads +type ReplicationLatency struct { + // Single & Multipart PUTs latency + UploadHistogram LastMinuteHistogram +} + +// Merge two replication latency into a new one +func (rl ReplicationLatency) merge(other ReplicationLatency) (newReplLatency ReplicationLatency) { + newReplLatency.UploadHistogram = rl.UploadHistogram.Merge(other.UploadHistogram) + return newReplLatency +} + +// Get upload latency of each object size range +func (rl ReplicationLatency) getUploadLatency() (ret map[string]uint64) { + ret = make(map[string]uint64) + avg := rl.UploadHistogram.GetAvgData() + for k, v := range avg { + // Convert nanoseconds to milliseconds + ret[sizeTagToString(k)] = uint64(v.avg() / time.Millisecond) + } + return ret +} + +// Update replication upload latency with a new value +func (rl *ReplicationLatency) update(size int64, duration time.Duration) { + rl.UploadHistogram.Add(size, duration) +} + +// ReplicationLastMinute has last minute replication counters +type ReplicationLastMinute struct { + LastMinute lastMinuteLatency +} + +func (rl ReplicationLastMinute) merge(other ReplicationLastMinute) (nl ReplicationLastMinute) { + nl = ReplicationLastMinute{rl.LastMinute.merge(other.LastMinute)} + return nl +} + +func (rl *ReplicationLastMinute) addsize(n int64) { + t := time.Now().Unix() + rl.LastMinute.addAll(t-1, AccElem{Total: t - 1, Size: n, N: 1}) +} + +func (rl *ReplicationLastMinute) String() string { + t := rl.LastMinute.getTotal() + return fmt.Sprintf("ReplicationLastMinute sz= %d, n=%d , dur=%d", t.Size, t.N, t.Total) +} + +func (rl *ReplicationLastMinute) getTotal() AccElem { + return rl.LastMinute.getTotal() +} + +// ReplicationLastHour keeps track of replication counts over the last hour +type ReplicationLastHour struct { + Totals [60]AccElem + LastMin int64 +} + +// Merge data of two ReplicationLastHour structure +func (l ReplicationLastHour) merge(o ReplicationLastHour) (merged ReplicationLastHour) { + if l.LastMin > o.LastMin { + o.forwardTo(l.LastMin) + merged.LastMin = l.LastMin + } else { + l.forwardTo(o.LastMin) + merged.LastMin = o.LastMin + } + + for i := range merged.Totals { + merged.Totals[i] = AccElem{ + Total: l.Totals[i].Total + o.Totals[i].Total, + N: l.Totals[i].N + o.Totals[i].N, + Size: l.Totals[i].Size + o.Totals[i].Size, + } + } + return merged +} + +// Add a new duration data +func (l *ReplicationLastHour) addsize(sz int64) { + minutes := time.Now().Unix() / 60 + l.forwardTo(minutes) + winIdx := minutes % 60 + l.Totals[winIdx].merge(AccElem{Total: minutes, Size: sz, N: 1}) + l.LastMin = minutes +} + +// Merge all recorded counts of last hour into one +func (l *ReplicationLastHour) getTotal() AccElem { + var res AccElem + minutes := time.Now().Unix() / 60 + l.forwardTo(minutes) + for _, elem := range l.Totals[:] { + res.merge(elem) + } + return res +} + +// forwardTo time t, clearing any entries in between. +func (l *ReplicationLastHour) forwardTo(t int64) { + if l.LastMin >= t { + return + } + if t-l.LastMin >= 60 { + l.Totals = [60]AccElem{} + return + } + for l.LastMin != t { + // Clear next element. + idx := (l.LastMin + 1) % 60 + l.Totals[idx] = AccElem{} + l.LastMin++ + } +} + +// BucketStatsMap captures bucket statistics for all buckets +type BucketStatsMap struct { + Stats map[string]BucketStats + Timestamp time.Time +} + +// BucketStats bucket statistics +type BucketStats struct { + Uptime int64 `json:"uptime"` + ReplicationStats BucketReplicationStats `json:"currStats"` // current replication stats since cluster startup + QueueStats ReplicationQueueStats `json:"queueStats"` // replication queue stats + ProxyStats ProxyMetric `json:"proxyStats"` +} + +// BucketReplicationStats represents inline replication statistics +// such as pending, failed and completed bytes in total for a bucket +type BucketReplicationStats struct { + Stats map[string]*BucketReplicationStat `json:",omitempty"` + // Completed size in bytes + ReplicatedSize int64 `json:"completedReplicationSize"` + // Total Replica size in bytes + ReplicaSize int64 `json:"replicaSize"` + // Total failed operations including metadata updates for various time frames + Failed madmin.TimedErrStats `json:"failed"` + + // Total number of completed operations + ReplicatedCount int64 `json:"replicationCount"` + // Total number of replica received + ReplicaCount int64 `json:"replicaCount"` + + // in Queue stats for bucket - from qCache + QStat InQueueMetric `json:"queued"` + // Deprecated fields + // Pending size in bytes + PendingSize int64 `json:"pendingReplicationSize"` + // Failed size in bytes + FailedSize int64 `json:"failedReplicationSize"` + // Total number of pending operations including metadata updates + PendingCount int64 `json:"pendingReplicationCount"` + // Total number of failed operations including metadata updates + FailedCount int64 `json:"failedReplicationCount"` +} + +func newBucketReplicationStats() *BucketReplicationStats { + return &BucketReplicationStats{ + Stats: make(map[string]*BucketReplicationStat), + } +} + +// Empty returns true if there are no target stats +func (brs *BucketReplicationStats) Empty() bool { + return len(brs.Stats) == 0 && brs.ReplicaSize == 0 +} + +// Clone creates a new BucketReplicationStats copy +func (brs BucketReplicationStats) Clone() (c BucketReplicationStats) { + // This is called only by replicationStats cache and already holds a + // read lock before calling Clone() + + c = brs + // We need to copy the map, so we do not reference the one in `brs`. + c.Stats = make(map[string]*BucketReplicationStat, len(brs.Stats)) + for arn, st := range brs.Stats { + // make a copy of `*st` + s := BucketReplicationStat{ + ReplicatedSize: st.ReplicatedSize, + ReplicaSize: st.ReplicaSize, + Latency: st.Latency, + BandWidthLimitInBytesPerSecond: st.BandWidthLimitInBytesPerSecond, + CurrentBandwidthInBytesPerSecond: st.CurrentBandwidthInBytesPerSecond, + XferRateLrg: st.XferRateLrg.Clone(), + XferRateSml: st.XferRateSml.Clone(), + ReplicatedCount: st.ReplicatedCount, + Failed: st.Failed, + FailStats: st.FailStats, + } + if s.Failed.ErrCounts == nil { + s.Failed.ErrCounts = make(map[string]int) + maps.Copy(s.Failed.ErrCounts, st.Failed.ErrCounts) + } + c.Stats[arn] = &s + } + return c +} + +// BucketReplicationStat represents inline replication statistics +// such as pending, failed and completed bytes in total for a bucket +// remote target +type BucketReplicationStat struct { + // Pending size in bytes + // PendingSize int64 `json:"pendingReplicationSize"` + // Completed size in bytes + ReplicatedSize int64 `json:"completedReplicationSize"` + // Total Replica size in bytes + ReplicaSize int64 `json:"replicaSize"` + // Collect stats for failures + FailStats RTimedMetrics `json:"-"` + + // Total number of failed operations including metadata updates in the last minute + Failed madmin.TimedErrStats `json:"failed"` + // Total number of completed operations + ReplicatedCount int64 `json:"replicationCount"` + // Replication latency information + Latency ReplicationLatency `json:"replicationLatency"` + // bandwidth limit for target + BandWidthLimitInBytesPerSecond int64 `json:"limitInBits"` + // current bandwidth reported + CurrentBandwidthInBytesPerSecond float64 `json:"currentBandwidth"` + // transfer rate for large uploads + XferRateLrg *XferStats `json:"-" msg:"lt"` + // transfer rate for small uploads + XferRateSml *XferStats `json:"-" msg:"st"` + + // Deprecated fields + // Pending size in bytes + PendingSize int64 `json:"pendingReplicationSize"` + // Failed size in bytes + FailedSize int64 `json:"failedReplicationSize"` + // Total number of pending operations including metadata updates + PendingCount int64 `json:"pendingReplicationCount"` + // Total number of failed operations including metadata updates + FailedCount int64 `json:"failedReplicationCount"` +} + +func (bs *BucketReplicationStat) hasReplicationUsage() bool { + return bs.FailStats.SinceUptime.Count > 0 || + bs.ReplicatedSize > 0 || + bs.ReplicaSize > 0 +} + +func (bs *BucketReplicationStat) updateXferRate(sz int64, duration time.Duration) { + if sz > minLargeObjSize { + bs.XferRateLrg.addSize(sz, duration) + } else { + bs.XferRateSml.addSize(sz, duration) + } +} + +// RMetricName - name of replication metric +type RMetricName string + +const ( + // Large - objects larger than 128MiB + Large RMetricName = "Large" + // Small - objects smaller than 128MiB + Small RMetricName = "Small" + // Total - metric pertaining to totals + Total RMetricName = "Total" +) + +// ReplQNodeStats holds queue stats for replication per node +type ReplQNodeStats struct { + NodeName string `json:"nodeName"` + Uptime int64 `json:"uptime"` + ActiveWorkers ActiveWorkerStat `json:"activeWorkers"` + XferStats map[RMetricName]XferStats `json:"transferSummary"` + TgtXferStats map[string]map[RMetricName]XferStats `json:"tgtTransferStats"` + QStats InQueueMetric `json:"queueStats"` + MRFStats ReplicationMRFStats `json:"mrfStats"` +} + +// getNodeQueueStats returns replication operational stats at the node level +func (r *ReplicationStats) getNodeQueueStats(bucket string) (qs ReplQNodeStats) { + qs.NodeName = globalLocalNodeName + qs.Uptime = UTCNow().Unix() - globalBootTime.Unix() + grs := globalReplicationStats.Load() + if grs != nil { + qs.ActiveWorkers = grs.ActiveWorkers() + } else { + qs.ActiveWorkers = ActiveWorkerStat{} + } + qs.XferStats = make(map[RMetricName]XferStats) + qs.QStats = r.qCache.getBucketStats(bucket) + qs.TgtXferStats = make(map[string]map[RMetricName]XferStats) + qs.MRFStats = ReplicationMRFStats{ + LastFailedCount: atomic.LoadUint64(&r.mrfStats.LastFailedCount), + } + + r.RLock() + defer r.RUnlock() + + brs, ok := r.Cache[bucket] + if !ok { + return qs + } + for arn := range brs.Stats { + qs.TgtXferStats[arn] = make(map[RMetricName]XferStats) + } + count := 0 + var totPeak float64 + // calculate large, small transfers and total transfer rates per replication target at bucket level + for arn, v := range brs.Stats { + lcurrTgt := v.XferRateLrg.curr() + scurrTgt := v.XferRateSml.curr() + totPeak = math.Max(math.Max(v.XferRateLrg.Peak, v.XferRateSml.Peak), totPeak) + totPeak = math.Max(math.Max(lcurrTgt, scurrTgt), totPeak) + tcount := 0 + if v.XferRateLrg.Peak > 0 { + tcount++ + } + if v.XferRateSml.Peak > 0 { + tcount++ + } + qs.TgtXferStats[arn][Large] = XferStats{ + Avg: v.XferRateLrg.Avg, + Curr: lcurrTgt, + Peak: math.Max(v.XferRateLrg.Peak, lcurrTgt), + } + qs.TgtXferStats[arn][Small] = XferStats{ + Avg: v.XferRateSml.Avg, + Curr: scurrTgt, + Peak: math.Max(v.XferRateSml.Peak, scurrTgt), + } + if tcount > 0 { + qs.TgtXferStats[arn][Total] = XferStats{ + Avg: (v.XferRateLrg.Avg + v.XferRateSml.Avg) / float64(tcount), + Curr: (scurrTgt + lcurrTgt) / float64(tcount), + Peak: totPeak, + } + } + } + // calculate large, small and total transfer rates for a minio node + var lavg, lcurr, lpeak, savg, scurr, speak, totpeak float64 + for _, v := range qs.TgtXferStats { + tot := v[Total] + lavg += v[Large].Avg + lcurr += v[Large].Curr + savg += v[Small].Avg + scurr += v[Small].Curr + totpeak = math.Max(math.Max(tot.Peak, totpeak), tot.Curr) + lpeak = math.Max(math.Max(v[Large].Peak, lpeak), v[Large].Curr) + speak = math.Max(math.Max(v[Small].Peak, speak), v[Small].Curr) + if lpeak > 0 || speak > 0 { + count++ + } + } + if count > 0 { + lrg := XferStats{ + Avg: lavg / float64(count), + Curr: lcurr / float64(count), + Peak: lpeak, + } + sml := XferStats{ + Avg: savg / float64(count), + Curr: scurr / float64(count), + Peak: speak, + } + qs.XferStats[Large] = lrg + qs.XferStats[Small] = sml + qs.XferStats[Total] = XferStats{ + Avg: (savg + lavg) / float64(count), + Curr: (lcurr + scurr) / float64(count), + Peak: totpeak, + } + } + return qs +} + +// populate queue totals for node and active workers in use for metrics +func (r *ReplicationStats) getNodeQueueStatsSummary() (qs ReplQNodeStats) { + qs.NodeName = globalLocalNodeName + qs.Uptime = UTCNow().Unix() - globalBootTime.Unix() + qs.ActiveWorkers = globalReplicationStats.Load().ActiveWorkers() + qs.XferStats = make(map[RMetricName]XferStats) + qs.QStats = r.qCache.getSiteStats() + qs.MRFStats = ReplicationMRFStats{ + LastFailedCount: atomic.LoadUint64(&r.mrfStats.LastFailedCount), + } + r.RLock() + defer r.RUnlock() + tx := newXferStats() + for _, brs := range r.Cache { + for _, v := range brs.Stats { + tx := tx.merge(*v.XferRateLrg) + tx = tx.merge(*v.XferRateSml) + } + } + qs.XferStats[Total] = *tx + return qs +} + +// ReplicationQueueStats holds overall queue stats for replication +type ReplicationQueueStats struct { + Nodes []ReplQNodeStats `json:"nodes"` + Uptime int64 `json:"uptime"` +} diff --git a/cmd/bucket-stats_gen.go b/cmd/bucket-stats_gen.go new file mode 100644 index 00000000..201c574c --- /dev/null +++ b/cmd/bucket-stats_gen.go @@ -0,0 +1,2393 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *BucketReplicationStat) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ReplicatedSize": + z.ReplicatedSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ReplicatedSize") + return + } + case "ReplicaSize": + z.ReplicaSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ReplicaSize") + return + } + case "FailStats": + err = z.FailStats.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "FailStats") + return + } + case "Failed": + err = z.Failed.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Failed") + return + } + case "ReplicatedCount": + z.ReplicatedCount, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ReplicatedCount") + return + } + case "Latency": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Latency") + return + } + for zb0002 > 0 { + zb0002-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "Latency") + return + } + switch msgp.UnsafeString(field) { + case "UploadHistogram": + err = z.Latency.UploadHistogram.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Latency", "UploadHistogram") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "Latency") + return + } + } + } + case "BandWidthLimitInBytesPerSecond": + z.BandWidthLimitInBytesPerSecond, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "BandWidthLimitInBytesPerSecond") + return + } + case "CurrentBandwidthInBytesPerSecond": + z.CurrentBandwidthInBytesPerSecond, err = dc.ReadFloat64() + if err != nil { + err = msgp.WrapError(err, "CurrentBandwidthInBytesPerSecond") + return + } + case "lt": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "XferRateLrg") + return + } + z.XferRateLrg = nil + } else { + if z.XferRateLrg == nil { + z.XferRateLrg = new(XferStats) + } + err = z.XferRateLrg.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "XferRateLrg") + return + } + } + case "st": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "XferRateSml") + return + } + z.XferRateSml = nil + } else { + if z.XferRateSml == nil { + z.XferRateSml = new(XferStats) + } + err = z.XferRateSml.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "XferRateSml") + return + } + } + case "PendingSize": + z.PendingSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "PendingSize") + return + } + case "FailedSize": + z.FailedSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "FailedSize") + return + } + case "PendingCount": + z.PendingCount, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "PendingCount") + return + } + case "FailedCount": + z.FailedCount, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "FailedCount") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *BucketReplicationStat) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 14 + // write "ReplicatedSize" + err = en.Append(0x8e, 0xae, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.ReplicatedSize) + if err != nil { + err = msgp.WrapError(err, "ReplicatedSize") + return + } + // write "ReplicaSize" + err = en.Append(0xab, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.ReplicaSize) + if err != nil { + err = msgp.WrapError(err, "ReplicaSize") + return + } + // write "FailStats" + err = en.Append(0xa9, 0x46, 0x61, 0x69, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x73) + if err != nil { + return + } + err = z.FailStats.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "FailStats") + return + } + // write "Failed" + err = en.Append(0xa6, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64) + if err != nil { + return + } + err = z.Failed.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Failed") + return + } + // write "ReplicatedCount" + err = en.Append(0xaf, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteInt64(z.ReplicatedCount) + if err != nil { + err = msgp.WrapError(err, "ReplicatedCount") + return + } + // write "Latency" + err = en.Append(0xa7, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79) + if err != nil { + return + } + // map header, size 1 + // write "UploadHistogram" + err = en.Append(0x81, 0xaf, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d) + if err != nil { + return + } + err = z.Latency.UploadHistogram.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Latency", "UploadHistogram") + return + } + // write "BandWidthLimitInBytesPerSecond" + err = en.Append(0xbe, 0x42, 0x61, 0x6e, 0x64, 0x57, 0x69, 0x64, 0x74, 0x68, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x49, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64) + if err != nil { + return + } + err = en.WriteInt64(z.BandWidthLimitInBytesPerSecond) + if err != nil { + err = msgp.WrapError(err, "BandWidthLimitInBytesPerSecond") + return + } + // write "CurrentBandwidthInBytesPerSecond" + err = en.Append(0xd9, 0x20, 0x43, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, 0x74, 0x68, 0x49, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64) + if err != nil { + return + } + err = en.WriteFloat64(z.CurrentBandwidthInBytesPerSecond) + if err != nil { + err = msgp.WrapError(err, "CurrentBandwidthInBytesPerSecond") + return + } + // write "lt" + err = en.Append(0xa2, 0x6c, 0x74) + if err != nil { + return + } + if z.XferRateLrg == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.XferRateLrg.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "XferRateLrg") + return + } + } + // write "st" + err = en.Append(0xa2, 0x73, 0x74) + if err != nil { + return + } + if z.XferRateSml == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.XferRateSml.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "XferRateSml") + return + } + } + // write "PendingSize" + err = en.Append(0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.PendingSize) + if err != nil { + err = msgp.WrapError(err, "PendingSize") + return + } + // write "FailedSize" + err = en.Append(0xaa, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.FailedSize) + if err != nil { + err = msgp.WrapError(err, "FailedSize") + return + } + // write "PendingCount" + err = en.Append(0xac, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x75, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteInt64(z.PendingCount) + if err != nil { + err = msgp.WrapError(err, "PendingCount") + return + } + // write "FailedCount" + err = en.Append(0xab, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteInt64(z.FailedCount) + if err != nil { + err = msgp.WrapError(err, "FailedCount") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *BucketReplicationStat) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 14 + // string "ReplicatedSize" + o = append(o, 0x8e, 0xae, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.ReplicatedSize) + // string "ReplicaSize" + o = append(o, 0xab, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.ReplicaSize) + // string "FailStats" + o = append(o, 0xa9, 0x46, 0x61, 0x69, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x73) + o, err = z.FailStats.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "FailStats") + return + } + // string "Failed" + o = append(o, 0xa6, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64) + o, err = z.Failed.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Failed") + return + } + // string "ReplicatedCount" + o = append(o, 0xaf, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendInt64(o, z.ReplicatedCount) + // string "Latency" + o = append(o, 0xa7, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79) + // map header, size 1 + // string "UploadHistogram" + o = append(o, 0x81, 0xaf, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d) + o, err = z.Latency.UploadHistogram.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Latency", "UploadHistogram") + return + } + // string "BandWidthLimitInBytesPerSecond" + o = append(o, 0xbe, 0x42, 0x61, 0x6e, 0x64, 0x57, 0x69, 0x64, 0x74, 0x68, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x49, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64) + o = msgp.AppendInt64(o, z.BandWidthLimitInBytesPerSecond) + // string "CurrentBandwidthInBytesPerSecond" + o = append(o, 0xd9, 0x20, 0x43, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, 0x74, 0x68, 0x49, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64) + o = msgp.AppendFloat64(o, z.CurrentBandwidthInBytesPerSecond) + // string "lt" + o = append(o, 0xa2, 0x6c, 0x74) + if z.XferRateLrg == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.XferRateLrg.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "XferRateLrg") + return + } + } + // string "st" + o = append(o, 0xa2, 0x73, 0x74) + if z.XferRateSml == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.XferRateSml.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "XferRateSml") + return + } + } + // string "PendingSize" + o = append(o, 0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.PendingSize) + // string "FailedSize" + o = append(o, 0xaa, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.FailedSize) + // string "PendingCount" + o = append(o, 0xac, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendInt64(o, z.PendingCount) + // string "FailedCount" + o = append(o, 0xab, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendInt64(o, z.FailedCount) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BucketReplicationStat) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ReplicatedSize": + z.ReplicatedSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicatedSize") + return + } + case "ReplicaSize": + z.ReplicaSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicaSize") + return + } + case "FailStats": + bts, err = z.FailStats.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "FailStats") + return + } + case "Failed": + bts, err = z.Failed.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Failed") + return + } + case "ReplicatedCount": + z.ReplicatedCount, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicatedCount") + return + } + case "Latency": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Latency") + return + } + for zb0002 > 0 { + zb0002-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "Latency") + return + } + switch msgp.UnsafeString(field) { + case "UploadHistogram": + bts, err = z.Latency.UploadHistogram.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Latency", "UploadHistogram") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "Latency") + return + } + } + } + case "BandWidthLimitInBytesPerSecond": + z.BandWidthLimitInBytesPerSecond, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "BandWidthLimitInBytesPerSecond") + return + } + case "CurrentBandwidthInBytesPerSecond": + z.CurrentBandwidthInBytesPerSecond, bts, err = msgp.ReadFloat64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "CurrentBandwidthInBytesPerSecond") + return + } + case "lt": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.XferRateLrg = nil + } else { + if z.XferRateLrg == nil { + z.XferRateLrg = new(XferStats) + } + bts, err = z.XferRateLrg.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "XferRateLrg") + return + } + } + case "st": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.XferRateSml = nil + } else { + if z.XferRateSml == nil { + z.XferRateSml = new(XferStats) + } + bts, err = z.XferRateSml.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "XferRateSml") + return + } + } + case "PendingSize": + z.PendingSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "PendingSize") + return + } + case "FailedSize": + z.FailedSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "FailedSize") + return + } + case "PendingCount": + z.PendingCount, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "PendingCount") + return + } + case "FailedCount": + z.FailedCount, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "FailedCount") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *BucketReplicationStat) Msgsize() (s int) { + s = 1 + 15 + msgp.Int64Size + 12 + msgp.Int64Size + 10 + z.FailStats.Msgsize() + 7 + z.Failed.Msgsize() + 16 + msgp.Int64Size + 8 + 1 + 16 + z.Latency.UploadHistogram.Msgsize() + 31 + msgp.Int64Size + 34 + msgp.Float64Size + 3 + if z.XferRateLrg == nil { + s += msgp.NilSize + } else { + s += z.XferRateLrg.Msgsize() + } + s += 3 + if z.XferRateSml == nil { + s += msgp.NilSize + } else { + s += z.XferRateSml.Msgsize() + } + s += 12 + msgp.Int64Size + 11 + msgp.Int64Size + 13 + msgp.Int64Size + 12 + msgp.Int64Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *BucketReplicationStats) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Stats": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Stats") + return + } + if z.Stats == nil { + z.Stats = make(map[string]*BucketReplicationStat, zb0002) + } else if len(z.Stats) > 0 { + clear(z.Stats) + } + for zb0002 > 0 { + zb0002-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Stats") + return + } + var za0002 *BucketReplicationStat + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "Stats", za0001) + return + } + za0002 = nil + } else { + if za0002 == nil { + za0002 = new(BucketReplicationStat) + } + err = za0002.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Stats", za0001) + return + } + } + z.Stats[za0001] = za0002 + } + case "ReplicatedSize": + z.ReplicatedSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ReplicatedSize") + return + } + case "ReplicaSize": + z.ReplicaSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ReplicaSize") + return + } + case "Failed": + err = z.Failed.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Failed") + return + } + case "ReplicatedCount": + z.ReplicatedCount, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ReplicatedCount") + return + } + case "ReplicaCount": + z.ReplicaCount, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ReplicaCount") + return + } + case "QStat": + err = z.QStat.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "QStat") + return + } + case "PendingSize": + z.PendingSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "PendingSize") + return + } + case "FailedSize": + z.FailedSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "FailedSize") + return + } + case "PendingCount": + z.PendingCount, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "PendingCount") + return + } + case "FailedCount": + z.FailedCount, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "FailedCount") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *BucketReplicationStats) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 11 + // write "Stats" + err = en.Append(0x8b, 0xa5, 0x53, 0x74, 0x61, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.Stats))) + if err != nil { + err = msgp.WrapError(err, "Stats") + return + } + for za0001, za0002 := range z.Stats { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "Stats") + return + } + if za0002 == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = za0002.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Stats", za0001) + return + } + } + } + // write "ReplicatedSize" + err = en.Append(0xae, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.ReplicatedSize) + if err != nil { + err = msgp.WrapError(err, "ReplicatedSize") + return + } + // write "ReplicaSize" + err = en.Append(0xab, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.ReplicaSize) + if err != nil { + err = msgp.WrapError(err, "ReplicaSize") + return + } + // write "Failed" + err = en.Append(0xa6, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64) + if err != nil { + return + } + err = z.Failed.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Failed") + return + } + // write "ReplicatedCount" + err = en.Append(0xaf, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteInt64(z.ReplicatedCount) + if err != nil { + err = msgp.WrapError(err, "ReplicatedCount") + return + } + // write "ReplicaCount" + err = en.Append(0xac, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x43, 0x6f, 0x75, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteInt64(z.ReplicaCount) + if err != nil { + err = msgp.WrapError(err, "ReplicaCount") + return + } + // write "QStat" + err = en.Append(0xa5, 0x51, 0x53, 0x74, 0x61, 0x74) + if err != nil { + return + } + err = z.QStat.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "QStat") + return + } + // write "PendingSize" + err = en.Append(0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.PendingSize) + if err != nil { + err = msgp.WrapError(err, "PendingSize") + return + } + // write "FailedSize" + err = en.Append(0xaa, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.FailedSize) + if err != nil { + err = msgp.WrapError(err, "FailedSize") + return + } + // write "PendingCount" + err = en.Append(0xac, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x75, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteInt64(z.PendingCount) + if err != nil { + err = msgp.WrapError(err, "PendingCount") + return + } + // write "FailedCount" + err = en.Append(0xab, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteInt64(z.FailedCount) + if err != nil { + err = msgp.WrapError(err, "FailedCount") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *BucketReplicationStats) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 11 + // string "Stats" + o = append(o, 0x8b, 0xa5, 0x53, 0x74, 0x61, 0x74, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.Stats))) + for za0001, za0002 := range z.Stats { + o = msgp.AppendString(o, za0001) + if za0002 == nil { + o = msgp.AppendNil(o) + } else { + o, err = za0002.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Stats", za0001) + return + } + } + } + // string "ReplicatedSize" + o = append(o, 0xae, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.ReplicatedSize) + // string "ReplicaSize" + o = append(o, 0xab, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.ReplicaSize) + // string "Failed" + o = append(o, 0xa6, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64) + o, err = z.Failed.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Failed") + return + } + // string "ReplicatedCount" + o = append(o, 0xaf, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendInt64(o, z.ReplicatedCount) + // string "ReplicaCount" + o = append(o, 0xac, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendInt64(o, z.ReplicaCount) + // string "QStat" + o = append(o, 0xa5, 0x51, 0x53, 0x74, 0x61, 0x74) + o, err = z.QStat.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "QStat") + return + } + // string "PendingSize" + o = append(o, 0xab, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.PendingSize) + // string "FailedSize" + o = append(o, 0xaa, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.FailedSize) + // string "PendingCount" + o = append(o, 0xac, 0x50, 0x65, 0x6e, 0x64, 0x69, 0x6e, 0x67, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendInt64(o, z.PendingCount) + // string "FailedCount" + o = append(o, 0xab, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendInt64(o, z.FailedCount) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BucketReplicationStats) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Stats": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Stats") + return + } + if z.Stats == nil { + z.Stats = make(map[string]*BucketReplicationStat, zb0002) + } else if len(z.Stats) > 0 { + clear(z.Stats) + } + for zb0002 > 0 { + var za0002 *BucketReplicationStat + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Stats") + return + } + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + za0002 = nil + } else { + if za0002 == nil { + za0002 = new(BucketReplicationStat) + } + bts, err = za0002.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Stats", za0001) + return + } + } + z.Stats[za0001] = za0002 + } + case "ReplicatedSize": + z.ReplicatedSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicatedSize") + return + } + case "ReplicaSize": + z.ReplicaSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicaSize") + return + } + case "Failed": + bts, err = z.Failed.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Failed") + return + } + case "ReplicatedCount": + z.ReplicatedCount, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicatedCount") + return + } + case "ReplicaCount": + z.ReplicaCount, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicaCount") + return + } + case "QStat": + bts, err = z.QStat.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "QStat") + return + } + case "PendingSize": + z.PendingSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "PendingSize") + return + } + case "FailedSize": + z.FailedSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "FailedSize") + return + } + case "PendingCount": + z.PendingCount, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "PendingCount") + return + } + case "FailedCount": + z.FailedCount, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "FailedCount") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *BucketReplicationStats) Msgsize() (s int) { + s = 1 + 6 + msgp.MapHeaderSize + if z.Stats != nil { + for za0001, za0002 := range z.Stats { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + if za0002 == nil { + s += msgp.NilSize + } else { + s += za0002.Msgsize() + } + } + } + s += 15 + msgp.Int64Size + 12 + msgp.Int64Size + 7 + z.Failed.Msgsize() + 16 + msgp.Int64Size + 13 + msgp.Int64Size + 6 + z.QStat.Msgsize() + 12 + msgp.Int64Size + 11 + msgp.Int64Size + 13 + msgp.Int64Size + 12 + msgp.Int64Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *BucketStats) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Uptime": + z.Uptime, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Uptime") + return + } + case "ReplicationStats": + err = z.ReplicationStats.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "ReplicationStats") + return + } + case "QueueStats": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "QueueStats") + return + } + for zb0002 > 0 { + zb0002-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "QueueStats") + return + } + switch msgp.UnsafeString(field) { + case "Nodes": + var zb0003 uint32 + zb0003, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "QueueStats", "Nodes") + return + } + if cap(z.QueueStats.Nodes) >= int(zb0003) { + z.QueueStats.Nodes = (z.QueueStats.Nodes)[:zb0003] + } else { + z.QueueStats.Nodes = make([]ReplQNodeStats, zb0003) + } + for za0001 := range z.QueueStats.Nodes { + err = z.QueueStats.Nodes[za0001].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "QueueStats", "Nodes", za0001) + return + } + } + case "Uptime": + z.QueueStats.Uptime, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "QueueStats", "Uptime") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "QueueStats") + return + } + } + } + case "ProxyStats": + err = z.ProxyStats.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "ProxyStats") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *BucketStats) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 4 + // write "Uptime" + err = en.Append(0x84, 0xa6, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.Uptime) + if err != nil { + err = msgp.WrapError(err, "Uptime") + return + } + // write "ReplicationStats" + err = en.Append(0xb0, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73) + if err != nil { + return + } + err = z.ReplicationStats.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "ReplicationStats") + return + } + // write "QueueStats" + err = en.Append(0xaa, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, 0x74, 0x61, 0x74, 0x73) + if err != nil { + return + } + // map header, size 2 + // write "Nodes" + err = en.Append(0x82, 0xa5, 0x4e, 0x6f, 0x64, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.QueueStats.Nodes))) + if err != nil { + err = msgp.WrapError(err, "QueueStats", "Nodes") + return + } + for za0001 := range z.QueueStats.Nodes { + err = z.QueueStats.Nodes[za0001].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "QueueStats", "Nodes", za0001) + return + } + } + // write "Uptime" + err = en.Append(0xa6, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.QueueStats.Uptime) + if err != nil { + err = msgp.WrapError(err, "QueueStats", "Uptime") + return + } + // write "ProxyStats" + err = en.Append(0xaa, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x53, 0x74, 0x61, 0x74, 0x73) + if err != nil { + return + } + err = z.ProxyStats.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "ProxyStats") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *BucketStats) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 4 + // string "Uptime" + o = append(o, 0x84, 0xa6, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65) + o = msgp.AppendInt64(o, z.Uptime) + // string "ReplicationStats" + o = append(o, 0xb0, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x73) + o, err = z.ReplicationStats.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "ReplicationStats") + return + } + // string "QueueStats" + o = append(o, 0xaa, 0x51, 0x75, 0x65, 0x75, 0x65, 0x53, 0x74, 0x61, 0x74, 0x73) + // map header, size 2 + // string "Nodes" + o = append(o, 0x82, 0xa5, 0x4e, 0x6f, 0x64, 0x65, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.QueueStats.Nodes))) + for za0001 := range z.QueueStats.Nodes { + o, err = z.QueueStats.Nodes[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "QueueStats", "Nodes", za0001) + return + } + } + // string "Uptime" + o = append(o, 0xa6, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65) + o = msgp.AppendInt64(o, z.QueueStats.Uptime) + // string "ProxyStats" + o = append(o, 0xaa, 0x50, 0x72, 0x6f, 0x78, 0x79, 0x53, 0x74, 0x61, 0x74, 0x73) + o, err = z.ProxyStats.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "ProxyStats") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BucketStats) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Uptime": + z.Uptime, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Uptime") + return + } + case "ReplicationStats": + bts, err = z.ReplicationStats.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicationStats") + return + } + case "QueueStats": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "QueueStats") + return + } + for zb0002 > 0 { + zb0002-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "QueueStats") + return + } + switch msgp.UnsafeString(field) { + case "Nodes": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "QueueStats", "Nodes") + return + } + if cap(z.QueueStats.Nodes) >= int(zb0003) { + z.QueueStats.Nodes = (z.QueueStats.Nodes)[:zb0003] + } else { + z.QueueStats.Nodes = make([]ReplQNodeStats, zb0003) + } + for za0001 := range z.QueueStats.Nodes { + bts, err = z.QueueStats.Nodes[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "QueueStats", "Nodes", za0001) + return + } + } + case "Uptime": + z.QueueStats.Uptime, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "QueueStats", "Uptime") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "QueueStats") + return + } + } + } + case "ProxyStats": + bts, err = z.ProxyStats.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ProxyStats") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *BucketStats) Msgsize() (s int) { + s = 1 + 7 + msgp.Int64Size + 17 + z.ReplicationStats.Msgsize() + 11 + 1 + 6 + msgp.ArrayHeaderSize + for za0001 := range z.QueueStats.Nodes { + s += z.QueueStats.Nodes[za0001].Msgsize() + } + s += 7 + msgp.Int64Size + 11 + z.ProxyStats.Msgsize() + return +} + +// DecodeMsg implements msgp.Decodable +func (z *BucketStatsMap) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Stats": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Stats") + return + } + if z.Stats == nil { + z.Stats = make(map[string]BucketStats, zb0002) + } else if len(z.Stats) > 0 { + clear(z.Stats) + } + for zb0002 > 0 { + zb0002-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Stats") + return + } + var za0002 BucketStats + err = za0002.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Stats", za0001) + return + } + z.Stats[za0001] = za0002 + } + case "Timestamp": + z.Timestamp, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "Timestamp") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *BucketStatsMap) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "Stats" + err = en.Append(0x82, 0xa5, 0x53, 0x74, 0x61, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.Stats))) + if err != nil { + err = msgp.WrapError(err, "Stats") + return + } + for za0001, za0002 := range z.Stats { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "Stats") + return + } + err = za0002.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Stats", za0001) + return + } + } + // write "Timestamp" + err = en.Append(0xa9, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70) + if err != nil { + return + } + err = en.WriteTime(z.Timestamp) + if err != nil { + err = msgp.WrapError(err, "Timestamp") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *BucketStatsMap) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "Stats" + o = append(o, 0x82, 0xa5, 0x53, 0x74, 0x61, 0x74, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.Stats))) + for za0001, za0002 := range z.Stats { + o = msgp.AppendString(o, za0001) + o, err = za0002.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Stats", za0001) + return + } + } + // string "Timestamp" + o = append(o, 0xa9, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70) + o = msgp.AppendTime(o, z.Timestamp) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BucketStatsMap) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Stats": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Stats") + return + } + if z.Stats == nil { + z.Stats = make(map[string]BucketStats, zb0002) + } else if len(z.Stats) > 0 { + clear(z.Stats) + } + for zb0002 > 0 { + var za0002 BucketStats + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Stats") + return + } + bts, err = za0002.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Stats", za0001) + return + } + z.Stats[za0001] = za0002 + } + case "Timestamp": + z.Timestamp, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Timestamp") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *BucketStatsMap) Msgsize() (s int) { + s = 1 + 6 + msgp.MapHeaderSize + if z.Stats != nil { + for za0001, za0002 := range z.Stats { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize() + } + } + s += 10 + msgp.TimeSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *RMetricName) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 string + zb0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = RMetricName(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z RMetricName) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteString(string(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z RMetricName) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendString(o, string(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *RMetricName) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 string + zb0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = RMetricName(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z RMetricName) Msgsize() (s int) { + s = msgp.StringPrefixSize + len(string(z)) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ReplQNodeStats) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "NodeName": + z.NodeName, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "NodeName") + return + } + case "Uptime": + z.Uptime, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Uptime") + return + } + case "ActiveWorkers": + err = z.ActiveWorkers.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "ActiveWorkers") + return + } + case "QStats": + err = z.QStats.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "QStats") + return + } + case "MRFStats": + err = z.MRFStats.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "MRFStats") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *ReplQNodeStats) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 5 + // write "NodeName" + err = en.Append(0x85, 0xa8, 0x4e, 0x6f, 0x64, 0x65, 0x4e, 0x61, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteString(z.NodeName) + if err != nil { + err = msgp.WrapError(err, "NodeName") + return + } + // write "Uptime" + err = en.Append(0xa6, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.Uptime) + if err != nil { + err = msgp.WrapError(err, "Uptime") + return + } + // write "ActiveWorkers" + err = en.Append(0xad, 0x41, 0x63, 0x74, 0x69, 0x76, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73) + if err != nil { + return + } + err = z.ActiveWorkers.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "ActiveWorkers") + return + } + // write "QStats" + err = en.Append(0xa6, 0x51, 0x53, 0x74, 0x61, 0x74, 0x73) + if err != nil { + return + } + err = z.QStats.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "QStats") + return + } + // write "MRFStats" + err = en.Append(0xa8, 0x4d, 0x52, 0x46, 0x53, 0x74, 0x61, 0x74, 0x73) + if err != nil { + return + } + err = z.MRFStats.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "MRFStats") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ReplQNodeStats) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 5 + // string "NodeName" + o = append(o, 0x85, 0xa8, 0x4e, 0x6f, 0x64, 0x65, 0x4e, 0x61, 0x6d, 0x65) + o = msgp.AppendString(o, z.NodeName) + // string "Uptime" + o = append(o, 0xa6, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65) + o = msgp.AppendInt64(o, z.Uptime) + // string "ActiveWorkers" + o = append(o, 0xad, 0x41, 0x63, 0x74, 0x69, 0x76, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73) + o, err = z.ActiveWorkers.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "ActiveWorkers") + return + } + // string "QStats" + o = append(o, 0xa6, 0x51, 0x53, 0x74, 0x61, 0x74, 0x73) + o, err = z.QStats.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "QStats") + return + } + // string "MRFStats" + o = append(o, 0xa8, 0x4d, 0x52, 0x46, 0x53, 0x74, 0x61, 0x74, 0x73) + o, err = z.MRFStats.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "MRFStats") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ReplQNodeStats) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "NodeName": + z.NodeName, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "NodeName") + return + } + case "Uptime": + z.Uptime, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Uptime") + return + } + case "ActiveWorkers": + bts, err = z.ActiveWorkers.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ActiveWorkers") + return + } + case "QStats": + bts, err = z.QStats.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "QStats") + return + } + case "MRFStats": + bts, err = z.MRFStats.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "MRFStats") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ReplQNodeStats) Msgsize() (s int) { + s = 1 + 9 + msgp.StringPrefixSize + len(z.NodeName) + 7 + msgp.Int64Size + 14 + z.ActiveWorkers.Msgsize() + 7 + z.QStats.Msgsize() + 9 + z.MRFStats.Msgsize() + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ReplicationLastHour) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Totals": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Totals") + return + } + if zb0002 != uint32(60) { + err = msgp.ArrayError{Wanted: uint32(60), Got: zb0002} + return + } + for za0001 := range z.Totals { + err = z.Totals[za0001].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Totals", za0001) + return + } + } + case "LastMin": + z.LastMin, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "LastMin") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *ReplicationLastHour) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "Totals" + err = en.Append(0x82, 0xa6, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(60)) + if err != nil { + err = msgp.WrapError(err, "Totals") + return + } + for za0001 := range z.Totals { + err = z.Totals[za0001].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Totals", za0001) + return + } + } + // write "LastMin" + err = en.Append(0xa7, 0x4c, 0x61, 0x73, 0x74, 0x4d, 0x69, 0x6e) + if err != nil { + return + } + err = en.WriteInt64(z.LastMin) + if err != nil { + err = msgp.WrapError(err, "LastMin") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ReplicationLastHour) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "Totals" + o = append(o, 0x82, 0xa6, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73) + o = msgp.AppendArrayHeader(o, uint32(60)) + for za0001 := range z.Totals { + o, err = z.Totals[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Totals", za0001) + return + } + } + // string "LastMin" + o = append(o, 0xa7, 0x4c, 0x61, 0x73, 0x74, 0x4d, 0x69, 0x6e) + o = msgp.AppendInt64(o, z.LastMin) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ReplicationLastHour) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Totals": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Totals") + return + } + if zb0002 != uint32(60) { + err = msgp.ArrayError{Wanted: uint32(60), Got: zb0002} + return + } + for za0001 := range z.Totals { + bts, err = z.Totals[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Totals", za0001) + return + } + } + case "LastMin": + z.LastMin, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "LastMin") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ReplicationLastHour) Msgsize() (s int) { + s = 1 + 7 + msgp.ArrayHeaderSize + for za0001 := range z.Totals { + s += z.Totals[za0001].Msgsize() + } + s += 8 + msgp.Int64Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ReplicationLastMinute) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "LastMinute": + err = z.LastMinute.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "LastMinute") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *ReplicationLastMinute) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "LastMinute" + err = en.Append(0x81, 0xaa, 0x4c, 0x61, 0x73, 0x74, 0x4d, 0x69, 0x6e, 0x75, 0x74, 0x65) + if err != nil { + return + } + err = z.LastMinute.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "LastMinute") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ReplicationLastMinute) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "LastMinute" + o = append(o, 0x81, 0xaa, 0x4c, 0x61, 0x73, 0x74, 0x4d, 0x69, 0x6e, 0x75, 0x74, 0x65) + o, err = z.LastMinute.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "LastMinute") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ReplicationLastMinute) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "LastMinute": + bts, err = z.LastMinute.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "LastMinute") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ReplicationLastMinute) Msgsize() (s int) { + s = 1 + 11 + z.LastMinute.Msgsize() + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ReplicationLatency) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "UploadHistogram": + err = z.UploadHistogram.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "UploadHistogram") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *ReplicationLatency) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "UploadHistogram" + err = en.Append(0x81, 0xaf, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d) + if err != nil { + return + } + err = z.UploadHistogram.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "UploadHistogram") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ReplicationLatency) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "UploadHistogram" + o = append(o, 0x81, 0xaf, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d) + o, err = z.UploadHistogram.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "UploadHistogram") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ReplicationLatency) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "UploadHistogram": + bts, err = z.UploadHistogram.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "UploadHistogram") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ReplicationLatency) Msgsize() (s int) { + s = 1 + 16 + z.UploadHistogram.Msgsize() + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ReplicationQueueStats) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Nodes": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Nodes") + return + } + if cap(z.Nodes) >= int(zb0002) { + z.Nodes = (z.Nodes)[:zb0002] + } else { + z.Nodes = make([]ReplQNodeStats, zb0002) + } + for za0001 := range z.Nodes { + err = z.Nodes[za0001].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Nodes", za0001) + return + } + } + case "Uptime": + z.Uptime, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Uptime") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *ReplicationQueueStats) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "Nodes" + err = en.Append(0x82, 0xa5, 0x4e, 0x6f, 0x64, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Nodes))) + if err != nil { + err = msgp.WrapError(err, "Nodes") + return + } + for za0001 := range z.Nodes { + err = z.Nodes[za0001].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Nodes", za0001) + return + } + } + // write "Uptime" + err = en.Append(0xa6, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.Uptime) + if err != nil { + err = msgp.WrapError(err, "Uptime") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ReplicationQueueStats) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "Nodes" + o = append(o, 0x82, 0xa5, 0x4e, 0x6f, 0x64, 0x65, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Nodes))) + for za0001 := range z.Nodes { + o, err = z.Nodes[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Nodes", za0001) + return + } + } + // string "Uptime" + o = append(o, 0xa6, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65) + o = msgp.AppendInt64(o, z.Uptime) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ReplicationQueueStats) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Nodes": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Nodes") + return + } + if cap(z.Nodes) >= int(zb0002) { + z.Nodes = (z.Nodes)[:zb0002] + } else { + z.Nodes = make([]ReplQNodeStats, zb0002) + } + for za0001 := range z.Nodes { + bts, err = z.Nodes[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Nodes", za0001) + return + } + } + case "Uptime": + z.Uptime, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Uptime") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ReplicationQueueStats) Msgsize() (s int) { + s = 1 + 6 + msgp.ArrayHeaderSize + for za0001 := range z.Nodes { + s += z.Nodes[za0001].Msgsize() + } + s += 7 + msgp.Int64Size + return +} diff --git a/cmd/bucket-stats_gen_test.go b/cmd/bucket-stats_gen_test.go new file mode 100644 index 00000000..18344b43 --- /dev/null +++ b/cmd/bucket-stats_gen_test.go @@ -0,0 +1,1027 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalBucketReplicationStat(t *testing.T) { + v := BucketReplicationStat{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBucketReplicationStat(b *testing.B) { + v := BucketReplicationStat{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBucketReplicationStat(b *testing.B) { + v := BucketReplicationStat{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBucketReplicationStat(b *testing.B) { + v := BucketReplicationStat{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBucketReplicationStat(t *testing.T) { + v := BucketReplicationStat{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBucketReplicationStat Msgsize() is inaccurate") + } + + vn := BucketReplicationStat{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBucketReplicationStat(b *testing.B) { + v := BucketReplicationStat{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBucketReplicationStat(b *testing.B) { + v := BucketReplicationStat{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBucketReplicationStats(t *testing.T) { + v := BucketReplicationStats{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBucketReplicationStats(b *testing.B) { + v := BucketReplicationStats{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBucketReplicationStats(b *testing.B) { + v := BucketReplicationStats{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBucketReplicationStats(b *testing.B) { + v := BucketReplicationStats{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBucketReplicationStats(t *testing.T) { + v := BucketReplicationStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBucketReplicationStats Msgsize() is inaccurate") + } + + vn := BucketReplicationStats{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBucketReplicationStats(b *testing.B) { + v := BucketReplicationStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBucketReplicationStats(b *testing.B) { + v := BucketReplicationStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBucketStats(t *testing.T) { + v := BucketStats{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBucketStats(b *testing.B) { + v := BucketStats{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBucketStats(b *testing.B) { + v := BucketStats{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBucketStats(b *testing.B) { + v := BucketStats{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBucketStats(t *testing.T) { + v := BucketStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBucketStats Msgsize() is inaccurate") + } + + vn := BucketStats{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBucketStats(b *testing.B) { + v := BucketStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBucketStats(b *testing.B) { + v := BucketStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBucketStatsMap(t *testing.T) { + v := BucketStatsMap{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBucketStatsMap(b *testing.B) { + v := BucketStatsMap{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBucketStatsMap(b *testing.B) { + v := BucketStatsMap{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBucketStatsMap(b *testing.B) { + v := BucketStatsMap{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBucketStatsMap(t *testing.T) { + v := BucketStatsMap{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBucketStatsMap Msgsize() is inaccurate") + } + + vn := BucketStatsMap{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBucketStatsMap(b *testing.B) { + v := BucketStatsMap{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBucketStatsMap(b *testing.B) { + v := BucketStatsMap{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalReplQNodeStats(t *testing.T) { + v := ReplQNodeStats{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgReplQNodeStats(b *testing.B) { + v := ReplQNodeStats{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgReplQNodeStats(b *testing.B) { + v := ReplQNodeStats{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalReplQNodeStats(b *testing.B) { + v := ReplQNodeStats{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeReplQNodeStats(t *testing.T) { + v := ReplQNodeStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeReplQNodeStats Msgsize() is inaccurate") + } + + vn := ReplQNodeStats{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeReplQNodeStats(b *testing.B) { + v := ReplQNodeStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeReplQNodeStats(b *testing.B) { + v := ReplQNodeStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalReplicationLastHour(t *testing.T) { + v := ReplicationLastHour{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgReplicationLastHour(b *testing.B) { + v := ReplicationLastHour{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgReplicationLastHour(b *testing.B) { + v := ReplicationLastHour{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalReplicationLastHour(b *testing.B) { + v := ReplicationLastHour{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeReplicationLastHour(t *testing.T) { + v := ReplicationLastHour{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeReplicationLastHour Msgsize() is inaccurate") + } + + vn := ReplicationLastHour{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeReplicationLastHour(b *testing.B) { + v := ReplicationLastHour{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeReplicationLastHour(b *testing.B) { + v := ReplicationLastHour{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalReplicationLastMinute(t *testing.T) { + v := ReplicationLastMinute{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgReplicationLastMinute(b *testing.B) { + v := ReplicationLastMinute{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgReplicationLastMinute(b *testing.B) { + v := ReplicationLastMinute{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalReplicationLastMinute(b *testing.B) { + v := ReplicationLastMinute{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeReplicationLastMinute(t *testing.T) { + v := ReplicationLastMinute{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeReplicationLastMinute Msgsize() is inaccurate") + } + + vn := ReplicationLastMinute{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeReplicationLastMinute(b *testing.B) { + v := ReplicationLastMinute{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeReplicationLastMinute(b *testing.B) { + v := ReplicationLastMinute{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalReplicationLatency(t *testing.T) { + v := ReplicationLatency{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgReplicationLatency(b *testing.B) { + v := ReplicationLatency{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgReplicationLatency(b *testing.B) { + v := ReplicationLatency{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalReplicationLatency(b *testing.B) { + v := ReplicationLatency{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeReplicationLatency(t *testing.T) { + v := ReplicationLatency{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeReplicationLatency Msgsize() is inaccurate") + } + + vn := ReplicationLatency{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeReplicationLatency(b *testing.B) { + v := ReplicationLatency{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeReplicationLatency(b *testing.B) { + v := ReplicationLatency{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalReplicationQueueStats(t *testing.T) { + v := ReplicationQueueStats{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgReplicationQueueStats(b *testing.B) { + v := ReplicationQueueStats{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgReplicationQueueStats(b *testing.B) { + v := ReplicationQueueStats{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalReplicationQueueStats(b *testing.B) { + v := ReplicationQueueStats{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeReplicationQueueStats(t *testing.T) { + v := ReplicationQueueStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeReplicationQueueStats Msgsize() is inaccurate") + } + + vn := ReplicationQueueStats{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeReplicationQueueStats(b *testing.B) { + v := ReplicationQueueStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeReplicationQueueStats(b *testing.B) { + v := ReplicationQueueStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/bucket-targets.go b/cmd/bucket-targets.go new file mode 100644 index 00000000..9bd67a05 --- /dev/null +++ b/cmd/bucket-targets.go @@ -0,0 +1,768 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "maps" + "net/url" + "sync" + "time" + + jsoniter "github.com/json-iterator/go" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/kms" +) + +const ( + defaultHealthCheckDuration = 5 * time.Second + // default interval for reload of all remote target endpoints + defaultHealthCheckReloadDuration = 30 * time.Minute +) + +type arnTarget struct { + Client *TargetClient + lastRefresh time.Time +} + +// arnErrs represents number of errors seen for a ARN and if update is in progress +// to refresh remote targets from bucket metadata. +type arnErrs struct { + count int64 + updateInProgress bool + bucket string +} + +// BucketTargetSys represents bucket targets subsystem +type BucketTargetSys struct { + sync.RWMutex + arnRemotesMap map[string]arnTarget + targetsMap map[string][]madmin.BucketTarget + hMutex sync.RWMutex + hc map[string]epHealth + hcClient *madmin.AnonymousClient + aMutex sync.RWMutex + arnErrsMap map[string]arnErrs // map of ARN to error count of failures to get target +} + +type latencyStat struct { + lastmin lastMinuteLatency + curr time.Duration + avg time.Duration + peak time.Duration + N int64 +} + +func (l *latencyStat) update(d time.Duration) { + l.lastmin.add(d) + l.N++ + if d > l.peak { + l.peak = d + } + l.curr = l.lastmin.getTotal().avg() + l.avg = time.Duration((int64(l.avg)*(l.N-1) + int64(l.curr)) / l.N) +} + +// epHealth struct represents health of a replication target endpoint. +type epHealth struct { + Endpoint string + Scheme string + Online bool + lastOnline time.Time + lastHCAt time.Time + offlineDuration time.Duration + latency latencyStat +} + +// isOffline returns current liveness result of remote target. Add endpoint to +// healthCheck map if missing and default to online status +func (sys *BucketTargetSys) isOffline(ep *url.URL) bool { + sys.hMutex.RLock() + defer sys.hMutex.RUnlock() + if h, ok := sys.hc[ep.Host]; ok { + return !h.Online + } + go sys.initHC(ep) + return false +} + +// markOffline sets endpoint to offline if network i/o timeout seen. +func (sys *BucketTargetSys) markOffline(ep *url.URL) { + sys.hMutex.Lock() + defer sys.hMutex.Unlock() + if h, ok := sys.hc[ep.Host]; ok { + h.Online = false + sys.hc[ep.Host] = h + } +} + +func (sys *BucketTargetSys) initHC(ep *url.URL) { + sys.hMutex.Lock() + sys.hc[ep.Host] = epHealth{ + Endpoint: ep.Host, + Scheme: ep.Scheme, + Online: true, + } + sys.hMutex.Unlock() +} + +// newHCClient initializes an anonymous client for performing health check on the remote endpoints +func newHCClient() *madmin.AnonymousClient { + clnt, e := madmin.NewAnonymousClientNoEndpoint() + if e != nil { + bugLogIf(GlobalContext, errors.New("Unable to initialize health check client")) + return nil + } + clnt.SetCustomTransport(globalRemoteTargetTransport) + return clnt +} + +// heartBeat performs liveness check on remote endpoints. +func (sys *BucketTargetSys) heartBeat(ctx context.Context) { + hcTimer := time.NewTimer(defaultHealthCheckDuration) + defer hcTimer.Stop() + for { + select { + case <-hcTimer.C: + sys.hMutex.RLock() + eps := make([]madmin.ServerProperties, 0, len(sys.hc)) + for _, ep := range sys.hc { + eps = append(eps, madmin.ServerProperties{Endpoint: ep.Endpoint, Scheme: ep.Scheme}) + } + sys.hMutex.RUnlock() + + if len(eps) > 0 { + cctx, cancel := context.WithTimeout(ctx, 30*time.Second) + m := make(map[string]epHealth, len(eps)) + start := time.Now() + + for result := range sys.hcClient.Alive(cctx, madmin.AliveOpts{}, eps...) { + var lastOnline time.Time + var offline time.Duration + // var deploymentID string + sys.hMutex.RLock() + prev, ok := sys.hc[result.Endpoint.Host] + sys.hMutex.RUnlock() + if ok { + if prev.Online != result.Online || !result.Online { + if !prev.lastHCAt.IsZero() { + offline = time.Since(prev.lastHCAt) + prev.offlineDuration + } else { + offline = prev.offlineDuration + } + } else if result.Online { + offline = prev.offlineDuration + } + } + lastOnline = prev.lastOnline + if result.Online { + lastOnline = time.Now() + } + l := prev.latency + l.update(time.Since(start)) + m[result.Endpoint.Host] = epHealth{ + Endpoint: result.Endpoint.Host, + Scheme: result.Endpoint.Scheme, + Online: result.Online, + lastOnline: lastOnline, + offlineDuration: offline, + lastHCAt: time.Now(), + latency: l, + } + } + cancel() + sys.hMutex.Lock() + sys.hc = m + sys.hMutex.Unlock() + } + hcTimer.Reset(defaultHealthCheckDuration) + case <-ctx.Done(): + return + } + } +} + +// periodically rebuild the healthCheck map from list of targets to clear +// out stale endpoints +func (sys *BucketTargetSys) reloadHealthCheckers(ctx context.Context) { + m := make(map[string]epHealth) + tgts := sys.ListTargets(ctx, "", "") + sys.hMutex.Lock() + for _, t := range tgts { + if _, ok := m[t.Endpoint]; !ok { + scheme := "http" + if t.Secure { + scheme = "https" + } + epHealth := epHealth{ + Online: true, + Endpoint: t.Endpoint, + Scheme: scheme, + } + if prev, ok := sys.hc[t.Endpoint]; ok { + epHealth.lastOnline = prev.lastOnline + epHealth.offlineDuration = prev.offlineDuration + epHealth.lastHCAt = prev.lastHCAt + epHealth.latency = prev.latency + } + m[t.Endpoint] = epHealth + } + } + // swap out the map + sys.hc = m + sys.hMutex.Unlock() +} + +func (sys *BucketTargetSys) healthStats() map[string]epHealth { + sys.hMutex.RLock() + defer sys.hMutex.RUnlock() + m := make(map[string]epHealth, len(sys.hc)) + maps.Copy(m, sys.hc) + return m +} + +// ListTargets lists bucket targets across tenant or for individual bucket, and returns +// results filtered by arnType +func (sys *BucketTargetSys) ListTargets(ctx context.Context, bucket, arnType string) (targets []madmin.BucketTarget) { + h := sys.healthStats() + + if bucket != "" { + if ts, err := sys.ListBucketTargets(ctx, bucket); err == nil { + for _, t := range ts.Targets { + if string(t.Type) == arnType || arnType == "" { + if hs, ok := h[t.URL().Host]; ok { + t.TotalDowntime = hs.offlineDuration + t.Online = hs.Online + t.LastOnline = hs.lastOnline + t.Latency = madmin.LatencyStat{ + Curr: hs.latency.curr, + Avg: hs.latency.avg, + Max: hs.latency.peak, + } + } + targets = append(targets, t.Clone()) + } + } + } + return targets + } + sys.RLock() + defer sys.RUnlock() + for _, tgts := range sys.targetsMap { + for _, t := range tgts { + if string(t.Type) == arnType || arnType == "" { + if hs, ok := h[t.URL().Host]; ok { + t.TotalDowntime = hs.offlineDuration + t.Online = hs.Online + t.LastOnline = hs.lastOnline + t.Latency = madmin.LatencyStat{ + Curr: hs.latency.curr, + Avg: hs.latency.avg, + Max: hs.latency.peak, + } + } + targets = append(targets, t.Clone()) + } + } + } + return targets +} + +// ListBucketTargets - gets list of bucket targets for this bucket. +func (sys *BucketTargetSys) ListBucketTargets(ctx context.Context, bucket string) (*madmin.BucketTargets, error) { + sys.RLock() + defer sys.RUnlock() + + tgts, ok := sys.targetsMap[bucket] + if ok { + return &madmin.BucketTargets{Targets: tgts}, nil + } + return nil, BucketRemoteTargetNotFound{Bucket: bucket} +} + +// Delete clears targets present for a bucket +func (sys *BucketTargetSys) Delete(bucket string) { + sys.Lock() + defer sys.Unlock() + tgts, ok := sys.targetsMap[bucket] + if !ok { + return + } + for _, t := range tgts { + delete(sys.arnRemotesMap, t.Arn) + } + delete(sys.targetsMap, bucket) +} + +// SetTarget - sets a new minio-go client target for this bucket. +func (sys *BucketTargetSys) SetTarget(ctx context.Context, bucket string, tgt *madmin.BucketTarget, update bool) error { + if !tgt.Type.IsValid() && !update { + return BucketRemoteArnTypeInvalid{Bucket: bucket} + } + clnt, err := sys.getRemoteTargetClient(tgt) + if err != nil { + return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket, Err: err} + } + // validate if target credentials are ok + exists, err := clnt.BucketExists(ctx, tgt.TargetBucket) + if err != nil { + switch minio.ToErrorResponse(err).Code { + case "NoSuchBucket": + return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket, Err: err} + case "AccessDenied": + return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, AccessKey: tgt.Credentials.AccessKey, Err: err} + } + return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, AccessKey: tgt.Credentials.AccessKey, Err: err} + } + if !exists { + return BucketRemoteTargetNotFound{Bucket: tgt.TargetBucket} + } + if tgt.Type == madmin.ReplicationService { + if !globalBucketVersioningSys.Enabled(bucket) { + return BucketReplicationSourceNotVersioned{Bucket: bucket} + } + vcfg, err := clnt.GetBucketVersioning(ctx, tgt.TargetBucket) + if err != nil { + return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, Err: err, AccessKey: tgt.Credentials.AccessKey} + } + if !vcfg.Enabled() { + return BucketRemoteTargetNotVersioned{Bucket: tgt.TargetBucket} + } + } + + // Check if target is a MinIO server and alive + hcCtx, cancel := context.WithTimeout(ctx, 3*time.Second) + scheme := "http" + if tgt.Secure { + scheme = "https" + } + result := <-sys.hcClient.Alive(hcCtx, madmin.AliveOpts{}, madmin.ServerProperties{ + Endpoint: tgt.Endpoint, + Scheme: scheme, + }) + + cancel() + if result.Error != nil { + return RemoteTargetConnectionErr{Bucket: tgt.TargetBucket, Err: result.Error, AccessKey: tgt.Credentials.AccessKey} + } + if !result.Online { + err := errors.New("Health check timed out after 3 seconds") + return RemoteTargetConnectionErr{Err: err} + } + + sys.Lock() + defer sys.Unlock() + + tgts := sys.targetsMap[bucket] + newtgts := make([]madmin.BucketTarget, len(tgts)) + found := false + for idx, t := range tgts { + if t.Type == tgt.Type { + if t.Arn == tgt.Arn { + if !update { + return BucketRemoteAlreadyExists{Bucket: t.TargetBucket} + } + newtgts[idx] = *tgt + found = true + continue + } + // fail if endpoint is already present in list of targets and not a matching ARN + if t.Endpoint == tgt.Endpoint { + return BucketRemoteAlreadyExists{Bucket: t.TargetBucket} + } + } + newtgts[idx] = t + } + if !found && !update { + newtgts = append(newtgts, *tgt) + } + + sys.targetsMap[bucket] = newtgts + sys.arnRemotesMap[tgt.Arn] = arnTarget{Client: clnt} + sys.updateBandwidthLimit(bucket, tgt.Arn, tgt.BandwidthLimit) + return nil +} + +func (sys *BucketTargetSys) updateBandwidthLimit(bucket, arn string, limit int64) { + if limit == 0 { + globalBucketMonitor.DeleteBucketThrottle(bucket, arn) + return + } + // Setup bandwidth throttling + + globalBucketMonitor.SetBandwidthLimit(bucket, arn, limit) +} + +// RemoveTarget - removes a remote bucket target for this source bucket. +func (sys *BucketTargetSys) RemoveTarget(ctx context.Context, bucket, arnStr string) error { + if arnStr == "" { + return BucketRemoteArnInvalid{Bucket: bucket} + } + + arn, err := madmin.ParseARN(arnStr) + if err != nil { + return BucketRemoteArnInvalid{Bucket: bucket} + } + + if arn.Type == madmin.ReplicationService { + // reject removal of remote target if replication configuration is present + rcfg, err := getReplicationConfig(ctx, bucket) + if err == nil && rcfg != nil { + for _, tgtArn := range rcfg.FilterTargetArns(replication.ObjectOpts{OpType: replication.AllReplicationType}) { + if err == nil && (tgtArn == arnStr || rcfg.RoleArn == arnStr) { + sys.RLock() + _, ok := sys.arnRemotesMap[arnStr] + sys.RUnlock() + if ok { + return BucketRemoteRemoveDisallowed{Bucket: bucket} + } + } + } + } + } + + // delete ARN type from list of matching targets + sys.Lock() + defer sys.Unlock() + found := false + tgts, ok := sys.targetsMap[bucket] + if !ok { + return BucketRemoteTargetNotFound{Bucket: bucket} + } + targets := make([]madmin.BucketTarget, 0, len(tgts)) + for _, tgt := range tgts { + if tgt.Arn != arnStr { + targets = append(targets, tgt) + continue + } + found = true + } + if !found { + return BucketRemoteTargetNotFound{Bucket: bucket} + } + sys.targetsMap[bucket] = targets + delete(sys.arnRemotesMap, arnStr) + sys.updateBandwidthLimit(bucket, arnStr, 0) + return nil +} + +func (sys *BucketTargetSys) markRefreshInProgress(bucket, arn string) { + sys.aMutex.Lock() + defer sys.aMutex.Unlock() + if v, ok := sys.arnErrsMap[arn]; !ok { + sys.arnErrsMap[arn] = arnErrs{ + updateInProgress: true, + count: v.count + 1, + bucket: bucket, + } + } +} + +func (sys *BucketTargetSys) markRefreshDone(bucket, arn string) { + sys.aMutex.Lock() + defer sys.aMutex.Unlock() + if v, ok := sys.arnErrsMap[arn]; ok { + sys.arnErrsMap[arn] = arnErrs{ + updateInProgress: false, + count: v.count, + bucket: bucket, + } + } +} + +func (sys *BucketTargetSys) isReloadingTarget(bucket, arn string) bool { + sys.aMutex.RLock() + defer sys.aMutex.RUnlock() + if v, ok := sys.arnErrsMap[arn]; ok { + return v.updateInProgress + } + return false +} + +func (sys *BucketTargetSys) incTargetErr(bucket, arn string) { + sys.aMutex.Lock() + defer sys.aMutex.Unlock() + if v, ok := sys.arnErrsMap[arn]; ok { + sys.arnErrsMap[arn] = arnErrs{ + updateInProgress: v.updateInProgress, + count: v.count + 1, + } + } +} + +// GetRemoteTargetClient returns minio-go client for replication target instance +func (sys *BucketTargetSys) GetRemoteTargetClient(bucket, arn string) *TargetClient { + sys.RLock() + tgt := sys.arnRemotesMap[arn] + sys.RUnlock() + + if tgt.Client != nil { + return tgt.Client + } + defer func() { // lazy refresh remote targets + if tgt.Client == nil && !sys.isReloadingTarget(bucket, arn) && (tgt.lastRefresh.Equal(timeSentinel) || tgt.lastRefresh.Before(UTCNow().Add(-5*time.Minute))) { + tgts, err := globalBucketMetadataSys.GetBucketTargetsConfig(bucket) + if err == nil { + sys.markRefreshInProgress(bucket, arn) + sys.UpdateAllTargets(bucket, tgts) + sys.markRefreshDone(bucket, arn) + } + } + sys.incTargetErr(bucket, arn) + }() + return nil +} + +// GetRemoteBucketTargetByArn returns BucketTarget for a ARN +func (sys *BucketTargetSys) GetRemoteBucketTargetByArn(ctx context.Context, bucket, arn string) madmin.BucketTarget { + sys.RLock() + defer sys.RUnlock() + var tgt madmin.BucketTarget + for _, t := range sys.targetsMap[bucket] { + if t.Arn == arn { + tgt = t.Clone() + tgt.Credentials = t.Credentials + return tgt + } + } + return tgt +} + +// NewBucketTargetSys - creates new replication system. +func NewBucketTargetSys(ctx context.Context) *BucketTargetSys { + sys := &BucketTargetSys{ + arnRemotesMap: make(map[string]arnTarget), + targetsMap: make(map[string][]madmin.BucketTarget), + arnErrsMap: make(map[string]arnErrs), + hc: make(map[string]epHealth), + hcClient: newHCClient(), + } + // reload healthCheck endpoints map periodically to remove stale endpoints from the map. + go func() { + rTimer := time.NewTimer(defaultHealthCheckReloadDuration) + defer rTimer.Stop() + for { + select { + case <-rTimer.C: + sys.reloadHealthCheckers(ctx) + rTimer.Reset(defaultHealthCheckReloadDuration) + case <-ctx.Done(): + return + } + } + }() + go sys.heartBeat(ctx) + return sys +} + +// UpdateAllTargets updates target to reflect metadata updates +func (sys *BucketTargetSys) UpdateAllTargets(bucket string, tgts *madmin.BucketTargets) { + if sys == nil { + return + } + sys.Lock() + defer sys.Unlock() + + // Remove existingtarget and arn association + if stgts, ok := sys.targetsMap[bucket]; ok { + for _, t := range stgts { + delete(sys.arnRemotesMap, t.Arn) + } + delete(sys.targetsMap, bucket) + } + + if tgts != nil { + for _, tgt := range tgts.Targets { + tgtClient, err := sys.getRemoteTargetClient(&tgt) + if err != nil { + continue + } + sys.arnRemotesMap[tgt.Arn] = arnTarget{ + Client: tgtClient, + lastRefresh: UTCNow(), + } + sys.updateBandwidthLimit(bucket, tgt.Arn, tgt.BandwidthLimit) + } + + if !tgts.Empty() { + sys.targetsMap[bucket] = tgts.Targets + } + } +} + +// create minio-go clients for buckets having remote targets +func (sys *BucketTargetSys) set(bucket string, meta BucketMetadata) { + cfg := meta.bucketTargetConfig + if cfg == nil || cfg.Empty() { + return + } + sys.Lock() + defer sys.Unlock() + for _, tgt := range cfg.Targets { + tgtClient, err := sys.getRemoteTargetClient(&tgt) + if err != nil { + replLogIf(GlobalContext, err) + continue + } + sys.arnRemotesMap[tgt.Arn] = arnTarget{Client: tgtClient} + sys.updateBandwidthLimit(bucket, tgt.Arn, tgt.BandwidthLimit) + } + sys.targetsMap[bucket] = cfg.Targets +} + +// Returns a minio-go Client configured to access remote host described in replication target config. +func (sys *BucketTargetSys) getRemoteTargetClient(tcfg *madmin.BucketTarget) (*TargetClient, error) { + config := tcfg.Credentials + creds := credentials.NewStaticV4(config.AccessKey, config.SecretKey, "") + + api, err := minio.New(tcfg.Endpoint, &minio.Options{ + Creds: creds, + Secure: tcfg.Secure, + Region: tcfg.Region, + Transport: globalRemoteTargetTransport, + }) + if err != nil { + return nil, err + } + api.SetAppInfo("minio-replication-target", ReleaseTag+" "+tcfg.Arn) + + hcDuration := defaultHealthCheckDuration + if tcfg.HealthCheckDuration >= 1 { // require minimum health check duration of 1 sec. + hcDuration = tcfg.HealthCheckDuration + } + tc := &TargetClient{ + Client: api, + healthCheckDuration: hcDuration, + replicateSync: tcfg.ReplicationSync, + Bucket: tcfg.TargetBucket, + StorageClass: tcfg.StorageClass, + disableProxy: tcfg.DisableProxy, + ARN: tcfg.Arn, + ResetID: tcfg.ResetID, + Endpoint: tcfg.Endpoint, + Secure: tcfg.Secure, + } + return tc, nil +} + +// getRemoteARN gets existing ARN for an endpoint or generates a new one. +func (sys *BucketTargetSys) getRemoteARN(bucket string, target *madmin.BucketTarget, deplID string) (arn string, exists bool) { + if target == nil { + return arn, exists + } + sys.RLock() + defer sys.RUnlock() + tgts := sys.targetsMap[bucket] + for _, tgt := range tgts { + if tgt.Type == target.Type && + tgt.TargetBucket == target.TargetBucket && + target.URL().String() == tgt.URL().String() && + tgt.Credentials.AccessKey == target.Credentials.AccessKey { + return tgt.Arn, true + } + } + if !target.Type.IsValid() { + return arn, exists + } + return generateARN(target, deplID), false +} + +// getRemoteARNForPeer returns the remote target for a peer site in site replication +func (sys *BucketTargetSys) getRemoteARNForPeer(bucket string, peer madmin.PeerInfo) string { + sys.RLock() + defer sys.RUnlock() + tgts := sys.targetsMap[bucket] + for _, target := range tgts { + ep, _ := url.Parse(peer.Endpoint) + if target.SourceBucket == bucket && + target.TargetBucket == bucket && + target.Endpoint == ep.Host && + target.Secure == (ep.Scheme == "https") && + target.Type == madmin.ReplicationService { + return target.Arn + } + } + return "" +} + +// generate ARN that is unique to this target type +func generateARN(t *madmin.BucketTarget, deplID string) string { + uuid := deplID + if uuid == "" { + uuid = mustGetUUID() + } + arn := madmin.ARN{ + Type: t.Type, + ID: uuid, + Region: t.Region, + Bucket: t.TargetBucket, + } + return arn.String() +} + +// Returns parsed target config. If KMS is configured, remote target is decrypted +func parseBucketTargetConfig(bucket string, cdata, cmetadata []byte) (*madmin.BucketTargets, error) { + var ( + data []byte + err error + t madmin.BucketTargets + meta map[string]string + ) + if len(cdata) == 0 { + return nil, nil + } + data = cdata + json := jsoniter.ConfigCompatibleWithStandardLibrary + if len(cmetadata) != 0 { + if err := json.Unmarshal(cmetadata, &meta); err != nil { + return nil, err + } + if crypto.S3.IsEncrypted(meta) { + if data, err = decryptBucketMetadata(cdata, bucket, meta, kms.Context{ + bucket: bucket, + bucketTargetsFile: bucketTargetsFile, + }); err != nil { + return nil, err + } + } + } + + if err = json.Unmarshal(data, &t); err != nil { + return nil, err + } + return &t, nil +} + +// TargetClient is the struct for remote target client. +type TargetClient struct { + *minio.Client + healthCheckDuration time.Duration + Bucket string // remote bucket target + replicateSync bool + StorageClass string // storage class on remote + disableProxy bool + ARN string // ARN to uniquely identify remote target + ResetID string + Endpoint string + Secure bool +} diff --git a/cmd/bucket-versioning-handler.go b/cmd/bucket-versioning-handler.go new file mode 100644 index 00000000..92b2c146 --- /dev/null +++ b/cmd/bucket-versioning-handler.go @@ -0,0 +1,162 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "encoding/base64" + "encoding/xml" + "io" + "net/http" + + humanize "github.com/dustin/go-humanize" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/bucket/versioning" + "github.com/minio/minio/internal/logger" + "github.com/minio/mux" + "github.com/minio/pkg/v3/policy" +) + +const ( + bucketVersioningConfig = "versioning.xml" + + // Maximum size of bucket versioning configuration payload sent to the PutBucketVersioningHandler. + maxBucketVersioningConfigSize = 1 * humanize.MiByte +) + +// PutBucketVersioningHandler - PUT Bucket Versioning. +// ---------- +func (api objectAPIHandlers) PutBucketVersioningHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PutBucketVersioning") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketVersioningAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + v, err := versioning.ParseConfig(io.LimitReader(r.Body, maxBucketVersioningConfigSize)) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if globalSiteReplicationSys.isEnabled() && !v.Enabled() { + writeErrorResponse(ctx, w, APIError{ + Code: "InvalidBucketState", + Description: "Cluster replication is enabled on this site, versioning cannot be suspended on bucket.", + HTTPStatusCode: http.StatusBadRequest, + }, r.URL) + return + } + + if rcfg, _ := globalBucketObjectLockSys.Get(bucket); rcfg.LockEnabled && (v.Suspended() || v.PrefixesExcluded()) { + writeErrorResponse(ctx, w, APIError{ + Code: "InvalidBucketState", + Description: "An Object Lock configuration is present on this bucket, versioning cannot be suspended.", + HTTPStatusCode: http.StatusBadRequest, + }, r.URL) + return + } + if rc, _ := getReplicationConfig(ctx, bucket); rc != nil && v.Suspended() { + writeErrorResponse(ctx, w, APIError{ + Code: "InvalidBucketState", + Description: "A replication configuration is present on this bucket, bucket wide versioning cannot be suspended.", + HTTPStatusCode: http.StatusBadRequest, + }, r.URL) + return + } + + configData, err := xml.Marshal(v) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + updatedAt, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Call site replication hook. + // + // We encode the xml bytes as base64 to ensure there are no encoding + // errors. + cfgStr := base64.StdEncoding.EncodeToString(configData) + replLogIf(ctx, globalSiteReplicationSys.BucketMetaHook(ctx, madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaTypeVersionConfig, + Bucket: bucket, + Versioning: &cfgStr, + UpdatedAt: updatedAt, + })) + + writeSuccessResponseHeadersOnly(w) +} + +// GetBucketVersioningHandler - GET Bucket Versioning. +// ---------- +func (api objectAPIHandlers) GetBucketVersioningHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetBucketVersioning") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketVersioningAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Check if bucket exists. + if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + config, err := globalBucketVersioningSys.Get(bucket) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + configData, err := xml.Marshal(config) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Write bucket versioning configuration to client + writeSuccessResponseXML(w, configData) +} diff --git a/cmd/bucket-versioning.go b/cmd/bucket-versioning.go new file mode 100644 index 00000000..5bfd44a4 --- /dev/null +++ b/cmd/bucket-versioning.go @@ -0,0 +1,83 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "strings" + + "github.com/minio/minio/internal/bucket/versioning" + "github.com/minio/minio/internal/logger" +) + +// BucketVersioningSys - policy subsystem. +type BucketVersioningSys struct{} + +// Enabled enabled versioning? +func (sys *BucketVersioningSys) Enabled(bucket string) bool { + vc, err := sys.Get(bucket) + if err != nil { + logger.CriticalIf(GlobalContext, err) + } + return vc.Enabled() +} + +// PrefixEnabled returns true is versioning is enabled at bucket level and if +// the given prefix doesn't match any excluded prefixes pattern. This is +// part of a MinIO versioning configuration extension. +func (sys *BucketVersioningSys) PrefixEnabled(bucket, prefix string) bool { + vc, err := sys.Get(bucket) + if err != nil { + logger.CriticalIf(GlobalContext, err) + } + return vc.PrefixEnabled(prefix) +} + +// Suspended suspended versioning? +func (sys *BucketVersioningSys) Suspended(bucket string) bool { + vc, err := sys.Get(bucket) + if err != nil { + logger.CriticalIf(GlobalContext, err) + } + return vc.Suspended() +} + +// PrefixSuspended returns true if the given prefix matches an excluded prefix +// pattern. This is part of a MinIO versioning configuration extension. +func (sys *BucketVersioningSys) PrefixSuspended(bucket, prefix string) bool { + vc, err := sys.Get(bucket) + if err != nil { + logger.CriticalIf(GlobalContext, err) + } + + return vc.PrefixSuspended(prefix) +} + +// Get returns stored bucket policy +func (sys *BucketVersioningSys) Get(bucket string) (*versioning.Versioning, error) { + if bucket == minioMetaBucket || strings.HasPrefix(bucket, minioMetaBucket) { + return &versioning.Versioning{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/"}, nil + } + + vcfg, _, err := globalBucketMetadataSys.GetVersioningConfig(bucket) + return vcfg, err +} + +// NewBucketVersioningSys - creates new versioning system. +func NewBucketVersioningSys() *BucketVersioningSys { + return &BucketVersioningSys{} +} diff --git a/cmd/build-constants.go b/cmd/build-constants.go new file mode 100644 index 00000000..7f46baff --- /dev/null +++ b/cmd/build-constants.go @@ -0,0 +1,69 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import "runtime" + +// DO NOT EDIT THIS FILE DIRECTLY. These are build-time constants +// set through ‘buildscripts/gen-ldflags.go’. +var ( + // GOPATH - GOPATH value at the time of build. + GOPATH = "" + + // GOROOT - GOROOT value at the time of build. + GOROOT = "" + + // Version - version time.RFC3339. + Version = "DEVELOPMENT.GOGET" + + // ReleaseTag - release tag in TAG.%Y-%m-%dT%H-%M-%SZ. + ReleaseTag = "DEVELOPMENT.GOGET" + + // CommitID - latest commit id. + CommitID = "DEVELOPMENT.GOGET" + + // ShortCommitID - first 12 characters from CommitID. + ShortCommitID = "DEVELOPMENT.GOGET" + + // CopyrightYear - dynamic value of the copyright end year + CopyrightYear = "0000" + + // MinioReleaseTagTimeLayout - release tag time layout. + MinioReleaseTagTimeLayout = "2006-01-02T15-04-05Z" + + // MinioOSARCH - OS and ARCH. + minioOSARCH = runtime.GOOS + "-" + runtime.GOARCH + + // MinioReleaseBaseURL - release url without os and arch. + MinioReleaseBaseURL = "https://dl.min.io/server/minio/release/" + + // MinioReleaseURL - release URL. + MinioReleaseURL = MinioReleaseBaseURL + minioOSARCH + SlashSeparator + + // MinioStoreName - MinIO store name. + MinioStoreName = "MinIO" + + // MinioUAName - MinIO user agent name. + MinioUAName = "MinIO" + + // MinioBannerName - MinIO banner name for startup message. + MinioBannerName = "MinIO Object Storage Server" + + // MinioLicense - MinIO server license. + MinioLicense = "GNU AGPLv3 - https://www.gnu.org/licenses/agpl-3.0.html" +) diff --git a/cmd/callhome.go b/cmd/callhome.go new file mode 100644 index 00000000..2a6d6695 --- /dev/null +++ b/cmd/callhome.go @@ -0,0 +1,199 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "compress/gzip" + "context" + "encoding/json" + "errors" + "fmt" + "math/rand" + "net/url" + "time" + + "github.com/minio/madmin-go/v3" +) + +var callhomeLeaderLockTimeout = newDynamicTimeout(30*time.Second, 10*time.Second) + +// initCallhome will start the callhome task in the background. +func initCallhome(ctx context.Context, objAPI ObjectLayer) { + if !globalCallhomeConfig.Enabled() { + return + } + + go func() { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + // Leader node (that successfully acquires the lock inside runCallhome) + // will keep performing the callhome. If the leader goes down for some reason, + // the lock will be released and another node will acquire it and take over + // because of this loop. + for { + if !globalCallhomeConfig.Enabled() { + return + } + + if !runCallhome(ctx, objAPI) { + // callhome was disabled or context was canceled + return + } + + // callhome running on a different node. + // sleep for some time and try again. + duration := max(time.Duration(r.Float64()*float64(globalCallhomeConfig.FrequencyDur())), + // Make sure to sleep at least a second to avoid high CPU ticks. + time.Second) + time.Sleep(duration) + } + }() +} + +func runCallhome(ctx context.Context, objAPI ObjectLayer) bool { + // Make sure only 1 callhome is running on the cluster. + locker := objAPI.NewNSLock(minioMetaBucket, "callhome/runCallhome.lock") + lkctx, err := locker.GetLock(ctx, callhomeLeaderLockTimeout) + if err != nil { + // lock timedout means some other node is the leader, + // cycle back return 'true' + return true + } + + ctx = lkctx.Context() + defer locker.Unlock(lkctx) + + // Perform callhome once and then keep running it at regular intervals. + performCallhome(ctx) + + callhomeTimer := time.NewTimer(globalCallhomeConfig.FrequencyDur()) + defer callhomeTimer.Stop() + + for { + if !globalCallhomeConfig.Enabled() { + // Stop the processing as callhome got disabled + return false + } + + select { + case <-ctx.Done(): + // indicates that we do not need to run callhome anymore + return false + case <-callhomeTimer.C: + if !globalCallhomeConfig.Enabled() { + // Stop the processing as callhome got disabled + return false + } + + performCallhome(ctx) + + // Reset the timer for next cycle. + callhomeTimer.Reset(globalCallhomeConfig.FrequencyDur()) + } + } +} + +func performCallhome(ctx context.Context) { + deadline := 10 * time.Second // Default deadline is 10secs for callhome + objectAPI := newObjectLayerFn() + if objectAPI == nil { + internalLogIf(ctx, errors.New("Callhome: object layer not ready")) + return + } + + healthCtx, healthCancel := context.WithTimeout(ctx, deadline) + defer healthCancel() + + healthInfoCh := make(chan madmin.HealthInfo) + + query := url.Values{} + for _, k := range madmin.HealthDataTypesList { + query.Set(string(k), "true") + } + + healthInfo := madmin.HealthInfo{ + TimeStamp: time.Now().UTC(), + Version: madmin.HealthInfoVersion, + Minio: madmin.MinioHealthInfo{ + Info: madmin.MinioInfo{ + DeploymentID: globalDeploymentID(), + }, + }, + } + + go fetchHealthInfo(healthCtx, objectAPI, &query, healthInfoCh, healthInfo) + + for { + select { + case hi, hasMore := <-healthInfoCh: + if !hasMore { + auditOptions := AuditLogOptions{Event: "callhome:diagnostics"} + // Received all data. Send to SUBNET and return + err := sendHealthInfo(ctx, healthInfo) + if err != nil { + internalLogIf(ctx, fmt.Errorf("Unable to perform callhome: %w", err)) + auditOptions.Error = err.Error() + } + auditLogInternal(ctx, auditOptions) + return + } + healthInfo = hi + case <-healthCtx.Done(): + return + } + } +} + +const ( + subnetHealthPath = "/api/health/upload" +) + +func sendHealthInfo(ctx context.Context, healthInfo madmin.HealthInfo) error { + url := globalSubnetConfig.BaseURL + subnetHealthPath + + filename := fmt.Sprintf("health_%s.json.gz", UTCNow().Format("20060102150405")) + url += "?filename=" + filename + + _, err := globalSubnetConfig.Upload(url, filename, createHealthJSONGzip(ctx, healthInfo)) + return err +} + +func createHealthJSONGzip(ctx context.Context, healthInfo madmin.HealthInfo) []byte { + var b bytes.Buffer + gzWriter := gzip.NewWriter(&b) + + header := struct { + Version string `json:"version"` + }{Version: healthInfo.Version} + + enc := json.NewEncoder(gzWriter) + if e := enc.Encode(header); e != nil { + internalLogIf(ctx, fmt.Errorf("Could not encode health info header: %w", e)) + return nil + } + + if e := enc.Encode(healthInfo); e != nil { + internalLogIf(ctx, fmt.Errorf("Could not encode health info: %w", e)) + return nil + } + + gzWriter.Flush() + gzWriter.Close() + + return b.Bytes() +} diff --git a/cmd/common-main.go b/cmd/common-main.go new file mode 100644 index 00000000..e57e190f --- /dev/null +++ b/cmd/common-main.go @@ -0,0 +1,1064 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bufio" + "bytes" + "context" + "crypto/x509" + "encoding/gob" + "errors" + "fmt" + "net" + "net/url" + "os" + "path" + "path/filepath" + "runtime" + "sort" + "strconv" + "strings" + "syscall" + "time" + + "github.com/dustin/go-humanize" + fcolor "github.com/fatih/color" + "github.com/go-openapi/loads" + "github.com/inconshreveable/mousetrap" + dns2 "github.com/miekg/dns" + "github.com/minio/cli" + consoleapi "github.com/minio/console/api" + "github.com/minio/console/api/operations" + consoleoauth2 "github.com/minio/console/pkg/auth/idp/oauth2" + consoleCerts "github.com/minio/console/pkg/certs" + "github.com/minio/kms-go/kes" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/color" + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/kms" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/certs" + "github.com/minio/pkg/v3/console" + "github.com/minio/pkg/v3/env" + xnet "github.com/minio/pkg/v3/net" + "golang.org/x/term" +) + +// serverDebugLog will enable debug printing +var ( + serverDebugLog = env.Get("_MINIO_SERVER_DEBUG", config.EnableOff) == config.EnableOn + currentReleaseTime time.Time + orchestrated = IsKubernetes() || IsDocker() +) + +func init() { + if !term.IsTerminal(int(os.Stdout.Fd())) || !term.IsTerminal(int(os.Stderr.Fd())) { + color.TurnOff() + } + if env.Get("NO_COLOR", "") != "" || env.Get("TERM", "") == "dumb" { + color.TurnOff() + } + + if runtime.GOOS == "windows" { + if mousetrap.StartedByExplorer() { + fmt.Printf("Don't double-click %s\n", os.Args[0]) + fmt.Println("You need to open cmd.exe/PowerShell and run it from the command line") + fmt.Println("Refer to the docs here on how to run it as a Windows Service https://github.com/minio/minio-service/tree/master/windows") + fmt.Println("Press the Enter Key to Exit") + fmt.Scanln() + os.Exit(1) + } + } + + logger.Init(GOPATH, GOROOT) + logger.RegisterError(config.FmtError) + + t, _ := minioVersionToReleaseTime(Version) + if !t.IsZero() { + globalVersionUnix = uint64(t.Unix()) + } + + globalIsCICD = env.Get("MINIO_CI_CD", "") != "" || env.Get("CI", "") != "" + + console.SetColor("Debug", fcolor.New()) + + gob.Register(StorageErr("")) + gob.Register(madmin.TimeInfo{}) + gob.Register(madmin.XFSErrorConfigs{}) + gob.Register(map[string]string{}) + gob.Register(map[string]any{}) + + // All minio-go and madmin-go API operations shall be performed only once, + // another way to look at this is we are turning off retries. + minio.MaxRetry = 1 + madmin.MaxRetry = 1 + + currentReleaseTime, _ = GetCurrentReleaseTime() +} + +const consolePrefix = "CONSOLE_" + +func minioConfigToConsoleFeatures() { + os.Setenv("CONSOLE_PBKDF_SALT", globalDeploymentID()) + os.Setenv("CONSOLE_PBKDF_PASSPHRASE", globalDeploymentID()) + if globalMinioEndpoint != "" { + os.Setenv("CONSOLE_MINIO_SERVER", globalMinioEndpoint) + } else { + // Explicitly set 127.0.0.1 so Console will automatically bypass TLS verification to the local S3 API. + // This will save users from providing a certificate with IP or FQDN SAN that points to the local host. + os.Setenv("CONSOLE_MINIO_SERVER", fmt.Sprintf("%s://127.0.0.1:%s", getURLScheme(globalIsTLS), globalMinioPort)) + } + if value := env.Get(config.EnvMinIOLogQueryURL, ""); value != "" { + os.Setenv("CONSOLE_LOG_QUERY_URL", value) + if value := env.Get(config.EnvMinIOLogQueryAuthToken, ""); value != "" { + os.Setenv("CONSOLE_LOG_QUERY_AUTH_TOKEN", value) + } + } + if value := env.Get(config.EnvBrowserRedirectURL, ""); value != "" { + os.Setenv("CONSOLE_BROWSER_REDIRECT_URL", value) + } + if value := env.Get(config.EnvConsoleDebugLogLevel, ""); value != "" { + os.Setenv("CONSOLE_DEBUG_LOGLEVEL", value) + } + // pass the console subpath configuration + if globalBrowserRedirectURL != nil { + subPath := path.Clean(pathJoin(strings.TrimSpace(globalBrowserRedirectURL.Path), SlashSeparator)) + if subPath != SlashSeparator { + os.Setenv("CONSOLE_SUBPATH", subPath) + } + } + // Enable if prometheus URL is set. + if value := env.Get(config.EnvMinIOPrometheusURL, ""); value != "" { + os.Setenv("CONSOLE_PROMETHEUS_URL", value) + if value := env.Get(config.EnvMinIOPrometheusJobID, "minio-job"); value != "" { + os.Setenv("CONSOLE_PROMETHEUS_JOB_ID", value) + // Support additional labels for more granular filtering. + if value := env.Get(config.EnvMinIOPrometheusExtraLabels, ""); value != "" { + os.Setenv("CONSOLE_PROMETHEUS_EXTRA_LABELS", value) + } + } + // Support Prometheus Auth Token + if value := env.Get(config.EnvMinIOPrometheusAuthToken, ""); value != "" { + os.Setenv("CONSOLE_PROMETHEUS_AUTH_TOKEN", value) + } + } + // Enable if LDAP is enabled. + if globalIAMSys.LDAPConfig.Enabled() { + os.Setenv("CONSOLE_LDAP_ENABLED", config.EnableOn) + } + // Handle animation in welcome page + if value := env.Get(config.EnvBrowserLoginAnimation, "on"); value != "" { + os.Setenv("CONSOLE_ANIMATED_LOGIN", value) + } + + // Pass on the session duration environment variable, else we will default to 12 hours + if valueSts := env.Get(config.EnvMinioStsDuration, ""); valueSts != "" { + os.Setenv("CONSOLE_STS_DURATION", valueSts) + } else if valueSession := env.Get(config.EnvBrowserSessionDuration, ""); valueSession != "" { + os.Setenv("CONSOLE_STS_DURATION", valueSession) + } + + os.Setenv("CONSOLE_MINIO_SITE_NAME", globalSite.Name()) + os.Setenv("CONSOLE_MINIO_SITE_REGION", globalSite.Region()) + os.Setenv("CONSOLE_MINIO_REGION", globalSite.Region()) + + os.Setenv("CONSOLE_CERT_PASSWD", env.Get("MINIO_CERT_PASSWD", "")) + + // This section sets Browser (console) stored config + if valueSCP := globalBrowserConfig.GetCSPolicy(); valueSCP != "" { + os.Setenv("CONSOLE_SECURE_CONTENT_SECURITY_POLICY", valueSCP) + } + + if hstsSeconds := globalBrowserConfig.GetHSTSSeconds(); hstsSeconds > 0 { + isubdom := globalBrowserConfig.IsHSTSIncludeSubdomains() + isprel := globalBrowserConfig.IsHSTSPreload() + os.Setenv("CONSOLE_SECURE_STS_SECONDS", strconv.Itoa(hstsSeconds)) + os.Setenv("CONSOLE_SECURE_STS_INCLUDE_SUB_DOMAINS", isubdom) + os.Setenv("CONSOLE_SECURE_STS_PRELOAD", isprel) + } + + if valueRefer := globalBrowserConfig.GetReferPolicy(); valueRefer != "" { + os.Setenv("CONSOLE_SECURE_REFERRER_POLICY", valueRefer) + } + + globalSubnetConfig.ApplyEnv() +} + +func buildOpenIDConsoleConfig() consoleoauth2.OpenIDPCfg { + pcfgs := globalIAMSys.OpenIDConfig.ProviderCfgs + m := make(map[string]consoleoauth2.ProviderConfig, len(pcfgs)) + for name, cfg := range pcfgs { + callback := getConsoleEndpoints()[0] + "/oauth_callback" + if cfg.RedirectURI != "" { + callback = cfg.RedirectURI + } + m[name] = consoleoauth2.ProviderConfig{ + URL: cfg.URL.String(), + DisplayName: cfg.DisplayName, + ClientID: cfg.ClientID, + ClientSecret: cfg.ClientSecret, + HMACSalt: globalDeploymentID(), + HMACPassphrase: cfg.ClientID, + Scopes: strings.Join(cfg.DiscoveryDoc.ScopesSupported, ","), + Userinfo: cfg.ClaimUserinfo, + RedirectCallbackDynamic: cfg.RedirectURIDynamic, + RedirectCallback: callback, + EndSessionEndpoint: cfg.DiscoveryDoc.EndSessionEndpoint, + RoleArn: cfg.GetRoleArn(), + } + } + return m +} + +func initConsoleServer() (*consoleapi.Server, error) { + // unset all console_ environment variables. + for _, cenv := range env.List(consolePrefix) { + os.Unsetenv(cenv) + } + + // enable all console environment variables + minioConfigToConsoleFeatures() + + // set certs dir to minio directory + consoleCerts.GlobalCertsDir = &consoleCerts.ConfigDir{ + Path: globalCertsDir.Get(), + } + consoleCerts.GlobalCertsCADir = &consoleCerts.ConfigDir{ + Path: globalCertsCADir.Get(), + } + + // set certs before other console initialization + consoleapi.GlobalRootCAs, consoleapi.GlobalPublicCerts, consoleapi.GlobalTLSCertsManager = globalRootCAs, globalPublicCerts, globalTLSCerts + + swaggerSpec, err := loads.Embedded(consoleapi.SwaggerJSON, consoleapi.FlatSwaggerJSON) + if err != nil { + return nil, err + } + + api := operations.NewConsoleAPI(swaggerSpec) + + if !serverDebugLog { + // Disable console logging if server debug log is not enabled + noLog := func(string, ...any) {} + + consoleapi.LogInfo = noLog + consoleapi.LogError = noLog + api.Logger = noLog + } + + // Pass in console application config. This needs to happen before the + // ConfigureAPI() call. + consoleapi.GlobalMinIOConfig = consoleapi.MinIOConfig{ + OpenIDProviders: buildOpenIDConsoleConfig(), + } + + server := consoleapi.NewServer(api) + // register all APIs + server.ConfigureAPI() + + consolePort, _ := strconv.Atoi(globalMinioConsolePort) + + server.Host = globalMinioConsoleHost + server.Port = consolePort + consoleapi.Port = globalMinioConsolePort + consoleapi.Hostname = globalMinioConsoleHost + + if globalIsTLS { + // If TLS certificates are provided enforce the HTTPS. + server.EnabledListeners = []string{"https"} + server.TLSPort = consolePort + // Need to store tls-port, tls-host un config variables so secure.middleware can read from there + consoleapi.TLSPort = globalMinioConsolePort + consoleapi.Hostname = globalMinioConsoleHost + } + + return server, nil +} + +// Check for updates and print a notification message +func checkUpdate(mode string) { + updateURL := minioReleaseInfoURL + if runtime.GOOS == globalWindowsOSName { + updateURL = minioReleaseWindowsInfoURL + } + + u, err := url.Parse(updateURL) + if err != nil { + return + } + + if currentReleaseTime.IsZero() { + return + } + + _, lrTime, err := getLatestReleaseTime(u, 2*time.Second, mode) + if err != nil { + return + } + + var older time.Duration + var downloadURL string + if lrTime.After(currentReleaseTime) { + older = lrTime.Sub(currentReleaseTime) + downloadURL = getDownloadURL(releaseTimeToReleaseTag(lrTime)) + } + + updateMsg := prepareUpdateMessage(downloadURL, older) + if updateMsg == "" { + return + } + + logger.Info(prepareUpdateMessage("Run `mc admin update ALIAS`", lrTime.Sub(currentReleaseTime))) +} + +func newConfigDir(dir string, dirSet bool, getDefaultDir func() string) (*ConfigDir, error) { + if dir == "" { + dir = getDefaultDir() + } + + if dir == "" { + if !dirSet { + return nil, fmt.Errorf("missing option must be provided") + } + return nil, fmt.Errorf("provided option cannot be empty") + } + + // Disallow relative paths, figure out absolute paths. + dirAbs, err := filepath.Abs(dir) + if err != nil { + return nil, err + } + err = mkdirAllIgnorePerm(dirAbs) + if err != nil { + return nil, fmt.Errorf("unable to create the directory `%s`: %w", dirAbs, err) + } + + return &ConfigDir{path: dirAbs}, nil +} + +func buildServerCtxt(ctx *cli.Context, ctxt *serverCtxt) (err error) { + // Get "json" flag from command line argument and + ctxt.JSON = ctx.IsSet("json") || ctx.GlobalIsSet("json") + // Get quiet flag from command line argument. + ctxt.Quiet = ctx.IsSet("quiet") || ctx.GlobalIsSet("quiet") + // Get anonymous flag from command line argument. + ctxt.Anonymous = ctx.IsSet("anonymous") || ctx.GlobalIsSet("anonymous") + // Fetch address option + ctxt.Addr = ctx.GlobalString("address") + if ctxt.Addr == "" || ctxt.Addr == ":"+GlobalMinioDefaultPort { + ctxt.Addr = ctx.String("address") + } + + // Fetch console address option + ctxt.ConsoleAddr = ctx.GlobalString("console-address") + if ctxt.ConsoleAddr == "" { + ctxt.ConsoleAddr = ctx.String("console-address") + } + + if cxml := ctx.String("crossdomain-xml"); cxml != "" { + buf, err := os.ReadFile(cxml) + if err != nil { + return err + } + ctxt.CrossDomainXML = string(buf) + } + + // Check "no-compat" flag from command line argument. + ctxt.StrictS3Compat = !ctx.IsSet("no-compat") && !ctx.GlobalIsSet("no-compat") + + switch { + case ctx.IsSet("config-dir"): + ctxt.ConfigDir = ctx.String("config-dir") + ctxt.configDirSet = true + case ctx.GlobalIsSet("config-dir"): + ctxt.ConfigDir = ctx.GlobalString("config-dir") + ctxt.configDirSet = true + } + + switch { + case ctx.IsSet("certs-dir"): + ctxt.CertsDir = ctx.String("certs-dir") + ctxt.certsDirSet = true + case ctx.GlobalIsSet("certs-dir"): + ctxt.CertsDir = ctx.GlobalString("certs-dir") + ctxt.certsDirSet = true + } + + memAvailable := availableMemory() + if ctx.IsSet("memlimit") || ctx.GlobalIsSet("memlimit") { + memlimit := ctx.String("memlimit") + if memlimit == "" { + memlimit = ctx.GlobalString("memlimit") + } + mlimit, err := humanize.ParseBytes(memlimit) + if err != nil { + return err + } + if mlimit > memAvailable { + logger.Info("WARNING: maximum memory available (%s) smaller than specified --memlimit=%s, ignoring --memlimit value", + humanize.IBytes(memAvailable), memlimit) + } + ctxt.MemLimit = mlimit + } else { + ctxt.MemLimit = memAvailable + } + + if memAvailable < ctxt.MemLimit { + ctxt.MemLimit = memAvailable + } + + ctxt.FTP = ctx.StringSlice("ftp") + ctxt.SFTP = ctx.StringSlice("sftp") + ctxt.Interface = ctx.String("interface") + ctxt.UserTimeout = ctx.Duration("conn-user-timeout") + ctxt.SendBufSize = ctx.Int("send-buf-size") + ctxt.RecvBufSize = ctx.Int("recv-buf-size") + ctxt.IdleTimeout = ctx.Duration("idle-timeout") + ctxt.UserTimeout = ctx.Duration("conn-user-timeout") + + if conf := ctx.String("config"); len(conf) > 0 { + err = mergeServerCtxtFromConfigFile(conf, ctxt) + } else { + err = mergeDisksLayoutFromArgs(serverCmdArgs(ctx), ctxt) + } + + return err +} + +func handleCommonArgs(ctxt serverCtxt) { + if ctxt.JSON { + logger.EnableJSON() + } + if ctxt.Quiet { + logger.EnableQuiet() + } + if ctxt.Anonymous { + logger.EnableAnonymous() + } + + consoleAddr := ctxt.ConsoleAddr + addr := ctxt.Addr + configDir := ctxt.ConfigDir + configSet := ctxt.configDirSet + certsDir := ctxt.CertsDir + certsSet := ctxt.certsDirSet + + if globalBrowserEnabled { + if consoleAddr == "" { + p, err := xnet.GetFreePort() + if err != nil { + logger.FatalIf(err, "Unable to get free port for Console UI on the host") + } + // hold the port + l, err := net.Listen("TCP", fmt.Sprintf(":%s", p.String())) + if err == nil { + defer l.Close() + } + consoleAddr = net.JoinHostPort("", p.String()) + } + + if _, _, err := net.SplitHostPort(consoleAddr); err != nil { + logger.FatalIf(err, "Unable to start listening on console port") + } + + if consoleAddr == addr { + logger.FatalIf(errors.New("--console-address cannot be same as --address"), "Unable to start the server") + } + } + + globalMinioHost, globalMinioPort = mustSplitHostPort(addr) + if globalMinioPort == "0" { + p, err := xnet.GetFreePort() + if err != nil { + logger.FatalIf(err, "Unable to get free port for S3 API on the host") + } + globalMinioPort = p.String() + globalDynamicAPIPort = true + } + + if globalBrowserEnabled { + globalMinioConsoleHost, globalMinioConsolePort = mustSplitHostPort(consoleAddr) + } + + if globalMinioPort == globalMinioConsolePort { + logger.FatalIf(errors.New("--console-address port cannot be same as --address port"), "Unable to start the server") + } + + globalMinioAddr = addr + + // Set all config, certs and CAs directories. + var err error + globalConfigDir, err = newConfigDir(configDir, configSet, defaultConfigDir.Get) + logger.FatalIf(err, "Unable to initialize the (deprecated) config directory") + globalCertsDir, err = newConfigDir(certsDir, certsSet, defaultCertsDir.Get) + logger.FatalIf(err, "Unable to initialize the certs directory") + + // Remove this code when we deprecate and remove config-dir. + // This code is to make sure we inherit from the config-dir + // option if certs-dir is not provided. + if !certsSet && configSet { + globalCertsDir = &ConfigDir{path: filepath.Join(globalConfigDir.Get(), certsDir)} + } + + globalCertsCADir = &ConfigDir{path: filepath.Join(globalCertsDir.Get(), certsCADir)} + + logger.FatalIf(mkdirAllIgnorePerm(globalCertsCADir.Get()), "Unable to create certs CA directory at %s", globalCertsCADir.Get()) +} + +func runDNSCache(ctx *cli.Context) { + dnsTTL := ctx.Duration("dns-cache-ttl") + // Check if we have configured a custom DNS cache TTL. + if dnsTTL <= 0 { + if orchestrated { + dnsTTL = 30 * time.Second + } else { + dnsTTL = 10 * time.Minute + } + } + + // Call to refresh will refresh names in cache. + go func() { + // Baremetal setups set DNS refresh window up to dnsTTL duration. + t := time.NewTicker(dnsTTL) + defer t.Stop() + for { + select { + case <-t.C: + globalDNSCache.Refresh() + + case <-GlobalContext.Done(): + return + } + } + }() +} + +type envKV struct { + Key string + Value string + Skip bool +} + +func (e envKV) String() string { + if e.Skip { + return "" + } + return fmt.Sprintf("%s=%s", e.Key, e.Value) +} + +func parsEnvEntry(envEntry string) (envKV, error) { + envEntry = strings.TrimSpace(envEntry) + if envEntry == "" { + // Skip all empty lines + return envKV{ + Skip: true, + }, nil + } + if strings.HasPrefix(envEntry, "#") { + // Skip commented lines + return envKV{ + Skip: true, + }, nil + } + envTokens := strings.SplitN(strings.TrimSpace(strings.TrimPrefix(envEntry, "export")), config.EnvSeparator, 2) + if len(envTokens) != 2 { + return envKV{}, fmt.Errorf("envEntry malformed; %s, expected to be of form 'KEY=value'", envEntry) + } + + key := envTokens[0] + val := envTokens[1] + + // Remove quotes from the value if found + if len(val) >= 2 { + quote := val[0] + if (quote == '"' || quote == '\'') && val[len(val)-1] == quote { + val = val[1 : len(val)-1] + } + } + + return envKV{ + Key: key, + Value: val, + }, nil +} + +// Similar to os.Environ returns a copy of strings representing +// the environment values from a file, in the form "key, value". +// in a structured form. +func minioEnvironFromFile(envConfigFile string) ([]envKV, error) { + f, err := Open(envConfigFile) + if err != nil { + return nil, err + } + defer f.Close() + var ekvs []envKV + scanner := bufio.NewScanner(f) + for scanner.Scan() { + ekv, err := parsEnvEntry(scanner.Text()) + if err != nil { + return nil, err + } + if ekv.Skip { + // Skips empty lines + continue + } + ekvs = append(ekvs, ekv) + } + if err = scanner.Err(); err != nil { + return nil, err + } + return ekvs, nil +} + +func readFromSecret(sp string) (string, error) { + // Supports reading path from docker secrets, filename is + // relative to /run/secrets/ position. + if isFile(pathJoin("/run/secrets/", sp)) { + sp = pathJoin("/run/secrets/", sp) + } + credBuf, err := os.ReadFile(sp) + if err != nil { + if os.IsNotExist(err) { // ignore if file doesn't exist. + return "", nil + } + return "", err + } + return string(bytes.TrimSpace(credBuf)), nil +} + +func loadEnvVarsFromFiles() { + if env.IsSet(config.EnvAccessKeyFile) { + accessKey, err := readFromSecret(env.Get(config.EnvAccessKeyFile, "")) + if err != nil { + logger.Fatal(config.ErrInvalidCredentials(err), + "Unable to validate credentials inherited from the secret file(s)") + } + if accessKey != "" { + os.Setenv(config.EnvRootUser, accessKey) + } + } + + if env.IsSet(config.EnvSecretKeyFile) { + secretKey, err := readFromSecret(env.Get(config.EnvSecretKeyFile, "")) + if err != nil { + logger.Fatal(config.ErrInvalidCredentials(err), + "Unable to validate credentials inherited from the secret file(s)") + } + if secretKey != "" { + os.Setenv(config.EnvRootPassword, secretKey) + } + } + + if env.IsSet(config.EnvRootUserFile) { + rootUser, err := readFromSecret(env.Get(config.EnvRootUserFile, "")) + if err != nil { + logger.Fatal(config.ErrInvalidCredentials(err), + "Unable to validate credentials inherited from the secret file(s)") + } + if rootUser != "" { + os.Setenv(config.EnvRootUser, rootUser) + } + } + + if env.IsSet(config.EnvRootPasswordFile) { + rootPassword, err := readFromSecret(env.Get(config.EnvRootPasswordFile, "")) + if err != nil { + logger.Fatal(config.ErrInvalidCredentials(err), + "Unable to validate credentials inherited from the secret file(s)") + } + if rootPassword != "" { + os.Setenv(config.EnvRootPassword, rootPassword) + } + } + + if env.IsSet(config.EnvConfigEnvFile) { + ekvs, err := minioEnvironFromFile(env.Get(config.EnvConfigEnvFile, "")) + if err != nil && !os.IsNotExist(err) { + logger.Fatal(err, "Unable to read the config environment file") + } + for _, ekv := range ekvs { + os.Setenv(ekv.Key, ekv.Value) + } + } +} + +func serverHandleEarlyEnvVars() { + var err error + globalBrowserEnabled, err = config.ParseBool(env.Get(config.EnvBrowser, config.EnableOn)) + if err != nil { + logger.Fatal(config.ErrInvalidBrowserValue(err), "Invalid MINIO_BROWSER value in environment variable") + } +} + +func serverHandleEnvVars() { + var err error + if globalBrowserEnabled { + if redirectURL := env.Get(config.EnvBrowserRedirectURL, ""); redirectURL != "" { + u, err := xnet.ParseHTTPURL(redirectURL) + if err != nil { + logger.Fatal(err, "Invalid MINIO_BROWSER_REDIRECT_URL value in environment variable") + } + // Look for if URL has invalid values and return error. + if !isValidURLEndpoint((*url.URL)(u)) { + err := fmt.Errorf("URL contains unexpected resources, expected URL to be one of http(s)://console.example.com or as a subpath via API endpoint http(s)://minio.example.com/minio format: %v", u) + logger.Fatal(err, "Invalid MINIO_BROWSER_REDIRECT_URL value is environment variable") + } + globalBrowserRedirectURL = u + } + globalBrowserRedirect = env.Get(config.EnvBrowserRedirect, config.EnableOn) == config.EnableOn + } + + if serverURL := env.Get(config.EnvMinIOServerURL, ""); serverURL != "" { + u, err := xnet.ParseHTTPURL(serverURL) + if err != nil { + logger.Fatal(err, "Invalid MINIO_SERVER_URL value in environment variable") + } + // Look for if URL has invalid values and return error. + if !isValidURLEndpoint((*url.URL)(u)) { + err := fmt.Errorf("URL contains unexpected resources, expected URL to be of http(s)://minio.example.com format: %v", u) + logger.Fatal(err, "Invalid MINIO_SERVER_URL value is environment variable") + } + u.Path = "" // remove any path component such as `/` + globalMinioEndpoint = u.String() + globalMinioEndpointURL = u + } + + globalFSOSync, err = config.ParseBool(env.Get(config.EnvFSOSync, config.EnableOff)) + if err != nil { + logger.Fatal(config.ErrInvalidFSOSyncValue(err), "Invalid MINIO_FS_OSYNC value in environment variable") + } + + rootDiskSize := env.Get(config.EnvRootDriveThresholdSize, "") + if rootDiskSize == "" { + rootDiskSize = env.Get(config.EnvRootDiskThresholdSize, "") + } + if rootDiskSize != "" { + size, err := humanize.ParseBytes(rootDiskSize) + if err != nil { + logger.Fatal(err, fmt.Sprintf("Invalid %s value in root drive threshold environment variable", rootDiskSize)) + } + globalRootDiskThreshold = size + } + + domains := env.Get(config.EnvDomain, "") + if len(domains) != 0 { + for domainName := range strings.SplitSeq(domains, config.ValueSeparator) { + if _, ok := dns2.IsDomainName(domainName); !ok { + logger.Fatal(config.ErrInvalidDomainValue(nil).Msgf("Unknown value `%s`", domainName), + "Invalid MINIO_DOMAIN value in environment variable") + } + globalDomainNames = append(globalDomainNames, domainName) + } + sort.Strings(globalDomainNames) + lcpSuf := lcpSuffix(globalDomainNames) + for _, domainName := range globalDomainNames { + if domainName == lcpSuf && len(globalDomainNames) > 1 { + logger.Fatal(config.ErrOverlappingDomainValue(nil).Msgf("Overlapping domains `%s` not allowed", globalDomainNames), + "Invalid MINIO_DOMAIN value in environment variable") + } + } + } + + publicIPs := env.Get(config.EnvPublicIPs, "") + if len(publicIPs) != 0 { + minioEndpoints := strings.Split(publicIPs, config.ValueSeparator) + domainIPs := set.NewStringSet() + for _, endpoint := range minioEndpoints { + if net.ParseIP(endpoint) == nil { + // Checking if the IP is a DNS entry. + addrs, err := globalDNSCache.LookupHost(GlobalContext, endpoint) + if err != nil { + logger.FatalIf(err, "Unable to initialize MinIO server with [%s] invalid entry found in MINIO_PUBLIC_IPS", endpoint) + } + for _, addr := range addrs { + domainIPs.Add(addr) + } + } + domainIPs.Add(endpoint) + } + updateDomainIPs(domainIPs) + } else { + // Add found interfaces IP address to global domain IPS, + // loopback addresses will be naturally dropped. + domainIPs := mustGetLocalIP4() + for _, host := range globalEndpoints.Hostnames() { + domainIPs.Add(host) + } + updateDomainIPs(domainIPs) + } + + // In place update is true by default if the MINIO_UPDATE is not set + // or is not set to 'off', if MINIO_UPDATE is set to 'off' then + // in-place update is off. + globalInplaceUpdateDisabled = strings.EqualFold(env.Get(config.EnvUpdate, config.EnableOn), config.EnableOff) + + // Check if the supported credential env vars, + // "MINIO_ROOT_USER" and "MINIO_ROOT_PASSWORD" are provided + // Warn user if deprecated environment variables, + // "MINIO_ACCESS_KEY" and "MINIO_SECRET_KEY", are defined + // Check all error conditions first + //nolint:gocritic + if !env.IsSet(config.EnvRootUser) && env.IsSet(config.EnvRootPassword) { + logger.Fatal(config.ErrMissingEnvCredentialRootUser(nil), "Unable to start MinIO") + } else if env.IsSet(config.EnvRootUser) && !env.IsSet(config.EnvRootPassword) { + logger.Fatal(config.ErrMissingEnvCredentialRootPassword(nil), "Unable to start MinIO") + } else if !env.IsSet(config.EnvRootUser) && !env.IsSet(config.EnvRootPassword) { + if !env.IsSet(config.EnvAccessKey) && env.IsSet(config.EnvSecretKey) { + logger.Fatal(config.ErrMissingEnvCredentialAccessKey(nil), "Unable to start MinIO") + } else if env.IsSet(config.EnvAccessKey) && !env.IsSet(config.EnvSecretKey) { + logger.Fatal(config.ErrMissingEnvCredentialSecretKey(nil), "Unable to start MinIO") + } + } + + globalEnableSyncBoot = env.Get("MINIO_SYNC_BOOT", config.EnableOff) == config.EnableOn +} + +func loadRootCredentials() auth.Credentials { + // At this point, either both environment variables + // are defined or both are not defined. + // Check both cases and authenticate them if correctly defined + var user, password string + var legacyCredentials bool + //nolint:gocritic + if env.IsSet(config.EnvRootUser) && env.IsSet(config.EnvRootPassword) { + user = env.Get(config.EnvRootUser, "") + password = env.Get(config.EnvRootPassword, "") + } else if env.IsSet(config.EnvAccessKey) && env.IsSet(config.EnvSecretKey) { + user = env.Get(config.EnvAccessKey, "") + password = env.Get(config.EnvSecretKey, "") + legacyCredentials = true + } else if globalServerCtxt.RootUser != "" && globalServerCtxt.RootPwd != "" { + user, password = globalServerCtxt.RootUser, globalServerCtxt.RootPwd + } + if user == "" || password == "" { + return auth.Credentials{} + } + cred, err := auth.CreateCredentials(user, password) + if err != nil { + if legacyCredentials { + logger.Fatal(config.ErrInvalidCredentials(err), + "Unable to validate credentials inherited from the shell environment") + } else { + logger.Fatal(config.ErrInvalidRootUserCredentials(err), + "Unable to validate credentials inherited from the shell environment") + } + } + if env.IsSet(config.EnvAccessKey) && env.IsSet(config.EnvSecretKey) { + msg := fmt.Sprintf("WARNING: %s and %s are deprecated.\n"+ + " Please use %s and %s", + config.EnvAccessKey, config.EnvSecretKey, + config.EnvRootUser, config.EnvRootPassword) + logger.Info(color.RedBold(msg)) + } + globalCredViaEnv = true + return cred +} + +// autoGenerateRootCredentials generates root credentials deterministically if +// a KMS is configured, no manual credentials have been specified and if root +// access is disabled. +func autoGenerateRootCredentials() auth.Credentials { + if GlobalKMS == nil { + return globalActiveCred + } + + aKey, err := GlobalKMS.MAC(GlobalContext, &kms.MACRequest{Message: []byte("root access key")}) + if IsErrIgnored(err, kes.ErrNotAllowed, kms.ErrNotSupported, errors.ErrUnsupported, kms.ErrPermission) { + // If we don't have permission to compute the HMAC, don't change the cred. + return globalActiveCred + } + if err != nil { + logger.Fatal(err, "Unable to generate root access key using KMS") + } + + sKey, err := GlobalKMS.MAC(GlobalContext, &kms.MACRequest{Message: []byte("root secret key")}) + if err != nil { + // Here, we must have permission. Otherwise, we would have failed earlier. + logger.Fatal(err, "Unable to generate root secret key using KMS") + } + + accessKey, err := auth.GenerateAccessKey(20, bytes.NewReader(aKey)) + if err != nil { + logger.Fatal(err, "Unable to generate root access key") + } + secretKey, err := auth.GenerateSecretKey(32, bytes.NewReader(sKey)) + if err != nil { + logger.Fatal(err, "Unable to generate root secret key") + } + + logger.Info("Automatically generated root access key and secret key with the KMS") + return auth.Credentials{ + AccessKey: accessKey, + SecretKey: secretKey, + } +} + +// Initialize KMS global variable after valiadating and loading the configuration. +// It depends on KMS env variables and global cli flags. +func handleKMSConfig() { + present, err := kms.IsPresent() + if err != nil { + logger.Fatal(err, "Invalid KMS configuration specified") + } + if !present { + return + } + + KMS, err := kms.Connect(GlobalContext, &kms.ConnectionOptions{ + CADir: globalCertsCADir.Get(), + }) + if err != nil { + logger.Fatal(err, "Failed to connect to KMS") + } + + if _, err = KMS.GenerateKey(GlobalContext, &kms.GenerateKeyRequest{}); errors.Is(err, kms.ErrKeyNotFound) { + err = KMS.CreateKey(GlobalContext, &kms.CreateKeyRequest{Name: KMS.DefaultKey}) + } + if err != nil && !errors.Is(err, kms.ErrKeyExists) && !errors.Is(err, kms.ErrPermission) { + logger.Fatal(err, "Failed to connect to KMS") + } + GlobalKMS = KMS +} + +func getTLSConfig() (x509Certs []*x509.Certificate, manager *certs.Manager, secureConn bool, err error) { + if !isFile(getPublicCertFile()) || !isFile(getPrivateKeyFile()) { + return nil, nil, false, nil + } + + if x509Certs, err = config.ParsePublicCertFile(getPublicCertFile()); err != nil { + return nil, nil, false, err + } + + manager, err = certs.NewManager(GlobalContext, getPublicCertFile(), getPrivateKeyFile(), config.LoadX509KeyPair) + if err != nil { + return nil, nil, false, err + } + + // MinIO has support for multiple certificates. It expects the following structure: + // certs/ + // │ + // ├─ public.crt + // ├─ private.key + // │ + // ├─ example.com/ + // │ │ + // │ ├─ public.crt + // │ └─ private.key + // └─ foobar.org/ + // │ + // ├─ public.crt + // └─ private.key + // ... + // + // Therefore, we read all filenames in the cert directory and check + // for each directory whether it contains a public.crt and private.key. + // If so, we try to add it to certificate manager. + root, err := Open(globalCertsDir.Get()) + if err != nil { + return nil, nil, false, err + } + defer root.Close() + + files, err := root.Readdir(-1) + if err != nil { + return nil, nil, false, err + } + for _, file := range files { + // Ignore all + // - regular files + // - "CAs" directory + // - any directory which starts with ".." + if file.Mode().IsRegular() || file.Name() == "CAs" || strings.HasPrefix(file.Name(), "..") { + continue + } + if file.Mode()&os.ModeSymlink == os.ModeSymlink { + file, err = Stat(filepath.Join(root.Name(), file.Name())) + if err != nil { + // not accessible ignore + continue + } + if !file.IsDir() { + continue + } + } + + var ( + certFile = filepath.Join(root.Name(), file.Name(), publicCertFile) + keyFile = filepath.Join(root.Name(), file.Name(), privateKeyFile) + ) + if !isFile(certFile) || !isFile(keyFile) { + continue + } + if err = manager.AddCertificate(certFile, keyFile); err != nil { + err = fmt.Errorf("Unable to load TLS certificate '%s,%s': %w", certFile, keyFile, err) + bootLogIf(GlobalContext, err, logger.ErrorKind) + } + } + secureConn = true + + // Certs might be symlinks, reload them every 10 seconds. + manager.UpdateReloadDuration(10 * time.Second) + + // syscall.SIGHUP to reload the certs. + manager.ReloadOnSignal(syscall.SIGHUP) + + return x509Certs, manager, secureConn, nil +} + +// contextCanceled returns whether a context is canceled. +func contextCanceled(ctx context.Context) bool { + select { + case <-ctx.Done(): + return true + default: + return false + } +} + +// bgContext returns a context that can be used for async operations. +// Cancellation/timeouts are removed, so parent cancellations/timeout will +// not propagate from parent. +// Context values are preserved. +// This can be used for goroutines that live beyond the parent context. +func bgContext(parent context.Context) context.Context { + return bgCtx{parent: parent} +} + +type bgCtx struct { + parent context.Context +} + +func (a bgCtx) Done() <-chan struct{} { + return nil +} + +func (a bgCtx) Err() error { + return nil +} + +func (a bgCtx) Deadline() (deadline time.Time, ok bool) { + return time.Time{}, false +} + +func (a bgCtx) Value(key any) any { + return a.parent.Value(key) +} diff --git a/cmd/common-main_test.go b/cmd/common-main_test.go new file mode 100644 index 00000000..9757267d --- /dev/null +++ b/cmd/common-main_test.go @@ -0,0 +1,183 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "errors" + "os" + "reflect" + "testing" +) + +func Test_readFromSecret(t *testing.T) { + testCases := []struct { + content string + expectedErr bool + expectedValue string + }{ + { + "value\n", + false, + "value", + }, + { + " \t\n Hello, Gophers \n\t\r\n", + false, + "Hello, Gophers", + }, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + tmpfile, err := os.CreateTemp(t.TempDir(), "testfile") + if err != nil { + t.Error(err) + } + tmpfile.WriteString(testCase.content) + tmpfile.Sync() + tmpfile.Close() + + value, err := readFromSecret(tmpfile.Name()) + if err != nil && !testCase.expectedErr { + t.Error(err) + } + if err == nil && testCase.expectedErr { + t.Error(errors.New("expected error, found success")) + } + if value != testCase.expectedValue { + t.Errorf("Expected %s, got %s", testCase.expectedValue, value) + } + }) + } +} + +func Test_minioEnvironFromFile(t *testing.T) { + testCases := []struct { + content string + expectedErr bool + expectedEkvs []envKV + }{ + { + ` +export MINIO_ROOT_USER=minio +export MINIO_ROOT_PASSWORD=minio123`, + false, + []envKV{ + { + Key: "MINIO_ROOT_USER", + Value: "minio", + }, + { + Key: "MINIO_ROOT_PASSWORD", + Value: "minio123", + }, + }, + }, + // Value with double quotes + { + `export MINIO_ROOT_USER="minio"`, + false, + []envKV{ + { + Key: "MINIO_ROOT_USER", + Value: "minio", + }, + }, + }, + // Value with single quotes + { + `export MINIO_ROOT_USER='minio'`, + false, + []envKV{ + { + Key: "MINIO_ROOT_USER", + Value: "minio", + }, + }, + }, + { + ` +MINIO_ROOT_USER=minio +MINIO_ROOT_PASSWORD=minio123`, + false, + []envKV{ + { + Key: "MINIO_ROOT_USER", + Value: "minio", + }, + { + Key: "MINIO_ROOT_PASSWORD", + Value: "minio123", + }, + }, + }, + { + ` +export MINIO_ROOT_USERminio +export MINIO_ROOT_PASSWORD=minio123`, + true, + nil, + }, + { + ` +# simple comment +# MINIO_ROOT_USER=minioadmin +# MINIO_ROOT_PASSWORD=minioadmin +MINIO_ROOT_USER=minio +MINIO_ROOT_PASSWORD=minio123`, + false, + []envKV{ + { + Key: "MINIO_ROOT_USER", + Value: "minio", + }, + { + Key: "MINIO_ROOT_PASSWORD", + Value: "minio123", + }, + }, + }, + } + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + tmpfile, err := os.CreateTemp(t.TempDir(), "testfile") + if err != nil { + t.Error(err) + } + tmpfile.WriteString(testCase.content) + tmpfile.Sync() + tmpfile.Close() + + ekvs, err := minioEnvironFromFile(tmpfile.Name()) + if err != nil && !testCase.expectedErr { + t.Error(err) + } + if err == nil && testCase.expectedErr { + t.Error(errors.New("expected error, found success")) + } + + if len(ekvs) != len(testCase.expectedEkvs) { + t.Errorf("expected %v keys, got %v keys", len(testCase.expectedEkvs), len(ekvs)) + } + + if !reflect.DeepEqual(ekvs, testCase.expectedEkvs) { + t.Errorf("expected %v, got %v", testCase.expectedEkvs, ekvs) + } + }) + } +} diff --git a/cmd/config-common.go b/cmd/config-common.go new file mode 100644 index 00000000..71140aa3 --- /dev/null +++ b/cmd/config-common.go @@ -0,0 +1,97 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "errors" + "io" + "net/http" + + "github.com/minio/minio/internal/hash" +) + +var errConfigNotFound = errors.New("config file not found") + +func readConfigWithMetadata(ctx context.Context, store objectIO, configFile string, opts ObjectOptions) ([]byte, ObjectInfo, error) { + r, err := store.GetObjectNInfo(ctx, minioMetaBucket, configFile, nil, http.Header{}, opts) + if err != nil { + if isErrObjectNotFound(err) { + return nil, ObjectInfo{}, errConfigNotFound + } + + return nil, ObjectInfo{}, err + } + defer r.Close() + + buf, err := io.ReadAll(r) + if err != nil { + return nil, ObjectInfo{}, err + } + if len(buf) == 0 { + return nil, ObjectInfo{}, errConfigNotFound + } + return buf, r.ObjInfo, nil +} + +func readConfig(ctx context.Context, store objectIO, configFile string) ([]byte, error) { + buf, _, err := readConfigWithMetadata(ctx, store, configFile, ObjectOptions{}) + return buf, err +} + +type objectDeleter interface { + DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) +} + +func deleteConfig(ctx context.Context, objAPI objectDeleter, configFile string) error { + _, err := objAPI.DeleteObject(ctx, minioMetaBucket, configFile, ObjectOptions{ + DeletePrefix: true, + DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls) + }) + if err != nil && isErrObjectNotFound(err) { + return errConfigNotFound + } + return err +} + +func saveConfigWithOpts(ctx context.Context, store objectIO, configFile string, data []byte, opts ObjectOptions) error { + hashReader, err := hash.NewReader(ctx, bytes.NewReader(data), int64(len(data)), "", getSHA256Hash(data), int64(len(data))) + if err != nil { + return err + } + + _, err = store.PutObject(ctx, minioMetaBucket, configFile, NewPutObjReader(hashReader), opts) + return err +} + +func saveConfig(ctx context.Context, store objectIO, configFile string, data []byte) error { + return saveConfigWithOpts(ctx, store, configFile, data, ObjectOptions{MaxParity: true}) +} + +func checkConfig(ctx context.Context, objAPI ObjectLayer, configFile string) error { + if _, err := objAPI.GetObjectInfo(ctx, minioMetaBucket, configFile, ObjectOptions{}); err != nil { + // Treat object not found as config not found. + if isErrObjectNotFound(err) { + return errConfigNotFound + } + + return err + } + return nil +} diff --git a/cmd/config-current.go b/cmd/config-current.go new file mode 100644 index 00000000..a87e2876 --- /dev/null +++ b/cmd/config-current.go @@ -0,0 +1,861 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "maps" + "strings" + "sync" + + "github.com/minio/minio/internal/config/browser" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/config/api" + "github.com/minio/minio/internal/config/batch" + "github.com/minio/minio/internal/config/callhome" + "github.com/minio/minio/internal/config/compress" + "github.com/minio/minio/internal/config/dns" + "github.com/minio/minio/internal/config/drive" + "github.com/minio/minio/internal/config/etcd" + "github.com/minio/minio/internal/config/heal" + xldap "github.com/minio/minio/internal/config/identity/ldap" + "github.com/minio/minio/internal/config/identity/openid" + idplugin "github.com/minio/minio/internal/config/identity/plugin" + xtls "github.com/minio/minio/internal/config/identity/tls" + "github.com/minio/minio/internal/config/ilm" + "github.com/minio/minio/internal/config/lambda" + "github.com/minio/minio/internal/config/notify" + "github.com/minio/minio/internal/config/policy/opa" + polplugin "github.com/minio/minio/internal/config/policy/plugin" + "github.com/minio/minio/internal/config/scanner" + "github.com/minio/minio/internal/config/storageclass" + "github.com/minio/minio/internal/config/subnet" + "github.com/minio/minio/internal/crypto" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/env" +) + +func initHelp() { + kvs := map[string]config.KVS{ + config.EtcdSubSys: etcd.DefaultKVS, + config.CompressionSubSys: compress.DefaultKVS, + config.IdentityLDAPSubSys: xldap.DefaultKVS, + config.IdentityOpenIDSubSys: openid.DefaultKVS, + config.IdentityTLSSubSys: xtls.DefaultKVS, + config.IdentityPluginSubSys: idplugin.DefaultKVS, + config.PolicyOPASubSys: opa.DefaultKVS, + config.PolicyPluginSubSys: polplugin.DefaultKVS, + config.SiteSubSys: config.DefaultSiteKVS, + config.RegionSubSys: config.DefaultRegionKVS, + config.APISubSys: api.DefaultKVS, + config.LoggerWebhookSubSys: logger.DefaultLoggerWebhookKVS, + config.AuditWebhookSubSys: logger.DefaultAuditWebhookKVS, + config.AuditKafkaSubSys: logger.DefaultAuditKafkaKVS, + config.ScannerSubSys: scanner.DefaultKVS, + config.SubnetSubSys: subnet.DefaultKVS, + config.CallhomeSubSys: callhome.DefaultKVS, + config.DriveSubSys: drive.DefaultKVS, + config.ILMSubSys: ilm.DefaultKVS, + config.BatchSubSys: batch.DefaultKVS, + config.BrowserSubSys: browser.DefaultKVS, + } + maps.Copy(kvs, notify.DefaultNotificationKVS) + maps.Copy(kvs, lambda.DefaultLambdaKVS) + if globalIsErasure { + kvs[config.StorageClassSubSys] = storageclass.DefaultKVS + kvs[config.HealSubSys] = heal.DefaultKVS + } + config.RegisterDefaultKVS(kvs) + + // Captures help for each sub-system + helpSubSys := config.HelpKVS{ + config.HelpKV{ + Key: config.SubnetSubSys, + Type: "string", + Description: "register Enterprise license for the cluster", + Optional: true, + }, + config.HelpKV{ + Key: config.CallhomeSubSys, + Type: "string", + Description: "enable callhome to MinIO SUBNET", + Optional: true, + }, + config.HelpKV{ + Key: config.DriveSubSys, + Description: "enable drive specific settings", + }, + config.HelpKV{ + Key: config.SiteSubSys, + Description: "label the server and its location", + }, + config.HelpKV{ + Key: config.APISubSys, + Description: "manage global HTTP API call specific features, such as throttling, authentication types, etc.", + }, + config.HelpKV{ + Key: config.ScannerSubSys, + Description: "manage namespace scanning for usage calculation, lifecycle, healing and more", + }, + config.HelpKV{ + Key: config.BatchSubSys, + Description: "manage batch job workers and wait times", + }, + config.HelpKV{ + Key: config.CompressionSubSys, + Description: "enable server side compression of objects", + }, + config.HelpKV{ + Key: config.IdentityOpenIDSubSys, + Description: "enable OpenID SSO support", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.IdentityLDAPSubSys, + Description: "enable LDAP SSO support", + }, + config.HelpKV{ + Key: config.IdentityTLSSubSys, + Description: "enable X.509 TLS certificate SSO support", + }, + config.HelpKV{ + Key: config.IdentityPluginSubSys, + Description: "enable Identity Plugin via external hook", + }, + config.HelpKV{ + Key: config.PolicyPluginSubSys, + Description: "enable Access Management Plugin for policy enforcement", + }, + config.HelpKV{ + Key: config.LoggerWebhookSubSys, + Description: "send server logs to webhook endpoints", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.AuditWebhookSubSys, + Description: "send audit logs to webhook endpoints", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.AuditKafkaSubSys, + Description: "send audit logs to kafka endpoints", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.NotifyWebhookSubSys, + Description: "publish bucket notifications to webhook endpoints", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.NotifyAMQPSubSys, + Description: "publish bucket notifications to AMQP endpoints", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.NotifyKafkaSubSys, + Description: "publish bucket notifications to Kafka endpoints", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.NotifyMQTTSubSys, + Description: "publish bucket notifications to MQTT endpoints", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.NotifyNATSSubSys, + Description: "publish bucket notifications to NATS endpoints", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.NotifyNSQSubSys, + Description: "publish bucket notifications to NSQ endpoints", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.NotifyMySQLSubSys, + Description: "publish bucket notifications to MySQL databases", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.NotifyPostgresSubSys, + Description: "publish bucket notifications to Postgres databases", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.NotifyESSubSys, + Description: "publish bucket notifications to Elasticsearch endpoints", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.NotifyRedisSubSys, + Description: "publish bucket notifications to Redis datastores", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.LambdaWebhookSubSys, + Description: "manage remote lambda functions", + MultipleTargets: true, + }, + config.HelpKV{ + Key: config.EtcdSubSys, + Description: "persist IAM assets externally to etcd", + }, + config.HelpKV{ + Key: config.BrowserSubSys, + Description: "manage Browser HTTP specific features, such as Security headers, etc.", + Optional: true, + }, + config.HelpKV{ + Key: config.ILMSubSys, + Description: "manage ILM settings for expiration and transition workers", + Optional: true, + }, + } + + if globalIsErasure { + helpSubSys = append(helpSubSys, config.HelpKV{ + Key: config.StorageClassSubSys, + Description: "define object level redundancy", + }, config.HelpKV{ + Key: config.HealSubSys, + Description: "manage object healing frequency and bitrot verification checks", + }) + } + + helpMap := map[string]config.HelpKVS{ + "": helpSubSys, // Help for all sub-systems. + config.SiteSubSys: config.SiteHelp, + config.RegionSubSys: config.RegionHelp, + config.APISubSys: api.Help, + config.StorageClassSubSys: storageclass.Help, + config.EtcdSubSys: etcd.Help, + config.CompressionSubSys: compress.Help, + config.HealSubSys: heal.Help, + config.BatchSubSys: batch.Help, + config.ScannerSubSys: scanner.Help, + config.IdentityOpenIDSubSys: openid.Help, + config.IdentityLDAPSubSys: xldap.Help, + config.IdentityTLSSubSys: xtls.Help, + config.IdentityPluginSubSys: idplugin.Help, + config.PolicyOPASubSys: opa.Help, + config.PolicyPluginSubSys: polplugin.Help, + config.LoggerWebhookSubSys: logger.Help, + config.AuditWebhookSubSys: logger.HelpWebhook, + config.AuditKafkaSubSys: logger.HelpKafka, + config.NotifyAMQPSubSys: notify.HelpAMQP, + config.NotifyKafkaSubSys: notify.HelpKafka, + config.NotifyMQTTSubSys: notify.HelpMQTT, + config.NotifyNATSSubSys: notify.HelpNATS, + config.NotifyNSQSubSys: notify.HelpNSQ, + config.NotifyMySQLSubSys: notify.HelpMySQL, + config.NotifyPostgresSubSys: notify.HelpPostgres, + config.NotifyRedisSubSys: notify.HelpRedis, + config.NotifyWebhookSubSys: notify.HelpWebhook, + config.NotifyESSubSys: notify.HelpES, + config.LambdaWebhookSubSys: lambda.HelpWebhook, + config.SubnetSubSys: subnet.HelpSubnet, + config.CallhomeSubSys: callhome.HelpCallhome, + config.DriveSubSys: drive.HelpDrive, + config.BrowserSubSys: browser.Help, + config.ILMSubSys: ilm.Help, + } + + config.RegisterHelpSubSys(helpMap) + + // save top-level help for deprecated sub-systems in a separate map. + deprecatedHelpKVMap := map[string]config.HelpKV{ + config.RegionSubSys: { + Key: config.RegionSubSys, + Description: "[DEPRECATED - use `site` instead] label the location of the server", + }, + config.PolicyOPASubSys: { + Key: config.PolicyOPASubSys, + Description: "[DEPRECATED - use `policy_plugin` instead] enable external OPA for policy enforcement", + }, + } + + config.RegisterHelpDeprecatedSubSys(deprecatedHelpKVMap) +} + +var ( + // globalServerConfig server config. + globalServerConfig config.Config + globalServerConfigMu sync.RWMutex +) + +func validateSubSysConfig(ctx context.Context, s config.Config, subSys string, objAPI ObjectLayer) error { + switch subSys { + case config.SiteSubSys: + if _, err := config.LookupSite(s[config.SiteSubSys][config.Default], s[config.RegionSubSys][config.Default]); err != nil { + return err + } + case config.APISubSys: + if _, err := api.LookupConfig(s[config.APISubSys][config.Default]); err != nil { + return err + } + case config.BatchSubSys: + if _, err := batch.LookupConfig(s[config.BatchSubSys][config.Default]); err != nil { + return err + } + case config.StorageClassSubSys: + if objAPI == nil { + return errServerNotInitialized + } + for _, setDriveCount := range objAPI.SetDriveCounts() { + if _, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount); err != nil { + return err + } + } + case config.CompressionSubSys: + if _, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default]); err != nil { + return err + } + case config.HealSubSys: + if _, err := heal.LookupConfig(s[config.HealSubSys][config.Default]); err != nil { + return err + } + case config.ScannerSubSys: + if _, err := scanner.LookupConfig(s[config.ScannerSubSys][config.Default]); err != nil { + return err + } + case config.EtcdSubSys: + etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs) + if err != nil { + return err + } + if etcdCfg.Enabled { + etcdClnt, err := etcd.New(etcdCfg) + if err != nil { + return err + } + etcdClnt.Close() + } + case config.IdentityOpenIDSubSys: + if _, err := openid.LookupConfig(s, + xhttp.WithUserAgent(NewHTTPTransport(), func() string { + return getUserAgent(getMinioMode()) + }), xhttp.DrainBody, globalSite.Region()); err != nil { + return err + } + case config.IdentityLDAPSubSys: + cfg, err := xldap.Lookup(s, globalRootCAs) + if err != nil { + return err + } + if cfg.Enabled() { + conn, cerr := cfg.LDAP.Connect() + if cerr != nil { + return cerr + } + conn.Close() + } + case config.IdentityTLSSubSys: + if _, err := xtls.Lookup(s[config.IdentityTLSSubSys][config.Default]); err != nil { + return err + } + case config.IdentityPluginSubSys: + if _, err := idplugin.LookupConfig(s[config.IdentityPluginSubSys][config.Default], + NewHTTPTransport(), xhttp.DrainBody, globalSite.Region()); err != nil { + return err + } + case config.SubnetSubSys: + if _, err := subnet.LookupConfig(s[config.SubnetSubSys][config.Default], nil); err != nil { + return err + } + case config.CallhomeSubSys: + cfg, err := callhome.LookupConfig(s[config.CallhomeSubSys][config.Default]) + if err != nil { + return err + } + // callhome cannot be enabled if license is not registered yet, throw an error. + if cfg.Enabled() && !globalSubnetConfig.Registered() { + return errors.New("Deployment is not registered with SUBNET. Please register the deployment via 'mc license register ALIAS'") + } + case config.DriveSubSys: + if _, err := drive.LookupConfig(s[config.DriveSubSys][config.Default]); err != nil { + return err + } + case config.PolicyOPASubSys: + // In case legacy OPA config is being set, we treat it as if the + // AuthZPlugin is being set. + subSys = config.PolicyPluginSubSys + fallthrough + case config.PolicyPluginSubSys: + if ppargs, err := polplugin.LookupConfig(s, GetDefaultConnSettings(), xhttp.DrainBody); err != nil { + return err + } else if ppargs.URL == nil { + // Check if legacy opa is configured. + if _, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default], + NewHTTPTransport(), xhttp.DrainBody); err != nil { + return err + } + } + case config.BrowserSubSys: + if _, err := browser.LookupConfig(s[config.BrowserSubSys][config.Default]); err != nil { + return err + } + default: + if config.LoggerSubSystems.Contains(subSys) { + if err := logger.ValidateSubSysConfig(ctx, s, subSys); err != nil { + return err + } + } + } + + if config.NotifySubSystems.Contains(subSys) { + if err := notify.TestSubSysNotificationTargets(ctx, s, subSys, NewHTTPTransport()); err != nil { + return err + } + } + + if config.LambdaSubSystems.Contains(subSys) { + if err := lambda.TestSubSysLambdaTargets(GlobalContext, s, subSys, NewHTTPTransport()); err != nil { + return err + } + } + + return nil +} + +func validateConfig(ctx context.Context, s config.Config, subSys string) error { + objAPI := newObjectLayerFn() + + // We must have a global lock for this so nobody else modifies env while we do. + defer env.LockSetEnv()() + + // Disable merging env values with config for validation. + env.SetEnvOff() + + // Enable env values to validate KMS. + defer env.SetEnvOn() + if subSys != "" { + return validateSubSysConfig(ctx, s, subSys, objAPI) + } + + // No sub-system passed. Validate all of them. + for _, ss := range config.SubSystems.ToSlice() { + if err := validateSubSysConfig(ctx, s, ss, objAPI); err != nil { + return err + } + } + + return nil +} + +func lookupConfigs(s config.Config, objAPI ObjectLayer) { + ctx := GlobalContext + + dnsURL, dnsUser, dnsPass, err := env.LookupEnv(config.EnvDNSWebhook) + if err != nil { + configLogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err)) + } + if err == nil && dnsURL != "" { + bootstrapTraceMsg("initialize remote bucket DNS store") + globalDNSConfig, err = dns.NewOperatorDNS(dnsURL, + dns.Authentication(dnsUser, dnsPass), + dns.RootCAs(globalRootCAs)) + if err != nil { + configLogIf(ctx, fmt.Errorf("Unable to initialize remote webhook DNS config %w", err)) + } + } + + etcdCfg, err := etcd.LookupConfig(s[config.EtcdSubSys][config.Default], globalRootCAs) + if err != nil { + configLogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err)) + } + + if etcdCfg.Enabled { + bootstrapTraceMsg("initialize etcd store") + globalEtcdClient, err = etcd.New(etcdCfg) + if err != nil { + configLogIf(ctx, fmt.Errorf("Unable to initialize etcd config: %w", err)) + } + + if len(globalDomainNames) != 0 && !globalDomainIPs.IsEmpty() && globalEtcdClient != nil { + if globalDNSConfig != nil { + // if global DNS is already configured, indicate with a warning, in case + // users are confused. + configLogIf(ctx, fmt.Errorf("DNS store is already configured with %s, etcd is not used for DNS store", globalDNSConfig)) + } else { + globalDNSConfig, err = dns.NewCoreDNS(etcdCfg.Config, + dns.DomainNames(globalDomainNames), + dns.DomainIPs(globalDomainIPs), + dns.DomainPort(globalMinioPort), + dns.CoreDNSPath(etcdCfg.CoreDNSPath), + ) + if err != nil { + configLogIf(ctx, fmt.Errorf("Unable to initialize DNS config for %s: %w", + globalDomainNames, err)) + } + } + } + } + + // Bucket federation is 'true' only when IAM assets are not namespaced + // per tenant and all tenants interested in globally available users + // if namespace was requested such as specifying etcdPathPrefix then + // we assume that users are interested in global bucket support + // but not federation. + globalBucketFederation = etcdCfg.PathPrefix == "" && etcdCfg.Enabled + + siteCfg, err := config.LookupSite(s[config.SiteSubSys][config.Default], s[config.RegionSubSys][config.Default]) + if err != nil { + configLogIf(ctx, fmt.Errorf("Invalid site configuration: %w", err)) + } + globalSite.Update(siteCfg) + + globalAutoEncryption = crypto.LookupAutoEncryption() // Enable auto-encryption if enabled + if globalAutoEncryption && GlobalKMS == nil { + logger.Fatal(errors.New("no KMS configured"), "MINIO_KMS_AUTO_ENCRYPTION requires a valid KMS configuration") + } + + transport := NewHTTPTransport() + + bootstrapTraceMsg("initialize the event notification targets") + globalNotifyTargetList, err = notify.FetchEnabledTargets(GlobalContext, s, transport) + if err != nil { + configLogIf(ctx, fmt.Errorf("Unable to initialize notification target(s): %w", err)) + } + + bootstrapTraceMsg("initialize the lambda targets") + globalLambdaTargetList, err = lambda.FetchEnabledTargets(GlobalContext, s, transport) + if err != nil { + configLogIf(ctx, fmt.Errorf("Unable to initialize lambda target(s): %w", err)) + } + + bootstrapTraceMsg("applying the dynamic configuration") + // Apply dynamic config values + if err := applyDynamicConfig(ctx, objAPI, s); err != nil { + configLogIf(ctx, err) + } +} + +func applyDynamicConfigForSubSys(ctx context.Context, objAPI ObjectLayer, s config.Config, subSys string) error { + if objAPI == nil { + return errServerNotInitialized + } + + var errs []error + setDriveCounts := objAPI.SetDriveCounts() + switch subSys { + case config.APISubSys: + apiConfig, err := api.LookupConfig(s[config.APISubSys][config.Default]) + if err != nil { + configLogIf(ctx, fmt.Errorf("Invalid api configuration: %w", err)) + } + + globalAPIConfig.init(apiConfig, setDriveCounts, objAPI.Legacy()) + setRemoteInstanceTransport(NewHTTPTransportWithTimeout(apiConfig.RemoteTransportDeadline)) + case config.CompressionSubSys: + cmpCfg, err := compress.LookupConfig(s[config.CompressionSubSys][config.Default]) + if err != nil { + return fmt.Errorf("Unable to setup Compression: %w", err) + } + globalCompressConfigMu.Lock() + globalCompressConfig = cmpCfg + globalCompressConfigMu.Unlock() + case config.HealSubSys: + healCfg, err := heal.LookupConfig(s[config.HealSubSys][config.Default]) + if err != nil { + errs = append(errs, fmt.Errorf("Unable to apply heal config: %w", err)) + } else { + globalHealConfig.Update(healCfg) + } + case config.BatchSubSys: + batchCfg, err := batch.LookupConfig(s[config.BatchSubSys][config.Default]) + if err != nil { + errs = append(errs, fmt.Errorf("Unable to apply batch config: %w", err)) + } else { + globalBatchConfig.Update(batchCfg) + } + case config.ScannerSubSys: + scannerCfg, err := scanner.LookupConfig(s[config.ScannerSubSys][config.Default]) + if err != nil { + errs = append(errs, fmt.Errorf("Unable to apply scanner config: %w", err)) + } else { + // update dynamic scanner values. + scannerIdleMode.Store(scannerCfg.IdleMode) + scannerCycle.Store(scannerCfg.Cycle) + scannerExcessObjectVersions.Store(scannerCfg.ExcessVersions) + scannerExcessFolders.Store(scannerCfg.ExcessFolders) + configLogIf(ctx, scannerSleeper.Update(scannerCfg.Delay, scannerCfg.MaxWait)) + } + case config.LoggerWebhookSubSys: + loggerCfg, err := logger.LookupConfigForSubSys(ctx, s, config.LoggerWebhookSubSys) + if err != nil { + configLogIf(ctx, fmt.Errorf("Unable to load logger webhook config: %w", err)) + } + userAgent := getUserAgent(getMinioMode()) + for n, l := range loggerCfg.HTTP { + if l.Enabled { + l.LogOnceIf = configLogOnceConsoleIf + l.UserAgent = userAgent + l.Transport = NewHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey) + } + loggerCfg.HTTP[n] = l + } + if errs := logger.UpdateHTTPWebhooks(ctx, loggerCfg.HTTP); len(errs) > 0 { + configLogIf(ctx, fmt.Errorf("Unable to update logger webhook config: %v", errs)) + } + case config.AuditWebhookSubSys: + loggerCfg, err := logger.LookupConfigForSubSys(ctx, s, config.AuditWebhookSubSys) + if err != nil { + configLogIf(ctx, fmt.Errorf("Unable to load audit webhook config: %w", err)) + } + userAgent := getUserAgent(getMinioMode()) + for n, l := range loggerCfg.AuditWebhook { + if l.Enabled { + l.LogOnceIf = configLogOnceConsoleIf + l.UserAgent = userAgent + l.Transport = NewHTTPTransportWithClientCerts(l.ClientCert, l.ClientKey) + } + loggerCfg.AuditWebhook[n] = l + } + + if errs := logger.UpdateAuditWebhooks(ctx, loggerCfg.AuditWebhook); len(errs) > 0 { + configLogIf(ctx, fmt.Errorf("Unable to update audit webhook targets: %v", errs)) + } + case config.AuditKafkaSubSys: + loggerCfg, err := logger.LookupConfigForSubSys(ctx, s, config.AuditKafkaSubSys) + if err != nil { + configLogIf(ctx, fmt.Errorf("Unable to load audit kafka config: %w", err)) + } + for n, l := range loggerCfg.AuditKafka { + if l.Enabled { + if l.TLS.Enable { + l.TLS.RootCAs = globalRootCAs + } + l.LogOnce = configLogOnceIf + loggerCfg.AuditKafka[n] = l + } + } + if errs := logger.UpdateAuditKafkaTargets(ctx, loggerCfg); len(errs) > 0 { + configLogIf(ctx, fmt.Errorf("Unable to update audit kafka targets: %v", errs)) + } + case config.StorageClassSubSys: + for i, setDriveCount := range setDriveCounts { + sc, err := storageclass.LookupConfig(s[config.StorageClassSubSys][config.Default], setDriveCount) + if err != nil { + configLogIf(ctx, fmt.Errorf("Unable to initialize storage class config: %w", err)) + break + } + if i == 0 { + globalStorageClass.Update(sc) + } + } + case config.SubnetSubSys: + subnetConfig, err := subnet.LookupConfig(s[config.SubnetSubSys][config.Default], globalRemoteTargetTransport) + if err != nil { + configLogIf(ctx, fmt.Errorf("Unable to parse subnet configuration: %w", err)) + } else { + globalSubnetConfig.Update(subnetConfig, globalIsCICD) + globalSubnetConfig.ApplyEnv() // update environment settings for Console UI + } + case config.CallhomeSubSys: + callhomeCfg, err := callhome.LookupConfig(s[config.CallhomeSubSys][config.Default]) + if err != nil { + configLogIf(ctx, fmt.Errorf("Unable to load callhome config: %w", err)) + } else { + enable := callhomeCfg.Enable && !globalCallhomeConfig.Enabled() + globalCallhomeConfig.Update(callhomeCfg) + if enable { + initCallhome(ctx, objAPI) + } + } + case config.DriveSubSys: + driveConfig, err := drive.LookupConfig(s[config.DriveSubSys][config.Default]) + if err != nil { + configLogIf(ctx, fmt.Errorf("Unable to load drive config: %w", err)) + } else { + if err = globalDriveConfig.Update(driveConfig); err != nil { + configLogIf(ctx, fmt.Errorf("Unable to update drive config: %v", err)) + } + } + case config.BrowserSubSys: + browserCfg, err := browser.LookupConfig(s[config.BrowserSubSys][config.Default]) + if err != nil { + errs = append(errs, fmt.Errorf("Unable to apply browser config: %w", err)) + } else { + globalBrowserConfig.Update(browserCfg) + } + case config.ILMSubSys: + ilmCfg, err := ilm.LookupConfig(s[config.ILMSubSys][config.Default]) + if err != nil { + errs = append(errs, fmt.Errorf("Unable to apply ilm config: %w", err)) + } else { + if globalTransitionState != nil { + globalTransitionState.UpdateWorkers(ilmCfg.TransitionWorkers) + } + if globalExpiryState != nil { + globalExpiryState.ResizeWorkers(ilmCfg.ExpirationWorkers) + } + globalILMConfig.update(ilmCfg) + } + } + globalServerConfigMu.Lock() + defer globalServerConfigMu.Unlock() + if globalServerConfig != nil { + globalServerConfig[subSys] = s[subSys] + } + if len(errs) > 0 { + return errors.Join(errs...) + } + return nil +} + +// applyDynamicConfig will apply dynamic config values. +// Dynamic systems should be in config.SubSystemsDynamic as well. +func applyDynamicConfig(ctx context.Context, objAPI ObjectLayer, s config.Config) error { + for subSys := range config.SubSystemsDynamic { + err := applyDynamicConfigForSubSys(ctx, objAPI, s, subSys) + if err != nil { + return err + } + } + return nil +} + +// Help - return sub-system level help +type Help struct { + SubSys string `json:"subSys"` + Description string `json:"description"` + MultipleTargets bool `json:"multipleTargets"` + KeysHelp config.HelpKVS `json:"keysHelp"` +} + +// GetHelp - returns help for sub-sys, a key for a sub-system or all the help. +func GetHelp(subSys, key string, envOnly bool) (Help, error) { + if len(subSys) == 0 { + return Help{KeysHelp: config.HelpSubSysMap[subSys]}, nil + } + subSystemValue := strings.SplitN(subSys, config.SubSystemSeparator, 2) + if len(subSystemValue) == 0 { + return Help{}, config.Errorf("invalid number of arguments %s", subSys) + } + + subSys = subSystemValue[0] + + subSysHelp, ok := config.HelpSubSysMap[""].Lookup(subSys) + if !ok { + subSysHelp, ok = config.HelpDeprecatedSubSysMap[subSys] + if !ok { + return Help{}, config.Errorf("unknown sub-system %s", subSys) + } + } + + h, ok := config.HelpSubSysMap[subSys] + if !ok { + return Help{}, config.Errorf("unknown sub-system %s", subSys) + } + if key != "" { + value, ok := h.Lookup(key) + if !ok { + return Help{}, config.Errorf("unknown key %s for sub-system %s", + key, subSys) + } + h = config.HelpKVS{value} + } + + help := config.HelpKVS{} + + // Only for multiple targets, make sure + // to list the ENV, for regular k/v EnableKey is + // implicit, for ENVs we cannot make it implicit. + if subSysHelp.MultipleTargets { + key := madmin.EnableKey + if envOnly { + key = config.EnvPrefix + strings.ToTitle(subSys) + config.EnvWordDelimiter + strings.ToTitle(madmin.EnableKey) + } + help = append(help, config.HelpKV{ + Key: key, + Description: fmt.Sprintf("enable %s target, default is 'off'", subSys), + Optional: false, + Type: "on|off", + }) + } + + for _, hkv := range h { + key := hkv.Key + if envOnly { + key = config.EnvPrefix + strings.ToTitle(subSys) + config.EnvWordDelimiter + strings.ToTitle(hkv.Key) + } + help = append(help, config.HelpKV{ + Key: key, + Description: hkv.Description, + Optional: hkv.Optional, + Type: hkv.Type, + }) + } + + return Help{ + SubSys: subSys, + Description: subSysHelp.Description, + MultipleTargets: subSysHelp.MultipleTargets, + KeysHelp: help, + }, nil +} + +func newServerConfig() config.Config { + return config.New() +} + +// newSrvConfig - initialize a new server config, saves env parameters if +// found, otherwise use default parameters +func newSrvConfig(objAPI ObjectLayer) error { + // Initialize server config. + srvCfg := newServerConfig() + + // hold the mutex lock before a new config is assigned. + globalServerConfigMu.Lock() + globalServerConfig = srvCfg + globalServerConfigMu.Unlock() + + // Save config into file. + return saveServerConfig(GlobalContext, objAPI, srvCfg) +} + +func getValidConfig(objAPI ObjectLayer) (config.Config, error) { + return readServerConfig(GlobalContext, objAPI, nil) +} + +// loadConfig - loads a new config from disk, overrides params +// from env if found and valid +// data is optional. If nil it will be loaded from backend. +func loadConfig(objAPI ObjectLayer, data []byte) error { + bootstrapTraceMsg("load the configuration") + srvCfg, err := readServerConfig(GlobalContext, objAPI, data) + if err != nil { + return err + } + + bootstrapTraceMsg("lookup the configuration") + // Override any values from ENVs. + lookupConfigs(srvCfg, objAPI) + + // hold the mutex lock before a new config is assigned. + globalServerConfigMu.Lock() + globalServerConfig = srvCfg + globalServerConfigMu.Unlock() + + return nil +} diff --git a/cmd/config-current_test.go b/cmd/config-current_test.go new file mode 100644 index 00000000..338cebbe --- /dev/null +++ b/cmd/config-current_test.go @@ -0,0 +1,67 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "os" + "testing" + + "github.com/minio/minio/internal/config" +) + +func TestServerConfig(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + objLayer, fsDir, err := prepareFS(ctx) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(fsDir) + + if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil { + t.Fatalf("Init Test config failed") + } + + if globalSite.Region() != globalMinioDefaultRegion { + t.Errorf("Expecting region `us-east-1` found %s", globalSite.Region()) + } + + // Set new region and verify. + config.SetRegion(globalServerConfig, "us-west-1") + site, err := config.LookupSite( + globalServerConfig[config.SiteSubSys][config.Default], + globalServerConfig[config.RegionSubSys][config.Default], + ) + if err != nil { + t.Fatal(err) + } + if site.Region() != "us-west-1" { + t.Errorf("Expecting region `us-west-1` found %s", globalSite.Region()) + } + + if err := saveServerConfig(t.Context(), objLayer, globalServerConfig); err != nil { + t.Fatalf("Unable to save updated config file %s", err) + } + + // Initialize server config. + if err := loadConfig(objLayer, nil); err != nil { + t.Fatalf("Unable to initialize from updated config file %s", err) + } +} diff --git a/cmd/config-dir.go b/cmd/config-dir.go new file mode 100644 index 00000000..b03b175b --- /dev/null +++ b/cmd/config-dir.go @@ -0,0 +1,108 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "os" + "path/filepath" + + homedir "github.com/mitchellh/go-homedir" +) + +const ( + // Default minio configuration directory where below configuration files/directories are stored. + defaultMinioConfigDir = ".minio" + + // Directory contains below files/directories for HTTPS configuration. + certsDir = "certs" + + // Directory contains all CA certificates other than system defaults for HTTPS. + certsCADir = "CAs" + + // Public certificate file for HTTPS. + publicCertFile = "public.crt" + + // Private key file for HTTPS. + privateKeyFile = "private.key" +) + +// ConfigDir - points to a user set directory. +type ConfigDir struct { + path string +} + +func getDefaultConfigDir() string { + homeDir, err := homedir.Dir() + if err != nil { + return "" + } + + return filepath.Join(homeDir, defaultMinioConfigDir) +} + +func getDefaultCertsDir() string { + return filepath.Join(getDefaultConfigDir(), certsDir) +} + +func getDefaultCertsCADir() string { + return filepath.Join(getDefaultCertsDir(), certsCADir) +} + +var ( + // Default config, certs and CA directories. + defaultConfigDir = &ConfigDir{path: getDefaultConfigDir()} + defaultCertsDir = &ConfigDir{path: getDefaultCertsDir()} + defaultCertsCADir = &ConfigDir{path: getDefaultCertsCADir()} + + // Points to current configuration directory -- deprecated, to be removed in future. + globalConfigDir = defaultConfigDir + // Points to current certs directory set by user with --certs-dir + globalCertsDir = defaultCertsDir + // Points to relative path to certs directory and is /CAs + globalCertsCADir = defaultCertsCADir +) + +// Get - returns current directory. +func (dir *ConfigDir) Get() string { + return dir.path +} + +// Attempts to create all directories, ignores any permission denied errors. +func mkdirAllIgnorePerm(path string) error { + err := os.MkdirAll(path, 0o700) + if err != nil { + // It is possible in kubernetes like deployments this directory + // is already mounted and is not writable, ignore any write errors. + if osIsPermission(err) { + err = nil + } + } + return err +} + +func getConfigFile() string { + return filepath.Join(globalConfigDir.Get(), minioConfigFile) +} + +func getPublicCertFile() string { + return filepath.Join(globalCertsDir.Get(), publicCertFile) +} + +func getPrivateKeyFile() string { + return filepath.Join(globalCertsDir.Get(), privateKeyFile) +} diff --git a/cmd/config-encrypted_test.go b/cmd/config-encrypted_test.go new file mode 100644 index 00000000..7b59ae73 --- /dev/null +++ b/cmd/config-encrypted_test.go @@ -0,0 +1,76 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "testing" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/auth" +) + +func TestDecryptData(t *testing.T) { + cred1 := auth.Credentials{ + AccessKey: "minio", + SecretKey: "minio123", + } + + cred2 := auth.Credentials{ + AccessKey: "minio", + SecretKey: "minio1234", + } + + data := []byte(`config data`) + edata1, err := madmin.EncryptData(cred1.String(), data) + if err != nil { + t.Fatal(err) + } + + edata2, err := madmin.EncryptData(cred2.String(), data) + if err != nil { + t.Fatal(err) + } + + tests := []struct { + edata []byte + cred auth.Credentials + success bool + }{ + {edata1, cred1, true}, + {edata2, cred2, true}, + {data, cred1, false}, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + ddata, err := madmin.DecryptData(test.cred.String(), bytes.NewReader(test.edata)) + if err != nil && test.success { + t.Errorf("Expected success, saw failure %v", err) + } + if err == nil && !test.success { + t.Error("Expected failure, saw success") + } + if test.success { + if !bytes.Equal(ddata, data) { + t.Errorf("Expected %s, got %s", string(data), string(ddata)) + } + } + }) + } +} diff --git a/cmd/config-migrate.go b/cmd/config-migrate.go new file mode 100644 index 00000000..0cceb185 --- /dev/null +++ b/cmd/config-migrate.go @@ -0,0 +1,189 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/json" + "errors" + "path" + "strings" + + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/config/compress" + xldap "github.com/minio/minio/internal/config/identity/ldap" + "github.com/minio/minio/internal/config/identity/openid" + "github.com/minio/minio/internal/config/notify" + "github.com/minio/minio/internal/config/policy/opa" + "github.com/minio/minio/internal/config/storageclass" + "github.com/minio/minio/internal/event/target" + "github.com/minio/minio/internal/logger" + xnet "github.com/minio/pkg/v3/net" + "github.com/minio/pkg/v3/quick" +) + +// Save config file to corresponding backend +func Save(configFile string, data any) error { + return quick.SaveConfig(data, configFile, globalEtcdClient) +} + +// Load config from backend +func Load(configFile string, data any) (quick.Config, error) { + return quick.LoadConfig(configFile, globalEtcdClient, data) +} + +func readConfigWithoutMigrate(ctx context.Context, objAPI ObjectLayer) (config.Config, error) { + // Construct path to config.json for the given bucket. + configFile := path.Join(minioConfigPrefix, minioConfigFile) + + configFiles := []string{ + getConfigFile(), + getConfigFile() + ".deprecated", + configFile, + } + + newServerCfg := func() (config.Config, error) { + // Initialize server config. + srvCfg := newServerConfig() + + return srvCfg, saveServerConfig(ctx, objAPI, srvCfg) + } + + var data []byte + var err error + + cfg := &serverConfigV33{} + for _, cfgFile := range configFiles { + if _, err = Load(cfgFile, cfg); err != nil { + if !osIsNotExist(err) && !osIsPermission(err) { + return nil, err + } + continue + } + data, _ = json.Marshal(cfg) + break + } + if osIsPermission(err) { + logger.Info("Older config found but is not readable %s, proceeding to read config from other places", err) + } + if osIsNotExist(err) || osIsPermission(err) || len(data) == 0 { + data, err = readConfig(GlobalContext, objAPI, configFile) + if err != nil { + // when config.json is not found, then we freshly initialize. + if errors.Is(err, errConfigNotFound) { + return newServerCfg() + } + return nil, err + } + + data, err = decryptData(data, configFile) + if err != nil { + return nil, err + } + + newCfg, err := readServerConfig(GlobalContext, objAPI, data) + if err == nil { + return newCfg, nil + } + + // Read older `.minio.sys/config/config.json`, if not + // possible just fail. + if err = json.Unmarshal(data, cfg); err != nil { + // Unable to parse old JSON simply re-initialize a new one. + return newServerCfg() + } + } + + if !globalCredViaEnv && cfg.Credential.IsValid() { + // Preserve older credential if we do not have + // root credentials set via environment variable. + globalActiveCred = cfg.Credential + } + + // Init compression config. For future migration, Compression config needs to be copied over from previous version. + switch cfg.Version { + case "29": + // V29 -> V30 + cfg.Compression.Enabled = false + cfg.Compression.Extensions = strings.Split(compress.DefaultExtensions, config.ValueSeparator) + cfg.Compression.MimeTypes = strings.Split(compress.DefaultMimeTypes, config.ValueSeparator) + case "30": + // V30 -> V31 + cfg.OpenID = openid.Config{} + cfg.Policy.OPA = opa.Args{ + URL: &xnet.URL{}, + AuthToken: "", + } + case "31": + // V31 -> V32 + cfg.Notify.NSQ = make(map[string]target.NSQArgs) + cfg.Notify.NSQ["1"] = target.NSQArgs{} + } + + // Move to latest. + cfg.Version = "33" + + newCfg := newServerConfig() + + config.SetRegion(newCfg, cfg.Region) + storageclass.SetStorageClass(newCfg, cfg.StorageClass) + + for k, loggerArgs := range cfg.Logger.HTTP { + logger.SetLoggerHTTP(newCfg, k, loggerArgs) + } + for k, auditArgs := range cfg.Logger.AuditWebhook { + logger.SetLoggerHTTPAudit(newCfg, k, auditArgs) + } + + xldap.SetIdentityLDAP(newCfg, cfg.LDAPServerConfig) + opa.SetPolicyOPAConfig(newCfg, cfg.Policy.OPA) + compress.SetCompressionConfig(newCfg, cfg.Compression) + + for k, args := range cfg.Notify.AMQP { + notify.SetNotifyAMQP(newCfg, k, args) + } + for k, args := range cfg.Notify.Elasticsearch { + notify.SetNotifyES(newCfg, k, args) + } + for k, args := range cfg.Notify.Kafka { + notify.SetNotifyKafka(newCfg, k, args) + } + for k, args := range cfg.Notify.MQTT { + notify.SetNotifyMQTT(newCfg, k, args) + } + for k, args := range cfg.Notify.MySQL { + notify.SetNotifyMySQL(newCfg, k, args) + } + for k, args := range cfg.Notify.NATS { + notify.SetNotifyNATS(newCfg, k, args) + } + for k, args := range cfg.Notify.NSQ { + notify.SetNotifyNSQ(newCfg, k, args) + } + for k, args := range cfg.Notify.PostgreSQL { + notify.SetNotifyPostgres(newCfg, k, args) + } + for k, args := range cfg.Notify.Redis { + notify.SetNotifyRedis(newCfg, k, args) + } + for k, args := range cfg.Notify.Webhook { + notify.SetNotifyWebhook(newCfg, k, args) + } + + return newCfg, nil +} diff --git a/cmd/config-versions.go b/cmd/config-versions.go new file mode 100644 index 00000000..020bfa44 --- /dev/null +++ b/cmd/config-versions.go @@ -0,0 +1,79 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/config/compress" + xldap "github.com/minio/minio/internal/config/identity/ldap" + "github.com/minio/minio/internal/config/identity/openid" + "github.com/minio/minio/internal/config/notify" + "github.com/minio/minio/internal/config/policy/opa" + "github.com/minio/minio/internal/config/storageclass" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/quick" +) + +// FileLogger is introduced to workaround the dependency about logrus +type FileLogger struct { + Enable bool `json:"enable"` + Filename string `json:"filename"` +} + +// ConsoleLogger is introduced to workaround the dependency about logrus +type ConsoleLogger struct { + Enable bool `json:"enable"` +} + +// serverConfigV33 is just like version '32', removes clientID from NATS and MQTT, and adds queueDir, queueLimit in all notification targets. +type serverConfigV33 struct { + quick.Config `json:"-"` // ignore interfaces + + Version string `json:"version"` + + // S3 API configuration. + Credential auth.Credentials `json:"credential"` + Region string `json:"region"` + Worm config.BoolFlag `json:"worm"` + + // Storage class configuration + StorageClass storageclass.Config `json:"storageclass"` + + // Notification queue configuration. + Notify notify.Config `json:"notify"` + + // Logger configuration + Logger logger.Config `json:"logger"` + + // Compression configuration + Compression compress.Config `json:"compress"` + + // OpenID configuration + OpenID openid.Config `json:"openid"` + + // External policy enforcements. + Policy struct { + // OPA configuration. + OPA opa.Args `json:"opa"` + + // Add new external policy enforcements here. + } `json:"policy"` + + LDAPServerConfig xldap.LegacyConfig `json:"ldapserverconfig"` +} diff --git a/cmd/config.go b/cmd/config.go new file mode 100644 index 00000000..7311d628 --- /dev/null +++ b/cmd/config.go @@ -0,0 +1,227 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "path" + "sort" + "strings" + + jsoniter "github.com/json-iterator/go" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/kms" +) + +const ( + minioConfigPrefix = "config" + minioConfigBucket = minioMetaBucket + SlashSeparator + minioConfigPrefix + kvPrefix = ".kv" + + // Captures all the previous SetKV operations and allows rollback. + minioConfigHistoryPrefix = minioConfigPrefix + "/history" + + // MinIO configuration file. + minioConfigFile = "config.json" +) + +func listServerConfigHistory(ctx context.Context, objAPI ObjectLayer, withData bool, count int) ( + []madmin.ConfigHistoryEntry, error, +) { + var configHistory []madmin.ConfigHistoryEntry + + // List all kvs + marker := "" + for { + res, err := objAPI.ListObjects(ctx, minioMetaBucket, minioConfigHistoryPrefix, marker, "", maxObjectList) + if err != nil { + return nil, err + } + for _, obj := range res.Objects { + cfgEntry := madmin.ConfigHistoryEntry{ + RestoreID: strings.TrimSuffix(path.Base(obj.Name), kvPrefix), + CreateTime: obj.ModTime, // ModTime is createTime for config history entries. + } + if withData { + data, err := readConfig(ctx, objAPI, obj.Name) + if err != nil { + // ignore history file if not readable. + continue + } + + data, err = decryptData(data, obj.Name) + if err != nil { + // ignore history file that cannot be loaded. + continue + } + + cfgEntry.Data = string(data) + } + configHistory = append(configHistory, cfgEntry) + count-- + if count == 0 { + break + } + } + if !res.IsTruncated { + // We are done here + break + } + marker = res.NextMarker + } + sort.Slice(configHistory, func(i, j int) bool { + return configHistory[i].CreateTime.Before(configHistory[j].CreateTime) + }) + return configHistory, nil +} + +func delServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV string) error { + historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV+kvPrefix) + _, err := objAPI.DeleteObject(ctx, minioMetaBucket, historyFile, ObjectOptions{ + DeletePrefix: true, + DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls) + }) + return err +} + +func readServerConfigHistory(ctx context.Context, objAPI ObjectLayer, uuidKV string) ([]byte, error) { + historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV+kvPrefix) + data, err := readConfig(ctx, objAPI, historyFile) + if err != nil { + return nil, err + } + + return decryptData(data, historyFile) +} + +func saveServerConfigHistory(ctx context.Context, objAPI ObjectLayer, kv []byte) error { + uuidKV := mustGetUUID() + kvPrefix + historyFile := pathJoin(minioConfigHistoryPrefix, uuidKV) + + if GlobalKMS != nil { + var err error + kv, err = config.EncryptBytes(GlobalKMS, kv, kms.Context{ + minioMetaBucket: path.Join(minioMetaBucket, historyFile), + }) + if err != nil { + return err + } + } + return saveConfig(ctx, objAPI, historyFile, kv) +} + +func saveServerConfig(ctx context.Context, objAPI ObjectLayer, cfg any) error { + data, err := json.Marshal(cfg) + if err != nil { + return err + } + + configFile := path.Join(minioConfigPrefix, minioConfigFile) + if GlobalKMS != nil { + data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{ + minioMetaBucket: path.Join(minioMetaBucket, configFile), + }) + if err != nil { + return err + } + } + return saveConfig(ctx, objAPI, configFile, data) +} + +// data is optional. If nil it will be loaded from backend. +func readServerConfig(ctx context.Context, objAPI ObjectLayer, data []byte) (config.Config, error) { + srvCfg := config.New() + var err error + if len(data) == 0 { + configFile := path.Join(minioConfigPrefix, minioConfigFile) + data, err = readConfig(ctx, objAPI, configFile) + if err != nil { + if errors.Is(err, errConfigNotFound) { + lookupConfigs(srvCfg, objAPI) + return srvCfg, nil + } + return nil, err + } + + data, err = decryptData(data, configFile) + if err != nil { + lookupConfigs(srvCfg, objAPI) + return nil, err + } + } + + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err := json.Unmarshal(data, &srvCfg); err != nil { + return nil, err + } + + // Add any missing entries + return srvCfg.Merge(), nil +} + +// ConfigSys - config system. +type ConfigSys struct{} + +// Init - initializes config system from config.json. +func (sys *ConfigSys) Init(objAPI ObjectLayer) error { + if objAPI == nil { + return errInvalidArgument + } + + return initConfig(objAPI) +} + +// NewConfigSys - creates new config system object. +func NewConfigSys() *ConfigSys { + return &ConfigSys{} +} + +// Initialize and load config from remote etcd or local config directory +func initConfig(objAPI ObjectLayer) (err error) { + bootstrapTraceMsg("load the configuration") + defer func() { + if err != nil { + bootstrapTraceMsg(fmt.Sprintf("loading configuration failed: %v", err)) + } + }() + + if objAPI == nil { + return errServerNotInitialized + } + + srvCfg, err := readConfigWithoutMigrate(GlobalContext, objAPI) + if err != nil { + return err + } + + bootstrapTraceMsg("lookup the configuration") + + // Override any values from ENVs. + lookupConfigs(srvCfg, objAPI) + + // hold the mutex lock before a new config is assigned. + globalServerConfigMu.Lock() + globalServerConfig = srvCfg + globalServerConfigMu.Unlock() + + return nil +} diff --git a/cmd/consolelogger.go b/cmd/consolelogger.go new file mode 100644 index 00000000..18488e19 --- /dev/null +++ b/cmd/consolelogger.go @@ -0,0 +1,205 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "container/ring" + "context" + "io" + "sync" + "sync/atomic" + + "github.com/minio/madmin-go/v3" + "github.com/minio/madmin-go/v3/logger/log" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/logger/target/console" + types "github.com/minio/minio/internal/logger/target/loggertypes" + "github.com/minio/minio/internal/pubsub" + xnet "github.com/minio/pkg/v3/net" +) + +// number of log messages to buffer +const defaultLogBufferCount = 10000 + +// HTTPConsoleLoggerSys holds global console logger state +type HTTPConsoleLoggerSys struct { + totalMessages int64 + failedMessages int64 + + sync.RWMutex + pubsub *pubsub.PubSub[log.Info, madmin.LogMask] + console *console.Target + nodeName string + logBuf *ring.Ring +} + +// NewConsoleLogger - creates new HTTPConsoleLoggerSys with all nodes subscribed to +// the console logging pub sub system +func NewConsoleLogger(ctx context.Context, w io.Writer) *HTTPConsoleLoggerSys { + return &HTTPConsoleLoggerSys{ + pubsub: pubsub.New[log.Info, madmin.LogMask](8), + console: console.New(w), + logBuf: ring.New(defaultLogBufferCount), + } +} + +// IsOnline always true in case of console logger +func (sys *HTTPConsoleLoggerSys) IsOnline(_ context.Context) bool { + return true +} + +// SetNodeName - sets the node name if any after distributed setup has initialized +func (sys *HTTPConsoleLoggerSys) SetNodeName(nodeName string) { + if !globalIsDistErasure { + sys.nodeName = "" + return + } + + host, err := xnet.ParseHost(globalLocalNodeName) + if err != nil { + logger.FatalIf(err, "Unable to start console logging subsystem") + } + + sys.nodeName = host.Name +} + +// HasLogListeners returns true if console log listeners are registered +// for this node or peers +func (sys *HTTPConsoleLoggerSys) HasLogListeners() bool { + return sys != nil && sys.pubsub.Subscribers() > 0 +} + +// Subscribe starts console logging for this node. +func (sys *HTTPConsoleLoggerSys) Subscribe(subCh chan log.Info, doneCh <-chan struct{}, node string, last int, logKind madmin.LogMask, filter func(entry log.Info) bool) error { + // Enable console logging for remote client. + if !sys.HasLogListeners() { + logger.AddSystemTarget(GlobalContext, sys) + } + + cnt := 0 + // by default send all console logs in the ring buffer unless node or limit query parameters + // are set. + var lastN []log.Info + if last > defaultLogBufferCount || last <= 0 { + last = defaultLogBufferCount + } + + lastN = make([]log.Info, last) + sys.RLock() + sys.logBuf.Do(func(p any) { + if p != nil { + lg, ok := p.(log.Info) + if ok && lg.SendLog(node, logKind) { + lastN[cnt%last] = lg + cnt++ + } + } + }) + sys.RUnlock() + // send last n console log messages in order filtered by node + if cnt > 0 { + for i := range last { + entry := lastN[(cnt+i)%last] + if (entry == log.Info{}) { + continue + } + select { + case subCh <- entry: + case <-doneCh: + return nil + } + } + } + return sys.pubsub.Subscribe(madmin.LogMaskAll, subCh, doneCh, filter) +} + +// Init if HTTPConsoleLoggerSys is valid, always returns nil right now +func (sys *HTTPConsoleLoggerSys) Init(_ context.Context) error { + return nil +} + +// Endpoint - dummy function for interface compatibility +func (sys *HTTPConsoleLoggerSys) Endpoint() string { + return sys.console.Endpoint() +} + +// String - stringer function for interface compatibility +func (sys *HTTPConsoleLoggerSys) String() string { + return logger.ConsoleLoggerTgt +} + +// Stats returns the target statistics. +func (sys *HTTPConsoleLoggerSys) Stats() types.TargetStats { + return types.TargetStats{ + TotalMessages: atomic.LoadInt64(&sys.totalMessages), + FailedMessages: atomic.LoadInt64(&sys.failedMessages), + QueueLength: 0, + } +} + +// Content returns the console stdout log +func (sys *HTTPConsoleLoggerSys) Content() (logs []log.Entry) { + sys.RLock() + sys.logBuf.Do(func(p any) { + if p != nil { + lg, ok := p.(log.Info) + if ok { + if (lg.Entry != log.Entry{}) { + logs = append(logs, lg.Entry) + } + } + } + }) + sys.RUnlock() + + return logs +} + +// Cancel - cancels the target +func (sys *HTTPConsoleLoggerSys) Cancel() { +} + +// Type - returns type of the target +func (sys *HTTPConsoleLoggerSys) Type() types.TargetType { + return types.TargetConsole +} + +// Send log message 'e' to console and publish to console +// log pubsub system +func (sys *HTTPConsoleLoggerSys) Send(ctx context.Context, entry any) error { + var lg log.Info + switch e := entry.(type) { + case log.Entry: + lg = log.Info{Entry: e, NodeName: sys.nodeName} + case string: + lg = log.Info{ConsoleMsg: e, NodeName: sys.nodeName} + } + atomic.AddInt64(&sys.totalMessages, 1) + + sys.pubsub.Publish(lg) + sys.Lock() + // add log to ring buffer + sys.logBuf.Value = lg + sys.logBuf = sys.logBuf.Next() + sys.Unlock() + err := sys.console.Send(entry) + if err != nil { + atomic.AddInt64(&sys.failedMessages, 1) + } + return err +} diff --git a/cmd/copy-part-range.go b/cmd/copy-part-range.go new file mode 100644 index 00000000..1c2f84b0 --- /dev/null +++ b/cmd/copy-part-range.go @@ -0,0 +1,72 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "net/http" + "net/url" +) + +// Writes S3 compatible copy part range error. +func writeCopyPartErr(ctx context.Context, w http.ResponseWriter, err error, url *url.URL) { + switch err { + case errInvalidRange: + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopyPartRange), url) + return + case errInvalidRangeSource: + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopyPartRangeSource), url) + return + default: + apiErr := errorCodes.ToAPIErr(ErrInvalidCopyPartRangeSource) + apiErr.Description = err.Error() + writeErrorResponse(ctx, w, apiErr, url) + return + } +} + +// Parses x-amz-copy-source-range for CopyObjectPart API. Its behavior +// is different from regular HTTP range header. It only supports the +// form `bytes=first-last` where first and last are zero-based byte +// offsets. See +// http://docs.aws.amazon.com/AmazonS3/latest/API/mpUploadUploadPartCopy.html +// for full details. This function treats an empty rangeString as +// referring to the whole resource. +func parseCopyPartRangeSpec(rangeString string) (hrange *HTTPRangeSpec, err error) { + hrange, err = parseRequestRangeSpec(rangeString) + if err != nil { + return nil, err + } + if hrange.IsSuffixLength || hrange.Start < 0 || hrange.End < 0 { + return nil, errInvalidRange + } + return hrange, nil +} + +// checkCopyPartRangeWithSize adds more check to the range string in case of +// copy object part. This API requires having specific start and end range values +// e.g. 'bytes=3-10'. Other use cases will be rejected. +func checkCopyPartRangeWithSize(rs *HTTPRangeSpec, resourceSize int64) (err error) { + if rs == nil { + return nil + } + if rs.IsSuffixLength || rs.Start >= resourceSize || rs.End >= resourceSize { + return errInvalidRangeSource + } + return nil +} diff --git a/cmd/copy-part-range_test.go b/cmd/copy-part-range_test.go new file mode 100644 index 00000000..afe8cd0c --- /dev/null +++ b/cmd/copy-part-range_test.go @@ -0,0 +1,95 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import "testing" + +// Test parseCopyPartRange() +func TestParseCopyPartRangeSpec(t *testing.T) { + // Test success cases. + successCases := []struct { + rangeString string + offsetBegin int64 + offsetEnd int64 + }{ + {"bytes=2-5", 2, 5}, + {"bytes=2-9", 2, 9}, + {"bytes=2-2", 2, 2}, + {"bytes=0000-0006", 0, 6}, + } + objectSize := int64(10) + + for _, successCase := range successCases { + rs, err := parseCopyPartRangeSpec(successCase.rangeString) + if err != nil { + t.Fatalf("expected: , got: %s", err) + } + + start, length, err1 := rs.GetOffsetLength(objectSize) + if err1 != nil { + t.Fatalf("expected: , got: %s", err1) + } + + if start != successCase.offsetBegin { + t.Fatalf("expected: %d, got: %d", successCase.offsetBegin, start) + } + + if start+length-1 != successCase.offsetEnd { + t.Fatalf("expected: %d, got: %d", successCase.offsetEnd, start+length-1) + } + } + + // Test invalid range strings. + invalidRangeStrings := []string{ + "bytes=8", + "bytes=5-2", + "bytes=+2-5", + "bytes=2-+5", + "bytes=2--5", + "bytes=-", + "2-5", + "bytes = 2-5", + "bytes=2 - 5", + "bytes=0-0,-1", + "bytes=2-5 ", + "bytes=-1", + "bytes=1-", + } + for _, rangeString := range invalidRangeStrings { + if _, err := parseCopyPartRangeSpec(rangeString); err == nil { + t.Fatalf("expected: an error, got: for range %s", rangeString) + } + } + + // Test error range strings. + errorRangeString := []string{ + "bytes=10-10", + "bytes=20-30", + } + for _, rangeString := range errorRangeString { + rs, err := parseCopyPartRangeSpec(rangeString) + if err == nil { + err1 := checkCopyPartRangeWithSize(rs, objectSize) + if err1 != errInvalidRangeSource { + t.Fatalf("expected: %s, got: %s", errInvalidRangeSource, err) + } + } else { + t.Fatalf("expected: %s, got: ", errInvalidRangeSource) + } + } +} diff --git a/cmd/crossdomain-xml-handler.go b/cmd/crossdomain-xml-handler.go new file mode 100644 index 00000000..78cd9652 --- /dev/null +++ b/cmd/crossdomain-xml-handler.go @@ -0,0 +1,48 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import "net/http" + +// Standard cross domain policy information located at https://s3.amazonaws.com/crossdomain.xml +const crossDomainXML = `` + +// Standard path where an app would find cross domain policy information. +const crossDomainXMLEntity = "/crossdomain.xml" + +// A cross-domain policy file is an XML document that grants a web client, such as Adobe Flash Player +// or Adobe Acrobat (though not necessarily limited to these), permission to handle data across domains. +// When clients request content hosted on a particular source domain and that content make requests +// directed towards a domain other than its own, the remote domain needs to host a cross-domain +// policy file that grants access to the source domain, allowing the client to continue the transaction. +func setCrossDomainPolicyMiddleware(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + cxml := crossDomainXML + if globalServerCtxt.CrossDomainXML != "" { + cxml = globalServerCtxt.CrossDomainXML + } + // Look for 'crossdomain.xml' in the incoming request. + if r.URL.Path == crossDomainXMLEntity { + // Write the standard cross domain policy xml. + w.Write([]byte(cxml)) + // Request completed, no need to serve to other handlers. + return + } + h.ServeHTTP(w, r) + }) +} diff --git a/cmd/crossdomain-xml-handler_test.go b/cmd/crossdomain-xml-handler_test.go new file mode 100644 index 00000000..a5f0cb5a --- /dev/null +++ b/cmd/crossdomain-xml-handler_test.go @@ -0,0 +1,42 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "net/http" + "net/http/httptest" + "testing" + + "github.com/minio/mux" +) + +// Test cross domain xml handler. +func TestCrossXMLHandler(t *testing.T) { + // Server initialization. + router := mux.NewRouter().SkipClean(true).UseEncodedPath() + handler := setCrossDomainPolicyMiddleware(router) + srv := httptest.NewServer(handler) + + resp, err := http.Get(srv.URL + crossDomainXMLEntity) + if err != nil { + t.Fatal(err) + } + if resp.StatusCode != http.StatusOK { + t.Fatal("Unexpected http status received", resp.Status) + } +} diff --git a/cmd/data-scanner-metric.go b/cmd/data-scanner-metric.go new file mode 100644 index 00000000..db0bd21b --- /dev/null +++ b/cmd/data-scanner-metric.go @@ -0,0 +1,344 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "strings" + "sync" + "sync/atomic" + "time" + "unsafe" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/bucket/lifecycle" +) + +//go:generate stringer -type=scannerMetric -trimprefix=scannerMetric $GOFILE + +type scannerMetric uint8 + +type scannerMetrics struct { + // All fields must be accessed atomically and aligned. + operations [scannerMetricLast]uint64 + latency [scannerMetricLastRealtime]lockedLastMinuteLatency + + // actions records actions performed. + actions [lifecycle.ActionCount]uint64 + actionsLatency [lifecycle.ActionCount]lockedLastMinuteLatency + + // currentPaths contains (string,*currentPathTracker) for each disk processing. + // Alignment not required. + currentPaths sync.Map + + cycleInfoMu sync.Mutex + cycleInfo *currentScannerCycle +} + +var globalScannerMetrics scannerMetrics + +const ( + // START Realtime metrics, that only to records + // last minute latencies and total operation count. + scannerMetricReadMetadata scannerMetric = iota + scannerMetricCheckMissing + scannerMetricSaveUsage + scannerMetricApplyAll + scannerMetricApplyVersion + scannerMetricTierObjSweep + scannerMetricHealCheck + scannerMetricILM + scannerMetricCheckReplication + scannerMetricYield + scannerMetricCleanAbandoned + scannerMetricApplyNonCurrent + scannerMetricHealAbandonedVersion + + // START Trace metrics: + scannerMetricStartTrace + scannerMetricScanObject // Scan object. All operations included. + scannerMetricHealAbandonedObject + + // END realtime metrics: + scannerMetricLastRealtime + + // Trace only metrics: + scannerMetricScanFolder // Scan a folder on disk, recursively. + scannerMetricScanCycle // Full cycle, cluster global + scannerMetricScanBucketDrive // Single bucket on one drive + scannerMetricCompactFolder // Folder compacted. + + // Must be last: + scannerMetricLast +) + +// log scanner action. +// Use for s > scannerMetricStartTrace +func (p *scannerMetrics) log(s scannerMetric, paths ...string) func(custom map[string]string) { + startTime := time.Now() + return func(custom map[string]string) { + duration := time.Since(startTime) + + atomic.AddUint64(&p.operations[s], 1) + if s < scannerMetricLastRealtime { + p.latency[s].add(duration) + } + + if s > scannerMetricStartTrace && globalTrace.NumSubscribers(madmin.TraceScanner) > 0 { + globalTrace.Publish(scannerTrace(s, startTime, duration, strings.Join(paths, " "), custom)) + } + } +} + +// time n scanner actions. +// Use for s < scannerMetricLastRealtime +func (p *scannerMetrics) timeN(s scannerMetric) func(n int) { + startTime := time.Now() + return func(n int) { + duration := time.Since(startTime) + + atomic.AddUint64(&p.operations[s], uint64(n)) + if s < scannerMetricLastRealtime { + p.latency[s].add(duration) + } + } +} + +// time a scanner action. +// Use for s < scannerMetricLastRealtime +func (p *scannerMetrics) time(s scannerMetric) func() { + startTime := time.Now() + return func() { + duration := time.Since(startTime) + + atomic.AddUint64(&p.operations[s], 1) + if s < scannerMetricLastRealtime { + p.latency[s].add(duration) + } + } +} + +// timeSize add time and size of a scanner action. +// Use for s < scannerMetricLastRealtime +func (p *scannerMetrics) timeSize(s scannerMetric) func(sz int) { + startTime := time.Now() + return func(sz int) { + duration := time.Since(startTime) + + atomic.AddUint64(&p.operations[s], 1) + if s < scannerMetricLastRealtime { + p.latency[s].addSize(duration, int64(sz)) + } + } +} + +// incTime will increment time on metric s with a specific duration. +// Use for s < scannerMetricLastRealtime +func (p *scannerMetrics) incTime(s scannerMetric, d time.Duration) { + atomic.AddUint64(&p.operations[s], 1) + if s < scannerMetricLastRealtime { + p.latency[s].add(d) + } +} + +// timeILM times an ILM action. +// lifecycle.NoneAction is ignored. +// Use for s < scannerMetricLastRealtime +func (p *scannerMetrics) timeILM(a lifecycle.Action) func(versions uint64) { + if a == lifecycle.NoneAction || a >= lifecycle.ActionCount { + return func(_ uint64) {} + } + startTime := time.Now() + return func(versions uint64) { + duration := time.Since(startTime) + atomic.AddUint64(&p.actions[a], versions) + p.actionsLatency[a].add(duration) + } +} + +type currentPathTracker struct { + name *unsafe.Pointer // contains atomically accessed *string +} + +// currentPathUpdater provides a lightweight update function for keeping track of +// current objects for each disk. +// Returns a function that can be used to update the current object +// and a function to call to when processing finished. +func (p *scannerMetrics) currentPathUpdater(disk, initial string) (update func(path string), done func()) { + initialPtr := unsafe.Pointer(&initial) + tracker := ¤tPathTracker{ + name: &initialPtr, + } + + p.currentPaths.Store(disk, tracker) + return func(path string) { + atomic.StorePointer(tracker.name, unsafe.Pointer(&path)) + }, func() { + p.currentPaths.Delete(disk) + } +} + +// getCurrentPaths returns the paths currently being processed. +func (p *scannerMetrics) getCurrentPaths() []string { + var res []string + prefix := globalLocalNodeName + "/" + p.currentPaths.Range(func(key, value any) bool { + // We are a bit paranoid, but better miss an entry than crash. + name, ok := key.(string) + if !ok { + return true + } + obj, ok := value.(*currentPathTracker) + if !ok { + return true + } + strptr := (*string)(atomic.LoadPointer(obj.name)) + if strptr != nil { + res = append(res, pathJoin(prefix, name, *strptr)) + } + return true + }) + return res +} + +// activeDrives returns the number of currently active disks. +// (since this is concurrent it may not be 100% reliable) +func (p *scannerMetrics) activeDrives() int { + var i int + p.currentPaths.Range(func(k, v any) bool { + i++ + return true + }) + return i +} + +// lifetime returns the lifetime count of the specified metric. +func (p *scannerMetrics) lifetime(m scannerMetric) uint64 { + if m >= scannerMetricLast { + return 0 + } + val := atomic.LoadUint64(&p.operations[m]) + return val +} + +// lastMinute returns the last minute statistics of a metric. +// m should be < scannerMetricLastRealtime +func (p *scannerMetrics) lastMinute(m scannerMetric) AccElem { + if m >= scannerMetricLastRealtime { + return AccElem{} + } + val := p.latency[m].total() + return val +} + +// lifetimeActions returns the lifetime count of the specified ilm metric. +func (p *scannerMetrics) lifetimeActions(a lifecycle.Action) uint64 { + if a == lifecycle.NoneAction || a >= lifecycle.ActionCount { + return 0 + } + val := atomic.LoadUint64(&p.actions[a]) + return val +} + +// lastMinuteActions returns the last minute statistics of an ilm metric. +func (p *scannerMetrics) lastMinuteActions(a lifecycle.Action) AccElem { + if a == lifecycle.NoneAction || a >= lifecycle.ActionCount { + return AccElem{} + } + val := p.actionsLatency[a].total() + return val +} + +// setCycle updates the current cycle metrics. +func (p *scannerMetrics) setCycle(c *currentScannerCycle) { + if c != nil { + c2 := c.clone() + c = &c2 + } + p.cycleInfoMu.Lock() + p.cycleInfo = c + p.cycleInfoMu.Unlock() +} + +// getCycle returns the current cycle metrics. +// If not nil, the returned value can safely be modified. +func (p *scannerMetrics) getCycle() *currentScannerCycle { + p.cycleInfoMu.Lock() + defer p.cycleInfoMu.Unlock() + if p.cycleInfo == nil { + return nil + } + c := p.cycleInfo.clone() + return &c +} + +func (p *scannerMetrics) report() madmin.ScannerMetrics { + var m madmin.ScannerMetrics + cycle := p.getCycle() + if cycle != nil { + m.CurrentCycle = cycle.current + m.CyclesCompletedAt = cycle.cycleCompleted + m.CurrentStarted = cycle.started + } + m.CollectedAt = time.Now() + m.ActivePaths = p.getCurrentPaths() + m.LifeTimeOps = make(map[string]uint64, scannerMetricLast) + for i := range scannerMetricLast { + if n := atomic.LoadUint64(&p.operations[i]); n > 0 { + m.LifeTimeOps[i.String()] = n + } + } + if len(m.LifeTimeOps) == 0 { + m.LifeTimeOps = nil + } + + m.LastMinute.Actions = make(map[string]madmin.TimedAction, scannerMetricLastRealtime) + for i := range scannerMetricLastRealtime { + lm := p.lastMinute(i) + if lm.N > 0 { + m.LastMinute.Actions[i.String()] = lm.asTimedAction() + } + } + if len(m.LastMinute.Actions) == 0 { + m.LastMinute.Actions = nil + } + + // ILM + m.LifeTimeILM = make(map[string]uint64) + for i := lifecycle.NoneAction + 1; i < lifecycle.ActionCount; i++ { + if n := atomic.LoadUint64(&p.actions[i]); n > 0 { + m.LifeTimeILM[i.String()] = n + } + } + if len(m.LifeTimeILM) == 0 { + m.LifeTimeILM = nil + } + + if len(m.LifeTimeILM) > 0 { + m.LastMinute.ILM = make(map[string]madmin.TimedAction, len(m.LifeTimeILM)) + for i := lifecycle.NoneAction + 1; i < lifecycle.ActionCount; i++ { + lm := p.lastMinuteActions(i) + if lm.N > 0 { + m.LastMinute.ILM[i.String()] = madmin.TimedAction{Count: uint64(lm.N), AccTime: uint64(lm.Total)} + } + } + if len(m.LastMinute.ILM) == 0 { + m.LastMinute.ILM = nil + } + } + return m +} diff --git a/cmd/data-scanner.go b/cmd/data-scanner.go new file mode 100644 index 00000000..288f4d71 --- /dev/null +++ b/cmd/data-scanner.go @@ -0,0 +1,1498 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/binary" + "encoding/json" + "errors" + "fmt" + "io/fs" + "math" + "math/rand" + "os" + "path" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/color" + "github.com/minio/minio/internal/config/heal" + "github.com/minio/minio/internal/event" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/pkg/v3/console" + uatomic "go.uber.org/atomic" +) + +const ( + dataScannerSleepPerFolder = time.Millisecond // Time to wait between folders. + dataUsageUpdateDirCycles = 16 // Visit all folders every n cycles. + dataScannerCompactLeastObject = 500 // Compact when there is less than this many objects in a branch. + dataScannerCompactAtChildren = 10000 // Compact when there are this many children in a branch. + dataScannerCompactAtFolders = dataScannerCompactAtChildren / 4 // Compact when this many subfolders in a single folder. + dataScannerForceCompactAtFolders = 250_000 // Compact when this many subfolders in a single folder (even top level). + dataScannerStartDelay = 1 * time.Minute // Time to wait on startup and between cycles. + + healDeleteDangling = true + healObjectSelectProb = 1024 // Overall probability of a file being scanned; one in n. +) + +var ( + globalHealConfig heal.Config + + // Sleeper values are updated when config is loaded. + scannerSleeper = newDynamicSleeper(2, time.Second, true) // Keep defaults same as config defaults + scannerCycle = uatomic.NewDuration(dataScannerStartDelay) + scannerIdleMode = uatomic.NewInt32(0) // default is throttled when idle + scannerExcessObjectVersions = uatomic.NewInt64(100) + scannerExcessObjectVersionsTotalSize = uatomic.NewInt64(1024 * 1024 * 1024 * 1024) // 1 TB + scannerExcessFolders = uatomic.NewInt64(50000) +) + +// initDataScanner will start the scanner in the background. +func initDataScanner(ctx context.Context, objAPI ObjectLayer) { + go func() { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + // Run the data scanner in a loop + for { + runDataScanner(ctx, objAPI) + duration := max(time.Duration(r.Float64()*float64(scannerCycle.Load())), + // Make sure to sleep at least a second to avoid high CPU ticks. + time.Second) + time.Sleep(duration) + } + }() +} + +func getCycleScanMode(currentCycle, bitrotStartCycle uint64, bitrotStartTime time.Time) madmin.HealScanMode { + bitrotCycle := globalHealConfig.BitrotScanCycle() + switch bitrotCycle { + case -1: + return madmin.HealNormalScan + case 0: + return madmin.HealDeepScan + } + + if currentCycle-bitrotStartCycle < healObjectSelectProb { + return madmin.HealDeepScan + } + + if time.Since(bitrotStartTime) > bitrotCycle { + return madmin.HealDeepScan + } + + return madmin.HealNormalScan +} + +type backgroundHealInfo struct { + BitrotStartTime time.Time `json:"bitrotStartTime"` + BitrotStartCycle uint64 `json:"bitrotStartCycle"` + CurrentScanMode madmin.HealScanMode `json:"currentScanMode"` +} + +func readBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer) backgroundHealInfo { + if globalIsErasureSD { + return backgroundHealInfo{} + } + + // Get last healing information + buf, err := readConfig(ctx, objAPI, backgroundHealInfoPath) + if err != nil { + if !errors.Is(err, errConfigNotFound) { + internalLogOnceIf(ctx, err, backgroundHealInfoPath) + } + return backgroundHealInfo{} + } + var info backgroundHealInfo + if err = json.Unmarshal(buf, &info); err != nil { + bugLogIf(ctx, err, backgroundHealInfoPath) + } + return info +} + +func saveBackgroundHealInfo(ctx context.Context, objAPI ObjectLayer, info backgroundHealInfo) { + if globalIsErasureSD { + return + } + + b, err := json.Marshal(info) + if err != nil { + bugLogIf(ctx, err) + return + } + // Get last healing information + err = saveConfig(ctx, objAPI, backgroundHealInfoPath, b) + if err != nil { + internalLogIf(ctx, err) + } +} + +// runDataScanner will start a data scanner. +// The function will block until the context is canceled. +// There should only ever be one scanner running per cluster. +func runDataScanner(ctx context.Context, objAPI ObjectLayer) { + ctx, cancel := globalLeaderLock.GetLock(ctx) + defer cancel() + + // Load current bloom cycle + var cycleInfo currentScannerCycle + + buf, _ := readConfig(ctx, objAPI, dataUsageBloomNamePath) + if len(buf) == 8 { + cycleInfo.next = binary.LittleEndian.Uint64(buf) + } else if len(buf) > 8 { + cycleInfo.next = binary.LittleEndian.Uint64(buf[:8]) + buf = buf[8:] + _, err := cycleInfo.UnmarshalMsg(buf) + bugLogIf(ctx, err) + } + + scannerTimer := time.NewTimer(scannerCycle.Load()) + defer scannerTimer.Stop() + defer globalScannerMetrics.setCycle(nil) + + for { + select { + case <-ctx.Done(): + return + case <-scannerTimer.C: + // Reset the timer for next cycle. + // If scanner takes longer we start at once. + scannerTimer.Reset(scannerCycle.Load()) + + stopFn := globalScannerMetrics.log(scannerMetricScanCycle) + cycleInfo.current = cycleInfo.next + cycleInfo.started = time.Now() + globalScannerMetrics.setCycle(&cycleInfo) + + bgHealInfo := readBackgroundHealInfo(ctx, objAPI) + scanMode := getCycleScanMode(cycleInfo.current, bgHealInfo.BitrotStartCycle, bgHealInfo.BitrotStartTime) + if bgHealInfo.CurrentScanMode != scanMode { + newHealInfo := bgHealInfo + newHealInfo.CurrentScanMode = scanMode + if scanMode == madmin.HealDeepScan { + newHealInfo.BitrotStartTime = time.Now().UTC() + newHealInfo.BitrotStartCycle = cycleInfo.current + } + saveBackgroundHealInfo(ctx, objAPI, newHealInfo) + } + + // Wait before starting next cycle and wait on startup. + results := make(chan DataUsageInfo, 1) + go storeDataUsageInBackend(ctx, objAPI, results) + err := objAPI.NSScanner(ctx, results, uint32(cycleInfo.current), scanMode) + scannerLogIf(ctx, err) + res := map[string]string{"cycle": strconv.FormatUint(cycleInfo.current, 10)} + if err != nil { + res["error"] = err.Error() + } + stopFn(res) + if err == nil { + // Store new cycle... + cycleInfo.next++ + cycleInfo.current = 0 + cycleInfo.cycleCompleted = append(cycleInfo.cycleCompleted, time.Now()) + if len(cycleInfo.cycleCompleted) > dataUsageUpdateDirCycles { + cycleInfo.cycleCompleted = cycleInfo.cycleCompleted[len(cycleInfo.cycleCompleted)-dataUsageUpdateDirCycles:] + } + globalScannerMetrics.setCycle(&cycleInfo) + tmp := make([]byte, 8, 8+cycleInfo.Msgsize()) + // Cycle for backward compat. + binary.LittleEndian.PutUint64(tmp, cycleInfo.next) + tmp, _ = cycleInfo.MarshalMsg(tmp) + err = saveConfig(ctx, objAPI, dataUsageBloomNamePath, tmp) + if err != nil { + scannerLogIf(ctx, fmt.Errorf("%w, Object %s", err, dataUsageBloomNamePath)) + } + } + } + } +} + +type cachedFolder struct { + name string + parent *dataUsageHash + objectHealProbDiv uint32 +} + +type folderScanner struct { + root string + getSize getSizeFn + oldCache dataUsageCache + newCache dataUsageCache + updateCache dataUsageCache + + dataUsageScannerDebug bool + healObjectSelect uint32 // Do a heal check on an object once every n cycles. Must divide into healFolderInclude + scanMode madmin.HealScanMode + + weSleep func() bool + shouldHeal func() bool + + disks []StorageAPI + disksQuorum int + + // If set updates will be sent regularly to this channel. + // Will not be closed when returned. + updates chan<- dataUsageEntry + lastUpdate time.Time + + // updateCurrentPath should be called whenever a new path is scanned. + updateCurrentPath func(string) +} + +// Cache structure and compaction: +// +// A cache structure will be kept with a tree of usages. +// The cache is a tree structure where each keeps track of its children. +// +// An uncompacted branch contains a count of the files only directly at the +// branch level, and contains link to children branches or leaves. +// +// The leaves are "compacted" based on a number of properties. +// A compacted leaf contains the totals of all files beneath it. +// +// A leaf is only scanned once every dataUsageUpdateDirCycles, +// rarer if the bloom filter for the path is clean and no lifecycles are applied. +// Skipped leaves have their totals transferred from the previous cycle. +// +// When selected there is a one in healObjectSelectProb that any object will be chosen for heal scan. +// +// Compaction happens when either: +// +// 1) The folder (and subfolders) contains less than dataScannerCompactLeastObject objects. +// 2) The folder itself contains more than dataScannerCompactAtFolders folders. +// 3) The folder only contains objects and no subfolders. +// +// A bucket root will never be compacted. +// +// Furthermore if a has more than dataScannerCompactAtChildren recursive children (uncompacted folders) +// the tree will be recursively scanned and the branches with the least number of objects will be +// compacted until the limit is reached. +// +// This ensures that any branch will never contain an unreasonable amount of other branches, +// and also that small branches with few objects don't take up unreasonable amounts of space. +// This keeps the cache size at a reasonable size for all buckets. +// +// Whenever a branch is scanned, it is assumed that it will be un-compacted +// before it hits any of the above limits. +// This will make the branch rebalance itself when scanned if the distribution of objects has changed. + +// scanDataFolder will scanner the basepath+cache.Info.Name and return an updated cache. +// The returned cache will always be valid, but may not be updated from the existing. +// Before each operation sleepDuration is called which can be used to temporarily halt the scanner. +// If the supplied context is canceled the function will return at the first chance. +func scanDataFolder(ctx context.Context, disks []StorageAPI, drive *xlStorage, cache dataUsageCache, getSize getSizeFn, scanMode madmin.HealScanMode, weSleep func() bool) (dataUsageCache, error) { + switch cache.Info.Name { + case "", dataUsageRoot: + return cache, errors.New("internal error: root scan attempted") + } + basePath := drive.drivePath + updatePath, closeDisk := globalScannerMetrics.currentPathUpdater(basePath, cache.Info.Name) + defer closeDisk() + + s := folderScanner{ + root: basePath, + getSize: getSize, + oldCache: cache, + newCache: dataUsageCache{Info: cache.Info}, + updateCache: dataUsageCache{Info: cache.Info}, + dataUsageScannerDebug: false, + healObjectSelect: 0, + scanMode: scanMode, + weSleep: weSleep, + updates: cache.Info.updates, + updateCurrentPath: updatePath, + disks: disks, + disksQuorum: len(disks) / 2, + } + + var skipHeal atomic.Bool + if !globalIsErasure || cache.Info.SkipHealing { + skipHeal.Store(true) + } + + // Check if we should do healing at all. + s.shouldHeal = func() bool { + if skipHeal.Load() { + return false + } + if s.healObjectSelect == 0 { + return false + } + if di, _ := drive.DiskInfo(ctx, DiskInfoOptions{}); di.Healing { + skipHeal.Store(true) + return false + } + return true + } + + // Enable healing in XL mode. + if globalIsErasure && !cache.Info.SkipHealing { + // Do a heal check on an object once every n cycles. Must divide into healFolderInclude + s.healObjectSelect = healObjectSelectProb + } + + done := ctx.Done() + + // Read top level in bucket. + select { + case <-done: + return cache, ctx.Err() + default: + } + root := dataUsageEntry{} + folder := cachedFolder{name: cache.Info.Name, objectHealProbDiv: 1} + err := s.scanFolder(ctx, folder, &root) + if err != nil { + // No useful information... + return cache, err + } + s.newCache.forceCompact(dataScannerCompactAtChildren) + s.newCache.Info.LastUpdate = UTCNow() + s.newCache.Info.NextCycle = cache.Info.NextCycle + return s.newCache, nil +} + +// sendUpdate() should be called on a regular basis when the newCache contains more recent total than previously. +// May or may not send an update upstream. +func (f *folderScanner) sendUpdate() { + // Send at most an update every minute. + if f.updates == nil || time.Since(f.lastUpdate) < time.Minute { + return + } + if flat := f.updateCache.sizeRecursive(f.newCache.Info.Name); flat != nil { + select { + case f.updates <- flat.clone(): + default: + } + f.lastUpdate = time.Now() + } +} + +// scanFolder will scan the provided folder. +// Files found in the folders will be added to f.newCache. +// If final is provided folders will be put into f.newFolders or f.existingFolders. +// If final is not provided the folders found are returned from the function. +func (f *folderScanner) scanFolder(ctx context.Context, folder cachedFolder, into *dataUsageEntry) error { + done := ctx.Done() + scannerLogPrefix := color.Green("folder-scanner:") + + noWait := func() {} + + thisHash := hashPath(folder.name) + // Store initial compaction state. + wasCompacted := into.Compacted + + for { + select { + case <-done: + return ctx.Err() + default: + } + var abandonedChildren dataUsageHashMap + if !into.Compacted { + abandonedChildren = f.oldCache.findChildrenCopy(thisHash) + } + + // If there are lifecycle rules for the prefix. + _, prefix := path2BucketObjectWithBasePath(f.root, folder.name) + var activeLifeCycle *lifecycle.Lifecycle + if f.oldCache.Info.lifeCycle != nil && f.oldCache.Info.lifeCycle.HasActiveRules(prefix) { + if f.dataUsageScannerDebug { + console.Debugf(scannerLogPrefix+" Prefix %q has active rules\n", prefix) + } + activeLifeCycle = f.oldCache.Info.lifeCycle + } + // If there are replication rules for the prefix. + var replicationCfg replicationConfig + if !f.oldCache.Info.replication.Empty() && f.oldCache.Info.replication.Config.HasActiveRules(prefix, true) { + replicationCfg = f.oldCache.Info.replication + } + + if f.weSleep() { + scannerSleeper.Sleep(ctx, dataScannerSleepPerFolder) + } + + var existingFolders, newFolders []cachedFolder + var foundObjects bool + err := readDirFn(pathJoin(f.root, folder.name), func(entName string, typ os.FileMode) error { + // Parse + entName = pathClean(pathJoin(folder.name, entName)) + if entName == "" || entName == folder.name { + if f.dataUsageScannerDebug { + console.Debugf(scannerLogPrefix+" no entity (%s,%s)\n", f.root, entName) + } + return nil + } + bucket, prefix := path2BucketObjectWithBasePath(f.root, entName) + if bucket == "" { + if f.dataUsageScannerDebug { + console.Debugf(scannerLogPrefix+" no bucket (%s,%s)\n", f.root, entName) + } + return errDoneForNow + } + + if isReservedOrInvalidBucket(bucket, false) { + if f.dataUsageScannerDebug { + console.Debugf(scannerLogPrefix+" invalid bucket: %v, entry: %v\n", bucket, entName) + } + return errDoneForNow + } + + select { + case <-done: + return errDoneForNow + default: + } + + if typ&os.ModeDir != 0 { + h := hashPath(entName) + _, exists := f.oldCache.Cache[h.Key()] + if h == thisHash { + return nil + } + this := cachedFolder{name: entName, parent: &thisHash, objectHealProbDiv: folder.objectHealProbDiv} + delete(abandonedChildren, h.Key()) // h.Key() already accounted for. + if exists { + existingFolders = append(existingFolders, this) + f.updateCache.copyWithChildren(&f.oldCache, h, &thisHash) + } else { + newFolders = append(newFolders, this) + } + return nil + } + + wait := noWait + if f.weSleep() { + // Dynamic time delay. + wait = scannerSleeper.Timer(ctx) + } + + // Get file size, ignore errors. + item := scannerItem{ + Path: pathJoin(f.root, entName), + Typ: typ, + bucket: bucket, + prefix: path.Dir(prefix), + objectName: path.Base(entName), + debug: f.dataUsageScannerDebug, + lifeCycle: activeLifeCycle, + replication: replicationCfg, + } + + item.heal.enabled = thisHash.modAlt(f.oldCache.Info.NextCycle/folder.objectHealProbDiv, f.healObjectSelect/folder.objectHealProbDiv) && f.shouldHeal() + item.heal.bitrot = f.scanMode == madmin.HealDeepScan + + sz, err := f.getSize(item) + if err != nil && err != errIgnoreFileContrib { + wait() // wait to proceed to next entry. + if err != errSkipFile && f.dataUsageScannerDebug { + console.Debugf(scannerLogPrefix+" getSize \"%v/%v\" returned err: %v\n", bucket, item.objectPath(), err) + } + return nil + } + + // successfully read means we have a valid object. + foundObjects = true + // Remove filename i.e is the meta file to construct object name + item.transformMetaDir() + + // Object already accounted for, remove from heal map, + // simply because getSize() function already heals the + // object. + delete(abandonedChildren, pathJoin(item.bucket, item.objectPath())) + + if err != errIgnoreFileContrib { + into.addSizes(sz) + into.Objects++ + } + + wait() // wait to proceed to next entry. + + return nil + }) + if err != nil { + return err + } + + if foundObjects && globalIsErasure { + // If we found an object in erasure mode, we skip subdirs (only datadirs)... + break + } + + // If we have many subfolders, compact ourself. + shouldCompact := f.newCache.Info.Name != folder.name && + len(existingFolders)+len(newFolders) >= dataScannerCompactAtFolders || + len(existingFolders)+len(newFolders) >= dataScannerForceCompactAtFolders + + if totalFolders := len(existingFolders) + len(newFolders); totalFolders > int(scannerExcessFolders.Load()) { + prefixName := strings.TrimSuffix(folder.name, "/") + "/" + sendEvent(eventArgs{ + EventName: event.PrefixManyFolders, + BucketName: f.root, + Object: ObjectInfo{ + Name: prefixName, + Size: int64(totalFolders), + }, + UserAgent: "Scanner", + Host: globalMinioHost, + }) + auditLogInternal(context.Background(), AuditLogOptions{ + Event: "scanner:manyprefixes", + APIName: "Scanner", + Bucket: f.root, + Object: prefixName, + Tags: map[string]string{ + "x-minio-prefixes-total": strconv.Itoa(totalFolders), + }, + }) + } + if !into.Compacted && shouldCompact { + into.Compacted = true + newFolders = append(newFolders, existingFolders...) + existingFolders = nil + if f.dataUsageScannerDebug { + console.Debugf(scannerLogPrefix+" Preemptively compacting: %v, entries: %v\n", folder.name, len(existingFolders)+len(newFolders)) + } + } + + scanFolder := func(folder cachedFolder) { + if contextCanceled(ctx) { + return + } + dst := into + if !into.Compacted { + dst = &dataUsageEntry{Compacted: false} + } + if err := f.scanFolder(ctx, folder, dst); err != nil { + return + } + if !into.Compacted { + h := dataUsageHash(folder.name) + into.addChild(h) + // We scanned a folder, optionally send update. + f.updateCache.deleteRecursive(h) + f.updateCache.copyWithChildren(&f.newCache, h, folder.parent) + f.sendUpdate() + } + } + + // Transfer existing + if !into.Compacted { + for _, folder := range existingFolders { + h := hashPath(folder.name) + f.updateCache.copyWithChildren(&f.oldCache, h, folder.parent) + } + } + // Scan new... + for _, folder := range newFolders { + h := hashPath(folder.name) + // Add new folders to the update tree so totals update for these. + if !into.Compacted { + var foundAny bool + parent := thisHash + for parent != hashPath(f.updateCache.Info.Name) { + e := f.updateCache.find(parent.Key()) + if e == nil || e.Compacted { + foundAny = true + break + } + next := f.updateCache.searchParent(parent) + if next == nil { + foundAny = true + break + } + parent = *next + } + if !foundAny { + // Add non-compacted empty entry. + f.updateCache.replaceHashed(h, &thisHash, dataUsageEntry{}) + } + } + f.updateCurrentPath(folder.name) + stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, folder.name) + scanFolder(folder) + stopFn(map[string]string{"type": "new"}) + + // Add new folders if this is new and we don't have existing. + if !into.Compacted { + parent := f.updateCache.find(thisHash.Key()) + if parent != nil && !parent.Compacted { + f.updateCache.deleteRecursive(h) + f.updateCache.copyWithChildren(&f.newCache, h, &thisHash) + } + } + } + + // Scan existing... + for _, folder := range existingFolders { + h := hashPath(folder.name) + // Check if we should skip scanning folder... + // We can only skip if we are not indexing into a compacted destination + // and the entry itself is compacted. + if !into.Compacted && f.oldCache.isCompacted(h) { + if !h.mod(f.oldCache.Info.NextCycle, dataUsageUpdateDirCycles) { + // Transfer and add as child... + f.newCache.copyWithChildren(&f.oldCache, h, folder.parent) + into.addChild(h) + continue + } + // Adjust the probability of healing. + // This first removes lowest x from the mod check and makes it x times more likely. + // So if duudc = 10 and we want heal check every 50 cycles, we check + // if (cycle/10) % (50/10) == 0, which would make heal checks run once every 50 cycles, + // if the objects are pre-selected as 1:10. + folder.objectHealProbDiv = dataUsageUpdateDirCycles + } + f.updateCurrentPath(folder.name) + stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, folder.name) + scanFolder(folder) + stopFn(map[string]string{"type": "existing"}) + } + + // Scan for healing + if len(abandonedChildren) == 0 || !f.shouldHeal() { + // If we are not heal scanning, return now. + break + } + + if len(f.disks) == 0 || f.disksQuorum == 0 { + break + } + + bgSeq, found := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID) + if !found { + break + } + + // Whatever remains in 'abandonedChildren' are folders at this level + // that existed in the previous run but wasn't found now. + // + // This may be because of 2 reasons: + // + // 1) The folder/object was deleted. + // 2) We come from another disk and this disk missed the write. + // + // We therefore perform a heal check. + // If that doesn't bring it back we remove the folder and assume it was deleted. + // This means that the next run will not look for it. + // How to resolve results. + resolver := metadataResolutionParams{ + dirQuorum: f.disksQuorum, + objQuorum: f.disksQuorum, + bucket: "", + strict: false, + } + + healObjectsPrefix := color.Green("healObjects:") + for k := range abandonedChildren { + if !f.shouldHeal() { + break + } + bucket, prefix := path2BucketObject(k) + stopFn := globalScannerMetrics.time(scannerMetricCheckMissing) + f.updateCurrentPath(k) + + if bucket != resolver.bucket { + // Bucket might be missing as well with abandoned children. + // make sure it is created first otherwise healing won't proceed + // for objects. + bgSeq.queueHealTask(healSource{ + bucket: bucket, + }, madmin.HealItemBucket) + } + + resolver.bucket = bucket + + foundObjs := false + ctx, cancel := context.WithCancel(ctx) + + err := listPathRaw(ctx, listPathRawOptions{ + disks: f.disks, + bucket: bucket, + path: prefix, + recursive: true, + reportNotFound: true, + minDisks: f.disksQuorum, + agreed: func(entry metaCacheEntry) { + f.updateCurrentPath(entry.name) + if f.dataUsageScannerDebug { + console.Debugf(healObjectsPrefix+" got agreement: %v\n", entry.name) + } + }, + // Some disks have data for this. + partial: func(entries metaCacheEntries, errs []error) { + if !f.shouldHeal() { + cancel() + return + } + entry, ok := entries.resolve(&resolver) + if !ok { + // check if we can get one entry at least + // proceed to heal nonetheless, since + // this object might be dangling. + entry, _ = entries.firstFound() + } + wait := noWait + if f.weSleep() { + // wait timer per object. + wait = scannerSleeper.Timer(ctx) + } + defer wait() + f.updateCurrentPath(entry.name) + stopFn := globalScannerMetrics.log(scannerMetricHealAbandonedObject, f.root, entry.name) + custom := make(map[string]string) + defer stopFn(custom) + + if f.dataUsageScannerDebug { + console.Debugf(healObjectsPrefix+" resolved to: %v, dir: %v\n", entry.name, entry.isDir()) + } + + if entry.isDir() { + return + } + + // We got an entry which we should be able to heal. + fiv, err := entry.fileInfoVersions(bucket) + if err != nil { + err := bgSeq.queueHealTask(healSource{ + bucket: bucket, + object: entry.name, + versionID: "", + }, madmin.HealItemObject) + if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) { + scannerLogIf(ctx, err) + } + foundObjs = foundObjs || err == nil + return + } + + custom["versions"] = fmt.Sprint(len(fiv.Versions)) + var successVersions, failVersions int + for _, ver := range fiv.Versions { + stopFn := globalScannerMetrics.timeSize(scannerMetricHealAbandonedVersion) + err := bgSeq.queueHealTask(healSource{ + bucket: bucket, + object: fiv.Name, + versionID: ver.VersionID, + }, madmin.HealItemObject) + stopFn(int(ver.Size)) + if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) { + if err != nil { + scannerLogIf(ctx, fmt.Errorf("%w, Object %s/%s/%s", err, bucket, fiv.Name, ver.VersionID)) + } + } + if err == nil { + successVersions++ + } else { + failVersions++ + } + foundObjs = foundObjs || err == nil + } + custom["success_versions"] = fmt.Sprint(successVersions) + custom["failed_versions"] = fmt.Sprint(failVersions) + }, + // Too many disks failed. + finished: func(errs []error) { + if f.dataUsageScannerDebug { + console.Debugf(healObjectsPrefix+" too many errors: %v\n", errs) + } + cancel() + }, + }) + + stopFn() + if f.dataUsageScannerDebug && err != nil && err != errFileNotFound { + console.Debugf(healObjectsPrefix+" checking returned value %v (%T)\n", err, err) + } + + // Add unless healing returned an error. + if foundObjs { + this := cachedFolder{name: k, parent: &thisHash, objectHealProbDiv: 1} + stopFn := globalScannerMetrics.log(scannerMetricScanFolder, f.root, this.name, "HEALED") + scanFolder(this) + stopFn(map[string]string{"type": "healed"}) + } + } + break + } + if !wasCompacted { + f.newCache.replaceHashed(thisHash, folder.parent, *into) + } + + if !into.Compacted && f.newCache.Info.Name != folder.name { + flat := f.newCache.sizeRecursive(thisHash.Key()) + flat.Compacted = true + var compact bool + if flat.Objects < dataScannerCompactLeastObject { + compact = true + } else { + // Compact if we only have objects as children... + compact = true + for k := range into.Children { + if v, ok := f.newCache.Cache[k]; ok { + if len(v.Children) > 0 || v.Objects > 1 { + compact = false + break + } + } + } + } + + if compact { + stop := globalScannerMetrics.log(scannerMetricCompactFolder, folder.name) + f.newCache.deleteRecursive(thisHash) + f.newCache.replaceHashed(thisHash, folder.parent, *flat) + total := map[string]string{ + "objects": strconv.FormatUint(flat.Objects, 10), + "size": strconv.FormatInt(flat.Size, 10), + } + if flat.Versions > 0 { + total["versions"] = strconv.FormatUint(flat.Versions, 10) + } + stop(total) + } + } + // Compact if too many children... + if !into.Compacted { + f.newCache.reduceChildrenOf(thisHash, dataScannerCompactAtChildren, f.newCache.Info.Name != folder.name) + } + if _, ok := f.updateCache.Cache[thisHash.Key()]; !wasCompacted && ok { + // Replace if existed before. + if flat := f.newCache.sizeRecursive(thisHash.Key()); flat != nil { + f.updateCache.deleteRecursive(thisHash) + f.updateCache.replaceHashed(thisHash, folder.parent, *flat) + } + } + + return nil +} + +// scannerItem represents each file while walking. +type scannerItem struct { + Path string + bucket string // Bucket. + prefix string // Only the prefix if any, does not have final object name. + objectName string // Only the object name without prefixes. + replication replicationConfig + lifeCycle *lifecycle.Lifecycle + Typ fs.FileMode + heal struct { + enabled bool + bitrot bool + } // Has the object been selected for heal check? + debug bool +} + +type sizeSummary struct { + totalSize int64 + versions uint64 + deleteMarkers uint64 + replicatedSize int64 + replicatedCount int64 + pendingSize int64 + failedSize int64 + replicaSize int64 + replicaCount int64 + pendingCount uint64 + failedCount uint64 + replTargetStats map[string]replTargetSizeSummary + tiers map[string]tierStats +} + +// replTargetSizeSummary holds summary of replication stats by target +type replTargetSizeSummary struct { + replicatedSize int64 + replicatedCount int64 + pendingSize int64 + failedSize int64 + pendingCount uint64 + failedCount uint64 +} + +type getSizeFn func(item scannerItem) (sizeSummary, error) + +// transformMetaDir will transform a directory to prefix/file.ext +func (i *scannerItem) transformMetaDir() { + split := strings.Split(i.prefix, SlashSeparator) + if len(split) > 1 { + i.prefix = pathJoin(split[:len(split)-1]...) + } else { + i.prefix = "" + } + // Object name is last element + i.objectName = split[len(split)-1] +} + +var applyActionsLogPrefix = color.Green("applyActions:") + +func (i *scannerItem) applyHealing(ctx context.Context, o ObjectLayer, oi ObjectInfo) (size int64) { + if i.debug { + if oi.VersionID != "" { + console.Debugf(applyActionsLogPrefix+" heal checking: %v/%v v(%s)\n", i.bucket, i.objectPath(), oi.VersionID) + } else { + console.Debugf(applyActionsLogPrefix+" heal checking: %v/%v\n", i.bucket, i.objectPath()) + } + } + scanMode := madmin.HealNormalScan + if i.heal.bitrot { + scanMode = madmin.HealDeepScan + } + healOpts := madmin.HealOpts{ + Remove: healDeleteDangling, + ScanMode: scanMode, + } + res, _ := o.HealObject(ctx, i.bucket, i.objectPath(), oi.VersionID, healOpts) + if res.ObjectSize > 0 { + return res.ObjectSize + } + return 0 +} + +func (i *scannerItem) alertExcessiveVersions(remainingVersions int, cumulativeSize int64) { + if remainingVersions >= int(scannerExcessObjectVersions.Load()) { + // Notify object accessed via a GET request. + sendEvent(eventArgs{ + EventName: event.ObjectManyVersions, + BucketName: i.bucket, + Object: ObjectInfo{ + Name: i.objectPath(), + }, + UserAgent: "Scanner", + Host: globalLocalNodeName, + RespElements: map[string]string{"x-minio-versions": strconv.Itoa(remainingVersions)}, + }) + + auditLogInternal(context.Background(), AuditLogOptions{ + Event: "scanner:manyversions", + APIName: "Scanner", + Bucket: i.bucket, + Object: i.objectPath(), + Tags: map[string]string{ + "x-minio-versions": strconv.Itoa(remainingVersions), + }, + }) + } + + // Check if the cumulative size of all versions of this object is high. + if cumulativeSize >= scannerExcessObjectVersionsTotalSize.Load() { + // Notify object accessed via a GET request. + sendEvent(eventArgs{ + EventName: event.ObjectLargeVersions, + BucketName: i.bucket, + Object: ObjectInfo{ + Name: i.objectPath(), + }, + UserAgent: "Scanner", + Host: globalLocalNodeName, + RespElements: map[string]string{ + "x-minio-versions-count": strconv.Itoa(remainingVersions), + "x-minio-versions-size": strconv.FormatInt(cumulativeSize, 10), + }, + }) + + auditLogInternal(context.Background(), AuditLogOptions{ + Event: "scanner:largeversions", + APIName: "Scanner", + Bucket: i.bucket, + Object: i.objectPath(), + Tags: map[string]string{ + "x-minio-versions-count": strconv.Itoa(remainingVersions), + "x-minio-versions-size": strconv.FormatInt(cumulativeSize, 10), + }, + }) + } +} + +type actionsAccountingFn func(oi ObjectInfo, sz, actualSz int64, sizeS *sizeSummary) + +// applyActions will apply lifecycle checks on to a scanned item. +// The resulting size on disk will always be returned. +// The metadata will be compared to consensus on the object layer before any changes are applied. +// If no metadata is supplied, -1 is returned if no action is taken. +func (i *scannerItem) applyActions(ctx context.Context, objAPI ObjectLayer, objInfos []ObjectInfo, lr lock.Retention, sizeS *sizeSummary, fn actionsAccountingFn) { + if len(objInfos) == 0 { + return + } + healActions := func(oi ObjectInfo, actualSz int64) int64 { + size := actualSz + if i.heal.enabled { + done := globalScannerMetrics.time(scannerMetricHealCheck) + size = i.applyHealing(ctx, objAPI, oi) + done() + + if healDeleteDangling { + done := globalScannerMetrics.time(scannerMetricCleanAbandoned) + err := objAPI.CheckAbandonedParts(ctx, i.bucket, i.objectPath(), madmin.HealOpts{Remove: healDeleteDangling}) + done() + if err != nil { + healingLogIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", i.bucket, i.objectPath(), err), i.objectPath()) + } + } + } + + // replicate only if lifecycle rules are not applied. + done := globalScannerMetrics.time(scannerMetricCheckReplication) + i.healReplication(ctx, oi.Clone(), sizeS) + done() + return size + } + + vc, err := globalBucketVersioningSys.Get(i.bucket) + if err != nil { + scannerLogOnceIf(ctx, err, i.bucket) + return + } + + // start ILM check timer + done := globalScannerMetrics.timeN(scannerMetricILM) + if i.lifeCycle == nil { // no ILM configured, apply healing and replication checks + var cumulativeSize int64 + for _, oi := range objInfos { + actualSz, err := oi.GetActualSize() + if err != nil { + scannerLogIf(ctx, err) + continue + } + size := healActions(oi, actualSz) + if fn != nil { // call accountingfn + fn(oi, size, actualSz, sizeS) + } + cumulativeSize += size + } + // end ILM check timer + done(len(objInfos)) + i.alertExcessiveVersions(len(objInfos), cumulativeSize) + return + } + objOpts := make([]lifecycle.ObjectOpts, len(objInfos)) + for i, oi := range objInfos { + objOpts[i] = oi.ToLifecycleOpts() + } + evaluator := lifecycle.NewEvaluator(*i.lifeCycle).WithLockRetention(&lr).WithReplicationConfig(i.replication.Config) + events, err := evaluator.Eval(objOpts) + if err != nil { + // This error indicates that the objOpts passed to Eval is invalid. + bugLogIf(ctx, err, i.bucket, i.objectPath()) + done(len(objInfos)) // end ILM check timer + return + } + done(len(objInfos)) // end ILM check timer + + var ( + toDel []ObjectToDelete + noncurrentEvents []lifecycle.Event + cumulativeSize int64 + ) + remainingVersions := len(objInfos) +eventLoop: + for idx, event := range events { + oi := objInfos[idx] + actualSz, err := oi.GetActualSize() + if i.debug { + scannerLogIf(ctx, err) + } + size := actualSz + switch event.Action { + case lifecycle.DeleteAllVersionsAction, lifecycle.DelMarkerDeleteAllVersionsAction: + remainingVersions = 0 + applyExpiryRule(event, lcEventSrc_Scanner, oi) + break eventLoop + + case lifecycle.DeleteAction, lifecycle.DeleteRestoredAction, lifecycle.DeleteRestoredVersionAction: + if !vc.PrefixEnabled(i.objectPath()) && event.Action == lifecycle.DeleteAction { + remainingVersions-- + size = 0 + } + applyExpiryRule(event, lcEventSrc_Scanner, oi) + + case lifecycle.DeleteVersionAction: // noncurrent versions expiration + opts := objOpts[idx] + remainingVersions-- + size = 0 + toDel = append(toDel, ObjectToDelete{ + ObjectV: ObjectV{ + ObjectName: opts.Name, + VersionID: opts.VersionID, + }, + }) + noncurrentEvents = append(noncurrentEvents, event) + + case lifecycle.TransitionAction, lifecycle.TransitionVersionAction: + applyTransitionRule(event, lcEventSrc_Scanner, oi) + + case lifecycle.NoneAction: + size = healActions(oi, actualSz) + } + // NB fn must be called for every object version except if it is + // expired or was a dangling object. + if fn != nil { + fn(oi, size, actualSz, sizeS) + } + cumulativeSize += size + } + + if len(toDel) > 0 { + globalExpiryState.enqueueNoncurrentVersions(i.bucket, toDel, noncurrentEvents) + } + i.alertExcessiveVersions(remainingVersions, cumulativeSize) +} + +func evalActionFromLifecycle(ctx context.Context, lc lifecycle.Lifecycle, lr lock.Retention, rcfg *replication.Config, obj ObjectInfo) lifecycle.Event { + event := lc.Eval(obj.ToLifecycleOpts()) + if serverDebugLog { + console.Debugf(applyActionsLogPrefix+" lifecycle: Secondary scan: %v\n", event.Action) + } + + switch event.Action { + case lifecycle.DeleteAllVersionsAction, lifecycle.DelMarkerDeleteAllVersionsAction: + // Skip if bucket has object locking enabled; To prevent the + // possibility of violating an object retention on one of the + // noncurrent versions of this object. + if lr.LockEnabled { + return lifecycle.Event{Action: lifecycle.NoneAction} + } + + case lifecycle.DeleteVersionAction, lifecycle.DeleteRestoredVersionAction: + // Defensive code, should never happen + if obj.VersionID == "" { + return lifecycle.Event{Action: lifecycle.NoneAction} + } + if lr.LockEnabled && enforceRetentionForDeletion(ctx, obj) { + if serverDebugLog { + if obj.VersionID != "" { + console.Debugf(applyActionsLogPrefix+" lifecycle: %s v(%s) is locked, not deleting\n", obj.Name, obj.VersionID) + } else { + console.Debugf(applyActionsLogPrefix+" lifecycle: %s is locked, not deleting\n", obj.Name) + } + } + return lifecycle.Event{Action: lifecycle.NoneAction} + } + if rcfg != nil && !obj.VersionPurgeStatus.Empty() && rcfg.HasActiveRules(obj.Name, true) { + return lifecycle.Event{Action: lifecycle.NoneAction} + } + } + + return event +} + +func applyTransitionRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo) bool { + if obj.DeleteMarker || obj.IsDir { + return false + } + globalTransitionState.queueTransitionTask(obj, event, src) + return true +} + +func applyExpiryOnTransitionedObject(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc) (ok bool) { + timeILM := globalScannerMetrics.timeILM(lcEvent.Action) + if err := expireTransitionedObject(ctx, objLayer, &obj, lcEvent, src); err != nil { + if isErrObjectNotFound(err) || isErrVersionNotFound(err) { + return false + } + ilmLogIf(ctx, fmt.Errorf("expireTransitionedObject(%s, %s): %w", obj.Bucket, obj.Name, err)) + return false + } + timeILM(1) + + // Notification already sent in *expireTransitionedObject*, just return 'true' here. + return true +} + +func applyExpiryOnNonTransitionedObjects(ctx context.Context, objLayer ObjectLayer, obj ObjectInfo, lcEvent lifecycle.Event, src lcEventSrc) (ok bool) { + traceFn := globalLifecycleSys.trace(obj) + opts := ObjectOptions{ + Expiration: ExpirationOptions{Expire: true}, + } + + if lcEvent.Action.DeleteVersioned() { + opts.VersionID = obj.VersionID + } + + opts.Versioned = globalBucketVersioningSys.PrefixEnabled(obj.Bucket, obj.Name) + opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(obj.Bucket, obj.Name) + + if lcEvent.Action.DeleteAll() { + opts.DeletePrefix = true + // use prefix delete on exact object (this is an optimization to avoid fan-out calls) + opts.DeletePrefixObject = true + } + var ( + dobj ObjectInfo + err error + ) + + timeILM := globalScannerMetrics.timeILM(lcEvent.Action) + defer func() { + if !ok { + return + } + + if lcEvent.Action != lifecycle.NoneAction { + numVersions := uint64(1) + if lcEvent.Action.DeleteAll() { + numVersions = uint64(obj.NumVersions) + } + timeILM(numVersions) + } + }() + + dobj, err = objLayer.DeleteObject(ctx, obj.Bucket, encodeDirObject(obj.Name), opts) + if err != nil { + if isErrObjectNotFound(err) || isErrVersionNotFound(err) { + traceFn(ILMExpiry, nil, nil) + return false + } + // Assume it is still there. + err := fmt.Errorf("DeleteObject(%s, %s): %w", obj.Bucket, obj.Name, err) + ilmLogOnceIf(ctx, err, "non-transition-expiry"+obj.Name) + traceFn(ILMExpiry, nil, err) + return false + } + if dobj.Name == "" { + dobj = obj + } + + tags := newLifecycleAuditEvent(src, lcEvent).Tags() + tags["version-id"] = dobj.VersionID + + // Send audit for the lifecycle delete operation + auditLogLifecycle(ctx, dobj, ILMExpiry, tags, traceFn) + + eventName := event.ObjectRemovedDelete + if obj.DeleteMarker { + eventName = event.ObjectRemovedDeleteMarkerCreated + } + switch lcEvent.Action { + case lifecycle.DeleteAllVersionsAction: + eventName = event.ObjectRemovedDeleteAllVersions + case lifecycle.DelMarkerDeleteAllVersionsAction: + eventName = event.ILMDelMarkerExpirationDelete + } + // Notify object deleted event. + sendEvent(eventArgs{ + EventName: eventName, + BucketName: dobj.Bucket, + Object: dobj, + UserAgent: "Internal: [ILM-Expiry]", + Host: globalLocalNodeName, + }) + + return true +} + +// Apply object, object version, restored object or restored object version action on the given object +func applyExpiryRule(event lifecycle.Event, src lcEventSrc, obj ObjectInfo) { + globalExpiryState.enqueueByDays(obj, event, src) +} + +// objectPath returns the prefix and object name. +func (i *scannerItem) objectPath() string { + return pathJoin(i.prefix, i.objectName) +} + +// healReplication will heal a scanned item that has failed replication. +func (i *scannerItem) healReplication(ctx context.Context, oi ObjectInfo, sizeS *sizeSummary) { + if oi.VersionID == "" { + return + } + if i.replication.Config == nil { + return + } + roi := queueReplicationHeal(ctx, oi.Bucket, oi, i.replication, 0) + if oi.DeleteMarker || !oi.VersionPurgeStatus.Empty() { + return + } + + if sizeS.replTargetStats == nil && len(roi.TargetStatuses) > 0 { + sizeS.replTargetStats = make(map[string]replTargetSizeSummary) + } + + for arn, tgtStatus := range roi.TargetStatuses { + tgtSizeS, ok := sizeS.replTargetStats[arn] + if !ok { + tgtSizeS = replTargetSizeSummary{} + } + switch tgtStatus { + case replication.Pending: + tgtSizeS.pendingCount++ + tgtSizeS.pendingSize += oi.Size + sizeS.pendingCount++ + sizeS.pendingSize += oi.Size + case replication.Failed: + tgtSizeS.failedSize += oi.Size + tgtSizeS.failedCount++ + sizeS.failedSize += oi.Size + sizeS.failedCount++ + case replication.Completed, replication.CompletedLegacy: + tgtSizeS.replicatedSize += oi.Size + tgtSizeS.replicatedCount++ + sizeS.replicatedSize += oi.Size + sizeS.replicatedCount++ + } + sizeS.replTargetStats[arn] = tgtSizeS + } + + if oi.ReplicationStatus == replication.Replica { + sizeS.replicaSize += oi.Size + sizeS.replicaCount++ + } +} + +type dynamicSleeper struct { + mu sync.RWMutex + + // Sleep factor + factor float64 + + // maximum sleep cap, + // set to <= 0 to disable. + maxSleep time.Duration + + // Don't sleep at all, if time taken is below this value. + // This is to avoid too small costly sleeps. + minSleep time.Duration + + // cycle will be closed + cycle chan struct{} + + // isScanner should be set when this is used by the scanner + // to record metrics. + isScanner bool +} + +// newDynamicSleeper +func newDynamicSleeper(factor float64, maxWait time.Duration, isScanner bool) *dynamicSleeper { + return &dynamicSleeper{ + factor: factor, + cycle: make(chan struct{}), + maxSleep: maxWait, + minSleep: 100 * time.Microsecond, + isScanner: isScanner, + } +} + +// Timer returns a timer that has started. +// When the returned function is called it will wait. +func (d *dynamicSleeper) Timer(ctx context.Context) func() { + t := time.Now() + return func() { + doneAt := time.Now() + d.Sleep(ctx, doneAt.Sub(t)) + } +} + +// Sleep sleeps the specified time multiplied by the sleep factor. +// If the factor is updated the sleep will be done again with the new factor. +func (d *dynamicSleeper) Sleep(ctx context.Context, base time.Duration) { + for { + // Grab current values + d.mu.RLock() + minWait, maxWait := d.minSleep, d.maxSleep + factor := d.factor + cycle := d.cycle + d.mu.RUnlock() + // Don't sleep for really small amount of time + wantSleep := time.Duration(float64(base) * factor) + if wantSleep <= minWait { + return + } + if maxWait > 0 && wantSleep > maxWait { + wantSleep = maxWait + } + timer := time.NewTimer(wantSleep) + select { + case <-ctx.Done(): + if !timer.Stop() { + <-timer.C + if d.isScanner { + globalScannerMetrics.incTime(scannerMetricYield, wantSleep) + } + } + return + case <-timer.C: + if d.isScanner { + globalScannerMetrics.incTime(scannerMetricYield, wantSleep) + } + return + case <-cycle: + if !timer.Stop() { + // We expired. + <-timer.C + if d.isScanner { + globalScannerMetrics.incTime(scannerMetricYield, wantSleep) + } + return + } + } + } +} + +// Update the current settings and cycle all waiting. +// Parameters are the same as in the constructor. +func (d *dynamicSleeper) Update(factor float64, maxWait time.Duration) error { + d.mu.Lock() + defer d.mu.Unlock() + if math.Abs(d.factor-factor) < 1e-10 && d.maxSleep == maxWait { + return nil + } + // Update values and cycle waiting. + xioutil.SafeClose(d.cycle) + d.factor = factor + d.maxSleep = maxWait + d.cycle = make(chan struct{}) + return nil +} + +const ( + // ILMExpiry - audit trail for ILM expiry + ILMExpiry = "ilm:expiry" + // ILMFreeVersionDelete - audit trail for ILM free-version delete + ILMFreeVersionDelete = "ilm:free-version-delete" + // ILMTransition - audit trail for ILM transitioning. + ILMTransition = " ilm:transition" +) + +func auditLogLifecycle(ctx context.Context, oi ObjectInfo, event string, tags map[string]string, traceFn func(event string, metadata map[string]string, err error)) { + var apiName string + switch event { + case ILMExpiry: + apiName = "ILMExpiry" + case ILMFreeVersionDelete: + apiName = "ILMFreeVersionDelete" + case ILMTransition: + apiName = "ILMTransition" + } + auditLogInternal(ctx, AuditLogOptions{ + Event: event, + APIName: apiName, + Bucket: oi.Bucket, + Object: oi.Name, + VersionID: oi.VersionID, + Tags: tags, + }) + traceFn(event, tags, nil) +} diff --git a/cmd/data-scanner_test.go b/cmd/data-scanner_test.go new file mode 100644 index 00000000..c45ad1e7 --- /dev/null +++ b/cmd/data-scanner_test.go @@ -0,0 +1,407 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "encoding/xml" + "fmt" + "slices" + "strings" + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/minio/minio/internal/amztime" + "github.com/minio/minio/internal/bucket/lifecycle" + objectlock "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/bucket/versioning" + xhttp "github.com/minio/minio/internal/http" +) + +func TestApplyNewerNoncurrentVersionsLimit(t *testing.T) { + // Prepare object layer + objAPI, disks, err := prepareErasure(t.Context(), 8) + if err != nil { + t.Fatalf("Failed to initialize object layer: %v", err) + } + defer removeRoots(disks) + setObjectLayer(objAPI) + + // Prepare bucket metadata + globalBucketMetadataSys = NewBucketMetadataSys() + globalBucketObjectLockSys = &BucketObjectLockSys{} + globalBucketVersioningSys = &BucketVersioningSys{} + + lcXML := ` + + + max-versions + Enabled + + 2 + + + + delete-all-versions + Enabled + + + del-all + true + + + + 1 + true + + + +` + lc, err := lifecycle.ParseLifecycleConfig(strings.NewReader(lcXML)) + if err != nil { + t.Fatalf("Failed to unmarshal lifecycle config: %v", err) + } + + vcfg := versioning.Versioning{ + Status: "Enabled", + } + vcfgXML, err := xml.Marshal(vcfg) + if err != nil { + t.Fatalf("Failed to marshal versioning config: %v", err) + } + + bucket := "bucket" + now := time.Now() + meta := BucketMetadata{ + Name: bucket, + Created: now, + LifecycleConfigXML: []byte(lcXML), + VersioningConfigXML: vcfgXML, + VersioningConfigUpdatedAt: now, + LifecycleConfigUpdatedAt: now, + lifecycleConfig: lc, + versioningConfig: &vcfg, + } + globalBucketMetadataSys.Set(bucket, meta) + // Prepare lifecycle expiration workers + es := newExpiryState(t.Context(), objAPI, 0) + globalExpiryState = es + + // Prepare object versions + obj := "obj-1" + // Simulate objects uploaded 30 hours ago + modTime := now.Add(-48 * time.Hour) + uuids := make([]uuid.UUID, 5) + for i := range uuids { + uuids[i] = uuid.UUID([16]byte{15: uint8(i + 1)}) + } + fivs := make([]FileInfo, 5) + objInfos := make([]ObjectInfo, 5) + objRetentionMeta := make(map[string]string) + objRetentionMeta[strings.ToLower(xhttp.AmzObjectLockMode)] = string(objectlock.RetCompliance) + // Set retain until date 12 hours into the future + objRetentionMeta[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(now.Add(12 * time.Hour)) + /* + objInfos: + version stack for obj-1 + v5 uuid-5 modTime + v4 uuid-4 modTime -1m + v3 uuid-3 modTime -2m + v2 uuid-2 modTime -3m + v1 uuid-1 modTime -4m + */ + for i := range 5 { + fivs[i] = FileInfo{ + Volume: bucket, + Name: obj, + VersionID: uuids[i].String(), + IsLatest: i == 0, + ModTime: modTime.Add(-1 * time.Duration(i) * time.Minute), + Size: 1 << 10, + NumVersions: 5, + } + objInfos[i] = fivs[i].ToObjectInfo(bucket, obj, true) + } + /* + lrObjInfos: objInfos with following modifications + version stack for obj-1 + v2 uuid-2 modTime -3m objRetentionMeta + */ + lrObjInfos := slices.Clone(objInfos) + lrObjInfos[3].UserDefined = objRetentionMeta + var lrWants []ObjectInfo + lrWants = append(lrWants, lrObjInfos[:4]...) + + /* + replObjInfos: objInfos with following modifications + version stack for obj-1 + v1 uuid-1 modTime -4m "VersionPurgeStatus: replication.VersionPurgePending" + */ + replObjInfos := slices.Clone(objInfos) + replObjInfos[4].VersionPurgeStatus = replication.VersionPurgePending + var replWants []ObjectInfo + replWants = append(replWants, replObjInfos[:3]...) + replWants = append(replWants, replObjInfos[4]) + + allVersExpObjInfos := slices.Clone(objInfos) + allVersExpObjInfos[0].UserTags = "del-all=true" + + replCfg := replication.Config{ + Rules: []replication.Rule{ + { + ID: "", + Status: "Enabled", + Priority: 1, + Destination: replication.Destination{ + ARN: "arn:minio:replication:::dest-bucket", + Bucket: "dest-bucket", + }, + }, + }, + } + lr := objectlock.Retention{ + Mode: objectlock.RetCompliance, + Validity: 12 * time.Hour, + LockEnabled: true, + } + + expiryWorker := func(wg *sync.WaitGroup, readyCh chan<- struct{}, taskCh <-chan expiryOp, gotExpired *[]ObjectToDelete) { + defer wg.Done() + // signal the calling goroutine that the worker is ready tor receive tasks + close(readyCh) + var expired []ObjectToDelete + for t := range taskCh { + switch v := t.(type) { + case noncurrentVersionsTask: + expired = append(expired, v.versions...) + case expiryTask: + expired = append(expired, ObjectToDelete{ + ObjectV: ObjectV{ + ObjectName: v.objInfo.Name, + VersionID: v.objInfo.VersionID, + }, + }) + } + } + if len(expired) > 0 { + *gotExpired = expired + } + } + tests := []struct { + replCfg replicationConfig + lr objectlock.Retention + objInfos []ObjectInfo + wants []ObjectInfo + wantExpired []ObjectToDelete + }{ + { + // With replication configured, version(s) with PENDING purge status + replCfg: replicationConfig{Config: &replCfg}, + objInfos: replObjInfos, + wants: replWants, + wantExpired: []ObjectToDelete{ + {ObjectV: ObjectV{ObjectName: obj, VersionID: objInfos[3].VersionID}}, + }, + }, + { + // With lock retention configured and version(s) with retention metadata + lr: lr, + objInfos: lrObjInfos, + wants: lrWants, + wantExpired: []ObjectToDelete{ + {ObjectV: ObjectV{ObjectName: obj, VersionID: objInfos[4].VersionID}}, + }, + }, + { + // With replication configured, but no versions with PENDING purge status + replCfg: replicationConfig{Config: &replCfg}, + objInfos: objInfos, + wants: objInfos[:3], + wantExpired: []ObjectToDelete{ + {ObjectV: ObjectV{ObjectName: obj, VersionID: objInfos[3].VersionID}}, + {ObjectV: ObjectV{ObjectName: obj, VersionID: objInfos[4].VersionID}}, + }, + }, + { + objInfos: allVersExpObjInfos, + wants: nil, + wantExpired: []ObjectToDelete{{ObjectV: ObjectV{ObjectName: obj, VersionID: allVersExpObjInfos[0].VersionID}}}, + }, + { + // When no versions are present, in practice this could be an object with only free versions + objInfos: nil, + wants: nil, + wantExpired: nil, + }, + } + for i, test := range tests { + t.Run(fmt.Sprintf("TestApplyNewerNoncurrentVersionsLimit-%d", i), func(t *testing.T) { + workers := []chan expiryOp{make(chan expiryOp)} + es.workers.Store(&workers) + workerReady := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + var gotExpired []ObjectToDelete + go expiryWorker(&wg, workerReady, workers[0], &gotExpired) + <-workerReady + + item := scannerItem{ + Path: obj, + bucket: bucket, + prefix: "", + objectName: obj, + lifeCycle: lc, + replication: test.replCfg, + } + + var ( + sizeS sizeSummary + gots []ObjectInfo + ) + item.applyActions(t.Context(), objAPI, test.objInfos, test.lr, &sizeS, func(oi ObjectInfo, sz, _ int64, _ *sizeSummary) { + if sz != 0 { + gots = append(gots, oi) + } + }) + + if len(gots) != len(test.wants) { + t.Fatalf("Expected %d objects but got %d", len(test.wants), len(gots)) + } + if slices.CompareFunc(gots, test.wants, func(g, w ObjectInfo) int { + if g.VersionID == w.VersionID { + return 0 + } + return -1 + }) != 0 { + t.Fatalf("Expected %v but got %v", test.wants, gots) + } + // verify the objects to be deleted + close(workers[0]) + wg.Wait() + if len(gotExpired) != len(test.wantExpired) { + t.Fatalf("Expected expiry of %d objects but got %d", len(test.wantExpired), len(gotExpired)) + } + if slices.CompareFunc(gotExpired, test.wantExpired, func(g, w ObjectToDelete) int { + if g.VersionID == w.VersionID { + return 0 + } + return -1 + }) != 0 { + t.Fatalf("Expected %v but got %v", test.wantExpired, gotExpired) + } + }) + } +} + +func TestEvalActionFromLifecycle(t *testing.T) { + // Tests cover only ExpiredObjectDeleteAllVersions and DelMarkerExpiration actions + numVersions := 4 + obj := ObjectInfo{ + Name: "foo", + ModTime: time.Now().Add(-31 * 24 * time.Hour), + Size: 100 << 20, + VersionID: uuid.New().String(), + IsLatest: true, + NumVersions: numVersions, + } + delMarker := ObjectInfo{ + Name: "foo-deleted", + ModTime: time.Now().Add(-61 * 24 * time.Hour), + Size: 0, + VersionID: uuid.New().String(), + IsLatest: true, + DeleteMarker: true, + NumVersions: numVersions, + } + + deleteAllILM := ` + + + 30 + true + + + Enabled + DeleteAllVersions + + ` + delMarkerILM := ` + + DelMarkerExpiration + + Enabled + + 60 + + + ` + deleteAllLc, err := lifecycle.ParseLifecycleConfig(strings.NewReader(deleteAllILM)) + if err != nil { + t.Fatalf("Failed to parse deleteAllILM test ILM policy %v", err) + } + delMarkerLc, err := lifecycle.ParseLifecycleConfig(strings.NewReader(delMarkerILM)) + if err != nil { + t.Fatalf("Failed to parse delMarkerILM test ILM policy %v", err) + } + tests := []struct { + ilm lifecycle.Lifecycle + retention *objectlock.Retention + obj ObjectInfo + want lifecycle.Action + }{ + { + // with object locking + ilm: *deleteAllLc, + retention: &objectlock.Retention{LockEnabled: true}, + obj: obj, + want: lifecycle.NoneAction, + }, + { + // without object locking + ilm: *deleteAllLc, + retention: &objectlock.Retention{}, + obj: obj, + want: lifecycle.DeleteAllVersionsAction, + }, + { + // with object locking + ilm: *delMarkerLc, + retention: &objectlock.Retention{LockEnabled: true}, + obj: delMarker, + want: lifecycle.NoneAction, + }, + { + // without object locking + ilm: *delMarkerLc, + retention: &objectlock.Retention{}, + obj: delMarker, + want: lifecycle.DelMarkerDeleteAllVersionsAction, + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("TestEvalAction-%d", i), func(t *testing.T) { + gotEvent := evalActionFromLifecycle(t.Context(), test.ilm, *test.retention, nil, test.obj) + if gotEvent.Action != test.want { + t.Fatalf("Expected %v but got %v", test.want, gotEvent.Action) + } + }) + } +} diff --git a/cmd/data-usage-cache.go b/cmd/data-usage-cache.go new file mode 100644 index 00000000..5752daae --- /dev/null +++ b/cmd/data-usage-cache.go @@ -0,0 +1,1323 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "maps" + "math/rand" + "net/http" + "path" + "path/filepath" + "sort" + "strings" + "time" + + "github.com/cespare/xxhash/v2" + "github.com/dustin/go-humanize" + "github.com/klauspost/compress/zstd" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/tinylib/msgp/msgp" + "github.com/valyala/bytebufferpool" +) + +//msgp:clearomitted + +//go:generate msgp -file $GOFILE -unexported + +// dataUsageHash is the hash type used. +type dataUsageHash string + +// sizeHistogramV1 is size histogram V1, which has fewer intervals esp. between +// 1024B and 1MiB. +type sizeHistogramV1 [dataUsageBucketLenV1]uint64 + +// sizeHistogram is a size histogram. +type sizeHistogram [dataUsageBucketLen]uint64 + +// versionsHistogram is a histogram of number of versions in an object. +type versionsHistogram [dataUsageVersionLen]uint64 + +type dataUsageEntry struct { + Children dataUsageHashMap `msg:"ch"` + // These fields do no include any children. + Size int64 `msg:"sz"` + Objects uint64 `msg:"os"` + Versions uint64 `msg:"vs"` // Versions that are not delete markers. + DeleteMarkers uint64 `msg:"dms"` + ObjSizes sizeHistogram `msg:"szs"` + ObjVersions versionsHistogram `msg:"vh"` + AllTierStats *allTierStats `msg:"ats,omitempty"` + Compacted bool `msg:"c"` +} + +// allTierStats is a collection of per-tier stats across all configured remote +// tiers. +type allTierStats struct { + Tiers map[string]tierStats `msg:"ts"` +} + +func newAllTierStats() *allTierStats { + return &allTierStats{ + Tiers: make(map[string]tierStats), + } +} + +func (ats *allTierStats) addSizes(tiers map[string]tierStats) { + for tier, st := range tiers { + ats.Tiers[tier] = ats.Tiers[tier].add(st) + } +} + +func (ats *allTierStats) merge(other *allTierStats) { + for tier, st := range other.Tiers { + ats.Tiers[tier] = ats.Tiers[tier].add(st) + } +} + +func (ats *allTierStats) clone() *allTierStats { + if ats == nil { + return nil + } + dst := *ats + dst.Tiers = make(map[string]tierStats, len(ats.Tiers)) + maps.Copy(dst.Tiers, ats.Tiers) + return &dst +} + +func (ats *allTierStats) populateStats(stats map[string]madmin.TierStats) { + if ats == nil { + return + } + + // Update stats for tiers as they become available. + for tier, st := range ats.Tiers { + stats[tier] = madmin.TierStats{ + TotalSize: st.TotalSize, + NumVersions: st.NumVersions, + NumObjects: st.NumObjects, + } + } +} + +// tierStats holds per-tier stats of a remote tier. +type tierStats struct { + TotalSize uint64 `msg:"ts"` + NumVersions int `msg:"nv"` + NumObjects int `msg:"no"` +} + +func (ts tierStats) add(u tierStats) tierStats { + return tierStats{ + TotalSize: ts.TotalSize + u.TotalSize, + NumVersions: ts.NumVersions + u.NumVersions, + NumObjects: ts.NumObjects + u.NumObjects, + } +} + +//msgp:encode ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7 +//msgp:marshal ignore dataUsageEntryV2 dataUsageEntryV3 dataUsageEntryV4 dataUsageEntryV5 dataUsageEntryV6 dataUsageEntryV7 + +//msgp:tuple dataUsageEntryV2 +type dataUsageEntryV2 struct { + // These fields do no include any children. + Size int64 + Objects uint64 + ObjSizes sizeHistogram + Children dataUsageHashMap +} + +//msgp:tuple dataUsageEntryV3 +type dataUsageEntryV3 struct { + // These fields do no include any children. + Size int64 + Objects uint64 + ObjSizes sizeHistogram + Children dataUsageHashMap +} + +//msgp:tuple dataUsageEntryV4 +type dataUsageEntryV4 struct { + Children dataUsageHashMap + // These fields do no include any children. + Size int64 + Objects uint64 + ObjSizes sizeHistogram +} + +//msgp:tuple dataUsageEntryV5 +type dataUsageEntryV5 struct { + Children dataUsageHashMap + // These fields do no include any children. + Size int64 + Objects uint64 + Versions uint64 // Versions that are not delete markers. + ObjSizes sizeHistogram + Compacted bool +} + +//msgp:tuple dataUsageEntryV6 +type dataUsageEntryV6 struct { + Children dataUsageHashMap + // These fields do no include any children. + Size int64 + Objects uint64 + Versions uint64 // Versions that are not delete markers. + ObjSizes sizeHistogram + Compacted bool +} + +type dataUsageEntryV7 struct { + Children dataUsageHashMap `msg:"ch"` + // These fields do no include any children. + Size int64 `msg:"sz"` + Objects uint64 `msg:"os"` + Versions uint64 `msg:"vs"` // Versions that are not delete markers. + DeleteMarkers uint64 `msg:"dms"` + ObjSizes sizeHistogramV1 `msg:"szs"` + ObjVersions versionsHistogram `msg:"vh"` + AllTierStats *allTierStats `msg:"ats,omitempty"` + Compacted bool `msg:"c"` +} + +// dataUsageCache contains a cache of data usage entries latest version. +type dataUsageCache struct { + Info dataUsageCacheInfo + Cache map[string]dataUsageEntry +} + +//msgp:encode ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7 +//msgp:marshal ignore dataUsageCacheV2 dataUsageCacheV3 dataUsageCacheV4 dataUsageCacheV5 dataUsageCacheV6 dataUsageCacheV7 + +// dataUsageCacheV2 contains a cache of data usage entries version 2. +type dataUsageCacheV2 struct { + Info dataUsageCacheInfo + Cache map[string]dataUsageEntryV2 +} + +// dataUsageCacheV3 contains a cache of data usage entries version 3. +type dataUsageCacheV3 struct { + Info dataUsageCacheInfo + Cache map[string]dataUsageEntryV3 +} + +// dataUsageCacheV4 contains a cache of data usage entries version 4. +type dataUsageCacheV4 struct { + Info dataUsageCacheInfo + Cache map[string]dataUsageEntryV4 +} + +// dataUsageCacheV5 contains a cache of data usage entries version 5. +type dataUsageCacheV5 struct { + Info dataUsageCacheInfo + Cache map[string]dataUsageEntryV5 +} + +// dataUsageCacheV6 contains a cache of data usage entries version 6. +type dataUsageCacheV6 struct { + Info dataUsageCacheInfo + Cache map[string]dataUsageEntryV6 +} + +// dataUsageCacheV7 contains a cache of data usage entries version 7. +type dataUsageCacheV7 struct { + Info dataUsageCacheInfo + Cache map[string]dataUsageEntryV7 +} + +//msgp:ignore dataUsageEntryInfo +type dataUsageEntryInfo struct { + Name string + Parent string + Entry dataUsageEntry +} + +type dataUsageCacheInfo struct { + // Name of the bucket. Also root element. + Name string + NextCycle uint32 + LastUpdate time.Time + // indicates if the disk is being healed and scanner + // should skip healing the disk + SkipHealing bool + + // Active lifecycle, if any on the bucket + lifeCycle *lifecycle.Lifecycle `msg:"-"` + + // optional updates channel. + // If set updates will be sent regularly to this channel. + // Will not be closed when returned. + updates chan<- dataUsageEntry `msg:"-"` + replication replicationConfig `msg:"-"` +} + +func (e *dataUsageEntry) addSizes(summary sizeSummary) { + e.Size += summary.totalSize + e.Versions += summary.versions + e.DeleteMarkers += summary.deleteMarkers + e.ObjSizes.add(summary.totalSize) + e.ObjVersions.add(summary.versions) + + if len(summary.tiers) != 0 { + if e.AllTierStats == nil { + e.AllTierStats = newAllTierStats() + } + e.AllTierStats.addSizes(summary.tiers) + } +} + +// merge other data usage entry into this, excluding children. +func (e *dataUsageEntry) merge(other dataUsageEntry) { + e.Objects += other.Objects + e.Versions += other.Versions + e.DeleteMarkers += other.DeleteMarkers + e.Size += other.Size + + for i, v := range other.ObjSizes[:] { + e.ObjSizes[i] += v + } + + for i, v := range other.ObjVersions[:] { + e.ObjVersions[i] += v + } + + if other.AllTierStats != nil && len(other.AllTierStats.Tiers) != 0 { + if e.AllTierStats == nil { + e.AllTierStats = newAllTierStats() + } + e.AllTierStats.merge(other.AllTierStats) + } +} + +// mod returns true if the hash mod cycles == cycle. +// If cycles is 0 false is always returned. +// If cycles is 1 true is always returned (as expected). +func (h dataUsageHash) mod(cycle uint32, cycles uint32) bool { + if cycles <= 1 { + return cycles == 1 + } + return uint32(xxhash.Sum64String(string(h)))%cycles == cycle%cycles +} + +// modAlt returns true if the hash mod cycles == cycle. +// This is out of sync with mod. +// If cycles is 0 false is always returned. +// If cycles is 1 true is always returned (as expected). +func (h dataUsageHash) modAlt(cycle uint32, cycles uint32) bool { + if cycles <= 1 { + return cycles == 1 + } + return uint32(xxhash.Sum64String(string(h))>>32)%(cycles) == cycle%cycles +} + +// addChild will add a child based on its hash. +// If it already exists it will not be added again. +func (e *dataUsageEntry) addChild(hash dataUsageHash) { + if _, ok := e.Children[hash.Key()]; ok { + return + } + if e.Children == nil { + e.Children = make(dataUsageHashMap, 1) + } + e.Children[hash.Key()] = struct{}{} +} + +// Create a clone of the entry. +func (e dataUsageEntry) clone() dataUsageEntry { + // We operate on a copy from the receiver. + if e.Children != nil { + ch := make(dataUsageHashMap, len(e.Children)) + maps.Copy(ch, e.Children) + e.Children = ch + } + + if e.AllTierStats != nil { + e.AllTierStats = e.AllTierStats.clone() + } + return e +} + +// find a path in the cache. +// Returns nil if not found. +func (d *dataUsageCache) find(path string) *dataUsageEntry { + due, ok := d.Cache[hashPath(path).Key()] + if !ok { + return nil + } + return &due +} + +// isCompacted returns whether an entry is compacted. +// Returns false if not found. +func (d *dataUsageCache) isCompacted(h dataUsageHash) bool { + due, ok := d.Cache[h.Key()] + if !ok { + return false + } + return due.Compacted +} + +// findChildrenCopy returns a copy of the children of the supplied hash. +func (d *dataUsageCache) findChildrenCopy(h dataUsageHash) dataUsageHashMap { + ch := d.Cache[h.String()].Children + res := make(dataUsageHashMap, len(ch)) + for k := range ch { + res[k] = struct{}{} + } + return res +} + +// searchParent will search for the parent of h. +// This is an O(N*N) operation if there is no parent or it cannot be guessed. +func (d *dataUsageCache) searchParent(h dataUsageHash) *dataUsageHash { + want := h.Key() + if idx := strings.LastIndexByte(want, '/'); idx >= 0 { + if v := d.find(want[:idx]); v != nil { + _, ok := v.Children[want] + if ok { + found := hashPath(want[:idx]) + return &found + } + } + } + for k, v := range d.Cache { + _, ok := v.Children[want] + if ok { + found := dataUsageHash(k) + return &found + } + } + return nil +} + +// deleteRecursive will delete an entry recursively, but not change its parent. +func (d *dataUsageCache) deleteRecursive(h dataUsageHash) { + if existing, ok := d.Cache[h.String()]; ok { + // Delete first if there should be a loop. + delete(d.Cache, h.Key()) + for child := range existing.Children { + d.deleteRecursive(dataUsageHash(child)) + } + } +} + +// dui converts the flattened version of the path to madmin.DataUsageInfo. +// As a side effect d will be flattened, use a clone if this is not ok. +func (d *dataUsageCache) dui(path string, buckets []BucketInfo) DataUsageInfo { + e := d.find(path) + if e == nil { + // No entry found, return empty. + return DataUsageInfo{} + } + flat := d.flatten(*e) + dui := DataUsageInfo{ + LastUpdate: d.Info.LastUpdate, + ObjectsTotalCount: flat.Objects, + VersionsTotalCount: flat.Versions, + DeleteMarkersTotalCount: flat.DeleteMarkers, + ObjectsTotalSize: uint64(flat.Size), + BucketsCount: uint64(len(e.Children)), + BucketsUsage: d.bucketsUsageInfo(buckets), + TierStats: d.tiersUsageInfo(buckets), + } + return dui +} + +// replace will add or replace an entry in the cache. +// If a parent is specified it will be added to that if not already there. +// If the parent does not exist, it will be added. +func (d *dataUsageCache) replace(path, parent string, e dataUsageEntry) { + hash := hashPath(path) + if d.Cache == nil { + d.Cache = make(map[string]dataUsageEntry, 100) + } + d.Cache[hash.Key()] = e + if parent != "" { + phash := hashPath(parent) + p := d.Cache[phash.Key()] + p.addChild(hash) + d.Cache[phash.Key()] = p + } +} + +// replaceHashed add or replaces an entry to the cache based on its hash. +// If a parent is specified it will be added to that if not already there. +// If the parent does not exist, it will be added. +func (d *dataUsageCache) replaceHashed(hash dataUsageHash, parent *dataUsageHash, e dataUsageEntry) { + if d.Cache == nil { + d.Cache = make(map[string]dataUsageEntry, 100) + } + d.Cache[hash.Key()] = e + if parent != nil { + p := d.Cache[parent.Key()] + p.addChild(hash) + d.Cache[parent.Key()] = p + } +} + +// copyWithChildren will copy entry with hash from src if it exists along with any children. +// If a parent is specified it will be added to that if not already there. +// If the parent does not exist, it will be added. +func (d *dataUsageCache) copyWithChildren(src *dataUsageCache, hash dataUsageHash, parent *dataUsageHash) { + if d.Cache == nil { + d.Cache = make(map[string]dataUsageEntry, 100) + } + e, ok := src.Cache[hash.String()] + if !ok { + return + } + d.Cache[hash.Key()] = e + for ch := range e.Children { + if ch == hash.Key() { + scannerLogIf(GlobalContext, errors.New("dataUsageCache.copyWithChildren: Circular reference")) + return + } + d.copyWithChildren(src, dataUsageHash(ch), &hash) + } + if parent != nil { + p := d.Cache[parent.Key()] + p.addChild(hash) + d.Cache[parent.Key()] = p + } +} + +// reduceChildrenOf will reduce the recursive number of children to the limit +// by compacting the children with the least number of objects. +func (d *dataUsageCache) reduceChildrenOf(path dataUsageHash, limit int, compactSelf bool) { + e, ok := d.Cache[path.Key()] + if !ok { + return + } + if e.Compacted { + return + } + // If direct children have more, compact all. + if len(e.Children) > limit && compactSelf { + flat := d.sizeRecursive(path.Key()) + flat.Compacted = true + d.deleteRecursive(path) + d.replaceHashed(path, nil, *flat) + return + } + total := d.totalChildrenRec(path.Key()) + if total < limit { + return + } + + // Appears to be printed with _MINIO_SERVER_DEBUG=off + // console.Debugf(" %d children found, compacting %v\n", total, path) + + leaves := make([]struct { + objects uint64 + path dataUsageHash + }, total) + // Collect current leaves that have children. + leaves = leaves[:0] + remove := total - limit + var add func(path dataUsageHash) + add = func(path dataUsageHash) { + e, ok := d.Cache[path.Key()] + if !ok { + return + } + if len(e.Children) == 0 { + return + } + sz := d.sizeRecursive(path.Key()) + leaves = append(leaves, struct { + objects uint64 + path dataUsageHash + }{objects: sz.Objects, path: path}) + for ch := range e.Children { + add(dataUsageHash(ch)) + } + } + + // Add path recursively. + add(path) + sort.Slice(leaves, func(i, j int) bool { + return leaves[i].objects < leaves[j].objects + }) + for remove > 0 && len(leaves) > 0 { + // Remove top entry. + e := leaves[0] + candidate := e.path + if candidate == path && !compactSelf { + // We should be the biggest, + // if we cannot compact ourself, we are done. + break + } + removing := d.totalChildrenRec(candidate.Key()) + flat := d.sizeRecursive(candidate.Key()) + if flat == nil { + leaves = leaves[1:] + continue + } + // Appears to be printed with _MINIO_SERVER_DEBUG=off + // console.Debugf("compacting %v, removing %d children\n", candidate, removing) + + flat.Compacted = true + d.deleteRecursive(candidate) + d.replaceHashed(candidate, nil, *flat) + + // Remove top entry and subtract removed children. + remove -= removing + leaves = leaves[1:] + } +} + +// forceCompact will force compact the cache of the top entry. +// If the number of children is more than limit*100, it will compact self. +// When above the limit a cleanup will also be performed to remove any possible abandoned entries. +func (d *dataUsageCache) forceCompact(limit int) { + if d == nil || len(d.Cache) <= limit { + return + } + top := hashPath(d.Info.Name).Key() + topE := d.find(top) + if topE == nil { + scannerLogIf(GlobalContext, errors.New("forceCompact: root not found")) + return + } + // If off by 2 orders of magnitude, compact self and log error. + if len(topE.Children) > dataScannerForceCompactAtFolders { + // If we still have too many children, compact self. + scannerLogOnceIf(GlobalContext, fmt.Errorf("forceCompact: %q has %d children. Force compacting. Expect reduced scanner performance", d.Info.Name, len(topE.Children)), d.Info.Name) + d.reduceChildrenOf(hashPath(d.Info.Name), limit, true) + } + if len(d.Cache) <= limit { + return + } + + // Check for abandoned entries. + found := make(map[string]struct{}, len(d.Cache)) + + // Mark all children recursively + var mark func(entry dataUsageEntry) + mark = func(entry dataUsageEntry) { + for k := range entry.Children { + found[k] = struct{}{} + if ch, ok := d.Cache[k]; ok { + mark(ch) + } + } + } + found[top] = struct{}{} + mark(*topE) + + // Delete all entries not found. + for k := range d.Cache { + if _, ok := found[k]; !ok { + delete(d.Cache, k) + } + } +} + +// StringAll returns a detailed string representation of all entries in the cache. +func (d *dataUsageCache) StringAll() string { + // Remove bloom filter from print. + s := fmt.Sprintf("info:%+v\n", d.Info) + for k, v := range d.Cache { + s += fmt.Sprintf("\t%v: %+v\n", k, v) + } + return strings.TrimSpace(s) +} + +// String returns a human readable representation of the string. +func (h dataUsageHash) String() string { + return string(h) +} + +// Key returns the key. +func (h dataUsageHash) Key() string { + return string(h) +} + +func (d *dataUsageCache) flattenChildrens(root dataUsageEntry) (m map[string]dataUsageEntry) { + m = make(map[string]dataUsageEntry) + for id := range root.Children { + e := d.Cache[id] + if len(e.Children) > 0 { + e = d.flatten(e) + } + m[id] = e + } + return m +} + +// flatten all children of the root into the root element and return it. +func (d *dataUsageCache) flatten(root dataUsageEntry) dataUsageEntry { + for id := range root.Children { + e := d.Cache[id] + if len(e.Children) > 0 { + e = d.flatten(e) + } + root.merge(e) + } + root.Children = nil + return root +} + +// add a size to the histogram. +func (h *sizeHistogram) add(size int64) { + // Fetch the histogram interval corresponding + // to the passed object size. + for i, interval := range ObjectsHistogramIntervals[:] { + if size >= interval.start && size <= interval.end { + h[i]++ + break + } + } +} + +// mergeV1 is used to migrate data usage cache from sizeHistogramV1 to +// sizeHistogram +func (h *sizeHistogram) mergeV1(v sizeHistogramV1) { + var oidx, nidx int + for oidx < len(v) { + intOld, intNew := ObjectsHistogramIntervalsV1[oidx], ObjectsHistogramIntervals[nidx] + // skip intervals that aren't common to both histograms + if intOld.start != intNew.start || intOld.end != intNew.end { + nidx++ + continue + } + h[nidx] += v[oidx] + oidx++ + nidx++ + } +} + +// toMap returns the map to a map[string]uint64. +func (h *sizeHistogram) toMap() map[string]uint64 { + res := make(map[string]uint64, dataUsageBucketLen) + var splCount uint64 + for i, count := range h { + szInt := ObjectsHistogramIntervals[i] + switch { + case humanize.KiByte == szInt.start && szInt.end == humanize.MiByte-1: + // spl interval: [1024B, 1MiB) + res[szInt.name] = splCount + case humanize.KiByte <= szInt.start && szInt.end <= humanize.MiByte-1: + // intervals that fall within the spl interval above; they + // appear earlier in this array of intervals, see + // ObjectsHistogramIntervals + splCount += count + fallthrough + default: + res[szInt.name] = count + } + } + return res +} + +// add a version count to the histogram. +func (h *versionsHistogram) add(versions uint64) { + // Fetch the histogram interval corresponding + // to the passed object size. + for i, interval := range ObjectsVersionCountIntervals[:] { + if versions >= uint64(interval.start) && versions <= uint64(interval.end) { + h[i]++ + break + } + } +} + +// toMap returns the map to a map[string]uint64. +func (h *versionsHistogram) toMap() map[string]uint64 { + res := make(map[string]uint64, dataUsageVersionLen) + for i, count := range h { + res[ObjectsVersionCountIntervals[i].name] = count + } + return res +} + +func (d *dataUsageCache) tiersUsageInfo(buckets []BucketInfo) *allTierStats { + dst := newAllTierStats() + for _, bucket := range buckets { + e := d.find(bucket.Name) + if e == nil { + continue + } + flat := d.flatten(*e) + if flat.AllTierStats == nil { + continue + } + dst.merge(flat.AllTierStats) + } + if len(dst.Tiers) == 0 { + return nil + } + return dst +} + +// bucketsUsageInfo returns the buckets usage info as a map, with +// key as bucket name +func (d *dataUsageCache) bucketsUsageInfo(buckets []BucketInfo) map[string]BucketUsageInfo { + dst := make(map[string]BucketUsageInfo, len(buckets)) + for _, bucket := range buckets { + e := d.find(bucket.Name) + if e == nil { + continue + } + flat := d.flatten(*e) + bui := BucketUsageInfo{ + Size: uint64(flat.Size), + VersionsCount: flat.Versions, + ObjectsCount: flat.Objects, + DeleteMarkersCount: flat.DeleteMarkers, + ObjectSizesHistogram: flat.ObjSizes.toMap(), + ObjectVersionsHistogram: flat.ObjVersions.toMap(), + } + dst[bucket.Name] = bui + } + return dst +} + +// sizeRecursive returns the path as a flattened entry. +func (d *dataUsageCache) sizeRecursive(path string) *dataUsageEntry { + root := d.find(path) + if root == nil || len(root.Children) == 0 { + return root + } + flat := d.flatten(*root) + return &flat +} + +// totalChildrenRec returns the total number of children recorded. +func (d *dataUsageCache) totalChildrenRec(path string) int { + root := d.find(path) + if root == nil || len(root.Children) == 0 { + return 0 + } + n := len(root.Children) + for ch := range root.Children { + n += d.totalChildrenRec(ch) + } + return n +} + +// root returns the root of the cache. +func (d *dataUsageCache) root() *dataUsageEntry { + return d.find(d.Info.Name) +} + +// rootHash returns the root of the cache. +func (d *dataUsageCache) rootHash() dataUsageHash { + return hashPath(d.Info.Name) +} + +// clone returns a copy of the cache with no references to the existing. +func (d *dataUsageCache) clone() dataUsageCache { + clone := dataUsageCache{ + Info: d.Info, + Cache: make(map[string]dataUsageEntry, len(d.Cache)), + } + for k, v := range d.Cache { + clone.Cache[k] = v.clone() + } + return clone +} + +// merge root of other into d. +// children of root will be flattened before being merged. +// Last update time will be set to the last updated. +func (d *dataUsageCache) merge(other dataUsageCache) { + existingRoot := d.root() + otherRoot := other.root() + if existingRoot == nil && otherRoot == nil { + return + } + if otherRoot == nil { + return + } + if existingRoot == nil { + *d = other.clone() + return + } + if other.Info.LastUpdate.After(d.Info.LastUpdate) { + d.Info.LastUpdate = other.Info.LastUpdate + } + existingRoot.merge(*otherRoot) + eHash := d.rootHash() + for key := range otherRoot.Children { + entry := other.Cache[key] + flat := other.flatten(entry) + existing := d.Cache[key] + // If not found, merging simply adds. + existing.merge(flat) + d.replaceHashed(dataUsageHash(key), &eHash, existing) + } +} + +type objectIO interface { + GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (reader *GetObjectReader, err error) + PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) +} + +// load the cache content with name from minioMetaBackgroundOpsBucket. +// Only backend errors are returned as errors. +// The loader is optimistic and has no locking, but tries 5 times before giving up. +// If the object is not found, a nil error with empty data usage cache is returned. +func (d *dataUsageCache) load(ctx context.Context, store objectIO, name string) error { + // By default, empty data usage cache + *d = dataUsageCache{} + + load := func(name string, timeout time.Duration) (bool, error) { + // Abandon if more than time.Minute, so we don't hold up scanner. + // drive timeout by default is 2 minutes, we do not need to wait longer. + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + r, err := store.GetObjectNInfo(ctx, minioMetaBucket, pathJoin(bucketMetaPrefix, name), nil, http.Header{}, ObjectOptions{NoLock: true}) + if err != nil { + switch err.(type) { + case ObjectNotFound, BucketNotFound: + r, err = store.GetObjectNInfo(ctx, dataUsageBucket, name, nil, http.Header{}, ObjectOptions{NoLock: true}) + if err != nil { + switch err.(type) { + case ObjectNotFound, BucketNotFound: + return false, nil + case InsufficientReadQuorum, StorageErr: + return true, nil + } + return false, err + } + err = d.deserialize(r) + r.Close() + return err != nil, nil + case InsufficientReadQuorum, StorageErr: + return true, nil + } + return false, err + } + err = d.deserialize(r) + r.Close() + return err != nil, nil + } + + // Caches are read+written without locks, + retries := 0 + for retries < 5 { + retry, err := load(name, time.Minute) + if err != nil { + return toObjectErr(err, dataUsageBucket, name) + } + if !retry { + break + } + retry, err = load(name+".bkp", 30*time.Second) + if err == nil && !retry { + // Only return when we have valid data from the backup + break + } + retries++ + time.Sleep(time.Duration(rand.Int63n(int64(time.Second)))) + } + + if retries == 5 { + scannerLogOnceIf(ctx, fmt.Errorf("maximum retry reached to load the data usage cache `%s`", name), "retry-loading-data-usage-cache") + } + + return nil +} + +// Maximum running concurrent saves on server. +var maxConcurrentScannerSaves = make(chan struct{}, 4) + +// save the content of the cache to minioMetaBackgroundOpsBucket with the provided name. +// Note that no locking is done when saving. +func (d *dataUsageCache) save(ctx context.Context, store objectIO, name string) error { + select { + case <-ctx.Done(): + return ctx.Err() + case maxConcurrentScannerSaves <- struct{}{}: + } + + buf := bytebufferpool.Get() + defer func() { + <-maxConcurrentScannerSaves + buf.Reset() + bytebufferpool.Put(buf) + }() + + if err := d.serializeTo(buf); err != nil { + return err + } + + save := func(name string, timeout time.Duration) error { + // Abandon if more than a minute, so we don't hold up scanner. + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + return saveConfig(ctx, store, pathJoin(bucketMetaPrefix, name), buf.Bytes()) + } + defer save(name+".bkp", 5*time.Second) // Keep a backup as well + + // drive timeout by default is 2 minutes, we do not need to wait longer. + return save(name, time.Minute) +} + +// dataUsageCacheVer indicates the cache version. +// Bumping the cache version will drop data from previous versions +// and write new data with the new version. +const ( + dataUsageCacheVerCurrent = 8 + dataUsageCacheVerV7 = 7 + dataUsageCacheVerV6 = 6 + dataUsageCacheVerV5 = 5 + dataUsageCacheVerV4 = 4 + dataUsageCacheVerV3 = 3 + dataUsageCacheVerV2 = 2 + dataUsageCacheVerV1 = 1 +) + +// serialize the contents of the cache. +func (d *dataUsageCache) serializeTo(dst io.Writer) error { + // Add version and compress. + _, err := dst.Write([]byte{dataUsageCacheVerCurrent}) + if err != nil { + return err + } + enc, err := zstd.NewWriter(dst, + zstd.WithEncoderLevel(zstd.SpeedFastest), + zstd.WithWindowSize(1<<20), + zstd.WithEncoderConcurrency(2)) + if err != nil { + return err + } + mEnc := msgp.NewWriter(enc) + err = d.EncodeMsg(mEnc) + if err != nil { + return err + } + err = mEnc.Flush() + if err != nil { + return err + } + err = enc.Close() + if err != nil { + return err + } + return nil +} + +// deserialize the supplied byte slice into the cache. +func (d *dataUsageCache) deserialize(r io.Reader) error { + var b [1]byte + n, _ := r.Read(b[:]) + if n != 1 { + return io.ErrUnexpectedEOF + } + ver := int(b[0]) + switch ver { + case dataUsageCacheVerV1: + return errors.New("cache version deprecated (will autoupdate)") + case dataUsageCacheVerV2: + // Zstd compressed. + dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2)) + if err != nil { + return err + } + defer dec.Close() + + dold := &dataUsageCacheV2{} + if err = dold.DecodeMsg(msgp.NewReader(dec)); err != nil { + return err + } + d.Info = dold.Info + d.Cache = make(map[string]dataUsageEntry, len(dold.Cache)) + for k, v := range dold.Cache { + d.Cache[k] = dataUsageEntry{ + Size: v.Size, + Objects: v.Objects, + ObjSizes: v.ObjSizes, + Children: v.Children, + Compacted: len(v.Children) == 0 && k != d.Info.Name, + } + } + return nil + case dataUsageCacheVerV3: + // Zstd compressed. + dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2)) + if err != nil { + return err + } + defer dec.Close() + dold := &dataUsageCacheV3{} + if err = dold.DecodeMsg(msgp.NewReader(dec)); err != nil { + return err + } + d.Info = dold.Info + d.Cache = make(map[string]dataUsageEntry, len(dold.Cache)) + for k, v := range dold.Cache { + due := dataUsageEntry{ + Size: v.Size, + Objects: v.Objects, + ObjSizes: v.ObjSizes, + Children: v.Children, + } + due.Compacted = len(due.Children) == 0 && k != d.Info.Name + + d.Cache[k] = due + } + return nil + case dataUsageCacheVerV4: + // Zstd compressed. + dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2)) + if err != nil { + return err + } + defer dec.Close() + dold := &dataUsageCacheV4{} + if err = dold.DecodeMsg(msgp.NewReader(dec)); err != nil { + return err + } + d.Info = dold.Info + d.Cache = make(map[string]dataUsageEntry, len(dold.Cache)) + for k, v := range dold.Cache { + due := dataUsageEntry{ + Size: v.Size, + Objects: v.Objects, + ObjSizes: v.ObjSizes, + Children: v.Children, + } + due.Compacted = len(due.Children) == 0 && k != d.Info.Name + + d.Cache[k] = due + } + return nil + case dataUsageCacheVerV5: + // Zstd compressed. + dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2)) + if err != nil { + return err + } + defer dec.Close() + dold := &dataUsageCacheV5{} + if err = dold.DecodeMsg(msgp.NewReader(dec)); err != nil { + return err + } + d.Info = dold.Info + d.Cache = make(map[string]dataUsageEntry, len(dold.Cache)) + for k, v := range dold.Cache { + due := dataUsageEntry{ + Size: v.Size, + Objects: v.Objects, + ObjSizes: v.ObjSizes, + Children: v.Children, + } + due.Compacted = len(due.Children) == 0 && k != d.Info.Name + + d.Cache[k] = due + } + return nil + case dataUsageCacheVerV6: + // Zstd compressed. + dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2)) + if err != nil { + return err + } + defer dec.Close() + dold := &dataUsageCacheV6{} + if err = dold.DecodeMsg(msgp.NewReader(dec)); err != nil { + return err + } + d.Info = dold.Info + d.Cache = make(map[string]dataUsageEntry, len(dold.Cache)) + for k, v := range dold.Cache { + due := dataUsageEntry{ + Children: v.Children, + Size: v.Size, + Objects: v.Objects, + Versions: v.Versions, + ObjSizes: v.ObjSizes, + Compacted: v.Compacted, + } + d.Cache[k] = due + } + return nil + case dataUsageCacheVerV7: + // Zstd compressed. + dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2)) + if err != nil { + return err + } + defer dec.Close() + dold := &dataUsageCacheV7{} + if err = dold.DecodeMsg(msgp.NewReader(dec)); err != nil { + return err + } + d.Info = dold.Info + d.Cache = make(map[string]dataUsageEntry, len(dold.Cache)) + for k, v := range dold.Cache { + var szHist sizeHistogram + szHist.mergeV1(v.ObjSizes) + d.Cache[k] = dataUsageEntry{ + Children: v.Children, + Size: v.Size, + Objects: v.Objects, + Versions: v.Versions, + ObjSizes: szHist, + Compacted: v.Compacted, + } + } + + return nil + case dataUsageCacheVerCurrent: + // Zstd compressed. + dec, err := zstd.NewReader(r, zstd.WithDecoderConcurrency(2)) + if err != nil { + return err + } + defer dec.Close() + return d.DecodeMsg(msgp.NewReader(dec)) + default: + return fmt.Errorf("dataUsageCache: unknown version: %d", ver) + } +} + +// Trim this from start+end of hashes. +var hashPathCutSet = dataUsageRoot + +func init() { + if dataUsageRoot != string(filepath.Separator) { + hashPathCutSet = dataUsageRoot + string(filepath.Separator) + } +} + +// hashPath calculates a hash of the provided string. +func hashPath(data string) dataUsageHash { + if data != dataUsageRoot { + data = strings.Trim(data, hashPathCutSet) + } + return dataUsageHash(path.Clean(data)) +} + +//msgp:ignore dataUsageHashMap +type dataUsageHashMap map[string]struct{} + +// DecodeMsg implements msgp.Decodable +func (z *dataUsageHashMap) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return err + } + if zb0002 == 0 { + *z = nil + return err + } + *z = make(dataUsageHashMap, zb0002) + for i := uint32(0); i < zb0002; i++ { + { + var zb0003 string + zb0003, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err) + return err + } + (*z)[zb0003] = struct{}{} + } + } + return err +} + +// EncodeMsg implements msgp.Encodable +func (z dataUsageHashMap) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteArrayHeader(uint32(len(z))) + if err != nil { + err = msgp.WrapError(err) + return err + } + for zb0004 := range z { + err = en.WriteString(zb0004) + if err != nil { + err = msgp.WrapError(err, zb0004) + return err + } + } + return err +} + +// MarshalMsg implements msgp.Marshaler +func (z dataUsageHashMap) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendArrayHeader(o, uint32(len(z))) + for zb0004 := range z { + o = msgp.AppendString(o, zb0004) + } + return o, err +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *dataUsageHashMap) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return o, err + } + if zb0002 == 0 { + *z = nil + return bts, nil + } + *z = make(dataUsageHashMap, zb0002) + for i := uint32(0); i < zb0002; i++ { + { + var zb0003 string + zb0003, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return o, err + } + (*z)[zb0003] = struct{}{} + } + } + o = bts + return o, err +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z dataUsageHashMap) Msgsize() (s int) { + s = msgp.ArrayHeaderSize + for zb0004 := range z { + s += msgp.StringPrefixSize + len(zb0004) + } + return s +} + +//msgp:encode ignore currentScannerCycle +//msgp:decode ignore currentScannerCycle + +type currentScannerCycle struct { + current uint64 + next uint64 + started time.Time + cycleCompleted []time.Time +} + +// clone returns a clone. +func (z currentScannerCycle) clone() currentScannerCycle { + z.cycleCompleted = append(make([]time.Time, 0, len(z.cycleCompleted)), z.cycleCompleted...) + return z +} diff --git a/cmd/data-usage-cache_gen.go b/cmd/data-usage-cache_gen.go new file mode 100644 index 00000000..df9d34a1 --- /dev/null +++ b/cmd/data-usage-cache_gen.go @@ -0,0 +1,3698 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "time" + + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *allTierStats) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ts": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Tiers") + return + } + if z.Tiers == nil { + z.Tiers = make(map[string]tierStats, zb0002) + } else if len(z.Tiers) > 0 { + clear(z.Tiers) + } + for zb0002 > 0 { + zb0002-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Tiers") + return + } + var za0002 tierStats + var zb0003 uint32 + zb0003, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001) + return + } + for zb0003 > 0 { + zb0003-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001) + return + } + switch msgp.UnsafeString(field) { + case "ts": + za0002.TotalSize, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001, "TotalSize") + return + } + case "nv": + za0002.NumVersions, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001, "NumVersions") + return + } + case "no": + za0002.NumObjects, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001, "NumObjects") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001) + return + } + } + } + z.Tiers[za0001] = za0002 + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *allTierStats) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "ts" + err = en.Append(0x81, 0xa2, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.Tiers))) + if err != nil { + err = msgp.WrapError(err, "Tiers") + return + } + for za0001, za0002 := range z.Tiers { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "Tiers") + return + } + // map header, size 3 + // write "ts" + err = en.Append(0x83, 0xa2, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteUint64(za0002.TotalSize) + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001, "TotalSize") + return + } + // write "nv" + err = en.Append(0xa2, 0x6e, 0x76) + if err != nil { + return + } + err = en.WriteInt(za0002.NumVersions) + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001, "NumVersions") + return + } + // write "no" + err = en.Append(0xa2, 0x6e, 0x6f) + if err != nil { + return + } + err = en.WriteInt(za0002.NumObjects) + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001, "NumObjects") + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *allTierStats) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "ts" + o = append(o, 0x81, 0xa2, 0x74, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.Tiers))) + for za0001, za0002 := range z.Tiers { + o = msgp.AppendString(o, za0001) + // map header, size 3 + // string "ts" + o = append(o, 0x83, 0xa2, 0x74, 0x73) + o = msgp.AppendUint64(o, za0002.TotalSize) + // string "nv" + o = append(o, 0xa2, 0x6e, 0x76) + o = msgp.AppendInt(o, za0002.NumVersions) + // string "no" + o = append(o, 0xa2, 0x6e, 0x6f) + o = msgp.AppendInt(o, za0002.NumObjects) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *allTierStats) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ts": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tiers") + return + } + if z.Tiers == nil { + z.Tiers = make(map[string]tierStats, zb0002) + } else if len(z.Tiers) > 0 { + clear(z.Tiers) + } + for zb0002 > 0 { + var za0002 tierStats + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tiers") + return + } + var zb0003 uint32 + zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001) + return + } + for zb0003 > 0 { + zb0003-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001) + return + } + switch msgp.UnsafeString(field) { + case "ts": + za0002.TotalSize, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001, "TotalSize") + return + } + case "nv": + za0002.NumVersions, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001, "NumVersions") + return + } + case "no": + za0002.NumObjects, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001, "NumObjects") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001) + return + } + } + } + z.Tiers[za0001] = za0002 + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *allTierStats) Msgsize() (s int) { + s = 1 + 3 + msgp.MapHeaderSize + if z.Tiers != nil { + for za0001, za0002 := range z.Tiers { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + 1 + 3 + msgp.Uint64Size + 3 + msgp.IntSize + 3 + msgp.IntSize + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *currentScannerCycle) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 4 + // string "current" + o = append(o, 0x84, 0xa7, 0x63, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74) + o = msgp.AppendUint64(o, z.current) + // string "next" + o = append(o, 0xa4, 0x6e, 0x65, 0x78, 0x74) + o = msgp.AppendUint64(o, z.next) + // string "started" + o = append(o, 0xa7, 0x73, 0x74, 0x61, 0x72, 0x74, 0x65, 0x64) + o = msgp.AppendTime(o, z.started) + // string "cycleCompleted" + o = append(o, 0xae, 0x63, 0x79, 0x63, 0x6c, 0x65, 0x43, 0x6f, 0x6d, 0x70, 0x6c, 0x65, 0x74, 0x65, 0x64) + o = msgp.AppendArrayHeader(o, uint32(len(z.cycleCompleted))) + for za0001 := range z.cycleCompleted { + o = msgp.AppendTime(o, z.cycleCompleted[za0001]) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *currentScannerCycle) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "current": + z.current, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "current") + return + } + case "next": + z.next, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "next") + return + } + case "started": + z.started, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "started") + return + } + case "cycleCompleted": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "cycleCompleted") + return + } + if cap(z.cycleCompleted) >= int(zb0002) { + z.cycleCompleted = (z.cycleCompleted)[:zb0002] + } else { + z.cycleCompleted = make([]time.Time, zb0002) + } + for za0001 := range z.cycleCompleted { + z.cycleCompleted[za0001], bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "cycleCompleted", za0001) + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *currentScannerCycle) Msgsize() (s int) { + s = 1 + 8 + msgp.Uint64Size + 5 + msgp.Uint64Size + 8 + msgp.TimeSize + 15 + msgp.ArrayHeaderSize + (len(z.cycleCompleted) * (msgp.TimeSize)) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *dataUsageCache) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Info": + err = z.Info.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + case "Cache": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + if z.Cache == nil { + z.Cache = make(map[string]dataUsageEntry, zb0002) + } else if len(z.Cache) > 0 { + clear(z.Cache) + } + for zb0002 > 0 { + zb0002-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + var za0002 dataUsageEntry + err = za0002.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Cache", za0001) + return + } + z.Cache[za0001] = za0002 + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *dataUsageCache) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "Info" + err = en.Append(0x82, 0xa4, 0x49, 0x6e, 0x66, 0x6f) + if err != nil { + return + } + err = z.Info.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + // write "Cache" + err = en.Append(0xa5, 0x43, 0x61, 0x63, 0x68, 0x65) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.Cache))) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + for za0001, za0002 := range z.Cache { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + err = za0002.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Cache", za0001) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *dataUsageCache) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "Info" + o = append(o, 0x82, 0xa4, 0x49, 0x6e, 0x66, 0x6f) + o, err = z.Info.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + // string "Cache" + o = append(o, 0xa5, 0x43, 0x61, 0x63, 0x68, 0x65) + o = msgp.AppendMapHeader(o, uint32(len(z.Cache))) + for za0001, za0002 := range z.Cache { + o = msgp.AppendString(o, za0001) + o, err = za0002.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Cache", za0001) + return + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *dataUsageCache) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Info": + bts, err = z.Info.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + case "Cache": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + if z.Cache == nil { + z.Cache = make(map[string]dataUsageEntry, zb0002) + } else if len(z.Cache) > 0 { + clear(z.Cache) + } + for zb0002 > 0 { + var za0002 dataUsageEntry + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + bts, err = za0002.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Cache", za0001) + return + } + z.Cache[za0001] = za0002 + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *dataUsageCache) Msgsize() (s int) { + s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize + if z.Cache != nil { + for za0001, za0002 := range z.Cache { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize() + } + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *dataUsageCacheInfo) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Name": + z.Name, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "NextCycle": + z.NextCycle, err = dc.ReadUint32() + if err != nil { + err = msgp.WrapError(err, "NextCycle") + return + } + case "LastUpdate": + z.LastUpdate, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "LastUpdate") + return + } + case "SkipHealing": + z.SkipHealing, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "SkipHealing") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *dataUsageCacheInfo) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 4 + // write "Name" + err = en.Append(0x84, 0xa4, 0x4e, 0x61, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteString(z.Name) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + // write "NextCycle" + err = en.Append(0xa9, 0x4e, 0x65, 0x78, 0x74, 0x43, 0x79, 0x63, 0x6c, 0x65) + if err != nil { + return + } + err = en.WriteUint32(z.NextCycle) + if err != nil { + err = msgp.WrapError(err, "NextCycle") + return + } + // write "LastUpdate" + err = en.Append(0xaa, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65) + if err != nil { + return + } + err = en.WriteTime(z.LastUpdate) + if err != nil { + err = msgp.WrapError(err, "LastUpdate") + return + } + // write "SkipHealing" + err = en.Append(0xab, 0x53, 0x6b, 0x69, 0x70, 0x48, 0x65, 0x61, 0x6c, 0x69, 0x6e, 0x67) + if err != nil { + return + } + err = en.WriteBool(z.SkipHealing) + if err != nil { + err = msgp.WrapError(err, "SkipHealing") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *dataUsageCacheInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 4 + // string "Name" + o = append(o, 0x84, 0xa4, 0x4e, 0x61, 0x6d, 0x65) + o = msgp.AppendString(o, z.Name) + // string "NextCycle" + o = append(o, 0xa9, 0x4e, 0x65, 0x78, 0x74, 0x43, 0x79, 0x63, 0x6c, 0x65) + o = msgp.AppendUint32(o, z.NextCycle) + // string "LastUpdate" + o = append(o, 0xaa, 0x4c, 0x61, 0x73, 0x74, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65) + o = msgp.AppendTime(o, z.LastUpdate) + // string "SkipHealing" + o = append(o, 0xab, 0x53, 0x6b, 0x69, 0x70, 0x48, 0x65, 0x61, 0x6c, 0x69, 0x6e, 0x67) + o = msgp.AppendBool(o, z.SkipHealing) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *dataUsageCacheInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Name": + z.Name, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "NextCycle": + z.NextCycle, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "NextCycle") + return + } + case "LastUpdate": + z.LastUpdate, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "LastUpdate") + return + } + case "SkipHealing": + z.SkipHealing, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SkipHealing") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *dataUsageCacheInfo) Msgsize() (s int) { + s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 10 + msgp.Uint32Size + 11 + msgp.TimeSize + 12 + msgp.BoolSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *dataUsageCacheV2) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Info": + err = z.Info.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + case "Cache": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + if z.Cache == nil { + z.Cache = make(map[string]dataUsageEntryV2, zb0002) + } else if len(z.Cache) > 0 { + clear(z.Cache) + } + for zb0002 > 0 { + zb0002-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + var za0002 dataUsageEntryV2 + err = za0002.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Cache", za0001) + return + } + z.Cache[za0001] = za0002 + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *dataUsageCacheV2) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Info": + bts, err = z.Info.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + case "Cache": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + if z.Cache == nil { + z.Cache = make(map[string]dataUsageEntryV2, zb0002) + } else if len(z.Cache) > 0 { + clear(z.Cache) + } + for zb0002 > 0 { + var za0002 dataUsageEntryV2 + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + bts, err = za0002.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Cache", za0001) + return + } + z.Cache[za0001] = za0002 + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *dataUsageCacheV2) Msgsize() (s int) { + s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize + if z.Cache != nil { + for za0001, za0002 := range z.Cache { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize() + } + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *dataUsageCacheV3) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Info": + err = z.Info.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + case "Cache": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + if z.Cache == nil { + z.Cache = make(map[string]dataUsageEntryV3, zb0002) + } else if len(z.Cache) > 0 { + clear(z.Cache) + } + for zb0002 > 0 { + zb0002-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + var za0002 dataUsageEntryV3 + err = za0002.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Cache", za0001) + return + } + z.Cache[za0001] = za0002 + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *dataUsageCacheV3) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Info": + bts, err = z.Info.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + case "Cache": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + if z.Cache == nil { + z.Cache = make(map[string]dataUsageEntryV3, zb0002) + } else if len(z.Cache) > 0 { + clear(z.Cache) + } + for zb0002 > 0 { + var za0002 dataUsageEntryV3 + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + bts, err = za0002.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Cache", za0001) + return + } + z.Cache[za0001] = za0002 + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *dataUsageCacheV3) Msgsize() (s int) { + s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize + if z.Cache != nil { + for za0001, za0002 := range z.Cache { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize() + } + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *dataUsageCacheV4) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Info": + err = z.Info.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + case "Cache": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + if z.Cache == nil { + z.Cache = make(map[string]dataUsageEntryV4, zb0002) + } else if len(z.Cache) > 0 { + clear(z.Cache) + } + for zb0002 > 0 { + zb0002-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + var za0002 dataUsageEntryV4 + err = za0002.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Cache", za0001) + return + } + z.Cache[za0001] = za0002 + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *dataUsageCacheV4) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Info": + bts, err = z.Info.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + case "Cache": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + if z.Cache == nil { + z.Cache = make(map[string]dataUsageEntryV4, zb0002) + } else if len(z.Cache) > 0 { + clear(z.Cache) + } + for zb0002 > 0 { + var za0002 dataUsageEntryV4 + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + bts, err = za0002.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Cache", za0001) + return + } + z.Cache[za0001] = za0002 + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *dataUsageCacheV4) Msgsize() (s int) { + s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize + if z.Cache != nil { + for za0001, za0002 := range z.Cache { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize() + } + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *dataUsageCacheV5) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Info": + err = z.Info.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + case "Cache": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + if z.Cache == nil { + z.Cache = make(map[string]dataUsageEntryV5, zb0002) + } else if len(z.Cache) > 0 { + clear(z.Cache) + } + for zb0002 > 0 { + zb0002-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + var za0002 dataUsageEntryV5 + err = za0002.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Cache", za0001) + return + } + z.Cache[za0001] = za0002 + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *dataUsageCacheV5) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Info": + bts, err = z.Info.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + case "Cache": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + if z.Cache == nil { + z.Cache = make(map[string]dataUsageEntryV5, zb0002) + } else if len(z.Cache) > 0 { + clear(z.Cache) + } + for zb0002 > 0 { + var za0002 dataUsageEntryV5 + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + bts, err = za0002.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Cache", za0001) + return + } + z.Cache[za0001] = za0002 + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *dataUsageCacheV5) Msgsize() (s int) { + s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize + if z.Cache != nil { + for za0001, za0002 := range z.Cache { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize() + } + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *dataUsageCacheV6) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Info": + err = z.Info.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + case "Cache": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + if z.Cache == nil { + z.Cache = make(map[string]dataUsageEntryV6, zb0002) + } else if len(z.Cache) > 0 { + clear(z.Cache) + } + for zb0002 > 0 { + zb0002-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + var za0002 dataUsageEntryV6 + err = za0002.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Cache", za0001) + return + } + z.Cache[za0001] = za0002 + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *dataUsageCacheV6) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Info": + bts, err = z.Info.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + case "Cache": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + if z.Cache == nil { + z.Cache = make(map[string]dataUsageEntryV6, zb0002) + } else if len(z.Cache) > 0 { + clear(z.Cache) + } + for zb0002 > 0 { + var za0002 dataUsageEntryV6 + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + bts, err = za0002.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Cache", za0001) + return + } + z.Cache[za0001] = za0002 + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *dataUsageCacheV6) Msgsize() (s int) { + s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize + if z.Cache != nil { + for za0001, za0002 := range z.Cache { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize() + } + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *dataUsageCacheV7) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Info": + err = z.Info.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + case "Cache": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + if z.Cache == nil { + z.Cache = make(map[string]dataUsageEntryV7, zb0002) + } else if len(z.Cache) > 0 { + clear(z.Cache) + } + for zb0002 > 0 { + zb0002-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + var za0002 dataUsageEntryV7 + err = za0002.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Cache", za0001) + return + } + z.Cache[za0001] = za0002 + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *dataUsageCacheV7) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Info": + bts, err = z.Info.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + case "Cache": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + if z.Cache == nil { + z.Cache = make(map[string]dataUsageEntryV7, zb0002) + } else if len(z.Cache) > 0 { + clear(z.Cache) + } + for zb0002 > 0 { + var za0002 dataUsageEntryV7 + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + bts, err = za0002.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Cache", za0001) + return + } + z.Cache[za0001] = za0002 + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *dataUsageCacheV7) Msgsize() (s int) { + s = 1 + 5 + z.Info.Msgsize() + 6 + msgp.MapHeaderSize + if z.Cache != nil { + for za0001, za0002 := range z.Cache { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize() + } + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *dataUsageEntry) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 1 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ch": + err = z.Children.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Children") + return + } + case "sz": + z.Size, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + case "os": + z.Objects, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + case "vs": + z.Versions, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + case "dms": + z.DeleteMarkers, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "DeleteMarkers") + return + } + case "szs": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "ObjSizes") + return + } + if zb0002 != uint32(dataUsageBucketLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002} + return + } + for za0001 := range z.ObjSizes { + z.ObjSizes[za0001], err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ObjSizes", za0001) + return + } + } + case "vh": + var zb0003 uint32 + zb0003, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "ObjVersions") + return + } + if zb0003 != uint32(dataUsageVersionLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003} + return + } + for za0002 := range z.ObjVersions { + z.ObjVersions[za0002], err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ObjVersions", za0002) + return + } + } + case "ats": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + z.AllTierStats = nil + } else { + if z.AllTierStats == nil { + z.AllTierStats = new(allTierStats) + } + var zb0004 uint32 + zb0004, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + for zb0004 > 0 { + zb0004-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + switch msgp.UnsafeString(field) { + case "ts": + var zb0005 uint32 + zb0005, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers") + return + } + if z.AllTierStats.Tiers == nil { + z.AllTierStats.Tiers = make(map[string]tierStats, zb0005) + } else if len(z.AllTierStats.Tiers) > 0 { + clear(z.AllTierStats.Tiers) + } + for zb0005 > 0 { + zb0005-- + var za0003 string + za0003, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers") + return + } + var za0004 tierStats + var zb0006 uint32 + zb0006, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003) + return + } + for zb0006 > 0 { + zb0006-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003) + return + } + switch msgp.UnsafeString(field) { + case "ts": + za0004.TotalSize, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "TotalSize") + return + } + case "nv": + za0004.NumVersions, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumVersions") + return + } + case "no": + za0004.NumObjects, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumObjects") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003) + return + } + } + } + z.AllTierStats.Tiers[za0003] = za0004 + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + } + } + } + zb0001Mask |= 0x1 + case "c": + z.Compacted, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Compacted") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if (zb0001Mask & 0x1) == 0 { + z.AllTierStats = nil + } + + return +} + +// EncodeMsg implements msgp.Encodable +func (z *dataUsageEntry) EncodeMsg(en *msgp.Writer) (err error) { + // check for omitted fields + zb0001Len := uint32(9) + var zb0001Mask uint16 /* 9 bits */ + _ = zb0001Mask + if z.AllTierStats == nil { + zb0001Len-- + zb0001Mask |= 0x80 + } + // variable map header, size zb0001Len + err = en.Append(0x80 | uint8(zb0001Len)) + if err != nil { + return + } + + // skip if no fields are to be emitted + if zb0001Len != 0 { + // write "ch" + err = en.Append(0xa2, 0x63, 0x68) + if err != nil { + return + } + err = z.Children.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Children") + return + } + // write "sz" + err = en.Append(0xa2, 0x73, 0x7a) + if err != nil { + return + } + err = en.WriteInt64(z.Size) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + // write "os" + err = en.Append(0xa2, 0x6f, 0x73) + if err != nil { + return + } + err = en.WriteUint64(z.Objects) + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + // write "vs" + err = en.Append(0xa2, 0x76, 0x73) + if err != nil { + return + } + err = en.WriteUint64(z.Versions) + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + // write "dms" + err = en.Append(0xa3, 0x64, 0x6d, 0x73) + if err != nil { + return + } + err = en.WriteUint64(z.DeleteMarkers) + if err != nil { + err = msgp.WrapError(err, "DeleteMarkers") + return + } + // write "szs" + err = en.Append(0xa3, 0x73, 0x7a, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(dataUsageBucketLen)) + if err != nil { + err = msgp.WrapError(err, "ObjSizes") + return + } + for za0001 := range z.ObjSizes { + err = en.WriteUint64(z.ObjSizes[za0001]) + if err != nil { + err = msgp.WrapError(err, "ObjSizes", za0001) + return + } + } + // write "vh" + err = en.Append(0xa2, 0x76, 0x68) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(dataUsageVersionLen)) + if err != nil { + err = msgp.WrapError(err, "ObjVersions") + return + } + for za0002 := range z.ObjVersions { + err = en.WriteUint64(z.ObjVersions[za0002]) + if err != nil { + err = msgp.WrapError(err, "ObjVersions", za0002) + return + } + } + if (zb0001Mask & 0x80) == 0 { // if not omitted + // write "ats" + err = en.Append(0xa3, 0x61, 0x74, 0x73) + if err != nil { + return + } + if z.AllTierStats == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + // map header, size 1 + // write "ts" + err = en.Append(0x81, 0xa2, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.AllTierStats.Tiers))) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers") + return + } + for za0003, za0004 := range z.AllTierStats.Tiers { + err = en.WriteString(za0003) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers") + return + } + // map header, size 3 + // write "ts" + err = en.Append(0x83, 0xa2, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteUint64(za0004.TotalSize) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "TotalSize") + return + } + // write "nv" + err = en.Append(0xa2, 0x6e, 0x76) + if err != nil { + return + } + err = en.WriteInt(za0004.NumVersions) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumVersions") + return + } + // write "no" + err = en.Append(0xa2, 0x6e, 0x6f) + if err != nil { + return + } + err = en.WriteInt(za0004.NumObjects) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumObjects") + return + } + } + } + } + // write "c" + err = en.Append(0xa1, 0x63) + if err != nil { + return + } + err = en.WriteBool(z.Compacted) + if err != nil { + err = msgp.WrapError(err, "Compacted") + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *dataUsageEntry) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // check for omitted fields + zb0001Len := uint32(9) + var zb0001Mask uint16 /* 9 bits */ + _ = zb0001Mask + if z.AllTierStats == nil { + zb0001Len-- + zb0001Mask |= 0x80 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + + // skip if no fields are to be emitted + if zb0001Len != 0 { + // string "ch" + o = append(o, 0xa2, 0x63, 0x68) + o, err = z.Children.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Children") + return + } + // string "sz" + o = append(o, 0xa2, 0x73, 0x7a) + o = msgp.AppendInt64(o, z.Size) + // string "os" + o = append(o, 0xa2, 0x6f, 0x73) + o = msgp.AppendUint64(o, z.Objects) + // string "vs" + o = append(o, 0xa2, 0x76, 0x73) + o = msgp.AppendUint64(o, z.Versions) + // string "dms" + o = append(o, 0xa3, 0x64, 0x6d, 0x73) + o = msgp.AppendUint64(o, z.DeleteMarkers) + // string "szs" + o = append(o, 0xa3, 0x73, 0x7a, 0x73) + o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen)) + for za0001 := range z.ObjSizes { + o = msgp.AppendUint64(o, z.ObjSizes[za0001]) + } + // string "vh" + o = append(o, 0xa2, 0x76, 0x68) + o = msgp.AppendArrayHeader(o, uint32(dataUsageVersionLen)) + for za0002 := range z.ObjVersions { + o = msgp.AppendUint64(o, z.ObjVersions[za0002]) + } + if (zb0001Mask & 0x80) == 0 { // if not omitted + // string "ats" + o = append(o, 0xa3, 0x61, 0x74, 0x73) + if z.AllTierStats == nil { + o = msgp.AppendNil(o) + } else { + // map header, size 1 + // string "ts" + o = append(o, 0x81, 0xa2, 0x74, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.AllTierStats.Tiers))) + for za0003, za0004 := range z.AllTierStats.Tiers { + o = msgp.AppendString(o, za0003) + // map header, size 3 + // string "ts" + o = append(o, 0x83, 0xa2, 0x74, 0x73) + o = msgp.AppendUint64(o, za0004.TotalSize) + // string "nv" + o = append(o, 0xa2, 0x6e, 0x76) + o = msgp.AppendInt(o, za0004.NumVersions) + // string "no" + o = append(o, 0xa2, 0x6e, 0x6f) + o = msgp.AppendInt(o, za0004.NumObjects) + } + } + } + // string "c" + o = append(o, 0xa1, 0x63) + o = msgp.AppendBool(o, z.Compacted) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *dataUsageEntry) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 1 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ch": + bts, err = z.Children.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Children") + return + } + case "sz": + z.Size, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + case "os": + z.Objects, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + case "vs": + z.Versions, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + case "dms": + z.DeleteMarkers, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "DeleteMarkers") + return + } + case "szs": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjSizes") + return + } + if zb0002 != uint32(dataUsageBucketLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002} + return + } + for za0001 := range z.ObjSizes { + z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjSizes", za0001) + return + } + } + case "vh": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjVersions") + return + } + if zb0003 != uint32(dataUsageVersionLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003} + return + } + for za0002 := range z.ObjVersions { + z.ObjVersions[za0002], bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjVersions", za0002) + return + } + } + case "ats": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.AllTierStats = nil + } else { + if z.AllTierStats == nil { + z.AllTierStats = new(allTierStats) + } + var zb0004 uint32 + zb0004, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + for zb0004 > 0 { + zb0004-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + switch msgp.UnsafeString(field) { + case "ts": + var zb0005 uint32 + zb0005, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers") + return + } + if z.AllTierStats.Tiers == nil { + z.AllTierStats.Tiers = make(map[string]tierStats, zb0005) + } else if len(z.AllTierStats.Tiers) > 0 { + clear(z.AllTierStats.Tiers) + } + for zb0005 > 0 { + var za0004 tierStats + zb0005-- + var za0003 string + za0003, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers") + return + } + var zb0006 uint32 + zb0006, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003) + return + } + for zb0006 > 0 { + zb0006-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003) + return + } + switch msgp.UnsafeString(field) { + case "ts": + za0004.TotalSize, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "TotalSize") + return + } + case "nv": + za0004.NumVersions, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumVersions") + return + } + case "no": + za0004.NumObjects, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumObjects") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003) + return + } + } + } + z.AllTierStats.Tiers[za0003] = za0004 + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + } + } + } + zb0001Mask |= 0x1 + case "c": + z.Compacted, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Compacted") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if (zb0001Mask & 0x1) == 0 { + z.AllTierStats = nil + } + + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *dataUsageEntry) Msgsize() (s int) { + s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 4 + if z.AllTierStats == nil { + s += msgp.NilSize + } else { + s += 1 + 3 + msgp.MapHeaderSize + if z.AllTierStats.Tiers != nil { + for za0003, za0004 := range z.AllTierStats.Tiers { + _ = za0004 + s += msgp.StringPrefixSize + len(za0003) + 1 + 3 + msgp.Uint64Size + 3 + msgp.IntSize + 3 + msgp.IntSize + } + } + } + s += 2 + msgp.BoolSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *dataUsageEntryV2) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0001 uint32 + zb0001, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 4 { + err = msgp.ArrayError{Wanted: 4, Got: zb0001} + return + } + z.Size, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + z.Objects, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "ObjSizes") + return + } + if zb0002 != uint32(dataUsageBucketLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002} + return + } + for za0001 := range z.ObjSizes { + z.ObjSizes[za0001], err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ObjSizes", za0001) + return + } + } + err = z.Children.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Children") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *dataUsageEntryV2) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0001 uint32 + zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 4 { + err = msgp.ArrayError{Wanted: 4, Got: zb0001} + return + } + z.Size, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + z.Objects, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjSizes") + return + } + if zb0002 != uint32(dataUsageBucketLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002} + return + } + for za0001 := range z.ObjSizes { + z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjSizes", za0001) + return + } + } + bts, err = z.Children.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Children") + return + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *dataUsageEntryV2) Msgsize() (s int) { + s = 1 + msgp.Int64Size + msgp.Uint64Size + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + z.Children.Msgsize() + return +} + +// DecodeMsg implements msgp.Decodable +func (z *dataUsageEntryV3) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0001 uint32 + zb0001, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 4 { + err = msgp.ArrayError{Wanted: 4, Got: zb0001} + return + } + z.Size, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + z.Objects, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "ObjSizes") + return + } + if zb0002 != uint32(dataUsageBucketLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002} + return + } + for za0001 := range z.ObjSizes { + z.ObjSizes[za0001], err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ObjSizes", za0001) + return + } + } + err = z.Children.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Children") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *dataUsageEntryV3) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0001 uint32 + zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 4 { + err = msgp.ArrayError{Wanted: 4, Got: zb0001} + return + } + z.Size, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + z.Objects, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjSizes") + return + } + if zb0002 != uint32(dataUsageBucketLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002} + return + } + for za0001 := range z.ObjSizes { + z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjSizes", za0001) + return + } + } + bts, err = z.Children.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Children") + return + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *dataUsageEntryV3) Msgsize() (s int) { + s = 1 + msgp.Int64Size + msgp.Uint64Size + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + z.Children.Msgsize() + return +} + +// DecodeMsg implements msgp.Decodable +func (z *dataUsageEntryV4) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0001 uint32 + zb0001, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 4 { + err = msgp.ArrayError{Wanted: 4, Got: zb0001} + return + } + err = z.Children.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Children") + return + } + z.Size, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + z.Objects, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "ObjSizes") + return + } + if zb0002 != uint32(dataUsageBucketLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002} + return + } + for za0001 := range z.ObjSizes { + z.ObjSizes[za0001], err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ObjSizes", za0001) + return + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *dataUsageEntryV4) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0001 uint32 + zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 4 { + err = msgp.ArrayError{Wanted: 4, Got: zb0001} + return + } + bts, err = z.Children.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Children") + return + } + z.Size, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + z.Objects, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjSizes") + return + } + if zb0002 != uint32(dataUsageBucketLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002} + return + } + for za0001 := range z.ObjSizes { + z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjSizes", za0001) + return + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *dataUsageEntryV4) Msgsize() (s int) { + s = 1 + z.Children.Msgsize() + msgp.Int64Size + msgp.Uint64Size + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *dataUsageEntryV5) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0001 uint32 + zb0001, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 6 { + err = msgp.ArrayError{Wanted: 6, Got: zb0001} + return + } + err = z.Children.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Children") + return + } + z.Size, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + z.Objects, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + z.Versions, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "ObjSizes") + return + } + if zb0002 != uint32(dataUsageBucketLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002} + return + } + for za0001 := range z.ObjSizes { + z.ObjSizes[za0001], err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ObjSizes", za0001) + return + } + } + z.Compacted, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Compacted") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *dataUsageEntryV5) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0001 uint32 + zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 6 { + err = msgp.ArrayError{Wanted: 6, Got: zb0001} + return + } + bts, err = z.Children.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Children") + return + } + z.Size, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + z.Objects, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + z.Versions, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjSizes") + return + } + if zb0002 != uint32(dataUsageBucketLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002} + return + } + for za0001 := range z.ObjSizes { + z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjSizes", za0001) + return + } + } + z.Compacted, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Compacted") + return + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *dataUsageEntryV5) Msgsize() (s int) { + s = 1 + z.Children.Msgsize() + msgp.Int64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + msgp.BoolSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *dataUsageEntryV6) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0001 uint32 + zb0001, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 6 { + err = msgp.ArrayError{Wanted: 6, Got: zb0001} + return + } + err = z.Children.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Children") + return + } + z.Size, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + z.Objects, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + z.Versions, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "ObjSizes") + return + } + if zb0002 != uint32(dataUsageBucketLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002} + return + } + for za0001 := range z.ObjSizes { + z.ObjSizes[za0001], err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ObjSizes", za0001) + return + } + } + z.Compacted, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Compacted") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *dataUsageEntryV6) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0001 uint32 + zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 6 { + err = msgp.ArrayError{Wanted: 6, Got: zb0001} + return + } + bts, err = z.Children.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Children") + return + } + z.Size, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + z.Objects, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + z.Versions, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjSizes") + return + } + if zb0002 != uint32(dataUsageBucketLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0002} + return + } + for za0001 := range z.ObjSizes { + z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjSizes", za0001) + return + } + } + z.Compacted, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Compacted") + return + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *dataUsageEntryV6) Msgsize() (s int) { + s = 1 + z.Children.Msgsize() + msgp.Int64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + msgp.BoolSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *dataUsageEntryV7) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 1 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ch": + err = z.Children.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Children") + return + } + case "sz": + z.Size, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + case "os": + z.Objects, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + case "vs": + z.Versions, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + case "dms": + z.DeleteMarkers, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "DeleteMarkers") + return + } + case "szs": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "ObjSizes") + return + } + if zb0002 != uint32(dataUsageBucketLenV1) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLenV1), Got: zb0002} + return + } + for za0001 := range z.ObjSizes { + z.ObjSizes[za0001], err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ObjSizes", za0001) + return + } + } + case "vh": + var zb0003 uint32 + zb0003, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "ObjVersions") + return + } + if zb0003 != uint32(dataUsageVersionLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003} + return + } + for za0002 := range z.ObjVersions { + z.ObjVersions[za0002], err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "ObjVersions", za0002) + return + } + } + case "ats": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + z.AllTierStats = nil + } else { + if z.AllTierStats == nil { + z.AllTierStats = new(allTierStats) + } + var zb0004 uint32 + zb0004, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + for zb0004 > 0 { + zb0004-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + switch msgp.UnsafeString(field) { + case "ts": + var zb0005 uint32 + zb0005, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers") + return + } + if z.AllTierStats.Tiers == nil { + z.AllTierStats.Tiers = make(map[string]tierStats, zb0005) + } else if len(z.AllTierStats.Tiers) > 0 { + clear(z.AllTierStats.Tiers) + } + for zb0005 > 0 { + zb0005-- + var za0003 string + za0003, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers") + return + } + var za0004 tierStats + var zb0006 uint32 + zb0006, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003) + return + } + for zb0006 > 0 { + zb0006-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003) + return + } + switch msgp.UnsafeString(field) { + case "ts": + za0004.TotalSize, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "TotalSize") + return + } + case "nv": + za0004.NumVersions, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumVersions") + return + } + case "no": + za0004.NumObjects, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumObjects") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003) + return + } + } + } + z.AllTierStats.Tiers[za0003] = za0004 + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + } + } + } + zb0001Mask |= 0x1 + case "c": + z.Compacted, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Compacted") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if (zb0001Mask & 0x1) == 0 { + z.AllTierStats = nil + } + + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *dataUsageEntryV7) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 1 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ch": + bts, err = z.Children.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Children") + return + } + case "sz": + z.Size, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + case "os": + z.Objects, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + case "vs": + z.Versions, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + case "dms": + z.DeleteMarkers, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "DeleteMarkers") + return + } + case "szs": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjSizes") + return + } + if zb0002 != uint32(dataUsageBucketLenV1) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLenV1), Got: zb0002} + return + } + for za0001 := range z.ObjSizes { + z.ObjSizes[za0001], bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjSizes", za0001) + return + } + } + case "vh": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjVersions") + return + } + if zb0003 != uint32(dataUsageVersionLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0003} + return + } + for za0002 := range z.ObjVersions { + z.ObjVersions[za0002], bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjVersions", za0002) + return + } + } + case "ats": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.AllTierStats = nil + } else { + if z.AllTierStats == nil { + z.AllTierStats = new(allTierStats) + } + var zb0004 uint32 + zb0004, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + for zb0004 > 0 { + zb0004-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + switch msgp.UnsafeString(field) { + case "ts": + var zb0005 uint32 + zb0005, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers") + return + } + if z.AllTierStats.Tiers == nil { + z.AllTierStats.Tiers = make(map[string]tierStats, zb0005) + } else if len(z.AllTierStats.Tiers) > 0 { + clear(z.AllTierStats.Tiers) + } + for zb0005 > 0 { + var za0004 tierStats + zb0005-- + var za0003 string + za0003, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers") + return + } + var zb0006 uint32 + zb0006, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003) + return + } + for zb0006 > 0 { + zb0006-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003) + return + } + switch msgp.UnsafeString(field) { + case "ts": + za0004.TotalSize, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "TotalSize") + return + } + case "nv": + za0004.NumVersions, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumVersions") + return + } + case "no": + za0004.NumObjects, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003, "NumObjects") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats", "Tiers", za0003) + return + } + } + } + z.AllTierStats.Tiers[za0003] = za0004 + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "AllTierStats") + return + } + } + } + } + zb0001Mask |= 0x1 + case "c": + z.Compacted, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Compacted") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if (zb0001Mask & 0x1) == 0 { + z.AllTierStats = nil + } + + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *dataUsageEntryV7) Msgsize() (s int) { + s = 1 + 3 + z.Children.Msgsize() + 3 + msgp.Int64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + (dataUsageBucketLenV1 * (msgp.Uint64Size)) + 3 + msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + 4 + if z.AllTierStats == nil { + s += msgp.NilSize + } else { + s += 1 + 3 + msgp.MapHeaderSize + if z.AllTierStats.Tiers != nil { + for za0003, za0004 := range z.AllTierStats.Tiers { + _ = za0004 + s += msgp.StringPrefixSize + len(za0003) + 1 + 3 + msgp.Uint64Size + 3 + msgp.IntSize + 3 + msgp.IntSize + } + } + } + s += 2 + msgp.BoolSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *dataUsageHash) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 string + zb0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = dataUsageHash(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z dataUsageHash) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteString(string(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z dataUsageHash) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendString(o, string(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *dataUsageHash) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 string + zb0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = dataUsageHash(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z dataUsageHash) Msgsize() (s int) { + s = msgp.StringPrefixSize + len(string(z)) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *sizeHistogram) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0001 uint32 + zb0001, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != uint32(dataUsageBucketLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0001} + return + } + for za0001 := range z { + z[za0001], err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, za0001) + return + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *sizeHistogram) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteArrayHeader(uint32(dataUsageBucketLen)) + if err != nil { + err = msgp.WrapError(err) + return + } + for za0001 := range z { + err = en.WriteUint64(z[za0001]) + if err != nil { + err = msgp.WrapError(err, za0001) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *sizeHistogram) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLen)) + for za0001 := range z { + o = msgp.AppendUint64(o, z[za0001]) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *sizeHistogram) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0001 uint32 + zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != uint32(dataUsageBucketLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLen), Got: zb0001} + return + } + for za0001 := range z { + z[za0001], bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, za0001) + return + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *sizeHistogram) Msgsize() (s int) { + s = msgp.ArrayHeaderSize + (dataUsageBucketLen * (msgp.Uint64Size)) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *sizeHistogramV1) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0001 uint32 + zb0001, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != uint32(dataUsageBucketLenV1) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLenV1), Got: zb0001} + return + } + for za0001 := range z { + z[za0001], err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, za0001) + return + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *sizeHistogramV1) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteArrayHeader(uint32(dataUsageBucketLenV1)) + if err != nil { + err = msgp.WrapError(err) + return + } + for za0001 := range z { + err = en.WriteUint64(z[za0001]) + if err != nil { + err = msgp.WrapError(err, za0001) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *sizeHistogramV1) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendArrayHeader(o, uint32(dataUsageBucketLenV1)) + for za0001 := range z { + o = msgp.AppendUint64(o, z[za0001]) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *sizeHistogramV1) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0001 uint32 + zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != uint32(dataUsageBucketLenV1) { + err = msgp.ArrayError{Wanted: uint32(dataUsageBucketLenV1), Got: zb0001} + return + } + for za0001 := range z { + z[za0001], bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, za0001) + return + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *sizeHistogramV1) Msgsize() (s int) { + s = msgp.ArrayHeaderSize + (dataUsageBucketLenV1 * (msgp.Uint64Size)) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *tierStats) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ts": + z.TotalSize, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "TotalSize") + return + } + case "nv": + z.NumVersions, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "NumVersions") + return + } + case "no": + z.NumObjects, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "NumObjects") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z tierStats) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "ts" + err = en.Append(0x83, 0xa2, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteUint64(z.TotalSize) + if err != nil { + err = msgp.WrapError(err, "TotalSize") + return + } + // write "nv" + err = en.Append(0xa2, 0x6e, 0x76) + if err != nil { + return + } + err = en.WriteInt(z.NumVersions) + if err != nil { + err = msgp.WrapError(err, "NumVersions") + return + } + // write "no" + err = en.Append(0xa2, 0x6e, 0x6f) + if err != nil { + return + } + err = en.WriteInt(z.NumObjects) + if err != nil { + err = msgp.WrapError(err, "NumObjects") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z tierStats) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "ts" + o = append(o, 0x83, 0xa2, 0x74, 0x73) + o = msgp.AppendUint64(o, z.TotalSize) + // string "nv" + o = append(o, 0xa2, 0x6e, 0x76) + o = msgp.AppendInt(o, z.NumVersions) + // string "no" + o = append(o, 0xa2, 0x6e, 0x6f) + o = msgp.AppendInt(o, z.NumObjects) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *tierStats) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ts": + z.TotalSize, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalSize") + return + } + case "nv": + z.NumVersions, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "NumVersions") + return + } + case "no": + z.NumObjects, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "NumObjects") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z tierStats) Msgsize() (s int) { + s = 1 + 3 + msgp.Uint64Size + 3 + msgp.IntSize + 3 + msgp.IntSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *versionsHistogram) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0001 uint32 + zb0001, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != uint32(dataUsageVersionLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0001} + return + } + for za0001 := range z { + z[za0001], err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, za0001) + return + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *versionsHistogram) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteArrayHeader(uint32(dataUsageVersionLen)) + if err != nil { + err = msgp.WrapError(err) + return + } + for za0001 := range z { + err = en.WriteUint64(z[za0001]) + if err != nil { + err = msgp.WrapError(err, za0001) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *versionsHistogram) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendArrayHeader(o, uint32(dataUsageVersionLen)) + for za0001 := range z { + o = msgp.AppendUint64(o, z[za0001]) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *versionsHistogram) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0001 uint32 + zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != uint32(dataUsageVersionLen) { + err = msgp.ArrayError{Wanted: uint32(dataUsageVersionLen), Got: zb0001} + return + } + for za0001 := range z { + z[za0001], bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, za0001) + return + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *versionsHistogram) Msgsize() (s int) { + s = msgp.ArrayHeaderSize + (dataUsageVersionLen * (msgp.Uint64Size)) + return +} diff --git a/cmd/data-usage-cache_gen_test.go b/cmd/data-usage-cache_gen_test.go new file mode 100644 index 00000000..b19ca6bb --- /dev/null +++ b/cmd/data-usage-cache_gen_test.go @@ -0,0 +1,972 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalallTierStats(t *testing.T) { + v := allTierStats{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgallTierStats(b *testing.B) { + v := allTierStats{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgallTierStats(b *testing.B) { + v := allTierStats{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalallTierStats(b *testing.B) { + v := allTierStats{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeallTierStats(t *testing.T) { + v := allTierStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeallTierStats Msgsize() is inaccurate") + } + + vn := allTierStats{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeallTierStats(b *testing.B) { + v := allTierStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeallTierStats(b *testing.B) { + v := allTierStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalcurrentScannerCycle(t *testing.T) { + v := currentScannerCycle{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgcurrentScannerCycle(b *testing.B) { + v := currentScannerCycle{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgcurrentScannerCycle(b *testing.B) { + v := currentScannerCycle{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalcurrentScannerCycle(b *testing.B) { + v := currentScannerCycle{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshaldataUsageCache(t *testing.T) { + v := dataUsageCache{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgdataUsageCache(b *testing.B) { + v := dataUsageCache{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgdataUsageCache(b *testing.B) { + v := dataUsageCache{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshaldataUsageCache(b *testing.B) { + v := dataUsageCache{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodedataUsageCache(t *testing.T) { + v := dataUsageCache{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodedataUsageCache Msgsize() is inaccurate") + } + + vn := dataUsageCache{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodedataUsageCache(b *testing.B) { + v := dataUsageCache{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodedataUsageCache(b *testing.B) { + v := dataUsageCache{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshaldataUsageCacheInfo(t *testing.T) { + v := dataUsageCacheInfo{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgdataUsageCacheInfo(b *testing.B) { + v := dataUsageCacheInfo{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgdataUsageCacheInfo(b *testing.B) { + v := dataUsageCacheInfo{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshaldataUsageCacheInfo(b *testing.B) { + v := dataUsageCacheInfo{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodedataUsageCacheInfo(t *testing.T) { + v := dataUsageCacheInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodedataUsageCacheInfo Msgsize() is inaccurate") + } + + vn := dataUsageCacheInfo{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodedataUsageCacheInfo(b *testing.B) { + v := dataUsageCacheInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodedataUsageCacheInfo(b *testing.B) { + v := dataUsageCacheInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshaldataUsageEntry(t *testing.T) { + v := dataUsageEntry{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgdataUsageEntry(b *testing.B) { + v := dataUsageEntry{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgdataUsageEntry(b *testing.B) { + v := dataUsageEntry{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshaldataUsageEntry(b *testing.B) { + v := dataUsageEntry{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodedataUsageEntry(t *testing.T) { + v := dataUsageEntry{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodedataUsageEntry Msgsize() is inaccurate") + } + + vn := dataUsageEntry{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodedataUsageEntry(b *testing.B) { + v := dataUsageEntry{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodedataUsageEntry(b *testing.B) { + v := dataUsageEntry{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalsizeHistogram(t *testing.T) { + v := sizeHistogram{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgsizeHistogram(b *testing.B) { + v := sizeHistogram{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgsizeHistogram(b *testing.B) { + v := sizeHistogram{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalsizeHistogram(b *testing.B) { + v := sizeHistogram{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodesizeHistogram(t *testing.T) { + v := sizeHistogram{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodesizeHistogram Msgsize() is inaccurate") + } + + vn := sizeHistogram{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodesizeHistogram(b *testing.B) { + v := sizeHistogram{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodesizeHistogram(b *testing.B) { + v := sizeHistogram{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalsizeHistogramV1(t *testing.T) { + v := sizeHistogramV1{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgsizeHistogramV1(b *testing.B) { + v := sizeHistogramV1{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgsizeHistogramV1(b *testing.B) { + v := sizeHistogramV1{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalsizeHistogramV1(b *testing.B) { + v := sizeHistogramV1{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodesizeHistogramV1(t *testing.T) { + v := sizeHistogramV1{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodesizeHistogramV1 Msgsize() is inaccurate") + } + + vn := sizeHistogramV1{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodesizeHistogramV1(b *testing.B) { + v := sizeHistogramV1{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodesizeHistogramV1(b *testing.B) { + v := sizeHistogramV1{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshaltierStats(t *testing.T) { + v := tierStats{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgtierStats(b *testing.B) { + v := tierStats{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgtierStats(b *testing.B) { + v := tierStats{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshaltierStats(b *testing.B) { + v := tierStats{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodetierStats(t *testing.T) { + v := tierStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodetierStats Msgsize() is inaccurate") + } + + vn := tierStats{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodetierStats(b *testing.B) { + v := tierStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodetierStats(b *testing.B) { + v := tierStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalversionsHistogram(t *testing.T) { + v := versionsHistogram{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgversionsHistogram(b *testing.B) { + v := versionsHistogram{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgversionsHistogram(b *testing.B) { + v := versionsHistogram{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalversionsHistogram(b *testing.B) { + v := versionsHistogram{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeversionsHistogram(t *testing.T) { + v := versionsHistogram{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeversionsHistogram Msgsize() is inaccurate") + } + + vn := versionsHistogram{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeversionsHistogram(b *testing.B) { + v := versionsHistogram{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeversionsHistogram(b *testing.B) { + v := versionsHistogram{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/data-usage-cache_test.go b/cmd/data-usage-cache_test.go new file mode 100644 index 00000000..69fbd9fe --- /dev/null +++ b/cmd/data-usage-cache_test.go @@ -0,0 +1,98 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "testing" + + "github.com/dustin/go-humanize" +) + +func TestSizeHistogramToMap(t *testing.T) { + tests := []struct { + sizes []int64 + want map[string]uint64 + }{ + { + sizes: []int64{100, 1000, 72_000, 100_000}, + want: map[string]uint64{ + "LESS_THAN_1024_B": 2, + "BETWEEN_64_KB_AND_256_KB": 2, + "BETWEEN_1024B_AND_1_MB": 2, + }, + }, + { + sizes: []int64{100, 1000, 2000, 100_000, 13 * humanize.MiByte}, + want: map[string]uint64{ + "LESS_THAN_1024_B": 2, + "BETWEEN_1024_B_AND_64_KB": 1, + "BETWEEN_64_KB_AND_256_KB": 1, + "BETWEEN_1024B_AND_1_MB": 2, + "BETWEEN_10_MB_AND_64_MB": 1, + }, + }, + } + for i, test := range tests { + t.Run(fmt.Sprintf("Test-%d", i), func(t *testing.T) { + var h sizeHistogram + for _, sz := range test.sizes { + h.add(sz) + } + got := h.toMap() + exp := test.want + // what is in exp is in got + for k := range exp { + if exp[k] != got[k] { + t.Fatalf("interval %s: Expected %d values but got %d values\n", k, exp[k], got[k]) + } + } + // what is absent in exp is absent in got too + for k := range got { + if _, ok := exp[k]; !ok && got[k] > 0 { + t.Fatalf("Unexpected interval: %s has value %d\n", k, got[k]) + } + } + }) + } +} + +func TestMigrateSizeHistogramFromV1(t *testing.T) { + tests := []struct { + v sizeHistogramV1 + want sizeHistogram + }{ + { + v: sizeHistogramV1{0: 10, 1: 20, 2: 3}, + want: sizeHistogram{0: 10, 5: 20, 6: 3}, + }, + { + v: sizeHistogramV1{0: 10, 1: 20, 2: 3, 3: 4, 4: 5, 5: 6, 6: 7}, + want: sizeHistogram{0: 10, 5: 20, 6: 3, 7: 4, 8: 5, 9: 6, 10: 7}, + }, + } + for i, test := range tests { + t.Run(fmt.Sprintf("test-%d", i), func(t *testing.T) { + var got sizeHistogram + got.mergeV1(test.v) + if got != test.want { + t.Fatalf("Expected %v but got %v", test.want, got) + } + }) + } +} diff --git a/cmd/data-usage-utils.go b/cmd/data-usage-utils.go new file mode 100644 index 00000000..1f6b3f11 --- /dev/null +++ b/cmd/data-usage-utils.go @@ -0,0 +1,169 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "sort" + "time" + + "github.com/minio/madmin-go/v3" +) + +// BucketTargetUsageInfo - bucket target usage info provides +// - replicated size for all objects sent to this target +// - replica size for all objects received from this target +// - replication pending size for all objects pending replication to this target +// - replication failed size for all objects failed replication to this target +// - replica pending count +// - replica failed count +type BucketTargetUsageInfo struct { + ReplicationPendingSize uint64 `json:"objectsPendingReplicationTotalSize"` + ReplicationFailedSize uint64 `json:"objectsFailedReplicationTotalSize"` + ReplicatedSize uint64 `json:"objectsReplicatedTotalSize"` + ReplicaSize uint64 `json:"objectReplicaTotalSize"` + ReplicationPendingCount uint64 `json:"objectsPendingReplicationCount"` + ReplicationFailedCount uint64 `json:"objectsFailedReplicationCount"` + ReplicatedCount uint64 `json:"objectsReplicatedCount"` +} + +// BucketUsageInfo - bucket usage info provides +// - total size of the bucket +// - total objects in a bucket +// - object size histogram per bucket +type BucketUsageInfo struct { + Size uint64 `json:"size"` + // Following five fields suffixed with V1 are here for backward compatibility + // Total Size for objects that have not yet been replicated + ReplicationPendingSizeV1 uint64 `json:"objectsPendingReplicationTotalSize"` + // Total size for objects that have witness one or more failures and will be retried + ReplicationFailedSizeV1 uint64 `json:"objectsFailedReplicationTotalSize"` + // Total size for objects that have been replicated to destination + ReplicatedSizeV1 uint64 `json:"objectsReplicatedTotalSize"` + // Total number of objects pending replication + ReplicationPendingCountV1 uint64 `json:"objectsPendingReplicationCount"` + // Total number of objects that failed replication + ReplicationFailedCountV1 uint64 `json:"objectsFailedReplicationCount"` + + ObjectsCount uint64 `json:"objectsCount"` + ObjectSizesHistogram map[string]uint64 `json:"objectsSizesHistogram"` + ObjectVersionsHistogram map[string]uint64 `json:"objectsVersionsHistogram"` + VersionsCount uint64 `json:"versionsCount"` + DeleteMarkersCount uint64 `json:"deleteMarkersCount"` + ReplicaSize uint64 `json:"objectReplicaTotalSize"` + ReplicaCount uint64 `json:"objectReplicaCount"` + ReplicationInfo map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"` +} + +// DataUsageInfo represents data usage stats of the underlying Object API +type DataUsageInfo struct { + TotalCapacity uint64 `json:"capacity,omitempty"` + TotalUsedCapacity uint64 `json:"usedCapacity,omitempty"` + TotalFreeCapacity uint64 `json:"freeCapacity,omitempty"` + + // LastUpdate is the timestamp of when the data usage info was last updated. + // This does not indicate a full scan. + LastUpdate time.Time `json:"lastUpdate"` + + // Objects total count across all buckets + ObjectsTotalCount uint64 `json:"objectsCount"` + + // Versions total count across all buckets + VersionsTotalCount uint64 `json:"versionsCount"` + + // Delete markers total count across all buckets + DeleteMarkersTotalCount uint64 `json:"deleteMarkersCount"` + + // Objects total size across all buckets + ObjectsTotalSize uint64 `json:"objectsTotalSize"` + ReplicationInfo map[string]BucketTargetUsageInfo `json:"objectsReplicationInfo"` + + // Total number of buckets in this cluster + BucketsCount uint64 `json:"bucketsCount"` + + // Buckets usage info provides following information across all buckets + // - total size of the bucket + // - total objects in a bucket + // - object size histogram per bucket + BucketsUsage map[string]BucketUsageInfo `json:"bucketsUsageInfo"` + // Deprecated kept here for backward compatibility reasons. + BucketSizes map[string]uint64 `json:"bucketsSizes"` + + // TierStats contains per-tier stats of all configured remote tiers + TierStats *allTierStats `json:"tierStats,omitempty"` +} + +func (dui DataUsageInfo) tierStats() []madmin.TierInfo { + if dui.TierStats == nil { + return nil + } + + if globalTierConfigMgr.Empty() { + return nil + } + + ts := make(map[string]madmin.TierStats) + dui.TierStats.populateStats(ts) + + infos := make([]madmin.TierInfo, 0, len(ts)) + for tier, stats := range ts { + infos = append(infos, madmin.TierInfo{ + Name: tier, + Type: globalTierConfigMgr.TierType(tier), + Stats: stats, + }) + } + + sort.Slice(infos, func(i, j int) bool { + if infos[i].Type == "internal" { + return true + } + if infos[j].Type == "internal" { + return false + } + return infos[i].Name < infos[j].Name + }) + return infos +} + +func (dui DataUsageInfo) tierMetrics() (metrics []MetricV2) { + if dui.TierStats == nil { + return nil + } + // e.g minio_cluster_ilm_transitioned_bytes{tier="S3TIER-1"}=136314880 + // minio_cluster_ilm_transitioned_objects{tier="S3TIER-1"}=1 + // minio_cluster_ilm_transitioned_versions{tier="S3TIER-1"}=3 + for tier, st := range dui.TierStats.Tiers { + metrics = append(metrics, MetricV2{ + Description: getClusterTransitionedBytesMD(), + Value: float64(st.TotalSize), + VariableLabels: map[string]string{"tier": tier}, + }) + metrics = append(metrics, MetricV2{ + Description: getClusterTransitionedObjectsMD(), + Value: float64(st.NumObjects), + VariableLabels: map[string]string{"tier": tier}, + }) + metrics = append(metrics, MetricV2{ + Description: getClusterTransitionedVersionsMD(), + Value: float64(st.NumVersions), + VariableLabels: map[string]string{"tier": tier}, + }) + } + + return metrics +} diff --git a/cmd/data-usage.go b/cmd/data-usage.go new file mode 100644 index 00000000..51227e10 --- /dev/null +++ b/cmd/data-usage.go @@ -0,0 +1,165 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "strings" + "time" + + jsoniter "github.com/json-iterator/go" + "github.com/minio/minio/internal/cachevalue" +) + +const ( + dataUsageRoot = SlashSeparator + dataUsageBucket = minioMetaBucket + SlashSeparator + bucketMetaPrefix + + dataUsageObjName = ".usage.json" + dataUsageObjNamePath = bucketMetaPrefix + SlashSeparator + dataUsageObjName + dataUsageBloomName = ".bloomcycle.bin" + dataUsageBloomNamePath = bucketMetaPrefix + SlashSeparator + dataUsageBloomName + + backgroundHealInfoPath = bucketMetaPrefix + SlashSeparator + ".background-heal.json" + + dataUsageCacheName = ".usage-cache.bin" +) + +// storeDataUsageInBackend will store all objects sent on the dui channel until closed. +func storeDataUsageInBackend(ctx context.Context, objAPI ObjectLayer, dui <-chan DataUsageInfo) { + attempts := 1 + for dataUsageInfo := range dui { + json := jsoniter.ConfigCompatibleWithStandardLibrary + dataUsageJSON, err := json.Marshal(dataUsageInfo) + if err != nil { + scannerLogIf(ctx, err) + continue + } + if attempts > 10 { + saveConfig(ctx, objAPI, dataUsageObjNamePath+".bkp", dataUsageJSON) // Save a backup every 10th update. + attempts = 1 + } + if err = saveConfig(ctx, objAPI, dataUsageObjNamePath, dataUsageJSON); err != nil { + scannerLogOnceIf(ctx, err, dataUsageObjNamePath) + } + attempts++ + } +} + +var prefixUsageCache = cachevalue.New[map[string]uint64]() + +// loadPrefixUsageFromBackend returns prefix usages found in passed buckets +// +// e.g.: /testbucket/prefix => 355601334 +func loadPrefixUsageFromBackend(ctx context.Context, objAPI ObjectLayer, bucket string) (map[string]uint64, error) { + z, ok := objAPI.(*erasureServerPools) + if !ok { + // Prefix usage is empty + return map[string]uint64{}, nil + } + + cache := dataUsageCache{} + + prefixUsageCache.InitOnce(30*time.Second, + // No need to fail upon Update() error, fallback to old value. + cachevalue.Opts{ReturnLastGood: true, NoWait: true}, + func(ctx context.Context) (map[string]uint64, error) { + m := make(map[string]uint64) + for _, pool := range z.serverPools { + for _, er := range pool.sets { + // Load bucket usage prefixes + ctx, done := context.WithTimeout(ctx, 2*time.Second) + ok := cache.load(ctx, er, bucket+slashSeparator+dataUsageCacheName) == nil + done() + if ok { + root := cache.find(bucket) + if root == nil { + // We dont have usage information for this bucket in this + // set, go to the next set + continue + } + + for id, usageInfo := range cache.flattenChildrens(*root) { + prefix := decodeDirObject(strings.TrimPrefix(id, bucket+slashSeparator)) + // decodeDirObject to avoid any __XLDIR__ objects + m[prefix] += uint64(usageInfo.Size) + } + } + } + } + return m, nil + }, + ) + + return prefixUsageCache.GetWithCtx(ctx) +} + +func loadDataUsageFromBackend(ctx context.Context, objAPI ObjectLayer) (DataUsageInfo, error) { + buf, err := readConfig(ctx, objAPI, dataUsageObjNamePath) + if err != nil { + buf, err = readConfig(ctx, objAPI, dataUsageObjNamePath+".bkp") + if err != nil { + if errors.Is(err, errConfigNotFound) { + return DataUsageInfo{}, nil + } + return DataUsageInfo{}, toObjectErr(err, minioMetaBucket, dataUsageObjNamePath) + } + } + + var dataUsageInfo DataUsageInfo + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err = json.Unmarshal(buf, &dataUsageInfo); err != nil { + return DataUsageInfo{}, err + } + // For forward compatibility reasons, we need to add this code. + if len(dataUsageInfo.BucketsUsage) == 0 { + dataUsageInfo.BucketsUsage = make(map[string]BucketUsageInfo, len(dataUsageInfo.BucketSizes)) + for bucket, size := range dataUsageInfo.BucketSizes { + dataUsageInfo.BucketsUsage[bucket] = BucketUsageInfo{Size: size} + } + } + + // For backward compatibility reasons, we need to add this code. + if len(dataUsageInfo.BucketSizes) == 0 { + dataUsageInfo.BucketSizes = make(map[string]uint64, len(dataUsageInfo.BucketsUsage)) + for bucket, bui := range dataUsageInfo.BucketsUsage { + dataUsageInfo.BucketSizes[bucket] = bui.Size + } + } + // For forward compatibility reasons, we need to add this code. + for bucket, bui := range dataUsageInfo.BucketsUsage { + if bui.ReplicatedSizeV1 > 0 || bui.ReplicationFailedCountV1 > 0 || + bui.ReplicationFailedSizeV1 > 0 || bui.ReplicationPendingCountV1 > 0 { + cfg, _ := getReplicationConfig(GlobalContext, bucket) + if cfg != nil && cfg.RoleArn != "" { + if dataUsageInfo.ReplicationInfo == nil { + dataUsageInfo.ReplicationInfo = make(map[string]BucketTargetUsageInfo) + } + dataUsageInfo.ReplicationInfo[cfg.RoleArn] = BucketTargetUsageInfo{ + ReplicationFailedSize: bui.ReplicationFailedSizeV1, + ReplicationFailedCount: bui.ReplicationFailedCountV1, + ReplicatedSize: bui.ReplicatedSizeV1, + ReplicationPendingCount: bui.ReplicationPendingCountV1, + ReplicationPendingSize: bui.ReplicationPendingSizeV1, + } + } + } + } + return dataUsageInfo, nil +} diff --git a/cmd/data-usage_test.go b/cmd/data-usage_test.go new file mode 100644 index 00000000..51a46e65 --- /dev/null +++ b/cmd/data-usage_test.go @@ -0,0 +1,631 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os" + "path" + "path/filepath" + "testing" + "time" + + "github.com/minio/minio/internal/cachevalue" +) + +type usageTestFile struct { + name string + size int +} + +func TestDataUsageUpdate(t *testing.T) { + base := t.TempDir() + const bucket = "bucket" + files := []usageTestFile{ + {name: "rootfile", size: 10000}, + {name: "rootfile2", size: 10000}, + {name: "dir1/d1file", size: 2000}, + {name: "dir2/d2file", size: 300}, + {name: "dir1/dira/dafile", size: 100000}, + {name: "dir1/dira/dbfile", size: 200000}, + {name: "dir1/dira/dirasub/dcfile", size: 1000000}, + {name: "dir1/dira/dirasub/sublevel3/dccccfile", size: 10}, + } + createUsageTestFiles(t, base, bucket, files) + + getSize := func(item scannerItem) (sizeS sizeSummary, err error) { + if item.Typ&os.ModeDir == 0 { + var s os.FileInfo + s, err = os.Stat(item.Path) + if err != nil { + return sizeS, err + } + sizeS.totalSize = s.Size() + sizeS.versions++ + return sizeS, nil + } + return sizeS, err + } + xls := xlStorage{drivePath: base, diskInfoCache: cachevalue.New[DiskInfo]()} + xls.diskInfoCache.InitOnce(time.Second, cachevalue.Opts{}, func(ctx context.Context) (DiskInfo, error) { + return DiskInfo{Total: 1 << 40, Free: 1 << 40}, nil + }) + weSleep := func() bool { return false } + + got, err := scanDataFolder(t.Context(), nil, &xls, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize, 0, weSleep) + if err != nil { + t.Fatal(err) + } + + // Test dirs + want := []struct { + path string + isNil bool + size, objs int + flatten bool + oSizes sizeHistogram + }{ + { + path: "/", + size: 1322310, + flatten: true, + objs: 8, + oSizes: sizeHistogram{0: 2, 1: 3, 2: 2, 4: 1}, + }, + { + path: "/", + size: 20000, + objs: 2, + oSizes: sizeHistogram{1: 2}, + }, + { + path: "/dir1", + size: 1302010, + objs: 5, + oSizes: sizeHistogram{0: 1, 1: 1, 2: 2, 4: 1}, + }, + { + path: "/dir1/dira", + isNil: true, + }, + { + path: "/nonexistying", + isNil: true, + }, + } + + for _, w := range want { + p := path.Join(bucket, w.path) + t.Run(p, func(t *testing.T) { + e := got.find(p) + if w.isNil { + if e != nil { + t.Error("want nil, got", e) + } + return + } + if e == nil { + t.Fatal("got nil result") + } + if w.flatten { + *e = got.flatten(*e) + } + if e.Size != int64(w.size) { + t.Error("got size", e.Size, "want", w.size) + } + if e.Objects != uint64(w.objs) { + t.Error("got objects", e.Objects, "want", w.objs) + } + if e.Versions != uint64(w.objs) { + t.Error("got versions", e.Versions, "want", w.objs) + } + if e.ObjSizes != w.oSizes { + t.Error("got histogram", e.ObjSizes, "want", w.oSizes) + } + }) + } + + files = []usageTestFile{ + { + name: "newfolder/afile", + size: 4, + }, + { + name: "newfolder/anotherone", + size: 1, + }, + { + name: "newfolder/anemptyone", + size: 0, + }, + { + name: "dir1/fileindir1", + size: 20000, + }, + { + name: "dir1/dirc/fileindirc", + size: 20000, + }, + { + name: "rootfile3", + size: 1000, + }, + { + name: "dir1/dira/dirasub/fileindira2", + size: 200, + }, + } + createUsageTestFiles(t, base, bucket, files) + err = os.RemoveAll(filepath.Join(base, bucket, "dir1/dira/dirasub/dcfile")) + if err != nil { + t.Fatal(err) + } + // Changed dir must be picked up in this many cycles. + for range dataUsageUpdateDirCycles { + got, err = scanDataFolder(t.Context(), nil, &xls, got, getSize, 0, weSleep) + got.Info.NextCycle++ + if err != nil { + t.Fatal(err) + } + } + + want = []struct { + path string + isNil bool + size, objs int + flatten bool + oSizes sizeHistogram + }{ + { + path: "/", + size: 363515, + flatten: true, + objs: 14, + oSizes: sizeHistogram{0: 7, 1: 5, 2: 2}, + }, + { + path: "/dir1", + size: 342210, + objs: 7, + flatten: false, + oSizes: sizeHistogram{0: 2, 1: 3, 2: 2}, + }, + { + path: "/newfolder", + size: 5, + objs: 3, + oSizes: sizeHistogram{0: 3}, + }, + { + path: "/nonexistying", + isNil: true, + }, + } + + for _, w := range want { + p := path.Join(bucket, w.path) + t.Run(p, func(t *testing.T) { + e := got.find(p) + if w.isNil { + if e != nil { + t.Error("want nil, got", e) + } + return + } + if e == nil { + t.Fatal("got nil result") + } + if w.flatten { + *e = got.flatten(*e) + } + if e.Size != int64(w.size) { + t.Error("got size", e.Size, "want", w.size) + } + if e.Objects != uint64(w.objs) { + t.Error("got objects", e.Objects, "want", w.objs) + } + if e.Versions != uint64(w.objs) { + t.Error("got versions", e.Versions, "want", w.objs) + } + if e.ObjSizes != w.oSizes { + t.Error("got histogram", e.ObjSizes, "want", w.oSizes) + } + }) + } +} + +func TestDataUsageUpdatePrefix(t *testing.T) { + base := t.TempDir() + scannerSleeper.Update(0, 0) + files := []usageTestFile{ + {name: "bucket/rootfile", size: 10000}, + {name: "bucket/rootfile2", size: 10000}, + {name: "bucket/dir1/d1file", size: 2000}, + {name: "bucket/dir2/d2file", size: 300}, + {name: "bucket/dir1/dira/dafile", size: 100000}, + {name: "bucket/dir1/dira/dbfile", size: 200000}, + {name: "bucket/dir1/dira/dirasub/dcfile", size: 1000000}, + {name: "bucket/dir1/dira/dirasub/sublevel3/dccccfile", size: 10}, + } + createUsageTestFiles(t, base, "", files) + const foldersBelow = 3 + const filesBelowT = dataScannerCompactLeastObject / 2 + const filesAboveT = dataScannerCompactAtFolders + 1 + const expectSize = foldersBelow*filesBelowT + filesAboveT + + generateUsageTestFiles(t, base, "bucket/dirwithalot", foldersBelow, filesBelowT, 1) + generateUsageTestFiles(t, base, "bucket/dirwithevenmore", filesAboveT, 1, 1) + + getSize := func(item scannerItem) (sizeS sizeSummary, err error) { + if item.Typ&os.ModeDir == 0 { + var s os.FileInfo + s, err = os.Stat(item.Path) + if err != nil { + return sizeS, err + } + sizeS.totalSize = s.Size() + sizeS.versions++ + return sizeS, err + } + return sizeS, err + } + + weSleep := func() bool { return false } + xls := xlStorage{drivePath: base, diskInfoCache: cachevalue.New[DiskInfo]()} + xls.diskInfoCache.InitOnce(time.Second, cachevalue.Opts{}, func(ctx context.Context) (DiskInfo, error) { + return DiskInfo{Total: 1 << 40, Free: 1 << 40}, nil + }) + + got, err := scanDataFolder(t.Context(), nil, &xls, dataUsageCache{Info: dataUsageCacheInfo{Name: "bucket"}}, getSize, 0, weSleep) + if err != nil { + t.Fatal(err) + } + if got.root() == nil { + t.Log("cached folders:") + for folder := range got.Cache { + t.Log("folder:", folder) + } + t.Fatal("got nil root.") + } + + // Test dirs + want := []struct { + path string + isNil bool + size, objs int + oSizes sizeHistogram + }{ + { + path: "flat", + size: 1322310 + expectSize, + objs: 8 + expectSize, + oSizes: sizeHistogram{0: 2 + expectSize, 1: 3, 2: 2, 4: 1}, + }, + { + path: "bucket/", + size: 20000, + objs: 2, + oSizes: sizeHistogram{1: 2}, + }, + { + // Gets compacted... + path: "bucket/dir1", + size: 1302010, + objs: 5, + oSizes: sizeHistogram{0: 1, 1: 1, 2: 2, 4: 1}, + }, + { + // Gets compacted at this level... + path: "bucket/dirwithalot/0", + size: filesBelowT, + objs: filesBelowT, + oSizes: sizeHistogram{0: filesBelowT}, + }, + { + // Gets compacted at this level (below obj threshold)... + path: "bucket/dirwithalot/0", + size: filesBelowT, + objs: filesBelowT, + oSizes: sizeHistogram{0: filesBelowT}, + }, + { + // Gets compacted at this level... + path: "bucket/dirwithevenmore", + size: filesAboveT, + objs: filesAboveT, + oSizes: sizeHistogram{0: filesAboveT}, + }, + { + path: "bucket/nonexistying", + isNil: true, + }, + } + + for _, w := range want { + t.Run(w.path, func(t *testing.T) { + e := got.find(w.path) + if w.path == "flat" { + f := got.flatten(*got.root()) + e = &f + } + if w.isNil { + if e != nil { + t.Error("want nil, got", e) + } + return + } + if e == nil { + t.Fatal("got nil result") + return + } + if e.Size != int64(w.size) { + t.Error("got size", e.Size, "want", w.size) + } + if e.Objects != uint64(w.objs) { + t.Error("got objects", e.Objects, "want", w.objs) + } + if e.Versions != uint64(w.objs) { + t.Error("got versions", e.Versions, "want", w.objs) + } + if e.ObjSizes != w.oSizes { + t.Error("got histogram", e.ObjSizes, "want", w.oSizes) + } + }) + } + + files = []usageTestFile{ + { + name: "bucket/newfolder/afile", + size: 4, + }, + { + name: "bucket/newfolder/anotherone", + size: 1, + }, + { + name: "bucket/newfolder/anemptyone", + size: 0, + }, + { + name: "bucket/dir1/fileindir1", + size: 20000, + }, + { + name: "bucket/dir1/dirc/fileindirc", + size: 20000, + }, + { + name: "bucket/rootfile3", + size: 1000, + }, + { + name: "bucket/dir1/dira/dirasub/fileindira2", + size: 200, + }, + } + + createUsageTestFiles(t, base, "", files) + err = os.RemoveAll(filepath.Join(base, "bucket/dir1/dira/dirasub/dcfile")) + if err != nil { + t.Fatal(err) + } + // Changed dir must be picked up in this many cycles. + for range dataUsageUpdateDirCycles { + got, err = scanDataFolder(t.Context(), nil, &xls, got, getSize, 0, weSleep) + got.Info.NextCycle++ + if err != nil { + t.Fatal(err) + } + } + + want = []struct { + path string + isNil bool + size, objs int + oSizes sizeHistogram + }{ + { + path: "flat", + size: 363515 + expectSize, + objs: 14 + expectSize, + oSizes: sizeHistogram{0: 7 + expectSize, 1: 5, 2: 2}, + }, + { + path: "bucket/dir1", + size: 342210, + objs: 7, + oSizes: sizeHistogram{0: 2, 1: 3, 2: 2}, + }, + { + path: "bucket/", + size: 21000, + objs: 3, + oSizes: sizeHistogram{0: 1, 1: 2}, + }, + { + path: "bucket/newfolder", + size: 5, + objs: 3, + oSizes: sizeHistogram{0: 3}, + }, + { + // Compacted into bucket/dir1 + path: "bucket/dir1/dira", + isNil: true, + }, + { + path: "bucket/nonexistying", + isNil: true, + }, + } + + for _, w := range want { + t.Run(w.path, func(t *testing.T) { + e := got.find(w.path) + if w.path == "flat" { + f := got.flatten(*got.root()) + e = &f + } + if w.isNil { + if e != nil { + t.Error("want nil, got", e) + } + return + } + if e == nil { + t.Error("got nil result") + return + } + if e.Size != int64(w.size) { + t.Error("got size", e.Size, "want", w.size) + } + if e.Objects != uint64(w.objs) { + t.Error("got objects", e.Objects, "want", w.objs) + } + if e.Versions != uint64(w.objs) { + t.Error("got versions", e.Versions, "want", w.objs) + } + if e.ObjSizes != w.oSizes { + t.Error("got histogram", e.ObjSizes, "want", w.oSizes) + } + }) + } +} + +func createUsageTestFiles(t *testing.T, base, bucket string, files []usageTestFile) { + for _, f := range files { + err := os.MkdirAll(filepath.Dir(filepath.Join(base, bucket, f.name)), os.ModePerm) + if err != nil { + t.Fatal(err) + } + err = os.WriteFile(filepath.Join(base, bucket, f.name), make([]byte, f.size), os.ModePerm) + if err != nil { + t.Fatal(err) + } + } +} + +// generateUsageTestFiles create nFolders * nFiles files of size bytes each. +func generateUsageTestFiles(t *testing.T, base, bucket string, nFolders, nFiles, size int) { + pl := make([]byte, size) + for i := range nFolders { + name := filepath.Join(base, bucket, fmt.Sprint(i), "0.txt") + err := os.MkdirAll(filepath.Dir(name), os.ModePerm) + if err != nil { + t.Fatal(err) + } + for j := range nFiles { + name := filepath.Join(base, bucket, fmt.Sprint(i), fmt.Sprint(j)+".txt") + err = os.WriteFile(name, pl, os.ModePerm) + if err != nil { + t.Fatal(err) + } + } + } +} + +func TestDataUsageCacheSerialize(t *testing.T) { + base := t.TempDir() + const bucket = "abucket" + files := []usageTestFile{ + {name: "rootfile", size: 10000}, + {name: "rootfile2", size: 10000}, + {name: "dir1/d1file", size: 2000}, + {name: "dir2/d2file", size: 300}, + {name: "dir2/d2file2", size: 300}, + {name: "dir2/d2file3/", size: 300}, + {name: "dir2/d2file4/", size: 300}, + {name: "dir2/d2file5", size: 300}, + {name: "dir1/dira/dafile", size: 100000}, + {name: "dir1/dira/dbfile", size: 200000}, + {name: "dir1/dira/dirasub/dcfile", size: 1000000}, + {name: "dir1/dira/dirasub/sublevel3/dccccfile", size: 10}, + {name: "dir1/dira/dirasub/sublevel3/dccccfile20", size: 20}, + {name: "dir1/dira/dirasub/sublevel3/dccccfile30", size: 30}, + {name: "dir1/dira/dirasub/sublevel3/dccccfile40", size: 40}, + } + createUsageTestFiles(t, base, bucket, files) + + getSize := func(item scannerItem) (sizeS sizeSummary, err error) { + if item.Typ&os.ModeDir == 0 { + var s os.FileInfo + s, err = os.Stat(item.Path) + if err != nil { + return sizeS, err + } + sizeS.versions++ + sizeS.totalSize = s.Size() + return sizeS, err + } + return sizeS, err + } + xls := xlStorage{drivePath: base, diskInfoCache: cachevalue.New[DiskInfo]()} + xls.diskInfoCache.InitOnce(time.Second, cachevalue.Opts{}, func(ctx context.Context) (DiskInfo, error) { + return DiskInfo{Total: 1 << 40, Free: 1 << 40}, nil + }) + weSleep := func() bool { return false } + want, err := scanDataFolder(t.Context(), nil, &xls, dataUsageCache{Info: dataUsageCacheInfo{Name: bucket}}, getSize, 0, weSleep) + if err != nil { + t.Fatal(err) + } + e := want.find("abucket/dir2") + want.replace("abucket/dir2", "", *e) + var buf bytes.Buffer + err = want.serializeTo(&buf) + if err != nil { + t.Fatal(err) + } + t.Log("serialized size:", buf.Len(), "bytes") + var got dataUsageCache + err = got.deserialize(&buf) + if err != nil { + t.Fatal(err) + } + if got.Info.LastUpdate.IsZero() { + t.Error("lastupdate not set") + } + + if !want.Info.LastUpdate.Equal(got.Info.LastUpdate) { + t.Fatalf("deserialize LastUpdate mismatch\nwant: %+v\ngot: %+v", want, got) + } + if len(want.Cache) != len(got.Cache) { + t.Errorf("deserialize mismatch length\nwant: %+v\ngot: %+v", len(want.Cache), len(got.Cache)) + } + for wkey, wval := range want.Cache { + gotv := got.Cache[wkey] + if !equalAsJSON(gotv, wval) { + t.Errorf("deserialize mismatch, key %v\nwant: %#v\ngot: %#v", wkey, wval, gotv) + } + } +} + +// equalAsJSON returns whether the values are equal when encoded as JSON. +func equalAsJSON(a, b any) bool { + aj, err := json.Marshal(a) + if err != nil { + panic(err) + } + bj, err := json.Marshal(b) + if err != nil { + panic(err) + } + return bytes.Equal(aj, bj) +} diff --git a/cmd/decommetric_string.go b/cmd/decommetric_string.go new file mode 100644 index 00000000..fb485ab0 --- /dev/null +++ b/cmd/decommetric_string.go @@ -0,0 +1,25 @@ +// Code generated by "stringer -type=decomMetric -trimprefix=decomMetric erasure-server-pool-decom.go"; DO NOT EDIT. + +package cmd + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[decomMetricDecommissionBucket-0] + _ = x[decomMetricDecommissionObject-1] + _ = x[decomMetricDecommissionRemoveObject-2] +} + +const _decomMetric_name = "DecommissionBucketDecommissionObjectDecommissionRemoveObject" + +var _decomMetric_index = [...]uint8{0, 18, 36, 60} + +func (i decomMetric) String() string { + if i >= decomMetric(len(_decomMetric_index)-1) { + return "decomMetric(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _decomMetric_name[_decomMetric_index[i]:_decomMetric_index[i+1]] +} diff --git a/cmd/dummy-data-generator_test.go b/cmd/dummy-data-generator_test.go new file mode 100644 index 00000000..07400b30 --- /dev/null +++ b/cmd/dummy-data-generator_test.go @@ -0,0 +1,181 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "errors" + "fmt" + "io" + "testing" +) + +var alphabets = []byte("abcdefghijklmnopqrstuvwxyz0123456789") + +// DummyDataGen returns a reader that repeats the bytes in `alphabets` +// upto the desired length. +type DummyDataGen struct { + b []byte + idx, length int64 +} + +// NewDummyDataGen returns a ReadSeeker over the first `totalLength` +// bytes from the infinite stream consisting of repeated +// concatenations of `alphabets`. +// +// The skipOffset (generally = 0) can be used to skip a given number +// of bytes from the beginning of the infinite stream. This is useful +// to compare such streams of bytes that may be split up, because: +// +// Given the function: +// +// f := func(r io.Reader) string { +// b, _ := io.ReadAll(r) +// return string(b) +// } +// +// for example, the following is true: +// +// f(NewDummyDataGen(100, 0)) == f(NewDummyDataGen(50, 0)) + f(NewDummyDataGen(50, 50)) +func NewDummyDataGen(totalLength, skipOffset int64) io.ReadSeeker { + if totalLength < 0 { + panic("Negative length passed to DummyDataGen!") + } + if skipOffset < 0 { + panic("Negative rotations are not allowed") + } + + skipOffset %= int64(len(alphabets)) + const multiply = 100 + as := bytes.Repeat(alphabets, multiply) + b := as[skipOffset : skipOffset+int64(len(alphabets)*(multiply-1))] + return &DummyDataGen{ + length: totalLength, + b: b, + } +} + +func (d *DummyDataGen) Read(b []byte) (n int, err error) { + k := len(b) + numLetters := int64(len(d.b)) + for k > 0 && d.idx < d.length { + w := copy(b[len(b)-k:], d.b[d.idx%numLetters:]) + k -= w + d.idx += int64(w) + n += w + } + if d.idx >= d.length { + extraBytes := d.idx - d.length + n -= int(extraBytes) + if n < 0 { + n = 0 + } + err = io.EOF + } + return n, err +} + +func (d *DummyDataGen) Seek(offset int64, whence int) (int64, error) { + switch whence { + case io.SeekStart: + if offset < 0 { + return 0, errors.New("Invalid offset") + } + d.idx = offset + case io.SeekCurrent: + if d.idx+offset < 0 { + return 0, errors.New("Invalid offset") + } + d.idx += offset + case io.SeekEnd: + if d.length+offset < 0 { + return 0, errors.New("Invalid offset") + } + d.idx = d.length + offset + } + return d.idx, nil +} + +func TestDummyDataGenerator(t *testing.T) { + readAll := func(r io.Reader) string { + b, _ := io.ReadAll(r) + return string(b) + } + checkEq := func(a, b string) { + if a != b { + t.Fatalf("Unexpected equality failure") + } + } + + checkEq(readAll(NewDummyDataGen(0, 0)), "") + + checkEq(readAll(NewDummyDataGen(10, 0)), readAll(NewDummyDataGen(10, int64(len(alphabets))))) + + checkEq(readAll(NewDummyDataGen(100, 0)), readAll(NewDummyDataGen(50, 0))+readAll(NewDummyDataGen(50, 50))) + + r := NewDummyDataGen(100, 0) + r.Seek(int64(len(alphabets)), 0) + checkEq(readAll(r), readAll(NewDummyDataGen(100-int64(len(alphabets)), 0))) +} + +// Compares all the bytes returned by the given readers. Any Read +// errors cause a `false` result. A string describing the error is +// also returned. +func cmpReaders(r1, r2 io.Reader) (bool, string) { + bufLen := 32 * 1024 + b1, b2 := make([]byte, bufLen), make([]byte, bufLen) + for i := 0; true; i++ { + n1, e1 := io.ReadFull(r1, b1) + n2, e2 := io.ReadFull(r2, b2) + if n1 != n2 { + return false, fmt.Sprintf("Read %d != %d bytes from the readers", n1, n2) + } + if !bytes.Equal(b1[:n1], b2[:n2]) { + return false, fmt.Sprintf("After reading %d equal buffers (32Kib each), we got the following two strings:\n%v\n%v\n", + i, b1, b2) + } + // Check if stream has ended + if (e1 == io.ErrUnexpectedEOF && e2 == io.ErrUnexpectedEOF) || (e1 == io.EOF && e2 == io.EOF) { + break + } + if e1 != nil || e2 != nil { + return false, fmt.Sprintf("Got unexpected error values: %v == %v", e1, e2) + } + } + return true, "" +} + +func TestCmpReaders(t *testing.T) { + { + r1 := bytes.NewReader([]byte("abc")) + r2 := bytes.NewReader([]byte("abc")) + ok, msg := cmpReaders(r1, r2) + if !ok || msg != "" { + t.Fatalf("unexpected") + } + } + + { + r1 := bytes.NewReader([]byte("abc")) + r2 := bytes.NewReader([]byte("abcd")) + ok, _ := cmpReaders(r1, r2) + if ok { + t.Fatalf("unexpected") + } + } +} diff --git a/cmd/dummy-handlers.go b/cmd/dummy-handlers.go new file mode 100644 index 00000000..685b7925 --- /dev/null +++ b/cmd/dummy-handlers.go @@ -0,0 +1,257 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "net/http" + + "github.com/minio/minio/internal/logger" + "github.com/minio/mux" + "github.com/minio/pkg/v3/policy" +) + +// Data types used for returning dummy tagging XML. +// These variables shouldn't be used elsewhere. +// They are only defined to be used in this file alone. + +// GetBucketWebsite - GET bucket website, a dummy api +func (api objectAPIHandlers) GetBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetBucketWebsite") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + // Allow GetBucketWebsite if policy action is set, since this is a dummy call + // we are simply re-purposing the bucketPolicyAction. + if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Validate if bucket exists, before proceeding further... + _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNoSuchWebsiteConfiguration), r.URL) +} + +// GetBucketAccelerate - GET bucket accelerate, a dummy api +func (api objectAPIHandlers) GetBucketAccelerateHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetBucketAccelerate") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + // Allow GetBucketAccelerate if policy action is set, since this is a dummy call + // we are simply re-purposing the bucketPolicyAction. + if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Validate if bucket exists, before proceeding further... + _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + const accelerateDefaultConfig = `` + writeSuccessResponseXML(w, []byte(accelerateDefaultConfig)) +} + +// GetBucketRequestPaymentHandler - GET bucket requestPayment, a dummy api +func (api objectAPIHandlers) GetBucketRequestPaymentHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetBucketRequestPayment") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + // Allow GetBucketRequestPaymentHandler if policy action is set, since this is a dummy call + // we are simply re-purposing the bucketPolicyAction. + if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Validate if bucket exists, before proceeding further... + _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + const requestPaymentDefaultConfig = `BucketOwner` + + writeSuccessResponseXML(w, []byte(requestPaymentDefaultConfig)) +} + +// GetBucketLoggingHandler - GET bucket logging, a dummy api +func (api objectAPIHandlers) GetBucketLoggingHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetBucketLogging") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + // Allow GetBucketLoggingHandler if policy action is set, since this is a dummy call + // we are simply re-purposing the bucketPolicyAction. + if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketPolicyAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Validate if bucket exists, before proceeding further... + _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + const loggingDefaultConfig = `` + writeSuccessResponseXML(w, []byte(loggingDefaultConfig)) +} + +// DeleteBucketWebsiteHandler - DELETE bucket website, a dummy api +func (api objectAPIHandlers) DeleteBucketWebsiteHandler(w http.ResponseWriter, r *http.Request) { + writeSuccessResponseHeadersOnly(w) +} + +// GetBucketCorsHandler - GET bucket cors, a dummy api +func (api objectAPIHandlers) GetBucketCorsHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetBucketCors") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.GetBucketCorsAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Validate if bucket exists, before proceeding further... + _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNoSuchCORSConfiguration), r.URL) +} + +// PutBucketCorsHandler - PUT bucket cors, a dummy api +func (api objectAPIHandlers) PutBucketCorsHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PutBucketCors") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.PutBucketCorsAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Validate if bucket exists, before proceeding further... + _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) +} + +// DeleteBucketCorsHandler - DELETE bucket cors, a dummy api +func (api objectAPIHandlers) DeleteBucketCorsHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "DeleteBucketCors") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.DeleteBucketCorsAction, bucket, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Validate if bucket exists, before proceeding further... + _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) +} diff --git a/cmd/dynamic-timeouts.go b/cmd/dynamic-timeouts.go new file mode 100644 index 00000000..9c4f297b --- /dev/null +++ b/cmd/dynamic-timeouts.go @@ -0,0 +1,155 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "math" + "sync" + "sync/atomic" + "time" +) + +const ( + dynamicTimeoutIncreaseThresholdPct = 0.33 // Upper threshold for failures in order to increase timeout + dynamicTimeoutDecreaseThresholdPct = 0.10 // Lower threshold for failures in order to decrease timeout + dynamicTimeoutLogSize = 16 + maxDuration = math.MaxInt64 + maxDynamicTimeout = 24 * time.Hour // Never set timeout bigger than this. +) + +// timeouts that are dynamically adapted based on actual usage results +type dynamicTimeout struct { + timeout int64 + minimum int64 + entries int64 + log [dynamicTimeoutLogSize]time.Duration + mutex sync.Mutex + retryInterval time.Duration +} + +type dynamicTimeoutOpts struct { + timeout time.Duration + minimum time.Duration + retryInterval time.Duration +} + +func newDynamicTimeoutWithOpts(opts dynamicTimeoutOpts) *dynamicTimeout { + dt := newDynamicTimeout(opts.timeout, opts.minimum) + dt.retryInterval = opts.retryInterval + return dt +} + +// newDynamicTimeout returns a new dynamic timeout initialized with timeout value +func newDynamicTimeout(timeout, minimum time.Duration) *dynamicTimeout { + if timeout <= 0 || minimum <= 0 { + panic("newDynamicTimeout: negative or zero timeout") + } + if minimum > timeout { + minimum = timeout + } + return &dynamicTimeout{timeout: int64(timeout), minimum: int64(minimum)} +} + +// Timeout returns the current timeout value +func (dt *dynamicTimeout) Timeout() time.Duration { + return time.Duration(atomic.LoadInt64(&dt.timeout)) +} + +func (dt *dynamicTimeout) RetryInterval() time.Duration { + return dt.retryInterval +} + +// LogSuccess logs the duration of a successful action that +// did not hit the timeout +func (dt *dynamicTimeout) LogSuccess(duration time.Duration) { + dt.logEntry(duration) +} + +// LogFailure logs an action that hit the timeout +func (dt *dynamicTimeout) LogFailure() { + dt.logEntry(maxDuration) +} + +// logEntry stores a log entry +func (dt *dynamicTimeout) logEntry(duration time.Duration) { + if duration < 0 { + return + } + entries := int(atomic.AddInt64(&dt.entries, 1)) + index := entries - 1 + if index < dynamicTimeoutLogSize { + dt.mutex.Lock() + dt.log[index] = duration + + // We leak entries while we copy + if entries == dynamicTimeoutLogSize { + // Make copy on stack in order to call adjust() + logCopy := [dynamicTimeoutLogSize]time.Duration{} + copy(logCopy[:], dt.log[:]) + + // reset log entries + atomic.StoreInt64(&dt.entries, 0) + dt.mutex.Unlock() + + dt.adjust(logCopy) + return + } + dt.mutex.Unlock() + } +} + +// adjust changes the value of the dynamic timeout based on the +// previous results +func (dt *dynamicTimeout) adjust(entries [dynamicTimeoutLogSize]time.Duration) { + failures, maxDur := 0, time.Duration(0) + for _, dur := range entries[:] { + if dur == maxDuration { + failures++ + } else if dur > maxDur { + maxDur = dur + } + } + + failPct := float64(failures) / float64(len(entries)) + + if failPct > dynamicTimeoutIncreaseThresholdPct { + // We are hitting the timeout too often, so increase the timeout by 25% + timeout := min( + // Set upper cap. + atomic.LoadInt64(&dt.timeout)*125/100, int64(maxDynamicTimeout)) + // Safety, shouldn't happen + if timeout < dt.minimum { + timeout = dt.minimum + } + atomic.StoreInt64(&dt.timeout, timeout) + } else if failPct < dynamicTimeoutDecreaseThresholdPct { + // We are hitting the timeout relatively few times, + // so decrease the timeout towards 25 % of maximum time spent. + maxDur = maxDur * 125 / 100 + + timeout := atomic.LoadInt64(&dt.timeout) + if maxDur < time.Duration(timeout) { + // Move 50% toward the max. + timeout = (int64(maxDur) + timeout) / 2 + } + if timeout < dt.minimum { + timeout = dt.minimum + } + atomic.StoreInt64(&dt.timeout, timeout) + } +} diff --git a/cmd/dynamic-timeouts_test.go b/cmd/dynamic-timeouts_test.go new file mode 100644 index 00000000..b353b983 --- /dev/null +++ b/cmd/dynamic-timeouts_test.go @@ -0,0 +1,214 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "math/rand" + "runtime" + "sync" + "testing" + "time" +) + +func TestDynamicTimeoutSingleIncrease(t *testing.T) { + timeout := newDynamicTimeout(time.Minute, time.Second) + + initial := timeout.Timeout() + + for range dynamicTimeoutLogSize { + timeout.LogFailure() + } + + adjusted := timeout.Timeout() + + if initial >= adjusted { + t.Errorf("Failure to increase timeout, expected %v to be more than %v", adjusted, initial) + } +} + +func TestDynamicTimeoutDualIncrease(t *testing.T) { + timeout := newDynamicTimeout(time.Minute, time.Second) + + initial := timeout.Timeout() + + for range dynamicTimeoutLogSize { + timeout.LogFailure() + } + + adjusted := timeout.Timeout() + + for range dynamicTimeoutLogSize { + timeout.LogFailure() + } + + adjustedAgain := timeout.Timeout() + + if initial >= adjusted || adjusted >= adjustedAgain { + t.Errorf("Failure to increase timeout multiple times") + } +} + +func TestDynamicTimeoutSingleDecrease(t *testing.T) { + timeout := newDynamicTimeout(time.Minute, time.Second) + + initial := timeout.Timeout() + + for range dynamicTimeoutLogSize { + timeout.LogSuccess(20 * time.Second) + } + + adjusted := timeout.Timeout() + + if initial <= adjusted { + t.Errorf("Failure to decrease timeout, expected %v to be less than %v", adjusted, initial) + } +} + +func TestDynamicTimeoutDualDecrease(t *testing.T) { + timeout := newDynamicTimeout(time.Minute, time.Second) + + initial := timeout.Timeout() + + for range dynamicTimeoutLogSize { + timeout.LogSuccess(20 * time.Second) + } + + adjusted := timeout.Timeout() + + for range dynamicTimeoutLogSize { + timeout.LogSuccess(20 * time.Second) + } + + adjustedAgain := timeout.Timeout() + + if initial <= adjusted || adjusted <= adjustedAgain { + t.Errorf("Failure to decrease timeout multiple times, initial: %v, adjusted: %v, again: %v", initial, adjusted, adjustedAgain) + } +} + +func TestDynamicTimeoutManyDecreases(t *testing.T) { + timeout := newDynamicTimeout(time.Minute, time.Second) + + initial := timeout.Timeout() + + const successTimeout = 20 * time.Second + for range 100 { + for range dynamicTimeoutLogSize { + timeout.LogSuccess(successTimeout) + } + } + + adjusted := timeout.Timeout() + // Check whether eventual timeout is between initial value and success timeout + if initial <= adjusted || adjusted <= successTimeout { + t.Errorf("Failure to decrease timeout appropriately") + } +} + +func TestDynamicTimeoutConcurrent(t *testing.T) { + // Race test. + timeout := newDynamicTimeout(time.Second, time.Millisecond) + var wg sync.WaitGroup + for i := 0; i < runtime.GOMAXPROCS(0); i++ { + wg.Add(1) + rng := rand.New(rand.NewSource(int64(i))) + go func() { + defer wg.Done() + for range 100 { + for range 100 { + timeout.LogSuccess(time.Duration(float64(time.Second) * rng.Float64())) + } + to := timeout.Timeout() + if to < time.Millisecond || to > time.Second { + panic(to) + } + } + }() + } + wg.Wait() +} + +func TestDynamicTimeoutHitMinimum(t *testing.T) { + const minimum = 30 * time.Second + timeout := newDynamicTimeout(time.Minute, minimum) + + initial := timeout.Timeout() + + const successTimeout = 20 * time.Second + for range 100 { + for range dynamicTimeoutLogSize { + timeout.LogSuccess(successTimeout) + } + } + + adjusted := timeout.Timeout() + // Check whether eventual timeout has hit the minimum value + if initial <= adjusted || adjusted != minimum { + t.Errorf("Failure to decrease timeout appropriately") + } +} + +func testDynamicTimeoutAdjust(t *testing.T, timeout *dynamicTimeout, f func() float64) { + const successTimeout = 20 * time.Second + + for range dynamicTimeoutLogSize { + rnd := f() + duration := max(time.Duration(float64(successTimeout)*rnd), 100*time.Millisecond) + if duration >= time.Minute { + timeout.LogFailure() + } else { + timeout.LogSuccess(duration) + } + } +} + +func TestDynamicTimeoutAdjustExponential(t *testing.T) { + timeout := newDynamicTimeout(time.Minute, time.Second) + + rand.Seed(0) + + initial := timeout.Timeout() + + for range 10 { + testDynamicTimeoutAdjust(t, timeout, rand.ExpFloat64) + } + + adjusted := timeout.Timeout() + if initial <= adjusted { + t.Errorf("Failure to decrease timeout, expected %v to be less than %v", adjusted, initial) + } +} + +func TestDynamicTimeoutAdjustNormalized(t *testing.T) { + timeout := newDynamicTimeout(time.Minute, time.Second) + + rand.Seed(0) + + initial := timeout.Timeout() + + for range 10 { + testDynamicTimeoutAdjust(t, timeout, func() float64 { + return 1.0 + rand.NormFloat64() + }) + } + + adjusted := timeout.Timeout() + if initial <= adjusted { + t.Errorf("Failure to decrease timeout, expected %v to be less than %v", adjusted, initial) + } +} diff --git a/cmd/encryption-v1.go b/cmd/encryption-v1.go new file mode 100644 index 00000000..c3da051a --- /dev/null +++ b/cmd/encryption-v1.go @@ -0,0 +1,1200 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bufio" + "bytes" + "context" + "crypto/hmac" + "crypto/rand" + "crypto/subtle" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "io" + "maps" + "net/http" + "path" + "strconv" + "strings" + + "github.com/minio/kms-go/kes" + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/etag" + "github.com/minio/minio/internal/hash" + "github.com/minio/minio/internal/hash/sha256" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/kms" + "github.com/minio/minio/internal/logger" + "github.com/minio/sio" +) + +var ( + // AWS errors for invalid SSE-C requests. + errEncryptedObject = errors.New("The object was stored using a form of SSE") + errInvalidSSEParameters = errors.New("The SSE-C key for key-rotation is not correct") // special access denied + errKMSNotConfigured = errors.New("KMS not configured for a server side encrypted objects") + errKMSKeyNotFound = errors.New("Unknown KMS key ID") + errKMSDefaultKeyAlreadyConfigured = errors.New("A default encryption already exists on KMS") + // Additional MinIO errors for SSE-C requests. + errObjectTampered = errors.New("The requested object was modified and may be compromised") + // error returned when invalid encryption parameters are specified + errInvalidEncryptionParameters = errors.New("The encryption parameters are not applicable to this object") + errInvalidEncryptionParametersSSEC = errors.New("SSE-C encryption parameters are not supported on this bucket") +) + +const ( + // SSECustomerKeySize is the size of valid client provided encryption keys in bytes. + // Currently AWS supports only AES256. So the SSE-C key size is fixed to 32 bytes. + SSECustomerKeySize = 32 + + // SSEIVSize is the size of the IV data + SSEIVSize = 32 // 32 bytes + + // SSEDAREPackageBlockSize - SSE dare package block size. + SSEDAREPackageBlockSize = 64 * 1024 // 64KiB bytes + + // SSEDAREPackageMetaSize - SSE dare package meta padding bytes. + SSEDAREPackageMetaSize = 32 // 32 bytes + +) + +// KMSKeyID returns in AWS compatible KMS KeyID() format. +func (o *ObjectInfo) KMSKeyID() string { return kmsKeyIDFromMetadata(o.UserDefined) } + +// KMSKeyID returns in AWS compatible KMS KeyID() format. +func (o *MultipartInfo) KMSKeyID() string { return kmsKeyIDFromMetadata(o.UserDefined) } + +// kmsKeyIDFromMetadata returns any AWS S3 KMS key ID in the +// metadata, if any. It returns an empty ID if no key ID is +// present. +func kmsKeyIDFromMetadata(metadata map[string]string) string { + const ARNPrefix = crypto.ARNPrefix + if len(metadata) == 0 { + return "" + } + kmsID, ok := metadata[crypto.MetaKeyID] + if !ok { + return "" + } + if strings.HasPrefix(kmsID, ARNPrefix) { + return kmsID + } + return ARNPrefix + kmsID +} + +// DecryptETags decryptes the ETag of all ObjectInfos using the KMS. +// +// It adjusts the size of all encrypted objects since encrypted +// objects are slightly larger due to encryption overhead. +// Further, it decrypts all single-part SSE-S3 encrypted objects +// and formats ETags of SSE-C / SSE-KMS encrypted objects to +// be AWS S3 compliant. +// +// DecryptETags uses a KMS bulk decryption API, if available, which +// is more efficient than decrypting ETags sequentially. +func DecryptETags(ctx context.Context, k *kms.KMS, objects []ObjectInfo) error { + const BatchSize = 250 // We process the objects in batches - 250 is a reasonable default. + var ( + metadata = make([]map[string]string, 0, BatchSize) + buckets = make([]string, 0, BatchSize) + names = make([]string, 0, BatchSize) + ) + for len(objects) > 0 { + N := min(len(objects), BatchSize) + batch := objects[:N] + + // We have to decrypt only ETags of SSE-S3 single-part + // objects. + // Therefore, we remember which objects (there index) + // in the current batch are single-part SSE-S3 objects. + metadata = metadata[:0:N] + buckets = buckets[:0:N] + names = names[:0:N] + SSES3SinglePartObjects := make(map[int]bool) + for i, object := range batch { + if kind, ok := crypto.IsEncrypted(object.UserDefined); ok && kind == crypto.S3 && !crypto.IsMultiPart(object.UserDefined) { + ETag, err := etag.Parse(object.ETag) + if err != nil { + continue + } + if ETag.IsEncrypted() { + SSES3SinglePartObjects[i] = true + metadata = append(metadata, object.UserDefined) + buckets = append(buckets, object.Bucket) + names = append(names, object.Name) + } + } + } + + // If there are no SSE-S3 single-part objects + // we can skip the decryption process. However, + // we still have to adjust the size and ETag + // of SSE-C and SSE-KMS objects. + if len(SSES3SinglePartObjects) == 0 { + for i := range batch { + size, err := batch[i].GetActualSize() + if err != nil { + return err + } + batch[i].Size = size + + if _, ok := crypto.IsEncrypted(batch[i].UserDefined); ok { + ETag, err := etag.Parse(batch[i].ETag) + if err != nil { + return err + } + batch[i].ETag = ETag.Format().String() + } + } + objects = objects[N:] + continue + } + + // There is at least one SSE-S3 single-part object. + // For all SSE-S3 single-part objects we have to + // fetch their decryption keys. We do this using + // a Bulk-Decryption API call, if available. + keys, err := crypto.S3.UnsealObjectKeys(ctx, k, metadata, buckets, names) + if err != nil { + return err + } + + // Now, we have to decrypt the ETags of SSE-S3 single-part + // objects and adjust the size and ETags of all encrypted + // objects. + for i := range batch { + size, err := batch[i].GetActualSize() + if err != nil { + return err + } + batch[i].Size = size + + if _, ok := crypto.IsEncrypted(batch[i].UserDefined); ok { + ETag, err := etag.Parse(batch[i].ETag) + if err != nil { + return err + } + if SSES3SinglePartObjects[i] { + ETag, err = etag.Decrypt(keys[0][:], ETag) + if err != nil { + return err + } + keys = keys[1:] + } + batch[i].ETag = ETag.Format().String() + } + } + objects = objects[N:] + } + return nil +} + +// isMultipart returns true if the current object is +// uploaded by the user using multipart mechanism: +// initiate new multipart, upload part, complete upload +func (o *ObjectInfo) isMultipart() bool { + _, encrypted := crypto.IsEncrypted(o.UserDefined) + if encrypted { + if !crypto.IsMultiPart(o.UserDefined) { + return false + } + for _, part := range o.Parts { + _, err := sio.DecryptedSize(uint64(part.Size)) + if err != nil { + return false + } + } + } + + // Further check if this object is uploaded using multipart mechanism + // by the user and it is not about Erasure internally splitting the + // object into parts in PutObject() + return len(o.ETag) != 32 +} + +// ParseSSECopyCustomerRequest parses the SSE-C header fields of the provided request. +// It returns the client provided key on success. +func ParseSSECopyCustomerRequest(h http.Header, metadata map[string]string) (key []byte, err error) { + if crypto.S3.IsEncrypted(metadata) && crypto.SSECopy.IsRequested(h) { + return nil, crypto.ErrIncompatibleEncryptionMethod + } + k, err := crypto.SSECopy.ParseHTTP(h) + return k[:], err +} + +// ParseSSECustomerRequest parses the SSE-C header fields of the provided request. +// It returns the client provided key on success. +func ParseSSECustomerRequest(r *http.Request) (key []byte, err error) { + return ParseSSECustomerHeader(r.Header) +} + +// ParseSSECustomerHeader parses the SSE-C header fields and returns +// the client provided key on success. +func ParseSSECustomerHeader(header http.Header) (key []byte, err error) { + if crypto.S3.IsRequested(header) && crypto.SSEC.IsRequested(header) { + return key, crypto.ErrIncompatibleEncryptionMethod + } + + k, err := crypto.SSEC.ParseHTTP(header) + return k[:], err +} + +// This function rotates old to new key. +func rotateKey(ctx context.Context, oldKey []byte, newKeyID string, newKey []byte, bucket, object string, metadata map[string]string, cryptoCtx kms.Context) error { + kind, _ := crypto.IsEncrypted(metadata) + switch kind { + case crypto.S3: + if GlobalKMS == nil { + return errKMSNotConfigured + } + keyID, kmsKey, sealedKey, err := crypto.S3.ParseMetadata(metadata) + if err != nil { + return err + } + oldKey, err := GlobalKMS.Decrypt(ctx, &kms.DecryptRequest{ + Name: keyID, + Ciphertext: kmsKey, + AssociatedData: kms.Context{bucket: path.Join(bucket, object)}, + }) + if err != nil { + return err + } + var objectKey crypto.ObjectKey + if err = objectKey.Unseal(oldKey, sealedKey, crypto.S3.String(), bucket, object); err != nil { + return err + } + + newKey, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{ + Name: GlobalKMS.DefaultKey, + AssociatedData: kms.Context{bucket: path.Join(bucket, object)}, + }) + if err != nil { + return err + } + sealedKey = objectKey.Seal(newKey.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object) + crypto.S3.CreateMetadata(metadata, newKey.KeyID, newKey.Ciphertext, sealedKey) + return nil + case crypto.S3KMS: + if GlobalKMS == nil { + return errKMSNotConfigured + } + objectKey, err := crypto.S3KMS.UnsealObjectKey(GlobalKMS, metadata, bucket, object) + if err != nil { + return err + } + + if len(cryptoCtx) == 0 { + _, _, _, cryptoCtx, err = crypto.S3KMS.ParseMetadata(metadata) + if err != nil { + return err + } + } + + // If the context does not contain the bucket key + // we must add it for key generation. However, + // the context must be stored exactly like the + // client provided it. Therefore, we create a copy + // of the client provided context and add the bucket + // key, if not present. + kmsCtx := kms.Context{} + maps.Copy(kmsCtx, cryptoCtx) + if _, ok := kmsCtx[bucket]; !ok { + kmsCtx[bucket] = path.Join(bucket, object) + } + newKey, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{ + Name: newKeyID, + AssociatedData: kmsCtx, + }) + if err != nil { + return err + } + + sealedKey := objectKey.Seal(newKey.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3KMS.String(), bucket, object) + crypto.S3KMS.CreateMetadata(metadata, newKey.KeyID, newKey.Ciphertext, sealedKey, cryptoCtx) + return nil + case crypto.SSEC: + sealedKey, err := crypto.SSEC.ParseMetadata(metadata) + if err != nil { + return err + } + + var objectKey crypto.ObjectKey + if err = objectKey.Unseal(oldKey, sealedKey, crypto.SSEC.String(), bucket, object); err != nil { + if subtle.ConstantTimeCompare(oldKey, newKey) == 1 { + return errInvalidSSEParameters // AWS returns special error for equal but invalid keys. + } + return crypto.ErrInvalidCustomerKey // To provide strict AWS S3 compatibility we return: access denied. + } + + if subtle.ConstantTimeCompare(oldKey, newKey) == 1 && sealedKey.Algorithm == crypto.SealAlgorithm { + return nil // don't rotate on equal keys if seal algorithm is latest + } + sealedKey = objectKey.Seal(newKey, sealedKey.IV, crypto.SSEC.String(), bucket, object) + crypto.SSEC.CreateMetadata(metadata, sealedKey) + return nil + default: + return errObjectTampered + } +} + +func newEncryptMetadata(ctx context.Context, kind crypto.Type, keyID string, key []byte, bucket, object string, metadata map[string]string, cryptoCtx kms.Context) (crypto.ObjectKey, error) { + var sealedKey crypto.SealedKey + switch kind { + case crypto.S3: + if GlobalKMS == nil { + return crypto.ObjectKey{}, errKMSNotConfigured + } + key, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{ + AssociatedData: kms.Context{bucket: path.Join(bucket, object)}, + }) + if err != nil { + return crypto.ObjectKey{}, err + } + + objectKey := crypto.GenerateKey(key.Plaintext, rand.Reader) + sealedKey = objectKey.Seal(key.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3.String(), bucket, object) + crypto.S3.CreateMetadata(metadata, key.KeyID, key.Ciphertext, sealedKey) + return objectKey, nil + case crypto.S3KMS: + if GlobalKMS == nil { + return crypto.ObjectKey{}, errKMSNotConfigured + } + + // If the context does not contain the bucket key + // we must add it for key generation. However, + // the context must be stored exactly like the + // client provided it. Therefore, we create a copy + // of the client provided context and add the bucket + // key, if not present. + kmsCtx := kms.Context{} + maps.Copy(kmsCtx, cryptoCtx) + if _, ok := kmsCtx[bucket]; !ok { + kmsCtx[bucket] = path.Join(bucket, object) + } + key, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{ + Name: keyID, + AssociatedData: kmsCtx, + }) + if err != nil { + if errors.Is(err, kes.ErrKeyNotFound) { + return crypto.ObjectKey{}, errKMSKeyNotFound + } + return crypto.ObjectKey{}, err + } + + objectKey := crypto.GenerateKey(key.Plaintext, rand.Reader) + sealedKey = objectKey.Seal(key.Plaintext, crypto.GenerateIV(rand.Reader), crypto.S3KMS.String(), bucket, object) + crypto.S3KMS.CreateMetadata(metadata, key.KeyID, key.Ciphertext, sealedKey, cryptoCtx) + return objectKey, nil + case crypto.SSEC: + objectKey := crypto.GenerateKey(key, rand.Reader) + sealedKey = objectKey.Seal(key, crypto.GenerateIV(rand.Reader), crypto.SSEC.String(), bucket, object) + crypto.SSEC.CreateMetadata(metadata, sealedKey) + return objectKey, nil + default: + return crypto.ObjectKey{}, fmt.Errorf("encryption type '%v' not supported", kind) + } +} + +func newEncryptReader(ctx context.Context, content io.Reader, kind crypto.Type, keyID string, key []byte, bucket, object string, metadata map[string]string, cryptoCtx kms.Context) (io.Reader, crypto.ObjectKey, error) { + objectEncryptionKey, err := newEncryptMetadata(ctx, kind, keyID, key, bucket, object, metadata, cryptoCtx) + if err != nil { + return nil, crypto.ObjectKey{}, err + } + + reader, err := sio.EncryptReader(content, sio.Config{Key: objectEncryptionKey[:], MinVersion: sio.Version20}) + if err != nil { + return nil, crypto.ObjectKey{}, crypto.ErrInvalidCustomerKey + } + + return reader, objectEncryptionKey, nil +} + +// set new encryption metadata from http request headers for SSE-C and generated key from KMS in the case of +// SSE-S3 +func setEncryptionMetadata(r *http.Request, bucket, object string, metadata map[string]string) (err error) { + var ( + key []byte + keyID string + kmsCtx kms.Context + ) + kind, _ := crypto.IsRequested(r.Header) + switch kind { + case crypto.SSEC: + key, err = ParseSSECustomerRequest(r) + if err != nil { + return err + } + case crypto.S3KMS: + keyID, kmsCtx, err = crypto.S3KMS.ParseHTTP(r.Header) + if err != nil { + return err + } + } + _, err = newEncryptMetadata(r.Context(), kind, keyID, key, bucket, object, metadata, kmsCtx) + return err +} + +// EncryptRequest takes the client provided content and encrypts the data +// with the client provided key. It also marks the object as client-side-encrypted +// and sets the correct headers. +func EncryptRequest(content io.Reader, r *http.Request, bucket, object string, metadata map[string]string) (io.Reader, crypto.ObjectKey, error) { + if r.ContentLength > encryptBufferThreshold { + // The encryption reads in blocks of 64KB. + // We add a buffer on bigger files to reduce the number of syscalls upstream. + content = bufio.NewReaderSize(content, encryptBufferSize) + } + + var ( + key []byte + keyID string + ctx kms.Context + err error + ) + kind, _ := crypto.IsRequested(r.Header) + if kind == crypto.SSEC { + key, err = ParseSSECustomerRequest(r) + if err != nil { + return nil, crypto.ObjectKey{}, err + } + } + if kind == crypto.S3KMS { + keyID, ctx, err = crypto.S3KMS.ParseHTTP(r.Header) + if err != nil { + return nil, crypto.ObjectKey{}, err + } + } + return newEncryptReader(r.Context(), content, kind, keyID, key, bucket, object, metadata, ctx) +} + +func decryptObjectMeta(key []byte, bucket, object string, metadata map[string]string) ([]byte, error) { + switch kind, _ := crypto.IsEncrypted(metadata); kind { + case crypto.S3: + if GlobalKMS == nil { + return nil, errKMSNotConfigured + } + objectKey, err := crypto.S3.UnsealObjectKey(GlobalKMS, metadata, bucket, object) + if err != nil { + return nil, err + } + return objectKey[:], nil + case crypto.S3KMS: + if GlobalKMS == nil { + return nil, errKMSNotConfigured + } + objectKey, err := crypto.S3KMS.UnsealObjectKey(GlobalKMS, metadata, bucket, object) + if err != nil { + return nil, err + } + return objectKey[:], nil + case crypto.SSEC: + sealedKey, err := crypto.SSEC.ParseMetadata(metadata) + if err != nil { + return nil, err + } + var objectKey crypto.ObjectKey + if err = objectKey.Unseal(key, sealedKey, crypto.SSEC.String(), bucket, object); err != nil { + return nil, err + } + return objectKey[:], nil + default: + return nil, errObjectTampered + } +} + +// Adding support for reader based interface + +// DecryptRequestWithSequenceNumberR - same as +// DecryptRequestWithSequenceNumber but with a reader +func DecryptRequestWithSequenceNumberR(client io.Reader, h http.Header, bucket, object string, seqNumber uint32, metadata map[string]string) (io.Reader, error) { + if crypto.SSEC.IsEncrypted(metadata) { + key, err := ParseSSECustomerHeader(h) + if err != nil { + return nil, err + } + return newDecryptReader(client, key, bucket, object, seqNumber, metadata) + } + return newDecryptReader(client, nil, bucket, object, seqNumber, metadata) +} + +// DecryptCopyRequestR - same as DecryptCopyRequest, but with a +// Reader +func DecryptCopyRequestR(client io.Reader, h http.Header, bucket, object string, seqNumber uint32, metadata map[string]string) (io.Reader, error) { + var ( + key []byte + err error + ) + if crypto.SSECopy.IsRequested(h) { + key, err = ParseSSECopyCustomerRequest(h, metadata) + if err != nil { + return nil, err + } + } + return newDecryptReader(client, key, bucket, object, seqNumber, metadata) +} + +func newDecryptReader(client io.Reader, key []byte, bucket, object string, seqNumber uint32, metadata map[string]string) (io.Reader, error) { + objectEncryptionKey, err := decryptObjectMeta(key, bucket, object, metadata) + if err != nil { + return nil, err + } + return newDecryptReaderWithObjectKey(client, objectEncryptionKey, seqNumber) +} + +func newDecryptReaderWithObjectKey(client io.Reader, objectEncryptionKey []byte, seqNumber uint32) (io.Reader, error) { + reader, err := sio.DecryptReader(client, sio.Config{ + Key: objectEncryptionKey, + SequenceNumber: seqNumber, + }) + if err != nil { + return nil, crypto.ErrInvalidCustomerKey + } + return reader, nil +} + +// DecryptBlocksRequestR - same as DecryptBlocksRequest but with a +// reader +func DecryptBlocksRequestR(inputReader io.Reader, h http.Header, seqNumber uint32, partStart int, oi ObjectInfo, copySource bool) (io.Reader, error) { + bucket, object := oi.Bucket, oi.Name + // Single part case + if !oi.isMultipart() { + var reader io.Reader + var err error + if copySource { + reader, err = DecryptCopyRequestR(inputReader, h, bucket, object, seqNumber, oi.UserDefined) + } else { + reader, err = DecryptRequestWithSequenceNumberR(inputReader, h, bucket, object, seqNumber, oi.UserDefined) + } + if err != nil { + return nil, err + } + return reader, nil + } + + partDecRelOffset := int64(seqNumber) * SSEDAREPackageBlockSize + partEncRelOffset := int64(seqNumber) * (SSEDAREPackageBlockSize + SSEDAREPackageMetaSize) + + w := &DecryptBlocksReader{ + reader: inputReader, + startSeqNum: seqNumber, + partDecRelOffset: partDecRelOffset, + partEncRelOffset: partEncRelOffset, + parts: oi.Parts, + partIndex: partStart, + } + + // In case of SSE-C, we have to decrypt the OEK using the client-provided key. + // In case of a SSE-C server-side copy, the client might provide two keys, + // one for the source and one for the target. This reader is the source. + var ssecClientKey []byte + if crypto.SSEC.IsEncrypted(oi.UserDefined) { + if copySource && crypto.SSECopy.IsRequested(h) { + key, err := crypto.SSECopy.ParseHTTP(h) + if err != nil { + return nil, err + } + ssecClientKey = key[:] + } else { + key, err := crypto.SSEC.ParseHTTP(h) + if err != nil { + return nil, err + } + ssecClientKey = key[:] + } + } + + // Decrypt the OEK once and reuse it for all subsequent parts. + objectEncryptionKey, err := decryptObjectMeta(ssecClientKey, bucket, object, oi.UserDefined) + if err != nil { + return nil, err + } + w.objectEncryptionKey = objectEncryptionKey + + if err := w.buildDecrypter(w.parts[w.partIndex].Number); err != nil { + return nil, err + } + return w, nil +} + +// DecryptBlocksReader - decrypts multipart parts, while implementing +// a io.Reader compatible interface. +type DecryptBlocksReader struct { + // Source of the encrypted content that will be decrypted + reader io.Reader + // Current decrypter for the current encrypted data block + decrypter io.Reader + // Start sequence number + startSeqNum uint32 + // Current part index + partIndex int + // Parts information + parts []ObjectPartInfo + + objectEncryptionKey []byte + partDecRelOffset, partEncRelOffset int64 +} + +func (d *DecryptBlocksReader) buildDecrypter(partID int) error { + var partIDbin [4]byte + binary.LittleEndian.PutUint32(partIDbin[:], uint32(partID)) // marshal part ID + + mac := hmac.New(sha256.New, d.objectEncryptionKey) // derive part encryption key from part ID and object key + mac.Write(partIDbin[:]) + partEncryptionKey := mac.Sum(nil) + + // Limit the reader, so the decryptor doesn't receive bytes + // from the next part (different DARE stream) + encLenToRead := d.parts[d.partIndex].Size - d.partEncRelOffset + decrypter, err := newDecryptReaderWithObjectKey(io.LimitReader(d.reader, encLenToRead), partEncryptionKey, d.startSeqNum) + if err != nil { + return err + } + + d.decrypter = decrypter + return nil +} + +func (d *DecryptBlocksReader) Read(p []byte) (int, error) { + var err error + var n1 int + decPartSize, _ := sio.DecryptedSize(uint64(d.parts[d.partIndex].Size)) + unreadPartLen := int64(decPartSize) - d.partDecRelOffset + if int64(len(p)) < unreadPartLen { + n1, err = d.decrypter.Read(p) + if err != nil { + return 0, err + } + d.partDecRelOffset += int64(n1) + } else { + n1, err = io.ReadFull(d.decrypter, p[:unreadPartLen]) + if err != nil { + return 0, err + } + + // We should now proceed to next part, reset all + // values appropriately. + d.partEncRelOffset = 0 + d.partDecRelOffset = 0 + d.startSeqNum = 0 + + d.partIndex++ + if d.partIndex == len(d.parts) { + return n1, io.EOF + } + + err = d.buildDecrypter(d.parts[d.partIndex].Number) + if err != nil { + return 0, err + } + + n1, err = d.decrypter.Read(p[n1:]) + if err != nil { + return 0, err + } + + d.partDecRelOffset += int64(n1) + } + return len(p), nil +} + +// DecryptedSize returns the size of the object after decryption in bytes. +// It returns an error if the object is not encrypted or marked as encrypted +// but has an invalid size. +func (o ObjectInfo) DecryptedSize() (int64, error) { + if _, ok := crypto.IsEncrypted(o.UserDefined); !ok { + return -1, errors.New("Cannot compute decrypted size of an unencrypted object") + } + + if !o.isMultipart() { + size, err := sio.DecryptedSize(uint64(o.Size)) + if err != nil { + err = errObjectTampered // assign correct error type + } + return int64(size), err + } + + var size int64 + for _, part := range o.Parts { + partSize, err := sio.DecryptedSize(uint64(part.Size)) + if err != nil { + return -1, errObjectTampered + } + size += int64(partSize) + } + return size, nil +} + +// DecryptETag decrypts the ETag that is part of given object +// with the given object encryption key. +// +// However, DecryptETag does not try to decrypt the ETag if +// it consists of a 128 bit hex value (32 hex chars) and exactly +// one '-' followed by a 32-bit number. +// This special case addresses randomly-generated ETags generated +// by the MinIO server when running in non-compat mode. These +// random ETags are not encrypt. +// +// Calling DecryptETag with a non-randomly generated ETag will +// fail. +func DecryptETag(key crypto.ObjectKey, object ObjectInfo) (string, error) { + if n := strings.Count(object.ETag, "-"); n > 0 { + if n != 1 { + return "", errObjectTampered + } + i := strings.IndexByte(object.ETag, '-') + if len(object.ETag[:i]) != 32 { + return "", errObjectTampered + } + if _, err := hex.DecodeString(object.ETag[:32]); err != nil { + return "", errObjectTampered + } + if _, err := strconv.ParseInt(object.ETag[i+1:], 10, 32); err != nil { + return "", errObjectTampered + } + return object.ETag, nil + } + + etag, err := hex.DecodeString(object.ETag) + if err != nil { + return "", err + } + etag, err = key.UnsealETag(etag) + if err != nil { + return "", err + } + return hex.EncodeToString(etag), nil +} + +// For encrypted objects, the ETag sent by client if available +// is stored in encrypted form in the backend. Decrypt the ETag +// if ETag was previously encrypted. +func getDecryptedETag(headers http.Header, objInfo ObjectInfo, copySource bool) (decryptedETag string) { + var ( + key [32]byte + err error + ) + // If ETag is contentMD5Sum return it as is. + if len(objInfo.ETag) == 32 { + return objInfo.ETag + } + + if crypto.IsMultiPart(objInfo.UserDefined) { + return objInfo.ETag + } + + if crypto.SSECopy.IsRequested(headers) { + key, err = crypto.SSECopy.ParseHTTP(headers) + if err != nil { + return objInfo.ETag + } + } + + // As per AWS S3 Spec, ETag for SSE-C encrypted objects need not be MD5Sum of the data. + // Since server side copy with same source and dest just replaces the ETag, we save + // encrypted content MD5Sum as ETag for both SSE-C and SSE-KMS, we standardize the ETag + // encryption across SSE-C and SSE-KMS, and only return last 32 bytes for SSE-C + if (crypto.SSEC.IsEncrypted(objInfo.UserDefined) || crypto.S3KMS.IsEncrypted(objInfo.UserDefined)) && !copySource { + return objInfo.ETag[len(objInfo.ETag)-32:] + } + + objectEncryptionKey, err := decryptObjectMeta(key[:], objInfo.Bucket, objInfo.Name, objInfo.UserDefined) + if err != nil { + return objInfo.ETag + } + return tryDecryptETag(objectEncryptionKey, objInfo.ETag, true) +} + +// helper to decrypt Etag given object encryption key and encrypted ETag +func tryDecryptETag(key []byte, encryptedETag string, sses3 bool) string { + // ETag for SSE-C or SSE-KMS encrypted objects need not be content MD5Sum.While encrypted + // md5sum is stored internally, return just the last 32 bytes of hex-encoded and + // encrypted md5sum string for SSE-C + if !sses3 { + return encryptedETag[len(encryptedETag)-32:] + } + var objectKey crypto.ObjectKey + copy(objectKey[:], key) + encBytes, err := hex.DecodeString(encryptedETag) + if err != nil { + return encryptedETag + } + etagBytes, err := objectKey.UnsealETag(encBytes) + if err != nil { + return encryptedETag + } + return hex.EncodeToString(etagBytes) +} + +// GetDecryptedRange - To decrypt the range (off, length) of the +// decrypted object stream, we need to read the range (encOff, +// encLength) of the encrypted object stream to decrypt it, and +// compute skipLen, the number of bytes to skip in the beginning of +// the encrypted range. +// +// In addition we also compute the object part number for where the +// requested range starts, along with the DARE sequence number within +// that part. For single part objects, the partStart will be 0. +func (o *ObjectInfo) GetDecryptedRange(rs *HTTPRangeSpec) (encOff, encLength, skipLen int64, seqNumber uint32, partStart int, err error) { + if _, ok := crypto.IsEncrypted(o.UserDefined); !ok { + err = errors.New("Object is not encrypted") + return encOff, encLength, skipLen, seqNumber, partStart, err + } + + if rs == nil { + // No range, so offsets refer to the whole object. + return 0, o.Size, 0, 0, 0, nil + } + + // Assemble slice of (decrypted) part sizes in `sizes` + var sizes []int64 + var decObjSize int64 // decrypted total object size + if o.isMultipart() { + sizes = make([]int64, len(o.Parts)) + for i, part := range o.Parts { + var partSize uint64 + partSize, err = sio.DecryptedSize(uint64(part.Size)) + if err != nil { + err = errObjectTampered + return encOff, encLength, skipLen, seqNumber, partStart, err + } + sizes[i] = int64(partSize) + decObjSize += int64(partSize) + } + } else { + var partSize uint64 + partSize, err = sio.DecryptedSize(uint64(o.Size)) + if err != nil { + err = errObjectTampered + return encOff, encLength, skipLen, seqNumber, partStart, err + } + sizes = []int64{int64(partSize)} + decObjSize = sizes[0] + } + + var off, length int64 + off, length, err = rs.GetOffsetLength(decObjSize) + if err != nil { + return encOff, encLength, skipLen, seqNumber, partStart, err + } + + // At this point, we have: + // + // 1. the decrypted part sizes in `sizes` (single element for + // single part object) and total decrypted object size `decObjSize` + // + // 2. the (decrypted) start offset `off` and (decrypted) + // length to read `length` + // + // These are the inputs to the rest of the algorithm below. + + // Locate the part containing the start of the required range + var partEnd int + var cumulativeSum, encCumulativeSum int64 + for i, size := range sizes { + if off < cumulativeSum+size { + partStart = i + break + } + cumulativeSum += size + encPartSize, _ := sio.EncryptedSize(uint64(size)) + encCumulativeSum += int64(encPartSize) + } + // partStart is always found in the loop above, + // because off is validated. + + sseDAREEncPackageBlockSize := int64(SSEDAREPackageBlockSize + SSEDAREPackageMetaSize) + startPkgNum := (off - cumulativeSum) / SSEDAREPackageBlockSize + + // Now we can calculate the number of bytes to skip + skipLen = (off - cumulativeSum) % SSEDAREPackageBlockSize + + encOff = encCumulativeSum + startPkgNum*sseDAREEncPackageBlockSize + // Locate the part containing the end of the required range + endOffset := off + length - 1 + for i1, size := range sizes[partStart:] { + i := partStart + i1 + if endOffset < cumulativeSum+size { + partEnd = i + break + } + cumulativeSum += size + encPartSize, _ := sio.EncryptedSize(uint64(size)) + encCumulativeSum += int64(encPartSize) + } + // partEnd is always found in the loop above, because off and + // length are validated. + endPkgNum := (endOffset - cumulativeSum) / SSEDAREPackageBlockSize + // Compute endEncOffset with one additional DARE package (so + // we read the package containing the last desired byte). + endEncOffset := encCumulativeSum + (endPkgNum+1)*sseDAREEncPackageBlockSize + // Check if the DARE package containing the end offset is a + // full sized package (as the last package in the part may be + // smaller) + lastPartSize, _ := sio.EncryptedSize(uint64(sizes[partEnd])) + if endEncOffset > encCumulativeSum+int64(lastPartSize) { + endEncOffset = encCumulativeSum + int64(lastPartSize) + } + encLength = endEncOffset - encOff + // Set the sequence number as the starting package number of + // the requested block + seqNumber = uint32(startPkgNum) + return encOff, encLength, skipLen, seqNumber, partStart, nil +} + +// EncryptedSize returns the size of the object after encryption. +// An encrypted object is always larger than a plain object +// except for zero size objects. +func (o *ObjectInfo) EncryptedSize() int64 { + size, err := sio.EncryptedSize(uint64(o.Size)) + if err != nil { + // This cannot happen since AWS S3 allows parts to be 5GB at most + // sio max. size is 256 TB + reqInfo := (&logger.ReqInfo{}).AppendTags("size", strconv.FormatUint(size, 10)) + ctx := logger.SetReqInfo(GlobalContext, reqInfo) + logger.CriticalIf(ctx, err) + } + return int64(size) +} + +// DecryptObjectInfo tries to decrypt the provided object if it is encrypted. +// It fails if the object is encrypted and the HTTP headers don't contain +// SSE-C headers or the object is not encrypted but SSE-C headers are provided. (AWS behavior) +// DecryptObjectInfo returns 'ErrNone' if the object is not encrypted or the +// decryption succeeded. +// +// DecryptObjectInfo also returns whether the object is encrypted or not. +func DecryptObjectInfo(info *ObjectInfo, r *http.Request) (encrypted bool, err error) { + // Directories are never encrypted. + if info.IsDir { + return false, nil + } + if r == nil { + return false, errInvalidArgument + } + + headers := r.Header + + // disallow X-Amz-Server-Side-Encryption header on HEAD and GET + switch r.Method { + case http.MethodGet, http.MethodHead: + if crypto.S3.IsRequested(headers) || crypto.S3KMS.IsRequested(headers) { + return false, errInvalidEncryptionParameters + } + } + + _, encrypted = crypto.IsEncrypted(info.UserDefined) + if !encrypted && crypto.SSEC.IsRequested(headers) && r.Header.Get(xhttp.AmzCopySource) == "" { + return false, errInvalidEncryptionParameters + } + + if encrypted { + if crypto.SSEC.IsEncrypted(info.UserDefined) { + if !crypto.SSEC.IsRequested(headers) && !crypto.SSECopy.IsRequested(headers) { + if r.Header.Get(xhttp.MinIOSourceReplicationRequest) != "true" { + return encrypted, errEncryptedObject + } + } + } + + if crypto.S3.IsEncrypted(info.UserDefined) && r.Header.Get(xhttp.AmzCopySource) == "" { + if crypto.SSEC.IsRequested(headers) || crypto.SSECopy.IsRequested(headers) { + return encrypted, errEncryptedObject + } + } + + if crypto.S3KMS.IsEncrypted(info.UserDefined) && r.Header.Get(xhttp.AmzCopySource) == "" { + if crypto.SSEC.IsRequested(headers) || crypto.SSECopy.IsRequested(headers) { + return encrypted, errEncryptedObject + } + } + + if _, err = info.DecryptedSize(); err != nil { + return encrypted, err + } + + if _, ok := crypto.IsEncrypted(info.UserDefined); ok && !crypto.IsMultiPart(info.UserDefined) { + info.ETag = getDecryptedETag(headers, *info, false) + } + } + + return encrypted, nil +} + +type ( + objectMetaEncryptFn func(baseKey string, data []byte) []byte + objectMetaDecryptFn func(baseKey string, data []byte) ([]byte, error) +) + +// metadataEncrypter returns a function that will read data from input, +// encrypt it using the provided key and return the result. +// 0 sized inputs are passed through. +func metadataEncrypter(key crypto.ObjectKey) objectMetaEncryptFn { + return func(baseKey string, data []byte) []byte { + if len(data) == 0 { + return data + } + var buffer bytes.Buffer + mac := hmac.New(sha256.New, key[:]) + mac.Write([]byte(baseKey)) + if _, err := sio.Encrypt(&buffer, bytes.NewReader(data), sio.Config{Key: mac.Sum(nil)}); err != nil { + logger.CriticalIf(context.Background(), errors.New("unable to encrypt using object key")) + } + return buffer.Bytes() + } +} + +// metadataDecrypter reverses metadataEncrypter. +func (o *ObjectInfo) metadataDecrypter(h http.Header) objectMetaDecryptFn { + return func(baseKey string, input []byte) ([]byte, error) { + if len(input) == 0 { + return input, nil + } + var key []byte + if crypto.SSECopy.IsRequested(h) { + sseCopyKey, err := crypto.SSECopy.ParseHTTP(h) + if err != nil { + return nil, err + } + key = sseCopyKey[:] + } else { + if k, err := crypto.SSEC.ParseHTTP(h); err == nil { + key = k[:] + } + } + key, err := decryptObjectMeta(key, o.Bucket, o.Name, o.UserDefined) + if err != nil { + return nil, err + } + mac := hmac.New(sha256.New, key) + mac.Write([]byte(baseKey)) + return sio.DecryptBuffer(nil, input, sio.Config{Key: mac.Sum(nil)}) + } +} + +// decryptPartsChecksums will attempt to decrypt and decode part checksums, and save +// only the decrypted part checksum values on ObjectInfo directly. +// if part > 0, and we have the checksum for the part that will be returned. +func (o *ObjectInfo) decryptPartsChecksums(h http.Header) { + data := o.Checksum + if len(data) == 0 { + return + } + if _, encrypted := crypto.IsEncrypted(o.UserDefined); encrypted { + decrypted, err := o.metadataDecrypter(h)("object-checksum", data) + if err != nil { + if !errors.Is(err, crypto.ErrSecretKeyMismatch) { + encLogIf(GlobalContext, err) + } + return + } + data = decrypted + } + cs := hash.ReadPartCheckSums(data) + if len(cs) == len(o.Parts) { + for i := range o.Parts { + o.Parts[i].Checksums = cs[i] + } + } +} + +// decryptChecksum will attempt to decrypt the ObjectInfo.Checksum, returns the decrypted value +// An error is only returned if it was encrypted and the decryption failed. +func (o *ObjectInfo) decryptChecksum(h http.Header) ([]byte, error) { + data := o.Checksum + if len(data) == 0 { + return data, nil + } + if _, encrypted := crypto.IsEncrypted(o.UserDefined); encrypted { + decrypted, err := o.metadataDecrypter(h)("object-checksum", data) + if err != nil { + return nil, err + } + data = decrypted + } + return data, nil +} + +// metadataEncryptFn provides an encryption function for metadata. +// Will return nil, nil if unencrypted. +func (o *ObjectInfo) metadataEncryptFn(headers http.Header) (objectMetaEncryptFn, error) { + kind, _ := crypto.IsEncrypted(o.UserDefined) + switch kind { + case crypto.SSEC: + if crypto.SSECopy.IsRequested(headers) { + key, err := crypto.SSECopy.ParseHTTP(headers) + if err != nil { + return nil, err + } + objectEncryptionKey, err := decryptObjectMeta(key[:], o.Bucket, o.Name, o.UserDefined) + if err != nil { + return nil, err + } + if len(objectEncryptionKey) == 32 { + var key crypto.ObjectKey + copy(key[:], objectEncryptionKey) + return metadataEncrypter(key), nil + } + return nil, errors.New("metadataEncryptFn: unexpected key size") + } + case crypto.S3, crypto.S3KMS: + objectEncryptionKey, err := decryptObjectMeta(nil, o.Bucket, o.Name, o.UserDefined) + if err != nil { + return nil, err + } + if len(objectEncryptionKey) == 32 { + var key crypto.ObjectKey + copy(key[:], objectEncryptionKey) + return metadataEncrypter(key), nil + } + return nil, errors.New("metadataEncryptFn: unexpected key size") + } + + return nil, nil +} + +// decryptChecksums will attempt to decode checksums and return it/them if set. +// if part > 0, and we have the checksum for the part that will be returned. +// Returns whether the checksum (main part 0) is a multipart checksum. +func (o *ObjectInfo) decryptChecksums(part int, h http.Header) (cs map[string]string, isMP bool) { + data := o.Checksum + if len(data) == 0 { + return nil, false + } + if part > 0 && !crypto.SSEC.IsEncrypted(o.UserDefined) { + // already decrypted in ToObjectInfo for multipart objects + for _, pi := range o.Parts { + if pi.Number == part { + return pi.Checksums, true + } + } + } + if _, encrypted := crypto.IsEncrypted(o.UserDefined); encrypted { + decrypted, err := o.metadataDecrypter(h)("object-checksum", data) + if err != nil { + if err != crypto.ErrSecretKeyMismatch { + encLogIf(GlobalContext, err) + } + return nil, part > 0 + } + data = decrypted + } + return hash.ReadCheckSums(data, part) +} diff --git a/cmd/encryption-v1_test.go b/cmd/encryption-v1_test.go new file mode 100644 index 00000000..95f78ef8 --- /dev/null +++ b/cmd/encryption-v1_test.go @@ -0,0 +1,648 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "encoding/base64" + "net/http" + "testing" + + humanize "github.com/dustin/go-humanize" + "github.com/minio/minio-go/v7/pkg/encrypt" + "github.com/minio/minio/internal/crypto" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/sio" +) + +var encryptRequestTests = []struct { + header map[string]string + metadata map[string]string +}{ + { + header: map[string]string{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: "AES256", + xhttp.AmzServerSideEncryptionCustomerKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", + xhttp.AmzServerSideEncryptionCustomerKeyMD5: "bY4wkxQejw9mUJfo72k53A==", + }, + metadata: map[string]string{}, + }, + { + header: map[string]string{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: "AES256", + xhttp.AmzServerSideEncryptionCustomerKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", + xhttp.AmzServerSideEncryptionCustomerKeyMD5: "bY4wkxQejw9mUJfo72k53A==", + }, + metadata: map[string]string{ + xhttp.AmzServerSideEncryptionCustomerKey: "XAm0dRrJsEsyPb1UuFNezv1bl9hxuYsgUVC/MUctE2k=", + }, + }, +} + +func TestEncryptRequest(t *testing.T) { + defer func(flag bool) { globalIsTLS = flag }(globalIsTLS) + globalIsTLS = true + for i, test := range encryptRequestTests { + content := bytes.NewReader(make([]byte, 64)) + req := &http.Request{Header: http.Header{}} + for k, v := range test.header { + req.Header.Set(k, v) + } + _, _, err := EncryptRequest(content, req, "bucket", "object", test.metadata) + if err != nil { + t.Fatalf("Test %d: Failed to encrypt request: %v", i, err) + } + if kdf, ok := test.metadata[crypto.MetaAlgorithm]; !ok { + t.Errorf("Test %d: ServerSideEncryptionKDF must be part of metadata: %v", i, kdf) + } + if iv, ok := test.metadata[crypto.MetaIV]; !ok { + t.Errorf("Test %d: crypto.SSEIV must be part of metadata: %v", i, iv) + } + if mac, ok := test.metadata[crypto.MetaSealedKeySSEC]; !ok { + t.Errorf("Test %d: ServerSideEncryptionKeyMAC must be part of metadata: %v", i, mac) + } + } +} + +var decryptObjectMetaTests = []struct { + info ObjectInfo + request *http.Request + expErr error +}{ + { + info: ObjectInfo{Size: 100}, + request: &http.Request{Header: http.Header{}}, + expErr: nil, + }, + { + info: ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.MetaAlgorithm: crypto.InsecureSealAlgorithm}}, + request: &http.Request{Header: http.Header{xhttp.AmzServerSideEncryption: []string{xhttp.AmzEncryptionAES}}}, + expErr: nil, + }, + { + info: ObjectInfo{Size: 0, UserDefined: map[string]string{crypto.MetaAlgorithm: crypto.InsecureSealAlgorithm}}, + request: &http.Request{Header: http.Header{xhttp.AmzServerSideEncryption: []string{xhttp.AmzEncryptionAES}}}, + expErr: nil, + }, + { + info: ObjectInfo{Size: 100, UserDefined: map[string]string{crypto.MetaSealedKeySSEC: "EAAfAAAAAAD7v1hQq3PFRUHsItalxmrJqrOq6FwnbXNarxOOpb8jTWONPPKyM3Gfjkjyj6NCf+aB/VpHCLCTBA=="}}, + request: &http.Request{Header: http.Header{}}, + expErr: errEncryptedObject, + }, + { + info: ObjectInfo{Size: 100, UserDefined: map[string]string{}}, + request: &http.Request{Method: http.MethodGet, Header: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES}}}, + expErr: errInvalidEncryptionParameters, + }, + { + info: ObjectInfo{Size: 100, UserDefined: map[string]string{}}, + request: &http.Request{Method: http.MethodHead, Header: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES}}}, + expErr: errInvalidEncryptionParameters, + }, + { + info: ObjectInfo{Size: 31, UserDefined: map[string]string{crypto.MetaAlgorithm: crypto.InsecureSealAlgorithm}}, + request: &http.Request{Header: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES}}}, + expErr: errObjectTampered, + }, +} + +func TestDecryptObjectInfo(t *testing.T) { + for i, test := range decryptObjectMetaTests { + if encrypted, err := DecryptObjectInfo(&test.info, test.request); err != test.expErr { + t.Errorf("Test %d: Decryption returned wrong error code: got %d , want %d", i, err, test.expErr) + } else if _, enc := crypto.IsEncrypted(test.info.UserDefined); encrypted && enc != encrypted { + t.Errorf("Test %d: Decryption thinks object is encrypted but it is not", i) + } else if !encrypted && enc != encrypted { + t.Errorf("Test %d: Decryption thinks object is not encrypted but it is", i) + } + } +} + +var decryptETagTests = []struct { + ObjectKey crypto.ObjectKey + ObjectInfo ObjectInfo + ShouldFail bool + ETag string +}{ + { + ObjectKey: [32]byte{}, + ObjectInfo: ObjectInfo{ETag: "20000f00f27834c9a2654927546df57f9e998187496394d4ee80f3d9978f85f3c7d81f72600cdbe03d80dc5a13d69354"}, + ETag: "8ad3fe6b84bf38489e95c701c84355b6", + }, + { + ObjectKey: [32]byte{}, + ObjectInfo: ObjectInfo{ETag: "20000f00f27834c9a2654927546df57f9e998187496394d4ee80f3d9978f85f3c7d81f72600cdbe03d80dc5a13d6935"}, + ETag: "", + ShouldFail: true, // ETag is not a valid hex value + }, + { + ObjectKey: [32]byte{}, + ObjectInfo: ObjectInfo{ETag: "00000f00f27834c9a2654927546df57f9e998187496394d4ee80f3d9978f85f3c7d81f72600cdbe03d80dc5a13d69354"}, + ETag: "", + ShouldFail: true, // modified ETag + }, + + // Special tests for ETags that end with a '-x' + { + ObjectKey: [32]byte{}, + ObjectInfo: ObjectInfo{ETag: "916516b396f0f4d4f2a0e7177557bec4-1"}, + ETag: "916516b396f0f4d4f2a0e7177557bec4-1", + }, + { + ObjectKey: [32]byte{}, + ObjectInfo: ObjectInfo{ETag: "916516b396f0f4d4f2a0e7177557bec4-738"}, + ETag: "916516b396f0f4d4f2a0e7177557bec4-738", + }, + { + ObjectKey: [32]byte{}, + ObjectInfo: ObjectInfo{ETag: "916516b396f0f4d4f2a0e7177557bec4-Q"}, + ETag: "", + ShouldFail: true, // Q is not a number + }, + { + ObjectKey: [32]byte{}, + ObjectInfo: ObjectInfo{ETag: "16516b396f0f4d4f2a0e7177557bec4-1"}, + ETag: "", + ShouldFail: true, // ETag prefix is not a valid hex value + }, + { + ObjectKey: [32]byte{}, + ObjectInfo: ObjectInfo{ETag: "16516b396f0f4d4f2a0e7177557bec4-1-2"}, + ETag: "", + ShouldFail: true, // ETag contains multiple: - + }, +} + +func TestDecryptETag(t *testing.T) { + for i, test := range decryptETagTests { + etag, err := DecryptETag(test.ObjectKey, test.ObjectInfo) + if err != nil && !test.ShouldFail { + t.Fatalf("Test %d: should succeed but failed: %v", i, err) + } + if err == nil && test.ShouldFail { + t.Fatalf("Test %d: should fail but succeeded", i) + } + if err == nil { + if etag != test.ETag { + t.Fatalf("Test %d: ETag mismatch: got %s - want %s", i, etag, test.ETag) + } + } + } +} + +// Tests for issue reproduced when getting the right encrypted +// offset of the object. +func TestGetDecryptedRange_Issue50(t *testing.T) { + rs, err := parseRequestRangeSpec("bytes=594870256-594870263") + if err != nil { + t.Fatal(err) + } + + objInfo := ObjectInfo{ + Bucket: "bucket", + Name: "object", + Size: 595160760, + UserDefined: map[string]string{ + crypto.MetaMultipart: "", + crypto.MetaIV: "HTexa=", + crypto.MetaAlgorithm: "DAREv2-HMAC-SHA256", + crypto.MetaSealedKeySSEC: "IAA8PGAA==", + ReservedMetadataPrefix + "actual-size": "594870264", + "content-type": "application/octet-stream", + "etag": "166b1545b4c1535294ee0686678bea8c-2", + }, + Parts: []ObjectPartInfo{ + { + Number: 1, + Size: 297580380, + ActualSize: 297435132, + }, + { + Number: 2, + Size: 297580380, + ActualSize: 297435132, + }, + }, + } + + encOff, encLength, skipLen, seqNumber, partStart, err := objInfo.GetDecryptedRange(rs) + if err != nil { + t.Fatalf("Test: failed %s", err) + } + if encOff != 595127964 { + t.Fatalf("Test: expected %d, got %d", 595127964, encOff) + } + if encLength != 32796 { + t.Fatalf("Test: expected %d, got %d", 32796, encLength) + } + if skipLen != 32756 { + t.Fatalf("Test: expected %d, got %d", 32756, skipLen) + } + if seqNumber != 4538 { + t.Fatalf("Test: expected %d, got %d", 4538, seqNumber) + } + if partStart != 1 { + t.Fatalf("Test: expected %d, got %d", 1, partStart) + } +} + +func TestGetDecryptedRange(t *testing.T) { + var ( + pkgSz = int64(64) * humanize.KiByte + minPartSz = int64(5) * humanize.MiByte + maxPartSz = int64(5) * humanize.GiByte + + getEncSize = func(s int64) int64 { + v, _ := sio.EncryptedSize(uint64(s)) + return int64(v) + } + udMap = func(isMulti bool) map[string]string { + m := map[string]string{ + crypto.MetaAlgorithm: crypto.InsecureSealAlgorithm, + crypto.MetaMultipart: "1", + } + if !isMulti { + delete(m, crypto.MetaMultipart) + } + return m + } + ) + + // Single part object tests + + mkSPObj := func(s int64) ObjectInfo { + return ObjectInfo{ + Size: getEncSize(s), + UserDefined: udMap(false), + } + } + + testSP := []struct { + decSz int64 + oi ObjectInfo + }{ + {0, mkSPObj(0)}, + {1, mkSPObj(1)}, + {pkgSz - 1, mkSPObj(pkgSz - 1)}, + {pkgSz, mkSPObj(pkgSz)}, + {2*pkgSz - 1, mkSPObj(2*pkgSz - 1)}, + {minPartSz, mkSPObj(minPartSz)}, + {maxPartSz, mkSPObj(maxPartSz)}, + } + + for i, test := range testSP { + { + // nil range + o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(nil) + if err != nil { + t.Errorf("Case %d: unexpected err: %v", i, err) + } + if skip != 0 || sn != 0 || ps != 0 || o != 0 || l != getEncSize(test.decSz) { + t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps) + } + } + + if test.decSz >= 10 { + // first 10 bytes + o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{false, 0, 9}) + if err != nil { + t.Errorf("Case %d: unexpected err: %v", i, err) + } + rLen := pkgSz + 32 + if test.decSz < pkgSz { + rLen = test.decSz + 32 + } + if skip != 0 || sn != 0 || ps != 0 || o != 0 || l != rLen { + t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps) + } + } + + kb32 := int64(32) * humanize.KiByte + if test.decSz >= (64+32)*humanize.KiByte { + // Skip the first 32Kib, and read the next 64Kib + o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{false, kb32, 3*kb32 - 1}) + if err != nil { + t.Errorf("Case %d: unexpected err: %v", i, err) + } + rLen := (pkgSz + 32) * 2 + if test.decSz < 2*pkgSz { + rLen = (pkgSz + 32) + (test.decSz - pkgSz + 32) + } + if skip != kb32 || sn != 0 || ps != 0 || o != 0 || l != rLen { + t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps) + } + } + + if test.decSz >= (64*2+32)*humanize.KiByte { + // Skip the first 96Kib and read the next 64Kib + o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{false, 3 * kb32, 5*kb32 - 1}) + if err != nil { + t.Errorf("Case %d: unexpected err: %v", i, err) + } + rLen := (pkgSz + 32) * 2 + if test.decSz-pkgSz < 2*pkgSz { + rLen = (pkgSz + 32) + (test.decSz - pkgSz + 32*2) + } + if skip != kb32 || sn != 1 || ps != 0 || o != pkgSz+32 || l != rLen { + t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps) + } + } + } + + // Multipart object tests + var ( + // make a multipart object-info given part sizes + mkMPObj = func(sizes []int64) ObjectInfo { + r := make([]ObjectPartInfo, len(sizes)) + sum := int64(0) + for i, s := range sizes { + r[i].Number = i + r[i].Size = getEncSize(s) + sum += r[i].Size + } + return ObjectInfo{ + Size: sum, + UserDefined: udMap(true), + Parts: r, + } + } + // Simple useful utilities + repeat = func(k int64, n int) []int64 { + a := []int64{} + for range n { + a = append(a, k) + } + return a + } + lsum = func(s []int64) int64 { + sum := int64(0) + for _, i := range s { + if i < 0 { + return -1 + } + sum += i + } + return sum + } + esum = func(oi ObjectInfo) int64 { + sum := int64(0) + for _, i := range oi.Parts { + sum += i.Size + } + return sum + } + ) + + s1 := []int64{5487701, 5487799, 3} + s2 := repeat(5487701, 5) + s3 := repeat(maxPartSz, 10000) + testMPs := []struct { + decSizes []int64 + oi ObjectInfo + }{ + {s1, mkMPObj(s1)}, + {s2, mkMPObj(s2)}, + {s3, mkMPObj(s3)}, + } + + // This function is a reference (re-)implementation of + // decrypted range computation, written solely for the purpose + // of the unit tests. + // + // `s` gives the decrypted part sizes, and the other + // parameters describe the desired read segment. When + // `isFromEnd` is true, `skipLen` argument is ignored. + decryptedRangeRef := func(s []int64, skipLen, readLen int64, isFromEnd bool) (o, l, skip int64, sn uint32, ps int) { + oSize := lsum(s) + if isFromEnd { + skipLen = oSize - readLen + } + if skipLen < 0 || readLen < 0 || oSize < 0 || skipLen+readLen > oSize { + t.Fatalf("Impossible read specified: %d %d %d", skipLen, readLen, oSize) + } + + var cumulativeSum, cumulativeEncSum int64 + toRead := readLen + readStart := false + for i, v := range s { + partOffset := int64(0) + partDarePkgOffset := int64(0) + if !readStart && cumulativeSum+v > skipLen { + // Read starts at the current part + readStart = true + + partOffset = skipLen - cumulativeSum + + // All return values except `l` are + // calculated here. + sn = uint32(partOffset / pkgSz) + skip = partOffset % pkgSz + ps = i + o = cumulativeEncSum + int64(sn)*(pkgSz+32) + + partDarePkgOffset = partOffset - skip + } + if readStart { + currentPartBytes := v - partOffset + currentPartDareBytes := v - partDarePkgOffset + if currentPartBytes < toRead { + toRead -= currentPartBytes + l += getEncSize(currentPartDareBytes) + } else { + // current part has the last + // byte required + lbPartOffset := partOffset + toRead - 1 + + // round up the lbPartOffset + // to the end of the + // corresponding DARE package + lbPkgEndOffset := min(lbPartOffset-(lbPartOffset%pkgSz)+pkgSz, v) + bytesToDrop := v - lbPkgEndOffset + + // Last segment to update `l` + l += getEncSize(currentPartDareBytes - bytesToDrop) + break + } + } + + cumulativeSum += v + cumulativeEncSum += getEncSize(v) + } + return o, l, skip, sn, ps + } + + for i, test := range testMPs { + { + // nil range + o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(nil) + if err != nil { + t.Errorf("Case %d: unexpected err: %v", i, err) + } + if o != 0 || l != esum(test.oi) || skip != 0 || sn != 0 || ps != 0 { + t.Errorf("Case %d: test failed: %d %d %d %d %d", i, o, l, skip, sn, ps) + } + } + + // Skip 1Mib and read 1Mib (in the decrypted object) + // + // The check below ensures the object is large enough + // for the read. + if lsum(test.decSizes) >= 2*humanize.MiByte { + skipLen, readLen := int64(1)*humanize.MiByte, int64(1)*humanize.MiByte + o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{false, skipLen, skipLen + readLen - 1}) + if err != nil { + t.Errorf("Case %d: unexpected err: %v", i, err) + } + + oRef, lRef, skipRef, snRef, psRef := decryptedRangeRef(test.decSizes, skipLen, readLen, false) + if o != oRef || l != lRef || skip != skipRef || sn != snRef || ps != psRef { + t.Errorf("Case %d: test failed: %d %d %d %d %d (Ref: %d %d %d %d %d)", + i, o, l, skip, sn, ps, oRef, lRef, skipRef, snRef, psRef) + } + } + + // Read the last 6Mib+1 bytes of the (decrypted) + // object + // + // The check below ensures the object is large enough + // for the read. + readLen := int64(6)*humanize.MiByte + 1 + if lsum(test.decSizes) >= readLen { + o, l, skip, sn, ps, err := test.oi.GetDecryptedRange(&HTTPRangeSpec{true, -readLen, -1}) + if err != nil { + t.Errorf("Case %d: unexpected err: %v", i, err) + } + + oRef, lRef, skipRef, snRef, psRef := decryptedRangeRef(test.decSizes, 0, readLen, true) + if o != oRef || l != lRef || skip != skipRef || sn != snRef || ps != psRef { + t.Errorf("Case %d: test failed: %d %d %d %d %d (Ref: %d %d %d %d %d)", + i, o, l, skip, sn, ps, oRef, lRef, skipRef, snRef, psRef) + } + } + } +} + +var getDefaultOptsTests = []struct { + headers http.Header + copySource bool + metadata map[string]string + encryptionType encrypt.Type + err error +}{ + { + headers: http.Header{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{"AES256"}, + xhttp.AmzServerSideEncryptionCustomerKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + copySource: false, + metadata: nil, + encryptionType: encrypt.SSEC, + err: nil, + }, // 0 + { + headers: http.Header{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{"AES256"}, + xhttp.AmzServerSideEncryptionCustomerKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + copySource: true, + metadata: nil, + encryptionType: "", + err: nil, + }, // 1 + { + headers: http.Header{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{"AES256"}, + xhttp.AmzServerSideEncryptionCustomerKey: []string{"Mz"}, + xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + copySource: false, + metadata: nil, + encryptionType: "", + err: crypto.ErrInvalidCustomerKey, + }, // 2 + { + headers: http.Header{xhttp.AmzServerSideEncryption: []string{"AES256"}}, + copySource: false, + metadata: nil, + encryptionType: encrypt.S3, + err: nil, + }, // 3 + { + headers: http.Header{}, + copySource: false, + metadata: map[string]string{ + crypto.MetaSealedKeyS3: base64.StdEncoding.EncodeToString(make([]byte, 64)), + crypto.MetaKeyID: "kms-key", + crypto.MetaDataEncryptionKey: "m-key", + }, + encryptionType: encrypt.S3, + err: nil, + }, // 4 + { + headers: http.Header{}, + copySource: true, + metadata: map[string]string{ + crypto.MetaSealedKeyS3: base64.StdEncoding.EncodeToString(make([]byte, 64)), + crypto.MetaKeyID: "kms-key", + crypto.MetaDataEncryptionKey: "m-key", + }, + encryptionType: "", + err: nil, + }, // 5 + { + headers: http.Header{ + xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: []string{"AES256"}, + xhttp.AmzServerSideEncryptionCopyCustomerKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + copySource: true, + metadata: nil, + encryptionType: encrypt.SSEC, + err: nil, + }, // 6 + { + headers: http.Header{ + xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: []string{"AES256"}, + xhttp.AmzServerSideEncryptionCopyCustomerKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + copySource: false, + metadata: nil, + encryptionType: "", + err: nil, + }, // 7 +} + +func TestGetDefaultOpts(t *testing.T) { + for i, test := range getDefaultOptsTests { + opts, err := getDefaultOpts(test.headers, test.copySource, test.metadata) + if test.err != err { + t.Errorf("Case %d: expected err: %v , actual err: %v", i, test.err, err) + } + if err == nil { + if opts.ServerSideEncryption == nil && test.encryptionType != "" { + t.Errorf("Case %d: expected opts to be of %v encryption type", i, test.encryptionType) + } + if opts.ServerSideEncryption != nil && test.encryptionType != opts.ServerSideEncryption.Type() { + t.Errorf("Case %d: expected opts to have encryption type %v but was %v ", i, test.encryptionType, opts.ServerSideEncryption.Type()) + } + } + } +} diff --git a/cmd/endpoint-ellipses.go b/cmd/endpoint-ellipses.go new file mode 100644 index 00000000..f1924945 --- /dev/null +++ b/cmd/endpoint-ellipses.go @@ -0,0 +1,523 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "errors" + "fmt" + "net/url" + "runtime" + "slices" + "strings" + + "github.com/cespare/xxhash/v2" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/config" + "github.com/minio/pkg/v3/ellipses" + "github.com/minio/pkg/v3/env" +) + +// This file implements and supports ellipses pattern for +// `minio server` command line arguments. + +// Endpoint set represents parsed ellipses values, also provides +// methods to get the sets of endpoints. +type endpointSet struct { + argPatterns []ellipses.ArgPattern + endpoints []string // Endpoints saved from previous GetEndpoints(). + setIndexes [][]uint64 // All the sets. +} + +// Supported set sizes this is used to find the optimal +// single set size. +var setSizes = []uint64{2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16} + +// getDivisibleSize - returns a greatest common divisor of +// all the ellipses sizes. +func getDivisibleSize(totalSizes []uint64) (result uint64) { + gcd := func(x, y uint64) uint64 { + for y != 0 { + x, y = y, x%y + } + return x + } + result = totalSizes[0] + for i := 1; i < len(totalSizes); i++ { + result = gcd(result, totalSizes[i]) + } + return result +} + +// isValidSetSize - checks whether given count is a valid set size for erasure coding. +var isValidSetSize = func(count uint64) bool { + return (count >= setSizes[0] && count <= setSizes[len(setSizes)-1]) +} + +func commonSetDriveCount(divisibleSize uint64, setCounts []uint64) (setSize uint64) { + // prefers setCounts to be sorted for optimal behavior. + if divisibleSize < setCounts[len(setCounts)-1] { + return divisibleSize + } + + // Figure out largest value of total_drives_in_erasure_set which results + // in least number of total_drives/total_drives_erasure_set ratio. + prevD := divisibleSize / setCounts[0] + for _, cnt := range setCounts { + if divisibleSize%cnt == 0 { + d := divisibleSize / cnt + if d <= prevD { + prevD = d + setSize = cnt + } + } + } + return setSize +} + +// possibleSetCountsWithSymmetry returns symmetrical setCounts based on the +// input argument patterns, the symmetry calculation is to ensure that +// we also use uniform number of drives common across all ellipses patterns. +func possibleSetCountsWithSymmetry(setCounts []uint64, argPatterns []ellipses.ArgPattern) []uint64 { + newSetCounts := make(map[uint64]struct{}) + for _, ss := range setCounts { + var symmetry bool + for _, argPattern := range argPatterns { + for _, p := range argPattern { + if uint64(len(p.Seq)) > ss { + symmetry = uint64(len(p.Seq))%ss == 0 + } else { + symmetry = ss%uint64(len(p.Seq)) == 0 + } + } + } + // With no arg patterns, it is expected that user knows + // the right symmetry, so either ellipses patterns are + // provided (recommended) or no ellipses patterns. + if _, ok := newSetCounts[ss]; !ok && (symmetry || argPatterns == nil) { + newSetCounts[ss] = struct{}{} + } + } + + setCounts = []uint64{} + for setCount := range newSetCounts { + setCounts = append(setCounts, setCount) + } + + // Not necessarily needed but it ensures to the readers + // eyes that we prefer a sorted setCount slice for the + // subsequent function to figure out the right common + // divisor, it avoids loops. + slices.Sort(setCounts) + + return setCounts +} + +// getSetIndexes returns list of indexes which provides the set size +// on each index, this function also determines the final set size +// The final set size has the affinity towards choosing smaller +// indexes (total sets) +func getSetIndexes(args []string, totalSizes []uint64, setDriveCount uint64, argPatterns []ellipses.ArgPattern) (setIndexes [][]uint64, err error) { + if len(totalSizes) == 0 || len(args) == 0 { + return nil, errInvalidArgument + } + + setIndexes = make([][]uint64, len(totalSizes)) + for _, totalSize := range totalSizes { + // Check if totalSize has minimum range upto setSize + if totalSize < setSizes[0] || totalSize < setDriveCount { + msg := fmt.Sprintf("Incorrect number of endpoints provided %s", args) + return nil, config.ErrInvalidNumberOfErasureEndpoints(nil).Msg(msg) + } + } + + commonSize := getDivisibleSize(totalSizes) + possibleSetCounts := func(setSize uint64) (ss []uint64) { + for _, s := range setSizes { + if setSize%s == 0 { + ss = append(ss, s) + } + } + return ss + } + + setCounts := possibleSetCounts(commonSize) + if len(setCounts) == 0 { + msg := fmt.Sprintf("Incorrect number of endpoints provided %s, number of drives %d is not divisible by any supported erasure set sizes %d", args, commonSize, setSizes) + return nil, config.ErrInvalidNumberOfErasureEndpoints(nil).Msg(msg) + } + + var setSize uint64 + // Custom set drive count allows to override automatic distribution. + // only meant if you want to further optimize drive distribution. + if setDriveCount > 0 { + msg := fmt.Sprintf("Invalid set drive count. Acceptable values for %d number drives are %d", commonSize, setCounts) + var found bool + for _, ss := range setCounts { + if ss == setDriveCount { + found = true + } + } + if !found { + return nil, config.ErrInvalidErasureSetSize(nil).Msg(msg) + } + + // No automatic symmetry calculation expected, user is on their own + setSize = setDriveCount + } else { + // Returns possible set counts with symmetry. + setCounts = possibleSetCountsWithSymmetry(setCounts, argPatterns) + + if len(setCounts) == 0 { + msg := fmt.Sprintf("No symmetric distribution detected with input endpoints provided %s, drives %d cannot be spread symmetrically by any supported erasure set sizes %d", args, commonSize, setSizes) + return nil, config.ErrInvalidNumberOfErasureEndpoints(nil).Msg(msg) + } + + // Final set size with all the symmetry accounted for. + setSize = commonSetDriveCount(commonSize, setCounts) + } + + // Check whether setSize is with the supported range. + if !isValidSetSize(setSize) { + msg := fmt.Sprintf("Incorrect number of endpoints provided %s, number of drives %d is not divisible by any supported erasure set sizes %d", args, commonSize, setSizes) + return nil, config.ErrInvalidNumberOfErasureEndpoints(nil).Msg(msg) + } + + for i := range totalSizes { + for j := uint64(0); j < totalSizes[i]/setSize; j++ { + setIndexes[i] = append(setIndexes[i], setSize) + } + } + + return setIndexes, nil +} + +// Returns all the expanded endpoints, each argument is expanded separately. +func (s *endpointSet) getEndpoints() (endpoints []string) { + if len(s.endpoints) != 0 { + return s.endpoints + } + for _, argPattern := range s.argPatterns { + for _, lbls := range argPattern.Expand() { + endpoints = append(endpoints, strings.Join(lbls, "")) + } + } + s.endpoints = endpoints + return endpoints +} + +// Get returns the sets representation of the endpoints +// this function also intelligently decides on what will +// be the right set size etc. +func (s endpointSet) Get() (sets [][]string) { + k := uint64(0) + endpoints := s.getEndpoints() + for i := range s.setIndexes { + for j := range s.setIndexes[i] { + sets = append(sets, endpoints[k:s.setIndexes[i][j]+k]) + k = s.setIndexes[i][j] + k + } + } + + return sets +} + +// Return the total size for each argument patterns. +func getTotalSizes(argPatterns []ellipses.ArgPattern) []uint64 { + var totalSizes []uint64 + for _, argPattern := range argPatterns { + var totalSize uint64 = 1 + for _, p := range argPattern { + totalSize *= uint64(len(p.Seq)) + } + totalSizes = append(totalSizes, totalSize) + } + return totalSizes +} + +// Parses all arguments and returns an endpointSet which is a collection +// of endpoints following the ellipses pattern, this is what is used +// by the object layer for initializing itself. +func parseEndpointSet(setDriveCount uint64, args ...string) (ep endpointSet, err error) { + argPatterns := make([]ellipses.ArgPattern, len(args)) + for i, arg := range args { + patterns, perr := ellipses.FindEllipsesPatterns(arg) + if perr != nil { + return endpointSet{}, config.ErrInvalidErasureEndpoints(nil).Msg(perr.Error()) + } + argPatterns[i] = patterns + } + + ep.setIndexes, err = getSetIndexes(args, getTotalSizes(argPatterns), setDriveCount, argPatterns) + if err != nil { + return endpointSet{}, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error()) + } + + ep.argPatterns = argPatterns + + return ep, nil +} + +// GetAllSets - parses all ellipses input arguments, expands them into +// corresponding list of endpoints chunked evenly in accordance with a +// specific set size. +// For example: {1...64} is divided into 4 sets each of size 16. +// This applies to even distributed setup syntax as well. +func GetAllSets(setDriveCount uint64, args ...string) ([][]string, error) { + var setArgs [][]string + if !ellipses.HasEllipses(args...) { + var setIndexes [][]uint64 + // Check if we have more one args. + if len(args) > 1 { + var err error + setIndexes, err = getSetIndexes(args, []uint64{uint64(len(args))}, setDriveCount, nil) + if err != nil { + return nil, err + } + } else { + // We are in FS setup, proceed forward. + setIndexes = [][]uint64{{uint64(len(args))}} + } + s := endpointSet{ + endpoints: args, + setIndexes: setIndexes, + } + setArgs = s.Get() + } else { + s, err := parseEndpointSet(setDriveCount, args...) + if err != nil { + return nil, err + } + setArgs = s.Get() + } + + uniqueArgs := set.NewStringSet() + for _, sargs := range setArgs { + for _, arg := range sargs { + if uniqueArgs.Contains(arg) { + return nil, config.ErrInvalidErasureEndpoints(nil).Msgf("Input args (%s) has duplicate ellipses", args) + } + uniqueArgs.Add(arg) + } + } + + return setArgs, nil +} + +// Override set drive count for manual distribution. +const ( + EnvErasureSetDriveCount = "MINIO_ERASURE_SET_DRIVE_COUNT" +) + +type node struct { + nodeName string + disks []string +} + +type endpointsList []node + +func (el *endpointsList) add(arg string) error { + u, err := url.Parse(arg) + if err != nil { + return err + } + found := false + list := *el + for i := range list { + if list[i].nodeName == u.Host { + list[i].disks = append(list[i].disks, u.String()) + found = true + break + } + } + if !found { + list = append(list, node{nodeName: u.Host, disks: []string{u.String()}}) + } + *el = list + return nil +} + +type poolArgs struct { + args []string + setDriveCount uint64 +} + +// buildDisksLayoutFromConfFile supports with and without ellipses transparently. +func buildDisksLayoutFromConfFile(pools []poolArgs) (layout disksLayout, err error) { + if len(pools) == 0 { + return layout, errInvalidArgument + } + + for _, list := range pools { + var endpointsList endpointsList + + for _, arg := range list.args { + switch { + case ellipses.HasList(arg): + patterns, err := ellipses.FindListPatterns(arg) + if err != nil { + return layout, err + } + for _, exp := range patterns.Expand() { + for _, ep := range exp { + if err := endpointsList.add(ep); err != nil { + return layout, err + } + } + } + case ellipses.HasEllipses(arg): + patterns, err := ellipses.FindEllipsesPatterns(arg) + if err != nil { + return layout, err + } + for _, exp := range patterns.Expand() { + if err := endpointsList.add(strings.Join(exp, "")); err != nil { + return layout, err + } + } + default: + if err := endpointsList.add(arg); err != nil { + return layout, err + } + } + } + + var stopping bool + var singleNode bool + var eps []string + + for i := 0; ; i++ { + for _, node := range endpointsList { + if node.nodeName == "" { + singleNode = true + } + + if len(node.disks) <= i { + stopping = true + continue + } + if stopping { + return layout, errors.New("number of disks per node does not match") + } + eps = append(eps, node.disks[i]) + } + if stopping { + break + } + } + + for _, node := range endpointsList { + if node.nodeName != "" && singleNode { + return layout, errors.New("all arguments must but either single node or distributed") + } + } + + setArgs, err := GetAllSets(list.setDriveCount, eps...) + if err != nil { + return layout, err + } + + h := xxhash.New() + for _, s := range setArgs { + for _, d := range s { + h.WriteString(d) + } + } + + layout.pools = append(layout.pools, poolDisksLayout{ + cmdline: fmt.Sprintf("hash:%x", h.Sum(nil)), + layout: setArgs, + }) + } + return layout, err +} + +// mergeDisksLayoutFromArgs supports with and without ellipses transparently. +func mergeDisksLayoutFromArgs(args []string, ctxt *serverCtxt) (err error) { + if len(args) == 0 { + return errInvalidArgument + } + + ok := true + for _, arg := range args { + ok = ok && !ellipses.HasEllipses(arg) + } + + var setArgs [][]string + + v, err := env.GetInt(EnvErasureSetDriveCount, 0) + if err != nil { + return err + } + setDriveCount := uint64(v) + + // None of the args have ellipses use the old style. + if ok { + setArgs, err = GetAllSets(setDriveCount, args...) + if err != nil { + return err + } + ctxt.Layout = disksLayout{ + legacy: true, + pools: []poolDisksLayout{{layout: setArgs, cmdline: strings.Join(args, " ")}}, + } + return err + } + + for _, arg := range args { + if !ellipses.HasEllipses(arg) && len(args) > 1 { + // TODO: support SNSD deployments to be decommissioned in future + return fmt.Errorf("all args must have ellipses for pool expansion (%w) args: %s", errInvalidArgument, args) + } + setArgs, err = GetAllSets(setDriveCount, arg) + if err != nil { + return err + } + ctxt.Layout.pools = append(ctxt.Layout.pools, poolDisksLayout{cmdline: arg, layout: setArgs}) + } + return err +} + +// CreateServerEndpoints - validates and creates new endpoints from input args, supports +// both ellipses and without ellipses transparently. +func createServerEndpoints(serverAddr string, poolArgs []poolDisksLayout, legacy bool) ( + endpointServerPools EndpointServerPools, setupType SetupType, err error, +) { + if len(poolArgs) == 0 { + return nil, -1, errInvalidArgument + } + + poolEndpoints, setupType, err := CreatePoolEndpoints(serverAddr, poolArgs...) + if err != nil { + return nil, -1, err + } + + for i, endpointList := range poolEndpoints { + if err = endpointServerPools.Add(PoolEndpoints{ + Legacy: legacy, + SetCount: len(poolArgs[i].layout), + DrivesPerSet: len(poolArgs[i].layout[0]), + Endpoints: endpointList, + Platform: fmt.Sprintf("OS: %s | Arch: %s", runtime.GOOS, runtime.GOARCH), + CmdLine: poolArgs[i].cmdline, + }); err != nil { + return nil, -1, err + } + } + + return endpointServerPools, setupType, nil +} diff --git a/cmd/endpoint-ellipses_test.go b/cmd/endpoint-ellipses_test.go new file mode 100644 index 00000000..6caaebbb --- /dev/null +++ b/cmd/endpoint-ellipses_test.go @@ -0,0 +1,649 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "reflect" + "testing" + + "github.com/minio/pkg/v3/ellipses" +) + +// Tests create endpoints with ellipses and without. +func TestCreateServerEndpoints(t *testing.T) { + testCases := []struct { + serverAddr string + args []string + success bool + }{ + // Invalid input. + {"", []string{}, false}, + // Range cannot be negative. + {":9000", []string{"/export1{-1...1}"}, false}, + // Range cannot start bigger than end. + {":9000", []string{"/export1{64...1}"}, false}, + // Range can only be numeric. + {":9000", []string{"/export1{a...z}"}, false}, + // Duplicate disks not allowed. + {":9000", []string{"/export1{1...32}", "/export1{1...32}"}, false}, + // Same host cannot export same disk on two ports - special case localhost. + {":9001", []string{"http://localhost:900{1...2}/export{1...64}"}, false}, + // Valid inputs. + {":9000", []string{"/export1"}, true}, + {":9000", []string{"/export1", "/export2", "/export3", "/export4"}, true}, + {":9000", []string{"/export1{1...64}"}, true}, + {":9000", []string{"/export1{01...64}"}, true}, + {":9000", []string{"/export1{1...32}", "/export1{33...64}"}, true}, + {":9001", []string{"http://localhost:9001/export{1...64}"}, true}, + {":9001", []string{"http://localhost:9001/export{01...64}"}, true}, + } + + for i, testCase := range testCases { + t.Run("", func(t *testing.T) { + srvCtxt := serverCtxt{} + err := mergeDisksLayoutFromArgs(testCase.args, &srvCtxt) + if err != nil && testCase.success { + t.Fatalf("Test %d: unexpected error: %v", i+1, err) + } + _, _, err = createServerEndpoints(testCase.serverAddr, srvCtxt.Layout.pools, srvCtxt.Layout.legacy) + if err != nil && testCase.success { + t.Errorf("Test %d: Expected success but failed instead %s", i+1, err) + } + if err == nil && !testCase.success { + t.Errorf("Test %d: Expected failure but passed instead", i+1) + } + }) + } +} + +func TestGetDivisibleSize(t *testing.T) { + testCases := []struct { + totalSizes []uint64 + result uint64 + }{ + {[]uint64{24, 32, 16}, 8}, + {[]uint64{32, 8, 4}, 4}, + {[]uint64{8, 8, 8}, 8}, + {[]uint64{24}, 24}, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + gotGCD := getDivisibleSize(testCase.totalSizes) + if testCase.result != gotGCD { + t.Errorf("Expected %v, got %v", testCase.result, gotGCD) + } + }) + } +} + +// Test tests calculating set indexes with ENV override for drive count. +func TestGetSetIndexesEnvOverride(t *testing.T) { + testCases := []struct { + args []string + totalSizes []uint64 + indexes [][]uint64 + envOverride uint64 + success bool + }{ + { + []string{"data{1...64}"}, + []uint64{64}, + [][]uint64{{8, 8, 8, 8, 8, 8, 8, 8}}, + 8, + true, + }, + { + []string{"http://host{1...2}/data{1...180}"}, + []uint64{360}, + [][]uint64{{15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}}, + 15, + true, + }, + { + []string{"http://host{1...12}/data{1...12}"}, + []uint64{144}, + [][]uint64{{12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12}}, + 12, + true, + }, + { + []string{"http://host{0...5}/data{1...28}"}, + []uint64{168}, + [][]uint64{{12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12}}, + 12, + true, + }, + // Incorrect custom set drive count. + { + []string{"http://host{0...5}/data{1...28}"}, + []uint64{168}, + nil, + 10, + false, + }, + // Failure not divisible number of disks. + { + []string{"http://host{1...11}/data{1...11}"}, + []uint64{121}, + [][]uint64{{11, 11, 11, 11, 11, 11, 11, 11, 11, 11, 11}}, + 11, + true, + }, + { + []string{"data{1...60}"}, + nil, + nil, + 8, + false, + }, + { + []string{"data{1...64}"}, + nil, + nil, + 64, + false, + }, + { + []string{"data{1...64}"}, + nil, + nil, + 2, + false, + }, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + argPatterns := make([]ellipses.ArgPattern, len(testCase.args)) + for i, arg := range testCase.args { + patterns, err := ellipses.FindEllipsesPatterns(arg) + if err != nil { + t.Fatalf("Unexpected failure %s", err) + } + argPatterns[i] = patterns + } + + gotIndexes, err := getSetIndexes(testCase.args, testCase.totalSizes, testCase.envOverride, argPatterns) + if err != nil && testCase.success { + t.Errorf("Expected success but failed instead %s", err) + } + if err == nil && !testCase.success { + t.Errorf("Expected failure but passed instead") + } + if !reflect.DeepEqual(testCase.indexes, gotIndexes) { + t.Errorf("Expected %v, got %v", testCase.indexes, gotIndexes) + } + }) + } +} + +// Test tests calculating set indexes. +func TestGetSetIndexes(t *testing.T) { + testCases := []struct { + args []string + totalSizes []uint64 + indexes [][]uint64 + success bool + }{ + // Invalid inputs. + { + []string{"data{1...17}/export{1...52}"}, + []uint64{14144}, + nil, + false, + }, + // Valid inputs. + { + []string{"data{1...3}"}, + []uint64{3}, + [][]uint64{{3}}, + true, + }, + { + []string{"data/controller1/export{1...2}, data/controller2/export{1...4}, data/controller3/export{1...8}"}, + []uint64{2, 4, 8}, + [][]uint64{{2}, {2, 2}, {2, 2, 2, 2}}, + true, + }, + { + []string{"data{1...27}"}, + []uint64{27}, + [][]uint64{{9, 9, 9}}, + true, + }, + { + []string{"http://host{1...3}/data{1...180}"}, + []uint64{540}, + [][]uint64{{15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15}}, + true, + }, + { + []string{"http://host{1...2}.rack{1...4}/data{1...180}"}, + []uint64{1440}, + [][]uint64{{16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16}}, + true, + }, + { + []string{"http://host{1...2}/data{1...180}"}, + []uint64{360}, + [][]uint64{{12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12, 12}}, + true, + }, + { + []string{"data/controller1/export{1...4}, data/controller2/export{1...8}, data/controller3/export{1...12}"}, + []uint64{4, 8, 12}, + [][]uint64{{4}, {4, 4}, {4, 4, 4}}, + true, + }, + { + []string{"data{1...64}"}, + []uint64{64}, + [][]uint64{{16, 16, 16, 16}}, + true, + }, + { + []string{"data{1...24}"}, + []uint64{24}, + [][]uint64{{12, 12}}, + true, + }, + { + []string{"data/controller{1...11}/export{1...8}"}, + []uint64{88}, + [][]uint64{{11, 11, 11, 11, 11, 11, 11, 11}}, + true, + }, + { + []string{"data{1...4}"}, + []uint64{4}, + [][]uint64{{4}}, + true, + }, + { + []string{"data/controller1/export{1...10}, data/controller2/export{1...10}, data/controller3/export{1...10}"}, + []uint64{10, 10, 10}, + [][]uint64{{10}, {10}, {10}}, + true, + }, + { + []string{"data{1...16}/export{1...52}"}, + []uint64{832}, + [][]uint64{{16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16, 16}}, + true, + }, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + argPatterns := make([]ellipses.ArgPattern, len(testCase.args)) + for i, arg := range testCase.args { + patterns, err := ellipses.FindEllipsesPatterns(arg) + if err != nil { + t.Fatalf("Unexpected failure %s", err) + } + argPatterns[i] = patterns + } + gotIndexes, err := getSetIndexes(testCase.args, testCase.totalSizes, 0, argPatterns) + if err != nil && testCase.success { + t.Errorf("Expected success but failed instead %s", err) + } + if err == nil && !testCase.success { + t.Errorf("Expected failure but passed instead") + } + if !reflect.DeepEqual(testCase.indexes, gotIndexes) { + t.Errorf("Expected %v, got %v", testCase.indexes, gotIndexes) + } + }) + } +} + +func getHexSequences(start int, number int, paddinglen int) (seq []string) { + for i := start; i <= number; i++ { + if paddinglen == 0 { + seq = append(seq, fmt.Sprintf("%x", i)) + } else { + seq = append(seq, fmt.Sprintf(fmt.Sprintf("%%0%dx", paddinglen), i)) + } + } + return seq +} + +func getSequences(start int, number int, paddinglen int) (seq []string) { + for i := start; i <= number; i++ { + if paddinglen == 0 { + seq = append(seq, fmt.Sprintf("%d", i)) + } else { + seq = append(seq, fmt.Sprintf(fmt.Sprintf("%%0%dd", paddinglen), i)) + } + } + return seq +} + +// Test tests parses endpoint ellipses input pattern. +func TestParseEndpointSet(t *testing.T) { + testCases := []struct { + arg string + es endpointSet + success bool + }{ + // Tests invalid inputs. + { + "...", + endpointSet{}, + false, + }, + // No range specified. + { + "{...}", + endpointSet{}, + false, + }, + // Invalid range. + { + "http://minio{2...3}/export/set{1...0}", + endpointSet{}, + false, + }, + // Range cannot be smaller than 4 minimum. + { + "/export{1..2}", + endpointSet{}, + false, + }, + // Unsupported characters. + { + "/export/test{1...2O}", + endpointSet{}, + false, + }, + // Tests valid inputs. + { + "{1...27}", + endpointSet{ + []ellipses.ArgPattern{ + []ellipses.Pattern{ + { + Prefix: "", + Suffix: "", + Seq: getSequences(1, 27, 0), + }, + }, + }, + nil, + [][]uint64{{9, 9, 9}}, + }, + true, + }, + { + "/export/set{1...64}", + endpointSet{ + []ellipses.ArgPattern{ + []ellipses.Pattern{ + { + Prefix: "/export/set", + Suffix: "", + Seq: getSequences(1, 64, 0), + }, + }, + }, + nil, + [][]uint64{{16, 16, 16, 16}}, + }, + true, + }, + // Valid input for distributed setup. + { + "http://minio{2...3}/export/set{1...64}", + endpointSet{ + []ellipses.ArgPattern{ + []ellipses.Pattern{ + { + Prefix: "", + Suffix: "", + Seq: getSequences(1, 64, 0), + }, + { + Prefix: "http://minio", + Suffix: "/export/set", + Seq: getSequences(2, 3, 0), + }, + }, + }, + nil, + [][]uint64{{16, 16, 16, 16, 16, 16, 16, 16}}, + }, + true, + }, + // Supporting some advanced cases. + { + "http://minio{1...64}.mydomain.net/data", + endpointSet{ + []ellipses.ArgPattern{ + []ellipses.Pattern{ + { + Prefix: "http://minio", + Suffix: ".mydomain.net/data", + Seq: getSequences(1, 64, 0), + }, + }, + }, + nil, + [][]uint64{{16, 16, 16, 16}}, + }, + true, + }, + { + "http://rack{1...4}.mydomain.minio{1...16}/data", + endpointSet{ + []ellipses.ArgPattern{ + []ellipses.Pattern{ + { + Prefix: "", + Suffix: "/data", + Seq: getSequences(1, 16, 0), + }, + { + Prefix: "http://rack", + Suffix: ".mydomain.minio", + Seq: getSequences(1, 4, 0), + }, + }, + }, + nil, + [][]uint64{{16, 16, 16, 16}}, + }, + true, + }, + // Supporting kubernetes cases. + { + "http://minio{0...15}.mydomain.net/data{0...1}", + endpointSet{ + []ellipses.ArgPattern{ + []ellipses.Pattern{ + { + Prefix: "", + Suffix: "", + Seq: getSequences(0, 1, 0), + }, + { + Prefix: "http://minio", + Suffix: ".mydomain.net/data", + Seq: getSequences(0, 15, 0), + }, + }, + }, + nil, + [][]uint64{{16, 16}}, + }, + true, + }, + // No host regex, just disks. + { + "http://server1/data{1...32}", + endpointSet{ + []ellipses.ArgPattern{ + []ellipses.Pattern{ + { + Prefix: "http://server1/data", + Suffix: "", + Seq: getSequences(1, 32, 0), + }, + }, + }, + nil, + [][]uint64{{16, 16}}, + }, + true, + }, + // No host regex, just disks with two position numerics. + { + "http://server1/data{01...32}", + endpointSet{ + []ellipses.ArgPattern{ + []ellipses.Pattern{ + { + Prefix: "http://server1/data", + Suffix: "", + Seq: getSequences(1, 32, 2), + }, + }, + }, + nil, + [][]uint64{{16, 16}}, + }, + true, + }, + // More than 2 ellipses are supported as well. + { + "http://minio{2...3}/export/set{1...64}/test{1...2}", + endpointSet{ + []ellipses.ArgPattern{ + []ellipses.Pattern{ + { + Prefix: "", + Suffix: "", + Seq: getSequences(1, 2, 0), + }, + { + Prefix: "", + Suffix: "/test", + Seq: getSequences(1, 64, 0), + }, + { + Prefix: "http://minio", + Suffix: "/export/set", + Seq: getSequences(2, 3, 0), + }, + }, + }, + nil, + [][]uint64{{ + 16, 16, 16, 16, 16, 16, 16, 16, + 16, 16, 16, 16, 16, 16, 16, 16, + }}, + }, + true, + }, + // More than 1 ellipses per argument for standalone setup. + { + "/export{1...10}/disk{1...10}", + endpointSet{ + []ellipses.ArgPattern{ + []ellipses.Pattern{ + { + Prefix: "", + Suffix: "", + Seq: getSequences(1, 10, 0), + }, + { + Prefix: "/export", + Suffix: "/disk", + Seq: getSequences(1, 10, 0), + }, + }, + }, + nil, + [][]uint64{{10, 10, 10, 10, 10, 10, 10, 10, 10, 10}}, + }, + true, + }, + // IPv6 ellipses with hexadecimal expansion + { + "http://[2001:3984:3989::{1...a}]/disk{1...10}", + endpointSet{ + []ellipses.ArgPattern{ + []ellipses.Pattern{ + { + Prefix: "", + Suffix: "", + Seq: getSequences(1, 10, 0), + }, + { + Prefix: "http://[2001:3984:3989::", + Suffix: "]/disk", + Seq: getHexSequences(1, 10, 0), + }, + }, + }, + nil, + [][]uint64{{10, 10, 10, 10, 10, 10, 10, 10, 10, 10}}, + }, + true, + }, + // IPv6 ellipses with hexadecimal expansion with 3 position numerics. + { + "http://[2001:3984:3989::{001...00a}]/disk{1...10}", + endpointSet{ + []ellipses.ArgPattern{ + []ellipses.Pattern{ + { + Prefix: "", + Suffix: "", + Seq: getSequences(1, 10, 0), + }, + { + Prefix: "http://[2001:3984:3989::", + Suffix: "]/disk", + Seq: getHexSequences(1, 10, 3), + }, + }, + }, + nil, + [][]uint64{{10, 10, 10, 10, 10, 10, 10, 10, 10, 10}}, + }, + true, + }, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + gotEs, err := parseEndpointSet(0, testCase.arg) + if err != nil && testCase.success { + t.Errorf("Expected success but failed instead %s", err) + } + if err == nil && !testCase.success { + t.Errorf("Expected failure but passed instead") + } + if !reflect.DeepEqual(testCase.es, gotEs) { + t.Errorf("Expected %v, got %v", testCase.es, gotEs) + } + }) + } +} diff --git a/cmd/endpoint.go b/cmd/endpoint.go new file mode 100644 index 00000000..dcfe2e61 --- /dev/null +++ b/cmd/endpoint.go @@ -0,0 +1,1262 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "net" + "net/http" + "net/url" + "path" + "path/filepath" + "reflect" + "runtime" + "slices" + "sort" + "strconv" + "strings" + "time" + + "github.com/dustin/go-humanize" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/mountinfo" + "github.com/minio/pkg/v3/env" + xnet "github.com/minio/pkg/v3/net" +) + +// EndpointType - enum for endpoint type. +type EndpointType int + +const ( + // PathEndpointType - path style endpoint type enum. + PathEndpointType EndpointType = iota + 1 + + // URLEndpointType - URL style endpoint type enum. + URLEndpointType +) + +// ProxyEndpoint - endpoint used for proxy redirects +// See proxyRequest() for details. +type ProxyEndpoint struct { + Endpoint + Transport http.RoundTripper +} + +// Node holds information about a node in this cluster +type Node struct { + *url.URL + Pools []int + IsLocal bool + GridHost string +} + +// Endpoint - any type of endpoint. +type Endpoint struct { + *url.URL + IsLocal bool + + PoolIdx, SetIdx, DiskIdx int +} + +// Equal returns true if endpoint == ep +func (endpoint Endpoint) Equal(ep Endpoint) bool { + if endpoint.IsLocal == ep.IsLocal && endpoint.PoolIdx == ep.PoolIdx && endpoint.SetIdx == ep.SetIdx && endpoint.DiskIdx == ep.DiskIdx { + if endpoint.Path == ep.Path && endpoint.Host == ep.Host { + return true + } + } + return false +} + +func (endpoint Endpoint) String() string { + if endpoint.Host == "" { + return endpoint.Path + } + + return endpoint.URL.String() +} + +// Type - returns type of endpoint. +func (endpoint Endpoint) Type() EndpointType { + if endpoint.Host == "" { + return PathEndpointType + } + + return URLEndpointType +} + +// HTTPS - returns true if secure for URLEndpointType. +func (endpoint Endpoint) HTTPS() bool { + return endpoint.Scheme == "https" +} + +// GridHost returns the host to be used for grid connections. +func (endpoint Endpoint) GridHost() string { + return fmt.Sprintf("%s://%s", endpoint.Scheme, endpoint.Host) +} + +// UpdateIsLocal - resolves the host and updates if it is local or not. +func (endpoint *Endpoint) UpdateIsLocal() (err error) { + if endpoint.Host != "" { + endpoint.IsLocal, err = isLocalHost(endpoint.Hostname(), endpoint.Port(), globalMinioPort) + if err != nil { + return err + } + } + return nil +} + +// SetPoolIndex sets a specific pool number to this node +func (endpoint *Endpoint) SetPoolIndex(i int) { + endpoint.PoolIdx = i +} + +// SetSetIndex sets a specific set number to this node +func (endpoint *Endpoint) SetSetIndex(i int) { + endpoint.SetIdx = i +} + +// SetDiskIndex sets a specific disk number to this node +func (endpoint *Endpoint) SetDiskIndex(i int) { + endpoint.DiskIdx = i +} + +func isValidURLEndpoint(u *url.URL) bool { + // URL style of endpoint. + // Valid URL style endpoint is + // - Scheme field must contain "http" or "https" + // - All field should be empty except Host and Path. + isURLOk := (u.Scheme == "http" || u.Scheme == "https") && + u.User == nil && u.Opaque == "" && !u.ForceQuery && + u.RawQuery == "" && u.Fragment == "" + return isURLOk +} + +// NewEndpoint - returns new endpoint based on given arguments. +func NewEndpoint(arg string) (ep Endpoint, e error) { + // isEmptyPath - check whether given path is not empty. + isEmptyPath := func(path string) bool { + return path == "" || path == SlashSeparator || path == `\` + } + + if isEmptyPath(arg) { + return ep, fmt.Errorf("empty or root endpoint is not supported") + } + + var isLocal bool + var host string + u, err := url.Parse(arg) + if err == nil && u.Host != "" { + // URL style of endpoint. + // Valid URL style endpoint is + // - Scheme field must contain "http" or "https" + // - All field should be empty except Host and Path. + if !isValidURLEndpoint(u) { + return ep, fmt.Errorf("invalid URL endpoint format") + } + + var port string + host, port, err = net.SplitHostPort(u.Host) + if err != nil { + if !strings.Contains(err.Error(), "missing port in address") { + return ep, fmt.Errorf("invalid URL endpoint format: %w", err) + } + + host = u.Host + } else { + var p int + p, err = strconv.Atoi(port) + if err != nil { + return ep, fmt.Errorf("invalid URL endpoint format: invalid port number") + } else if p < 1 || p > 65535 { + return ep, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535") + } + } + if i := strings.Index(host, "%"); i > -1 { + host = host[:i] + } + + if host == "" { + return ep, fmt.Errorf("invalid URL endpoint format: empty host name") + } + + // As this is path in the URL, we should use path package, not filepath package. + // On MS Windows, filepath.Clean() converts into Windows path style ie `/foo` becomes `\foo` + u.Path = path.Clean(u.Path) + if isEmptyPath(u.Path) { + return ep, fmt.Errorf("empty or root path is not supported in URL endpoint") + } + + // On windows having a preceding SlashSeparator will cause problems, if the + // command line already has C:/= len(endpoints) { + return "" + } + return endpoints[i].String() +} + +// GetAllStrings - returns allstring of all endpoints +func (endpoints Endpoints) GetAllStrings() (all []string) { + for _, e := range endpoints { + all = append(all, e.String()) + } + return all +} + +func hostResolveToLocalhost(endpoint Endpoint) bool { + hostIPs, err := getHostIP(endpoint.Hostname()) + if err != nil { + return false + } + var loopback int + for _, hostIP := range hostIPs.ToSlice() { + if net.ParseIP(hostIP).IsLoopback() { + loopback++ + } + } + return loopback == len(hostIPs) +} + +// UpdateIsLocal - resolves the host and discovers the local host. +func (endpoints Endpoints) UpdateIsLocal() error { + var epsResolved int + var foundLocal bool + resolvedList := make([]bool, len(endpoints)) + // Mark the starting time + startTime := time.Now() + keepAliveTicker := time.NewTicker(500 * time.Millisecond) + defer keepAliveTicker.Stop() + for !foundLocal && (epsResolved != len(endpoints)) { + // Break if the local endpoint is found already Or all the endpoints are resolved. + + // Retry infinitely on Kubernetes and Docker swarm. + // This is needed as the remote hosts are sometime + // not available immediately. + select { + case <-globalOSSignalCh: + return fmt.Errorf("The endpoint resolution got interrupted") + default: + for i, resolved := range resolvedList { + if resolved { + // Continue if host is already resolved. + continue + } + + if endpoints[i].Host == "" { + resolvedList[i] = true + endpoints[i].IsLocal = true + epsResolved++ + if !foundLocal { + foundLocal = true + } + continue + } + + // Log the message to console about the host resolving + reqInfo := (&logger.ReqInfo{}).AppendTags( + "host", + endpoints[i].Hostname(), + ) + + if orchestrated && hostResolveToLocalhost(endpoints[i]) { + // time elapsed + timeElapsed := time.Since(startTime) + // log error only if more than a second has elapsed + if timeElapsed > time.Second { + reqInfo.AppendTags("elapsedTime", + humanize.RelTime(startTime, + startTime.Add(timeElapsed), + "elapsed", + "", + )) + ctx := logger.SetReqInfo(GlobalContext, + reqInfo) + bootLogOnceIf(ctx, fmt.Errorf("%s resolves to localhost in a containerized deployment, waiting for it to resolve to a valid IP", + endpoints[i].Hostname()), endpoints[i].Hostname(), logger.ErrorKind) + } + + continue + } + + // return err if not Docker or Kubernetes + // We use IsDocker() to check for Docker environment + // We use IsKubernetes() to check for Kubernetes environment + isLocal, err := isLocalHost(endpoints[i].Hostname(), + endpoints[i].Port(), + globalMinioPort, + ) + if err != nil && !orchestrated { + return err + } + if err != nil { + // time elapsed + timeElapsed := time.Since(startTime) + // log error only if more than a second has elapsed + if timeElapsed > time.Second { + reqInfo.AppendTags("elapsedTime", + humanize.RelTime(startTime, + startTime.Add(timeElapsed), + "elapsed", + "", + )) + ctx := logger.SetReqInfo(GlobalContext, + reqInfo) + bootLogOnceIf(ctx, err, endpoints[i].Hostname(), logger.ErrorKind) + } + } else { + resolvedList[i] = true + endpoints[i].IsLocal = isLocal + epsResolved++ + if !foundLocal { + foundLocal = isLocal + } + } + } + + // Wait for the tick, if the there exist a local endpoint in discovery. + // Non docker/kubernetes environment we do not need to wait. + if !foundLocal && orchestrated { + <-keepAliveTicker.C + } + } + } + + // On Kubernetes/Docker setups DNS resolves inappropriately sometimes + // where there are situations same endpoints with multiple disks + // come online indicating either one of them is local and some + // of them are not local. This situation can never happen and + // its only a possibility in orchestrated deployments with dynamic + // DNS. Following code ensures that we treat if one of the endpoint + // says its local for a given host - it is true for all endpoints + // for the same host. Following code ensures that this assumption + // is true and it works in all scenarios and it is safe to assume + // for a given host. + endpointLocalMap := make(map[string]bool) + for _, ep := range endpoints { + if ep.IsLocal { + endpointLocalMap[ep.Host] = ep.IsLocal + } + } + for i := range endpoints { + endpoints[i].IsLocal = endpointLocalMap[endpoints[i].Host] + } + return nil +} + +// NewEndpoints - returns new endpoint list based on input args. +func NewEndpoints(args ...string) (endpoints Endpoints, err error) { + var endpointType EndpointType + var scheme string + + uniqueArgs := set.NewStringSet() + // Loop through args and adds to endpoint list. + for i, arg := range args { + endpoint, err := NewEndpoint(arg) + if err != nil { + return nil, fmt.Errorf("'%s': %s", arg, err.Error()) + } + + // All endpoints have to be same type and scheme if applicable. + //nolint:gocritic + if i == 0 { + endpointType = endpoint.Type() + scheme = endpoint.Scheme + } else if endpoint.Type() != endpointType { + return nil, fmt.Errorf("mixed style endpoints are not supported") + } else if endpoint.Scheme != scheme { + return nil, fmt.Errorf("mixed scheme is not supported") + } + + arg = endpoint.String() + if uniqueArgs.Contains(arg) { + return nil, fmt.Errorf("duplicate endpoints found") + } + uniqueArgs.Add(arg) + endpoints = append(endpoints, endpoint) + } + + return endpoints, nil +} + +// Checks if there are any cross device mounts. +func checkCrossDeviceMounts(endpoints Endpoints) (err error) { + var absPaths []string + for _, endpoint := range endpoints { + if endpoint.IsLocal { + var absPath string + absPath, err = filepath.Abs(endpoint.Path) + if err != nil { + return err + } + absPaths = append(absPaths, absPath) + } + } + return mountinfo.CheckCrossDevice(absPaths) +} + +// PoolEndpointList is a temporary type to holds the list of endpoints +type PoolEndpointList []Endpoints + +// UpdateIsLocal - resolves all hosts and discovers which are local +func (p PoolEndpointList) UpdateIsLocal() error { + var epsResolved int + var epCount int + + for _, endpoints := range p { + epCount += len(endpoints) + } + + var foundLocal bool + resolvedList := make(map[Endpoint]bool) + + // Mark the starting time + startTime := time.Now() + keepAliveTicker := time.NewTicker(1 * time.Second) + defer keepAliveTicker.Stop() + for !foundLocal && (epsResolved != epCount) { + // Break if the local endpoint is found already Or all the endpoints are resolved. + + // Retry infinitely on Kubernetes and Docker swarm. + // This is needed as the remote hosts are sometime + // not available immediately. + select { + case <-globalOSSignalCh: + return fmt.Errorf("The endpoint resolution got interrupted") + default: + for i, endpoints := range p { + for j, endpoint := range endpoints { + if resolvedList[endpoint] { + // Continue if host is already resolved. + continue + } + + if endpoint.Host == "" || (orchestrated && env.Get("_MINIO_SERVER_LOCAL", "") == endpoint.Host) { + if !foundLocal { + foundLocal = true + } + endpoint.IsLocal = true + endpoints[j] = endpoint + epsResolved++ + resolvedList[endpoint] = true + continue + } + + // Log the message to console about the host resolving + reqInfo := (&logger.ReqInfo{}).AppendTags( + "host", + endpoint.Hostname(), + ) + + if orchestrated && hostResolveToLocalhost(endpoint) { + // time elapsed + timeElapsed := time.Since(startTime) + // log error only if more than a second has elapsed + if timeElapsed > time.Second { + reqInfo.AppendTags("elapsedTime", + humanize.RelTime(startTime, + startTime.Add(timeElapsed), + "elapsed", + "", + )) + ctx := logger.SetReqInfo(GlobalContext, + reqInfo) + bootLogOnceIf(ctx, fmt.Errorf("%s resolves to localhost in a containerized deployment, waiting for it to resolve to a valid IP", + endpoint.Hostname()), endpoint.Hostname(), logger.ErrorKind) + } + continue + } + + // return err if not Docker or Kubernetes + // We use IsDocker() to check for Docker environment + // We use IsKubernetes() to check for Kubernetes environment + isLocal, err := isLocalHost(endpoint.Hostname(), + endpoint.Port(), + globalMinioPort, + ) + if err != nil && !orchestrated { + return err + } + if err != nil { + // time elapsed + timeElapsed := time.Since(startTime) + // log error only if more than a second has elapsed + if timeElapsed > time.Second { + reqInfo.AppendTags("elapsedTime", + humanize.RelTime(startTime, + startTime.Add(timeElapsed), + "elapsed", + "", + )) + ctx := logger.SetReqInfo(GlobalContext, + reqInfo) + bootLogOnceIf(ctx, fmt.Errorf("Unable to resolve DNS for %s: %w", endpoint, err), endpoint.Hostname(), logger.ErrorKind) + } + } else { + resolvedList[endpoint] = true + endpoint.IsLocal = isLocal + epsResolved++ + if !foundLocal { + foundLocal = isLocal + } + endpoints[j] = endpoint + } + } + + p[i] = endpoints + + // Wait for the tick, if the there exist a local endpoint in discovery. + // Non docker/kubernetes environment we do not need to wait. + if !foundLocal && orchestrated { + <-keepAliveTicker.C + } + } + } + } + + // On Kubernetes/Docker setups DNS resolves inappropriately sometimes + // where there are situations same endpoints with multiple disks + // come online indicating either one of them is local and some + // of them are not local. This situation can never happen and + // its only a possibility in orchestrated deployments with dynamic + // DNS. Following code ensures that we treat if one of the endpoint + // says its local for a given host - it is true for all endpoints + // for the same host. Following code ensures that this assumption + // is true and it works in all scenarios and it is safe to assume + // for a given host. + for i, endpoints := range p { + endpointLocalMap := make(map[string]bool) + for _, ep := range endpoints { + if ep.IsLocal { + endpointLocalMap[ep.Host] = ep.IsLocal + } + } + for i := range endpoints { + endpoints[i].IsLocal = endpointLocalMap[endpoints[i].Host] + } + p[i] = endpoints + } + + return nil +} + +func isEmptyLayout(poolsLayout ...poolDisksLayout) bool { + return len(poolsLayout) == 0 || len(poolsLayout[0].layout) == 0 || len(poolsLayout[0].layout[0]) == 0 || len(poolsLayout[0].layout[0][0]) == 0 +} + +func isSingleDriveLayout(poolsLayout ...poolDisksLayout) bool { + return len(poolsLayout) == 1 && len(poolsLayout[0].layout) == 1 && len(poolsLayout[0].layout[0]) == 1 +} + +// CreatePoolEndpoints creates a list of endpoints per pool, resolves their relevant hostnames and +// discovers those are local or remote. +func CreatePoolEndpoints(serverAddr string, poolsLayout ...poolDisksLayout) ([]Endpoints, SetupType, error) { + var setupType SetupType + + if isEmptyLayout(poolsLayout...) { + return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg("invalid number of endpoints") + } + + // Check whether serverAddr is valid for this host. + if err := CheckLocalServerAddr(serverAddr); err != nil { + return nil, setupType, err + } + + _, serverAddrPort := mustSplitHostPort(serverAddr) + + poolEndpoints := make(PoolEndpointList, len(poolsLayout)) + + // For single arg, return single drive EC setup. + if isSingleDriveLayout(poolsLayout...) { + endpoint, err := NewEndpoint(poolsLayout[0].layout[0][0]) + if err != nil { + return nil, setupType, err + } + if err := endpoint.UpdateIsLocal(); err != nil { + return nil, setupType, err + } + if endpoint.Type() != PathEndpointType { + return nil, setupType, config.ErrInvalidEndpoint(nil).Msg("use path style endpoint for single node setup") + } + + endpoint.SetPoolIndex(0) + endpoint.SetSetIndex(0) + endpoint.SetDiskIndex(0) + + var endpoints Endpoints + endpoints = append(endpoints, endpoint) + setupType = ErasureSDSetupType + + poolEndpoints[0] = endpoints + // Check for cross device mounts if any. + if err = checkCrossDeviceMounts(endpoints); err != nil { + return nil, setupType, config.ErrInvalidEndpoint(nil).Msg(err.Error()) + } + + return poolEndpoints, setupType, nil + } + + uniqueArgs := set.NewStringSet() + + for poolIdx, pool := range poolsLayout { + var endpoints Endpoints + for setIdx, setLayout := range pool.layout { + // Convert args to endpoints + eps, err := NewEndpoints(setLayout...) + if err != nil { + return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error()) + } + + // Check for cross device mounts if any. + if err = checkCrossDeviceMounts(eps); err != nil { + return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error()) + } + + for diskIdx := range eps { + eps[diskIdx].SetPoolIndex(poolIdx) + eps[diskIdx].SetSetIndex(setIdx) + eps[diskIdx].SetDiskIndex(diskIdx) + } + + endpoints = append(endpoints, eps...) + } + + if len(endpoints) == 0 { + return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg("invalid number of endpoints") + } + + poolEndpoints[poolIdx] = endpoints + } + + if err := poolEndpoints.UpdateIsLocal(); err != nil { + return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(err.Error()) + } + + for i, endpoints := range poolEndpoints { + // Here all endpoints are URL style. + endpointPathSet := set.NewStringSet() + localEndpointCount := 0 + localServerHostSet := set.NewStringSet() + localPortSet := set.NewStringSet() + + for _, endpoint := range endpoints { + endpointPathSet.Add(endpoint.Path) + if endpoint.IsLocal && endpoint.Host != "" { + localServerHostSet.Add(endpoint.Hostname()) + + _, port, err := net.SplitHostPort(endpoint.Host) + if err != nil { + port = serverAddrPort + } + localPortSet.Add(port) + + localEndpointCount++ + } + } + + reverseProxy := (env.Get("_MINIO_REVERSE_PROXY", "") != "") && ((env.Get("MINIO_CI_CD", "") != "") || (env.Get("CI", "") != "")) + // If not orchestrated + // and not setup in reverse proxy + if !orchestrated && !reverseProxy { + // Check whether same path is not used in endpoints of a host on different port. + // Only verify this on baremetal setups, DNS is not available in orchestrated + // environments so we can't do much here. + pathIPMap := make(map[string]set.StringSet) + hostIPCache := make(map[string]set.StringSet) + for _, endpoint := range endpoints { + host := endpoint.Hostname() + var hostIPSet set.StringSet + if host != "" { + var ok bool + hostIPSet, ok = hostIPCache[host] + if !ok { + var err error + hostIPSet, err = getHostIP(host) + if err != nil { + return nil, setupType, config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("host '%s' cannot resolve: %s", host, err)) + } + hostIPCache[host] = hostIPSet + } + } + if IPSet, ok := pathIPMap[endpoint.Path]; ok { + if !IPSet.Intersection(hostIPSet).IsEmpty() { + return nil, setupType, + config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("same path '%s' can not be served by different port on same address", endpoint.Path)) + } + pathIPMap[endpoint.Path] = IPSet.Union(hostIPSet) + } else { + pathIPMap[endpoint.Path] = hostIPSet + } + } + } + + // Check whether same path is used for more than 1 local endpoints. + { + localPathSet := set.CreateStringSet() + for _, endpoint := range endpoints { + if !endpoint.IsLocal { + continue + } + if localPathSet.Contains(endpoint.Path) { + return nil, setupType, + config.ErrInvalidErasureEndpoints(nil).Msg(fmt.Sprintf("path '%s' cannot be served by different address on same server", endpoint.Path)) + } + localPathSet.Add(endpoint.Path) + } + } + + // Add missing port in all endpoints. + for i := range endpoints { + if endpoints[i].Host != "" { + _, port, err := net.SplitHostPort(endpoints[i].Host) + if err != nil { + endpoints[i].Host = net.JoinHostPort(endpoints[i].Host, serverAddrPort) + } else if endpoints[i].IsLocal && serverAddrPort != port { + // If endpoint is local, but port is different than serverAddrPort, then make it as remote. + endpoints[i].IsLocal = false + } + } + } + + // All endpoints are pointing to local host + if len(endpoints) == localEndpointCount { + // If all endpoints have same port number, Just treat it as local erasure setup + // using URL style endpoints. + if len(localPortSet) == 1 { + if len(localServerHostSet) > 1 { + return nil, setupType, + config.ErrInvalidErasureEndpoints(nil).Msg("all local endpoints should not have different hostnames/ips") + } + } + + // Even though all endpoints are local, but those endpoints use different ports. + // This means it is DistErasure setup. + } + + for _, endpoint := range endpoints { + if endpoint.Host != "" { + uniqueArgs.Add(endpoint.Host) + } else { + uniqueArgs.Add(net.JoinHostPort("localhost", serverAddrPort)) + } + } + + poolEndpoints[i] = endpoints + } + + publicIPs := env.Get(config.EnvPublicIPs, "") + if len(publicIPs) == 0 { + updateDomainIPs(uniqueArgs) + } + + erasureType := len(uniqueArgs.ToSlice()) == 1 + + for _, endpoints := range poolEndpoints { + // Return Erasure setup when all endpoints are path style. + if endpoints[0].Type() == PathEndpointType { + setupType = ErasureSetupType + break + } + if endpoints[0].Type() == URLEndpointType { + if erasureType { + setupType = ErasureSetupType + } else { + setupType = DistErasureSetupType + } + break + } + } + + return poolEndpoints, setupType, nil +} + +// GetLocalPeer - returns local peer value, returns globalMinioAddr +// for FS and Erasure mode. In case of distributed server return +// the first element from the set of peers which indicate that +// they are local. There is always one entry that is local +// even with repeated server endpoints. +func GetLocalPeer(endpointServerPools EndpointServerPools, host, port string) (localPeer string) { + peerSet := set.NewStringSet() + for _, ep := range endpointServerPools { + for _, endpoint := range ep.Endpoints { + if endpoint.Type() != URLEndpointType { + continue + } + if endpoint.IsLocal && endpoint.Host != "" { + peerSet.Add(endpoint.Host) + } + } + } + if peerSet.IsEmpty() { + // Local peer can be empty in FS or Erasure coded mode. + // If so, return globalMinioHost + globalMinioPort value. + if host != "" { + return net.JoinHostPort(host, port) + } + + return net.JoinHostPort("127.0.0.1", port) + } + return peerSet.ToSlice()[0] +} + +// GetProxyEndpointLocalIndex returns index of the local proxy endpoint +func GetProxyEndpointLocalIndex(proxyEps []ProxyEndpoint) int { + for i, pep := range proxyEps { + if pep.IsLocal { + return i + } + } + return -1 +} + +// GetProxyEndpoints - get all endpoints that can be used to proxy list request. +func GetProxyEndpoints(endpointServerPools EndpointServerPools, transport http.RoundTripper) []ProxyEndpoint { + var proxyEps []ProxyEndpoint + + proxyEpSet := set.NewStringSet() + + for _, ep := range endpointServerPools { + for _, endpoint := range ep.Endpoints { + if endpoint.Type() != URLEndpointType { + continue + } + + host := endpoint.Host + if proxyEpSet.Contains(host) { + continue + } + proxyEpSet.Add(host) + + proxyEps = append(proxyEps, ProxyEndpoint{ + Endpoint: endpoint, + Transport: transport, + }) + } + } + return proxyEps +} + +func updateDomainIPs(endPoints set.StringSet) { + ipList := set.NewStringSet() + for e := range endPoints { + host, port, err := net.SplitHostPort(e) + if err != nil { + if strings.Contains(err.Error(), "missing port in address") { + host = e + port = globalMinioPort + } else { + continue + } + } + + if net.ParseIP(host) == nil { + IPs, err := getHostIP(host) + if err != nil { + continue + } + + IPsWithPort := IPs.ApplyFunc(func(ip string) string { + return net.JoinHostPort(ip, port) + }) + + ipList = ipList.Union(IPsWithPort) + } + + ipList.Add(net.JoinHostPort(host, port)) + } + + globalDomainIPs = ipList.FuncMatch(func(ip string, matchString string) bool { + host, _, err := net.SplitHostPort(ip) + if err != nil { + host = ip + } + return !net.ParseIP(host).IsLoopback() && host != "localhost" + }, "") +} diff --git a/cmd/endpoint_contrib_test.go b/cmd/endpoint_contrib_test.go new file mode 100644 index 00000000..6b81f08b --- /dev/null +++ b/cmd/endpoint_contrib_test.go @@ -0,0 +1,63 @@ +/* + * MinIO Object Storage (c) 2021 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "testing" + + "github.com/minio/minio-go/v7/pkg/set" +) + +func TestUpdateDomainIPs(t *testing.T) { + tempGlobalMinioPort := globalMinioPort + defer func() { + globalMinioPort = tempGlobalMinioPort + }() + globalMinioPort = "9000" + + tempGlobalDomainIPs := globalDomainIPs + defer func() { + globalDomainIPs = tempGlobalDomainIPs + }() + + ipv4TestCases := []struct { + endPoints set.StringSet + expectedResult set.StringSet + }{ + {set.NewStringSet(), set.NewStringSet()}, + {set.CreateStringSet("localhost"), set.NewStringSet()}, + {set.CreateStringSet("localhost", "10.0.0.1"), set.CreateStringSet("10.0.0.1:9000")}, + {set.CreateStringSet("localhost:9001", "10.0.0.1"), set.CreateStringSet("10.0.0.1:9000")}, + {set.CreateStringSet("localhost", "10.0.0.1:9001"), set.CreateStringSet("10.0.0.1:9001")}, + {set.CreateStringSet("localhost:9000", "10.0.0.1:9001"), set.CreateStringSet("10.0.0.1:9001")}, + + {set.CreateStringSet("10.0.0.1", "10.0.0.2"), set.CreateStringSet("10.0.0.1:9000", "10.0.0.2:9000")}, + {set.CreateStringSet("10.0.0.1:9001", "10.0.0.2"), set.CreateStringSet("10.0.0.1:9001", "10.0.0.2:9000")}, + {set.CreateStringSet("10.0.0.1", "10.0.0.2:9002"), set.CreateStringSet("10.0.0.1:9000", "10.0.0.2:9002")}, + {set.CreateStringSet("10.0.0.1:9001", "10.0.0.2:9002"), set.CreateStringSet("10.0.0.1:9001", "10.0.0.2:9002")}, + } + + for _, testCase := range ipv4TestCases { + globalDomainIPs = nil + + updateDomainIPs(testCase.endPoints) + + if !testCase.expectedResult.Equals(globalDomainIPs) { + t.Fatalf("error: expected = %s, got = %s", testCase.expectedResult, globalDomainIPs) + } + } +} diff --git a/cmd/endpoint_test.go b/cmd/endpoint_test.go new file mode 100644 index 00000000..1eb034ed --- /dev/null +++ b/cmd/endpoint_test.go @@ -0,0 +1,429 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "net" + "net/url" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func TestNewEndpoint(t *testing.T) { + u2, _ := url.Parse("https://example.org/path") + u4, _ := url.Parse("http://192.168.253.200/path") + rootSlashFoo, _ := filepath.Abs("/foo") + testCases := []struct { + arg string + expectedEndpoint Endpoint + expectedType EndpointType + expectedErr error + }{ + {"/foo", Endpoint{&url.URL{Path: rootSlashFoo}, true, -1, -1, -1}, PathEndpointType, nil}, + {"https://example.org/path", Endpoint{u2, false, -1, -1, -1}, URLEndpointType, nil}, + {"http://192.168.253.200/path", Endpoint{u4, false, -1, -1, -1}, URLEndpointType, nil}, + {"", Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")}, + {SlashSeparator, Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")}, + {`\`, Endpoint{}, -1, fmt.Errorf("empty or root endpoint is not supported")}, + {"c://foo", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format")}, + {"ftp://foo", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format")}, + {"http://server/path?location", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format")}, + {"http://:/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: invalid port number")}, + {"http://:8080/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: empty host name")}, + {"http://server:/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: invalid port number")}, + {"https://93.184.216.34:808080/path", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: port number must be between 1 to 65535")}, + {"http://server:8080//", Endpoint{}, -1, fmt.Errorf("empty or root path is not supported in URL endpoint")}, + {"http://server:8080/", Endpoint{}, -1, fmt.Errorf("empty or root path is not supported in URL endpoint")}, + {"192.168.1.210:9000", Endpoint{}, -1, fmt.Errorf("invalid URL endpoint format: missing scheme http or https")}, + } + + for i, test := range testCases { + t.Run(fmt.Sprint("case-", i), func(t *testing.T) { + endpoint, err := NewEndpoint(test.arg) + if err == nil { + err = endpoint.UpdateIsLocal() + } + + switch { + case test.expectedErr == nil: + if err != nil { + t.Errorf("error: expected = , got = %v", err) + } + case err == nil: + t.Errorf("error: expected = %v, got = ", test.expectedErr) + case test.expectedErr.Error() != err.Error(): + t.Errorf("error: expected = %v, got = %v", test.expectedErr, err) + } + + if err == nil { + if (test.expectedEndpoint.URL == nil) != (endpoint.URL == nil) { + t.Errorf("endpoint url: expected = %#v, got = %#v", test.expectedEndpoint.URL, endpoint.URL) + return + } else if test.expectedEndpoint.URL.String() != endpoint.URL.String() { + t.Errorf("endpoint url: expected = %#v, got = %#v", test.expectedEndpoint.URL.String(), endpoint.URL.String()) + return + } + if !reflect.DeepEqual(test.expectedEndpoint, endpoint) { + t.Errorf("endpoint: expected = %#v, got = %#v", test.expectedEndpoint, endpoint) + } + } + + if err == nil && test.expectedType != endpoint.Type() { + t.Errorf("type: expected = %+v, got = %+v", test.expectedType, endpoint.Type()) + } + }) + } +} + +func TestNewEndpoints(t *testing.T) { + testCases := []struct { + args []string + expectedErr error + }{ + {[]string{"/d1", "/d2", "/d3", "/d4"}, nil}, + {[]string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}, nil}, + {[]string{"http://example.org/d1", "http://example.com/d1", "http://example.net/d1", "http://example.edu/d1"}, nil}, + {[]string{"http://localhost/d1", "http://localhost/d2", "http://example.org/d1", "http://example.org/d2"}, nil}, + {[]string{"https://localhost:9000/d1", "https://localhost:9001/d2", "https://localhost:9002/d3", "https://localhost:9003/d4"}, nil}, + // // It is valid WRT endpoint list that same path is expected with different port on same server. + {[]string{"https://127.0.0.1:9000/d1", "https://127.0.0.1:9001/d1", "https://127.0.0.1:9002/d1", "https://127.0.0.1:9003/d1"}, nil}, + {[]string{"d1", "d2", "d3", "d1"}, fmt.Errorf("duplicate endpoints found")}, + {[]string{"d1", "d2", "d3", "./d1"}, fmt.Errorf("duplicate endpoints found")}, + {[]string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d1", "http://localhost/d4"}, fmt.Errorf("duplicate endpoints found")}, + {[]string{"ftp://server/d1", "http://server/d2", "http://server/d3", "http://server/d4"}, fmt.Errorf("'ftp://server/d1': invalid URL endpoint format")}, + {[]string{"d1", "http://localhost/d2", "d3", "d4"}, fmt.Errorf("mixed style endpoints are not supported")}, + {[]string{"http://example.org/d1", "https://example.com/d1", "http://example.net/d1", "https://example.edut/d1"}, fmt.Errorf("mixed scheme is not supported")}, + {[]string{"192.168.1.210:9000/tmp/dir0", "192.168.1.210:9000/tmp/dir1", "192.168.1.210:9000/tmp/dir2", "192.168.110:9000/tmp/dir3"}, fmt.Errorf("'192.168.1.210:9000/tmp/dir0': invalid URL endpoint format: missing scheme http or https")}, + } + + for _, testCase := range testCases { + _, err := NewEndpoints(testCase.args...) + switch { + case testCase.expectedErr == nil: + if err != nil { + t.Fatalf("error: expected = , got = %v", err) + } + case err == nil: + t.Fatalf("error: expected = %v, got = ", testCase.expectedErr) + case testCase.expectedErr.Error() != err.Error(): + t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err) + } + } +} + +func TestCreateEndpoints(t *testing.T) { + tempGlobalMinioPort := globalMinioPort + defer func() { + globalMinioPort = tempGlobalMinioPort + }() + globalMinioPort = "9000" + + // Filter ipList by IPs those do not start with '127.'. + nonLoopBackIPs := localIP4.FuncMatch(func(ip string, matchString string) bool { + return !net.ParseIP(ip).IsLoopback() + }, "") + if len(nonLoopBackIPs) == 0 { + t.Fatalf("No non-loop back IP address found for this host") + } + nonLoopBackIP := nonLoopBackIPs.ToSlice()[0] + + mustAbs := func(s string) string { + s, err := filepath.Abs(s) + if err != nil { + t.Fatal(err) + } + return s + } + getExpectedEndpoints := func(args []string, prefix string) ([]*url.URL, []bool) { + var URLs []*url.URL + var localFlags []bool + for _, arg := range args { + u, _ := url.Parse(arg) + URLs = append(URLs, u) + localFlags = append(localFlags, strings.HasPrefix(arg, prefix)) + } + + return URLs, localFlags + } + + case1Endpoint1 := "http://" + nonLoopBackIP + "/d1" + case1Endpoint2 := "http://" + nonLoopBackIP + "/d2" + args := []string{ + "http://" + nonLoopBackIP + ":10000/d1", + "http://" + nonLoopBackIP + ":10000/d2", + "http://example.org:10000/d3", + "http://example.com:10000/d4", + } + case1URLs, case1LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":10000/") + + case2Endpoint1 := "http://" + nonLoopBackIP + "/d1" + case2Endpoint2 := "http://" + nonLoopBackIP + ":9000/d2" + args = []string{ + "http://" + nonLoopBackIP + ":10000/d1", + "http://" + nonLoopBackIP + ":9000/d2", + "http://example.org:10000/d3", + "http://example.com:10000/d4", + } + case2URLs, case2LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":10000/") + + case3Endpoint1 := "http://" + nonLoopBackIP + "/d1" + args = []string{ + "http://" + nonLoopBackIP + ":80/d1", + "http://example.org:9000/d2", + "http://example.com:80/d3", + "http://example.net:80/d4", + } + case3URLs, case3LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":80/") + + case4Endpoint1 := "http://" + nonLoopBackIP + "/d1" + args = []string{ + "http://" + nonLoopBackIP + ":9000/d1", + "http://example.org:9000/d2", + "http://example.com:9000/d3", + "http://example.net:9000/d4", + } + case4URLs, case4LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":9000/") + + case5Endpoint1 := "http://" + nonLoopBackIP + ":9000/d1" + case5Endpoint2 := "http://" + nonLoopBackIP + ":9001/d2" + case5Endpoint3 := "http://" + nonLoopBackIP + ":9002/d3" + case5Endpoint4 := "http://" + nonLoopBackIP + ":9003/d4" + args = []string{ + case5Endpoint1, + case5Endpoint2, + case5Endpoint3, + case5Endpoint4, + } + case5URLs, case5LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":9000/") + + case6Endpoint := "http://" + nonLoopBackIP + ":9003/d4" + args = []string{ + "http://localhost:9000/d1", + "http://localhost:9001/d2", + "http://127.0.0.1:9002/d3", + case6Endpoint, + } + case6URLs, case6LocalFlags := getExpectedEndpoints(args, "http://"+nonLoopBackIP+":9003/") + + testCases := []struct { + serverAddr string + args []string + expectedServerAddr string + expectedEndpoints Endpoints + expectedSetupType SetupType + expectedErr error + }{ + {"localhost", []string{}, "", Endpoints{}, -1, fmt.Errorf("address localhost: missing port in address")}, + + // Erasure Single Drive + {"localhost:9000", []string{"http://localhost/d1"}, "", Endpoints{}, -1, fmt.Errorf("use path style endpoint for SD setup")}, + {":443", []string{"/d1"}, ":443", Endpoints{Endpoint{URL: &url.URL{Path: mustAbs("/d1")}, IsLocal: true}}, ErasureSDSetupType, nil}, + {"localhost:10000", []string{"/d1"}, "localhost:10000", Endpoints{Endpoint{URL: &url.URL{Path: mustAbs("/d1")}, IsLocal: true}}, ErasureSDSetupType, nil}, + {"localhost:9000", []string{"https://127.0.0.1:9000/d1", "https://localhost:9001/d1", "https://example.com/d1", "https://example.com/d2"}, "", Endpoints{}, -1, fmt.Errorf("path '/d1' can not be served by different port on same address")}, + + // Erasure Setup with PathEndpointType + { + ":1234", + []string{"/d1", "/d2", "/d3", "/d4"}, + ":1234", + Endpoints{ + Endpoint{URL: &url.URL{Path: mustAbs("/d1")}, IsLocal: true}, + Endpoint{URL: &url.URL{Path: mustAbs("/d2")}, IsLocal: true}, + Endpoint{URL: &url.URL{Path: mustAbs("/d3")}, IsLocal: true}, + Endpoint{URL: &url.URL{Path: mustAbs("/d4")}, IsLocal: true}, + }, + ErasureSetupType, nil, + }, + // DistErasure Setup with URLEndpointType + {":9000", []string{"http://localhost/d1", "http://localhost/d2", "http://localhost/d3", "http://localhost/d4"}, ":9000", Endpoints{ + Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost:9000", Path: "/d1"}, IsLocal: true}, + Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost:9000", Path: "/d2"}, IsLocal: true}, + Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost:9000", Path: "/d3"}, IsLocal: true}, + Endpoint{URL: &url.URL{Scheme: "http", Host: "localhost:9000", Path: "/d4"}, IsLocal: true}, + }, ErasureSetupType, nil}, + // DistErasure Setup with URLEndpointType having mixed naming to local host. + {"127.0.0.1:10000", []string{"http://localhost/d1", "http://localhost/d2", "http://127.0.0.1/d3", "http://127.0.0.1/d4"}, "", Endpoints{}, -1, fmt.Errorf("all local endpoints should not have different hostnames/ips")}, + + {":9001", []string{"http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export", "http://" + nonLoopBackIP + ":9001/export", "http://10.0.0.2:9001/export"}, "", Endpoints{}, -1, fmt.Errorf("path '/export' can not be served by different port on same address")}, + + {":9000", []string{"http://127.0.0.1:9000/export", "http://" + nonLoopBackIP + ":9000/export", "http://10.0.0.1:9000/export", "http://10.0.0.2:9000/export"}, "", Endpoints{}, -1, fmt.Errorf("path '/export' cannot be served by different address on same server")}, + + // DistErasure type + {"127.0.0.1:10000", []string{case1Endpoint1, case1Endpoint2, "http://example.org/d3", "http://example.com/d4"}, "127.0.0.1:10000", Endpoints{ + Endpoint{URL: case1URLs[0], IsLocal: case1LocalFlags[0]}, + Endpoint{URL: case1URLs[1], IsLocal: case1LocalFlags[1]}, + Endpoint{URL: case1URLs[2], IsLocal: case1LocalFlags[2]}, + Endpoint{URL: case1URLs[3], IsLocal: case1LocalFlags[3]}, + }, DistErasureSetupType, nil}, + + {"127.0.0.1:10000", []string{case2Endpoint1, case2Endpoint2, "http://example.org/d3", "http://example.com/d4"}, "127.0.0.1:10000", Endpoints{ + Endpoint{URL: case2URLs[0], IsLocal: case2LocalFlags[0]}, + Endpoint{URL: case2URLs[1], IsLocal: case2LocalFlags[1]}, + Endpoint{URL: case2URLs[2], IsLocal: case2LocalFlags[2]}, + Endpoint{URL: case2URLs[3], IsLocal: case2LocalFlags[3]}, + }, DistErasureSetupType, nil}, + + {":80", []string{case3Endpoint1, "http://example.org:9000/d2", "http://example.com/d3", "http://example.net/d4"}, ":80", Endpoints{ + Endpoint{URL: case3URLs[0], IsLocal: case3LocalFlags[0]}, + Endpoint{URL: case3URLs[1], IsLocal: case3LocalFlags[1]}, + Endpoint{URL: case3URLs[2], IsLocal: case3LocalFlags[2]}, + Endpoint{URL: case3URLs[3], IsLocal: case3LocalFlags[3]}, + }, DistErasureSetupType, nil}, + + {":9000", []string{case4Endpoint1, "http://example.org/d2", "http://example.com/d3", "http://example.net/d4"}, ":9000", Endpoints{ + Endpoint{URL: case4URLs[0], IsLocal: case4LocalFlags[0]}, + Endpoint{URL: case4URLs[1], IsLocal: case4LocalFlags[1]}, + Endpoint{URL: case4URLs[2], IsLocal: case4LocalFlags[2]}, + Endpoint{URL: case4URLs[3], IsLocal: case4LocalFlags[3]}, + }, DistErasureSetupType, nil}, + + {":9000", []string{case5Endpoint1, case5Endpoint2, case5Endpoint3, case5Endpoint4}, ":9000", Endpoints{ + Endpoint{URL: case5URLs[0], IsLocal: case5LocalFlags[0]}, + Endpoint{URL: case5URLs[1], IsLocal: case5LocalFlags[1]}, + Endpoint{URL: case5URLs[2], IsLocal: case5LocalFlags[2]}, + Endpoint{URL: case5URLs[3], IsLocal: case5LocalFlags[3]}, + }, DistErasureSetupType, nil}, + + // DistErasure Setup using only local host. + {":9003", []string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://127.0.0.1:9002/d3", case6Endpoint}, ":9003", Endpoints{ + Endpoint{URL: case6URLs[0], IsLocal: case6LocalFlags[0]}, + Endpoint{URL: case6URLs[1], IsLocal: case6LocalFlags[1]}, + Endpoint{URL: case6URLs[2], IsLocal: case6LocalFlags[2]}, + Endpoint{URL: case6URLs[3], IsLocal: case6LocalFlags[3]}, + }, DistErasureSetupType, nil}, + } + + for i, testCase := range testCases { + testCase := testCase + t.Run("", func(t *testing.T) { + var srvCtxt serverCtxt + err := mergeDisksLayoutFromArgs(testCase.args, &srvCtxt) + if err != nil && testCase.expectedErr == nil { + t.Errorf("Test %d: unexpected error: %v", i+1, err) + } + pools, setupType, err := CreatePoolEndpoints(testCase.serverAddr, srvCtxt.Layout.pools...) + if err == nil && testCase.expectedErr != nil { + t.Errorf("Test %d: expected = %v, got = ", i+1, testCase.expectedErr) + } + if err == nil { + if setupType != testCase.expectedSetupType { + t.Errorf("Test %d: setupType: expected = %v, got = %v", i+1, testCase.expectedSetupType, setupType) + } + endpoints := pools[0] + if len(endpoints) != len(testCase.expectedEndpoints) { + t.Errorf("Test %d: endpoints: expected = %d, got = %d", i+1, len(testCase.expectedEndpoints), + len(endpoints)) + } else { + for i, endpoint := range endpoints { + if testCase.expectedEndpoints[i].String() != endpoint.String() { + t.Errorf("Test %d: endpoints: expected = %s, got = %s", + i+1, + testCase.expectedEndpoints[i], + endpoint) + } + } + } + } + if err != nil && testCase.expectedErr == nil { + t.Errorf("Test %d: error: expected = , got = %v, testCase: %v", i+1, err, testCase) + } + }) + } +} + +// Tests get local peer functionality, local peer is supposed to only return one entry per minio service. +// So it means that if you have say localhost:9000 and localhost:9001 as endpointArgs then localhost:9001 +// is considered a remote service from localhost:9000 perspective. +func TestGetLocalPeer(t *testing.T) { + tempGlobalMinioPort := globalMinioPort + defer func() { + globalMinioPort = tempGlobalMinioPort + }() + globalMinioPort = "9000" + + testCases := []struct { + endpointArgs []string + expectedResult string + }{ + {[]string{"/d1", "/d2", "d3", "d4"}, "127.0.0.1:9000"}, + { + []string{"http://localhost:9000/d1", "http://localhost:9000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, + "localhost:9000", + }, + { + []string{"http://localhost:9000/d1", "http://example.org:9000/d2", "http://example.com:9000/d3", "http://example.net:9000/d4"}, + "localhost:9000", + }, + { + []string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://localhost:9002/d3", "http://localhost:9003/d4"}, + "localhost:9000", + }, + } + + for i, testCase := range testCases { + zendpoints := mustGetPoolEndpoints(0, testCase.endpointArgs...) + if !zendpoints[0].Endpoints[0].IsLocal { + if err := zendpoints[0].Endpoints.UpdateIsLocal(); err != nil { + t.Fatalf("error: expected = , got = %v", err) + } + } + localPeer := GetLocalPeer(zendpoints, "", "9000") + if localPeer != testCase.expectedResult { + t.Fatalf("Test %d: expected: %v, got: %v", i+1, testCase.expectedResult, localPeer) + } + } +} + +func TestGetRemotePeers(t *testing.T) { + tempGlobalMinioPort := globalMinioPort + defer func() { + globalMinioPort = tempGlobalMinioPort + }() + globalMinioPort = "9000" + + testCases := []struct { + endpointArgs []string + expectedResult []string + expectedLocal string + }{ + {[]string{"/d1", "/d2", "d3", "d4"}, []string{}, ""}, + {[]string{"http://localhost:9000/d1", "http://localhost:9000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000", "localhost:9000"}, "localhost:9000"}, + {[]string{"http://localhost:9000/d1", "http://localhost:10000/d2", "http://example.org:9000/d3", "http://example.com:9000/d4"}, []string{"example.com:9000", "example.org:9000", "localhost:10000", "localhost:9000"}, "localhost:9000"}, + {[]string{"http://localhost:9000/d1", "http://example.org:9000/d2", "http://example.com:9000/d3", "http://example.net:9000/d4"}, []string{"example.com:9000", "example.net:9000", "example.org:9000", "localhost:9000"}, "localhost:9000"}, + {[]string{"http://localhost:9000/d1", "http://localhost:9001/d2", "http://localhost:9002/d3", "http://localhost:9003/d4"}, []string{"localhost:9000", "localhost:9001", "localhost:9002", "localhost:9003"}, "localhost:9000"}, + } + + for _, testCase := range testCases { + zendpoints := mustGetPoolEndpoints(0, testCase.endpointArgs...) + if !zendpoints[0].Endpoints[0].IsLocal { + if err := zendpoints[0].Endpoints.UpdateIsLocal(); err != nil { + t.Errorf("error: expected = , got = %v", err) + } + } + remotePeers, local := zendpoints.peers() + if !reflect.DeepEqual(remotePeers, testCase.expectedResult) { + t.Errorf("expected: %v, got: %v", testCase.expectedResult, remotePeers) + } + if local != testCase.expectedLocal { + t.Errorf("expected: %v, got: %v", testCase.expectedLocal, local) + } + } +} diff --git a/cmd/erasure-coding.go b/cmd/erasure-coding.go new file mode 100644 index 00000000..3294458e --- /dev/null +++ b/cmd/erasure-coding.go @@ -0,0 +1,206 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/hex" + "fmt" + "os" + "reflect" + "sync" + + "github.com/cespare/xxhash/v2" + "github.com/klauspost/reedsolomon" + "github.com/minio/minio/internal/logger" +) + +// Erasure - erasure encoding details. +type Erasure struct { + encoder func() reedsolomon.Encoder + dataBlocks, parityBlocks int + blockSize int64 +} + +// NewErasure creates a new ErasureStorage. +func NewErasure(ctx context.Context, dataBlocks, parityBlocks int, blockSize int64) (e Erasure, err error) { + // Check the parameters for sanity now. + if dataBlocks <= 0 || parityBlocks < 0 { + return e, reedsolomon.ErrInvShardNum + } + + if dataBlocks+parityBlocks > 256 { + return e, reedsolomon.ErrMaxShardNum + } + + e = Erasure{ + dataBlocks: dataBlocks, + parityBlocks: parityBlocks, + blockSize: blockSize, + } + + // Encoder when needed. + var enc reedsolomon.Encoder + var once sync.Once + e.encoder = func() reedsolomon.Encoder { + once.Do(func() { + e, err := reedsolomon.New(dataBlocks, parityBlocks, reedsolomon.WithAutoGoroutines(int(e.ShardSize()))) + if err != nil { + // Error conditions should be checked above. + panic(err) + } + enc = e + }) + return enc + } + return e, err +} + +// EncodeData encodes the given data and returns the erasure-coded data. +// It returns an error if the erasure coding failed. +func (e *Erasure) EncodeData(ctx context.Context, data []byte) ([][]byte, error) { + if len(data) == 0 { + return make([][]byte, e.dataBlocks+e.parityBlocks), nil + } + encoded, err := e.encoder().Split(data) + if err != nil { + return nil, err + } + if err = e.encoder().Encode(encoded); err != nil { + return nil, err + } + return encoded, nil +} + +// DecodeDataBlocks decodes the given erasure-coded data. +// It only decodes the data blocks but does not verify them. +// It returns an error if the decoding failed. +func (e *Erasure) DecodeDataBlocks(data [][]byte) error { + isZero := 0 + for _, b := range data { + if len(b) == 0 { + isZero++ + break + } + } + if isZero == 0 || isZero == len(data) { + // If all are zero, payload is 0 bytes. + return nil + } + return e.encoder().ReconstructData(data) +} + +// DecodeDataAndParityBlocks decodes the given erasure-coded data and verifies it. +// It returns an error if the decoding failed. +func (e *Erasure) DecodeDataAndParityBlocks(ctx context.Context, data [][]byte) error { + return e.encoder().Reconstruct(data) +} + +// ShardSize - returns actual shared size from erasure blockSize. +func (e *Erasure) ShardSize() int64 { + return ceilFrac(e.blockSize, int64(e.dataBlocks)) +} + +// ShardFileSize - returns final erasure size from original size. +func (e *Erasure) ShardFileSize(totalLength int64) int64 { + if totalLength == 0 { + return 0 + } + if totalLength == -1 { + return -1 + } + numShards := totalLength / e.blockSize + lastBlockSize := totalLength % e.blockSize + lastShardSize := ceilFrac(lastBlockSize, int64(e.dataBlocks)) + return numShards*e.ShardSize() + lastShardSize +} + +// ShardFileOffset - returns the effective offset where erasure reading begins. +func (e *Erasure) ShardFileOffset(startOffset, length, totalLength int64) int64 { + shardSize := e.ShardSize() + shardFileSize := e.ShardFileSize(totalLength) + endShard := (startOffset + length) / e.blockSize + tillOffset := min(endShard*shardSize+shardSize, shardFileSize) + return tillOffset +} + +// erasureSelfTest performs a self-test to ensure that erasure +// algorithms compute expected erasure codes. If any algorithm +// produces an incorrect value it fails with a hard error. +// +// erasureSelfTest tries to catch any issue in the erasure implementation +// early instead of silently corrupting data. +func erasureSelfTest() { + // Approx runtime ~1ms + var testConfigs [][2]uint8 + for total := uint8(4); total < 16; total++ { + for data := total / 2; data < total; data++ { + parity := total - data + testConfigs = append(testConfigs, [2]uint8{data, parity}) + } + } + got := make(map[[2]uint8]map[ErasureAlgo]uint64, len(testConfigs)) + // Copied from output of fmt.Printf("%#v", got) at the end. + want := map[[2]uint8]map[ErasureAlgo]uint64{{0x2, 0x2}: {0x1: 0x23fb21be2496f5d3}, {0x2, 0x3}: {0x1: 0xa5cd5600ba0d8e7c}, {0x3, 0x1}: {0x1: 0x60ab052148b010b4}, {0x3, 0x2}: {0x1: 0xe64927daef76435a}, {0x3, 0x3}: {0x1: 0x672f6f242b227b21}, {0x3, 0x4}: {0x1: 0x571e41ba23a6dc6}, {0x4, 0x1}: {0x1: 0x524eaa814d5d86e2}, {0x4, 0x2}: {0x1: 0x62b9552945504fef}, {0x4, 0x3}: {0x1: 0xcbf9065ee053e518}, {0x4, 0x4}: {0x1: 0x9a07581dcd03da8}, {0x4, 0x5}: {0x1: 0xbf2d27b55370113f}, {0x5, 0x1}: {0x1: 0xf71031a01d70daf}, {0x5, 0x2}: {0x1: 0x8e5845859939d0f4}, {0x5, 0x3}: {0x1: 0x7ad9161acbb4c325}, {0x5, 0x4}: {0x1: 0xc446b88830b4f800}, {0x5, 0x5}: {0x1: 0xabf1573cc6f76165}, {0x5, 0x6}: {0x1: 0x7b5598a85045bfb8}, {0x6, 0x1}: {0x1: 0xe2fc1e677cc7d872}, {0x6, 0x2}: {0x1: 0x7ed133de5ca6a58e}, {0x6, 0x3}: {0x1: 0x39ef92d0a74cc3c0}, {0x6, 0x4}: {0x1: 0xcfc90052bc25d20}, {0x6, 0x5}: {0x1: 0x71c96f6baeef9c58}, {0x6, 0x6}: {0x1: 0x4b79056484883e4c}, {0x6, 0x7}: {0x1: 0xb1a0e2427ac2dc1a}, {0x7, 0x1}: {0x1: 0x937ba2b7af467a22}, {0x7, 0x2}: {0x1: 0x5fd13a734d27d37a}, {0x7, 0x3}: {0x1: 0x3be2722d9b66912f}, {0x7, 0x4}: {0x1: 0x14c628e59011be3d}, {0x7, 0x5}: {0x1: 0xcc3b39ad4c083b9f}, {0x7, 0x6}: {0x1: 0x45af361b7de7a4ff}, {0x7, 0x7}: {0x1: 0x456cc320cec8a6e6}, {0x7, 0x8}: {0x1: 0x1867a9f4db315b5c}, {0x8, 0x1}: {0x1: 0xbc5756b9a9ade030}, {0x8, 0x2}: {0x1: 0xdfd7d9d0b3e36503}, {0x8, 0x3}: {0x1: 0x72bb72c2cdbcf99d}, {0x8, 0x4}: {0x1: 0x3ba5e9b41bf07f0}, {0x8, 0x5}: {0x1: 0xd7dabc15800f9d41}, {0x8, 0x6}: {0x1: 0xb482a6169fd270f}, {0x8, 0x7}: {0x1: 0x50748e0099d657e8}, {0x9, 0x1}: {0x1: 0xc77ae0144fcaeb6e}, {0x9, 0x2}: {0x1: 0x8a86c7dbebf27b68}, {0x9, 0x3}: {0x1: 0xa64e3be6d6fe7e92}, {0x9, 0x4}: {0x1: 0x239b71c41745d207}, {0x9, 0x5}: {0x1: 0x2d0803094c5a86ce}, {0x9, 0x6}: {0x1: 0xa3c2539b3af84874}, {0xa, 0x1}: {0x1: 0x7d30d91b89fcec21}, {0xa, 0x2}: {0x1: 0xfa5af9aa9f1857a3}, {0xa, 0x3}: {0x1: 0x84bc4bda8af81f90}, {0xa, 0x4}: {0x1: 0x6c1cba8631de994a}, {0xa, 0x5}: {0x1: 0x4383e58a086cc1ac}, {0xb, 0x1}: {0x1: 0x4ed2929a2df690b}, {0xb, 0x2}: {0x1: 0xecd6f1b1399775c0}, {0xb, 0x3}: {0x1: 0xc78cfbfc0dc64d01}, {0xb, 0x4}: {0x1: 0xb2643390973702d6}, {0xc, 0x1}: {0x1: 0x3b2a88686122d082}, {0xc, 0x2}: {0x1: 0xfd2f30a48a8e2e9}, {0xc, 0x3}: {0x1: 0xd5ce58368ae90b13}, {0xd, 0x1}: {0x1: 0x9c88e2a9d1b8fff8}, {0xd, 0x2}: {0x1: 0xcb8460aa4cf6613}, {0xe, 0x1}: {0x1: 0x78a28bbaec57996e}} + var testData [256]byte + for i := range testData { + testData[i] = byte(i) + } + ok := true + for algo := invalidErasureAlgo + 1; algo < lastErasureAlgo; algo++ { + for _, conf := range testConfigs { + failOnErr := func(err error) { + if err != nil { + logger.Fatal(errSelfTestFailure, "%v: error on self-test [d:%d,p:%d]: %v. Unsafe to start server.\n", algo, conf[0], conf[1], err) + } + } + e, err := NewErasure(context.Background(), int(conf[0]), int(conf[1]), blockSizeV2) + failOnErr(err) + encoded, err := e.EncodeData(GlobalContext, testData[:]) + failOnErr(err) + hash := xxhash.New() + for i, data := range encoded { + // Write index to keep track of sizes of each. + _, err = hash.Write([]byte{byte(i)}) + failOnErr(err) + _, err = hash.Write(data) + failOnErr(err) + got[conf] = map[ErasureAlgo]uint64{algo: hash.Sum64()} + } + + if a, b := want[conf], got[conf]; !reflect.DeepEqual(a, b) { + fmt.Fprintf(os.Stderr, "%v: error on self-test [d:%d,p:%d]: want %#v, got %#v\n", algo, conf[0], conf[1], a, b) + ok = false + continue + } + // Delete first shard and reconstruct... + first := encoded[0] + encoded[0] = nil + failOnErr(e.DecodeDataBlocks(encoded)) + if a, b := first, encoded[0]; !bytes.Equal(a, b) { + fmt.Fprintf(os.Stderr, "%v: error on self-test [d:%d,p:%d]: want %#v, got %#v\n", algo, conf[0], conf[1], hex.EncodeToString(a), hex.EncodeToString(b)) + ok = false + continue + } + } + } + if !ok { + logger.Fatal(errSelfTestFailure, "Erasure Coding self test failed") + } +} diff --git a/cmd/erasure-common.go b/cmd/erasure-common.go new file mode 100644 index 00000000..7146766a --- /dev/null +++ b/cmd/erasure-common.go @@ -0,0 +1,84 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "math/rand" + "sync" + "time" +) + +func (er erasureObjects) getOnlineDisks() (newDisks []StorageAPI) { + disks := er.getDisks() + var wg sync.WaitGroup + var mu sync.Mutex + r := rand.New(rand.NewSource(time.Now().UnixNano())) + for _, i := range r.Perm(len(disks)) { + wg.Add(1) + go func() { + defer wg.Done() + if disks[i] == nil { + return + } + di, err := disks[i].DiskInfo(context.Background(), DiskInfoOptions{}) + if err != nil || di.Healing { + // - Do not consume disks which are not reachable + // unformatted or simply not accessible for some reason. + // + // - Do not consume disks which are being healed + // + // - Future: skip busy disks + return + } + + mu.Lock() + newDisks = append(newDisks, disks[i]) + mu.Unlock() + }() + } + wg.Wait() + return newDisks +} + +func (er erasureObjects) getOnlineLocalDisks() (newDisks []StorageAPI) { + disks := er.getOnlineDisks() + + // Based on the random shuffling return back randomized disks. + r := rand.New(rand.NewSource(time.Now().UnixNano())) + + for _, i := range r.Perm(len(disks)) { + if disks[i] != nil && disks[i].IsLocal() { + newDisks = append(newDisks, disks[i]) + } + } + + return newDisks +} + +func (er erasureObjects) getLocalDisks() (newDisks []StorageAPI) { + disks := er.getDisks() + // Based on the random shuffling return back randomized disks. + r := rand.New(rand.NewSource(time.Now().UnixNano())) + for _, i := range r.Perm(len(disks)) { + if disks[i] != nil && disks[i].IsLocal() { + newDisks = append(newDisks, disks[i]) + } + } + return newDisks +} diff --git a/cmd/erasure-decode.go b/cmd/erasure-decode.go new file mode 100644 index 00000000..f0cc90ab --- /dev/null +++ b/cmd/erasure-decode.go @@ -0,0 +1,364 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "sync" + "sync/atomic" + + xioutil "github.com/minio/minio/internal/ioutil" +) + +// Reads in parallel from readers. +type parallelReader struct { + readers []io.ReaderAt + orgReaders []io.ReaderAt + dataBlocks int + offset int64 + shardSize int64 + shardFileSize int64 + buf [][]byte + readerToBuf []int + stashBuffer []byte +} + +// newParallelReader returns parallelReader. +func newParallelReader(readers []io.ReaderAt, e Erasure, offset, totalLength int64) *parallelReader { + r2b := make([]int, len(readers)) + for i := range r2b { + r2b[i] = i + } + bufs := make([][]byte, len(readers)) + shardSize := int(e.ShardSize()) + var b []byte + + // We should always have enough capacity, but older objects may be bigger + // we do not need stashbuffer for them. + if globalBytePoolCap.Load().WidthCap() >= len(readers)*shardSize { + // Fill buffers + b = globalBytePoolCap.Load().Get() + // Seed the buffers. + for i := range bufs { + bufs[i] = b[i*shardSize : (i+1)*shardSize] + } + } + + return ¶llelReader{ + readers: readers, + orgReaders: readers, + dataBlocks: e.dataBlocks, + offset: (offset / e.blockSize) * e.ShardSize(), + shardSize: e.ShardSize(), + shardFileSize: e.ShardFileSize(totalLength), + buf: make([][]byte, len(readers)), + readerToBuf: r2b, + stashBuffer: b, + } +} + +// Done will release any resources used by the parallelReader. +func (p *parallelReader) Done() { + if p.stashBuffer != nil { + globalBytePoolCap.Load().Put(p.stashBuffer) + p.stashBuffer = nil + } +} + +// preferReaders can mark readers as preferred. +// These will be chosen before others. +func (p *parallelReader) preferReaders(prefer []bool) { + if len(prefer) != len(p.orgReaders) { + return + } + // Copy so we don't change our input. + tmp := make([]io.ReaderAt, len(p.orgReaders)) + copy(tmp, p.orgReaders) + p.readers = tmp + // next is the next non-preferred index. + next := 0 + for i, ok := range prefer { + if !ok || p.readers[i] == nil { + continue + } + if i == next { + next++ + continue + } + // Move reader with index i to index next. + // Do this by swapping next and i + p.readers[next], p.readers[i] = p.readers[i], p.readers[next] + p.readerToBuf[next] = i + p.readerToBuf[i] = next + next++ + } +} + +// Returns if buf can be erasure decoded. +func (p *parallelReader) canDecode(buf [][]byte) bool { + bufCount := 0 + for _, b := range buf { + if len(b) > 0 { + bufCount++ + } + } + return bufCount >= p.dataBlocks +} + +// Read reads from readers in parallel. Returns p.dataBlocks number of bufs. +func (p *parallelReader) Read(dst [][]byte) ([][]byte, error) { + newBuf := dst + if len(dst) != len(p.readers) { + newBuf = make([][]byte, len(p.readers)) + } else { + for i := range newBuf { + newBuf[i] = newBuf[i][:0] + } + } + var newBufLK sync.RWMutex + + if p.offset+p.shardSize > p.shardFileSize { + p.shardSize = p.shardFileSize - p.offset + } + if p.shardSize == 0 { + return newBuf, nil + } + + readTriggerCh := make(chan bool, len(p.readers)) + defer xioutil.SafeClose(readTriggerCh) // close the channel upon return + + for i := 0; i < p.dataBlocks; i++ { + // Setup read triggers for p.dataBlocks number of reads so that it reads in parallel. + readTriggerCh <- true + } + + disksNotFound := int32(0) + bitrotHeal := int32(0) // Atomic bool flag. + missingPartsHeal := int32(0) // Atomic bool flag. + readerIndex := 0 + var wg sync.WaitGroup + // if readTrigger is true, it implies next disk.ReadAt() should be tried + // if readTrigger is false, it implies previous disk.ReadAt() was successful and there is no need + // to try reading the next disk. + for readTrigger := range readTriggerCh { + newBufLK.RLock() + canDecode := p.canDecode(newBuf) + newBufLK.RUnlock() + if canDecode { + break + } + if readerIndex == len(p.readers) { + break + } + if !readTrigger { + continue + } + wg.Add(1) + go func(i int) { + defer wg.Done() + rr := p.readers[i] + if rr == nil { + // Since reader is nil, trigger another read. + readTriggerCh <- true + return + } + bufIdx := p.readerToBuf[i] + if p.buf[bufIdx] == nil { + // Reading first time on this disk, hence the buffer needs to be allocated. + // Subsequent reads will reuse this buffer. + p.buf[bufIdx] = make([]byte, p.shardSize) + } + // For the last shard, the shardsize might be less than previous shard sizes. + // Hence the following statement ensures that the buffer size is reset to the right size. + p.buf[bufIdx] = p.buf[bufIdx][:p.shardSize] + n, err := rr.ReadAt(p.buf[bufIdx], p.offset) + if err != nil { + switch { + case errors.Is(err, errFileNotFound): + atomic.StoreInt32(&missingPartsHeal, 1) + case errors.Is(err, errFileCorrupt): + atomic.StoreInt32(&bitrotHeal, 1) + case errors.Is(err, errDiskNotFound): + atomic.AddInt32(&disksNotFound, 1) + } + + // This will be communicated upstream. + p.orgReaders[bufIdx] = nil + if br, ok := p.readers[i].(io.Closer); ok { + br.Close() + } + p.readers[i] = nil + + // Since ReadAt returned error, trigger another read. + readTriggerCh <- true + return + } + newBufLK.Lock() + newBuf[bufIdx] = p.buf[bufIdx][:n] + newBufLK.Unlock() + // Since ReadAt returned success, there is no need to trigger another read. + readTriggerCh <- false + }(readerIndex) + readerIndex++ + } + wg.Wait() + if p.canDecode(newBuf) { + p.offset += p.shardSize + if missingPartsHeal == 1 { + return newBuf, errFileNotFound + } else if bitrotHeal == 1 { + return newBuf, errFileCorrupt + } + return newBuf, nil + } + + // If we cannot decode, just return read quorum error. + return nil, fmt.Errorf("%w (offline-disks=%d/%d)", errErasureReadQuorum, disksNotFound, len(p.readers)) +} + +// Decode reads from readers, reconstructs data if needed and writes the data to the writer. +// A set of preferred drives can be supplied. In that case they will be used and the data reconstructed. +func (e Erasure) Decode(ctx context.Context, writer io.Writer, readers []io.ReaderAt, offset, length, totalLength int64, prefer []bool) (written int64, derr error) { + if offset < 0 || length < 0 { + return -1, errInvalidArgument + } + if offset+length > totalLength { + return -1, errInvalidArgument + } + + if length == 0 { + return 0, nil + } + + reader := newParallelReader(readers, e, offset, totalLength) + if len(prefer) == len(readers) { + reader.preferReaders(prefer) + } + defer reader.Done() + + startBlock := offset / e.blockSize + endBlock := (offset + length) / e.blockSize + + var bytesWritten int64 + var bufs [][]byte + for block := startBlock; block <= endBlock; block++ { + var blockOffset, blockLength int64 + switch { + case startBlock == endBlock: + blockOffset = offset % e.blockSize + blockLength = length + case block == startBlock: + blockOffset = offset % e.blockSize + blockLength = e.blockSize - blockOffset + case block == endBlock: + blockOffset = 0 + blockLength = (offset + length) % e.blockSize + default: + blockOffset = 0 + blockLength = e.blockSize + } + if blockLength == 0 { + break + } + + var err error + bufs, err = reader.Read(bufs) + if len(bufs) > 0 { + // Set only if there are be enough data for reconstruction. + // and only for expected errors, also set once. + if errors.Is(err, errFileNotFound) || errors.Is(err, errFileCorrupt) { + if derr == nil { + derr = err + } + } + } else if err != nil { + // For all errors that cannot be reconstructed fail the read operation. + return -1, err + } + + if err = e.DecodeDataBlocks(bufs); err != nil { + return -1, err + } + + n, err := writeDataBlocks(ctx, writer, bufs, e.dataBlocks, blockOffset, blockLength) + if err != nil { + return -1, err + } + + bytesWritten += n + } + + if bytesWritten != length { + return bytesWritten, errLessData + } + + return bytesWritten, derr +} + +// Heal reads from readers, reconstruct shards and writes the data to the writers. +func (e Erasure) Heal(ctx context.Context, writers []io.Writer, readers []io.ReaderAt, totalLength int64, prefer []bool) (derr error) { + if len(writers) != e.parityBlocks+e.dataBlocks { + return errInvalidArgument + } + + reader := newParallelReader(readers, e, 0, totalLength) + if len(readers) == len(prefer) { + reader.preferReaders(prefer) + } + defer reader.Done() + + startBlock := int64(0) + endBlock := totalLength / e.blockSize + if totalLength%e.blockSize != 0 { + endBlock++ + } + + var bufs [][]byte + for block := startBlock; block < endBlock; block++ { + var err error + bufs, err = reader.Read(bufs) + if len(bufs) > 0 { + if errors.Is(err, errFileNotFound) || errors.Is(err, errFileCorrupt) { + if derr == nil { + derr = err + } + } + } else if err != nil { + return err + } + + if err = e.DecodeDataAndParityBlocks(ctx, bufs); err != nil { + return err + } + + w := multiWriter{ + writers: writers, + writeQuorum: 1, + errs: make([]error, len(writers)), + } + + if err = w.Write(ctx, bufs); err != nil { + return err + } + } + + return derr +} diff --git a/cmd/erasure-decode_test.go b/cmd/erasure-decode_test.go new file mode 100644 index 00000000..229047e1 --- /dev/null +++ b/cmd/erasure-decode_test.go @@ -0,0 +1,385 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + crand "crypto/rand" + "io" + "math/rand" + "testing" + + "github.com/dustin/go-humanize" +) + +func (a badDisk) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) { + return 0, errFaultyDisk +} + +var erasureDecodeTests = []struct { + dataBlocks int + onDisks, offDisks int + blocksize, data int64 + offset int64 + length int64 + algorithm BitrotAlgorithm + shouldFail, shouldFailQuorum bool +}{ + {dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false}, // 0 + {dataBlocks: 3, onDisks: 6, offDisks: 0, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: SHA256, shouldFail: false, shouldFailQuorum: false}, // 1 + {dataBlocks: 4, onDisks: 8, offDisks: 0, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 2 + {dataBlocks: 5, onDisks: 10, offDisks: 0, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 1, length: oneMiByte - 1, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false}, // 3 + {dataBlocks: 6, onDisks: 12, offDisks: 0, blocksize: int64(oneMiByte), data: oneMiByte, offset: oneMiByte, length: 0, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false}, + // 4 + {dataBlocks: 7, onDisks: 14, offDisks: 0, blocksize: int64(oneMiByte), data: oneMiByte, offset: 3, length: 1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 5 + {dataBlocks: 8, onDisks: 16, offDisks: 0, blocksize: int64(oneMiByte), data: oneMiByte, offset: 4, length: 8 * 1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 6 + {dataBlocks: 7, onDisks: 14, offDisks: 7, blocksize: int64(blockSizeV2), data: oneMiByte, offset: oneMiByte, length: 1, algorithm: DefaultBitrotAlgorithm, shouldFail: true, shouldFailQuorum: false}, // 7 + {dataBlocks: 6, onDisks: 12, offDisks: 6, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 8 + {dataBlocks: 5, onDisks: 10, offDisks: 5, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false}, // 9 + {dataBlocks: 4, onDisks: 8, offDisks: 4, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: SHA256, shouldFail: false, shouldFailQuorum: false}, // 10 + {dataBlocks: 3, onDisks: 6, offDisks: 3, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 11 + {dataBlocks: 2, onDisks: 4, offDisks: 2, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 12 + {dataBlocks: 2, onDisks: 4, offDisks: 1, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 13 + {dataBlocks: 3, onDisks: 6, offDisks: 2, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 14 + {dataBlocks: 4, onDisks: 8, offDisks: 3, blocksize: int64(2 * oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 15 + {dataBlocks: 5, onDisks: 10, offDisks: 6, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true}, // 16 + {dataBlocks: 5, onDisks: 10, offDisks: 2, blocksize: int64(blockSizeV2), data: 2 * oneMiByte, offset: oneMiByte, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 17 + {dataBlocks: 5, onDisks: 10, offDisks: 1, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false}, // 18 + {dataBlocks: 6, onDisks: 12, offDisks: 3, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: SHA256, shouldFail: false, shouldFailQuorum: false}, + // 19 + {dataBlocks: 6, onDisks: 12, offDisks: 7, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true}, // 20 + {dataBlocks: 8, onDisks: 16, offDisks: 8, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 21 + {dataBlocks: 8, onDisks: 16, offDisks: 9, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true}, // 22 + {dataBlocks: 8, onDisks: 16, offDisks: 7, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 23 + {dataBlocks: 2, onDisks: 4, offDisks: 1, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 24 + {dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 0, length: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 25 + {dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV2), data: int64(blockSizeV2) + 1, offset: 0, length: int64(blockSizeV2) + 1, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false}, // 26 + {dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV2), data: int64(2 * blockSizeV2), offset: 12, length: int64(blockSizeV2) + 17, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false}, // 27 + {dataBlocks: 3, onDisks: 6, offDisks: 0, blocksize: int64(blockSizeV2), data: int64(2 * blockSizeV2), offset: 1023, length: int64(blockSizeV2) + 1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 28 + {dataBlocks: 4, onDisks: 8, offDisks: 0, blocksize: int64(blockSizeV2), data: int64(2 * blockSizeV2), offset: 11, length: int64(blockSizeV2) + 2*1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 29 + {dataBlocks: 6, onDisks: 12, offDisks: 0, blocksize: int64(blockSizeV2), data: int64(2 * blockSizeV2), offset: 512, length: int64(blockSizeV2) + 8*1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 30 + {dataBlocks: 8, onDisks: 16, offDisks: 0, blocksize: int64(blockSizeV2), data: int64(2 * blockSizeV2), offset: int64(blockSizeV2), length: int64(blockSizeV2) - 1, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 31 + {dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV2), data: int64(oneMiByte), offset: -1, length: 3, algorithm: DefaultBitrotAlgorithm, shouldFail: true, shouldFailQuorum: false}, // 32 + {dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV2), data: int64(oneMiByte), offset: 1024, length: -1, algorithm: DefaultBitrotAlgorithm, shouldFail: true, shouldFailQuorum: false}, // 33 + {dataBlocks: 4, onDisks: 6, offDisks: 0, blocksize: int64(blockSizeV2), data: int64(blockSizeV2), offset: 0, length: int64(blockSizeV2), algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false}, // 34 + {dataBlocks: 4, onDisks: 6, offDisks: 1, blocksize: int64(blockSizeV2), data: int64(2 * blockSizeV2), offset: 12, length: int64(blockSizeV2) + 17, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false}, // 35 + {dataBlocks: 4, onDisks: 6, offDisks: 3, blocksize: int64(blockSizeV2), data: int64(2 * blockSizeV2), offset: 1023, length: int64(blockSizeV2) + 1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true}, // 36 + {dataBlocks: 8, onDisks: 12, offDisks: 4, blocksize: int64(blockSizeV2), data: int64(2 * blockSizeV2), offset: 11, length: int64(blockSizeV2) + 2*1024, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 37 +} + +func TestErasureDecode(t *testing.T) { + for i, test := range erasureDecodeTests { + setup, err := newErasureTestSetup(t, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize) + if err != nil { + t.Fatalf("Test %d: failed to create test setup: %v", i, err) + } + erasure, err := NewErasure(t.Context(), test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize) + if err != nil { + t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err) + } + disks := setup.disks + data := make([]byte, test.data) + if _, err = io.ReadFull(crand.Reader, data); err != nil { + t.Fatalf("Test %d: failed to generate random test data: %v", i, err) + } + + writeAlgorithm := test.algorithm + if !test.algorithm.Available() { + writeAlgorithm = DefaultBitrotAlgorithm + } + buffer := make([]byte, test.blocksize, 2*test.blocksize) + writers := make([]io.Writer, len(disks)) + for i, disk := range disks { + writers[i] = newBitrotWriter(disk, "", "testbucket", "object", erasure.ShardFileSize(test.data), writeAlgorithm, erasure.ShardSize()) + } + n, err := erasure.Encode(t.Context(), bytes.NewReader(data), writers, buffer, erasure.dataBlocks+1) + closeBitrotWriters(writers) + if err != nil { + t.Fatalf("Test %d: failed to create erasure test file: %v", i, err) + } + if n != test.data { + t.Fatalf("Test %d: failed to create erasure test file", i) + } + for i, w := range writers { + if w == nil { + disks[i] = nil + } + } + + // Get the checksums of the current part. + bitrotReaders := make([]io.ReaderAt, len(disks)) + for index, disk := range disks { + if disk == OfflineDisk { + continue + } + tillOffset := erasure.ShardFileOffset(test.offset, test.length, test.data) + + bitrotReaders[index] = newBitrotReader(disk, nil, "testbucket", "object", tillOffset, writeAlgorithm, bitrotWriterSum(writers[index]), erasure.ShardSize()) + } + + writer := bytes.NewBuffer(nil) + _, err = erasure.Decode(t.Context(), writer, bitrotReaders, test.offset, test.length, test.data, nil) + closeBitrotReaders(bitrotReaders) + if err != nil && !test.shouldFail { + t.Errorf("Test %d: should pass but failed with: %v", i, err) + } + if err == nil && test.shouldFail { + t.Errorf("Test %d: should fail but it passed", i) + } + if err == nil { + if content := writer.Bytes(); !bytes.Equal(content, data[test.offset:test.offset+test.length]) { + t.Errorf("Test %d: read returns wrong file content.", i) + } + } + + for i, r := range bitrotReaders { + if r == nil { + disks[i] = OfflineDisk + } + } + if err == nil && !test.shouldFail { + bitrotReaders = make([]io.ReaderAt, len(disks)) + for index, disk := range disks { + if disk == OfflineDisk { + continue + } + tillOffset := erasure.ShardFileOffset(test.offset, test.length, test.data) + bitrotReaders[index] = newBitrotReader(disk, nil, "testbucket", "object", tillOffset, writeAlgorithm, bitrotWriterSum(writers[index]), erasure.ShardSize()) + } + for j := range disks[:test.offDisks] { + if bitrotReaders[j] == nil { + continue + } + switch r := bitrotReaders[j].(type) { + case *wholeBitrotReader: + r.disk = badDisk{nil} + case *streamingBitrotReader: + r.disk = badDisk{nil} + } + } + if test.offDisks > 0 { + bitrotReaders[0] = nil + } + writer.Reset() + _, err = erasure.Decode(t.Context(), writer, bitrotReaders, test.offset, test.length, test.data, nil) + closeBitrotReaders(bitrotReaders) + if err != nil && !test.shouldFailQuorum { + t.Errorf("Test %d: should pass but failed with: %v", i, err) + } + if err == nil && test.shouldFailQuorum { + t.Errorf("Test %d: should fail but it passed", i) + } + if !test.shouldFailQuorum { + if content := writer.Bytes(); !bytes.Equal(content, data[test.offset:test.offset+test.length]) { + t.Errorf("Test %d: read returns wrong file content", i) + } + } + } + } +} + +// Test erasureDecode with random offset and lengths. +// This test is t.Skip()ed as it a long time to run, hence should be run +// explicitly after commenting out t.Skip() +func TestErasureDecodeRandomOffsetLength(t *testing.T) { + if testing.Short() { + t.Skip() + } + // Initialize environment needed for the test. + dataBlocks := 7 + parityBlocks := 7 + blockSize := int64(1 * humanize.MiByte) + setup, err := newErasureTestSetup(t, dataBlocks, parityBlocks, blockSize) + if err != nil { + t.Error(err) + return + } + disks := setup.disks + erasure, err := NewErasure(t.Context(), dataBlocks, parityBlocks, blockSize) + if err != nil { + t.Fatalf("failed to create ErasureStorage: %v", err) + } + // Prepare a slice of 5MiB with random data. + data := make([]byte, 5*humanize.MiByte) + length := int64(len(data)) + _, err = rand.Read(data) + if err != nil { + t.Fatal(err) + } + + writers := make([]io.Writer, len(disks)) + for i, disk := range disks { + if disk == nil { + continue + } + writers[i] = newBitrotWriter(disk, "", "testbucket", "object", erasure.ShardFileSize(length), DefaultBitrotAlgorithm, erasure.ShardSize()) + } + + // 10000 iterations with random offsets and lengths. + iterations := 10000 + + // Create a test file to read from. + buffer := make([]byte, blockSize, 2*blockSize) + n, err := erasure.Encode(t.Context(), bytes.NewReader(data), writers, buffer, erasure.dataBlocks+1) + closeBitrotWriters(writers) + if err != nil { + t.Fatal(err) + } + if n != length { + t.Errorf("erasureCreateFile returned %d, expected %d", n, length) + } + + // To generate random offset/length. + r := rand.New(rand.NewSource(UTCNow().UnixNano())) + + buf := &bytes.Buffer{} + + // Verify erasure.Decode() for random offsets and lengths. + for range iterations { + offset := r.Int63n(length) + readLen := r.Int63n(length - offset) + + expected := data[offset : offset+readLen] + + // Get the checksums of the current part. + bitrotReaders := make([]io.ReaderAt, len(disks)) + for index, disk := range disks { + if disk == OfflineDisk { + continue + } + tillOffset := erasure.ShardFileOffset(offset, readLen, length) + bitrotReaders[index] = newStreamingBitrotReader(disk, nil, "testbucket", "object", tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize()) + } + _, err = erasure.Decode(t.Context(), buf, bitrotReaders, offset, readLen, length, nil) + closeBitrotReaders(bitrotReaders) + if err != nil { + t.Fatal(err, offset, readLen) + } + got := buf.Bytes() + if !bytes.Equal(expected, got) { + t.Fatalf("read data is different from what was expected, offset=%d length=%d", offset, readLen) + } + buf.Reset() + } +} + +// Benchmarks + +func benchmarkErasureDecode(data, parity, dataDown, parityDown int, size int64, b *testing.B) { + setup, err := newErasureTestSetup(b, data, parity, blockSizeV2) + if err != nil { + b.Fatalf("failed to create test setup: %v", err) + } + disks := setup.disks + erasure, err := NewErasure(context.Background(), data, parity, blockSizeV2) + if err != nil { + b.Fatalf("failed to create ErasureStorage: %v", err) + } + + writers := make([]io.Writer, len(disks)) + for i, disk := range disks { + if disk == nil { + continue + } + writers[i] = newBitrotWriter(disk, "", "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize()) + } + + content := make([]byte, size) + buffer := make([]byte, blockSizeV2, 2*blockSizeV2) + _, err = erasure.Encode(context.Background(), bytes.NewReader(content), writers, buffer, erasure.dataBlocks+1) + closeBitrotWriters(writers) + if err != nil { + b.Fatalf("failed to create erasure test file: %v", err) + } + + for i := range dataDown { + writers[i] = nil + } + for i := data; i < data+parityDown; i++ { + writers[i] = nil + } + + b.SetBytes(size) + b.ReportAllocs() + for b.Loop() { + bitrotReaders := make([]io.ReaderAt, len(disks)) + for index, disk := range disks { + if writers[index] == nil { + continue + } + tillOffset := erasure.ShardFileOffset(0, size, size) + bitrotReaders[index] = newStreamingBitrotReader(disk, nil, "testbucket", "object", tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize()) + } + if _, err = erasure.Decode(context.Background(), bytes.NewBuffer(content[:0]), bitrotReaders, 0, size, size, nil); err != nil { + panic(err) + } + closeBitrotReaders(bitrotReaders) + } +} + +func BenchmarkErasureDecodeQuick(b *testing.B) { + const size = 12 * 1024 * 1024 + b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 0, 0, size, b) }) + b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 0, 1, size, b) }) + b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 1, 0, size, b) }) + b.Run(" X0|X0 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 1, 1, size, b) }) +} + +func BenchmarkErasureDecode_4_64KB(b *testing.B) { + const size = 64 * 1024 + b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 0, 0, size, b) }) + b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 0, 1, size, b) }) + b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 1, 0, size, b) }) + b.Run(" X0|X0 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 1, 1, size, b) }) + b.Run(" 00|XX ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 0, 2, size, b) }) + b.Run(" XX|00 ", func(b *testing.B) { benchmarkErasureDecode(2, 2, 2, 0, size, b) }) +} + +func BenchmarkErasureDecode_8_20MB(b *testing.B) { + const size = 20 * 1024 * 1024 + b.Run(" 0000|0000 ", func(b *testing.B) { benchmarkErasureDecode(4, 4, 0, 0, size, b) }) + b.Run(" 0000|X000 ", func(b *testing.B) { benchmarkErasureDecode(4, 4, 0, 1, size, b) }) + b.Run(" X000|0000 ", func(b *testing.B) { benchmarkErasureDecode(4, 4, 1, 0, size, b) }) + b.Run(" X000|X000 ", func(b *testing.B) { benchmarkErasureDecode(4, 4, 1, 1, size, b) }) + b.Run(" 0000|XXXX ", func(b *testing.B) { benchmarkErasureDecode(4, 4, 0, 4, size, b) }) + b.Run(" XX00|XX00 ", func(b *testing.B) { benchmarkErasureDecode(4, 4, 2, 2, size, b) }) + b.Run(" XXXX|0000 ", func(b *testing.B) { benchmarkErasureDecode(4, 4, 4, 0, size, b) }) +} + +func BenchmarkErasureDecode_12_30MB(b *testing.B) { + const size = 30 * 1024 * 1024 + b.Run(" 000000|000000 ", func(b *testing.B) { benchmarkErasureDecode(6, 6, 0, 0, size, b) }) + b.Run(" 000000|X00000 ", func(b *testing.B) { benchmarkErasureDecode(6, 6, 0, 1, size, b) }) + b.Run(" X00000|000000 ", func(b *testing.B) { benchmarkErasureDecode(6, 6, 1, 0, size, b) }) + b.Run(" X00000|X00000 ", func(b *testing.B) { benchmarkErasureDecode(6, 6, 1, 1, size, b) }) + b.Run(" 000000|XXXXXX ", func(b *testing.B) { benchmarkErasureDecode(6, 6, 0, 6, size, b) }) + b.Run(" XXX000|XXX000 ", func(b *testing.B) { benchmarkErasureDecode(6, 6, 3, 3, size, b) }) + b.Run(" XXXXXX|000000 ", func(b *testing.B) { benchmarkErasureDecode(6, 6, 6, 0, size, b) }) +} + +func BenchmarkErasureDecode_16_40MB(b *testing.B) { + const size = 40 * 1024 * 1024 + b.Run(" 00000000|00000000 ", func(b *testing.B) { benchmarkErasureDecode(8, 8, 0, 0, size, b) }) + b.Run(" 00000000|X0000000 ", func(b *testing.B) { benchmarkErasureDecode(8, 8, 0, 1, size, b) }) + b.Run(" X0000000|00000000 ", func(b *testing.B) { benchmarkErasureDecode(8, 8, 1, 0, size, b) }) + b.Run(" X0000000|X0000000 ", func(b *testing.B) { benchmarkErasureDecode(8, 8, 1, 1, size, b) }) + b.Run(" 00000000|XXXXXXXX ", func(b *testing.B) { benchmarkErasureDecode(8, 8, 0, 8, size, b) }) + b.Run(" XXXX0000|XXXX0000 ", func(b *testing.B) { benchmarkErasureDecode(8, 8, 4, 4, size, b) }) + b.Run(" XXXXXXXX|00000000 ", func(b *testing.B) { benchmarkErasureDecode(8, 8, 8, 0, size, b) }) +} diff --git a/cmd/erasure-encode.go b/cmd/erasure-encode.go new file mode 100644 index 00000000..215ac172 --- /dev/null +++ b/cmd/erasure-encode.go @@ -0,0 +1,110 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "fmt" + "io" +) + +// Writes to multiple writers +type multiWriter struct { + writers []io.Writer + writeQuorum int + errs []error +} + +// Write writes data to writers. +func (p *multiWriter) Write(ctx context.Context, blocks [][]byte) error { + for i := range p.writers { + if p.errs[i] != nil { + continue + } + if p.writers[i] == nil { + p.errs[i] = errDiskNotFound + continue + } + var n int + n, p.errs[i] = p.writers[i].Write(blocks[i]) + if p.errs[i] == nil { + if n != len(blocks[i]) { + p.errs[i] = io.ErrShortWrite + p.writers[i] = nil + } + } else { + p.writers[i] = nil + } + } + + // If nilCount >= p.writeQuorum, we return nil. This is because HealFile() uses + // CreateFile with p.writeQuorum=1 to accommodate healing of single disk. + // i.e if we do no return here in such a case, reduceWriteQuorumErrs() would + // return a quorum error to HealFile(). + nilCount := countErrs(p.errs, nil) + if nilCount >= p.writeQuorum { + return nil + } + + writeErr := reduceWriteQuorumErrs(ctx, p.errs, objectOpIgnoredErrs, p.writeQuorum) + return fmt.Errorf("%w (offline-disks=%d/%d)", writeErr, countErrs(p.errs, errDiskNotFound), len(p.writers)) +} + +// Encode reads from the reader, erasure-encodes the data and writes to the writers. +func (e *Erasure) Encode(ctx context.Context, src io.Reader, writers []io.Writer, buf []byte, quorum int) (total int64, err error) { + writer := &multiWriter{ + writers: writers, + writeQuorum: quorum, + errs: make([]error, len(writers)), + } + + for { + var blocks [][]byte + n, err := io.ReadFull(src, buf) + if err != nil { + if !IsErrIgnored(err, []error{ + io.EOF, + io.ErrUnexpectedEOF, + }...) { + return 0, err + } + } + + eof := err == io.EOF || err == io.ErrUnexpectedEOF + if n == 0 && total != 0 { + // Reached EOF, nothing more to be done. + break + } + + // We take care of the situation where if n == 0 and total == 0 by creating empty data and parity files. + blocks, err = e.EncodeData(ctx, buf[:n]) + if err != nil { + return 0, err + } + + if err = writer.Write(ctx, blocks); err != nil { + return 0, err + } + + total += int64(n) + if eof { + break + } + } + return total, nil +} diff --git a/cmd/erasure-encode_test.go b/cmd/erasure-encode_test.go new file mode 100644 index 00000000..54a37b71 --- /dev/null +++ b/cmd/erasure-encode_test.go @@ -0,0 +1,243 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/rand" + "io" + "testing" + + "github.com/dustin/go-humanize" +) + +type badDisk struct{ StorageAPI } + +func (a badDisk) String() string { + return "bad-disk" +} + +func (a badDisk) AppendFile(ctx context.Context, volume string, path string, buf []byte) error { + return errFaultyDisk +} + +func (a badDisk) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) { + return nil, errFaultyDisk +} + +func (a badDisk) CreateFile(ctx context.Context, origvolume, volume, path string, size int64, reader io.Reader) error { + return errFaultyDisk +} + +func (badDisk) Hostname() string { + return "" +} + +const oneMiByte = 1 * humanize.MiByte + +var erasureEncodeTests = []struct { + dataBlocks int + onDisks, offDisks int + blocksize, data int64 + offset int + algorithm BitrotAlgorithm + shouldFail, shouldFailQuorum bool +}{ + {dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 0, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false}, // 0 + {dataBlocks: 3, onDisks: 6, offDisks: 0, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 1, algorithm: SHA256, shouldFail: false, shouldFailQuorum: false}, // 1 + {dataBlocks: 4, onDisks: 8, offDisks: 2, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 2, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 2 + {dataBlocks: 5, onDisks: 10, offDisks: 3, blocksize: int64(blockSizeV2), data: oneMiByte, offset: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false}, // 3 + {dataBlocks: 6, onDisks: 12, offDisks: 4, blocksize: int64(blockSizeV2), data: oneMiByte, offset: oneMiByte, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false}, // 4 + {dataBlocks: 7, onDisks: 14, offDisks: 5, blocksize: int64(blockSizeV2), data: 0, offset: 0, shouldFail: false, algorithm: SHA256, shouldFailQuorum: false}, // 5 + {dataBlocks: 8, onDisks: 16, offDisks: 7, blocksize: int64(blockSizeV2), data: 0, offset: 0, shouldFail: false, algorithm: DefaultBitrotAlgorithm, shouldFailQuorum: false}, // 6 + {dataBlocks: 2, onDisks: 4, offDisks: 2, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 0, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: true}, // 7 + {dataBlocks: 4, onDisks: 8, offDisks: 4, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 0, algorithm: SHA256, shouldFail: false, shouldFailQuorum: true}, // 8 + {dataBlocks: 7, onDisks: 14, offDisks: 7, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true}, // 9 + {dataBlocks: 8, onDisks: 16, offDisks: 8, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true}, // 10 + {dataBlocks: 5, onDisks: 10, offDisks: 3, blocksize: int64(oneMiByte), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 11 + {dataBlocks: 3, onDisks: 6, offDisks: 1, blocksize: int64(blockSizeV2), data: oneMiByte, offset: oneMiByte / 2, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 12 + {dataBlocks: 2, onDisks: 4, offDisks: 0, blocksize: int64(oneMiByte / 2), data: oneMiByte, offset: oneMiByte/2 + 1, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 13 + {dataBlocks: 4, onDisks: 8, offDisks: 0, blocksize: int64(oneMiByte - 1), data: oneMiByte, offset: oneMiByte - 1, algorithm: BLAKE2b512, shouldFail: false, shouldFailQuorum: false}, // 14 + {dataBlocks: 8, onDisks: 12, offDisks: 2, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 2, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 15 + {dataBlocks: 8, onDisks: 10, offDisks: 1, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 16 + {dataBlocks: 10, onDisks: 14, offDisks: 0, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 17, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 17 + {dataBlocks: 2, onDisks: 6, offDisks: 2, blocksize: int64(oneMiByte), data: oneMiByte, offset: oneMiByte / 2, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: false}, // 18 + {dataBlocks: 10, onDisks: 16, offDisks: 8, blocksize: int64(blockSizeV2), data: oneMiByte, offset: 0, algorithm: DefaultBitrotAlgorithm, shouldFail: false, shouldFailQuorum: true}, // 19 +} + +func TestErasureEncode(t *testing.T) { + for i, test := range erasureEncodeTests { + setup, err := newErasureTestSetup(t, test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize) + if err != nil { + t.Fatalf("Test %d: failed to create test setup: %v", i, err) + } + disks := setup.disks + erasure, err := NewErasure(t.Context(), test.dataBlocks, test.onDisks-test.dataBlocks, test.blocksize) + if err != nil { + t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err) + } + buffer := make([]byte, test.blocksize, 2*test.blocksize) + + data := make([]byte, test.data) + if _, err = io.ReadFull(rand.Reader, data); err != nil { + t.Fatalf("Test %d: failed to generate random test data: %v", i, err) + } + writers := make([]io.Writer, len(disks)) + for i, disk := range disks { + if disk == OfflineDisk { + continue + } + writers[i] = newBitrotWriter(disk, "", "testbucket", "object", erasure.ShardFileSize(int64(len(data[test.offset:]))), test.algorithm, erasure.ShardSize()) + } + n, err := erasure.Encode(t.Context(), bytes.NewReader(data[test.offset:]), writers, buffer, erasure.dataBlocks+1) + closeBitrotWriters(writers) + if err != nil && !test.shouldFail { + t.Errorf("Test %d: should pass but failed with: %v", i, err) + } + if err == nil && test.shouldFail { + t.Errorf("Test %d: should fail but it passed", i) + } + for i, w := range writers { + if w == nil { + disks[i] = OfflineDisk + } + } + if err == nil { + if length := int64(len(data[test.offset:])); n != length { + t.Errorf("Test %d: invalid number of bytes written: got: #%d want #%d", i, n, length) + } + writers := make([]io.Writer, len(disks)) + for i, disk := range disks { + if disk == nil { + continue + } + writers[i] = newBitrotWriter(disk, "", "testbucket", "object2", erasure.ShardFileSize(int64(len(data[test.offset:]))), test.algorithm, erasure.ShardSize()) + } + for j := range disks[:test.offDisks] { + switch w := writers[j].(type) { + case *wholeBitrotWriter: + w.disk = badDisk{nil} + case *streamingBitrotWriter: + w.closeWithErr(errFaultyDisk) + } + } + if test.offDisks > 0 { + writers[0] = nil + } + n, err = erasure.Encode(t.Context(), bytes.NewReader(data[test.offset:]), writers, buffer, erasure.dataBlocks+1) + closeBitrotWriters(writers) + if err != nil && !test.shouldFailQuorum { + t.Errorf("Test %d: should pass but failed with: %v", i, err) + } + if err == nil && test.shouldFailQuorum { + t.Errorf("Test %d: should fail but it passed", i) + } + if err == nil { + if length := int64(len(data[test.offset:])); n != length { + t.Errorf("Test %d: invalid number of bytes written: got: #%d want #%d", i, n, length) + } + } + } + } +} + +// Benchmarks + +func benchmarkErasureEncode(data, parity, dataDown, parityDown int, size int64, b *testing.B) { + setup, err := newErasureTestSetup(b, data, parity, blockSizeV2) + if err != nil { + b.Fatalf("failed to create test setup: %v", err) + } + erasure, err := NewErasure(context.Background(), data, parity, blockSizeV2) + if err != nil { + b.Fatalf("failed to create ErasureStorage: %v", err) + } + disks := setup.disks + buffer := make([]byte, blockSizeV2, 2*blockSizeV2) + content := make([]byte, size) + + for i := range dataDown { + disks[i] = OfflineDisk + } + for i := data; i < data+parityDown; i++ { + disks[i] = OfflineDisk + } + + b.SetBytes(size) + b.ReportAllocs() + for b.Loop() { + writers := make([]io.Writer, len(disks)) + for i, disk := range disks { + if disk == OfflineDisk { + continue + } + disk.Delete(context.Background(), "testbucket", "object", DeleteOptions{ + Recursive: false, + Immediate: false, + }) + writers[i] = newBitrotWriter(disk, "", "testbucket", "object", erasure.ShardFileSize(size), DefaultBitrotAlgorithm, erasure.ShardSize()) + } + _, err := erasure.Encode(context.Background(), bytes.NewReader(content), writers, buffer, erasure.dataBlocks+1) + closeBitrotWriters(writers) + if err != nil { + panic(err) + } + } +} + +func BenchmarkErasureEncodeQuick(b *testing.B) { + const size = 12 * 1024 * 1024 + b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureEncode(2, 2, 0, 0, size, b) }) + b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureEncode(2, 2, 0, 1, size, b) }) + b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureEncode(2, 2, 1, 0, size, b) }) +} + +func BenchmarkErasureEncode_4_64KB(b *testing.B) { + const size = 64 * 1024 + b.Run(" 00|00 ", func(b *testing.B) { benchmarkErasureEncode(2, 2, 0, 0, size, b) }) + b.Run(" 00|X0 ", func(b *testing.B) { benchmarkErasureEncode(2, 2, 0, 1, size, b) }) + b.Run(" X0|00 ", func(b *testing.B) { benchmarkErasureEncode(2, 2, 1, 0, size, b) }) +} + +func BenchmarkErasureEncode_8_20MB(b *testing.B) { + const size = 20 * 1024 * 1024 + b.Run(" 0000|0000 ", func(b *testing.B) { benchmarkErasureEncode(4, 4, 0, 0, size, b) }) + b.Run(" 0000|X000 ", func(b *testing.B) { benchmarkErasureEncode(4, 4, 0, 1, size, b) }) + b.Run(" X000|0000 ", func(b *testing.B) { benchmarkErasureEncode(4, 4, 1, 0, size, b) }) + b.Run(" 0000|XXX0 ", func(b *testing.B) { benchmarkErasureEncode(4, 4, 0, 3, size, b) }) + b.Run(" XXX0|0000 ", func(b *testing.B) { benchmarkErasureEncode(4, 4, 3, 0, size, b) }) +} + +func BenchmarkErasureEncode_12_30MB(b *testing.B) { + const size = 30 * 1024 * 1024 + b.Run(" 000000|000000 ", func(b *testing.B) { benchmarkErasureEncode(6, 6, 0, 0, size, b) }) + b.Run(" 000000|X00000 ", func(b *testing.B) { benchmarkErasureEncode(6, 6, 0, 1, size, b) }) + b.Run(" X00000|000000 ", func(b *testing.B) { benchmarkErasureEncode(6, 6, 1, 0, size, b) }) + b.Run(" 000000|XXXXX0 ", func(b *testing.B) { benchmarkErasureEncode(6, 6, 0, 5, size, b) }) + b.Run(" XXXXX0|000000 ", func(b *testing.B) { benchmarkErasureEncode(6, 6, 5, 0, size, b) }) +} + +func BenchmarkErasureEncode_16_40MB(b *testing.B) { + const size = 40 * 1024 * 1024 + b.Run(" 00000000|00000000 ", func(b *testing.B) { benchmarkErasureEncode(8, 8, 0, 0, size, b) }) + b.Run(" 00000000|X0000000 ", func(b *testing.B) { benchmarkErasureEncode(8, 8, 0, 1, size, b) }) + b.Run(" X0000000|00000000 ", func(b *testing.B) { benchmarkErasureEncode(8, 8, 1, 0, size, b) }) + b.Run(" 00000000|XXXXXXX0 ", func(b *testing.B) { benchmarkErasureEncode(8, 8, 0, 7, size, b) }) + b.Run(" XXXXXXX0|00000000 ", func(b *testing.B) { benchmarkErasureEncode(8, 8, 7, 0, size, b) }) +} diff --git a/cmd/erasure-errors.go b/cmd/erasure-errors.go new file mode 100644 index 00000000..a54f72ba --- /dev/null +++ b/cmd/erasure-errors.go @@ -0,0 +1,29 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import "errors" + +// errErasureReadQuorum - did not meet read quorum. +var errErasureReadQuorum = errors.New("Read failed. Insufficient number of drives online") + +// errErasureWriteQuorum - did not meet write quorum. +var errErasureWriteQuorum = errors.New("Write failed. Insufficient number of drives online") + +// errNoHealRequired - returned when healing is attempted on a previously healed disks. +var errNoHealRequired = errors.New("No healing is required") diff --git a/cmd/erasure-heal_test.go b/cmd/erasure-heal_test.go new file mode 100644 index 00000000..cb6b25fd --- /dev/null +++ b/cmd/erasure-heal_test.go @@ -0,0 +1,156 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "crypto/rand" + "io" + "os" + "testing" +) + +var erasureHealTests = []struct { + dataBlocks, disks int + + // number of offline disks is also number of staleDisks for + // erasure reconstruction in this test + offDisks int + + // bad disks are online disks which return errors + badDisks, badStaleDisks int + + blocksize, size int64 + algorithm BitrotAlgorithm + shouldFail bool +}{ + {dataBlocks: 2, disks: 4, offDisks: 1, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV2), size: oneMiByte, algorithm: SHA256, shouldFail: false}, // 0 + {dataBlocks: 3, disks: 6, offDisks: 2, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV2), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false}, // 1 + {dataBlocks: 4, disks: 8, offDisks: 2, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV2), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false}, // 2 + {dataBlocks: 5, disks: 10, offDisks: 3, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV2), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false}, // 3 + {dataBlocks: 6, disks: 12, offDisks: 2, badDisks: 3, badStaleDisks: 0, blocksize: int64(blockSizeV2), size: oneMiByte, algorithm: SHA256, shouldFail: false}, // 4 + {dataBlocks: 7, disks: 14, offDisks: 4, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV2), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false}, // 5 + {dataBlocks: 8, disks: 16, offDisks: 6, badDisks: 1, badStaleDisks: 1, blocksize: int64(blockSizeV2), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false}, // 6 + {dataBlocks: 7, disks: 14, offDisks: 2, badDisks: 3, badStaleDisks: 0, blocksize: int64(oneMiByte / 2), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false}, // 7 + {dataBlocks: 6, disks: 12, offDisks: 1, badDisks: 0, badStaleDisks: 1, blocksize: int64(oneMiByte - 1), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: true}, // 8 + {dataBlocks: 5, disks: 10, offDisks: 3, badDisks: 0, badStaleDisks: 3, blocksize: int64(oneMiByte / 2), size: oneMiByte, algorithm: SHA256, shouldFail: true}, // 9 + {dataBlocks: 4, disks: 8, offDisks: 1, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV2), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false}, // 10 + {dataBlocks: 2, disks: 4, offDisks: 1, badDisks: 0, badStaleDisks: 1, blocksize: int64(blockSizeV2), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: true}, // 11 + {dataBlocks: 6, disks: 12, offDisks: 8, badDisks: 3, badStaleDisks: 0, blocksize: int64(blockSizeV2), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: true}, // 12 + {dataBlocks: 7, disks: 14, offDisks: 3, badDisks: 4, badStaleDisks: 0, blocksize: int64(blockSizeV2), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false}, // 13 + {dataBlocks: 7, disks: 14, offDisks: 6, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV2), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false}, // 14 + {dataBlocks: 8, disks: 16, offDisks: 4, badDisks: 5, badStaleDisks: 0, blocksize: int64(blockSizeV2), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: true}, // 15 + {dataBlocks: 2, disks: 4, offDisks: 1, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV2), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false}, // 16 + {dataBlocks: 12, disks: 16, offDisks: 2, badDisks: 1, badStaleDisks: 0, blocksize: int64(blockSizeV2), size: oneMiByte, algorithm: DefaultBitrotAlgorithm, shouldFail: false}, // 17 + {dataBlocks: 6, disks: 8, offDisks: 1, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV2), size: oneMiByte, algorithm: BLAKE2b512, shouldFail: false}, // 18 + {dataBlocks: 2, disks: 4, offDisks: 1, badDisks: 0, badStaleDisks: 0, blocksize: int64(blockSizeV2), size: oneMiByte * 64, algorithm: SHA256, shouldFail: false}, // 19 +} + +func TestErasureHeal(t *testing.T) { + for i, test := range erasureHealTests { + if test.offDisks < test.badStaleDisks { + // test case sanity check + t.Fatalf("Test %d: Bad test case - number of stale drives cannot be less than number of badstale drives", i) + } + + // create some test data + setup, err := newErasureTestSetup(t, test.dataBlocks, test.disks-test.dataBlocks, test.blocksize) + if err != nil { + t.Fatalf("Test %d: failed to setup Erasure environment: %v", i, err) + } + disks := setup.disks + erasure, err := NewErasure(t.Context(), test.dataBlocks, test.disks-test.dataBlocks, test.blocksize) + if err != nil { + t.Fatalf("Test %d: failed to create ErasureStorage: %v", i, err) + } + data := make([]byte, test.size) + if _, err = io.ReadFull(rand.Reader, data); err != nil { + t.Fatalf("Test %d: failed to create random test data: %v", i, err) + } + buffer := make([]byte, test.blocksize, 2*test.blocksize) + writers := make([]io.Writer, len(disks)) + for i, disk := range disks { + writers[i] = newBitrotWriter(disk, "", "testbucket", "testobject", erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize()) + } + _, err = erasure.Encode(t.Context(), bytes.NewReader(data), writers, buffer, erasure.dataBlocks+1) + closeBitrotWriters(writers) + if err != nil { + t.Fatalf("Test %d: failed to create random test data: %v", i, err) + } + + readers := make([]io.ReaderAt, len(disks)) + for i, disk := range disks { + shardFilesize := erasure.ShardFileSize(test.size) + readers[i] = newBitrotReader(disk, nil, "testbucket", "testobject", shardFilesize, test.algorithm, bitrotWriterSum(writers[i]), erasure.ShardSize()) + } + + // setup stale disks for the test case + staleDisks := make([]StorageAPI, len(disks)) + copy(staleDisks, disks) + for j := range staleDisks { + if j < test.offDisks { + readers[j] = nil + } else { + staleDisks[j] = nil + } + } + for j := 0; j < test.badDisks; j++ { + switch r := readers[test.offDisks+j].(type) { + case *streamingBitrotReader: + r.disk = badDisk{nil} + case *wholeBitrotReader: + r.disk = badDisk{nil} + } + } + for j := 0; j < test.badStaleDisks; j++ { + staleDisks[j] = badDisk{nil} + } + + staleWriters := make([]io.Writer, len(staleDisks)) + for i, disk := range staleDisks { + if disk == nil { + continue + } + os.Remove(pathJoin(disk.String(), "testbucket", "testobject")) + staleWriters[i] = newBitrotWriter(disk, "", "testbucket", "testobject", erasure.ShardFileSize(test.size), test.algorithm, erasure.ShardSize()) + } + + // test case setup is complete - now call Heal() + err = erasure.Heal(t.Context(), staleWriters, readers, test.size, nil) + closeBitrotReaders(readers) + closeBitrotWriters(staleWriters) + if err != nil && !test.shouldFail { + t.Errorf("Test %d: should pass but it failed with: %v", i, err) + } + if err == nil && test.shouldFail { + t.Errorf("Test %d: should fail but it passed", i) + } + if err == nil { + // Verify that checksums of staleDisks + // match expected values + for i := range staleWriters { + if staleWriters[i] == nil { + continue + } + if !bytes.Equal(bitrotWriterSum(staleWriters[i]), bitrotWriterSum(writers[i])) { + t.Errorf("Test %d: heal returned different bitrot checksums", i) + } + } + } + } +} diff --git a/cmd/erasure-healing-common.go b/cmd/erasure-healing-common.go new file mode 100644 index 00000000..0bde3107 --- /dev/null +++ b/cmd/erasure-healing-common.go @@ -0,0 +1,440 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "slices" + "time" + + "github.com/minio/madmin-go/v3" +) + +func commonETags(etags []string) (etag string, maxima int) { + etagOccurrenceMap := make(map[string]int, len(etags)) + + // Ignore the uuid sentinel and count the rest. + for _, etag := range etags { + if etag == "" { + continue + } + etagOccurrenceMap[etag]++ + } + + maxima = 0 // Counter for remembering max occurrence of elements. + latest := "" + + // Find the common cardinality from previously collected + // occurrences of elements. + for etag, count := range etagOccurrenceMap { + if count < maxima { + continue + } + + // We are at or above maxima + if count > maxima { + maxima = count + latest = etag + } + } + + // Return the collected common max time, with maxima + return latest, maxima +} + +// commonTime returns a maximally occurring time from a list of time. +func commonTimeAndOccurrence(times []time.Time, group time.Duration) (maxTime time.Time, maxima int) { + timeOccurrenceMap := make(map[int64]int, len(times)) + groupNano := group.Nanoseconds() + // Ignore the uuid sentinel and count the rest. + for _, t := range times { + if t.Equal(timeSentinel) || t.IsZero() { + continue + } + nano := t.UnixNano() + if group > 0 { + for k := range timeOccurrenceMap { + if k == nano { + // We add to ourself later + continue + } + diff := k - nano + if diff < 0 { + diff = -diff + } + // We are within the limit + if diff < groupNano { + timeOccurrenceMap[k]++ + } + } + } + // Add ourself... + timeOccurrenceMap[nano]++ + } + + maxima = 0 // Counter for remembering max occurrence of elements. + latest := int64(0) + + // Find the common cardinality from previously collected + // occurrences of elements. + for nano, count := range timeOccurrenceMap { + if count < maxima { + continue + } + + // We are at or above maxima + if count > maxima || nano > latest { + maxima = count + latest = nano + } + } + + // Return the collected common max time, with maxima + return time.Unix(0, latest).UTC(), maxima +} + +// commonTime returns a maximally occurring time from a list of time if it +// occurs >= quorum, else return timeSentinel +func commonTime(modTimes []time.Time, quorum int) time.Time { + if modTime, count := commonTimeAndOccurrence(modTimes, 0); count >= quorum { + return modTime + } + + return timeSentinel +} + +func commonETag(etags []string, quorum int) string { + if etag, count := commonETags(etags); count >= quorum { + return etag + } + return "" +} + +// Beginning of unix time is treated as sentinel value here. +var ( + timeSentinel = time.Unix(0, 0).UTC() + timeSentinel1970 = time.Unix(0, 1).UTC() // 1970 used for special cases when xlmeta.version == 0 +) + +// Boot modTimes up to disk count, setting the value to time sentinel. +func bootModtimes(diskCount int) []time.Time { + modTimes := make([]time.Time, diskCount) + // Boots up all the modtimes. + for i := range modTimes { + modTimes[i] = timeSentinel + } + return modTimes +} + +func listObjectETags(partsMetadata []FileInfo, errs []error, quorum int) (etags []string) { + etags = make([]string, len(partsMetadata)) + vidMap := map[string]int{} + for index, metadata := range partsMetadata { + if errs[index] != nil { + continue + } + vid := metadata.VersionID + if metadata.VersionID == "" { + vid = nullVersionID + } + vidMap[vid]++ + etags[index] = metadata.Metadata["etag"] + } + + for _, count := range vidMap { + // do we have enough common versions + // that have enough quorum to satisfy + // the etag. + if count >= quorum { + return etags + } + } + + return make([]string, len(partsMetadata)) +} + +// Extracts list of times from FileInfo slice and returns, skips +// slice elements which have errors. +func listObjectModtimes(partsMetadata []FileInfo, errs []error) (modTimes []time.Time) { + modTimes = bootModtimes(len(partsMetadata)) + for index, metadata := range partsMetadata { + if errs[index] != nil { + continue + } + // Once the file is found, save the uuid saved on disk. + modTimes[index] = metadata.ModTime + } + return modTimes +} + +func filterOnlineDisksInplace(fi FileInfo, partsMetadata []FileInfo, onlineDisks []StorageAPI) { + for i, meta := range partsMetadata { + if fi.XLV1 == meta.XLV1 { + continue + } + onlineDisks[i] = nil + } +} + +// Notes: +// There are 5 possible states a disk could be in, +// 1. __online__ - has the latest copy of xl.meta - returned by listOnlineDisks +// +// 2. __offline__ - err == errDiskNotFound +// +// 3. __availableWithParts__ - has the latest copy of xl.meta and has all +// parts with checksums matching; returned by disksWithAllParts +// +// 4. __outdated__ - returned by outDatedDisk, provided []StorageAPI +// returned by diskWithAllParts is passed for latestDisks. +// - has an old copy of xl.meta +// - doesn't have xl.meta (errFileNotFound) +// - has the latest xl.meta but one or more parts are corrupt +// +// 5. __missingParts__ - has the latest copy of xl.meta but has some parts +// missing. This is identified separately since this may need manual +// inspection to understand the root cause. E.g, this could be due to +// backend filesystem corruption. + +// listOnlineDisks - returns +// - a slice of disks where disk having 'older' xl.meta (or nothing) +// are set to nil. +// - latest (in time) of the maximally occurring modTime(s), which has at least quorum occurrences. +func listOnlineDisks(disks []StorageAPI, partsMetadata []FileInfo, errs []error, quorum int) (onlineDisks []StorageAPI, modTime time.Time, etag string) { + onlineDisks = make([]StorageAPI, len(disks)) + + // List all the file commit ids from parts metadata. + modTimes := listObjectModtimes(partsMetadata, errs) + + // Reduce list of UUIDs to a single common value. + modTime = commonTime(modTimes, quorum) + + if modTime.IsZero() || modTime.Equal(timeSentinel) { + etags := listObjectETags(partsMetadata, errs, quorum) + + etag = commonETag(etags, quorum) + + if etag != "" { // allow this fallback only if a non-empty etag is found. + for index, e := range etags { + if partsMetadata[index].IsValid() && e == etag { + onlineDisks[index] = disks[index] + } else { + onlineDisks[index] = nil + } + } + return onlineDisks, modTime, etag + } + } + + // Create a new online disks slice, which have common uuid. + for index, t := range modTimes { + if partsMetadata[index].IsValid() && t.Equal(modTime) { + onlineDisks[index] = disks[index] + } else { + onlineDisks[index] = nil + } + } + + return onlineDisks, modTime, "" +} + +// Convert verify or check parts returned error to integer representation +func convPartErrToInt(err error) int { + err = unwrapAll(err) + switch err { + case nil: + return checkPartSuccess + case errFileNotFound, errFileVersionNotFound: + return checkPartFileNotFound + case errFileCorrupt: + return checkPartFileCorrupt + case errVolumeNotFound: + return checkPartVolumeNotFound + case errDiskNotFound: + return checkPartDiskNotFound + default: + return checkPartUnknown + } +} + +func partNeedsHealing(partErrs []int) bool { + return slices.IndexFunc(partErrs, func(i int) bool { return i != checkPartSuccess && i != checkPartUnknown }) > -1 +} + +func countPartNotSuccess(partErrs []int) (c int) { + for _, pe := range partErrs { + if pe != checkPartSuccess { + c++ + } + } + return c +} + +// checkObjectWithAllParts sets partsMetadata and onlineDisks when xl.meta is inexistant/corrupted or outdated +// it also checks if the status of each part (corrupted, missing, ok) in each drive +func checkObjectWithAllParts(ctx context.Context, onlineDisks []StorageAPI, partsMetadata []FileInfo, + errs []error, latestMeta FileInfo, filterByETag bool, bucket, object string, + scanMode madmin.HealScanMode, +) (dataErrsByDisk map[int][]int, dataErrsByPart map[int][]int) { + dataErrsByDisk = make(map[int][]int, len(onlineDisks)) + for i := range onlineDisks { + dataErrsByDisk[i] = make([]int, len(latestMeta.Parts)) + } + + dataErrsByPart = make(map[int][]int, len(latestMeta.Parts)) + for i := range latestMeta.Parts { + dataErrsByPart[i] = make([]int, len(onlineDisks)) + } + + inconsistent := 0 + for i, meta := range partsMetadata { + if !meta.IsValid() { + // Since for majority of the cases erasure.Index matches with erasure.Distribution we can + // consider the offline disks as consistent. + continue + } + if !meta.Deleted { + if len(meta.Erasure.Distribution) != len(onlineDisks) { + // Erasure distribution seems to have lesser + // number of items than number of online disks. + inconsistent++ + continue + } + if meta.Erasure.Distribution[i] != meta.Erasure.Index { + // Mismatch indexes with distribution order + inconsistent++ + } + } + } + + erasureDistributionReliable := inconsistent <= len(partsMetadata)/2 + + metaErrs := make([]error, len(errs)) + + for i := range onlineDisks { + if errs[i] != nil { + metaErrs[i] = errs[i] + continue + } + if onlineDisks[i] == OfflineDisk { + metaErrs[i] = errDiskNotFound + continue + } + + meta := partsMetadata[i] + corrupted := false + if filterByETag { + corrupted = meta.Metadata["etag"] != latestMeta.Metadata["etag"] + } else { + corrupted = !meta.ModTime.Equal(latestMeta.ModTime) || meta.DataDir != latestMeta.DataDir + } + + if corrupted { + metaErrs[i] = errFileCorrupt + partsMetadata[i] = FileInfo{} + onlineDisks[i] = nil + continue + } + + if erasureDistributionReliable { + if !meta.IsValid() { + partsMetadata[i] = FileInfo{} + metaErrs[i] = errFileCorrupt + onlineDisks[i] = nil + continue + } + + if !meta.Deleted { + if len(meta.Erasure.Distribution) != len(onlineDisks) { + // Erasure distribution is not the same as onlineDisks + // attempt a fix if possible, assuming other entries + // might have the right erasure distribution. + partsMetadata[i] = FileInfo{} + metaErrs[i] = errFileCorrupt + onlineDisks[i] = nil + continue + } + } + } + } + + // Copy meta errors to part errors + for i, err := range metaErrs { + if err != nil { + partErr := convPartErrToInt(err) + for p := range latestMeta.Parts { + dataErrsByPart[p][i] = partErr + } + } + } + + for i, onlineDisk := range onlineDisks { + if metaErrs[i] != nil { + continue + } + + meta := partsMetadata[i] + if meta.Deleted || meta.IsRemote() { + continue + } + + // Always check data, if we got it. + if (len(meta.Data) > 0 || meta.Size == 0) && len(meta.Parts) > 0 { + checksumInfo := meta.Erasure.GetChecksumInfo(meta.Parts[0].Number) + verifyErr := bitrotVerify(bytes.NewReader(meta.Data), + int64(len(meta.Data)), + meta.Erasure.ShardFileSize(meta.Size), + checksumInfo.Algorithm, + checksumInfo.Hash, meta.Erasure.ShardSize()) + dataErrsByPart[0][i] = convPartErrToInt(verifyErr) + continue + } + + var ( + verifyErr error + verifyResp *CheckPartsResp + ) + + switch scanMode { + case madmin.HealDeepScan: + // disk has a valid xl.meta but may not have all the + // parts. This is considered an outdated disk, since + // it needs healing too. + verifyResp, verifyErr = onlineDisk.VerifyFile(ctx, bucket, object, meta) + default: + verifyResp, verifyErr = onlineDisk.CheckParts(ctx, bucket, object, meta) + } + + for p := range latestMeta.Parts { + if verifyErr != nil { + dataErrsByPart[p][i] = convPartErrToInt(verifyErr) + } else { + dataErrsByPart[p][i] = verifyResp.Results[p] + } + } + } + + // Build dataErrs by disk from dataErrs by part + for part, disks := range dataErrsByPart { + for disk := range disks { + dataErrsByDisk[disk][part] = dataErrsByPart[part][disk] + } + } + return dataErrsByDisk, dataErrsByPart +} diff --git a/cmd/erasure-healing-common_test.go b/cmd/erasure-healing-common_test.go new file mode 100644 index 00000000..04909b2d --- /dev/null +++ b/cmd/erasure-healing-common_test.go @@ -0,0 +1,774 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "fmt" + "os" + "path/filepath" + "runtime" + "testing" + "time" + + "github.com/minio/madmin-go/v3" +) + +// Returns the latest updated FileInfo files and error in case of failure. +func getLatestFileInfo(ctx context.Context, partsMetadata []FileInfo, defaultParityCount int, errs []error) (FileInfo, error) { + // There should be at least half correct entries, if not return failure + expectedRQuorum := len(partsMetadata) / 2 + if defaultParityCount == 0 { + // if parity count is '0', we expected all entries to be present. + expectedRQuorum = len(partsMetadata) + } + + reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, expectedRQuorum) + if reducedErr != nil { + return FileInfo{}, reducedErr + } + + // List all the file commit ids from parts metadata. + modTimes := listObjectModtimes(partsMetadata, errs) + + // Count all latest updated FileInfo values + var count int + var latestFileInfo FileInfo + + // Reduce list of UUIDs to a single common value - i.e. the last updated Time + modTime := commonTime(modTimes, expectedRQuorum) + + if modTime.IsZero() || modTime.Equal(timeSentinel) { + return FileInfo{}, errErasureReadQuorum + } + + // Iterate through all the modTimes and count the FileInfo(s) with latest time. + for index, t := range modTimes { + if partsMetadata[index].IsValid() && t.Equal(modTime) { + latestFileInfo = partsMetadata[index] + count++ + } + } + + if !latestFileInfo.IsValid() { + return FileInfo{}, errErasureReadQuorum + } + + if count < latestFileInfo.Erasure.DataBlocks { + return FileInfo{}, errErasureReadQuorum + } + + return latestFileInfo, nil +} + +// validates functionality provided to find most common +// time occurrence from a list of time. +func TestCommonTime(t *testing.T) { + // List of test cases for common modTime. + testCases := []struct { + times []time.Time + time time.Time + quorum int + }{ + { + // 1. Tests common times when slice has varying time elements. + []time.Time{ + time.Unix(0, 1).UTC(), + time.Unix(0, 2).UTC(), + time.Unix(0, 3).UTC(), + time.Unix(0, 3).UTC(), + time.Unix(0, 2).UTC(), + time.Unix(0, 3).UTC(), + time.Unix(0, 1).UTC(), + }, + time.Unix(0, 3).UTC(), + 3, + }, + { + // 2. Tests common time obtained when all elements are equal. + []time.Time{ + time.Unix(0, 3).UTC(), + time.Unix(0, 3).UTC(), + time.Unix(0, 3).UTC(), + time.Unix(0, 3).UTC(), + time.Unix(0, 3).UTC(), + time.Unix(0, 3).UTC(), + time.Unix(0, 3).UTC(), + }, + time.Unix(0, 3).UTC(), + 4, + }, + { + // 3. Tests common time obtained when elements have a mixture of + // sentinel values and don't have read quorum on any of the values. + []time.Time{ + time.Unix(0, 3).UTC(), + time.Unix(0, 3).UTC(), + time.Unix(0, 2).UTC(), + time.Unix(0, 1).UTC(), + time.Unix(0, 3).UTC(), + time.Unix(0, 4).UTC(), + time.Unix(0, 3).UTC(), + timeSentinel, + timeSentinel, + timeSentinel, + }, + timeSentinel, + 5, + }, + } + + // Tests all the testcases, and validates them against expected + // common modtime. Tests fail if modtime does not match. + for i, testCase := range testCases { + // Obtain a common mod time from modTimes slice. + ctime := commonTime(testCase.times, testCase.quorum) + if !testCase.time.Equal(ctime) { + t.Errorf("Test case %d, expect to pass but failed. Wanted modTime: %s, got modTime: %s\n", i+1, testCase.time, ctime) + } + } +} + +// TestListOnlineDisks - checks if listOnlineDisks and outDatedDisks +// are consistent with each other. +func TestListOnlineDisks(t *testing.T) { + if runtime.GOOS == globalWindowsOSName { + t.Skip() + } + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + obj, disks, err := prepareErasure16(ctx) + if err != nil { + t.Fatalf("Prepare Erasure backend failed - %v", err) + } + setObjectLayer(obj) + defer obj.Shutdown(t.Context()) + defer removeRoots(disks) + + type tamperKind int + const ( + noTamper tamperKind = iota + deletePart + corruptPart + ) + + timeSentinel := time.Unix(1, 0).UTC() + threeNanoSecs := time.Unix(3, 0).UTC() + fourNanoSecs := time.Unix(4, 0).UTC() + modTimesThreeNone := make([]time.Time, 16) + modTimesThreeFour := make([]time.Time, 16) + for i := range 16 { + // Have 13 good xl.meta, 12 for default parity count = 4 (EC:4) and one + // to be tampered with. + if i > 12 { + modTimesThreeFour[i] = fourNanoSecs + modTimesThreeNone[i] = timeSentinel + continue + } + modTimesThreeFour[i] = threeNanoSecs + modTimesThreeNone[i] = threeNanoSecs + } + + testCases := []struct { + modTimes []time.Time + expectedTime time.Time + errs []error + _tamperBackend tamperKind + }{ + { + modTimes: modTimesThreeFour, + expectedTime: threeNanoSecs, + errs: []error{ + nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, + }, + _tamperBackend: noTamper, + }, + { + modTimes: modTimesThreeNone, + expectedTime: threeNanoSecs, + errs: []error{ + // Disks that have a valid xl.meta. + nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, + // Some disks can't access xl.meta. + errFileNotFound, errDiskAccessDenied, errDiskNotFound, + }, + _tamperBackend: deletePart, + }, + { + modTimes: modTimesThreeNone, + expectedTime: threeNanoSecs, + errs: []error{ + // Disks that have a valid xl.meta. + nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, + // Some disks don't have xl.meta. + errDiskNotFound, errFileNotFound, errFileNotFound, + }, + _tamperBackend: corruptPart, + }, + } + + bucket := "bucket" + err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{}) + if err != nil { + t.Fatalf("Failed to make a bucket %v", err) + } + + object := "object" + data := bytes.Repeat([]byte("a"), smallFileThreshold*32) + z := obj.(*erasureServerPools) + + erasureDisks, err := z.GetDisks(0, 0) + if err != nil { + t.Fatal(err) + } + + for i, test := range testCases { + t.Run(fmt.Sprintf("case-%d", i), func(t *testing.T) { + _, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}) + if err != nil { + t.Fatalf("Failed to putObject %v", err) + } + + partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true) + fi, err := getLatestFileInfo(ctx, partsMetadata, z.serverPools[0].sets[0].defaultParityCount, errs) + if err != nil { + t.Fatalf("Failed to getLatestFileInfo %v", err) + } + + for j := range partsMetadata { + if errs[j] != nil { + t.Fatalf("expected error to be nil: %s", errs[j]) + } + partsMetadata[j].ModTime = test.modTimes[j] + } + + tamperedIndex := -1 + switch test._tamperBackend { + case deletePart: + for index, err := range test.errs { + if err != nil { + continue + } + // Remove a part from a disk + // which has a valid xl.meta, + // and check if that disk + // appears in outDatedDisks. + tamperedIndex = index + dErr := erasureDisks[index].Delete(t.Context(), bucket, pathJoin(object, fi.DataDir, "part.1"), DeleteOptions{ + Recursive: false, + Immediate: false, + }) + if dErr != nil { + t.Fatalf("Failed to delete %s - %v", filepath.Join(object, "part.1"), dErr) + } + break + } + case corruptPart: + for index, err := range test.errs { + if err != nil { + continue + } + // Corrupt a part from a disk + // which has a valid xl.meta, + // and check if that disk + // appears in outDatedDisks. + tamperedIndex = index + filePath := pathJoin(erasureDisks[index].String(), bucket, object, fi.DataDir, "part.1") + f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_SYNC, 0) + if err != nil { + t.Fatalf("Failed to open %s: %s\n", filePath, err) + } + f.WriteString("oops") // Will cause bitrot error + f.Close() + break + } + } + + rQuorum := len(errs) - z.serverPools[0].sets[0].defaultParityCount + onlineDisks, modTime, _ := listOnlineDisks(erasureDisks, partsMetadata, test.errs, rQuorum) + if !modTime.Equal(test.expectedTime) { + t.Fatalf("Expected modTime to be equal to %v but was found to be %v", + test.expectedTime, modTime) + } + _, _ = checkObjectWithAllParts(ctx, onlineDisks, partsMetadata, + test.errs, fi, false, bucket, object, madmin.HealDeepScan) + + if test._tamperBackend != noTamper { + if tamperedIndex != -1 && onlineDisks[tamperedIndex] != nil { + t.Fatalf("Drive (%v) with part.1 missing is not a drive with available data", + erasureDisks[tamperedIndex]) + } + } + }) + } +} + +// TestListOnlineDisksSmallObjects - checks if listOnlineDisks and outDatedDisks +// are consistent with each other. +func TestListOnlineDisksSmallObjects(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + obj, disks, err := prepareErasure16(ctx) + if err != nil { + t.Fatalf("Prepare Erasure backend failed - %v", err) + } + setObjectLayer(obj) + defer obj.Shutdown(t.Context()) + defer removeRoots(disks) + + type tamperKind int + const ( + noTamper tamperKind = iota + deletePart tamperKind = iota + corruptPart tamperKind = iota + ) + timeSentinel := time.Unix(1, 0).UTC() + threeNanoSecs := time.Unix(3, 0).UTC() + fourNanoSecs := time.Unix(4, 0).UTC() + modTimesThreeNone := make([]time.Time, 16) + modTimesThreeFour := make([]time.Time, 16) + for i := range 16 { + // Have 13 good xl.meta, 12 for default parity count = 4 (EC:4) and one + // to be tampered with. + if i > 12 { + modTimesThreeFour[i] = fourNanoSecs + modTimesThreeNone[i] = timeSentinel + continue + } + modTimesThreeFour[i] = threeNanoSecs + modTimesThreeNone[i] = threeNanoSecs + } + + testCases := []struct { + modTimes []time.Time + expectedTime time.Time + errs []error + _tamperBackend tamperKind + }{ + { + modTimes: modTimesThreeFour, + expectedTime: threeNanoSecs, + errs: []error{ + nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, nil, nil, nil, + }, + _tamperBackend: noTamper, + }, + { + modTimes: modTimesThreeNone, + expectedTime: threeNanoSecs, + errs: []error{ + // Disks that have a valid xl.meta. + nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, + // Some disks can't access xl.meta. + errFileNotFound, errDiskAccessDenied, errDiskNotFound, + }, + _tamperBackend: deletePart, + }, + { + modTimes: modTimesThreeNone, + expectedTime: threeNanoSecs, + errs: []error{ + // Disks that have a valid xl.meta. + nil, nil, nil, nil, nil, nil, nil, nil, + nil, nil, nil, nil, nil, + // Some disks don't have xl.meta. + errDiskNotFound, errFileNotFound, errFileNotFound, + }, + _tamperBackend: corruptPart, + }, + } + + bucket := "bucket" + err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{}) + if err != nil { + t.Fatalf("Failed to make a bucket %v", err) + } + + object := "object" + data := bytes.Repeat([]byte("a"), smallFileThreshold/2) + z := obj.(*erasureServerPools) + + erasureDisks, err := z.GetDisks(0, 0) + if err != nil { + t.Fatal(err) + } + + for i, test := range testCases { + t.Run(fmt.Sprintf("case-%d", i), func(t *testing.T) { + _, err := obj.PutObject(ctx, bucket, object, + mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}) + if err != nil { + t.Fatalf("Failed to putObject %v", err) + } + + partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", true, true) + fi, err := getLatestFileInfo(ctx, partsMetadata, z.serverPools[0].sets[0].defaultParityCount, errs) + if err != nil { + t.Fatalf("Failed to getLatestFileInfo %v", err) + } + + for j := range partsMetadata { + if errs[j] != nil { + t.Fatalf("expected error to be nil: %s", errs[j]) + } + partsMetadata[j].ModTime = test.modTimes[j] + } + + if erasureDisks, err = writeUniqueFileInfo(ctx, erasureDisks, "", bucket, object, partsMetadata, diskCount(erasureDisks)); err != nil { + t.Fatal(ctx, err) + } + + tamperedIndex := -1 + switch test._tamperBackend { + case deletePart: + for index, err := range test.errs { + if err != nil { + continue + } + // Remove a part from a disk + // which has a valid xl.meta, + // and check if that disk + // appears in outDatedDisks. + tamperedIndex = index + dErr := erasureDisks[index].Delete(t.Context(), bucket, pathJoin(object, xlStorageFormatFile), DeleteOptions{ + Recursive: false, + Immediate: false, + }) + if dErr != nil { + t.Fatalf("Failed to delete %s - %v", pathJoin(object, xlStorageFormatFile), dErr) + } + break + } + case corruptPart: + for index, err := range test.errs { + if err != nil { + continue + } + // Corrupt a part from a disk + // which has a valid xl.meta, + // and check if that disk + // appears in outDatedDisks. + tamperedIndex = index + filePath := pathJoin(erasureDisks[index].String(), bucket, object, xlStorageFormatFile) + f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_SYNC, 0) + if err != nil { + t.Fatalf("Failed to open %s: %s\n", filePath, err) + } + f.WriteString("oops") // Will cause bitrot error + f.Close() + break + } + } + + rQuorum := len(errs) - z.serverPools[0].sets[0].defaultParityCount + onlineDisks, modTime, _ := listOnlineDisks(erasureDisks, partsMetadata, test.errs, rQuorum) + if !modTime.Equal(test.expectedTime) { + t.Fatalf("Expected modTime to be equal to %v but was found to be %v", + test.expectedTime, modTime) + } + + _, _ = checkObjectWithAllParts(ctx, onlineDisks, partsMetadata, + test.errs, fi, false, bucket, object, madmin.HealDeepScan) + + if test._tamperBackend != noTamper { + if tamperedIndex != -1 && onlineDisks[tamperedIndex] != nil { + t.Fatalf("Drive (%v) with part.1 missing is not a drive with available data", + erasureDisks[tamperedIndex]) + } + } + }) + } +} + +func TestDisksWithAllParts(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + obj, disks, err := prepareErasure16(ctx) + if err != nil { + t.Fatalf("Prepare Erasure backend failed - %v", err) + } + setObjectLayer(obj) + defer obj.Shutdown(t.Context()) + defer removeRoots(disks) + + bucket := "bucket" + object := "object" + // make data with more than one part + partCount := 3 + data := bytes.Repeat([]byte("a"), 6*1024*1024*partCount) + z := obj.(*erasureServerPools) + s := z.serverPools[0].sets[0] + erasureDisks := s.getDisks() + err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{}) + if err != nil { + t.Fatalf("Failed to make a bucket %v", err) + } + + _, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}) + if err != nil { + t.Fatalf("Failed to putObject %v", err) + } + + _, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true) + readQuorum := len(erasureDisks) / 2 + if reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); reducedErr != nil { + t.Fatalf("Failed to read xl meta data %v", reducedErr) + } + + // Test 1: Test that all disks are returned without any failures with + // unmodified meta data + erasureDisks = s.getDisks() + partsMetadata, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true) + if err != nil { + t.Fatalf("Failed to read xl meta data %v", err) + } + + fi, err := getLatestFileInfo(ctx, partsMetadata, s.defaultParityCount, errs) + if err != nil { + t.Fatalf("Failed to get quorum consistent fileInfo %v", err) + } + + erasureDisks, _, _ = listOnlineDisks(erasureDisks, partsMetadata, errs, readQuorum) + + dataErrsPerDisk, _ := checkObjectWithAllParts(ctx, erasureDisks, partsMetadata, + errs, fi, false, bucket, object, madmin.HealDeepScan) + + for diskIndex, disk := range erasureDisks { + if partNeedsHealing(dataErrsPerDisk[diskIndex]) { + t.Errorf("Unexpected error: %v", dataErrsPerDisk[diskIndex]) + } + + if disk == nil { + t.Errorf("Drive erroneously filtered, driveIndex: %d", diskIndex) + } + } + + // Test 2: Not synchronized modtime + erasureDisks = s.getDisks() + partsMetadataBackup := partsMetadata[0] + partsMetadata[0].ModTime = partsMetadata[0].ModTime.Add(-1 * time.Hour) + + errs = make([]error, len(erasureDisks)) + _, _ = checkObjectWithAllParts(ctx, erasureDisks, partsMetadata, + errs, fi, false, bucket, object, madmin.HealDeepScan) + + for diskIndex, disk := range erasureDisks { + if diskIndex == 0 && disk != nil { + t.Errorf("Drive not filtered as expected, drive: %d", diskIndex) + } + if diskIndex != 0 && disk == nil { + t.Errorf("Drive erroneously filtered, driveIndex: %d", diskIndex) + } + } + partsMetadata[0] = partsMetadataBackup // Revert before going to the next test + + // Test 3: Not synchronized DataDir + erasureDisks = s.getDisks() + partsMetadataBackup = partsMetadata[1] + partsMetadata[1].DataDir = "foo-random" + + errs = make([]error, len(erasureDisks)) + _, _ = checkObjectWithAllParts(ctx, erasureDisks, partsMetadata, + errs, fi, false, bucket, object, madmin.HealDeepScan) + + for diskIndex, disk := range erasureDisks { + if diskIndex == 1 && disk != nil { + t.Errorf("Drive not filtered as expected, drive: %d", diskIndex) + } + if diskIndex != 1 && disk == nil { + t.Errorf("Drive erroneously filtered, driveIndex: %d", diskIndex) + } + } + partsMetadata[1] = partsMetadataBackup // Revert before going to the next test + + // Test 4: key = disk index, value = part name with hash mismatch + erasureDisks = s.getDisks() + diskFailures := make(map[int]string) + diskFailures[0] = "part.1" + diskFailures[3] = "part.1" + diskFailures[15] = "part.1" + + for diskIndex, partName := range diskFailures { + for i := range partsMetadata[diskIndex].Parts { + if fmt.Sprintf("part.%d", i+1) == partName { + filePath := pathJoin(erasureDisks[diskIndex].String(), bucket, object, partsMetadata[diskIndex].DataDir, partName) + f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_SYNC, 0) + if err != nil { + t.Fatalf("Failed to open %s: %s\n", filePath, err) + } + f.WriteString("oops") // Will cause bitrot error + f.Close() + } + } + } + + errs = make([]error, len(erasureDisks)) + dataErrsPerDisk, _ = checkObjectWithAllParts(ctx, erasureDisks, partsMetadata, + errs, fi, false, bucket, object, madmin.HealDeepScan) + + for diskIndex := range erasureDisks { + if _, ok := diskFailures[diskIndex]; ok { + if !partNeedsHealing(dataErrsPerDisk[diskIndex]) { + t.Errorf("Disk expected to be healed, driveIndex: %d", diskIndex) + } + } else { + if partNeedsHealing(dataErrsPerDisk[diskIndex]) { + t.Errorf("Disk not expected to be healed, driveIndex: %d", diskIndex) + } + } + } +} + +func TestCommonParities(t *testing.T) { + // This test uses two FileInfo values that represent the same object but + // have different parities. They occur in equal number of drives, but only + // one has read quorum. commonParity should pick the parity corresponding to + // the FileInfo which has read quorum. + fi1 := FileInfo{ + Volume: "mybucket", + Name: "myobject", + VersionID: "", + IsLatest: true, + Deleted: false, + ExpireRestored: false, + DataDir: "4a01d9dd-0c5e-4103-88f8-b307c57d212e", + XLV1: false, + ModTime: time.Date(2023, time.March, 15, 11, 18, 4, 989906961, time.UTC), + Size: 329289, Mode: 0x0, WrittenByVersion: 0x63c77756, + Metadata: map[string]string{ + "content-type": "application/octet-stream", "etag": "f205307ef9f50594c4b86d9c246bee86", "x-minio-internal-erasure-upgraded": "5->6", "x-minio-internal-inline-data": "true", + }, + Parts: []ObjectPartInfo{ + { + ETag: "", + Number: 1, + Size: 329289, + ActualSize: 329289, + ModTime: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), + Index: []uint8(nil), + Checksums: map[string]string(nil), + }, + }, + Erasure: ErasureInfo{ + Algorithm: "ReedSolomon", + DataBlocks: 6, + ParityBlocks: 6, + BlockSize: 1048576, + Index: 1, + Distribution: []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}, + Checksums: []ChecksumInfo{{PartNumber: 1, Algorithm: 0x3, Hash: []uint8{}}}, + }, + NumVersions: 1, + Idx: 0, + } + + fi2 := FileInfo{ + Volume: "mybucket", + Name: "myobject", + VersionID: "", + IsLatest: true, + Deleted: false, + DataDir: "6f5c106d-9d28-4c85-a7f4-eac56225876b", + ModTime: time.Date(2023, time.March, 15, 19, 57, 30, 492530160, time.UTC), + Size: 329289, + Mode: 0x0, + WrittenByVersion: 0x63c77756, + Metadata: map[string]string{"content-type": "application/octet-stream", "etag": "f205307ef9f50594c4b86d9c246bee86", "x-minio-internal-inline-data": "true"}, + Parts: []ObjectPartInfo{ + { + ETag: "", + Number: 1, + Size: 329289, + ActualSize: 329289, + ModTime: time.Date(1, time.January, 1, 0, 0, 0, 0, time.UTC), + Index: []uint8(nil), + Checksums: map[string]string(nil), + }, + }, + Erasure: ErasureInfo{ + Algorithm: "ReedSolomon", + DataBlocks: 7, + ParityBlocks: 5, + BlockSize: 1048576, + Index: 2, + Distribution: []int{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12}, + Checksums: []ChecksumInfo{ + {PartNumber: 1, Algorithm: 0x3, Hash: []uint8{}}, + }, + }, + NumVersions: 1, + Idx: 0, + } + + fiDel := FileInfo{ + Volume: "mybucket", + Name: "myobject", + VersionID: "", + IsLatest: true, + Deleted: true, + ModTime: time.Date(2023, time.March, 15, 19, 57, 30, 492530160, time.UTC), + Mode: 0x0, + WrittenByVersion: 0x63c77756, + NumVersions: 1, + Idx: 0, + } + + tests := []struct { + fi1, fi2 FileInfo + }{ + { + fi1: fi1, + fi2: fi2, + }, + { + fi1: fi1, + fi2: fiDel, + }, + } + for idx, test := range tests { + var metaArr []FileInfo + for i := range 12 { + fi := test.fi1 + if i%2 == 0 { + fi = test.fi2 + } + metaArr = append(metaArr, fi) + } + + parities := listObjectParities(metaArr, make([]error, len(metaArr))) + parity := commonParity(parities, 5) + var match int + for _, fi := range metaArr { + if fi.Erasure.ParityBlocks == parity { + match++ + } + } + if match < len(metaArr)-parity { + t.Fatalf("Test %d: Expected %d drives with parity=%d, but got %d", idx, len(metaArr)-parity, parity, match) + } + } +} diff --git a/cmd/erasure-healing.go b/cmd/erasure-healing.go new file mode 100644 index 00000000..9ea507f7 --- /dev/null +++ b/cmd/erasure-healing.go @@ -0,0 +1,1137 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "strconv" + "strings" + "sync" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/grid" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/sync/errgroup" + "github.com/puzpuzpuz/xsync/v3" +) + +//go:generate stringer -type=healingMetric -trimprefix=healingMetric $GOFILE + +type healingMetric uint8 + +const ( + healingMetricBucket healingMetric = iota + healingMetricObject + healingMetricCheckAbandonedParts +) + +// List a prefix or a single object versions and heal +func (er erasureObjects) listAndHeal(ctx context.Context, bucket, prefix string, recursive bool, scanMode madmin.HealScanMode, healEntry func(string, metaCacheEntry, madmin.HealScanMode) error) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + disks, _ := er.getOnlineDisksWithHealing(false) + if len(disks) == 0 { + return errors.New("listAndHeal: No non-healing drives found") + } + + // How to resolve partial results. + resolver := metadataResolutionParams{ + dirQuorum: 1, + objQuorum: 1, + bucket: bucket, + strict: false, // Allow less strict matching. + } + + path := baseDirFromPrefix(prefix) + filterPrefix := strings.Trim(strings.TrimPrefix(prefix, path), slashSeparator) + if path == prefix { + filterPrefix = "" + } + + lopts := listPathRawOptions{ + disks: disks, + bucket: bucket, + path: path, + filterPrefix: filterPrefix, + recursive: recursive, + forwardTo: "", + minDisks: 1, + reportNotFound: false, + agreed: func(entry metaCacheEntry) { + if !recursive && prefix != entry.name { + return + } + if err := healEntry(bucket, entry, scanMode); err != nil { + cancel() + } + }, + partial: func(entries metaCacheEntries, _ []error) { + entry, ok := entries.resolve(&resolver) + if !ok { + // check if we can get one entry at least + // proceed to heal nonetheless. + entry, _ = entries.firstFound() + } + if !recursive && prefix != entry.name { + return + } + if err := healEntry(bucket, *entry, scanMode); err != nil { + cancel() + return + } + }, + finished: nil, + } + + if err := listPathRaw(ctx, lopts); err != nil { + return fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts) + } + + return nil +} + +// listAllBuckets lists all buckets from all disks. It also +// returns the occurrence of each buckets in all disks +func listAllBuckets(ctx context.Context, storageDisks []StorageAPI, healBuckets *xsync.MapOf[string, VolInfo], readQuorum int) error { + g := errgroup.WithNErrs(len(storageDisks)) + for index := range storageDisks { + g.Go(func() error { + if storageDisks[index] == nil { + // we ignore disk not found errors + return nil + } + volsInfo, err := storageDisks[index].ListVols(ctx) + if err != nil { + return err + } + + for _, volInfo := range volsInfo { + // StorageAPI can send volume names which are + // incompatible with buckets - these are + // skipped, like the meta-bucket. + if isReservedOrInvalidBucket(volInfo.Name, false) { + continue + } + + healBuckets.Compute(volInfo.Name, func(oldValue VolInfo, loaded bool) (newValue VolInfo, del bool) { + if loaded { + newValue = oldValue + newValue.count = oldValue.count + 1 + return newValue, false + } + return VolInfo{ + Name: volInfo.Name, + Created: volInfo.Created, + count: 1, + }, false + }) + } + + return nil + }, index) + } + + if err := reduceReadQuorumErrs(ctx, g.Wait(), bucketMetadataOpIgnoredErrs, readQuorum); err != nil { + return err + } + + healBuckets.Range(func(volName string, volInfo VolInfo) bool { + if volInfo.count < readQuorum { + healBuckets.Delete(volName) + } + return true + }) + + return nil +} + +var ( + errLegacyXLMeta = errors.New("legacy XL meta") + errOutdatedXLMeta = errors.New("outdated XL meta") + errPartCorrupt = errors.New("part corrupt") + errPartMissing = errors.New("part missing") +) + +// Only heal on disks where we are sure that healing is needed. We can expand +// this list as and when we figure out more errors can be added to this list safely. +func shouldHealObjectOnDisk(erErr error, partsErrs []int, meta FileInfo, latestMeta FileInfo) (bool, bool, error) { + if errors.Is(erErr, errFileNotFound) || errors.Is(erErr, errFileVersionNotFound) || errors.Is(erErr, errFileCorrupt) { + return true, true, erErr + } + if erErr == nil { + if meta.XLV1 { + // Legacy means heal always + // always check first. + return true, true, errLegacyXLMeta + } + if !latestMeta.Equals(meta) { + return true, true, errOutdatedXLMeta + } + if !meta.Deleted && !meta.IsRemote() { + // If xl.meta was read fine but there may be problem with the part.N files. + for _, partErr := range partsErrs { + if partErr == checkPartFileNotFound { + return true, false, errPartMissing + } + if partErr == checkPartFileCorrupt { + return true, false, errPartCorrupt + } + } + } + return false, false, nil + } + return false, false, erErr +} + +const ( + xMinIOHealing = ReservedMetadataPrefix + "healing" + xMinIODataMov = ReservedMetadataPrefix + "data-mov" +) + +// SetHealing marks object (version) as being healed. +// Note: this is to be used only from healObject +func (fi *FileInfo) SetHealing() { + if fi.Metadata == nil { + fi.Metadata = make(map[string]string) + } + fi.Metadata[xMinIOHealing] = "true" +} + +// Healing returns true if object is being healed (i.e fi is being passed down +// from healObject) +func (fi FileInfo) Healing() bool { + _, ok := fi.Metadata[xMinIOHealing] + return ok +} + +// SetDataMov marks object (version) as being currently +// in movement, such as decommissioning or rebalance. +func (fi *FileInfo) SetDataMov() { + if fi.Metadata == nil { + fi.Metadata = make(map[string]string) + } + fi.Metadata[xMinIODataMov] = "true" +} + +// DataMov returns true if object is being in movement +func (fi FileInfo) DataMov() bool { + _, ok := fi.Metadata[xMinIODataMov] + return ok +} + +func (er *erasureObjects) auditHealObject(ctx context.Context, bucket, object, versionID string, result madmin.HealResultItem, err error) { + if len(logger.AuditTargets()) == 0 { + return + } + + opts := AuditLogOptions{ + Event: "HealObject", + Bucket: bucket, + Object: decodeDirObject(object), + VersionID: versionID, + } + if err != nil { + opts.Error = err.Error() + } + + b, a := result.GetCorruptedCounts() + if b > 0 && b == a { + opts.Error = fmt.Sprintf("unable to heal %d corrupted blocks on drives", b) + } + + b, a = result.GetMissingCounts() + if b > 0 && b == a { + opts.Error = fmt.Sprintf("unable to heal %d missing blocks on drives", b) + } + + opts.Tags = map[string]string{ + "healObject": auditObjectOp{ + Name: opts.Object, + Pool: er.poolIndex + 1, + Set: er.setIndex + 1, + }.String(), + } + + auditLogInternal(ctx, opts) +} + +func objectErrToDriveState(reason error) string { + switch { + case reason == nil: + return madmin.DriveStateOk + case IsErr(reason, errDiskNotFound): + return madmin.DriveStateOffline + case IsErr(reason, errFileNotFound, errFileVersionNotFound, errVolumeNotFound, errPartMissing, errOutdatedXLMeta, errLegacyXLMeta): + return madmin.DriveStateMissing + case IsErr(reason, errFileCorrupt, errPartCorrupt): + return madmin.DriveStateCorrupt + default: + return fmt.Sprintf("%s (%s)", madmin.DriveStateUnknown, reason.Error()) + } +} + +// Heals an object by re-writing corrupt/missing erasure blocks. +func (er *erasureObjects) healObject(ctx context.Context, bucket string, object string, versionID string, opts madmin.HealOpts) (result madmin.HealResultItem, err error) { + dryRun := opts.DryRun + scanMode := opts.ScanMode + + storageDisks := er.getDisks() + storageEndpoints := er.getEndpoints() + + defer func() { + er.auditHealObject(ctx, bucket, object, versionID, result, err) + }() + + if globalTrace.NumSubscribers(madmin.TraceHealing) > 0 { + startTime := time.Now() + defer func() { + healTrace(healingMetricObject, startTime, bucket, object, &opts, err, &result) + }() + } + + // Initialize heal result object + result = madmin.HealResultItem{ + Type: madmin.HealItemObject, + Bucket: bucket, + Object: object, + VersionID: versionID, + DiskCount: len(storageDisks), + } + + if !opts.NoLock { + lk := er.NewNSLock(bucket, object) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + return result, err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + } + + // Re-read when we have lock... + partsMetadata, errs := readAllFileInfo(ctx, storageDisks, "", bucket, object, versionID, true, true) + if isAllNotFound(errs) { + err := errFileNotFound + if versionID != "" { + err = errFileVersionNotFound + } + // Nothing to do, file is already gone. + return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints, + errs, bucket, object, versionID), err + } + + readQuorum, _, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount) + if err != nil { + m, derr := er.deleteIfDangling(ctx, bucket, object, partsMetadata, errs, nil, ObjectOptions{ + VersionID: versionID, + }) + errs = make([]error, len(errs)) + if derr == nil { + derr = errFileNotFound + if versionID != "" { + derr = errFileVersionNotFound + } + // We did find a new danging object + return er.defaultHealResult(m, storageDisks, storageEndpoints, + errs, bucket, object, versionID), derr + } + return er.defaultHealResult(m, storageDisks, storageEndpoints, + errs, bucket, object, versionID), err + } + + result.ParityBlocks = result.DiskCount - readQuorum + result.DataBlocks = readQuorum + + // List of disks having latest version of the object xl.meta + // (by modtime). + onlineDisks, quorumModTime, quorumETag := listOnlineDisks(storageDisks, partsMetadata, errs, readQuorum) + + // Latest FileInfo for reference. If a valid metadata is not + // present, it is as good as object not found. + latestMeta, err := pickValidFileInfo(ctx, partsMetadata, quorumModTime, quorumETag, readQuorum) + if err != nil { + return result, err + } + + // No modtime quorum + filterDisksByETag := quorumETag != "" + + dataErrsByDisk, dataErrsByPart := checkObjectWithAllParts(ctx, onlineDisks, partsMetadata, + errs, latestMeta, filterDisksByETag, bucket, object, scanMode) + + var erasure Erasure + if !latestMeta.Deleted && !latestMeta.IsRemote() { + // Initialize erasure coding + erasure, err = NewErasure(ctx, latestMeta.Erasure.DataBlocks, + latestMeta.Erasure.ParityBlocks, latestMeta.Erasure.BlockSize) + if err != nil { + return result, err + } + } + + result.ObjectSize, err = latestMeta.ToObjectInfo(bucket, object, true).GetActualSize() + if err != nil { + return result, err + } + + // Loop to find number of disks with valid data, per-drive + // data state and a list of outdated disks on which data needs + // to be healed. + outDatedDisks := make([]StorageAPI, len(storageDisks)) + disksToHealCount, xlMetaToHealCount := 0, 0 + for i := range onlineDisks { + yes, isMeta, reason := shouldHealObjectOnDisk(errs[i], dataErrsByDisk[i], partsMetadata[i], latestMeta) + if yes { + outDatedDisks[i] = storageDisks[i] + disksToHealCount++ + if isMeta { + xlMetaToHealCount++ + } + } + + driveState := objectErrToDriveState(reason) + + result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{ + UUID: "", + Endpoint: storageEndpoints[i].String(), + State: driveState, + }) + result.After.Drives = append(result.After.Drives, madmin.HealDriveInfo{ + UUID: "", + Endpoint: storageEndpoints[i].String(), + State: driveState, + }) + } + + if disksToHealCount == 0 { + // Nothing to heal! + return result, nil + } + + // After this point, only have to repair data on disk - so + // return if it is a dry-run + if dryRun { + return result, nil + } + + cannotHeal := !latestMeta.XLV1 && !latestMeta.Deleted && xlMetaToHealCount > latestMeta.Erasure.ParityBlocks + if cannotHeal && quorumETag != "" { + // This is an object that is supposed to be removed by the dangling code + // but we noticed that ETag is the same for all objects, let's give it a shot + cannotHeal = false + } + + if !latestMeta.Deleted && !latestMeta.IsRemote() { + // check if there is a part that lost its quorum + for _, partErrs := range dataErrsByPart { + if countPartNotSuccess(partErrs) > latestMeta.Erasure.ParityBlocks { + cannotHeal = true + break + } + } + } + + if cannotHeal { + // Allow for dangling deletes, on versions that have DataDir missing etc. + // this would end up restoring the correct readable versions. + m, err := er.deleteIfDangling(ctx, bucket, object, partsMetadata, errs, dataErrsByPart, ObjectOptions{ + VersionID: versionID, + }) + errs = make([]error, len(errs)) + if err == nil { + err = errFileNotFound + if versionID != "" { + err = errFileVersionNotFound + } + // We did find a new danging object + return er.defaultHealResult(m, storageDisks, storageEndpoints, + errs, bucket, object, versionID), err + } + for i := range errs { + errs[i] = err + } + return er.defaultHealResult(m, storageDisks, storageEndpoints, + errs, bucket, object, versionID), err + } + + cleanFileInfo := func(fi FileInfo) FileInfo { + // Returns a copy of the 'fi' with erasure index, checksums and inline data niled. + nfi := fi + if !nfi.IsRemote() { + nfi.Data = nil + nfi.Erasure.Index = 0 + nfi.Erasure.Checksums = nil + } + return nfi + } + + // We write at temporary location and then rename to final location. + tmpID := mustGetUUID() + migrateDataDir := mustGetUUID() + + if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(onlineDisks) { + err := fmt.Errorf("unexpected file distribution (%v) from online disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)", + latestMeta.Erasure.Distribution, onlineDisks, bucket, object, versionID) + healingLogOnceIf(ctx, err, "heal-object-online-disks") + return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs, + bucket, object, versionID), err + } + + latestDisks := shuffleDisks(onlineDisks, latestMeta.Erasure.Distribution) + + if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(outDatedDisks) { + err := fmt.Errorf("unexpected file distribution (%v) from outdated disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)", + latestMeta.Erasure.Distribution, outDatedDisks, bucket, object, versionID) + healingLogOnceIf(ctx, err, "heal-object-outdated-disks") + return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs, + bucket, object, versionID), err + } + + outDatedDisks = shuffleDisks(outDatedDisks, latestMeta.Erasure.Distribution) + + if !latestMeta.Deleted && len(latestMeta.Erasure.Distribution) != len(partsMetadata) { + err := fmt.Errorf("unexpected file distribution (%v) from metadata entries (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)", + latestMeta.Erasure.Distribution, len(partsMetadata), bucket, object, versionID) + healingLogOnceIf(ctx, err, "heal-object-metadata-entries") + return er.defaultHealResult(latestMeta, storageDisks, storageEndpoints, errs, + bucket, object, versionID), err + } + + partsMetadata = shufflePartsMetadata(partsMetadata, latestMeta.Erasure.Distribution) + + copyPartsMetadata := make([]FileInfo, len(partsMetadata)) + for i := range latestDisks { + if latestDisks[i] == nil { + continue + } + copyPartsMetadata[i] = partsMetadata[i] + } + + for i := range outDatedDisks { + if outDatedDisks[i] == nil { + continue + } + // Make sure to write the FileInfo information + // that is expected to be in quorum. + partsMetadata[i] = cleanFileInfo(latestMeta) + } + + // source data dir shall be empty in case of XLV1 + // differentiate it with dstDataDir for readability + // srcDataDir is the one used with newBitrotReader() + // to read existing content. + srcDataDir := latestMeta.DataDir + dstDataDir := latestMeta.DataDir + if latestMeta.XLV1 { + dstDataDir = migrateDataDir + } + + var inlineBuffers []*bytes.Buffer + if !latestMeta.Deleted && !latestMeta.IsRemote() { + if latestMeta.InlineData() { + inlineBuffers = make([]*bytes.Buffer, len(outDatedDisks)) + } + + erasureInfo := latestMeta.Erasure + for partIndex := 0; partIndex < len(latestMeta.Parts); partIndex++ { + partSize := latestMeta.Parts[partIndex].Size + partActualSize := latestMeta.Parts[partIndex].ActualSize + partModTime := latestMeta.Parts[partIndex].ModTime + partNumber := latestMeta.Parts[partIndex].Number + partIdx := latestMeta.Parts[partIndex].Index + partChecksums := latestMeta.Parts[partIndex].Checksums + tillOffset := erasure.ShardFileOffset(0, partSize, partSize) + readers := make([]io.ReaderAt, len(latestDisks)) + prefer := make([]bool, len(latestDisks)) + checksumAlgo := erasureInfo.GetChecksumInfo(partNumber).Algorithm + for i, disk := range latestDisks { + if disk == OfflineDisk { + continue + } + thisPartErrs := shuffleCheckParts(dataErrsByPart[partIndex], latestMeta.Erasure.Distribution) + if thisPartErrs[i] != checkPartSuccess { + continue + } + checksumInfo := copyPartsMetadata[i].Erasure.GetChecksumInfo(partNumber) + partPath := pathJoin(object, srcDataDir, fmt.Sprintf("part.%d", partNumber)) + readers[i] = newBitrotReader(disk, copyPartsMetadata[i].Data, bucket, partPath, tillOffset, checksumAlgo, + checksumInfo.Hash, erasure.ShardSize()) + prefer[i] = disk.Hostname() == "" + } + writers := make([]io.Writer, len(outDatedDisks)) + for i, disk := range outDatedDisks { + if disk == OfflineDisk { + continue + } + partPath := pathJoin(tmpID, dstDataDir, fmt.Sprintf("part.%d", partNumber)) + if len(inlineBuffers) > 0 { + buf := grid.GetByteBufferCap(int(erasure.ShardFileSize(latestMeta.Size)) + 64) + inlineBuffers[i] = bytes.NewBuffer(buf[:0]) + defer grid.PutByteBuffer(buf) + + writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize()) + } else { + writers[i] = newBitrotWriter(disk, bucket, minioMetaTmpBucket, partPath, + tillOffset, DefaultBitrotAlgorithm, erasure.ShardSize()) + } + } + + // Heal each part. erasure.Heal() will write the healed + // part to .minio/tmp/uuid/ which needs to be renamed + // later to the final location. + err = erasure.Heal(ctx, writers, readers, partSize, prefer) + closeBitrotReaders(readers) + closeErrs := closeBitrotWriters(writers) + if err != nil { + return result, err + } + + // outDatedDisks that had write errors should not be + // written to for remaining parts, so we nil it out. + for i, disk := range outDatedDisks { + if disk == OfflineDisk { + continue + } + + // A non-nil stale disk which did not receive + // a healed part checksum had a write error. + if writers[i] == nil { + outDatedDisks[i] = nil + disksToHealCount-- + continue + } + + // A non-nil stale disk which got error on Close() + if closeErrs[i] != nil { + outDatedDisks[i] = nil + disksToHealCount-- + continue + } + + partsMetadata[i].DataDir = dstDataDir + partsMetadata[i].AddObjectPart(partNumber, "", partSize, partActualSize, partModTime, partIdx, partChecksums) + if len(inlineBuffers) > 0 && inlineBuffers[i] != nil { + partsMetadata[i].Data = inlineBuffers[i].Bytes() + partsMetadata[i].SetInlineData() + } else { + partsMetadata[i].Data = nil + } + } + + // If all disks are having errors, we give up. + if disksToHealCount == 0 { + return result, fmt.Errorf("all drives had write errors, unable to heal %s/%s", bucket, object) + } + } + } + + defer er.deleteAll(context.Background(), minioMetaTmpBucket, tmpID) + + // Rename from tmp location to the actual location. + for i, disk := range outDatedDisks { + if disk == OfflineDisk { + continue + } + + // record the index of the updated disks + partsMetadata[i].Erasure.Index = i + 1 + + // Attempt a rename now from healed data to final location. + partsMetadata[i].SetHealing() + + if _, err = disk.RenameData(ctx, minioMetaTmpBucket, tmpID, partsMetadata[i], bucket, object, RenameOptions{}); err != nil { + return result, err + } + + // - Remove any remaining parts from outdated disks from before transition. + if partsMetadata[i].IsRemote() { + rmDataDir := partsMetadata[i].DataDir + disk.Delete(ctx, bucket, pathJoin(encodeDirObject(object), rmDataDir), DeleteOptions{ + Immediate: true, + Recursive: true, + }) + } + + for i, v := range result.Before.Drives { + if v.Endpoint == disk.Endpoint().String() { + result.After.Drives[i].State = madmin.DriveStateOk + } + } + } + + return result, nil +} + +// checkAbandonedParts will check if an object has abandoned parts, +// meaning data-dirs or inlined data that are no longer referenced by the xl.meta +// Errors are generally ignored by this function. +func (er *erasureObjects) checkAbandonedParts(ctx context.Context, bucket string, object string, opts madmin.HealOpts) (err error) { + if !opts.Remove || opts.DryRun { + return nil + } + + if globalTrace.NumSubscribers(madmin.TraceHealing) > 0 { + startTime := time.Now() + defer func() { + healTrace(healingMetricCheckAbandonedParts, startTime, bucket, object, nil, err, nil) + }() + } + if !opts.NoLock { + lk := er.NewNSLock(bucket, object) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + return err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + } + var wg sync.WaitGroup + for _, disk := range er.getDisks() { + if disk != nil { + wg.Add(1) + go func(disk StorageAPI) { + defer wg.Done() + _ = disk.CleanAbandonedData(ctx, bucket, object) + }(disk) + } + } + wg.Wait() + return nil +} + +// healObjectDir - heals object directory specifically, this special call +// is needed since we do not have a special backend format for directories. +func (er *erasureObjects) healObjectDir(ctx context.Context, bucket, object string, dryRun bool, remove bool) (hr madmin.HealResultItem, err error) { + storageDisks := er.getDisks() + storageEndpoints := er.getEndpoints() + + // Initialize heal result object + hr = madmin.HealResultItem{ + Type: madmin.HealItemObject, + Bucket: bucket, + Object: object, + DiskCount: len(storageDisks), + ParityBlocks: er.defaultParityCount, + DataBlocks: len(storageDisks) - er.defaultParityCount, + ObjectSize: 0, + } + + hr.Before.Drives = make([]madmin.HealDriveInfo, len(storageDisks)) + hr.After.Drives = make([]madmin.HealDriveInfo, len(storageDisks)) + + errs := statAllDirs(ctx, storageDisks, bucket, object) + danglingObject := isObjectDirDangling(errs) + if danglingObject { + if !dryRun && remove { + var wg sync.WaitGroup + // Remove versions in bulk for each disk + for index, disk := range storageDisks { + if disk == nil { + continue + } + wg.Add(1) + go func(index int, disk StorageAPI) { + defer wg.Done() + _ = disk.Delete(ctx, bucket, object, DeleteOptions{ + Recursive: false, + Immediate: false, + }) + }(index, disk) + } + wg.Wait() + } + } + + // Prepare object creation in all disks + for i, err := range errs { + drive := storageEndpoints[i].String() + switch err { + case nil: + hr.Before.Drives[i] = madmin.HealDriveInfo{Endpoint: drive, State: madmin.DriveStateOk} + hr.After.Drives[i] = madmin.HealDriveInfo{Endpoint: drive, State: madmin.DriveStateOk} + case errDiskNotFound: + hr.Before.Drives[i] = madmin.HealDriveInfo{State: madmin.DriveStateOffline} + hr.After.Drives[i] = madmin.HealDriveInfo{State: madmin.DriveStateOffline} + case errVolumeNotFound, errFileNotFound: + // Bucket or prefix/directory not found + hr.Before.Drives[i] = madmin.HealDriveInfo{Endpoint: drive, State: madmin.DriveStateMissing} + hr.After.Drives[i] = madmin.HealDriveInfo{Endpoint: drive, State: madmin.DriveStateMissing} + default: + hr.Before.Drives[i] = madmin.HealDriveInfo{Endpoint: drive, State: madmin.DriveStateCorrupt} + hr.After.Drives[i] = madmin.HealDriveInfo{Endpoint: drive, State: madmin.DriveStateCorrupt} + } + } + if danglingObject || isAllNotFound(errs) { + // Nothing to do, file is already gone. + return hr, errFileNotFound + } + + if dryRun { + // Quit without try to heal the object dir + return hr, nil + } + + for i, err := range errs { + if err == errVolumeNotFound || err == errFileNotFound { + // Bucket or prefix/directory not found + merr := storageDisks[i].MakeVol(ctx, pathJoin(bucket, object)) + switch merr { + case nil, errVolumeExists: + hr.After.Drives[i].State = madmin.DriveStateOk + case errDiskNotFound: + hr.After.Drives[i].State = madmin.DriveStateOffline + default: + hr.After.Drives[i].State = madmin.DriveStateCorrupt + } + } + } + return hr, nil +} + +// Populates default heal result item entries with possible values when we are returning prematurely. +// This is to ensure that in any circumstance we are not returning empty arrays with wrong values. +func (er *erasureObjects) defaultHealResult(lfi FileInfo, storageDisks []StorageAPI, storageEndpoints []Endpoint, errs []error, bucket, object, versionID string) madmin.HealResultItem { + // Initialize heal result object + result := madmin.HealResultItem{ + Type: madmin.HealItemObject, + Bucket: bucket, + Object: object, + ObjectSize: lfi.Size, + VersionID: versionID, + DiskCount: len(storageDisks), + } + + if lfi.IsValid() { + result.ParityBlocks = lfi.Erasure.ParityBlocks + } else { + // Default to most common configuration for erasure blocks. + result.ParityBlocks = er.defaultParityCount + } + result.DataBlocks = len(storageDisks) - result.ParityBlocks + + for index, disk := range storageDisks { + if disk == nil { + result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{ + UUID: "", + Endpoint: storageEndpoints[index].String(), + State: madmin.DriveStateOffline, + }) + result.After.Drives = append(result.After.Drives, madmin.HealDriveInfo{ + UUID: "", + Endpoint: storageEndpoints[index].String(), + State: madmin.DriveStateOffline, + }) + continue + } + driveState := objectErrToDriveState(errs[index]) + result.Before.Drives = append(result.Before.Drives, madmin.HealDriveInfo{ + UUID: "", + Endpoint: storageEndpoints[index].String(), + State: driveState, + }) + result.After.Drives = append(result.After.Drives, madmin.HealDriveInfo{ + UUID: "", + Endpoint: storageEndpoints[index].String(), + State: driveState, + }) + } + + return result +} + +// Stat all directories. +func statAllDirs(ctx context.Context, storageDisks []StorageAPI, bucket, prefix string) []error { + g := errgroup.WithNErrs(len(storageDisks)) + for index, disk := range storageDisks { + if disk == nil { + continue + } + index := index + g.Go(func() error { + entries, err := storageDisks[index].ListDir(ctx, "", bucket, prefix, 1) + if err != nil { + return err + } + if len(entries) > 0 { + return errVolumeNotEmpty + } + return nil + }, index) + } + + return g.Wait() +} + +func isAllVolumeNotFound(errs []error) bool { + return countErrs(errs, errVolumeNotFound) == len(errs) +} + +// isAllNotFound will return if any element of the error slice is not +// errFileNotFound, errFileVersionNotFound or errVolumeNotFound. +// A 0 length slice will always return false. +func isAllNotFound(errs []error) bool { + for _, err := range errs { + if err != nil { + switch err.Error() { + case errFileNotFound.Error(): + fallthrough + case errVolumeNotFound.Error(): + fallthrough + case errFileVersionNotFound.Error(): + continue + } + } + return false + } + return len(errs) > 0 +} + +// isAllBucketsNotFound will return true if all the errors are either errFileNotFound +// or errFileCorrupt +// A 0 length slice will always return false. +func isAllBucketsNotFound(errs []error) bool { + if len(errs) == 0 { + return false + } + notFoundCount := 0 + for _, err := range errs { + if err != nil { + if errors.Is(err, errVolumeNotFound) { + notFoundCount++ + } else if isErrBucketNotFound(err) { + notFoundCount++ + } + } + } + return len(errs) == notFoundCount +} + +// ObjectDir is considered dangling/corrupted if any only +// if total disks - a combination of corrupted and missing +// files is lesser than N/2+1 number of disks. +// If no files were found false will be returned. +func isObjectDirDangling(errs []error) (ok bool) { + var found int + var notFound int + var foundNotEmpty int + var otherFound int + for _, readErr := range errs { + switch readErr { + case nil: + found++ + case errFileNotFound, errVolumeNotFound: + notFound++ + case errVolumeNotEmpty: + foundNotEmpty++ + default: + otherFound++ + } + } + found = found + foundNotEmpty + otherFound + return found < notFound && found > 0 +} + +func danglingMetaErrsCount(cerrs []error) (notFoundCount int, nonActionableCount int) { + for _, readErr := range cerrs { + if readErr == nil { + continue + } + switch { + case errors.Is(readErr, errFileNotFound) || errors.Is(readErr, errFileVersionNotFound): + notFoundCount++ + default: + // All other errors are non-actionable + nonActionableCount++ + } + } + return notFoundCount, nonActionableCount +} + +func danglingPartErrsCount(results []int) (notFoundCount int, nonActionableCount int) { + for _, partResult := range results { + switch partResult { + case checkPartSuccess: + continue + case checkPartFileNotFound: + notFoundCount++ + default: + // All other errors are non-actionable + nonActionableCount++ + } + } + return notFoundCount, nonActionableCount +} + +// Object is considered dangling/corrupted if and only +// if total disks - a combination of corrupted and missing +// files is lesser than number of data blocks. +func isObjectDangling(metaArr []FileInfo, errs []error, dataErrsByPart map[int][]int) (validMeta FileInfo, ok bool) { + // We can consider an object data not reliable + // when xl.meta is not found in read quorum disks. + // or when xl.meta is not readable in read quorum disks. + notFoundMetaErrs, nonActionableMetaErrs := danglingMetaErrsCount(errs) + + notFoundPartsErrs, nonActionablePartsErrs := 0, 0 + for _, dataErrs := range dataErrsByPart { + if nf, na := danglingPartErrsCount(dataErrs); nf > notFoundPartsErrs { + notFoundPartsErrs, nonActionablePartsErrs = nf, na + } + } + + for _, m := range metaArr { + if m.IsValid() { + validMeta = m + break + } + } + + if !validMeta.IsValid() { + // validMeta is invalid because all xl.meta is missing apparently + // we should figure out if dataDirs are also missing > dataBlocks. + dataBlocks := (len(metaArr) + 1) / 2 + if notFoundPartsErrs > dataBlocks { + // Not using parity to ensure that we do not delete + // any valid content, if any is recoverable. But if + // notFoundDataDirs are already greater than the data + // blocks all bets are off and it is safe to purge. + // + // This is purely a defensive code, ideally parityBlocks + // is sufficient, however we can't know that since we + // do have the FileInfo{}. + return validMeta, true + } + + // We have no idea what this file is, leave it as is. + return validMeta, false + } + + if nonActionableMetaErrs > 0 || nonActionablePartsErrs > 0 { + return validMeta, false + } + + if validMeta.Deleted { + // notFoundPartsErrs is ignored since + // - delete marker does not have any parts + dataBlocks := (len(errs) + 1) / 2 + return validMeta, notFoundMetaErrs > dataBlocks + } + + // TODO: It is possible to replay the object via just single + // xl.meta file, considering quorum number of data-dirs are still + // present on other drives. + // + // However this requires a bit of a rewrite, leave this up for + // future work. + if notFoundMetaErrs > 0 && notFoundMetaErrs > validMeta.Erasure.ParityBlocks { + // All xl.meta is beyond parity blocks missing, this is dangling + return validMeta, true + } + + if !validMeta.IsRemote() && notFoundPartsErrs > 0 && notFoundPartsErrs > validMeta.Erasure.ParityBlocks { + // All data-dir is beyond parity blocks missing, this is dangling + return validMeta, true + } + + return validMeta, false +} + +// HealObject - heal the given object, automatically deletes the object if stale/corrupted if `remove` is true. +func (er erasureObjects) HealObject(ctx context.Context, bucket, object, versionID string, opts madmin.HealOpts) (hr madmin.HealResultItem, err error) { + // Create context that also contains information about the object and bucket. + // The top level handler might not have this information. + reqInfo := logger.GetReqInfo(ctx) + var newReqInfo *logger.ReqInfo + if reqInfo != nil { + newReqInfo = logger.NewReqInfo(reqInfo.RemoteHost, reqInfo.UserAgent, reqInfo.DeploymentID, reqInfo.RequestID, reqInfo.API, bucket, object) + } else { + newReqInfo = logger.NewReqInfo("", "", globalDeploymentID(), "", "Heal", bucket, object) + } + healCtx := logger.SetReqInfo(GlobalContext, newReqInfo) + + // Healing directories handle it separately. + if HasSuffix(object, SlashSeparator) { + hr, err := er.healObjectDir(healCtx, bucket, object, opts.DryRun, opts.Remove) + return hr, toObjectErr(err, bucket, object) + } + + storageDisks := er.getDisks() + storageEndpoints := er.getEndpoints() + + // When versionID is empty, we read directly from the `null` versionID for healing. + if versionID == "" { + versionID = nullVersionID + } + + // Perform quick read without lock. + // This allows to quickly check if all is ok or all are missing. + _, errs := readAllFileInfo(healCtx, storageDisks, "", bucket, object, versionID, false, false) + if isAllNotFound(errs) { + err := errFileNotFound + if versionID != "" { + err = errFileVersionNotFound + } + // Nothing to do, file is already gone. + return er.defaultHealResult(FileInfo{}, storageDisks, storageEndpoints, + errs, bucket, object, versionID), toObjectErr(err, bucket, object, versionID) + } + + // Heal the object. + hr, err = er.healObject(healCtx, bucket, object, versionID, opts) + if errors.Is(err, errFileCorrupt) && opts.ScanMode != madmin.HealDeepScan { + // Instead of returning an error when a bitrot error is detected + // during a normal heal scan, heal again with bitrot flag enabled. + opts.ScanMode = madmin.HealDeepScan + hr, err = er.healObject(healCtx, bucket, object, versionID, opts) + } + return hr, toObjectErr(err, bucket, object, versionID) +} + +// healTrace sends healing results to trace output. +func healTrace(funcName healingMetric, startTime time.Time, bucket, object string, opts *madmin.HealOpts, err error, result *madmin.HealResultItem) { + tr := madmin.TraceInfo{ + TraceType: madmin.TraceHealing, + Time: startTime, + NodeName: globalLocalNodeName, + FuncName: "heal." + funcName.String(), + Duration: time.Since(startTime), + Path: pathJoin(bucket, decodeDirObject(object)), + } + if opts != nil { + tr.Custom = map[string]string{ + "dry": fmt.Sprint(opts.DryRun), + "remove": fmt.Sprint(opts.Remove), + "mode": fmt.Sprint(opts.ScanMode), + } + if result != nil { + tr.Custom["version-id"] = result.VersionID + tr.Custom["disks"] = strconv.Itoa(result.DiskCount) + tr.Bytes = result.ObjectSize + } + } + if err != nil { + tr.Error = err.Error() + } + tr.HealResult = result + globalTrace.Publish(tr) +} diff --git a/cmd/erasure-healing_test.go b/cmd/erasure-healing_test.go new file mode 100644 index 00000000..c19fddd0 --- /dev/null +++ b/cmd/erasure-healing_test.go @@ -0,0 +1,1769 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/rand" + "crypto/sha256" + "errors" + "io" + "os" + "path" + "reflect" + "testing" + "time" + + "github.com/dustin/go-humanize" + uuid2 "github.com/google/uuid" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/config/storageclass" +) + +// Tests isObjectDangling function +func TestIsObjectDangling(t *testing.T) { + fi := newFileInfo("test-object", 2, 2) + fi.Erasure.Index = 1 + + ifi := newFileInfo("test-object", 2, 2) + ifi.SetInlineData() + ifi.Erasure.Index = 1 + + testCases := []struct { + name string + metaArr []FileInfo + errs []error + dataErrs map[int][]int + expectedMeta FileInfo + expectedDangling bool + }{ + { + name: "FileInfoExists-case1", + metaArr: []FileInfo{ + {}, + {}, + fi, + fi, + }, + errs: []error{ + errFileNotFound, + errDiskNotFound, + nil, + nil, + }, + dataErrs: nil, + expectedMeta: fi, + expectedDangling: false, + }, + { + name: "FileInfoExists-case2", + metaArr: []FileInfo{ + {}, + {}, + fi, + fi, + }, + errs: []error{ + errFileNotFound, + errFileNotFound, + nil, + nil, + }, + dataErrs: nil, + expectedMeta: fi, + expectedDangling: false, + }, + { + name: "FileInfoUndecided-case1", + metaArr: []FileInfo{ + {}, + {}, + {}, + fi, + }, + errs: []error{ + errFileNotFound, + errDiskNotFound, + errDiskNotFound, + nil, + }, + dataErrs: nil, + expectedMeta: fi, + expectedDangling: false, + }, + { + name: "FileInfoUndecided-case2", + metaArr: []FileInfo{}, + errs: []error{ + errFileNotFound, + errDiskNotFound, + errDiskNotFound, + errFileNotFound, + }, + dataErrs: nil, + expectedMeta: FileInfo{}, + expectedDangling: false, + }, + { + name: "FileInfoUndecided-case3(file deleted)", + metaArr: []FileInfo{}, + errs: []error{ + errFileNotFound, + errFileNotFound, + errFileNotFound, + errFileNotFound, + }, + dataErrs: nil, + expectedMeta: FileInfo{}, + expectedDangling: false, + }, + { + name: "FileInfoUnDecided-case4", + metaArr: []FileInfo{ + {}, + {}, + {}, + ifi, + }, + errs: []error{ + errFileNotFound, + errFileCorrupt, + errFileCorrupt, + nil, + }, + dataErrs: nil, + expectedMeta: ifi, + expectedDangling: false, + }, + { + name: "FileInfoUnDecided-case5-(ignore errFileCorrupt error)", + metaArr: []FileInfo{ + {}, + {}, + {}, + fi, + }, + errs: []error{ + errFileNotFound, + errFileCorrupt, + nil, + nil, + }, + dataErrs: map[int][]int{ + 0: {checkPartFileCorrupt, checkPartFileNotFound, checkPartSuccess, checkPartFileCorrupt}, + }, + expectedMeta: fi, + expectedDangling: false, + }, + { + name: "FileInfoUnDecided-case6-(data-dir intact)", + metaArr: []FileInfo{ + {}, + {}, + {}, + fi, + }, + errs: []error{ + errFileNotFound, + errFileNotFound, + errFileNotFound, + nil, + }, + dataErrs: map[int][]int{ + 0: {checkPartFileNotFound, checkPartFileCorrupt, checkPartSuccess, checkPartSuccess}, + }, + expectedMeta: fi, + expectedDangling: false, + }, + { + name: "FileInfoDecided-case1", + metaArr: []FileInfo{ + {}, + {}, + {}, + ifi, + }, + errs: []error{ + errFileNotFound, + errFileNotFound, + errFileNotFound, + nil, + }, + dataErrs: nil, + expectedMeta: ifi, + expectedDangling: true, + }, + { + name: "FileInfoDecided-case2-delete-marker", + metaArr: []FileInfo{ + {}, + {}, + {}, + {Deleted: true}, + }, + errs: []error{ + errFileNotFound, + errFileNotFound, + errFileNotFound, + nil, + }, + dataErrs: nil, + expectedMeta: FileInfo{Deleted: true}, + expectedDangling: true, + }, + { + name: "FileInfoDecided-case3-(enough data-dir missing)", + metaArr: []FileInfo{ + {}, + {}, + {}, + fi, + }, + errs: []error{ + errFileNotFound, + errFileNotFound, + nil, + nil, + }, + dataErrs: map[int][]int{ + 0: {checkPartFileNotFound, checkPartFileNotFound, checkPartSuccess, checkPartFileNotFound}, + }, + expectedMeta: fi, + expectedDangling: true, + }, + { + name: "FileInfoDecided-case4-(missing data-dir for part 2)", + metaArr: []FileInfo{ + {}, + {}, + {}, + fi, + }, + errs: []error{ + errFileNotFound, + errFileNotFound, + nil, + nil, + }, + dataErrs: map[int][]int{ + 0: {checkPartSuccess, checkPartSuccess, checkPartSuccess, checkPartSuccess}, + 1: {checkPartSuccess, checkPartFileNotFound, checkPartFileNotFound, checkPartFileNotFound}, + }, + expectedMeta: fi, + expectedDangling: true, + }, + + { + name: "FileInfoDecided-case4-(enough data-dir existing for each part)", + metaArr: []FileInfo{ + {}, + {}, + {}, + fi, + }, + errs: []error{ + errFileNotFound, + errFileNotFound, + nil, + nil, + }, + dataErrs: map[int][]int{ + 0: {checkPartFileNotFound, checkPartSuccess, checkPartSuccess, checkPartSuccess}, + 1: {checkPartSuccess, checkPartFileNotFound, checkPartSuccess, checkPartSuccess}, + 2: {checkPartSuccess, checkPartSuccess, checkPartFileNotFound, checkPartSuccess}, + 3: {checkPartSuccess, checkPartSuccess, checkPartSuccess, checkPartFileNotFound}, + }, + expectedMeta: fi, + expectedDangling: false, + }, + + // Add new cases as seen + } + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + gotMeta, dangling := isObjectDangling(testCase.metaArr, testCase.errs, testCase.dataErrs) + if !gotMeta.Equals(testCase.expectedMeta) { + t.Errorf("Expected %#v, got %#v", testCase.expectedMeta, gotMeta) + } + if dangling != testCase.expectedDangling { + t.Errorf("Expected dangling %t, got %t", testCase.expectedDangling, dangling) + } + }) + } +} + +// Tests both object and bucket healing. +func TestHealing(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + obj, fsDirs, err := prepareErasure16(ctx) + if err != nil { + t.Fatal(err) + } + defer obj.Shutdown(t.Context()) + + // initialize the server and obtain the credentials and root. + // credentials are necessary to sign the HTTP request. + if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil { + t.Fatalf("Unable to initialize server config. %s", err) + } + + defer removeRoots(fsDirs) + + z := obj.(*erasureServerPools) + er := z.serverPools[0].sets[0] + + // Create "bucket" + err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{}) + if err != nil { + t.Fatal(err) + } + + bucket := "bucket" + object := "object" + + data := make([]byte, 1*humanize.MiByte) + length := int64(len(data)) + _, err = rand.Read(data) + if err != nil { + t.Fatal(err) + } + + _, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), length, "", ""), ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + + disk := er.getDisks()[0] + fileInfoPreHeal, err := disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true}) + if err != nil { + t.Fatal(err) + } + + // Remove the object - to simulate the case where the disk was down when the object + // was created. + err = removeAll(pathJoin(disk.String(), bucket, object)) + if err != nil { + t.Fatal(err) + } + + // Checking abandoned parts should do nothing + err = er.checkAbandonedParts(ctx, bucket, object, madmin.HealOpts{ScanMode: madmin.HealNormalScan, Remove: true}) + if err != nil { + t.Fatal(err) + } + + _, err = er.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan}) + if err != nil { + t.Fatal(err) + } + + fileInfoPostHeal, err := disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true}) + if err != nil { + t.Fatal(err) + } + + // After heal the meta file should be as expected. + if !fileInfoPreHeal.Equals(fileInfoPostHeal) { + t.Fatal("HealObject failed") + } + + err = os.RemoveAll(path.Join(fsDirs[0], bucket, object, "xl.meta")) + if err != nil { + t.Fatal(err) + } + + // Write xl.meta with different modtime to simulate the case where a disk had + // gone down when an object was replaced by a new object. + fileInfoOutDated := fileInfoPreHeal + fileInfoOutDated.ModTime = time.Now() + err = disk.WriteMetadata(t.Context(), "", bucket, object, fileInfoOutDated) + if err != nil { + t.Fatal(err) + } + + _, err = er.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealDeepScan}) + if err != nil { + t.Fatal(err) + } + + fileInfoPostHeal, err = disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true}) + if err != nil { + t.Fatal(err) + } + + // After heal the meta file should be as expected. + if !fileInfoPreHeal.Equals(fileInfoPostHeal) { + t.Fatal("HealObject failed") + } + + uuid, _ := uuid2.NewRandom() + for _, drive := range fsDirs { + dir := path.Join(drive, bucket, object, uuid.String()) + err = os.MkdirAll(dir, os.ModePerm) + if err != nil { + t.Fatal(err) + } + err = os.WriteFile(pathJoin(dir, "part.1"), []byte("some data"), os.ModePerm) + if err != nil { + t.Fatal(err) + } + } + + // This should remove all the unreferenced parts. + err = er.checkAbandonedParts(ctx, bucket, object, madmin.HealOpts{ScanMode: madmin.HealNormalScan, Remove: true}) + if err != nil { + t.Fatal(err) + } + + for _, drive := range fsDirs { + dir := path.Join(drive, bucket, object, uuid.String()) + _, err := os.ReadFile(pathJoin(dir, "part.1")) + if err == nil { + t.Fatal("expected data dit to be cleaned up") + } + } + + // Remove the bucket - to simulate the case where bucket was + // created when the disk was down. + err = os.RemoveAll(path.Join(fsDirs[0], bucket)) + if err != nil { + t.Fatal(err) + } + // This would create the bucket. + _, err = obj.HealBucket(ctx, bucket, madmin.HealOpts{ + DryRun: false, + Remove: false, + }) + if err != nil { + t.Fatal(err) + } + // Stat the bucket to make sure that it was created. + _, err = er.getDisks()[0].StatVol(t.Context(), bucket) + if err != nil { + t.Fatal(err) + } +} + +// Tests both object and bucket healing. +func TestHealingVersioned(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + obj, fsDirs, err := prepareErasure16(ctx) + if err != nil { + t.Fatal(err) + } + defer obj.Shutdown(t.Context()) + + // initialize the server and obtain the credentials and root. + // credentials are necessary to sign the HTTP request. + if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil { + t.Fatalf("Unable to initialize server config. %s", err) + } + + defer removeRoots(fsDirs) + + z := obj.(*erasureServerPools) + er := z.serverPools[0].sets[0] + + // Create "bucket" + err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{VersioningEnabled: true}) + if err != nil { + t.Fatal(err) + } + + bucket := "bucket" + object := "object" + + data := make([]byte, 1*humanize.MiByte) + length := int64(len(data)) + _, err = rand.Read(data) + if err != nil { + t.Fatal(err) + } + + oi1, err := obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), length, "", ""), ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + // 2nd version. + _, _ = rand.Read(data) + oi2, err := obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), length, "", ""), ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + + disk := er.getDisks()[0] + fileInfoPreHeal1, err := disk.ReadVersion(t.Context(), "", bucket, object, oi1.VersionID, ReadOptions{ReadData: false, Healing: true}) + if err != nil { + t.Fatal(err) + } + fileInfoPreHeal2, err := disk.ReadVersion(t.Context(), "", bucket, object, oi2.VersionID, ReadOptions{ReadData: false, Healing: true}) + if err != nil { + t.Fatal(err) + } + + // Remove the object - to simulate the case where the disk was down when the object + // was created. + err = removeAll(pathJoin(disk.String(), bucket, object)) + if err != nil { + t.Fatal(err) + } + + // Checking abandoned parts should do nothing + err = er.checkAbandonedParts(ctx, bucket, object, madmin.HealOpts{ScanMode: madmin.HealNormalScan, Remove: true}) + if err != nil { + t.Fatal(err) + } + + _, err = er.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan}) + if err != nil { + t.Fatal(err) + } + + fileInfoPostHeal1, err := disk.ReadVersion(t.Context(), "", bucket, object, oi1.VersionID, ReadOptions{ReadData: false, Healing: true}) + if err != nil { + t.Fatal(err) + } + fileInfoPostHeal2, err := disk.ReadVersion(t.Context(), "", bucket, object, oi2.VersionID, ReadOptions{ReadData: false, Healing: true}) + if err != nil { + t.Fatal(err) + } + + // After heal the meta file should be as expected. + if !fileInfoPreHeal1.Equals(fileInfoPostHeal1) { + t.Fatal("HealObject failed") + } + if !fileInfoPreHeal1.Equals(fileInfoPostHeal2) { + t.Fatal("HealObject failed") + } + + err = os.RemoveAll(path.Join(fsDirs[0], bucket, object, "xl.meta")) + if err != nil { + t.Fatal(err) + } + + // Write xl.meta with different modtime to simulate the case where a disk had + // gone down when an object was replaced by a new object. + fileInfoOutDated := fileInfoPreHeal1 + fileInfoOutDated.ModTime = time.Now() + err = disk.WriteMetadata(t.Context(), "", bucket, object, fileInfoOutDated) + if err != nil { + t.Fatal(err) + } + + _, err = er.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealDeepScan}) + if err != nil { + t.Fatal(err) + } + + fileInfoPostHeal1, err = disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true}) + if err != nil { + t.Fatal(err) + } + + // After heal the meta file should be as expected. + if !fileInfoPreHeal1.Equals(fileInfoPostHeal1) { + t.Fatal("HealObject failed") + } + + fileInfoPostHeal2, err = disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true}) + if err != nil { + t.Fatal(err) + } + + // After heal the meta file should be as expected. + if !fileInfoPreHeal2.Equals(fileInfoPostHeal2) { + t.Fatal("HealObject failed") + } + + uuid, _ := uuid2.NewRandom() + for _, drive := range fsDirs { + dir := path.Join(drive, bucket, object, uuid.String()) + err = os.MkdirAll(dir, os.ModePerm) + if err != nil { + t.Fatal(err) + } + err = os.WriteFile(pathJoin(dir, "part.1"), []byte("some data"), os.ModePerm) + if err != nil { + t.Fatal(err) + } + } + + // This should remove all the unreferenced parts. + err = er.checkAbandonedParts(ctx, bucket, object, madmin.HealOpts{ScanMode: madmin.HealNormalScan, Remove: true}) + if err != nil { + t.Fatal(err) + } + + for _, drive := range fsDirs { + dir := path.Join(drive, bucket, object, uuid.String()) + _, err := os.ReadFile(pathJoin(dir, "part.1")) + if err == nil { + t.Fatal("expected data dit to be cleaned up") + } + } + + // Remove the bucket - to simulate the case where bucket was + // created when the disk was down. + err = os.RemoveAll(path.Join(fsDirs[0], bucket)) + if err != nil { + t.Fatal(err) + } + // This would create the bucket. + _, err = obj.HealBucket(ctx, bucket, madmin.HealOpts{ + DryRun: false, + Remove: false, + }) + if err != nil { + t.Fatal(err) + } + // Stat the bucket to make sure that it was created. + _, err = er.getDisks()[0].StatVol(t.Context(), bucket) + if err != nil { + t.Fatal(err) + } +} + +func TestHealingDanglingObject(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + resetGlobalHealState() + defer resetGlobalHealState() + + // Set globalStorageClass.STANDARD to EC:4 for this test + saveSC := globalStorageClass + defer func() { + globalStorageClass.Update(saveSC) + }() + globalStorageClass.Update(storageclass.Config{ + Standard: storageclass.StorageClass{ + Parity: 4, + }, + }) + + nDisks := 16 + fsDirs, err := getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + + defer removeRoots(fsDirs) + + // Everything is fine, should return nil + objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, fsDirs...)) + if err != nil { + t.Fatal(err) + } + + setObjectLayer(objLayer) + + bucket := getRandomBucketName() + object := getRandomObjectName() + data := bytes.Repeat([]byte("a"), 128*1024) + + err = objLayer.MakeBucket(ctx, bucket, MakeBucketOptions{}) + if err != nil { + t.Fatalf("Failed to make a bucket - %v", err) + } + + disks := objLayer.(*erasureServerPools).serverPools[0].erasureDisks[0] + orgDisks := append([]StorageAPI{}, disks...) + + // Enable versioning. + globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, []byte(`Enabled`)) + + _, err = objLayer.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{ + Versioned: true, + }) + if err != nil { + t.Fatal(err) + } + + setDisks := func(newDisks ...StorageAPI) { + objLayer.(*erasureServerPools).serverPools[0].erasureDisksMu.Lock() + copy(disks, newDisks) + objLayer.(*erasureServerPools).serverPools[0].erasureDisksMu.Unlock() + } + getDisk := func(n int) StorageAPI { + objLayer.(*erasureServerPools).serverPools[0].erasureDisksMu.Lock() + disk := disks[n] + objLayer.(*erasureServerPools).serverPools[0].erasureDisksMu.Unlock() + return disk + } + + // Remove 4 disks. + setDisks(nil, nil, nil, nil) + + // Create delete marker under quorum. + objInfo, err := objLayer.DeleteObject(ctx, bucket, object, ObjectOptions{Versioned: true}) + if err != nil { + t.Fatal(err) + } + + // Restore... + setDisks(orgDisks[:4]...) + + fileInfoPreHeal, err := disks[0].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true}) + if err != nil { + t.Fatal(err) + } + + if fileInfoPreHeal.NumVersions != 1 { + t.Fatalf("Expected versions 1, got %d", fileInfoPreHeal.NumVersions) + } + + if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Recursive: true, Remove: true}, + func(bucket, object, vid string, scanMode madmin.HealScanMode) error { + _, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{ScanMode: scanMode, Remove: true}) + return err + }); err != nil { + t.Fatal(err) + } + + fileInfoPostHeal, err := disks[0].ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true}) + if err != nil { + t.Fatal(err) + } + + if fileInfoPostHeal.NumVersions != 2 { + t.Fatalf("Expected versions 2, got %d", fileInfoPreHeal.NumVersions) + } + + if objInfo.DeleteMarker { + if _, err = objLayer.DeleteObject(ctx, bucket, object, ObjectOptions{ + Versioned: true, + VersionID: objInfo.VersionID, + }); err != nil { + t.Fatal(err) + } + } + + setDisks(nil, nil, nil, nil) + + rd := mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", "") + _, err = objLayer.PutObject(ctx, bucket, object, rd, ObjectOptions{ + Versioned: true, + }) + if err != nil { + t.Fatal(err) + } + + setDisks(orgDisks[:4]...) + disk := getDisk(0) + fileInfoPreHeal, err = disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true}) + if err != nil { + t.Fatal(err) + } + + if fileInfoPreHeal.NumVersions != 1 { + t.Fatalf("Expected versions 1, got %d", fileInfoPreHeal.NumVersions) + } + + if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Recursive: true, Remove: true}, + func(bucket, object, vid string, scanMode madmin.HealScanMode) error { + _, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{ScanMode: scanMode, Remove: true}) + return err + }); err != nil { + t.Fatal(err) + } + + disk = getDisk(0) + fileInfoPostHeal, err = disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true}) + if err != nil { + t.Fatal(err) + } + + if fileInfoPostHeal.NumVersions != 2 { + t.Fatalf("Expected versions 2, got %d", fileInfoPreHeal.NumVersions) + } + + rd = mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", "") + objInfo, err = objLayer.PutObject(ctx, bucket, object, rd, ObjectOptions{ + Versioned: true, + }) + if err != nil { + t.Fatal(err) + } + + setDisks(nil, nil, nil, nil) + + // Create delete marker under quorum. + _, err = objLayer.DeleteObject(ctx, bucket, object, ObjectOptions{ + Versioned: true, + VersionID: objInfo.VersionID, + }) + if err != nil { + t.Fatal(err) + } + + setDisks(orgDisks[:4]...) + + disk = getDisk(0) + fileInfoPreHeal, err = disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true}) + if err != nil { + t.Fatal(err) + } + + if fileInfoPreHeal.NumVersions != 3 { + t.Fatalf("Expected versions 3, got %d", fileInfoPreHeal.NumVersions) + } + + if err = objLayer.HealObjects(ctx, bucket, "", madmin.HealOpts{Recursive: true, Remove: true}, + func(bucket, object, vid string, scanMode madmin.HealScanMode) error { + _, err := objLayer.HealObject(ctx, bucket, object, vid, madmin.HealOpts{ScanMode: scanMode, Remove: true}) + return err + }); err != nil { + t.Fatal(err) + } + + disk = getDisk(0) + fileInfoPostHeal, err = disk.ReadVersion(t.Context(), "", bucket, object, "", ReadOptions{ReadData: false, Healing: true}) + if err != nil { + t.Fatal(err) + } + + if fileInfoPostHeal.NumVersions != 2 { + t.Fatalf("Expected versions 2, got %d", fileInfoPreHeal.NumVersions) + } +} + +func TestHealCorrectQuorum(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + resetGlobalHealState() + defer resetGlobalHealState() + + nDisks := 32 + fsDirs, err := getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + + defer removeRoots(fsDirs) + + pools := mustGetPoolEndpoints(0, fsDirs[:16]...) + pools = append(pools, mustGetPoolEndpoints(1, fsDirs[16:]...)...) + + // Everything is fine, should return nil + objLayer, _, err := initObjectLayer(ctx, pools) + if err != nil { + t.Fatal(err) + } + + bucket := getRandomBucketName() + object := getRandomObjectName() + data := bytes.Repeat([]byte("a"), 5*1024*1024) + var opts ObjectOptions + + err = objLayer.MakeBucket(ctx, bucket, MakeBucketOptions{}) + if err != nil { + t.Fatalf("Failed to make a bucket - %v", err) + } + + // Create an object with multiple parts uploaded in decreasing + // part number. + res, err := objLayer.NewMultipartUpload(ctx, bucket, object, opts) + if err != nil { + t.Fatalf("Failed to create a multipart upload - %v", err) + } + + var uploadedParts []CompletePart + for _, partID := range []int{2, 1} { + pInfo, err1 := objLayer.PutObjectPart(ctx, bucket, object, res.UploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts) + if err1 != nil { + t.Fatalf("Failed to upload a part - %v", err1) + } + uploadedParts = append(uploadedParts, CompletePart{ + PartNumber: pInfo.PartNumber, + ETag: pInfo.ETag, + }) + } + + _, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, res.UploadID, uploadedParts, ObjectOptions{}) + if err != nil { + t.Fatalf("Failed to complete multipart upload - got: %v", err) + } + + cfgFile := pathJoin(bucketMetaPrefix, bucket, ".test.bin") + if err = saveConfig(ctx, objLayer, cfgFile, data); err != nil { + t.Fatal(err) + } + + hopts := madmin.HealOpts{ + DryRun: false, + Remove: true, + ScanMode: madmin.HealNormalScan, + } + + // Test 1: Remove the object backend files from the first disk. + z := objLayer.(*erasureServerPools) + for _, set := range z.serverPools { + er := set.sets[0] + erasureDisks := er.getDisks() + + fileInfos, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true) + nfi, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs) + if errors.Is(err, errFileNotFound) { + continue + } + if err != nil { + t.Fatalf("Failed to getLatestFileInfo - %v", err) + } + + for i := 0; i < nfi.Erasure.ParityBlocks; i++ { + erasureDisks[i].Delete(t.Context(), bucket, pathJoin(object, xlStorageFormatFile), DeleteOptions{ + Recursive: false, + Immediate: false, + }) + } + + // Try healing now, it should heal the content properly. + _, err = objLayer.HealObject(ctx, bucket, object, "", hopts) + if err != nil { + t.Fatal(err) + } + + fileInfos, errs = readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true) + if countErrs(errs, nil) != len(fileInfos) { + t.Fatal("Expected all xl.meta healed, but partial heal detected") + } + + fileInfos, errs = readAllFileInfo(ctx, erasureDisks, "", minioMetaBucket, cfgFile, "", false, true) + nfi, err = getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs) + if errors.Is(err, errFileNotFound) { + continue + } + if err != nil { + t.Fatalf("Failed to getLatestFileInfo - %v", err) + } + + for i := 0; i < nfi.Erasure.ParityBlocks; i++ { + erasureDisks[i].Delete(t.Context(), minioMetaBucket, pathJoin(cfgFile, xlStorageFormatFile), DeleteOptions{ + Recursive: false, + Immediate: false, + }) + } + + // Try healing now, it should heal the content properly. + _, err = objLayer.HealObject(ctx, minioMetaBucket, cfgFile, "", hopts) + if err != nil { + t.Fatal(err) + } + + fileInfos, errs = readAllFileInfo(ctx, erasureDisks, "", minioMetaBucket, cfgFile, "", false, true) + if countErrs(errs, nil) != len(fileInfos) { + t.Fatal("Expected all xl.meta healed, but partial heal detected") + } + } +} + +func TestHealObjectCorruptedPools(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + resetGlobalHealState() + defer resetGlobalHealState() + + nDisks := 32 + fsDirs, err := getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + + defer removeRoots(fsDirs) + + pools := mustGetPoolEndpoints(0, fsDirs[:16]...) + pools = append(pools, mustGetPoolEndpoints(1, fsDirs[16:]...)...) + + // Everything is fine, should return nil + objLayer, _, err := initObjectLayer(ctx, pools) + if err != nil { + t.Fatal(err) + } + + bucket := getRandomBucketName() + object := getRandomObjectName() + data := bytes.Repeat([]byte("a"), 5*1024*1024) + var opts ObjectOptions + + err = objLayer.MakeBucket(ctx, bucket, MakeBucketOptions{}) + if err != nil { + t.Fatalf("Failed to make a bucket - %v", err) + } + + // Upload a multipart object in the second pool + z := objLayer.(*erasureServerPools) + set := z.serverPools[1] + + res, err := set.NewMultipartUpload(ctx, bucket, object, opts) + if err != nil { + t.Fatalf("Failed to create a multipart upload - %v", err) + } + uploadID := res.UploadID + + var uploadedParts []CompletePart + for _, partID := range []int{2, 1} { + pInfo, err1 := set.PutObjectPart(ctx, bucket, object, uploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts) + if err1 != nil { + t.Fatalf("Failed to upload a part - %v", err1) + } + uploadedParts = append(uploadedParts, CompletePart{ + PartNumber: pInfo.PartNumber, + ETag: pInfo.ETag, + }) + } + + _, err = set.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, ObjectOptions{}) + if err != nil { + t.Fatalf("Failed to complete multipart upload - %v", err) + } + + // Test 1: Remove the object backend files from the first disk. + er := set.sets[0] + erasureDisks := er.getDisks() + firstDisk := erasureDisks[0] + err = firstDisk.Delete(t.Context(), bucket, pathJoin(object, xlStorageFormatFile), DeleteOptions{ + Recursive: false, + Immediate: false, + }) + if err != nil { + t.Fatalf("Failed to delete a file - %v", err) + } + + _, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan}) + if err != nil { + t.Fatalf("Failed to heal object - %v", err) + } + + fileInfos, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true) + fi, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs) + if err != nil { + t.Fatalf("Failed to getLatestFileInfo - %v", err) + } + + if _, err = firstDisk.StatInfoFile(t.Context(), bucket, object+"/"+xlStorageFormatFile, false); err != nil { + t.Errorf("Expected xl.meta file to be present but stat failed - %v", err) + } + + err = firstDisk.Delete(t.Context(), bucket, pathJoin(object, fi.DataDir, "part.1"), DeleteOptions{ + Recursive: false, + Immediate: false, + }) + if err != nil { + t.Errorf("Failure during deleting part.1 - %v", err) + } + + err = firstDisk.WriteAll(t.Context(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte{}) + if err != nil { + t.Errorf("Failure during creating part.1 - %v", err) + } + + _, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan}) + if err != nil { + t.Errorf("Expected nil but received %v", err) + } + + fileInfos, errs = readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true) + nfi, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs) + if err != nil { + t.Fatalf("Failed to getLatestFileInfo - %v", err) + } + + if !reflect.DeepEqual(fi, nfi) { + t.Fatalf("FileInfo not equal after healing: %v != %v", fi, nfi) + } + + err = firstDisk.Delete(t.Context(), bucket, pathJoin(object, fi.DataDir, "part.1"), DeleteOptions{ + Recursive: false, + Immediate: false, + }) + if err != nil { + t.Errorf("Failure during deleting part.1 - %v", err) + } + + bdata := bytes.Repeat([]byte("b"), int(nfi.Size)) + err = firstDisk.WriteAll(t.Context(), bucket, pathJoin(object, fi.DataDir, "part.1"), bdata) + if err != nil { + t.Errorf("Failure during creating part.1 - %v", err) + } + + _, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan}) + if err != nil { + t.Errorf("Expected nil but received %v", err) + } + + fileInfos, errs = readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true) + nfi, err = getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs) + if err != nil { + t.Fatalf("Failed to getLatestFileInfo - %v", err) + } + + if !reflect.DeepEqual(fi, nfi) { + t.Fatalf("FileInfo not equal after healing: %v != %v", fi, nfi) + } + + // Test 4: checks if HealObject returns an error when xl.meta is not found + // in more than read quorum number of disks, to create a corrupted situation. + for i := 0; i <= nfi.Erasure.DataBlocks; i++ { + erasureDisks[i].Delete(t.Context(), bucket, pathJoin(object, xlStorageFormatFile), DeleteOptions{ + Recursive: false, + Immediate: false, + }) + } + + // Try healing now, expect to receive errFileNotFound. + _, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan}) + if err != nil { + if _, ok := err.(ObjectNotFound); !ok { + t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err) + } + } + + // since majority of xl.meta's are not available, object should be successfully deleted. + _, err = objLayer.GetObjectInfo(ctx, bucket, object, ObjectOptions{}) + if _, ok := err.(ObjectNotFound); !ok { + t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err) + } + + for i := 0; i < (nfi.Erasure.DataBlocks + nfi.Erasure.ParityBlocks); i++ { + stats, _ := erasureDisks[i].StatInfoFile(t.Context(), bucket, pathJoin(object, xlStorageFormatFile), false) + if len(stats) != 0 { + t.Errorf("Expected xl.meta file to be not present, but succeeded") + } + } +} + +func TestHealObjectCorruptedXLMeta(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + resetGlobalHealState() + defer resetGlobalHealState() + + nDisks := 16 + fsDirs, err := getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + + defer removeRoots(fsDirs) + + // Everything is fine, should return nil + objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, fsDirs...)) + if err != nil { + t.Fatal(err) + } + + bucket := getRandomBucketName() + object := getRandomObjectName() + data := bytes.Repeat([]byte("a"), 5*1024*1024) + var opts ObjectOptions + + err = objLayer.MakeBucket(ctx, bucket, MakeBucketOptions{}) + if err != nil { + t.Fatalf("Failed to make a bucket - %v", err) + } + + // Create an object with multiple parts uploaded in decreasing + // part number. + res, err := objLayer.NewMultipartUpload(ctx, bucket, object, opts) + if err != nil { + t.Fatalf("Failed to create a multipart upload - %v", err) + } + + var uploadedParts []CompletePart + for _, partID := range []int{2, 1} { + pInfo, err1 := objLayer.PutObjectPart(ctx, bucket, object, res.UploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts) + if err1 != nil { + t.Fatalf("Failed to upload a part - %v", err1) + } + uploadedParts = append(uploadedParts, CompletePart{ + PartNumber: pInfo.PartNumber, + ETag: pInfo.ETag, + }) + } + + _, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, res.UploadID, uploadedParts, ObjectOptions{}) + if err != nil { + t.Fatalf("Failed to complete multipart upload - %v", err) + } + + z := objLayer.(*erasureServerPools) + er := z.serverPools[0].sets[0] + erasureDisks := er.getDisks() + firstDisk := erasureDisks[0] + + // Test 1: Remove the object backend files from the first disk. + fileInfos, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true) + fi, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs) + if err != nil { + t.Fatalf("Failed to getLatestFileInfo - %v", err) + } + + err = firstDisk.Delete(t.Context(), bucket, pathJoin(object, xlStorageFormatFile), DeleteOptions{ + Recursive: false, + Immediate: false, + }) + if err != nil { + t.Fatalf("Failed to delete a file - %v", err) + } + + _, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan}) + if err != nil { + t.Fatalf("Failed to heal object - %v", err) + } + + if _, err = firstDisk.StatInfoFile(t.Context(), bucket, object+"/"+xlStorageFormatFile, false); err != nil { + t.Errorf("Expected xl.meta file to be present but stat failed - %v", err) + } + + fileInfos, errs = readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true) + nfi1, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs) + if err != nil { + t.Fatalf("Failed to getLatestFileInfo - %v", err) + } + + if !reflect.DeepEqual(fi, nfi1) { + t.Fatalf("FileInfo not equal after healing") + } + + // Test 2: Test with a corrupted xl.meta + err = firstDisk.WriteAll(t.Context(), bucket, pathJoin(object, xlStorageFormatFile), []byte("abcd")) + if err != nil { + t.Errorf("Failure during creating part.1 - %v", err) + } + + _, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan}) + if err != nil { + t.Errorf("Expected nil but received %v", err) + } + + fileInfos, errs = readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true) + nfi2, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs) + if err != nil { + t.Fatalf("Failed to getLatestFileInfo - %v", err) + } + + if !reflect.DeepEqual(fi, nfi2) { + t.Fatalf("FileInfo not equal after healing") + } + + // Test 3: checks if HealObject returns an error when xl.meta is not found + // in more than read quorum number of disks, to create a corrupted situation. + for i := 0; i <= nfi2.Erasure.DataBlocks; i++ { + erasureDisks[i].Delete(t.Context(), bucket, pathJoin(object, xlStorageFormatFile), DeleteOptions{ + Recursive: false, + Immediate: false, + }) + } + + // Try healing now, expect to receive errFileNotFound. + _, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{DryRun: false, Remove: true, ScanMode: madmin.HealDeepScan}) + if err != nil { + if _, ok := err.(ObjectNotFound); !ok { + t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err) + } + } + + // since majority of xl.meta's are not available, object should be successfully deleted. + _, err = objLayer.GetObjectInfo(ctx, bucket, object, ObjectOptions{}) + if _, ok := err.(ObjectNotFound); !ok { + t.Errorf("Expect %v but received %v", ObjectNotFound{Bucket: bucket, Object: object}, err) + } +} + +func TestHealObjectCorruptedParts(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + resetGlobalHealState() + defer resetGlobalHealState() + + nDisks := 16 + fsDirs, err := getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + + defer removeRoots(fsDirs) + + // Everything is fine, should return nil + objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, fsDirs...)) + if err != nil { + t.Fatal(err) + } + + bucket := getRandomBucketName() + object := getRandomObjectName() + data := bytes.Repeat([]byte("a"), 5*1024*1024) + var opts ObjectOptions + + err = objLayer.MakeBucket(ctx, bucket, MakeBucketOptions{}) + if err != nil { + t.Fatalf("Failed to make a bucket - %v", err) + } + + // Create an object with multiple parts uploaded in decreasing + // part number. + res, err := objLayer.NewMultipartUpload(ctx, bucket, object, opts) + if err != nil { + t.Fatalf("Failed to create a multipart upload - %v", err) + } + + var uploadedParts []CompletePart + for _, partID := range []int{2, 1} { + pInfo, err1 := objLayer.PutObjectPart(ctx, bucket, object, res.UploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts) + if err1 != nil { + t.Fatalf("Failed to upload a part - %v", err1) + } + uploadedParts = append(uploadedParts, CompletePart{ + PartNumber: pInfo.PartNumber, + ETag: pInfo.ETag, + }) + } + + _, err = objLayer.CompleteMultipartUpload(ctx, bucket, object, res.UploadID, uploadedParts, ObjectOptions{}) + if err != nil { + t.Fatalf("Failed to complete multipart upload - %v", err) + } + + // Test 1: Remove the object backend files from the first disk. + z := objLayer.(*erasureServerPools) + er := z.serverPools[0].sets[0] + erasureDisks := er.getDisks() + firstDisk := erasureDisks[0] + secondDisk := erasureDisks[1] + + fileInfos, errs := readAllFileInfo(ctx, erasureDisks, "", bucket, object, "", false, true) + fi, err := getLatestFileInfo(ctx, fileInfos, er.defaultParityCount, errs) + if err != nil { + t.Fatalf("Failed to getLatestFileInfo - %v", err) + } + + part1Disk1Origin, err := firstDisk.ReadAll(t.Context(), bucket, pathJoin(object, fi.DataDir, "part.1")) + if err != nil { + t.Fatalf("Failed to read a file - %v", err) + } + + part1Disk2Origin, err := secondDisk.ReadAll(t.Context(), bucket, pathJoin(object, fi.DataDir, "part.1")) + if err != nil { + t.Fatalf("Failed to read a file - %v", err) + } + + // Test 1, remove part.1 + err = firstDisk.Delete(t.Context(), bucket, pathJoin(object, fi.DataDir, "part.1"), DeleteOptions{ + Recursive: false, + Immediate: false, + }) + if err != nil { + t.Fatalf("Failed to delete a file - %v", err) + } + + _, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan}) + if err != nil { + t.Fatalf("Failed to heal object - %v", err) + } + + part1Replaced, err := firstDisk.ReadAll(t.Context(), bucket, pathJoin(object, fi.DataDir, "part.1")) + if err != nil { + t.Fatalf("Failed to read a file - %v", err) + } + + if !reflect.DeepEqual(part1Disk1Origin, part1Replaced) { + t.Fatalf("part.1 not healed correctly") + } + + // Test 2, Corrupt part.1 + err = firstDisk.WriteAll(t.Context(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte("foobytes")) + if err != nil { + t.Fatalf("Failed to write a file - %v", err) + } + + _, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan}) + if err != nil { + t.Fatalf("Failed to heal object - %v", err) + } + + part1Replaced, err = firstDisk.ReadAll(t.Context(), bucket, pathJoin(object, fi.DataDir, "part.1")) + if err != nil { + t.Fatalf("Failed to read a file - %v", err) + } + + if !reflect.DeepEqual(part1Disk1Origin, part1Replaced) { + t.Fatalf("part.1 not healed correctly") + } + + // Test 3, Corrupt one part and remove data in another disk + err = firstDisk.WriteAll(t.Context(), bucket, pathJoin(object, fi.DataDir, "part.1"), []byte("foobytes")) + if err != nil { + t.Fatalf("Failed to write a file - %v", err) + } + + err = secondDisk.Delete(t.Context(), bucket, object, DeleteOptions{ + Recursive: true, + Immediate: false, + }) + if err != nil { + t.Fatalf("Failed to delete a file - %v", err) + } + + _, err = objLayer.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan}) + if err != nil { + t.Fatalf("Failed to heal object - %v", err) + } + + partReconstructed, err := firstDisk.ReadAll(t.Context(), bucket, pathJoin(object, fi.DataDir, "part.1")) + if err != nil { + t.Fatalf("Failed to read a file - %v", err) + } + + if !reflect.DeepEqual(part1Disk1Origin, partReconstructed) { + t.Fatalf("part.1 not healed correctly") + } + + partReconstructed, err = secondDisk.ReadAll(t.Context(), bucket, pathJoin(object, fi.DataDir, "part.1")) + if err != nil { + t.Fatalf("Failed to read a file - %v", err) + } + + if !reflect.DeepEqual(part1Disk2Origin, partReconstructed) { + t.Fatalf("part.1 not healed correctly") + } +} + +// Tests healing of object. +func TestHealObjectErasure(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + nDisks := 16 + fsDirs, err := getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + + defer removeRoots(fsDirs) + + // Everything is fine, should return nil + obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, fsDirs...)) + if err != nil { + t.Fatal(err) + } + + bucket := "bucket" + object := "object" + data := bytes.Repeat([]byte("a"), 5*1024*1024) + var opts ObjectOptions + + err = obj.MakeBucket(ctx, bucket, MakeBucketOptions{}) + if err != nil { + t.Fatalf("Failed to make a bucket - %v", err) + } + + // Create an object with multiple parts uploaded in decreasing + // part number. + res, err := obj.NewMultipartUpload(ctx, bucket, object, opts) + if err != nil { + t.Fatalf("Failed to create a multipart upload - %v", err) + } + + var uploadedParts []CompletePart + for _, partID := range []int{2, 1} { + pInfo, err1 := obj.PutObjectPart(ctx, bucket, object, res.UploadID, partID, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts) + if err1 != nil { + t.Fatalf("Failed to upload a part - %v", err1) + } + uploadedParts = append(uploadedParts, CompletePart{ + PartNumber: pInfo.PartNumber, + ETag: pInfo.ETag, + }) + } + + // Remove the object backend files from the first disk. + z := obj.(*erasureServerPools) + er := z.serverPools[0].sets[0] + firstDisk := er.getDisks()[0] + + _, err = obj.CompleteMultipartUpload(ctx, bucket, object, res.UploadID, uploadedParts, ObjectOptions{}) + if err != nil { + t.Fatalf("Failed to complete multipart upload - %v", err) + } + + // Delete the whole object folder + err = firstDisk.Delete(t.Context(), bucket, object, DeleteOptions{ + Recursive: true, + Immediate: false, + }) + if err != nil { + t.Fatalf("Failed to delete a file - %v", err) + } + + _, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan}) + if err != nil { + t.Fatalf("Failed to heal object - %v", err) + } + + if _, err = firstDisk.StatInfoFile(t.Context(), bucket, object+"/"+xlStorageFormatFile, false); err != nil { + t.Errorf("Expected xl.meta file to be present but stat failed - %v", err) + } + + erasureDisks := er.getDisks() + z.serverPools[0].erasureDisksMu.Lock() + er.getDisks = func() []StorageAPI { + // Nil more than half the disks, to remove write quorum. + for i := 0; i <= len(erasureDisks)/2; i++ { + err := erasureDisks[i].Delete(t.Context(), bucket, object, DeleteOptions{ + Recursive: true, + Immediate: false, + }) + if err != nil { + t.Fatalf("Failed to delete a file - %v", err) + } + } + return erasureDisks + } + z.serverPools[0].erasureDisksMu.Unlock() + + // Try healing now, expect to receive errDiskNotFound. + _, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{ + ScanMode: madmin.HealDeepScan, + }) + // since majority of xl.meta's are not available, object quorum + // can't be read properly will be deleted automatically and + // err is nil + if !isErrObjectNotFound(err) { + t.Fatal(err) + } +} + +// Tests healing of empty directories +func TestHealEmptyDirectoryErasure(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + nDisks := 16 + fsDirs, err := getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + defer removeRoots(fsDirs) + + // Everything is fine, should return nil + obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, fsDirs...)) + if err != nil { + t.Fatal(err) + } + + bucket := "bucket" + object := "empty-dir/" + var opts ObjectOptions + + err = obj.MakeBucket(ctx, bucket, MakeBucketOptions{}) + if err != nil { + t.Fatalf("Failed to make a bucket - %v", err) + } + + // Upload an empty directory + _, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, + bytes.NewReader([]byte{}), 0, "", ""), opts) + if err != nil { + t.Fatal(err) + } + + // Remove the object backend files from the first disk. + z := obj.(*erasureServerPools) + er := z.serverPools[0].sets[0] + firstDisk := er.getDisks()[0] + err = firstDisk.DeleteVol(t.Context(), pathJoin(bucket, encodeDirObject(object)), true) + if err != nil { + t.Fatalf("Failed to delete a file - %v", err) + } + + // Heal the object + hr, err := obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan}) + if err != nil { + t.Fatalf("Failed to heal object - %v", err) + } + + // Check if the empty directory is restored in the first disk + _, err = firstDisk.StatVol(t.Context(), pathJoin(bucket, encodeDirObject(object))) + if err != nil { + t.Fatalf("Expected object to be present but stat failed - %v", err) + } + + // Check the state of the object in the first disk (should be missing) + if hr.Before.Drives[0].State != madmin.DriveStateMissing { + t.Fatalf("Unexpected drive state: %v", hr.Before.Drives[0].State) + } + + // Check the state of all other disks (should be ok) + for i, h := range append(hr.Before.Drives[1:], hr.After.Drives...) { + if h.State != madmin.DriveStateOk { + t.Fatalf("Unexpected drive state (%d): %v", i+1, h.State) + } + } + + // Heal the same object again + hr, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{ScanMode: madmin.HealNormalScan}) + if err != nil { + t.Fatalf("Failed to heal object - %v", err) + } + + // Check that Before & After states are all okay + for i, h := range append(hr.Before.Drives, hr.After.Drives...) { + if h.State != madmin.DriveStateOk { + t.Fatalf("Unexpected drive state (%d): %v", i+1, h.State) + } + } +} + +func TestHealLastDataShard(t *testing.T) { + tests := []struct { + name string + dataSize int64 + }{ + {"4KiB", 4 * humanize.KiByte}, + {"64KiB", 64 * humanize.KiByte}, + {"128KiB", 128 * humanize.KiByte}, + {"1MiB", 1 * humanize.MiByte}, + {"5MiB", 5 * humanize.MiByte}, + {"10MiB", 10 * humanize.MiByte}, + {"5MiB-1KiB", 5*humanize.MiByte - 1*humanize.KiByte}, + {"10MiB-1Kib", 10*humanize.MiByte - 1*humanize.KiByte}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + nDisks := 16 + fsDirs, err := getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + + defer removeRoots(fsDirs) + + obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, fsDirs...)) + if err != nil { + t.Fatal(err) + } + bucket := "bucket" + object := "object" + + data := make([]byte, test.dataSize) + _, err = rand.Read(data) + if err != nil { + t.Fatal(err) + } + var opts ObjectOptions + + err = obj.MakeBucket(ctx, bucket, MakeBucketOptions{}) + if err != nil { + t.Fatalf("Failed to make a bucket - %v", err) + } + + _, err = obj.PutObject(ctx, bucket, object, + mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts) + if err != nil { + t.Fatal(err) + } + + actualH := sha256.New() + _, err = io.Copy(actualH, bytes.NewReader(data)) + if err != nil { + return + } + actualSha256 := actualH.Sum(nil) + + z := obj.(*erasureServerPools) + er := z.serverPools[0].getHashedSet(object) + + disks := er.getDisks() + distribution := hashOrder(pathJoin(bucket, object), nDisks) + shuffledDisks := shuffleDisks(disks, distribution) + + // remove last data shard + err = removeAll(pathJoin(shuffledDisks[11].String(), bucket, object)) + if err != nil { + t.Fatalf("Failed to delete a file - %v", err) + } + _, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{ + ScanMode: madmin.HealNormalScan, + }) + if err != nil { + t.Fatal(err) + } + + firstGr, err := obj.GetObjectNInfo(ctx, bucket, object, nil, nil, ObjectOptions{NoLock: true}) + if err != nil { + t.Fatal(err) + } + defer firstGr.Close() + + firstHealedH := sha256.New() + _, err = io.Copy(firstHealedH, firstGr) + if err != nil { + t.Fatal(err) + } + firstHealedDataSha256 := firstHealedH.Sum(nil) + + if !bytes.Equal(actualSha256, firstHealedDataSha256) { + t.Fatalf("object healed wrong, expected %x, got %x", + actualSha256, firstHealedDataSha256) + } + + // remove another data shard + if err = removeAll(pathJoin(shuffledDisks[1].String(), bucket, object)); err != nil { + t.Fatalf("Failed to delete a file - %v", err) + } + + _, err = obj.HealObject(ctx, bucket, object, "", madmin.HealOpts{ + ScanMode: madmin.HealNormalScan, + }) + if err != nil { + t.Fatal(err) + } + + secondGr, err := obj.GetObjectNInfo(ctx, bucket, object, nil, nil, ObjectOptions{NoLock: true}) + if err != nil { + t.Fatal(err) + } + defer secondGr.Close() + + secondHealedH := sha256.New() + _, err = io.Copy(secondHealedH, secondGr) + if err != nil { + t.Fatal(err) + } + secondHealedDataSha256 := secondHealedH.Sum(nil) + + if !bytes.Equal(actualSha256, secondHealedDataSha256) { + t.Fatalf("object healed wrong, expected %x, got %x", + actualSha256, secondHealedDataSha256) + } + }) + } +} diff --git a/cmd/erasure-metadata-utils.go b/cmd/erasure-metadata-utils.go new file mode 100644 index 00000000..1409d99e --- /dev/null +++ b/cmd/erasure-metadata-utils.go @@ -0,0 +1,381 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/binary" + "errors" + "hash/crc32" + + "github.com/minio/pkg/v3/sync/errgroup" +) + +// counterMap type adds GetValueWithQuorum method to a map[T]int used to count occurrences of values of type T. +type counterMap[T comparable] map[T]int + +// GetValueWithQuorum returns the first key which occurs >= quorum number of times. +func (c counterMap[T]) GetValueWithQuorum(quorum int) (T, bool) { + var zero T + for x, count := range c { + if count >= quorum { + return x, true + } + } + return zero, false +} + +// figure out the most commonVersions across disk that satisfies +// the 'writeQuorum' this function returns "" if quorum cannot +// be achieved and disks have too many inconsistent versions. +func reduceCommonVersions(diskVersions [][]byte, writeQuorum int) (versions []byte) { + diskVersionsCount := make(map[uint64]int) + for _, versions := range diskVersions { + if len(versions) > 0 { + diskVersionsCount[binary.BigEndian.Uint64(versions)]++ + } + } + + var commonVersions uint64 + maxCnt := 0 + for versions, count := range diskVersionsCount { + if maxCnt < count { + maxCnt = count + commonVersions = versions + } + } + + if maxCnt >= writeQuorum { + for _, versions := range diskVersions { + if binary.BigEndian.Uint64(versions) == commonVersions { + return versions + } + } + } + + return []byte{} +} + +// figure out the most commonVersions across disk that satisfies +// the 'writeQuorum' this function returns '0' if quorum cannot +// be achieved and disks have too many inconsistent versions. +func reduceCommonDataDir(dataDirs []string, writeQuorum int) (dataDir string) { + dataDirsCount := make(map[string]int) + for _, ddir := range dataDirs { + dataDirsCount[ddir]++ + } + + maxCnt := 0 + for ddir, count := range dataDirsCount { + if maxCnt < count { + maxCnt = count + dataDir = ddir + } + } + + if maxCnt >= writeQuorum { + return dataDir + } + + return "" +} + +// Returns number of errors that occurred the most (incl. nil) and the +// corresponding error value. NB When there is more than one error value that +// occurs maximum number of times, the error value returned depends on how +// golang's map orders keys. This doesn't affect correctness as long as quorum +// value is greater than or equal to simple majority, since none of the equally +// maximal values would occur quorum or more number of times. +func reduceErrs(errs []error, ignoredErrs []error) (maxCount int, maxErr error) { + errorCounts := make(map[error]int) + for _, err := range errs { + if IsErrIgnored(err, ignoredErrs...) { + continue + } + // Errors due to context cancellation may be wrapped - group them by context.Canceled. + if errors.Is(err, context.Canceled) { + errorCounts[context.Canceled]++ + continue + } + errorCounts[err]++ + } + + maxCnt := 0 + for err, count := range errorCounts { + switch { + case maxCnt < count: + maxCnt = count + maxErr = err + + // Prefer `nil` over other error values with the same + // number of occurrences. + case maxCnt == count && err == nil: + maxErr = err + } + } + return maxCnt, maxErr +} + +// reduceQuorumErrs behaves like reduceErrs by only for returning +// values of maximally occurring errors validated against a generic +// quorum number that can be read or write quorum depending on usage. +func reduceQuorumErrs(ctx context.Context, errs []error, ignoredErrs []error, quorum int, quorumErr error) error { + if contextCanceled(ctx) { + return context.Canceled + } + maxCount, maxErr := reduceErrs(errs, ignoredErrs) + if maxCount >= quorum { + return maxErr + } + return quorumErr +} + +// reduceReadQuorumErrs behaves like reduceErrs but only for returning +// values of maximally occurring errors validated against readQuorum. +func reduceReadQuorumErrs(ctx context.Context, errs []error, ignoredErrs []error, readQuorum int) (maxErr error) { + return reduceQuorumErrs(ctx, errs, ignoredErrs, readQuorum, errErasureReadQuorum) +} + +// reduceWriteQuorumErrs behaves like reduceErrs but only for returning +// values of maximally occurring errors validated against writeQuorum. +func reduceWriteQuorumErrs(ctx context.Context, errs []error, ignoredErrs []error, writeQuorum int) (maxErr error) { + return reduceQuorumErrs(ctx, errs, ignoredErrs, writeQuorum, errErasureWriteQuorum) +} + +// Similar to 'len(slice)' but returns the actual elements count +// skipping the unallocated elements. +func diskCount(disks []StorageAPI) int { + diskCount := 0 + for _, disk := range disks { + if disk == nil { + continue + } + diskCount++ + } + return diskCount +} + +// hashOrder - hashes input key to return consistent +// hashed integer slice. Returned integer order is salted +// with an input key. This results in consistent order. +// NOTE: collisions are fine, we are not looking for uniqueness +// in the slices returned. +func hashOrder(key string, cardinality int) []int { + if cardinality <= 0 { + // Returns an empty int slice for cardinality < 0. + return nil + } + + nums := make([]int, cardinality) + keyCrc := crc32.Checksum([]byte(key), crc32.IEEETable) + + start := int(keyCrc % uint32(cardinality)) + for i := 1; i <= cardinality; i++ { + nums[i-1] = 1 + ((start + i) % cardinality) + } + return nums +} + +// Reads all `xl.meta` metadata as a FileInfo slice. +// Returns error slice indicating the failed metadata reads. +func readAllFileInfo(ctx context.Context, disks []StorageAPI, origbucket string, bucket, object, versionID string, readData, healing bool) ([]FileInfo, []error) { + metadataArray := make([]FileInfo, len(disks)) + + opts := ReadOptions{ + ReadData: readData, + Healing: healing, + } + + g := errgroup.WithNErrs(len(disks)) + // Read `xl.meta` in parallel across disks. + for index := range disks { + g.Go(func() (err error) { + if disks[index] == nil { + return errDiskNotFound + } + metadataArray[index], err = disks[index].ReadVersion(ctx, origbucket, bucket, object, versionID, opts) + return err + }, index) + } + + return metadataArray, g.Wait() +} + +// shuffleDisksAndPartsMetadataByIndex this function should be always used by GetObjectNInfo() +// and CompleteMultipartUpload code path, it is not meant to be used with PutObject, +// NewMultipartUpload metadata shuffling. +func shuffleDisksAndPartsMetadataByIndex(disks []StorageAPI, metaArr []FileInfo, fi FileInfo) (shuffledDisks []StorageAPI, shuffledPartsMetadata []FileInfo) { + shuffledDisks = make([]StorageAPI, len(disks)) + shuffledPartsMetadata = make([]FileInfo, len(disks)) + distribution := fi.Erasure.Distribution + + var inconsistent int + for i, meta := range metaArr { + if disks[i] == nil { + // Assuming offline drives as inconsistent, + // to be safe and fallback to original + // distribution order. + inconsistent++ + continue + } + if !meta.IsValid() { + inconsistent++ + continue + } + if meta.XLV1 != fi.XLV1 { + inconsistent++ + continue + } + // check if erasure distribution order matches the index + // position if this is not correct we discard the disk + // and move to collect others + if distribution[i] != meta.Erasure.Index { + inconsistent++ // keep track of inconsistent entries + continue + } + shuffledDisks[meta.Erasure.Index-1] = disks[i] + shuffledPartsMetadata[meta.Erasure.Index-1] = metaArr[i] + } + + // Inconsistent meta info is with in the limit of + // expected quorum, proceed with EcIndex based + // disk order. + if inconsistent < fi.Erasure.ParityBlocks { + return shuffledDisks, shuffledPartsMetadata + } + + // fall back to original distribution based order. + return shuffleDisksAndPartsMetadata(disks, metaArr, fi) +} + +// Return shuffled partsMetadata depending on fi.Distribution. +// additional validation is attempted and invalid metadata is +// automatically skipped only when fi.ModTime is non-zero +// indicating that this is called during read-phase +func shuffleDisksAndPartsMetadata(disks []StorageAPI, partsMetadata []FileInfo, fi FileInfo) (shuffledDisks []StorageAPI, shuffledPartsMetadata []FileInfo) { + shuffledDisks = make([]StorageAPI, len(disks)) + shuffledPartsMetadata = make([]FileInfo, len(partsMetadata)) + distribution := fi.Erasure.Distribution + + init := fi.ModTime.IsZero() + // Shuffle slice xl metadata for expected distribution. + for index := range partsMetadata { + if disks[index] == nil { + continue + } + if !init && !partsMetadata[index].IsValid() { + // Check for parts metadata validity for only + // fi.ModTime is not empty - ModTime is always set, + // if object was ever written previously. + continue + } + if !init && fi.XLV1 != partsMetadata[index].XLV1 { + continue + } + blockIndex := distribution[index] + shuffledPartsMetadata[blockIndex-1] = partsMetadata[index] + shuffledDisks[blockIndex-1] = disks[index] + } + return shuffledDisks, shuffledPartsMetadata +} + +func shuffleWithDist[T any](input []T, distribution []int) []T { + if distribution == nil { + return input + } + shuffled := make([]T, len(input)) + for index := range input { + blockIndex := distribution[index] + shuffled[blockIndex-1] = input[index] + } + return shuffled +} + +// Return shuffled partsMetadata depending on distribution. +func shufflePartsMetadata(partsMetadata []FileInfo, distribution []int) []FileInfo { + return shuffleWithDist[FileInfo](partsMetadata, distribution) +} + +// shuffleCheckParts - shuffle CheckParts slice depending on the +// erasure distribution. +func shuffleCheckParts(parts []int, distribution []int) []int { + return shuffleWithDist[int](parts, distribution) +} + +// shuffleDisks - shuffle input disks slice depending on the +// erasure distribution. Return shuffled slice of disks with +// their expected distribution. +func shuffleDisks(disks []StorageAPI, distribution []int) []StorageAPI { + return shuffleWithDist[StorageAPI](disks, distribution) +} + +// evalDisks - returns a new slice of disks where nil is set if +// the corresponding error in errs slice is not nil +func evalDisks(disks []StorageAPI, errs []error) []StorageAPI { + if len(errs) != len(disks) { + bugLogIf(GlobalContext, errors.New("unexpected drives/errors slice length")) + return nil + } + newDisks := make([]StorageAPI, len(disks)) + for index := range errs { + if errs[index] == nil { + newDisks[index] = disks[index] + } else { + newDisks[index] = nil + } + } + return newDisks +} + +// Errors specifically generated by calculatePartSizeFromIdx function. +var ( + errPartSizeZero = errors.New("Part size cannot be zero") + errPartSizeIndex = errors.New("Part index cannot be smaller than 1") +) + +// calculatePartSizeFromIdx calculates the part size according to input index. +// returns error if totalSize is -1, partSize is 0, partIndex is 0. +func calculatePartSizeFromIdx(ctx context.Context, totalSize int64, partSize int64, partIndex int) (currPartSize int64, err error) { + if totalSize < -1 { + return 0, errInvalidArgument + } + if partSize == 0 { + return 0, errPartSizeZero + } + if partIndex < 1 { + return 0, errPartSizeIndex + } + if totalSize == -1 { + return -1, nil + } + if totalSize > 0 { + // Compute the total count of parts + partsCount := totalSize/partSize + 1 + // Return the part's size + switch { + case int64(partIndex) < partsCount: + currPartSize = partSize + case int64(partIndex) == partsCount: + // Size of last part + currPartSize = totalSize % partSize + default: + currPartSize = 0 + } + } + return currPartSize, nil +} diff --git a/cmd/erasure-metadata-utils_test.go b/cmd/erasure-metadata-utils_test.go new file mode 100644 index 00000000..b14cc25d --- /dev/null +++ b/cmd/erasure-metadata-utils_test.go @@ -0,0 +1,234 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/hex" + "fmt" + "math/rand" + "reflect" + "testing" +) + +// Tests caclculating disk count. +func TestDiskCount(t *testing.T) { + testCases := []struct { + disks []StorageAPI + diskCount int + }{ + // Test case - 1 + { + disks: []StorageAPI{&xlStorage{}, &xlStorage{}, &xlStorage{}, &xlStorage{}}, + diskCount: 4, + }, + // Test case - 2 + { + disks: []StorageAPI{nil, &xlStorage{}, &xlStorage{}, &xlStorage{}}, + diskCount: 3, + }, + } + for i, testCase := range testCases { + cdiskCount := diskCount(testCase.disks) + if cdiskCount != testCase.diskCount { + t.Errorf("Test %d: Expected %d, got %d", i+1, testCase.diskCount, cdiskCount) + } + } +} + +// Test for reduceErrs, reduceErr reduces collection +// of errors into a single maximal error with in the list. +func TestReduceErrs(t *testing.T) { + canceledErrs := make([]error, 0, 5) + for i := range 5 { + canceledErrs = append(canceledErrs, fmt.Errorf("error %d: %w", i, context.Canceled)) + } + // List all of all test cases to validate various cases of reduce errors. + testCases := []struct { + errs []error + ignoredErrs []error + err error + }{ + // Validate if have reduced properly. + {[]error{ + errDiskNotFound, + errDiskNotFound, + errDiskFull, + }, []error{}, errErasureReadQuorum}, + // Validate if have no consensus. + {[]error{ + errDiskFull, + errDiskNotFound, + nil, nil, + }, []error{}, errErasureReadQuorum}, + // Validate if have consensus and errors ignored. + {[]error{ + errVolumeNotFound, + errVolumeNotFound, + errVolumeNotFound, + errVolumeNotFound, + errVolumeNotFound, + errDiskNotFound, + errDiskNotFound, + }, []error{errDiskNotFound}, errVolumeNotFound}, + {[]error{}, []error{}, errErasureReadQuorum}, + { + []error{ + errFileNotFound, errFileNotFound, errFileNotFound, + errFileNotFound, errFileNotFound, nil, nil, nil, nil, nil, + }, + nil, nil, + }, + // Checks if wrapped context cancellation errors are grouped as one. + {canceledErrs, nil, context.Canceled}, + } + // Validates list of all the testcases for returning valid errors. + for i, testCase := range testCases { + gotErr := reduceReadQuorumErrs(t.Context(), testCase.errs, testCase.ignoredErrs, 5) + if gotErr != testCase.err { + t.Errorf("Test %d : expected %s, got %s", i+1, testCase.err, gotErr) + } + gotNewErr := reduceWriteQuorumErrs(t.Context(), testCase.errs, testCase.ignoredErrs, 6) + if gotNewErr != errErasureWriteQuorum { + t.Errorf("Test %d : expected %s, got %s", i+1, errErasureWriteQuorum, gotErr) + } + } +} + +// TestHashOrder - test order of ints in array +func TestHashOrder(t *testing.T) { + testCases := []struct { + objectName string + hashedOrder []int + }{ + // cases which should pass the test. + // passing in valid object name. + {"object", []int{14, 15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13}}, + {"The Shining Script .pdf", []int{16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15}}, + {"Cost Benefit Analysis (2009-2010).pptx", []int{15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}}, + {"117Gn8rfHL2ACARPAhaFd0AGzic9pUbIA/5OCn5A", []int{3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2}}, + {"SHØRT", []int{11, 12, 13, 14, 15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}}, + {"There are far too many object names, and far too few bucket names!", []int{15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}}, + {"a/b/c/", []int{3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2}}, + {"/a/b/c", []int{6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 1, 2, 3, 4, 5}}, + {string([]byte{0xff, 0xfe, 0xfd}), []int{15, 16, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14}}, + } + + // Tests hashing order to be consistent. + for i, testCase := range testCases { + hashedOrder := hashOrder(testCase.objectName, 16) + if !reflect.DeepEqual(testCase.hashedOrder, hashedOrder) { + t.Errorf("Test case %d: Expected \"%v\" but failed \"%v\"", i+1, testCase.hashedOrder, hashedOrder) + } + } + + // Tests hashing order to fail for when order is '-1'. + if hashedOrder := hashOrder("This will fail", -1); hashedOrder != nil { + t.Errorf("Test: Expect \"nil\" but failed \"%#v\"", hashedOrder) + } + + if hashedOrder := hashOrder("This will fail", 0); hashedOrder != nil { + t.Errorf("Test: Expect \"nil\" but failed \"%#v\"", hashedOrder) + } +} + +func TestShuffleDisks(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + nDisks := 16 + disks, err := getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...)) + if err != nil { + removeRoots(disks) + t.Fatal(err) + } + defer removeRoots(disks) + z := objLayer.(*erasureServerPools) + testShuffleDisks(t, z) +} + +// Test shuffleDisks which returns shuffled slice of disks for their actual distribution. +func testShuffleDisks(t *testing.T, z *erasureServerPools) { + disks := z.serverPools[0].GetDisks(0)() + distribution := []int{16, 14, 12, 10, 8, 6, 4, 2, 1, 3, 5, 7, 9, 11, 13, 15} + shuffledDisks := shuffleDisks(disks, distribution) + // From the "distribution" above you can notice that: + // 1st data block is in the 9th disk (i.e distribution index 8) + // 2nd data block is in the 8th disk (i.e distribution index 7) and so on. + if shuffledDisks[0] != disks[8] || + shuffledDisks[1] != disks[7] || + shuffledDisks[2] != disks[9] || + shuffledDisks[3] != disks[6] || + shuffledDisks[4] != disks[10] || + shuffledDisks[5] != disks[5] || + shuffledDisks[6] != disks[11] || + shuffledDisks[7] != disks[4] || + shuffledDisks[8] != disks[12] || + shuffledDisks[9] != disks[3] || + shuffledDisks[10] != disks[13] || + shuffledDisks[11] != disks[2] || + shuffledDisks[12] != disks[14] || + shuffledDisks[13] != disks[1] || + shuffledDisks[14] != disks[15] || + shuffledDisks[15] != disks[0] { + t.Errorf("shuffleDisks returned incorrect order.") + } +} + +// TestEvalDisks tests the behavior of evalDisks +func TestEvalDisks(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + nDisks := 16 + disks, err := getRandomDisks(nDisks) + if err != nil { + t.Fatal(err) + } + objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, disks...)) + if err != nil { + removeRoots(disks) + t.Fatal(err) + } + defer removeRoots(disks) + z := objLayer.(*erasureServerPools) + testShuffleDisks(t, z) +} + +func Test_hashOrder(t *testing.T) { + for x := 1; x < 17; x++ { + t.Run(fmt.Sprintf("%d", x), func(t *testing.T) { + var first [17]int + rng := rand.New(rand.NewSource(0)) + var tmp [16]byte + rng.Read(tmp[:]) + prefix := hex.EncodeToString(tmp[:]) + for range 10000 { + rng.Read(tmp[:]) + + y := hashOrder(fmt.Sprintf("%s/%x", prefix, hex.EncodeToString(tmp[:3])), x) + first[y[0]]++ + } + t.Log("first:", first[:x]) + }) + } +} diff --git a/cmd/erasure-metadata.go b/cmd/erasure-metadata.go new file mode 100644 index 00000000..f4b8798b --- /dev/null +++ b/cmd/erasure-metadata.go @@ -0,0 +1,684 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/hex" + "fmt" + "sort" + "strings" + "time" + + "github.com/minio/minio/internal/amztime" + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/hash/sha256" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/pkg/v3/sync/errgroup" +) + +// Object was stored with additional erasure codes due to degraded system at upload time +const minIOErasureUpgraded = "x-minio-internal-erasure-upgraded" + +const erasureAlgorithm = "rs-vandermonde" + +// GetChecksumInfo - get checksum of a part. +func (e ErasureInfo) GetChecksumInfo(partNumber int) (ckSum ChecksumInfo) { + for _, sum := range e.Checksums { + if sum.PartNumber == partNumber { + // Return the checksum + return sum + } + } + return ChecksumInfo{Algorithm: DefaultBitrotAlgorithm} +} + +// ShardFileSize - returns final erasure size from original size. +func (e ErasureInfo) ShardFileSize(totalLength int64) int64 { + if totalLength == 0 { + return 0 + } + if totalLength == -1 { + return -1 + } + numShards := totalLength / e.BlockSize + lastBlockSize := totalLength % e.BlockSize + lastShardSize := ceilFrac(lastBlockSize, int64(e.DataBlocks)) + return numShards*e.ShardSize() + lastShardSize +} + +// ShardSize - returns actual shared size from erasure blockSize. +func (e ErasureInfo) ShardSize() int64 { + return ceilFrac(e.BlockSize, int64(e.DataBlocks)) +} + +// IsValid - tells if erasure info fields are valid. +func (fi FileInfo) IsValid() bool { + if fi.Deleted { + // Delete marker has no data, no need to check + // for erasure coding information + return true + } + dataBlocks := fi.Erasure.DataBlocks + parityBlocks := fi.Erasure.ParityBlocks + correctIndexes := (fi.Erasure.Index > 0 && + fi.Erasure.Index <= dataBlocks+parityBlocks && + len(fi.Erasure.Distribution) == (dataBlocks+parityBlocks)) + return ((dataBlocks >= parityBlocks) && + (dataBlocks > 0) && (parityBlocks >= 0) && + correctIndexes) +} + +// ToObjectInfo - Converts metadata to object info. +func (fi FileInfo) ToObjectInfo(bucket, object string, versioned bool) ObjectInfo { + object = decodeDirObject(object) + versionID := fi.VersionID + if versioned && versionID == "" { + versionID = nullVersionID + } + + objInfo := ObjectInfo{ + IsDir: HasSuffix(object, SlashSeparator), + Bucket: bucket, + Name: object, + ParityBlocks: fi.Erasure.ParityBlocks, + DataBlocks: fi.Erasure.DataBlocks, + VersionID: versionID, + IsLatest: fi.IsLatest, + DeleteMarker: fi.Deleted, + Size: fi.Size, + ModTime: fi.ModTime, + Legacy: fi.XLV1, + ContentType: fi.Metadata["content-type"], + ContentEncoding: fi.Metadata["content-encoding"], + NumVersions: fi.NumVersions, + SuccessorModTime: fi.SuccessorModTime, + CacheControl: fi.Metadata["cache-control"], + } + + if exp, ok := fi.Metadata["expires"]; ok { + if t, err := amztime.ParseHeader(exp); err == nil { + objInfo.Expires = t.UTC() + } + } + + // Extract etag from metadata. + objInfo.ETag = extractETag(fi.Metadata) + + // Add user tags to the object info + tags := fi.Metadata[xhttp.AmzObjectTagging] + if len(tags) != 0 { + objInfo.UserTags = tags + } + + // Add replication status to the object info + objInfo.ReplicationStatusInternal = fi.ReplicationState.ReplicationStatusInternal + objInfo.VersionPurgeStatusInternal = fi.ReplicationState.VersionPurgeStatusInternal + objInfo.ReplicationStatus = fi.ReplicationStatus() + if objInfo.ReplicationStatus.Empty() { // overlay x-amx-replication-status if present for replicas + if st, ok := fi.Metadata[xhttp.AmzBucketReplicationStatus]; ok && st == string(replication.Replica) { + objInfo.ReplicationStatus = replication.StatusType(st) + } + } + objInfo.VersionPurgeStatus = fi.VersionPurgeStatus() + + objInfo.TransitionedObject = TransitionedObject{ + Name: fi.TransitionedObjName, + VersionID: fi.TransitionVersionID, + Status: fi.TransitionStatus, + FreeVersion: fi.TierFreeVersion(), + Tier: fi.TransitionTier, + } + + // etag/md5Sum has already been extracted. We need to + // remove to avoid it from appearing as part of + // response headers. e.g, X-Minio-* or X-Amz-*. + // Tags have also been extracted, we remove that as well. + objInfo.UserDefined = cleanMetadata(fi.Metadata) + + // All the parts per object. + objInfo.Parts = fi.Parts + + // Update storage class + if fi.TransitionTier != "" { + objInfo.StorageClass = fi.TransitionTier + } else if sc, ok := fi.Metadata[xhttp.AmzStorageClass]; ok { + objInfo.StorageClass = sc + } else { + objInfo.StorageClass = globalMinioDefaultStorageClass + } + + // set restore status for transitioned object + restoreHdr, ok := fi.Metadata[xhttp.AmzRestore] + if ok { + if restoreStatus, err := parseRestoreObjStatus(restoreHdr); err == nil { + objInfo.RestoreOngoing = restoreStatus.Ongoing() + objInfo.RestoreExpires, _ = restoreStatus.Expiry() + } + } + objInfo.Checksum = fi.Checksum + objInfo.decryptPartsChecksums(nil) + objInfo.Inlined = fi.InlineData() + // Success. + return objInfo +} + +// TransitionInfoEquals returns true if transition related information are equal, false otherwise. +func (fi FileInfo) TransitionInfoEquals(ofi FileInfo) bool { + switch { + case fi.TransitionStatus != ofi.TransitionStatus, + fi.TransitionTier != ofi.TransitionTier, + fi.TransitionedObjName != ofi.TransitionedObjName, + fi.TransitionVersionID != ofi.TransitionVersionID: + return false + } + return true +} + +// MetadataEquals returns true if FileInfos Metadata maps are equal, false otherwise. +func (fi FileInfo) MetadataEquals(ofi FileInfo) bool { + if len(fi.Metadata) != len(ofi.Metadata) { + return false + } + for k, v := range fi.Metadata { + if ov, ok := ofi.Metadata[k]; !ok || ov != v { + return false + } + } + return true +} + +// ReplicationInfoEquals returns true if server-side replication related fields are equal, false otherwise. +func (fi FileInfo) ReplicationInfoEquals(ofi FileInfo) bool { + switch { + case fi.MarkDeleted != ofi.MarkDeleted, + !fi.ReplicationState.Equal(ofi.ReplicationState): + return false + } + return true +} + +// objectPartIndex - returns the index of matching object part number. +// Returns -1 if the part cannot be found. +func objectPartIndex(parts []ObjectPartInfo, partNumber int) int { + for i, part := range parts { + if partNumber == part.Number { + return i + } + } + return -1 +} + +// objectPartIndexNums returns the index of the specified part number. +// Returns -1 if the part cannot be found. +func objectPartIndexNums(parts []int, partNumber int) int { + for i, part := range parts { + if part != 0 && partNumber == part { + return i + } + } + return -1 +} + +// AddObjectPart - add a new object part in order. +func (fi *FileInfo) AddObjectPart(partNumber int, partETag string, partSize, actualSize int64, modTime time.Time, idx []byte, checksums map[string]string) { + partInfo := ObjectPartInfo{ + Number: partNumber, + ETag: partETag, + Size: partSize, + ActualSize: actualSize, + ModTime: modTime, + Index: idx, + Checksums: checksums, + } + + // Update part info if it already exists. + for i, part := range fi.Parts { + if partNumber == part.Number { + fi.Parts[i] = partInfo + return + } + } + + // Proceed to include new part info. + fi.Parts = append(fi.Parts, partInfo) + + // Parts in FileInfo should be in sorted order by part number. + sort.Slice(fi.Parts, func(i, j int) bool { return fi.Parts[i].Number < fi.Parts[j].Number }) +} + +// ObjectToPartOffset - translate offset of an object to offset of its individual part. +func (fi FileInfo) ObjectToPartOffset(ctx context.Context, offset int64) (partIndex int, partOffset int64, err error) { + if offset == 0 { + // Special case - if offset is 0, then partIndex and partOffset are always 0. + return 0, 0, nil + } + partOffset = offset + // Seek until object offset maps to a particular part offset. + for i, part := range fi.Parts { + partIndex = i + // Offset is smaller than size we have reached the proper part offset. + if partOffset < part.Size { + return partIndex, partOffset, nil + } + // Continue to towards the next part. + partOffset -= part.Size + } + internalLogIf(ctx, InvalidRange{}) + // Offset beyond the size of the object return InvalidRange. + return 0, 0, InvalidRange{} +} + +func findFileInfoInQuorum(ctx context.Context, metaArr []FileInfo, modTime time.Time, etag string, quorum int) (FileInfo, error) { + // with less quorum return error. + if quorum < 1 { + return FileInfo{}, InsufficientReadQuorum{Err: errErasureReadQuorum, Type: RQInsufficientOnlineDrives} + } + metaHashes := make([]string, len(metaArr)) + h := sha256.New() + for i, meta := range metaArr { + if !meta.IsValid() { + continue + } + etagOnly := modTime.Equal(timeSentinel) && (etag != "" && etag == meta.Metadata["etag"]) + mtimeValid := meta.ModTime.Equal(modTime) + if mtimeValid || etagOnly { + fmt.Fprintf(h, "%v", meta.XLV1) + for _, part := range meta.Parts { + fmt.Fprintf(h, "part.%d", part.Number) + fmt.Fprintf(h, "part.%d", part.Size) + } + // Previously we checked if we had quorum on DataDir value. + // We have removed this check to allow reading objects with different DataDir + // values in a few drives (due to a rebalance-stop race bug) + // provided their their etags or ModTimes match. + + if !meta.Deleted && meta.Size != 0 { + fmt.Fprintf(h, "%v+%v", meta.Erasure.DataBlocks, meta.Erasure.ParityBlocks) + fmt.Fprintf(h, "%v", meta.Erasure.Distribution) + } + + if meta.IsRemote() { + // ILM transition fields + fmt.Fprint(h, meta.TransitionStatus) + fmt.Fprint(h, meta.TransitionTier) + fmt.Fprint(h, meta.TransitionedObjName) + fmt.Fprint(h, meta.TransitionVersionID) + } + + // If metadata says encrypted, ask for it in quorum. + if etyp, ok := crypto.IsEncrypted(meta.Metadata); ok { + fmt.Fprint(h, etyp) + } + + // If compressed, look for compressed FileInfo only + if meta.IsCompressed() { + fmt.Fprint(h, meta.Metadata[ReservedMetadataPrefix+"compression"]) + } + + metaHashes[i] = hex.EncodeToString(h.Sum(nil)) + h.Reset() + } + } + + metaHashCountMap := make(map[string]int) + for _, hash := range metaHashes { + if hash == "" { + continue + } + metaHashCountMap[hash]++ + } + + maxHash := "" + maxCount := 0 + for hash, count := range metaHashCountMap { + if count > maxCount { + maxCount = count + maxHash = hash + } + } + + if maxCount < quorum { + return FileInfo{}, InsufficientReadQuorum{Err: errErasureReadQuorum, Type: RQInconsistentMeta} + } + + // objProps represents properties that go beyond a single version + type objProps struct { + succModTime time.Time + numVersions int + } + // Find the successor mod time and numVersions in quorum, otherwise leave the + // candidate as found + otherPropsMap := make(counterMap[objProps]) + var candidate FileInfo + var found bool + for i, hash := range metaHashes { + if hash == maxHash { + if metaArr[i].IsValid() { + if !found { + candidate = metaArr[i] + found = true + } + props := objProps{ + succModTime: metaArr[i].SuccessorModTime, + numVersions: metaArr[i].NumVersions, + } + otherPropsMap[props]++ + } + } + } + + if found { + // Update candidate FileInfo with succModTime and numVersions in quorum when available + if props, ok := otherPropsMap.GetValueWithQuorum(quorum); ok { + candidate.SuccessorModTime = props.succModTime + candidate.IsLatest = props.succModTime.IsZero() + candidate.NumVersions = props.numVersions + } + return candidate, nil + } + return FileInfo{}, InsufficientReadQuorum{Err: errErasureReadQuorum, Type: RQInconsistentMeta} +} + +// pickValidFileInfo - picks one valid FileInfo content and returns from a +// slice of FileInfo. +func pickValidFileInfo(ctx context.Context, metaArr []FileInfo, modTime time.Time, etag string, quorum int) (FileInfo, error) { + return findFileInfoInQuorum(ctx, metaArr, modTime, etag, quorum) +} + +func writeAllMetadataWithRevert(ctx context.Context, disks []StorageAPI, origbucket, bucket, prefix string, files []FileInfo, quorum int, revert bool) ([]StorageAPI, error) { + g := errgroup.WithNErrs(len(disks)) + + // Start writing `xl.meta` to all disks in parallel. + for index := range disks { + g.Go(func() error { + if disks[index] == nil { + return errDiskNotFound + } + // Pick one FileInfo for a disk at index. + fi := files[index] + fi.Erasure.Index = index + 1 + if fi.IsValid() { + return disks[index].WriteMetadata(ctx, origbucket, bucket, prefix, fi) + } + return errFileCorrupt + }, index) + } + + // Wait for all the routines. + mErrs := g.Wait() + + err := reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, quorum) + if err != nil && revert { + ng := errgroup.WithNErrs(len(disks)) + for index := range disks { + if mErrs[index] != nil { + continue + } + index := index + ng.Go(func() error { + if disks[index] == nil { + return errDiskNotFound + } + return disks[index].Delete(ctx, bucket, pathJoin(prefix, xlStorageFormatFile), DeleteOptions{ + Recursive: true, + }) + }, index) + } + ng.Wait() + } + + return evalDisks(disks, mErrs), err +} + +func writeAllMetadata(ctx context.Context, disks []StorageAPI, origbucket, bucket, prefix string, files []FileInfo, quorum int) ([]StorageAPI, error) { + return writeAllMetadataWithRevert(ctx, disks, origbucket, bucket, prefix, files, quorum, true) +} + +// writeUniqueFileInfo - writes unique `xl.meta` content for each disk concurrently. +func writeUniqueFileInfo(ctx context.Context, disks []StorageAPI, origbucket, bucket, prefix string, files []FileInfo, quorum int) ([]StorageAPI, error) { + return writeAllMetadataWithRevert(ctx, disks, origbucket, bucket, prefix, files, quorum, false) +} + +func commonParity(parities []int, defaultParityCount int) int { + N := len(parities) + + occMap := make(map[int]int) + for _, p := range parities { + occMap[p]++ + } + + var maxOcc, cparity int + for parity, occ := range occMap { + if parity == -1 { + // Ignore non defined parity + continue + } + + readQuorum := N - parity + if defaultParityCount > 0 && parity == 0 { + // In this case, parity == 0 implies that this object version is a + // delete marker + readQuorum = N/2 + 1 + } + if occ < readQuorum { + // Ignore this parity since we don't have enough shards for read quorum + continue + } + + if occ > maxOcc { + maxOcc = occ + cparity = parity + } + } + + if maxOcc == 0 { + // Did not found anything useful + return -1 + } + return cparity +} + +func listObjectParities(partsMetadata []FileInfo, errs []error) (parities []int) { + totalShards := len(partsMetadata) + parities = make([]int, len(partsMetadata)) + for index, metadata := range partsMetadata { + if errs[index] != nil { + parities[index] = -1 + continue + } + if !metadata.IsValid() { + parities[index] = -1 + continue + } + //nolint:gocritic + // Delete marker or zero byte objects take highest parity. + if metadata.Deleted || metadata.Size == 0 { + parities[index] = totalShards / 2 + } else if metadata.TransitionStatus == lifecycle.TransitionComplete { + // For tiered objects, read quorum is N/2+1 to ensure simple majority on xl.meta. + // It is not equal to EcM because the data integrity is entrusted with the warm tier. + // However, we never go below EcM, in case of a EcM=EcN setup. + parities[index] = max(totalShards-(totalShards/2+1), metadata.Erasure.ParityBlocks) + } else { + parities[index] = metadata.Erasure.ParityBlocks + } + } + return parities +} + +// Returns per object readQuorum and writeQuorum +// readQuorum is the min required disks to read data. +// writeQuorum is the min required disks to write data. +func objectQuorumFromMeta(ctx context.Context, partsMetaData []FileInfo, errs []error, defaultParityCount int) (objectReadQuorum, objectWriteQuorum int, err error) { + // There should be at least half correct entries, if not return failure + expectedRQuorum := len(partsMetaData) / 2 + if defaultParityCount == 0 { + // if parity count is '0', we expected all entries to be present. + expectedRQuorum = len(partsMetaData) + } + + reducedErr := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, expectedRQuorum) + if reducedErr != nil { + return -1, -1, reducedErr + } + + // special case when parity is '0' + if defaultParityCount == 0 { + return len(partsMetaData), len(partsMetaData), nil + } + + parities := listObjectParities(partsMetaData, errs) + parityBlocks := commonParity(parities, defaultParityCount) + if parityBlocks < 0 { + return -1, -1, InsufficientReadQuorum{Err: errErasureReadQuorum, Type: RQInsufficientOnlineDrives} + } + + dataBlocks := len(partsMetaData) - parityBlocks + + writeQuorum := dataBlocks + if dataBlocks == parityBlocks { + writeQuorum++ + } + + // Since all the valid erasure code meta updated at the same time are equivalent, pass dataBlocks + // from latestFileInfo to get the quorum + return dataBlocks, writeQuorum, nil +} + +const ( + tierFVID = "tier-free-versionID" + tierFVMarker = "tier-free-marker" + tierSkipFVID = "tier-skip-fvid" +) + +// SetTierFreeVersionID sets free-version's versionID. This method is used by +// object layer to pass down a versionID to set for a free-version that may be +// created. +func (fi *FileInfo) SetTierFreeVersionID(versionID string) { + if fi.Metadata == nil { + fi.Metadata = make(map[string]string) + } + fi.Metadata[ReservedMetadataPrefixLower+tierFVID] = versionID +} + +// TierFreeVersionID returns the free-version's version id. +func (fi *FileInfo) TierFreeVersionID() string { + return fi.Metadata[ReservedMetadataPrefixLower+tierFVID] +} + +// SetTierFreeVersion sets fi as a free-version. This method is used by +// lower layers to indicate a free-version. +func (fi *FileInfo) SetTierFreeVersion() { + if fi.Metadata == nil { + fi.Metadata = make(map[string]string) + } + fi.Metadata[ReservedMetadataPrefixLower+tierFVMarker] = "" +} + +// SetSkipTierFreeVersion indicates to skip adding a tier free version id. +// Note: Used only when expiring tiered objects and the remote content has +// already been scheduled for deletion +func (fi *FileInfo) SetSkipTierFreeVersion() { + if fi.Metadata == nil { + fi.Metadata = make(map[string]string) + } + fi.Metadata[ReservedMetadataPrefixLower+tierSkipFVID] = "" +} + +// SkipTierFreeVersion returns true if set, false otherwise. +// See SetSkipTierVersion for its purpose. +func (fi *FileInfo) SkipTierFreeVersion() bool { + _, ok := fi.Metadata[ReservedMetadataPrefixLower+tierSkipFVID] + return ok +} + +// TierFreeVersion returns true if version is a free-version. +func (fi *FileInfo) TierFreeVersion() bool { + _, ok := fi.Metadata[ReservedMetadataPrefixLower+tierFVMarker] + return ok +} + +// IsRestoreObjReq returns true if fi corresponds to a RestoreObject request. +func (fi *FileInfo) IsRestoreObjReq() bool { + if restoreHdr, ok := fi.Metadata[xhttp.AmzRestore]; ok { + if restoreStatus, err := parseRestoreObjStatus(restoreHdr); err == nil { + if !restoreStatus.Ongoing() { + return true + } + } + } + return false +} + +// VersionPurgeStatus returns overall version purge status for this object version across targets +func (fi *FileInfo) VersionPurgeStatus() VersionPurgeStatusType { + return fi.ReplicationState.CompositeVersionPurgeStatus() +} + +// ReplicationStatus returns overall version replication status for this object version across targets +func (fi *FileInfo) ReplicationStatus() replication.StatusType { + return fi.ReplicationState.CompositeReplicationStatus() +} + +// DeleteMarkerReplicationStatus returns overall replication status for this delete marker version across targets +func (fi *FileInfo) DeleteMarkerReplicationStatus() replication.StatusType { + if fi.Deleted { + return fi.ReplicationState.CompositeReplicationStatus() + } + return replication.StatusType("") +} + +// GetInternalReplicationState is a wrapper method to fetch internal replication state from the map m +func GetInternalReplicationState(m map[string][]byte) ReplicationState { + m1 := make(map[string]string, len(m)) + for k, v := range m { + m1[k] = string(v) + } + return getInternalReplicationState(m1) +} + +// getInternalReplicationState fetches internal replication state from the map m +func getInternalReplicationState(m map[string]string) ReplicationState { + d := ReplicationState{} + for k, v := range m { + switch { + case equals(k, ReservedMetadataPrefixLower+ReplicationTimestamp): + d.ReplicaTimeStamp, _ = amztime.ParseReplicationTS(v) + case equals(k, ReservedMetadataPrefixLower+ReplicaTimestamp): + d.ReplicaTimeStamp, _ = amztime.ParseReplicationTS(v) + case equals(k, ReservedMetadataPrefixLower+ReplicaStatus): + d.ReplicaStatus = replication.StatusType(v) + case equals(k, ReservedMetadataPrefixLower+ReplicationStatus): + d.ReplicationStatusInternal = v + d.Targets = replicationStatusesMap(v) + case equals(k, VersionPurgeStatusKey): + d.VersionPurgeStatusInternal = v + d.PurgeTargets = versionPurgeStatusesMap(v) + case strings.HasPrefix(k, ReservedMetadataPrefixLower+ReplicationReset): + arn := strings.TrimPrefix(k, fmt.Sprintf("%s-", ReservedMetadataPrefixLower+ReplicationReset)) + if d.ResetStatusesMap == nil { + d.ResetStatusesMap = make(map[string]string, 1) + } + d.ResetStatusesMap[arn] = v + } + } + return d +} diff --git a/cmd/erasure-metadata_test.go b/cmd/erasure-metadata_test.go new file mode 100644 index 00000000..76ee2e10 --- /dev/null +++ b/cmd/erasure-metadata_test.go @@ -0,0 +1,482 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "slices" + "strconv" + "testing" + "time" + + "github.com/dustin/go-humanize" +) + +const ActualSize = 1000 + +// Test FileInfo.AddObjectPart() +func TestAddObjectPart(t *testing.T) { + testCases := []struct { + partNum int + expectedIndex int + }{ + {1, 0}, + {2, 1}, + {4, 2}, + {5, 3}, + {7, 4}, + // Insert part. + {3, 2}, + // Replace existing part. + {4, 3}, + // Missing part. + {6, -1}, + } + + // Setup. + fi := newFileInfo("test-object", 8, 8) + fi.Erasure.Index = 1 + if !fi.IsValid() { + t.Fatalf("unable to get xl meta") + } + + // Test them. + for _, testCase := range testCases { + if testCase.expectedIndex > -1 { + partNumString := strconv.Itoa(testCase.partNum) + fi.AddObjectPart(testCase.partNum, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize, UTCNow(), nil, nil) + } + + if index := objectPartIndex(fi.Parts, testCase.partNum); index != testCase.expectedIndex { + t.Fatalf("%+v: expected = %d, got: %d", testCase, testCase.expectedIndex, index) + } + } +} + +// Test objectPartIndex(). generates a sample FileInfo data and asserts +// the output of objectPartIndex() with the expected value. +func TestObjectPartIndex(t *testing.T) { + testCases := []struct { + partNum int + expectedIndex int + }{ + {2, 1}, + {1, 0}, + {5, 3}, + {4, 2}, + {7, 4}, + } + + // Setup. + fi := newFileInfo("test-object", 8, 8) + fi.Erasure.Index = 1 + if !fi.IsValid() { + t.Fatalf("unable to get xl meta") + } + + // Add some parts for testing. + for _, testCase := range testCases { + partNumString := strconv.Itoa(testCase.partNum) + fi.AddObjectPart(testCase.partNum, "etag."+partNumString, int64(testCase.partNum+humanize.MiByte), ActualSize, UTCNow(), nil, nil) + } + + // Add failure test case. + testCases = append(testCases, struct { + partNum int + expectedIndex int + }{6, -1}) + + // Test them. + for _, testCase := range testCases { + if index := objectPartIndex(fi.Parts, testCase.partNum); index != testCase.expectedIndex { + t.Fatalf("%+v: expected = %d, got: %d", testCase, testCase.expectedIndex, index) + } + } +} + +// Test FileInfo.ObjectToPartOffset(). +func TestObjectToPartOffset(t *testing.T) { + // Setup. + fi := newFileInfo("test-object", 8, 8) + fi.Erasure.Index = 1 + if !fi.IsValid() { + t.Fatalf("unable to get xl meta") + } + + // Add some parts for testing. + // Total size of all parts is 5,242,899 bytes. + for _, partNum := range []int{1, 2, 4, 5, 7} { + partNumString := strconv.Itoa(partNum) + fi.AddObjectPart(partNum, "etag."+partNumString, int64(partNum+humanize.MiByte), ActualSize, UTCNow(), nil, nil) + } + + testCases := []struct { + offset int64 + expectedIndex int + expectedOffset int64 + expectedErr error + }{ + {0, 0, 0, nil}, + {1 * humanize.MiByte, 0, 1 * humanize.MiByte, nil}, + {1 + humanize.MiByte, 1, 0, nil}, + {2 + humanize.MiByte, 1, 1, nil}, + // Its valid for zero sized object. + {-1, 0, -1, nil}, + // Max fffset is always (size - 1). + {(1 + 2 + 4 + 5 + 7) + (5 * humanize.MiByte) - 1, 4, 1048582, nil}, + // Error if offset is size. + {(1 + 2 + 4 + 5 + 7) + (5 * humanize.MiByte), 0, 0, InvalidRange{}}, + } + + // Test them. + for _, testCase := range testCases { + index, offset, err := fi.ObjectToPartOffset(t.Context(), testCase.offset) + if err != testCase.expectedErr { + t.Fatalf("%+v: expected = %s, got: %s", testCase, testCase.expectedErr, err) + } + if index != testCase.expectedIndex { + t.Fatalf("%+v: index: expected = %d, got: %d", testCase, testCase.expectedIndex, index) + } + if offset != testCase.expectedOffset { + t.Fatalf("%+v: offset: expected = %d, got: %d", testCase, testCase.expectedOffset, offset) + } + } +} + +func TestFindFileInfoInQuorum(t *testing.T) { + getNFInfo := func(n int, quorum int, t int64, dataDir string, succModTimes []time.Time, numVersions []int) []FileInfo { + fi := newFileInfo("test", 8, 8) + fi.AddObjectPart(1, "etag", 100, 100, UTCNow(), nil, nil) + fi.ModTime = time.Unix(t, 0) + fi.DataDir = dataDir + fis := make([]FileInfo, n) + for i := range fis { + fis[i] = fi + fis[i].Erasure.Index = i + 1 + if succModTimes != nil { + fis[i].SuccessorModTime = succModTimes[i] + fis[i].IsLatest = succModTimes[i].IsZero() + } + if numVersions != nil { + fis[i].NumVersions = numVersions[i] + } + quorum-- + if quorum == 0 { + break + } + } + return fis + } + + commonSuccModTime := time.Date(2023, time.August, 25, 0, 0, 0, 0, time.UTC) + succModTimesInQuorum := make([]time.Time, 16) + succModTimesNoQuorum := make([]time.Time, 16) + commonNumVersions := 2 + numVersionsInQuorum := make([]int, 16) + numVersionsNoQuorum := make([]int, 16) + for i := range 16 { + if i < 4 { + continue + } + succModTimesInQuorum[i] = commonSuccModTime + numVersionsInQuorum[i] = commonNumVersions + if i < 9 { + continue + } + succModTimesNoQuorum[i] = commonSuccModTime + numVersionsNoQuorum[i] = commonNumVersions + } + tests := []struct { + fis []FileInfo + modTime time.Time + succmodTimes []time.Time + numVersions []int + expectedErr error + expectedQuorum int + expectedSuccModTime time.Time + expectedNumVersions int + expectedIsLatest bool + }{ + { + fis: getNFInfo(16, 16, 1603863445, "36a21454-a2ca-11eb-bbaa-93a81c686f21", nil, nil), + modTime: time.Unix(1603863445, 0), + expectedErr: nil, + expectedQuorum: 8, + }, + { + fis: getNFInfo(16, 7, 1603863445, "36a21454-a2ca-11eb-bbaa-93a81c686f21", nil, nil), + modTime: time.Unix(1603863445, 0), + expectedErr: InsufficientReadQuorum{}, + expectedQuorum: 8, + }, + { + fis: getNFInfo(16, 16, 1603863445, "36a21454-a2ca-11eb-bbaa-93a81c686f21", nil, nil), + modTime: time.Unix(1603863445, 0), + expectedErr: InsufficientReadQuorum{}, + expectedQuorum: 0, + }, + { + fis: getNFInfo(16, 16, 1603863445, "36a21454-a2ca-11eb-bbaa-93a81c686f21", succModTimesInQuorum, nil), + modTime: time.Unix(1603863445, 0), + succmodTimes: succModTimesInQuorum, + expectedErr: nil, + expectedQuorum: 12, + expectedSuccModTime: commonSuccModTime, + expectedIsLatest: false, + }, + { + fis: getNFInfo(16, 16, 1603863445, "36a21454-a2ca-11eb-bbaa-93a81c686f21", succModTimesNoQuorum, nil), + modTime: time.Unix(1603863445, 0), + succmodTimes: succModTimesNoQuorum, + expectedErr: nil, + expectedQuorum: 12, + expectedSuccModTime: time.Time{}, + expectedIsLatest: true, + }, + { + fis: getNFInfo(16, 16, 1603863445, "36a21454-a2ca-11eb-bbaa-93a81c686f21", nil, numVersionsInQuorum), + modTime: time.Unix(1603863445, 0), + numVersions: numVersionsInQuorum, + expectedErr: nil, + expectedQuorum: 12, + expectedIsLatest: true, + expectedNumVersions: 2, + }, + { + fis: getNFInfo(16, 16, 1603863445, "36a21454-a2ca-11eb-bbaa-93a81c686f21", nil, numVersionsNoQuorum), + modTime: time.Unix(1603863445, 0), + numVersions: numVersionsNoQuorum, + expectedErr: nil, + expectedQuorum: 12, + expectedIsLatest: true, + expectedNumVersions: 0, + }, + } + + for _, test := range tests { + t.Run("", func(t *testing.T) { + fi, err := findFileInfoInQuorum(t.Context(), test.fis, test.modTime, "", test.expectedQuorum) + _, ok1 := err.(InsufficientReadQuorum) + _, ok2 := test.expectedErr.(InsufficientReadQuorum) + if ok1 != ok2 { + t.Errorf("Expected %s, got %s", test.expectedErr, err) + } + if test.succmodTimes != nil { + if !test.expectedSuccModTime.Equal(fi.SuccessorModTime) { + t.Errorf("Expected successor mod time to be %v but got %v", test.expectedSuccModTime, fi.SuccessorModTime) + } + if test.expectedIsLatest != fi.IsLatest { + t.Errorf("Expected IsLatest to be %v but got %v", test.expectedIsLatest, fi.IsLatest) + } + } + if test.numVersions != nil && test.expectedNumVersions > 0 { + if test.expectedNumVersions != fi.NumVersions { + t.Errorf("Expected Numversions to be %d but got %d", test.expectedNumVersions, fi.NumVersions) + } + } + }) + } +} + +func TestTransitionInfoEquals(t *testing.T) { + inputs := []struct { + tier string + remoteObjName string + remoteVersionID string + status string + }{ + { + tier: "S3TIER-1", + remoteObjName: mustGetUUID(), + remoteVersionID: mustGetUUID(), + status: "complete", + }, + { + tier: "S3TIER-2", + remoteObjName: mustGetUUID(), + remoteVersionID: mustGetUUID(), + status: "complete", + }, + } + + var i uint + for i = range uint(8) { + fi := FileInfo{ + TransitionTier: inputs[0].tier, + TransitionedObjName: inputs[0].remoteObjName, + TransitionVersionID: inputs[0].remoteVersionID, + TransitionStatus: inputs[0].status, + } + ofi := fi + if i&(1<<0) != 0 { + ofi.TransitionTier = inputs[1].tier + } + if i&(1<<1) != 0 { + ofi.TransitionedObjName = inputs[1].remoteObjName + } + if i&(1<<2) != 0 { + ofi.TransitionVersionID = inputs[1].remoteVersionID + } + actual := fi.TransitionInfoEquals(ofi) + if i == 0 && !actual { + t.Fatalf("Test %d: Expected FileInfo's transition info to be equal: fi %v ofi %v", i, fi, ofi) + } + if i != 0 && actual { + t.Fatalf("Test %d: Expected FileInfo's transition info to be inequal: fi %v ofi %v", i, fi, ofi) + } + } + fi := FileInfo{ + TransitionTier: inputs[0].tier, + TransitionedObjName: inputs[0].remoteObjName, + TransitionVersionID: inputs[0].remoteVersionID, + TransitionStatus: inputs[0].status, + } + ofi := FileInfo{} + if fi.TransitionInfoEquals(ofi) { + t.Fatalf("Expected to be inequal: fi %v ofi %v", fi, ofi) + } +} + +func TestSkipTierFreeVersion(t *testing.T) { + fi := newFileInfo("object", 8, 8) + fi.SetSkipTierFreeVersion() + if ok := fi.SkipTierFreeVersion(); !ok { + t.Fatal("Expected SkipTierFreeVersion to be set on FileInfo but wasn't") + } +} + +func TestListObjectParities(t *testing.T) { + mkMetaArr := func(N, parity, agree int) []FileInfo { + fi := newFileInfo("obj-1", N-parity, parity) + fi.TransitionTier = "WARM-TIER" + fi.TransitionedObjName = mustGetUUID() + fi.TransitionStatus = "complete" + fi.Size = 1 << 20 + + metaArr := make([]FileInfo, N) + for i := range N { + fi.Erasure.Index = i + 1 + metaArr[i] = fi + if i < agree { + continue + } + metaArr[i].TransitionTier, metaArr[i].TransitionedObjName = "", "" + metaArr[i].TransitionStatus = "" + } + return metaArr + } + mkParities := func(N, agreedParity, disagreedParity, agree int) []int { + ps := make([]int, N) + for i := range N { + if i < agree { + ps[i] = agreedParity + continue + } + ps[i] = disagreedParity // disagree + } + return ps + } + + mkTest := func(N, parity, agree int) (res struct { + metaArr []FileInfo + errs []error + parities []int + parity int + }, + ) { + res.metaArr = mkMetaArr(N, parity, agree) + res.parities = mkParities(N, N-(N/2+1), parity, agree) + res.errs = make([]error, N) + if agree >= N/2+1 { // simple majority consensus + res.parity = N - (N/2 + 1) + } else { + res.parity = -1 + } + return res + } + + nonTieredTest := func(N, parity, agree int) (res struct { + metaArr []FileInfo + errs []error + parities []int + parity int + }, + ) { + fi := newFileInfo("obj-1", N-parity, parity) + fi.Size = 1 << 20 + metaArr := make([]FileInfo, N) + parities := make([]int, N) + for i := range N { + fi.Erasure.Index = i + 1 + metaArr[i] = fi + parities[i] = parity + if i < agree { + continue + } + metaArr[i].Erasure.Index = 0 // creates invalid fi on remaining drives + parities[i] = -1 // invalid fi are assigned parity -1 + } + res.metaArr = metaArr + res.parities = parities + res.errs = make([]error, N) + if agree >= N-parity { + res.parity = parity + } else { + res.parity = -1 + } + + return res + } + tests := []struct { + metaArr []FileInfo + errs []error + parities []int + parity int + }{ + // More than simple majority consensus + mkTest(15, 3, 11), + // No simple majority consensus + mkTest(15, 3, 7), + // Exact simple majority consensus + mkTest(15, 3, 8), + // More than simple majority consensus + mkTest(16, 4, 11), + // No simple majority consensus + mkTest(16, 4, 8), + // Exact simple majority consensus + mkTest(16, 4, 9), + // non-tiered object require read quorum of EcM + nonTieredTest(15, 3, 12), + // non-tiered object with fewer than EcM in consensus + nonTieredTest(15, 3, 11), + // non-tiered object require read quorum of EcM + nonTieredTest(16, 4, 12), + // non-tiered object with fewer than EcM in consensus + nonTieredTest(16, 4, 11), + } + for i, test := range tests { + t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) { + if got := listObjectParities(test.metaArr, test.errs); !slices.Equal(got, test.parities) { + t.Fatalf("Expected parities %v but got %v", test.parities, got) + } + if got := commonParity(test.parities, len(test.metaArr)/2); got != test.parity { + t.Fatalf("Expected common parity %v but got %v", test.parity, got) + } + }) + } +} diff --git a/cmd/erasure-multipart.go b/cmd/erasure-multipart.go new file mode 100644 index 00000000..2d85e9dc --- /dev/null +++ b/cmd/erasure-multipart.go @@ -0,0 +1,1528 @@ +// Copyright (c) 2015-2025 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/base64" + "errors" + "fmt" + "io" + "os" + "path" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/klauspost/readahead" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/config/storageclass" + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/mimedb" + "github.com/minio/pkg/v3/sync/errgroup" + "github.com/minio/sio" +) + +func (er erasureObjects) getUploadIDDir(bucket, object, uploadID string) string { + uploadUUID := uploadID + uploadBytes, err := base64.RawURLEncoding.DecodeString(uploadID) + if err == nil { + slc := strings.SplitN(string(uploadBytes), ".", 2) + if len(slc) == 2 { + uploadUUID = slc[1] + } + } + return pathJoin(er.getMultipartSHADir(bucket, object), uploadUUID) +} + +func (er erasureObjects) getMultipartSHADir(bucket, object string) string { + return getSHA256Hash([]byte(pathJoin(bucket, object))) +} + +// checkUploadIDExists - verify if a given uploadID exists and is valid. +func (er erasureObjects) checkUploadIDExists(ctx context.Context, bucket, object, uploadID string, write bool) (fi FileInfo, metArr []FileInfo, err error) { + defer func() { + if errors.Is(err, errFileNotFound) { + err = errUploadIDNotFound + } + }() + + uploadIDPath := er.getUploadIDDir(bucket, object, uploadID) + + storageDisks := er.getDisks() + + // Read metadata associated with the object from all disks. + partsMetadata, errs := readAllFileInfo(ctx, storageDisks, bucket, minioMetaMultipartBucket, + uploadIDPath, "", false, false) + + readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, partsMetadata, errs, er.defaultParityCount) + if err != nil { + return fi, nil, err + } + + if readQuorum < 0 { + return fi, nil, errErasureReadQuorum + } + + if writeQuorum < 0 { + return fi, nil, errErasureWriteQuorum + } + + quorum := readQuorum + if write { + quorum = writeQuorum + } + + // List all online disks. + _, modTime, etag := listOnlineDisks(storageDisks, partsMetadata, errs, quorum) + + if write { + err = reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum) + } else { + err = reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum) + } + if err != nil { + return fi, nil, err + } + + // Pick one from the first valid metadata. + fi, err = pickValidFileInfo(ctx, partsMetadata, modTime, etag, quorum) + return fi, partsMetadata, err +} + +// cleanupMultipartPath removes all extraneous files and parts from the multipart folder, this is used per CompleteMultipart. +// do not use this function outside of completeMultipartUpload() +func (er erasureObjects) cleanupMultipartPath(ctx context.Context, paths ...string) { + storageDisks := er.getDisks() + + g := errgroup.WithNErrs(len(storageDisks)) + for index, disk := range storageDisks { + if disk == nil { + continue + } + index := index + g.Go(func() error { + _ = storageDisks[index].DeleteBulk(ctx, minioMetaMultipartBucket, paths...) + return nil + }, index) + } + g.Wait() +} + +// Clean-up the old multipart uploads. Should be run in a Go routine. +func (er erasureObjects) cleanupStaleUploads(ctx context.Context) { + // run multiple cleanup's local to this server. + var wg sync.WaitGroup + for _, disk := range er.getLocalDisks() { + if disk != nil { + wg.Add(1) + go func(disk StorageAPI) { + defer wg.Done() + er.cleanupStaleUploadsOnDisk(ctx, disk) + }(disk) + } + } + wg.Wait() +} + +func (er erasureObjects) deleteAll(ctx context.Context, bucket, prefix string) { + var wg sync.WaitGroup + for _, disk := range er.getDisks() { + if disk == nil { + continue + } + wg.Add(1) + go func(disk StorageAPI) { + defer wg.Done() + disk.Delete(ctx, bucket, prefix, DeleteOptions{ + Recursive: true, + Immediate: false, + }) + }(disk) + } + wg.Wait() +} + +// Remove the old multipart uploads on the given disk. +func (er erasureObjects) cleanupStaleUploadsOnDisk(ctx context.Context, disk StorageAPI) { + drivePath := disk.Endpoint().Path + + readDirFn(pathJoin(drivePath, minioMetaMultipartBucket), func(shaDir string, typ os.FileMode) error { + readDirFn(pathJoin(drivePath, minioMetaMultipartBucket, shaDir), func(uploadIDDir string, typ os.FileMode) error { + uploadIDPath := pathJoin(shaDir, uploadIDDir) + var modTime time.Time + // Upload IDs are of the form base64_url(x), we can extract the time from the UUID. + if b64, err := base64.RawURLEncoding.DecodeString(uploadIDDir); err == nil { + if split := strings.Split(string(b64), "x"); len(split) == 2 { + t, err := strconv.ParseInt(split[1], 10, 64) + if err == nil { + modTime = time.Unix(0, t) + } + } + } + // Fallback for older uploads without time in the ID. + if modTime.IsZero() { + wait := deleteMultipartCleanupSleeper.Timer(ctx) + fi, err := disk.ReadVersion(ctx, "", minioMetaMultipartBucket, uploadIDPath, "", ReadOptions{}) + if err != nil { + return nil + } + modTime = fi.ModTime + wait() + } + if time.Since(modTime) < globalAPIConfig.getStaleUploadsExpiry() { + return nil + } + w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) + return w.Run(func() error { + wait := deleteMultipartCleanupSleeper.Timer(ctx) + pathUUID := mustGetUUID() + targetPath := pathJoin(drivePath, minioMetaTmpDeletedBucket, pathUUID) + renameAll(pathJoin(drivePath, minioMetaMultipartBucket, uploadIDPath), targetPath, pathJoin(drivePath, minioMetaBucket)) + wait() + return nil + }) + }) + // Get the modtime of the shaDir. + vi, err := disk.StatVol(ctx, pathJoin(minioMetaMultipartBucket, shaDir)) + if err != nil { + return nil + } + // Modtime is returned in the Created field. See (*xlStorage).StatVol + if time.Since(vi.Created) < globalAPIConfig.getStaleUploadsExpiry() { + return nil + } + w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) + return w.Run(func() error { + wait := deleteMultipartCleanupSleeper.Timer(ctx) + pathUUID := mustGetUUID() + targetPath := pathJoin(drivePath, minioMetaTmpDeletedBucket, pathUUID) + + // We are not deleting shaDir recursively here, if shaDir is empty + // and its older then we can happily delete it. + Rename(pathJoin(drivePath, minioMetaMultipartBucket, shaDir), targetPath) + wait() + return nil + }) + }) + + readDirFn(pathJoin(drivePath, minioMetaTmpBucket), func(tmpDir string, typ os.FileMode) error { + if strings.HasPrefix(tmpDir, ".trash") { + // do not remove .trash/ here, it has its own routines + return nil + } + vi, err := disk.StatVol(ctx, pathJoin(minioMetaTmpBucket, tmpDir)) + if err != nil { + return nil + } + w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) + return w.Run(func() error { + wait := deleteMultipartCleanupSleeper.Timer(ctx) + if time.Since(vi.Created) > globalAPIConfig.getStaleUploadsExpiry() { + pathUUID := mustGetUUID() + targetPath := pathJoin(drivePath, minioMetaTmpDeletedBucket, pathUUID) + + renameAll(pathJoin(drivePath, minioMetaTmpBucket, tmpDir), targetPath, pathJoin(drivePath, minioMetaBucket)) + } + wait() + return nil + }) + }) +} + +// ListMultipartUploads - lists all the pending multipart +// uploads for a particular object in a bucket. +// +// Implements minimal S3 compatible ListMultipartUploads API. We do +// not support prefix based listing, this is a deliberate attempt +// towards simplification of multipart APIs. +// The resulting ListMultipartsInfo structure is unmarshalled directly as XML. +func (er erasureObjects) ListMultipartUploads(ctx context.Context, bucket, object, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) { + auditObjectErasureSet(ctx, "ListMultipartUploads", object, &er) + + result.MaxUploads = maxUploads + result.KeyMarker = keyMarker + result.Prefix = object + result.Delimiter = delimiter + + var uploadIDs []string + var disk StorageAPI + disks := er.getOnlineLocalDisks() + if len(disks) == 0 { + // If no local, get non-healing disks. + var ok bool + if disks, ok = er.getOnlineDisksWithHealing(false); !ok { + disks = er.getOnlineDisks() + } + } + + for _, disk = range disks { + if disk == nil { + continue + } + if !disk.IsOnline() { + continue + } + uploadIDs, err = disk.ListDir(ctx, bucket, minioMetaMultipartBucket, er.getMultipartSHADir(bucket, object), -1) + if err != nil { + if errors.Is(err, errDiskNotFound) { + continue + } + if errors.Is(err, errFileNotFound) { + return result, nil + } + return result, toObjectErr(err, bucket, object) + } + break + } + + for i := range uploadIDs { + uploadIDs[i] = strings.TrimSuffix(uploadIDs[i], SlashSeparator) + } + + // S3 spec says uploadIDs should be sorted based on initiated time, we need + // to read the metadata entry. + var uploads []MultipartInfo + + populatedUploadIDs := set.NewStringSet() + + for _, uploadID := range uploadIDs { + if populatedUploadIDs.Contains(uploadID) { + continue + } + // If present, use time stored in ID. + startTime := time.Now() + if split := strings.Split(uploadID, "x"); len(split) == 2 { + t, err := strconv.ParseInt(split[1], 10, 64) + if err == nil { + startTime = time.Unix(0, t) + } + } + uploads = append(uploads, MultipartInfo{ + Bucket: bucket, + Object: object, + UploadID: base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadID)), + Initiated: startTime, + }) + populatedUploadIDs.Add(uploadID) + } + + sort.Slice(uploads, func(i int, j int) bool { + return uploads[i].Initiated.Before(uploads[j].Initiated) + }) + + uploadIndex := 0 + if uploadIDMarker != "" { + for uploadIndex < len(uploads) { + if uploads[uploadIndex].UploadID != uploadIDMarker { + uploadIndex++ + continue + } + if uploads[uploadIndex].UploadID == uploadIDMarker { + uploadIndex++ + break + } + uploadIndex++ + } + } + for uploadIndex < len(uploads) { + result.Uploads = append(result.Uploads, uploads[uploadIndex]) + result.NextUploadIDMarker = uploads[uploadIndex].UploadID + uploadIndex++ + if len(result.Uploads) == maxUploads { + break + } + } + + result.IsTruncated = uploadIndex < len(uploads) + + if !result.IsTruncated { + result.NextKeyMarker = "" + result.NextUploadIDMarker = "" + } + + return result, nil +} + +// newMultipartUpload - wrapper for initializing a new multipart +// request; returns a unique upload id. +// +// Internally this function creates 'uploads.json' associated for the +// incoming object at +// '.minio.sys/multipart/bucket/object/uploads.json' on all the +// disks. `uploads.json` carries metadata regarding on-going multipart +// operation(s) on the object. +func (er erasureObjects) newMultipartUpload(ctx context.Context, bucket string, object string, opts ObjectOptions) (*NewMultipartUploadResult, error) { + if opts.CheckPrecondFn != nil { + if !opts.NoLock { + ns := er.NewNSLock(bucket, object) + lkctx, err := ns.GetLock(ctx, globalOperationTimeout) + if err != nil { + return nil, err + } + ctx = lkctx.Context() + defer ns.Unlock(lkctx) + opts.NoLock = true + } + + obj, err := er.getObjectInfo(ctx, bucket, object, opts) + if err == nil && opts.CheckPrecondFn(obj) { + return nil, PreConditionFailed{} + } + if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) && !isErrReadQuorum(err) { + return nil, err + } + + // if object doesn't exist return error for If-Match conditional requests + // If-None-Match should be allowed to proceed for non-existent objects + if err != nil && opts.HasIfMatch && (isErrObjectNotFound(err) || isErrVersionNotFound(err)) { + return nil, err + } + } + + userDefined := cloneMSS(opts.UserDefined) + if opts.PreserveETag != "" { + userDefined["etag"] = opts.PreserveETag + } + onlineDisks := er.getDisks() + + // Get parity and data drive count based on storage class metadata + parityDrives := globalStorageClass.GetParityForSC(userDefined[xhttp.AmzStorageClass]) + if parityDrives < 0 { + parityDrives = er.defaultParityCount + } + + if globalStorageClass.AvailabilityOptimized() { + // If we have offline disks upgrade the number of erasure codes for this object. + parityOrig := parityDrives + + var offlineDrives int + for _, disk := range onlineDisks { + if disk == nil || !disk.IsOnline() { + parityDrives++ + offlineDrives++ + continue + } + } + + if offlineDrives >= (len(onlineDisks)+1)/2 { + // if offline drives are more than 50% of the drives + // we have no quorum, we shouldn't proceed just + // fail at that point. + return nil, toObjectErr(errErasureWriteQuorum, bucket, object) + } + + if parityDrives >= len(onlineDisks)/2 { + parityDrives = len(onlineDisks) / 2 + } + + if parityOrig != parityDrives { + userDefined[minIOErasureUpgraded] = strconv.Itoa(parityOrig) + "->" + strconv.Itoa(parityDrives) + } + } + + dataDrives := len(onlineDisks) - parityDrives + + // we now know the number of blocks this object needs for data and parity. + // establish the writeQuorum using this data + writeQuorum := dataDrives + if dataDrives == parityDrives { + writeQuorum++ + } + + // Initialize parts metadata + partsMetadata := make([]FileInfo, len(onlineDisks)) + + fi := newFileInfo(pathJoin(bucket, object), dataDrives, parityDrives) + fi.VersionID = opts.VersionID + if opts.Versioned && fi.VersionID == "" { + fi.VersionID = mustGetUUID() + } + fi.DataDir = mustGetUUID() + + if ckSum := userDefined[ReplicationSsecChecksumHeader]; ckSum != "" { + v, err := base64.StdEncoding.DecodeString(ckSum) + if err == nil { + fi.Checksum = v + } + delete(userDefined, ReplicationSsecChecksumHeader) + } + + // Initialize erasure metadata. + for index := range partsMetadata { + partsMetadata[index] = fi + } + + // Guess content-type from the extension if possible. + if userDefined["content-type"] == "" { + userDefined["content-type"] = mimedb.TypeByExtension(path.Ext(object)) + } + + // if storageClass is standard no need to save it as part of metadata. + if userDefined[xhttp.AmzStorageClass] == storageclass.STANDARD { + delete(userDefined, xhttp.AmzStorageClass) + } + + if opts.WantChecksum != nil && opts.WantChecksum.Type.IsSet() { + userDefined[hash.MinIOMultipartChecksum] = opts.WantChecksum.Type.String() + userDefined[hash.MinIOMultipartChecksumType] = opts.WantChecksum.Type.ObjType() + } + + modTime := opts.MTime + if opts.MTime.IsZero() { + modTime = UTCNow() + } + + onlineDisks, partsMetadata = shuffleDisksAndPartsMetadata(onlineDisks, partsMetadata, fi) + + // Fill all the necessary metadata. + // Update `xl.meta` content on each disks. + for index := range partsMetadata { + partsMetadata[index].Fresh = true + partsMetadata[index].ModTime = modTime + partsMetadata[index].Metadata = userDefined + } + uploadUUID := fmt.Sprintf("%sx%d", mustGetUUID(), modTime.UnixNano()) + uploadID := base64.RawURLEncoding.EncodeToString(fmt.Appendf(nil, "%s.%s", globalDeploymentID(), uploadUUID)) + uploadIDPath := er.getUploadIDDir(bucket, object, uploadUUID) + + // Write updated `xl.meta` to all disks. + if _, err := writeAllMetadata(ctx, onlineDisks, bucket, minioMetaMultipartBucket, uploadIDPath, partsMetadata, writeQuorum); err != nil { + return nil, toObjectErr(err, bucket, object) + } + + return &NewMultipartUploadResult{ + UploadID: uploadID, + ChecksumAlgo: userDefined[hash.MinIOMultipartChecksum], + ChecksumType: userDefined[hash.MinIOMultipartChecksumType], + }, nil +} + +// NewMultipartUpload - initialize a new multipart upload, returns a +// unique id. The unique id returned here is of UUID form, for each +// subsequent request each UUID is unique. +// +// Implements S3 compatible initiate multipart API. +func (er erasureObjects) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (*NewMultipartUploadResult, error) { + if !opts.NoAuditLog { + auditObjectErasureSet(ctx, "NewMultipartUpload", object, &er) + } + + return er.newMultipartUpload(ctx, bucket, object, opts) +} + +// renamePart - renames multipart part to its relevant location under uploadID. +func (er erasureObjects) renamePart(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry, dstBucket, dstEntry string, optsMeta []byte, writeQuorum int, skipParent string) ([]StorageAPI, error) { + paths := []string{ + dstEntry, + dstEntry + ".meta", + } + + // cleanup existing paths first across all drives. + er.cleanupMultipartPath(ctx, paths...) + + g := errgroup.WithNErrs(len(disks)) + + // Rename file on all underlying storage disks. + for index := range disks { + g.Go(func() error { + if disks[index] == nil { + return errDiskNotFound + } + return disks[index].RenamePart(ctx, srcBucket, srcEntry, dstBucket, dstEntry, optsMeta, skipParent) + }, index) + } + + // Wait for all renames to finish. + errs := g.Wait() + + err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum) + if err != nil { + er.cleanupMultipartPath(ctx, paths...) + } + + // We can safely allow RenameFile errors up to len(er.getDisks()) - writeQuorum + // otherwise return failure. Cleanup successful renames. + return evalDisks(disks, errs), err +} + +// PutObjectPart - reads incoming stream and internally erasure codes +// them. This call is similar to single put operation but it is part +// of the multipart transaction. +// +// Implements S3 compatible Upload Part API. +func (er erasureObjects) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, r *PutObjReader, opts ObjectOptions) (pi PartInfo, err error) { + if !opts.NoAuditLog { + auditObjectErasureSet(ctx, "PutObjectPart", object, &er) + } + + data := r.Reader + // Validate input data size and it can never be less than zero. + if data.Size() < -1 { + bugLogIf(ctx, errInvalidArgument, logger.ErrorKind) + return pi, toObjectErr(errInvalidArgument) + } + + uploadIDPath := er.getUploadIDDir(bucket, object, uploadID) + // Validates if upload ID exists. + fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true) + if err != nil { + if errors.Is(err, errVolumeNotFound) { + return pi, toObjectErr(err, bucket) + } + return pi, toObjectErr(err, bucket, object, uploadID) + } + + onlineDisks := er.getDisks() + writeQuorum := fi.WriteQuorum(er.defaultWQuorum()) + + if cs := fi.Metadata[hash.MinIOMultipartChecksum]; cs != "" { + if r.ContentCRCType().String() != cs { + return pi, InvalidArgument{ + Bucket: bucket, + Object: fi.Name, + Err: fmt.Errorf("checksum missing, want %q, got %q", cs, r.ContentCRCType().String()), + } + } + } + onlineDisks = shuffleDisks(onlineDisks, fi.Erasure.Distribution) + + // Need a unique name for the part being written in minioMetaBucket to + // accommodate concurrent PutObjectPart requests + + partSuffix := fmt.Sprintf("part.%d", partID) + // Random UUID and timestamp for temporary part file. + tmpPart := fmt.Sprintf("%sx%d", mustGetUUID(), time.Now().UnixNano()) + tmpPartPath := pathJoin(tmpPart, partSuffix) + + // Delete the temporary object part. If PutObjectPart succeeds there would be nothing to delete. + defer func() { + if countOnlineDisks(onlineDisks) != len(onlineDisks) { + er.deleteAll(context.Background(), minioMetaTmpBucket, tmpPart) + } + }() + + erasure, err := NewErasure(ctx, fi.Erasure.DataBlocks, fi.Erasure.ParityBlocks, fi.Erasure.BlockSize) + if err != nil { + return pi, toObjectErr(err, bucket, object) + } + + // Fetch buffer for I/O, returns from the pool if not allocates a new one and returns. + var buffer []byte + switch size := data.Size(); { + case size == 0: + buffer = make([]byte, 1) // Allocate at least a byte to reach EOF + case size >= fi.Erasure.BlockSize || size == -1: + if int64(globalBytePoolCap.Load().Width()) < fi.Erasure.BlockSize { + buffer = make([]byte, fi.Erasure.BlockSize, 2*fi.Erasure.BlockSize) + } else { + buffer = globalBytePoolCap.Load().Get() + defer globalBytePoolCap.Load().Put(buffer) + } + case size < fi.Erasure.BlockSize: + // No need to allocate fully fi.Erasure.BlockSize buffer if the incoming data is smaller. + buffer = make([]byte, size, 2*size+int64(fi.Erasure.ParityBlocks+fi.Erasure.DataBlocks-1)) + } + + if len(buffer) > int(fi.Erasure.BlockSize) { + buffer = buffer[:fi.Erasure.BlockSize] + } + writers := make([]io.Writer, len(onlineDisks)) + for i, disk := range onlineDisks { + if disk == nil { + continue + } + writers[i] = newBitrotWriter(disk, bucket, minioMetaTmpBucket, tmpPartPath, erasure.ShardFileSize(data.Size()), DefaultBitrotAlgorithm, erasure.ShardSize()) + } + + toEncode := io.Reader(data) + if data.Size() > bigFileThreshold { + // Add input readahead. + // We use 2 buffers, so we always have a full buffer of input. + pool := globalBytePoolCap.Load() + bufA := pool.Get() + bufB := pool.Get() + defer pool.Put(bufA) + defer pool.Put(bufB) + ra, err := readahead.NewReaderBuffer(data, [][]byte{bufA[:fi.Erasure.BlockSize], bufB[:fi.Erasure.BlockSize]}) + if err == nil { + toEncode = ra + defer ra.Close() + } + } + + n, err := erasure.Encode(ctx, toEncode, writers, buffer, writeQuorum) + closeErrs := closeBitrotWriters(writers) + if err != nil { + return pi, toObjectErr(err, bucket, object) + } + if closeErr := reduceWriteQuorumErrs(ctx, closeErrs, objectOpIgnoredErrs, writeQuorum); closeErr != nil { + return pi, toObjectErr(closeErr, bucket, object) + } + + // Should return IncompleteBody{} error when reader has fewer bytes + // than specified in request header. + if n < data.Size() { + return pi, IncompleteBody{Bucket: bucket, Object: object} + } + + for i := range writers { + if writers[i] == nil { + onlineDisks[i] = nil + } + } + + // Rename temporary part file to its final location. + partPath := pathJoin(uploadIDPath, fi.DataDir, partSuffix) + + md5hex := r.MD5CurrentHexString() + if opts.PreserveETag != "" { + md5hex = opts.PreserveETag + } + + var index []byte + if opts.IndexCB != nil { + index = opts.IndexCB() + } + + actualSize := data.ActualSize() + if actualSize < 0 { + _, encrypted := crypto.IsEncrypted(fi.Metadata) + compressed := fi.IsCompressed() + switch { + case compressed: + // ... nothing changes for compressed stream. + // if actualSize is -1 we have no known way to + // determine what is the actualSize. + case encrypted: + decSize, err := sio.DecryptedSize(uint64(n)) + if err == nil { + actualSize = int64(decSize) + } + default: + actualSize = n + } + } + + partInfo := ObjectPartInfo{ + Number: partID, + ETag: md5hex, + Size: n, + ActualSize: actualSize, + ModTime: UTCNow(), + Index: index, + Checksums: r.ContentCRC(), + } + + partFI, err := partInfo.MarshalMsg(nil) + if err != nil { + return pi, toObjectErr(err, minioMetaMultipartBucket, partPath) + } + + // Serialize concurrent part uploads. + partIDLock := er.NewNSLock(bucket, pathJoin(object, uploadID, strconv.Itoa(partID))) + plkctx, err := partIDLock.GetLock(ctx, globalOperationTimeout) + if err != nil { + return PartInfo{}, err + } + + ctx = plkctx.Context() + defer partIDLock.Unlock(plkctx) + + // Read lock for upload id, only held while reading the upload metadata. + uploadIDRLock := er.NewNSLock(bucket, pathJoin(object, uploadID)) + rlkctx, err := uploadIDRLock.GetRLock(ctx, globalOperationTimeout) + if err != nil { + return PartInfo{}, err + } + ctx = rlkctx.Context() + defer uploadIDRLock.RUnlock(rlkctx) + + onlineDisks, err = er.renamePart(ctx, onlineDisks, minioMetaTmpBucket, tmpPartPath, minioMetaMultipartBucket, partPath, partFI, writeQuorum, uploadIDPath) + if err != nil { + if errors.Is(err, errUploadIDNotFound) { + return pi, toObjectErr(errUploadIDNotFound, bucket, object, uploadID) + } + if errors.Is(err, errFileNotFound) { + // An in-quorum errFileNotFound means that client stream + // prematurely closed and we do not find any xl.meta or + // part.1's - in such a scenario we must return as if client + // disconnected. This means that erasure.Encode() CreateFile() + // did not do anything. + return pi, IncompleteBody{Bucket: bucket, Object: object} + } + + return pi, toObjectErr(err, minioMetaMultipartBucket, partPath) + } + + // Return success. + return PartInfo{ + PartNumber: partInfo.Number, + ETag: partInfo.ETag, + LastModified: partInfo.ModTime, + Size: partInfo.Size, + ActualSize: partInfo.ActualSize, + ChecksumCRC32: partInfo.Checksums["CRC32"], + ChecksumCRC32C: partInfo.Checksums["CRC32C"], + ChecksumSHA1: partInfo.Checksums["SHA1"], + ChecksumSHA256: partInfo.Checksums["SHA256"], + ChecksumCRC64NVME: partInfo.Checksums["CRC64NVME"], + }, nil +} + +// GetMultipartInfo returns multipart metadata uploaded during newMultipartUpload, used +// by callers to verify object states +// - encrypted +// - compressed +// Does not contain currently uploaded parts by design. +func (er erasureObjects) GetMultipartInfo(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (MultipartInfo, error) { + if !opts.NoAuditLog { + auditObjectErasureSet(ctx, "GetMultipartInfo", object, &er) + } + + result := MultipartInfo{ + Bucket: bucket, + Object: object, + UploadID: uploadID, + } + + fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false) + if err != nil { + if errors.Is(err, errVolumeNotFound) { + return result, toObjectErr(err, bucket) + } + return result, toObjectErr(err, bucket, object, uploadID) + } + + result.UserDefined = cloneMSS(fi.Metadata) + return result, nil +} + +func (er erasureObjects) listParts(ctx context.Context, onlineDisks []StorageAPI, partPath string, readQuorum int) ([]int, error) { + g := errgroup.WithNErrs(len(onlineDisks)) + + objectParts := make([][]string, len(onlineDisks)) + // List uploaded parts from drives. + for index := range onlineDisks { + g.Go(func() (err error) { + if onlineDisks[index] == nil { + return errDiskNotFound + } + objectParts[index], err = onlineDisks[index].ListDir(ctx, minioMetaMultipartBucket, minioMetaMultipartBucket, partPath, -1) + return err + }, index) + } + + if err := reduceReadQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, readQuorum); err != nil { + return nil, err + } + + partQuorumMap := make(map[int]int) + for _, driveParts := range objectParts { + partsWithMetaCount := make(map[int]int, len(driveParts)) + // part files can be either part.N or part.N.meta + for _, partPath := range driveParts { + var partNum int + if _, err := fmt.Sscanf(partPath, "part.%d", &partNum); err == nil { + partsWithMetaCount[partNum]++ + continue + } + if _, err := fmt.Sscanf(partPath, "part.%d.meta", &partNum); err == nil { + partsWithMetaCount[partNum]++ + } + } + // Include only part.N.meta files with corresponding part.N + for partNum, cnt := range partsWithMetaCount { + if cnt < 2 { + continue + } + partQuorumMap[partNum]++ + } + } + + var partNums []int + for partNum, count := range partQuorumMap { + if count < readQuorum { + continue + } + partNums = append(partNums, partNum) + } + + sort.Ints(partNums) + return partNums, nil +} + +// ListObjectParts - lists all previously uploaded parts for a given +// object and uploadID. Takes additional input of part-number-marker +// to indicate where the listing should begin from. +// +// Implements S3 compatible ListObjectParts API. The resulting +// ListPartsInfo structure is marshaled directly into XML and +// replied back to the client. +func (er erasureObjects) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker, maxParts int, opts ObjectOptions) (result ListPartsInfo, err error) { + if !opts.NoAuditLog { + auditObjectErasureSet(ctx, "ListObjectParts", object, &er) + } + + fi, _, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, false) + if err != nil { + return result, toObjectErr(err, bucket, object, uploadID) + } + + uploadIDPath := er.getUploadIDDir(bucket, object, uploadID) + if partNumberMarker < 0 { + partNumberMarker = 0 + } + + // Limit output to maxPartsList. + if maxParts > maxPartsList { + maxParts = maxPartsList + } + + // Populate the result stub. + result.Bucket = bucket + result.Object = object + result.UploadID = uploadID + result.MaxParts = maxParts + result.PartNumberMarker = partNumberMarker + result.UserDefined = cloneMSS(fi.Metadata) + result.ChecksumAlgorithm = fi.Metadata[hash.MinIOMultipartChecksum] + result.ChecksumType = fi.Metadata[hash.MinIOMultipartChecksumType] + + if maxParts == 0 { + return result, nil + } + + onlineDisks := er.getDisks() + readQuorum := fi.ReadQuorum(er.defaultRQuorum()) + // Read Part info for all parts + partPath := pathJoin(uploadIDPath, fi.DataDir) + SlashSeparator + + // List parts in quorum + partNums, err := er.listParts(ctx, onlineDisks, partPath, readQuorum) + if err != nil { + // This means that fi.DataDir, is not yet populated so we + // return an empty response. + if errors.Is(err, errFileNotFound) { + return result, nil + } + return result, toObjectErr(err, bucket, object, uploadID) + } + + if len(partNums) == 0 { + return result, nil + } + + start := objectPartIndexNums(partNums, partNumberMarker) + if partNumberMarker > 0 && start == -1 { + // Marker not present among what is present on the + // server, we return an empty list. + return result, nil + } + + if partNumberMarker > 0 && start != -1 { + if start+1 >= len(partNums) { + // Marker indicates that we are the end + // of the list, so we simply return empty + return result, nil + } + + partNums = partNums[start+1:] + } + + result.Parts = make([]PartInfo, 0, len(partNums)) + partMetaPaths := make([]string, len(partNums)) + for i, part := range partNums { + partMetaPaths[i] = pathJoin(partPath, fmt.Sprintf("part.%d.meta", part)) + } + + // Read parts in quorum + objParts, err := readParts(ctx, onlineDisks, minioMetaMultipartBucket, partMetaPaths, + partNums, readQuorum) + if err != nil { + return result, toObjectErr(err, bucket, object, uploadID) + } + + count := maxParts + for _, objPart := range objParts { + result.Parts = append(result.Parts, PartInfo{ + PartNumber: objPart.Number, + LastModified: objPart.ModTime, + ETag: objPart.ETag, + Size: objPart.Size, + ActualSize: objPart.ActualSize, + ChecksumCRC32: objPart.Checksums["CRC32"], + ChecksumCRC32C: objPart.Checksums["CRC32C"], + ChecksumSHA1: objPart.Checksums["SHA1"], + ChecksumSHA256: objPart.Checksums["SHA256"], + ChecksumCRC64NVME: objPart.Checksums["CRC64NVME"], + }) + count-- + if count == 0 { + break + } + } + + if len(objParts) > len(result.Parts) { + result.IsTruncated = true + // Make sure to fill next part number marker if IsTruncated is true for subsequent listing. + result.NextPartNumberMarker = result.Parts[len(result.Parts)-1].PartNumber + } + + return result, nil +} + +func readParts(ctx context.Context, disks []StorageAPI, bucket string, partMetaPaths []string, partNumbers []int, readQuorum int) ([]ObjectPartInfo, error) { + g := errgroup.WithNErrs(len(disks)) + + objectPartInfos := make([][]*ObjectPartInfo, len(disks)) + // Rename file on all underlying storage disks. + for index := range disks { + g.Go(func() (err error) { + if disks[index] == nil { + return errDiskNotFound + } + objectPartInfos[index], err = disks[index].ReadParts(ctx, bucket, partMetaPaths...) + return err + }, index) + } + + if err := reduceReadQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, readQuorum); err != nil { + return nil, err + } + + partInfosInQuorum := make([]ObjectPartInfo, len(partMetaPaths)) + for pidx := range partMetaPaths { + // partMetaQuorumMap uses + // - path/to/part.N as key to collate errors from failed drives. + // - part ETag to collate part metadata + partMetaQuorumMap := make(map[string]int, len(partNumbers)) + var pinfos []*ObjectPartInfo + for idx := range disks { + if len(objectPartInfos[idx]) != len(partMetaPaths) { + partMetaQuorumMap[partMetaPaths[pidx]]++ + continue + } + + pinfo := objectPartInfos[idx][pidx] + if pinfo != nil && pinfo.ETag != "" { + pinfos = append(pinfos, pinfo) + partMetaQuorumMap[pinfo.ETag]++ + continue + } + partMetaQuorumMap[partMetaPaths[pidx]]++ + } + + var maxQuorum int + var maxETag string + var maxPartMeta string + for etag, quorum := range partMetaQuorumMap { + if maxQuorum < quorum { + maxQuorum = quorum + maxETag = etag + maxPartMeta = etag + } + } + // found is a representative ObjectPartInfo which either has the maximally occurring ETag or an error. + var found *ObjectPartInfo + for _, pinfo := range pinfos { + if pinfo == nil { + continue + } + if maxETag != "" && pinfo.ETag == maxETag { + found = pinfo + break + } + if pinfo.ETag == "" && maxPartMeta != "" && path.Base(maxPartMeta) == fmt.Sprintf("part.%d.meta", pinfo.Number) { + found = pinfo + break + } + } + + if found != nil && found.ETag != "" && partMetaQuorumMap[maxETag] >= readQuorum { + partInfosInQuorum[pidx] = *found + continue + } + partInfosInQuorum[pidx] = ObjectPartInfo{ + Number: partNumbers[pidx], + Error: InvalidPart{ + PartNumber: partNumbers[pidx], + }.Error(), + } + } + return partInfosInQuorum, nil +} + +func objPartToPartErr(part ObjectPartInfo) error { + if strings.Contains(part.Error, "file not found") { + return InvalidPart{PartNumber: part.Number} + } + if strings.Contains(part.Error, "Specified part could not be found") { + return InvalidPart{PartNumber: part.Number} + } + if strings.Contains(part.Error, errErasureReadQuorum.Error()) { + return errErasureReadQuorum + } + return errors.New(part.Error) +} + +// CompleteMultipartUpload - completes an ongoing multipart +// transaction after receiving all the parts indicated by the client. +// Returns an md5sum calculated by concatenating all the individual +// md5sums of all the parts. +// +// Implements S3 compatible Complete multipart API. +func (er erasureObjects) CompleteMultipartUpload(ctx context.Context, bucket string, object string, uploadID string, parts []CompletePart, opts ObjectOptions) (oi ObjectInfo, err error) { + if !opts.NoAuditLog { + auditObjectErasureSet(ctx, "CompleteMultipartUpload", object, &er) + } + + if opts.CheckPrecondFn != nil { + if !opts.NoLock { + ns := er.NewNSLock(bucket, object) + lkctx, err := ns.GetLock(ctx, globalOperationTimeout) + if err != nil { + return ObjectInfo{}, err + } + ctx = lkctx.Context() + defer ns.Unlock(lkctx) + opts.NoLock = true + } + + obj, err := er.getObjectInfo(ctx, bucket, object, opts) + if err == nil && opts.CheckPrecondFn(obj) { + return ObjectInfo{}, PreConditionFailed{} + } + if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) && !isErrReadQuorum(err) { + return ObjectInfo{}, err + } + + // if object doesn't exist return error for If-Match conditional requests + // If-None-Match should be allowed to proceed for non-existent objects + if err != nil && opts.HasIfMatch && (isErrObjectNotFound(err) || isErrVersionNotFound(err)) { + return ObjectInfo{}, err + } + } + + fi, partsMetadata, err := er.checkUploadIDExists(ctx, bucket, object, uploadID, true) + if err != nil { + if errors.Is(err, errVolumeNotFound) { + return oi, toObjectErr(err, bucket) + } + return oi, toObjectErr(err, bucket, object, uploadID) + } + + uploadIDPath := er.getUploadIDDir(bucket, object, uploadID) + onlineDisks := er.getDisks() + writeQuorum := fi.WriteQuorum(er.defaultWQuorum()) + readQuorum := fi.ReadQuorum(er.defaultRQuorum()) + + // Read Part info for all parts + partPath := pathJoin(uploadIDPath, fi.DataDir) + SlashSeparator + partMetaPaths := make([]string, len(parts)) + partNumbers := make([]int, len(parts)) + for idx, part := range parts { + partMetaPaths[idx] = pathJoin(partPath, fmt.Sprintf("part.%d.meta", part.PartNumber)) + partNumbers[idx] = part.PartNumber + } + + partInfoFiles, err := readParts(ctx, onlineDisks, minioMetaMultipartBucket, partMetaPaths, partNumbers, readQuorum) + if err != nil { + return oi, err + } + + if len(partInfoFiles) != len(parts) { + // Should only happen through internal error + err := fmt.Errorf("unexpected part result count: %d, want %d", len(partInfoFiles), len(parts)) + bugLogIf(ctx, err) + return oi, toObjectErr(err, bucket, object) + } + + // Checksum type set when upload started. + var checksumType hash.ChecksumType + if cs := fi.Metadata[hash.MinIOMultipartChecksum]; cs != "" { + checksumType = hash.NewChecksumType(cs, fi.Metadata[hash.MinIOMultipartChecksumType]) + if opts.WantChecksum != nil && !opts.WantChecksum.Type.Is(checksumType) { + return oi, InvalidArgument{ + Bucket: bucket, + Object: fi.Name, + Err: fmt.Errorf("checksum type mismatch. got %q (%s) expected %q (%s)", checksumType.String(), checksumType.ObjType(), opts.WantChecksum.Type.String(), opts.WantChecksum.Type.ObjType()), + } + } + checksumType |= hash.ChecksumMultipart | hash.ChecksumIncludesMultipart + } + + var checksumCombined []byte + + // However, in case of encryption, the persisted part ETags don't match + // what we have sent to the client during PutObjectPart. The reason is + // that ETags are encrypted. Hence, the client will send a list of complete + // part ETags of which may not match the ETag of any part. For example + // ETag (client): 30902184f4e62dd8f98f0aaff810c626 + // ETag (server-internal): 20000f00ce5dc16e3f3b124f586ae1d88e9caa1c598415c2759bbb50e84a59f630902184f4e62dd8f98f0aaff810c626 + // + // Therefore, we adjust all ETags sent by the client to match what is stored + // on the backend. + kind, _ := crypto.IsEncrypted(fi.Metadata) + + var objectEncryptionKey []byte + switch kind { + case crypto.SSEC: + if checksumType.IsSet() { + if opts.EncryptFn == nil { + return oi, crypto.ErrMissingCustomerKey + } + baseKey := opts.EncryptFn("", nil) + if len(baseKey) != 32 { + return oi, crypto.ErrInvalidCustomerKey + } + objectEncryptionKey, err = decryptObjectMeta(baseKey, bucket, object, fi.Metadata) + if err != nil { + return oi, err + } + } + case crypto.S3, crypto.S3KMS: + objectEncryptionKey, err = decryptObjectMeta(nil, bucket, object, fi.Metadata) + if err != nil { + return oi, err + } + } + if len(objectEncryptionKey) == 32 { + var key crypto.ObjectKey + copy(key[:], objectEncryptionKey) + opts.EncryptFn = metadataEncrypter(key) + } + + for idx, part := range partInfoFiles { + if part.Error != "" { + err = objPartToPartErr(part) + bugLogIf(ctx, err) + return oi, err + } + + if parts[idx].PartNumber != part.Number { + internalLogIf(ctx, fmt.Errorf("part.%d.meta has incorrect corresponding part number: expected %d, got %d", parts[idx].PartNumber, parts[idx].PartNumber, part.Number)) + return oi, InvalidPart{ + PartNumber: part.Number, + } + } + + // Add the current part. + fi.AddObjectPart(part.Number, part.ETag, part.Size, part.ActualSize, part.ModTime, part.Index, part.Checksums) + } + + // Calculate full object size. + var objectSize int64 + + // Calculate consolidated actual size. + var objectActualSize int64 + + // Order online disks in accordance with distribution order. + // Order parts metadata in accordance with distribution order. + onlineDisks, partsMetadata = shuffleDisksAndPartsMetadataByIndex(onlineDisks, partsMetadata, fi) + + // Save current erasure metadata for validation. + currentFI := fi + + // Allocate parts similar to incoming slice. + fi.Parts = make([]ObjectPartInfo, len(parts)) + + var checksum hash.Checksum + checksum.Type = checksumType + + // Validate each part and then commit to disk. + for i, part := range parts { + partIdx := objectPartIndex(currentFI.Parts, part.PartNumber) + // All parts should have same part number. + if partIdx == -1 { + invp := InvalidPart{ + PartNumber: part.PartNumber, + GotETag: part.ETag, + } + return oi, invp + } + expPart := currentFI.Parts[partIdx] + + // ensure that part ETag is canonicalized to strip off extraneous quotes + part.ETag = canonicalizeETag(part.ETag) + expETag := tryDecryptETag(objectEncryptionKey, expPart.ETag, kind == crypto.S3) + if expETag != part.ETag { + invp := InvalidPart{ + PartNumber: part.PartNumber, + ExpETag: expETag, + GotETag: part.ETag, + } + return oi, invp + } + + if checksumType.IsSet() { + crc := expPart.Checksums[checksumType.String()] + if crc == "" { + return oi, InvalidPart{ + PartNumber: part.PartNumber, + } + } + wantCS := map[string]string{ + hash.ChecksumCRC32.String(): part.ChecksumCRC32, + hash.ChecksumCRC32C.String(): part.ChecksumCRC32C, + hash.ChecksumSHA1.String(): part.ChecksumSHA1, + hash.ChecksumSHA256.String(): part.ChecksumSHA256, + hash.ChecksumCRC64NVME.String(): part.ChecksumCRC64NVME, + } + if wantCS[checksumType.String()] != crc { + return oi, InvalidPart{ + PartNumber: part.PartNumber, + ExpETag: wantCS[checksumType.String()], + GotETag: crc, + } + } + cs := hash.NewChecksumString(checksumType.String(), crc) + if !cs.Valid() { + return oi, InvalidPart{ + PartNumber: part.PartNumber, + } + } + if checksumType.FullObjectRequested() { + if err := checksum.AddPart(*cs, expPart.ActualSize); err != nil { + return oi, InvalidPart{ + PartNumber: part.PartNumber, + ExpETag: "", + GotETag: err.Error(), + } + } + } + checksumCombined = append(checksumCombined, cs.Raw...) + } + + // All parts except the last part has to be at least 5MB. + if (i < len(parts)-1) && !isMinAllowedPartSize(currentFI.Parts[partIdx].ActualSize) { + return oi, PartTooSmall{ + PartNumber: part.PartNumber, + PartSize: expPart.ActualSize, + PartETag: part.ETag, + } + } + + // Save for total object size. + objectSize += expPart.Size + + // Save the consolidated actual size. + objectActualSize += expPart.ActualSize + + // Add incoming parts. + fi.Parts[i] = ObjectPartInfo{ + Number: part.PartNumber, + Size: expPart.Size, + ActualSize: expPart.ActualSize, + ModTime: expPart.ModTime, + Index: expPart.Index, + Checksums: nil, // Not transferred since we do not need it. + } + } + + if opts.WantChecksum != nil { + if checksumType.FullObjectRequested() { + if opts.WantChecksum.Encoded != checksum.Encoded { + err := hash.ChecksumMismatch{ + Want: opts.WantChecksum.Encoded, + Got: checksum.Encoded, + } + return oi, err + } + } else { + err := opts.WantChecksum.Matches(checksumCombined, len(parts)) + if err != nil { + return oi, err + } + } + } + + // Accept encrypted checksum from incoming request. + if opts.UserDefined[ReplicationSsecChecksumHeader] != "" { + if v, err := base64.StdEncoding.DecodeString(opts.UserDefined[ReplicationSsecChecksumHeader]); err == nil { + fi.Checksum = v + } + delete(opts.UserDefined, ReplicationSsecChecksumHeader) + } + + if checksumType.IsSet() { + checksumType |= hash.ChecksumMultipart | hash.ChecksumIncludesMultipart + checksum.Type = checksumType + if !checksumType.FullObjectRequested() { + checksum = *hash.NewChecksumFromData(checksumType, checksumCombined) + } + fi.Checksum = checksum.AppendTo(nil, checksumCombined) + if opts.EncryptFn != nil { + fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum) + } + } + // Remove superfluous internal headers. + delete(fi.Metadata, hash.MinIOMultipartChecksum) + delete(fi.Metadata, hash.MinIOMultipartChecksumType) + + // Save the final object size and modtime. + fi.Size = objectSize + fi.ModTime = opts.MTime + if opts.MTime.IsZero() { + fi.ModTime = UTCNow() + } + + // Save successfully calculated md5sum. + // for replica, newMultipartUpload would have already sent the replication ETag + if fi.Metadata["etag"] == "" { + if opts.UserDefined["etag"] != "" { + fi.Metadata["etag"] = opts.UserDefined["etag"] + } else { // fallback if not already calculated in handler. + fi.Metadata["etag"] = getCompleteMultipartMD5(parts) + } + } + + // Save the consolidated actual size. + if opts.ReplicationRequest { + if v := opts.UserDefined[ReservedMetadataPrefix+"Actual-Object-Size"]; v != "" { + fi.Metadata[ReservedMetadataPrefix+"actual-size"] = v + } + } else { + fi.Metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(objectActualSize, 10) + } + + if opts.DataMovement { + fi.SetDataMov() + } + + // Update all erasure metadata, make sure to not modify fields like + // checksum which are different on each disks. + for index := range partsMetadata { + if partsMetadata[index].IsValid() { + partsMetadata[index].Size = fi.Size + partsMetadata[index].ModTime = fi.ModTime + partsMetadata[index].Metadata = fi.Metadata + partsMetadata[index].Parts = fi.Parts + partsMetadata[index].Checksum = fi.Checksum + partsMetadata[index].Versioned = opts.Versioned || opts.VersionSuspended + } + } + + paths := make([]string, 0, len(currentFI.Parts)) + // Remove parts that weren't present in CompleteMultipartUpload request. + for _, curpart := range currentFI.Parts { + paths = append(paths, pathJoin(uploadIDPath, currentFI.DataDir, fmt.Sprintf("part.%d.meta", curpart.Number))) + + if objectPartIndex(fi.Parts, curpart.Number) == -1 { + // Delete the missing part files. e.g, + // Request 1: NewMultipart + // Request 2: PutObjectPart 1 + // Request 3: PutObjectPart 2 + // Request 4: CompleteMultipartUpload --part 2 + // N.B. 1st part is not present. This part should be removed from the storage. + paths = append(paths, pathJoin(uploadIDPath, currentFI.DataDir, fmt.Sprintf("part.%d", curpart.Number))) + } + } + + if !opts.NoLock { + lk := er.NewNSLock(bucket, object) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + return ObjectInfo{}, err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + } + + er.cleanupMultipartPath(ctx, paths...) // cleanup all part.N.meta, and skipped part.N's before final rename(). + + defer func() { + if err == nil { + er.deleteAll(context.Background(), minioMetaMultipartBucket, uploadIDPath) + } + }() + + // Rename the multipart object to final location. + onlineDisks, versions, oldDataDir, err := renameData(ctx, onlineDisks, minioMetaMultipartBucket, uploadIDPath, + partsMetadata, bucket, object, writeQuorum) + if err != nil { + return oi, toObjectErr(err, bucket, object, uploadID) + } + + if err = er.commitRenameDataDir(ctx, bucket, object, oldDataDir, onlineDisks, writeQuorum); err != nil { + return ObjectInfo{}, toObjectErr(err, bucket, object, uploadID) + } + + if !opts.Speedtest && len(versions) > 0 { + globalMRFState.addPartialOp(PartialOperation{ + Bucket: bucket, + Object: object, + Queued: time.Now(), + Versions: versions, + SetIndex: er.setIndex, + PoolIndex: er.poolIndex, + }) + } + + if !opts.Speedtest && len(versions) == 0 { + // Check if there is any offline disk and add it to the MRF list + for _, disk := range onlineDisks { + if disk != nil && disk.IsOnline() { + continue + } + er.addPartial(bucket, object, fi.VersionID) + break + } + } + + for i := range len(onlineDisks) { + if onlineDisks[i] != nil && onlineDisks[i].IsOnline() { + // Object info is the same in all disks, so we can pick + // the first meta from online disk + fi = partsMetadata[i] + break + } + } + + // we are adding a new version to this object under the namespace lock, so this is the latest version. + fi.IsLatest = true + + // Success, return object info. + return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil +} + +// AbortMultipartUpload - aborts an ongoing multipart operation +// signified by the input uploadID. This is an atomic operation +// doesn't require clients to initiate multiple such requests. +// +// All parts are purged from all disks and reference to the uploadID +// would be removed from the system, rollback is not possible on this +// operation. +func (er erasureObjects) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (err error) { + if !opts.NoAuditLog { + auditObjectErasureSet(ctx, "AbortMultipartUpload", object, &er) + } + + // Cleanup all uploaded parts. + defer er.deleteAll(ctx, minioMetaMultipartBucket, er.getUploadIDDir(bucket, object, uploadID)) + + // Validates if upload ID exists. + _, _, err = er.checkUploadIDExists(ctx, bucket, object, uploadID, false) + return toObjectErr(err, bucket, object, uploadID) +} diff --git a/cmd/erasure-object.go b/cmd/erasure-object.go new file mode 100644 index 00000000..33fd3dc1 --- /dev/null +++ b/cmd/erasure-object.go @@ -0,0 +1,2599 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/base64" + "errors" + "fmt" + "io" + "maps" + "net/http" + "path" + "runtime" + "slices" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/klauspost/readahead" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7/pkg/tags" + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/config/storageclass" + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/grid" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/mimedb" + "github.com/minio/pkg/v3/sync/errgroup" + "github.com/minio/sio" +) + +// list all errors which can be ignored in object operations. +var objectOpIgnoredErrs = append(baseIgnoredErrs, errDiskAccessDenied, errUnformattedDisk, errDiskOngoingReq) + +// Object Operations + +func countOnlineDisks(onlineDisks []StorageAPI) (online int) { + for _, onlineDisk := range onlineDisks { + if onlineDisk != nil && onlineDisk.IsOnline() { + online++ + } + } + return online +} + +// CopyObject - copy object source object to destination object. +// if source object and destination object are same we only +// update metadata. +func (er erasureObjects) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (oi ObjectInfo, err error) { + if !dstOpts.NoAuditLog { + auditObjectErasureSet(ctx, "CopyObject", dstObject, &er) + } + + // This call shouldn't be used for anything other than metadata updates or adding self referential versions. + if !srcInfo.metadataOnly { + return oi, NotImplemented{} + } + + if !dstOpts.NoLock { + lk := er.NewNSLock(dstBucket, dstObject) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + return oi, err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + } + // Read metadata associated with the object from all disks. + storageDisks := er.getDisks() + + var metaArr []FileInfo + var errs []error + + // Read metadata associated with the object from all disks. + if srcOpts.VersionID != "" { + metaArr, errs = readAllFileInfo(ctx, storageDisks, "", srcBucket, srcObject, srcOpts.VersionID, true, false) + } else { + metaArr, errs = readAllXL(ctx, storageDisks, srcBucket, srcObject, true, false) + } + + readQuorum, writeQuorum, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount) + if err != nil { + if shouldCheckForDangling(err, errs, srcBucket) { + _, derr := er.deleteIfDangling(context.Background(), srcBucket, srcObject, metaArr, errs, nil, srcOpts) + if derr == nil { + if srcOpts.VersionID != "" { + err = errFileVersionNotFound + } else { + err = errFileNotFound + } + } + } + return ObjectInfo{}, toObjectErr(err, srcBucket, srcObject) + } + + // List all online disks. + onlineDisks, modTime, etag := listOnlineDisks(storageDisks, metaArr, errs, readQuorum) + + // Pick latest valid metadata. + fi, err := pickValidFileInfo(ctx, metaArr, modTime, etag, readQuorum) + if err != nil { + return oi, toObjectErr(err, srcBucket, srcObject) + } + if fi.Deleted { + if srcOpts.VersionID == "" { + return oi, toObjectErr(errFileNotFound, srcBucket, srcObject) + } + return fi.ToObjectInfo(srcBucket, srcObject, srcOpts.Versioned || srcOpts.VersionSuspended), toObjectErr(errMethodNotAllowed, srcBucket, srcObject) + } + + filterOnlineDisksInplace(fi, metaArr, onlineDisks) + + versionID := srcInfo.VersionID + if srcInfo.versionOnly { + versionID = dstOpts.VersionID + // preserve destination versionId if specified. + if versionID == "" { + versionID = mustGetUUID() + fi.IsLatest = true // we are creating a new version so this is latest. + } + } + + modTime = UTCNow() // We only preserve modTime if dstOpts.MTime is true. + // in all other cases mtime is latest. + + fi.VersionID = versionID // set any new versionID we might have created + fi.ModTime = modTime // set modTime for the new versionID + if !dstOpts.MTime.IsZero() { + modTime = dstOpts.MTime + fi.ModTime = dstOpts.MTime + } + // check inline before overwriting metadata. + inlineData := fi.InlineData() + + fi.Metadata = srcInfo.UserDefined + srcInfo.UserDefined["etag"] = srcInfo.ETag + + freeVersionID := fi.TierFreeVersionID() + freeVersionMarker := fi.TierFreeVersion() + + // Update `xl.meta` content on each disks. + for index := range metaArr { + if metaArr[index].IsValid() { + metaArr[index].ModTime = modTime + metaArr[index].VersionID = versionID + if !metaArr[index].InlineData() { + // If the data is not inlined, we may end up incorrectly + // inlining the data here, that leads to an inconsistent + // situation where some objects are were not inlined + // were now inlined, make sure to `nil` the Data such + // that xl.meta is written as expected. + metaArr[index].Data = nil + } + metaArr[index].Metadata = srcInfo.UserDefined + // Preserve existing values + if inlineData { + metaArr[index].SetInlineData() + } + if freeVersionID != "" { + metaArr[index].SetTierFreeVersionID(freeVersionID) + } + if freeVersionMarker { + metaArr[index].SetTierFreeVersion() + } + } + } + + // Write unique `xl.meta` for each disk. + if _, err = writeUniqueFileInfo(ctx, onlineDisks, "", srcBucket, srcObject, metaArr, writeQuorum); err != nil { + return oi, toObjectErr(err, srcBucket, srcObject) + } + + return fi.ToObjectInfo(srcBucket, srcObject, srcOpts.Versioned || srcOpts.VersionSuspended), nil +} + +// GetObjectNInfo - returns object info and an object +// Read(Closer). When err != nil, the returned reader is always nil. +func (er erasureObjects) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, err error) { + if !opts.NoAuditLog { + auditObjectErasureSet(ctx, "GetObject", object, &er) + } + + var unlockOnDefer bool + nsUnlocker := func() {} + defer func() { + if unlockOnDefer { + nsUnlocker() + } + }() + + // Acquire lock + if !opts.NoLock { + lock := er.NewNSLock(bucket, object) + lkctx, err := lock.GetRLock(ctx, globalOperationTimeout) + if err != nil { + return nil, err + } + ctx = lkctx.Context() + + // Release lock when the metadata is verified, and reader + // is ready to be read. + // + // This is possible to be lock free because + // - xl.meta for inlined objects has already read the data + // into memory, any mutation on xl.meta subsequently is + // inconsequential to the overall read operation. + // - xl.meta metadata is still verified for quorum under lock() + // however writing the response doesn't need to serialize + // concurrent writers + unlockOnDefer = true + nsUnlocker = func() { lock.RUnlock(lkctx) } + } + + fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts, true) + if err != nil { + return nil, toObjectErr(err, bucket, object) + } + + objInfo := fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended) + if objInfo.DeleteMarker { + if opts.VersionID == "" { + return &GetObjectReader{ + ObjInfo: objInfo, + }, toObjectErr(errFileNotFound, bucket, object) + } + // Make sure to return object info to provide extra information. + return &GetObjectReader{ + ObjInfo: objInfo, + }, toObjectErr(errMethodNotAllowed, bucket, object) + } + + // Set NoDecryption for SSE-C objects and if replication request + if crypto.SSEC.IsEncrypted(objInfo.UserDefined) && opts.ReplicationRequest { + opts.NoDecryption = true + } + + if objInfo.Size == 0 { + if _, _, err := rs.GetOffsetLength(objInfo.Size); err != nil { + // Make sure to return object info to provide extra information. + return &GetObjectReader{ + ObjInfo: objInfo, + }, err + } + + // Zero byte objects don't even need to further initialize pipes etc. + return NewGetObjectReaderFromReader(bytes.NewReader(nil), objInfo, opts) + } + + if objInfo.IsRemote() { + gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, h, objInfo, opts) + if err != nil { + return nil, err + } + unlockOnDefer = false + return gr.WithCleanupFuncs(nsUnlocker), nil + } + + fn, off, length, err := NewGetObjectReader(rs, objInfo, opts, h) + if err != nil { + return nil, err + } + + if unlockOnDefer { + unlockOnDefer = fi.InlineData() || len(fi.Data) > 0 + } + + pr, pw := xioutil.WaitPipe() + go func() { + pw.CloseWithError(er.getObjectWithFileInfo(ctx, bucket, object, off, length, pw, fi, metaArr, onlineDisks)) + }() + + // Cleanup function to cause the go routine above to exit, in + // case of incomplete read. + pipeCloser := func() { + pr.CloseWithError(nil) + } + + if !unlockOnDefer { + return fn(pr, h, pipeCloser, nsUnlocker) + } + + return fn(pr, h, pipeCloser) +} + +func (er erasureObjects) getObjectWithFileInfo(ctx context.Context, bucket, object string, startOffset int64, length int64, writer io.Writer, fi FileInfo, metaArr []FileInfo, onlineDisks []StorageAPI) error { + // Reorder online disks based on erasure distribution order. + // Reorder parts metadata based on erasure distribution order. + onlineDisks, metaArr = shuffleDisksAndPartsMetadataByIndex(onlineDisks, metaArr, fi) + + // For negative length read everything. + if length < 0 { + length = fi.Size - startOffset + } + + // Reply back invalid range if the input offset and length fall out of range. + if startOffset > fi.Size || startOffset+length > fi.Size { + return InvalidRange{startOffset, length, fi.Size} + } + + // Get start part index and offset. + partIndex, partOffset, err := fi.ObjectToPartOffset(ctx, startOffset) + if err != nil { + return InvalidRange{startOffset, length, fi.Size} + } + + // Calculate endOffset according to length + endOffset := startOffset + if length > 0 { + endOffset += length - 1 + } + + // Get last part index to read given length. + lastPartIndex, _, err := fi.ObjectToPartOffset(ctx, endOffset) + if err != nil { + return InvalidRange{startOffset, length, fi.Size} + } + + var totalBytesRead int64 + erasure, err := NewErasure(ctx, fi.Erasure.DataBlocks, fi.Erasure.ParityBlocks, fi.Erasure.BlockSize) + if err != nil { + return toObjectErr(err, bucket, object) + } + + var healOnce sync.Once + + for ; partIndex <= lastPartIndex; partIndex++ { + if length == totalBytesRead { + break + } + + partNumber := fi.Parts[partIndex].Number + + // Save the current part name and size. + partSize := fi.Parts[partIndex].Size + + partLength := partSize - partOffset + // partLength should be adjusted so that we don't write more data than what was requested. + if partLength > (length - totalBytesRead) { + partLength = length - totalBytesRead + } + + tillOffset := erasure.ShardFileOffset(partOffset, partLength, partSize) + // Get the checksums of the current part. + readers := make([]io.ReaderAt, len(onlineDisks)) + prefer := make([]bool, len(onlineDisks)) + for index, disk := range onlineDisks { + if disk == OfflineDisk { + continue + } + if !metaArr[index].IsValid() { + continue + } + if !metaArr[index].Erasure.Equal(fi.Erasure) { + continue + } + checksumInfo := metaArr[index].Erasure.GetChecksumInfo(partNumber) + partPath := pathJoin(object, metaArr[index].DataDir, fmt.Sprintf("part.%d", partNumber)) + readers[index] = newBitrotReader(disk, metaArr[index].Data, bucket, partPath, tillOffset, + checksumInfo.Algorithm, checksumInfo.Hash, erasure.ShardSize()) + + // Prefer local disks + prefer[index] = disk.Hostname() == "" + } + + written, err := erasure.Decode(ctx, writer, readers, partOffset, partLength, partSize, prefer) + // Note: we should not be defer'ing the following closeBitrotReaders() call as + // we are inside a for loop i.e if we use defer, we would accumulate a lot of open files by the time + // we return from this function. + closeBitrotReaders(readers) + if err != nil { + // If we have successfully written all the content that was asked + // by the client, but we still see an error - this would mean + // that we have some parts or data blocks missing or corrupted + // - attempt a heal to successfully heal them for future calls. + if written == partLength { + if errors.Is(err, errFileNotFound) || errors.Is(err, errFileCorrupt) { + healOnce.Do(func() { + globalMRFState.addPartialOp(PartialOperation{ + Bucket: bucket, + Object: object, + VersionID: fi.VersionID, + Queued: time.Now(), + SetIndex: er.setIndex, + PoolIndex: er.poolIndex, + BitrotScan: errors.Is(err, errFileCorrupt), + }) + }) + // Healing is triggered and we have written + // successfully the content to client for + // the specific part, we should `nil` this error + // and proceed forward, instead of throwing errors. + err = nil + } + } + if err != nil { + return toObjectErr(err, bucket, object) + } + } + // Track total bytes read from disk and written to the client. + totalBytesRead += partLength + // partOffset will be valid only for the first part, hence reset it to 0 for + // the remaining parts. + partOffset = 0 + } // End of read all parts loop. + // Return success. + return nil +} + +// GetObjectInfo - reads object metadata and replies back ObjectInfo. +func (er erasureObjects) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (info ObjectInfo, err error) { + if !opts.NoAuditLog { + auditObjectErasureSet(ctx, "GetObjectInfo", object, &er) + } + + if !opts.NoLock { + // Lock the object before reading. + lk := er.NewNSLock(bucket, object) + lkctx, err := lk.GetRLock(ctx, globalOperationTimeout) + if err != nil { + return ObjectInfo{}, err + } + ctx = lkctx.Context() + defer lk.RUnlock(lkctx) + } + + return er.getObjectInfo(ctx, bucket, object, opts) +} + +func auditDanglingObjectDeletion(ctx context.Context, bucket, object, versionID string, tags map[string]string) { + if len(logger.AuditTargets()) == 0 { + return + } + + opts := AuditLogOptions{ + Event: "DeleteDanglingObject", + Bucket: bucket, + Object: object, + VersionID: versionID, + Tags: tags, + } + + auditLogInternal(ctx, opts) +} + +func joinErrs(errs []error) string { + var s string + for i := range s { + if s != "" { + s += "," + } + if errs[i] == nil { + s += "" + } else { + s += errs[i].Error() + } + } + return s +} + +func (er erasureObjects) deleteIfDangling(ctx context.Context, bucket, object string, metaArr []FileInfo, errs []error, dataErrsByPart map[int][]int, opts ObjectOptions) (FileInfo, error) { + m, ok := isObjectDangling(metaArr, errs, dataErrsByPart) + if !ok { + // We only come here if we cannot figure out if the object + // can be deleted safely, in such a scenario return ReadQuorum error. + return FileInfo{}, errErasureReadQuorum + } + tags := make(map[string]string, 16) + tags["set"] = strconv.Itoa(er.setIndex) + tags["pool"] = strconv.Itoa(er.poolIndex) + tags["merrs"] = joinErrs(errs) + tags["derrs"] = fmt.Sprintf("%v", dataErrsByPart) + if m.IsValid() { + tags["sz"] = strconv.FormatInt(m.Size, 10) + tags["mt"] = m.ModTime.Format(iso8601Format) + tags["d:p"] = fmt.Sprintf("%d:%d", m.Erasure.DataBlocks, m.Erasure.ParityBlocks) + } else { + tags["invalid"] = "1" + tags["d:p"] = fmt.Sprintf("%d:%d", er.setDriveCount-er.defaultParityCount, er.defaultParityCount) + } + + // count the number of offline disks + offline := 0 + for i := range len(errs) { + var found bool + switch { + case errors.Is(errs[i], errDiskNotFound): + found = true + default: + for p := range dataErrsByPart { + if dataErrsByPart[p][i] == checkPartDiskNotFound { + found = true + break + } + } + } + if found { + offline++ + } + } + if offline > 0 { + tags["offline"] = strconv.Itoa(offline) + } + + _, file, line, cok := runtime.Caller(1) + if cok { + tags["caller"] = fmt.Sprintf("%s:%d", file, line) + } + + defer auditDanglingObjectDeletion(ctx, bucket, object, m.VersionID, tags) + + fi := FileInfo{ + VersionID: m.VersionID, + } + if opts.VersionID != "" { + fi.VersionID = opts.VersionID + } + fi.SetTierFreeVersionID(mustGetUUID()) + disks := er.getDisks() + g := errgroup.WithNErrs(len(disks)) + for index := range disks { + g.Go(func() error { + if disks[index] == nil { + return errDiskNotFound + } + return disks[index].DeleteVersion(ctx, bucket, object, fi, false, DeleteOptions{}) + }, index) + } + + for index, err := range g.Wait() { + var errStr string + if err != nil { + errStr = err.Error() + } else { + errStr = "" + } + tags[fmt.Sprintf("ddisk-%d", index)] = errStr + } + + return m, nil +} + +func fileInfoFromRaw(ri RawFileInfo, bucket, object string, readData, inclFreeVers bool) (FileInfo, error) { + return getFileInfo(ri.Buf, bucket, object, "", fileInfoOpts{ + Data: readData, + InclFreeVersions: inclFreeVers, + }) +} + +func readAllRawFileInfo(ctx context.Context, disks []StorageAPI, bucket, object string, readData bool) ([]RawFileInfo, []error) { + rawFileInfos := make([]RawFileInfo, len(disks)) + g := errgroup.WithNErrs(len(disks)) + for index := range disks { + g.Go(func() (err error) { + if disks[index] == nil { + return errDiskNotFound + } + rf, err := disks[index].ReadXL(ctx, bucket, object, readData) + if err != nil { + return err + } + rawFileInfos[index] = rf + return nil + }, index) + } + + return rawFileInfos, g.Wait() +} + +func pickLatestQuorumFilesInfo(ctx context.Context, rawFileInfos []RawFileInfo, errs []error, bucket, object string, readData, inclFreeVers bool) ([]FileInfo, []error) { + metadataArray := make([]*xlMetaV2, len(rawFileInfos)) + metaFileInfos := make([]FileInfo, len(rawFileInfos)) + metadataShallowVersions := make([][]xlMetaV2ShallowVersion, len(rawFileInfos)) + var v2bufs [][]byte + if !readData { + v2bufs = make([][]byte, len(rawFileInfos)) + } + + // Read `xl.meta` in parallel across disks. + for index := range rawFileInfos { + rf := rawFileInfos[index] + if rf.Buf == nil { + continue + } + if !readData { + // Save the buffer so we can reuse it. + v2bufs[index] = rf.Buf + } + + var xl xlMetaV2 + if err := xl.LoadOrConvert(rf.Buf); err != nil { + errs[index] = err + continue + } + metadataArray[index] = &xl + metaFileInfos[index] = FileInfo{} + } + + for index := range metadataArray { + if metadataArray[index] != nil { + metadataShallowVersions[index] = metadataArray[index].versions + } + } + + readQuorum := (len(rawFileInfos) + 1) / 2 + meta := &xlMetaV2{versions: mergeXLV2Versions(readQuorum, false, 1, metadataShallowVersions...)} + lfi, err := meta.ToFileInfo(bucket, object, "", inclFreeVers, true) + if err != nil { + for i := range errs { + if errs[i] == nil { + errs[i] = err + } + } + return metaFileInfos, errs + } + if !lfi.IsValid() { + for i := range errs { + if errs[i] == nil { + errs[i] = errFileCorrupt + } + } + return metaFileInfos, errs + } + + versionID := lfi.VersionID + if versionID == "" { + versionID = nullVersionID + } + + for index := range metadataArray { + if metadataArray[index] == nil { + continue + } + + // make sure to preserve this for diskmtime based healing bugfix. + metaFileInfos[index], errs[index] = metadataArray[index].ToFileInfo(bucket, object, versionID, inclFreeVers, true) + if errs[index] != nil { + continue + } + + if readData { + metaFileInfos[index].Data = metadataArray[index].data.find(versionID) + } + } + if !readData { + for i := range v2bufs { + metaDataPoolPut(v2bufs[i]) + } + } + + // Return all the metadata. + return metaFileInfos, errs +} + +// Checking if an object is dangling costs some IOPS; hence implementing this function +// which decides which condition it is useful to check if an object is dangling +// +// errs: errors from reading xl.meta in all disks +// err: reduced errs +// bucket: the object name in question +func shouldCheckForDangling(err error, errs []error, bucket string) bool { + // Avoid data in .minio.sys for now + if bucket == minioMetaBucket { + return false + } + switch { + // Check if we have a read quorum issue + case errors.Is(err, errErasureReadQuorum): + return true + // Check if the object is non-existent on most disks but not all of them + case (errors.Is(err, errFileNotFound) || errors.Is(err, errFileVersionNotFound)) && (countErrs(errs, nil) > 0): + return true + } + return false +} + +func readAllXL(ctx context.Context, disks []StorageAPI, bucket, object string, readData, inclFreeVers bool) ([]FileInfo, []error) { + rawFileInfos, errs := readAllRawFileInfo(ctx, disks, bucket, object, readData) + return pickLatestQuorumFilesInfo(ctx, rawFileInfos, errs, bucket, object, readData, inclFreeVers) +} + +func (er erasureObjects) getObjectFileInfo(ctx context.Context, bucket, object string, opts ObjectOptions, readData bool) (FileInfo, []FileInfo, []StorageAPI, error) { + rawArr := make([]RawFileInfo, er.setDriveCount) + metaArr := make([]FileInfo, er.setDriveCount) + errs := make([]error, er.setDriveCount) + for i := range errs { + errs[i] = errDiskOngoingReq + } + + done := make(chan bool, er.setDriveCount) + disks := er.getDisks() + + ropts := ReadOptions{ + ReadData: readData, + InclFreeVersions: opts.InclFreeVersions, + Healing: false, + } + + mrfCheck := make(chan FileInfo) + defer xioutil.SafeClose(mrfCheck) + + var rw sync.Mutex + + // Ask for all disks first; + go func() { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + wg := sync.WaitGroup{} + for i, disk := range disks { + if disk == nil { + done <- false + continue + } + if !disk.IsOnline() { + done <- false + continue + } + wg.Add(1) + go func(i int, disk StorageAPI) { + defer wg.Done() + + var ( + fi FileInfo + rfi RawFileInfo + err error + ) + + if opts.VersionID != "" { + // Read a specific version ID + fi, err = disk.ReadVersion(ctx, "", bucket, object, opts.VersionID, ropts) + } else { + // Read the latest version + rfi, err = disk.ReadXL(ctx, bucket, object, readData) + if err == nil { + fi, err = fileInfoFromRaw(rfi, bucket, object, readData, opts.InclFreeVersions) + } + } + + rw.Lock() + rawArr[i] = rfi + metaArr[i], errs[i] = fi, err + rw.Unlock() + + done <- err == nil + }(i, disk) + } + + wg.Wait() + xioutil.SafeClose(done) + + fi, ok := <-mrfCheck + if !ok { + return + } + + if fi.Deleted { + return + } + + // if one of the disk is offline, return right here no need + // to attempt a heal on the object. + if countErrs(errs, errDiskNotFound) > 0 { + return + } + + var missingBlocks int + for i := range errs { + if IsErr(errs[i], + errFileNotFound, + errFileVersionNotFound, + errFileCorrupt, + ) { + missingBlocks++ + } + } + + // if missing metadata can be reconstructed, attempt to reconstruct. + // additionally do not heal delete markers inline, let them be + // healed upon regular heal process. + if missingBlocks > 0 && missingBlocks < fi.Erasure.DataBlocks { + globalMRFState.addPartialOp(PartialOperation{ + Bucket: fi.Volume, + Object: fi.Name, + VersionID: fi.VersionID, + Queued: time.Now(), + SetIndex: er.setIndex, + PoolIndex: er.poolIndex, + }) + } + }() + + validResp := 0 + totalResp := 0 + + // minDisks value is only to reduce the number of calls + // to the disks; this value is not accurate because we do + // not know the storage class of the object yet + minDisks := 0 + if p := globalStorageClass.GetParityForSC(""); p > -1 { + minDisks = er.setDriveCount - p + } else { + minDisks = er.setDriveCount - er.defaultParityCount + } + + if minDisks == er.setDriveCount/2 { + // when data and parity are same we must atleast + // wait for response from 1 extra drive to avoid + // split-brain. + minDisks++ + } + + calcQuorum := func(metaArr []FileInfo, errs []error) (FileInfo, []FileInfo, []StorageAPI, time.Time, string, error) { + readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount) + if err != nil { + return FileInfo{}, nil, nil, time.Time{}, "", err + } + if err := reduceReadQuorumErrs(ctx, errs, objectOpIgnoredErrs, readQuorum); err != nil { + return FileInfo{}, nil, nil, time.Time{}, "", err + } + onlineDisks, modTime, etag := listOnlineDisks(disks, metaArr, errs, readQuorum) + fi, err := pickValidFileInfo(ctx, metaArr, modTime, etag, readQuorum) + if err != nil { + return FileInfo{}, nil, nil, time.Time{}, "", err + } + + onlineMeta := make([]FileInfo, len(metaArr)) + for i, disk := range onlineDisks { + if disk != nil { + onlineMeta[i] = metaArr[i] + } + } + + return fi, onlineMeta, onlineDisks, modTime, etag, nil + } + + var ( + modTime time.Time + etag string + fi FileInfo + onlineMeta []FileInfo + onlineDisks []StorageAPI + err error + ) + + for success := range done { + totalResp++ + if success { + validResp++ + } + + if totalResp >= minDisks && opts.FastGetObjInfo { + rw.Lock() + ok := countErrs(errs, errFileNotFound) >= minDisks || countErrs(errs, errFileVersionNotFound) >= minDisks + rw.Unlock() + if ok { + err = errFileNotFound + if opts.VersionID != "" { + err = errFileVersionNotFound + } + break + } + } + + if totalResp < er.setDriveCount { + if !opts.FastGetObjInfo { + continue + } + if validResp < minDisks { + continue + } + } + + rw.Lock() + // when its a versioned bucket and empty versionID - at totalResp == setDriveCount + // we must use rawFileInfo to resolve versions to figure out the latest version. + if opts.VersionID == "" && totalResp == er.setDriveCount { + fi, onlineMeta, onlineDisks, modTime, etag, err = calcQuorum(pickLatestQuorumFilesInfo(ctx, + rawArr, errs, bucket, object, readData, opts.InclFreeVersions)) + } else { + fi, onlineMeta, onlineDisks, modTime, etag, err = calcQuorum(metaArr, errs) + } + rw.Unlock() + if err == nil && (fi.InlineData() || len(fi.Data) > 0) { + break + } + } + + if err != nil { + // We can only look for dangling if we received all the responses, if we did + // not we simply ignore it, since we can't tell for sure if its dangling object. + if totalResp == er.setDriveCount && shouldCheckForDangling(err, errs, bucket) { + _, derr := er.deleteIfDangling(context.Background(), bucket, object, metaArr, errs, nil, opts) + if derr == nil { + if opts.VersionID != "" { + err = errFileVersionNotFound + } else { + err = errFileNotFound + } + } + } + // when we have insufficient read quorum and inconsistent metadata return + // file not found, since we can't possibly have a way to recover this object + // anyway. + if v, ok := err.(InsufficientReadQuorum); ok && v.Type == RQInconsistentMeta { + if opts.VersionID != "" { + err = errFileVersionNotFound + } else { + err = errFileNotFound + } + } + return fi, nil, nil, toObjectErr(err, bucket, object) + } + + if !fi.Deleted && len(fi.Erasure.Distribution) != len(onlineDisks) { + err := fmt.Errorf("unexpected file distribution (%v) from online disks (%v), looks like backend disks have been manually modified refusing to heal %s/%s(%s)", + fi.Erasure.Distribution, onlineDisks, bucket, object, opts.VersionID) + storageLogOnceIf(ctx, err, "get-object-file-info-manually-modified") + return fi, nil, nil, toObjectErr(err, bucket, object, opts.VersionID) + } + + filterOnlineDisksInplace(fi, onlineMeta, onlineDisks) + for i := range onlineMeta { + // verify metadata is valid, it has similar erasure info + // as well as common modtime, if modtime is not possible + // verify if it has common "etag" at least. + if onlineMeta[i].IsValid() && onlineMeta[i].Erasure.Equal(fi.Erasure) { + ok := onlineMeta[i].ModTime.Equal(modTime) + if modTime.IsZero() || modTime.Equal(timeSentinel) { + ok = etag != "" && etag == fi.Metadata["etag"] + } + if ok { + continue + } + } // in all other cases metadata is corrupt, do not read from it. + + onlineMeta[i] = FileInfo{} + onlineDisks[i] = nil + } + + select { + case mrfCheck <- fi.ShallowCopy(): + case <-ctx.Done(): + return fi, onlineMeta, onlineDisks, toObjectErr(ctx.Err(), bucket, object) + } + + return fi, onlineMeta, onlineDisks, nil +} + +// getObjectInfo - wrapper for reading object metadata and constructs ObjectInfo. +func (er erasureObjects) getObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) { + fi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false) + if err != nil { + return objInfo, toObjectErr(err, bucket, object) + } + objInfo = fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended) + if fi.Deleted { + if opts.VersionID == "" || opts.DeleteMarker { + return objInfo, toObjectErr(errFileNotFound, bucket, object) + } + // Make sure to return object info to provide extra information. + return objInfo, toObjectErr(errMethodNotAllowed, bucket, object) + } + + return objInfo, nil +} + +// getObjectInfoAndQuorum - wrapper for reading object metadata and constructs ObjectInfo, additionally returns write quorum for the object. +func (er erasureObjects) getObjectInfoAndQuorum(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, wquorum int, err error) { + fi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false) + if err != nil { + return objInfo, er.defaultWQuorum(), toObjectErr(err, bucket, object) + } + + wquorum = fi.WriteQuorum(er.defaultWQuorum()) + + objInfo = fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended) + if !fi.VersionPurgeStatus().Empty() && opts.VersionID != "" { + // Make sure to return object info to provide extra information. + return objInfo, wquorum, toObjectErr(errMethodNotAllowed, bucket, object) + } + + if fi.Deleted { + if opts.VersionID == "" || opts.DeleteMarker { + return objInfo, wquorum, toObjectErr(errFileNotFound, bucket, object) + } + // Make sure to return object info to provide extra information. + return objInfo, wquorum, toObjectErr(errMethodNotAllowed, bucket, object) + } + + return objInfo, wquorum, nil +} + +// Similar to rename but renames data from srcEntry to dstEntry at dataDir +func renameData(ctx context.Context, disks []StorageAPI, srcBucket, srcEntry string, metadata []FileInfo, dstBucket, dstEntry string, writeQuorum int) ([]StorageAPI, []byte, string, error) { + g := errgroup.WithNErrs(len(disks)) + + fvID := mustGetUUID() + for index := range disks { + metadata[index].SetTierFreeVersionID(fvID) + } + + diskVersions := make([][]byte, len(disks)) + dataDirs := make([]string, len(disks)) + // Rename file on all underlying storage disks. + for index := range disks { + g.Go(func() error { + if disks[index] == nil { + return errDiskNotFound + } + + // Pick one FileInfo for a disk at index. + fi := metadata[index] + // Assign index when index is initialized + if fi.Erasure.Index == 0 { + fi.Erasure.Index = index + 1 + } + + if !fi.IsValid() { + return errFileCorrupt + } + resp, err := disks[index].RenameData(ctx, srcBucket, srcEntry, fi, dstBucket, dstEntry, RenameOptions{}) + if err != nil { + return err + } + diskVersions[index] = resp.Sign + dataDirs[index] = resp.OldDataDir + return nil + }, index) + } + + // Wait for all renames to finish. + errs := g.Wait() + + err := reduceWriteQuorumErrs(ctx, errs, objectOpIgnoredErrs, writeQuorum) + if err != nil { + dg := errgroup.WithNErrs(len(disks)) + for index, nerr := range errs { + if nerr != nil { + continue + } + index := index + // When we are going to return error, attempt to delete success + // on some of the drives, if we cannot we do not have to notify + // caller this dangling object will be now scheduled to be removed + // via active healing. + dg.Go(func() error { + return disks[index].DeleteVersion(context.Background(), dstBucket, dstEntry, metadata[index], false, DeleteOptions{ + UndoWrite: true, + OldDataDir: dataDirs[index], + }) + }, index) + } + dg.Wait() + } + var dataDir string + var versions []byte + if err == nil { + versions = reduceCommonVersions(diskVersions, writeQuorum) + for index, dversions := range diskVersions { + if errs[index] != nil { + continue + } + if !bytes.Equal(dversions, versions) { + if len(dversions) > len(versions) { + versions = dversions + } + break + } + } + dataDir = reduceCommonDataDir(dataDirs, writeQuorum) + } + + // We can safely allow RenameData errors up to len(er.getDisks()) - writeQuorum + // otherwise return failure. + return evalDisks(disks, errs), versions, dataDir, err +} + +func (er erasureObjects) putMetacacheObject(ctx context.Context, key string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) { + data := r.Reader + + // No metadata is set, allocate a new one. + if opts.UserDefined == nil { + opts.UserDefined = make(map[string]string) + } + + storageDisks := er.getDisks() + // Get parity and data drive count based on storage class metadata + parityDrives := globalStorageClass.GetParityForSC(opts.UserDefined[xhttp.AmzStorageClass]) + if parityDrives < 0 { + parityDrives = er.defaultParityCount + } + dataDrives := len(storageDisks) - parityDrives + + // we now know the number of blocks this object needs for data and parity. + // writeQuorum is dataBlocks + 1 + writeQuorum := dataDrives + if dataDrives == parityDrives { + writeQuorum++ + } + + // Validate input data size and it can never be less than zero. + if data.Size() < -1 { + bugLogIf(ctx, errInvalidArgument, logger.ErrorKind) + return ObjectInfo{}, toObjectErr(errInvalidArgument) + } + + // Initialize parts metadata + partsMetadata := make([]FileInfo, len(storageDisks)) + + fi := newFileInfo(pathJoin(minioMetaBucket, key), dataDrives, parityDrives) + fi.DataDir = mustGetUUID() + + // Initialize erasure metadata. + for index := range partsMetadata { + partsMetadata[index] = fi + } + + // Order disks according to erasure distribution + var onlineDisks []StorageAPI + onlineDisks, partsMetadata = shuffleDisksAndPartsMetadata(storageDisks, partsMetadata, fi) + + erasure, err := NewErasure(ctx, fi.Erasure.DataBlocks, fi.Erasure.ParityBlocks, fi.Erasure.BlockSize) + if err != nil { + return ObjectInfo{}, toObjectErr(err, minioMetaBucket, key) + } + + // Fetch buffer for I/O, returns from the pool if not allocates a new one and returns. + var buffer []byte + switch size := data.Size(); { + case size == 0: + buffer = make([]byte, 1) // Allocate at least a byte to reach EOF + case size >= fi.Erasure.BlockSize: + buffer = globalBytePoolCap.Load().Get() + defer globalBytePoolCap.Load().Put(buffer) + case size < fi.Erasure.BlockSize: + // No need to allocate fully blockSizeV1 buffer if the incoming data is smaller. + buffer = make([]byte, size, 2*size+int64(fi.Erasure.ParityBlocks+fi.Erasure.DataBlocks-1)) + } + + if len(buffer) > int(fi.Erasure.BlockSize) { + buffer = buffer[:fi.Erasure.BlockSize] + } + + writers := make([]io.Writer, len(onlineDisks)) + inlineBuffers := make([]*bytes.Buffer, len(onlineDisks)) + for i, disk := range onlineDisks { + if disk == nil { + continue + } + if disk.IsOnline() { + buf := grid.GetByteBufferCap(int(erasure.ShardFileSize(data.Size())) + 64) + inlineBuffers[i] = bytes.NewBuffer(buf[:0]) + defer grid.PutByteBuffer(buf) + writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize()) + } + } + + n, erasureErr := erasure.Encode(ctx, data, writers, buffer, writeQuorum) + closeErrs := closeBitrotWriters(writers) + if erasureErr != nil { + return ObjectInfo{}, toObjectErr(erasureErr, minioMetaBucket, key) + } + + if closeErr := reduceWriteQuorumErrs(ctx, closeErrs, objectOpIgnoredErrs, writeQuorum); closeErr != nil { + return ObjectInfo{}, toObjectErr(closeErr, minioMetaBucket, key) + } + + // Should return IncompleteBody{} error when reader has fewer bytes + // than specified in request header. + if n < data.Size() { + return ObjectInfo{}, IncompleteBody{Bucket: minioMetaBucket, Object: key} + } + var index []byte + if opts.IndexCB != nil { + index = opts.IndexCB() + } + + modTime := UTCNow() + + for i, w := range writers { + if w == nil { + // Make sure to avoid writing to disks which we couldn't complete in erasure.Encode() + onlineDisks[i] = nil + continue + } + partsMetadata[i].Data = inlineBuffers[i].Bytes() + partsMetadata[i].AddObjectPart(1, "", n, data.ActualSize(), modTime, index, nil) + } + + // Fill all the necessary metadata. + // Update `xl.meta` content on each disks. + for index := range partsMetadata { + partsMetadata[index].Size = n + partsMetadata[index].Fresh = true + partsMetadata[index].ModTime = modTime + partsMetadata[index].Metadata = opts.UserDefined + } + + // Set an additional header when data is inlined. + for index := range partsMetadata { + partsMetadata[index].SetInlineData() + } + + for i := range len(onlineDisks) { + if onlineDisks[i] != nil && onlineDisks[i].IsOnline() { + // Object info is the same in all disks, so we can pick + // the first meta from online disk + fi = partsMetadata[i] + break + } + } + + if _, err = writeUniqueFileInfo(ctx, onlineDisks, "", minioMetaBucket, key, partsMetadata, writeQuorum); err != nil { + return ObjectInfo{}, toObjectErr(err, minioMetaBucket, key) + } + + return fi.ToObjectInfo(minioMetaBucket, key, opts.Versioned || opts.VersionSuspended), nil +} + +// PutObject - creates an object upon reading from the input stream +// until EOF, erasure codes the data across all disk and additionally +// writes `xl.meta` which carries the necessary metadata for future +// object operations. +func (er erasureObjects) PutObject(ctx context.Context, bucket string, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) { + return er.putObject(ctx, bucket, object, data, opts) +} + +// putObject wrapper for erasureObjects PutObject +func (er erasureObjects) putObject(ctx context.Context, bucket string, object string, r *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) { + if !opts.NoAuditLog { + auditObjectErasureSet(ctx, "PutObject", object, &er) + } + + data := r.Reader + + if opts.CheckPrecondFn != nil { + if !opts.NoLock { + ns := er.NewNSLock(bucket, object) + lkctx, err := ns.GetLock(ctx, globalOperationTimeout) + if err != nil { + return ObjectInfo{}, err + } + ctx = lkctx.Context() + defer ns.Unlock(lkctx) + opts.NoLock = true + } + + obj, err := er.getObjectInfo(ctx, bucket, object, opts) + if err == nil && opts.CheckPrecondFn(obj) { + return objInfo, PreConditionFailed{} + } + if err != nil && !isErrVersionNotFound(err) && !isErrObjectNotFound(err) && !isErrReadQuorum(err) { + return objInfo, err + } + + // if object doesn't exist return error for If-Match conditional requests + // If-None-Match should be allowed to proceed for non-existent objects + if err != nil && opts.HasIfMatch && (isErrObjectNotFound(err) || isErrVersionNotFound(err)) { + return objInfo, err + } + } + + // Validate input data size and it can never be less than -1. + if data.Size() < -1 { + bugLogIf(ctx, errInvalidArgument, logger.ErrorKind) + return ObjectInfo{}, toObjectErr(errInvalidArgument) + } + + userDefined := cloneMSS(opts.UserDefined) + + storageDisks := er.getDisks() + + // Get parity and data drive count based on storage class metadata + parityDrives := globalStorageClass.GetParityForSC(userDefined[xhttp.AmzStorageClass]) + if parityDrives < 0 { + parityDrives = er.defaultParityCount + } + if opts.MaxParity { + parityDrives = len(storageDisks) / 2 + } + if !opts.MaxParity && globalStorageClass.AvailabilityOptimized() { + // If we have offline disks upgrade the number of erasure codes for this object. + parityOrig := parityDrives + + var offlineDrives int + for _, disk := range storageDisks { + if disk == nil || !disk.IsOnline() { + parityDrives++ + offlineDrives++ + continue + } + } + + if offlineDrives >= (len(storageDisks)+1)/2 { + // if offline drives are more than 50% of the drives + // we have no quorum, we shouldn't proceed just + // fail at that point. + return ObjectInfo{}, toObjectErr(errErasureWriteQuorum, bucket, object) + } + + if parityDrives >= len(storageDisks)/2 { + parityDrives = len(storageDisks) / 2 + } + + if parityOrig != parityDrives { + userDefined[minIOErasureUpgraded] = strconv.Itoa(parityOrig) + "->" + strconv.Itoa(parityDrives) + } + } + dataDrives := len(storageDisks) - parityDrives + + // we now know the number of blocks this object needs for data and parity. + // writeQuorum is dataBlocks + 1 + writeQuorum := dataDrives + if dataDrives == parityDrives { + writeQuorum++ + } + + // Initialize parts metadata + partsMetadata := make([]FileInfo, len(storageDisks)) + + fi := newFileInfo(pathJoin(bucket, object), dataDrives, parityDrives) + fi.VersionID = opts.VersionID + if opts.Versioned && fi.VersionID == "" { + fi.VersionID = mustGetUUID() + } + + fi.DataDir = mustGetUUID() + if ckSum := userDefined[ReplicationSsecChecksumHeader]; ckSum != "" { + if v, err := base64.StdEncoding.DecodeString(ckSum); err == nil { + fi.Checksum = v + } + delete(userDefined, ReplicationSsecChecksumHeader) + } + uniqueID := mustGetUUID() + tempObj := uniqueID + + // Initialize erasure metadata. + for index := range partsMetadata { + partsMetadata[index] = fi + } + + // Order disks according to erasure distribution + var onlineDisks []StorageAPI + onlineDisks, partsMetadata = shuffleDisksAndPartsMetadata(storageDisks, partsMetadata, fi) + + erasure, err := NewErasure(ctx, fi.Erasure.DataBlocks, fi.Erasure.ParityBlocks, fi.Erasure.BlockSize) + if err != nil { + return ObjectInfo{}, toObjectErr(err, bucket, object) + } + + // Fetch buffer for I/O, returns from the pool if not allocates a new one and returns. + var buffer []byte + switch size := data.Size(); { + case size == 0: + buffer = make([]byte, 1) // Allocate at least a byte to reach EOF + case size >= fi.Erasure.BlockSize || size == -1: + buffer = globalBytePoolCap.Load().Get() + defer globalBytePoolCap.Load().Put(buffer) + case size < fi.Erasure.BlockSize: + // No need to allocate fully blockSizeV1 buffer if the incoming data is smaller. + buffer = make([]byte, size, 2*size+int64(fi.Erasure.ParityBlocks+fi.Erasure.DataBlocks-1)) + } + + if len(buffer) > int(fi.Erasure.BlockSize) { + buffer = buffer[:fi.Erasure.BlockSize] + } + + partName := "part.1" + tempErasureObj := pathJoin(uniqueID, fi.DataDir, partName) + + defer er.deleteAll(context.Background(), minioMetaTmpBucket, tempObj) + + var inlineBuffers []*bytes.Buffer + if globalStorageClass.ShouldInline(erasure.ShardFileSize(data.ActualSize()), opts.Versioned) { + inlineBuffers = make([]*bytes.Buffer, len(onlineDisks)) + } + + shardFileSize := erasure.ShardFileSize(data.Size()) + writers := make([]io.Writer, len(onlineDisks)) + for i, disk := range onlineDisks { + if disk == nil { + continue + } + + if !disk.IsOnline() { + continue + } + + if len(inlineBuffers) > 0 { + buf := grid.GetByteBufferCap(int(shardFileSize) + 64) + inlineBuffers[i] = bytes.NewBuffer(buf[:0]) + defer grid.PutByteBuffer(buf) + writers[i] = newStreamingBitrotWriterBuffer(inlineBuffers[i], DefaultBitrotAlgorithm, erasure.ShardSize()) + continue + } + + writers[i] = newBitrotWriter(disk, bucket, minioMetaTmpBucket, tempErasureObj, shardFileSize, DefaultBitrotAlgorithm, erasure.ShardSize()) + } + + toEncode := io.Reader(data) + if data.Size() >= bigFileThreshold { + // We use 2 buffers, so we always have a full buffer of input. + pool := globalBytePoolCap.Load() + bufA := pool.Get() + bufB := pool.Get() + defer pool.Put(bufA) + defer pool.Put(bufB) + ra, err := readahead.NewReaderBuffer(data, [][]byte{bufA[:fi.Erasure.BlockSize], bufB[:fi.Erasure.BlockSize]}) + if err == nil { + toEncode = ra + defer ra.Close() + } + bugLogIf(ctx, err) + } + n, erasureErr := erasure.Encode(ctx, toEncode, writers, buffer, writeQuorum) + closeErrs := closeBitrotWriters(writers) + if erasureErr != nil { + return ObjectInfo{}, toObjectErr(erasureErr, bucket, object) + } + + if closeErr := reduceWriteQuorumErrs(ctx, closeErrs, objectOpIgnoredErrs, writeQuorum); closeErr != nil { + return ObjectInfo{}, toObjectErr(closeErr, bucket, object) + } + + // Should return IncompleteBody{} error when reader has fewer bytes + // than specified in request header. + if n < data.Size() { + return ObjectInfo{}, IncompleteBody{Bucket: bucket, Object: object} + } + + var compIndex []byte + if opts.IndexCB != nil { + compIndex = opts.IndexCB() + } + + modTime := opts.MTime + if opts.MTime.IsZero() { + modTime = UTCNow() + } + + kind, encrypted := crypto.IsEncrypted(userDefined) + actualSize := data.ActualSize() + if actualSize < 0 { + compressed := fi.IsCompressed() + switch { + case compressed: + // ... nothing changes for compressed stream. + // if actualSize is -1 we have no known way to + // determine what is the actualSize. + case encrypted: + decSize, err := sio.DecryptedSize(uint64(n)) + if err == nil { + actualSize = int64(decSize) + } + default: + actualSize = n + } + } + // If ServerSideChecksum is wanted for this object, it takes precedence + // over opts.WantChecksum. + if opts.WantServerSideChecksumType.IsSet() { + serverSideChecksum := r.RawServerSideChecksumResult() + if serverSideChecksum != nil { + fi.Checksum = serverSideChecksum.AppendTo(nil, nil) + if opts.EncryptFn != nil { + fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum) + } + } + } else if fi.Checksum == nil && opts.WantChecksum != nil { + // Trailing headers checksums should now be filled. + fi.Checksum = opts.WantChecksum.AppendTo(nil, nil) + if opts.EncryptFn != nil { + fi.Checksum = opts.EncryptFn("object-checksum", fi.Checksum) + } + } + for i, w := range writers { + if w == nil { + onlineDisks[i] = nil + continue + } + if len(inlineBuffers) > 0 && inlineBuffers[i] != nil { + partsMetadata[i].Data = inlineBuffers[i].Bytes() + } else { + partsMetadata[i].Data = nil + } + // No need to add checksum to part. We already have it on the object. + partsMetadata[i].AddObjectPart(1, "", n, actualSize, modTime, compIndex, nil) + partsMetadata[i].Versioned = opts.Versioned || opts.VersionSuspended + partsMetadata[i].Checksum = fi.Checksum + } + + userDefined["etag"] = r.MD5CurrentHexString() + if opts.PreserveETag != "" { + if !opts.ReplicationRequest { + userDefined["etag"] = opts.PreserveETag + } else if kind != crypto.S3 { + // if we have a replication request + // and SSE-S3 is specified do not preserve + // the incoming etag. + userDefined["etag"] = opts.PreserveETag + } + } + + // Guess content-type from the extension if possible. + if userDefined["content-type"] == "" { + userDefined["content-type"] = mimedb.TypeByExtension(path.Ext(object)) + } + + // if storageClass is standard no need to save it as part of metadata. + if userDefined[xhttp.AmzStorageClass] == storageclass.STANDARD { + delete(userDefined, xhttp.AmzStorageClass) + } + + // Fill all the necessary metadata. + // Update `xl.meta` content on each disks. + for index := range partsMetadata { + partsMetadata[index].Metadata = userDefined + partsMetadata[index].Size = n + partsMetadata[index].ModTime = modTime + if len(inlineBuffers) > 0 { + partsMetadata[index].SetInlineData() + } + if opts.DataMovement { + partsMetadata[index].SetDataMov() + } + } + + if !opts.NoLock { + lk := er.NewNSLock(bucket, object) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + return ObjectInfo{}, err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + } + + // Rename the successfully written temporary object to final location. + onlineDisks, versions, oldDataDir, err := renameData(ctx, onlineDisks, minioMetaTmpBucket, tempObj, partsMetadata, bucket, object, writeQuorum) + if err != nil { + if errors.Is(err, errFileNotFound) { + // An in-quorum errFileNotFound means that client stream + // prematurely closed and we do not find any xl.meta or + // part.1's - in such a scenario we must return as if client + // disconnected. This means that erasure.Encode() CreateFile() + // did not do anything. + return ObjectInfo{}, IncompleteBody{Bucket: bucket, Object: object} + } + return ObjectInfo{}, toObjectErr(err, bucket, object) + } + + if err = er.commitRenameDataDir(ctx, bucket, object, oldDataDir, onlineDisks, writeQuorum); err != nil { + return ObjectInfo{}, toObjectErr(err, bucket, object) + } + + for i := range len(onlineDisks) { + if onlineDisks[i] != nil && onlineDisks[i].IsOnline() { + // Object info is the same in all disks, so we can pick + // the first meta from online disk + fi = partsMetadata[i] + break + } + } + + // For speedtest objects do not attempt to heal them. + if !opts.Speedtest { + // When there is versions disparity we are healing + // the content implicitly for all versions, we can + // avoid triggering another MRF heal for offline drives. + if len(versions) == 0 { + // Whether a disk was initially or becomes offline + // during this upload, send it to the MRF list. + for i := range len(onlineDisks) { + if onlineDisks[i] != nil && onlineDisks[i].IsOnline() { + continue + } + + er.addPartial(bucket, object, fi.VersionID) + break + } + } else { + globalMRFState.addPartialOp(PartialOperation{ + Bucket: bucket, + Object: object, + Queued: time.Now(), + Versions: versions, + SetIndex: er.setIndex, + PoolIndex: er.poolIndex, + }) + } + } + + fi.ReplicationState = opts.PutReplicationState() + + // we are adding a new version to this object under the namespace lock, so this is the latest version. + fi.IsLatest = true + + return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil +} + +func (er erasureObjects) deleteObjectVersion(ctx context.Context, bucket, object string, fi FileInfo, forceDelMarker bool) error { + disks := er.getDisks() + // Assume (N/2 + 1) quorum for Delete() + // this is a theoretical assumption such that + // for delete's we do not need to honor storage + // class for objects that have reduced quorum + // due to storage class - this only needs to be honored + // for Read() requests alone that we already do. + writeQuorum := len(disks)/2 + 1 + + g := errgroup.WithNErrs(len(disks)) + for index := range disks { + g.Go(func() error { + if disks[index] == nil { + return errDiskNotFound + } + return disks[index].DeleteVersion(ctx, bucket, object, fi, forceDelMarker, DeleteOptions{}) + }, index) + } + // return errors if any during deletion + return reduceWriteQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, writeQuorum) +} + +// DeleteObjects deletes objects/versions in bulk, this function will still automatically split objects list +// into smaller bulks if some object names are found to be duplicated in the delete list, splitting +// into smaller bulks will avoid holding twice the write lock of the duplicated object names. +func (er erasureObjects) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) { + if !opts.NoAuditLog { + for _, obj := range objects { + auditObjectErasureSet(ctx, "DeleteObjects", obj.ObjectName, &er) + } + } + + errs := make([]error, len(objects)) + dobjects := make([]DeletedObject, len(objects)) + writeQuorums := make([]int, len(objects)) + + storageDisks := er.getDisks() + + for i := range objects { + // Assume (N/2 + 1) quorums for all objects + // this is a theoretical assumption such that + // for delete's we do not need to honor storage + // class for objects which have reduced quorum + // storage class only needs to be honored for + // Read() requests alone which we already do. + writeQuorums[i] = len(storageDisks)/2 + 1 + } + + versionsMap := make(map[string]FileInfoVersions, len(objects)) + for i := range objects { + // Construct the FileInfo data that needs to be preserved on the disk. + vr := FileInfo{ + Name: objects[i].ObjectName, + VersionID: objects[i].VersionID, + ReplicationState: objects[i].ReplicationState(), + // save the index to set correct error at this index. + Idx: i, + } + vr.SetTierFreeVersionID(mustGetUUID()) + // VersionID is not set means delete is not specific about + // any version, look for if the bucket is versioned or not. + if objects[i].VersionID == "" { + // MinIO extension to bucket version configuration + suspended := opts.VersionSuspended + versioned := opts.Versioned + if opts.PrefixEnabledFn != nil { + versioned = opts.PrefixEnabledFn(objects[i].ObjectName) + } + if versioned || suspended { + // Bucket is versioned and no version was explicitly + // mentioned for deletes, create a delete marker instead. + vr.ModTime = UTCNow() + vr.Deleted = true + // Versioning suspended means that we add a `null` version + // delete marker, if not add a new version for this delete + // marker. + if versioned { + vr.VersionID = mustGetUUID() + } + } + } + // De-dup same object name to collect multiple versions for same object. + v, ok := versionsMap[objects[i].ObjectName] + if ok { + v.Versions = append(v.Versions, vr) + } else { + v = FileInfoVersions{ + Name: vr.Name, + Versions: []FileInfo{vr}, + } + } + if vr.Deleted { + dobjects[i] = DeletedObject{ + DeleteMarker: vr.Deleted, + DeleteMarkerVersionID: vr.VersionID, + DeleteMarkerMTime: DeleteMarkerMTime{vr.ModTime}, + ObjectName: vr.Name, + ReplicationState: vr.ReplicationState, + } + } else { + dobjects[i] = DeletedObject{ + ObjectName: vr.Name, + VersionID: vr.VersionID, + ReplicationState: vr.ReplicationState, + } + } + versionsMap[objects[i].ObjectName] = v + } + + dedupVersions := make([]FileInfoVersions, 0, len(versionsMap)) + for _, fivs := range versionsMap { + // Removal of existing versions and adding a delete marker in the same + // request is supported. At the same time, we cannot allow adding + // two delete markers on top of any object. To avoid this situation, + // we will sort deletions to execute existing deletion first, + // then add only one delete marker if requested + sort.SliceStable(fivs.Versions, func(i, j int) bool { + return !fivs.Versions[i].Deleted + }) + if idx := slices.IndexFunc(fivs.Versions, func(fi FileInfo) bool { + return fi.Deleted + }); idx > -1 { + fivs.Versions = fivs.Versions[:idx+1] + } + dedupVersions = append(dedupVersions, fivs) + } + + // Initialize list of errors. + delObjErrs := make([][]error, len(storageDisks)) + + var wg sync.WaitGroup + // Remove versions in bulk for each disk + for index, disk := range storageDisks { + wg.Add(1) + go func(index int, disk StorageAPI) { + defer wg.Done() + delObjErrs[index] = make([]error, len(objects)) + if disk == nil { + for i := range objects { + delObjErrs[index][i] = errDiskNotFound + } + return + } + errs := disk.DeleteVersions(ctx, bucket, dedupVersions, DeleteOptions{}) + for i, err := range errs { + if err == nil { + continue + } + for _, v := range dedupVersions[i].Versions { + delObjErrs[index][v.Idx] = err + } + } + }(index, disk) + } + wg.Wait() + + // Reduce errors for each object + for objIndex := range objects { + diskErrs := make([]error, len(storageDisks)) + // Iterate over disks to fetch the error + // of deleting of the current object + for i := range delObjErrs { + // delObjErrs[i] is not nil when disks[i] is also not nil + if delObjErrs[i] != nil { + diskErrs[i] = delObjErrs[i][objIndex] + } + } + err := reduceWriteQuorumErrs(ctx, diskErrs, objectOpIgnoredErrs, writeQuorums[objIndex]) + if err == nil { + dobjects[objIndex].found = true + } else if isErrVersionNotFound(err) || isErrObjectNotFound(err) { + if !dobjects[objIndex].DeleteMarker { + err = nil + } + } + if objects[objIndex].VersionID != "" { + errs[objIndex] = toObjectErr(err, bucket, objects[objIndex].ObjectName, objects[objIndex].VersionID) + } else { + errs[objIndex] = toObjectErr(err, bucket, objects[objIndex].ObjectName) + } + } + + // Check failed deletes across multiple objects + for i, dobj := range dobjects { + // This object errored, we should attempt a heal just in case. + if errs[i] != nil && !isErrVersionNotFound(errs[i]) && !isErrObjectNotFound(errs[i]) { + // all other direct versionId references we should + // ensure no dangling file is left over. + er.addPartial(bucket, dobj.ObjectName, dobj.VersionID) + continue + } + + // Check if there is any offline disk and add it to the MRF list + for _, disk := range storageDisks { + if disk != nil && disk.IsOnline() { + // Skip attempted heal on online disks. + continue + } + + // all other direct versionId references we should + // ensure no dangling file is left over. + er.addPartial(bucket, dobj.ObjectName, dobj.VersionID) + break + } + } + + return dobjects, errs +} + +func (er erasureObjects) commitRenameDataDir(ctx context.Context, bucket, object, dataDir string, onlineDisks []StorageAPI, writeQuorum int) error { + if dataDir == "" { + return nil + } + g := errgroup.WithNErrs(len(onlineDisks)) + for index := range onlineDisks { + g.Go(func() error { + if onlineDisks[index] == nil { + return nil + } + return onlineDisks[index].Delete(ctx, bucket, pathJoin(object, dataDir), DeleteOptions{ + Recursive: true, + }) + }, index) + } + + return reduceWriteQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, writeQuorum) +} + +func (er erasureObjects) deletePrefix(ctx context.Context, bucket, prefix string) error { + disks := er.getDisks() + // Assume (N/2 + 1) quorum for Delete() + // this is a theoretical assumption such that + // for delete's we do not need to honor storage + // class for objects that have reduced quorum + // due to storage class - this only needs to be honored + // for Read() requests alone that we already do. + writeQuorum := len(disks)/2 + 1 + + g := errgroup.WithNErrs(len(disks)) + for index := range disks { + g.Go(func() error { + if disks[index] == nil { + return nil + } + return disks[index].Delete(ctx, bucket, prefix, DeleteOptions{ + Recursive: true, + Immediate: true, + }) + }, index) + } + + // return errors if any during deletion + return reduceWriteQuorumErrs(ctx, g.Wait(), objectOpIgnoredErrs, writeQuorum) +} + +// DeleteObject - deletes an object, this call doesn't necessary reply +// any error as it is not necessary for the handler to reply back a +// response to the client request. +func (er erasureObjects) DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) { + if !opts.NoAuditLog { + auditObjectErasureSet(ctx, "DeleteObject", object, &er) + } + + var lc *lifecycle.Lifecycle + var rcfg lock.Retention + var replcfg *replication.Config + if opts.Expiration.Expire { + // Check if the current bucket has a configured lifecycle policy + lc, err = globalLifecycleSys.Get(bucket) + if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) { + return objInfo, err + } + rcfg, err = globalBucketObjectLockSys.Get(bucket) + if err != nil { + return objInfo, err + } + replcfg, err = getReplicationConfig(ctx, bucket) + if err != nil { + return objInfo, err + } + } + + // expiration attempted on a bucket with no lifecycle + // rules shall be rejected. + if lc == nil && opts.Expiration.Expire { + if opts.VersionID != "" { + return objInfo, VersionNotFound{ + Bucket: bucket, + Object: object, + VersionID: opts.VersionID, + } + } + return objInfo, ObjectNotFound{ + Bucket: bucket, + Object: object, + } + } + + if opts.DeletePrefix { + if opts.Expiration.Expire { + // Expire all versions expiration must still verify the state() on disk + // via a getObjectInfo() call as follows, any read quorum issues we + // must not proceed further for safety reasons. attempt a MRF heal + // while we see such quorum errors. + goi, _, gerr := er.getObjectInfoAndQuorum(ctx, bucket, object, opts) + if gerr != nil && goi.Name == "" { + if _, ok := gerr.(InsufficientReadQuorum); ok { + // Add an MRF heal for next time. + er.addPartial(bucket, object, opts.VersionID) + + return objInfo, InsufficientWriteQuorum{} + } + return objInfo, gerr + } + + // Add protection and re-verify the ILM rules for qualification + // based on the latest objectInfo and see if the object still + // qualifies for deletion. + if gerr == nil { + var isErr bool + evt := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, goi) + switch evt.Action { + case lifecycle.DeleteAllVersionsAction, lifecycle.DelMarkerDeleteAllVersionsAction: + // opts.DeletePrefix is used only in the above lifecycle Expiration actions. + default: + // object has been modified since lifecycle action was previously evaluated + isErr = true + } + if isErr { + if goi.VersionID != "" { + return goi, VersionNotFound{ + Bucket: bucket, + Object: object, + VersionID: goi.VersionID, + } + } + return goi, ObjectNotFound{ + Bucket: bucket, + Object: object, + } + } + } + } // Delete marker and any latest that qualifies shall be expired permanently. + + return ObjectInfo{}, toObjectErr(er.deletePrefix(ctx, bucket, object), bucket, object) + } + + storageDisks := er.getDisks() + versionFound := true + objInfo = ObjectInfo{VersionID: opts.VersionID} // version id needed in Delete API response. + goi, _, gerr := er.getObjectInfoAndQuorum(ctx, bucket, object, opts) + tryDel := false + if gerr != nil && goi.Name == "" { + if _, ok := gerr.(InsufficientReadQuorum); ok { + if opts.Versioned || opts.VersionSuspended || countOnlineDisks(storageDisks) < len(storageDisks)/2+1 { + // Add an MRF heal for next time. + er.addPartial(bucket, object, opts.VersionID) + return objInfo, InsufficientWriteQuorum{} + } + tryDel = true // only for unversioned objects if there is write quorum + } + // For delete marker replication, versionID being replicated will not exist on disk + if opts.DeleteMarker { + versionFound = false + } else if !tryDel { + return objInfo, gerr + } + } + + if opts.EvalMetadataFn != nil { + dsc, err := opts.EvalMetadataFn(&goi, gerr) + if err != nil { + return ObjectInfo{}, err + } + if dsc.ReplicateAny() { + opts.SetDeleteReplicationState(dsc, opts.VersionID) + goi.replicationDecision = opts.DeleteReplication.ReplicateDecisionStr + } + } + + if opts.EvalRetentionBypassFn != nil { + if err := opts.EvalRetentionBypassFn(goi, gerr); err != nil { + return ObjectInfo{}, err + } + } + + if opts.Expiration.Expire { + if gerr == nil { + evt := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, goi) + var isErr bool + switch evt.Action { + case lifecycle.NoneAction: + isErr = true + case lifecycle.TransitionAction, lifecycle.TransitionVersionAction: + isErr = true + } + if isErr { + if goi.VersionID != "" { + return goi, VersionNotFound{ + Bucket: bucket, + Object: object, + VersionID: goi.VersionID, + } + } + return goi, ObjectNotFound{ + Bucket: bucket, + Object: object, + } + } + } + } + + // Determine whether to mark object deleted for replication + markDelete := goi.VersionID != "" + + // Default deleteMarker to true if object is under versioning + deleteMarker := opts.Versioned + + if opts.VersionID != "" { + // case where replica version needs to be deleted on target cluster + if versionFound && opts.DeleteMarkerReplicationStatus() == replication.Replica { + markDelete = false + } + if opts.VersionPurgeStatus().Empty() && opts.DeleteMarkerReplicationStatus().Empty() { + markDelete = false + } + if opts.VersionPurgeStatus() == replication.VersionPurgeComplete { + markDelete = false + } + // now, since VersionPurgeStatus() is already set, we can let the + // lower layers decide this. This fixes a regression that was introduced + // in PR #14555 where !VersionPurgeStatus.Empty() is automatically + // considered as Delete marker true to avoid listing such objects by + // regular ListObjects() calls. However for delete replication this + // ends up being a problem because "upon" a successful delete this + // ends up creating a new delete marker that is spurious and unnecessary. + // + // Regression introduced by #14555 was reintroduced in #15564 + if versionFound { + if !goi.VersionPurgeStatus.Empty() { + deleteMarker = false + } else if !goi.DeleteMarker { // implies a versioned delete of object + deleteMarker = false + } + } + } + + modTime := opts.MTime + if opts.MTime.IsZero() { + modTime = UTCNow() + } + fvID := mustGetUUID() + + defer func() { + // attempt a heal before returning if there are offline disks + // for both del marker and permanent delete situations. + for _, disk := range storageDisks { + if disk != nil && disk.IsOnline() { + continue + } + er.addPartial(bucket, object, opts.VersionID) + break + } + }() + + if markDelete && (opts.Versioned || opts.VersionSuspended) { + if !deleteMarker { + // versioning suspended means we add `null` version as + // delete marker, if its not decided already. + deleteMarker = opts.VersionSuspended && opts.VersionID == "" + } + fi := FileInfo{ + Name: object, + Deleted: deleteMarker, + MarkDeleted: markDelete, + ModTime: modTime, + ReplicationState: opts.DeleteReplication, + TransitionStatus: opts.Transition.Status, + ExpireRestored: opts.Transition.ExpireRestored, + } + fi.SetTierFreeVersionID(fvID) + if opts.SkipFreeVersion { + fi.SetSkipTierFreeVersion() + } + if opts.VersionID != "" { + fi.VersionID = opts.VersionID + } else if opts.Versioned { + fi.VersionID = mustGetUUID() + } + // versioning suspended means we add `null` version as + // delete marker. Add delete marker, since we don't have + // any version specified explicitly. Or if a particular + // version id needs to be replicated. + if err = er.deleteObjectVersion(ctx, bucket, object, fi, opts.DeleteMarker); err != nil { + return objInfo, toObjectErr(err, bucket, object) + } + oi := fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended) + oi.replicationDecision = goi.replicationDecision + return oi, nil + } + + // Delete the object version on all disks. + dfi := FileInfo{ + Name: object, + VersionID: opts.VersionID, + MarkDeleted: markDelete, + Deleted: deleteMarker, + ModTime: modTime, + ReplicationState: opts.DeleteReplication, + TransitionStatus: opts.Transition.Status, + ExpireRestored: opts.Transition.ExpireRestored, + } + dfi.SetTierFreeVersionID(fvID) + if opts.SkipFreeVersion { + dfi.SetSkipTierFreeVersion() + } + if err = er.deleteObjectVersion(ctx, bucket, object, dfi, opts.DeleteMarker); err != nil { + return objInfo, toObjectErr(err, bucket, object) + } + + return dfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil +} + +// Send the successful but partial upload/delete, however ignore +// if the channel is blocked by other items. +func (er erasureObjects) addPartial(bucket, object, versionID string) { + globalMRFState.addPartialOp(PartialOperation{ + Bucket: bucket, + Object: object, + VersionID: versionID, + Queued: time.Now(), + }) +} + +func (er erasureObjects) PutObjectMetadata(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) { + if !opts.NoLock { + // Lock the object before updating metadata. + lk := er.NewNSLock(bucket, object) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + return ObjectInfo{}, err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + } + + disks := er.getDisks() + + var metaArr []FileInfo + var errs []error + + // Read metadata associated with the object from all disks. + if opts.VersionID != "" { + metaArr, errs = readAllFileInfo(ctx, disks, "", bucket, object, opts.VersionID, false, false) + } else { + metaArr, errs = readAllXL(ctx, disks, bucket, object, false, false) + } + + readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount) + if err != nil { + if shouldCheckForDangling(err, errs, bucket) { + _, derr := er.deleteIfDangling(context.Background(), bucket, object, metaArr, errs, nil, opts) + if derr == nil { + if opts.VersionID != "" { + err = errFileVersionNotFound + } else { + err = errFileNotFound + } + } + } + return ObjectInfo{}, toObjectErr(err, bucket, object) + } + + // List all online disks. + onlineDisks, modTime, etag := listOnlineDisks(disks, metaArr, errs, readQuorum) + + // Pick latest valid metadata. + fi, err := pickValidFileInfo(ctx, metaArr, modTime, etag, readQuorum) + if err != nil { + return ObjectInfo{}, toObjectErr(err, bucket, object) + } + + if fi.Deleted { + return ObjectInfo{}, toObjectErr(errMethodNotAllowed, bucket, object) + } + + filterOnlineDisksInplace(fi, metaArr, onlineDisks) + + // if version-id is not specified retention is supposed to be set on the latest object. + if opts.VersionID == "" { + opts.VersionID = fi.VersionID + } + + objInfo := fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended) + if opts.EvalMetadataFn != nil { + if _, err := opts.EvalMetadataFn(&objInfo, err); err != nil { + return ObjectInfo{}, err + } + } + maps.Copy(fi.Metadata, objInfo.UserDefined) + fi.ModTime = opts.MTime + fi.VersionID = opts.VersionID + + if err = er.updateObjectMeta(ctx, bucket, object, fi, onlineDisks); err != nil { + return ObjectInfo{}, toObjectErr(err, bucket, object) + } + + return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil +} + +// PutObjectTags - replace or add tags to an existing object +func (er erasureObjects) PutObjectTags(ctx context.Context, bucket, object string, tags string, opts ObjectOptions) (ObjectInfo, error) { + if !opts.NoLock { + // Lock the object before updating tags. + lk := er.NewNSLock(bucket, object) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + return ObjectInfo{}, err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + } + + disks := er.getDisks() + + var metaArr []FileInfo + var errs []error + + // Read metadata associated with the object from all disks. + if opts.VersionID != "" { + metaArr, errs = readAllFileInfo(ctx, disks, "", bucket, object, opts.VersionID, false, false) + } else { + metaArr, errs = readAllXL(ctx, disks, bucket, object, false, false) + } + + readQuorum, _, err := objectQuorumFromMeta(ctx, metaArr, errs, er.defaultParityCount) + if err != nil { + if shouldCheckForDangling(err, errs, bucket) { + _, derr := er.deleteIfDangling(context.Background(), bucket, object, metaArr, errs, nil, opts) + if derr == nil { + if opts.VersionID != "" { + err = errFileVersionNotFound + } else { + err = errFileNotFound + } + } + } + return ObjectInfo{}, toObjectErr(err, bucket, object) + } + + // List all online disks. + onlineDisks, modTime, etag := listOnlineDisks(disks, metaArr, errs, readQuorum) + + // Pick latest valid metadata. + fi, err := pickValidFileInfo(ctx, metaArr, modTime, etag, readQuorum) + if err != nil { + return ObjectInfo{}, toObjectErr(err, bucket, object) + } + if fi.Deleted { + if opts.VersionID == "" { + return ObjectInfo{}, toObjectErr(errFileNotFound, bucket, object) + } + return ObjectInfo{}, toObjectErr(errMethodNotAllowed, bucket, object) + } + + filterOnlineDisksInplace(fi, metaArr, onlineDisks) + + fi.Metadata[xhttp.AmzObjectTagging] = tags + fi.ReplicationState = opts.PutReplicationState() + maps.Copy(fi.Metadata, opts.UserDefined) + + if err = er.updateObjectMeta(ctx, bucket, object, fi, onlineDisks); err != nil { + return ObjectInfo{}, toObjectErr(err, bucket, object) + } + + return fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended), nil +} + +func (er erasureObjects) updateObjectMetaWithOpts(ctx context.Context, bucket, object string, fi FileInfo, onlineDisks []StorageAPI, opts UpdateMetadataOpts) error { + if len(fi.Metadata) == 0 { + return nil + } + + g := errgroup.WithNErrs(len(onlineDisks)) + + // Start writing `xl.meta` to all disks in parallel. + for index := range onlineDisks { + g.Go(func() error { + if onlineDisks[index] == nil { + return errDiskNotFound + } + return onlineDisks[index].UpdateMetadata(ctx, bucket, object, fi, opts) + }, index) + } + + // Wait for all the routines. + mErrs := g.Wait() + + return reduceWriteQuorumErrs(ctx, mErrs, objectOpIgnoredErrs, fi.WriteQuorum(er.defaultWQuorum())) +} + +// updateObjectMeta will update the metadata of a file. +func (er erasureObjects) updateObjectMeta(ctx context.Context, bucket, object string, fi FileInfo, onlineDisks []StorageAPI) error { + return er.updateObjectMetaWithOpts(ctx, bucket, object, fi, onlineDisks, UpdateMetadataOpts{}) +} + +// DeleteObjectTags - delete object tags from an existing object +func (er erasureObjects) DeleteObjectTags(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) { + return er.PutObjectTags(ctx, bucket, object, "", opts) +} + +// GetObjectTags - get object tags from an existing object +func (er erasureObjects) GetObjectTags(ctx context.Context, bucket, object string, opts ObjectOptions) (*tags.Tags, error) { + // GetObjectInfo will return tag value as well + oi, err := er.GetObjectInfo(ctx, bucket, object, opts) + if err != nil { + return nil, err + } + + return tags.ParseObjectTags(oi.UserTags) +} + +// TransitionObject - transition object content to target tier. +func (er erasureObjects) TransitionObject(ctx context.Context, bucket, object string, opts ObjectOptions) error { + tgtClient, err := globalTierConfigMgr.getDriver(ctx, opts.Transition.Tier) + if err != nil { + return err + } + + if !opts.NoLock { + // Acquire write lock before starting to transition the object. + lk := er.NewNSLock(bucket, object) + lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout) + if err != nil { + return err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + } + + fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, bucket, object, opts, true) + if err != nil { + return toObjectErr(err, bucket, object) + } + if fi.Deleted { + if opts.VersionID == "" { + return toObjectErr(errFileNotFound, bucket, object) + } + // Make sure to return object info to provide extra information. + return toObjectErr(errMethodNotAllowed, bucket, object) + } + // verify that the object queued for transition is identical to that on disk. + if !opts.MTime.Equal(fi.ModTime) || !strings.EqualFold(opts.Transition.ETag, extractETag(fi.Metadata)) { + return toObjectErr(errFileNotFound, bucket, object) + } + // if object already transitioned, return + if fi.TransitionStatus == lifecycle.TransitionComplete { + return nil + } + + if fi.XLV1 { + if _, err = er.HealObject(ctx, bucket, object, "", madmin.HealOpts{NoLock: true}); err != nil { + return err + } + // Fetch FileInfo again. HealObject migrates object the latest + // format. Among other things this changes fi.DataDir and + // possibly fi.Data (if data is inlined). + fi, metaArr, onlineDisks, err = er.getObjectFileInfo(ctx, bucket, object, opts, true) + if err != nil { + return toObjectErr(err, bucket, object) + } + } + traceFn := globalLifecycleSys.trace(fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended)) + + destObj, err := genTransitionObjName(bucket) + if err != nil { + traceFn(ILMTransition, nil, err) + return err + } + + pr, pw := xioutil.WaitPipe() + go func() { + err := er.getObjectWithFileInfo(ctx, bucket, object, 0, fi.Size, pw, fi, metaArr, onlineDisks) + pw.CloseWithError(err) + }() + + var rv remoteVersionID + rv, err = tgtClient.PutWithMeta(ctx, destObj, pr, fi.Size, map[string]string{ + "name": object, // preserve the original name of the object on the remote tier object metadata. + // this is just for future reverse lookup() purposes (applies only for new objects) + // does not apply retro-actively on already transitioned objects. + }) + pr.CloseWithError(err) + if err != nil { + traceFn(ILMTransition, nil, err) + return err + } + fi.TransitionStatus = lifecycle.TransitionComplete + fi.TransitionedObjName = destObj + fi.TransitionTier = opts.Transition.Tier + fi.TransitionVersionID = string(rv) + eventName := event.ObjectTransitionComplete + + storageDisks := er.getDisks() + + if err = er.deleteObjectVersion(ctx, bucket, object, fi, false); err != nil { + eventName = event.ObjectTransitionFailed + } + + for _, disk := range storageDisks { + if disk != nil && disk.IsOnline() { + continue + } + er.addPartial(bucket, object, opts.VersionID) + break + } + + objInfo := fi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended) + sendEvent(eventArgs{ + EventName: eventName, + BucketName: bucket, + Object: objInfo, + UserAgent: "Internal: [ILM-Transition]", + Host: globalLocalNodeName, + }) + tags := opts.LifecycleAuditEvent.Tags() + auditLogLifecycle(ctx, objInfo, ILMTransition, tags, traceFn) + return err +} + +// RestoreTransitionedObject - restore transitioned object content locally on this cluster. +// This is similar to PostObjectRestore from AWS GLACIER +// storage class. When PostObjectRestore API is called, a temporary copy of the object +// is restored locally to the bucket on source cluster until the restore expiry date. +// The copy that was transitioned continues to reside in the transitioned tier. +func (er erasureObjects) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error { + return er.restoreTransitionedObject(ctx, bucket, object, opts) +} + +// update restore status header in the metadata +func (er erasureObjects) updateRestoreMetadata(ctx context.Context, bucket, object string, objInfo ObjectInfo, opts ObjectOptions) error { + oi := objInfo.Clone() + oi.metadataOnly = true // Perform only metadata updates. + + // allow retry in the case of failure to restore + delete(oi.UserDefined, xhttp.AmzRestore) + + if _, err := er.CopyObject(ctx, bucket, object, bucket, object, oi, ObjectOptions{ + VersionID: oi.VersionID, + }, ObjectOptions{ + VersionID: oi.VersionID, + }); err != nil { + storageLogIf(ctx, fmt.Errorf("Unable to update transition restore metadata for %s/%s(%s): %s", bucket, object, oi.VersionID, err)) + return err + } + return nil +} + +// restoreTransitionedObject for multipart object chunks the file stream from remote tier into the same number of parts +// as in the xl.meta for this version and rehydrates the part.n into the fi.DataDir for this version as in the xl.meta +func (er erasureObjects) restoreTransitionedObject(ctx context.Context, bucket string, object string, opts ObjectOptions) error { + setRestoreHeaderFn := func(oi ObjectInfo, rerr error) error { + if rerr == nil { + return nil // nothing to do; restore object was successful + } + er.updateRestoreMetadata(ctx, bucket, object, oi, opts) + return rerr + } + var oi ObjectInfo + // get the file info on disk for transitioned object + actualfi, _, _, err := er.getObjectFileInfo(ctx, bucket, object, opts, false) + if err != nil { + return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object)) + } + + oi = actualfi.ToObjectInfo(bucket, object, opts.Versioned || opts.VersionSuspended) + ropts := putRestoreOpts(bucket, object, opts.Transition.RestoreRequest, oi) + if len(oi.Parts) == 1 { + var rs *HTTPRangeSpec + gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, oi, opts) + if err != nil { + return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object)) + } + defer gr.Close() + hashReader, err := hash.NewReader(ctx, gr, gr.ObjInfo.Size, "", "", gr.ObjInfo.Size) + if err != nil { + return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object)) + } + pReader := NewPutObjReader(hashReader) + _, err = er.PutObject(ctx, bucket, object, pReader, ropts) + return setRestoreHeaderFn(oi, toObjectErr(err, bucket, object)) + } + + res, err := er.NewMultipartUpload(ctx, bucket, object, ropts) + if err != nil { + return setRestoreHeaderFn(oi, err) + } + + var uploadedParts []CompletePart + var rs *HTTPRangeSpec + // get reader from the warm backend - note that even in the case of encrypted objects, this stream is still encrypted. + gr, err := getTransitionedObjectReader(ctx, bucket, object, rs, http.Header{}, oi, opts) + if err != nil { + return setRestoreHeaderFn(oi, err) + } + defer gr.Close() + + // rehydrate the parts back on disk as per the original xl.meta prior to transition + for _, partInfo := range oi.Parts { + hr, err := hash.NewReader(ctx, io.LimitReader(gr, partInfo.Size), partInfo.Size, "", "", partInfo.Size) + if err != nil { + return setRestoreHeaderFn(oi, err) + } + pInfo, err := er.PutObjectPart(ctx, bucket, object, res.UploadID, partInfo.Number, NewPutObjReader(hr), ObjectOptions{}) + if err != nil { + return setRestoreHeaderFn(oi, err) + } + if pInfo.Size != partInfo.Size { + return setRestoreHeaderFn(oi, InvalidObjectState{Bucket: bucket, Object: object}) + } + uploadedParts = append(uploadedParts, CompletePart{ + PartNumber: pInfo.PartNumber, + ETag: pInfo.ETag, + }) + } + _, err = er.CompleteMultipartUpload(ctx, bucket, object, res.UploadID, uploadedParts, ObjectOptions{ + MTime: oi.ModTime, + }) + return setRestoreHeaderFn(oi, err) +} + +// DecomTieredObject - moves tiered object to another pool during decommissioning. +func (er erasureObjects) DecomTieredObject(ctx context.Context, bucket, object string, fi FileInfo, opts ObjectOptions) error { + if opts.UserDefined == nil { + opts.UserDefined = make(map[string]string) + } + // overlay Erasure info for this set of disks + storageDisks := er.getDisks() + // Get parity and data drive count based on storage class metadata + parityDrives := globalStorageClass.GetParityForSC(opts.UserDefined[xhttp.AmzStorageClass]) + if parityDrives < 0 { + parityDrives = er.defaultParityCount + } + dataDrives := len(storageDisks) - parityDrives + + // we now know the number of blocks this object needs for data and parity. + // writeQuorum is dataBlocks + 1 + writeQuorum := dataDrives + if dataDrives == parityDrives { + writeQuorum++ + } + + // Initialize parts metadata + partsMetadata := make([]FileInfo, len(storageDisks)) + + fi2 := newFileInfo(pathJoin(bucket, object), dataDrives, parityDrives) + fi.Erasure = fi2.Erasure + // Initialize erasure metadata. + for index := range partsMetadata { + partsMetadata[index] = fi + partsMetadata[index].Erasure.Index = index + 1 + } + + // Order disks according to erasure distribution + var onlineDisks []StorageAPI + onlineDisks, partsMetadata = shuffleDisksAndPartsMetadata(storageDisks, partsMetadata, fi) + + if _, err := writeUniqueFileInfo(ctx, onlineDisks, "", bucket, object, partsMetadata, writeQuorum); err != nil { + return toObjectErr(err, bucket, object) + } + + return nil +} diff --git a/cmd/erasure-object_test.go b/cmd/erasure-object_test.go new file mode 100644 index 00000000..03f452f5 --- /dev/null +++ b/cmd/erasure-object_test.go @@ -0,0 +1,1283 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/md5" + crand "crypto/rand" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "strconv" + "testing" + + "github.com/dustin/go-humanize" + "github.com/minio/minio/internal/config/storageclass" +) + +func TestRepeatPutObjectPart(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + var objLayer ObjectLayer + var disks []string + var err error + var opts ObjectOptions + + objLayer, disks, err = prepareErasure16(ctx) + if err != nil { + t.Fatal(err) + } + + // cleaning up of temporary test directories + defer objLayer.Shutdown(t.Context()) + defer removeRoots(disks) + + err = objLayer.MakeBucket(ctx, "bucket1", MakeBucketOptions{}) + if err != nil { + t.Fatal(err) + } + + res, err := objLayer.NewMultipartUpload(ctx, "bucket1", "mpartObj1", opts) + if err != nil { + t.Fatal(err) + } + fiveMBBytes := bytes.Repeat([]byte("a"), 5*humanize.MiByte) + md5Hex := getMD5Hash(fiveMBBytes) + _, err = objLayer.PutObjectPart(ctx, "bucket1", "mpartObj1", res.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(fiveMBBytes), 5*humanize.MiByte, md5Hex, ""), opts) + if err != nil { + t.Fatal(err) + } + // PutObjectPart should succeed even if part already exists. ref: https://github.com/minio/minio/issues/1930 + _, err = objLayer.PutObjectPart(ctx, "bucket1", "mpartObj1", res.UploadID, 1, mustGetPutObjReader(t, bytes.NewReader(fiveMBBytes), 5*humanize.MiByte, md5Hex, ""), opts) + if err != nil { + t.Fatal(err) + } +} + +func TestErasureDeleteObjectBasic(t *testing.T) { + testCases := []struct { + bucket string + object string + expectedErr error + }{ + {".test", "dir/obj", BucketNameInvalid{Bucket: ".test"}}, + {"----", "dir/obj", BucketNameInvalid{Bucket: "----"}}, + {"bucket", "", ObjectNameInvalid{Bucket: "bucket", Object: ""}}, + {"bucket", "doesnotexist", ObjectNotFound{Bucket: "bucket", Object: "doesnotexist"}}, + {"bucket", "dir/doesnotexist", ObjectNotFound{Bucket: "bucket", Object: "dir/doesnotexist"}}, + {"bucket", "dir", ObjectNotFound{Bucket: "bucket", Object: "dir"}}, + {"bucket", "dir/", ObjectNotFound{Bucket: "bucket", Object: "dir/"}}, + {"bucket", "dir/obj", nil}, + } + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // Create an instance of xl backend + xl, fsDirs, err := prepareErasure16(ctx) + if err != nil { + t.Fatal(err) + } + defer xl.Shutdown(t.Context()) + + err = xl.MakeBucket(ctx, "bucket", MakeBucketOptions{}) + if err != nil { + t.Fatal(err) + } + + // Create object "dir/obj" under bucket "bucket" for Test 7 to pass + _, err = xl.PutObject(ctx, "bucket", "dir/obj", mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{}) + if err != nil { + t.Fatalf("Erasure Object upload failed: %s", err) + } + for _, test := range testCases { + t.Run("", func(t *testing.T) { + _, err := xl.GetObjectInfo(ctx, "bucket", "dir/obj", ObjectOptions{}) + if err != nil { + t.Fatal("dir/obj not found before last test") + } + _, actualErr := xl.DeleteObject(ctx, test.bucket, test.object, ObjectOptions{}) + if test.expectedErr != nil && actualErr != test.expectedErr { + t.Errorf("Expected to fail with %s, but failed with %s", test.expectedErr, actualErr) + } + if test.expectedErr == nil && actualErr != nil { + t.Errorf("Expected to pass, but failed with %s", actualErr) + } + }) + } + // Cleanup backend directories + removeRoots(fsDirs) +} + +func TestDeleteObjectsVersionedTwoPools(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + obj, fsDirs, err := prepareErasurePools() + if err != nil { + t.Fatal("Unable to initialize 'Erasure' object layer.", err) + } + // Remove all dirs. + for _, dir := range fsDirs { + defer os.RemoveAll(dir) + } + + bucketName := "bucket" + objectName := "myobject" + err = obj.MakeBucket(ctx, bucketName, MakeBucketOptions{ + VersioningEnabled: true, + }) + if err != nil { + t.Fatal(err) + } + + z, ok := obj.(*erasureServerPools) + if !ok { + t.Fatal("unexpected object layer type") + } + + versions := make([]string, 2) + for i := range z.serverPools { + objInfo, err := z.serverPools[i].PutObject(ctx, bucketName, objectName, + mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{ + Versioned: true, + }) + if err != nil { + t.Fatalf("Erasure Object upload failed: %s", err) + } + versions[i] = objInfo.VersionID + } + + // Remove and check the version in the second pool, then + // remove and check the version in the first pool + for testIdx, vid := range []string{versions[1], versions[0]} { + names := []ObjectToDelete{ + { + ObjectV: ObjectV{ + ObjectName: objectName, + VersionID: vid, + }, + }, + } + _, delErrs := obj.DeleteObjects(ctx, bucketName, names, ObjectOptions{ + Versioned: true, + }) + for i := range delErrs { + if delErrs[i] != nil { + t.Errorf("Test %d: Failed to remove object `%v` with the error: `%v`", testIdx, names[i], delErrs[i]) + } + _, statErr := obj.GetObjectInfo(ctx, bucketName, objectName, ObjectOptions{ + VersionID: names[i].VersionID, + }) + switch statErr.(type) { + case VersionNotFound: + default: + t.Errorf("Test %d: Object %s is not removed", testIdx, objectName) + } + } + } +} + +func TestDeleteObjectsVersioned(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + obj, fsDirs, err := prepareErasure(ctx, 16) + if err != nil { + t.Fatal("Unable to initialize 'Erasure' object layer.", err) + } + // Remove all dirs. + for _, dir := range fsDirs { + defer os.RemoveAll(dir) + } + + type testCaseType struct { + bucket string + object string + } + + bucketName := "bucket" + testCases := []testCaseType{ + {bucketName, "dir/obj1"}, + {bucketName, "dir/obj1"}, + } + + err = obj.MakeBucket(ctx, bucketName, MakeBucketOptions{ + VersioningEnabled: true, + }) + if err != nil { + t.Fatal(err) + } + + names := make([]ObjectToDelete, len(testCases)) + for i, testCase := range testCases { + objInfo, err := obj.PutObject(ctx, testCase.bucket, testCase.object, + mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{ + Versioned: true, + }) + if err != nil { + t.Fatalf("Erasure Object upload failed: %s", err) + } + names[i] = ObjectToDelete{ + ObjectV: ObjectV{ + ObjectName: objInfo.Name, + VersionID: objInfo.VersionID, + }, + } + } + names = append(names, ObjectToDelete{ + ObjectV: ObjectV{ + ObjectName: "dir/obj1", + VersionID: mustGetUUID(), // add a non-existent UUID. + }, + }) + + _, delErrs := obj.DeleteObjects(ctx, bucketName, names, ObjectOptions{ + Versioned: true, + }) + for i := range delErrs { + if delErrs[i] != nil { + t.Errorf("Failed to remove object `%v` with the error: `%v`", names[i], delErrs[i]) + } + } + + for i, test := range testCases { + _, statErr := obj.GetObjectInfo(ctx, test.bucket, test.object, ObjectOptions{ + VersionID: names[i].VersionID, + }) + switch statErr.(type) { + case VersionNotFound: + default: + t.Fatalf("Object %s is not removed", test.bucket+SlashSeparator+test.object) + } + } + + if _, err = os.ReadFile(pathJoin(fsDirs[0], bucketName, "dir/obj1", "xl.meta")); err == nil { + t.Fatalf("xl.meta still present after removal") + } +} + +func TestErasureDeleteObjectsErasureSet(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + obj, fsDirs, err := prepareErasureSets32(ctx) + if err != nil { + t.Fatal("Unable to initialize 'Erasure' object layer.", err) + } + + setObjectLayer(obj) + initConfigSubsystem(ctx, obj) + + // Remove all dirs. + for _, dir := range fsDirs { + defer os.RemoveAll(dir) + } + + type testCaseType struct { + bucket string + object string + } + + bucketName := "bucket" + testCases := []testCaseType{ + {bucketName, "dir/obj1"}, + {bucketName, "dir/obj2"}, + {bucketName, "obj3"}, + {bucketName, "obj_4"}, + } + + if err = obj.MakeBucket(ctx, bucketName, MakeBucketOptions{}); err != nil { + t.Fatal(err) + } + + for _, testCase := range testCases { + _, err = obj.PutObject(ctx, testCase.bucket, testCase.object, + mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), ObjectOptions{}) + if err != nil { + t.Fatalf("Erasure Object upload failed: %s", err) + } + } + + toObjectNames := func(testCases []testCaseType) []ObjectToDelete { + names := make([]ObjectToDelete, len(testCases)) + for i := range testCases { + names[i] = ObjectToDelete{ + ObjectV: ObjectV{ + ObjectName: testCases[i].object, + }, + } + } + return names + } + + objectNames := toObjectNames(testCases) + _, delErrs := obj.DeleteObjects(ctx, bucketName, objectNames, ObjectOptions{}) + + for i := range delErrs { + if delErrs[i] != nil { + t.Errorf("Failed to remove object `%v` with the error: `%v`", objectNames[i], delErrs[i]) + } + } + + for _, test := range testCases { + _, statErr := obj.GetObjectInfo(ctx, test.bucket, test.object, ObjectOptions{}) + switch statErr.(type) { + case ObjectNotFound: + default: + t.Fatalf("Object %s is not removed", test.bucket+SlashSeparator+test.object) + } + } +} + +func TestErasureDeleteObjectDiskNotFound(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // Create an instance of xl backend. + obj, fsDirs, err := prepareErasure16(ctx) + if err != nil { + t.Fatal(err) + } + // Cleanup backend directories + defer obj.Shutdown(t.Context()) + defer removeRoots(fsDirs) + + z := obj.(*erasureServerPools) + xl := z.serverPools[0].sets[0] + + // Create "bucket" + err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{}) + if err != nil { + t.Fatal(err) + } + + bucket := "bucket" + object := "object" + opts := ObjectOptions{} + // Create object "obj" under bucket "bucket". + _, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts) + if err != nil { + t.Fatal(err) + } + + erasureDisks := xl.getDisks() + z.serverPools[0].erasureDisksMu.Lock() + xl.getDisks = func() []StorageAPI { + for i := range erasureDisks[:6] { + erasureDisks[i] = newNaughtyDisk(erasureDisks[i], nil, errFaultyDisk) + } + return erasureDisks + } + + z.serverPools[0].erasureDisksMu.Unlock() + _, err = obj.DeleteObject(ctx, bucket, object, ObjectOptions{}) + if !errors.Is(err, errErasureWriteQuorum) { + t.Fatal(err) + } + + // Create "obj" under "bucket". + _, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts) + if err != nil { + t.Fatal(err) + } + + // Remove one more disk to 'lose' quorum, by taking 2 more drives offline. + erasureDisks = xl.getDisks() + z.serverPools[0].erasureDisksMu.Lock() + xl.getDisks = func() []StorageAPI { + erasureDisks[7] = nil + erasureDisks[8] = nil + return erasureDisks + } + + z.serverPools[0].erasureDisksMu.Unlock() + _, err = obj.DeleteObject(ctx, bucket, object, ObjectOptions{}) + // since majority of disks are not available, metaquorum is not achieved and hence errErasureWriteQuorum error + if !errors.Is(err, errErasureWriteQuorum) { + t.Errorf("Expected deleteObject to fail with %v, but failed with %v", toObjectErr(errErasureWriteQuorum, bucket, object), err) + } +} + +func TestErasureDeleteObjectDiskNotFoundErasure4(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // Create an instance of xl backend. + obj, fsDirs, err := prepareErasure16(ctx) + if err != nil { + t.Fatal(err) + } + // Cleanup backend directories + defer obj.Shutdown(t.Context()) + defer removeRoots(fsDirs) + + z := obj.(*erasureServerPools) + xl := z.serverPools[0].sets[0] + + // Create "bucket" + err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{}) + if err != nil { + t.Fatal(err) + } + + bucket := "bucket" + object := "object" + opts := ObjectOptions{} + // Create object "obj" under bucket "bucket". + _, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts) + if err != nil { + t.Fatal(err) + } + // Upload a good object + _, err = obj.DeleteObject(ctx, bucket, object, ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + + // Create "obj" under "bucket". + _, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts) + if err != nil { + t.Fatal(err) + } + + // Remove disks to 'lose' quorum for object, by setting 5 to nil. + erasureDisks := xl.getDisks() + z.serverPools[0].erasureDisksMu.Lock() + xl.getDisks = func() []StorageAPI { + for i := range erasureDisks[:5] { + erasureDisks[i] = newNaughtyDisk(erasureDisks[i], nil, errFaultyDisk) + } + return erasureDisks + } + + z.serverPools[0].erasureDisksMu.Unlock() + _, err = obj.DeleteObject(ctx, bucket, object, ObjectOptions{}) + // since majority of disks are not available, metaquorum is not achieved and hence errErasureWriteQuorum error + if !errors.Is(err, errErasureWriteQuorum) { + t.Errorf("Expected deleteObject to fail with %v, but failed with %v", toObjectErr(errErasureWriteQuorum, bucket, object), err) + } +} + +func TestErasureDeleteObjectDiskNotFoundErr(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // Create an instance of xl backend. + obj, fsDirs, err := prepareErasure16(ctx) + if err != nil { + t.Fatal(err) + } + // Cleanup backend directories + defer obj.Shutdown(t.Context()) + defer removeRoots(fsDirs) + + z := obj.(*erasureServerPools) + xl := z.serverPools[0].sets[0] + + // Create "bucket" + err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{}) + if err != nil { + t.Fatal(err) + } + + bucket := "bucket" + object := "object" + opts := ObjectOptions{} + // Create object "obj" under bucket "bucket". + _, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts) + if err != nil { + t.Fatal(err) + } + // for a 16 disk setup, EC is 4, but will be upgraded up to 8. + // Remove 4 disks. + erasureDisks := xl.getDisks() + z.serverPools[0].erasureDisksMu.Lock() + xl.getDisks = func() []StorageAPI { + for i := range erasureDisks[:4] { + erasureDisks[i] = newNaughtyDisk(erasureDisks[i], nil, errFaultyDisk) + } + return erasureDisks + } + + z.serverPools[0].erasureDisksMu.Unlock() + _, err = obj.DeleteObject(ctx, bucket, object, ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + + // Create "obj" under "bucket". + _, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts) + if err != nil { + t.Fatal(err) + } + + // Object was uploaded with 4 known bad drives, so we should still be able to lose 3 drives and still write to the object. + erasureDisks = xl.getDisks() + z.serverPools[0].erasureDisksMu.Lock() + xl.getDisks = func() []StorageAPI { + erasureDisks[7] = nil + erasureDisks[8] = nil + erasureDisks[9] = nil + return erasureDisks + } + + z.serverPools[0].erasureDisksMu.Unlock() + _, err = obj.DeleteObject(ctx, bucket, object, ObjectOptions{}) + // since majority of disks are available, metaquorum achieved. + if err != nil { + t.Errorf("Expected deleteObject to not fail, but failed with %v", err) + } +} + +func TestGetObjectNoQuorum(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // Create an instance of xl backend. + obj, fsDirs, err := prepareErasure16(ctx) + if err != nil { + t.Fatal(err) + } + // Cleanup backend directories. + defer obj.Shutdown(t.Context()) + defer removeRoots(fsDirs) + + z := obj.(*erasureServerPools) + xl := z.serverPools[0].sets[0] + + // Create "bucket" + err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{}) + if err != nil { + t.Fatal(err) + } + + bucket := "bucket" + object := "object" + opts := ObjectOptions{} + buf := make([]byte, smallFileThreshold*16) + if _, err = io.ReadFull(crand.Reader, buf); err != nil { + t.Fatal(err) + } + + // Test use case 1: All disks are online, xl.meta are present, but data are missing + _, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(buf), int64(len(buf)), "", ""), opts) + if err != nil { + t.Fatal(err) + } + + for _, disk := range xl.getDisks() { + files, _ := disk.ListDir(ctx, "", bucket, object, -1) + for _, file := range files { + if file != "xl.meta" { + disk.Delete(ctx, bucket, pathJoin(object, file), DeleteOptions{ + Recursive: true, + Immediate: false, + }) + } + } + } + + gr, err := xl.GetObjectNInfo(ctx, bucket, object, nil, nil, opts) + if err != nil { + if err != toObjectErr(errErasureReadQuorum, bucket, object) { + t.Errorf("Expected GetObject to fail with %v, but failed with %v", toObjectErr(errErasureReadQuorum, bucket, object), err) + } + } + if gr != nil { + _, err = io.Copy(io.Discard, gr) + if err != toObjectErr(errErasureReadQuorum, bucket, object) { + t.Errorf("Expected GetObject to fail with %v, but failed with %v", toObjectErr(errErasureReadQuorum, bucket, object), err) + } + gr.Close() + } + + // Test use case 2: Make 9 disks offline, which leaves less than quorum number of disks + // in a 16 disk Erasure setup. The original disks are 'replaced' with + // naughtyDisks that fail after 'f' successful StorageAPI method + // invocations, where f - [0,2) + + // Create "object" under "bucket". + _, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(buf), int64(len(buf)), "", ""), opts) + if err != nil { + t.Fatal(err) + } + + for f := range 2 { + diskErrors := make(map[int]error) + for i := 0; i <= f; i++ { + diskErrors[i] = nil + } + erasureDisks := xl.getDisks() + for i := range erasureDisks[:9] { + switch diskType := erasureDisks[i].(type) { + case *naughtyDisk: + erasureDisks[i] = newNaughtyDisk(diskType.disk, diskErrors, errFaultyDisk) + default: + erasureDisks[i] = newNaughtyDisk(erasureDisks[i], diskErrors, errFaultyDisk) + } + } + z.serverPools[0].erasureDisksMu.Lock() + xl.getDisks = func() []StorageAPI { + return erasureDisks + } + z.serverPools[0].erasureDisksMu.Unlock() + // Fetch object from store. + gr, err := xl.GetObjectNInfo(ctx, bucket, object, nil, nil, opts) + if err != nil { + if err != toObjectErr(errErasureReadQuorum, bucket, object) { + t.Errorf("Expected GetObject to fail with %v, but failed with %v", toObjectErr(errErasureReadQuorum, bucket, object), err) + } + } + if gr != nil { + _, err = io.Copy(io.Discard, gr) + if err != toObjectErr(errErasureReadQuorum, bucket, object) { + t.Errorf("Expected GetObject to fail with %v, but failed with %v", toObjectErr(errErasureReadQuorum, bucket, object), err) + } + gr.Close() + } + } +} + +func TestHeadObjectNoQuorum(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // Create an instance of xl backend. + obj, fsDirs, err := prepareErasure16(ctx) + if err != nil { + t.Fatal(err) + } + // Cleanup backend directories. + defer obj.Shutdown(t.Context()) + defer removeRoots(fsDirs) + + z := obj.(*erasureServerPools) + xl := z.serverPools[0].sets[0] + + // Create "bucket" + err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{}) + if err != nil { + t.Fatal(err) + } + + bucket := "bucket" + object := "object" + opts := ObjectOptions{} + + // Test use case 1: All disks are online, xl.meta are present, but data are missing + _, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts) + if err != nil { + t.Fatal(err) + } + for _, disk := range xl.getDisks() { + files, _ := disk.ListDir(ctx, "", bucket, object, -1) + for _, file := range files { + if file != "xl.meta" { + disk.Delete(ctx, bucket, pathJoin(object, file), DeleteOptions{ + Recursive: true, + Immediate: false, + }) + } + } + } + + _, err = xl.GetObjectInfo(ctx, bucket, object, opts) + if err != nil { + t.Errorf("Expected StatObject to succeed if data dir are not found, but failed with %v", err) + } + + // Test use case 2: Make 9 disks offline, which leaves less than quorum number of disks + // in a 16 disk Erasure setup. The original disks are 'replaced' with + // naughtyDisks that fail after 'f' successful StorageAPI method + // invocations, where f - [0,2) + + // Create "object" under "bucket". + _, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader([]byte("abcd")), int64(len("abcd")), "", ""), opts) + if err != nil { + t.Fatal(err) + } + + erasureDisks := xl.getDisks() + for i := range erasureDisks[:10] { + erasureDisks[i] = nil + } + + z.serverPools[0].erasureDisksMu.Lock() + xl.getDisks = func() []StorageAPI { + return erasureDisks + } + z.serverPools[0].erasureDisksMu.Unlock() + + // Fetch object from store. + _, err = xl.GetObjectInfo(ctx, bucket, object, opts) + if err != toObjectErr(errErasureReadQuorum, bucket, object) { + t.Errorf("Expected getObjectInfo to fail with %v, but failed with %v", toObjectErr(errErasureWriteQuorum, bucket, object), err) + } +} + +func TestPutObjectNoQuorum(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // Create an instance of xl backend. + obj, fsDirs, err := prepareErasure16(ctx) + if err != nil { + t.Fatal(err) + } + + // Cleanup backend directories. + defer obj.Shutdown(t.Context()) + defer removeRoots(fsDirs) + + z := obj.(*erasureServerPools) + xl := z.serverPools[0].sets[0] + + // Create "bucket" + err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{}) + if err != nil { + t.Fatal(err) + } + + bucket := "bucket" + object := "object" + opts := ObjectOptions{} + // Create "object" under "bucket". + _, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(bytes.Repeat([]byte{'a'}, smallFileThreshold*16)), smallFileThreshold*16, "", ""), opts) + if err != nil { + t.Fatal(err) + } + + // Make 9 disks offline, which leaves less than quorum number of disks + // in a 16 disk Erasure setup. The original disks are 'replaced' with + // naughtyDisks that fail after 'f' successful StorageAPI method + // invocations, where f - [0,4) + for f := range 2 { + diskErrors := make(map[int]error) + for i := 0; i <= f; i++ { + diskErrors[i] = nil + } + erasureDisks := xl.getDisks() + for i := range erasureDisks[:9] { + switch diskType := erasureDisks[i].(type) { + case *naughtyDisk: + erasureDisks[i] = newNaughtyDisk(diskType.disk, diskErrors, errFaultyDisk) + default: + erasureDisks[i] = newNaughtyDisk(erasureDisks[i], diskErrors, errFaultyDisk) + } + } + z.serverPools[0].erasureDisksMu.Lock() + xl.getDisks = func() []StorageAPI { + return erasureDisks + } + z.serverPools[0].erasureDisksMu.Unlock() + // Upload new content to same object "object" + _, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(bytes.Repeat([]byte{byte(f)}, smallFileThreshold*16)), smallFileThreshold*16, "", ""), opts) + if !errors.Is(err, errErasureWriteQuorum) { + t.Errorf("Expected putObject to fail with %v, but failed with %v", toObjectErr(errErasureWriteQuorum, bucket, object), err) + } + } +} + +func TestPutObjectNoQuorumSmall(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // Create an instance of xl backend. + obj, fsDirs, err := prepareErasure16(ctx) + if err != nil { + t.Fatal(err) + } + + // Cleanup backend directories. + defer obj.Shutdown(t.Context()) + defer removeRoots(fsDirs) + + z := obj.(*erasureServerPools) + xl := z.serverPools[0].sets[0] + + // Create "bucket" + err = obj.MakeBucket(ctx, "bucket", MakeBucketOptions{}) + if err != nil { + t.Fatal(err) + } + + bucket := "bucket" + object := "object" + opts := ObjectOptions{} + // Create "object" under "bucket". + _, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(bytes.Repeat([]byte{'a'}, smallFileThreshold/2)), smallFileThreshold/2, "", ""), opts) + if err != nil { + t.Fatal(err) + } + + // Make 9 disks offline, which leaves less than quorum number of disks + // in a 16 disk Erasure setup. The original disks are 'replaced' with + // naughtyDisks that fail after 'f' successful StorageAPI method + // invocations, where f - [0,2) + for f := range 2 { + t.Run("exec-"+strconv.Itoa(f), func(t *testing.T) { + diskErrors := make(map[int]error) + for i := 0; i <= f; i++ { + diskErrors[i] = nil + } + erasureDisks := xl.getDisks() + for i := range erasureDisks[:9] { + switch diskType := erasureDisks[i].(type) { + case *naughtyDisk: + erasureDisks[i] = newNaughtyDisk(diskType.disk, diskErrors, errFaultyDisk) + default: + erasureDisks[i] = newNaughtyDisk(erasureDisks[i], diskErrors, errFaultyDisk) + } + } + z.serverPools[0].erasureDisksMu.Lock() + xl.getDisks = func() []StorageAPI { + return erasureDisks + } + z.serverPools[0].erasureDisksMu.Unlock() + // Upload new content to same object "object" + _, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(bytes.Repeat([]byte{byte(f)}, smallFileThreshold/2)), smallFileThreshold/2, "", ""), opts) + if !errors.Is(err, errErasureWriteQuorum) { + t.Errorf("Expected putObject to fail with %v, but failed with %v", toObjectErr(errErasureWriteQuorum, bucket, object), err) + } + }) + } +} + +// Test PutObject twice, one small and another bigger +// than small data threshold and checks reading them again +func TestPutObjectSmallInlineData(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + const numberOfDisks = 4 + + // Create an instance of xl backend. + obj, fsDirs, err := prepareErasure(ctx, numberOfDisks) + if err != nil { + t.Fatal(err) + } + + // Cleanup backend directories. + defer obj.Shutdown(t.Context()) + defer removeRoots(fsDirs) + + bucket := "bucket" + object := "object" + + // Create "bucket" + err = obj.MakeBucket(ctx, bucket, MakeBucketOptions{}) + if err != nil { + t.Fatal(err) + } + + // Test: Upload a small file and read it. + smallData := []byte{'a'} + _, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(smallData), int64(len(smallData)), "", ""), ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + gr, err := obj.GetObjectNInfo(ctx, bucket, object, nil, nil, ObjectOptions{}) + if err != nil { + t.Fatalf("Expected GetObject to succeed, but failed with %v", err) + } + output := bytes.NewBuffer([]byte{}) + _, err = io.Copy(output, gr) + if err != nil { + t.Fatalf("Expected GetObject reading data to succeed, but failed with %v", err) + } + gr.Close() + if !bytes.Equal(output.Bytes(), smallData) { + t.Fatalf("Corrupted data is found") + } + + // Test: Upload a file bigger than the small file threshold + // under the same bucket & key name and try to read it again. + + output.Reset() + bigData := bytes.Repeat([]byte{'b'}, smallFileThreshold*numberOfDisks/2) + + _, err = obj.PutObject(ctx, bucket, object, mustGetPutObjReader(t, bytes.NewReader(bigData), int64(len(bigData)), "", ""), ObjectOptions{}) + if err != nil { + t.Fatal(err) + } + gr, err = obj.GetObjectNInfo(ctx, bucket, object, nil, nil, ObjectOptions{}) + if err != nil { + t.Fatalf("Expected GetObject to succeed, but failed with %v", err) + } + _, err = io.Copy(output, gr) + if err != nil { + t.Fatalf("Expected GetObject reading data to succeed, but failed with %v", err) + } + gr.Close() + if !bytes.Equal(output.Bytes(), bigData) { + t.Fatalf("Corrupted data found") + } +} + +func TestObjectQuorumFromMeta(t *testing.T) { + ExecObjectLayerTestWithDirs(t, testObjectQuorumFromMeta) +} + +func testObjectQuorumFromMeta(obj ObjectLayer, instanceType string, dirs []string, t TestErrHandler) { + bucket := getRandomBucketName() + + var opts ObjectOptions + // make data with more than one part + partCount := 3 + data := bytes.Repeat([]byte("a"), 6*1024*1024*partCount) + + z := obj.(*erasureServerPools) + xl := z.serverPools[0].sets[0] + erasureDisks := xl.getDisks() + + ctx, cancel := context.WithCancel(GlobalContext) + defer cancel() + + err := obj.MakeBucket(ctx, bucket, MakeBucketOptions{}) + if err != nil { + t.Fatalf("Failed to make a bucket %v", err) + } + + // Object for test case 1 - No StorageClass defined, no MetaData in PutObject + object1 := "object1" + globalStorageClass.Update(storageclass.Config{ + RRS: storageclass.StorageClass{ + Parity: 2, + }, + Standard: storageclass.StorageClass{ + Parity: 4, + }, + }) + _, err = obj.PutObject(ctx, bucket, object1, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), opts) + if err != nil { + t.Fatalf("Failed to putObject %v", err) + } + + parts1, errs1 := readAllFileInfo(ctx, erasureDisks, "", bucket, object1, "", false, false) + parts1SC := globalStorageClass + + // Object for test case 2 - No StorageClass defined, MetaData in PutObject requesting RRS Class + object2 := "object2" + metadata2 := make(map[string]string) + metadata2["x-amz-storage-class"] = storageclass.RRS + _, err = obj.PutObject(ctx, bucket, object2, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{UserDefined: metadata2}) + if err != nil { + t.Fatalf("Failed to putObject %v", err) + } + + parts2, errs2 := readAllFileInfo(ctx, erasureDisks, "", bucket, object2, "", false, false) + parts2SC := globalStorageClass + + // Object for test case 3 - No StorageClass defined, MetaData in PutObject requesting Standard Storage Class + object3 := "object3" + metadata3 := make(map[string]string) + metadata3["x-amz-storage-class"] = storageclass.STANDARD + _, err = obj.PutObject(ctx, bucket, object3, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{UserDefined: metadata3}) + if err != nil { + t.Fatalf("Failed to putObject %v", err) + } + + parts3, errs3 := readAllFileInfo(ctx, erasureDisks, "", bucket, object3, "", false, false) + parts3SC := globalStorageClass + + // Object for test case 4 - Standard StorageClass defined as Parity 6, MetaData in PutObject requesting Standard Storage Class + object4 := "object4" + metadata4 := make(map[string]string) + metadata4["x-amz-storage-class"] = storageclass.STANDARD + globalStorageClass.Update(storageclass.Config{ + Standard: storageclass.StorageClass{ + Parity: 6, + }, + }) + + _, err = obj.PutObject(ctx, bucket, object4, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{UserDefined: metadata4}) + if err != nil { + t.Fatalf("Failed to putObject %v", err) + } + + parts4, errs4 := readAllFileInfo(ctx, erasureDisks, "", bucket, object4, "", false, false) + parts4SC := storageclass.Config{ + Standard: storageclass.StorageClass{ + Parity: 6, + }, + } + + // Object for test case 5 - RRS StorageClass defined as Parity 2, MetaData in PutObject requesting RRS Class + // Reset global storage class flags + object5 := "object5" + metadata5 := make(map[string]string) + metadata5["x-amz-storage-class"] = storageclass.RRS + globalStorageClass.Update(storageclass.Config{ + RRS: storageclass.StorageClass{ + Parity: 2, + }, + }) + + _, err = obj.PutObject(ctx, bucket, object5, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{UserDefined: metadata5}) + if err != nil { + t.Fatalf("Failed to putObject %v", err) + } + + parts5, errs5 := readAllFileInfo(ctx, erasureDisks, "", bucket, object5, "", false, false) + parts5SC := globalStorageClass + + // Object for test case 6 - RRS StorageClass defined as Parity 2, MetaData in PutObject requesting Standard Storage Class + object6 := "object6" + metadata6 := make(map[string]string) + metadata6["x-amz-storage-class"] = storageclass.STANDARD + globalStorageClass.Update(storageclass.Config{ + Standard: storageclass.StorageClass{ + Parity: 4, + }, + RRS: storageclass.StorageClass{ + Parity: 2, + }, + }) + + _, err = obj.PutObject(ctx, bucket, object6, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{UserDefined: metadata6}) + if err != nil { + t.Fatalf("Failed to putObject %v", err) + } + + parts6, errs6 := readAllFileInfo(ctx, erasureDisks, "", bucket, object6, "", false, false) + parts6SC := storageclass.Config{ + RRS: storageclass.StorageClass{ + Parity: 2, + }, + } + + // Object for test case 7 - Standard StorageClass defined as Parity 5, MetaData in PutObject requesting RRS Class + // Reset global storage class flags + object7 := "object7" + metadata7 := make(map[string]string) + metadata7["x-amz-storage-class"] = storageclass.STANDARD + globalStorageClass.Update(storageclass.Config{ + Standard: storageclass.StorageClass{ + Parity: 5, + }, + }) + + _, err = obj.PutObject(ctx, bucket, object7, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{UserDefined: metadata7}) + if err != nil { + t.Fatalf("Failed to putObject %v", err) + } + + parts7, errs7 := readAllFileInfo(ctx, erasureDisks, "", bucket, object7, "", false, false) + parts7SC := storageclass.Config{ + Standard: storageclass.StorageClass{ + Parity: 5, + }, + } + + tests := []struct { + parts []FileInfo + errs []error + expectedReadQuorum int + expectedWriteQuorum int + storageClassCfg storageclass.Config + expectedError error + }{ + {parts1, errs1, 12, 12, parts1SC, nil}, + {parts2, errs2, 14, 14, parts2SC, nil}, + {parts3, errs3, 12, 12, parts3SC, nil}, + {parts4, errs4, 10, 10, parts4SC, nil}, + {parts5, errs5, 14, 14, parts5SC, nil}, + {parts6, errs6, 12, 12, parts6SC, nil}, + {parts7, errs7, 11, 11, parts7SC, nil}, + } + for _, tt := range tests { + t.(*testing.T).Run("", func(t *testing.T) { + globalStorageClass.Update(tt.storageClassCfg) + actualReadQuorum, actualWriteQuorum, err := objectQuorumFromMeta(ctx, tt.parts, tt.errs, storageclass.DefaultParityBlocks(len(erasureDisks))) + if tt.expectedError != nil && err == nil { + t.Errorf("Expected %s, got %s", tt.expectedError, err) + } + if tt.expectedError == nil && err != nil { + t.Errorf("Expected %s, got %s", tt.expectedError, err) + } + if tt.expectedReadQuorum != actualReadQuorum { + t.Errorf("Expected Read Quorum %d, got %d", tt.expectedReadQuorum, actualReadQuorum) + } + if tt.expectedWriteQuorum != actualWriteQuorum { + t.Errorf("Expected Write Quorum %d, got %d", tt.expectedWriteQuorum, actualWriteQuorum) + } + }) + } +} + +// In some deployments, one object has data inlined in one disk and not inlined in other disks. +func TestGetObjectInlineNotInline(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // Create a backend with 4 disks named disk{1...4}, this name convention + // because we will unzip some object data from a sample archive. + const numDisks = 4 + path := t.TempDir() + + var fsDirs []string + for i := 1; i <= numDisks; i++ { + fsDirs = append(fsDirs, filepath.Join(path, fmt.Sprintf("disk%d", i))) + } + + objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, fsDirs...)) + if err != nil { + removeRoots(fsDirs) + t.Fatal(err) + } + + // cleaning up of temporary test directories + defer objLayer.Shutdown(t.Context()) + defer removeRoots(fsDirs) + + // Create a testbucket + err = objLayer.MakeBucket(ctx, "testbucket", MakeBucketOptions{}) + if err != nil { + t.Fatal(err) + } + + // Unzip sample object data to the existing disks + err = unzipArchive("testdata/xl-meta-inline-notinline.zip", path) + if err != nil { + t.Fatal(err) + } + + // Try to read the object and check its md5sum + gr, err := objLayer.GetObjectNInfo(ctx, "testbucket", "file", nil, nil, ObjectOptions{}) + if err != nil { + t.Fatalf("Expected GetObject to succeed, but failed with %v", err) + } + + h := md5.New() + _, err = io.Copy(h, gr) + if err != nil { + t.Fatalf("Expected GetObject reading data to succeed, but failed with %v", err) + } + gr.Close() + + const expectedHash = "fffb6377948ebea75ad2b8058e849ef5" + foundHash := fmt.Sprintf("%x", h.Sum(nil)) + if foundHash != expectedHash { + t.Fatalf("Expected data to have md5sum = `%s`, found `%s`", expectedHash, foundHash) + } +} + +// Test reading an object with some outdated data in some disks +func TestGetObjectWithOutdatedDisks(t *testing.T) { + if runtime.GOOS == globalWindowsOSName { + t.Skip() + } + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + // Create an instance of xl backend. + obj, fsDirs, err := prepareErasure(ctx, 6) + if err != nil { + t.Fatal(err) + } + + // Cleanup backend directories. + defer obj.Shutdown(t.Context()) + defer removeRoots(fsDirs) + + z := obj.(*erasureServerPools) + sets := z.serverPools[0] + xl := sets.sets[0] + + origErasureDisks := xl.getDisks() + + testCases := []struct { + bucket string + versioned bool + object string + content []byte + }{ + {"bucket1", false, "object1", []byte("aaaaaaaaaaaaaaaa")}, + {"bucket2", false, "object2", bytes.Repeat([]byte{'a'}, smallFileThreshold*2)}, + {"bucket3", true, "version1", []byte("aaaaaaaaaaaaaaaa")}, + {"bucket4", true, "version2", bytes.Repeat([]byte{'a'}, smallFileThreshold*2)}, + } + + for i, testCase := range testCases { + // Step 1: create a bucket + err = z.MakeBucket(ctx, testCase.bucket, MakeBucketOptions{VersioningEnabled: testCase.versioned}) + if err != nil { + t.Fatalf("Test %d: Failed to create a bucket: %v", i+1, err) + } + + // Step 2: Upload an object with a random content + initialData := bytes.Repeat([]byte{'b'}, len(testCase.content)) + sets.erasureDisksMu.Lock() + xl.getDisks = func() []StorageAPI { return origErasureDisks } + sets.erasureDisksMu.Unlock() + _, err = z.PutObject(ctx, testCase.bucket, testCase.object, mustGetPutObjReader(t, bytes.NewReader(initialData), int64(len(initialData)), "", ""), + ObjectOptions{Versioned: testCase.versioned}) + if err != nil { + t.Fatalf("Test %d: Failed to upload a random object: %v", i+1, err) + } + + // Step 3: Upload the object with some disks offline + sets.erasureDisksMu.Lock() + xl.getDisks = func() []StorageAPI { + disks := make([]StorageAPI, len(origErasureDisks)) + copy(disks, origErasureDisks) + disks[0] = nil + disks[1] = nil + return disks + } + sets.erasureDisksMu.Unlock() + got, err := z.PutObject(ctx, testCase.bucket, testCase.object, mustGetPutObjReader(t, bytes.NewReader(testCase.content), int64(len(testCase.content)), "", ""), + ObjectOptions{Versioned: testCase.versioned}) + if err != nil { + t.Fatalf("Test %d: Failed to upload the final object: %v", i+1, err) + } + + // Step 4: Try to read the object back and check its md5sum + sets.erasureDisksMu.Lock() + xl.getDisks = func() []StorageAPI { return origErasureDisks } + sets.erasureDisksMu.Unlock() + gr, err := z.GetObjectNInfo(ctx, testCase.bucket, testCase.object, nil, nil, ObjectOptions{VersionID: got.VersionID}) + if err != nil { + t.Fatalf("Expected GetObject to succeed, but failed with %v", err) + } + + h := md5.New() + h.Write(testCase.content) + expectedHash := h.Sum(nil) + + h.Reset() + _, err = io.Copy(h, gr) + if err != nil { + t.Fatalf("Test %d: Failed to calculate md5sum of the object: %v", i+1, err) + } + gr.Close() + foundHash := h.Sum(nil) + + if !bytes.Equal(foundHash, expectedHash) { + t.Fatalf("Test %d: Expected data to have md5sum = `%x`, found `%x`", i+1, expectedHash, foundHash) + } + } +} diff --git a/cmd/erasure-server-pool-decom.go b/cmd/erasure-server-pool-decom.go new file mode 100644 index 00000000..834b9fb8 --- /dev/null +++ b/cmd/erasure-server-pool-decom.go @@ -0,0 +1,1544 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "math/rand" + "net/http" + "slices" + "sort" + "strings" + "time" + + "github.com/dustin/go-humanize" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/bucket/lifecycle" + objectlock "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/bucket/versioning" + "github.com/minio/minio/internal/hash" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/console" + "github.com/minio/pkg/v3/env" + "github.com/minio/pkg/v3/workers" +) + +// PoolDecommissionInfo currently decommissioning information +type PoolDecommissionInfo struct { + StartTime time.Time `json:"startTime" msg:"st"` + StartSize int64 `json:"startSize" msg:"ss"` + TotalSize int64 `json:"totalSize" msg:"ts"` + CurrentSize int64 `json:"currentSize" msg:"cs"` + + Complete bool `json:"complete" msg:"cmp"` + Failed bool `json:"failed" msg:"fl"` + Canceled bool `json:"canceled" msg:"cnl"` + + // Internal information. + QueuedBuckets []string `json:"-" msg:"bkts"` + DecommissionedBuckets []string `json:"-" msg:"dbkts"` + + // Last bucket/object decommissioned. + Bucket string `json:"-" msg:"bkt"` + // Captures prefix that is currently being + // decommissioned inside the 'Bucket' + Prefix string `json:"-" msg:"pfx"` + Object string `json:"-" msg:"obj"` + + // Verbose information + ItemsDecommissioned int64 `json:"objectsDecommissioned" msg:"id"` + ItemsDecommissionFailed int64 `json:"objectsDecommissionedFailed" msg:"idf"` + BytesDone int64 `json:"bytesDecommissioned" msg:"bd"` + BytesFailed int64 `json:"bytesDecommissionedFailed" msg:"bf"` +} + +// Clone make a copy of PoolDecommissionInfo +func (pd *PoolDecommissionInfo) Clone() *PoolDecommissionInfo { + if pd == nil { + return nil + } + return &PoolDecommissionInfo{ + StartTime: pd.StartTime, + StartSize: pd.StartSize, + TotalSize: pd.TotalSize, + CurrentSize: pd.CurrentSize, + Complete: pd.Complete, + Failed: pd.Failed, + Canceled: pd.Canceled, + QueuedBuckets: pd.QueuedBuckets, + DecommissionedBuckets: pd.DecommissionedBuckets, + Bucket: pd.Bucket, + Prefix: pd.Prefix, + Object: pd.Object, + ItemsDecommissioned: pd.ItemsDecommissioned, + ItemsDecommissionFailed: pd.ItemsDecommissionFailed, + BytesDone: pd.BytesDone, + BytesFailed: pd.BytesFailed, + } +} + +// bucketPop should be called when a bucket is done decommissioning. +// Adds the bucket to the list of decommissioned buckets and updates resume numbers. +func (pd *PoolDecommissionInfo) bucketPop(bucket string) bool { + pd.DecommissionedBuckets = append(pd.DecommissionedBuckets, bucket) + for i, b := range pd.QueuedBuckets { + if b == bucket { + // Bucket is done. + pd.QueuedBuckets = append(pd.QueuedBuckets[:i], pd.QueuedBuckets[i+1:]...) + // Clear tracker info. + if pd.Bucket == bucket { + pd.Bucket = "" // empty this out for next bucket + pd.Prefix = "" // empty this out for the next bucket + pd.Object = "" // empty this out for next object + } + return true + } + } + return false +} + +func (pd *PoolDecommissionInfo) isBucketDecommissioned(bucket string) bool { + return slices.Contains(pd.DecommissionedBuckets, bucket) +} + +func (pd *PoolDecommissionInfo) bucketPush(bucket decomBucketInfo) { + for _, b := range pd.QueuedBuckets { + if pd.isBucketDecommissioned(b) { + return + } + if b == bucket.String() { + return + } + } + pd.QueuedBuckets = append(pd.QueuedBuckets, bucket.String()) + pd.Bucket = bucket.Name + pd.Prefix = bucket.Prefix +} + +// PoolStatus captures current pool status +type PoolStatus struct { + ID int `json:"id" msg:"id"` + CmdLine string `json:"cmdline" msg:"cl"` + LastUpdate time.Time `json:"lastUpdate" msg:"lu"` + Decommission *PoolDecommissionInfo `json:"decommissionInfo,omitempty" msg:"dec"` +} + +// Clone returns a copy of PoolStatus +func (ps PoolStatus) Clone() PoolStatus { + return PoolStatus{ + ID: ps.ID, + CmdLine: ps.CmdLine, + LastUpdate: ps.LastUpdate, + Decommission: ps.Decommission.Clone(), + } +} + +//go:generate msgp -file $GOFILE -unexported +type poolMeta struct { + Version int `msg:"v"` + Pools []PoolStatus `msg:"pls"` + + // Value should not be saved when we have not loaded anything yet. + dontSave bool `msg:"-"` +} + +// A decommission resumable tells us if decommission is worth +// resuming upon restart of a cluster. +func (p *poolMeta) returnResumablePools() []PoolStatus { + var newPools []PoolStatus + for _, pool := range p.Pools { + if pool.Decommission == nil { + continue + } + if pool.Decommission.Complete || pool.Decommission.Canceled { + // Do not resume decommission upon startup for + // - decommission complete + // - decommission canceled + continue + } // In all other situations we need to resume + newPools = append(newPools, pool) + } + return newPools +} + +func (p *poolMeta) DecommissionComplete(idx int) bool { + if p.Pools[idx].Decommission != nil && !p.Pools[idx].Decommission.Complete { + p.Pools[idx].LastUpdate = UTCNow() + p.Pools[idx].Decommission.Complete = true + p.Pools[idx].Decommission.Failed = false + p.Pools[idx].Decommission.Canceled = false + return true + } + return false +} + +func (p *poolMeta) DecommissionFailed(idx int) bool { + if p.Pools[idx].Decommission != nil && !p.Pools[idx].Decommission.Failed { + p.Pools[idx].LastUpdate = UTCNow() + p.Pools[idx].Decommission.StartTime = time.Time{} + p.Pools[idx].Decommission.Complete = false + p.Pools[idx].Decommission.Failed = true + p.Pools[idx].Decommission.Canceled = false + return true + } + return false +} + +func (p *poolMeta) DecommissionCancel(idx int) bool { + if p.Pools[idx].Decommission != nil && !p.Pools[idx].Decommission.Canceled { + p.Pools[idx].LastUpdate = UTCNow() + p.Pools[idx].Decommission.StartTime = time.Time{} + p.Pools[idx].Decommission.Complete = false + p.Pools[idx].Decommission.Failed = false + p.Pools[idx].Decommission.Canceled = true + return true + } + return false +} + +func (p poolMeta) isBucketDecommissioned(idx int, bucket string) bool { + return p.Pools[idx].Decommission.isBucketDecommissioned(bucket) +} + +func (p *poolMeta) BucketDone(idx int, bucket decomBucketInfo) bool { + if p.Pools[idx].Decommission == nil { + // Decommission not in progress. + return false + } + return p.Pools[idx].Decommission.bucketPop(bucket.String()) +} + +func (p poolMeta) ResumeBucketObject(idx int) (bucket, object string) { + if p.Pools[idx].Decommission != nil { + bucket = p.Pools[idx].Decommission.Bucket + object = p.Pools[idx].Decommission.Object + } + return bucket, object +} + +func (p *poolMeta) TrackCurrentBucketObject(idx int, bucket string, object string) { + if p.Pools[idx].Decommission == nil { + // Decommission not in progress. + return + } + p.Pools[idx].Decommission.Bucket = bucket + p.Pools[idx].Decommission.Object = object +} + +func (p *poolMeta) PendingBuckets(idx int) []decomBucketInfo { + if p.Pools[idx].Decommission == nil { + // Decommission not in progress. + return nil + } + + decomBuckets := make([]decomBucketInfo, len(p.Pools[idx].Decommission.QueuedBuckets)) + for i := range decomBuckets { + bucket, prefix := path2BucketObject(p.Pools[idx].Decommission.QueuedBuckets[i]) + decomBuckets[i] = decomBucketInfo{ + Name: bucket, + Prefix: prefix, + } + } + + return decomBuckets +} + +//msgp:ignore decomBucketInfo +type decomBucketInfo struct { + Name string + Prefix string +} + +func (db decomBucketInfo) String() string { + return pathJoin(db.Name, db.Prefix) +} + +func (p *poolMeta) QueueBuckets(idx int, buckets []decomBucketInfo) { + // add new queued buckets + for _, bucket := range buckets { + p.Pools[idx].Decommission.bucketPush(bucket) + } +} + +var ( + errDecommissionAlreadyRunning = errors.New("decommission is already in progress") + errDecommissionComplete = errors.New("decommission is complete, please remove the servers from command-line") + errDecommissionNotStarted = errors.New("decommission is not in progress") +) + +func (p *poolMeta) Decommission(idx int, pi poolSpaceInfo) error { + // Return an error when there is decommission on going - the user needs + // to explicitly cancel it first in order to restart decommissioning again. + if p.Pools[idx].Decommission != nil && + !p.Pools[idx].Decommission.Complete && + !p.Pools[idx].Decommission.Failed && + !p.Pools[idx].Decommission.Canceled { + return errDecommissionAlreadyRunning + } + + now := UTCNow() + p.Pools[idx].LastUpdate = now + p.Pools[idx].Decommission = &PoolDecommissionInfo{ + StartTime: now, + StartSize: pi.Free, + CurrentSize: pi.Free, + TotalSize: pi.Total, + } + return nil +} + +func (p poolMeta) IsSuspended(idx int) bool { + if idx >= len(p.Pools) { + // We don't really know if the pool is suspended or not, since it doesn't exist. + return false + } + return p.Pools[idx].Decommission != nil +} + +func (p *poolMeta) validate(pools []*erasureSets) (bool, error) { + type poolInfo struct { + position int + completed bool + decomStarted bool // started but not finished yet + } + + rememberedPools := make(map[string]poolInfo) + for idx, pool := range p.Pools { + complete := false + decomStarted := false + if pool.Decommission != nil { + if pool.Decommission.Complete { + complete = true + } + decomStarted = true + } + rememberedPools[pool.CmdLine] = poolInfo{ + position: idx, + completed: complete, + decomStarted: decomStarted, + } + } + + specifiedPools := make(map[string]int) + for idx, pool := range pools { + specifiedPools[pool.endpoints.CmdLine] = idx + } + + var update bool + // Check if specified pools need to be removed from decommissioned pool. + for k := range specifiedPools { + pi, ok := rememberedPools[k] + if !ok { + // we do not have the pool anymore that we previously remembered, since all + // the CLI checks out we can allow updates since we are mostly adding a pool here. + update = true + } + if ok && pi.completed { + logger.LogIf(GlobalContext, "decommission", fmt.Errorf("pool(%s) = %s is decommissioned, please remove from server command line", humanize.Ordinal(pi.position+1), k)) + } + } + + if len(specifiedPools) == len(rememberedPools) { + for k, pi := range rememberedPools { + pos, ok := specifiedPools[k] + if ok && pos != pi.position { + update = true // pool order is changing, its okay to allow it. + } + } + } + + if !update { + update = len(specifiedPools) != len(rememberedPools) + } + + return update, nil +} + +func (p *poolMeta) load(ctx context.Context, pool *erasureSets, pools []*erasureSets) error { + data, err := readConfig(ctx, pool, poolMetaName) + if err != nil { + if errors.Is(err, errConfigNotFound) || isErrObjectNotFound(err) { + return nil + } + return err + } + if len(data) == 0 { + // Seems to be empty create a new poolMeta object. + return nil + } + if len(data) <= 4 { + return fmt.Errorf("poolMeta: no data") + } + // Read header + switch binary.LittleEndian.Uint16(data[0:2]) { + case poolMetaFormat: + default: + return fmt.Errorf("poolMeta: unknown format: %d", binary.LittleEndian.Uint16(data[0:2])) + } + switch binary.LittleEndian.Uint16(data[2:4]) { + case poolMetaVersion: + default: + return fmt.Errorf("poolMeta: unknown version: %d", binary.LittleEndian.Uint16(data[2:4])) + } + + // OK, parse data. + if _, err = p.UnmarshalMsg(data[4:]); err != nil { + return err + } + + switch p.Version { + case poolMetaVersionV1: + default: + return fmt.Errorf("unexpected pool meta version: %d", p.Version) + } + + return nil +} + +func (p *poolMeta) CountItem(idx int, size int64, failed bool) { + pd := p.Pools[idx].Decommission + if pd == nil { + return + } + if failed { + pd.ItemsDecommissionFailed++ + pd.BytesFailed += size + } else { + pd.ItemsDecommissioned++ + pd.BytesDone += size + } + p.Pools[idx].Decommission = pd +} + +func (p *poolMeta) updateAfter(ctx context.Context, idx int, pools []*erasureSets, duration time.Duration) (bool, error) { + if p.Pools[idx].Decommission == nil { + return false, errInvalidArgument + } + now := UTCNow() + if now.Sub(p.Pools[idx].LastUpdate) >= duration { + if serverDebugLog { + console.Debugf("decommission: persisting poolMeta on drive: threshold:%s, poolMeta:%#v\n", now.Sub(p.Pools[idx].LastUpdate), p.Pools[idx]) + } + p.Pools[idx].LastUpdate = now + if err := p.save(ctx, pools); err != nil { + return false, err + } + return true, nil + } + return false, nil +} + +func (p poolMeta) save(ctx context.Context, pools []*erasureSets) error { + if p.dontSave { + return nil + } + data := make([]byte, 4, p.Msgsize()+4) + + // Initialize the header. + binary.LittleEndian.PutUint16(data[0:2], poolMetaFormat) + binary.LittleEndian.PutUint16(data[2:4], poolMetaVersion) + + buf, err := p.MarshalMsg(data) + if err != nil { + return err + } + + // Saves on all pools to make sure decommissioning of first pool is allowed. + for i, eset := range pools { + if err = saveConfig(ctx, eset, poolMetaName, buf); err != nil { + if !errors.Is(err, context.Canceled) { + storageLogIf(ctx, fmt.Errorf("saving pool.bin for pool index %d failed with: %v", i, err)) + } + return err + } + } + return nil +} + +const ( + poolMetaName = "pool.bin" + poolMetaFormat = 1 + poolMetaVersionV1 = 1 + poolMetaVersion = poolMetaVersionV1 +) + +// Init() initializes pools and saves additional information about them +// in 'pool.bin', this is eventually used for decommissioning the pool. +func (z *erasureServerPools) Init(ctx context.Context) error { + // Load rebalance metadata if present + if err := z.loadRebalanceMeta(ctx); err == nil { + // Start rebalance routine if we can reload rebalance metadata. + z.StartRebalance() + } + + meta := poolMeta{} + if err := meta.load(ctx, z.serverPools[0], z.serverPools); err != nil { + return err + } + + update, err := meta.validate(z.serverPools) + if err != nil { + return err + } + + // if no update is needed return right away. + if !update { + z.poolMetaMutex.Lock() + z.poolMeta = meta + z.poolMetaMutex.Unlock() + } else { + newMeta := newPoolMeta(z, meta) + if err = newMeta.save(ctx, z.serverPools); err != nil { + return err + } + z.poolMetaMutex.Lock() + z.poolMeta = newMeta + z.poolMetaMutex.Unlock() + } + + pools := meta.returnResumablePools() + poolIndices := make([]int, 0, len(pools)) + for _, pool := range pools { + idx := globalEndpoints.GetPoolIdx(pool.CmdLine) + if idx == -1 { + return fmt.Errorf("unexpected state present for decommission status pool(%s) not found", pool.CmdLine) + } + poolIndices = append(poolIndices, idx) + } + + if len(poolIndices) > 0 && globalEndpoints[poolIndices[0]].Endpoints[0].IsLocal { + go func() { + // Resume decommissioning of pools, but wait 3 minutes for cluster to stabilize. + if err := sleepContext(ctx, 3*time.Minute); err != nil { + return + } + r := rand.New(rand.NewSource(time.Now().UnixNano())) + for { + if err := z.Decommission(ctx, poolIndices...); err != nil { + if errors.Is(err, errDecommissionAlreadyRunning) { + // A previous decommission running found restart it. + for _, idx := range poolIndices { + z.doDecommissionInRoutine(ctx, idx) + } + return + } + if configRetriableErrors(err) { + decomLogIf(ctx, fmt.Errorf("Unable to resume decommission of pools %v: %w: retrying..", pools, err)) + time.Sleep(time.Second + time.Duration(r.Float64()*float64(5*time.Second))) + continue + } + decomLogIf(ctx, fmt.Errorf("Unable to resume decommission of pool %v: %w", pools, err)) + return + } + } + }() + } + + return nil +} + +func newPoolMeta(z *erasureServerPools, prevMeta poolMeta) poolMeta { + newMeta := poolMeta{} // to update write poolMeta fresh. + // looks like new pool was added we need to update, + // or this is a fresh installation (or an existing + // installation with pool removed) + newMeta.Version = poolMetaVersion + for idx, pool := range z.serverPools { + var skip bool + for _, currentPool := range prevMeta.Pools { + // Preserve any current pool status. + if currentPool.CmdLine == pool.endpoints.CmdLine { + currentPool.ID = idx + newMeta.Pools = append(newMeta.Pools, currentPool) + skip = true + break + } + } + if skip { + continue + } + newMeta.Pools = append(newMeta.Pools, PoolStatus{ + CmdLine: pool.endpoints.CmdLine, + ID: idx, + LastUpdate: UTCNow(), + }) + } + return newMeta +} + +func (z *erasureServerPools) IsDecommissionRunning() bool { + z.poolMetaMutex.RLock() + defer z.poolMetaMutex.RUnlock() + meta := z.poolMeta + for _, pool := range meta.Pools { + if pool.Decommission != nil && + !pool.Decommission.Complete && + !pool.Decommission.Failed && + !pool.Decommission.Canceled { + return true + } + } + + return false +} + +func (z *erasureServerPools) decommissionObject(ctx context.Context, idx int, bucket string, gr *GetObjectReader) (err error) { + objInfo := gr.ObjInfo + + defer func() { + gr.Close() + auditLogDecom(ctx, "DecomCopyData", objInfo.Bucket, objInfo.Name, objInfo.VersionID, err) + }() + + actualSize, err := objInfo.GetActualSize() + if err != nil { + return err + } + + if objInfo.isMultipart() { + res, err := z.NewMultipartUpload(ctx, bucket, objInfo.Name, ObjectOptions{ + VersionID: objInfo.VersionID, + UserDefined: objInfo.UserDefined, + NoAuditLog: true, + SrcPoolIdx: idx, + DataMovement: true, + }) + if err != nil { + return fmt.Errorf("decommissionObject: NewMultipartUpload() %w", err) + } + defer z.AbortMultipartUpload(ctx, bucket, objInfo.Name, res.UploadID, ObjectOptions{NoAuditLog: true}) + parts := make([]CompletePart, len(objInfo.Parts)) + for i, part := range objInfo.Parts { + hr, err := hash.NewReader(ctx, io.LimitReader(gr, part.Size), part.Size, "", "", part.ActualSize) + if err != nil { + return fmt.Errorf("decommissionObject: hash.NewReader() %w", err) + } + pi, err := z.PutObjectPart(ctx, bucket, objInfo.Name, res.UploadID, + part.Number, + NewPutObjReader(hr), + ObjectOptions{ + PreserveETag: part.ETag, // Preserve original ETag to ensure same metadata. + IndexCB: func() []byte { + return part.Index // Preserve part Index to ensure decompression works. + }, + NoAuditLog: true, + }) + if err != nil { + return fmt.Errorf("decommissionObject: PutObjectPart() %w", err) + } + parts[i] = CompletePart{ + ETag: pi.ETag, + PartNumber: pi.PartNumber, + ChecksumCRC32: pi.ChecksumCRC32, + ChecksumCRC32C: pi.ChecksumCRC32C, + ChecksumSHA256: pi.ChecksumSHA256, + ChecksumSHA1: pi.ChecksumSHA1, + } + } + _, err = z.CompleteMultipartUpload(ctx, bucket, objInfo.Name, res.UploadID, parts, ObjectOptions{ + SrcPoolIdx: idx, + DataMovement: true, + MTime: objInfo.ModTime, + NoAuditLog: true, + }) + if err != nil { + err = fmt.Errorf("decommissionObject: CompleteMultipartUpload() %w", err) + } + return err + } + + hr, err := hash.NewReader(ctx, io.LimitReader(gr, objInfo.Size), objInfo.Size, "", "", actualSize) + if err != nil { + return fmt.Errorf("decommissionObject: hash.NewReader() %w", err) + } + + _, err = z.PutObject(ctx, + bucket, + objInfo.Name, + NewPutObjReader(hr), + ObjectOptions{ + DataMovement: true, + SrcPoolIdx: idx, + VersionID: objInfo.VersionID, + MTime: objInfo.ModTime, + UserDefined: objInfo.UserDefined, + PreserveETag: objInfo.ETag, // Preserve original ETag to ensure same metadata. + IndexCB: func() []byte { + return objInfo.Parts[0].Index // Preserve part Index to ensure decompression works. + }, + NoAuditLog: true, + }) + if err != nil { + err = fmt.Errorf("decommissionObject: PutObject() %w", err) + } + return err +} + +// versionsSorter sorts FileInfo slices by version. +// +//msgp:ignore versionsSorter +type versionsSorter []FileInfo + +func (v versionsSorter) reverse() { + sort.Slice(v, func(i, j int) bool { + return v[i].ModTime.Before(v[j].ModTime) + }) +} + +func (set *erasureObjects) listObjectsToDecommission(ctx context.Context, bi decomBucketInfo, fn func(entry metaCacheEntry)) error { + disks, _ := set.getOnlineDisksWithHealing(false) + if len(disks) == 0 { + return fmt.Errorf("no online drives found for set with endpoints %s", set.getEndpoints()) + } + + // However many we ask, versions must exist on ~50% + listingQuorum := (set.setDriveCount + 1) / 2 + + // How to resolve partial results. + resolver := metadataResolutionParams{ + dirQuorum: listingQuorum, // make sure to capture all quorum ratios + objQuorum: listingQuorum, // make sure to capture all quorum ratios + bucket: bi.Name, + } + + err := listPathRaw(ctx, listPathRawOptions{ + disks: disks, + bucket: bi.Name, + path: bi.Prefix, + recursive: true, + forwardTo: "", + minDisks: listingQuorum, + reportNotFound: false, + agreed: fn, + partial: func(entries metaCacheEntries, _ []error) { + entry, ok := entries.resolve(&resolver) + if ok { + fn(*entry) + } + }, + finished: nil, + }) + return err +} + +func (z *erasureServerPools) decommissionPool(ctx context.Context, idx int, pool *erasureSets, bi decomBucketInfo) error { + ctx = logger.SetReqInfo(ctx, &logger.ReqInfo{}) + + const envDecomWorkers = "_MINIO_DECOMMISSION_WORKERS" + workerSize, err := env.GetInt(envDecomWorkers, len(pool.sets)) + if err != nil { + decomLogIf(ctx, fmt.Errorf("invalid workers value err: %v, defaulting to %d", err, len(pool.sets))) + workerSize = len(pool.sets) + } + + // Each decom worker needs one List() goroutine/worker + // add that many extra workers. + workerSize += len(pool.sets) + + wk, err := workers.New(workerSize) + if err != nil { + return err + } + + var vc *versioning.Versioning + var lc *lifecycle.Lifecycle + var lr objectlock.Retention + var rcfg *replication.Config + if bi.Name != minioMetaBucket { + vc, err = globalBucketVersioningSys.Get(bi.Name) + if err != nil { + return err + } + + // Check if the current bucket has a configured lifecycle policy + lc, err = globalLifecycleSys.Get(bi.Name) + if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bi.Name}) { + return err + } + + // Check if bucket is object locked. + lr, err = globalBucketObjectLockSys.Get(bi.Name) + if err != nil { + return err + } + + rcfg, err = getReplicationConfig(ctx, bi.Name) + if err != nil { + return err + } + } + + for setIdx, set := range pool.sets { + filterLifecycle := func(bucket, object string, fi FileInfo) bool { + if lc == nil { + return false + } + versioned := vc != nil && vc.Versioned(object) + objInfo := fi.ToObjectInfo(bucket, object, versioned) + + evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo) + switch { + case evt.Action.DeleteRestored(): // if restored copy has expired, delete it synchronously + applyExpiryOnTransitionedObject(ctx, z, objInfo, evt, lcEventSrc_Decom) + return false + case evt.Action.Delete(): + globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_Decom) + return true + default: + return false + } + } + + decommissionEntry := func(entry metaCacheEntry) { + defer wk.Give() + + if entry.isDir() { + return + } + + fivs, err := entry.fileInfoVersions(bi.Name) + if err != nil { + return + } + + // We need a reversed order for decommissioning, + // to create the appropriate stack. + versionsSorter(fivs.Versions).reverse() + + var decommissioned, expired int + for _, version := range fivs.Versions { + stopFn := globalDecommissionMetrics.log(decomMetricDecommissionObject, idx, bi.Name, version.Name, version.VersionID) + // Apply lifecycle rules on the objects that are expired. + if filterLifecycle(bi.Name, version.Name, version) { + expired++ + decommissioned++ + stopFn(version.Size, errors.New("ILM expired object/version will be skipped")) + continue + } + + // any object with only single DEL marker we don't need + // to decommission, just skip it, this also includes + // any other versions that have already expired. + remainingVersions := len(fivs.Versions) - expired + if version.Deleted && remainingVersions == 1 && rcfg == nil { + decommissioned++ + stopFn(version.Size, errors.New("DELETE marked object with no other non-current versions will be skipped")) + continue + } + + versionID := version.VersionID + if versionID == "" { + versionID = nullVersionID + } + + var failure, ignore bool + if version.Deleted { + _, err := z.DeleteObject(ctx, + bi.Name, + version.Name, + ObjectOptions{ + // Since we are preserving a delete marker, we have to make sure this is always true. + // regardless of the current configuration of the bucket we must preserve all versions + // on the pool being decommissioned. + Versioned: true, + VersionID: versionID, + MTime: version.ModTime, + DeleteReplication: version.ReplicationState, + SrcPoolIdx: idx, + DataMovement: true, + DeleteMarker: true, // make sure we create a delete marker + SkipDecommissioned: true, // make sure we skip the decommissioned pool + NoAuditLog: true, + }) + if err != nil { + // This can happen when rebalance stop races with ongoing rebalance workers. + // These rebalance failures can be ignored. + if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) { + ignore = true + stopFn(0, nil) + continue + } + } + stopFn(version.Size, err) + if err != nil { + decomLogIf(ctx, err) + failure = true + } + z.poolMetaMutex.Lock() + z.poolMeta.CountItem(idx, 0, failure) + z.poolMetaMutex.Unlock() + if !failure { + // Success keep a count. + decommissioned++ + } + auditLogDecom(ctx, "DecomCopyDeleteMarker", bi.Name, version.Name, versionID, err) + continue + } + + // gr.Close() is ensured by decommissionObject(). + for range 3 { + if version.IsRemote() { + if err := z.DecomTieredObject(ctx, bi.Name, version.Name, version, ObjectOptions{ + VersionID: versionID, + MTime: version.ModTime, + UserDefined: version.Metadata, + SrcPoolIdx: idx, + DataMovement: true, + }); err != nil { + if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) { + ignore = true + stopFn(0, nil) + } + } + if !ignore { + stopFn(version.Size, err) + failure = err != nil + decomLogIf(ctx, err) + } + break + } + gr, err := set.GetObjectNInfo(ctx, + bi.Name, + encodeDirObject(version.Name), + nil, + http.Header{}, + ObjectOptions{ + VersionID: versionID, + NoDecryption: true, + NoLock: true, + NoAuditLog: true, + }) + if isErrObjectNotFound(err) || isErrVersionNotFound(err) { + // object deleted by the application, nothing to do here we move on. + ignore = true + stopFn(0, nil) + break + } + if err != nil && !ignore { + // if usage-cache.bin is not readable log and ignore it. + if bi.Name == minioMetaBucket && strings.Contains(version.Name, dataUsageCacheName) { + ignore = true + stopFn(version.Size, err) + decomLogIf(ctx, err) + break + } + } + if err != nil { + failure = true + decomLogIf(ctx, err) + stopFn(version.Size, err) + continue + } + if err = z.decommissionObject(ctx, idx, bi.Name, gr); err != nil { + if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) { + ignore = true + stopFn(0, nil) + break + } + stopFn(version.Size, err) + failure = true + decomLogIf(ctx, err) + continue + } + stopFn(version.Size, nil) + failure = false + break + } + if ignore { + continue + } + z.poolMetaMutex.Lock() + z.poolMeta.CountItem(idx, version.Size, failure) + z.poolMetaMutex.Unlock() + if failure { + break // break out on first error + } + decommissioned++ + } + + // if all versions were decommissioned, then we can delete the object versions. + if decommissioned == len(fivs.Versions) { + stopFn := globalDecommissionMetrics.log(decomMetricDecommissionRemoveObject, idx, bi.Name, entry.name) + _, err := set.DeleteObject(ctx, + bi.Name, + encodeDirObject(entry.name), + ObjectOptions{ + DeletePrefix: true, // use prefix delete to delete all versions at once. + DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls) + NoAuditLog: true, + }, + ) + stopFn(0, err) + auditLogDecom(ctx, "DecomDeleteObject", bi.Name, entry.name, "", err) + if err != nil { + decomLogIf(ctx, err) + } + } + z.poolMetaMutex.Lock() + z.poolMeta.TrackCurrentBucketObject(idx, bi.Name, entry.name) + ok, err := z.poolMeta.updateAfter(ctx, idx, z.serverPools, 30*time.Second) + decomLogIf(ctx, err) + if ok { + globalNotificationSys.ReloadPoolMeta(ctx) + } + z.poolMetaMutex.Unlock() + } + + wk.Take() + go func(setIdx int) { + defer wk.Give() + // We will perpetually retry listing if it fails, since we cannot + // possibly give up in this matter + for !contextCanceled(ctx) { + err := set.listObjectsToDecommission(ctx, bi, + func(entry metaCacheEntry) { + wk.Take() + go decommissionEntry(entry) + }, + ) + if err == nil || errors.Is(err, context.Canceled) || errors.Is(err, errVolumeNotFound) { + break + } + setN := humanize.Ordinal(setIdx + 1) + retryDur := time.Duration(rand.Float64() * float64(5*time.Second)) + decomLogOnceIf(ctx, fmt.Errorf("listing objects from %s set failed with %v, retrying in %v", setN, err, retryDur), "decom-listing-failed"+setN) + time.Sleep(retryDur) + } + }(setIdx) + } + wk.Wait() + return nil +} + +//msgp:ignore decomMetrics +type decomMetrics struct{} + +var globalDecommissionMetrics decomMetrics + +//msgp:ignore decomMetric +//go:generate stringer -type=decomMetric -trimprefix=decomMetric $GOFILE +type decomMetric uint8 + +const ( + decomMetricDecommissionBucket decomMetric = iota + decomMetricDecommissionObject + decomMetricDecommissionRemoveObject +) + +func decomTrace(d decomMetric, poolIdx int, startTime time.Time, duration time.Duration, path string, err error, sz int64) madmin.TraceInfo { + var errStr string + if err != nil { + errStr = err.Error() + } + return madmin.TraceInfo{ + TraceType: madmin.TraceDecommission, + Time: startTime, + NodeName: globalLocalNodeName, + FuncName: fmt.Sprintf("decommission.%s (pool-id=%d)", d.String(), poolIdx), + Duration: duration, + Path: path, + Error: errStr, + Bytes: sz, + } +} + +func (m *decomMetrics) log(d decomMetric, poolIdx int, paths ...string) func(z int64, err error) { + startTime := time.Now() + return func(sz int64, err error) { + duration := time.Since(startTime) + if globalTrace.NumSubscribers(madmin.TraceDecommission) > 0 { + globalTrace.Publish(decomTrace(d, poolIdx, startTime, duration, strings.Join(paths, " "), err, sz)) + } + } +} + +func (z *erasureServerPools) decommissionInBackground(ctx context.Context, idx int) error { + pool := z.serverPools[idx] + z.poolMetaMutex.RLock() + pending := z.poolMeta.PendingBuckets(idx) + z.poolMetaMutex.RUnlock() + + for _, bucket := range pending { + z.poolMetaMutex.RLock() + isDecommissioned := z.poolMeta.isBucketDecommissioned(idx, bucket.String()) + z.poolMetaMutex.RUnlock() + if isDecommissioned { + if serverDebugLog { + console.Debugln("decommission: already done, moving on", bucket) + } + + z.poolMetaMutex.Lock() + if z.poolMeta.BucketDone(idx, bucket) { + // remove from pendingBuckets and persist. + decomLogIf(ctx, z.poolMeta.save(ctx, z.serverPools)) + } + z.poolMetaMutex.Unlock() + continue + } + if serverDebugLog { + console.Debugln("decommission: currently on bucket", bucket.Name) + } + stopFn := globalDecommissionMetrics.log(decomMetricDecommissionBucket, idx, bucket.Name) + if err := z.decommissionPool(ctx, idx, pool, bucket); err != nil { + stopFn(0, err) + return err + } + stopFn(0, nil) + + z.poolMetaMutex.Lock() + if z.poolMeta.BucketDone(idx, bucket) { + decomLogIf(ctx, z.poolMeta.save(ctx, z.serverPools)) + } + z.poolMetaMutex.Unlock() + } + return nil +} + +func (z *erasureServerPools) checkAfterDecom(ctx context.Context, idx int) error { + buckets, err := z.getBucketsToDecommission(ctx) + if err != nil { + return err + } + + pool := z.serverPools[idx] + for _, set := range pool.sets { + for _, bi := range buckets { + var vc *versioning.Versioning + var lc *lifecycle.Lifecycle + var lr objectlock.Retention + var rcfg *replication.Config + if bi.Name != minioMetaBucket { + vc, err = globalBucketVersioningSys.Get(bi.Name) + if err != nil { + return err + } + + // Check if the current bucket has a configured lifecycle policy + lc, err = globalLifecycleSys.Get(bi.Name) + if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bi.Name}) { + return err + } + + // Check if bucket is object locked. + lr, err = globalBucketObjectLockSys.Get(bi.Name) + if err != nil { + return err + } + + rcfg, err = getReplicationConfig(ctx, bi.Name) + if err != nil { + return err + } + } + + filterLifecycle := func(bucket, object string, fi FileInfo) bool { + if lc == nil { + return false + } + versioned := vc != nil && vc.Versioned(object) + objInfo := fi.ToObjectInfo(bucket, object, versioned) + + evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo) + switch { + case evt.Action.DeleteRestored(): // if restored copy has expired,delete it synchronously + applyExpiryOnTransitionedObject(ctx, z, objInfo, evt, lcEventSrc_Decom) + return false + case evt.Action.Delete(): + globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_Decom) + return true + default: + return false + } + } + + var versionsFound int + if err = set.listObjectsToDecommission(ctx, bi, func(entry metaCacheEntry) { + if !entry.isObject() { + return + } + + // `.usage-cache.bin` still exists, must be not readable ignore it. + if bi.Name == minioMetaBucket && strings.Contains(entry.name, dataUsageCacheName) { + // skipping bucket usage cache name, as its autogenerated. + return + } + + fivs, err := entry.fileInfoVersions(bi.Name) + if err != nil { + return + } + + var ignored int + + for _, version := range fivs.Versions { + // Apply lifecycle rules on the objects that are expired. + if filterLifecycle(bi.Name, version.Name, version) { + ignored++ + continue + } + if version.Deleted { + ignored++ + continue + } + } + + versionsFound += len(fivs.Versions) - ignored + }); err != nil { + return err + } + + if versionsFound > 0 { + return fmt.Errorf("at least %d object(s)/version(s) were found in bucket `%s` after decommissioning", versionsFound, bi.Name) + } + } + } + + return nil +} + +func (z *erasureServerPools) doDecommissionInRoutine(ctx context.Context, idx int) { + z.poolMetaMutex.Lock() + var dctx context.Context + dctx, z.decommissionCancelers[idx] = context.WithCancel(GlobalContext) + z.poolMetaMutex.Unlock() + + // Generate an empty request info so it can be directly modified later by audit + dctx = logger.SetReqInfo(dctx, &logger.ReqInfo{}) + + if err := z.decommissionInBackground(dctx, idx); err != nil { + decomLogIf(GlobalContext, err) + decomLogIf(GlobalContext, z.DecommissionFailed(dctx, idx)) + return + } + + z.poolMetaMutex.Lock() + failed := z.poolMeta.Pools[idx].Decommission.ItemsDecommissionFailed > 0 || contextCanceled(dctx) + poolCmdLine := z.poolMeta.Pools[idx].CmdLine + z.poolMetaMutex.Unlock() + + if !failed { + decomLogEvent(dctx, "Decommissioning complete for pool '%s', verifying for any pending objects", poolCmdLine) + err := z.checkAfterDecom(dctx, idx) + if err != nil { + decomLogIf(ctx, err) + failed = true + } + } + + if failed { + // Decommission failed indicate as such. + decomLogIf(GlobalContext, z.DecommissionFailed(dctx, idx)) + } else { + // Complete the decommission.. + decomLogIf(GlobalContext, z.CompleteDecommission(dctx, idx)) + } +} + +func (z *erasureServerPools) IsSuspended(idx int) bool { + z.poolMetaMutex.RLock() + defer z.poolMetaMutex.RUnlock() + return z.poolMeta.IsSuspended(idx) +} + +// Decommission - start decommission session. +func (z *erasureServerPools) Decommission(ctx context.Context, indices ...int) error { + if len(indices) == 0 { + return errInvalidArgument + } + + if z.SinglePool() { + return errInvalidArgument + } + + // Make pool unwritable before decommissioning. + if err := z.StartDecommission(ctx, indices...); err != nil { + return err + } + + go func() { + for _, idx := range indices { + // decommission all pools serially one after + // the other. + z.doDecommissionInRoutine(ctx, idx) + } + }() + + // Successfully started decommissioning. + return nil +} + +type decomError struct { + Err string +} + +func (d decomError) Error() string { + return d.Err +} + +type poolSpaceInfo struct { + Free int64 + Total int64 + Used int64 +} + +func (z *erasureServerPools) getDecommissionPoolSpaceInfo(idx int) (pi poolSpaceInfo, err error) { + if idx < 0 { + return pi, errInvalidArgument + } + if idx+1 > len(z.serverPools) { + return pi, errInvalidArgument + } + + info := z.serverPools[idx].StorageInfo(context.Background()) + info.Backend = z.BackendInfo() + + usableTotal := int64(GetTotalUsableCapacity(info.Disks, info)) + usableFree := int64(GetTotalUsableCapacityFree(info.Disks, info)) + return poolSpaceInfo{ + Total: usableTotal, + Free: usableFree, + Used: usableTotal - usableFree, + }, nil +} + +func (z *erasureServerPools) Status(ctx context.Context, idx int) (PoolStatus, error) { + if idx < 0 { + return PoolStatus{}, errInvalidArgument + } + + pi, err := z.getDecommissionPoolSpaceInfo(idx) + if err != nil { + return PoolStatus{}, err + } + + z.poolMetaMutex.RLock() + defer z.poolMetaMutex.RUnlock() + + poolInfo := z.poolMeta.Pools[idx].Clone() + if poolInfo.Decommission != nil { + poolInfo.Decommission.TotalSize = pi.Total + poolInfo.Decommission.CurrentSize = pi.Free + } else { + poolInfo.Decommission = &PoolDecommissionInfo{ + TotalSize: pi.Total, + CurrentSize: pi.Free, + } + } + return poolInfo, nil +} + +func (z *erasureServerPools) ReloadPoolMeta(ctx context.Context) (err error) { + meta := poolMeta{} + + if err = meta.load(ctx, z.serverPools[0], z.serverPools); err != nil { + return err + } + + z.poolMetaMutex.Lock() + defer z.poolMetaMutex.Unlock() + + z.poolMeta = meta + return nil +} + +func (z *erasureServerPools) DecommissionCancel(ctx context.Context, idx int) (err error) { + if idx < 0 { + return errInvalidArgument + } + + if z.SinglePool() { + return errInvalidArgument + } + + z.poolMetaMutex.Lock() + defer z.poolMetaMutex.Unlock() + + fn := z.decommissionCancelers[idx] + if fn == nil { + // canceling a decommission before it started return an error. + return errDecommissionNotStarted + } + + defer fn() // cancel any active thread. + + if z.poolMeta.DecommissionCancel(idx) { + if err = z.poolMeta.save(ctx, z.serverPools); err != nil { + return err + } + globalNotificationSys.ReloadPoolMeta(ctx) + } + + return nil +} + +func (z *erasureServerPools) DecommissionFailed(ctx context.Context, idx int) (err error) { + if idx < 0 { + return errInvalidArgument + } + + if z.SinglePool() { + return errInvalidArgument + } + + z.poolMetaMutex.Lock() + defer z.poolMetaMutex.Unlock() + + if z.poolMeta.DecommissionFailed(idx) { + if fn := z.decommissionCancelers[idx]; fn != nil { + defer fn() + } // cancel any active thread. + + if err = z.poolMeta.save(ctx, z.serverPools); err != nil { + return err + } + globalNotificationSys.ReloadPoolMeta(ctx) + } + return nil +} + +func (z *erasureServerPools) CompleteDecommission(ctx context.Context, idx int) (err error) { + if idx < 0 { + return errInvalidArgument + } + + if z.SinglePool() { + return errInvalidArgument + } + + z.poolMetaMutex.Lock() + defer z.poolMetaMutex.Unlock() + + if z.poolMeta.DecommissionComplete(idx) { + if fn := z.decommissionCancelers[idx]; fn != nil { + defer fn() + } // cancel any active thread. + + if err = z.poolMeta.save(ctx, z.serverPools); err != nil { + return err + } + globalNotificationSys.ReloadPoolMeta(ctx) + } + return nil +} + +func (z *erasureServerPools) getBucketsToDecommission(ctx context.Context) ([]decomBucketInfo, error) { + buckets, err := z.ListBuckets(ctx, BucketOptions{}) + if err != nil { + return nil, err + } + + decomBuckets := make([]decomBucketInfo, len(buckets)) + for i := range buckets { + decomBuckets[i] = decomBucketInfo{ + Name: buckets[i].Name, + } + } + + // Buckets data are dispersed in multiple zones/sets, make + // sure to decommission the necessary metadata. + decomMetaBuckets := []decomBucketInfo{ + { + Name: minioMetaBucket, + Prefix: minioConfigPrefix, + }, + { + Name: minioMetaBucket, + Prefix: bucketMetaPrefix, + }, + } + + return append(decomMetaBuckets, decomBuckets...), nil +} + +func (z *erasureServerPools) StartDecommission(ctx context.Context, indices ...int) (err error) { + if len(indices) == 0 { + return errInvalidArgument + } + + if z.SinglePool() { + return errInvalidArgument + } + + decomBuckets, err := z.getBucketsToDecommission(ctx) + if err != nil { + return err + } + + for _, bucket := range decomBuckets { + z.HealBucket(ctx, bucket.Name, madmin.HealOpts{}) + } + + // Create .minio.sys/config, .minio.sys/buckets paths if missing, + // this code is present to avoid any missing meta buckets on other + // pools. + for _, metaBucket := range []string{ + pathJoin(minioMetaBucket, minioConfigPrefix), + pathJoin(minioMetaBucket, bucketMetaPrefix), + } { + var bucketExists BucketExists + if err = z.MakeBucket(ctx, metaBucket, MakeBucketOptions{}); err != nil { + if !errors.As(err, &bucketExists) { + return err + } + } + } + + z.poolMetaMutex.Lock() + defer z.poolMetaMutex.Unlock() + + for _, idx := range indices { + pi, err := z.getDecommissionPoolSpaceInfo(idx) + if err != nil { + return err + } + + if err = z.poolMeta.Decommission(idx, pi); err != nil { + return err + } + + z.poolMeta.QueueBuckets(idx, decomBuckets) + } + + if err = z.poolMeta.save(ctx, z.serverPools); err != nil { + return err + } + + globalNotificationSys.ReloadPoolMeta(ctx) + + return nil +} + +func auditLogDecom(ctx context.Context, apiName, bucket, object, versionID string, err error) { + errStr := "" + if err != nil { + errStr = err.Error() + } + auditLogInternal(ctx, AuditLogOptions{ + Event: "decommission", + APIName: apiName, + Bucket: bucket, + Object: object, + VersionID: versionID, + Error: errStr, + }) +} diff --git a/cmd/erasure-server-pool-decom_gen.go b/cmd/erasure-server-pool-decom_gen.go new file mode 100644 index 00000000..a4c6b56f --- /dev/null +++ b/cmd/erasure-server-pool-decom_gen.go @@ -0,0 +1,1213 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *PoolDecommissionInfo) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "st": + z.StartTime, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "StartTime") + return + } + case "ss": + z.StartSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "StartSize") + return + } + case "ts": + z.TotalSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "TotalSize") + return + } + case "cs": + z.CurrentSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "CurrentSize") + return + } + case "cmp": + z.Complete, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Complete") + return + } + case "fl": + z.Failed, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Failed") + return + } + case "cnl": + z.Canceled, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Canceled") + return + } + case "bkts": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "QueuedBuckets") + return + } + if cap(z.QueuedBuckets) >= int(zb0002) { + z.QueuedBuckets = (z.QueuedBuckets)[:zb0002] + } else { + z.QueuedBuckets = make([]string, zb0002) + } + for za0001 := range z.QueuedBuckets { + z.QueuedBuckets[za0001], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "QueuedBuckets", za0001) + return + } + } + case "dbkts": + var zb0003 uint32 + zb0003, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "DecommissionedBuckets") + return + } + if cap(z.DecommissionedBuckets) >= int(zb0003) { + z.DecommissionedBuckets = (z.DecommissionedBuckets)[:zb0003] + } else { + z.DecommissionedBuckets = make([]string, zb0003) + } + for za0002 := range z.DecommissionedBuckets { + z.DecommissionedBuckets[za0002], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DecommissionedBuckets", za0002) + return + } + } + case "bkt": + z.Bucket, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "pfx": + z.Prefix, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + case "obj": + z.Object, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + case "id": + z.ItemsDecommissioned, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ItemsDecommissioned") + return + } + case "idf": + z.ItemsDecommissionFailed, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ItemsDecommissionFailed") + return + } + case "bd": + z.BytesDone, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "BytesDone") + return + } + case "bf": + z.BytesFailed, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "BytesFailed") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *PoolDecommissionInfo) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 16 + // write "st" + err = en.Append(0xde, 0x0, 0x10, 0xa2, 0x73, 0x74) + if err != nil { + return + } + err = en.WriteTime(z.StartTime) + if err != nil { + err = msgp.WrapError(err, "StartTime") + return + } + // write "ss" + err = en.Append(0xa2, 0x73, 0x73) + if err != nil { + return + } + err = en.WriteInt64(z.StartSize) + if err != nil { + err = msgp.WrapError(err, "StartSize") + return + } + // write "ts" + err = en.Append(0xa2, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteInt64(z.TotalSize) + if err != nil { + err = msgp.WrapError(err, "TotalSize") + return + } + // write "cs" + err = en.Append(0xa2, 0x63, 0x73) + if err != nil { + return + } + err = en.WriteInt64(z.CurrentSize) + if err != nil { + err = msgp.WrapError(err, "CurrentSize") + return + } + // write "cmp" + err = en.Append(0xa3, 0x63, 0x6d, 0x70) + if err != nil { + return + } + err = en.WriteBool(z.Complete) + if err != nil { + err = msgp.WrapError(err, "Complete") + return + } + // write "fl" + err = en.Append(0xa2, 0x66, 0x6c) + if err != nil { + return + } + err = en.WriteBool(z.Failed) + if err != nil { + err = msgp.WrapError(err, "Failed") + return + } + // write "cnl" + err = en.Append(0xa3, 0x63, 0x6e, 0x6c) + if err != nil { + return + } + err = en.WriteBool(z.Canceled) + if err != nil { + err = msgp.WrapError(err, "Canceled") + return + } + // write "bkts" + err = en.Append(0xa4, 0x62, 0x6b, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.QueuedBuckets))) + if err != nil { + err = msgp.WrapError(err, "QueuedBuckets") + return + } + for za0001 := range z.QueuedBuckets { + err = en.WriteString(z.QueuedBuckets[za0001]) + if err != nil { + err = msgp.WrapError(err, "QueuedBuckets", za0001) + return + } + } + // write "dbkts" + err = en.Append(0xa5, 0x64, 0x62, 0x6b, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.DecommissionedBuckets))) + if err != nil { + err = msgp.WrapError(err, "DecommissionedBuckets") + return + } + for za0002 := range z.DecommissionedBuckets { + err = en.WriteString(z.DecommissionedBuckets[za0002]) + if err != nil { + err = msgp.WrapError(err, "DecommissionedBuckets", za0002) + return + } + } + // write "bkt" + err = en.Append(0xa3, 0x62, 0x6b, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Bucket) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + // write "pfx" + err = en.Append(0xa3, 0x70, 0x66, 0x78) + if err != nil { + return + } + err = en.WriteString(z.Prefix) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + // write "obj" + err = en.Append(0xa3, 0x6f, 0x62, 0x6a) + if err != nil { + return + } + err = en.WriteString(z.Object) + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + // write "id" + err = en.Append(0xa2, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteInt64(z.ItemsDecommissioned) + if err != nil { + err = msgp.WrapError(err, "ItemsDecommissioned") + return + } + // write "idf" + err = en.Append(0xa3, 0x69, 0x64, 0x66) + if err != nil { + return + } + err = en.WriteInt64(z.ItemsDecommissionFailed) + if err != nil { + err = msgp.WrapError(err, "ItemsDecommissionFailed") + return + } + // write "bd" + err = en.Append(0xa2, 0x62, 0x64) + if err != nil { + return + } + err = en.WriteInt64(z.BytesDone) + if err != nil { + err = msgp.WrapError(err, "BytesDone") + return + } + // write "bf" + err = en.Append(0xa2, 0x62, 0x66) + if err != nil { + return + } + err = en.WriteInt64(z.BytesFailed) + if err != nil { + err = msgp.WrapError(err, "BytesFailed") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *PoolDecommissionInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 16 + // string "st" + o = append(o, 0xde, 0x0, 0x10, 0xa2, 0x73, 0x74) + o = msgp.AppendTime(o, z.StartTime) + // string "ss" + o = append(o, 0xa2, 0x73, 0x73) + o = msgp.AppendInt64(o, z.StartSize) + // string "ts" + o = append(o, 0xa2, 0x74, 0x73) + o = msgp.AppendInt64(o, z.TotalSize) + // string "cs" + o = append(o, 0xa2, 0x63, 0x73) + o = msgp.AppendInt64(o, z.CurrentSize) + // string "cmp" + o = append(o, 0xa3, 0x63, 0x6d, 0x70) + o = msgp.AppendBool(o, z.Complete) + // string "fl" + o = append(o, 0xa2, 0x66, 0x6c) + o = msgp.AppendBool(o, z.Failed) + // string "cnl" + o = append(o, 0xa3, 0x63, 0x6e, 0x6c) + o = msgp.AppendBool(o, z.Canceled) + // string "bkts" + o = append(o, 0xa4, 0x62, 0x6b, 0x74, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.QueuedBuckets))) + for za0001 := range z.QueuedBuckets { + o = msgp.AppendString(o, z.QueuedBuckets[za0001]) + } + // string "dbkts" + o = append(o, 0xa5, 0x64, 0x62, 0x6b, 0x74, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.DecommissionedBuckets))) + for za0002 := range z.DecommissionedBuckets { + o = msgp.AppendString(o, z.DecommissionedBuckets[za0002]) + } + // string "bkt" + o = append(o, 0xa3, 0x62, 0x6b, 0x74) + o = msgp.AppendString(o, z.Bucket) + // string "pfx" + o = append(o, 0xa3, 0x70, 0x66, 0x78) + o = msgp.AppendString(o, z.Prefix) + // string "obj" + o = append(o, 0xa3, 0x6f, 0x62, 0x6a) + o = msgp.AppendString(o, z.Object) + // string "id" + o = append(o, 0xa2, 0x69, 0x64) + o = msgp.AppendInt64(o, z.ItemsDecommissioned) + // string "idf" + o = append(o, 0xa3, 0x69, 0x64, 0x66) + o = msgp.AppendInt64(o, z.ItemsDecommissionFailed) + // string "bd" + o = append(o, 0xa2, 0x62, 0x64) + o = msgp.AppendInt64(o, z.BytesDone) + // string "bf" + o = append(o, 0xa2, 0x62, 0x66) + o = msgp.AppendInt64(o, z.BytesFailed) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *PoolDecommissionInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "st": + z.StartTime, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "StartTime") + return + } + case "ss": + z.StartSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "StartSize") + return + } + case "ts": + z.TotalSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalSize") + return + } + case "cs": + z.CurrentSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "CurrentSize") + return + } + case "cmp": + z.Complete, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Complete") + return + } + case "fl": + z.Failed, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Failed") + return + } + case "cnl": + z.Canceled, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Canceled") + return + } + case "bkts": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "QueuedBuckets") + return + } + if cap(z.QueuedBuckets) >= int(zb0002) { + z.QueuedBuckets = (z.QueuedBuckets)[:zb0002] + } else { + z.QueuedBuckets = make([]string, zb0002) + } + for za0001 := range z.QueuedBuckets { + z.QueuedBuckets[za0001], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "QueuedBuckets", za0001) + return + } + } + case "dbkts": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DecommissionedBuckets") + return + } + if cap(z.DecommissionedBuckets) >= int(zb0003) { + z.DecommissionedBuckets = (z.DecommissionedBuckets)[:zb0003] + } else { + z.DecommissionedBuckets = make([]string, zb0003) + } + for za0002 := range z.DecommissionedBuckets { + z.DecommissionedBuckets[za0002], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DecommissionedBuckets", za0002) + return + } + } + case "bkt": + z.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "pfx": + z.Prefix, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + case "obj": + z.Object, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + case "id": + z.ItemsDecommissioned, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ItemsDecommissioned") + return + } + case "idf": + z.ItemsDecommissionFailed, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ItemsDecommissionFailed") + return + } + case "bd": + z.BytesDone, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "BytesDone") + return + } + case "bf": + z.BytesFailed, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "BytesFailed") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *PoolDecommissionInfo) Msgsize() (s int) { + s = 3 + 3 + msgp.TimeSize + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size + 4 + msgp.BoolSize + 3 + msgp.BoolSize + 4 + msgp.BoolSize + 5 + msgp.ArrayHeaderSize + for za0001 := range z.QueuedBuckets { + s += msgp.StringPrefixSize + len(z.QueuedBuckets[za0001]) + } + s += 6 + msgp.ArrayHeaderSize + for za0002 := range z.DecommissionedBuckets { + s += msgp.StringPrefixSize + len(z.DecommissionedBuckets[za0002]) + } + s += 4 + msgp.StringPrefixSize + len(z.Bucket) + 4 + msgp.StringPrefixSize + len(z.Prefix) + 4 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.Int64Size + 4 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.Int64Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *PoolStatus) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.ID, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + case "cl": + z.CmdLine, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "CmdLine") + return + } + case "lu": + z.LastUpdate, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "LastUpdate") + return + } + case "dec": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "Decommission") + return + } + z.Decommission = nil + } else { + if z.Decommission == nil { + z.Decommission = new(PoolDecommissionInfo) + } + err = z.Decommission.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Decommission") + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *PoolStatus) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 4 + // write "id" + err = en.Append(0x84, 0xa2, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteInt(z.ID) + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + // write "cl" + err = en.Append(0xa2, 0x63, 0x6c) + if err != nil { + return + } + err = en.WriteString(z.CmdLine) + if err != nil { + err = msgp.WrapError(err, "CmdLine") + return + } + // write "lu" + err = en.Append(0xa2, 0x6c, 0x75) + if err != nil { + return + } + err = en.WriteTime(z.LastUpdate) + if err != nil { + err = msgp.WrapError(err, "LastUpdate") + return + } + // write "dec" + err = en.Append(0xa3, 0x64, 0x65, 0x63) + if err != nil { + return + } + if z.Decommission == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.Decommission.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Decommission") + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *PoolStatus) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 4 + // string "id" + o = append(o, 0x84, 0xa2, 0x69, 0x64) + o = msgp.AppendInt(o, z.ID) + // string "cl" + o = append(o, 0xa2, 0x63, 0x6c) + o = msgp.AppendString(o, z.CmdLine) + // string "lu" + o = append(o, 0xa2, 0x6c, 0x75) + o = msgp.AppendTime(o, z.LastUpdate) + // string "dec" + o = append(o, 0xa3, 0x64, 0x65, 0x63) + if z.Decommission == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.Decommission.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Decommission") + return + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *PoolStatus) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.ID, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + case "cl": + z.CmdLine, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "CmdLine") + return + } + case "lu": + z.LastUpdate, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "LastUpdate") + return + } + case "dec": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.Decommission = nil + } else { + if z.Decommission == nil { + z.Decommission = new(PoolDecommissionInfo) + } + bts, err = z.Decommission.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Decommission") + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *PoolStatus) Msgsize() (s int) { + s = 1 + 3 + msgp.IntSize + 3 + msgp.StringPrefixSize + len(z.CmdLine) + 3 + msgp.TimeSize + 4 + if z.Decommission == nil { + s += msgp.NilSize + } else { + s += z.Decommission.Msgsize() + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *decomError) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Err": + z.Err, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Err") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z decomError) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "Err" + err = en.Append(0x81, 0xa3, 0x45, 0x72, 0x72) + if err != nil { + return + } + err = en.WriteString(z.Err) + if err != nil { + err = msgp.WrapError(err, "Err") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z decomError) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "Err" + o = append(o, 0x81, 0xa3, 0x45, 0x72, 0x72) + o = msgp.AppendString(o, z.Err) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *decomError) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Err": + z.Err, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Err") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z decomError) Msgsize() (s int) { + s = 1 + 4 + msgp.StringPrefixSize + len(z.Err) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *poolMeta) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "v": + z.Version, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Version") + return + } + case "pls": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Pools") + return + } + if cap(z.Pools) >= int(zb0002) { + z.Pools = (z.Pools)[:zb0002] + } else { + z.Pools = make([]PoolStatus, zb0002) + } + for za0001 := range z.Pools { + err = z.Pools[za0001].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Pools", za0001) + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *poolMeta) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "v" + err = en.Append(0x82, 0xa1, 0x76) + if err != nil { + return + } + err = en.WriteInt(z.Version) + if err != nil { + err = msgp.WrapError(err, "Version") + return + } + // write "pls" + err = en.Append(0xa3, 0x70, 0x6c, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Pools))) + if err != nil { + err = msgp.WrapError(err, "Pools") + return + } + for za0001 := range z.Pools { + err = z.Pools[za0001].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Pools", za0001) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *poolMeta) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "v" + o = append(o, 0x82, 0xa1, 0x76) + o = msgp.AppendInt(o, z.Version) + // string "pls" + o = append(o, 0xa3, 0x70, 0x6c, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Pools))) + for za0001 := range z.Pools { + o, err = z.Pools[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Pools", za0001) + return + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *poolMeta) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "v": + z.Version, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Version") + return + } + case "pls": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Pools") + return + } + if cap(z.Pools) >= int(zb0002) { + z.Pools = (z.Pools)[:zb0002] + } else { + z.Pools = make([]PoolStatus, zb0002) + } + for za0001 := range z.Pools { + bts, err = z.Pools[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Pools", za0001) + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *poolMeta) Msgsize() (s int) { + s = 1 + 2 + msgp.IntSize + 4 + msgp.ArrayHeaderSize + for za0001 := range z.Pools { + s += z.Pools[za0001].Msgsize() + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *poolSpaceInfo) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Free": + z.Free, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Free") + return + } + case "Total": + z.Total, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Total") + return + } + case "Used": + z.Used, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Used") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z poolSpaceInfo) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "Free" + err = en.Append(0x83, 0xa4, 0x46, 0x72, 0x65, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.Free) + if err != nil { + err = msgp.WrapError(err, "Free") + return + } + // write "Total" + err = en.Append(0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c) + if err != nil { + return + } + err = en.WriteInt64(z.Total) + if err != nil { + err = msgp.WrapError(err, "Total") + return + } + // write "Used" + err = en.Append(0xa4, 0x55, 0x73, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteInt64(z.Used) + if err != nil { + err = msgp.WrapError(err, "Used") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z poolSpaceInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "Free" + o = append(o, 0x83, 0xa4, 0x46, 0x72, 0x65, 0x65) + o = msgp.AppendInt64(o, z.Free) + // string "Total" + o = append(o, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c) + o = msgp.AppendInt64(o, z.Total) + // string "Used" + o = append(o, 0xa4, 0x55, 0x73, 0x65, 0x64) + o = msgp.AppendInt64(o, z.Used) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *poolSpaceInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Free": + z.Free, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Free") + return + } + case "Total": + z.Total, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Total") + return + } + case "Used": + z.Used, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Used") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z poolSpaceInfo) Msgsize() (s int) { + s = 1 + 5 + msgp.Int64Size + 6 + msgp.Int64Size + 5 + msgp.Int64Size + return +} diff --git a/cmd/erasure-server-pool-decom_gen_test.go b/cmd/erasure-server-pool-decom_gen_test.go new file mode 100644 index 00000000..47a08be1 --- /dev/null +++ b/cmd/erasure-server-pool-decom_gen_test.go @@ -0,0 +1,575 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalPoolDecommissionInfo(t *testing.T) { + v := PoolDecommissionInfo{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgPoolDecommissionInfo(b *testing.B) { + v := PoolDecommissionInfo{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgPoolDecommissionInfo(b *testing.B) { + v := PoolDecommissionInfo{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalPoolDecommissionInfo(b *testing.B) { + v := PoolDecommissionInfo{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodePoolDecommissionInfo(t *testing.T) { + v := PoolDecommissionInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodePoolDecommissionInfo Msgsize() is inaccurate") + } + + vn := PoolDecommissionInfo{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodePoolDecommissionInfo(b *testing.B) { + v := PoolDecommissionInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodePoolDecommissionInfo(b *testing.B) { + v := PoolDecommissionInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalPoolStatus(t *testing.T) { + v := PoolStatus{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgPoolStatus(b *testing.B) { + v := PoolStatus{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgPoolStatus(b *testing.B) { + v := PoolStatus{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalPoolStatus(b *testing.B) { + v := PoolStatus{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodePoolStatus(t *testing.T) { + v := PoolStatus{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodePoolStatus Msgsize() is inaccurate") + } + + vn := PoolStatus{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodePoolStatus(b *testing.B) { + v := PoolStatus{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodePoolStatus(b *testing.B) { + v := PoolStatus{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshaldecomError(t *testing.T) { + v := decomError{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgdecomError(b *testing.B) { + v := decomError{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgdecomError(b *testing.B) { + v := decomError{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshaldecomError(b *testing.B) { + v := decomError{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodedecomError(t *testing.T) { + v := decomError{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodedecomError Msgsize() is inaccurate") + } + + vn := decomError{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodedecomError(b *testing.B) { + v := decomError{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodedecomError(b *testing.B) { + v := decomError{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalpoolMeta(t *testing.T) { + v := poolMeta{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgpoolMeta(b *testing.B) { + v := poolMeta{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgpoolMeta(b *testing.B) { + v := poolMeta{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalpoolMeta(b *testing.B) { + v := poolMeta{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodepoolMeta(t *testing.T) { + v := poolMeta{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodepoolMeta Msgsize() is inaccurate") + } + + vn := poolMeta{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodepoolMeta(b *testing.B) { + v := poolMeta{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodepoolMeta(b *testing.B) { + v := poolMeta{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalpoolSpaceInfo(t *testing.T) { + v := poolSpaceInfo{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgpoolSpaceInfo(b *testing.B) { + v := poolSpaceInfo{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgpoolSpaceInfo(b *testing.B) { + v := poolSpaceInfo{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalpoolSpaceInfo(b *testing.B) { + v := poolSpaceInfo{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodepoolSpaceInfo(t *testing.T) { + v := poolSpaceInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodepoolSpaceInfo Msgsize() is inaccurate") + } + + vn := poolSpaceInfo{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodepoolSpaceInfo(b *testing.B) { + v := poolSpaceInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodepoolSpaceInfo(b *testing.B) { + v := poolSpaceInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/erasure-server-pool-decom_test.go b/cmd/erasure-server-pool-decom_test.go new file mode 100644 index 00000000..7f4c8ef1 --- /dev/null +++ b/cmd/erasure-server-pool-decom_test.go @@ -0,0 +1,195 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "testing" +) + +func prepareErasurePools() (ObjectLayer, []string, error) { + nDisks := 32 + fsDirs, err := getRandomDisks(nDisks) + if err != nil { + return nil, nil, err + } + + pools := mustGetPoolEndpoints(0, fsDirs[:16]...) + pools = append(pools, mustGetPoolEndpoints(1, fsDirs[16:]...)...) + + objLayer, _, err := initObjectLayer(context.Background(), pools) + if err != nil { + removeRoots(fsDirs) + return nil, nil, err + } + return objLayer, fsDirs, nil +} + +func TestPoolMetaValidate(t *testing.T) { + objLayer1, fsDirs, err := prepareErasurePools() + if err != nil { + t.Fatal(err) + } + defer removeRoots(fsDirs) + + meta := objLayer1.(*erasureServerPools).poolMeta + pools := objLayer1.(*erasureServerPools).serverPools + + objLayer2, fsDirs, err := prepareErasurePools() + if err != nil { + t.Fatalf("Initialization of object layer failed for Erasure setup: %s", err) + } + defer removeRoots(fsDirs) + + newPools := objLayer2.(*erasureServerPools).serverPools + reducedPools := pools[1:] + orderChangePools := []*erasureSets{ + pools[1], + pools[0], + } + + var nmeta1 poolMeta + nmeta1.Version = poolMetaVersion + nmeta1.Pools = append(nmeta1.Pools, meta.Pools...) + for i, pool := range nmeta1.Pools { + if i == 0 { + nmeta1.Pools[i] = PoolStatus{ + CmdLine: pool.CmdLine, + ID: i, + LastUpdate: UTCNow(), + Decommission: &PoolDecommissionInfo{ + Complete: true, + }, + } + } + } + + var nmeta2 poolMeta + nmeta2.Version = poolMetaVersion + nmeta2.Pools = append(nmeta2.Pools, meta.Pools...) + for i, pool := range nmeta2.Pools { + if i == 0 { + nmeta2.Pools[i] = PoolStatus{ + CmdLine: pool.CmdLine, + ID: i, + LastUpdate: UTCNow(), + Decommission: &PoolDecommissionInfo{ + Complete: false, + }, + } + } + } + + testCases := []struct { + meta poolMeta + pools []*erasureSets + expectedUpdate bool + expectedErr bool + name string + }{ + { + meta: meta, + pools: pools, + name: "Correct", + expectedErr: false, + expectedUpdate: false, + }, + { + meta: meta, + pools: newPools, + name: "Correct-Update", + expectedErr: false, + expectedUpdate: true, + }, + { + meta: meta, + pools: reducedPools, + name: "Correct-Update", + expectedErr: false, + expectedUpdate: true, + }, + { + meta: meta, + pools: orderChangePools, + name: "Invalid-Orderchange", + expectedErr: false, + expectedUpdate: true, + }, + { + meta: nmeta1, + pools: pools, + name: "Invalid-Completed-Pool-Not-Removed", + expectedErr: false, + expectedUpdate: false, + }, + { + meta: nmeta2, + pools: pools, + name: "Correct-Decom-Pending", + expectedErr: false, + expectedUpdate: false, + }, + { + meta: nmeta2, + pools: reducedPools, + name: "Invalid-Decom-Pending-Pool-Removal", + expectedErr: false, + expectedUpdate: true, + }, + { + meta: nmeta1, + pools: reducedPools, + name: "Correct-Decom-Pool-Removed", + expectedErr: false, + expectedUpdate: true, + }, + { + meta: poolMeta{}, // no-pool info available fresh setup. + pools: pools, + name: "Correct-Fresh-Setup", + expectedErr: false, + expectedUpdate: true, + }, + { + meta: nmeta2, + pools: orderChangePools, + name: "Invalid-Orderchange-Decom", + expectedErr: false, + expectedUpdate: true, + }, + } + + t.Parallel() + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + update, err := testCase.meta.validate(testCase.pools) + if testCase.expectedErr { + t.Log(err) + } + if err != nil && !testCase.expectedErr { + t.Errorf("Expected success, but found %s", err) + } + if err == nil && testCase.expectedErr { + t.Error("Expected error, but got `nil`") + } + if update != testCase.expectedUpdate { + t.Errorf("Expected %t, got %t", testCase.expectedUpdate, update) + } + }) + } +} diff --git a/cmd/erasure-server-pool-rebalance.go b/cmd/erasure-server-pool-rebalance.go new file mode 100644 index 00000000..c037f0dd --- /dev/null +++ b/cmd/erasure-server-pool-rebalance.go @@ -0,0 +1,1036 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "math" + "math/rand" + "net/http" + "strings" + "time" + + "github.com/dustin/go-humanize" + "github.com/lithammer/shortuuid/v4" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/bucket/lifecycle" + objectlock "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/bucket/versioning" + "github.com/minio/minio/internal/hash" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/env" + "github.com/minio/pkg/v3/workers" +) + +//go:generate msgp -file $GOFILE -unexported + +// rebalanceStats contains per-pool rebalance statistics like number of objects, +// versions and bytes rebalanced out of a pool +type rebalanceStats struct { + InitFreeSpace uint64 `json:"initFreeSpace" msg:"ifs"` // Pool free space at the start of rebalance + InitCapacity uint64 `json:"initCapacity" msg:"ic"` // Pool capacity at the start of rebalance + + Buckets []string `json:"buckets" msg:"bus"` // buckets being rebalanced or to be rebalanced + RebalancedBuckets []string `json:"rebalancedBuckets" msg:"rbs"` // buckets rebalanced + Bucket string `json:"bucket" msg:"bu"` // Last rebalanced bucket + Object string `json:"object" msg:"ob"` // Last rebalanced object + NumObjects uint64 `json:"numObjects" msg:"no"` // Number of objects rebalanced + NumVersions uint64 `json:"numVersions" msg:"nv"` // Number of versions rebalanced + Bytes uint64 `json:"bytes" msg:"bs"` // Number of bytes rebalanced + Participating bool `json:"participating" msg:"par"` + Info rebalanceInfo `json:"info" msg:"inf"` +} + +func (rs *rebalanceStats) update(bucket string, fi FileInfo) { + if fi.IsLatest { + rs.NumObjects++ + } + + rs.NumVersions++ + onDiskSz := int64(0) + if !fi.Deleted { + onDiskSz = fi.Size * int64(fi.Erasure.DataBlocks+fi.Erasure.ParityBlocks) / int64(fi.Erasure.DataBlocks) + } + rs.Bytes += uint64(onDiskSz) + rs.Bucket = bucket + rs.Object = fi.Name +} + +type rstats []*rebalanceStats + +//go:generate stringer -type=rebalStatus -trimprefix=rebal $GOFILE +type rebalStatus uint8 + +const ( + rebalNone rebalStatus = iota + rebalStarted + rebalCompleted + rebalStopped + rebalFailed +) + +type rebalanceInfo struct { + StartTime time.Time `msg:"startTs"` // Time at which rebalance-start was issued + EndTime time.Time `msg:"stopTs"` // Time at which rebalance operation completed or rebalance-stop was called + Status rebalStatus `msg:"status"` // Current state of rebalance operation. One of Started|Stopped|Completed|Failed. +} + +// rebalanceMeta contains information pertaining to an ongoing rebalance operation. +type rebalanceMeta struct { + StoppedAt time.Time `msg:"stopTs"` // Time when rebalance-stop was issued. + ID string `msg:"id"` // ID of the ongoing rebalance operation + PercentFreeGoal float64 `msg:"pf"` // Computed from total free space and capacity at the start of rebalance + PoolStats []*rebalanceStats `msg:"rss"` // Per-pool rebalance stats keyed by pool index +} + +var errRebalanceNotStarted = errors.New("rebalance not started") + +func (z *erasureServerPools) loadRebalanceMeta(ctx context.Context) error { + r := &rebalanceMeta{} + if err := r.load(ctx, z.serverPools[0]); err == nil { + z.rebalMu.Lock() + z.rebalMeta = r + z.updateRebalanceStats(ctx) + z.rebalMu.Unlock() + } else if !errors.Is(err, errConfigNotFound) { + rebalanceLogIf(ctx, fmt.Errorf("failed to load rebalance metadata, continue to restart rebalance as needed: %w", err)) + } + return nil +} + +// updates rebalance.bin from let's say 2 pool setup in the middle +// of a rebalance, was expanded can cause z.rebalMeta to be outdated +// due to a missing new pool. This function tries to handle this +// scenario, albeit rare it seems to have occurred in the wild. +// +// since we do not explicitly disallow it, but it is okay for them +// expand and then we continue to rebalance. +func (z *erasureServerPools) updateRebalanceStats(ctx context.Context) error { + var ok bool + for i := range z.serverPools { + if z.findIndex(i) == -1 { + // Also ensure to initialize rebalanceStats to indicate + // its a new pool that can receive rebalanced data. + z.rebalMeta.PoolStats = append(z.rebalMeta.PoolStats, &rebalanceStats{}) + ok = true + } + } + if ok { + return z.rebalMeta.save(ctx, z.serverPools[0]) + } + + return nil +} + +func (z *erasureServerPools) findIndex(index int) int { + if z.rebalMeta == nil { + return 0 + } + for i := range len(z.rebalMeta.PoolStats) { + if i == index { + return index + } + } + return -1 +} + +// initRebalanceMeta initializes rebalance metadata for a new rebalance +// operation and saves it in the object store. +func (z *erasureServerPools) initRebalanceMeta(ctx context.Context, buckets []string) (arn string, err error) { + r := &rebalanceMeta{ + ID: shortuuid.New(), + PoolStats: make([]*rebalanceStats, len(z.serverPools)), + } + + // Fetch disk capacity and available space. + si := z.StorageInfo(ctx, true) + diskStats := make([]struct { + AvailableSpace uint64 + TotalSpace uint64 + }, len(z.serverPools)) + var totalCap, totalFree uint64 + for _, disk := range si.Disks { + // Ignore invalid. + if disk.PoolIndex < 0 || len(diskStats) <= disk.PoolIndex { + // https://github.com/minio/minio/issues/16500 + continue + } + totalCap += disk.TotalSpace + totalFree += disk.AvailableSpace + + diskStats[disk.PoolIndex].AvailableSpace += disk.AvailableSpace + diskStats[disk.PoolIndex].TotalSpace += disk.TotalSpace + } + r.PercentFreeGoal = float64(totalFree) / float64(totalCap) + + now := time.Now() + for idx := range z.serverPools { + r.PoolStats[idx] = &rebalanceStats{ + Buckets: make([]string, len(buckets)), + RebalancedBuckets: make([]string, 0, len(buckets)), + InitFreeSpace: diskStats[idx].AvailableSpace, + InitCapacity: diskStats[idx].TotalSpace, + } + copy(r.PoolStats[idx].Buckets, buckets) + + if pfi := float64(diskStats[idx].AvailableSpace) / float64(diskStats[idx].TotalSpace); pfi < r.PercentFreeGoal { + r.PoolStats[idx].Participating = true + r.PoolStats[idx].Info = rebalanceInfo{ + StartTime: now, + Status: rebalStarted, + } + } + } + + err = r.save(ctx, z.serverPools[0]) + if err != nil { + return arn, err + } + + z.rebalMeta = r + return r.ID, nil +} + +func (z *erasureServerPools) updatePoolStats(poolIdx int, bucket string, fi FileInfo) { + z.rebalMu.Lock() + defer z.rebalMu.Unlock() + + r := z.rebalMeta + if r == nil { + return + } + + r.PoolStats[poolIdx].update(bucket, fi) +} + +const ( + rebalMetaName = "rebalance.bin" + rebalMetaFmt = 1 + rebalMetaVer = 1 +) + +func (z *erasureServerPools) nextRebalBucket(poolIdx int) (string, bool) { + z.rebalMu.RLock() + defer z.rebalMu.RUnlock() + + r := z.rebalMeta + if r == nil { + return "", false + } + + ps := r.PoolStats[poolIdx] + if ps == nil { + return "", false + } + + if ps.Info.Status == rebalCompleted || !ps.Participating { + return "", false + } + + if len(ps.Buckets) == 0 { + return "", false + } + + return ps.Buckets[0], true +} + +func (z *erasureServerPools) bucketRebalanceDone(bucket string, poolIdx int) { + z.rebalMu.Lock() + defer z.rebalMu.Unlock() + + if z.rebalMeta == nil { + return + } + + ps := z.rebalMeta.PoolStats[poolIdx] + if ps == nil { + return + } + + for i, b := range ps.Buckets { + if b == bucket { + ps.Buckets = append(ps.Buckets[:i], ps.Buckets[i+1:]...) + ps.RebalancedBuckets = append(ps.RebalancedBuckets, bucket) + break + } + } +} + +func (r *rebalanceMeta) load(ctx context.Context, store objectIO) error { + return r.loadWithOpts(ctx, store, ObjectOptions{}) +} + +func (r *rebalanceMeta) loadWithOpts(ctx context.Context, store objectIO, opts ObjectOptions) error { + data, _, err := readConfigWithMetadata(ctx, store, rebalMetaName, opts) + if err != nil { + return err + } + + if len(data) == 0 { + return nil + } + if len(data) <= 4 { + return fmt.Errorf("rebalanceMeta: no data") + } + + // Read header + switch binary.LittleEndian.Uint16(data[0:2]) { + case rebalMetaFmt: + default: + return fmt.Errorf("rebalanceMeta: unknown format: %d", binary.LittleEndian.Uint16(data[0:2])) + } + switch binary.LittleEndian.Uint16(data[2:4]) { + case rebalMetaVer: + default: + return fmt.Errorf("rebalanceMeta: unknown version: %d", binary.LittleEndian.Uint16(data[2:4])) + } + + // OK, parse data. + if _, err = r.UnmarshalMsg(data[4:]); err != nil { + return err + } + + return nil +} + +func (r *rebalanceMeta) saveWithOpts(ctx context.Context, store objectIO, opts ObjectOptions) error { + if r == nil { + return nil + } + + data := make([]byte, 4, r.Msgsize()+4) + + // Initialize the header. + binary.LittleEndian.PutUint16(data[0:2], rebalMetaFmt) + binary.LittleEndian.PutUint16(data[2:4], rebalMetaVer) + + buf, err := r.MarshalMsg(data) + if err != nil { + return err + } + + return saveConfigWithOpts(ctx, store, rebalMetaName, buf, opts) +} + +func (r *rebalanceMeta) save(ctx context.Context, store objectIO) error { + return r.saveWithOpts(ctx, store, ObjectOptions{}) +} + +func (z *erasureServerPools) IsRebalanceStarted(ctx context.Context) bool { + _ = z.loadRebalanceMeta(ctx) + z.rebalMu.RLock() + defer z.rebalMu.RUnlock() + + r := z.rebalMeta + if r == nil { + return false + } + if !r.StoppedAt.IsZero() { + return false + } + for _, ps := range r.PoolStats { + if ps.Participating && ps.Info.Status != rebalCompleted { + return true + } + } + return false +} + +func (z *erasureServerPools) IsPoolRebalancing(poolIndex int) bool { + z.rebalMu.RLock() + defer z.rebalMu.RUnlock() + + if r := z.rebalMeta; r != nil { + if !r.StoppedAt.IsZero() { + return false + } + ps := r.PoolStats[poolIndex] + return ps.Participating && ps.Info.Status == rebalStarted + } + return false +} + +func (z *erasureServerPools) rebalanceBuckets(ctx context.Context, poolIdx int) (err error) { + doneCh := make(chan error, 1) + defer xioutil.SafeClose(doneCh) + + // Save rebalance.bin periodically. + go func() { + // Update rebalance.bin periodically once every 5-10s, chosen randomly + // to avoid multiple pool leaders herding to update around the same + // time. + r := rand.New(rand.NewSource(time.Now().UnixNano())) + randSleepFor := func() time.Duration { + return 5*time.Second + time.Duration(float64(5*time.Second)*r.Float64()) + } + + timer := time.NewTimer(randSleepFor()) + defer timer.Stop() + + var ( + quit bool + traceMsg string + notify bool // if status changed, notify nodes to reload rebalance metadata + ) + + for { + select { + case rebalErr := <-doneCh: + quit = true + notify = true + now := time.Now() + var status rebalStatus + + switch { + case errors.Is(rebalErr, context.Canceled): + status = rebalStopped + traceMsg = fmt.Sprintf("stopped at %s", now) + case rebalErr == nil: + status = rebalCompleted + traceMsg = fmt.Sprintf("completed at %s", now) + default: + status = rebalFailed + traceMsg = fmt.Sprintf("stopped at %s with err: %v", now, rebalErr) + } + + z.rebalMu.Lock() + z.rebalMeta.PoolStats[poolIdx].Info.Status = status + z.rebalMeta.PoolStats[poolIdx].Info.EndTime = now + z.rebalMu.Unlock() + + case <-timer.C: + notify = false + traceMsg = fmt.Sprintf("saved at %s", time.Now()) + } + + stopFn := globalRebalanceMetrics.log(rebalanceMetricSaveMetadata, poolIdx, traceMsg) + err := z.saveRebalanceStats(GlobalContext, poolIdx, rebalSaveStats) + stopFn(0, err) + if err == nil && notify { + globalNotificationSys.LoadRebalanceMeta(GlobalContext, false) + } + rebalanceLogIf(GlobalContext, err) + + if quit { + return + } + + timer.Reset(randSleepFor()) + } + }() + + rebalanceLogEvent(ctx, "Pool %d rebalancing is started", poolIdx+1) + + for { + select { + case <-ctx.Done(): + doneCh <- ctx.Err() + return err + default: + } + + bucket, ok := z.nextRebalBucket(poolIdx) + if !ok { + // no more buckets to rebalance or target free_space/capacity reached + break + } + + stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceBucket, poolIdx, bucket) + if err = z.rebalanceBucket(ctx, bucket, poolIdx); err != nil { + stopFn(0, err) + if errors.Is(err, errServerNotInitialized) || errors.Is(err, errBucketMetadataNotInitialized) { + continue + } + rebalanceLogIf(GlobalContext, err) + doneCh <- err + return err + } + stopFn(0, nil) + z.bucketRebalanceDone(bucket, poolIdx) + } + + rebalanceLogEvent(GlobalContext, "Pool %d rebalancing is done", poolIdx+1) + + return err +} + +func (z *erasureServerPools) checkIfRebalanceDone(poolIdx int) bool { + z.rebalMu.Lock() + defer z.rebalMu.Unlock() + + // check if enough objects have been rebalanced + r := z.rebalMeta + poolStats := r.PoolStats[poolIdx] + if poolStats.Info.Status == rebalCompleted { + return true + } + + pfi := float64(poolStats.InitFreeSpace+poolStats.Bytes) / float64(poolStats.InitCapacity) + // Mark pool rebalance as done if within 5% from PercentFreeGoal. + if diff := math.Abs(pfi - r.PercentFreeGoal); diff <= 0.05 { + r.PoolStats[poolIdx].Info.Status = rebalCompleted + r.PoolStats[poolIdx].Info.EndTime = time.Now() + return true + } + + return false +} + +func (set *erasureObjects) listObjectsToRebalance(ctx context.Context, bucketName string, fn func(entry metaCacheEntry)) error { + disks, _ := set.getOnlineDisksWithHealing(false) + if len(disks) == 0 { + return fmt.Errorf("no online drives found for set with endpoints %s", set.getEndpoints()) + } + + // However many we ask, versions must exist on ~50% + listingQuorum := (set.setDriveCount + 1) / 2 + + // How to resolve partial results. + resolver := metadataResolutionParams{ + dirQuorum: listingQuorum, // make sure to capture all quorum ratios + objQuorum: listingQuorum, // make sure to capture all quorum ratios + bucket: bucketName, + } + + err := listPathRaw(ctx, listPathRawOptions{ + disks: disks, + bucket: bucketName, + recursive: true, + forwardTo: "", + minDisks: listingQuorum, + reportNotFound: false, + agreed: fn, + partial: func(entries metaCacheEntries, _ []error) { + entry, ok := entries.resolve(&resolver) + if ok { + fn(*entry) + } + }, + finished: nil, + }) + return err +} + +// rebalanceBucket rebalances objects under bucket in poolIdx pool +func (z *erasureServerPools) rebalanceBucket(ctx context.Context, bucket string, poolIdx int) (err error) { + ctx = logger.SetReqInfo(ctx, &logger.ReqInfo{}) + + var vc *versioning.Versioning + var lc *lifecycle.Lifecycle + var lr objectlock.Retention + var rcfg *replication.Config + if bucket != minioMetaBucket { + vc, err = globalBucketVersioningSys.Get(bucket) + if err != nil { + return err + } + + // Check if the current bucket has a configured lifecycle policy + lc, err = globalLifecycleSys.Get(bucket) + if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) { + return err + } + + // Check if bucket is object locked. + lr, err = globalBucketObjectLockSys.Get(bucket) + if err != nil { + return err + } + + rcfg, err = getReplicationConfig(ctx, bucket) + if err != nil { + return err + } + } + + pool := z.serverPools[poolIdx] + + const envRebalanceWorkers = "_MINIO_REBALANCE_WORKERS" + workerSize, err := env.GetInt(envRebalanceWorkers, len(pool.sets)) + if err != nil { + rebalanceLogIf(ctx, fmt.Errorf("invalid workers value err: %v, defaulting to %d", err, len(pool.sets))) + workerSize = len(pool.sets) + } + + // Each decom worker needs one List() goroutine/worker + // add that many extra workers. + workerSize += len(pool.sets) + + wk, err := workers.New(workerSize) + if err != nil { + return err + } + + for setIdx, set := range pool.sets { + filterLifecycle := func(bucket, object string, fi FileInfo) bool { + if lc == nil { + return false + } + versioned := vc != nil && vc.Versioned(object) + objInfo := fi.ToObjectInfo(bucket, object, versioned) + + evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo) + if evt.Action.Delete() { + globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_Rebal) + return true + } + return false + } + + rebalanceEntry := func(entry metaCacheEntry) { + defer wk.Give() + + if entry.isDir() { + return + } + + // rebalance on poolIdx has reached its goal + if z.checkIfRebalanceDone(poolIdx) { + return + } + + fivs, err := entry.fileInfoVersions(bucket) + if err != nil { + return + } + + // We need a reversed order for rebalance, + // to create the appropriate stack. + versionsSorter(fivs.Versions).reverse() + + var rebalanced, expired int + for _, version := range fivs.Versions { + stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceObject, poolIdx, bucket, version.Name, version.VersionID) + + // Skip transitioned objects for now. TBD + if version.IsRemote() { + stopFn(version.Size, errors.New("ILM Tiered version will be skipped for now")) + continue + } + + // Apply lifecycle rules on the objects that are expired. + if filterLifecycle(bucket, version.Name, version) { + expired++ + stopFn(version.Size, errors.New("ILM expired object/version will be skipped")) + continue + } + + // any object with only single DEL marker we don't need + // to rebalance, just skip it, this also includes + // any other versions that have already expired. + remainingVersions := len(fivs.Versions) - expired + if version.Deleted && remainingVersions == 1 { + rebalanced++ + stopFn(version.Size, errors.New("DELETE marked object with no other non-current versions will be skipped")) + continue + } + + versionID := version.VersionID + if versionID == "" { + versionID = nullVersionID + } + + var failure, ignore bool + if version.Deleted { + _, err := z.DeleteObject(ctx, + bucket, + version.Name, + ObjectOptions{ + Versioned: true, + VersionID: versionID, + MTime: version.ModTime, + DeleteReplication: version.ReplicationState, + SrcPoolIdx: poolIdx, + DataMovement: true, + DeleteMarker: true, // make sure we create a delete marker + SkipRebalancing: true, // make sure we skip the decommissioned pool + NoAuditLog: true, + }) + // This can happen when rebalance stop races with ongoing rebalance workers. + // These rebalance failures can be ignored. + if err != nil { + // This can happen when rebalance stop races with ongoing rebalance workers. + // These rebalance failures can be ignored. + if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) { + ignore = true + stopFn(0, nil) + continue + } + } + stopFn(version.Size, err) + rebalanceLogIf(ctx, err) + failure = err != nil + if !failure { + z.updatePoolStats(poolIdx, bucket, version) + rebalanced++ + } + auditLogRebalance(ctx, "Rebalance:DeleteMarker", bucket, version.Name, versionID, err) + continue + } + + for range 3 { + // GetObjectReader.Close is called by rebalanceObject + gr, err := set.GetObjectNInfo(ctx, + bucket, + encodeDirObject(version.Name), + nil, + http.Header{}, + ObjectOptions{ + VersionID: versionID, + NoDecryption: true, + NoLock: true, + NoAuditLog: true, + }) + if isErrObjectNotFound(err) || isErrVersionNotFound(err) { + // object deleted by the application, nothing to do here we move on. + ignore = true + stopFn(0, nil) + break + } + if err != nil { + failure = true + rebalanceLogIf(ctx, err) + stopFn(0, err) + continue + } + + if err = z.rebalanceObject(ctx, poolIdx, bucket, gr); err != nil { + // This can happen when rebalance stop races with ongoing rebalance workers. + // These rebalance failures can be ignored. + if isErrObjectNotFound(err) || isErrVersionNotFound(err) || isDataMovementOverWriteErr(err) { + ignore = true + stopFn(0, nil) + break + } + failure = true + rebalanceLogIf(ctx, err) + stopFn(version.Size, err) + continue + } + + stopFn(version.Size, nil) + failure = false + break + } + if ignore { + continue + } + if failure { + break // break out on first error + } + z.updatePoolStats(poolIdx, bucket, version) + rebalanced++ + } + + // if all versions were rebalanced, we can delete the object versions. + if rebalanced == len(fivs.Versions) { + stopFn := globalRebalanceMetrics.log(rebalanceMetricRebalanceRemoveObject, poolIdx, bucket, entry.name) + _, err := set.DeleteObject(ctx, + bucket, + encodeDirObject(entry.name), + ObjectOptions{ + DeletePrefix: true, // use prefix delete to delete all versions at once. + DeletePrefixObject: true, // use prefix delete on exact object (this is an optimization to avoid fan-out calls) + NoAuditLog: true, + }, + ) + stopFn(0, err) + auditLogRebalance(ctx, "Rebalance:DeleteObject", bucket, entry.name, "", err) + if err != nil { + rebalanceLogIf(ctx, err) + } + } + } + + wk.Take() + go func(setIdx int) { + defer wk.Give() + err := set.listObjectsToRebalance(ctx, bucket, + func(entry metaCacheEntry) { + wk.Take() + go rebalanceEntry(entry) + }, + ) + if err == nil || errors.Is(err, context.Canceled) { + return + } + setN := humanize.Ordinal(setIdx + 1) + rebalanceLogIf(ctx, fmt.Errorf("listing objects from %s set failed with %v", setN, err), "rebalance-listing-failed"+setN) + }(setIdx) + } + + wk.Wait() + return nil +} + +type rebalSaveOpts uint8 + +const ( + rebalSaveStats rebalSaveOpts = iota + rebalSaveStoppedAt +) + +func (z *erasureServerPools) saveRebalanceStats(ctx context.Context, poolIdx int, opts rebalSaveOpts) error { + lock := z.serverPools[0].NewNSLock(minioMetaBucket, rebalMetaName) + lkCtx, err := lock.GetLock(ctx, globalOperationTimeout) + if err != nil { + rebalanceLogIf(ctx, fmt.Errorf("failed to acquire write lock on %s/%s: %w", minioMetaBucket, rebalMetaName, err)) + return err + } + defer lock.Unlock(lkCtx) + + ctx = lkCtx.Context() + noLockOpts := ObjectOptions{NoLock: true} + r := &rebalanceMeta{} + err = r.loadWithOpts(ctx, z.serverPools[0], noLockOpts) + if err != nil && !errors.Is(err, errConfigNotFound) { + return err + } + + z.rebalMu.Lock() + defer z.rebalMu.Unlock() + + // if not found, we store the memory metadata back + // when rebalance status changed, will notify all nodes update status to memory, we can treat the memory metadata is the latest status + if errors.Is(err, errConfigNotFound) { + r = z.rebalMeta + } + + switch opts { + case rebalSaveStoppedAt: + r.StoppedAt = time.Now() + case rebalSaveStats: + if z.rebalMeta != nil { + r.PoolStats[poolIdx] = z.rebalMeta.PoolStats[poolIdx] + } + } + z.rebalMeta = r + + return z.rebalMeta.saveWithOpts(ctx, z.serverPools[0], noLockOpts) +} + +func auditLogRebalance(ctx context.Context, apiName, bucket, object, versionID string, err error) { + errStr := "" + if err != nil { + errStr = err.Error() + } + auditLogInternal(ctx, AuditLogOptions{ + Event: "rebalance", + APIName: apiName, + Bucket: bucket, + Object: object, + VersionID: versionID, + Error: errStr, + }) +} + +func (z *erasureServerPools) rebalanceObject(ctx context.Context, poolIdx int, bucket string, gr *GetObjectReader) (err error) { + oi := gr.ObjInfo + + defer func() { + gr.Close() + auditLogRebalance(ctx, "RebalanceCopyData", oi.Bucket, oi.Name, oi.VersionID, err) + }() + + actualSize, err := oi.GetActualSize() + if err != nil { + return err + } + + if oi.isMultipart() { + res, err := z.NewMultipartUpload(ctx, bucket, oi.Name, ObjectOptions{ + VersionID: oi.VersionID, + UserDefined: oi.UserDefined, + NoAuditLog: true, + DataMovement: true, + SrcPoolIdx: poolIdx, + }) + if err != nil { + return fmt.Errorf("rebalanceObject: NewMultipartUpload() %w", err) + } + defer z.AbortMultipartUpload(ctx, bucket, oi.Name, res.UploadID, ObjectOptions{NoAuditLog: true}) + + parts := make([]CompletePart, len(oi.Parts)) + for i, part := range oi.Parts { + hr, err := hash.NewReader(ctx, io.LimitReader(gr, part.Size), part.Size, "", "", part.ActualSize) + if err != nil { + return fmt.Errorf("rebalanceObject: hash.NewReader() %w", err) + } + pi, err := z.PutObjectPart(ctx, bucket, oi.Name, res.UploadID, + part.Number, + NewPutObjReader(hr), + ObjectOptions{ + PreserveETag: part.ETag, // Preserve original ETag to ensure same metadata. + IndexCB: func() []byte { + return part.Index // Preserve part Index to ensure decompression works. + }, + NoAuditLog: true, + }) + if err != nil { + return fmt.Errorf("rebalanceObject: PutObjectPart() %w", err) + } + parts[i] = CompletePart{ + ETag: pi.ETag, + PartNumber: pi.PartNumber, + } + } + _, err = z.CompleteMultipartUpload(ctx, bucket, oi.Name, res.UploadID, parts, ObjectOptions{ + DataMovement: true, + MTime: oi.ModTime, + NoAuditLog: true, + }) + if err != nil { + err = fmt.Errorf("rebalanceObject: CompleteMultipartUpload() %w", err) + } + return err + } + + hr, err := hash.NewReader(ctx, gr, oi.Size, "", "", actualSize) + if err != nil { + return fmt.Errorf("rebalanceObject: hash.NewReader() %w", err) + } + + _, err = z.PutObject(ctx, + bucket, + oi.Name, + NewPutObjReader(hr), + ObjectOptions{ + SrcPoolIdx: poolIdx, + DataMovement: true, + VersionID: oi.VersionID, + MTime: oi.ModTime, + UserDefined: oi.UserDefined, + PreserveETag: oi.ETag, // Preserve original ETag to ensure same metadata. + IndexCB: func() []byte { + return oi.Parts[0].Index // Preserve part Index to ensure decompression works. + }, + NoAuditLog: true, + }) + if err != nil { + err = fmt.Errorf("rebalanceObject: PutObject() %w", err) + } + return err +} + +func (z *erasureServerPools) StartRebalance() { + z.rebalMu.Lock() + if z.rebalMeta == nil || !z.rebalMeta.StoppedAt.IsZero() { // rebalance not running, nothing to do + z.rebalMu.Unlock() + return + } + ctx, cancel := context.WithCancel(GlobalContext) + z.rebalCancel = cancel // to be used when rebalance-stop is called + z.rebalMu.Unlock() + + z.rebalMu.RLock() + participants := make([]bool, len(z.rebalMeta.PoolStats)) + for i, ps := range z.rebalMeta.PoolStats { + // skip pools which have completed rebalancing + if ps.Info.Status != rebalStarted { + continue + } + + participants[i] = ps.Participating + } + z.rebalMu.RUnlock() + + for poolIdx, doRebalance := range participants { + if !doRebalance { + continue + } + // nothing to do if this node is not pool's first node (i.e pool's rebalance 'leader'). + if !globalEndpoints[poolIdx].Endpoints[0].IsLocal { + continue + } + + go func(idx int) { + stopfn := globalRebalanceMetrics.log(rebalanceMetricRebalanceBuckets, idx) + err := z.rebalanceBuckets(ctx, idx) + stopfn(0, err) + }(poolIdx) + } +} + +// StopRebalance signals the rebalance goroutine running on this node (if any) +// to stop, using the context.CancelFunc(s) saved at the time ofStartRebalance. +func (z *erasureServerPools) StopRebalance() error { + z.rebalMu.Lock() + defer z.rebalMu.Unlock() + + r := z.rebalMeta + if r == nil { // rebalance not running in this node, nothing to do + return nil + } + + if cancel := z.rebalCancel; cancel != nil { + cancel() + z.rebalCancel = nil + } + return nil +} + +// for rebalance trace support +type rebalanceMetrics struct{} + +var globalRebalanceMetrics rebalanceMetrics + +//go:generate stringer -type=rebalanceMetric -trimprefix=rebalanceMetric $GOFILE +type rebalanceMetric uint8 + +const ( + rebalanceMetricRebalanceBuckets rebalanceMetric = iota + rebalanceMetricRebalanceBucket + rebalanceMetricRebalanceObject + rebalanceMetricRebalanceRemoveObject + rebalanceMetricSaveMetadata +) + +var errDataMovementSrcDstPoolSame = errors.New("source and destination pool are the same") + +func rebalanceTrace(r rebalanceMetric, poolIdx int, startTime time.Time, duration time.Duration, err error, path string, sz int64) madmin.TraceInfo { + var errStr string + if err != nil { + errStr = err.Error() + } + return madmin.TraceInfo{ + TraceType: madmin.TraceRebalance, + Time: startTime, + NodeName: globalLocalNodeName, + FuncName: fmt.Sprintf("rebalance.%s (pool-id=%d)", r.String(), poolIdx), + Duration: duration, + Path: path, + Error: errStr, + Bytes: sz, + } +} + +func (p *rebalanceMetrics) log(r rebalanceMetric, poolIdx int, paths ...string) func(sz int64, err error) { + startTime := time.Now() + return func(sz int64, err error) { + duration := time.Since(startTime) + if globalTrace.NumSubscribers(madmin.TraceRebalance) > 0 { + globalTrace.Publish(rebalanceTrace(r, poolIdx, startTime, duration, err, strings.Join(paths, " "), sz)) + } + } +} diff --git a/cmd/erasure-server-pool-rebalance_gen.go b/cmd/erasure-server-pool-rebalance_gen.go new file mode 100644 index 00000000..842adc11 --- /dev/null +++ b/cmd/erasure-server-pool-rebalance_gen.go @@ -0,0 +1,1230 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *rebalSaveOpts) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 uint8 + zb0001, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = rebalSaveOpts(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z rebalSaveOpts) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteUint8(uint8(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z rebalSaveOpts) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendUint8(o, uint8(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *rebalSaveOpts) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 uint8 + zb0001, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = rebalSaveOpts(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z rebalSaveOpts) Msgsize() (s int) { + s = msgp.Uint8Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *rebalStatus) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 uint8 + zb0001, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = rebalStatus(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z rebalStatus) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteUint8(uint8(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z rebalStatus) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendUint8(o, uint8(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *rebalStatus) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 uint8 + zb0001, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = rebalStatus(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z rebalStatus) Msgsize() (s int) { + s = msgp.Uint8Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *rebalanceInfo) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "startTs": + z.StartTime, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "StartTime") + return + } + case "stopTs": + z.EndTime, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "EndTime") + return + } + case "status": + { + var zb0002 uint8 + zb0002, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err, "Status") + return + } + z.Status = rebalStatus(zb0002) + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z rebalanceInfo) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "startTs" + err = en.Append(0x83, 0xa7, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x73) + if err != nil { + return + } + err = en.WriteTime(z.StartTime) + if err != nil { + err = msgp.WrapError(err, "StartTime") + return + } + // write "stopTs" + err = en.Append(0xa6, 0x73, 0x74, 0x6f, 0x70, 0x54, 0x73) + if err != nil { + return + } + err = en.WriteTime(z.EndTime) + if err != nil { + err = msgp.WrapError(err, "EndTime") + return + } + // write "status" + err = en.Append(0xa6, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73) + if err != nil { + return + } + err = en.WriteUint8(uint8(z.Status)) + if err != nil { + err = msgp.WrapError(err, "Status") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z rebalanceInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "startTs" + o = append(o, 0x83, 0xa7, 0x73, 0x74, 0x61, 0x72, 0x74, 0x54, 0x73) + o = msgp.AppendTime(o, z.StartTime) + // string "stopTs" + o = append(o, 0xa6, 0x73, 0x74, 0x6f, 0x70, 0x54, 0x73) + o = msgp.AppendTime(o, z.EndTime) + // string "status" + o = append(o, 0xa6, 0x73, 0x74, 0x61, 0x74, 0x75, 0x73) + o = msgp.AppendUint8(o, uint8(z.Status)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *rebalanceInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "startTs": + z.StartTime, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "StartTime") + return + } + case "stopTs": + z.EndTime, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "EndTime") + return + } + case "status": + { + var zb0002 uint8 + zb0002, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Status") + return + } + z.Status = rebalStatus(zb0002) + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z rebalanceInfo) Msgsize() (s int) { + s = 1 + 8 + msgp.TimeSize + 7 + msgp.TimeSize + 7 + msgp.Uint8Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *rebalanceMeta) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "stopTs": + z.StoppedAt, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "StoppedAt") + return + } + case "id": + z.ID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + case "pf": + z.PercentFreeGoal, err = dc.ReadFloat64() + if err != nil { + err = msgp.WrapError(err, "PercentFreeGoal") + return + } + case "rss": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "PoolStats") + return + } + if cap(z.PoolStats) >= int(zb0002) { + z.PoolStats = (z.PoolStats)[:zb0002] + } else { + z.PoolStats = make([]*rebalanceStats, zb0002) + } + for za0001 := range z.PoolStats { + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "PoolStats", za0001) + return + } + z.PoolStats[za0001] = nil + } else { + if z.PoolStats[za0001] == nil { + z.PoolStats[za0001] = new(rebalanceStats) + } + err = z.PoolStats[za0001].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "PoolStats", za0001) + return + } + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *rebalanceMeta) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 4 + // write "stopTs" + err = en.Append(0x84, 0xa6, 0x73, 0x74, 0x6f, 0x70, 0x54, 0x73) + if err != nil { + return + } + err = en.WriteTime(z.StoppedAt) + if err != nil { + err = msgp.WrapError(err, "StoppedAt") + return + } + // write "id" + err = en.Append(0xa2, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteString(z.ID) + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + // write "pf" + err = en.Append(0xa2, 0x70, 0x66) + if err != nil { + return + } + err = en.WriteFloat64(z.PercentFreeGoal) + if err != nil { + err = msgp.WrapError(err, "PercentFreeGoal") + return + } + // write "rss" + err = en.Append(0xa3, 0x72, 0x73, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.PoolStats))) + if err != nil { + err = msgp.WrapError(err, "PoolStats") + return + } + for za0001 := range z.PoolStats { + if z.PoolStats[za0001] == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.PoolStats[za0001].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "PoolStats", za0001) + return + } + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *rebalanceMeta) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 4 + // string "stopTs" + o = append(o, 0x84, 0xa6, 0x73, 0x74, 0x6f, 0x70, 0x54, 0x73) + o = msgp.AppendTime(o, z.StoppedAt) + // string "id" + o = append(o, 0xa2, 0x69, 0x64) + o = msgp.AppendString(o, z.ID) + // string "pf" + o = append(o, 0xa2, 0x70, 0x66) + o = msgp.AppendFloat64(o, z.PercentFreeGoal) + // string "rss" + o = append(o, 0xa3, 0x72, 0x73, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.PoolStats))) + for za0001 := range z.PoolStats { + if z.PoolStats[za0001] == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.PoolStats[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "PoolStats", za0001) + return + } + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *rebalanceMeta) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "stopTs": + z.StoppedAt, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "StoppedAt") + return + } + case "id": + z.ID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + case "pf": + z.PercentFreeGoal, bts, err = msgp.ReadFloat64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "PercentFreeGoal") + return + } + case "rss": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "PoolStats") + return + } + if cap(z.PoolStats) >= int(zb0002) { + z.PoolStats = (z.PoolStats)[:zb0002] + } else { + z.PoolStats = make([]*rebalanceStats, zb0002) + } + for za0001 := range z.PoolStats { + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.PoolStats[za0001] = nil + } else { + if z.PoolStats[za0001] == nil { + z.PoolStats[za0001] = new(rebalanceStats) + } + bts, err = z.PoolStats[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "PoolStats", za0001) + return + } + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *rebalanceMeta) Msgsize() (s int) { + s = 1 + 7 + msgp.TimeSize + 3 + msgp.StringPrefixSize + len(z.ID) + 3 + msgp.Float64Size + 4 + msgp.ArrayHeaderSize + for za0001 := range z.PoolStats { + if z.PoolStats[za0001] == nil { + s += msgp.NilSize + } else { + s += z.PoolStats[za0001].Msgsize() + } + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *rebalanceMetric) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 uint8 + zb0001, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = rebalanceMetric(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z rebalanceMetric) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteUint8(uint8(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z rebalanceMetric) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendUint8(o, uint8(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *rebalanceMetric) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 uint8 + zb0001, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = rebalanceMetric(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z rebalanceMetric) Msgsize() (s int) { + s = msgp.Uint8Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *rebalanceMetrics) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z rebalanceMetrics) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 0 + _ = z + err = en.Append(0x80) + if err != nil { + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z rebalanceMetrics) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 0 + _ = z + o = append(o, 0x80) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *rebalanceMetrics) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z rebalanceMetrics) Msgsize() (s int) { + s = 1 + return +} + +// DecodeMsg implements msgp.Decodable +func (z *rebalanceStats) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ifs": + z.InitFreeSpace, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "InitFreeSpace") + return + } + case "ic": + z.InitCapacity, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "InitCapacity") + return + } + case "bus": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Buckets") + return + } + if cap(z.Buckets) >= int(zb0002) { + z.Buckets = (z.Buckets)[:zb0002] + } else { + z.Buckets = make([]string, zb0002) + } + for za0001 := range z.Buckets { + z.Buckets[za0001], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Buckets", za0001) + return + } + } + case "rbs": + var zb0003 uint32 + zb0003, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "RebalancedBuckets") + return + } + if cap(z.RebalancedBuckets) >= int(zb0003) { + z.RebalancedBuckets = (z.RebalancedBuckets)[:zb0003] + } else { + z.RebalancedBuckets = make([]string, zb0003) + } + for za0002 := range z.RebalancedBuckets { + z.RebalancedBuckets[za0002], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "RebalancedBuckets", za0002) + return + } + } + case "bu": + z.Bucket, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "ob": + z.Object, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + case "no": + z.NumObjects, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "NumObjects") + return + } + case "nv": + z.NumVersions, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "NumVersions") + return + } + case "bs": + z.Bytes, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Bytes") + return + } + case "par": + z.Participating, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Participating") + return + } + case "inf": + err = z.Info.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *rebalanceStats) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 11 + // write "ifs" + err = en.Append(0x8b, 0xa3, 0x69, 0x66, 0x73) + if err != nil { + return + } + err = en.WriteUint64(z.InitFreeSpace) + if err != nil { + err = msgp.WrapError(err, "InitFreeSpace") + return + } + // write "ic" + err = en.Append(0xa2, 0x69, 0x63) + if err != nil { + return + } + err = en.WriteUint64(z.InitCapacity) + if err != nil { + err = msgp.WrapError(err, "InitCapacity") + return + } + // write "bus" + err = en.Append(0xa3, 0x62, 0x75, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Buckets))) + if err != nil { + err = msgp.WrapError(err, "Buckets") + return + } + for za0001 := range z.Buckets { + err = en.WriteString(z.Buckets[za0001]) + if err != nil { + err = msgp.WrapError(err, "Buckets", za0001) + return + } + } + // write "rbs" + err = en.Append(0xa3, 0x72, 0x62, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.RebalancedBuckets))) + if err != nil { + err = msgp.WrapError(err, "RebalancedBuckets") + return + } + for za0002 := range z.RebalancedBuckets { + err = en.WriteString(z.RebalancedBuckets[za0002]) + if err != nil { + err = msgp.WrapError(err, "RebalancedBuckets", za0002) + return + } + } + // write "bu" + err = en.Append(0xa2, 0x62, 0x75) + if err != nil { + return + } + err = en.WriteString(z.Bucket) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + // write "ob" + err = en.Append(0xa2, 0x6f, 0x62) + if err != nil { + return + } + err = en.WriteString(z.Object) + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + // write "no" + err = en.Append(0xa2, 0x6e, 0x6f) + if err != nil { + return + } + err = en.WriteUint64(z.NumObjects) + if err != nil { + err = msgp.WrapError(err, "NumObjects") + return + } + // write "nv" + err = en.Append(0xa2, 0x6e, 0x76) + if err != nil { + return + } + err = en.WriteUint64(z.NumVersions) + if err != nil { + err = msgp.WrapError(err, "NumVersions") + return + } + // write "bs" + err = en.Append(0xa2, 0x62, 0x73) + if err != nil { + return + } + err = en.WriteUint64(z.Bytes) + if err != nil { + err = msgp.WrapError(err, "Bytes") + return + } + // write "par" + err = en.Append(0xa3, 0x70, 0x61, 0x72) + if err != nil { + return + } + err = en.WriteBool(z.Participating) + if err != nil { + err = msgp.WrapError(err, "Participating") + return + } + // write "inf" + err = en.Append(0xa3, 0x69, 0x6e, 0x66) + if err != nil { + return + } + err = z.Info.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *rebalanceStats) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 11 + // string "ifs" + o = append(o, 0x8b, 0xa3, 0x69, 0x66, 0x73) + o = msgp.AppendUint64(o, z.InitFreeSpace) + // string "ic" + o = append(o, 0xa2, 0x69, 0x63) + o = msgp.AppendUint64(o, z.InitCapacity) + // string "bus" + o = append(o, 0xa3, 0x62, 0x75, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Buckets))) + for za0001 := range z.Buckets { + o = msgp.AppendString(o, z.Buckets[za0001]) + } + // string "rbs" + o = append(o, 0xa3, 0x72, 0x62, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.RebalancedBuckets))) + for za0002 := range z.RebalancedBuckets { + o = msgp.AppendString(o, z.RebalancedBuckets[za0002]) + } + // string "bu" + o = append(o, 0xa2, 0x62, 0x75) + o = msgp.AppendString(o, z.Bucket) + // string "ob" + o = append(o, 0xa2, 0x6f, 0x62) + o = msgp.AppendString(o, z.Object) + // string "no" + o = append(o, 0xa2, 0x6e, 0x6f) + o = msgp.AppendUint64(o, z.NumObjects) + // string "nv" + o = append(o, 0xa2, 0x6e, 0x76) + o = msgp.AppendUint64(o, z.NumVersions) + // string "bs" + o = append(o, 0xa2, 0x62, 0x73) + o = msgp.AppendUint64(o, z.Bytes) + // string "par" + o = append(o, 0xa3, 0x70, 0x61, 0x72) + o = msgp.AppendBool(o, z.Participating) + // string "inf" + o = append(o, 0xa3, 0x69, 0x6e, 0x66) + o, err = z.Info.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *rebalanceStats) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ifs": + z.InitFreeSpace, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "InitFreeSpace") + return + } + case "ic": + z.InitCapacity, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "InitCapacity") + return + } + case "bus": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Buckets") + return + } + if cap(z.Buckets) >= int(zb0002) { + z.Buckets = (z.Buckets)[:zb0002] + } else { + z.Buckets = make([]string, zb0002) + } + for za0001 := range z.Buckets { + z.Buckets[za0001], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Buckets", za0001) + return + } + } + case "rbs": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "RebalancedBuckets") + return + } + if cap(z.RebalancedBuckets) >= int(zb0003) { + z.RebalancedBuckets = (z.RebalancedBuckets)[:zb0003] + } else { + z.RebalancedBuckets = make([]string, zb0003) + } + for za0002 := range z.RebalancedBuckets { + z.RebalancedBuckets[za0002], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "RebalancedBuckets", za0002) + return + } + } + case "bu": + z.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "ob": + z.Object, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + case "no": + z.NumObjects, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "NumObjects") + return + } + case "nv": + z.NumVersions, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "NumVersions") + return + } + case "bs": + z.Bytes, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bytes") + return + } + case "par": + z.Participating, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Participating") + return + } + case "inf": + bts, err = z.Info.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Info") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *rebalanceStats) Msgsize() (s int) { + s = 1 + 4 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.ArrayHeaderSize + for za0001 := range z.Buckets { + s += msgp.StringPrefixSize + len(z.Buckets[za0001]) + } + s += 4 + msgp.ArrayHeaderSize + for za0002 := range z.RebalancedBuckets { + s += msgp.StringPrefixSize + len(z.RebalancedBuckets[za0002]) + } + s += 3 + msgp.StringPrefixSize + len(z.Bucket) + 3 + msgp.StringPrefixSize + len(z.Object) + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 3 + msgp.Uint64Size + 4 + msgp.BoolSize + 4 + z.Info.Msgsize() + return +} + +// DecodeMsg implements msgp.Decodable +func (z *rstats) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if cap((*z)) >= int(zb0002) { + (*z) = (*z)[:zb0002] + } else { + (*z) = make(rstats, zb0002) + } + for zb0001 := range *z { + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, zb0001) + return + } + (*z)[zb0001] = nil + } else { + if (*z)[zb0001] == nil { + (*z)[zb0001] = new(rebalanceStats) + } + err = (*z)[zb0001].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, zb0001) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z rstats) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteArrayHeader(uint32(len(z))) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0003 := range z { + if z[zb0003] == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z[zb0003].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, zb0003) + return + } + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z rstats) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendArrayHeader(o, uint32(len(z))) + for zb0003 := range z { + if z[zb0003] == nil { + o = msgp.AppendNil(o) + } else { + o, err = z[zb0003].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, zb0003) + return + } + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *rstats) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if cap((*z)) >= int(zb0002) { + (*z) = (*z)[:zb0002] + } else { + (*z) = make(rstats, zb0002) + } + for zb0001 := range *z { + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + (*z)[zb0001] = nil + } else { + if (*z)[zb0001] == nil { + (*z)[zb0001] = new(rebalanceStats) + } + bts, err = (*z)[zb0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, zb0001) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z rstats) Msgsize() (s int) { + s = msgp.ArrayHeaderSize + for zb0003 := range z { + if z[zb0003] == nil { + s += msgp.NilSize + } else { + s += z[zb0003].Msgsize() + } + } + return +} diff --git a/cmd/erasure-server-pool-rebalance_gen_test.go b/cmd/erasure-server-pool-rebalance_gen_test.go new file mode 100644 index 00000000..4365ccfa --- /dev/null +++ b/cmd/erasure-server-pool-rebalance_gen_test.go @@ -0,0 +1,575 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalrebalanceInfo(t *testing.T) { + v := rebalanceInfo{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgrebalanceInfo(b *testing.B) { + v := rebalanceInfo{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgrebalanceInfo(b *testing.B) { + v := rebalanceInfo{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalrebalanceInfo(b *testing.B) { + v := rebalanceInfo{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecoderebalanceInfo(t *testing.T) { + v := rebalanceInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecoderebalanceInfo Msgsize() is inaccurate") + } + + vn := rebalanceInfo{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncoderebalanceInfo(b *testing.B) { + v := rebalanceInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecoderebalanceInfo(b *testing.B) { + v := rebalanceInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalrebalanceMeta(t *testing.T) { + v := rebalanceMeta{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgrebalanceMeta(b *testing.B) { + v := rebalanceMeta{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgrebalanceMeta(b *testing.B) { + v := rebalanceMeta{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalrebalanceMeta(b *testing.B) { + v := rebalanceMeta{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecoderebalanceMeta(t *testing.T) { + v := rebalanceMeta{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecoderebalanceMeta Msgsize() is inaccurate") + } + + vn := rebalanceMeta{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncoderebalanceMeta(b *testing.B) { + v := rebalanceMeta{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecoderebalanceMeta(b *testing.B) { + v := rebalanceMeta{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalrebalanceMetrics(t *testing.T) { + v := rebalanceMetrics{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgrebalanceMetrics(b *testing.B) { + v := rebalanceMetrics{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgrebalanceMetrics(b *testing.B) { + v := rebalanceMetrics{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalrebalanceMetrics(b *testing.B) { + v := rebalanceMetrics{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecoderebalanceMetrics(t *testing.T) { + v := rebalanceMetrics{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecoderebalanceMetrics Msgsize() is inaccurate") + } + + vn := rebalanceMetrics{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncoderebalanceMetrics(b *testing.B) { + v := rebalanceMetrics{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecoderebalanceMetrics(b *testing.B) { + v := rebalanceMetrics{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalrebalanceStats(t *testing.T) { + v := rebalanceStats{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgrebalanceStats(b *testing.B) { + v := rebalanceStats{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgrebalanceStats(b *testing.B) { + v := rebalanceStats{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalrebalanceStats(b *testing.B) { + v := rebalanceStats{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecoderebalanceStats(t *testing.T) { + v := rebalanceStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecoderebalanceStats Msgsize() is inaccurate") + } + + vn := rebalanceStats{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncoderebalanceStats(b *testing.B) { + v := rebalanceStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecoderebalanceStats(b *testing.B) { + v := rebalanceStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalrstats(t *testing.T) { + v := rstats{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgrstats(b *testing.B) { + v := rstats{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgrstats(b *testing.B) { + v := rstats{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalrstats(b *testing.B) { + v := rstats{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecoderstats(t *testing.T) { + v := rstats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecoderstats Msgsize() is inaccurate") + } + + vn := rstats{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncoderstats(b *testing.B) { + v := rstats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecoderstats(b *testing.B) { + v := rstats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/erasure-server-pool.go b/cmd/erasure-server-pool.go new file mode 100644 index 00000000..6b12ec4f --- /dev/null +++ b/cmd/erasure-server-pool.go @@ -0,0 +1,3005 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "math/rand" + "net/http" + "path" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/dustin/go-humanize" + "github.com/google/uuid" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio-go/v7/pkg/tags" + "github.com/minio/minio/internal/bpool" + "github.com/minio/minio/internal/cachevalue" + "github.com/minio/minio/internal/config/storageclass" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/sync/errgroup" + "github.com/minio/pkg/v3/wildcard" + "github.com/minio/pkg/v3/workers" + "github.com/puzpuzpuz/xsync/v3" +) + +type erasureServerPools struct { + poolMetaMutex sync.RWMutex + poolMeta poolMeta + + rebalMu sync.RWMutex + rebalMeta *rebalanceMeta + rebalCancel context.CancelFunc + + deploymentID [16]byte + distributionAlgo string + + serverPools []*erasureSets + + // Active decommission canceler + decommissionCancelers []context.CancelFunc + + s3Peer *S3PeerSys + + mpCache *xsync.MapOf[string, MultipartInfo] +} + +func (z *erasureServerPools) SinglePool() bool { + return len(z.serverPools) == 1 +} + +// Initialize new pool of erasure sets. +func newErasureServerPools(ctx context.Context, endpointServerPools EndpointServerPools) (ObjectLayer, error) { + var ( + deploymentID string + commonParityDrives int + err error + + formats = make([]*formatErasureV3, len(endpointServerPools)) + storageDisks = make([][]StorageAPI, len(endpointServerPools)) + z = &erasureServerPools{ + serverPools: make([]*erasureSets, len(endpointServerPools)), + s3Peer: NewS3PeerSys(endpointServerPools), + distributionAlgo: formatErasureVersionV3DistributionAlgoV3, + } + ) + + // Maximum number of reusable buffers per node at any given point in time. + n := uint64(1024) // single node single/multiple drives set this to 1024 entries + + if globalIsDistErasure { + n = 2048 + } + + // Avoid allocating more than half of the available memory + if maxN := availableMemory() / (blockSizeV2 * 2); n > maxN { + n = maxN + } + + if globalIsCICD || strconv.IntSize == 32 { + n = 256 // 256MiB for CI/CD environments is sufficient or on 32bit platforms. + } + + // Initialize byte pool once for all sets, bpool size is set to + // setCount * setDriveCount with each memory upto blockSizeV2. + buffers := bpool.NewBytePoolCap(n, blockSizeV2, blockSizeV2*2) + if n >= 16384 { + // pre-populate buffers only n >= 16384 which is (32Gi/2Mi) + // for all setups smaller than this avoid pre-alloc. + buffers.Populate() + } + globalBytePoolCap.Store(buffers) + + var localDrives []StorageAPI + local := endpointServerPools.FirstLocal() + for i, ep := range endpointServerPools { + // If storage class is not set during startup, default values are used + // -- Default for Reduced Redundancy Storage class is, parity = 2 + // -- Default for Standard Storage class is, parity = 2 - disks 4, 5 + // -- Default for Standard Storage class is, parity = 3 - disks 6, 7 + // -- Default for Standard Storage class is, parity = 4 - disks 8 to 16 + if commonParityDrives == 0 { + commonParityDrives, err = ecDrivesNoConfig(ep.DrivesPerSet) + if err != nil { + return nil, err + } + } + + if err = storageclass.ValidateParity(commonParityDrives, ep.DrivesPerSet); err != nil { + return nil, fmt.Errorf("parity validation returned an error: %w <- (%d, %d), for pool(%s)", err, commonParityDrives, ep.DrivesPerSet, humanize.Ordinal(i+1)) + } + + bootstrapTrace("waitForFormatErasure: loading disks", func() { + storageDisks[i], formats[i], err = waitForFormatErasure(local, ep.Endpoints, i+1, + ep.SetCount, ep.DrivesPerSet, deploymentID) + }) + if err != nil { + return nil, err + } + + if deploymentID == "" { + // all pools should have same deployment ID + deploymentID = formats[i].ID + } + + // Validate if users brought different DeploymentID pools. + if deploymentID != formats[i].ID { + return nil, fmt.Errorf("all pools must have same deployment ID - expected %s, got %s for pool(%s)", deploymentID, formats[i].ID, humanize.Ordinal(i+1)) + } + + bootstrapTrace(fmt.Sprintf("newErasureSets: initializing %s pool", humanize.Ordinal(i+1)), func() { + z.serverPools[i], err = newErasureSets(ctx, ep, storageDisks[i], formats[i], commonParityDrives, i) + }) + if err != nil { + return nil, err + } + + if deploymentID != "" && bytes.Equal(z.deploymentID[:], []byte{}) { + z.deploymentID = uuid.MustParse(deploymentID) + } + + for _, storageDisk := range storageDisks[i] { + if storageDisk != nil && storageDisk.IsLocal() { + localDrives = append(localDrives, storageDisk) + } + } + } + + if !globalIsDistErasure { + globalLocalDrivesMu.Lock() + globalLocalDrivesMap = make(map[string]StorageAPI, len(localDrives)) + for _, drive := range localDrives { + globalLocalDrivesMap[drive.Endpoint().String()] = drive + } + globalLocalDrivesMu.Unlock() + } + + z.decommissionCancelers = make([]context.CancelFunc, len(z.serverPools)) + + // Initialize the pool meta, but set it to not save. + // When z.Init below has loaded the poolmeta will be initialized, + // and allowed to save. + z.poolMeta = newPoolMeta(z, poolMeta{}) + z.poolMeta.dontSave = true + + bootstrapTrace("newSharedLock", func() { + globalLeaderLock = newSharedLock(GlobalContext, z, "leader.lock") + }) + + // Start self healing after the object initialization + // so various tasks will be useful + bootstrapTrace("initAutoHeal", func() { + initAutoHeal(GlobalContext, z) + }) + + // initialize the object layer. + defer setObjectLayer(z) + + r := rand.New(rand.NewSource(time.Now().UnixNano())) + attempt := 1 + for { + var err error + bootstrapTrace(fmt.Sprintf("poolMeta.Init: loading pool metadata, attempt: %d", attempt), func() { + err = z.Init(ctx) // Initializes all pools. + }) + if err != nil { + if !configRetriableErrors(err) { + logger.Fatal(err, "Unable to initialize backend") + } + retry := time.Duration(r.Float64() * float64(5*time.Second)) + storageLogIf(ctx, fmt.Errorf("Unable to initialize backend: %w, retrying in %s", err, retry)) + time.Sleep(retry) + attempt++ + continue + } + break + } + + // initialize the incomplete uploads cache + z.mpCache = xsync.NewMapOf[string, MultipartInfo]() + + go z.cleanupStaleMPCache(ctx) + + return z, nil +} + +func (z *erasureServerPools) cleanupStaleMPCache(ctx context.Context) { + timer := time.NewTimer(globalAPIConfig.getStaleUploadsCleanupInterval()) + defer timer.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-timer.C: + z.mpCache.Range(func(id string, info MultipartInfo) bool { + if time.Since(info.Initiated) >= globalAPIConfig.getStaleUploadsExpiry() { + z.mpCache.Delete(id) + // No need to notify to peers, each node will delete its own cache. + } + return true + }) + + // Reset for the next interval + timer.Reset(globalAPIConfig.getStaleUploadsCleanupInterval()) + } + } +} + +func (z *erasureServerPools) NewNSLock(bucket string, objects ...string) RWLocker { + return z.serverPools[0].NewNSLock(bucket, objects...) +} + +// GetDisksID will return disks by their ID. +func (z *erasureServerPools) GetDisksID(ids ...string) []StorageAPI { + idMap := make(map[string]struct{}, len(ids)) + for _, id := range ids { + idMap[id] = struct{}{} + } + res := make([]StorageAPI, 0, len(idMap)) + for _, s := range z.serverPools { + for _, set := range s.sets { + for _, disk := range set.getDisks() { + if disk == OfflineDisk { + continue + } + if id, _ := disk.GetDiskID(); id != "" { + if _, ok := idMap[id]; ok { + res = append(res, disk) + } + } + } + } + } + return res +} + +// GetRawData will return all files with a given raw path to the callback. +// Errors are ignored, only errors from the callback are returned. +// For now only direct file paths are supported. +func (z *erasureServerPools) GetRawData(ctx context.Context, volume, file string, fn func(r io.Reader, host string, disk string, filename string, info StatInfo) error) error { + found := 0 + for _, s := range z.serverPools { + for _, set := range s.sets { + for _, disk := range set.getDisks() { + if disk == OfflineDisk { + continue + } + stats, err := disk.StatInfoFile(ctx, volume, file, true) + if err != nil { + continue + } + for _, si := range stats { + found++ + var r io.ReadCloser + if !si.Dir { + r, err = disk.ReadFileStream(ctx, volume, si.Name, 0, si.Size) + if err != nil { + continue + } + } else { + r = io.NopCloser(bytes.NewBuffer([]byte{})) + } + // Keep disk path instead of ID, to ensure that the downloaded zip file can be + // easily automated with `minio server hostname{1...n}/disk{1...m}`. + err = fn(r, disk.Hostname(), disk.Endpoint().Path, pathJoin(volume, si.Name), si) + r.Close() + if err != nil { + return err + } + } + } + } + } + if found == 0 { + return errFileNotFound + } + return nil +} + +// Return the disks belonging to the poolIdx, and setIdx. +func (z *erasureServerPools) GetDisks(poolIdx, setIdx int) ([]StorageAPI, error) { + if poolIdx < len(z.serverPools) && setIdx < len(z.serverPools[poolIdx].sets) { + return z.serverPools[poolIdx].sets[setIdx].getDisks(), nil + } + return nil, fmt.Errorf("Matching pool %s, set %s not found", humanize.Ordinal(poolIdx+1), humanize.Ordinal(setIdx+1)) +} + +// Return the count of disks in each pool +func (z *erasureServerPools) SetDriveCounts() []int { + setDriveCounts := make([]int, len(z.serverPools)) + for i := range z.serverPools { + setDriveCounts[i] = z.serverPools[i].SetDriveCount() + } + return setDriveCounts +} + +type serverPoolsAvailableSpace []poolAvailableSpace + +type poolAvailableSpace struct { + Index int + Available uint64 // in bytes + MaxUsedPct int // Used disk percentage of most filled disk, rounded down. +} + +// TotalAvailable - total available space +func (p serverPoolsAvailableSpace) TotalAvailable() uint64 { + total := uint64(0) + for _, z := range p { + total += z.Available + } + return total +} + +// FilterMaxUsed will filter out any pools that has used percent bigger than max, +// unless all have that, in which case all are preserved. +func (p serverPoolsAvailableSpace) FilterMaxUsed(maxUsed int) { + // We aren't modifying p, only entries in it, so we don't need to receive a pointer. + if len(p) <= 1 { + // Nothing to do. + return + } + var ok bool + for _, z := range p { + if z.Available > 0 && z.MaxUsedPct < maxUsed { + ok = true + break + } + } + if !ok { + // All above limit. + // Do not modify + return + } + + // Remove entries that are above. + for i, z := range p { + if z.Available > 0 && z.MaxUsedPct < maxUsed { + continue + } + p[i].Available = 0 + } +} + +// getAvailablePoolIdx will return an index that can hold size bytes. +// -1 is returned if no serverPools have available space for the size given. +func (z *erasureServerPools) getAvailablePoolIdx(ctx context.Context, bucket, object string, size int64) int { + serverPools := z.getServerPoolsAvailableSpace(ctx, bucket, object, size) + serverPools.FilterMaxUsed(100 - (100 * diskReserveFraction)) + total := serverPools.TotalAvailable() + if total == 0 { + return -1 + } + // choose when we reach this many + choose := rand.Uint64() % total + atTotal := uint64(0) + for _, pool := range serverPools { + atTotal += pool.Available + if atTotal > choose && pool.Available > 0 { + return pool.Index + } + } + // Should not happen, but print values just in case. + storageLogIf(ctx, fmt.Errorf("reached end of serverPools (total: %v, atTotal: %v, choose: %v)", total, atTotal, choose)) + return -1 +} + +// getServerPoolsAvailableSpace will return the available space of each pool after storing the content. +// If there is not enough space the pool will return 0 bytes available. +// The size of each will be multiplied by the number of sets. +// Negative sizes are seen as 0 bytes. +func (z *erasureServerPools) getServerPoolsAvailableSpace(ctx context.Context, bucket, object string, size int64) serverPoolsAvailableSpace { + serverPools := make(serverPoolsAvailableSpace, len(z.serverPools)) + + storageInfos := make([][]*DiskInfo, len(z.serverPools)) + nSets := make([]int, len(z.serverPools)) + g := errgroup.WithNErrs(len(z.serverPools)) + for index := range z.serverPools { + // Skip suspended pools or pools participating in rebalance for any new + // I/O. + if z.IsSuspended(index) || z.IsPoolRebalancing(index) { + continue + } + pool := z.serverPools[index] + nSets[index] = pool.setCount + g.Go(func() error { + // Get the set where it would be placed. + storageInfos[index] = getDiskInfos(ctx, pool.getHashedSet(object).getDisks()...) + return nil + }, index) + } + + // Wait for the go routines. + g.Wait() + + for i, zinfo := range storageInfos { + if zinfo == nil { + serverPools[i] = poolAvailableSpace{Index: i} + continue + } + var available uint64 + if !isMinioMetaBucketName(bucket) { + if avail, err := hasSpaceFor(zinfo, size); err != nil || !avail { + serverPools[i] = poolAvailableSpace{Index: i} + continue + } + } + var maxUsedPct int + for _, disk := range zinfo { + if disk == nil || disk.Total == 0 { + continue + } + available += disk.Total - disk.Used + + // set maxUsedPct to the value from the disk with the least space percentage. + if pctUsed := int(disk.Used * 100 / disk.Total); pctUsed > maxUsedPct { + maxUsedPct = pctUsed + } + } + + // Since we are comparing pools that may have a different number of sets + // we multiply by the number of sets in the pool. + // This will compensate for differences in set sizes + // when choosing destination pool. + // Different set sizes are already compensated by less disks. + available *= uint64(nSets[i]) + + serverPools[i] = poolAvailableSpace{ + Index: i, + Available: available, + MaxUsedPct: maxUsedPct, + } + } + return serverPools +} + +// PoolObjInfo represents the state of current object version per pool +type PoolObjInfo struct { + Index int + ObjInfo ObjectInfo + Err error +} + +type poolErrs struct { + Index int + Err error +} + +func (z *erasureServerPools) getPoolInfoExistingWithOpts(ctx context.Context, bucket, object string, opts ObjectOptions) (PoolObjInfo, []poolErrs, error) { + var noReadQuorumPools []poolErrs + poolObjInfos := make([]PoolObjInfo, len(z.serverPools)) + poolOpts := make([]ObjectOptions, len(z.serverPools)) + for i := range z.serverPools { + poolOpts[i] = opts + } + + var wg sync.WaitGroup + for i, pool := range z.serverPools { + wg.Add(1) + go func(i int, pool *erasureSets, opts ObjectOptions) { + defer wg.Done() + // remember the pool index, we may sort the slice original index might be lost. + pinfo := PoolObjInfo{ + Index: i, + } + // do not remove this check as it can lead to inconsistencies + // for all callers of bucket replication. + if !opts.MetadataChg { + opts.VersionID = "" + } + pinfo.ObjInfo, pinfo.Err = pool.GetObjectInfo(ctx, bucket, object, opts) + poolObjInfos[i] = pinfo + }(i, pool, poolOpts[i]) + } + wg.Wait() + + // Sort the objInfos such that we always serve latest + // this is a defensive change to handle any duplicate + // content that may have been created, we always serve + // the latest object. + sort.Slice(poolObjInfos, func(i, j int) bool { + mtime1 := poolObjInfos[i].ObjInfo.ModTime + mtime2 := poolObjInfos[j].ObjInfo.ModTime + return mtime1.After(mtime2) + }) + + defPool := PoolObjInfo{Index: -1} + for _, pinfo := range poolObjInfos { + // skip all objects from suspended pools if asked by the + // caller. + if opts.SkipDecommissioned && z.IsSuspended(pinfo.Index) { + continue + } + // Skip object if it's from pools participating in a rebalance operation. + if opts.SkipRebalancing && z.IsPoolRebalancing(pinfo.Index) { + continue + } + if pinfo.Err == nil { + // found a pool + return pinfo, z.poolsWithObject(poolObjInfos, opts), nil + } + + if isErrReadQuorum(pinfo.Err) && !opts.MetadataChg { + // read quorum is returned when the object is visibly + // present but its unreadable, we simply ask the writes to + // schedule to this pool instead. If there is no quorum + // it will fail anyways, however if there is quorum available + // with enough disks online but sufficiently inconsistent to + // break parity threshold, allow them to be overwritten + // or allow new versions to be added. + + return pinfo, z.poolsWithObject(poolObjInfos, opts), nil + } + defPool = pinfo + if !isErrObjectNotFound(pinfo.Err) && !isErrVersionNotFound(pinfo.Err) { + return pinfo, noReadQuorumPools, pinfo.Err + } + + // No object exists or its a delete marker, + // check objInfo to confirm. + if pinfo.ObjInfo.DeleteMarker && pinfo.ObjInfo.Name != "" { + return pinfo, noReadQuorumPools, nil + } + } + if opts.ReplicationRequest && opts.DeleteMarker && defPool.Index >= 0 { + // If the request is a delete marker replication request, return a default pool + // in cases where the object does not exist. + // This is to ensure that the delete marker is replicated to the destination. + return defPool, noReadQuorumPools, nil + } + return PoolObjInfo{}, noReadQuorumPools, toObjectErr(errFileNotFound, bucket, object) +} + +// return all pools with read quorum error or no error for an object with given opts.Note that this array is +// returned in the order of recency of object ModTime. +func (z *erasureServerPools) poolsWithObject(pools []PoolObjInfo, opts ObjectOptions) (errs []poolErrs) { + for _, pool := range pools { + if opts.SkipDecommissioned && z.IsSuspended(pool.Index) { + continue + } + // Skip object if it's from pools participating in a rebalance operation. + if opts.SkipRebalancing && z.IsPoolRebalancing(pool.Index) { + continue + } + if isErrReadQuorum(pool.Err) || pool.Err == nil { + errs = append(errs, poolErrs{Err: pool.Err, Index: pool.Index}) + } + } + return errs +} + +func (z *erasureServerPools) getPoolIdxExistingWithOpts(ctx context.Context, bucket, object string, opts ObjectOptions) (idx int, err error) { + pinfo, _, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, opts) + if err != nil { + return -1, err + } + return pinfo.Index, nil +} + +// getPoolIdxExistingNoLock returns the (first) found object pool index containing an object. +// If the object exists, but the latest version is a delete marker, the index with it is still returned. +// If the object does not exist ObjectNotFound error is returned. +// If any other error is found, it is returned. +// The check is skipped if there is only one pool, and 0, nil is always returned in that case. +func (z *erasureServerPools) getPoolIdxExistingNoLock(ctx context.Context, bucket, object string) (idx int, err error) { + return z.getPoolIdxExistingWithOpts(ctx, bucket, object, ObjectOptions{ + NoLock: true, + SkipDecommissioned: true, + SkipRebalancing: true, + }) +} + +func (z *erasureServerPools) getPoolIdxNoLock(ctx context.Context, bucket, object string, size int64) (idx int, err error) { + idx, err = z.getPoolIdxExistingNoLock(ctx, bucket, object) + if err != nil && !isErrObjectNotFound(err) { + return idx, err + } + + if isErrObjectNotFound(err) { + idx = z.getAvailablePoolIdx(ctx, bucket, object, size) + if idx < 0 { + return -1, toObjectErr(errDiskFull) + } + } + + return idx, nil +} + +// getPoolIdx returns the found previous object and its corresponding pool idx, +// if none are found falls back to most available space pool, this function is +// designed to be only used by PutObject, CopyObject (newObject creation) and NewMultipartUpload. +func (z *erasureServerPools) getPoolIdx(ctx context.Context, bucket, object string, size int64) (idx int, err error) { + pinfo, _, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, ObjectOptions{ + SkipDecommissioned: true, + SkipRebalancing: true, + }) + + if err != nil && !isErrObjectNotFound(err) { + return -1, err + } + + idx = pinfo.Index + if isErrObjectNotFound(err) || pinfo.Err == nil { + // will generate a temp object + idx = z.getAvailablePoolIdx(ctx, bucket, object, size) + if idx < 0 { + return -1, toObjectErr(errDiskFull) + } + } + + return idx, nil +} + +func (z *erasureServerPools) Shutdown(ctx context.Context) error { + g := errgroup.WithNErrs(len(z.serverPools)) + + for index := range z.serverPools { + g.Go(func() error { + return z.serverPools[index].Shutdown(ctx) + }, index) + } + + for _, err := range g.Wait() { + if err != nil { + storageLogIf(ctx, err) + } + // let's the rest shutdown + } + return nil +} + +// Legacy returns 'true' if distribution algo is CRCMOD +func (z *erasureServerPools) Legacy() (ok bool) { + ok = true + for _, set := range z.serverPools { + ok = ok && set.Legacy() + } + return ok +} + +func (z *erasureServerPools) BackendInfo() (b madmin.BackendInfo) { + b.Type = madmin.Erasure + + scParity := globalStorageClass.GetParityForSC(storageclass.STANDARD) + if scParity < 0 { + scParity = z.serverPools[0].defaultParityCount + } + rrSCParity := globalStorageClass.GetParityForSC(storageclass.RRS) + + // Data blocks can vary per pool, but parity is same. + for i, setDriveCount := range z.SetDriveCounts() { + b.StandardSCData = append(b.StandardSCData, setDriveCount-scParity) + b.RRSCData = append(b.RRSCData, setDriveCount-rrSCParity) + b.DrivesPerSet = append(b.DrivesPerSet, setDriveCount) + b.TotalSets = append(b.TotalSets, z.serverPools[i].setCount) + } + + b.StandardSCParity = scParity + b.RRSCParity = rrSCParity + return b +} + +func (z *erasureServerPools) LocalStorageInfo(ctx context.Context, metrics bool) StorageInfo { + var storageInfo StorageInfo + + storageInfos := make([]StorageInfo, len(z.serverPools)) + g := errgroup.WithNErrs(len(z.serverPools)) + for index := range z.serverPools { + g.Go(func() error { + storageInfos[index] = z.serverPools[index].LocalStorageInfo(ctx, metrics) + return nil + }, index) + } + + // Wait for the go routines. + g.Wait() + + storageInfo.Backend = z.BackendInfo() + for _, lstorageInfo := range storageInfos { + storageInfo.Disks = append(storageInfo.Disks, lstorageInfo.Disks...) + } + + return storageInfo +} + +func (z *erasureServerPools) StorageInfo(ctx context.Context, metrics bool) StorageInfo { + return globalNotificationSys.StorageInfo(ctx, z, metrics) +} + +func (z *erasureServerPools) NSScanner(ctx context.Context, updates chan<- DataUsageInfo, wantCycle uint32, healScanMode madmin.HealScanMode) error { + // Updates must be closed before we return. + defer xioutil.SafeClose(updates) + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + var wg sync.WaitGroup + var mu sync.Mutex + var results []dataUsageCache + var firstErr error + + allBuckets, err := z.ListBuckets(ctx, BucketOptions{}) + if err != nil { + return err + } + + if len(allBuckets) == 0 { + updates <- DataUsageInfo{} // no buckets found update data usage to reflect latest state + return nil + } + totalResults := 0 + resultIndex := -1 + for _, z := range z.serverPools { + totalResults += len(z.sets) + } + results = make([]dataUsageCache, totalResults) + // Collect for each set in serverPools. + for _, z := range z.serverPools { + for _, erObj := range z.sets { + resultIndex++ + wg.Add(1) + go func(i int, erObj *erasureObjects) { + updates := make(chan dataUsageCache, 1) + defer xioutil.SafeClose(updates) + // Start update collector. + go func() { + defer wg.Done() + for info := range updates { + mu.Lock() + results[i] = info + mu.Unlock() + } + }() + // Start scanner. Blocks until done. + err := erObj.nsScanner(ctx, allBuckets, wantCycle, updates, healScanMode) + if err != nil { + scannerLogIf(ctx, err) + mu.Lock() + if firstErr == nil { + firstErr = err + } + // Cancel remaining... + cancel() + mu.Unlock() + return + } + }(resultIndex, erObj) + } + } + updateCloser := make(chan chan struct{}) + go func() { + updateTicker := time.NewTicker(30 * time.Second) + defer updateTicker.Stop() + var lastUpdate time.Time + + // We need to merge since we will get the same buckets from each pool. + // Therefore to get the exact bucket sizes we must merge before we can convert. + var allMerged dataUsageCache + + update := func() { + mu.Lock() + defer mu.Unlock() + + allMerged = dataUsageCache{Info: dataUsageCacheInfo{Name: dataUsageRoot}} + for _, info := range results { + if info.Info.LastUpdate.IsZero() { + // Not filled yet. + return + } + allMerged.merge(info) + } + if allMerged.root() != nil && allMerged.Info.LastUpdate.After(lastUpdate) { + updates <- allMerged.dui(allMerged.Info.Name, allBuckets) + lastUpdate = allMerged.Info.LastUpdate + } + } + for { + select { + case <-ctx.Done(): + return + case v := <-updateCloser: + update() + xioutil.SafeClose(v) + return + case <-updateTicker.C: + update() + } + } + }() + + wg.Wait() + ch := make(chan struct{}) + select { + case updateCloser <- ch: + <-ch + case <-ctx.Done(): + mu.Lock() + if firstErr == nil { + firstErr = ctx.Err() + } + defer mu.Unlock() + } + return firstErr +} + +// MakeBucket - creates a new bucket across all serverPools simultaneously +// even if one of the sets fail to create buckets, we proceed all the successful +// operations. +func (z *erasureServerPools) MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error { + // Verify if bucket is valid. + if !isMinioMetaBucketName(bucket) { + if err := s3utils.CheckValidBucketNameStrict(bucket); err != nil { + return BucketNameInvalid{Bucket: bucket} + } + + if !opts.NoLock { + // Lock the bucket name before creating. + lk := z.NewNSLock(minioMetaTmpBucket, bucket+".lck") + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + return err + } + + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + } + } + + if err := z.s3Peer.MakeBucket(ctx, bucket, opts); err != nil { + if _, ok := err.(BucketExists); !ok { + // Delete created buckets, ignoring errors. + z.DeleteBucket(context.Background(), bucket, DeleteBucketOptions{ + NoLock: true, + NoRecreate: true, + }) + } + return err + } + + // If it doesn't exist we get a new, so ignore errors + meta := newBucketMetadata(bucket) + meta.SetCreatedAt(opts.CreatedAt) + if opts.LockEnabled { + meta.VersioningConfigXML = enabledBucketVersioningConfig + meta.ObjectLockConfigXML = enabledBucketObjectLockConfig + } + + if opts.VersioningEnabled { + meta.VersioningConfigXML = enabledBucketVersioningConfig + } + + if err := meta.Save(context.Background(), z); err != nil { + return toObjectErr(err, bucket) + } + + globalBucketMetadataSys.Set(bucket, meta) + + // Success. + return nil +} + +func (z *erasureServerPools) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, err error) { + if err = checkGetObjArgs(ctx, bucket, object); err != nil { + return nil, err + } + + // This is a special call attempted first to check for SOS-API calls. + gr, err = veeamSOSAPIGetObject(ctx, bucket, object, rs, opts) + if err == nil { + return gr, nil + } + + // reset any error to 'nil' and any reader to be 'nil' + gr = nil + err = nil + + object = encodeDirObject(object) + + if z.SinglePool() { + return z.serverPools[0].GetObjectNInfo(ctx, bucket, object, rs, h, opts) + } + + var unlockOnDefer bool + nsUnlocker := func() {} + defer func() { + if unlockOnDefer { + nsUnlocker() + } + }() + + // Acquire lock + if !opts.NoLock { + lock := z.NewNSLock(bucket, object) + lkctx, err := lock.GetRLock(ctx, globalOperationTimeout) + if err != nil { + return nil, err + } + ctx = lkctx.Context() + nsUnlocker = func() { lock.RUnlock(lkctx) } + unlockOnDefer = true + } + + checkPrecondFn := opts.CheckPrecondFn + opts.CheckPrecondFn = nil // do not need to apply pre-conditions at lower layer. + opts.NoLock = true // no locks needed at lower levels for getObjectInfo() + objInfo, zIdx, err := z.getLatestObjectInfoWithIdx(ctx, bucket, object, opts) + if err != nil { + if objInfo.DeleteMarker { + if opts.VersionID == "" { + return &GetObjectReader{ + ObjInfo: objInfo, + }, toObjectErr(errFileNotFound, bucket, object) + } + // Make sure to return object info to provide extra information. + return &GetObjectReader{ + ObjInfo: objInfo, + }, toObjectErr(errMethodNotAllowed, bucket, object) + } + return nil, err + } + + // check preconditions before reading the stream. + if checkPrecondFn != nil && checkPrecondFn(objInfo) { + return nil, PreConditionFailed{} + } + + opts.NoLock = true + gr, err = z.serverPools[zIdx].GetObjectNInfo(ctx, bucket, object, rs, h, opts) + if err != nil { + return nil, err + } + + if unlockOnDefer { + unlockOnDefer = gr.ObjInfo.Inlined + } + + if !unlockOnDefer { + return gr.WithCleanupFuncs(nsUnlocker), nil + } + return gr, nil +} + +// getLatestObjectInfoWithIdx returns the objectInfo of the latest object from multiple pools (this function +// is present in-case there were duplicate writes to both pools, this function also returns the +// additional index where the latest object exists, that is used to start the GetObject stream. +func (z *erasureServerPools) getLatestObjectInfoWithIdx(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, int, error) { + object = encodeDirObject(object) + results := make([]struct { + zIdx int + oi ObjectInfo + err error + }, len(z.serverPools)) + var wg sync.WaitGroup + for i, pool := range z.serverPools { + wg.Add(1) + go func(i int, pool *erasureSets) { + defer wg.Done() + results[i].zIdx = i + results[i].oi, results[i].err = pool.GetObjectInfo(ctx, bucket, object, opts) + }(i, pool) + } + wg.Wait() + + // Sort the objInfos such that we always serve latest + // this is a defensive change to handle any duplicate + // content that may have been created, we always serve + // the latest object. + sort.Slice(results, func(i, j int) bool { + a, b := results[i], results[j] + if a.oi.ModTime.Equal(b.oi.ModTime) { + // On tiebreak, select the lowest pool index. + return a.zIdx < b.zIdx + } + return a.oi.ModTime.After(b.oi.ModTime) + }) + + for _, res := range results { + err := res.err + if err == nil { + return res.oi, res.zIdx, nil + } + if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) { + // some errors such as MethodNotAllowed for delete marker + // should be returned upwards. + return res.oi, res.zIdx, err + } + // When its a delete marker and versionID is empty + // we should simply return the error right away. + if res.oi.DeleteMarker && opts.VersionID == "" { + return res.oi, res.zIdx, err + } + } + + object = decodeDirObject(object) + if opts.VersionID != "" { + return ObjectInfo{}, -1, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID} + } + return ObjectInfo{}, -1, ObjectNotFound{Bucket: bucket, Object: object} +} + +func (z *erasureServerPools) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) { + if err = checkGetObjArgs(ctx, bucket, object); err != nil { + return objInfo, err + } + + // This is a special call attempted first to check for SOS-API calls. + objInfo, err = veeamSOSAPIHeadObject(ctx, bucket, object, opts) + if err == nil { + return objInfo, nil + } + + // reset any error to 'nil', and object info to be empty. + err = nil + objInfo = ObjectInfo{} + + object = encodeDirObject(object) + + if z.SinglePool() { + return z.serverPools[0].GetObjectInfo(ctx, bucket, object, opts) + } + + if !opts.NoLock { + opts.NoLock = true // avoid taking locks at lower levels for multi-pool setups. + + // Lock the object before reading. + lk := z.NewNSLock(bucket, object) + lkctx, err := lk.GetRLock(ctx, globalOperationTimeout) + if err != nil { + return ObjectInfo{}, err + } + ctx = lkctx.Context() + defer lk.RUnlock(lkctx) + } + + objInfo, _, err = z.getLatestObjectInfoWithIdx(ctx, bucket, object, opts) + return objInfo, err +} + +// PutObject - writes an object to least used erasure pool. +func (z *erasureServerPools) PutObject(ctx context.Context, bucket string, object string, data *PutObjReader, opts ObjectOptions) (ObjectInfo, error) { + // Validate put object input args. + if err := checkPutObjectArgs(ctx, bucket, object); err != nil { + return ObjectInfo{}, err + } + + object = encodeDirObject(object) + if z.SinglePool() { + _, err := z.getPoolIdx(ctx, bucket, object, data.Size()) + if err != nil { + return ObjectInfo{}, err + } + return z.serverPools[0].PutObject(ctx, bucket, object, data, opts) + } + + idx, err := z.getPoolIdx(ctx, bucket, object, data.Size()) + if err != nil { + return ObjectInfo{}, err + } + + if opts.DataMovement && idx == opts.SrcPoolIdx { + return ObjectInfo{}, DataMovementOverwriteErr{ + Bucket: bucket, + Object: object, + VersionID: opts.VersionID, + Err: errDataMovementSrcDstPoolSame, + } + } + + return z.serverPools[idx].PutObject(ctx, bucket, object, data, opts) +} + +func (z *erasureServerPools) deletePrefix(ctx context.Context, bucket string, prefix string) error { + for _, pool := range z.serverPools { + if _, err := pool.DeleteObject(ctx, bucket, prefix, ObjectOptions{DeletePrefix: true}); err != nil { + return err + } + } + return nil +} + +func (z *erasureServerPools) DeleteObject(ctx context.Context, bucket string, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) { + if err = checkDelObjArgs(ctx, bucket, object); err != nil { + return objInfo, err + } + + if !opts.DeletePrefix { // DeletePrefix handles dir object encoding differently. + object = encodeDirObject(object) + } + + // Acquire a write lock before deleting the object. + lk := z.NewNSLock(bucket, object) + lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout) + if err != nil { + return ObjectInfo{}, err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + + if opts.DeletePrefix { + return ObjectInfo{}, z.deletePrefix(ctx, bucket, object) + } + + gopts := opts + gopts.NoLock = true + + pinfo, noReadQuorumPools, err := z.getPoolInfoExistingWithOpts(ctx, bucket, object, gopts) + if err != nil { + if _, ok := err.(InsufficientReadQuorum); ok { + return objInfo, InsufficientWriteQuorum{} + } + return objInfo, err + } + + // Delete marker already present we are not going to create new delete markers. + if pinfo.ObjInfo.DeleteMarker && opts.VersionID == "" { + pinfo.ObjInfo.Name = decodeDirObject(object) + return pinfo.ObjInfo, nil + } + + // Datamovement must never be allowed on the same pool. + if opts.DataMovement && opts.SrcPoolIdx == pinfo.Index { + return pinfo.ObjInfo, DataMovementOverwriteErr{ + Bucket: bucket, + Object: object, + VersionID: opts.VersionID, + Err: errDataMovementSrcDstPoolSame, + } + } + + if opts.DataMovement { + objInfo, err = z.serverPools[pinfo.Index].DeleteObject(ctx, bucket, object, opts) + objInfo.Name = decodeDirObject(object) + return objInfo, err + } + + // Delete concurrently in all server pools with read quorum error for unversioned objects. + if len(noReadQuorumPools) > 0 && !opts.Versioned && !opts.VersionSuspended { + return z.deleteObjectFromAllPools(ctx, bucket, object, opts, noReadQuorumPools) + } + + // All replication requests needs to go to pool with the object. + if opts.ReplicationRequest { + objInfo, err = z.serverPools[pinfo.Index].DeleteObject(ctx, bucket, object, opts) + objInfo.Name = decodeDirObject(object) + return objInfo, err + } + + for _, pool := range z.serverPools { + objInfo, err := pool.DeleteObject(ctx, bucket, object, opts) + if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) { + objInfo.Name = decodeDirObject(object) + return objInfo, err + } + if err == nil { + objInfo.Name = decodeDirObject(object) + return objInfo, nil + } + } + + objInfo.Name = decodeDirObject(object) + if opts.VersionID != "" { + return objInfo, VersionNotFound{Bucket: bucket, Object: object, VersionID: opts.VersionID} + } + return objInfo, ObjectNotFound{Bucket: bucket, Object: object} +} + +func (z *erasureServerPools) deleteObjectFromAllPools(ctx context.Context, bucket string, object string, opts ObjectOptions, poolIndices []poolErrs) (objInfo ObjectInfo, err error) { + derrs := make([]error, len(poolIndices)) + dobjects := make([]ObjectInfo, len(poolIndices)) + + // Delete concurrently in all server pools that reported no error or read quorum error + // where the read quorum issue is from metadata inconsistency. + var wg sync.WaitGroup + for idx, pe := range poolIndices { + if v, ok := pe.Err.(InsufficientReadQuorum); ok && v.Type != RQInconsistentMeta { + derrs[idx] = InsufficientWriteQuorum{} + continue + } + wg.Add(1) + pool := z.serverPools[pe.Index] + go func(idx int, pool *erasureSets) { + defer wg.Done() + dobjects[idx], derrs[idx] = pool.DeleteObject(ctx, bucket, object, opts) + }(idx, pool) + } + wg.Wait() + + // the poolIndices array is pre-sorted in order of latest ModTime, we care only about pool with latest object though + // the delete call tries to clean up other pools during DeleteObject call. + objInfo = dobjects[0] + objInfo.Name = decodeDirObject(object) + err = derrs[0] + return objInfo, err +} + +func (z *erasureServerPools) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) { + derrs := make([]error, len(objects)) + dobjects := make([]DeletedObject, len(objects)) + objSets := set.NewStringSet() + for i := range derrs { + objects[i].ObjectName = encodeDirObject(objects[i].ObjectName) + + derrs[i] = checkDelObjArgs(ctx, bucket, objects[i].ObjectName) + objSets.Add(objects[i].ObjectName) + } + + // Acquire a bulk write lock across 'objects' + multiDeleteLock := z.NewNSLock(bucket, objSets.ToSlice()...) + lkctx, err := multiDeleteLock.GetLock(ctx, globalOperationTimeout) + if err != nil { + for i := range derrs { + derrs[i] = err + } + return dobjects, derrs + } + ctx = lkctx.Context() + defer multiDeleteLock.Unlock(lkctx) + + dObjectsByPool := make([][]DeletedObject, len(z.serverPools)) + dErrsByPool := make([][]error, len(z.serverPools)) + + eg := errgroup.WithNErrs(len(z.serverPools)).WithConcurrency(len(z.serverPools)) + for i, pool := range z.serverPools { + pool := pool + eg.Go(func() error { + dObjectsByPool[i], dErrsByPool[i] = pool.DeleteObjects(ctx, bucket, objects, opts) + return nil + }, i) + } + eg.Wait() // wait to check all the pools. + + for i := range dobjects { + // Iterate over pools + for pool := range z.serverPools { + if dErrsByPool[pool][i] == nil && dObjectsByPool[pool][i].found { + // A fast exit when the object is found and removed + dobjects[i] = dObjectsByPool[pool][i] + derrs[i] = nil + break + } + if derrs[i] == nil { + // No error related to this object is found, assign this pool result + // whether it is nil because there is no object found or because of + // some other errors such erasure quorum errors. + dobjects[i] = dObjectsByPool[pool][i] + derrs[i] = dErrsByPool[pool][i] + } + } + } + + for i := range dobjects { + dobjects[i].ObjectName = decodeDirObject(dobjects[i].ObjectName) + } + return dobjects, derrs +} + +func (z *erasureServerPools) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) { + if err := checkCopyObjArgs(ctx, srcBucket, srcObject); err != nil { + return ObjectInfo{}, err + } + if err := checkCopyObjArgs(ctx, dstBucket, dstObject); err != nil { + return ObjectInfo{}, err + } + + srcObject = encodeDirObject(srcObject) + dstObject = encodeDirObject(dstObject) + + cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject)) + + if !dstOpts.NoLock { + ns := z.NewNSLock(dstBucket, dstObject) + lkctx, err := ns.GetLock(ctx, globalOperationTimeout) + if err != nil { + return ObjectInfo{}, err + } + ctx = lkctx.Context() + defer ns.Unlock(lkctx) + dstOpts.NoLock = true + } + + poolIdx, err := z.getPoolIdxNoLock(ctx, dstBucket, dstObject, srcInfo.Size) + if err != nil { + return objInfo, err + } + + if cpSrcDstSame && srcInfo.metadataOnly { + // Version ID is set for the destination and source == destination version ID. + if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID { + return z.serverPools[poolIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts) + } + // Destination is not versioned and source version ID is empty + // perform an in-place update. + if !dstOpts.Versioned && srcOpts.VersionID == "" { + return z.serverPools[poolIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts) + } + // Destination is versioned, source is not destination version, + // as a special case look for if the source object is not legacy + // from older format, for older format we will rewrite them as + // newer using PutObject() - this is an optimization to save space + if dstOpts.Versioned && srcOpts.VersionID != dstOpts.VersionID && !srcInfo.Legacy { + // CopyObject optimization where we don't create an entire copy + // of the content, instead we add a reference. + srcInfo.versionOnly = true + return z.serverPools[poolIdx].CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts) + } + } + + putOpts := ObjectOptions{ + ServerSideEncryption: dstOpts.ServerSideEncryption, + UserDefined: srcInfo.UserDefined, + Versioned: dstOpts.Versioned, + VersionID: dstOpts.VersionID, + MTime: dstOpts.MTime, + NoLock: true, + EncryptFn: dstOpts.EncryptFn, + WantChecksum: dstOpts.WantChecksum, + WantServerSideChecksumType: dstOpts.WantServerSideChecksumType, + } + + return z.serverPools[poolIdx].PutObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts) +} + +func (z *erasureServerPools) ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (ListObjectsInfo, error) { + return z.listObjectsGeneric(ctx, bucket, prefix, marker, delimiter, maxKeys, true) +} + +func (z *erasureServerPools) ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (ListObjectsV2Info, error) { + marker := continuationToken + if marker == "" { + marker = startAfter + } + + loi, err := z.listObjectsGeneric(ctx, bucket, prefix, marker, delimiter, maxKeys, false) + if err != nil { + return ListObjectsV2Info{}, err + } + + listObjectsV2Info := ListObjectsV2Info{ + IsTruncated: loi.IsTruncated, + ContinuationToken: continuationToken, + NextContinuationToken: loi.NextMarker, + Objects: loi.Objects, + Prefixes: loi.Prefixes, + } + return listObjectsV2Info, err +} + +func (z *erasureServerPools) ListObjectVersions(ctx context.Context, bucket, prefix, marker, versionMarker, delimiter string, maxKeys int) (ListObjectVersionsInfo, error) { + loi := ListObjectVersionsInfo{} + if marker == "" && versionMarker != "" { + return loi, NotImplemented{} + } + + opts := listPathOptions{ + Bucket: bucket, + Prefix: prefix, + Separator: delimiter, + Limit: maxKeysPlusOne(maxKeys, marker != ""), + Marker: marker, + InclDeleted: true, + AskDisks: globalAPIConfig.getListQuorum(), + Versioned: true, + } + + // Shortcut for APN/1.0 Veeam/1.0 Backup/10.0 + // It requests unique blocks with a specific prefix. + // We skip scanning the parent directory for + // more objects matching the prefix. + if isVeeamClient(ctx) && strings.HasSuffix(prefix, ".blk") { + opts.BaseDir = prefix + opts.Transient = true + } + + // set bucket metadata in opts + opts.setBucketMeta(ctx) + + merged, err := z.listPath(ctx, &opts) + if err != nil && err != io.EOF { + if !isErrBucketNotFound(err) { + storageLogOnceIf(ctx, err, "erasure-list-objects-path-"+bucket) + } + return loi, toObjectErr(err, bucket) + } + defer merged.truncate(0) // Release when returning + + if contextCanceled(ctx) { + return ListObjectVersionsInfo{}, ctx.Err() + } + + if versionMarker == "" { + o := listPathOptions{Marker: marker} + // If we are not looking for a specific version skip it. + + o.parseMarker() + merged.forwardPast(o.Marker) + } + objects := merged.fileInfoVersions(bucket, prefix, delimiter, versionMarker) + loi.IsTruncated = err == nil && len(objects) > 0 + if maxKeys > 0 && len(objects) > maxKeys { + objects = objects[:maxKeys] + loi.IsTruncated = true + } + for _, obj := range objects { + if obj.IsDir && obj.ModTime.IsZero() && delimiter != "" { + // Only add each once. + // With slash delimiter we only get the directory once. + found := false + if delimiter != slashSeparator { + for _, p := range loi.Prefixes { + if found { + break + } + found = p == obj.Name + } + } + if !found { + loi.Prefixes = append(loi.Prefixes, obj.Name) + } + } else { + loi.Objects = append(loi.Objects, obj) + } + } + if loi.IsTruncated { + last := objects[len(objects)-1] + loi.NextMarker = opts.encodeMarker(last.Name) + loi.NextVersionIDMarker = last.VersionID + } + return loi, nil +} + +func maxKeysPlusOne(maxKeys int, addOne bool) int { + if maxKeys < 0 || maxKeys > maxObjectList { + maxKeys = maxObjectList + } + if addOne { + maxKeys++ + } + return maxKeys +} + +func (z *erasureServerPools) listObjectsGeneric(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int, v1 bool) (loi ListObjectsInfo, err error) { + opts := listPathOptions{ + V1: v1, + Bucket: bucket, + Prefix: prefix, + Separator: delimiter, + Limit: maxKeysPlusOne(maxKeys, marker != ""), + Marker: marker, + InclDeleted: false, + AskDisks: globalAPIConfig.getListQuorum(), + } + opts.setBucketMeta(ctx) + listFn := func(ctx context.Context, opts listPathOptions, limitTo int) (ListObjectsInfo, error) { + var loi ListObjectsInfo + merged, err := z.listPath(ctx, &opts) + if err != nil && err != io.EOF { + if !isErrBucketNotFound(err) { + storageLogOnceIf(ctx, err, "erasure-list-objects-path-"+bucket) + } + return loi, toObjectErr(err, bucket) + } + merged.forwardPast(opts.Marker) + defer merged.truncate(0) // Release when returning + + if contextCanceled(ctx) { + return ListObjectsInfo{}, ctx.Err() + } + + // Default is recursive, if delimiter is set then list non recursive. + objects := merged.fileInfos(bucket, prefix, delimiter) + loi.IsTruncated = err == nil && len(objects) > 0 + if limitTo > 0 && len(objects) > limitTo { + objects = objects[:limitTo] + loi.IsTruncated = true + } + for _, obj := range objects { + if obj.IsDir && obj.ModTime.IsZero() && delimiter != "" { + // Only add each once. + // With slash delimiter we only get the directory once. + found := false + if delimiter != slashSeparator { + for _, p := range loi.Prefixes { + if found { + break + } + found = p == obj.Name + } + } + if !found { + loi.Prefixes = append(loi.Prefixes, obj.Name) + } + } else { + loi.Objects = append(loi.Objects, obj) + } + } + if loi.IsTruncated { + last := objects[len(objects)-1] + loi.NextMarker = last.Name + } + + if loi.IsTruncated && merged.lastSkippedEntry > loi.NextMarker { + // An object hidden by ILM was found during a truncated listing. Set the next marker + // as the last skipped entry if it is lexically higher loi.NextMarker as an optimization + loi.NextMarker = merged.lastSkippedEntry + } + + if loi.NextMarker != "" { + loi.NextMarker = opts.encodeMarker(loi.NextMarker) + } + return loi, nil + } + ri := logger.GetReqInfo(ctx) + hadoop := ri != nil && strings.Contains(ri.UserAgent, "Hadoop ") && strings.Contains(ri.UserAgent, "scala/") + matches := func() bool { + if prefix == "" { + return false + } + // List of standard files supported by s3a + // that involves a List() on a directory + // where directory is actually an object on + // namespace. + for _, k := range []string{ + "_SUCCESS/", + ".parquet/", + ".csv/", + ".json/", + ".avro/", + ".orc/", + ".txt/", + // Add any other files in future + } { + if strings.HasSuffix(prefix, k) { + return true + } + } + return false + } + + if hadoop && delimiter == SlashSeparator && maxKeys == 2 && marker == "" { + // Optimization for Spark/Hadoop workload where spark sends a garbage + // request of this kind + // + // GET /testbucket/?list-type=2&delimiter=%2F&max-keys=2&prefix=parquet%2F_SUCCESS%2F&fetch-owner=false + // + // Here spark is expecting that the List() return empty instead, so from MinIO's point + // of view if we simply do a GetObjectInfo() on this prefix by treating it as an object + // We save a lot of calls over the network. + // + // This happens repeatedly for all objects that are created concurrently() avoiding this + // as a List() call is an important performance improvement. + // + // Spark based s3a committers are a big enough use-case to have this optimization. + // + // A sample code to see the improvements is as follows, this sample code is + // simply a read on JSON from MinIO and write it back as "parquet". + // + // import org.apache.spark.sql.SparkSession + // import org.apache.spark.sql.types.{IntegerType, StringType, StructField, StructType} + // object SparkJSONRead { + // def main(args: Array[String]): Unit = { + // val spark:SparkSession = SparkSession.builder() + // .appName("SparkByExample") + // .master("local[1]").getOrCreate() + // + // spark.sparkContext.setLogLevel("ERROR") + // spark.sparkContext.hadoopConfiguration.set("fs.s3a.endpoint", "http://minio-lb:9000") + // spark.sparkContext.hadoopConfiguration.set("fs.s3a.path.style.access", "true") + // spark.sparkContext.hadoopConfiguration.set("fs.s3a.access.key", "minioadmin") + // spark.sparkContext.hadoopConfiguration.set("fs.s3a.secret.key", "minioadmin") + // + // val df = spark.read.json("s3a://testbucket/s3.json") + // + // df.write.parquet("s3a://testbucket/parquet/") + // } + // } + if matches() { + objInfo, err := z.GetObjectInfo(ctx, bucket, path.Dir(prefix), ObjectOptions{NoLock: true}) + if err == nil || objInfo.IsLatest && objInfo.DeleteMarker { + if opts.Lifecycle != nil { + evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, opts.Replication.Config, objInfo) + if evt.Action.Delete() { + globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_s3ListObjects) + if !evt.Action.DeleteRestored() { + // Skip entry if ILM action was DeleteVersionAction or DeleteAction + return loi, nil + } + } + } + return loi, nil + } + if isErrBucketNotFound(err) { + return loi, err + } + if contextCanceled(ctx) { + return ListObjectsInfo{}, ctx.Err() + } + } + // Hadoop makes the max-keys=2 listing call just to find if the directory is empty or not, or in the case + // of an object to check for object existence. For versioned buckets, MinIO's non-recursive + // call will report top level prefixes in deleted state, whereas spark/hadoop interpret this as non-empty + // and throw a 404 exception. This is especially a problem for spark jobs overwriting the same partition + // repeatedly. This workaround recursively lists the top 3 entries including delete markers to reflect the + // correct state of the directory in the list results. + if strings.HasSuffix(opts.Prefix, SlashSeparator) { + li, err := listFn(ctx, opts, maxKeys) + if err != nil { + return loi, err + } + if len(li.Objects) == 0 { + prefixes := li.Prefixes[:0] + for _, prefix := range li.Prefixes { + objInfo, _ := z.GetObjectInfo(ctx, bucket, pathJoin(prefix, "_SUCCESS"), ObjectOptions{NoLock: true}) + if objInfo.IsLatest && objInfo.DeleteMarker { + continue + } + prefixes = append(prefixes, prefix) + } + if len(prefixes) > 0 { + objInfo, _ := z.GetObjectInfo(ctx, bucket, pathJoin(opts.Prefix, "_SUCCESS"), ObjectOptions{NoLock: true}) + if objInfo.IsLatest && objInfo.DeleteMarker { + return loi, nil + } + } + li.Prefixes = prefixes + } + return li, nil + } + } + + if len(prefix) > 0 && maxKeys == 1 && marker == "" { + // Optimization for certain applications like + // - Cohesity + // - Actifio, Splunk etc. + // which send ListObjects requests where the actual object + // itself is the prefix and max-keys=1 in such scenarios + // we can simply verify locally if such an object exists + // to avoid the need for ListObjects(). + objInfo, err := z.GetObjectInfo(ctx, bucket, prefix, ObjectOptions{NoLock: true}) + if err == nil { + if opts.Lifecycle != nil { + evt := evalActionFromLifecycle(ctx, *opts.Lifecycle, opts.Retention, opts.Replication.Config, objInfo) + if evt.Action.Delete() { + globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_s3ListObjects) + if !evt.Action.DeleteRestored() { + // Skip entry if ILM action was DeleteVersionAction or DeleteAction + return loi, nil + } + } + } + loi.Objects = append(loi.Objects, objInfo) + return loi, nil + } + if isErrBucketNotFound(err) { + return ListObjectsInfo{}, err + } + if contextCanceled(ctx) { + return ListObjectsInfo{}, ctx.Err() + } + } + return listFn(ctx, opts, maxKeys) +} + +func (z *erasureServerPools) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (ListMultipartsInfo, error) { + if err := checkListMultipartArgs(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter); err != nil { + return ListMultipartsInfo{}, err + } + + poolResult := ListMultipartsInfo{} + poolResult.MaxUploads = maxUploads + poolResult.KeyMarker = keyMarker + poolResult.Prefix = prefix + poolResult.Delimiter = delimiter + + // if no prefix provided, return the list from cache + if prefix == "" { + if _, err := z.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + return ListMultipartsInfo{}, toObjectErr(err, bucket) + } + + z.mpCache.Range(func(_ string, mp MultipartInfo) bool { + if mp.Bucket == bucket { + poolResult.Uploads = append(poolResult.Uploads, mp) + } + return true + }) + sort.Slice(poolResult.Uploads, func(i int, j int) bool { + return poolResult.Uploads[i].Initiated.Before(poolResult.Uploads[j].Initiated) + }) + return poolResult, nil + } + + if z.SinglePool() { + return z.serverPools[0].ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads) + } + + for idx, pool := range z.serverPools { + if z.IsSuspended(idx) { + continue + } + result, err := pool.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, + delimiter, maxUploads) + if err != nil { + return result, err + } + poolResult.Uploads = append(poolResult.Uploads, result.Uploads...) + } + + return poolResult, nil +} + +// Initiate a new multipart upload on a hashedSet based on object name. +func (z *erasureServerPools) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (mp *NewMultipartUploadResult, err error) { + if err := checkNewMultipartArgs(ctx, bucket, object); err != nil { + return nil, err + } + + defer func() { + if err == nil && mp != nil { + z.mpCache.Store(mp.UploadID, MultipartInfo{ + Bucket: bucket, + Object: object, + UploadID: mp.UploadID, + Initiated: time.Now(), + }) + } + }() + + if z.SinglePool() { + return z.serverPools[0].NewMultipartUpload(ctx, bucket, object, opts) + } + + for idx, pool := range z.serverPools { + if z.IsSuspended(idx) || z.IsPoolRebalancing(idx) { + continue + } + + result, err := pool.ListMultipartUploads(ctx, bucket, object, "", "", "", maxUploadsList) + if err != nil { + return nil, err + } + // If there is a multipart upload with the same bucket/object name, + // create the new multipart in the same pool, this will avoid + // creating two multiparts uploads in two different pools + if len(result.Uploads) != 0 { + return z.serverPools[idx].NewMultipartUpload(ctx, bucket, object, opts) + } + } + + // any parallel writes on the object will block for this poolIdx + // to return since this holds a read lock on the namespace. + idx, err := z.getPoolIdx(ctx, bucket, object, -1) + if err != nil { + return nil, err + } + + if opts.DataMovement && idx == opts.SrcPoolIdx { + return nil, DataMovementOverwriteErr{ + Bucket: bucket, + Object: object, + VersionID: opts.VersionID, + Err: errDataMovementSrcDstPoolSame, + } + } + + return z.serverPools[idx].NewMultipartUpload(ctx, bucket, object, opts) +} + +// Copies a part of an object from source hashedSet to destination hashedSet. +func (z *erasureServerPools) CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (PartInfo, error) { + if err := checkNewMultipartArgs(ctx, srcBucket, srcObject); err != nil { + return PartInfo{}, err + } + + return z.PutObjectPart(ctx, destBucket, destObject, uploadID, partID, + srcInfo.PutObjReader, dstOpts) +} + +// PutObjectPart - writes part of an object to hashedSet based on the object name. +func (z *erasureServerPools) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (PartInfo, error) { + if err := checkPutObjectPartArgs(ctx, bucket, object, uploadID); err != nil { + return PartInfo{}, err + } + + if z.SinglePool() { + _, err := z.getPoolIdx(ctx, bucket, object, data.Size()) + if err != nil { + return PartInfo{}, err + } + return z.serverPools[0].PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts) + } + + for idx, pool := range z.serverPools { + if z.IsSuspended(idx) { + continue + } + pi, err := pool.PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts) + if err == nil { + return pi, nil + } + if _, ok := err.(InvalidUploadID); ok { + // Look for information on the next pool + continue + } + // Any other unhandled errors such as quorum return. + return PartInfo{}, err + } + + return PartInfo{}, InvalidUploadID{ + Bucket: bucket, + Object: object, + UploadID: uploadID, + } +} + +func (z *erasureServerPools) GetMultipartInfo(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (MultipartInfo, error) { + if err := checkListPartsArgs(ctx, bucket, object, uploadID); err != nil { + return MultipartInfo{}, err + } + + uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID)) + lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout) + if err != nil { + return MultipartInfo{}, err + } + ctx = lkctx.Context() + defer uploadIDLock.RUnlock(lkctx) + + if z.SinglePool() { + return z.serverPools[0].GetMultipartInfo(ctx, bucket, object, uploadID, opts) + } + + for idx, pool := range z.serverPools { + if z.IsSuspended(idx) { + continue + } + mi, err := pool.GetMultipartInfo(ctx, bucket, object, uploadID, opts) + if err == nil { + return mi, nil + } + if _, ok := err.(InvalidUploadID); ok { + // upload id not found, continue to the next pool. + continue + } + // any other unhandled error return right here. + return MultipartInfo{}, err + } + + return MultipartInfo{}, InvalidUploadID{ + Bucket: bucket, + Object: object, + UploadID: uploadID, + } +} + +// ListObjectParts - lists all uploaded parts to an object in hashedSet. +func (z *erasureServerPools) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int, opts ObjectOptions) (ListPartsInfo, error) { + if err := checkListPartsArgs(ctx, bucket, object, uploadID); err != nil { + return ListPartsInfo{}, err + } + + uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID)) + lkctx, err := uploadIDLock.GetRLock(ctx, globalOperationTimeout) + if err != nil { + return ListPartsInfo{}, err + } + ctx = lkctx.Context() + defer uploadIDLock.RUnlock(lkctx) + + if z.SinglePool() { + return z.serverPools[0].ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts) + } + + for idx, pool := range z.serverPools { + if z.IsSuspended(idx) { + continue + } + result, err := pool.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts) + if err == nil { + return result, nil + } + if _, ok := err.(InvalidUploadID); ok { + continue + } + return ListPartsInfo{}, err + } + + return ListPartsInfo{}, InvalidUploadID{ + Bucket: bucket, + Object: object, + UploadID: uploadID, + } +} + +// Aborts an in-progress multipart operation on hashedSet based on the object name. +func (z *erasureServerPools) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (err error) { + if err := checkAbortMultipartArgs(ctx, bucket, object, uploadID); err != nil { + return err + } + + defer func() { + if err == nil { + z.mpCache.Delete(uploadID) + globalNotificationSys.DeleteUploadID(ctx, uploadID) + } + }() + + lk := z.NewNSLock(bucket, pathJoin(object, uploadID)) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + return err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + + if z.SinglePool() { + return z.serverPools[0].AbortMultipartUpload(ctx, bucket, object, uploadID, opts) + } + + for idx, pool := range z.serverPools { + if z.IsSuspended(idx) { + continue + } + err := pool.AbortMultipartUpload(ctx, bucket, object, uploadID, opts) + if err == nil { + return nil + } + if _, ok := err.(InvalidUploadID); ok { + // upload id not found move to next pool + continue + } + return err + } + return InvalidUploadID{ + Bucket: bucket, + Object: object, + UploadID: uploadID, + } +} + +// CompleteMultipartUpload - completes a pending multipart transaction, on hashedSet based on object name. +func (z *erasureServerPools) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error) { + if err = checkCompleteMultipartArgs(ctx, bucket, object, uploadID); err != nil { + return objInfo, err + } + + defer func() { + if err == nil { + z.mpCache.Delete(uploadID) + globalNotificationSys.DeleteUploadID(ctx, uploadID) + } + }() + + // Hold write locks to verify uploaded parts, also disallows any + // parallel PutObjectPart() requests. + uploadIDLock := z.NewNSLock(bucket, pathJoin(object, uploadID)) + wlkctx, err := uploadIDLock.GetLock(ctx, globalOperationTimeout) + if err != nil { + return objInfo, err + } + ctx = wlkctx.Context() + defer uploadIDLock.Unlock(wlkctx) + + if z.SinglePool() { + return z.serverPools[0].CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts) + } + + for idx, pool := range z.serverPools { + if z.IsSuspended(idx) { + continue + } + objInfo, err = pool.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts) + if err == nil { + return objInfo, nil + } + if _, ok := err.(InvalidUploadID); ok { + // upload id not found move to next pool + continue + } + return objInfo, err + } + + return objInfo, InvalidUploadID{ + Bucket: bucket, + Object: object, + UploadID: uploadID, + } +} + +// GetBucketInfo - returns bucket info from one of the erasure coded serverPools. +func (z *erasureServerPools) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bucketInfo BucketInfo, err error) { + bucketInfo, err = z.s3Peer.GetBucketInfo(ctx, bucket, opts) + if err != nil { + return bucketInfo, toObjectErr(err, bucket) + } + + meta, err := globalBucketMetadataSys.Get(bucket) + if err == nil { + bucketInfo.Created = meta.Created + bucketInfo.Versioning = meta.Versioning() + bucketInfo.ObjectLocking = meta.ObjectLocking() + } + return bucketInfo, nil +} + +// ClearUploadID deletes given uploadID from cache +func (z *erasureServerPools) ClearUploadID(uploadID string) error { + z.mpCache.Delete(uploadID) + return nil +} + +// DeleteBucket - deletes a bucket on all serverPools simultaneously, +// even if one of the serverPools fail to delete buckets, we proceed to +// undo a successful operation. +func (z *erasureServerPools) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error { + if isMinioMetaBucketName(bucket) { + return BucketNameInvalid{Bucket: bucket} + } + + // Verify if bucket is valid. + if err := s3utils.CheckValidBucketName(bucket); err != nil { + return BucketNameInvalid{Bucket: bucket} + } + + if !opts.NoLock { + // Lock the bucket name before creating. + lk := z.NewNSLock(minioMetaTmpBucket, bucket+".lck") + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + return err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + } + + if !opts.Force { + results := make(chan itemOrErr[ObjectInfo]) + + ctx, cancel := context.WithTimeout(ctx, time.Minute) + defer cancel() + err := z.Walk(ctx, bucket, "", results, WalkOptions{Limit: 1}) + if err != nil { + s3LogIf(ctx, fmt.Errorf("unable to verify if the bucket %s is empty: %w", bucket, err)) + return toObjectErr(err, bucket) + } + + select { + case <-ctx.Done(): + return ctx.Err() + case r, found := <-results: + if found { + if r.Err != nil { + s3LogIf(ctx, fmt.Errorf("unable to verify if the bucket %s is empty: %w", bucket, r.Err)) + return toObjectErr(r.Err, bucket) + } + return toObjectErr(errVolumeNotEmpty, bucket) + } + } + + // Always pass force to the lower level + opts.Force = true + } + + err := z.s3Peer.DeleteBucket(ctx, bucket, opts) + if err == nil || isErrBucketNotFound(err) { + // If site replication is configured, hold on to deleted bucket state until sites sync + if opts.SRDeleteOp == MarkDelete { + z.s3Peer.MakeBucket(context.Background(), pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, bucket), MakeBucketOptions{}) + } + } + + if err == nil { + // Purge the entire bucket metadata entirely. + z.deleteAll(context.Background(), minioMetaBucket, pathJoin(bucketMetaPrefix, bucket)) + } + + return toObjectErr(err, bucket) +} + +// deleteAll will rename bucket+prefix unconditionally across all disks to +// minioMetaTmpDeletedBucket + unique uuid, +// Note that set distribution is ignored so it should only be used in cases where +// data is not distributed across sets. Errors are logged but individual +// disk failures are not returned. +func (z *erasureServerPools) deleteAll(ctx context.Context, bucket, prefix string) { + for _, servers := range z.serverPools { + for _, set := range servers.sets { + set.deleteAll(ctx, bucket, prefix) + } + } +} + +var listBucketsCache = cachevalue.New[[]BucketInfo]() + +// List all buckets from one of the serverPools, we are not doing merge +// sort here just for simplification. As per design it is assumed +// that all buckets are present on all serverPools. +func (z *erasureServerPools) ListBuckets(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error) { + if opts.Cached { + listBucketsCache.InitOnce(time.Second, + cachevalue.Opts{ReturnLastGood: true, NoWait: true}, + func(ctx context.Context) ([]BucketInfo, error) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + buckets, err = z.s3Peer.ListBuckets(ctx, opts) + if err != nil { + return nil, err + } + if !opts.NoMetadata { + for i := range buckets { + createdAt, err := globalBucketMetadataSys.CreatedAt(buckets[i].Name) + if err == nil { + buckets[i].Created = createdAt + } + } + } + return buckets, nil + }, + ) + + return listBucketsCache.GetWithCtx(ctx) + } + + buckets, err = z.s3Peer.ListBuckets(ctx, opts) + if err != nil { + return nil, err + } + + if !opts.NoMetadata { + for i := range buckets { + createdAt, err := globalBucketMetadataSys.CreatedAt(buckets[i].Name) + if err == nil { + buckets[i].Created = createdAt + } + } + } + return buckets, nil +} + +func (z *erasureServerPools) HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) { + // Acquire lock on format.json + formatLock := z.NewNSLock(minioMetaBucket, formatConfigFile) + lkctx, err := formatLock.GetLock(ctx, globalOperationTimeout) + if err != nil { + return madmin.HealResultItem{}, err + } + ctx = lkctx.Context() + defer formatLock.Unlock(lkctx) + + r := madmin.HealResultItem{ + Type: madmin.HealItemMetadata, + Detail: "disk-format", + } + + var countNoHeal int + for _, pool := range z.serverPools { + result, err := pool.HealFormat(ctx, dryRun) + if err != nil && !errors.Is(err, errNoHealRequired) { + healingLogOnceIf(ctx, err, "erasure-heal-format") + continue + } + // Count errNoHealRequired across all serverPools, + // to return appropriate error to the caller + if errors.Is(err, errNoHealRequired) { + countNoHeal++ + } + r.DiskCount += result.DiskCount + r.SetCount += result.SetCount + r.Before.Drives = append(r.Before.Drives, result.Before.Drives...) + r.After.Drives = append(r.After.Drives, result.After.Drives...) + } + + // No heal returned by all serverPools, return errNoHealRequired + if countNoHeal == len(z.serverPools) { + return r, errNoHealRequired + } + + return r, nil +} + +func (z *erasureServerPools) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, error) { + // .metadata.bin healing is not needed here, it is automatically healed via read() call. + return z.s3Peer.HealBucket(ctx, bucket, opts) +} + +// Walk a bucket, optionally prefix recursively, until we have returned +// all the contents of the provided bucket+prefix. +func (z *erasureServerPools) Walk(ctx context.Context, bucket, prefix string, results chan<- itemOrErr[ObjectInfo], opts WalkOptions) error { + if err := checkListObjsArgs(ctx, bucket, prefix, ""); err != nil { + xioutil.SafeClose(results) + return err + } + parentCtx := ctx + ctx, cancelCause := context.WithCancelCause(ctx) + var entries []chan metaCacheEntry + + for poolIdx, erasureSet := range z.serverPools { + for setIdx, set := range erasureSet.sets { + listOut := make(chan metaCacheEntry, 1) + entries = append(entries, listOut) + disks, infos, _ := set.getOnlineDisksWithHealingAndInfo(true) + if len(disks) == 0 { + xioutil.SafeClose(results) + err := fmt.Errorf("Walk: no online disks found in (set:%d pool:%d) %w", setIdx, poolIdx, errErasureReadQuorum) + cancelCause(err) + return err + } + go func() { + defer xioutil.SafeClose(listOut) + send := func(e metaCacheEntry) { + if e.isDir() { + // Ignore directories. + return + } + select { + case listOut <- e: + case <-ctx.Done(): + } + } + + askDisks := getListQuorum(opts.AskDisks, set.setDriveCount) + if askDisks == -1 { + newDisks := getQuorumDisks(disks, infos, (len(disks)+1)/2) + if newDisks != nil { + // If we found disks signature in quorum, we proceed to list + // from a single drive, shuffling of the drives is subsequently. + disks = newDisks + askDisks = 1 + } else { + // If we did not find suitable disks, perform strict quorum listing + // as no disk agrees on quorum anymore. + askDisks = getListQuorum("strict", set.setDriveCount) + } + } + + // Special case: ask all disks if the drive count is 4 + if set.setDriveCount == 4 || askDisks > len(disks) { + askDisks = len(disks) // use all available drives + } + + var fallbackDisks []StorageAPI + if askDisks > 0 && len(disks) > askDisks { + rand.Shuffle(len(disks), func(i, j int) { + disks[i], disks[j] = disks[j], disks[i] + }) + fallbackDisks = disks[askDisks:] + disks = disks[:askDisks] + } + + requestedVersions := 0 + if opts.LatestOnly { + requestedVersions = 1 + } + + // However many we ask, versions must exist on ~50% + listingQuorum := (askDisks + 1) / 2 + + // How to resolve partial results. + resolver := metadataResolutionParams{ + dirQuorum: listingQuorum, + objQuorum: listingQuorum, + bucket: bucket, + requestedVersions: requestedVersions, + } + + path := baseDirFromPrefix(prefix) + filterPrefix := strings.Trim(strings.TrimPrefix(prefix, path), slashSeparator) + if path == prefix { + filterPrefix = "" + } + + lopts := listPathRawOptions{ + disks: disks, + fallbackDisks: fallbackDisks, + bucket: bucket, + path: path, + filterPrefix: filterPrefix, + recursive: true, + forwardTo: opts.Marker, + perDiskLimit: opts.Limit, + minDisks: listingQuorum, + reportNotFound: false, + agreed: send, + partial: func(entries metaCacheEntries, _ []error) { + entry, ok := entries.resolve(&resolver) + if ok { + send(*entry) + } + }, + finished: nil, + } + + if err := listPathRaw(ctx, lopts); err != nil { + cancelCause(fmt.Errorf("listPathRaw returned %w: opts(%#v)", err, lopts)) + return + } + }() + } + } + + // Convert and filter merged entries. + merged := make(chan metaCacheEntry, 100) + vcfg, _ := globalBucketVersioningSys.Get(bucket) + errCh := make(chan error, 1) + go func() { + sentErr := false + sendErr := func(err error) { + if !sentErr { + select { + case results <- itemOrErr[ObjectInfo]{Err: err}: + sentErr = true + case <-parentCtx.Done(): + } + } + } + defer func() { + select { + case <-ctx.Done(): + sendErr(ctx.Err()) + default: + } + xioutil.SafeClose(results) + cancelCause(nil) + }() + send := func(oi ObjectInfo) bool { + select { + case results <- itemOrErr[ObjectInfo]{Item: oi}: + return true + case <-ctx.Done(): + sendErr(context.Cause(ctx)) + return false + } + } + for entry := range merged { + if opts.LatestOnly { + fi, err := entry.fileInfo(bucket) + if err != nil { + sendErr(err) + return + } + if opts.Filter != nil { + if opts.Filter(fi) { + if !send(fi.ToObjectInfo(bucket, fi.Name, vcfg != nil && vcfg.Versioned(fi.Name))) { + return + } + } + } else { + if !send(fi.ToObjectInfo(bucket, fi.Name, vcfg != nil && vcfg.Versioned(fi.Name))) { + return + } + } + continue + } + fivs, err := entry.fileInfoVersions(bucket) + if err != nil { + sendErr(err) + return + } + + // Note: entry.fileInfoVersions returns versions sorted in reverse chronological order based on ModTime + if opts.VersionsSort == WalkVersionsSortAsc { + versionsSorter(fivs.Versions).reverse() + } + + for _, version := range fivs.Versions { + if opts.Filter != nil { + if opts.Filter(version) { + if !send(version.ToObjectInfo(bucket, version.Name, vcfg != nil && vcfg.Versioned(version.Name))) { + return + } + } + } else { + if !send(version.ToObjectInfo(bucket, version.Name, vcfg != nil && vcfg.Versioned(version.Name))) { + return + } + } + } + } + if err := <-errCh; err != nil { + sendErr(err) + } + }() + go func() { + defer close(errCh) + // Merge all entries from all disks. + // We leave quorum at 1, since entries are already resolved to have the desired quorum. + // mergeEntryChannels will close 'merged' channel upon completion or cancellation. + errCh <- mergeEntryChannels(ctx, entries, merged, 1) + }() + + return nil +} + +// HealObjectFn closure function heals the object. +type HealObjectFn func(bucket, object, versionID string, scanMode madmin.HealScanMode) error + +// List a prefix or a single object versions and heal +func (z *erasureServerPools) HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, healObjectFn HealObjectFn) error { + healEntry := func(bucket string, entry metaCacheEntry, scanMode madmin.HealScanMode) error { + if entry.isDir() { + return nil + } + // We might land at .metacache, .trash, .multipart + // no need to heal them skip, only when bucket + // is '.minio.sys' + if bucket == minioMetaBucket { + if wildcard.Match("buckets/*/.metacache/*", entry.name) { + return nil + } + if wildcard.Match("tmp/*", entry.name) { + return nil + } + if wildcard.Match("multipart/*", entry.name) { + return nil + } + if wildcard.Match("tmp-old/*", entry.name) { + return nil + } + } + fivs, err := entry.fileInfoVersions(bucket) + if err != nil { + return healObjectFn(bucket, entry.name, "", scanMode) + } + if opts.Remove && !opts.DryRun { + err := z.CheckAbandonedParts(ctx, bucket, entry.name, opts) + if err != nil { + healingLogIf(ctx, fmt.Errorf("unable to check object %s/%s for abandoned data: %w", bucket, entry.name, err)) + } + } + for _, version := range fivs.Versions { + err := healObjectFn(bucket, version.Name, version.VersionID, scanMode) + if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) { + return err + } + } + + return nil + } + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + poolErrs := make([][]error, len(z.serverPools)) + for idx, erasureSet := range z.serverPools { + if opts.Pool != nil && *opts.Pool != idx { + continue + } + if z.IsSuspended(idx) { + continue + } + errs := make([]error, len(erasureSet.sets)) + wk, _ := workers.New(3) + for idx, set := range erasureSet.sets { + if opts.Set != nil && *opts.Set != idx { + continue + } + wk.Take() + go func(idx int, set *erasureObjects) { + defer wk.Give() + + errs[idx] = set.listAndHeal(ctx, bucket, prefix, opts.Recursive, opts.ScanMode, healEntry) + }(idx, set) + } + wk.Wait() + poolErrs[idx] = errs + } + for _, errs := range poolErrs { + for _, err := range errs { + if err == nil { + continue + } + return err + } + } + return nil +} + +func (z *erasureServerPools) HealObject(ctx context.Context, bucket, object, versionID string, opts madmin.HealOpts) (madmin.HealResultItem, error) { + object = encodeDirObject(object) + + errs := make([]error, len(z.serverPools)) + results := make([]madmin.HealResultItem, len(z.serverPools)) + var wg sync.WaitGroup + for idx, pool := range z.serverPools { + if z.IsSuspended(idx) { + continue + } + wg.Add(1) + go func(idx int, pool *erasureSets) { + defer wg.Done() + result, err := pool.HealObject(ctx, bucket, object, versionID, opts) + result.Object = decodeDirObject(result.Object) + errs[idx] = err + results[idx] = result + }(idx, pool) + } + wg.Wait() + + // Return the first nil error + for idx, err := range errs { + if err == nil { + return results[idx], nil + } + } + + // No pool returned a nil error, return the first non 'not found' error + for idx, err := range errs { + if !isErrObjectNotFound(err) && !isErrVersionNotFound(err) { + return results[idx], err + } + } + + hr := madmin.HealResultItem{ + Type: madmin.HealItemObject, + Bucket: bucket, + Object: object, + VersionID: versionID, + } + + // At this stage, all errors are 'not found' + if versionID != "" { + return hr, VersionNotFound{ + Bucket: bucket, + Object: object, + VersionID: versionID, + } + } + return hr, ObjectNotFound{ + Bucket: bucket, + Object: object, + } +} + +func (z *erasureServerPools) getPoolAndSet(id string) (poolIdx, setIdx, diskIdx int, err error) { + for poolIdx := range z.serverPools { + format := z.serverPools[poolIdx].format + for setIdx, set := range format.Erasure.Sets { + for i, diskID := range set { + if diskID == id { + return poolIdx, setIdx, i, nil + } + } + } + } + return -1, -1, -1, fmt.Errorf("DriveID(%s) %w", id, errDiskNotFound) +} + +const ( + vmware = "VMWare" +) + +// HealthOptions takes input options to return specific information +type HealthOptions struct { + Maintenance bool + DeploymentType string + NoLogging bool +} + +// HealthResult returns the current state of the system, also +// additionally with any specific heuristic information which +// was queried +type HealthResult struct { + Healthy bool + HealthyRead bool + HealingDrives int + ESHealth []struct { + Maintenance bool + PoolID, SetID int + Healthy bool + HealthyRead bool + HealthyDrives int + HealingDrives int + ReadQuorum int + WriteQuorum int + } + WriteQuorum int + ReadQuorum int + UsingDefaults bool +} + +func (hr HealthResult) String() string { + var str strings.Builder + for i, es := range hr.ESHealth { + str.WriteString("(Pool: ") + str.WriteString(strconv.Itoa(es.PoolID)) + str.WriteString(" Set: ") + str.WriteString(strconv.Itoa(es.SetID)) + str.WriteString(" Healthy: ") + str.WriteString(strconv.FormatBool(es.Healthy)) + if i == 0 { + str.WriteString(")") + } else { + str.WriteString(") | ") + } + } + return str.String() +} + +// Health - returns current status of the object layer health, +// provides if write access exists across sets, additionally +// can be used to query scenarios if health may be lost +// if this node is taken down by an external orchestrator. +func (z *erasureServerPools) Health(ctx context.Context, opts HealthOptions) HealthResult { + reqInfo := (&logger.ReqInfo{}).AppendTags("maintenance", strconv.FormatBool(opts.Maintenance)) + + type setInfo struct { + online int + healing int + } + + var drivesHealing int + + erasureSetUpCount := make([][]setInfo, len(z.serverPools)) + for i := range z.serverPools { + erasureSetUpCount[i] = make([]setInfo, len(z.serverPools[i].sets)) + } + + storageInfo := z.StorageInfo(ctx, false) + + for _, disk := range storageInfo.Disks { + if opts.Maintenance { + globalLocalDrivesMu.RLock() + _, ok := globalLocalDrivesMap[disk.Endpoint] + globalLocalDrivesMu.RUnlock() + if ok { + continue + } + } + + if disk.PoolIndex > -1 && disk.SetIndex > -1 { + if disk.State == madmin.DriveStateOk { + si := erasureSetUpCount[disk.PoolIndex][disk.SetIndex] + si.online++ + if disk.Healing { + si.healing++ + drivesHealing++ + } + erasureSetUpCount[disk.PoolIndex][disk.SetIndex] = si + } + } + } + + b := z.BackendInfo() + poolReadQuorums := make([]int, len(b.StandardSCData)) + poolWriteQuorums := make([]int, len(b.StandardSCData)) + for i, data := range b.StandardSCData { + poolReadQuorums[i] = data + poolWriteQuorums[i] = data + if data == b.StandardSCParity { + poolWriteQuorums[i] = data + 1 + } + } + + // Check if disks are healing on in-case of VMware vsphere deployments. + if opts.Maintenance && opts.DeploymentType == vmware { + if drivesHealing > 0 { + healingLogIf(logger.SetReqInfo(ctx, reqInfo), fmt.Errorf("Total drives to be healed %d", drivesHealing)) + } + } + + var usingDefaults bool + if globalStorageClass.GetParityForSC(storageclass.STANDARD) < 0 { + usingDefaults = true + } + + var maximumWriteQuorum int + for _, writeQuorum := range poolWriteQuorums { + if maximumWriteQuorum == 0 { + maximumWriteQuorum = writeQuorum + } + if writeQuorum > maximumWriteQuorum { + maximumWriteQuorum = writeQuorum + } + } + + var maximumReadQuorum int + for _, readQuorum := range poolReadQuorums { + if maximumReadQuorum == 0 { + maximumReadQuorum = readQuorum + } + if readQuorum > maximumReadQuorum { + maximumReadQuorum = readQuorum + } + } + + result := HealthResult{ + Healthy: true, + HealthyRead: true, + WriteQuorum: maximumWriteQuorum, + ReadQuorum: maximumReadQuorum, + UsingDefaults: usingDefaults, // indicates if config was not initialized and we are using defaults on this node. + } + + for poolIdx := range erasureSetUpCount { + for setIdx := range erasureSetUpCount[poolIdx] { + result.ESHealth = append(result.ESHealth, struct { + Maintenance bool + PoolID, SetID int + Healthy bool + HealthyRead bool + HealthyDrives, HealingDrives int + ReadQuorum, WriteQuorum int + }{ + Maintenance: opts.Maintenance, + SetID: setIdx, + PoolID: poolIdx, + Healthy: erasureSetUpCount[poolIdx][setIdx].online >= poolWriteQuorums[poolIdx], + HealthyRead: erasureSetUpCount[poolIdx][setIdx].online >= poolReadQuorums[poolIdx], + HealthyDrives: erasureSetUpCount[poolIdx][setIdx].online, + HealingDrives: erasureSetUpCount[poolIdx][setIdx].healing, + ReadQuorum: poolReadQuorums[poolIdx], + WriteQuorum: poolWriteQuorums[poolIdx], + }) + + healthy := erasureSetUpCount[poolIdx][setIdx].online >= poolWriteQuorums[poolIdx] + if !healthy && !opts.NoLogging { + storageLogIf(logger.SetReqInfo(ctx, reqInfo), + fmt.Errorf("Write quorum could not be established on pool: %d, set: %d, expected write quorum: %d, drives-online: %d", + poolIdx, setIdx, poolWriteQuorums[poolIdx], erasureSetUpCount[poolIdx][setIdx].online), logger.FatalKind) + } + result.Healthy = result.Healthy && healthy + + healthyRead := erasureSetUpCount[poolIdx][setIdx].online >= poolReadQuorums[poolIdx] + if !healthyRead && !opts.NoLogging { + storageLogIf(logger.SetReqInfo(ctx, reqInfo), + fmt.Errorf("Read quorum could not be established on pool: %d, set: %d, expected read quorum: %d, drives-online: %d", + poolIdx, setIdx, poolReadQuorums[poolIdx], erasureSetUpCount[poolIdx][setIdx].online)) + } + result.HealthyRead = result.HealthyRead && healthyRead + } + } + + if opts.Maintenance { + result.Healthy = result.Healthy && drivesHealing == 0 + result.HealthyRead = result.HealthyRead && drivesHealing == 0 + result.HealingDrives = drivesHealing + } + + return result +} + +// PutObjectMetadata - replace or add tags to an existing object +func (z *erasureServerPools) PutObjectMetadata(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) { + object = encodeDirObject(object) + if z.SinglePool() { + return z.serverPools[0].PutObjectMetadata(ctx, bucket, object, opts) + } + + if !opts.NoLock { + // Lock the object before updating metadata. + lk := z.NewNSLock(bucket, object) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + return ObjectInfo{}, err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + } + + opts.MetadataChg = true + opts.NoLock = true + // We don't know the size here set 1GiB at least. + idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts) + if err != nil { + return ObjectInfo{}, err + } + + return z.serverPools[idx].PutObjectMetadata(ctx, bucket, object, opts) +} + +// PutObjectTags - replace or add tags to an existing object +func (z *erasureServerPools) PutObjectTags(ctx context.Context, bucket, object string, tags string, opts ObjectOptions) (ObjectInfo, error) { + object = encodeDirObject(object) + if z.SinglePool() { + return z.serverPools[0].PutObjectTags(ctx, bucket, object, tags, opts) + } + + if !opts.NoLock { + // Lock the object before updating tags. + lk := z.NewNSLock(bucket, object) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + return ObjectInfo{}, err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + } + + opts.MetadataChg = true + opts.NoLock = true + + // We don't know the size here set 1GiB at least. + idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts) + if err != nil { + return ObjectInfo{}, err + } + + return z.serverPools[idx].PutObjectTags(ctx, bucket, object, tags, opts) +} + +// DeleteObjectTags - delete object tags from an existing object +func (z *erasureServerPools) DeleteObjectTags(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) { + object = encodeDirObject(object) + if z.SinglePool() { + return z.serverPools[0].DeleteObjectTags(ctx, bucket, object, opts) + } + + if !opts.NoLock { + // Lock the object before deleting tags. + lk := z.NewNSLock(bucket, object) + lkctx, err := lk.GetLock(ctx, globalOperationTimeout) + if err != nil { + return ObjectInfo{}, err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + } + + opts.MetadataChg = true + opts.NoLock = true + idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts) + if err != nil { + return ObjectInfo{}, err + } + + return z.serverPools[idx].DeleteObjectTags(ctx, bucket, object, opts) +} + +// GetObjectTags - get object tags from an existing object +func (z *erasureServerPools) GetObjectTags(ctx context.Context, bucket, object string, opts ObjectOptions) (*tags.Tags, error) { + object = encodeDirObject(object) + if z.SinglePool() { + return z.serverPools[0].GetObjectTags(ctx, bucket, object, opts) + } + + oi, _, err := z.getLatestObjectInfoWithIdx(ctx, bucket, object, opts) + if err != nil { + return nil, err + } + + return tags.ParseObjectTags(oi.UserTags) +} + +// TransitionObject - transition object content to target tier. +func (z *erasureServerPools) TransitionObject(ctx context.Context, bucket, object string, opts ObjectOptions) error { + object = encodeDirObject(object) + if z.SinglePool() { + return z.serverPools[0].TransitionObject(ctx, bucket, object, opts) + } + + if !opts.NoLock { + // Acquire write lock before starting to transition the object. + lk := z.NewNSLock(bucket, object) + lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout) + if err != nil { + return err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + } + + // Avoid transitioning an object from a pool being decommissioned. + opts.SkipDecommissioned = true + opts.NoLock = true + idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts) + if err != nil { + return err + } + + return z.serverPools[idx].TransitionObject(ctx, bucket, object, opts) +} + +// RestoreTransitionedObject - restore transitioned object content locally on this cluster. +func (z *erasureServerPools) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error { + object = encodeDirObject(object) + if z.SinglePool() { + return z.serverPools[0].RestoreTransitionedObject(ctx, bucket, object, opts) + } + + if !opts.NoLock { + // Acquire write lock before restoring transitioned object + lk := z.NewNSLock(bucket, object) + lkctx, err := lk.GetLock(ctx, globalDeleteOperationTimeout) + if err != nil { + return err + } + ctx = lkctx.Context() + defer lk.Unlock(lkctx) + } + + // Avoid restoring object from a pool being decommissioned. + opts.SkipDecommissioned = true + opts.NoLock = true + idx, err := z.getPoolIdxExistingWithOpts(ctx, bucket, object, opts) + if err != nil { + return err + } + + return z.serverPools[idx].RestoreTransitionedObject(ctx, bucket, object, opts) +} + +func (z *erasureServerPools) CheckAbandonedParts(ctx context.Context, bucket, object string, opts madmin.HealOpts) error { + object = encodeDirObject(object) + if z.SinglePool() { + return z.serverPools[0].CheckAbandonedParts(ctx, bucket, object, opts) + } + errs := make([]error, len(z.serverPools)) + var wg sync.WaitGroup + for idx, pool := range z.serverPools { + if z.IsSuspended(idx) { + continue + } + wg.Add(1) + go func(idx int, pool *erasureSets) { + defer wg.Done() + err := pool.CheckAbandonedParts(ctx, bucket, object, opts) + if err != nil && !isErrObjectNotFound(err) && !isErrVersionNotFound(err) { + errs[idx] = err + } + }(idx, pool) + } + wg.Wait() + for _, err := range errs { + return err + } + return nil +} + +// DecomTieredObject - moves tiered object to another pool during decommissioning. +func (z *erasureServerPools) DecomTieredObject(ctx context.Context, bucket, object string, fi FileInfo, opts ObjectOptions) error { + object = encodeDirObject(object) + if z.SinglePool() { + return fmt.Errorf("error decommissioning %s/%s", bucket, object) + } + if !opts.NoLock { + ns := z.NewNSLock(bucket, object) + lkctx, err := ns.GetLock(ctx, globalOperationTimeout) + if err != nil { + return err + } + ctx = lkctx.Context() + defer ns.Unlock(lkctx) + opts.NoLock = true + } + idx, err := z.getPoolIdxNoLock(ctx, bucket, object, fi.Size) + if err != nil { + return err + } + + if opts.DataMovement && idx == opts.SrcPoolIdx { + return DataMovementOverwriteErr{ + Bucket: bucket, + Object: object, + VersionID: opts.VersionID, + Err: errDataMovementSrcDstPoolSame, + } + } + + return z.serverPools[idx].DecomTieredObject(ctx, bucket, object, fi, opts) +} diff --git a/cmd/erasure-sets.go b/cmd/erasure-sets.go new file mode 100644 index 00000000..9ceb8085 --- /dev/null +++ b/cmd/erasure-sets.go @@ -0,0 +1,1193 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "hash/crc32" + "math/rand" + "net/http" + "reflect" + "strings" + "sync" + "time" + + "github.com/dchest/siphash" + "github.com/google/uuid" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio-go/v7/pkg/tags" + "github.com/minio/minio/internal/dsync" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/console" + "github.com/minio/pkg/v3/sync/errgroup" + "github.com/puzpuzpuz/xsync/v3" +) + +// setsDsyncLockers is encapsulated type for Close() +type setsDsyncLockers [][]dsync.NetLocker + +// erasureSets implements ObjectLayer combining a static list of erasure coded +// object sets. NOTE: There is no dynamic scaling allowed or intended in +// current design. +type erasureSets struct { + sets []*erasureObjects + + // Reference format. + format *formatErasureV3 + + // erasureDisks mutex to lock erasureDisks. + erasureDisksMu sync.RWMutex + + // Re-ordered list of disks per set. + erasureDisks [][]StorageAPI + + // Distributed locker clients. + erasureLockers setsDsyncLockers + + // Distributed lock owner (constant per running instance). + erasureLockOwner string + + // List of endpoints provided on the command line. + endpoints PoolEndpoints + + // String version of all the endpoints, an optimization + // to avoid url.String() conversion taking CPU on + // large disk setups. + endpointStrings []string + + // Total number of sets and the number of disks per set. + setCount, setDriveCount int + defaultParityCount int + + poolIndex int + + // Distribution algorithm of choice. + distributionAlgo string + deploymentID [16]byte + + lastConnectDisksOpTime time.Time +} + +var staleUploadsCleanupIntervalChangedCh = make(chan struct{}) + +func (s *erasureSets) getDiskMap() map[Endpoint]StorageAPI { + diskMap := make(map[Endpoint]StorageAPI) + + s.erasureDisksMu.RLock() + defer s.erasureDisksMu.RUnlock() + + for i := range s.setCount { + for j := 0; j < s.setDriveCount; j++ { + disk := s.erasureDisks[i][j] + if disk == OfflineDisk { + continue + } + if !disk.IsOnline() { + continue + } + diskMap[disk.Endpoint()] = disk + } + } + return diskMap +} + +// Initializes a new StorageAPI from the endpoint argument, returns +// StorageAPI and also `format` which exists on the disk. +func connectEndpoint(endpoint Endpoint) (StorageAPI, *formatErasureV3, error) { + disk, err := newStorageAPI(endpoint, storageOpts{ + cleanUp: false, + healthCheck: false, + }) + if err != nil { + return nil, nil, err + } + + format, err := loadFormatErasure(disk, false) + if err != nil { + disk.Close() + return nil, nil, fmt.Errorf("Drive: %s returned %w", disk, err) // make sure to '%w' to wrap the error + } + + disk.Close() + disk, err = newStorageAPI(endpoint, storageOpts{ + cleanUp: true, + healthCheck: true, + }) + if err != nil { + return nil, nil, err + } + + return disk, format, nil +} + +// findDiskIndex - returns the i,j'th position of the input `diskID` against the reference +// format, after successful validation. +// - i'th position is the set index +// - j'th position is the disk index in the current set +func findDiskIndexByDiskID(refFormat *formatErasureV3, diskID string) (int, int, error) { + if diskID == "" { + return -1, -1, errDiskNotFound + } + if diskID == offlineDiskUUID { + return -1, -1, fmt.Errorf("DriveID: %s is offline", diskID) + } + for i := range len(refFormat.Erasure.Sets) { + for j := 0; j < len(refFormat.Erasure.Sets[0]); j++ { + if refFormat.Erasure.Sets[i][j] == diskID { + return i, j, nil + } + } + } + + return -1, -1, fmt.Errorf("DriveID: %s not found", diskID) +} + +// findDiskIndex - returns the i,j'th position of the input `format` against the reference +// format, after successful validation. +// - i'th position is the set index +// - j'th position is the disk index in the current set +func findDiskIndex(refFormat, format *formatErasureV3) (int, int, error) { + if err := formatErasureV3Check(refFormat, format); err != nil { + return 0, 0, err + } + + if format.Erasure.This == offlineDiskUUID { + return -1, -1, fmt.Errorf("DriveID: %s is offline", format.Erasure.This) + } + + for i := range len(refFormat.Erasure.Sets) { + for j := 0; j < len(refFormat.Erasure.Sets[0]); j++ { + if refFormat.Erasure.Sets[i][j] == format.Erasure.This { + return i, j, nil + } + } + } + + return -1, -1, fmt.Errorf("DriveID: %s not found", format.Erasure.This) +} + +// Legacy returns 'true' if distribution algo is CRCMOD +func (s *erasureSets) Legacy() (ok bool) { + return s.distributionAlgo == formatErasureVersionV2DistributionAlgoV1 +} + +// connectDisks - attempt to connect all the endpoints, loads format +// and re-arranges the disks in proper position. +func (s *erasureSets) connectDisks(log bool) { + defer func() { + s.lastConnectDisksOpTime = time.Now() + }() + + var wg sync.WaitGroup + diskMap := s.getDiskMap() + for _, endpoint := range s.endpoints.Endpoints { + cdisk := diskMap[endpoint] + if cdisk != nil && cdisk.IsOnline() { + if s.lastConnectDisksOpTime.IsZero() { + continue + } + + // An online-disk means its a valid disk but it may be a re-connected disk + // we verify that here based on LastConn(), however we make sure to avoid + // putting it back into the s.erasureDisks by re-placing the disk again. + _, setIndex, _ := cdisk.GetDiskLoc() + if setIndex != -1 { + continue + } + } + if cdisk != nil { + // Close previous offline disk. + cdisk.Close() + } + + wg.Add(1) + go func(endpoint Endpoint) { + defer wg.Done() + disk, format, err := connectEndpoint(endpoint) + if err != nil { + if endpoint.IsLocal && errors.Is(err, errUnformattedDisk) { + globalBackgroundHealState.pushHealLocalDisks(endpoint) + } else if !errors.Is(err, errDriveIsRoot) { + if log { + printEndpointError(endpoint, err, true) + } + } + return + } + if disk.IsLocal() { + h := disk.Healing() + if h != nil && !h.Finished { + globalBackgroundHealState.pushHealLocalDisks(disk.Endpoint()) + } + } + s.erasureDisksMu.Lock() + setIndex, diskIndex, err := findDiskIndex(s.format, format) + if err != nil { + printEndpointError(endpoint, err, false) + disk.Close() + s.erasureDisksMu.Unlock() + return + } + + if currentDisk := s.erasureDisks[setIndex][diskIndex]; currentDisk != nil { + if !reflect.DeepEqual(currentDisk.Endpoint(), disk.Endpoint()) { + err = fmt.Errorf("Detected unexpected drive ordering refusing to use the drive: expecting %s, found %s, refusing to use the drive", + currentDisk.Endpoint(), disk.Endpoint()) + printEndpointError(endpoint, err, false) + disk.Close() + s.erasureDisksMu.Unlock() + return + } + s.erasureDisks[setIndex][diskIndex].Close() + } + + disk.SetDiskID(format.Erasure.This) + s.erasureDisks[setIndex][diskIndex] = disk + + if disk.IsLocal() { + globalLocalDrivesMu.Lock() + if globalIsDistErasure { + globalLocalSetDrives[s.poolIndex][setIndex][diskIndex] = disk + } + globalLocalDrivesMap[disk.Endpoint().String()] = disk + globalLocalDrivesMu.Unlock() + } + s.erasureDisksMu.Unlock() + }(endpoint) + } + + wg.Wait() +} + +// monitorAndConnectEndpoints this is a monitoring loop to keep track of disconnected +// endpoints by reconnecting them and making sure to place them into right position in +// the set topology, this monitoring happens at a given monitoring interval. +func (s *erasureSets) monitorAndConnectEndpoints(ctx context.Context, monitorInterval time.Duration) { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + + time.Sleep(time.Duration(r.Float64() * float64(time.Second))) + + // Pre-emptively connect the disks if possible. + s.connectDisks(false) + + monitor := time.NewTimer(monitorInterval) + defer monitor.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-monitor.C: + if serverDebugLog { + console.Debugln("running drive monitoring") + } + + s.connectDisks(true) + + // Reset the timer for next interval + monitor.Reset(monitorInterval) + } + } +} + +func (s *erasureSets) GetLockers(setIndex int) func() ([]dsync.NetLocker, string) { + return func() ([]dsync.NetLocker, string) { + lockers := make([]dsync.NetLocker, len(s.erasureLockers[setIndex])) + copy(lockers, s.erasureLockers[setIndex]) + return lockers, s.erasureLockOwner + } +} + +func (s *erasureSets) GetEndpointStrings(setIndex int) func() []string { + return func() []string { + eps := make([]string, s.setDriveCount) + copy(eps, s.endpointStrings[setIndex*s.setDriveCount:setIndex*s.setDriveCount+s.setDriveCount]) + return eps + } +} + +func (s *erasureSets) GetEndpoints(setIndex int) func() []Endpoint { + return func() []Endpoint { + eps := make([]Endpoint, s.setDriveCount) + copy(eps, s.endpoints.Endpoints[setIndex*s.setDriveCount:setIndex*s.setDriveCount+s.setDriveCount]) + return eps + } +} + +// GetDisks returns a closure for a given set, which provides list of disks per set. +func (s *erasureSets) GetDisks(setIndex int) func() []StorageAPI { + return func() []StorageAPI { + s.erasureDisksMu.RLock() + defer s.erasureDisksMu.RUnlock() + disks := make([]StorageAPI, s.setDriveCount) + copy(disks, s.erasureDisks[setIndex]) + return disks + } +} + +// defaultMonitorConnectEndpointInterval is the interval to monitor endpoint connections. +// Must be bigger than defaultMonitorNewDiskInterval. +const defaultMonitorConnectEndpointInterval = defaultMonitorNewDiskInterval + time.Second*5 + +// Initialize new set of erasure coded sets. +func newErasureSets(ctx context.Context, endpoints PoolEndpoints, storageDisks []StorageAPI, format *formatErasureV3, defaultParityCount, poolIdx int) (*erasureSets, error) { + setCount := len(format.Erasure.Sets) + setDriveCount := len(format.Erasure.Sets[0]) + + endpointStrings := make([]string, len(endpoints.Endpoints)) + for i, endpoint := range endpoints.Endpoints { + endpointStrings[i] = endpoint.String() + } + + // Initialize the erasure sets instance. + s := &erasureSets{ + sets: make([]*erasureObjects, setCount), + erasureDisks: make([][]StorageAPI, setCount), + erasureLockers: make([][]dsync.NetLocker, setCount), + erasureLockOwner: globalLocalNodeName, + endpoints: endpoints, + endpointStrings: endpointStrings, + setCount: setCount, + setDriveCount: setDriveCount, + defaultParityCount: defaultParityCount, + format: format, + distributionAlgo: format.Erasure.DistributionAlgo, + deploymentID: uuid.MustParse(format.ID), + poolIndex: poolIdx, + } + + mutex := newNSLock(globalIsDistErasure) + + for i := range setCount { + s.erasureDisks[i] = make([]StorageAPI, setDriveCount) + } + + erasureLockers := map[string]dsync.NetLocker{} + for _, endpoint := range endpoints.Endpoints { + if _, ok := erasureLockers[endpoint.Host]; !ok { + erasureLockers[endpoint.Host] = newLockAPI(endpoint) + } + } + + var wg sync.WaitGroup + var lk sync.Mutex + for i := range setCount { + lockerEpSet := set.NewStringSet() + for j := range setDriveCount { + wg.Add(1) + go func(i int, endpoint Endpoint) { + defer wg.Done() + + lk.Lock() + // Only add lockers only one per endpoint and per erasure set. + if locker, ok := erasureLockers[endpoint.Host]; ok && !lockerEpSet.Contains(endpoint.Host) { + lockerEpSet.Add(endpoint.Host) + s.erasureLockers[i] = append(s.erasureLockers[i], locker) + } + lk.Unlock() + }(i, endpoints.Endpoints[i*setDriveCount+j]) + } + } + wg.Wait() + + for i := range setCount { + wg.Add(1) + go func(i int) { + defer wg.Done() + + var innerWg sync.WaitGroup + for j := range setDriveCount { + disk := storageDisks[i*setDriveCount+j] + if disk == nil { + continue + } + + if disk.IsLocal() && globalIsDistErasure { + globalLocalDrivesMu.RLock() + ldisk := globalLocalSetDrives[poolIdx][i][j] + if ldisk == nil { + globalLocalDrivesMu.RUnlock() + continue + } + disk.Close() + disk = ldisk + globalLocalDrivesMu.RUnlock() + } + + innerWg.Add(1) + go func(disk StorageAPI, i, j int) { + defer innerWg.Done() + diskID, err := disk.GetDiskID() + if err != nil { + if !errors.Is(err, errUnformattedDisk) { + bootLogIf(ctx, err) + } + return + } + if diskID == "" { + return + } + s.erasureDisks[i][j] = disk + }(disk, i, j) + } + + innerWg.Wait() + + // Initialize erasure objects for a given set. + s.sets[i] = &erasureObjects{ + setIndex: i, + poolIndex: poolIdx, + setDriveCount: setDriveCount, + defaultParityCount: defaultParityCount, + getDisks: s.GetDisks(i), + getLockers: s.GetLockers(i), + getEndpoints: s.GetEndpoints(i), + getEndpointStrings: s.GetEndpointStrings(i), + nsMutex: mutex, + } + }(i) + } + + wg.Wait() + + // start cleanup stale uploads go-routine. + go s.cleanupStaleUploads(ctx) + + // start cleanup of deleted objects. + go s.cleanupDeletedObjects(ctx) + + // Start the disk monitoring and connect routine. + if !globalIsTesting { + go s.monitorAndConnectEndpoints(ctx, defaultMonitorConnectEndpointInterval) + } + + return s, nil +} + +// cleanup ".trash/" folder every 5m minutes with sufficient sleep cycles, between each +// deletes a dynamic sleeper is used with a factor of 10 ratio with max delay between +// deletes to be 2 seconds. +func (s *erasureSets) cleanupDeletedObjects(ctx context.Context) { + timer := time.NewTimer(globalAPIConfig.getDeleteCleanupInterval()) + defer timer.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-timer.C: + var wg sync.WaitGroup + for _, set := range s.sets { + wg.Add(1) + go func(set *erasureObjects) { + defer wg.Done() + if set == nil { + return + } + set.cleanupDeletedObjects(ctx) + }(set) + } + wg.Wait() + + // Reset for the next interval + timer.Reset(globalAPIConfig.getDeleteCleanupInterval()) + } + } +} + +func (s *erasureSets) cleanupStaleUploads(ctx context.Context) { + timer := time.NewTimer(globalAPIConfig.getStaleUploadsCleanupInterval()) + defer timer.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-timer.C: + var wg sync.WaitGroup + for _, set := range s.sets { + wg.Add(1) + go func(set *erasureObjects) { + defer wg.Done() + if set == nil { + return + } + set.cleanupStaleUploads(ctx) + }(set) + } + wg.Wait() + case <-staleUploadsCleanupIntervalChangedCh: + } + + // Reset for the next interval + timer.Reset(globalAPIConfig.getStaleUploadsCleanupInterval()) + } +} + +type auditObjectOp struct { + Name string `json:"name"` + Pool int `json:"poolId"` + Set int `json:"setId"` +} + +func (op auditObjectOp) String() string { + // Flatten the auditObjectOp + return fmt.Sprintf("name=%s,pool=%d,set=%d", op.Name, op.Pool, op.Set) +} + +// Add erasure set information to the current context +func auditObjectErasureSet(ctx context.Context, api, object string, set *erasureObjects) { + if len(logger.AuditTargets()) == 0 { + return + } + + op := auditObjectOp{ + Name: decodeDirObject(object), + Pool: set.poolIndex + 1, + Set: set.setIndex + 1, + } + + logger.GetReqInfo(ctx).AppendTags(api, op.String()) +} + +// NewNSLock - initialize a new namespace RWLocker instance. +func (s *erasureSets) NewNSLock(bucket string, objects ...string) RWLocker { + return s.sets[0].NewNSLock(bucket, objects...) +} + +// SetDriveCount returns the current drives per set. +func (s *erasureSets) SetDriveCount() int { + return s.setDriveCount +} + +// ParityCount returns the default parity count used while erasure +// coding objects +func (s *erasureSets) ParityCount() int { + return s.defaultParityCount +} + +// StorageInfo - combines output of StorageInfo across all erasure coded object sets. +func (s *erasureSets) StorageInfo(ctx context.Context) StorageInfo { + var storageInfo madmin.StorageInfo + + storageInfos := make([]madmin.StorageInfo, len(s.sets)) + + g := errgroup.WithNErrs(len(s.sets)) + for index := range s.sets { + g.Go(func() error { + storageInfos[index] = s.sets[index].StorageInfo(ctx) + return nil + }, index) + } + + // Wait for the go routines. + g.Wait() + + for _, lstorageInfo := range storageInfos { + storageInfo.Disks = append(storageInfo.Disks, lstorageInfo.Disks...) + } + + return storageInfo +} + +// StorageInfo - combines output of StorageInfo across all erasure coded object sets. +func (s *erasureSets) LocalStorageInfo(ctx context.Context, metrics bool) StorageInfo { + var storageInfo StorageInfo + + storageInfos := make([]StorageInfo, len(s.sets)) + + g := errgroup.WithNErrs(len(s.sets)) + for index := range s.sets { + g.Go(func() error { + storageInfos[index] = s.sets[index].LocalStorageInfo(ctx, metrics) + return nil + }, index) + } + + // Wait for the go routines. + g.Wait() + + for _, lstorageInfo := range storageInfos { + storageInfo.Disks = append(storageInfo.Disks, lstorageInfo.Disks...) + } + + return storageInfo +} + +// Shutdown shutsdown all erasure coded sets in parallel +// returns error upon first error. +func (s *erasureSets) Shutdown(ctx context.Context) error { + g := errgroup.WithNErrs(len(s.sets)) + + for index := range s.sets { + g.Go(func() error { + return s.sets[index].Shutdown(ctx) + }, index) + } + + for _, err := range g.Wait() { + if err != nil { + return err + } + } + return nil +} + +// hashes the key returning an integer based on the input algorithm. +// This function currently supports +// - CRCMOD +// - SIPMOD +// - all new algos. +func sipHashMod(key string, cardinality int, id [16]byte) int { + if cardinality <= 0 { + return -1 + } + // use the faster version as per siphash docs + // https://github.com/dchest/siphash#usage + k0, k1 := binary.LittleEndian.Uint64(id[0:8]), binary.LittleEndian.Uint64(id[8:16]) + sum64 := siphash.Hash(k0, k1, []byte(key)) + return int(sum64 % uint64(cardinality)) +} + +func crcHashMod(key string, cardinality int) int { + if cardinality <= 0 { + return -1 + } + keyCrc := crc32.Checksum([]byte(key), crc32.IEEETable) + return int(keyCrc % uint32(cardinality)) +} + +func hashKey(algo string, key string, cardinality int, id [16]byte) int { + switch algo { + case formatErasureVersionV2DistributionAlgoV1: + return crcHashMod(key, cardinality) + case formatErasureVersionV3DistributionAlgoV2, formatErasureVersionV3DistributionAlgoV3: + return sipHashMod(key, cardinality, id) + default: + // Unknown algorithm returns -1, also if cardinality is lesser than 0. + return -1 + } +} + +// Returns always a same erasure coded set for a given input. +func (s *erasureSets) getHashedSetIndex(input string) int { + return hashKey(s.distributionAlgo, input, len(s.sets), s.deploymentID) +} + +// Returns always a same erasure coded set for a given input. +func (s *erasureSets) getHashedSet(input string) (set *erasureObjects) { + return s.sets[s.getHashedSetIndex(input)] +} + +// listDeletedBuckets lists deleted buckets from all disks. +func listDeletedBuckets(ctx context.Context, storageDisks []StorageAPI, delBuckets *xsync.MapOf[string, VolInfo], readQuorum int) error { + g := errgroup.WithNErrs(len(storageDisks)) + for index := range storageDisks { + g.Go(func() error { + if storageDisks[index] == nil { + // we ignore disk not found errors + return nil + } + volsInfo, err := storageDisks[index].ListDir(ctx, "", minioMetaBucket, pathJoin(bucketMetaPrefix, deletedBucketsPrefix), -1) + if err != nil { + if errors.Is(err, errFileNotFound) { + return nil + } + return err + } + for _, volName := range volsInfo { + vi, err := storageDisks[index].StatVol(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, volName)) + if err == nil { + vi.Name = strings.TrimSuffix(volName, SlashSeparator) + delBuckets.Store(volName, vi) + } + } + return nil + }, index) + } + return reduceReadQuorumErrs(ctx, g.Wait(), bucketMetadataOpIgnoredErrs, readQuorum) +} + +// --- Object Operations --- + +// GetObjectNInfo - returns object info and locked object ReadCloser +func (s *erasureSets) GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (gr *GetObjectReader, err error) { + set := s.getHashedSet(object) + return set.GetObjectNInfo(ctx, bucket, object, rs, h, opts) +} + +// PutObject - writes an object to hashedSet based on the object name. +func (s *erasureSets) PutObject(ctx context.Context, bucket string, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) { + set := s.getHashedSet(object) + return set.PutObject(ctx, bucket, object, data, opts) +} + +// GetObjectInfo - reads object metadata from the hashedSet based on the object name. +func (s *erasureSets) GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) { + set := s.getHashedSet(object) + return set.GetObjectInfo(ctx, bucket, object, opts) +} + +func (s *erasureSets) deletePrefix(ctx context.Context, bucket string, prefix string) error { + var wg sync.WaitGroup + wg.Add(len(s.sets)) + for _, s := range s.sets { + go func(s *erasureObjects) { + defer wg.Done() + // This is a force delete, no reason to throw errors. + s.DeleteObject(ctx, bucket, prefix, ObjectOptions{DeletePrefix: true}) + }(s) + } + wg.Wait() + return nil +} + +// DeleteObject - deletes an object from the hashedSet based on the object name. +func (s *erasureSets) DeleteObject(ctx context.Context, bucket string, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) { + if opts.DeletePrefix && !opts.DeletePrefixObject { + err := s.deletePrefix(ctx, bucket, object) + return ObjectInfo{}, err + } + set := s.getHashedSet(object) + return set.DeleteObject(ctx, bucket, object, opts) +} + +// DeleteObjects - bulk delete of objects +// Bulk delete is only possible within one set. For that purpose +// objects are group by set first, and then bulk delete is invoked +// for each set, the error response of each delete will be returned +func (s *erasureSets) DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) { + type delObj struct { + // Set index associated to this object + setIndex int + // Original index from the list of arguments + // where this object is passed + origIndex int + // object to delete + object ObjectToDelete + } + + // Transform []delObj to the list of object names + toNames := func(delObjs []delObj) []ObjectToDelete { + objs := make([]ObjectToDelete, len(delObjs)) + for i, obj := range delObjs { + objs[i] = obj.object + } + return objs + } + + // The result of delete operation on all passed objects + delErrs := make([]error, len(objects)) + + // The result of delete objects + delObjects := make([]DeletedObject, len(objects)) + + // A map between a set and its associated objects + objSetMap := make(map[int][]delObj) + + // Group objects by set index + for i, object := range objects { + index := s.getHashedSetIndex(object.ObjectName) + objSetMap[index] = append(objSetMap[index], delObj{setIndex: index, origIndex: i, object: object}) + } + + // Invoke bulk delete on objects per set and save + // the result of the delete operation + var wg sync.WaitGroup + var mu sync.Mutex + wg.Add(len(objSetMap)) + for setIdx, objsGroup := range objSetMap { + go func(set *erasureObjects, group []delObj) { + defer wg.Done() + dobjects, errs := set.DeleteObjects(ctx, bucket, toNames(group), opts) + mu.Lock() + defer mu.Unlock() + for i, obj := range group { + delErrs[obj.origIndex] = errs[i] + delObjects[obj.origIndex] = dobjects[i] + } + }(s.sets[setIdx], objsGroup) + } + wg.Wait() + + return delObjects, delErrs +} + +// CopyObject - copies objects from one hashedSet to another hashedSet, on server side. +func (s *erasureSets) CopyObject(ctx context.Context, srcBucket, srcObject, dstBucket, dstObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) { + srcSet := s.getHashedSet(srcObject) + dstSet := s.getHashedSet(dstObject) + + cpSrcDstSame := srcSet == dstSet + // Check if this request is only metadata update. + if cpSrcDstSame && srcInfo.metadataOnly { + // Version ID is set for the destination and source == destination version ID. + // perform an in-place update. + if dstOpts.VersionID != "" && srcOpts.VersionID == dstOpts.VersionID { + srcInfo.Reader.Close() // We are not interested in the reader stream at this point close it. + return srcSet.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts) + } + // Destination is not versioned and source version ID is empty + // perform an in-place update. + if !dstOpts.Versioned && srcOpts.VersionID == "" { + srcInfo.Reader.Close() // We are not interested in the reader stream at this point close it. + return srcSet.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts) + } + // CopyObject optimization where we don't create an entire copy + // of the content, instead we add a reference, we disallow legacy + // objects to be self referenced in this manner so make sure + // that we actually create a new dataDir for legacy objects. + if dstOpts.Versioned && srcOpts.VersionID != dstOpts.VersionID && !srcInfo.Legacy { + srcInfo.versionOnly = true + srcInfo.Reader.Close() // We are not interested in the reader stream at this point close it. + return srcSet.CopyObject(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts) + } + } + + putOpts := ObjectOptions{ + ServerSideEncryption: dstOpts.ServerSideEncryption, + UserDefined: srcInfo.UserDefined, + Versioned: dstOpts.Versioned, + VersionID: dstOpts.VersionID, + MTime: dstOpts.MTime, + EncryptFn: dstOpts.EncryptFn, + WantChecksum: dstOpts.WantChecksum, + WantServerSideChecksumType: dstOpts.WantServerSideChecksumType, + } + + return dstSet.putObject(ctx, dstBucket, dstObject, srcInfo.PutObjReader, putOpts) +} + +func (s *erasureSets) ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) { + // In list multipart uploads we are going to treat input prefix as the object, + // this means that we are not supporting directory navigation. + set := s.getHashedSet(prefix) + return set.ListMultipartUploads(ctx, bucket, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads) +} + +// Initiate a new multipart upload on a hashedSet based on object name. +func (s *erasureSets) NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (res *NewMultipartUploadResult, err error) { + set := s.getHashedSet(object) + return set.NewMultipartUpload(ctx, bucket, object, opts) +} + +// PutObjectPart - writes part of an object to hashedSet based on the object name. +func (s *erasureSets) PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error) { + set := s.getHashedSet(object) + return set.PutObjectPart(ctx, bucket, object, uploadID, partID, data, opts) +} + +// GetMultipartInfo - return multipart metadata info uploaded at hashedSet. +func (s *erasureSets) GetMultipartInfo(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (result MultipartInfo, err error) { + set := s.getHashedSet(object) + return set.GetMultipartInfo(ctx, bucket, object, uploadID, opts) +} + +// ListObjectParts - lists all uploaded parts to an object in hashedSet. +func (s *erasureSets) ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int, opts ObjectOptions) (result ListPartsInfo, err error) { + set := s.getHashedSet(object) + return set.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts) +} + +// Aborts an in-progress multipart operation on hashedSet based on the object name. +func (s *erasureSets) AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error { + set := s.getHashedSet(object) + return set.AbortMultipartUpload(ctx, bucket, object, uploadID, opts) +} + +// CompleteMultipartUpload - completes a pending multipart transaction, on hashedSet based on object name. +func (s *erasureSets) CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error) { + set := s.getHashedSet(object) + return set.CompleteMultipartUpload(ctx, bucket, object, uploadID, uploadedParts, opts) +} + +/* + +All disks online +----------------- +- All Unformatted - format all and return success. +- Some Unformatted - format all and return success. +- Any JBOD inconsistent - return failure +- Some are corrupt (missing format.json) - return failure +- Any unrecognized disks - return failure + +Some disks are offline and we have quorum. +----------------- +- Some unformatted - format all and return success, + treat disks offline as corrupted. +- Any JBOD inconsistent - return failure +- Some are corrupt (missing format.json) +- Any unrecognized disks - return failure + +No read quorum +----------------- +failure for all cases. + +// Pseudo code for managing `format.json`. + +// Generic checks. +if (no quorum) return error +if (any disk is corrupt) return error // Always error +if (jbod inconsistent) return error // Always error. +if (disks not recognized) // Always error. + +// Specific checks. +if (all disks online) + if (all disks return format.json) + if (jbod consistent) + if (all disks recognized) + return + else + if (all disks return format.json not found) + return error + else (some disks return format.json not found) + (heal format) + return + fi + fi +else + if (some disks return format.json not found) + // Offline disks are marked as dead. + (heal format) // Offline disks should be marked as dead. + return success + fi +fi +*/ + +func formatsToDrivesInfo(endpoints Endpoints, formats []*formatErasureV3, sErrs []error) (beforeDrives []madmin.HealDriveInfo) { + beforeDrives = make([]madmin.HealDriveInfo, len(endpoints)) + // Existing formats are available (i.e. ok), so save it in + // result, also populate disks to be healed. + for i, format := range formats { + drive := endpoints.GetString(i) + state := madmin.DriveStateCorrupt + switch { + case format != nil: + state = madmin.DriveStateOk + case sErrs[i] == errUnformattedDisk: + state = madmin.DriveStateMissing + case sErrs[i] == errDiskNotFound: + state = madmin.DriveStateOffline + } + beforeDrives[i] = madmin.HealDriveInfo{ + UUID: func() string { + if format != nil { + return format.Erasure.This + } + return "" + }(), + Endpoint: drive, + State: state, + } + } + + return beforeDrives +} + +// HealFormat - heals missing `format.json` on fresh unformatted disks. +func (s *erasureSets) HealFormat(ctx context.Context, dryRun bool) (res madmin.HealResultItem, err error) { + storageDisks, _ := initStorageDisksWithErrors(s.endpoints.Endpoints, storageOpts{ + cleanUp: false, + healthCheck: false, + }) + + defer func(storageDisks []StorageAPI) { + if err != nil { + closeStorageDisks(storageDisks...) + } + }(storageDisks) + + formats, sErrs := loadFormatErasureAll(storageDisks, true) + if err = checkFormatErasureValues(formats, storageDisks, s.setDriveCount); err != nil { + return madmin.HealResultItem{}, err + } + + refFormat, err := getFormatErasureInQuorum(formats) + if err != nil { + return res, err + } + + // Prepare heal-result + res = madmin.HealResultItem{ + Type: madmin.HealItemMetadata, + Detail: "disk-format", + DiskCount: s.setCount * s.setDriveCount, + SetCount: s.setCount, + } + + // Fetch all the drive info status. + beforeDrives := formatsToDrivesInfo(s.endpoints.Endpoints, formats, sErrs) + + res.After.Drives = make([]madmin.HealDriveInfo, len(beforeDrives)) + res.Before.Drives = make([]madmin.HealDriveInfo, len(beforeDrives)) + // Copy "after" drive state too from before. + for k, v := range beforeDrives { + res.Before.Drives[k] = v + res.After.Drives[k] = v + } + + if countErrs(sErrs, errUnformattedDisk) == 0 { + return res, errNoHealRequired + } + + if !reflect.DeepEqual(s.format, refFormat) { + // Format is corrupted and unrecognized by the running instance. + healingLogIf(ctx, fmt.Errorf("Unable to heal the newly replaced drives due to format.json inconsistencies, please engage MinIO support for further assistance: %w", + errCorruptedFormat)) + return res, errCorruptedFormat + } + + formatOpID := mustGetUUID() + + // Initialize a new set of set formats which will be written to disk. + newFormatSets, currentDisksInfo := newHealFormatSets(refFormat, s.setCount, s.setDriveCount, formats, sErrs) + + if !dryRun { + tmpNewFormats := make([]*formatErasureV3, s.setCount*s.setDriveCount) + for i := range newFormatSets { + for j := range newFormatSets[i] { + if newFormatSets[i][j] == nil { + continue + } + res.After.Drives[i*s.setDriveCount+j].UUID = newFormatSets[i][j].Erasure.This + res.After.Drives[i*s.setDriveCount+j].State = madmin.DriveStateOk + tmpNewFormats[i*s.setDriveCount+j] = newFormatSets[i][j] + } + } + + // Save new formats `format.json` on unformatted disks. + for index, format := range tmpNewFormats { + if storageDisks[index] == nil || format == nil { + continue + } + if err := saveFormatErasure(storageDisks[index], format, formatOpID); err != nil { + healingLogIf(ctx, fmt.Errorf("Drive %s failed to write updated 'format.json': %v", storageDisks[index], err)) + storageDisks[index].Close() + tmpNewFormats[index] = nil // this disk failed to write new format + } + } + + s.erasureDisksMu.Lock() + + for index, format := range tmpNewFormats { + if format == nil { + continue + } + + m, n, err := findDiskIndexByDiskID(refFormat, format.Erasure.This) + if err != nil { + healingLogIf(ctx, err) + continue + } + + if s.erasureDisks[m][n] != nil { + s.erasureDisks[m][n].Close() + } + + if disk := storageDisks[index]; disk != nil { + if disk.IsLocal() { + xldisk, ok := disk.(*xlStorageDiskIDCheck) + if ok { + _, commonDeletes := calcCommonWritesDeletes(currentDisksInfo[m], (s.setDriveCount+1)/2) + xldisk.totalDeletes.Store(commonDeletes) + xldisk.storage.setDeleteAttribute(commonDeletes) + + if globalDriveMonitoring { + go xldisk.monitorDiskWritable(xldisk.diskCtx) + } + } + } else { + disk.Close() // Close the remote storage client, re-initialize with healthchecks. + disk, err = newStorageRESTClient(disk.Endpoint(), true, globalGrid.Load()) + if err != nil { + continue + } + } + + s.erasureDisks[m][n] = disk + + if disk.IsLocal() { + globalLocalDrivesMu.Lock() + if globalIsDistErasure { + globalLocalSetDrives[s.poolIndex][m][n] = disk + } + globalLocalDrivesMap[disk.Endpoint().String()] = disk + globalLocalDrivesMu.Unlock() + } + } + } + + s.erasureDisksMu.Unlock() + } + + return res, nil +} + +// HealObject - heals inconsistent object on a hashedSet based on object name. +func (s *erasureSets) HealObject(ctx context.Context, bucket, object, versionID string, opts madmin.HealOpts) (madmin.HealResultItem, error) { + return s.getHashedSet(object).HealObject(ctx, bucket, object, versionID, opts) +} + +// PutObjectMetadata - replace or add metadata to an existing object/version +func (s *erasureSets) PutObjectMetadata(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) { + er := s.getHashedSet(object) + return er.PutObjectMetadata(ctx, bucket, object, opts) +} + +// DecomTieredObject - moves tiered object to another pool during decommissioning. +func (s *erasureSets) DecomTieredObject(ctx context.Context, bucket, object string, fi FileInfo, opts ObjectOptions) error { + er := s.getHashedSet(object) + return er.DecomTieredObject(ctx, bucket, object, fi, opts) +} + +// PutObjectTags - replace or add tags to an existing object +func (s *erasureSets) PutObjectTags(ctx context.Context, bucket, object string, tags string, opts ObjectOptions) (ObjectInfo, error) { + er := s.getHashedSet(object) + return er.PutObjectTags(ctx, bucket, object, tags, opts) +} + +// DeleteObjectTags - delete object tags from an existing object +func (s *erasureSets) DeleteObjectTags(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) { + er := s.getHashedSet(object) + return er.DeleteObjectTags(ctx, bucket, object, opts) +} + +// GetObjectTags - get object tags from an existing object +func (s *erasureSets) GetObjectTags(ctx context.Context, bucket, object string, opts ObjectOptions) (*tags.Tags, error) { + er := s.getHashedSet(object) + return er.GetObjectTags(ctx, bucket, object, opts) +} + +// TransitionObject - transition object content to target tier. +func (s *erasureSets) TransitionObject(ctx context.Context, bucket, object string, opts ObjectOptions) error { + return s.getHashedSet(object).TransitionObject(ctx, bucket, object, opts) +} + +// RestoreTransitionedObject - restore transitioned object content locally on this cluster. +func (s *erasureSets) RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error { + return s.getHashedSet(object).RestoreTransitionedObject(ctx, bucket, object, opts) +} + +// CheckAbandonedParts - check object for abandoned parts. +func (s *erasureSets) CheckAbandonedParts(ctx context.Context, bucket, object string, opts madmin.HealOpts) error { + return s.getHashedSet(object).checkAbandonedParts(ctx, bucket, object, opts) +} diff --git a/cmd/erasure-sets_test.go b/cmd/erasure-sets_test.go new file mode 100644 index 00000000..63311d55 --- /dev/null +++ b/cmd/erasure-sets_test.go @@ -0,0 +1,237 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/google/uuid" +) + +var testUUID = uuid.MustParse("f5c58c61-7175-4018-ab5e-a94fe9c2de4e") + +func BenchmarkCrcHash(b *testing.B) { + cases := []struct { + key int + }{ + {16}, + {64}, + {128}, + {256}, + {512}, + {1024}, + } + for _, testCase := range cases { + key := randString(testCase.key) + b.Run("", func(b *testing.B) { + b.SetBytes(1024) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + crcHashMod(key, 16) + } + }) + } +} + +func BenchmarkSipHash(b *testing.B) { + cases := []struct { + key int + }{ + {16}, + {64}, + {128}, + {256}, + {512}, + {1024}, + } + for _, testCase := range cases { + key := randString(testCase.key) + b.Run("", func(b *testing.B) { + b.SetBytes(1024) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + sipHashMod(key, 16, testUUID) + } + }) + } +} + +// TestSipHashMod - test sip hash. +func TestSipHashMod(t *testing.T) { + testCases := []struct { + objectName string + sipHash int + }{ + // cases which should pass the test. + // passing in valid object name. + {"object", 37}, + {"The Shining Script .pdf", 38}, + {"Cost Benefit Analysis (2009-2010).pptx", 59}, + {"117Gn8rfHL2ACARPAhaFd0AGzic9pUbIA/5OCn5A", 35}, + {"SHØRT", 49}, + {"There are far too many object names, and far too few bucket names!", 8}, + {"a/b/c/", 159}, + {"/a/b/c", 96}, + {string([]byte{0xff, 0xfe, 0xfd}), 147}, + } + + // Tests hashing order to be consistent. + for i, testCase := range testCases { + if sipHashElement := hashKey("SIPMOD", testCase.objectName, 200, testUUID); sipHashElement != testCase.sipHash { + t.Errorf("Test case %d: Expected \"%v\" but failed \"%v\"", i+1, testCase.sipHash, sipHashElement) + } + } + + if sipHashElement := hashKey("SIPMOD", "This will fail", -1, testUUID); sipHashElement != -1 { + t.Errorf("Test: Expected \"-1\" but got \"%v\"", sipHashElement) + } + + if sipHashElement := hashKey("SIPMOD", "This will fail", 0, testUUID); sipHashElement != -1 { + t.Errorf("Test: Expected \"-1\" but got \"%v\"", sipHashElement) + } + + if sipHashElement := hashKey("UNKNOWN", "This will fail", 0, testUUID); sipHashElement != -1 { + t.Errorf("Test: Expected \"-1\" but got \"%v\"", sipHashElement) + } +} + +// TestCrcHashMod - test crc hash. +func TestCrcHashMod(t *testing.T) { + testCases := []struct { + objectName string + crcHash int + }{ + // cases which should pass the test. + // passing in valid object name. + {"object", 28}, + {"The Shining Script .pdf", 142}, + {"Cost Benefit Analysis (2009-2010).pptx", 133}, + {"117Gn8rfHL2ACARPAhaFd0AGzic9pUbIA/5OCn5A", 185}, + {"SHØRT", 97}, + {"There are far too many object names, and far too few bucket names!", 101}, + {"a/b/c/", 193}, + {"/a/b/c", 116}, + {string([]byte{0xff, 0xfe, 0xfd}), 61}, + } + + // Tests hashing order to be consistent. + for i, testCase := range testCases { + if crcHashElement := hashKey("CRCMOD", testCase.objectName, 200, testUUID); crcHashElement != testCase.crcHash { + t.Errorf("Test case %d: Expected \"%v\" but failed \"%v\"", i+1, testCase.crcHash, crcHashElement) + } + } + + if crcHashElement := hashKey("CRCMOD", "This will fail", -1, testUUID); crcHashElement != -1 { + t.Errorf("Test: Expected \"-1\" but got \"%v\"", crcHashElement) + } + + if crcHashElement := hashKey("CRCMOD", "This will fail", 0, testUUID); crcHashElement != -1 { + t.Errorf("Test: Expected \"-1\" but got \"%v\"", crcHashElement) + } + + if crcHashElement := hashKey("UNKNOWN", "This will fail", 0, testUUID); crcHashElement != -1 { + t.Errorf("Test: Expected \"-1\" but got \"%v\"", crcHashElement) + } +} + +// TestNewErasure - tests initialization of all input disks +// and constructs a valid `Erasure` object +func TestNewErasureSets(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + nDisks := 16 // Maximum disks. + var erasureDisks []string + for range nDisks { + // Do not attempt to create this path, the test validates + // so that newErasureSets initializes non existing paths + // and successfully returns initialized object layer. + disk := filepath.Join(globalTestTmpDir, "minio-"+nextSuffix()) + erasureDisks = append(erasureDisks, disk) + defer os.RemoveAll(disk) + } + + endpoints := mustGetNewEndpoints(0, 16, erasureDisks...) + _, _, err := waitForFormatErasure(true, endpoints, 1, 0, 16, "") + if err != errInvalidArgument { + t.Fatalf("Expecting error, got %s", err) + } + + _, _, err = waitForFormatErasure(true, nil, 1, 1, 16, "") + if err != errInvalidArgument { + t.Fatalf("Expecting error, got %s", err) + } + + // Initializes all erasure disks + storageDisks, format, err := waitForFormatErasure(true, endpoints, 1, 1, 16, "") + if err != nil { + t.Fatalf("Unable to format drives for erasure, %s", err) + } + + ep := PoolEndpoints{Endpoints: endpoints} + + parity, err := ecDrivesNoConfig(16) + if err != nil { + t.Fatalf("Unexpected error during EC drive config: %v", err) + } + if _, err := newErasureSets(ctx, ep, storageDisks, format, parity, 0); err != nil { + t.Fatalf("Unable to initialize erasure") + } +} + +// TestHashedLayer - tests the hashed layer which will be returned +// consistently for a given object name. +func TestHashedLayer(t *testing.T) { + // Test distribution with 16 sets. + var objs [16]*erasureObjects + for i := range objs { + objs[i] = &erasureObjects{} + } + + sets := &erasureSets{sets: objs[:], distributionAlgo: "CRCMOD"} + + testCases := []struct { + objectName string + expectedObj *erasureObjects + }{ + // cases which should pass the test. + // passing in valid object name. + {"object", objs[12]}, + {"The Shining Script .pdf", objs[14]}, + {"Cost Benefit Analysis (2009-2010).pptx", objs[13]}, + {"117Gn8rfHL2ACARPAhaFd0AGzic9pUbIA/5OCn5A", objs[1]}, + {"SHØRT", objs[9]}, + {"There are far too many object names, and far too few bucket names!", objs[13]}, + {"a/b/c/", objs[1]}, + {"/a/b/c", objs[4]}, + {string([]byte{0xff, 0xfe, 0xfd}), objs[13]}, + } + + // Tests hashing order to be consistent. + for i, testCase := range testCases { + gotObj := sets.getHashedSet(testCase.objectName) + if gotObj != testCase.expectedObj { + t.Errorf("Test case %d: Expected \"%#v\" but failed \"%#v\"", i+1, testCase.expectedObj, gotObj) + } + } +} diff --git a/cmd/erasure-utils.go b/cmd/erasure-utils.go new file mode 100644 index 00000000..35385fed --- /dev/null +++ b/cmd/erasure-utils.go @@ -0,0 +1,118 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/base64" + "fmt" + "io" + "strings" + + "github.com/klauspost/reedsolomon" +) + +// getDataBlockLen - get length of data blocks from encoded blocks. +func getDataBlockLen(enBlocks [][]byte, dataBlocks int) int { + size := 0 + // Figure out the data block length. + for _, block := range enBlocks[:dataBlocks] { + size += len(block) + } + return size +} + +// Writes all the data blocks from encoded blocks until requested +// outSize length. Provides a way to skip bytes until the offset. +func writeDataBlocks(ctx context.Context, dst io.Writer, enBlocks [][]byte, dataBlocks int, offset int64, length int64) (int64, error) { + // Offset and out size cannot be negative. + if offset < 0 || length < 0 { + return 0, errUnexpected + } + + // Do we have enough blocks? + if len(enBlocks) < dataBlocks { + return 0, reedsolomon.ErrTooFewShards + } + + // Do we have enough data? + if int64(getDataBlockLen(enBlocks, dataBlocks)) < length { + return 0, reedsolomon.ErrShortData + } + + // Counter to decrement total left to write. + write := length + + // Counter to increment total written. + var totalWritten int64 + + // Write all data blocks to dst. + for _, block := range enBlocks[:dataBlocks] { + // Skip blocks until we have reached our offset. + if offset >= int64(len(block)) { + // Decrement offset. + offset -= int64(len(block)) + continue + } + + // Skip until offset. + block = block[offset:] + + // Reset the offset for next iteration to read everything + // from subsequent blocks. + offset = 0 + + // We have written all the blocks, write the last remaining block. + if write < int64(len(block)) { + n, err := dst.Write(block[:write]) + if err != nil { + return 0, err + } + totalWritten += int64(n) + break + } + + // Copy the block. + n, err := dst.Write(block) + if err != nil { + return 0, err + } + + // Decrement output size. + write -= int64(n) + + // Increment written. + totalWritten += int64(n) + } + + // Success. + return totalWritten, nil +} + +// returns deploymentID from uploadID +func getDeplIDFromUpload(uploadID string) (string, error) { + uploadBytes, err := base64.RawURLEncoding.DecodeString(uploadID) + if err != nil { + return "", fmt.Errorf("error parsing uploadID %s (%w)", uploadID, err) + } + slc := strings.SplitN(string(uploadBytes), ".", 2) + if len(slc) != 2 { + return "", fmt.Errorf("uploadID %s has incorrect format", uploadID) + } + return slc[0], nil +} diff --git a/cmd/erasure.go b/cmd/erasure.go new file mode 100644 index 00000000..4e6674c3 --- /dev/null +++ b/cmd/erasure.go @@ -0,0 +1,572 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "maps" + "math/rand" + "os" + "runtime" + "sort" + "sync" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/dsync" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/pkg/v3/sync/errgroup" +) + +// list all errors that can be ignore in a bucket operation. +var bucketOpIgnoredErrs = append(baseIgnoredErrs, errDiskAccessDenied, errUnformattedDisk) + +// list all errors that can be ignored in a bucket metadata operation. +var bucketMetadataOpIgnoredErrs = append(bucketOpIgnoredErrs, errVolumeNotFound) + +// OfflineDisk represents an unavailable disk. +var OfflineDisk StorageAPI // zero value is nil + +// erasureObjects - Implements ER object layer. +type erasureObjects struct { + setDriveCount int + defaultParityCount int + + setIndex int + poolIndex int + + // getDisks returns list of storageAPIs. + getDisks func() []StorageAPI + + // getLockers returns list of remote and local lockers. + getLockers func() ([]dsync.NetLocker, string) + + // getEndpoints returns list of endpoint belonging this set. + // some may be local and some remote. + getEndpoints func() []Endpoint + + // getEndpoints returns list of endpoint strings belonging this set. + // some may be local and some remote. + getEndpointStrings func() []string + + // Locker mutex map. + nsMutex *nsLockMap +} + +// NewNSLock - initialize a new namespace RWLocker instance. +func (er erasureObjects) NewNSLock(bucket string, objects ...string) RWLocker { + return er.nsMutex.NewNSLock(er.getLockers, bucket, objects...) +} + +// Shutdown function for object storage interface. +func (er erasureObjects) Shutdown(ctx context.Context) error { + // Add any object layer shutdown activities here. + closeStorageDisks(er.getDisks()...) + return nil +} + +// defaultWQuorum write quorum based on setDriveCount and defaultParityCount +func (er erasureObjects) defaultWQuorum() int { + dataCount := er.setDriveCount - er.defaultParityCount + if dataCount == er.defaultParityCount { + return dataCount + 1 + } + return dataCount +} + +// defaultRQuorum read quorum based on setDriveCount and defaultParityCount +func (er erasureObjects) defaultRQuorum() int { + return er.setDriveCount - er.defaultParityCount +} + +func diskErrToDriveState(err error) (state string) { + switch { + case errors.Is(err, errDiskNotFound) || errors.Is(err, context.DeadlineExceeded): + state = madmin.DriveStateOffline + case errors.Is(err, errCorruptedFormat) || errors.Is(err, errCorruptedBackend): + state = madmin.DriveStateCorrupt + case errors.Is(err, errUnformattedDisk): + state = madmin.DriveStateUnformatted + case errors.Is(err, errDiskAccessDenied): + state = madmin.DriveStatePermission + case errors.Is(err, errFaultyDisk): + state = madmin.DriveStateFaulty + case errors.Is(err, errDriveIsRoot): + state = madmin.DriveStateRootMount + case err == nil: + state = madmin.DriveStateOk + default: + state = fmt.Sprintf("%s (cause: %s)", madmin.DriveStateUnknown, err) + } + + return state +} + +func getOnlineOfflineDisksStats(disksInfo []madmin.Disk) (onlineDisks, offlineDisks madmin.BackendDisks) { + onlineDisks = make(madmin.BackendDisks) + offlineDisks = make(madmin.BackendDisks) + + for _, disk := range disksInfo { + ep := disk.Endpoint + if _, ok := offlineDisks[ep]; !ok { + offlineDisks[ep] = 0 + } + if _, ok := onlineDisks[ep]; !ok { + onlineDisks[ep] = 0 + } + } + + // Wait for the routines. + for _, disk := range disksInfo { + ep := disk.Endpoint + state := disk.State + if state != madmin.DriveStateOk && state != madmin.DriveStateUnformatted { + offlineDisks[ep]++ + continue + } + onlineDisks[ep]++ + } + + rootDiskCount := 0 + for _, di := range disksInfo { + if di.RootDisk { + rootDiskCount++ + } + } + + // Count offline disks as well to ensure consistent + // reportability of offline drives on local setups. + if len(disksInfo) == (rootDiskCount + offlineDisks.Sum()) { + // Success. + return onlineDisks, offlineDisks + } + + // Root disk should be considered offline + for i := range disksInfo { + ep := disksInfo[i].Endpoint + if disksInfo[i].RootDisk { + offlineDisks[ep]++ + onlineDisks[ep]-- + } + } + + return onlineDisks, offlineDisks +} + +// getDisksInfo - fetch disks info across all other storage API. +func getDisksInfo(disks []StorageAPI, endpoints []Endpoint, metrics bool) (disksInfo []madmin.Disk) { + disksInfo = make([]madmin.Disk, len(disks)) + + g := errgroup.WithNErrs(len(disks)) + for index := range disks { + g.Go(func() error { + di := madmin.Disk{ + Endpoint: endpoints[index].String(), + PoolIndex: endpoints[index].PoolIdx, + SetIndex: endpoints[index].SetIdx, + DiskIndex: endpoints[index].DiskIdx, + Local: endpoints[index].IsLocal, + } + if disks[index] == OfflineDisk { + di.State = diskErrToDriveState(errDiskNotFound) + disksInfo[index] = di + return nil + } + info, err := disks[index].DiskInfo(context.TODO(), DiskInfoOptions{Metrics: metrics}) + di.DrivePath = info.MountPath + di.TotalSpace = info.Total + di.UsedSpace = info.Used + di.AvailableSpace = info.Free + di.UUID = info.ID + di.Major = info.Major + di.Minor = info.Minor + di.RootDisk = info.RootDisk + di.Healing = info.Healing + di.Scanning = info.Scanning + di.State = diskErrToDriveState(err) + di.FreeInodes = info.FreeInodes + di.UsedInodes = info.UsedInodes + if hi := disks[index].Healing(); hi != nil { + hd := hi.toHealingDisk() + di.HealInfo = &hd + } + di.Metrics = &madmin.DiskMetrics{ + LastMinute: make(map[string]madmin.TimedAction, len(info.Metrics.LastMinute)), + APICalls: make(map[string]uint64, len(info.Metrics.APICalls)), + TotalErrorsAvailability: info.Metrics.TotalErrorsAvailability, + TotalErrorsTimeout: info.Metrics.TotalErrorsTimeout, + TotalWaiting: info.Metrics.TotalWaiting, + } + for k, v := range info.Metrics.LastMinute { + if v.N > 0 { + di.Metrics.LastMinute[k] = v.asTimedAction() + } + } + maps.Copy(di.Metrics.APICalls, info.Metrics.APICalls) + if info.Total > 0 { + di.Utilization = float64(info.Used / info.Total * 100) + } + disksInfo[index] = di + return nil + }, index) + } + + g.Wait() + return disksInfo +} + +// Get an aggregated storage info across all disks. +func getStorageInfo(disks []StorageAPI, endpoints []Endpoint, metrics bool) StorageInfo { + disksInfo := getDisksInfo(disks, endpoints, metrics) + + // Sort so that the first element is the smallest. + sort.Slice(disksInfo, func(i, j int) bool { + return disksInfo[i].TotalSpace < disksInfo[j].TotalSpace + }) + + storageInfo := StorageInfo{ + Disks: disksInfo, + } + + storageInfo.Backend.Type = madmin.Erasure + return storageInfo +} + +// StorageInfo - returns underlying storage statistics. +func (er erasureObjects) StorageInfo(ctx context.Context) StorageInfo { + disks := er.getDisks() + endpoints := er.getEndpoints() + return getStorageInfo(disks, endpoints, true) +} + +// LocalStorageInfo - returns underlying local storage statistics. +func (er erasureObjects) LocalStorageInfo(ctx context.Context, metrics bool) StorageInfo { + disks := er.getDisks() + endpoints := er.getEndpoints() + + var localDisks []StorageAPI + var localEndpoints []Endpoint + + for i, endpoint := range endpoints { + if endpoint.IsLocal { + localDisks = append(localDisks, disks[i]) + localEndpoints = append(localEndpoints, endpoint) + } + } + + return getStorageInfo(localDisks, localEndpoints, metrics) +} + +// getOnlineDisksWithHealingAndInfo - returns online disks and overall healing status. +// Disks are ordered in the following groups: +// - Non-scanning disks +// - Non-healing disks +// - Healing disks (if inclHealing is true) +func (er erasureObjects) getOnlineDisksWithHealingAndInfo(inclHealing bool) (newDisks []StorageAPI, newInfos []DiskInfo, healing int) { + var wg sync.WaitGroup + disks := er.getDisks() + infos := make([]DiskInfo, len(disks)) + r := rand.New(rand.NewSource(time.Now().UnixNano())) + for _, i := range r.Perm(len(disks)) { + wg.Add(1) + go func() { + defer wg.Done() + + disk := disks[i] + if disk == nil { + infos[i].Error = errDiskNotFound.Error() + return + } + + di, err := disk.DiskInfo(context.Background(), DiskInfoOptions{}) + infos[i] = di + if err != nil { + // - Do not consume disks which are not reachable + // unformatted or simply not accessible for some reason. + infos[i].Error = err.Error() + } + }() + } + wg.Wait() + + var scanningDisks, healingDisks []StorageAPI + var scanningInfos, healingInfos []DiskInfo + + for i, info := range infos { + // Check if one of the drives in the set is being healed. + // this information is used by scanner to skip healing + // this erasure set while it calculates the usage. + if info.Error != "" || disks[i] == nil { + continue + } + if info.Healing { + healing++ + if inclHealing { + healingDisks = append(healingDisks, disks[i]) + healingInfos = append(healingInfos, infos[i]) + } + continue + } + + if !info.Scanning { + newDisks = append(newDisks, disks[i]) + newInfos = append(newInfos, infos[i]) + } else { + scanningDisks = append(scanningDisks, disks[i]) + scanningInfos = append(scanningInfos, infos[i]) + } + } + + // Prefer non-scanning disks over disks which are currently being scanned. + newDisks = append(newDisks, scanningDisks...) + newInfos = append(newInfos, scanningInfos...) + + /// Then add healing disks. + newDisks = append(newDisks, healingDisks...) + newInfos = append(newInfos, healingInfos...) + + return newDisks, newInfos, healing +} + +func (er erasureObjects) getOnlineDisksWithHealing(inclHealing bool) ([]StorageAPI, bool) { + newDisks, _, healing := er.getOnlineDisksWithHealingAndInfo(inclHealing) + return newDisks, healing > 0 +} + +// Clean-up previously deleted objects. from .minio.sys/tmp/.trash/ +func (er erasureObjects) cleanupDeletedObjects(ctx context.Context) { + var wg sync.WaitGroup + for _, disk := range er.getLocalDisks() { + if disk == nil { + continue + } + wg.Add(1) + go func(disk StorageAPI) { + defer wg.Done() + drivePath := disk.Endpoint().Path + readDirFn(pathJoin(drivePath, minioMetaTmpDeletedBucket), func(ddir string, typ os.FileMode) error { + w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) + return w.Run(func() error { + wait := deleteCleanupSleeper.Timer(ctx) + removeAll(pathJoin(drivePath, minioMetaTmpDeletedBucket, ddir)) + wait() + return nil + }) + }) + }(disk) + } + wg.Wait() +} + +// nsScanner will start scanning buckets and send updated totals as they are traversed. +// Updates are sent on a regular basis and the caller *must* consume them. +func (er erasureObjects) nsScanner(ctx context.Context, buckets []BucketInfo, wantCycle uint32, updates chan<- dataUsageCache, healScanMode madmin.HealScanMode) error { + if len(buckets) == 0 { + return nil + } + + // Collect disks we can use. + disks, healing := er.getOnlineDisksWithHealing(false) + if len(disks) == 0 { + scannerLogIf(ctx, errors.New("data-scanner: all drives are offline or being healed, skipping scanner cycle")) + return nil + } + + // Load bucket totals + oldCache := dataUsageCache{} + if err := oldCache.load(ctx, er, dataUsageCacheName); err != nil { + return err + } + + // New cache.. + cache := dataUsageCache{ + Info: dataUsageCacheInfo{ + Name: dataUsageRoot, + NextCycle: oldCache.Info.NextCycle, + }, + Cache: make(map[string]dataUsageEntry, len(oldCache.Cache)), + } + + // Put all buckets into channel. + bucketCh := make(chan BucketInfo, len(buckets)) + + // Shuffle buckets to ensure total randomness of buckets, being scanned. + // Otherwise same set of buckets get scanned across erasure sets always. + // at any given point in time. This allows different buckets to be scanned + // in different order per erasure set, this wider spread is needed when + // there are lots of buckets with different order of objects in them. + r := rand.New(rand.NewSource(time.Now().UnixNano())) + permutes := r.Perm(len(buckets)) + // Add new buckets first + for _, idx := range permutes { + b := buckets[idx] + if e := oldCache.find(b.Name); e == nil { + bucketCh <- b + } + } + for _, idx := range permutes { + b := buckets[idx] + if e := oldCache.find(b.Name); e != nil { + cache.replace(b.Name, dataUsageRoot, *e) + bucketCh <- b + } + } + xioutil.SafeClose(bucketCh) + + bucketResults := make(chan dataUsageEntryInfo, len(disks)) + + // Start async collector/saver. + // This goroutine owns the cache. + var saverWg sync.WaitGroup + saverWg.Add(1) + go func() { + // Add jitter to the update time so multiple sets don't sync up. + updateTime := 30*time.Second + time.Duration(float64(10*time.Second)*rand.Float64()) + t := time.NewTicker(updateTime) + defer t.Stop() + defer saverWg.Done() + var lastSave time.Time + + for { + select { + case <-t.C: + if cache.Info.LastUpdate.Equal(lastSave) { + continue + } + scannerLogOnceIf(ctx, cache.save(ctx, er, dataUsageCacheName), "nsscanner-cache-update") + updates <- cache.clone() + + lastSave = cache.Info.LastUpdate + case v, ok := <-bucketResults: + if !ok { + // Save final state... + cache.Info.NextCycle = wantCycle + cache.Info.LastUpdate = time.Now() + scannerLogOnceIf(ctx, cache.save(ctx, er, dataUsageCacheName), "nsscanner-channel-closed") + updates <- cache.clone() + return + } + cache.replace(v.Name, v.Parent, v.Entry) + cache.Info.LastUpdate = time.Now() + } + } + }() + + // Restrict parallelism for disk usage scanner + // upto GOMAXPROCS if GOMAXPROCS is < len(disks) + maxProcs := runtime.GOMAXPROCS(0) + if maxProcs < len(disks) { + disks = disks[:maxProcs] + } + + // Start one scanner per disk + var wg sync.WaitGroup + wg.Add(len(disks)) + + for i := range disks { + go func(i int) { + defer wg.Done() + disk := disks[i] + + for bucket := range bucketCh { + select { + case <-ctx.Done(): + return + default: + } + + // Load cache for bucket + cacheName := pathJoin(bucket.Name, dataUsageCacheName) + cache := dataUsageCache{} + scannerLogIf(ctx, cache.load(ctx, er, cacheName)) + if cache.Info.Name == "" { + cache.Info.Name = bucket.Name + } + cache.Info.SkipHealing = healing + cache.Info.NextCycle = wantCycle + if cache.Info.Name != bucket.Name { + cache.Info = dataUsageCacheInfo{ + Name: bucket.Name, + LastUpdate: time.Time{}, + NextCycle: wantCycle, + } + } + // Collect updates. + updates := make(chan dataUsageEntry, 1) + var wg sync.WaitGroup + wg.Add(1) + go func(name string) { + defer wg.Done() + for update := range updates { + select { + case <-ctx.Done(): + case bucketResults <- dataUsageEntryInfo{ + Name: name, + Parent: dataUsageRoot, + Entry: update, + }: + } + } + }(cache.Info.Name) + // Calc usage + before := cache.Info.LastUpdate + var err error + cache, err = disk.NSScanner(ctx, cache, updates, healScanMode, nil) + if err != nil { + if !cache.Info.LastUpdate.IsZero() && cache.Info.LastUpdate.After(before) { + scannerLogIf(ctx, cache.save(ctx, er, cacheName)) + } else { + scannerLogIf(ctx, err) + } + // This ensures that we don't close + // bucketResults channel while the + // updates-collector goroutine still + // holds a reference to this. + wg.Wait() + continue + } + + wg.Wait() + // Flatten for upstream, but save full state. + var root dataUsageEntry + if r := cache.root(); r != nil { + root = cache.flatten(*r) + } + select { + case <-ctx.Done(): + return + case bucketResults <- dataUsageEntryInfo{ + Name: cache.Info.Name, + Parent: dataUsageRoot, + Entry: root, + }: + } + + // Save cache + scannerLogIf(ctx, cache.save(ctx, er, cacheName)) + } + }(i) + } + wg.Wait() + xioutil.SafeClose(bucketResults) + saverWg.Wait() + + return nil +} diff --git a/cmd/erasure_test.go b/cmd/erasure_test.go new file mode 100644 index 00000000..078771fd --- /dev/null +++ b/cmd/erasure_test.go @@ -0,0 +1,135 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "crypto/rand" + "io" + "testing" +) + +var erasureEncodeDecodeTests = []struct { + dataBlocks, parityBlocks int + missingData, missingParity int + reconstructParity bool + shouldFail bool +}{ + {dataBlocks: 2, parityBlocks: 2, missingData: 0, missingParity: 0, reconstructParity: true, shouldFail: false}, + {dataBlocks: 3, parityBlocks: 3, missingData: 1, missingParity: 0, reconstructParity: true, shouldFail: false}, + {dataBlocks: 4, parityBlocks: 4, missingData: 2, missingParity: 0, reconstructParity: false, shouldFail: false}, + {dataBlocks: 5, parityBlocks: 5, missingData: 0, missingParity: 1, reconstructParity: true, shouldFail: false}, + {dataBlocks: 6, parityBlocks: 6, missingData: 0, missingParity: 2, reconstructParity: true, shouldFail: false}, + {dataBlocks: 7, parityBlocks: 7, missingData: 1, missingParity: 1, reconstructParity: false, shouldFail: false}, + {dataBlocks: 8, parityBlocks: 8, missingData: 3, missingParity: 2, reconstructParity: false, shouldFail: false}, + {dataBlocks: 2, parityBlocks: 2, missingData: 2, missingParity: 1, reconstructParity: true, shouldFail: true}, + {dataBlocks: 4, parityBlocks: 2, missingData: 2, missingParity: 2, reconstructParity: false, shouldFail: true}, + {dataBlocks: 8, parityBlocks: 4, missingData: 2, missingParity: 2, reconstructParity: false, shouldFail: false}, +} + +func TestErasureEncodeDecode(t *testing.T) { + data := make([]byte, 256) + if _, err := io.ReadFull(rand.Reader, data); err != nil { + t.Fatalf("Failed to read random data: %v", err) + } + for i, test := range erasureEncodeDecodeTests { + buffer := make([]byte, len(data), 2*len(data)) + copy(buffer, data) + + erasure, err := NewErasure(t.Context(), test.dataBlocks, test.parityBlocks, blockSizeV2) + if err != nil { + t.Fatalf("Test %d: failed to create erasure: %v", i, err) + } + encoded, err := erasure.EncodeData(t.Context(), buffer) + if err != nil { + t.Fatalf("Test %d: failed to encode data: %v", i, err) + } + + for j := range encoded[:test.missingData] { + encoded[j] = nil + } + for j := test.dataBlocks; j < test.dataBlocks+test.missingParity; j++ { + encoded[j] = nil + } + + if test.reconstructParity { + err = erasure.DecodeDataAndParityBlocks(t.Context(), encoded) + } else { + err = erasure.DecodeDataBlocks(encoded) + } + + if err == nil && test.shouldFail { + t.Errorf("Test %d: test should fail but it passed", i) + } + if err != nil && !test.shouldFail { + t.Errorf("Test %d: test should pass but it failed: %v", i, err) + } + + decoded := encoded + if !test.shouldFail { + if test.reconstructParity { + for j := range decoded { + if decoded[j] == nil { + t.Errorf("Test %d: failed to reconstruct shard %d", i, j) + } + } + } else { + for j := range decoded[:test.dataBlocks] { + if decoded[j] == nil { + t.Errorf("Test %d: failed to reconstruct data shard %d", i, j) + } + } + } + + decodedData := new(bytes.Buffer) + if _, err = writeDataBlocks(t.Context(), decodedData, decoded, test.dataBlocks, 0, int64(len(data))); err != nil { + t.Errorf("Test %d: failed to write data blocks: %v", i, err) + } + if !bytes.Equal(decodedData.Bytes(), data) { + t.Errorf("Test %d: Decoded data does not match original data: got: %v want: %v", i, decodedData.Bytes(), data) + } + } + } +} + +// Setup for erasureCreateFile and erasureReadFile tests. +type erasureTestSetup struct { + dataBlocks int + parityBlocks int + blockSize int64 + diskPaths []string + disks []StorageAPI +} + +// Returns an initialized setup for erasure tests. +func newErasureTestSetup(tb testing.TB, dataBlocks int, parityBlocks int, blockSize int64) (*erasureTestSetup, error) { + diskPaths := make([]string, dataBlocks+parityBlocks) + disks := make([]StorageAPI, len(diskPaths)) + var err error + for i := range diskPaths { + disks[i], diskPaths[i], err = newXLStorageTestSetup(tb) + if err != nil { + return nil, err + } + err = disks[i].MakeVol(tb.Context(), "testbucket") + if err != nil { + return nil, err + } + } + return &erasureTestSetup{dataBlocks, parityBlocks, blockSize, diskPaths, disks}, nil +} diff --git a/cmd/etcd.go b/cmd/etcd.go new file mode 100644 index 00000000..e3cdebb7 --- /dev/null +++ b/cmd/etcd.go @@ -0,0 +1,91 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + + etcd "go.etcd.io/etcd/client/v3" +) + +var errEtcdUnreachable = errors.New("etcd is unreachable, please check your endpoints") + +func etcdErrToErr(err error, etcdEndpoints []string) error { + if err == nil { + return nil + } + switch err { + case context.DeadlineExceeded: + return fmt.Errorf("%w %s", errEtcdUnreachable, etcdEndpoints) + default: + return fmt.Errorf("unexpected error %w from etcd, please check your endpoints %s", err, etcdEndpoints) + } +} + +func saveKeyEtcdWithTTL(ctx context.Context, client *etcd.Client, key string, data []byte, ttl int64) error { + timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout) + defer cancel() + lease, err := client.Grant(timeoutCtx, ttl) + if err != nil { + return etcdErrToErr(err, client.Endpoints()) + } + _, err = client.Put(timeoutCtx, key, string(data), etcd.WithLease(lease.ID)) + etcdLogIf(ctx, err) + return etcdErrToErr(err, client.Endpoints()) +} + +func saveKeyEtcd(ctx context.Context, client *etcd.Client, key string, data []byte, opts ...options) error { + timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout) + defer cancel() + if len(opts) > 0 { + return saveKeyEtcdWithTTL(ctx, client, key, data, opts[0].ttl) + } + _, err := client.Put(timeoutCtx, key, string(data)) + etcdLogIf(ctx, err) + return etcdErrToErr(err, client.Endpoints()) +} + +func deleteKeyEtcd(ctx context.Context, client *etcd.Client, key string) error { + timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout) + defer cancel() + + _, err := client.Delete(timeoutCtx, key) + etcdLogIf(ctx, err) + return etcdErrToErr(err, client.Endpoints()) +} + +func readKeyEtcd(ctx context.Context, client *etcd.Client, key string) ([]byte, error) { + timeoutCtx, cancel := context.WithTimeout(ctx, defaultContextTimeout) + defer cancel() + resp, err := client.Get(timeoutCtx, key) + if err != nil { + etcdLogOnceIf(ctx, err, "etcd-retrieve-keys") + return nil, etcdErrToErr(err, client.Endpoints()) + } + if resp.Count == 0 { + return nil, errConfigNotFound + } + for _, ev := range resp.Kvs { + if string(ev.Key) == key { + return ev.Value, nil + } + } + return nil, errConfigNotFound +} diff --git a/cmd/event-notification.go b/cmd/event-notification.go new file mode 100644 index 00000000..ceda47ef --- /dev/null +++ b/cmd/event-notification.go @@ -0,0 +1,260 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "fmt" + "net/url" + "runtime" + "strings" + "sync" + + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/event" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/pubsub" + "github.com/minio/pkg/v3/policy" +) + +// EventNotifier - notifies external systems about events in MinIO. +type EventNotifier struct { + sync.RWMutex + targetList *event.TargetList + bucketRulesMap map[string]event.RulesMap +} + +// NewEventNotifier - creates new event notification object. +func NewEventNotifier(ctx context.Context) *EventNotifier { + // targetList/bucketRulesMap/bucketRemoteTargetRulesMap are populated by NotificationSys.InitBucketTargets() + return &EventNotifier{ + targetList: event.NewTargetList(ctx), + bucketRulesMap: make(map[string]event.RulesMap), + } +} + +// GetARNList - returns available ARNs. +func (evnot *EventNotifier) GetARNList() []string { + arns := []string{} + if evnot == nil { + return arns + } + region := globalSite.Region() + for targetID := range evnot.targetList.TargetMap() { + // httpclient target is part of ListenNotification + // which doesn't need to be listed as part of the ARN list + // This list is only meant for external targets, filter + // this out pro-actively. + if !strings.HasPrefix(targetID.ID, "httpclient+") { + arns = append(arns, targetID.ToARN(region).String()) + } + } + + return arns +} + +// Loads notification policies for all buckets into EventNotifier. +func (evnot *EventNotifier) set(bucket string, meta BucketMetadata) { + config := meta.notificationConfig + if config == nil { + return + } + region := globalSite.Region() + config.SetRegion(region) + if err := config.Validate(region, globalEventNotifier.targetList); err != nil { + if _, ok := err.(*event.ErrARNNotFound); !ok { + internalLogIf(GlobalContext, err) + } + } + evnot.AddRulesMap(bucket, config.ToRulesMap()) +} + +// Targets returns all the registered targets +func (evnot *EventNotifier) Targets() []event.Target { + return evnot.targetList.Targets() +} + +// InitBucketTargets - initializes event notification system from notification.xml of all buckets. +func (evnot *EventNotifier) InitBucketTargets(ctx context.Context, objAPI ObjectLayer) error { + if objAPI == nil { + return errServerNotInitialized + } + + if err := evnot.targetList.Add(globalNotifyTargetList.Targets()...); err != nil { + return err + } + evnot.targetList = evnot.targetList.Init(runtime.GOMAXPROCS(0)) // TODO: make this configurable (y4m4) + return nil +} + +// AddRulesMap - adds rules map for bucket name. +func (evnot *EventNotifier) AddRulesMap(bucketName string, rulesMap event.RulesMap) { + evnot.Lock() + defer evnot.Unlock() + + rulesMap = rulesMap.Clone() + + // Do not add for an empty rulesMap. + if len(rulesMap) == 0 { + delete(evnot.bucketRulesMap, bucketName) + } else { + evnot.bucketRulesMap[bucketName] = rulesMap + } +} + +// RemoveNotification - removes all notification configuration for bucket name. +func (evnot *EventNotifier) RemoveNotification(bucketName string) { + evnot.Lock() + defer evnot.Unlock() + + delete(evnot.bucketRulesMap, bucketName) +} + +// RemoveAllBucketTargets - closes and removes all notification targets. +func (evnot *EventNotifier) RemoveAllBucketTargets() { + evnot.Lock() + defer evnot.Unlock() + + targetIDSet := event.NewTargetIDSet() + for k := range evnot.targetList.TargetMap() { + targetIDSet[k] = struct{}{} + } + evnot.targetList.Remove(targetIDSet) +} + +// Send - sends the event to all registered notification targets +func (evnot *EventNotifier) Send(args eventArgs) { + evnot.RLock() + targetIDSet := evnot.bucketRulesMap[args.BucketName].Match(args.EventName, args.Object.Name) + evnot.RUnlock() + + if len(targetIDSet) == 0 { + return + } + + // If MINIO_API_SYNC_EVENTS is set, send events synchronously. + evnot.targetList.Send(args.ToEvent(true), targetIDSet, globalAPIConfig.isSyncEventsEnabled()) +} + +type eventArgs struct { + EventName event.Name + BucketName string + Object ObjectInfo + ReqParams map[string]string + RespElements map[string]string + Host string + UserAgent string +} + +// ToEvent - converts to notification event. +func (args eventArgs) ToEvent(escape bool) event.Event { + eventTime := UTCNow() + uniqueID := fmt.Sprintf("%X", eventTime.UnixNano()) + if !args.Object.ModTime.IsZero() { + uniqueID = fmt.Sprintf("%X", args.Object.ModTime.UnixNano()) + } + + respElements := map[string]string{ + "x-amz-request-id": args.RespElements["requestId"], + "x-amz-id-2": args.RespElements["nodeId"], + "x-minio-origin-endpoint": func() string { + if globalMinioEndpoint != "" { + return globalMinioEndpoint + } + return getAPIEndpoints()[0] + }(), // MinIO specific custom elements. + } + + // Add deployment as part of response elements. + respElements["x-minio-deployment-id"] = globalDeploymentID() + if args.RespElements["content-length"] != "" { + respElements["content-length"] = args.RespElements["content-length"] + } + + keyName := args.Object.Name + if escape { + keyName = url.QueryEscape(args.Object.Name) + } + + newEvent := event.Event{ + EventVersion: "2.0", + EventSource: "minio:s3", + AwsRegion: args.ReqParams["region"], + EventTime: eventTime.Format(event.AMZTimeFormat), + EventName: args.EventName, + UserIdentity: event.Identity{PrincipalID: args.ReqParams["principalId"]}, + RequestParameters: args.ReqParams, + ResponseElements: respElements, + S3: event.Metadata{ + SchemaVersion: "1.0", + ConfigurationID: "Config", + Bucket: event.Bucket{ + Name: args.BucketName, + OwnerIdentity: event.Identity{PrincipalID: args.ReqParams["principalId"]}, + ARN: policy.ResourceARNPrefix + args.BucketName, + }, + Object: event.Object{ + Key: keyName, + VersionID: args.Object.VersionID, + Sequencer: uniqueID, + }, + }, + Source: event.Source{ + Host: args.Host, + UserAgent: args.UserAgent, + }, + } + + isRemovedEvent := args.EventName == event.ObjectRemovedDelete || + args.EventName == event.ObjectRemovedDeleteMarkerCreated || + args.EventName == event.ObjectRemovedNoOP + + if !isRemovedEvent { + newEvent.S3.Object.ETag = args.Object.ETag + newEvent.S3.Object.Size = args.Object.Size + newEvent.S3.Object.ContentType = args.Object.ContentType + newEvent.S3.Object.UserMetadata = make(map[string]string, len(args.Object.UserDefined)) + for k, v := range args.Object.UserDefined { + if stringsHasPrefixFold(strings.ToLower(k), ReservedMetadataPrefixLower) { + continue + } + newEvent.S3.Object.UserMetadata[k] = v + } + } + + return newEvent +} + +func sendEvent(args eventArgs) { + // avoid generating a notification for REPLICA creation event. + if _, ok := args.ReqParams[xhttp.MinIOSourceReplicationRequest]; ok { + return + } + + args.Object.Size, _ = args.Object.GetActualSize() + + // remove sensitive encryption entries in metadata. + crypto.RemoveSensitiveEntries(args.Object.UserDefined) + crypto.RemoveInternalEntries(args.Object.UserDefined) + + if globalHTTPListen.NumSubscribers(pubsub.MaskFromMaskable(args.EventName)) > 0 { + globalHTTPListen.Publish(args.ToEvent(false)) + } + + globalEventNotifier.Send(args) +} diff --git a/cmd/fmt-gen.go b/cmd/fmt-gen.go new file mode 100644 index 00000000..c616c4dc --- /dev/null +++ b/cmd/fmt-gen.go @@ -0,0 +1,119 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "encoding/json" + "log" + "os" + "path/filepath" + + "github.com/klauspost/compress/zip" + "github.com/minio/cli" +) + +var fmtGenFlags = []cli.Flag{ + cli.IntFlag{ + Name: "parity", + Usage: "specify erasure code parity", + }, + cli.StringFlag{ + Name: "deployment-id", + Usage: "deployment-id of the MinIO cluster for which format.json is needed", + }, + cli.StringFlag{ + Name: "address", + Value: ":" + GlobalMinioDefaultPort, + Usage: "bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname", + EnvVar: "MINIO_ADDRESS", + }, +} + +var fmtGenCmd = cli.Command{ + Name: "fmt-gen", + Usage: "Generate format.json files for an erasure server pool", + Flags: append(fmtGenFlags, GlobalFlags...), + Action: fmtGenMain, + Hidden: true, + CustomHelpTemplate: `NAME: + {{.HelpName}} - {{.Usage}} + +USAGE: + {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}DIR1 [DIR2..] + {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}DIR{1...64} + {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}DIR{1...64} DIR{65...128} + +DIR: + DIR points to a directory on a filesystem. When you want to combine + multiple drives into a single large system, pass one directory per + filesystem separated by space. You may also use a '...' convention + to abbreviate the directory arguments. Remote directories in a + distributed setup are encoded as HTTP(s) URIs. +{{if .VisibleFlags}} +FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}}{{end}} +EXAMPLES: + 1. Generate format.json.zip containing format.json files for all drives in a distributed MinIO server pool of 32 nodes with 32 drives each. + {{.Prompt}} {{.HelpName}} http://node{1...32}.example.com/mnt/export{1...32} + +`, +} + +func fmtGenMain(ctxt *cli.Context) { + deploymentID := ctxt.String("deployment-id") + err := buildServerCtxt(ctxt, &globalServerCtxt) + if err != nil { + log.Fatalln(err) + } + handleCommonArgs(globalServerCtxt) + pools, _, err := createServerEndpoints(globalMinioAddr, globalServerCtxt.Layout.pools, globalServerCtxt.Layout.legacy) + if err != nil { + log.Fatalln(err) + } + + zipFile, err := os.Create("format.json.zip") + if err != nil { + log.Fatalf("failed to create format.json.zip: %v", err) + } + defer zipFile.Close() + fmtZipW := zip.NewWriter(zipFile) + defer fmtZipW.Close() + for _, pool := range pools { // for each pool + setCount, setDriveCount := pool.SetCount, pool.DrivesPerSet + format := newFormatErasureV3(setCount, setDriveCount) + format.ID = deploymentID + for i := range setCount { // for each erasure set + for j := range setDriveCount { + newFormat := format.Clone() + newFormat.Erasure.This = format.Erasure.Sets[i][j] + if deploymentID != "" { + newFormat.ID = deploymentID + } + drive := pool.Endpoints[i*setDriveCount+j] + fmtBytes, err := json.Marshal(newFormat) + if err != nil { + //nolint:gocritic + log.Fatalf("failed to marshal format.json for %s: %v", drive.String(), err) + } + fmtJSON := filepath.Join(drive.Host, drive.Path, minioMetaBucket, "format.json") + embedFileInZip(fmtZipW, fmtJSON, fmtBytes, 0o600) + } + } + } +} diff --git a/cmd/format-erasure.go b/cmd/format-erasure.go new file mode 100644 index 00000000..6ff6d9e9 --- /dev/null +++ b/cmd/format-erasure.go @@ -0,0 +1,720 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/fs" + "os" + "sync" + + "github.com/dustin/go-humanize" + "github.com/minio/minio/internal/color" + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/config/storageclass" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/sync/errgroup" +) + +const ( + // Represents Erasure backend. + formatBackendErasure = "xl" + + // Represents Erasure backend - single drive + formatBackendErasureSingle = "xl-single" + + // formatErasureV1.Erasure.Version - version '1'. + formatErasureVersionV1 = "1" + + // formatErasureV2.Erasure.Version - version '2'. + formatErasureVersionV2 = "2" + + // formatErasureV3.Erasure.Version - version '3'. + formatErasureVersionV3 = "3" + + // Distribution algorithm used, legacy + formatErasureVersionV2DistributionAlgoV1 = "CRCMOD" + + // Distributed algorithm used, with N/2 default parity + formatErasureVersionV3DistributionAlgoV2 = "SIPMOD" + + // Distributed algorithm used, with EC:4 default parity + formatErasureVersionV3DistributionAlgoV3 = "SIPMOD+PARITY" +) + +// Offline disk UUID represents an offline disk. +const offlineDiskUUID = "ffffffff-ffff-ffff-ffff-ffffffffffff" + +// Used to detect the version of "xl" format. +type formatErasureVersionDetect struct { + Erasure struct { + Version string `json:"version"` + } `json:"xl"` +} + +// Represents the V1 backend disk structure version +// under `.minio.sys` and actual data namespace. +// formatErasureV1 - structure holds format config version '1'. +type formatErasureV1 struct { + formatMetaV1 + Erasure struct { + Version string `json:"version"` // Version of 'xl' format. + Disk string `json:"drive"` // Disk field carries assigned disk uuid. + // JBOD field carries the input disk order generated the first + // time when fresh disks were supplied. + JBOD []string `json:"jbod"` + } `json:"xl"` // Erasure field holds xl format. +} + +// Represents the V2 backend disk structure version +// under `.minio.sys` and actual data namespace. +// formatErasureV2 - structure holds format config version '2'. +// The V2 format to support "large bucket" support where a bucket +// can span multiple erasure sets. +type formatErasureV2 struct { + formatMetaV1 + Erasure struct { + Version string `json:"version"` // Version of 'xl' format. + This string `json:"this"` // This field carries assigned disk uuid. + // Sets field carries the input disk order generated the first + // time when fresh disks were supplied, it is a two dimensional + // array second dimension represents list of disks used per set. + Sets [][]string `json:"sets"` + // Distribution algorithm represents the hashing algorithm + // to pick the right set index for an object. + DistributionAlgo string `json:"distributionAlgo"` + } `json:"xl"` +} + +// formatErasureV3 struct is same as formatErasureV2 struct except that formatErasureV3.Erasure.Version is "3" indicating +// the simplified multipart backend which is a flat hierarchy now. +// In .minio.sys/multipart we have: +// sha256(bucket/object)/uploadID/[xl.meta, part.1, part.2 ....] +type formatErasureV3 struct { + formatMetaV1 + Erasure struct { + Version string `json:"version"` // Version of 'xl' format. + This string `json:"this"` // This field carries assigned disk uuid. + // Sets field carries the input disk order generated the first + // time when fresh disks were supplied, it is a two dimensional + // array second dimension represents list of disks used per set. + Sets [][]string `json:"sets"` + // Distribution algorithm represents the hashing algorithm + // to pick the right set index for an object. + DistributionAlgo string `json:"distributionAlgo"` + } `json:"xl"` + Info DiskInfo `json:"-"` +} + +func (f *formatErasureV3) Drives() (drives int) { + for _, set := range f.Erasure.Sets { + drives += len(set) + } + return drives +} + +func (f *formatErasureV3) Clone() *formatErasureV3 { + b, err := json.Marshal(f) + if err != nil { + panic(err) + } + var dst formatErasureV3 + if err = json.Unmarshal(b, &dst); err != nil { + panic(err) + } + return &dst +} + +// Returns formatErasure.Erasure.Version +func newFormatErasureV3(numSets int, setLen int) *formatErasureV3 { + format := &formatErasureV3{} + format.Version = formatMetaVersionV1 + format.Format = formatBackendErasure + if setLen == 1 { + format.Format = formatBackendErasureSingle + } + format.ID = mustGetUUID() + format.Erasure.Version = formatErasureVersionV3 + format.Erasure.DistributionAlgo = formatErasureVersionV3DistributionAlgoV3 + format.Erasure.Sets = make([][]string, numSets) + + for i := range numSets { + format.Erasure.Sets[i] = make([]string, setLen) + for j := range setLen { + format.Erasure.Sets[i][j] = mustGetUUID() + } + } + return format +} + +// Returns format Erasure version after reading `format.json`, returns +// successfully the version only if the backend is Erasure. +func formatGetBackendErasureVersion(b []byte) (string, error) { + meta := &formatMetaV1{} + if err := json.Unmarshal(b, meta); err != nil { + return "", err + } + if meta.Version != formatMetaVersionV1 { + return "", fmt.Errorf(`format.Version expected: %s, got: %s`, formatMetaVersionV1, meta.Version) + } + if meta.Format != formatBackendErasure && meta.Format != formatBackendErasureSingle { + return "", fmt.Errorf(`found backend type %s, expected %s or %s - to migrate to a supported backend visit https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-migrate-fs-gateway.html`, meta.Format, formatBackendErasure, formatBackendErasureSingle) + } + // Erasure backend found, proceed to detect version. + format := &formatErasureVersionDetect{} + if err := json.Unmarshal(b, format); err != nil { + return "", err + } + return format.Erasure.Version, nil +} + +// Migrates all previous versions to latest version of `format.json`, +// this code calls migration in sequence, such as V1 is migrated to V2 +// first before it V2 migrates to V3.n +func formatErasureMigrate(export string) ([]byte, fs.FileInfo, error) { + formatPath := pathJoin(export, minioMetaBucket, formatConfigFile) + formatData, formatFi, err := xioutil.ReadFileWithFileInfo(formatPath) + if err != nil { + return nil, nil, err + } + + version, err := formatGetBackendErasureVersion(formatData) + if err != nil { + return nil, nil, fmt.Errorf("Drive %s: %w", export, err) + } + + migrate := func(formatPath string, formatData []byte) ([]byte, fs.FileInfo, error) { + if err = os.WriteFile(formatPath, formatData, 0o666); err != nil { + return nil, nil, err + } + formatFi, err := Lstat(formatPath) + if err != nil { + return nil, nil, err + } + return formatData, formatFi, nil + } + + switch version { + case formatErasureVersionV1: + formatData, err = formatErasureMigrateV1ToV2(formatData, version) + if err != nil { + return nil, nil, fmt.Errorf("Drive %s: %w", export, err) + } + // Migrate successful v1 => v2, proceed to v2 => v3 + version = formatErasureVersionV2 + fallthrough + case formatErasureVersionV2: + formatData, err = formatErasureMigrateV2ToV3(formatData, export, version) + if err != nil { + return nil, nil, fmt.Errorf("Drive %s: %w", export, err) + } + // Migrate successful v2 => v3, v3 is latest + // version = formatXLVersionV3 + return migrate(formatPath, formatData) + case formatErasureVersionV3: + // v3 is the latest version, return. + return formatData, formatFi, nil + } + return nil, nil, fmt.Errorf(`Disk %s: unknown format version %s`, export, version) +} + +// Migrates version V1 of format.json to version V2 of format.json, +// migration fails upon any error. +func formatErasureMigrateV1ToV2(data []byte, version string) ([]byte, error) { + if version != formatErasureVersionV1 { + return nil, fmt.Errorf(`format version expected %s, found %s`, formatErasureVersionV1, version) + } + + formatV1 := &formatErasureV1{} + if err := json.Unmarshal(data, formatV1); err != nil { + return nil, err + } + + formatV2 := &formatErasureV2{} + formatV2.Version = formatMetaVersionV1 + formatV2.Format = formatBackendErasure + formatV2.Erasure.Version = formatErasureVersionV2 + formatV2.Erasure.DistributionAlgo = formatErasureVersionV2DistributionAlgoV1 + formatV2.Erasure.This = formatV1.Erasure.Disk + formatV2.Erasure.Sets = make([][]string, 1) + formatV2.Erasure.Sets[0] = make([]string, len(formatV1.Erasure.JBOD)) + copy(formatV2.Erasure.Sets[0], formatV1.Erasure.JBOD) + + return json.Marshal(formatV2) +} + +// Migrates V2 for format.json to V3 (Flat hierarchy for multipart) +func formatErasureMigrateV2ToV3(data []byte, export, version string) ([]byte, error) { + if version != formatErasureVersionV2 { + return nil, fmt.Errorf(`format version expected %s, found %s`, formatErasureVersionV2, version) + } + + formatV2 := &formatErasureV2{} + if err := json.Unmarshal(data, formatV2); err != nil { + return nil, err + } + + tmpOld := pathJoin(export, minioMetaTmpDeletedBucket, mustGetUUID()) + if err := renameAll(pathJoin(export, minioMetaMultipartBucket), + tmpOld, export); err != nil && err != errFileNotFound { + bootLogIf(GlobalContext, fmt.Errorf("unable to rename (%s -> %s) %w, drive may be faulty please investigate", + pathJoin(export, minioMetaMultipartBucket), + tmpOld, + osErrToFileErr(err))) + } + + // format-V2 struct is exactly same as format-V1 except that version is "3" + // which indicates the simplified multipart backend. + formatV3 := formatErasureV3{} + formatV3.Version = formatV2.Version + formatV3.Format = formatV2.Format + formatV3.Erasure = formatV2.Erasure + formatV3.Erasure.Version = formatErasureVersionV3 + + return json.Marshal(formatV3) +} + +// countErrs - count a specific error. +func countErrs(errs []error, err error) int { + i := 0 + for _, err1 := range errs { + if err1 == err || errors.Is(err1, err) { + i++ + } + } + return i +} + +// Does all errors indicate we need to initialize all disks?. +func shouldInitErasureDisks(errs []error) bool { + return countErrs(errs, errUnformattedDisk) == len(errs) +} + +// Check if unformatted disks are equal to 50%+1 of all the drives. +func quorumUnformattedDisks(errs []error) bool { + return countErrs(errs, errUnformattedDisk) >= (len(errs)/2)+1 +} + +// loadFormatErasureAll - load all format config from all input disks in parallel. +func loadFormatErasureAll(storageDisks []StorageAPI, heal bool) ([]*formatErasureV3, []error) { + // Initialize list of errors. + g := errgroup.WithNErrs(len(storageDisks)) + + // Initialize format configs. + formats := make([]*formatErasureV3, len(storageDisks)) + + // Load format from each disk in parallel + for index := range storageDisks { + g.Go(func() error { + if storageDisks[index] == nil { + return errDiskNotFound + } + format, err := loadFormatErasure(storageDisks[index], heal) + if err != nil { + return err + } + formats[index] = format + if !heal { + // If no healing required, make the disks valid and + // online. + storageDisks[index].SetDiskID(format.Erasure.This) + } + return nil + }, index) + } + + // Return all formats and errors if any. + return formats, g.Wait() +} + +func saveFormatErasure(disk StorageAPI, format *formatErasureV3, healID string) error { + if disk == nil || format == nil { + return errDiskNotFound + } + + // Marshal and write to disk. + formatData, err := json.Marshal(format) + if err != nil { + return err + } + + tmpFormat := mustGetUUID() + + // Purge any existing temporary file, okay to ignore errors here. + defer disk.Delete(context.TODO(), minioMetaBucket, tmpFormat, DeleteOptions{ + Recursive: false, + Immediate: false, + }) + + // write to unique file. + if err = disk.WriteAll(context.TODO(), minioMetaBucket, tmpFormat, formatData); err != nil { + return err + } + + // Rename file `uuid.json` --> `format.json`. + if err = disk.RenameFile(context.TODO(), minioMetaBucket, tmpFormat, minioMetaBucket, formatConfigFile); err != nil { + return err + } + + disk.SetDiskID(format.Erasure.This) + if healID != "" { + ctx := context.Background() + ht := initHealingTracker(disk, healID) + return ht.save(ctx) + } + return nil +} + +// loadFormatErasure - loads format.json from disk. +func loadFormatErasure(disk StorageAPI, heal bool) (format *formatErasureV3, err error) { + data, err := disk.ReadAll(context.TODO(), minioMetaBucket, formatConfigFile) + if err != nil { + // 'file not found' and 'volume not found' as + // same. 'volume not found' usually means its a fresh disk. + if errors.Is(err, errFileNotFound) || errors.Is(err, errVolumeNotFound) { + return nil, errUnformattedDisk + } + return nil, err + } + + // Try to decode format json into formatConfigV1 struct. + format = &formatErasureV3{} + if err = json.Unmarshal(data, format); err != nil { + return nil, err + } + + if heal { + info, err := disk.DiskInfo(context.Background(), DiskInfoOptions{NoOp: heal}) + if err != nil { + return nil, err + } + format.Info = info + } + + // Success. + return format, nil +} + +// Valid formatErasure basic versions. +func checkFormatErasureValue(formatErasure *formatErasureV3, disk StorageAPI) error { + // Validate format version and format type. + if formatErasure.Version != formatMetaVersionV1 { + return fmt.Errorf("Unsupported version of backend format [%s] found on %s", formatErasure.Version, disk) + } + if formatErasure.Format != formatBackendErasure && formatErasure.Format != formatBackendErasureSingle { + return fmt.Errorf("Unsupported backend format [%s] found on %s", formatErasure.Format, disk) + } + if formatErasure.Erasure.Version != formatErasureVersionV3 { + return fmt.Errorf("Unsupported Erasure backend format found [%s] on %s", formatErasure.Erasure.Version, disk) + } + return nil +} + +// Check all format values. +func checkFormatErasureValues(formats []*formatErasureV3, disks []StorageAPI, setDriveCount int) error { + for i, formatErasure := range formats { + if formatErasure == nil { + continue + } + if err := checkFormatErasureValue(formatErasure, disks[i]); err != nil { + return err + } + if len(formats) != len(formatErasure.Erasure.Sets)*len(formatErasure.Erasure.Sets[0]) { + return fmt.Errorf("%s drive is already being used in another erasure deployment. (Number of drives specified: %d but the number of drives found in the %s drive's format.json: %d)", + disks[i], len(formats), humanize.Ordinal(i+1), len(formatErasure.Erasure.Sets)*len(formatErasure.Erasure.Sets[0])) + } + if len(formatErasure.Erasure.Sets[0]) != setDriveCount { + return fmt.Errorf("%s drive is already formatted with %d drives per erasure set. This cannot be changed to %d", disks[i], len(formatErasure.Erasure.Sets[0]), setDriveCount) + } + } + return nil +} + +// Get backend Erasure format in quorum `format.json`. +func getFormatErasureInQuorum(formats []*formatErasureV3) (*formatErasureV3, error) { + formatCountMap := make(map[int]int, len(formats)) + for _, format := range formats { + if format == nil { + continue + } + formatCountMap[format.Drives()]++ + } + + maxDrives := 0 + maxCount := 0 + for drives, count := range formatCountMap { + if count > maxCount { + maxCount = count + maxDrives = drives + } + } + + if maxDrives == 0 { + return nil, errErasureReadQuorum + } + + if maxCount < len(formats)/2 { + return nil, errErasureReadQuorum + } + + for i, format := range formats { + if format == nil { + continue + } + if format.Drives() == maxDrives { + format := formats[i].Clone() + format.Erasure.This = "" + return format, nil + } + } + + return nil, errErasureReadQuorum +} + +func formatErasureV3Check(reference *formatErasureV3, format *formatErasureV3) error { + tmpFormat := format.Clone() + this := tmpFormat.Erasure.This + tmpFormat.Erasure.This = "" + if len(reference.Erasure.Sets) != len(format.Erasure.Sets) { + return fmt.Errorf("Expected number of sets %d, got %d", len(reference.Erasure.Sets), len(format.Erasure.Sets)) + } + + // Make sure that the sets match. + for i := range reference.Erasure.Sets { + if len(reference.Erasure.Sets[i]) != len(format.Erasure.Sets[i]) { + return fmt.Errorf("Each set should be of same size, expected %d got %d", + len(reference.Erasure.Sets[i]), len(format.Erasure.Sets[i])) + } + for j := range reference.Erasure.Sets[i] { + if reference.Erasure.Sets[i][j] != format.Erasure.Sets[i][j] { + return fmt.Errorf("UUID on positions %d:%d do not match with, expected %s got %s: (%w)", + i, j, reference.Erasure.Sets[i][j], format.Erasure.Sets[i][j], errInconsistentDisk) + } + } + } + + // Make sure that the diskID is found in the set. + for i := range len(tmpFormat.Erasure.Sets) { + for j := 0; j < len(tmpFormat.Erasure.Sets[i]); j++ { + if this == tmpFormat.Erasure.Sets[i][j] { + return nil + } + } + } + return fmt.Errorf("DriveID %s not found in any drive sets %s", this, format.Erasure.Sets) +} + +// saveFormatErasureAll - populates `format.json` on disks in its order. +func saveFormatErasureAll(ctx context.Context, storageDisks []StorageAPI, formats []*formatErasureV3) error { + g := errgroup.WithNErrs(len(storageDisks)) + + // Write `format.json` to all disks. + for index := range storageDisks { + g.Go(func() error { + if formats[index] == nil { + return errDiskNotFound + } + return saveFormatErasure(storageDisks[index], formats[index], "") + }, index) + } + + // Wait for the routines to finish. + return reduceWriteQuorumErrs(ctx, g.Wait(), nil, len(storageDisks)) +} + +// relinquishes the underlying connection for all storage disks. +func closeStorageDisks(storageDisks ...StorageAPI) { + var wg sync.WaitGroup + for _, disk := range storageDisks { + if disk == nil { + continue + } + wg.Add(1) + go func(disk StorageAPI) { + defer wg.Done() + disk.Close() + }(disk) + } + wg.Wait() +} + +// Initialize storage disks for each endpoint. +// Errors are returned for each endpoint with matching index. +func initStorageDisksWithErrors(endpoints Endpoints, opts storageOpts) ([]StorageAPI, []error) { + // Bootstrap disks. + storageDisks := make([]StorageAPI, len(endpoints)) + g := errgroup.WithNErrs(len(endpoints)) + for index := range endpoints { + g.Go(func() (err error) { + storageDisks[index], err = newStorageAPI(endpoints[index], opts) + return err + }, index) + } + return storageDisks, g.Wait() +} + +// formatErasureV3ThisEmpty - find out if '.This' field is empty +// in any of the input `formats`, if yes return true. +func formatErasureV3ThisEmpty(formats []*formatErasureV3) bool { + for _, format := range formats { + if format == nil { + continue + } + // NOTE: This code is specifically needed when migrating version + // V1 to V2 to V3, in a scenario such as this we only need to handle + // single sets since we never used to support multiple sets in releases + // with V1 format version. + if len(format.Erasure.Sets) > 1 { + continue + } + if format.Erasure.This == "" { + return true + } + } + return false +} + +// fixFormatErasureV3 - fix format Erasure configuration on all disks. +func fixFormatErasureV3(storageDisks []StorageAPI, endpoints Endpoints, formats []*formatErasureV3) error { + g := errgroup.WithNErrs(len(formats)) + for i := range formats { + g.Go(func() error { + if formats[i] == nil || !endpoints[i].IsLocal { + return nil + } + // NOTE: This code is specifically needed when migrating version + // V1 to V2 to V3, in a scenario such as this we only need to handle + // single sets since we never used to support multiple sets in releases + // with V1 format version. + if len(formats[i].Erasure.Sets) > 1 { + return nil + } + if formats[i].Erasure.This == "" { + formats[i].Erasure.This = formats[i].Erasure.Sets[0][i] + // Heal the drive if drive has .This empty. + if err := saveFormatErasure(storageDisks[i], formats[i], mustGetUUID()); err != nil { + return err + } + } + return nil + }, i) + } + for _, err := range g.Wait() { + if err != nil { + return err + } + } + return nil +} + +// initFormatErasure - save Erasure format configuration on all disks. +func initFormatErasure(ctx context.Context, storageDisks []StorageAPI, setCount, setDriveCount int, deploymentID string, sErrs []error) (*formatErasureV3, error) { + format := newFormatErasureV3(setCount, setDriveCount) + formats := make([]*formatErasureV3, len(storageDisks)) + wantAtMost, err := ecDrivesNoConfig(setDriveCount) + if err != nil { + return nil, err + } + + for i := range setCount { + hostCount := make(map[string]int, setDriveCount) + for j := range setDriveCount { + disk := storageDisks[i*setDriveCount+j] + newFormat := format.Clone() + newFormat.Erasure.This = format.Erasure.Sets[i][j] + if deploymentID != "" { + newFormat.ID = deploymentID + } + hostCount[disk.Hostname()]++ + formats[i*setDriveCount+j] = newFormat + } + var once sync.Once + for host, count := range hostCount { + if count > wantAtMost { + if host == "" { + host = "local" + } + once.Do(func() { + if len(hostCount) == 1 { + return + } + logger.Info(" * Set %v:", i+1) + for j := range setDriveCount { + disk := storageDisks[i*setDriveCount+j] + logger.Info(" - Drive: %s", disk.String()) + } + }) + logger.Info(color.Yellow("WARNING:")+" Host %v has more than %v drives of set. "+ + "A host failure will result in data becoming unavailable.", host, wantAtMost) + } + } + } + + // Save formats `format.json` across all disks. + if err := saveFormatErasureAll(ctx, storageDisks, formats); err != nil { + return nil, err + } + + return getFormatErasureInQuorum(formats) +} + +// ecDrivesNoConfig returns the erasure coded drives in a set if no config has been set. +// It will attempt to read it from env variable and fall back to drives/2. +func ecDrivesNoConfig(setDriveCount int) (int, error) { + sc, err := storageclass.LookupConfig(config.KVS{}, setDriveCount) + if err != nil { + return 0, err + } + return sc.GetParityForSC(storageclass.STANDARD), nil +} + +// Initialize a new set of set formats which will be written to all disks. +func newHealFormatSets(refFormat *formatErasureV3, setCount, setDriveCount int, formats []*formatErasureV3, errs []error) ([][]*formatErasureV3, [][]DiskInfo) { + newFormats := make([][]*formatErasureV3, setCount) + for i := range refFormat.Erasure.Sets { + newFormats[i] = make([]*formatErasureV3, setDriveCount) + } + currentDisksInfo := make([][]DiskInfo, setCount) + for i := range refFormat.Erasure.Sets { + currentDisksInfo[i] = make([]DiskInfo, setDriveCount) + } + for i := range refFormat.Erasure.Sets { + for j := range refFormat.Erasure.Sets[i] { + if errors.Is(errs[i*setDriveCount+j], errUnformattedDisk) { + newFormats[i][j] = &formatErasureV3{} + newFormats[i][j].ID = refFormat.ID + newFormats[i][j].Format = refFormat.Format + newFormats[i][j].Version = refFormat.Version + newFormats[i][j].Erasure.This = refFormat.Erasure.Sets[i][j] + newFormats[i][j].Erasure.Sets = refFormat.Erasure.Sets + newFormats[i][j].Erasure.Version = refFormat.Erasure.Version + newFormats[i][j].Erasure.DistributionAlgo = refFormat.Erasure.DistributionAlgo + } + if format := formats[i*setDriveCount+j]; format != nil && (errs[i*setDriveCount+j] == nil) { + if format.Info.Endpoint != "" { + currentDisksInfo[i][j] = format.Info + } + } + } + } + return newFormats, currentDisksInfo +} diff --git a/cmd/format-erasure_test.go b/cmd/format-erasure_test.go new file mode 100644 index 00000000..732f6728 --- /dev/null +++ b/cmd/format-erasure_test.go @@ -0,0 +1,507 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "os" + "reflect" + "testing" +) + +// tests fixFormatErasureV3 - fix format.json on all disks. +func TestFixFormatV3(t *testing.T) { + erasureDirs, err := getRandomDisks(8) + if err != nil { + t.Fatal(err) + } + for _, erasureDir := range erasureDirs { + defer os.RemoveAll(erasureDir) + } + endpoints := mustGetNewEndpoints(0, 8, erasureDirs...) + + storageDisks, errs := initStorageDisksWithErrors(endpoints, storageOpts{cleanUp: false, healthCheck: false}) + for _, err := range errs { + if err != nil && err != errDiskNotFound { + t.Fatal(err) + } + } + + format := newFormatErasureV3(1, 8) + format.Erasure.DistributionAlgo = formatErasureVersionV2DistributionAlgoV1 + formats := make([]*formatErasureV3, 8) + + for j := range 8 { + newFormat := format.Clone() + newFormat.Erasure.This = format.Erasure.Sets[0][j] + formats[j] = newFormat + } + + formats[1] = nil + expThis := formats[2].Erasure.This + formats[2].Erasure.This = "" + if err := fixFormatErasureV3(storageDisks, endpoints, formats); err != nil { + t.Fatal(err) + } + + newFormats, errs := loadFormatErasureAll(storageDisks, false) + for _, err := range errs { + if err != nil && err != errUnformattedDisk { + t.Fatal(err) + } + } + gotThis := newFormats[2].Erasure.This + if expThis != gotThis { + t.Fatalf("expected uuid %s, got %s", expThis, gotThis) + } +} + +// tests formatErasureV3ThisEmpty conditions. +func TestFormatErasureEmpty(t *testing.T) { + format := newFormatErasureV3(1, 16) + format.Erasure.DistributionAlgo = formatErasureVersionV2DistributionAlgoV1 + formats := make([]*formatErasureV3, 16) + + for j := range 16 { + newFormat := format.Clone() + newFormat.Erasure.This = format.Erasure.Sets[0][j] + formats[j] = newFormat + } + + // empty format to indicate disk not found, but this + // empty should return false. + formats[0] = nil + + if ok := formatErasureV3ThisEmpty(formats); ok { + t.Fatalf("expected value false, got %t", ok) + } + + formats[2].Erasure.This = "" + if ok := formatErasureV3ThisEmpty(formats); !ok { + t.Fatalf("expected value true, got %t", ok) + } +} + +// Tests xl format migration. +func TestFormatErasureMigrate(t *testing.T) { + // Get test root. + rootPath := t.TempDir() + + m := &formatErasureV1{} + m.Format = formatBackendErasure + m.Version = formatMetaVersionV1 + m.Erasure.Version = formatErasureVersionV1 + m.Erasure.Disk = mustGetUUID() + m.Erasure.JBOD = []string{m.Erasure.Disk, mustGetUUID(), mustGetUUID(), mustGetUUID()} + + b, err := json.Marshal(m) + if err != nil { + t.Fatal(err) + } + + if err = os.MkdirAll(pathJoin(rootPath, minioMetaBucket), os.FileMode(0o755)); err != nil { + t.Fatal(err) + } + + if err = os.WriteFile(pathJoin(rootPath, minioMetaBucket, formatConfigFile), b, os.FileMode(0o644)); err != nil { + t.Fatal(err) + } + + formatData, _, err := formatErasureMigrate(rootPath) + if err != nil { + t.Fatal(err) + } + + migratedVersion, err := formatGetBackendErasureVersion(formatData) + if err != nil { + t.Fatal(err) + } + + if migratedVersion != formatErasureVersionV3 { + t.Fatalf("expected version: %s, got: %s", formatErasureVersionV3, migratedVersion) + } + + b, err = os.ReadFile(pathJoin(rootPath, minioMetaBucket, formatConfigFile)) + if err != nil { + t.Fatal(err) + } + formatV3 := &formatErasureV3{} + if err = json.Unmarshal(b, formatV3); err != nil { + t.Fatal(err) + } + if formatV3.Erasure.This != m.Erasure.Disk { + t.Fatalf("expected drive uuid: %s, got: %s", m.Erasure.Disk, formatV3.Erasure.This) + } + if len(formatV3.Erasure.Sets) != 1 { + t.Fatalf("expected single set after migrating from v1 to v3, but found %d", len(formatV3.Erasure.Sets)) + } + if !reflect.DeepEqual(formatV3.Erasure.Sets[0], m.Erasure.JBOD) { + t.Fatalf("expected drive uuid: %v, got: %v", m.Erasure.JBOD, formatV3.Erasure.Sets[0]) + } + + m = &formatErasureV1{} + m.Format = "unknown" + m.Version = formatMetaVersionV1 + m.Erasure.Version = formatErasureVersionV1 + m.Erasure.Disk = mustGetUUID() + m.Erasure.JBOD = []string{m.Erasure.Disk, mustGetUUID(), mustGetUUID(), mustGetUUID()} + + b, err = json.Marshal(m) + if err != nil { + t.Fatal(err) + } + + if err = os.WriteFile(pathJoin(rootPath, minioMetaBucket, formatConfigFile), b, os.FileMode(0o644)); err != nil { + t.Fatal(err) + } + + if _, _, err = formatErasureMigrate(rootPath); err == nil { + t.Fatal("Expected to fail with unexpected backend format") + } + + m = &formatErasureV1{} + m.Format = formatBackendErasure + m.Version = formatMetaVersionV1 + m.Erasure.Version = "30" + m.Erasure.Disk = mustGetUUID() + m.Erasure.JBOD = []string{m.Erasure.Disk, mustGetUUID(), mustGetUUID(), mustGetUUID()} + + b, err = json.Marshal(m) + if err != nil { + t.Fatal(err) + } + + if err = os.WriteFile(pathJoin(rootPath, minioMetaBucket, formatConfigFile), b, os.FileMode(0o644)); err != nil { + t.Fatal(err) + } + + if _, _, err = formatErasureMigrate(rootPath); err == nil { + t.Fatal("Expected to fail with unexpected backend format version number") + } +} + +// Tests check format xl value. +func TestCheckFormatErasureValue(t *testing.T) { + testCases := []struct { + format *formatErasureV3 + success bool + }{ + // Invalid Erasure format version "2". + { + &formatErasureV3{ + formatMetaV1: formatMetaV1{ + Version: "2", + Format: "Erasure", + }, + Erasure: struct { + Version string `json:"version"` + This string `json:"this"` + Sets [][]string `json:"sets"` + DistributionAlgo string `json:"distributionAlgo"` + }{ + Version: "2", + }, + }, + false, + }, + // Invalid Erasure format "Unknown". + { + &formatErasureV3{ + formatMetaV1: formatMetaV1{ + Version: "1", + Format: "Unknown", + }, + Erasure: struct { + Version string `json:"version"` + This string `json:"this"` + Sets [][]string `json:"sets"` + DistributionAlgo string `json:"distributionAlgo"` + }{ + Version: "2", + }, + }, + false, + }, + // Invalid Erasure format version "0". + { + &formatErasureV3{ + formatMetaV1: formatMetaV1{ + Version: "1", + Format: "Erasure", + }, + Erasure: struct { + Version string `json:"version"` + This string `json:"this"` + Sets [][]string `json:"sets"` + DistributionAlgo string `json:"distributionAlgo"` + }{ + Version: "0", + }, + }, + false, + }, + } + + // Valid all test cases. + for i, testCase := range testCases { + if err := checkFormatErasureValue(testCase.format, nil); err != nil && testCase.success { + t.Errorf("Test %d: Expected failure %s", i+1, err) + } + } +} + +// Tests getFormatErasureInQuorum() +func TestGetFormatErasureInQuorumCheck(t *testing.T) { + setCount := 2 + setDriveCount := 16 + + format := newFormatErasureV3(setCount, setDriveCount) + format.Erasure.DistributionAlgo = formatErasureVersionV2DistributionAlgoV1 + formats := make([]*formatErasureV3, 32) + + for i := range setCount { + for j := range setDriveCount { + newFormat := format.Clone() + newFormat.Erasure.This = format.Erasure.Sets[i][j] + formats[i*setDriveCount+j] = newFormat + } + } + + // Return a format from list of formats in quorum. + quorumFormat, err := getFormatErasureInQuorum(formats) + if err != nil { + t.Fatal(err) + } + + // Check if the reference format and input formats are same. + if err = formatErasureV3Check(quorumFormat, formats[0]); err != nil { + t.Fatal(err) + } + + // QuorumFormat has .This field empty on purpose, expect a failure. + if err = formatErasureV3Check(formats[0], quorumFormat); err == nil { + t.Fatal("Unexpected success") + } + + formats[0] = nil + quorumFormat, err = getFormatErasureInQuorum(formats) + if err != nil { + t.Fatal(err) + } + + badFormat := *quorumFormat + badFormat.Erasure.Sets = nil + if err = formatErasureV3Check(quorumFormat, &badFormat); err == nil { + t.Fatal("Unexpected success") + } + + badFormatUUID := *quorumFormat + badFormatUUID.Erasure.Sets[0][0] = "bad-uuid" + if err = formatErasureV3Check(quorumFormat, &badFormatUUID); err == nil { + t.Fatal("Unexpected success") + } + + badFormatSetSize := *quorumFormat + badFormatSetSize.Erasure.Sets[0] = nil + if err = formatErasureV3Check(quorumFormat, &badFormatSetSize); err == nil { + t.Fatal("Unexpected success") + } + + for i := range formats { + if i < 17 { + formats[i] = nil + } + } + if _, err = getFormatErasureInQuorum(formats); err == nil { + t.Fatal("Unexpected success") + } +} + +// Get backend Erasure format in quorum `format.json`. +func getFormatErasureInQuorumOld(formats []*formatErasureV3) (*formatErasureV3, error) { + formatHashes := make([]string, len(formats)) + for i, format := range formats { + if format == nil { + continue + } + h := sha256.New() + for _, set := range format.Erasure.Sets { + for _, diskID := range set { + h.Write([]byte(diskID)) + } + } + formatHashes[i] = hex.EncodeToString(h.Sum(nil)) + } + + formatCountMap := make(map[string]int) + for _, hash := range formatHashes { + if hash == "" { + continue + } + formatCountMap[hash]++ + } + + maxHash := "" + maxCount := 0 + for hash, count := range formatCountMap { + if count > maxCount { + maxCount = count + maxHash = hash + } + } + + if maxCount < len(formats)/2 { + return nil, errErasureReadQuorum + } + + for i, hash := range formatHashes { + if hash == maxHash { + format := formats[i].Clone() + format.Erasure.This = "" + return format, nil + } + } + + return nil, errErasureReadQuorum +} + +func BenchmarkGetFormatErasureInQuorumOld(b *testing.B) { + setCount := 200 + setDriveCount := 15 + + format := newFormatErasureV3(setCount, setDriveCount) + format.Erasure.DistributionAlgo = formatErasureVersionV2DistributionAlgoV1 + formats := make([]*formatErasureV3, 15*200) + + for i := range setCount { + for j := range setDriveCount { + newFormat := format.Clone() + newFormat.Erasure.This = format.Erasure.Sets[i][j] + formats[i*setDriveCount+j] = newFormat + } + } + + b.ReportAllocs() + + for b.Loop() { + _, _ = getFormatErasureInQuorumOld(formats) + } +} + +func BenchmarkGetFormatErasureInQuorum(b *testing.B) { + setCount := 200 + setDriveCount := 15 + + format := newFormatErasureV3(setCount, setDriveCount) + format.Erasure.DistributionAlgo = formatErasureVersionV2DistributionAlgoV1 + formats := make([]*formatErasureV3, 15*200) + + for i := range setCount { + for j := range setDriveCount { + newFormat := format.Clone() + newFormat.Erasure.This = format.Erasure.Sets[i][j] + formats[i*setDriveCount+j] = newFormat + } + } + + b.ReportAllocs() + + for b.Loop() { + _, _ = getFormatErasureInQuorum(formats) + } +} + +// Initialize new format sets. +func TestNewFormatSets(t *testing.T) { + setCount := 2 + setDriveCount := 16 + + format := newFormatErasureV3(setCount, setDriveCount) + format.Erasure.DistributionAlgo = formatErasureVersionV2DistributionAlgoV1 + formats := make([]*formatErasureV3, 32) + errs := make([]error, 32) + + for i := range setCount { + for j := range setDriveCount { + newFormat := format.Clone() + newFormat.Erasure.This = format.Erasure.Sets[i][j] + formats[i*setDriveCount+j] = newFormat + } + } + + quorumFormat, err := getFormatErasureInQuorum(formats) + if err != nil { + t.Fatal(err) + } + + // 16th disk is unformatted. + errs[15] = errUnformattedDisk + + newFormats, _ := newHealFormatSets(quorumFormat, setCount, setDriveCount, formats, errs) + if newFormats == nil { + t.Fatal("Unexpected failure") + } + + // Check if deployment IDs are preserved. + for i := range newFormats { + for j := range newFormats[i] { + if newFormats[i][j] == nil { + continue + } + if newFormats[i][j].ID != quorumFormat.ID { + t.Fatal("Deployment id in the new format is lost") + } + } + } +} + +func BenchmarkInitStorageDisks256(b *testing.B) { + benchmarkInitStorageDisksN(b, 256) +} + +func BenchmarkInitStorageDisks1024(b *testing.B) { + benchmarkInitStorageDisksN(b, 1024) +} + +func BenchmarkInitStorageDisks2048(b *testing.B) { + benchmarkInitStorageDisksN(b, 2048) +} + +func BenchmarkInitStorageDisksMax(b *testing.B) { + benchmarkInitStorageDisksN(b, 32*204) +} + +func benchmarkInitStorageDisksN(b *testing.B, nDisks int) { + b.ResetTimer() + b.ReportAllocs() + + fsDirs, err := getRandomDisks(nDisks) + if err != nil { + b.Fatal(err) + } + + endpoints := mustGetNewEndpoints(0, 16, fsDirs...) + b.RunParallel(func(pb *testing.PB) { + endpoints := endpoints + for pb.Next() { + initStorageDisksWithErrors(endpoints, storageOpts{cleanUp: false, healthCheck: false}) + } + }) +} diff --git a/cmd/format-meta.go b/cmd/format-meta.go new file mode 100644 index 00000000..dd558407 --- /dev/null +++ b/cmd/format-meta.go @@ -0,0 +1,52 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +// Format related consts +const ( + // Format config file carries backend format specific details. + formatConfigFile = "format.json" +) + +const ( + // Version of the formatMetaV1 + formatMetaVersionV1 = "1" +) + +// format.json currently has the format: +// { +// "version": "1", +// "format": "XXXXX", +// "XXXXX": { +// +// } +// } +// Here "XXXXX" depends on the backend, currently we have "fs" and "xl" implementations. +// formatMetaV1 should be inherited by backend format structs. Please look at format-fs.go +// and format-xl.go for details. + +// Ideally we will never have a situation where we will have to change the +// fields of this struct and deal with related migration. +type formatMetaV1 struct { + // Version of the format config. + Version string `json:"version"` + // Format indicates the backend format type, supports two values 'xl' and 'fs'. + Format string `json:"format"` + // ID is the identifier for the minio deployment + ID string `json:"id"` +} diff --git a/cmd/format_string.go b/cmd/format_string.go new file mode 100644 index 00000000..ce068152 --- /dev/null +++ b/cmd/format_string.go @@ -0,0 +1,28 @@ +// Code generated by "stringer -type=format -trimprefix=format untar.go"; DO NOT EDIT. + +package cmd + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[formatUnknown-0] + _ = x[formatGzip-1] + _ = x[formatZstd-2] + _ = x[formatLZ4-3] + _ = x[formatS2-4] + _ = x[formatBZ2-5] +} + +const _format_name = "UnknownGzipZstdLZ4S2BZ2" + +var _format_index = [...]uint8{0, 7, 11, 15, 18, 20, 23} + +func (i format) String() string { + if i < 0 || i >= format(len(_format_index)-1) { + return "format(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _format_name[_format_index[i]:_format_index[i+1]] +} diff --git a/cmd/ftp-server-driver.go b/cmd/ftp-server-driver.go new file mode 100644 index 00000000..7f21eeb8 --- /dev/null +++ b/cmd/ftp-server-driver.go @@ -0,0 +1,561 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/subtle" + "errors" + "fmt" + "io" + "net" + "net/http" + "os" + "path" + "strings" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio/internal/auth" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/pkg/v3/mimedb" + ftp "goftp.io/server/v2" +) + +var _ ftp.Driver = &ftpDriver{} + +// ftpDriver implements ftpDriver to store files in minio +type ftpDriver struct { + endpoint string +} + +// NewFTPDriver implements ftp.Driver interface +func NewFTPDriver() ftp.Driver { + return &ftpDriver{endpoint: fmt.Sprintf("127.0.0.1:%s", globalMinioPort)} +} + +func buildMinioPath(p string) string { + return strings.TrimPrefix(p, SlashSeparator) +} + +func buildMinioDir(p string) string { + v := buildMinioPath(p) + if !strings.HasSuffix(v, SlashSeparator) { + return v + SlashSeparator + } + return v +} + +type minioFileInfo struct { + p string + info minio.ObjectInfo + isDir bool +} + +func (m *minioFileInfo) Name() string { + return m.p +} + +func (m *minioFileInfo) Size() int64 { + return m.info.Size +} + +func (m *minioFileInfo) Mode() os.FileMode { + if m.isDir { + return os.ModeDir + } + return os.ModePerm +} + +var minFileDate = time.Date(1980, 1, 1, 0, 0, 0, 0, time.UTC) // Workaround for Filezilla + +func (m *minioFileInfo) ModTime() time.Time { + if !m.info.LastModified.IsZero() { + return m.info.LastModified + } + return minFileDate +} + +func (m *minioFileInfo) IsDir() bool { + return m.isDir +} + +func (m *minioFileInfo) Sys() any { + return nil +} + +//msgp:ignore ftpMetrics +type ftpMetrics struct{} + +var globalFtpMetrics ftpMetrics + +func ftpTrace(s *ftp.Context, startTime time.Time, source, objPath string, err error, sz int64) madmin.TraceInfo { + var errStr string + if err != nil { + errStr = err.Error() + } + return madmin.TraceInfo{ + TraceType: madmin.TraceFTP, + Time: startTime, + NodeName: globalLocalNodeName, + FuncName: s.Cmd, + Duration: time.Since(startTime), + Path: objPath, + Error: errStr, + Bytes: sz, + Custom: map[string]string{ + "user": s.Sess.LoginUser(), + "cmd": s.Cmd, + "param": s.Param, + "login": fmt.Sprintf("%t", s.Sess.IsLogin()), + "source": source, + }, + } +} + +func (m *ftpMetrics) log(s *ftp.Context, paths ...string) func(sz int64, err error) { + startTime := time.Now() + source := getSource(2) + return func(sz int64, err error) { + globalTrace.Publish(ftpTrace(s, startTime, source, strings.Join(paths, " "), err, sz)) + } +} + +// Stat implements ftpDriver +func (driver *ftpDriver) Stat(ctx *ftp.Context, objPath string) (fi os.FileInfo, err error) { + stopFn := globalFtpMetrics.log(ctx, objPath) + defer stopFn(0, err) + + if objPath == SlashSeparator { + return &minioFileInfo{ + p: SlashSeparator, + isDir: true, + }, nil + } + + bucket, object := path2BucketObject(objPath) + if bucket == "" { + return nil, errors.New("bucket name cannot be empty") + } + + clnt, err := driver.getMinIOClient(ctx) + if err != nil { + return nil, err + } + + if object == "" { + ok, err := clnt.BucketExists(context.Background(), bucket) + if err != nil { + return nil, err + } + if !ok { + return nil, os.ErrNotExist + } + return &minioFileInfo{ + p: pathClean(bucket), + info: minio.ObjectInfo{Key: bucket}, + isDir: true, + }, nil + } + + objInfo, err := clnt.StatObject(context.Background(), bucket, object, minio.StatObjectOptions{}) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + // dummy return to satisfy LIST (stat -> list) behavior. + return &minioFileInfo{ + p: pathClean(object), + info: minio.ObjectInfo{Key: object}, + isDir: true, + }, nil + } + return nil, err + } + + isDir := strings.HasSuffix(objInfo.Key, SlashSeparator) + return &minioFileInfo{ + p: pathClean(object), + info: objInfo, + isDir: isDir, + }, nil +} + +// ListDir implements ftpDriver +func (driver *ftpDriver) ListDir(ctx *ftp.Context, objPath string, callback func(os.FileInfo) error) (err error) { + stopFn := globalFtpMetrics.log(ctx, objPath) + defer stopFn(0, err) + + clnt, err := driver.getMinIOClient(ctx) + if err != nil { + return err + } + + cctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + bucket, prefix := path2BucketObject(objPath) + if bucket == "" { + buckets, err := clnt.ListBuckets(cctx) + if err != nil { + return err + } + + for _, bucket := range buckets { + info := minioFileInfo{ + p: pathClean(bucket.Name), + info: minio.ObjectInfo{Key: retainSlash(bucket.Name), LastModified: bucket.CreationDate}, + isDir: true, + } + if err := callback(&info); err != nil { + return err + } + } + + return nil + } + + prefix = retainSlash(prefix) + + for object := range clnt.ListObjects(cctx, bucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: false, + }) { + if object.Err != nil { + return object.Err + } + + if object.Key == prefix { + continue + } + + isDir := strings.HasSuffix(object.Key, SlashSeparator) + info := minioFileInfo{ + p: pathClean(strings.TrimPrefix(object.Key, prefix)), + info: object, + isDir: isDir, + } + + if err := callback(&info); err != nil { + return err + } + } + + return nil +} + +func (driver *ftpDriver) CheckPasswd(c *ftp.Context, username, password string) (ok bool, err error) { + stopFn := globalFtpMetrics.log(c, username) + defer stopFn(0, err) + + if globalIAMSys.LDAPConfig.Enabled() { + sa, _, err := globalIAMSys.getServiceAccount(context.Background(), username) + if err != nil && !errors.Is(err, errNoSuchServiceAccount) { + return false, err + } + if errors.Is(err, errNoSuchServiceAccount) { + lookupRes, groupDistNames, err := globalIAMSys.LDAPConfig.Bind(username, password) + if err != nil { + return false, err + } + ldapPolicies, _ := globalIAMSys.PolicyDBGet(lookupRes.NormDN, groupDistNames...) + return len(ldapPolicies) > 0, nil + } + return subtle.ConstantTimeCompare([]byte(sa.Credentials.SecretKey), []byte(password)) == 1, nil + } + + ui, ok := globalIAMSys.GetUser(context.Background(), username) + if !ok { + return false, nil + } + return subtle.ConstantTimeCompare([]byte(ui.Credentials.SecretKey), []byte(password)) == 1, nil +} + +func (driver *ftpDriver) getMinIOClient(ctx *ftp.Context) (*minio.Client, error) { + tr := http.RoundTripper(globalRemoteFTPClientTransport) + if host, _, err := net.SplitHostPort(ctx.Sess.RemoteAddr().String()); err == nil { + tr = forwardForTransport{tr: tr, fwd: host} + } + ui, ok := globalIAMSys.GetUser(context.Background(), ctx.Sess.LoginUser()) + if !ok && !globalIAMSys.LDAPConfig.Enabled() { + return nil, errNoSuchUser + } + if !ok && globalIAMSys.LDAPConfig.Enabled() { + sa, _, err := globalIAMSys.getServiceAccount(context.Background(), ctx.Sess.LoginUser()) + if err != nil && !errors.Is(err, errNoSuchServiceAccount) { + return nil, err + } + + var mcreds *credentials.Credentials + if errors.Is(err, errNoSuchServiceAccount) { + lookupResult, targetGroups, err := globalIAMSys.LDAPConfig.LookupUserDN(ctx.Sess.LoginUser()) + if err != nil { + return nil, err + } + ldapPolicies, _ := globalIAMSys.PolicyDBGet(lookupResult.NormDN, targetGroups...) + if len(ldapPolicies) == 0 { + return nil, errAuthentication + } + expiryDur, err := globalIAMSys.LDAPConfig.GetExpiryDuration("") + if err != nil { + return nil, err + } + claims := make(map[string]any) + claims[expClaim] = UTCNow().Add(expiryDur).Unix() + + claims[ldapUser] = lookupResult.NormDN + claims[ldapActualUser] = lookupResult.ActualDN + claims[ldapUserN] = ctx.Sess.LoginUser() + + // Add LDAP attributes that were looked up into the claims. + for attribKey, attribValue := range lookupResult.Attributes { + claims[ldapAttribPrefix+attribKey] = attribValue + } + + cred, err := auth.GetNewCredentialsWithMetadata(claims, globalActiveCred.SecretKey) + if err != nil { + return nil, err + } + + // Set the parent of the temporary access key, this is useful + // in obtaining service accounts by this cred. + cred.ParentUser = lookupResult.NormDN + + // Set this value to LDAP groups, LDAP user can be part + // of large number of groups + cred.Groups = targetGroups + + // Set the newly generated credentials, policyName is empty on purpose + // LDAP policies are applied automatically using their ldapUser, ldapGroups + // mapping. + updatedAt, err := globalIAMSys.SetTempUser(context.Background(), cred.AccessKey, cred, "") + if err != nil { + return nil, err + } + + // Call hook for site replication. + replLogIf(context.Background(), globalSiteReplicationSys.IAMChangeHook(context.Background(), madmin.SRIAMItem{ + Type: madmin.SRIAMItemSTSAcc, + STSCredential: &madmin.SRSTSCredential{ + AccessKey: cred.AccessKey, + SecretKey: cred.SecretKey, + SessionToken: cred.SessionToken, + ParentUser: cred.ParentUser, + }, + UpdatedAt: updatedAt, + })) + + mcreds = credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken) + } else { + mcreds = credentials.NewStaticV4(sa.Credentials.AccessKey, sa.Credentials.SecretKey, "") + } + + return minio.New(driver.endpoint, &minio.Options{ + Creds: mcreds, + Secure: globalIsTLS, + Transport: tr, + TrailingHeaders: true, + }) + } + + // ok == true - at this point + + if ui.Credentials.IsTemp() { + // Temporary credentials are not allowed. + return nil, errAuthentication + } + + return minio.New(driver.endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(ui.Credentials.AccessKey, ui.Credentials.SecretKey, ""), + Secure: globalIsTLS, + Transport: tr, + TrailingHeaders: true, + }) +} + +// DeleteDir implements ftpDriver +func (driver *ftpDriver) DeleteDir(ctx *ftp.Context, objPath string) (err error) { + stopFn := globalFtpMetrics.log(ctx, objPath) + defer stopFn(0, err) + + bucket, prefix := path2BucketObject(objPath) + if bucket == "" { + return errors.New("deleting all buckets not allowed") + } + + clnt, err := driver.getMinIOClient(ctx) + if err != nil { + return err + } + + cctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + if prefix == "" { + // if all objects are not deleted yet this call may fail. + return clnt.RemoveBucket(cctx, bucket) + } + + objectsCh := make(chan minio.ObjectInfo) + + // Send object names that are needed to be removed to objectsCh + go func() { + defer xioutil.SafeClose(objectsCh) + opts := minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + } + for object := range clnt.ListObjects(cctx, bucket, opts) { + if object.Err != nil { + return + } + objectsCh <- object + } + }() + + // Call RemoveObjects API + for err := range clnt.RemoveObjects(context.Background(), bucket, objectsCh, minio.RemoveObjectsOptions{}) { + if err.Err != nil { + return err.Err + } + } + + return nil +} + +// DeleteFile implements ftpDriver +func (driver *ftpDriver) DeleteFile(ctx *ftp.Context, objPath string) (err error) { + stopFn := globalFtpMetrics.log(ctx, objPath) + defer stopFn(0, err) + + bucket, object := path2BucketObject(objPath) + if bucket == "" { + return errors.New("bucket name cannot be empty") + } + + clnt, err := driver.getMinIOClient(ctx) + if err != nil { + return err + } + + return clnt.RemoveObject(context.Background(), bucket, object, minio.RemoveObjectOptions{}) +} + +// Rename implements ftpDriver +func (driver *ftpDriver) Rename(ctx *ftp.Context, fromObjPath string, toObjPath string) (err error) { + stopFn := globalFtpMetrics.log(ctx, fromObjPath, toObjPath) + defer stopFn(0, err) + + return NotImplemented{} +} + +// MakeDir implements ftpDriver +func (driver *ftpDriver) MakeDir(ctx *ftp.Context, objPath string) (err error) { + stopFn := globalFtpMetrics.log(ctx, objPath) + defer stopFn(0, err) + + bucket, prefix := path2BucketObject(objPath) + if bucket == "" { + return errors.New("bucket name cannot be empty") + } + + clnt, err := driver.getMinIOClient(ctx) + if err != nil { + return err + } + + if prefix == "" { + return clnt.MakeBucket(context.Background(), bucket, minio.MakeBucketOptions{Region: globalSite.Region()}) + } + + dirPath := buildMinioDir(prefix) + + _, err = clnt.PutObject(context.Background(), bucket, dirPath, bytes.NewReader([]byte("")), 0, minio.PutObjectOptions{ + DisableContentSha256: true, + }) + return err +} + +// GetFile implements ftpDriver +func (driver *ftpDriver) GetFile(ctx *ftp.Context, objPath string, offset int64) (n int64, rc io.ReadCloser, err error) { + stopFn := globalFtpMetrics.log(ctx, objPath) + defer stopFn(n, err) + + bucket, object := path2BucketObject(objPath) + if bucket == "" { + return 0, nil, errors.New("bucket name cannot be empty") + } + + clnt, err := driver.getMinIOClient(ctx) + if err != nil { + return 0, nil, err + } + + opts := minio.GetObjectOptions{} + obj, err := clnt.GetObject(context.Background(), bucket, object, opts) + if err != nil { + return 0, nil, err + } + defer func() { + if err != nil && obj != nil { + obj.Close() + } + }() + + _, err = obj.Seek(offset, io.SeekStart) + if err != nil { + return 0, nil, err + } + + info, err := obj.Stat() + if err != nil { + return 0, nil, err + } + n = info.Size - offset + return n, obj, nil +} + +// PutFile implements ftpDriver +func (driver *ftpDriver) PutFile(ctx *ftp.Context, objPath string, data io.Reader, offset int64) (n int64, err error) { + stopFn := globalFtpMetrics.log(ctx, objPath) + defer stopFn(n, err) + + bucket, object := path2BucketObject(objPath) + if bucket == "" { + return 0, errors.New("bucket name cannot be empty") + } + + if offset != -1 { + // FTP - APPEND not implemented + return 0, NotImplemented{} + } + + clnt, err := driver.getMinIOClient(ctx) + if err != nil { + return 0, err + } + + info, err := clnt.PutObject(context.Background(), bucket, object, data, -1, minio.PutObjectOptions{ + ContentType: mimedb.TypeByExtension(path.Ext(object)), + DisableContentSha256: true, + Checksum: minio.ChecksumFullObjectCRC32C, + }) + n = info.Size + return n, err +} diff --git a/cmd/ftp-server.go b/cmd/ftp-server.go new file mode 100644 index 00000000..562336e3 --- /dev/null +++ b/cmd/ftp-server.go @@ -0,0 +1,171 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "net" + "strconv" + "strings" + + "github.com/minio/minio/internal/logger" + ftp "goftp.io/server/v2" +) + +var globalRemoteFTPClientTransport = NewRemoteTargetHTTPTransport(true)() + +// minioLogger use an instance of this to log in a standard format +type minioLogger struct{} + +// Print implement Logger +func (log *minioLogger) Print(sessionID string, message any) { + if serverDebugLog { + fmt.Printf("%s %s\n", sessionID, message) + } +} + +// Printf implement Logger +func (log *minioLogger) Printf(sessionID string, format string, v ...any) { + if serverDebugLog { + if sessionID != "" { + fmt.Printf("%s %s\n", sessionID, fmt.Sprintf(format, v...)) + } else { + fmt.Printf(format+"\n", v...) + } + } +} + +// PrintCommand implement Logger +func (log *minioLogger) PrintCommand(sessionID string, command string, params string) { + if serverDebugLog { + if command == "PASS" { + fmt.Printf("%s > PASS ****\n", sessionID) + } else { + fmt.Printf("%s > %s %s\n", sessionID, command, params) + } + } +} + +// PrintResponse implement Logger +func (log *minioLogger) PrintResponse(sessionID string, code int, message string) { + if serverDebugLog { + fmt.Printf("%s < %d %s\n", sessionID, code, message) + } +} + +func startFTPServer(args []string) { + var ( + port int + publicIP string + portRange string + tlsPrivateKey string + tlsPublicCert string + forceTLS bool + ) + + var err error + for _, arg := range args { + tokens := strings.SplitN(arg, "=", 2) + if len(tokens) != 2 { + logger.Fatal(fmt.Errorf("invalid arguments passed to --ftp=%s", arg), "unable to start FTP server") + } + switch tokens[0] { + case "address": + host, portStr, err := net.SplitHostPort(tokens[1]) + if err != nil { + logger.Fatal(fmt.Errorf("invalid arguments passed to --ftp=%s (%v)", arg, err), "unable to start FTP server") + } + port, err = strconv.Atoi(portStr) + if err != nil { + logger.Fatal(fmt.Errorf("invalid arguments passed to --ftp=%s (%v)", arg, err), "unable to start FTP server") + } + if port < 1 || port > 65535 { + logger.Fatal(fmt.Errorf("invalid arguments passed to --ftp=%s, (port number must be between 1 to 65535)", arg), "unable to start FTP server") + } + publicIP = host + case "passive-port-range": + portRange = tokens[1] + case "tls-private-key": + tlsPrivateKey = tokens[1] + case "tls-public-cert": + tlsPublicCert = tokens[1] + case "force-tls": + forceTLS, err = strconv.ParseBool(tokens[1]) + if err != nil { + logger.Fatal(fmt.Errorf("invalid arguments passed to --ftp=%s (%v)", arg, err), "unable to start FTP server") + } + } + } + + // Verify if only partial inputs are given for FTP(secure) + { + if tlsPrivateKey == "" && tlsPublicCert != "" { + logger.Fatal(fmt.Errorf("invalid TLS arguments provided missing private key --ftp=\"tls-private-key=path/to/private.key\""), "unable to start FTP server") + } + + if tlsPrivateKey != "" && tlsPublicCert == "" { + logger.Fatal(fmt.Errorf("invalid TLS arguments provided missing public cert --ftp=\"tls-public-cert=path/to/public.crt\""), "unable to start FTP server") + } + if port == 0 { + port = 8021 // Default FTP port, since no port was given. + } + } + + // If no TLS certs were provided, server is running in TLS for S3 API + // we automatically make FTP also run under TLS mode. + if globalIsTLS && tlsPrivateKey == "" && tlsPublicCert == "" { + tlsPrivateKey = getPrivateKeyFile() + tlsPublicCert = getPublicCertFile() + } + + tls := tlsPrivateKey != "" && tlsPublicCert != "" + + if forceTLS && !tls { + logger.Fatal(fmt.Errorf("invalid TLS arguments provided. force-tls, but missing private key --ftp=\"tls-private-key=path/to/private.key\""), "unable to start FTP server") + } + + name := "MinIO FTP Server" + if tls { + name = "MinIO FTP(Secure) Server" + } + + ftpServer, err := ftp.NewServer(&ftp.Options{ + Name: name, + WelcomeMessage: fmt.Sprintf("Welcome to '%s' FTP Server Version='%s' License='%s'", MinioStoreName, MinioLicense, Version), + Driver: NewFTPDriver(), + Port: port, + Perm: ftp.NewSimplePerm("nobody", "nobody"), + TLS: tls, + KeyFile: tlsPrivateKey, + CertFile: tlsPublicCert, + ExplicitFTPS: tls, + Logger: &minioLogger{}, + PassivePorts: portRange, + PublicIP: publicIP, + ForceTLS: forceTLS, + }) + if err != nil { + logger.Fatal(err, "unable to initialize FTP server") + } + + logger.Info(fmt.Sprintf("%s listening on %s", name, net.JoinHostPort(publicIP, strconv.Itoa(port)))) + + if err = ftpServer.ListenAndServe(); err != nil { + logger.Fatal(err, "unable to start FTP server") + } +} diff --git a/cmd/generic-handlers.go b/cmd/generic-handlers.go new file mode 100644 index 00000000..88a11166 --- /dev/null +++ b/cmd/generic-handlers.go @@ -0,0 +1,632 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "net" + "net/http" + "path" + "runtime/debug" + "slices" + "strings" + "sync/atomic" + "time" + "unicode" + + "github.com/dustin/go-humanize" + "github.com/minio/minio-go/v7/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/grid" + xnet "github.com/minio/pkg/v3/net" + + "github.com/minio/minio/internal/amztime" + "github.com/minio/minio/internal/config/dns" + "github.com/minio/minio/internal/crypto" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/mcontext" +) + +const ( + // Maximum allowed form data field values. 64MiB is a guessed practical value + // which is more than enough to accommodate any form data fields and headers. + requestFormDataSize = 64 * humanize.MiByte + + // For any HTTP request, request body should be not more than 16GiB + requestFormDataSize + // where, 16GiB is the maximum allowed object size for object upload. + requestMaxBodySize = globalMaxObjectSize + requestFormDataSize + + // Maximum size for http headers - See: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html + maxHeaderSize = 8 * 1024 + + // Maximum size for user-defined metadata - See: https://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html + maxUserDataSize = 2 * 1024 + + // maxBuckets upto 500000 for any MinIO deployment. + maxBuckets = 500 * 1000 +) + +// ReservedMetadataPrefix is the prefix of a metadata key which +// is reserved and for internal use only. +const ( + ReservedMetadataPrefix = "X-Minio-Internal-" + ReservedMetadataPrefixLower = "x-minio-internal-" +) + +// containsReservedMetadata returns true if the http.Header contains +// keys which are treated as metadata but are reserved for internal use +// and must not set by clients +func containsReservedMetadata(header http.Header) bool { + for key := range header { + if _, ok := validSSEReplicationHeaders[key]; ok { + return false + } + if stringsHasPrefixFold(key, ReservedMetadataPrefix) { + return true + } + } + return false +} + +// isHTTPHeaderSizeTooLarge returns true if the provided +// header is larger than 8 KB or the user-defined metadata +// is larger than 2 KB. +func isHTTPHeaderSizeTooLarge(header http.Header) bool { + var size, usersize int + for key := range header { + length := len(key) + len(header.Get(key)) + size += length + for _, prefix := range userMetadataKeyPrefixes { + if stringsHasPrefixFold(key, prefix) { + usersize += length + break + } + } + if usersize > maxUserDataSize || size > maxHeaderSize { + return true + } + } + return false +} + +// Limits body and header to specific allowed maximum limits as per S3/MinIO API requirements. +func setRequestLimitMiddleware(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt) + + // Reject unsupported reserved metadata first before validation. + if containsReservedMetadata(r.Header) { + if ok { + tc.FuncName = "handler.ValidRequest" + tc.ResponseRecorder.LogErrBody = true + } + + defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r)) + writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrUnsupportedMetadata), r.URL) + return + } + + if isHTTPHeaderSizeTooLarge(r.Header) { + if ok { + tc.FuncName = "handler.ValidRequest" + tc.ResponseRecorder.LogErrBody = true + } + + defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r)) + writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrMetadataTooLarge), r.URL) + atomic.AddUint64(&globalHTTPStats.rejectedRequestsHeader, 1) + return + } + // Restricting read data to a given maximum length + r.Body = http.MaxBytesReader(w, r.Body, requestMaxBodySize) + h.ServeHTTP(w, r) + }) +} + +// Reserved bucket. +const ( + minioReservedBucket = "minio" + minioReservedBucketPath = SlashSeparator + minioReservedBucket + + loginPathPrefix = SlashSeparator + "login" +) + +func guessIsBrowserReq(r *http.Request) bool { + aType := getRequestAuthType(r) + return strings.Contains(r.Header.Get("User-Agent"), "Mozilla") && + globalBrowserEnabled && aType == authTypeAnonymous +} + +func setBrowserRedirectMiddleware(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + read := r.Method == http.MethodGet || r.Method == http.MethodHead + // Re-direction is handled specifically for browser requests. + if !guessIsHealthCheckReq(r) && guessIsBrowserReq(r) && read && globalBrowserRedirect { + // Fetch the redirect location if any. + if u := getRedirectLocation(r); u != nil { + // Employ a temporary re-direct. + http.Redirect(w, r, u.String(), http.StatusTemporaryRedirect) + return + } + } + h.ServeHTTP(w, r) + }) +} + +var redirectPrefixes = map[string]struct{}{ + "favicon-16x16.png": {}, + "favicon-32x32.png": {}, + "favicon-96x96.png": {}, + "index.html": {}, + minioReservedBucket: {}, +} + +// Fetch redirect location if urlPath satisfies certain +// criteria. Some special names are considered to be +// redirectable, this is purely internal function and +// serves only limited purpose on redirect-handler for +// browser requests. +func getRedirectLocation(r *http.Request) *xnet.URL { + resource, err := getResource(r.URL.Path, r.Host, globalDomainNames) + if err != nil { + return nil + } + bucket, _ := path2BucketObject(resource) + _, redirect := redirectPrefixes[path.Clean(bucket)] + if redirect || resource == slashSeparator { + if globalBrowserRedirectURL != nil { + return globalBrowserRedirectURL + } + xhost, err := xnet.ParseHost(r.Host) + if err != nil { + return nil + } + return &xnet.URL{ + Host: net.JoinHostPort(xhost.Name, globalMinioConsolePort), + Scheme: func() string { + scheme := "http" + if r.TLS != nil { + scheme = "https" + } + return scheme + }(), + } + } + return nil +} + +// guessIsHealthCheckReq - returns true if incoming request looks +// like healthCheck request +func guessIsHealthCheckReq(req *http.Request) bool { + if req == nil { + return false + } + aType := getRequestAuthType(req) + return aType == authTypeAnonymous && (req.Method == http.MethodGet || req.Method == http.MethodHead) && + (req.URL.Path == healthCheckPathPrefix+healthCheckLivenessPath || + req.URL.Path == healthCheckPathPrefix+healthCheckReadinessPath || + req.URL.Path == healthCheckPathPrefix+healthCheckClusterPath || + req.URL.Path == healthCheckPathPrefix+healthCheckClusterReadPath) +} + +// guessIsMetricsReq - returns true if incoming request looks +// like metrics request +func guessIsMetricsReq(req *http.Request) bool { + if req == nil { + return false + } + aType := getRequestAuthType(req) + return (aType == authTypeAnonymous || aType == authTypeJWT) && + req.URL.Path == minioReservedBucketPath+prometheusMetricsPathLegacy || + req.URL.Path == minioReservedBucketPath+prometheusMetricsV2ClusterPath || + req.URL.Path == minioReservedBucketPath+prometheusMetricsV2NodePath || + req.URL.Path == minioReservedBucketPath+prometheusMetricsV2BucketPath || + req.URL.Path == minioReservedBucketPath+prometheusMetricsV2ResourcePath || + strings.HasPrefix(req.URL.Path, minioReservedBucketPath+metricsV3Path) +} + +// guessIsRPCReq - returns true if the request is for an RPC endpoint. +func guessIsRPCReq(req *http.Request) bool { + if req == nil { + return false + } + if req.Method == http.MethodGet && req.URL != nil { + switch req.URL.Path { + case grid.RoutePath, grid.RouteLockPath: + return true + } + } + + return (req.Method == http.MethodPost || req.Method == http.MethodGet) && + strings.HasPrefix(req.URL.Path, minioReservedBucketPath+SlashSeparator) +} + +// Check to allow access to the reserved "bucket" `/minio` for Admin +// API requests. +func isAdminReq(r *http.Request) bool { + return strings.HasPrefix(r.URL.Path, adminPathPrefix) +} + +// Check to allow access to the reserved "bucket" `/minio` for KMS +// API requests. +func isKMSReq(r *http.Request) bool { + return strings.HasPrefix(r.URL.Path, kmsPathPrefix) +} + +// Supported Amz date headers. +var amzDateHeaders = []string{ + // Do not change this order, x-amz-date value should be + // validated first. + "x-amz-date", + "date", +} + +// parseAmzDateHeader - parses supported amz date headers, in +// supported amz date formats. +func parseAmzDateHeader(req *http.Request) (time.Time, APIErrorCode) { + for _, amzDateHeader := range amzDateHeaders { + amzDateStr := req.Header.Get(amzDateHeader) + if amzDateStr != "" { + t, err := amztime.Parse(amzDateStr) + if err != nil { + return time.Time{}, ErrMalformedDate + } + return t, ErrNone + } + } + // Date header missing. + return time.Time{}, ErrMissingDateHeader +} + +func hasBadHost(host string) error { + if globalIsCICD && strings.TrimSpace(host) == "" { + // under CI/CD test setups ignore empty hosts as invalid hosts + return nil + } + _, err := xnet.ParseHost(host) + return err +} + +// Check if the incoming path has bad path components, +// such as ".." and "." +func hasBadPathComponent(path string) bool { + n := len(path) + if n > 32<<10 { + // At 32K we are beyond reasonable. + return true + } + i := 0 + // Skip leading slashes (for sake of Windows \ is included as well) + for i < n && (path[i] == SlashSeparatorChar || path[i] == '\\') { + i++ + } + + for i < n { + // Find the next segment + start := i + for i < n && path[i] != SlashSeparatorChar && path[i] != '\\' { + i++ + } + + // Trim whitespace of segment + segmentStart, segmentEnd := start, i + for segmentStart < segmentEnd && unicode.IsSpace(rune(path[segmentStart])) { + segmentStart++ + } + for segmentEnd > segmentStart && unicode.IsSpace(rune(path[segmentEnd-1])) { + segmentEnd-- + } + + // Check for ".." or "." + switch { + case segmentEnd-segmentStart == 2 && path[segmentStart] == '.' && path[segmentStart+1] == '.': + return true + case segmentEnd-segmentStart == 1 && path[segmentStart] == '.': + return true + } + i++ + } + return false +} + +// Check if client is sending a malicious request. +func hasMultipleAuth(r *http.Request) bool { + authTypeCount := 0 + for _, hasValidAuth := range []func(*http.Request) bool{ + isRequestSignatureV2, isRequestPresignedSignatureV2, + isRequestSignatureV4, isRequestPresignedSignatureV4, + isRequestJWT, isRequestPostPolicySignatureV4, + } { + if hasValidAuth(r) { + authTypeCount++ + } + } + return authTypeCount > 1 +} + +// requestValidityHandler validates all the incoming paths for +// any malicious requests. +func setRequestValidityMiddleware(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt) + + if err := hasBadHost(r.Host); err != nil { + if ok { + tc.FuncName = "handler.ValidRequest" + tc.ResponseRecorder.LogErrBody = true + } + + defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r)) + invalidReq := errorCodes.ToAPIErr(ErrInvalidRequest) + invalidReq.Description = fmt.Sprintf("%s (%s)", invalidReq.Description, err) + writeErrorResponse(r.Context(), w, invalidReq, r.URL) + atomic.AddUint64(&globalHTTPStats.rejectedRequestsInvalid, 1) + return + } + + // Check for bad components in URL path. + if hasBadPathComponent(r.URL.Path) { + if ok { + tc.FuncName = "handler.ValidRequest" + tc.ResponseRecorder.LogErrBody = true + } + + defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r)) + writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidResourceName), r.URL) + atomic.AddUint64(&globalHTTPStats.rejectedRequestsInvalid, 1) + return + } + // Check for bad components in URL query values. + for k, vv := range r.Form { + if k == "delimiter" { // delimiters are allowed to have `.` or `..` + continue + } + if slices.ContainsFunc(vv, hasBadPathComponent) { + if ok { + tc.FuncName = "handler.ValidRequest" + tc.ResponseRecorder.LogErrBody = true + } + + defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r)) + writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidResourceName), r.URL) + atomic.AddUint64(&globalHTTPStats.rejectedRequestsInvalid, 1) + return + } + } + if hasMultipleAuth(r) { + if ok { + tc.FuncName = "handler.Auth" + tc.ResponseRecorder.LogErrBody = true + } + + defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r)) + invalidReq := errorCodes.ToAPIErr(ErrInvalidRequest) + invalidReq.Description = fmt.Sprintf("%s (request has multiple authentication types, please use one)", invalidReq.Description) + writeErrorResponse(r.Context(), w, invalidReq, r.URL) + atomic.AddUint64(&globalHTTPStats.rejectedRequestsInvalid, 1) + return + } + // For all other requests reject access to reserved buckets + bucketName, _ := request2BucketObjectName(r) + if isMinioReservedBucket(bucketName) || isMinioMetaBucket(bucketName) { + if !guessIsRPCReq(r) && !guessIsBrowserReq(r) && !guessIsHealthCheckReq(r) && !guessIsMetricsReq(r) && !isAdminReq(r) && !isKMSReq(r) { + if ok { + tc.FuncName = "handler.ValidRequest" + tc.ResponseRecorder.LogErrBody = true + } + defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r)) + writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrAllAccessDisabled), r.URL) + return + } + } else { + // Validate bucket names if it is not empty + if bucketName != "" && s3utils.CheckValidBucketNameStrict(bucketName) != nil { + if ok { + tc.FuncName = "handler.ValidRequest" + tc.ResponseRecorder.LogErrBody = true + } + defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r)) + writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInvalidBucketName), r.URL) + return + } + } + // Deny SSE-C requests if not made over TLS + if !globalIsTLS && (crypto.SSEC.IsRequested(r.Header) || crypto.SSECopy.IsRequested(r.Header)) { + if r.Method == http.MethodHead { + if ok { + tc.FuncName = "handler.ValidRequest" + tc.ResponseRecorder.LogErrBody = false + } + + defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r)) + writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrInsecureSSECustomerRequest)) + } else { + if ok { + tc.FuncName = "handler.ValidRequest" + tc.ResponseRecorder.LogErrBody = true + } + + defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r)) + writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInsecureSSECustomerRequest), r.URL) + } + return + } + h.ServeHTTP(w, r) + }) +} + +// setBucketForwardingMiddleware middleware forwards the path style requests +// on a bucket to the right bucket location, bucket to IP configuration +// is obtained from centralized etcd configuration service. +func setBucketForwardingMiddleware(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if origin := w.Header().Get("Access-Control-Allow-Origin"); origin == "null" { + // This is a workaround change to ensure that "Origin: null" + // incoming request to a response back as "*" instead of "null" + w.Header().Set("Access-Control-Allow-Origin", "*") + } + if globalDNSConfig == nil || !globalBucketFederation || + guessIsHealthCheckReq(r) || guessIsMetricsReq(r) || + guessIsRPCReq(r) || guessIsLoginSTSReq(r) || isAdminReq(r) || isKMSReq(r) { + h.ServeHTTP(w, r) + return + } + + bucket, object := request2BucketObjectName(r) + + // Requests in federated setups for STS type calls which are + // performed at '/' resource should be routed by the muxer, + // the assumption is simply such that requests without a bucket + // in a federated setup cannot be proxied, so serve them at + // current server. + if bucket == "" { + h.ServeHTTP(w, r) + return + } + + // MakeBucket requests should be handled at current endpoint + if r.Method == http.MethodPut && bucket != "" && object == "" && r.URL.RawQuery == "" { + h.ServeHTTP(w, r) + return + } + + // CopyObject requests should be handled at current endpoint as path style + // requests have target bucket and object in URI and source details are in + // header fields + if r.Method == http.MethodPut && r.Header.Get(xhttp.AmzCopySource) != "" { + bucket, object = path2BucketObject(r.Header.Get(xhttp.AmzCopySource)) + if bucket == "" || object == "" { + h.ServeHTTP(w, r) + return + } + } + sr, err := globalDNSConfig.Get(bucket) + if err != nil { + defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r)) + if err == dns.ErrNoEntriesFound { + writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrNoSuchBucket), r.URL) + } else { + writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL) + } + return + } + if globalDomainIPs.Intersection(set.CreateStringSet(getHostsSlice(sr)...)).IsEmpty() { + r.URL.Scheme = "http" + if globalIsTLS { + r.URL.Scheme = "https" + } + r.URL.Host = getHostFromSrv(sr) + // Make sure we remove any existing headers before + // proxying the request to another node. + for k := range w.Header() { + w.Header().Del(k) + } + globalForwarder.ServeHTTP(w, r) + return + } + h.ServeHTTP(w, r) + }) +} + +// addCustomHeadersMiddleware adds various HTTP(S) response headers. +// Security Headers enable various security protections behaviors in the client's browser. +func addCustomHeadersMiddleware(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + header := w.Header() + header.Set("X-XSS-Protection", "1; mode=block") // Prevents against XSS attacks + header.Set("X-Content-Type-Options", "nosniff") // Prevent mime-sniff + header.Set("Strict-Transport-Security", "max-age=31536000; includeSubDomains") // HSTS mitigates variants of MITM attacks + + // Previously, this value was set right before a response was sent to + // the client. So, logger and Error response XML were not using this + // value. This is set here so that this header can be logged as + // part of the log entry, Error response XML and auditing. + // Set custom headers such as x-amz-request-id for each request. + w.Header().Set(xhttp.AmzRequestID, mustGetRequestID(UTCNow())) + if globalLocalNodeName != "" { + w.Header().Set(xhttp.AmzRequestHostID, globalLocalNodeNameHex) + } + h.ServeHTTP(w, r) + }) +} + +// criticalErrorHandler handles panics and fatal errors by +// `panic(logger.ErrCritical)` as done by `logger.CriticalIf`. +// +// It should be always the first / highest HTTP handler. +func setCriticalErrorHandler(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer func() { + if rec := recover(); rec == logger.ErrCritical { // handle + stack := debug.Stack() + logger.Error("critical: \"%s %s\": %v\n%s", r.Method, r.URL, rec, string(stack)) + writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInternalError), r.URL) + return + } else if rec != nil { + stack := debug.Stack() + logger.Error("panic: \"%s %s\": %v\n%s", r.Method, r.URL, rec, string(stack)) + // Try to write an error response, upstream may not have written header. + writeErrorResponse(r.Context(), w, errorCodes.ToAPIErr(ErrInternalError), r.URL) + return + } + }() + h.ServeHTTP(w, r) + }) +} + +// setUploadForwardingMiddleware middleware forwards multiparts requests +// in a site replication setup to peer that initiated the upload +func setUploadForwardingMiddleware(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if !globalSiteReplicationSys.isEnabled() || + guessIsHealthCheckReq(r) || guessIsMetricsReq(r) || + guessIsRPCReq(r) || guessIsLoginSTSReq(r) || isAdminReq(r) { + h.ServeHTTP(w, r) + return + } + + bucket, object := request2BucketObjectName(r) + uploadID := r.Form.Get(xhttp.UploadID) + + if bucket != "" && object != "" && uploadID != "" { + deplID, err := getDeplIDFromUpload(uploadID) + if err != nil { + h.ServeHTTP(w, r) + return + } + remote, self := globalSiteReplicationSys.getPeerForUpload(deplID) + if self { + h.ServeHTTP(w, r) + return + } + r.URL.Scheme = remote.EndpointURL.Scheme + r.URL.Host = remote.EndpointURL.Host + // Make sure we remove any existing headers before + // proxying the request to another node. + for k := range w.Header() { + w.Header().Del(k) + } + ctx := newContext(r, w, "SiteReplicationUploadForwarding") + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + globalForwarder.ServeHTTP(w, r) + return + } + h.ServeHTTP(w, r) + }) +} diff --git a/cmd/generic-handlers_contrib.go b/cmd/generic-handlers_contrib.go new file mode 100644 index 00000000..2a2b07e9 --- /dev/null +++ b/cmd/generic-handlers_contrib.go @@ -0,0 +1,32 @@ +/* + * MinIO Object Storage (c) 2021 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package cmd + +import ( + "net/http" + "strings" +) + +// guessIsLoginSTSReq - returns true if incoming request is Login STS user +func guessIsLoginSTSReq(req *http.Request) bool { + if req == nil { + return false + } + return strings.HasPrefix(req.URL.Path, loginPathPrefix) || + (req.Method == http.MethodPost && req.URL.Path == SlashSeparator && + getRequestAuthType(req) == authTypeSTS) +} diff --git a/cmd/generic-handlers_test.go b/cmd/generic-handlers_test.go new file mode 100644 index 00000000..c2d71ae9 --- /dev/null +++ b/cmd/generic-handlers_test.go @@ -0,0 +1,210 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "testing" + + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/grid" + xhttp "github.com/minio/minio/internal/http" +) + +// Tests request guess function for net/rpc requests. +func TestGuessIsRPC(t *testing.T) { + if guessIsRPCReq(nil) { + t.Fatal("Unexpected return for nil request") + } + + u, err := url.Parse("http://localhost:9000/minio/lock") + if err != nil { + t.Fatal(err) + } + + r := &http.Request{ + Proto: "HTTP/1.0", + Method: http.MethodPost, + URL: u, + } + if !guessIsRPCReq(r) { + t.Fatal("Test shouldn't fail for a possible net/rpc request.") + } + r = &http.Request{ + Proto: "HTTP/1.1", + Method: http.MethodGet, + URL: u, + } + if !guessIsRPCReq(r) { + t.Fatal("Test shouldn't fail for a possible net/rpc request.") + } + r = &http.Request{ + Proto: "HTTP/1.1", + Method: http.MethodGet, + URL: &url.URL{Path: grid.RoutePath}, + } + if !guessIsRPCReq(r) { + t.Fatal("Grid RPC path not detected") + } + r = &http.Request{ + Proto: "HTTP/1.1", + Method: http.MethodGet, + URL: &url.URL{Path: grid.RouteLockPath}, + } + if !guessIsRPCReq(r) { + t.Fatal("Grid RPC path not detected") + } +} + +var isHTTPHeaderSizeTooLargeTests = []struct { + header http.Header + shouldFail bool +}{ + {header: generateHeader(0, 0), shouldFail: false}, + {header: generateHeader(1024, 0), shouldFail: false}, + {header: generateHeader(2048, 0), shouldFail: false}, + {header: generateHeader(8*1024+1, 0), shouldFail: true}, + {header: generateHeader(0, 1024), shouldFail: false}, + {header: generateHeader(0, 2048), shouldFail: true}, + {header: generateHeader(0, 2048+1), shouldFail: true}, +} + +func generateHeader(size, usersize int) http.Header { + header := http.Header{} + for i := range size { + header.Set(strconv.Itoa(i), "") + } + userlength := 0 + for i := 0; userlength < usersize; i++ { + userlength += len(userMetadataKeyPrefixes[0] + strconv.Itoa(i)) + header.Set(userMetadataKeyPrefixes[0]+strconv.Itoa(i), "") + } + return header +} + +func TestIsHTTPHeaderSizeTooLarge(t *testing.T) { + for i, test := range isHTTPHeaderSizeTooLargeTests { + if res := isHTTPHeaderSizeTooLarge(test.header); res != test.shouldFail { + t.Errorf("Test %d: Expected %v got %v", i, res, test.shouldFail) + } + } +} + +var containsReservedMetadataTests = []struct { + header http.Header + shouldFail bool +}{ + { + header: http.Header{"X-Minio-Key": []string{"value"}}, + }, + { + header: http.Header{crypto.MetaIV: []string{"iv"}}, + shouldFail: false, + }, + { + header: http.Header{crypto.MetaAlgorithm: []string{crypto.InsecureSealAlgorithm}}, + shouldFail: false, + }, + { + header: http.Header{crypto.MetaSealedKeySSEC: []string{"mac"}}, + shouldFail: false, + }, + { + header: http.Header{ReservedMetadataPrefix + "Key": []string{"value"}}, + shouldFail: true, + }, +} + +func TestContainsReservedMetadata(t *testing.T) { + for _, test := range containsReservedMetadataTests { + t.Run("", func(t *testing.T) { + contains := containsReservedMetadata(test.header) + if contains && !test.shouldFail { + t.Errorf("contains reserved header but should not fail") + } else if !contains && test.shouldFail { + t.Errorf("does not contain reserved header but failed") + } + }) + } +} + +var sseTLSHandlerTests = []struct { + URL *url.URL + Header http.Header + IsTLS, ShouldFail bool +}{ + {URL: &url.URL{}, Header: http.Header{}, IsTLS: false, ShouldFail: false}, // 0 + {URL: &url.URL{}, Header: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{"AES256"}}, IsTLS: false, ShouldFail: true}, // 1 + {URL: &url.URL{}, Header: http.Header{xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{"AES256"}}, IsTLS: true, ShouldFail: false}, // 2 + {URL: &url.URL{}, Header: http.Header{xhttp.AmzServerSideEncryptionCustomerKey: []string{""}}, IsTLS: true, ShouldFail: false}, // 3 + {URL: &url.URL{}, Header: http.Header{xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm: []string{""}}, IsTLS: false, ShouldFail: true}, // 4 +} + +func TestSSETLSHandler(t *testing.T) { + defer func(isSSL bool) { globalIsTLS = isSSL }(globalIsTLS) // reset globalIsTLS after test + + var okHandler http.HandlerFunc = func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + } + for i, test := range sseTLSHandlerTests { + globalIsTLS = test.IsTLS + + w := httptest.NewRecorder() + r := new(http.Request) + r.Header = test.Header + r.URL = test.URL + + h := setRequestValidityMiddleware(okHandler) + h.ServeHTTP(w, r) + + switch { + case test.ShouldFail && w.Code == http.StatusOK: + t.Errorf("Test %d: should fail but status code is HTTP %d", i, w.Code) + case !test.ShouldFail && w.Code != http.StatusOK: + t.Errorf("Test %d: should not fail but status code is HTTP %d and not 200 OK", i, w.Code) + } + } +} + +func Benchmark_hasBadPathComponent(t *testing.B) { + tests := []struct { + name string + input string + want bool + }{ + {name: "empty", input: "", want: false}, + {name: "backslashes", input: `\a\a\ \\ \\\\\\\`, want: false}, + {name: "long", input: strings.Repeat("a/", 2000), want: false}, + {name: "long-fail", input: strings.Repeat("a/", 2000) + "../..", want: true}, + } + for _, tt := range tests { + t.Run(tt.name, func(b *testing.B) { + b.SetBytes(int64(len(tt.input))) + b.ReportAllocs() + for b.Loop() { + if got := hasBadPathComponent(tt.input); got != tt.want { + t.Fatalf("hasBadPathComponent() = %v, want %v", got, tt.want) + } + } + }) + } +} diff --git a/cmd/global-heal.go b/cmd/global-heal.go new file mode 100644 index 00000000..57cce16e --- /dev/null +++ b/cmd/global-heal.go @@ -0,0 +1,594 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "math/rand" + "runtime" + "sort" + "time" + + "github.com/dustin/go-humanize" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/bucket/lifecycle" + objectlock "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/bucket/versioning" + "github.com/minio/minio/internal/color" + "github.com/minio/minio/internal/config/storageclass" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/console" + "github.com/minio/pkg/v3/wildcard" + "github.com/minio/pkg/v3/workers" +) + +const ( + bgHealingUUID = "0000-0000-0000-0000" +) + +// NewBgHealSequence creates a background healing sequence +// operation which scans all objects and heal them. +func newBgHealSequence() *healSequence { + reqInfo := &logger.ReqInfo{API: "BackgroundHeal"} + ctx, cancelCtx := context.WithCancel(logger.SetReqInfo(GlobalContext, reqInfo)) + + hs := madmin.HealOpts{ + // Remove objects that do not have read-quorum + Remove: healDeleteDangling, + } + + return &healSequence{ + startTime: UTCNow(), + clientToken: bgHealingUUID, + // run-background heal with reserved bucket + bucket: minioReservedBucket, + settings: hs, + currentStatus: healSequenceStatus{ + Summary: healNotStartedStatus, + HealSettings: hs, + }, + cancelCtx: cancelCtx, + ctx: ctx, + reportProgress: false, + scannedItemsMap: make(map[madmin.HealItemType]int64), + healedItemsMap: make(map[madmin.HealItemType]int64), + healFailedItemsMap: make(map[madmin.HealItemType]int64), + } +} + +// getLocalBackgroundHealStatus will return the heal status of the local node +func getLocalBackgroundHealStatus(ctx context.Context, o ObjectLayer) (madmin.BgHealState, bool) { + if globalBackgroundHealState == nil { + return madmin.BgHealState{}, false + } + + bgSeq, ok := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID) + if !ok { + return madmin.BgHealState{}, false + } + + status := madmin.BgHealState{ + ScannedItemsCount: bgSeq.getScannedItemsCount(), + } + + healDisksMap := map[string]struct{}{} + for _, ep := range getLocalDisksToHeal() { + healDisksMap[ep.String()] = struct{}{} + } + + if o == nil { + healing := globalBackgroundHealState.getLocalHealingDisks() + for _, disk := range healing { + status.HealDisks = append(status.HealDisks, disk.Endpoint) + } + + return status, true + } + + si := o.LocalStorageInfo(ctx, true) + + indexed := make(map[string][]madmin.Disk) + for _, disk := range si.Disks { + setIdx := fmt.Sprintf("%d-%d", disk.PoolIndex, disk.SetIndex) + indexed[setIdx] = append(indexed[setIdx], disk) + } + + for id, disks := range indexed { + ss := madmin.SetStatus{ + ID: id, + SetIndex: disks[0].SetIndex, + PoolIndex: disks[0].PoolIndex, + } + for _, disk := range disks { + ss.Disks = append(ss.Disks, disk) + if disk.Healing { + ss.HealStatus = "Healing" + ss.HealPriority = "high" + status.HealDisks = append(status.HealDisks, disk.Endpoint) + } + } + sortDisks(ss.Disks) + status.Sets = append(status.Sets, ss) + } + sort.Slice(status.Sets, func(i, j int) bool { + return status.Sets[i].ID < status.Sets[j].ID + }) + + backendInfo := o.BackendInfo() + status.SCParity = make(map[string]int) + status.SCParity[storageclass.STANDARD] = backendInfo.StandardSCParity + status.SCParity[storageclass.RRS] = backendInfo.RRSCParity + + return status, true +} + +type healEntryResult struct { + bytes uint64 + success bool + skipped bool + entryDone bool + name string +} + +// healErasureSet lists and heals all objects in a specific erasure set +func (er *erasureObjects) healErasureSet(ctx context.Context, buckets []string, tracker *healingTracker) error { + bgSeq, found := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID) + if !found { + return errors.New("no local healing sequence initialized, unable to heal the drive") + } + + scanMode := madmin.HealNormalScan + + // Make sure to copy since `buckets slice` + // is modified in place by tracker. + healBuckets := make([]string, len(buckets)) + copy(healBuckets, buckets) + + objAPI := newObjectLayerFn() + if objAPI == nil { + return errServerNotInitialized + } + + started := tracker.Started + if started.IsZero() || started.Equal(timeSentinel) { + healingLogIf(ctx, fmt.Errorf("unexpected tracker healing start time found: %v", started)) + started = time.Time{} + } + + // Final tracer update before quitting + defer func() { + tracker.setObject("") + tracker.setBucket("") + healingLogIf(ctx, tracker.update(ctx)) + }() + + for _, bucket := range healBuckets { + if err := bgSeq.healBucket(objAPI, bucket, true); err != nil { + // Log bucket healing error if any, we shall retry again. + healingLogIf(ctx, err) + } + } + + info, err := tracker.disk.DiskInfo(ctx, DiskInfoOptions{}) + if err != nil { + return fmt.Errorf("unable to get disk information before healing it: %w", err) + } + + var numHealers uint64 + + if numCores := uint64(runtime.GOMAXPROCS(0)); info.NRRequests > numCores { + numHealers = numCores / 4 + } else { + numHealers = info.NRRequests / 4 + } + if numHealers < 4 { + numHealers = 4 + } + // allow overriding this value as well.. + if v := globalHealConfig.GetWorkers(); v > 0 { + numHealers = uint64(v) + } + + healingLogEvent(ctx, "Healing drive '%s' - use %d parallel workers.", tracker.disk.String(), numHealers) + + jt, _ := workers.New(int(numHealers)) + + healEntryDone := func(name string) healEntryResult { + return healEntryResult{ + entryDone: true, + name: name, + } + } + + healEntrySuccess := func(sz uint64) healEntryResult { + return healEntryResult{ + bytes: sz, + success: true, + } + } + + healEntryFailure := func(sz uint64) healEntryResult { + return healEntryResult{ + bytes: sz, + } + } + + healEntrySkipped := func(sz uint64) healEntryResult { + return healEntryResult{ + bytes: sz, + skipped: true, + } + } + + // Collect updates to tracker from concurrent healEntry calls + results := make(chan healEntryResult, 1000) + quitting := make(chan struct{}) + defer func() { + close(results) + <-quitting + }() + + go func() { + for res := range results { + if res.entryDone { + tracker.setObject(res.name) + if time.Since(tracker.getLastUpdate()) > time.Minute { + healingLogIf(ctx, tracker.update(ctx)) + } + continue + } + + tracker.updateProgress(res.success, res.skipped, res.bytes) + } + + healingLogIf(ctx, tracker.update(ctx)) + close(quitting) + }() + + var retErr error + + // Heal all buckets with all objects + for _, bucket := range healBuckets { + if tracker.isHealed(bucket) { + continue + } + + var forwardTo string + // If we resume to the same bucket, forward to last known item. + b := tracker.getBucket() + if b == bucket { + forwardTo = tracker.getObject() + } + if b != "" { + // Reset to where last bucket ended if resuming. + tracker.resume() + } + tracker.setObject("") + tracker.setBucket(bucket) + // Heal current bucket again in case if it is failed + // in the beginning of erasure set healing + if err := bgSeq.healBucket(objAPI, bucket, true); err != nil { + // Set this such that when we return this function + // we let the caller retry this disk again for the + // buckets that failed healing. + retErr = err + healingLogIf(ctx, err) + continue + } + + var ( + vc *versioning.Versioning + lc *lifecycle.Lifecycle + lr objectlock.Retention + rcfg *replication.Config + ) + + if !isMinioMetaBucketName(bucket) { + vc, err = globalBucketVersioningSys.Get(bucket) + if err != nil { + retErr = err + healingLogIf(ctx, err) + continue + } + // Check if the current bucket has a configured lifecycle policy + lc, err = globalLifecycleSys.Get(bucket) + if err != nil && !errors.Is(err, BucketLifecycleNotFound{Bucket: bucket}) { + retErr = err + healingLogIf(ctx, err) + continue + } + // Check if bucket is object locked. + lr, err = globalBucketObjectLockSys.Get(bucket) + if err != nil { + retErr = err + healingLogIf(ctx, err) + continue + } + rcfg, err = getReplicationConfig(ctx, bucket) + if err != nil { + retErr = err + healingLogIf(ctx, err) + continue + } + } + + if serverDebugLog { + console.Debugf(color.Green("healDrive:")+" healing bucket %s content on %s erasure set\n", + bucket, humanize.Ordinal(er.setIndex+1)) + } + + disks, _, healing := er.getOnlineDisksWithHealingAndInfo(true) + if len(disks) == healing { + // All drives in this erasure set were reformatted for some reasons, abort healing and mark it as successful + healingLogIf(ctx, errors.New("all drives are in healing state, aborting..")) + return nil + } + + disks = disks[:len(disks)-healing] // healing drives are always at the end of the list + + if len(disks) < er.setDriveCount/2 { + return fmt.Errorf("not enough drives (found=%d, healing=%d, total=%d) are available to heal `%s`", len(disks), healing, er.setDriveCount, tracker.disk.String()) + } + + rand.Shuffle(len(disks), func(i, j int) { + disks[i], disks[j] = disks[j], disks[i] + }) + + filterLifecycle := func(bucket, object string, fi FileInfo) bool { + if lc == nil { + return false + } + versioned := vc != nil && vc.Versioned(object) + objInfo := fi.ToObjectInfo(bucket, object, versioned) + + evt := evalActionFromLifecycle(ctx, *lc, lr, rcfg, objInfo) + switch { + case evt.Action.DeleteRestored(): // if restored copy has expired,delete it synchronously + applyExpiryOnTransitionedObject(ctx, newObjectLayerFn(), objInfo, evt, lcEventSrc_Heal) + return false + case evt.Action.Delete(): + globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_Heal) + return true + default: + return false + } + } + + send := func(result healEntryResult) bool { + select { + case <-ctx.Done(): + if !contextCanceled(ctx) { + healingLogIf(ctx, ctx.Err()) + } + return false + case results <- result: + bgSeq.countScanned(madmin.HealItemObject) + return true + } + } + + // Note: updates from healEntry to tracker must be sent on results channel. + healEntry := func(bucket string, entry metaCacheEntry) { + defer jt.Give() + + if entry.name == "" && len(entry.metadata) == 0 { + // ignore entries that don't have metadata. + return + } + if entry.isDir() { + // ignore healing entry.name's with `/` suffix. + return + } + + // We might land at .metacache, .trash, .multipart + // no need to heal them skip, only when bucket + // is '.minio.sys' + if bucket == minioMetaBucket { + if wildcard.Match("buckets/*/.metacache/*", entry.name) { + return + } + if wildcard.Match("tmp/.trash/*", entry.name) { + return + } + if wildcard.Match("multipart/*", entry.name) { + return + } + } + + // erasureObjects layer needs object names to be encoded + encodedEntryName := encodeDirObject(entry.name) + + var result healEntryResult + fivs, err := entry.fileInfoVersions(bucket) + if err != nil { + res, err := er.HealObject(ctx, bucket, encodedEntryName, "", + madmin.HealOpts{ + ScanMode: scanMode, + Remove: healDeleteDangling, + }) + if err != nil { + if isErrObjectNotFound(err) || isErrVersionNotFound(err) { + // queueing happens across namespace, ignore + // objects that are not found. + return + } + result = healEntryFailure(0) + bgSeq.countFailed(madmin.HealItemObject) + healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", bucket, entry.name, err)) + } else { + bgSeq.countHealed(madmin.HealItemObject) + result = healEntrySuccess(uint64(res.ObjectSize)) + } + + send(result) + return + } + + var versionNotFound int + for _, version := range fivs.Versions { + // Ignore healing a version if: + // - It is uploaded after the drive healing is started + // - An object that is already expired by ILM rule. + if !started.IsZero() && version.ModTime.After(started) || filterLifecycle(bucket, version.Name, version) { + versionNotFound++ + if !send(healEntrySkipped(uint64(version.Size))) { + return + } + continue + } + + res, err := er.HealObject(ctx, bucket, encodedEntryName, + version.VersionID, madmin.HealOpts{ + ScanMode: scanMode, + Remove: healDeleteDangling, + }) + if err != nil { + if isErrObjectNotFound(err) || isErrVersionNotFound(err) { + // queueing happens across namespace, ignore + // objects that are not found. + versionNotFound++ + continue + } + } else { + // Look for the healing results + if res.After.Drives[tracker.DiskIndex].State != madmin.DriveStateOk { + err = fmt.Errorf("unexpected after heal state: %s", res.After.Drives[tracker.DiskIndex].State) + } + } + + if err == nil { + bgSeq.countHealed(madmin.HealItemObject) + result = healEntrySuccess(uint64(version.Size)) + } else { + bgSeq.countFailed(madmin.HealItemObject) + result = healEntryFailure(uint64(version.Size)) + if version.VersionID != "" { + healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s (version-id=%s): %w", + bucket, version.Name, version.VersionID, err)) + } else { + healingLogIf(ctx, fmt.Errorf("unable to heal object %s/%s: %w", + bucket, version.Name, err)) + } + } + + if !send(result) { + return + } + } + + // All versions resulted in 'ObjectNotFound/VersionNotFound' + if versionNotFound == len(fivs.Versions) { + return + } + + send(healEntryDone(entry.name)) + + // Wait and proceed if there are active requests + waitForLowHTTPReq() + } + + // How to resolve partial results. + resolver := metadataResolutionParams{ + dirQuorum: 1, + objQuorum: 1, + bucket: bucket, + } + + err = listPathRaw(ctx, listPathRawOptions{ + disks: disks, + bucket: bucket, + recursive: true, + forwardTo: forwardTo, + minDisks: 1, + reportNotFound: false, + agreed: func(entry metaCacheEntry) { + jt.Take() + go healEntry(bucket, entry) + }, + partial: func(entries metaCacheEntries, _ []error) { + entry, ok := entries.resolve(&resolver) + if !ok { + // check if we can get one entry at least + // proceed to heal nonetheless. + entry, _ = entries.firstFound() + } + jt.Take() + go healEntry(bucket, *entry) + }, + finished: func(errs []error) { + success := countErrs(errs, nil) + if success < len(disks)/2+1 { + retErr = fmt.Errorf("one or more errors reported during listing: %v", errors.Join(errs...)) + } + }, + }) + jt.Wait() // synchronize all the concurrent heal jobs + if err != nil { + // Set this such that when we return this function + // we let the caller retry this disk again for the + // buckets it failed to list. + retErr = err + } + + if retErr != nil { + healingLogIf(ctx, fmt.Errorf("listing failed with: %v on bucket: %v", retErr, bucket)) + continue + } + + select { + // If context is canceled don't mark as done... + case <-ctx.Done(): + return ctx.Err() + default: + tracker.bucketDone(bucket) + healingLogIf(ctx, tracker.update(ctx)) + } + } + if retErr != nil { + return retErr + } + + // Last sanity check + if len(tracker.QueuedBuckets) > 0 { + return fmt.Errorf("not all buckets were healed: %v", tracker.QueuedBuckets) + } + + return nil +} + +func healBucket(bucket string, scan madmin.HealScanMode) error { + // Get background heal sequence to send elements to heal + bgSeq, ok := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID) + if ok { + return bgSeq.queueHealTask(healSource{bucket: bucket}, madmin.HealItemBucket) + } + return nil +} + +// healObject sends the given object/version to the background healing workers +func healObject(bucket, object, versionID string, scan madmin.HealScanMode) error { + // Get background heal sequence to send elements to heal + bgSeq, ok := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID) + if ok { + return bgSeq.healObject(bucket, object, versionID, scan) + } + return nil +} diff --git a/cmd/globals.go b/cmd/globals.go new file mode 100644 index 00000000..734e6abe --- /dev/null +++ b/cmd/globals.go @@ -0,0 +1,491 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "crypto/x509" + "errors" + "net/http" + "os" + "sync" + "time" + + consoleapi "github.com/minio/console/api" + "github.com/minio/dnscache" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/bpool" + "github.com/minio/minio/internal/bucket/bandwidth" + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/config/browser" + "github.com/minio/minio/internal/grid" + "github.com/minio/minio/internal/handlers" + "github.com/minio/minio/internal/kms" + "go.uber.org/atomic" + + "github.com/dustin/go-humanize" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/config/callhome" + "github.com/minio/minio/internal/config/compress" + "github.com/minio/minio/internal/config/dns" + "github.com/minio/minio/internal/config/drive" + idplugin "github.com/minio/minio/internal/config/identity/plugin" + polplugin "github.com/minio/minio/internal/config/policy/plugin" + "github.com/minio/minio/internal/config/storageclass" + "github.com/minio/minio/internal/config/subnet" + xhttp "github.com/minio/minio/internal/http" + etcd "go.etcd.io/etcd/client/v3" + + levent "github.com/minio/minio/internal/config/lambda/event" + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/pubsub" + "github.com/minio/pkg/v3/certs" + "github.com/minio/pkg/v3/env" + xnet "github.com/minio/pkg/v3/net" +) + +// minio configuration related constants. +const ( + GlobalMinioDefaultPort = "9000" + + globalMinioDefaultRegion = "" + // This is a sha256 output of ``arn:aws:iam::minio:user/admin``, + // this is kept in present form to be compatible with S3 owner ID + // requirements - + // + // ``` + // The canonical user ID is the Amazon S3–only concept. + // It is 64-character obfuscated version of the account ID. + // ``` + // http://docs.aws.amazon.com/AmazonS3/latest/dev/example-walkthroughs-managing-access-example4.html + globalMinioDefaultOwnerID = "02d6176db174dc93cb1b899f7c6078f08654445fe8cf1b6ce98d8855f66bdbf4" + globalMinioDefaultStorageClass = "STANDARD" + globalWindowsOSName = "windows" + globalMacOSName = "darwin" + globalMinioModeFS = "mode-server-fs" + globalMinioModeErasureSD = "mode-server-xl-single" + globalMinioModeErasure = "mode-server-xl" + globalMinioModeDistErasure = "mode-server-distributed-xl" + globalDirSuffix = "__XLDIR__" + globalDirSuffixWithSlash = globalDirSuffix + slashSeparator + + // Add new global values here. +) + +const ( + // Limit fields size (except file) to 1Mib since Policy document + // can reach that size according to https://aws.amazon.com/articles/1434 + maxFormFieldSize = int64(1 * humanize.MiByte) + + // The maximum allowed time difference between the incoming request + // date and server date during signature verification. + globalMaxSkewTime = 15 * time.Minute // 15 minutes skew allowed. + + // GlobalStaleUploadsExpiry - Expiry duration after which the uploads in multipart, + // tmp directory are deemed stale. + GlobalStaleUploadsExpiry = time.Hour * 24 // 24 hrs. + + // GlobalStaleUploadsCleanupInterval - Cleanup interval when the stale uploads cleanup is initiated. + GlobalStaleUploadsCleanupInterval = time.Hour * 6 // 6 hrs. + + // Refresh interval to update in-memory iam config cache. + globalRefreshIAMInterval = 10 * time.Minute + + // Limit of location constraint XML for unauthenticated PUT bucket operations. + maxLocationConstraintSize = 3 * humanize.MiByte + + // Maximum size of default bucket encryption configuration allowed + maxBucketSSEConfigSize = 1 * humanize.MiByte + + // diskFillFraction is the fraction of a disk we allow to be filled. + diskFillFraction = 0.99 + + // diskReserveFraction is the fraction of a disk where we will fill other server pools first. + // If all pools reach this, we will use all pools with regular placement. + diskReserveFraction = 0.15 + + // diskAssumeUnknownSize is the size to assume when an unknown size upload is requested. + diskAssumeUnknownSize = 1 << 30 + + // diskMinInodes is the minimum number of inodes we want free on a disk to perform writes. + diskMinInodes = 1000 + + // tlsClientSessionCacheSize is the cache size for client sessions. + tlsClientSessionCacheSize = 100 +) + +func init() { + // Injected to prevent circular dependency. + pubsub.GetByteBuffer = grid.GetByteBuffer +} + +type poolDisksLayout struct { + cmdline string + layout [][]string +} + +type disksLayout struct { + legacy bool + pools []poolDisksLayout +} + +type serverCtxt struct { + JSON, Quiet bool + Anonymous bool + StrictS3Compat bool + Addr, ConsoleAddr string + ConfigDir, CertsDir string + configDirSet, certsDirSet bool + Interface string + + RootUser, RootPwd string + + FTP []string + SFTP []string + + MemLimit uint64 + + UserTimeout time.Duration + IdleTimeout time.Duration + ReadHeaderTimeout time.Duration + MaxIdleConnsPerHost int + + SendBufSize, RecvBufSize int + CrossDomainXML string + // The layout of disks as interpreted + Layout disksLayout +} + +var ( + // Global user opts context + globalServerCtxt serverCtxt + + // Indicates if the running minio server is distributed setup. + globalIsDistErasure = false + + // Indicates if the running minio server is an erasure-code backend. + globalIsErasure = false + + // Indicates if the running minio server is in single drive XL mode. + globalIsErasureSD = false + + // Indicates if server code should go through testing path. + globalIsTesting = false + + // This flag is set to 'true' by default + globalBrowserEnabled = true + + // Custom browser redirect URL, not set by default + // and it is automatically deduced. + globalBrowserRedirectURL *xnet.URL + + // Disable redirect, default is enabled. + globalBrowserRedirect bool + + // globalBrowserConfig Browser user configurable settings + globalBrowserConfig browser.Config + + // This flag is set to 'true' when MINIO_UPDATE env is set to 'off'. Default is false. + globalInplaceUpdateDisabled = false + + // Captures site name and region + globalSite config.Site + + // MinIO local server address (in `host:port` format) + globalMinioAddr = "" + + // MinIO default port, can be changed through command line. + globalMinioPort = GlobalMinioDefaultPort + globalMinioConsolePort = "13333" + + // Holds the host that was passed using --address + globalMinioHost = "" + // Holds the host that was passed using --console-address + globalMinioConsoleHost = "" + + // Holds the possible host endpoint. + globalMinioEndpoint = "" + globalMinioEndpointURL *xnet.URL + + // globalConfigSys server config system. + globalConfigSys *ConfigSys + + globalNotificationSys *NotificationSys + + globalEventNotifier *EventNotifier + globalNotifyTargetList *event.TargetList + globalLambdaTargetList *levent.TargetList + + globalBucketMetadataSys *BucketMetadataSys + globalBucketMonitor *bandwidth.Monitor + globalPolicySys *PolicySys + globalIAMSys *IAMSys + globalBytePoolCap atomic.Pointer[bpool.BytePoolCap] + + globalLifecycleSys *LifecycleSys + globalBucketSSEConfigSys *BucketSSEConfigSys + globalBucketTargetSys *BucketTargetSys + // globalAPIConfig controls S3 API requests throttling, + // healthCheck readiness deadlines and cors settings. + globalAPIConfig = apiConfig{listQuorum: "strict", rootAccess: true} + + globalStorageClass storageclass.Config + + globalAuthNPlugin *idplugin.AuthNPlugin + + // CA root certificates, a nil value means system certs pool will be used + globalRootCAs *x509.CertPool + + // IsSSL indicates if the server is configured with SSL. + globalIsTLS bool + + globalTLSCerts *certs.Manager + + globalHTTPServer *xhttp.Server + globalTCPOptions xhttp.TCPOptions + globalHTTPServerErrorCh = make(chan error) + globalOSSignalCh = make(chan os.Signal, 1) + + // global Trace system to send HTTP request/response + // and Storage/OS calls info to registered listeners. + globalTrace = pubsub.New[madmin.TraceInfo, madmin.TraceType](8) + + // global Listen system to send S3 API events to registered listeners + globalHTTPListen = pubsub.New[event.Event, pubsub.Mask](0) + + // global console system to send console logs to + // registered listeners + globalConsoleSys *HTTPConsoleLoggerSys + + // All unique drives for this deployment + globalEndpoints EndpointServerPools + // All unique nodes for this deployment + globalNodes []Node + + // The name of this local node, fetched from arguments + globalLocalNodeName string + globalLocalNodeNameHex string + globalNodeNamesHex = make(map[string]struct{}) + + // The global subnet config + globalSubnetConfig subnet.Config + + // The global callhome config + globalCallhomeConfig callhome.Config + + // The global drive config + globalDriveConfig drive.Config + + // Global server's network statistics + globalConnStats = newConnStats() + + // Global HTTP request statistics + globalHTTPStats = newHTTPStats() + + // Global bucket network and API statistics + globalBucketConnStats = newBucketConnStats() + globalBucketHTTPStats = newBucketHTTPStats() + + // Time when the server is started + globalBootTime = UTCNow() + + globalActiveCred auth.Credentials + globalNodeAuthToken string + globalSiteReplicatorCred siteReplicatorCred + + // Captures if root credentials are set via ENV. + globalCredViaEnv bool + + globalPublicCerts []*x509.Certificate + + globalDomainNames []string // Root domains for virtual host style requests + globalDomainIPs set.StringSet // Root domain IP address(s) for a distributed MinIO deployment + + globalOperationTimeout = newDynamicTimeout(10*time.Minute, 5*time.Minute) // default timeout for general ops + globalDeleteOperationTimeout = newDynamicTimeout(5*time.Minute, 1*time.Minute) // default time for delete ops + + globalBucketObjectLockSys *BucketObjectLockSys + globalBucketQuotaSys *BucketQuotaSys + globalBucketVersioningSys *BucketVersioningSys + + // Allocated etcd endpoint for config and bucket DNS. + globalEtcdClient *etcd.Client + + // Cluster replication manager. + globalSiteReplicationSys SiteReplicationSys + + // Cluster replication resync metrics + globalSiteResyncMetrics *siteResyncMetrics + + // Is set to true when Bucket federation is requested + // and is 'true' when etcdConfig.PathPrefix is empty + globalBucketFederation bool + + // Allocated DNS config wrapper over etcd client. + globalDNSConfig dns.Store + + // GlobalKMS initialized KMS configuration + GlobalKMS *kms.KMS + + // Common lock for various subsystems performing the leader tasks + globalLeaderLock *sharedLock + + // Auto-Encryption, if enabled, turns any non-SSE-C request + // into an SSE-S3 request. If enabled a valid, non-empty KMS + // configuration must be present. + globalAutoEncryption bool + + // Is compression enabled? + globalCompressConfigMu sync.Mutex + globalCompressConfig compress.Config + + // Some standard object extensions which we strictly dis-allow for compression. + standardExcludeCompressExtensions = []string{".gz", ".bz2", ".rar", ".zip", ".7z", ".xz", ".mp4", ".mkv", ".mov", ".jpg", ".png", ".gif"} + + // Some standard content-types which we strictly dis-allow for compression. + standardExcludeCompressContentTypes = []string{"video/*", "audio/*", "application/zip", "application/x-gzip", "application/x-zip-compressed", " application/x-compress", "application/x-spoon"} + + // AuthZ Plugin system. + globalAuthZPlugin *polplugin.AuthZPlugin + + // Deployment ID - unique per deployment + globalDeploymentIDPtr atomic.Pointer[string] + globalDeploymentID = func() string { + ptr := globalDeploymentIDPtr.Load() + if ptr == nil { + return "" + } + return *ptr + } + + globalAllHealState = newHealState(GlobalContext, true) + + // The always present healing routine ready to heal objects + globalBackgroundHealRoutine = newHealRoutine() + globalBackgroundHealState = newHealState(GlobalContext, false) + + globalMRFState = newMRFState() + + // If writes to FS backend should be O_SYNC. + globalFSOSync bool + + globalProxyEndpoints []ProxyEndpoint + + globalInternodeTransport http.RoundTripper + + globalRemoteTargetTransport http.RoundTripper + + globalDNSCache = &dnscache.Resolver{ + Timeout: 5 * time.Second, + } + + globalForwarder *handlers.Forwarder + + globalTierConfigMgr *TierConfigMgr + + globalConsoleSrv *consoleapi.Server + + // handles service freeze or un-freeze S3 API calls. + globalServiceFreeze atomic.Value + + // Only needed for tracking + globalServiceFreezeCnt int32 + globalServiceFreezeMu sync.Mutex // Updates. + + // Map of local drives to this node, this is set during server startup, + // disk reconnect and mutated by HealFormat. Hold globalLocalDrivesMu to access. + globalLocalDrivesMap map[string]StorageAPI + globalLocalDrivesMu sync.RWMutex + + globalDriveMonitoring = env.Get("_MINIO_DRIVE_ACTIVE_MONITORING", config.EnableOn) == config.EnableOn + + // Is MINIO_CI_CD set? + globalIsCICD bool + + globalRootDiskThreshold uint64 + + // Used for collecting stats for netperf + globalNetPerfMinDuration = time.Second * 10 + globalNetPerfRX netPerfRX + globalSiteNetPerfRX netPerfRX + globalObjectPerfBucket = "minio-perf-test-tmp-bucket" + globalObjectPerfUserMetadata = "X-Amz-Meta-Minio-Object-Perf" // Clients can set this to bypass S3 API service freeze. Used by object pref tests. + + // MinIO version unix timestamp + globalVersionUnix uint64 + + // MinIO client + globalMinioClient *minio.Client + + // Public key for subnet confidential information + subnetAdminPublicKey = []byte("-----BEGIN PUBLIC KEY-----\nMIIBCgKCAQEAyC+ol5v0FP+QcsR6d1KypR/063FInmNEFsFzbEwlHQyEQN3O7kNI\nwVDN1vqp1wDmJYmv4VZGRGzfFw1q+QV7K1TnysrEjrqpVxfxzDQCoUadAp8IxLLc\ns2fjyDNxnZjoC6fTID9C0khKnEa5fPZZc3Ihci9SiCGkPmyUyCGVSxWXIKqL2Lrj\nyDc0pGeEhWeEPqw6q8X2jvTC246tlzqpDeNsPbcv2KblXRcKniQNbBrizT37CKHQ\nM6hc9kugrZbFuo8U5/4RQvZPJnx/DVjLDyoKo2uzuVQs4s+iBrA5sSSLp8rPED/3\n6DgWw3e244Dxtrg972dIT1IOqgn7KUJzVQIDAQAB\n-----END PUBLIC KEY-----") + subnetAdminPublicKeyDev = []byte("-----BEGIN PUBLIC KEY-----\nMIIBCgKCAQEArhQYXQd6zI4uagtVfthAPOt6i4AYHnEWCoNeAovM4MNl42I9uQFh\n3VHkbWj9Gpx9ghf6PgRgK+8FcFvy+StmGcXpDCiFywXX24uNhcZjscX1C4Esk0BW\nidfI2eXYkOlymD4lcK70SVgJvC693Qa7Z3FE1KU8Nfv2bkxEE4bzOkojX9t6a3+J\nR8X6Z2U8EMlH1qxJPgiPogELhWP0qf2Lq7GwSAflo1Tj/ytxvD12WrnE0Rrj/8yP\nSnp7TbYm91KocKMExlmvx3l2XPLxeU8nf9U0U+KOmorejD3MDMEPF+tlk9LB3JWP\nZqYYe38rfALVTn4RVJriUcNOoEpEyC0WEwIDAQAB\n-----END PUBLIC KEY-----") + + // dynamic sleeper to avoid thundering herd for trash folder expunge routine + deleteCleanupSleeper = newDynamicSleeper(5, 25*time.Millisecond, false) + + // dynamic sleeper for multipart expiration routine + deleteMultipartCleanupSleeper = newDynamicSleeper(5, 25*time.Millisecond, false) + + // Is MINIO_SYNC_BOOT set? + globalEnableSyncBoot bool + + // Contains NIC interface name used for internode communication + globalInternodeInterface string + globalInternodeInterfaceOnce sync.Once + + // Set last client perf extra time (get lock, and validate) + globalLastClientPerfExtraTime int64 + + // Captures all batch jobs metrics globally + globalBatchJobsMetrics batchJobMetrics + + // Indicates if server was started as `--address ":0"` + globalDynamicAPIPort bool + + // Add new variable global values here. +) + +var globalAuthPluginMutex sync.Mutex + +func newGlobalAuthNPluginFn() *idplugin.AuthNPlugin { + globalAuthPluginMutex.Lock() + defer globalAuthPluginMutex.Unlock() + return globalAuthNPlugin +} + +func newGlobalAuthZPluginFn() *polplugin.AuthZPlugin { + globalAuthPluginMutex.Lock() + defer globalAuthPluginMutex.Unlock() + return globalAuthZPlugin +} + +func setGlobalAuthNPlugin(authn *idplugin.AuthNPlugin) { + globalAuthPluginMutex.Lock() + globalAuthNPlugin = authn + globalAuthPluginMutex.Unlock() +} + +func setGlobalAuthZPlugin(authz *polplugin.AuthZPlugin) { + globalAuthPluginMutex.Lock() + globalAuthZPlugin = authz + globalAuthPluginMutex.Unlock() +} + +var errSelfTestFailure = errors.New("self test failed. unsafe to start server") diff --git a/cmd/grid.go b/cmd/grid.go new file mode 100644 index 00000000..0b442267 --- /dev/null +++ b/cmd/grid.go @@ -0,0 +1,107 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "crypto/tls" + "sync/atomic" + + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/grid" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/rest" +) + +// globalGrid is the global grid manager. +var globalGrid atomic.Pointer[grid.Manager] + +// globalLockGrid is the global lock grid manager. +var globalLockGrid atomic.Pointer[grid.Manager] + +// globalGridStart is a channel that will block startup of grid connections until closed. +var globalGridStart = make(chan struct{}) + +// globalLockGridStart is a channel that will block startup of lock grid connections until closed. +var globalLockGridStart = make(chan struct{}) + +func initGlobalGrid(ctx context.Context, eps EndpointServerPools) error { + hosts, local := eps.GridHosts() + lookupHost := globalDNSCache.LookupHost + g, err := grid.NewManager(ctx, grid.ManagerOptions{ + // Pass Dialer for websocket grid, make sure we do not + // provide any DriveOPTimeout() function, as that is not + // useful over persistent connections. + Dialer: grid.ConnectWS( + grid.ContextDialer(xhttp.DialContextWithLookupHost(lookupHost, xhttp.NewInternodeDialContext(rest.DefaultTimeout, globalTCPOptions.ForWebsocket()))), + newCachedAuthToken(), + &tls.Config{ + RootCAs: globalRootCAs, + CipherSuites: crypto.TLSCiphers(), + CurvePreferences: crypto.TLSCurveIDs(), + }), + Local: local, + Hosts: hosts, + AuthToken: validateStorageRequestToken, + AuthFn: newCachedAuthToken(), + BlockConnect: globalGridStart, + // Record incoming and outgoing bytes. + Incoming: globalConnStats.incInternodeInputBytes, + Outgoing: globalConnStats.incInternodeOutputBytes, + TraceTo: globalTrace, + RoutePath: grid.RoutePath, + }) + if err != nil { + return err + } + globalGrid.Store(g) + return nil +} + +func initGlobalLockGrid(ctx context.Context, eps EndpointServerPools) error { + hosts, local := eps.GridHosts() + lookupHost := globalDNSCache.LookupHost + g, err := grid.NewManager(ctx, grid.ManagerOptions{ + // Pass Dialer for websocket grid, make sure we do not + // provide any DriveOPTimeout() function, as that is not + // useful over persistent connections. + Dialer: grid.ConnectWSWithRoutePath( + grid.ContextDialer(xhttp.DialContextWithLookupHost(lookupHost, xhttp.NewInternodeDialContext(rest.DefaultTimeout, globalTCPOptions.ForWebsocket()))), + newCachedAuthToken(), + &tls.Config{ + RootCAs: globalRootCAs, + CipherSuites: crypto.TLSCiphers(), + CurvePreferences: crypto.TLSCurveIDs(), + }, grid.RouteLockPath), + Local: local, + Hosts: hosts, + AuthToken: validateStorageRequestToken, + AuthFn: newCachedAuthToken(), + BlockConnect: globalGridStart, + // Record incoming and outgoing bytes. + Incoming: globalConnStats.incInternodeInputBytes, + Outgoing: globalConnStats.incInternodeOutputBytes, + TraceTo: globalTrace, + RoutePath: grid.RouteLockPath, + }) + if err != nil { + return err + } + globalLockGrid.Store(g) + return nil +} diff --git a/cmd/handler-api.go b/cmd/handler-api.go new file mode 100644 index 00000000..09790a60 --- /dev/null +++ b/cmd/handler-api.go @@ -0,0 +1,420 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "math" + "net/http" + "os" + "runtime" + "slices" + "strconv" + "strings" + "sync" + "time" + + "github.com/dustin/go-humanize" + "github.com/shirou/gopsutil/v3/mem" + + "github.com/minio/minio/internal/config/api" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/mcontext" +) + +type apiConfig struct { + mu sync.RWMutex + + requestsPool chan struct{} + clusterDeadline time.Duration + listQuorum string + corsAllowOrigins []string + replicationPriority string + replicationMaxWorkers int + replicationMaxLWorkers int + transitionWorkers int + + staleUploadsExpiry time.Duration + staleUploadsCleanupInterval time.Duration + deleteCleanupInterval time.Duration + enableODirect bool + gzipObjects bool + rootAccess bool + syncEvents bool + objectMaxVersions int64 +} + +const ( + cgroupV1MemLimitFile = "/sys/fs/cgroup/memory/memory.limit_in_bytes" + cgroupV2MemLimitFile = "/sys/fs/cgroup/memory.max" +) + +func cgroupMemLimit() (limit uint64) { + buf, err := os.ReadFile(cgroupV2MemLimitFile) + if err != nil { + buf, err = os.ReadFile(cgroupV1MemLimitFile) + } + if err != nil { + return 0 + } + limit, err = strconv.ParseUint(strings.TrimSpace(string(buf)), 10, 64) + if err != nil { + // The kernel can return valid but non integer values + // but still, no need to interpret more + return 0 + } + if limit >= 100*humanize.TiByte { + // No limit set, or unreasonably high. Ignore + return 0 + } + return limit +} + +func availableMemory() (available uint64) { + available = 2048 * blockSizeV2 * 2 // Default to 4 GiB when we can't find the limits. + + if runtime.GOOS == "linux" { + // Honor cgroup limits if set. + limit := cgroupMemLimit() + if limit > 0 { + // A valid value is found, return its 90% + available = (limit * 9) / 10 + return available + } + } // for all other platforms limits are based on virtual memory. + + memStats, err := mem.VirtualMemory() + if err != nil { + return available + } + + // A valid value is available return its 90% + available = (memStats.Available * 9) / 10 + return available +} + +func (t *apiConfig) init(cfg api.Config, setDriveCounts []int, legacy bool) { + t.mu.Lock() + defer t.mu.Unlock() + + clusterDeadline := cfg.ClusterDeadline + if clusterDeadline == 0 { + clusterDeadline = 10 * time.Second + } + t.clusterDeadline = clusterDeadline + corsAllowOrigin := cfg.CorsAllowOrigin + if len(corsAllowOrigin) == 0 { + corsAllowOrigin = []string{"*"} + } + t.corsAllowOrigins = corsAllowOrigin + + var apiRequestsMaxPerNode int + if cfg.RequestsMax <= 0 { + maxSetDrives := slices.Max(setDriveCounts) + + // Returns 75% of max memory allowed + maxMem := globalServerCtxt.MemLimit + + // max requests per node is calculated as + // total_ram / ram_per_request + blockSize := xioutil.LargeBlock + xioutil.SmallBlock + if legacy { + // ram_per_request is (1MiB+32KiB) * driveCount \ + // + 2 * 10MiB (default erasure block size v1) + 2 * 1MiB (default erasure block size v2) + apiRequestsMaxPerNode = int(maxMem / uint64(maxSetDrives*blockSize+int(blockSizeV1*2+blockSizeV2*2))) + } else { + // ram_per_request is (1MiB+32KiB) * driveCount \ + // + 2 * 1MiB (default erasure block size v2) + apiRequestsMaxPerNode = int(maxMem / uint64(maxSetDrives*blockSize+int(blockSizeV2*2))) + } + } else { + apiRequestsMaxPerNode = cfg.RequestsMax + if n := totalNodeCount(); n > 0 { + apiRequestsMaxPerNode /= n + } + } + + if globalIsDistErasure { + logger.Info("Configured max API requests per node based on available memory: %d", apiRequestsMaxPerNode) + } + + if cap(t.requestsPool) != apiRequestsMaxPerNode { + // Only replace if needed. + // Existing requests will use the previous limit, + // but new requests will use the new limit. + // There will be a short overlap window, + // but this shouldn't last long. + t.requestsPool = make(chan struct{}, apiRequestsMaxPerNode) + } + listQuorum := cfg.ListQuorum + if listQuorum == "" { + listQuorum = "strict" + } + t.listQuorum = listQuorum + if r := globalReplicationPool.GetNonBlocking(); r != nil && + (cfg.ReplicationPriority != t.replicationPriority || cfg.ReplicationMaxWorkers != t.replicationMaxWorkers || cfg.ReplicationMaxLWorkers != t.replicationMaxLWorkers) { + r.ResizeWorkerPriority(cfg.ReplicationPriority, cfg.ReplicationMaxWorkers, cfg.ReplicationMaxLWorkers) + } + t.replicationPriority = cfg.ReplicationPriority + t.replicationMaxWorkers = cfg.ReplicationMaxWorkers + t.replicationMaxLWorkers = cfg.ReplicationMaxLWorkers + + // N B api.transition_workers will be deprecated + if globalTransitionState != nil { + globalTransitionState.UpdateWorkers(cfg.TransitionWorkers) + } + t.transitionWorkers = cfg.TransitionWorkers + + t.staleUploadsExpiry = cfg.StaleUploadsExpiry + t.deleteCleanupInterval = cfg.DeleteCleanupInterval + t.enableODirect = cfg.EnableODirect + t.gzipObjects = cfg.GzipObjects + t.rootAccess = cfg.RootAccess + t.syncEvents = cfg.SyncEvents + t.objectMaxVersions = cfg.ObjectMaxVersions + + if t.staleUploadsCleanupInterval != cfg.StaleUploadsCleanupInterval { + t.staleUploadsCleanupInterval = cfg.StaleUploadsCleanupInterval + + // signal that cleanup interval has changed + select { + case staleUploadsCleanupIntervalChangedCh <- struct{}{}: + default: // in case the channel is blocked... + } + } +} + +func (t *apiConfig) odirectEnabled() bool { + t.mu.RLock() + defer t.mu.RUnlock() + + return t.enableODirect +} + +func (t *apiConfig) shouldGzipObjects() bool { + t.mu.RLock() + defer t.mu.RUnlock() + + return t.gzipObjects +} + +func (t *apiConfig) permitRootAccess() bool { + t.mu.RLock() + defer t.mu.RUnlock() + + return t.rootAccess +} + +func (t *apiConfig) getListQuorum() string { + t.mu.RLock() + defer t.mu.RUnlock() + + if t.listQuorum == "" { + return "strict" + } + + return t.listQuorum +} + +func (t *apiConfig) getCorsAllowOrigins() []string { + t.mu.RLock() + defer t.mu.RUnlock() + + if len(t.corsAllowOrigins) == 0 { + return []string{"*"} + } + + corsAllowOrigins := make([]string, len(t.corsAllowOrigins)) + copy(corsAllowOrigins, t.corsAllowOrigins) + return corsAllowOrigins +} + +func (t *apiConfig) getStaleUploadsCleanupInterval() time.Duration { + t.mu.RLock() + defer t.mu.RUnlock() + + if t.staleUploadsCleanupInterval == 0 { + return 6 * time.Hour // default 6 hours + } + + return t.staleUploadsCleanupInterval +} + +func (t *apiConfig) getStaleUploadsExpiry() time.Duration { + t.mu.RLock() + defer t.mu.RUnlock() + + if t.staleUploadsExpiry == 0 { + return 24 * time.Hour // default 24 hours + } + + return t.staleUploadsExpiry +} + +func (t *apiConfig) getDeleteCleanupInterval() time.Duration { + t.mu.RLock() + defer t.mu.RUnlock() + + if t.deleteCleanupInterval == 0 { + return 5 * time.Minute // every 5 minutes + } + + return t.deleteCleanupInterval +} + +func (t *apiConfig) getClusterDeadline() time.Duration { + t.mu.RLock() + defer t.mu.RUnlock() + + if t.clusterDeadline == 0 { + return 10 * time.Second + } + + return t.clusterDeadline +} + +func (t *apiConfig) getRequestsPoolCapacity() int { + t.mu.RLock() + defer t.mu.RUnlock() + + return cap(t.requestsPool) +} + +func (t *apiConfig) getRequestsPool() chan struct{} { + t.mu.RLock() + defer t.mu.RUnlock() + + if t.requestsPool == nil { + return nil + } + + return t.requestsPool +} + +// maxClients throttles the S3 API calls +func maxClients(f http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + globalHTTPStats.incS3RequestsIncoming() + + if r.Header.Get(globalObjectPerfUserMetadata) == "" { + if val := globalServiceFreeze.Load(); val != nil { + if unlock, ok := val.(chan struct{}); ok && unlock != nil { + // Wait until unfrozen. + select { + case <-unlock: + case <-r.Context().Done(): + // if client canceled we don't need to wait here forever. + return + } + } + } + } + + globalHTTPStats.addRequestsInQueue(1) + pool := globalAPIConfig.getRequestsPool() + if pool == nil { + globalHTTPStats.addRequestsInQueue(-1) + f.ServeHTTP(w, r) + return + } + + if tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt); ok { + tc.FuncName = "s3.MaxClients" + } + + w.Header().Set("X-RateLimit-Limit", strconv.Itoa(cap(pool))) + w.Header().Set("X-RateLimit-Remaining", strconv.Itoa(cap(pool)-len(pool))) + + ctx := r.Context() + select { + case pool <- struct{}{}: + defer func() { <-pool }() + globalHTTPStats.addRequestsInQueue(-1) + if contextCanceled(ctx) { + w.WriteHeader(499) + return + } + f.ServeHTTP(w, r) + case <-r.Context().Done(): + globalHTTPStats.addRequestsInQueue(-1) + // When the client disconnects before getting the S3 handler + // status code response, set the status code to 499 so this request + // will be properly audited and traced. + w.WriteHeader(499) + default: + globalHTTPStats.addRequestsInQueue(-1) + if contextCanceled(ctx) { + w.WriteHeader(499) + return + } + // Send a http timeout message + writeErrorResponse(ctx, w, + errorCodes.ToAPIErr(ErrTooManyRequests), + r.URL) + } + } +} + +func (t *apiConfig) getReplicationOpts() replicationPoolOpts { + t.mu.RLock() + defer t.mu.RUnlock() + + if t.replicationPriority == "" { + return replicationPoolOpts{ + Priority: "auto", + MaxWorkers: WorkerMaxLimit, + MaxLWorkers: LargeWorkerCount, + } + } + + return replicationPoolOpts{ + Priority: t.replicationPriority, + MaxWorkers: t.replicationMaxWorkers, + MaxLWorkers: t.replicationMaxLWorkers, + } +} + +func (t *apiConfig) getTransitionWorkers() int { + t.mu.RLock() + defer t.mu.RUnlock() + + if t.transitionWorkers <= 0 { + return runtime.GOMAXPROCS(0) / 2 + } + + return t.transitionWorkers +} + +func (t *apiConfig) isSyncEventsEnabled() bool { + t.mu.RLock() + defer t.mu.RUnlock() + + return t.syncEvents +} + +func (t *apiConfig) getObjectMaxVersions() int64 { + t.mu.RLock() + defer t.mu.RUnlock() + + if t.objectMaxVersions <= 0 { + // defaults to 'IntMax' when unset. + return math.MaxInt64 + } + + return t.objectMaxVersions +} diff --git a/cmd/handler-utils.go b/cmd/handler-utils.go new file mode 100644 index 00000000..7752cfec --- /dev/null +++ b/cmd/handler-utils.go @@ -0,0 +1,504 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/textproto" + "regexp" + "strings" + "sync/atomic" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/handlers" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/mcontext" + xnet "github.com/minio/pkg/v3/net" +) + +const ( + copyDirective = "COPY" + replaceDirective = "REPLACE" + accessDirective = "ACCESS" +) + +// Parses location constraint from the incoming reader. +func parseLocationConstraint(r *http.Request) (location string, s3Error APIErrorCode) { + // If the request has no body with content-length set to 0, + // we do not have to validate location constraint. Bucket will + // be created at default region. + locationConstraint := createBucketLocationConfiguration{} + err := xmlDecoder(r.Body, &locationConstraint, r.ContentLength) + if err != nil && r.ContentLength != 0 { + internalLogOnceIf(GlobalContext, err, "location-constraint-xml-parsing") + // Treat all other failures as XML parsing errors. + return "", ErrMalformedXML + } // else for both err as nil or io.EOF + location = locationConstraint.Location + if location == "" { + location = globalSite.Region() + } + if !isValidLocation(location) { + return location, ErrInvalidRegion + } + + return location, ErrNone +} + +// Validates input location is same as configured region +// of MinIO server. +func isValidLocation(location string) bool { + region := globalSite.Region() + return region == "" || region == location +} + +// Supported headers that needs to be extracted. +var supportedHeaders = []string{ + "content-type", + "cache-control", + "content-language", + "content-encoding", + "content-disposition", + "x-amz-storage-class", + xhttp.AmzStorageClass, + xhttp.AmzObjectTagging, + "expires", + xhttp.AmzBucketReplicationStatus, + "X-Minio-Replication-Server-Side-Encryption-Sealed-Key", + "X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm", + "X-Minio-Replication-Server-Side-Encryption-Iv", + "X-Minio-Replication-Encrypted-Multipart", + "X-Minio-Replication-Actual-Object-Size", + ReplicationSsecChecksumHeader, + // Add more supported headers here. +} + +// mapping of internal headers to allowed replication headers +var validSSEReplicationHeaders = map[string]string{ + "X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "X-Minio-Replication-Server-Side-Encryption-Sealed-Key", + "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm", + "X-Minio-Internal-Server-Side-Encryption-Iv": "X-Minio-Replication-Server-Side-Encryption-Iv", + "X-Minio-Internal-Encrypted-Multipart": "X-Minio-Replication-Encrypted-Multipart", + "X-Minio-Internal-Actual-Object-Size": "X-Minio-Replication-Actual-Object-Size", + // Add more supported headers here. +} + +// mapping of replication headers to internal headers +var replicationToInternalHeaders = map[string]string{ + "X-Minio-Replication-Server-Side-Encryption-Sealed-Key": "X-Minio-Internal-Server-Side-Encryption-Sealed-Key", + "X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm": "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm", + "X-Minio-Replication-Server-Side-Encryption-Iv": "X-Minio-Internal-Server-Side-Encryption-Iv", + "X-Minio-Replication-Encrypted-Multipart": "X-Minio-Internal-Encrypted-Multipart", + "X-Minio-Replication-Actual-Object-Size": "X-Minio-Internal-Actual-Object-Size", + ReplicationSsecChecksumHeader: ReplicationSsecChecksumHeader, + // Add more supported headers here. +} + +// isDirectiveValid - check if tagging-directive is valid. +func isDirectiveValid(v string) bool { + // Check if set metadata-directive is valid. + return isDirectiveCopy(v) || isDirectiveReplace(v) +} + +// Check if the directive COPY is requested. +func isDirectiveCopy(value string) bool { + // By default if directive is not set we + // treat it as 'COPY' this function returns true. + return value == copyDirective || value == "" +} + +// Check if the directive REPLACE is requested. +func isDirectiveReplace(value string) bool { + return value == replaceDirective +} + +// userMetadataKeyPrefixes contains the prefixes of used-defined metadata keys. +// All values stored with a key starting with one of the following prefixes +// must be extracted from the header. +var userMetadataKeyPrefixes = []string{ + "x-amz-meta-", + "x-minio-meta-", +} + +// extractMetadataFromReq extracts metadata from HTTP header and HTTP queryString. +func extractMetadataFromReq(ctx context.Context, r *http.Request) (metadata map[string]string, err error) { + return extractMetadata(ctx, textproto.MIMEHeader(r.Form), textproto.MIMEHeader(r.Header)) +} + +func extractMetadata(ctx context.Context, mimesHeader ...textproto.MIMEHeader) (metadata map[string]string, err error) { + metadata = make(map[string]string) + + for _, hdr := range mimesHeader { + // Extract all query values. + err = extractMetadataFromMime(ctx, hdr, metadata) + if err != nil { + return nil, err + } + } + + // Set content-type to default value if it is not set. + if _, ok := metadata[strings.ToLower(xhttp.ContentType)]; !ok { + metadata[strings.ToLower(xhttp.ContentType)] = "binary/octet-stream" + } + + // https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w + for k := range metadata { + if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) { + delete(metadata, k) + } + } + + if contentEncoding, ok := metadata[strings.ToLower(xhttp.ContentEncoding)]; ok { + contentEncoding = trimAwsChunkedContentEncoding(contentEncoding) + if contentEncoding != "" { + // Make sure to trim and save the content-encoding + // parameter for a streaming signature which is set + // to a custom value for example: "aws-chunked,gzip". + metadata[strings.ToLower(xhttp.ContentEncoding)] = contentEncoding + } else { + // Trimmed content encoding is empty when the header + // value is set to "aws-chunked" only. + + // Make sure to delete the content-encoding parameter + // for a streaming signature which is set to value + // for example: "aws-chunked" + delete(metadata, strings.ToLower(xhttp.ContentEncoding)) + } + } + + // Success. + return metadata, nil +} + +// extractMetadata extracts metadata from map values. +func extractMetadataFromMime(ctx context.Context, v textproto.MIMEHeader, m map[string]string) error { + if v == nil { + bugLogIf(ctx, errInvalidArgument) + return errInvalidArgument + } + + nv := make(textproto.MIMEHeader, len(v)) + for k, kv := range v { + // Canonicalize all headers, to remove any duplicates. + nv[http.CanonicalHeaderKey(k)] = kv + } + + // Save all supported headers. + for _, supportedHeader := range supportedHeaders { + value, ok := nv[http.CanonicalHeaderKey(supportedHeader)] + if ok { + if v, ok := replicationToInternalHeaders[supportedHeader]; ok { + m[v] = strings.Join(value, ",") + } else { + m[supportedHeader] = strings.Join(value, ",") + } + } + } + + for key := range v { + for _, prefix := range userMetadataKeyPrefixes { + if !stringsHasPrefixFold(key, prefix) { + continue + } + value, ok := nv[http.CanonicalHeaderKey(key)] + if ok { + m[key] = strings.Join(value, ",") + break + } + } + } + return nil +} + +// Returns access credentials in the request Authorization header. +func getReqAccessCred(r *http.Request, region string) (cred auth.Credentials) { + cred, _, _ = getReqAccessKeyV4(r, region, serviceS3) + if cred.AccessKey == "" { + cred, _, _ = getReqAccessKeyV2(r) + } + return cred +} + +// Extract request params to be sent with event notification. +func extractReqParams(r *http.Request) map[string]string { + if r == nil { + return nil + } + + region := globalSite.Region() + cred := getReqAccessCred(r, region) + + principalID := cred.AccessKey + if cred.ParentUser != "" { + principalID = cred.ParentUser + } + + // Success. + m := map[string]string{ + "region": region, + "principalId": principalID, + "sourceIPAddress": handlers.GetSourceIP(r), + // Add more fields here. + } + if rangeField := r.Header.Get(xhttp.Range); rangeField != "" { + m["range"] = rangeField + } + + if _, ok := r.Header[xhttp.MinIOSourceReplicationRequest]; ok { + m[xhttp.MinIOSourceReplicationRequest] = "" + } + return m +} + +// Extract response elements to be sent with event notification. +func extractRespElements(w http.ResponseWriter) map[string]string { + if w == nil { + return map[string]string{} + } + return map[string]string{ + "requestId": w.Header().Get(xhttp.AmzRequestID), + "nodeId": w.Header().Get(xhttp.AmzRequestHostID), + "content-length": w.Header().Get(xhttp.ContentLength), + // Add more fields here. + } +} + +// Trims away `aws-chunked` from the content-encoding header if present. +// Streaming signature clients can have custom content-encoding such as +// `aws-chunked,gzip` here we need to only save `gzip`. +// For more refer http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html +func trimAwsChunkedContentEncoding(contentEnc string) (trimmedContentEnc string) { + if contentEnc == "" { + return contentEnc + } + var newEncs []string + for enc := range strings.SplitSeq(contentEnc, ",") { + if enc != streamingContentEncoding { + newEncs = append(newEncs, enc) + } + } + return strings.Join(newEncs, ",") +} + +func collectInternodeStats(f http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + f.ServeHTTP(w, r) + + tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt) + if !ok || tc == nil { + return + } + + globalConnStats.incInternodeInputBytes(int64(tc.RequestRecorder.Size())) + globalConnStats.incInternodeOutputBytes(int64(tc.ResponseRecorder.Size())) + } +} + +func collectAPIStats(api string, f http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + resource, err := getResource(r.URL.Path, r.Host, globalDomainNames) + if err != nil { + defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r)) + + apiErr := errorCodes.ToAPIErr(ErrUnsupportedHostHeader) + apiErr.Description = fmt.Sprintf("%s: %v", apiErr.Description, err) + + writeErrorResponse(r.Context(), w, apiErr, r.URL) + return + } + + bucket, _ := path2BucketObject(resource) + + meta, err := globalBucketMetadataSys.Get(bucket) // check if this bucket exists. + countBktStat := bucket != "" && bucket != minioReservedBucket && err == nil && !meta.Created.IsZero() + if countBktStat { + globalBucketHTTPStats.updateHTTPStats(bucket, api, nil) + } + + globalHTTPStats.currentS3Requests.Inc(api) + f.ServeHTTP(w, r) + globalHTTPStats.currentS3Requests.Dec(api) + + tc, _ := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt) + if tc != nil { + globalHTTPStats.updateStats(api, tc.ResponseRecorder) + globalConnStats.incS3InputBytes(int64(tc.RequestRecorder.Size())) + globalConnStats.incS3OutputBytes(int64(tc.ResponseRecorder.Size())) + + if countBktStat { + globalBucketConnStats.incS3InputBytes(bucket, int64(tc.RequestRecorder.Size())) + globalBucketConnStats.incS3OutputBytes(bucket, int64(tc.ResponseRecorder.Size())) + globalBucketHTTPStats.updateHTTPStats(bucket, api, tc.ResponseRecorder) + } + } + } +} + +// Returns "/bucketName/objectName" for path-style or virtual-host-style requests. +func getResource(path string, host string, domains []string) (string, error) { + if len(domains) == 0 { + return path, nil + } + + // If virtual-host-style is enabled construct the "resource" properly. + xhost, err := xnet.ParseHost(host) + if err != nil { + return "", err + } + + for _, domain := range domains { + if xhost.Name == minioReservedBucket+"."+domain { + continue + } + if !strings.HasSuffix(xhost.Name, "."+domain) { + continue + } + bucket := strings.TrimSuffix(xhost.Name, "."+domain) + return SlashSeparator + pathJoin(bucket, path), nil + } + return path, nil +} + +var regexVersion = regexp.MustCompile(`^/minio.*/(v\d+)/.*`) + +func extractAPIVersion(r *http.Request) string { + if matches := regexVersion.FindStringSubmatch(r.URL.Path); len(matches) > 1 { + return matches[1] + } + return "unknown" +} + +func methodNotAllowedHandler(api string) func(w http.ResponseWriter, r *http.Request) { + return errorResponseHandler +} + +// If none of the http routes match respond with appropriate errors +func errorResponseHandler(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodOptions { + return + } + desc := "Do not upgrade one server at a time - please follow the recommended guidelines mentioned here https://github.com/minio/minio#upgrading-minio for your environment" + switch { + case strings.HasPrefix(r.URL.Path, peerRESTPrefix): + writeErrorResponseString(r.Context(), w, APIError{ + Code: "XMinioPeerVersionMismatch", + Description: desc, + HTTPStatusCode: http.StatusUpgradeRequired, + }, r.URL) + case strings.HasPrefix(r.URL.Path, storageRESTPrefix): + writeErrorResponseString(r.Context(), w, APIError{ + Code: "XMinioStorageVersionMismatch", + Description: desc, + HTTPStatusCode: http.StatusUpgradeRequired, + }, r.URL) + case strings.HasPrefix(r.URL.Path, adminPathPrefix): + var desc string + version := extractAPIVersion(r) + switch version { + case "v1", madmin.AdminAPIVersionV2: + desc = fmt.Sprintf("Server expects client requests with 'admin' API version '%s', found '%s', please upgrade the client to latest releases", madmin.AdminAPIVersion, version) + case madmin.AdminAPIVersion: + desc = fmt.Sprintf("This 'admin' API is not supported by server in '%s'", getMinioMode()) + default: + desc = fmt.Sprintf("Unexpected client 'admin' API version found '%s', expected '%s', please downgrade the client to older releases", version, madmin.AdminAPIVersion) + } + writeErrorResponseJSON(r.Context(), w, APIError{ + Code: "XMinioAdminVersionMismatch", + Description: desc, + HTTPStatusCode: http.StatusUpgradeRequired, + }, r.URL) + default: + defer logger.AuditLog(r.Context(), w, r, mustGetClaimsFromToken(r)) + defer atomic.AddUint64(&globalHTTPStats.rejectedRequestsInvalid, 1) + + // When we are not running in S3 Express mode, generate appropriate error + // for x-amz-write-offset HEADER specified. + if _, ok := r.Header[xhttp.AmzWriteOffsetBytes]; ok { + tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt) + if ok { + tc.FuncName = "s3.AppendObject" + tc.ResponseRecorder.LogErrBody = true + } + + writeErrorResponse(r.Context(), w, getAPIError(ErrNotImplemented), r.URL) + return + } + + tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt) + if ok { + tc.FuncName = "s3.ValidRequest" + tc.ResponseRecorder.LogErrBody = true + } + + writeErrorResponse(r.Context(), w, APIError{ + Code: "BadRequest", + Description: fmt.Sprintf("An unsupported API call for method: %s at '%s'", + r.Method, r.URL.Path), + HTTPStatusCode: http.StatusBadRequest, + }, r.URL) + } +} + +// gets host name for current node +func getHostName(r *http.Request) (hostName string) { + if globalIsDistErasure { + hostName = globalLocalNodeName + } else { + hostName = r.Host + } + return hostName +} + +// Proxy any request to an endpoint. +func proxyRequest(ctx context.Context, w http.ResponseWriter, r *http.Request, ep ProxyEndpoint, returnErr bool) (success bool) { + success = true + + // Make sure we remove any existing headers before + // proxying the request to another node. + for k := range w.Header() { + w.Header().Del(k) + } + + f := handlers.NewForwarder(&handlers.Forwarder{ + PassHost: true, + RoundTripper: ep.Transport, + ErrorHandler: func(w http.ResponseWriter, r *http.Request, err error) { + success = false + if err != nil && !errors.Is(err, context.Canceled) { + proxyLogIf(GlobalContext, err) + } + if returnErr { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + } + }, + }) + + r.URL.Scheme = "http" + if globalIsTLS { + r.URL.Scheme = "https" + } + + r.URL.Host = ep.Host + f.ServeHTTP(w, r) + return success +} diff --git a/cmd/handler-utils_test.go b/cmd/handler-utils_test.go new file mode 100644 index 00000000..517f93fc --- /dev/null +++ b/cmd/handler-utils_test.go @@ -0,0 +1,203 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/xml" + "io" + "net/http" + "net/textproto" + "os" + "reflect" + "testing" + + "github.com/minio/minio/internal/config" +) + +// Tests validate bucket LocationConstraint. +func TestIsValidLocationConstraint(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + obj, fsDir, err := prepareFS(ctx) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(fsDir) + if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil { + t.Fatal(err) + } + + // Corrupted XML + malformedReq := &http.Request{ + Body: io.NopCloser(bytes.NewReader([]byte("<>"))), + ContentLength: int64(len("<>")), + } + + // Not an XML + badRequest := &http.Request{ + Body: io.NopCloser(bytes.NewReader([]byte("garbage"))), + ContentLength: int64(len("garbage")), + } + + // generates the input request with XML bucket configuration set to the request body. + createExpectedRequest := func(req *http.Request, location string) *http.Request { + createBucketConfig := createBucketLocationConfiguration{} + createBucketConfig.Location = location + createBucketConfigBytes, _ := xml.Marshal(createBucketConfig) + createBucketConfigBuffer := bytes.NewReader(createBucketConfigBytes) + req.Body = io.NopCloser(createBucketConfigBuffer) + req.ContentLength = int64(createBucketConfigBuffer.Len()) + return req + } + + testCases := []struct { + request *http.Request + serverConfigRegion string + expectedCode APIErrorCode + }{ + // Test case - 1. + {createExpectedRequest(&http.Request{}, "eu-central-1"), globalMinioDefaultRegion, ErrNone}, + // Test case - 2. + // In case of empty request body ErrNone is returned. + {createExpectedRequest(&http.Request{}, ""), globalMinioDefaultRegion, ErrNone}, + // Test case - 3 + // In case of garbage request body ErrMalformedXML is returned. + {badRequest, globalMinioDefaultRegion, ErrMalformedXML}, + // Test case - 4 + // In case of invalid XML request body ErrMalformedXML is returned. + {malformedReq, globalMinioDefaultRegion, ErrMalformedXML}, + } + + for i, testCase := range testCases { + config.SetRegion(globalServerConfig, testCase.serverConfigRegion) + _, actualCode := parseLocationConstraint(testCase.request) + if testCase.expectedCode != actualCode { + t.Errorf("Test %d: Expected the APIErrCode to be %d, but instead found %d", i+1, testCase.expectedCode, actualCode) + } + } +} + +// Tests validate metadata extraction from http headers. +func TestExtractMetadataHeaders(t *testing.T) { + testCases := []struct { + header http.Header + metadata map[string]string + shouldFail bool + }{ + // Validate if there a known 'content-type'. + { + header: http.Header{ + "Content-Type": []string{"image/png"}, + }, + metadata: map[string]string{ + "content-type": "image/png", + }, + shouldFail: false, + }, + // Validate if there are no keys to extract. + { + header: http.Header{ + "Test-1": []string{"123"}, + }, + metadata: map[string]string{}, + shouldFail: false, + }, + // Validate that there are all headers extracted + { + header: http.Header{ + "X-Amz-Meta-Appid": []string{"amz-meta"}, + "X-Minio-Meta-Appid": []string{"minio-meta"}, + }, + metadata: map[string]string{ + "X-Amz-Meta-Appid": "amz-meta", + "X-Minio-Meta-Appid": "minio-meta", + }, + shouldFail: false, + }, + // Fail if header key is not in canonicalized form + { + header: http.Header{ + "x-amz-meta-appid": []string{"amz-meta"}, + }, + metadata: map[string]string{ + "x-amz-meta-appid": "amz-meta", + }, + shouldFail: false, + }, + // Support multiple values + { + header: http.Header{ + "x-amz-meta-key": []string{"amz-meta1", "amz-meta2"}, + }, + metadata: map[string]string{ + "x-amz-meta-key": "amz-meta1,amz-meta2", + }, + shouldFail: false, + }, + // Empty header input returns empty metadata. + { + header: nil, + metadata: nil, + shouldFail: true, + }, + } + + // Validate if the extracting headers. + for i, testCase := range testCases { + metadata := make(map[string]string) + err := extractMetadataFromMime(t.Context(), textproto.MIMEHeader(testCase.header), metadata) + if err != nil && !testCase.shouldFail { + t.Fatalf("Test %d failed to extract metadata: %v", i+1, err) + } + if err == nil && testCase.shouldFail { + t.Fatalf("Test %d should fail, but it passed", i+1) + } + if err == nil && !reflect.DeepEqual(metadata, testCase.metadata) { + t.Fatalf("Test %d failed: Expected \"%#v\", got \"%#v\"", i+1, testCase.metadata, metadata) + } + } +} + +// Test getResource() +func TestGetResource(t *testing.T) { + testCases := []struct { + p string + host string + domains []string + expectedResource string + }{ + {"/a/b/c", "test.mydomain.com", []string{"mydomain.com"}, "/test/a/b/c"}, + {"/a/b/c", "[2001:0db8:85a3:0000:0000:8a2e:0370:7334]:17000", []string{"mydomain.com"}, "/a/b/c"}, + {"/a/b/c", "[2001:0db8:85a3:0000:0000:8a2e:0370:7334]", []string{"mydomain.com"}, "/a/b/c"}, + {"/a/b/c", "192.168.1.1:9000", []string{"mydomain.com"}, "/a/b/c"}, + {"/a/b/c", "test.mydomain.com", []string{"notmydomain.com"}, "/a/b/c"}, + {"/a/b/c", "test.mydomain.com", nil, "/a/b/c"}, + } + for i, test := range testCases { + gotResource, err := getResource(test.p, test.host, test.domains) + if err != nil { + t.Fatal(err) + } + if gotResource != test.expectedResource { + t.Fatalf("test %d: expected %s got %s", i+1, test.expectedResource, gotResource) + } + } +} diff --git a/cmd/hasher.go b/cmd/hasher.go new file mode 100644 index 00000000..a5e5d7d1 --- /dev/null +++ b/cmd/hasher.go @@ -0,0 +1,49 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "crypto/md5" + "encoding/hex" + + "github.com/minio/minio/internal/hash/sha256" +) + +// getSHA256Hash returns SHA-256 hash in hex encoding of given data. +func getSHA256Hash(data []byte) string { + return hex.EncodeToString(getSHA256Sum(data)) +} + +// getSHA256Hash returns SHA-256 sum of given data. +func getSHA256Sum(data []byte) []byte { + hash := sha256.New() + hash.Write(data) + return hash.Sum(nil) +} + +// getMD5Sum returns MD5 sum of given data. +func getMD5Sum(data []byte) []byte { + hash := md5.New() + hash.Write(data) + return hash.Sum(nil) +} + +// getMD5Hash returns MD5 hash in hex encoding of given data. +func getMD5Hash(data []byte) string { + return hex.EncodeToString(getMD5Sum(data)) +} diff --git a/cmd/healingmetric_string.go b/cmd/healingmetric_string.go new file mode 100644 index 00000000..012fc7aa --- /dev/null +++ b/cmd/healingmetric_string.go @@ -0,0 +1,25 @@ +// Code generated by "stringer -type=healingMetric -trimprefix=healingMetric erasure-healing.go"; DO NOT EDIT. + +package cmd + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[healingMetricBucket-0] + _ = x[healingMetricObject-1] + _ = x[healingMetricCheckAbandonedParts-2] +} + +const _healingMetric_name = "BucketObjectCheckAbandonedParts" + +var _healingMetric_index = [...]uint8{0, 6, 12, 31} + +func (i healingMetric) String() string { + if i >= healingMetric(len(_healingMetric_index)-1) { + return "healingMetric(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _healingMetric_name[_healingMetric_index[i]:_healingMetric_index[i+1]] +} diff --git a/cmd/healthcheck-handler.go b/cmd/healthcheck-handler.go new file mode 100644 index 00000000..12368d1d --- /dev/null +++ b/cmd/healthcheck-handler.go @@ -0,0 +1,212 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "net/http" + "strconv" + "time" + + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/kms" +) + +const unavailable = "offline" + +func checkHealth(w http.ResponseWriter) ObjectLayer { + objLayer := newObjectLayerFn() + if objLayer == nil { + w.Header().Set(xhttp.MinIOServerStatus, unavailable) + writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone) + return nil + } + + if !globalBucketMetadataSys.Initialized() { + w.Header().Set(xhttp.MinIOServerStatus, "bucket-metadata-offline") + writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone) + return nil + } + + if !globalIAMSys.Initialized() { + w.Header().Set(xhttp.MinIOServerStatus, "iam-offline") + writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone) + return nil + } + + return objLayer +} + +// ClusterCheckHandler returns if the server is ready for requests. +func ClusterCheckHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "ClusterCheckHandler") + + objLayer := checkHealth(w) + if objLayer == nil { + return + } + + ctx, cancel := context.WithTimeout(ctx, globalAPIConfig.getClusterDeadline()) + defer cancel() + + opts := HealthOptions{ + Maintenance: r.Form.Get("maintenance") == "true", + DeploymentType: r.Form.Get("deployment-type"), + } + result := objLayer.Health(ctx, opts) + w.Header().Set(xhttp.MinIOWriteQuorum, strconv.Itoa(result.WriteQuorum)) + w.Header().Set(xhttp.MinIOStorageClassDefaults, strconv.FormatBool(result.UsingDefaults)) + // return how many drives are being healed if any + if result.HealingDrives > 0 { + w.Header().Set(xhttp.MinIOHealingDrives, strconv.Itoa(result.HealingDrives)) + } + if !result.Healthy { + // As a maintenance call we are purposefully asked to be taken + // down, this is for orchestrators to know if we can safely + // take this server down, return appropriate error. + if opts.Maintenance { + writeResponse(w, http.StatusPreconditionFailed, nil, mimeNone) + } else { + writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone) + } + return + } + writeResponse(w, http.StatusOK, nil, mimeNone) +} + +// ClusterReadCheckHandler returns if the server is ready for requests. +func ClusterReadCheckHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "ClusterReadCheckHandler") + + objLayer := checkHealth(w) + if objLayer == nil { + return + } + + ctx, cancel := context.WithTimeout(ctx, globalAPIConfig.getClusterDeadline()) + defer cancel() + + opts := HealthOptions{ + Maintenance: r.Form.Get("maintenance") == "true", + DeploymentType: r.Form.Get("deployment-type"), + } + result := objLayer.Health(ctx, opts) + w.Header().Set(xhttp.MinIOReadQuorum, strconv.Itoa(result.ReadQuorum)) + w.Header().Set(xhttp.MinIOStorageClassDefaults, strconv.FormatBool(result.UsingDefaults)) + // return how many drives are being healed if any + if result.HealingDrives > 0 { + w.Header().Set(xhttp.MinIOHealingDrives, strconv.Itoa(result.HealingDrives)) + } + if !result.HealthyRead { + // As a maintenance call we are purposefully asked to be taken + // down, this is for orchestrators to know if we can safely + // take this server down, return appropriate error. + if opts.Maintenance { + writeResponse(w, http.StatusPreconditionFailed, nil, mimeNone) + } else { + writeResponse(w, http.StatusServiceUnavailable, nil, mimeNone) + } + return + } + writeResponse(w, http.StatusOK, nil, mimeNone) +} + +// ReadinessCheckHandler checks whether MinIO is up and ready to serve requests. +// It also checks whether the KMS is available and whether etcd is reachable, +// if configured. +func ReadinessCheckHandler(w http.ResponseWriter, r *http.Request) { + if objLayer := newObjectLayerFn(); objLayer == nil { + w.Header().Set(xhttp.MinIOServerStatus, unavailable) // Service not initialized yet + } + if r.Header.Get(xhttp.MinIOPeerCall) != "" { + writeResponse(w, http.StatusOK, nil, mimeNone) + return + } + + if int(globalHTTPStats.loadRequestsInQueue()) > globalAPIConfig.getRequestsPoolCapacity() { + apiErr := getAPIError(ErrBusy) + switch r.Method { + case http.MethodHead: + writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone) + case http.MethodGet: + writeErrorResponse(r.Context(), w, apiErr, r.URL) + } + return + } + + // Verify if KMS is reachable if its configured + if GlobalKMS != nil { + ctx, cancel := context.WithTimeout(r.Context(), time.Minute) + defer cancel() + + if _, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{AssociatedData: kms.Context{"healthcheck": ""}}); err != nil { + switch r.Method { + case http.MethodHead: + apiErr := toAPIError(r.Context(), err) + writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone) + case http.MethodGet: + writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL) + } + return + } + } + + if globalEtcdClient != nil { + // Borrowed from + // https://github.com/etcd-io/etcd/blob/main/etcdctl/ctlv3/command/ep_command.go#L118 + ctx, cancel := context.WithTimeout(r.Context(), defaultContextTimeout) + defer cancel() + if _, err := globalEtcdClient.Get(ctx, "health"); err != nil { + // etcd unreachable throw an error.. + switch r.Method { + case http.MethodHead: + apiErr := toAPIError(r.Context(), err) + writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone) + case http.MethodGet: + writeErrorResponse(r.Context(), w, toAPIError(r.Context(), err), r.URL) + } + return + } + } + writeResponse(w, http.StatusOK, nil, mimeNone) +} + +// LivenessCheckHandler checks whether MinIO is up. It differs from the +// readiness handler since a failing liveness check causes pod restarts +// in K8S environments. Therefore, it does not contact external systems. +func LivenessCheckHandler(w http.ResponseWriter, r *http.Request) { + if objLayer := newObjectLayerFn(); objLayer == nil { + w.Header().Set(xhttp.MinIOServerStatus, unavailable) // Service not initialized yet + } + if r.Header.Get(xhttp.MinIOPeerCall) != "" { + writeResponse(w, http.StatusOK, nil, mimeNone) + return + } + + if int(globalHTTPStats.loadRequestsInQueue()) > globalAPIConfig.getRequestsPoolCapacity() { + apiErr := getAPIError(ErrBusy) + switch r.Method { + case http.MethodHead: + writeResponse(w, apiErr.HTTPStatusCode, nil, mimeNone) + case http.MethodGet: + writeErrorResponse(r.Context(), w, apiErr, r.URL) + } + return + } + writeResponse(w, http.StatusOK, nil, mimeNone) +} diff --git a/cmd/healthcheck-router.go b/cmd/healthcheck-router.go new file mode 100644 index 00000000..d9adbfd2 --- /dev/null +++ b/cmd/healthcheck-router.go @@ -0,0 +1,53 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "net/http" + + "github.com/minio/mux" +) + +const ( + healthCheckPath = "/health" + healthCheckLivenessPath = "/live" + healthCheckReadinessPath = "/ready" + healthCheckClusterPath = "/cluster" + healthCheckClusterReadPath = "/cluster/read" + healthCheckPathPrefix = minioReservedBucketPath + healthCheckPath +) + +// registerHealthCheckRouter - add handler functions for liveness and readiness routes. +func registerHealthCheckRouter(router *mux.Router) { + // Healthcheck router + healthRouter := router.PathPrefix(healthCheckPathPrefix).Subrouter() + + // Cluster check handler to verify cluster is active + healthRouter.Methods(http.MethodGet).Path(healthCheckClusterPath).HandlerFunc(httpTraceAll(ClusterCheckHandler)) + healthRouter.Methods(http.MethodHead).Path(healthCheckClusterPath).HandlerFunc(httpTraceAll(ClusterCheckHandler)) + healthRouter.Methods(http.MethodGet).Path(healthCheckClusterReadPath).HandlerFunc(httpTraceAll(ClusterReadCheckHandler)) + healthRouter.Methods(http.MethodHead).Path(healthCheckClusterReadPath).HandlerFunc(httpTraceAll(ClusterReadCheckHandler)) + + // Liveness handler + healthRouter.Methods(http.MethodGet).Path(healthCheckLivenessPath).HandlerFunc(httpTraceAll(LivenessCheckHandler)) + healthRouter.Methods(http.MethodHead).Path(healthCheckLivenessPath).HandlerFunc(httpTraceAll(LivenessCheckHandler)) + + // Readiness handler + healthRouter.Methods(http.MethodGet).Path(healthCheckReadinessPath).HandlerFunc(httpTraceAll(ReadinessCheckHandler)) + healthRouter.Methods(http.MethodHead).Path(healthCheckReadinessPath).HandlerFunc(httpTraceAll(ReadinessCheckHandler)) +} diff --git a/cmd/http-stats.go b/cmd/http-stats.go new file mode 100644 index 00000000..18a24e58 --- /dev/null +++ b/cmd/http-stats.go @@ -0,0 +1,458 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "net/http" + "strings" + "sync" + "sync/atomic" + + xhttp "github.com/minio/minio/internal/http" + "github.com/prometheus/client_golang/prometheus" +) + +// connStats - Network statistics +// Count total input/output transferred bytes during +// the server's life. +type connStats struct { + internodeInputBytes uint64 + internodeOutputBytes uint64 + s3InputBytes uint64 + s3OutputBytes uint64 +} + +// Increase internode total input bytes +func (s *connStats) incInternodeInputBytes(n int64) { + atomic.AddUint64(&s.internodeInputBytes, uint64(n)) +} + +// Increase internode total output bytes +func (s *connStats) incInternodeOutputBytes(n int64) { + atomic.AddUint64(&s.internodeOutputBytes, uint64(n)) +} + +// Return internode total input bytes +func (s *connStats) getInternodeInputBytes() uint64 { + return atomic.LoadUint64(&s.internodeInputBytes) +} + +// Return total output bytes +func (s *connStats) getInternodeOutputBytes() uint64 { + return atomic.LoadUint64(&s.internodeOutputBytes) +} + +// Increase S3 total input bytes +func (s *connStats) incS3InputBytes(n int64) { + atomic.AddUint64(&s.s3InputBytes, uint64(n)) +} + +// Increase S3 total output bytes +func (s *connStats) incS3OutputBytes(n int64) { + atomic.AddUint64(&s.s3OutputBytes, uint64(n)) +} + +// Return S3 total input bytes +func (s *connStats) getS3InputBytes() uint64 { + return atomic.LoadUint64(&s.s3InputBytes) +} + +// Return S3 total output bytes +func (s *connStats) getS3OutputBytes() uint64 { + return atomic.LoadUint64(&s.s3OutputBytes) +} + +// Return connection stats (total input/output bytes and total s3 input/output bytes) +func (s *connStats) toServerConnStats() serverConnStats { + return serverConnStats{ + internodeInputBytes: s.getInternodeInputBytes(), // Traffic internode received + internodeOutputBytes: s.getInternodeOutputBytes(), // Traffic internode sent + s3InputBytes: s.getS3InputBytes(), // Traffic S3 received + s3OutputBytes: s.getS3OutputBytes(), // Traffic S3 sent + } +} + +// Prepare new ConnStats structure +func newConnStats() *connStats { + return &connStats{} +} + +type bucketS3RXTX struct { + s3InputBytes uint64 + s3OutputBytes uint64 +} + +type bucketHTTPAPIStats struct { + currentS3Requests *HTTPAPIStats + totalS3Requests *HTTPAPIStats + totalS34xxErrors *HTTPAPIStats + totalS35xxErrors *HTTPAPIStats + totalS3Canceled *HTTPAPIStats +} + +type bucketHTTPStats struct { + sync.RWMutex + httpStats map[string]bucketHTTPAPIStats +} + +func newBucketHTTPStats() *bucketHTTPStats { + return &bucketHTTPStats{ + httpStats: make(map[string]bucketHTTPAPIStats), + } +} + +func (bh *bucketHTTPStats) delete(bucket string) { + bh.Lock() + defer bh.Unlock() + + delete(bh.httpStats, bucket) +} + +func (bh *bucketHTTPStats) updateHTTPStats(bucket, api string, w *xhttp.ResponseRecorder) { + if bh == nil { + return + } + + if w != nil { + // Increment the prometheus http request response histogram with API, Bucket + bucketHTTPRequestsDuration.With(prometheus.Labels{ + "api": api, + "bucket": bucket, + }).Observe(w.TTFB().Seconds()) + } + + bh.Lock() + defer bh.Unlock() + + hstats, ok := bh.httpStats[bucket] + if !ok { + hstats = bucketHTTPAPIStats{ + currentS3Requests: &HTTPAPIStats{}, + totalS3Requests: &HTTPAPIStats{}, + totalS3Canceled: &HTTPAPIStats{}, + totalS34xxErrors: &HTTPAPIStats{}, + totalS35xxErrors: &HTTPAPIStats{}, + } + } + + if w == nil { // when response recorder nil, this is an active request + hstats.currentS3Requests.Inc(api) + bh.httpStats[bucket] = hstats + return + } // else { + hstats.currentS3Requests.Dec(api) // decrement this once we have the response recorder. + + hstats.totalS3Requests.Inc(api) + code := w.StatusCode + + switch { + case code == 0: + case code == 499: + // 499 is a good error, shall be counted as canceled. + hstats.totalS3Canceled.Inc(api) + case code >= http.StatusBadRequest: + if code >= http.StatusInternalServerError { + hstats.totalS35xxErrors.Inc(api) + } else { + hstats.totalS34xxErrors.Inc(api) + } + } + + bh.httpStats[bucket] = hstats +} + +func (bh *bucketHTTPStats) load(bucket string) bucketHTTPAPIStats { + if bh == nil { + return bucketHTTPAPIStats{ + currentS3Requests: &HTTPAPIStats{}, + totalS3Requests: &HTTPAPIStats{}, + totalS3Canceled: &HTTPAPIStats{}, + totalS34xxErrors: &HTTPAPIStats{}, + totalS35xxErrors: &HTTPAPIStats{}, + } + } + + bh.RLock() + defer bh.RUnlock() + + val, ok := bh.httpStats[bucket] + if ok { + return val + } + + return bucketHTTPAPIStats{ + currentS3Requests: &HTTPAPIStats{}, + totalS3Requests: &HTTPAPIStats{}, + totalS3Canceled: &HTTPAPIStats{}, + totalS34xxErrors: &HTTPAPIStats{}, + totalS35xxErrors: &HTTPAPIStats{}, + } +} + +type bucketConnStats struct { + sync.RWMutex + stats map[string]*bucketS3RXTX +} + +func newBucketConnStats() *bucketConnStats { + return &bucketConnStats{ + stats: make(map[string]*bucketS3RXTX), + } +} + +// Increase S3 total input bytes for input bucket +func (s *bucketConnStats) incS3InputBytes(bucket string, n int64) { + s.Lock() + defer s.Unlock() + stats, ok := s.stats[bucket] + if !ok { + stats = &bucketS3RXTX{ + s3InputBytes: uint64(n), + } + } else { + stats.s3InputBytes += uint64(n) + } + s.stats[bucket] = stats +} + +// Increase S3 total output bytes for input bucket +func (s *bucketConnStats) incS3OutputBytes(bucket string, n int64) { + s.Lock() + defer s.Unlock() + stats, ok := s.stats[bucket] + if !ok { + stats = &bucketS3RXTX{ + s3OutputBytes: uint64(n), + } + } else { + stats.s3OutputBytes += uint64(n) + } + s.stats[bucket] = stats +} + +type inOutBytes struct { + In uint64 + Out uint64 +} + +// Return S3 total input bytes for input bucket +func (s *bucketConnStats) getS3InOutBytes() map[string]inOutBytes { + s.RLock() + defer s.RUnlock() + + if len(s.stats) == 0 { + return nil + } + + bucketStats := make(map[string]inOutBytes, len(s.stats)) + for k, v := range s.stats { + bucketStats[k] = inOutBytes{ + In: v.s3InputBytes, + Out: v.s3OutputBytes, + } + } + return bucketStats +} + +// Return S3 total input/output bytes for each +func (s *bucketConnStats) getBucketS3InOutBytes(buckets []string) map[string]inOutBytes { + s.RLock() + defer s.RUnlock() + + if len(s.stats) == 0 || len(buckets) == 0 { + return nil + } + + bucketStats := make(map[string]inOutBytes, len(buckets)) + for _, bucket := range buckets { + if stats, ok := s.stats[bucket]; ok { + bucketStats[bucket] = inOutBytes{ + In: stats.s3InputBytes, + Out: stats.s3OutputBytes, + } + } + } + + return bucketStats +} + +// delete metrics once bucket is deleted. +func (s *bucketConnStats) delete(bucket string) { + s.Lock() + defer s.Unlock() + + delete(s.stats, bucket) +} + +// HTTPAPIStats holds statistics information about +// a given API in the requests. +type HTTPAPIStats struct { + apiStats map[string]int + sync.RWMutex +} + +// Inc increments the api stats counter. +func (stats *HTTPAPIStats) Inc(api string) { + if stats == nil { + return + } + stats.Lock() + defer stats.Unlock() + if stats.apiStats == nil { + stats.apiStats = make(map[string]int) + } + stats.apiStats[api]++ +} + +// Dec increments the api stats counter. +func (stats *HTTPAPIStats) Dec(api string) { + if stats == nil { + return + } + stats.Lock() + defer stats.Unlock() + if val, ok := stats.apiStats[api]; ok && val > 0 { + stats.apiStats[api]-- + } +} + +// Get returns the current counter on input API string +func (stats *HTTPAPIStats) Get(api string) int { + if stats == nil { + return 0 + } + + stats.RLock() + defer stats.RUnlock() + + val, ok := stats.apiStats[api] + if ok { + return val + } + + return 0 +} + +// Load returns the recorded stats. +func (stats *HTTPAPIStats) Load(toLower bool) map[string]int { + if stats == nil { + return map[string]int{} + } + + stats.RLock() + defer stats.RUnlock() + + apiStats := make(map[string]int, len(stats.apiStats)) + for k, v := range stats.apiStats { + if toLower { + k = strings.ToLower(k) + } + apiStats[k] = v + } + return apiStats +} + +// HTTPStats holds statistics information about +// HTTP requests made by all clients +type HTTPStats struct { + s3RequestsInQueue int32 // ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG + _ int32 // For 64 bits alignment + s3RequestsIncoming uint64 + rejectedRequestsAuth uint64 + rejectedRequestsTime uint64 + rejectedRequestsHeader uint64 + rejectedRequestsInvalid uint64 + currentS3Requests HTTPAPIStats + totalS3Requests HTTPAPIStats + totalS3Errors HTTPAPIStats + totalS34xxErrors HTTPAPIStats + totalS35xxErrors HTTPAPIStats + totalS3Canceled HTTPAPIStats +} + +func (st *HTTPStats) loadRequestsInQueue() int32 { + return atomic.LoadInt32(&st.s3RequestsInQueue) +} + +func (st *HTTPStats) addRequestsInQueue(i int32) { + atomic.AddInt32(&st.s3RequestsInQueue, i) +} + +func (st *HTTPStats) incS3RequestsIncoming() { + // Golang automatically resets to zero if this overflows + atomic.AddUint64(&st.s3RequestsIncoming, 1) +} + +// Converts http stats into struct to be sent back to the client. +func (st *HTTPStats) toServerHTTPStats(toLowerKeys bool) ServerHTTPStats { + serverStats := ServerHTTPStats{} + serverStats.S3RequestsIncoming = atomic.SwapUint64(&st.s3RequestsIncoming, 0) + serverStats.S3RequestsInQueue = atomic.LoadInt32(&st.s3RequestsInQueue) + serverStats.TotalS3RejectedAuth = atomic.LoadUint64(&st.rejectedRequestsAuth) + serverStats.TotalS3RejectedTime = atomic.LoadUint64(&st.rejectedRequestsTime) + serverStats.TotalS3RejectedHeader = atomic.LoadUint64(&st.rejectedRequestsHeader) + serverStats.TotalS3RejectedInvalid = atomic.LoadUint64(&st.rejectedRequestsInvalid) + serverStats.CurrentS3Requests = ServerHTTPAPIStats{ + APIStats: st.currentS3Requests.Load(toLowerKeys), + } + serverStats.TotalS3Requests = ServerHTTPAPIStats{ + APIStats: st.totalS3Requests.Load(toLowerKeys), + } + serverStats.TotalS3Errors = ServerHTTPAPIStats{ + APIStats: st.totalS3Errors.Load(toLowerKeys), + } + serverStats.TotalS34xxErrors = ServerHTTPAPIStats{ + APIStats: st.totalS34xxErrors.Load(toLowerKeys), + } + serverStats.TotalS35xxErrors = ServerHTTPAPIStats{ + APIStats: st.totalS35xxErrors.Load(toLowerKeys), + } + serverStats.TotalS3Canceled = ServerHTTPAPIStats{ + APIStats: st.totalS3Canceled.Load(toLowerKeys), + } + return serverStats +} + +// Update statistics from http request and response data +func (st *HTTPStats) updateStats(api string, w *xhttp.ResponseRecorder) { + st.totalS3Requests.Inc(api) + + // Increment the prometheus http request response histogram with appropriate label + httpRequestsDuration.With(prometheus.Labels{"api": api}).Observe(w.TTFB().Seconds()) + + code := w.StatusCode + + switch { + case code == 0: + case code == 499: + // 499 is a good error, shall be counted as canceled. + st.totalS3Canceled.Inc(api) + case code >= http.StatusBadRequest: + st.totalS3Errors.Inc(api) + if code >= http.StatusInternalServerError { + st.totalS35xxErrors.Inc(api) + } else { + st.totalS34xxErrors.Inc(api) + } + } +} + +// Prepare new HTTPStats structure +func newHTTPStats() *HTTPStats { + return &HTTPStats{} +} diff --git a/cmd/http-tracer.go b/cmd/http-tracer.go new file mode 100644 index 00000000..e7ad7462 --- /dev/null +++ b/cmd/http-tracer.go @@ -0,0 +1,200 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "net" + "net/http" + "reflect" + "regexp" + "runtime" + "strconv" + "strings" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/handlers" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/mcontext" +) + +var ldapPwdRegex = regexp.MustCompile("(^.*?)LDAPPassword=([^&]*?)(&(.*?))?$") + +// redact LDAP password if part of string +func redactLDAPPwd(s string) string { + parts := ldapPwdRegex.FindStringSubmatch(s) + if len(parts) > 3 { + return parts[1] + "LDAPPassword=*REDACTED*" + parts[3] + } + return s +} + +// getOpName sanitizes the operation name for mc +func getOpName(name string) (op string) { + op = strings.TrimPrefix(name, "github.com/minio/minio/cmd.") + op = strings.TrimSuffix(op, "Handler-fm") + op = strings.Replace(op, "objectAPIHandlers", "s3", 1) + op = strings.Replace(op, "adminAPIHandlers", "admin", 1) + op = strings.Replace(op, "(*storageRESTServer)", "storageR", 1) + op = strings.Replace(op, "(*peerRESTServer)", "peer", 1) + op = strings.Replace(op, "(*lockRESTServer)", "lockR", 1) + op = strings.Replace(op, "(*stsAPIHandlers)", "sts", 1) + op = strings.Replace(op, "(*peerS3Server)", "s3", 1) + op = strings.Replace(op, "ClusterCheckHandler", "health.Cluster", 1) + op = strings.Replace(op, "ClusterReadCheckHandler", "health.ClusterRead", 1) + op = strings.Replace(op, "LivenessCheckHandler", "health.Liveness", 1) + op = strings.Replace(op, "ReadinessCheckHandler", "health.Readiness", 1) + op = strings.Replace(op, "-fm", "", 1) + return op +} + +// If trace is enabled, execute the request if it is traced by other handlers +// otherwise, generate a trace event with request information but no response. +func httpTracerMiddleware(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Setup a http request response recorder - this is needed for + // http stats requests and audit if enabled. + respRecorder := xhttp.NewResponseRecorder(w) + + // Setup a http request body recorder + reqRecorder := &xhttp.RequestRecorder{Reader: r.Body} + r.Body = reqRecorder + + // Create tracing data structure and associate it to the request context + tc := mcontext.TraceCtxt{ + AmzReqID: w.Header().Get(xhttp.AmzRequestID), + RequestRecorder: reqRecorder, + ResponseRecorder: respRecorder, + } + + r = r.WithContext(context.WithValue(r.Context(), mcontext.ContextTraceKey, &tc)) + + reqStartTime := time.Now().UTC() + h.ServeHTTP(respRecorder, r) + reqEndTime := time.Now().UTC() + + if globalTrace.NumSubscribers(madmin.TraceS3|madmin.TraceInternal) == 0 { + // no subscribers nothing to trace. + return + } + + tt := madmin.TraceInternal + if strings.HasPrefix(tc.FuncName, "s3.") { + tt = madmin.TraceS3 + } + + // Calculate input body size with headers + reqHeaders := r.Header.Clone() + reqHeaders.Set("Host", r.Host) + if len(r.TransferEncoding) == 0 { + reqHeaders.Set("Content-Length", strconv.Itoa(int(r.ContentLength))) + } else { + reqHeaders.Set("Transfer-Encoding", strings.Join(r.TransferEncoding, ",")) + } + inputBytes := reqRecorder.Size() + for k, v := range reqHeaders { + inputBytes += len(k) + len(v) + } + + // Calculate node name + nodeName := r.Host + if globalIsDistErasure { + nodeName = globalLocalNodeName + } + if host, port, err := net.SplitHostPort(nodeName); err == nil { + if port == "443" || port == "80" { + nodeName = host + } + } + + // Calculate reqPath + reqPath := r.URL.RawPath + if reqPath == "" { + reqPath = r.URL.Path + } + + // Calculate function name + funcName := tc.FuncName + if funcName == "" { + funcName = "" + } + + t := madmin.TraceInfo{ + TraceType: tt, + FuncName: funcName, + NodeName: nodeName, + Time: reqStartTime, + Duration: reqEndTime.Sub(respRecorder.StartTime), + Path: reqPath, + Bytes: int64(inputBytes + respRecorder.Size()), + HTTP: &madmin.TraceHTTPStats{ + ReqInfo: madmin.TraceRequestInfo{ + Time: reqStartTime, + Proto: r.Proto, + Method: r.Method, + RawQuery: redactLDAPPwd(r.URL.RawQuery), + Client: handlers.GetSourceIP(r), + Headers: reqHeaders, + Path: reqPath, + Body: reqRecorder.Data(), + }, + RespInfo: madmin.TraceResponseInfo{ + Time: reqEndTime, + Headers: respRecorder.Header().Clone(), + StatusCode: respRecorder.StatusCode, + Body: respRecorder.Body(), + }, + CallStats: madmin.TraceCallStats{ + Latency: reqEndTime.Sub(respRecorder.StartTime), + InputBytes: inputBytes, + OutputBytes: respRecorder.Size(), + TimeToFirstByte: respRecorder.TTFB(), + }, + }, + } + + globalTrace.Publish(t) + }) +} + +func httpTrace(f http.HandlerFunc, logBody bool) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt) + if !ok { + // Tracing is not enabled for this request + f.ServeHTTP(w, r) + return + } + + tc.FuncName = getOpName(runtime.FuncForPC(reflect.ValueOf(f).Pointer()).Name()) + tc.RequestRecorder.LogBody = logBody + tc.ResponseRecorder.LogAllBody = logBody + tc.ResponseRecorder.LogErrBody = true + + f.ServeHTTP(w, r) + } +} + +func httpTraceAll(f http.HandlerFunc) http.HandlerFunc { + return httpTrace(f, true) +} + +func httpTraceHdrs(f http.HandlerFunc) http.HandlerFunc { + return httpTrace(f, false) +} diff --git a/cmd/http-tracer_test.go b/cmd/http-tracer_test.go new file mode 100644 index 00000000..a05e1de7 --- /dev/null +++ b/cmd/http-tracer_test.go @@ -0,0 +1,184 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "sync" + "testing" + "time" + + xhttp "github.com/minio/minio/internal/http" +) + +// Test redactLDAPPwd() +func TestRedactLDAPPwd(t *testing.T) { + testCases := []struct { + query string + expectedQuery string + }{ + {"", ""}, + { + "?Action=AssumeRoleWithLDAPIdentity&LDAPUsername=myusername&LDAPPassword=can+youreadthis%3F&Version=2011-06-15", + "?Action=AssumeRoleWithLDAPIdentity&LDAPUsername=myusername&LDAPPassword=*REDACTED*&Version=2011-06-15", + }, + { + "LDAPPassword=can+youreadthis%3F&Version=2011-06-15&?Action=AssumeRoleWithLDAPIdentity&LDAPUsername=myusername", + "LDAPPassword=*REDACTED*&Version=2011-06-15&?Action=AssumeRoleWithLDAPIdentity&LDAPUsername=myusername", + }, + { + "?Action=AssumeRoleWithLDAPIdentity&LDAPUsername=myusername&Version=2011-06-15&LDAPPassword=can+youreadthis%3F", + "?Action=AssumeRoleWithLDAPIdentity&LDAPUsername=myusername&Version=2011-06-15&LDAPPassword=*REDACTED*", + }, + { + "?x=y&a=b", + "?x=y&a=b", + }, + } + for i, test := range testCases { + gotQuery := redactLDAPPwd(test.query) + if gotQuery != test.expectedQuery { + t.Fatalf("test %d: expected %s got %s", i+1, test.expectedQuery, gotQuery) + } + } +} + +// TestHTTPStatsRaceCondition tests the race condition fix for HTTPStats. +// This test specifically addresses the race between: +// - Write operations via updateStats. +// - Read operations via toServerHTTPStats(false). +func TestRaulStatsRaceCondition(t *testing.T) { + httpStats := newHTTPStats() + // Simulate the concurrent scenario from the original race condition: + // Multiple HTTP request handlers updating stats concurrently, + // while background processes are reading the stats for persistence. + const numWriters = 100 // Simulate many HTTP request handlers. + const numReaders = 50 // Simulate background stats readers. + const opsPerGoroutine = 100 + + var wg sync.WaitGroup + for i := range numWriters { + wg.Add(1) + go func(writerID int) { + defer wg.Done() + for j := 0; j < opsPerGoroutine; j++ { + switch j % 4 { + case 0: + httpStats.updateStats("GetObject", &xhttp.ResponseRecorder{}) + case 1: + httpStats.totalS3Requests.Inc("PutObject") + case 2: + httpStats.totalS3Errors.Inc("DeleteObject") + case 3: + httpStats.currentS3Requests.Inc("ListObjects") + } + } + }(i) + } + + for i := range numReaders { + wg.Add(1) + go func(readerID int) { + defer wg.Done() + for range opsPerGoroutine { + _ = httpStats.toServerHTTPStats(false) + _ = httpStats.totalS3Requests.Load(false) + _ = httpStats.currentS3Requests.Load(false) + time.Sleep(1 * time.Microsecond) + } + }(i) + } + wg.Wait() + + finalStats := httpStats.toServerHTTPStats(false) + totalRequests := 0 + for _, v := range finalStats.TotalS3Requests.APIStats { + totalRequests += v + } + if totalRequests == 0 { + t.Error("Expected some total requests to be recorded, but got zero") + } + t.Logf("Total requests recorded: %d", totalRequests) + t.Logf("Race condition test passed - no races detected") +} + +// TestHTTPAPIStatsRaceCondition tests concurrent access to HTTPAPIStats specifically. +func TestRaulHTTPAPIStatsRaceCondition(t *testing.T) { + stats := &HTTPAPIStats{} + const numGoroutines = 50 + const opsPerGoroutine = 1000 + + var wg sync.WaitGroup + for i := range numGoroutines { + wg.Add(1) + go func(id int) { + defer wg.Done() + for j := 0; j < opsPerGoroutine; j++ { + stats.Inc("TestAPI") + } + }(i) + } + + for i := range numGoroutines / 2 { + wg.Add(1) + go func(id int) { + defer wg.Done() + for range opsPerGoroutine / 2 { + _ = stats.Load(false) + } + }(i) + } + wg.Wait() + + finalStats := stats.Load(false) + expected := numGoroutines * opsPerGoroutine + actual := finalStats["TestAPI"] + if actual != expected { + t.Errorf("Race condition detected: expected %d, got %d (lost %d increments)", + expected, actual, expected-actual) + } +} + +// TestBucketHTTPStatsRaceCondition tests concurrent access to bucket-level HTTP stats. +func TestRaulBucketHTTPStatsRaceCondition(t *testing.T) { + bucketStats := newBucketHTTPStats() + const numGoroutines = 50 + const opsPerGoroutine = 100 + + var wg sync.WaitGroup + for i := range numGoroutines { + wg.Add(1) + go func(id int) { + defer wg.Done() + bucketName := "test-bucket" + + for range opsPerGoroutine { + bucketStats.updateHTTPStats(bucketName, "GetObject", nil) + recorder := &xhttp.ResponseRecorder{} + bucketStats.updateHTTPStats(bucketName, "GetObject", recorder) + _ = bucketStats.load(bucketName) + } + }(i) + } + wg.Wait() + + stats := bucketStats.load("test-bucket") + if stats.totalS3Requests == nil { + t.Error("Expected bucket stats to be initialized") + } + t.Logf("Bucket HTTP stats race test passed") +} diff --git a/cmd/httprange.go b/cmd/httprange.go new file mode 100644 index 00000000..80e64f76 --- /dev/null +++ b/cmd/httprange.go @@ -0,0 +1,197 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "errors" + "fmt" + "strconv" + "strings" +) + +const ( + byteRangePrefix = "bytes=" +) + +// HTTPRangeSpec represents a range specification as supported by S3 GET +// object request. +// +// Case 1: Not present -> represented by a nil RangeSpec +// Case 2: bytes=1-10 (absolute start and end offsets) -> RangeSpec{false, 1, 10} +// Case 3: bytes=10- (absolute start offset with end offset unspecified) -> RangeSpec{false, 10, -1} +// Case 4: bytes=-30 (suffix length specification) -> RangeSpec{true, -30, -1} +type HTTPRangeSpec struct { + // Does the range spec refer to a suffix of the object? + IsSuffixLength bool + + // Start and end offset specified in range spec + Start, End int64 +} + +// GetLength - get length of range +func (h *HTTPRangeSpec) GetLength(resourceSize int64) (rangeLength int64, err error) { + switch { + case resourceSize < 0: + return 0, errors.New("Resource size cannot be negative") + + case h == nil: + rangeLength = resourceSize + + case h.IsSuffixLength: + specifiedLen := -h.Start + rangeLength = min(specifiedLen, resourceSize) + + case h.Start >= resourceSize: + return 0, InvalidRange{ + OffsetBegin: h.Start, + OffsetEnd: h.End, + ResourceSize: resourceSize, + } + + case h.End > -1: + end := h.End + if resourceSize <= end { + end = resourceSize - 1 + } + rangeLength = end - h.Start + 1 + + case h.End == -1: + rangeLength = resourceSize - h.Start + + default: + return 0, errors.New("Unexpected range specification case") + } + + return rangeLength, nil +} + +// GetOffsetLength computes the start offset and length of the range +// given the size of the resource +func (h *HTTPRangeSpec) GetOffsetLength(resourceSize int64) (start, length int64, err error) { + if h == nil { + // No range specified, implies whole object. + return 0, resourceSize, nil + } + + length, err = h.GetLength(resourceSize) + if err != nil { + return 0, 0, err + } + + start = h.Start + if h.IsSuffixLength { + start = max(resourceSize+h.Start, 0) + } + return start, length, nil +} + +// Parse a HTTP range header value into a HTTPRangeSpec +func parseRequestRangeSpec(rangeString string) (hrange *HTTPRangeSpec, err error) { + // Return error if given range string doesn't start with byte range prefix. + if !strings.HasPrefix(rangeString, byteRangePrefix) { + return nil, fmt.Errorf("'%s' does not start with '%s'", rangeString, byteRangePrefix) + } + + // Trim byte range prefix. + byteRangeString := strings.TrimPrefix(rangeString, byteRangePrefix) + + // Check if range string contains delimiter '-', else return error. eg. "bytes=8" + sepIndex := strings.Index(byteRangeString, "-") + if sepIndex == -1 { + return nil, fmt.Errorf("'%s' does not have a valid range value", rangeString) + } + + offsetBeginString := byteRangeString[:sepIndex] + offsetBegin := int64(-1) + // Convert offsetBeginString only if its not empty. + if len(offsetBeginString) > 0 { + if offsetBeginString[0] == '+' { + return nil, fmt.Errorf("Byte position ('%s') must not have a sign", offsetBeginString) + } else if offsetBegin, err = strconv.ParseInt(offsetBeginString, 10, 64); err != nil { + return nil, fmt.Errorf("'%s' does not have a valid first byte position value", rangeString) + } else if offsetBegin < 0 { + return nil, fmt.Errorf("First byte position is negative ('%d')", offsetBegin) + } + } + + offsetEndString := byteRangeString[sepIndex+1:] + offsetEnd := int64(-1) + // Convert offsetEndString only if its not empty. + if len(offsetEndString) > 0 { + if offsetEndString[0] == '+' { + return nil, fmt.Errorf("Byte position ('%s') must not have a sign", offsetEndString) + } else if offsetEnd, err = strconv.ParseInt(offsetEndString, 10, 64); err != nil { + return nil, fmt.Errorf("'%s' does not have a valid last byte position value", rangeString) + } else if offsetEnd < 0 { + return nil, fmt.Errorf("Last byte position is negative ('%d')", offsetEnd) + } + } + + switch { + case offsetBegin > -1 && offsetEnd > -1: + if offsetBegin > offsetEnd { + return nil, errInvalidRange + } + return &HTTPRangeSpec{false, offsetBegin, offsetEnd}, nil + case offsetBegin > -1: + return &HTTPRangeSpec{false, offsetBegin, -1}, nil + case offsetEnd > -1: + if offsetEnd == 0 { + return nil, errInvalidRange + } + return &HTTPRangeSpec{true, -offsetEnd, -1}, nil + default: + // rangeString contains first and last byte positions missing. eg. "bytes=-" + return nil, fmt.Errorf("'%s' does not have valid range value", rangeString) + } +} + +// String returns stringified representation of range for a particular resource size. +func (h *HTTPRangeSpec) String(resourceSize int64) string { + if h == nil { + return "" + } + off, length, err := h.GetOffsetLength(resourceSize) + if err != nil { + return "" + } + return fmt.Sprintf("%d-%d", off, off+length-1) +} + +// ToHeader returns the Range header value. +func (h *HTTPRangeSpec) ToHeader() (string, error) { + if h == nil { + return "", nil + } + start := strconv.Itoa(int(h.Start)) + end := strconv.Itoa(int(h.End)) + switch { + case h.Start >= 0 && h.End >= 0: + if h.Start > h.End { + return "", errInvalidRange + } + case h.IsSuffixLength: + end = strconv.Itoa(int(h.Start * -1)) + start = "" + case h.Start > -1: + end = "" + default: + return "", errors.New("does not have valid range value") + } + return fmt.Sprintf("bytes=%s-%s", start, end), nil +} diff --git a/cmd/httprange_test.go b/cmd/httprange_test.go new file mode 100644 index 00000000..ea13a380 --- /dev/null +++ b/cmd/httprange_test.go @@ -0,0 +1,146 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "testing" +) + +func TestHTTPRequestRangeSpec(t *testing.T) { + resourceSize := int64(10) + validRangeSpecs := []struct { + spec string + expOffset, expLength int64 + }{ + {"bytes=0-", 0, 10}, + {"bytes=1-", 1, 9}, + {"bytes=0-9", 0, 10}, + {"bytes=1-10", 1, 9}, + {"bytes=1-1", 1, 1}, + {"bytes=2-5", 2, 4}, + {"bytes=-5", 5, 5}, + {"bytes=-1", 9, 1}, + {"bytes=-1000", 0, 10}, + } + for i, testCase := range validRangeSpecs { + rs, err := parseRequestRangeSpec(testCase.spec) + if err != nil { + t.Errorf("unexpected err: %v", err) + } + o, l, err := rs.GetOffsetLength(resourceSize) + if err != nil { + t.Errorf("unexpected err: %v", err) + } + if o != testCase.expOffset || l != testCase.expLength { + t.Errorf("Case %d: got bad offset/length: %d,%d expected: %d,%d", + i, o, l, testCase.expOffset, testCase.expLength) + } + } + + unparsableRangeSpecs := []string{ + "bytes=-", + "bytes==", + "bytes==1-10", + "bytes=", + "bytes=aa", + "aa", + "", + "bytes=1-10-", + "bytes=1--10", + "bytes=-1-10", + "bytes=0-+3", + "bytes=+3-+5", + "bytes=10-11,12-10", // Unsupported by S3/MinIO (valid in RFC) + } + for i, urs := range unparsableRangeSpecs { + rs, err := parseRequestRangeSpec(urs) + if err == nil { + t.Errorf("Case %d: Did not get an expected error - got %v", i, rs) + } + if isErrInvalidRange(err) { + t.Errorf("Case %d: Got invalid range error instead of a parse error", i) + } + if rs != nil { + t.Errorf("Case %d: Got non-nil rs though err != nil: %v", i, rs) + } + } + + invalidRangeSpecs := []string{ + "bytes=5-3", + "bytes=10-10", + "bytes=10-", + "bytes=100-", + "bytes=-0", + } + for i, irs := range invalidRangeSpecs { + var err1, err2 error + var rs *HTTPRangeSpec + var o, l int64 + rs, err1 = parseRequestRangeSpec(irs) + if err1 == nil { + o, l, err2 = rs.GetOffsetLength(resourceSize) + } + if isErrInvalidRange(err1) || (err1 == nil && isErrInvalidRange(err2)) { + continue + } + t.Errorf("Case %d: Expected errInvalidRange but: %v %v %d %d %v", i, rs, err1, o, l, err2) + } +} + +func TestHTTPRequestRangeToHeader(t *testing.T) { + validRangeSpecs := []struct { + spec string + errExpected bool + }{ + {"bytes=0-", false}, + {"bytes=1-", false}, + + {"bytes=0-9", false}, + {"bytes=1-10", false}, + {"bytes=1-1", false}, + {"bytes=2-5", false}, + + {"bytes=-5", false}, + {"bytes=-1", false}, + {"bytes=-1000", false}, + {"bytes=", true}, + {"bytes= ", true}, + {"byte=", true}, + {"bytes=A-B", true}, + {"bytes=1-B", true}, + {"bytes=B-1", true}, + {"bytes=-1-1", true}, + } + for i, testCase := range validRangeSpecs { + rs, err := parseRequestRangeSpec(testCase.spec) + if err != nil { + if !testCase.errExpected || err == nil && testCase.errExpected { + t.Errorf("unexpected err: %v", err) + } + continue + } + h, err := rs.ToHeader() + if err != nil && !testCase.errExpected || err == nil && testCase.errExpected { + t.Errorf("expected error with invalid range: %v", err) + } + if h != testCase.spec { + t.Errorf("Case %d: translated to incorrect header: %s expected: %s", + i, h, testCase.spec) + } + } +} diff --git a/cmd/iam-etcd-store.go b/cmd/iam-etcd-store.go new file mode 100644 index 00000000..850606fd --- /dev/null +++ b/cmd/iam-etcd-store.go @@ -0,0 +1,502 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/json" + "errors" + "path" + "strings" + "sync" + "time" + + jsoniter "github.com/json-iterator/go" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/kms" + "github.com/puzpuzpuz/xsync/v3" + "go.etcd.io/etcd/api/v3/mvccpb" + etcd "go.etcd.io/etcd/client/v3" +) + +var defaultContextTimeout = 30 * time.Second + +func etcdKvsToSet(prefix string, kvs []*mvccpb.KeyValue) set.StringSet { + users := set.NewStringSet() + for _, kv := range kvs { + user := extractPathPrefixAndSuffix(string(kv.Key), prefix, path.Base(string(kv.Key))) + users.Add(user) + } + return users +} + +// Extract path string by stripping off the `prefix` value and the suffix, +// value, usually in the following form. +// +// s := "config/iam/users/foo/config.json" +// prefix := "config/iam/users/" +// suffix := "config.json" +// result is foo +func extractPathPrefixAndSuffix(s string, prefix string, suffix string) string { + return pathClean(strings.TrimSuffix(strings.TrimPrefix(s, prefix), suffix)) +} + +// IAMEtcdStore implements IAMStorageAPI +type IAMEtcdStore struct { + sync.RWMutex + + *iamCache + + usersSysType UsersSysType + + client *etcd.Client +} + +func newIAMEtcdStore(client *etcd.Client, usersSysType UsersSysType) *IAMEtcdStore { + return &IAMEtcdStore{ + iamCache: newIamCache(), + client: client, + usersSysType: usersSysType, + } +} + +func (ies *IAMEtcdStore) rlock() *iamCache { + ies.RLock() + return ies.iamCache +} + +func (ies *IAMEtcdStore) runlock() { + ies.RUnlock() +} + +func (ies *IAMEtcdStore) lock() *iamCache { + ies.Lock() + return ies.iamCache +} + +func (ies *IAMEtcdStore) unlock() { + ies.Unlock() +} + +func (ies *IAMEtcdStore) getUsersSysType() UsersSysType { + return ies.usersSysType +} + +func (ies *IAMEtcdStore) saveIAMConfig(ctx context.Context, item any, itemPath string, opts ...options) error { + data, err := json.Marshal(item) + if err != nil { + return err + } + if GlobalKMS != nil { + data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{ + minioMetaBucket: path.Join(minioMetaBucket, itemPath), + }) + if err != nil { + return err + } + } + return saveKeyEtcd(ctx, ies.client, itemPath, data, opts...) +} + +func getIAMConfig(item any, data []byte, itemPath string) error { + data, err := decryptData(data, itemPath) + if err != nil { + return err + } + json := jsoniter.ConfigCompatibleWithStandardLibrary + return json.Unmarshal(data, item) +} + +func (ies *IAMEtcdStore) loadIAMConfig(ctx context.Context, item any, path string) error { + data, err := readKeyEtcd(ctx, ies.client, path) + if err != nil { + return err + } + return getIAMConfig(item, data, path) +} + +func (ies *IAMEtcdStore) loadIAMConfigBytes(ctx context.Context, path string) ([]byte, error) { + data, err := readKeyEtcd(ctx, ies.client, path) + if err != nil { + return nil, err + } + return decryptData(data, path) +} + +func (ies *IAMEtcdStore) deleteIAMConfig(ctx context.Context, path string) error { + return deleteKeyEtcd(ctx, ies.client, path) +} + +func (ies *IAMEtcdStore) loadPolicyDocWithRetry(ctx context.Context, policy string, m map[string]PolicyDoc, _ int) error { + return ies.loadPolicyDoc(ctx, policy, m) +} + +func (ies *IAMEtcdStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]PolicyDoc) error { + data, err := ies.loadIAMConfigBytes(ctx, getPolicyDocPath(policy)) + if err != nil { + if err == errConfigNotFound { + return errNoSuchPolicy + } + return err + } + + var p PolicyDoc + err = p.parseJSON(data) + if err != nil { + return err + } + + m[policy] = p + return nil +} + +func (ies *IAMEtcdStore) getPolicyDocKV(ctx context.Context, kvs *mvccpb.KeyValue, m map[string]PolicyDoc) error { + data, err := decryptData(kvs.Value, string(kvs.Key)) + if err != nil { + if err == errConfigNotFound { + return errNoSuchPolicy + } + return err + } + + var p PolicyDoc + err = p.parseJSON(data) + if err != nil { + return err + } + + policy := extractPathPrefixAndSuffix(string(kvs.Key), iamConfigPoliciesPrefix, path.Base(string(kvs.Key))) + m[policy] = p + return nil +} + +func (ies *IAMEtcdStore) loadPolicyDocs(ctx context.Context, m map[string]PolicyDoc) error { + ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout) + defer cancel() + // Retrieve all keys and values to avoid too many calls to etcd in case of + // a large number of policies + r, err := ies.client.Get(ctx, iamConfigPoliciesPrefix, etcd.WithPrefix()) + if err != nil { + return err + } + + // Parse all values to construct the policies data model. + for _, kvs := range r.Kvs { + if err = ies.getPolicyDocKV(ctx, kvs, m); err != nil && !errors.Is(err, errNoSuchPolicy) { + return err + } + } + return nil +} + +func (ies *IAMEtcdStore) getUserKV(ctx context.Context, userkv *mvccpb.KeyValue, userType IAMUserType, m map[string]UserIdentity, basePrefix string) error { + var u UserIdentity + err := getIAMConfig(&u, userkv.Value, string(userkv.Key)) + if err != nil { + if err == errConfigNotFound { + return errNoSuchUser + } + return err + } + user := extractPathPrefixAndSuffix(string(userkv.Key), basePrefix, path.Base(string(userkv.Key))) + return ies.addUser(ctx, user, userType, u, m) +} + +func (ies *IAMEtcdStore) addUser(ctx context.Context, user string, userType IAMUserType, u UserIdentity, m map[string]UserIdentity) error { + if u.Credentials.IsExpired() { + // Delete expired identity. + deleteKeyEtcd(ctx, ies.client, getUserIdentityPath(user, userType)) + deleteKeyEtcd(ctx, ies.client, getMappedPolicyPath(user, userType, false)) + return nil + } + if u.Credentials.AccessKey == "" { + u.Credentials.AccessKey = user + } + if u.Credentials.SessionToken != "" { + jwtClaims, err := extractJWTClaims(u) + if err != nil { + if u.Credentials.IsTemp() { + // We should delete such that the client can re-request + // for the expiring credentials. + deleteKeyEtcd(ctx, ies.client, getUserIdentityPath(user, userType)) + deleteKeyEtcd(ctx, ies.client, getMappedPolicyPath(user, userType, false)) + } + return nil + } + u.Credentials.Claims = jwtClaims.Map() + } + if u.Credentials.Description == "" { + u.Credentials.Description = u.Credentials.Comment + } + + m[user] = u + return nil +} + +func (ies *IAMEtcdStore) loadSecretKey(ctx context.Context, user string, userType IAMUserType) (string, error) { + var u UserIdentity + err := ies.loadIAMConfig(ctx, &u, getUserIdentityPath(user, userType)) + if err != nil { + if errors.Is(err, errConfigNotFound) { + return "", errNoSuchUser + } + return "", err + } + return u.Credentials.SecretKey, nil +} + +func (ies *IAMEtcdStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]UserIdentity) error { + var u UserIdentity + err := ies.loadIAMConfig(ctx, &u, getUserIdentityPath(user, userType)) + if err != nil { + if err == errConfigNotFound { + return errNoSuchUser + } + return err + } + return ies.addUser(ctx, user, userType, u, m) +} + +func (ies *IAMEtcdStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]UserIdentity) error { + var basePrefix string + switch userType { + case svcUser: + basePrefix = iamConfigServiceAccountsPrefix + case stsUser: + basePrefix = iamConfigSTSPrefix + default: + basePrefix = iamConfigUsersPrefix + } + + cctx, cancel := context.WithTimeout(ctx, defaultContextTimeout) + defer cancel() + + // Retrieve all keys and values to avoid too many calls to etcd in case of + // a large number of users + r, err := ies.client.Get(cctx, basePrefix, etcd.WithPrefix()) + if err != nil { + return err + } + + // Parse all users values to create the proper data model + for _, userKv := range r.Kvs { + if err = ies.getUserKV(ctx, userKv, userType, m, basePrefix); err != nil && err != errNoSuchUser { + return err + } + } + return nil +} + +func (ies *IAMEtcdStore) loadGroup(ctx context.Context, group string, m map[string]GroupInfo) error { + var gi GroupInfo + err := ies.loadIAMConfig(ctx, &gi, getGroupInfoPath(group)) + if err != nil { + if err == errConfigNotFound { + return errNoSuchGroup + } + return err + } + m[group] = gi + return nil +} + +func (ies *IAMEtcdStore) loadGroups(ctx context.Context, m map[string]GroupInfo) error { + cctx, cancel := context.WithTimeout(ctx, defaultContextTimeout) + defer cancel() + + r, err := ies.client.Get(cctx, iamConfigGroupsPrefix, etcd.WithPrefix(), etcd.WithKeysOnly()) + if err != nil { + return err + } + + groups := etcdKvsToSet(iamConfigGroupsPrefix, r.Kvs) + + // Reload config for all groups. + for _, group := range groups.ToSlice() { + if err = ies.loadGroup(ctx, group, m); err != nil && err != errNoSuchGroup { + return err + } + } + return nil +} + +func (ies *IAMEtcdStore) loadMappedPolicyWithRetry(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy], retries int) error { + return ies.loadMappedPolicy(ctx, name, userType, isGroup, m) +} + +func (ies *IAMEtcdStore) loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error { + var p MappedPolicy + err := ies.loadIAMConfig(ctx, &p, getMappedPolicyPath(name, userType, isGroup)) + if err != nil { + if err == errConfigNotFound { + return errNoSuchPolicy + } + return err + } + m.Store(name, p) + return nil +} + +func getMappedPolicy(kv *mvccpb.KeyValue, m *xsync.MapOf[string, MappedPolicy], basePrefix string) error { + var p MappedPolicy + err := getIAMConfig(&p, kv.Value, string(kv.Key)) + if err != nil { + if err == errConfigNotFound { + return errNoSuchPolicy + } + return err + } + name := extractPathPrefixAndSuffix(string(kv.Key), basePrefix, ".json") + m.Store(name, p) + return nil +} + +func (ies *IAMEtcdStore) loadMappedPolicies(ctx context.Context, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error { + cctx, cancel := context.WithTimeout(ctx, defaultContextTimeout) + defer cancel() + var basePrefix string + if isGroup { + basePrefix = iamConfigPolicyDBGroupsPrefix + } else { + switch userType { + case svcUser: + basePrefix = iamConfigPolicyDBServiceAccountsPrefix + case stsUser: + basePrefix = iamConfigPolicyDBSTSUsersPrefix + default: + basePrefix = iamConfigPolicyDBUsersPrefix + } + } + // Retrieve all keys and values to avoid too many calls to etcd in case of + // a large number of policy mappings + r, err := ies.client.Get(cctx, basePrefix, etcd.WithPrefix()) + if err != nil { + return err + } + + // Parse all policies mapping to create the proper data model + for _, kv := range r.Kvs { + if err = getMappedPolicy(kv, m, basePrefix); err != nil && !errors.Is(err, errNoSuchPolicy) { + return err + } + } + return nil +} + +func (ies *IAMEtcdStore) savePolicyDoc(ctx context.Context, policyName string, p PolicyDoc) error { + return ies.saveIAMConfig(ctx, &p, getPolicyDocPath(policyName)) +} + +func (ies *IAMEtcdStore) saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp MappedPolicy, opts ...options) error { + return ies.saveIAMConfig(ctx, mp, getMappedPolicyPath(name, userType, isGroup), opts...) +} + +func (ies *IAMEtcdStore) saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u UserIdentity, opts ...options) error { + return ies.saveIAMConfig(ctx, u, getUserIdentityPath(name, userType), opts...) +} + +func (ies *IAMEtcdStore) saveGroupInfo(ctx context.Context, name string, gi GroupInfo) error { + return ies.saveIAMConfig(ctx, gi, getGroupInfoPath(name)) +} + +func (ies *IAMEtcdStore) deletePolicyDoc(ctx context.Context, name string) error { + err := ies.deleteIAMConfig(ctx, getPolicyDocPath(name)) + if err == errConfigNotFound { + err = errNoSuchPolicy + } + return err +} + +func (ies *IAMEtcdStore) deleteMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool) error { + err := ies.deleteIAMConfig(ctx, getMappedPolicyPath(name, userType, isGroup)) + if err == errConfigNotFound { + err = errNoSuchPolicy + } + return err +} + +func (ies *IAMEtcdStore) deleteUserIdentity(ctx context.Context, name string, userType IAMUserType) error { + err := ies.deleteIAMConfig(ctx, getUserIdentityPath(name, userType)) + if err == errConfigNotFound { + err = errNoSuchUser + } + return err +} + +func (ies *IAMEtcdStore) deleteGroupInfo(ctx context.Context, name string) error { + err := ies.deleteIAMConfig(ctx, getGroupInfoPath(name)) + if err == errConfigNotFound { + err = errNoSuchGroup + } + return err +} + +func (ies *IAMEtcdStore) watch(ctx context.Context, keyPath string) <-chan iamWatchEvent { + ch := make(chan iamWatchEvent) + + // go routine to read events from the etcd watch channel and send them + // down `ch` + go func() { + for { + outerLoop: + watchCh := ies.client.Watch(ctx, + keyPath, etcd.WithPrefix(), etcd.WithKeysOnly()) + + for { + select { + case <-ctx.Done(): + return + case watchResp, ok := <-watchCh: + if !ok { + time.Sleep(1 * time.Second) + // Upon an error on watch channel + // re-init the watch channel. + goto outerLoop + } + if err := watchResp.Err(); err != nil { + iamLogIf(ctx, err) + // log and retry. + time.Sleep(1 * time.Second) + // Upon an error on watch channel + // re-init the watch channel. + goto outerLoop + } + for _, event := range watchResp.Events { + isCreateEvent := event.IsModify() || event.IsCreate() + isDeleteEvent := event.Type == etcd.EventTypeDelete + + switch { + case isCreateEvent: + ch <- iamWatchEvent{ + isCreated: true, + keyPath: string(event.Kv.Key), + } + case isDeleteEvent: + ch <- iamWatchEvent{ + isCreated: false, + keyPath: string(event.Kv.Key), + } + } + } + } + } + } + }() + return ch +} diff --git a/cmd/iam-etcd-store_test.go b/cmd/iam-etcd-store_test.go new file mode 100644 index 00000000..75958abb --- /dev/null +++ b/cmd/iam-etcd-store_test.go @@ -0,0 +1,40 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "testing" +) + +func TestExtractPrefixAndSuffix(t *testing.T) { + specs := []struct { + path, prefix, suffix string + expected string + }{ + {"config/iam/groups/foo.json", "config/iam/groups/", ".json", "foo"}, + {"config/iam/groups/./foo.json", "config/iam/groups/", ".json", "foo"}, + {"config/iam/groups/foo/config.json", "config/iam/groups/", "/config.json", "foo"}, + {"config/iam/groups/foo/config.json", "config/iam/groups/", "config.json", "foo"}, + } + for i, test := range specs { + result := extractPathPrefixAndSuffix(test.path, test.prefix, test.suffix) + if result != test.expected { + t.Errorf("unexpected result on test[%v]: expected[%s] but had [%s]", i, test.expected, result) + } + } +} diff --git a/cmd/iam-object-store.go b/cmd/iam-object-store.go new file mode 100644 index 00000000..89931148 --- /dev/null +++ b/cmd/iam-object-store.go @@ -0,0 +1,913 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "errors" + "fmt" + "maps" + "path" + "strings" + "sync" + "time" + "unicode/utf8" + + jsoniter "github.com/json-iterator/go" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/config" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/kms" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/sync/errgroup" + "github.com/puzpuzpuz/xsync/v3" +) + +// IAMObjectStore implements IAMStorageAPI +type IAMObjectStore struct { + // Protect access to storage within the current server. + sync.RWMutex + + *iamCache + + usersSysType UsersSysType + + objAPI ObjectLayer +} + +func newIAMObjectStore(objAPI ObjectLayer, usersSysType UsersSysType) *IAMObjectStore { + return &IAMObjectStore{ + iamCache: newIamCache(), + objAPI: objAPI, + usersSysType: usersSysType, + } +} + +func (iamOS *IAMObjectStore) rlock() *iamCache { + iamOS.RLock() + return iamOS.iamCache +} + +func (iamOS *IAMObjectStore) runlock() { + iamOS.RUnlock() +} + +func (iamOS *IAMObjectStore) lock() *iamCache { + iamOS.Lock() + return iamOS.iamCache +} + +func (iamOS *IAMObjectStore) unlock() { + iamOS.Unlock() +} + +func (iamOS *IAMObjectStore) getUsersSysType() UsersSysType { + return iamOS.usersSysType +} + +func (iamOS *IAMObjectStore) saveIAMConfig(ctx context.Context, item any, objPath string, opts ...options) error { + json := jsoniter.ConfigCompatibleWithStandardLibrary + data, err := json.Marshal(item) + if err != nil { + return err + } + if GlobalKMS != nil { + data, err = config.EncryptBytes(GlobalKMS, data, kms.Context{ + minioMetaBucket: path.Join(minioMetaBucket, objPath), + }) + if err != nil { + return err + } + } + return saveConfig(ctx, iamOS.objAPI, objPath, data) +} + +func decryptData(data []byte, objPath string) ([]byte, error) { + if utf8.Valid(data) { + return data, nil + } + + pdata, err := madmin.DecryptData(globalActiveCred.String(), bytes.NewReader(data)) + if err == nil { + return pdata, nil + } + if GlobalKMS != nil { + pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{ + minioMetaBucket: path.Join(minioMetaBucket, objPath), + }) + if err == nil { + return pdata, nil + } + pdata, err = config.DecryptBytes(GlobalKMS, data, kms.Context{ + minioMetaBucket: objPath, + }) + if err == nil { + return pdata, nil + } + } + return nil, err +} + +func (iamOS *IAMObjectStore) loadIAMConfigBytesWithMetadata(ctx context.Context, objPath string) ([]byte, ObjectInfo, error) { + data, meta, err := readConfigWithMetadata(ctx, iamOS.objAPI, objPath, ObjectOptions{}) + if err != nil { + return nil, meta, err + } + data, err = decryptData(data, objPath) + if err != nil { + return nil, meta, err + } + return data, meta, nil +} + +func (iamOS *IAMObjectStore) loadIAMConfig(ctx context.Context, item any, objPath string) error { + data, _, err := iamOS.loadIAMConfigBytesWithMetadata(ctx, objPath) + if err != nil { + return err + } + json := jsoniter.ConfigCompatibleWithStandardLibrary + return json.Unmarshal(data, item) +} + +func (iamOS *IAMObjectStore) deleteIAMConfig(ctx context.Context, path string) error { + return deleteConfig(ctx, iamOS.objAPI, path) +} + +func (iamOS *IAMObjectStore) loadPolicyDocWithRetry(ctx context.Context, policy string, m map[string]PolicyDoc, retries int) error { + for { + retry: + data, objInfo, err := iamOS.loadIAMConfigBytesWithMetadata(ctx, getPolicyDocPath(policy)) + if err != nil { + if err == errConfigNotFound { + return errNoSuchPolicy + } + retries-- + if retries <= 0 { + return err + } + time.Sleep(500 * time.Millisecond) + goto retry + } + + var p PolicyDoc + err = p.parseJSON(data) + if err != nil { + return err + } + + if p.Version == 0 { + // This means that policy was in the old version (without any + // timestamp info). We fetch the mod time of the file and save + // that as create and update date. + p.CreateDate = objInfo.ModTime + p.UpdateDate = objInfo.ModTime + } + + m[policy] = p + return nil + } +} + +func (iamOS *IAMObjectStore) loadPolicy(ctx context.Context, policy string) (PolicyDoc, error) { + var p PolicyDoc + + data, objInfo, err := iamOS.loadIAMConfigBytesWithMetadata(ctx, getPolicyDocPath(policy)) + if err != nil { + if err == errConfigNotFound { + return p, errNoSuchPolicy + } + return p, err + } + + err = p.parseJSON(data) + if err != nil { + return p, err + } + + if p.Version == 0 { + // This means that policy was in the old version (without any + // timestamp info). We fetch the mod time of the file and save + // that as create and update date. + p.CreateDate = objInfo.ModTime + p.UpdateDate = objInfo.ModTime + } + + return p, nil +} + +func (iamOS *IAMObjectStore) loadPolicyDoc(ctx context.Context, policy string, m map[string]PolicyDoc) error { + p, err := iamOS.loadPolicy(ctx, policy) + if err != nil { + return err + } + m[policy] = p + return nil +} + +func (iamOS *IAMObjectStore) loadPolicyDocs(ctx context.Context, m map[string]PolicyDoc) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPoliciesPrefix) { + if item.Err != nil { + return item.Err + } + + policyName := path.Dir(item.Item) + if err := iamOS.loadPolicyDoc(ctx, policyName, m); err != nil && !errors.Is(err, errNoSuchPolicy) { + return err + } + } + return nil +} + +func (iamOS *IAMObjectStore) loadSecretKey(ctx context.Context, user string, userType IAMUserType) (string, error) { + var u UserIdentity + err := iamOS.loadIAMConfig(ctx, &u, getUserIdentityPath(user, userType)) + if err != nil { + if errors.Is(err, errConfigNotFound) { + return "", errNoSuchUser + } + return "", err + } + return u.Credentials.SecretKey, nil +} + +func (iamOS *IAMObjectStore) loadUserIdentity(ctx context.Context, user string, userType IAMUserType) (UserIdentity, error) { + var u UserIdentity + err := iamOS.loadIAMConfig(ctx, &u, getUserIdentityPath(user, userType)) + if err != nil { + if err == errConfigNotFound { + return u, errNoSuchUser + } + return u, err + } + + if u.Credentials.IsExpired() { + // Delete expired identity - ignoring errors here. + iamOS.deleteIAMConfig(ctx, getUserIdentityPath(user, userType)) + iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(user, userType, false)) + return u, errNoSuchUser + } + + if u.Credentials.AccessKey == "" { + u.Credentials.AccessKey = user + } + + if u.Credentials.SessionToken != "" { + jwtClaims, err := extractJWTClaims(u) + if err != nil { + if u.Credentials.IsTemp() { + // We should delete such that the client can re-request + // for the expiring credentials. + iamOS.deleteIAMConfig(ctx, getUserIdentityPath(user, userType)) + iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(user, userType, false)) + } + return u, errNoSuchUser + } + u.Credentials.Claims = jwtClaims.Map() + } + + if u.Credentials.Description == "" { + u.Credentials.Description = u.Credentials.Comment + } + + return u, nil +} + +func (iamOS *IAMObjectStore) loadUserConcurrent(ctx context.Context, userType IAMUserType, users ...string) ([]UserIdentity, error) { + userIdentities := make([]UserIdentity, len(users)) + g := errgroup.WithNErrs(len(users)) + + for index := range users { + g.Go(func() error { + userName := path.Dir(users[index]) + user, err := iamOS.loadUserIdentity(ctx, userName, userType) + if err != nil && !errors.Is(err, errNoSuchUser) { + return fmt.Errorf("unable to load the user `%s`: %w", userName, err) + } + userIdentities[index] = user + return nil + }, index) + } + + err := errors.Join(g.Wait()...) + return userIdentities, err +} + +func (iamOS *IAMObjectStore) loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]UserIdentity) error { + u, err := iamOS.loadUserIdentity(ctx, user, userType) + if err != nil { + return err + } + m[user] = u + return nil +} + +func (iamOS *IAMObjectStore) loadUsers(ctx context.Context, userType IAMUserType, m map[string]UserIdentity) error { + var basePrefix string + switch userType { + case svcUser: + basePrefix = iamConfigServiceAccountsPrefix + case stsUser: + basePrefix = iamConfigSTSPrefix + default: + basePrefix = iamConfigUsersPrefix + } + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + for item := range listIAMConfigItems(ctx, iamOS.objAPI, basePrefix) { + if item.Err != nil { + return item.Err + } + + userName := path.Dir(item.Item) + if err := iamOS.loadUser(ctx, userName, userType, m); err != nil && err != errNoSuchUser { + return err + } + } + return nil +} + +func (iamOS *IAMObjectStore) loadGroup(ctx context.Context, group string, m map[string]GroupInfo) error { + var g GroupInfo + err := iamOS.loadIAMConfig(ctx, &g, getGroupInfoPath(group)) + if err != nil { + if err == errConfigNotFound { + return errNoSuchGroup + } + return err + } + m[group] = g + return nil +} + +func (iamOS *IAMObjectStore) loadGroups(ctx context.Context, m map[string]GroupInfo) error { + ctx, cancel := context.WithCancel(ctx) + defer cancel() + for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigGroupsPrefix) { + if item.Err != nil { + return item.Err + } + + group := path.Dir(item.Item) + if err := iamOS.loadGroup(ctx, group, m); err != nil && err != errNoSuchGroup { + return err + } + } + return nil +} + +func (iamOS *IAMObjectStore) loadMappedPolicyWithRetry(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy], retries int) error { + for { + retry: + var p MappedPolicy + err := iamOS.loadIAMConfig(ctx, &p, getMappedPolicyPath(name, userType, isGroup)) + if err != nil { + if err == errConfigNotFound { + return errNoSuchPolicy + } + retries-- + if retries <= 0 { + return err + } + time.Sleep(500 * time.Millisecond) + goto retry + } + + m.Store(name, p) + return nil + } +} + +func (iamOS *IAMObjectStore) loadMappedPolicyInternal(ctx context.Context, name string, userType IAMUserType, isGroup bool) (MappedPolicy, error) { + var p MappedPolicy + err := iamOS.loadIAMConfig(ctx, &p, getMappedPolicyPath(name, userType, isGroup)) + if err != nil { + if err == errConfigNotFound { + return p, errNoSuchPolicy + } + return p, err + } + return p, nil +} + +func (iamOS *IAMObjectStore) loadMappedPolicyConcurrent(ctx context.Context, userType IAMUserType, isGroup bool, users ...string) ([]MappedPolicy, error) { + mappedPolicies := make([]MappedPolicy, len(users)) + g := errgroup.WithNErrs(len(users)) + + for index := range users { + g.Go(func() error { + userName := strings.TrimSuffix(users[index], ".json") + userMP, err := iamOS.loadMappedPolicyInternal(ctx, userName, userType, isGroup) + if err != nil && !errors.Is(err, errNoSuchPolicy) { + return fmt.Errorf("unable to load the user policy map `%s`: %w", userName, err) + } + mappedPolicies[index] = userMP + return nil + }, index) + } + + err := errors.Join(g.Wait()...) + return mappedPolicies, err +} + +func (iamOS *IAMObjectStore) loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error { + p, err := iamOS.loadMappedPolicyInternal(ctx, name, userType, isGroup) + if err != nil { + return err + } + m.Store(name, p) + return nil +} + +func (iamOS *IAMObjectStore) loadMappedPolicies(ctx context.Context, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error { + var basePath string + if isGroup { + basePath = iamConfigPolicyDBGroupsPrefix + } else { + switch userType { + case svcUser: + basePath = iamConfigPolicyDBServiceAccountsPrefix + case stsUser: + basePath = iamConfigPolicyDBSTSUsersPrefix + default: + basePath = iamConfigPolicyDBUsersPrefix + } + } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + for item := range listIAMConfigItems(ctx, iamOS.objAPI, basePath) { + if item.Err != nil { + return item.Err + } + + policyFile := item.Item + userOrGroupName := strings.TrimSuffix(policyFile, ".json") + if err := iamOS.loadMappedPolicy(ctx, userOrGroupName, userType, isGroup, m); err != nil && !errors.Is(err, errNoSuchPolicy) { + return err + } + } + return nil +} + +var ( + usersListKey = "users/" + svcAccListKey = "service-accounts/" + groupsListKey = "groups/" + policiesListKey = "policies/" + stsListKey = "sts/" + policyDBPrefix = "policydb/" + policyDBUsersListKey = "policydb/users/" + policyDBSTSUsersListKey = "policydb/sts-users/" + policyDBGroupsListKey = "policydb/groups/" +) + +func findSecondIndex(s string, substr string) int { + first := strings.Index(s, substr) + if first == -1 { + return -1 + } + second := strings.Index(s[first+1:], substr) + if second == -1 { + return -1 + } + return first + second + 1 +} + +// splitPath splits a path into a top-level directory and a child item. The +// parent directory retains the trailing slash. +func splitPath(s string, secondIndex bool) (string, string) { + var i int + if secondIndex { + i = findSecondIndex(s, "/") + } else { + i = strings.Index(s, "/") + } + if i == -1 { + return s, "" + } + // Include the trailing slash in the parent directory. + return s[:i+1], s[i+1:] +} + +func (iamOS *IAMObjectStore) listAllIAMConfigItems(ctx context.Context) (res map[string][]string, err error) { + res = make(map[string][]string) + ctx, cancel := context.WithCancel(ctx) + defer cancel() + for item := range listIAMConfigItems(ctx, iamOS.objAPI, iamConfigPrefix+SlashSeparator) { + if item.Err != nil { + return nil, item.Err + } + + secondIndex := strings.HasPrefix(item.Item, policyDBPrefix) + listKey, trimmedItem := splitPath(item.Item, secondIndex) + if listKey == iamFormatFile { + continue + } + + res[listKey] = append(res[listKey], trimmedItem) + } + + return res, nil +} + +const ( + maxIAMLoadOpTime = 5 * time.Second +) + +func (iamOS *IAMObjectStore) loadPolicyDocConcurrent(ctx context.Context, policies ...string) ([]PolicyDoc, error) { + policyDocs := make([]PolicyDoc, len(policies)) + g := errgroup.WithNErrs(len(policies)) + + for index := range policies { + g.Go(func() error { + policyName := path.Dir(policies[index]) + policyDoc, err := iamOS.loadPolicy(ctx, policyName) + if err != nil && !errors.Is(err, errNoSuchPolicy) { + return fmt.Errorf("unable to load the policy doc `%s`: %w", policyName, err) + } + policyDocs[index] = policyDoc + return nil + }, index) + } + + err := errors.Join(g.Wait()...) + return policyDocs, err +} + +// Assumes cache is locked by caller. +func (iamOS *IAMObjectStore) loadAllFromObjStore(ctx context.Context, cache *iamCache, firstTime bool) error { + bootstrapTraceMsgFirstTime := func(s string) { + if firstTime { + bootstrapTraceMsg(s) + } + } + + if iamOS.objAPI == nil { + return errServerNotInitialized + } + + bootstrapTraceMsgFirstTime("loading all IAM items") + + setDefaultCannedPolicies(cache.iamPolicyDocsMap) + + listStartTime := UTCNow() + listedConfigItems, err := iamOS.listAllIAMConfigItems(ctx) + if err != nil { + return fmt.Errorf("unable to list IAM data: %w", err) + } + if took := time.Since(listStartTime); took > maxIAMLoadOpTime { + var s strings.Builder + for k, v := range listedConfigItems { + s.WriteString(fmt.Sprintf(" %s: %d items\n", k, len(v))) + } + logger.Info("listAllIAMConfigItems took %.2fs with contents:\n%s", took.Seconds(), s.String()) + } + + // Loads things in the same order as `LoadIAMCache()` + + bootstrapTraceMsgFirstTime("loading policy documents") + + policyLoadStartTime := UTCNow() + policiesList := listedConfigItems[policiesListKey] + count := 32 // number of parallel IAM loaders + for { + if len(policiesList) < count { + policyDocs, err := iamOS.loadPolicyDocConcurrent(ctx, policiesList...) + if err != nil { + return err + } + for index := range policiesList { + if policyDocs[index].Policy.Version != "" { + policyName := path.Dir(policiesList[index]) + cache.iamPolicyDocsMap[policyName] = policyDocs[index] + } + } + break + } + + policyDocs, err := iamOS.loadPolicyDocConcurrent(ctx, policiesList[:count]...) + if err != nil { + return err + } + + for index := range policiesList[:count] { + if policyDocs[index].Policy.Version != "" { + policyName := path.Dir(policiesList[index]) + cache.iamPolicyDocsMap[policyName] = policyDocs[index] + } + } + + policiesList = policiesList[count:] + } + + if took := time.Since(policyLoadStartTime); took > maxIAMLoadOpTime { + logger.Info("Policy docs load took %.2fs (for %d items)", took.Seconds(), len(policiesList)) + } + + if iamOS.usersSysType == MinIOUsersSysType { + bootstrapTraceMsgFirstTime("loading regular IAM users") + regUsersLoadStartTime := UTCNow() + regUsersList := listedConfigItems[usersListKey] + + for { + if len(regUsersList) < count { + users, err := iamOS.loadUserConcurrent(ctx, regUser, regUsersList...) + if err != nil { + return err + } + for index := range regUsersList { + if users[index].Credentials.AccessKey != "" { + userName := path.Dir(regUsersList[index]) + cache.iamUsersMap[userName] = users[index] + } + } + break + } + + users, err := iamOS.loadUserConcurrent(ctx, regUser, regUsersList[:count]...) + if err != nil { + return err + } + + for index := range regUsersList[:count] { + if users[index].Credentials.AccessKey != "" { + userName := path.Dir(regUsersList[index]) + cache.iamUsersMap[userName] = users[index] + } + } + + regUsersList = regUsersList[count:] + } + + if took := time.Since(regUsersLoadStartTime); took > maxIAMLoadOpTime { + actualLoaded := len(cache.iamUsersMap) + logger.Info("Reg. users load took %.2fs (for %d items with %d expired items)", took.Seconds(), + len(regUsersList), len(regUsersList)-actualLoaded) + } + + bootstrapTraceMsgFirstTime("loading regular IAM groups") + groupsLoadStartTime := UTCNow() + groupsList := listedConfigItems[groupsListKey] + for _, item := range groupsList { + group := path.Dir(item) + if err := iamOS.loadGroup(ctx, group, cache.iamGroupsMap); err != nil && err != errNoSuchGroup { + return fmt.Errorf("unable to load the group: %w", err) + } + } + if took := time.Since(groupsLoadStartTime); took > maxIAMLoadOpTime { + logger.Info("Groups load took %.2fs (for %d items)", took.Seconds(), len(groupsList)) + } + } + + bootstrapTraceMsgFirstTime("loading user policy mapping") + userPolicyMappingLoadStartTime := UTCNow() + userPolicyMappingsList := listedConfigItems[policyDBUsersListKey] + for { + if len(userPolicyMappingsList) < count { + mappedPolicies, err := iamOS.loadMappedPolicyConcurrent(ctx, regUser, false, userPolicyMappingsList...) + if err != nil { + return err + } + + for index := range userPolicyMappingsList { + if mappedPolicies[index].Policies != "" { + userName := strings.TrimSuffix(userPolicyMappingsList[index], ".json") + cache.iamUserPolicyMap.Store(userName, mappedPolicies[index]) + } + } + + break + } + + mappedPolicies, err := iamOS.loadMappedPolicyConcurrent(ctx, regUser, false, userPolicyMappingsList[:count]...) + if err != nil { + return err + } + + for index := range userPolicyMappingsList[:count] { + if mappedPolicies[index].Policies != "" { + userName := strings.TrimSuffix(userPolicyMappingsList[index], ".json") + cache.iamUserPolicyMap.Store(userName, mappedPolicies[index]) + } + } + + userPolicyMappingsList = userPolicyMappingsList[count:] + } + + if took := time.Since(userPolicyMappingLoadStartTime); took > maxIAMLoadOpTime { + logger.Info("User policy mappings load took %.2fs (for %d items)", took.Seconds(), len(userPolicyMappingsList)) + } + + bootstrapTraceMsgFirstTime("loading group policy mapping") + groupPolicyMappingLoadStartTime := UTCNow() + groupPolicyMappingsList := listedConfigItems[policyDBGroupsListKey] + for _, item := range groupPolicyMappingsList { + groupName := strings.TrimSuffix(item, ".json") + if err := iamOS.loadMappedPolicy(ctx, groupName, regUser, true, cache.iamGroupPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) { + return fmt.Errorf("unable to load the policy mapping for the group: %w", err) + } + } + if took := time.Since(groupPolicyMappingLoadStartTime); took > maxIAMLoadOpTime { + logger.Info("Group policy mappings load took %.2fs (for %d items)", took.Seconds(), len(groupPolicyMappingsList)) + } + + bootstrapTraceMsgFirstTime("loading service accounts") + svcAccLoadStartTime := UTCNow() + svcAccList := listedConfigItems[svcAccListKey] + svcUsersMap := make(map[string]UserIdentity, len(svcAccList)) + for _, item := range svcAccList { + userName := path.Dir(item) + if err := iamOS.loadUser(ctx, userName, svcUser, svcUsersMap); err != nil && err != errNoSuchUser { + return fmt.Errorf("unable to load the service account: %w", err) + } + } + if took := time.Since(svcAccLoadStartTime); took > maxIAMLoadOpTime { + logger.Info("Service accounts load took %.2fs (for %d items with %d expired items)", took.Seconds(), + len(svcAccList), len(svcAccList)-len(svcUsersMap)) + } + + bootstrapTraceMsg("loading STS account policy mapping") + stsPolicyMappingLoadStartTime := UTCNow() + var stsPolicyMappingsCount int + for _, svcAcc := range svcUsersMap { + svcParent := svcAcc.Credentials.ParentUser + if _, ok := cache.iamUsersMap[svcParent]; !ok { + stsPolicyMappingsCount++ + // If a service account's parent user is not in iamUsersMap, the + // parent is an STS account. Such accounts may have a policy mapped + // on the parent user, so we load them. This is not needed for the + // initial server startup, however, it is needed for the case where + // the STS account's policy mapping (for example in LDAP mode) may + // be changed and the user's policy mapping in memory is stale + // (because the policy change notification was missed by the current + // server). + // + // The "policy not found" error is ignored because the STS account may + // not have a policy mapped via its parent (for e.g. in + // OIDC/AssumeRoleWithCustomToken/AssumeRoleWithCertificate). + err := iamOS.loadMappedPolicy(ctx, svcParent, stsUser, false, cache.iamSTSPolicyMap) + if err != nil && !errors.Is(err, errNoSuchPolicy) { + return fmt.Errorf("unable to load the policy mapping for the STS user: %w", err) + } + } + } + if took := time.Since(stsPolicyMappingLoadStartTime); took > maxIAMLoadOpTime { + logger.Info("STS policy mappings load took %.2fs (for %d items)", took.Seconds(), stsPolicyMappingsCount) + } + + // Copy svcUsersMap to cache.iamUsersMap + maps.Copy(cache.iamUsersMap, svcUsersMap) + + cache.buildUserGroupMemberships() + + purgeStart := time.Now() + + // Purge expired STS credentials. + + // Scan STS users on disk and purge expired ones. + stsAccountsFromStore := map[string]UserIdentity{} + stsAccPoliciesFromStore := xsync.NewMapOf[string, MappedPolicy]() + for _, item := range listedConfigItems[stsListKey] { + userName := path.Dir(item) + // loadUser() will delete expired user during the load. + err := iamOS.loadUser(ctx, userName, stsUser, stsAccountsFromStore) + if err != nil && !errors.Is(err, errNoSuchUser) { + iamLogIf(ctx, err) + } + // No need to return errors for failed expiration of STS users + } + + // Loading the STS policy mappings from disk ensures that stale entries + // (removed during loadUser() in the loop above) are removed from memory. + for _, item := range listedConfigItems[policyDBSTSUsersListKey] { + stsName := strings.TrimSuffix(item, ".json") + err := iamOS.loadMappedPolicy(ctx, stsName, stsUser, false, stsAccPoliciesFromStore) + if err != nil && !errors.Is(err, errNoSuchPolicy) { + iamLogIf(ctx, err) + } + // No need to return errors for failed expiration of STS users + } + + took := time.Since(purgeStart).Seconds() + if took > maxDurationSecondsForLog { + // Log if we took a lot of time to load. + logger.Info("IAM expired STS purge took %.2fs", took) + } + + // Store the newly populated map in the iam cache. This takes care of + // removing stale entries from the existing map. + cache.iamSTSAccountsMap = stsAccountsFromStore + + stsAccPoliciesFromStore.Range(func(k string, v MappedPolicy) bool { + cache.iamSTSPolicyMap.Store(k, v) + return true + }) + + return nil +} + +func (iamOS *IAMObjectStore) savePolicyDoc(ctx context.Context, policyName string, p PolicyDoc) error { + return iamOS.saveIAMConfig(ctx, &p, getPolicyDocPath(policyName)) +} + +func (iamOS *IAMObjectStore) saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp MappedPolicy, opts ...options) error { + return iamOS.saveIAMConfig(ctx, mp, getMappedPolicyPath(name, userType, isGroup), opts...) +} + +func (iamOS *IAMObjectStore) saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u UserIdentity, opts ...options) error { + return iamOS.saveIAMConfig(ctx, u, getUserIdentityPath(name, userType), opts...) +} + +func (iamOS *IAMObjectStore) saveGroupInfo(ctx context.Context, name string, gi GroupInfo) error { + return iamOS.saveIAMConfig(ctx, gi, getGroupInfoPath(name)) +} + +func (iamOS *IAMObjectStore) deletePolicyDoc(ctx context.Context, name string) error { + err := iamOS.deleteIAMConfig(ctx, getPolicyDocPath(name)) + if err == errConfigNotFound { + err = errNoSuchPolicy + } + return err +} + +func (iamOS *IAMObjectStore) deleteMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool) error { + err := iamOS.deleteIAMConfig(ctx, getMappedPolicyPath(name, userType, isGroup)) + if err == errConfigNotFound { + err = errNoSuchPolicy + } + return err +} + +func (iamOS *IAMObjectStore) deleteUserIdentity(ctx context.Context, name string, userType IAMUserType) error { + err := iamOS.deleteIAMConfig(ctx, getUserIdentityPath(name, userType)) + if err == errConfigNotFound { + err = errNoSuchUser + } + return err +} + +func (iamOS *IAMObjectStore) deleteGroupInfo(ctx context.Context, name string) error { + err := iamOS.deleteIAMConfig(ctx, getGroupInfoPath(name)) + if err == errConfigNotFound { + err = errNoSuchGroup + } + return err +} + +// Lists objects in the minioMetaBucket at the given path prefix. All returned +// items have the pathPrefix removed from their names. +func listIAMConfigItems(ctx context.Context, objAPI ObjectLayer, pathPrefix string) <-chan itemOrErr[string] { + ch := make(chan itemOrErr[string]) + + go func() { + defer xioutil.SafeClose(ch) + + // Allocate new results channel to receive ObjectInfo. + objInfoCh := make(chan itemOrErr[ObjectInfo]) + + if err := objAPI.Walk(ctx, minioMetaBucket, pathPrefix, objInfoCh, WalkOptions{}); err != nil { + select { + case ch <- itemOrErr[string]{Err: err}: + case <-ctx.Done(): + } + return + } + + for obj := range objInfoCh { + if obj.Err != nil { + select { + case ch <- itemOrErr[string]{Err: obj.Err}: + case <-ctx.Done(): + return + } + } + item := strings.TrimPrefix(obj.Item.Name, pathPrefix) + item = strings.TrimSuffix(item, SlashSeparator) + select { + case ch <- itemOrErr[string]{Item: item}: + case <-ctx.Done(): + return + } + } + }() + + return ch +} diff --git a/cmd/iam-object-store_test.go b/cmd/iam-object-store_test.go new file mode 100644 index 00000000..dc0ce132 --- /dev/null +++ b/cmd/iam-object-store_test.go @@ -0,0 +1,53 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "testing" +) + +func TestSplitPath(t *testing.T) { + cases := []struct { + path string + secondIndex bool + expectedListKey, expectedItem string + }{ + {"format.json", false, "format.json", ""}, + {"users/tester.json", false, "users/", "tester.json"}, + {"groups/test/group.json", false, "groups/", "test/group.json"}, + {"policydb/groups/testgroup.json", true, "policydb/groups/", "testgroup.json"}, + { + "policydb/sts-users/uid=slash/user,ou=people,ou=swengg,dc=min,dc=io.json", true, + "policydb/sts-users/", "uid=slash/user,ou=people,ou=swengg,dc=min,dc=io.json", + }, + { + "policydb/sts-users/uid=slash/user/twice,ou=people,ou=swengg,dc=min,dc=io.json", true, + "policydb/sts-users/", "uid=slash/user/twice,ou=people,ou=swengg,dc=min,dc=io.json", + }, + { + "policydb/groups/cn=project/d,ou=groups,ou=swengg,dc=min,dc=io.json", true, + "policydb/groups/", "cn=project/d,ou=groups,ou=swengg,dc=min,dc=io.json", + }, + } + for i, test := range cases { + listKey, item := splitPath(test.path, test.secondIndex) + if listKey != test.expectedListKey || item != test.expectedItem { + t.Errorf("unexpected result on test[%v]: expected[%s, %s] but got [%s, %s]", i, test.expectedListKey, test.expectedItem, listKey, item) + } + } +} diff --git a/cmd/iam-store.go b/cmd/iam-store.go new file mode 100644 index 00000000..46465e00 --- /dev/null +++ b/cmd/iam-store.go @@ -0,0 +1,3072 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "maps" + "path" + "sort" + "strings" + "sync" + "time" + + jsoniter "github.com/json-iterator/go" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/config/identity/openid" + "github.com/minio/minio/internal/jwt" + "github.com/minio/pkg/v3/env" + "github.com/minio/pkg/v3/policy" + "github.com/minio/pkg/v3/sync/errgroup" + "github.com/puzpuzpuz/xsync/v3" + "golang.org/x/sync/singleflight" +) + +const ( + // IAM configuration directory. + iamConfigPrefix = minioConfigPrefix + "/iam" + + // IAM users directory. + iamConfigUsersPrefix = iamConfigPrefix + "/users/" + + // IAM service accounts directory. + iamConfigServiceAccountsPrefix = iamConfigPrefix + "/service-accounts/" + + // IAM groups directory. + iamConfigGroupsPrefix = iamConfigPrefix + "/groups/" + + // IAM policies directory. + iamConfigPoliciesPrefix = iamConfigPrefix + "/policies/" + + // IAM sts directory. + iamConfigSTSPrefix = iamConfigPrefix + "/sts/" + + // IAM Policy DB prefixes. + iamConfigPolicyDBPrefix = iamConfigPrefix + "/policydb/" + iamConfigPolicyDBUsersPrefix = iamConfigPolicyDBPrefix + "users/" + iamConfigPolicyDBSTSUsersPrefix = iamConfigPolicyDBPrefix + "sts-users/" + iamConfigPolicyDBServiceAccountsPrefix = iamConfigPolicyDBPrefix + "service-accounts/" + iamConfigPolicyDBGroupsPrefix = iamConfigPolicyDBPrefix + "groups/" + + // IAM identity file which captures identity credentials. + iamIdentityFile = "identity.json" + + // IAM policy file which provides policies for each users. + iamPolicyFile = "policy.json" + + // IAM group members file + iamGroupMembersFile = "members.json" + + // IAM format file + iamFormatFile = "format.json" + + iamFormatVersion1 = 1 + + minServiceAccountExpiry time.Duration = 15 * time.Minute + maxServiceAccountExpiry time.Duration = 365 * 24 * time.Hour +) + +var errInvalidSvcAcctExpiration = errors.New("invalid service account expiration") + +type iamFormat struct { + Version int `json:"version"` +} + +func newIAMFormatVersion1() iamFormat { + return iamFormat{Version: iamFormatVersion1} +} + +func getIAMFormatFilePath() string { + return iamConfigPrefix + SlashSeparator + iamFormatFile +} + +func getUserIdentityPath(user string, userType IAMUserType) string { + var basePath string + switch userType { + case svcUser: + basePath = iamConfigServiceAccountsPrefix + case stsUser: + basePath = iamConfigSTSPrefix + default: + basePath = iamConfigUsersPrefix + } + return pathJoin(basePath, user, iamIdentityFile) +} + +func saveIAMFormat(ctx context.Context, store IAMStorageAPI) error { + bootstrapTraceMsg("Load IAM format file") + var iamFmt iamFormat + path := getIAMFormatFilePath() + if err := store.loadIAMConfig(ctx, &iamFmt, path); err != nil && !errors.Is(err, errConfigNotFound) { + // if IAM format + return err + } + + if iamFmt.Version >= iamFormatVersion1 { + // Nothing to do. + return nil + } + + bootstrapTraceMsg("Write IAM format file") + // Save iam format to version 1. + return store.saveIAMConfig(ctx, newIAMFormatVersion1(), path) +} + +func getGroupInfoPath(group string) string { + return pathJoin(iamConfigGroupsPrefix, group, iamGroupMembersFile) +} + +func getPolicyDocPath(name string) string { + return pathJoin(iamConfigPoliciesPrefix, name, iamPolicyFile) +} + +func getMappedPolicyPath(name string, userType IAMUserType, isGroup bool) string { + if isGroup { + return pathJoin(iamConfigPolicyDBGroupsPrefix, name+".json") + } + switch userType { + case svcUser: + return pathJoin(iamConfigPolicyDBServiceAccountsPrefix, name+".json") + case stsUser: + return pathJoin(iamConfigPolicyDBSTSUsersPrefix, name+".json") + default: + return pathJoin(iamConfigPolicyDBUsersPrefix, name+".json") + } +} + +// UserIdentity represents a user's secret key and their status +type UserIdentity struct { + Version int `json:"version"` + Credentials auth.Credentials `json:"credentials"` + UpdatedAt time.Time `json:"updatedAt"` +} + +func newUserIdentity(cred auth.Credentials) UserIdentity { + return UserIdentity{Version: 1, Credentials: cred, UpdatedAt: UTCNow()} +} + +// GroupInfo contains info about a group +type GroupInfo struct { + Version int `json:"version"` + Status string `json:"status"` + Members []string `json:"members"` + UpdatedAt time.Time `json:"updatedAt"` +} + +func newGroupInfo(members []string) GroupInfo { + return GroupInfo{Version: 1, Status: statusEnabled, Members: members, UpdatedAt: UTCNow()} +} + +// MappedPolicy represents a policy name mapped to a user or group +type MappedPolicy struct { + Version int `json:"version"` + Policies string `json:"policy"` + UpdatedAt time.Time `json:"updatedAt"` +} + +// mappedPoliciesToMap copies the map of mapped policies to a regular map. +func mappedPoliciesToMap(m *xsync.MapOf[string, MappedPolicy]) map[string]MappedPolicy { + policies := make(map[string]MappedPolicy, m.Size()) + m.Range(func(k string, v MappedPolicy) bool { + policies[k] = v + return true + }) + return policies +} + +// converts a mapped policy into a slice of distinct policies +func (mp MappedPolicy) toSlice() []string { + var policies []string + for policy := range strings.SplitSeq(mp.Policies, ",") { + if strings.TrimSpace(policy) == "" { + continue + } + policies = append(policies, policy) + } + return policies +} + +func (mp MappedPolicy) policySet() set.StringSet { + return set.CreateStringSet(mp.toSlice()...) +} + +func newMappedPolicy(policy string) MappedPolicy { + return MappedPolicy{Version: 1, Policies: policy, UpdatedAt: UTCNow()} +} + +// PolicyDoc represents an IAM policy with some metadata. +type PolicyDoc struct { + Version int `json:",omitempty"` + Policy policy.Policy + CreateDate time.Time + UpdateDate time.Time +} + +func newPolicyDoc(p policy.Policy) PolicyDoc { + now := UTCNow().Round(time.Millisecond) + return PolicyDoc{ + Version: 1, + Policy: p, + CreateDate: now, + UpdateDate: now, + } +} + +// defaultPolicyDoc - used to wrap a default policy as PolicyDoc. +func defaultPolicyDoc(p policy.Policy) PolicyDoc { + return PolicyDoc{ + Version: 1, + Policy: p, + } +} + +func (d *PolicyDoc) update(p policy.Policy) { + now := UTCNow().Round(time.Millisecond) + d.UpdateDate = now + if d.CreateDate.IsZero() { + d.CreateDate = now + } + d.Policy = p +} + +// parseJSON parses both the old and the new format for storing policy +// definitions. +// +// The on-disk format of policy definitions has changed (around early 12/2021) +// from policy.Policy to PolicyDoc. To avoid a migration, loading supports +// both the old and the new formats. +func (d *PolicyDoc) parseJSON(data []byte) error { + json := jsoniter.ConfigCompatibleWithStandardLibrary + var doc PolicyDoc + err := json.Unmarshal(data, &doc) + if err != nil { + err2 := json.Unmarshal(data, &doc.Policy) + if err2 != nil { + // Just return the first error. + return err + } + d.Policy = doc.Policy + return nil + } + *d = doc + return nil +} + +// key options +type options struct { + ttl int64 // expiry in seconds +} + +type iamWatchEvent struct { + isCreated bool // !isCreated implies a delete event. + keyPath string +} + +// iamCache contains in-memory cache of IAM data. +type iamCache struct { + updatedAt time.Time + + // map of policy names to policy definitions + iamPolicyDocsMap map[string]PolicyDoc + + // map of regular username to credentials + iamUsersMap map[string]UserIdentity + // map of regular username to policy names + iamUserPolicyMap *xsync.MapOf[string, MappedPolicy] + + // STS accounts are loaded on demand and not via the periodic IAM reload. + // map of STS access key to credentials + iamSTSAccountsMap map[string]UserIdentity + // map of STS access key to policy names + iamSTSPolicyMap *xsync.MapOf[string, MappedPolicy] + + // map of group names to group info + iamGroupsMap map[string]GroupInfo + // map of user names to groups they are a member of + iamUserGroupMemberships map[string]set.StringSet + // map of group names to policy names + iamGroupPolicyMap *xsync.MapOf[string, MappedPolicy] +} + +func newIamCache() *iamCache { + return &iamCache{ + iamPolicyDocsMap: map[string]PolicyDoc{}, + iamUsersMap: map[string]UserIdentity{}, + iamUserPolicyMap: xsync.NewMapOf[string, MappedPolicy](), + iamSTSAccountsMap: map[string]UserIdentity{}, + iamSTSPolicyMap: xsync.NewMapOf[string, MappedPolicy](), + iamGroupsMap: map[string]GroupInfo{}, + iamUserGroupMemberships: map[string]set.StringSet{}, + iamGroupPolicyMap: xsync.NewMapOf[string, MappedPolicy](), + } +} + +// buildUserGroupMemberships - builds the memberships map. IMPORTANT: +// Assumes that c.Lock is held by caller. +func (c *iamCache) buildUserGroupMemberships() { + for group, gi := range c.iamGroupsMap { + c.updateGroupMembershipsMap(group, &gi) + } +} + +// updateGroupMembershipsMap - updates the memberships map for a +// group. IMPORTANT: Assumes c.Lock() is held by caller. +func (c *iamCache) updateGroupMembershipsMap(group string, gi *GroupInfo) { + if gi == nil { + return + } + for _, member := range gi.Members { + v := c.iamUserGroupMemberships[member] + if v == nil { + v = set.CreateStringSet(group) + } else { + v.Add(group) + } + c.iamUserGroupMemberships[member] = v + } +} + +// removeGroupFromMembershipsMap - removes the group from every member +// in the cache. IMPORTANT: Assumes c.Lock() is held by caller. +func (c *iamCache) removeGroupFromMembershipsMap(group string) { + for member, groups := range c.iamUserGroupMemberships { + if !groups.Contains(group) { + continue + } + groups.Remove(group) + c.iamUserGroupMemberships[member] = groups + } +} + +func (c *iamCache) policyDBGetGroups(store *IAMStoreSys, userPolicyPresent bool, groups ...string) ([]string, error) { + var policies []string + for _, group := range groups { + if store.getUsersSysType() == MinIOUsersSysType { + g, ok := c.iamGroupsMap[group] + if !ok { + continue + } + + // Group is disabled, so we return no policy - this + // ensures the request is denied. + if g.Status == statusDisabled { + continue + } + } + + policy, ok := c.iamGroupPolicyMap.Load(group) + if !ok { + continue + } + + policies = append(policies, policy.toSlice()...) + } + + found := len(policies) > 0 + if found { + return policies, nil + } + + if userPolicyPresent { + // if user mapping present and no group policies found + // rely on user policy for access, instead of fallback. + return nil, nil + } + + var mu sync.Mutex + + // no mappings found, fallback for all groups. + g := errgroup.WithNErrs(len(groups)).WithConcurrency(10) // load like 10 groups at a time. + + for index := range groups { + g.Go(func() error { + err := store.loadMappedPolicy(context.TODO(), groups[index], regUser, true, c.iamGroupPolicyMap) + if err != nil && !errors.Is(err, errNoSuchPolicy) { + return err + } + if errors.Is(err, errNoSuchPolicy) { + return nil + } + policy, _ := c.iamGroupPolicyMap.Load(groups[index]) + mu.Lock() + policies = append(policies, policy.toSlice()...) + mu.Unlock() + return nil + }, index) + } + + err := errors.Join(g.Wait()...) + return policies, err +} + +// policyDBGet - lower-level helper; does not take locks. +// +// If a group is passed, it returns policies associated with the group. +// +// If a user is passed, it returns policies of the user along with any groups +// that the server knows the user is a member of. +// +// In LDAP users mode, the server does not store any group membership +// information in IAM (i.e sys.iam*Map) - this info is stored only in the STS +// generated credentials. Thus we skip looking up group memberships, user map, +// and group map and check the appropriate policy maps directly. +func (c *iamCache) policyDBGet(store *IAMStoreSys, name string, isGroup bool, policyPresent bool) ([]string, time.Time, error) { + if isGroup { + if store.getUsersSysType() == MinIOUsersSysType { + g, ok := c.iamGroupsMap[name] + if !ok { + if err := store.loadGroup(context.Background(), name, c.iamGroupsMap); err != nil { + return nil, time.Time{}, err + } + g, ok = c.iamGroupsMap[name] + if !ok { + return nil, time.Time{}, errNoSuchGroup + } + } + + // Group is disabled, so we return no policy - this + // ensures the request is denied. + if g.Status == statusDisabled { + return nil, time.Time{}, nil + } + } + + policy, ok := c.iamGroupPolicyMap.Load(name) + if ok { + return policy.toSlice(), policy.UpdatedAt, nil + } + if !policyPresent { + if err := store.loadMappedPolicy(context.TODO(), name, regUser, true, c.iamGroupPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) { + return nil, time.Time{}, err + } + policy, _ = c.iamGroupPolicyMap.Load(name) + return policy.toSlice(), policy.UpdatedAt, nil + } + return nil, time.Time{}, nil + } + + // returned policy could be empty, we use set to de-duplicate. + var policies set.StringSet + var updatedAt time.Time + + if store.getUsersSysType() == LDAPUsersSysType { + // For LDAP policy mapping is part of STS users, we only need to lookup + // those mappings. + mp, ok := c.iamSTSPolicyMap.Load(name) + if !ok { + // Attempt to load parent user mapping for STS accounts + if err := store.loadMappedPolicy(context.TODO(), name, stsUser, false, c.iamSTSPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) { + return nil, time.Time{}, err + } + mp, _ = c.iamSTSPolicyMap.Load(name) + } + policies = set.CreateStringSet(mp.toSlice()...) + updatedAt = mp.UpdatedAt + } else { + // When looking for a user's policies, we also check if the user + // and the groups they are member of are enabled. + u, ok := c.iamUsersMap[name] + if ok { + if !u.Credentials.IsValid() { + return nil, time.Time{}, nil + } + } + + // For internal IDP regular/service account user accounts, the policy + // mapping is iamUserPolicyMap. For STS accounts, the parent user would be + // passed here and we lookup the mapping in iamSTSPolicyMap. + mp, ok := c.iamUserPolicyMap.Load(name) + if !ok { + if err := store.loadMappedPolicy(context.TODO(), name, regUser, false, c.iamUserPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) { + return nil, time.Time{}, err + } + mp, ok = c.iamUserPolicyMap.Load(name) + if !ok { + // Since user "name" could be a parent user of an STS account, we look up + // mappings for those too. + mp, ok = c.iamSTSPolicyMap.Load(name) + if !ok { + // Attempt to load parent user mapping for STS accounts + if err := store.loadMappedPolicy(context.TODO(), name, stsUser, false, c.iamSTSPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) { + return nil, time.Time{}, err + } + mp, _ = c.iamSTSPolicyMap.Load(name) + } + } + } + policies = set.CreateStringSet(mp.toSlice()...) + + for _, group := range u.Credentials.Groups { + g, ok := c.iamGroupsMap[group] + if ok { + // Group is disabled, so we return no policy - this + // ensures the request is denied. + if g.Status == statusDisabled { + return nil, time.Time{}, nil + } + } + + policy, ok := c.iamGroupPolicyMap.Load(group) + if !ok { + if err := store.loadMappedPolicy(context.TODO(), group, regUser, true, c.iamGroupPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) { + return nil, time.Time{}, err + } + policy, _ = c.iamGroupPolicyMap.Load(group) + } + + for _, p := range policy.toSlice() { + policies.Add(p) + } + } + updatedAt = mp.UpdatedAt + } + + for _, group := range c.iamUserGroupMemberships[name].ToSlice() { + if store.getUsersSysType() == MinIOUsersSysType { + g, ok := c.iamGroupsMap[group] + if ok { + // Group is disabled, so we return no policy - this + // ensures the request is denied. + if g.Status == statusDisabled { + return nil, time.Time{}, nil + } + } + } + + policy, ok := c.iamGroupPolicyMap.Load(group) + if !ok { + if err := store.loadMappedPolicy(context.TODO(), group, regUser, true, c.iamGroupPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) { + return nil, time.Time{}, err + } + policy, _ = c.iamGroupPolicyMap.Load(group) + } + + for _, p := range policy.toSlice() { + policies.Add(p) + } + } + + return policies.ToSlice(), updatedAt, nil +} + +func (c *iamCache) updateUserWithClaims(key string, u UserIdentity) error { + if u.Credentials.SessionToken != "" { + jwtClaims, err := extractJWTClaims(u) + if err != nil { + return err + } + u.Credentials.Claims = jwtClaims.Map() + } + if u.Credentials.IsTemp() && !u.Credentials.IsServiceAccount() { + c.iamSTSAccountsMap[key] = u + } else { + c.iamUsersMap[key] = u + } + c.updatedAt = time.Now() + return nil +} + +// IAMStorageAPI defines an interface for the IAM persistence layer +type IAMStorageAPI interface { + // The role of the read-write lock is to prevent go routines from + // concurrently reading and writing the IAM storage. The (r)lock() + // functions return the iamCache. The cache can be safely written to + // only when returned by `lock()`. + lock() *iamCache + unlock() + rlock() *iamCache + runlock() + getUsersSysType() UsersSysType + loadPolicyDoc(ctx context.Context, policy string, m map[string]PolicyDoc) error + loadPolicyDocWithRetry(ctx context.Context, policy string, m map[string]PolicyDoc, retries int) error + loadPolicyDocs(ctx context.Context, m map[string]PolicyDoc) error + loadUser(ctx context.Context, user string, userType IAMUserType, m map[string]UserIdentity) error + loadSecretKey(ctx context.Context, user string, userType IAMUserType) (string, error) + loadUsers(ctx context.Context, userType IAMUserType, m map[string]UserIdentity) error + loadGroup(ctx context.Context, group string, m map[string]GroupInfo) error + loadGroups(ctx context.Context, m map[string]GroupInfo) error + loadMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error + loadMappedPolicyWithRetry(ctx context.Context, name string, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy], retries int) error + loadMappedPolicies(ctx context.Context, userType IAMUserType, isGroup bool, m *xsync.MapOf[string, MappedPolicy]) error + saveIAMConfig(ctx context.Context, item any, path string, opts ...options) error + loadIAMConfig(ctx context.Context, item any, path string) error + deleteIAMConfig(ctx context.Context, path string) error + savePolicyDoc(ctx context.Context, policyName string, p PolicyDoc) error + saveMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool, mp MappedPolicy, opts ...options) error + saveUserIdentity(ctx context.Context, name string, userType IAMUserType, u UserIdentity, opts ...options) error + saveGroupInfo(ctx context.Context, group string, gi GroupInfo) error + deletePolicyDoc(ctx context.Context, policyName string) error + deleteMappedPolicy(ctx context.Context, name string, userType IAMUserType, isGroup bool) error + deleteUserIdentity(ctx context.Context, name string, userType IAMUserType) error + deleteGroupInfo(ctx context.Context, name string) error +} + +// iamStorageWatcher is implemented by `IAMStorageAPI` implementers that +// additionally support watching storage for changes. +type iamStorageWatcher interface { + watch(ctx context.Context, keyPath string) <-chan iamWatchEvent +} + +// Set default canned policies only if not already overridden by users. +func setDefaultCannedPolicies(policies map[string]PolicyDoc) { + for _, v := range policy.DefaultPolicies { + if _, ok := policies[v.Name]; !ok { + policies[v.Name] = defaultPolicyDoc(v.Definition) + } + } +} + +// LoadIAMCache reads all IAM items and populates a new iamCache object and +// replaces the in-memory cache object. +func (store *IAMStoreSys) LoadIAMCache(ctx context.Context, firstTime bool) error { + bootstrapTraceMsgFirstTime := func(s string) { + if firstTime { + bootstrapTraceMsg(s) + } + } + bootstrapTraceMsgFirstTime("loading IAM data") + + newCache := newIamCache() + + loadedAt := time.Now() + + if iamOS, ok := store.IAMStorageAPI.(*IAMObjectStore); ok { + err := iamOS.loadAllFromObjStore(ctx, newCache, firstTime) + if err != nil { + return err + } + } else { + // Only non-object IAM store (i.e. only etcd backend). + bootstrapTraceMsgFirstTime("loading policy documents") + if err := store.loadPolicyDocs(ctx, newCache.iamPolicyDocsMap); err != nil { + return err + } + + // Sets default canned policies, if none are set. + setDefaultCannedPolicies(newCache.iamPolicyDocsMap) + + if store.getUsersSysType() == MinIOUsersSysType { + bootstrapTraceMsgFirstTime("loading regular users") + if err := store.loadUsers(ctx, regUser, newCache.iamUsersMap); err != nil { + return err + } + bootstrapTraceMsgFirstTime("loading regular groups") + if err := store.loadGroups(ctx, newCache.iamGroupsMap); err != nil { + return err + } + } + + bootstrapTraceMsgFirstTime("loading user policy mapping") + // load polices mapped to users + if err := store.loadMappedPolicies(ctx, regUser, false, newCache.iamUserPolicyMap); err != nil { + return err + } + + bootstrapTraceMsgFirstTime("loading group policy mapping") + // load policies mapped to groups + if err := store.loadMappedPolicies(ctx, regUser, true, newCache.iamGroupPolicyMap); err != nil { + return err + } + + bootstrapTraceMsgFirstTime("loading service accounts") + // load service accounts + if err := store.loadUsers(ctx, svcUser, newCache.iamUsersMap); err != nil { + return err + } + + newCache.buildUserGroupMemberships() + } + + cache := store.lock() + defer store.unlock() + + // We should only update the in-memory cache if there were no changes + // to the in-memory cache since the disk loading began. If there + // were changes to the in-memory cache we should wait for the next + // cycle until we can safely update the in-memory cache. + // + // An in-memory cache must be replaced only if we know for sure that the + // values loaded from disk are not stale. They might be stale if the + // cached.updatedAt is more recent than the refresh cycle began. + if cache.updatedAt.Before(loadedAt) || firstTime { + // No one has updated anything since the config was loaded, + // so we just replace whatever is on the disk into memory. + cache.iamGroupPolicyMap = newCache.iamGroupPolicyMap + cache.iamGroupsMap = newCache.iamGroupsMap + cache.iamPolicyDocsMap = newCache.iamPolicyDocsMap + cache.iamUserGroupMemberships = newCache.iamUserGroupMemberships + cache.iamUserPolicyMap = newCache.iamUserPolicyMap + cache.iamUsersMap = newCache.iamUsersMap + // For STS policy map, we need to merge the new cache with the existing + // cache because the periodic IAM reload is partial. The periodic load + // here is to account for STS policy mapping changes that should apply + // for service accounts derived from such STS accounts (i.e. LDAP STS + // accounts). + newCache.iamSTSPolicyMap.Range(func(k string, v MappedPolicy) bool { + cache.iamSTSPolicyMap.Store(k, v) + return true + }) + + cache.updatedAt = time.Now() + } + + return nil +} + +// IAMStoreSys contains IAMStorageAPI to add higher-level methods on the storage +// layer. +type IAMStoreSys struct { + IAMStorageAPI + + group *singleflight.Group + policy *singleflight.Group +} + +// HasWatcher - returns if the storage system has a watcher. +func (store *IAMStoreSys) HasWatcher() bool { + _, ok := store.IAMStorageAPI.(iamStorageWatcher) + return ok +} + +// GetUser - fetches credential from memory. +func (store *IAMStoreSys) GetUser(user string) (UserIdentity, bool) { + cache := store.rlock() + defer store.runlock() + + u, ok := cache.iamUsersMap[user] + if !ok { + // Check the sts map + u, ok = cache.iamSTSAccountsMap[user] + } + return u, ok +} + +// GetMappedPolicy - fetches mapped policy from memory. +func (store *IAMStoreSys) GetMappedPolicy(name string, isGroup bool) (MappedPolicy, bool) { + cache := store.rlock() + defer store.runlock() + + if isGroup { + v, ok := cache.iamGroupPolicyMap.Load(name) + return v, ok + } + return cache.iamUserPolicyMap.Load(name) +} + +// GroupNotificationHandler - updates in-memory cache on notification of +// change (e.g. peer notification for object storage and etcd watch +// notification). +func (store *IAMStoreSys) GroupNotificationHandler(ctx context.Context, group string) error { + cache := store.lock() + defer store.unlock() + + err := store.loadGroup(ctx, group, cache.iamGroupsMap) + if err != nil && err != errNoSuchGroup { + return err + } + + if err == errNoSuchGroup { + // group does not exist - so remove from memory. + cache.removeGroupFromMembershipsMap(group) + delete(cache.iamGroupsMap, group) + cache.iamGroupPolicyMap.Delete(group) + + cache.updatedAt = time.Now() + return nil + } + + gi := cache.iamGroupsMap[group] + + // Updating the group memberships cache happens in two steps: + // + // 1. Remove the group from each user's list of memberships. + // 2. Add the group to each member's list of memberships. + // + // This ensures that regardless of members being added or + // removed, the cache stays current. + cache.removeGroupFromMembershipsMap(group) + cache.updateGroupMembershipsMap(group, &gi) + cache.updatedAt = time.Now() + return nil +} + +// PolicyDBGet - fetches policies associated with the given user or group, and +// additional groups if provided. +func (store *IAMStoreSys) PolicyDBGet(name string, groups ...string) ([]string, error) { + if name == "" { + return nil, errInvalidArgument + } + + cache := store.rlock() + defer store.runlock() + + getPolicies := func() ([]string, error) { + policies, _, err := cache.policyDBGet(store, name, false, false) + if err != nil { + return nil, err + } + + userPolicyPresent := len(policies) > 0 + + groupPolicies, err := cache.policyDBGetGroups(store, userPolicyPresent, groups...) + if err != nil { + return nil, err + } + + policies = append(policies, groupPolicies...) + return policies, nil + } + if store.policy != nil { + val, err, _ := store.policy.Do(name, func() (any, error) { + return getPolicies() + }) + if err != nil { + return nil, err + } + res, ok := val.([]string) + if !ok { + return nil, errors.New("unexpected policy type") + } + return res, nil + } + return getPolicies() +} + +// AddUsersToGroup - adds users to group, creating the group if needed. +func (store *IAMStoreSys) AddUsersToGroup(ctx context.Context, group string, members []string) (updatedAt time.Time, err error) { + if group == "" { + return updatedAt, errInvalidArgument + } + + cache := store.lock() + defer store.unlock() + + // Validate that all members exist. + for _, member := range members { + u, ok := cache.iamUsersMap[member] + if !ok { + return updatedAt, errNoSuchUser + } + cr := u.Credentials + if cr.IsTemp() || cr.IsServiceAccount() { + return updatedAt, errIAMActionNotAllowed + } + } + + gi, ok := cache.iamGroupsMap[group] + if !ok { + // Set group as enabled by default when it doesn't + // exist. + gi = newGroupInfo(members) + } else { + gi.Members = set.CreateStringSet(append(gi.Members, members...)...).ToSlice() + gi.UpdatedAt = UTCNow() + } + + if err := store.saveGroupInfo(ctx, group, gi); err != nil { + return updatedAt, err + } + + cache.iamGroupsMap[group] = gi + + // update user-group membership map + for _, member := range members { + gset := cache.iamUserGroupMemberships[member] + if gset == nil { + gset = set.CreateStringSet(group) + } else { + gset.Add(group) + } + cache.iamUserGroupMemberships[member] = gset + } + + cache.updatedAt = time.Now() + return gi.UpdatedAt, nil +} + +// helper function - does not take any locks. Updates only cache if +// updateCacheOnly is set. +func removeMembersFromGroup(ctx context.Context, store *IAMStoreSys, cache *iamCache, group string, members []string, updateCacheOnly bool) (updatedAt time.Time, err error) { + gi, ok := cache.iamGroupsMap[group] + if !ok { + return updatedAt, errNoSuchGroup + } + + s := set.CreateStringSet(gi.Members...) + d := set.CreateStringSet(members...) + gi.Members = s.Difference(d).ToSlice() + + if !updateCacheOnly { + err := store.saveGroupInfo(ctx, group, gi) + if err != nil { + return updatedAt, err + } + } + gi.UpdatedAt = UTCNow() + cache.iamGroupsMap[group] = gi + + // update user-group membership map + for _, member := range members { + gset := cache.iamUserGroupMemberships[member] + if gset == nil { + continue + } + gset.Remove(group) + cache.iamUserGroupMemberships[member] = gset + } + + cache.updatedAt = time.Now() + return gi.UpdatedAt, nil +} + +// RemoveUsersFromGroup - removes users from group, deleting it if it is empty. +func (store *IAMStoreSys) RemoveUsersFromGroup(ctx context.Context, group string, members []string) (updatedAt time.Time, err error) { + if group == "" { + return updatedAt, errInvalidArgument + } + + cache := store.lock() + defer store.unlock() + + // Validate that all members exist. + for _, member := range members { + u, ok := cache.iamUsersMap[member] + if !ok { + return updatedAt, errNoSuchUser + } + cr := u.Credentials + if cr.IsTemp() || cr.IsServiceAccount() { + return updatedAt, errIAMActionNotAllowed + } + } + + gi, ok := cache.iamGroupsMap[group] + if !ok { + return updatedAt, errNoSuchGroup + } + + // Check if attempting to delete a non-empty group. + if len(members) == 0 && len(gi.Members) != 0 { + return updatedAt, errGroupNotEmpty + } + + if len(members) == 0 { + // len(gi.Members) == 0 here. + + // Remove the group from storage. First delete the + // mapped policy. No-mapped-policy case is ignored. + if err := store.deleteMappedPolicy(ctx, group, regUser, true); err != nil && !errors.Is(err, errNoSuchPolicy) { + return updatedAt, err + } + if err := store.deleteGroupInfo(ctx, group); err != nil && err != errNoSuchGroup { + return updatedAt, err + } + + // Delete from server memory + delete(cache.iamGroupsMap, group) + cache.iamGroupPolicyMap.Delete(group) + cache.updatedAt = time.Now() + return cache.updatedAt, nil + } + + return removeMembersFromGroup(ctx, store, cache, group, members, false) +} + +// SetGroupStatus - updates group status +func (store *IAMStoreSys) SetGroupStatus(ctx context.Context, group string, enabled bool) (updatedAt time.Time, err error) { + if group == "" { + return updatedAt, errInvalidArgument + } + + cache := store.lock() + defer store.unlock() + + gi, ok := cache.iamGroupsMap[group] + if !ok { + return updatedAt, errNoSuchGroup + } + + if enabled { + gi.Status = statusEnabled + } else { + gi.Status = statusDisabled + } + gi.UpdatedAt = UTCNow() + if err := store.saveGroupInfo(ctx, group, gi); err != nil { + return gi.UpdatedAt, err + } + + cache.iamGroupsMap[group] = gi + cache.updatedAt = time.Now() + + return gi.UpdatedAt, nil +} + +// GetGroupDescription - builds up group description +func (store *IAMStoreSys) GetGroupDescription(group string) (gd madmin.GroupDesc, err error) { + cache := store.rlock() + defer store.runlock() + + ps, updatedAt, err := cache.policyDBGet(store, group, true, false) + if err != nil { + return gd, err + } + + policy := strings.Join(ps, ",") + + if store.getUsersSysType() != MinIOUsersSysType { + return madmin.GroupDesc{ + Name: group, + Policy: policy, + UpdatedAt: updatedAt, + }, nil + } + + gi, ok := cache.iamGroupsMap[group] + if !ok { + return gd, errNoSuchGroup + } + + return madmin.GroupDesc{ + Name: group, + Status: gi.Status, + Members: gi.Members, + Policy: policy, + UpdatedAt: gi.UpdatedAt, + }, nil +} + +// updateGroups updates the group from the persistent store, and also related policy mapping if any. +func (store *IAMStoreSys) updateGroups(ctx context.Context, cache *iamCache) (res []string, err error) { + groupSet := set.NewStringSet() + if iamOS, ok := store.IAMStorageAPI.(*IAMObjectStore); ok { + listedConfigItems, err := iamOS.listAllIAMConfigItems(ctx) + if err != nil { + return nil, err + } + if store.getUsersSysType() == MinIOUsersSysType { + groupsList := listedConfigItems[groupsListKey] + for _, item := range groupsList { + group := path.Dir(item) + if err = iamOS.loadGroup(ctx, group, cache.iamGroupsMap); err != nil && !errors.Is(err, errNoSuchGroup) { + return nil, fmt.Errorf("unable to load the group: %w", err) + } + groupSet.Add(group) + } + } + + groupPolicyMappingsList := listedConfigItems[policyDBGroupsListKey] + for _, item := range groupPolicyMappingsList { + group := strings.TrimSuffix(item, ".json") + if err = iamOS.loadMappedPolicy(ctx, group, regUser, true, cache.iamGroupPolicyMap); err != nil && !errors.Is(err, errNoSuchPolicy) { + return nil, fmt.Errorf("unable to load the policy mapping for the group: %w", err) + } + groupSet.Add(group) + } + + return groupSet.ToSlice(), nil + } + + // For etcd just return from cache. + for k := range cache.iamGroupsMap { + groupSet.Add(k) + } + + cache.iamGroupPolicyMap.Range(func(k string, v MappedPolicy) bool { + groupSet.Add(k) + return true + }) + + return groupSet.ToSlice(), nil +} + +// ListGroups - lists groups. Since this is not going to be a frequent +// operation, we fetch this info from storage, and refresh the cache as well. +func (store *IAMStoreSys) ListGroups(ctx context.Context) (res []string, err error) { + cache := store.lock() + defer store.unlock() + + return store.updateGroups(ctx, cache) +} + +// listGroups - lists groups - fetch groups from cache +func (store *IAMStoreSys) listGroups(ctx context.Context) (res []string, err error) { + cache := store.rlock() + defer store.runlock() + + if store.getUsersSysType() == MinIOUsersSysType { + for k := range cache.iamGroupsMap { + res = append(res, k) + } + } + + if store.getUsersSysType() == LDAPUsersSysType { + cache.iamGroupPolicyMap.Range(func(k string, _ MappedPolicy) bool { + res = append(res, k) + return true + }) + } + return res, err +} + +// PolicyDBUpdate - adds or removes given policies to/from the user or group's +// policy associations. +func (store *IAMStoreSys) PolicyDBUpdate(ctx context.Context, name string, isGroup bool, + userType IAMUserType, policies []string, isAttach bool) (updatedAt time.Time, + addedOrRemoved, effectivePolicies []string, err error, +) { + if name == "" { + err = errInvalidArgument + return updatedAt, addedOrRemoved, effectivePolicies, err + } + + cache := store.lock() + defer store.unlock() + + // Load existing policy mapping + var mp MappedPolicy + if !isGroup { + if userType == stsUser { + stsMap := xsync.NewMapOf[string, MappedPolicy]() + + // Attempt to load parent user mapping for STS accounts + store.loadMappedPolicy(context.TODO(), name, stsUser, false, stsMap) + + mp, _ = stsMap.Load(name) + } else { + mp, _ = cache.iamUserPolicyMap.Load(name) + } + } else { + if store.getUsersSysType() == MinIOUsersSysType { + g, ok := cache.iamGroupsMap[name] + if !ok { + err = errNoSuchGroup + return updatedAt, addedOrRemoved, effectivePolicies, err + } + + if g.Status == statusDisabled { + err = errGroupDisabled + return updatedAt, addedOrRemoved, effectivePolicies, err + } + } + mp, _ = cache.iamGroupPolicyMap.Load(name) + } + + // Compute net policy change effect and updated policy mapping + existingPolicySet := mp.policySet() + policiesToUpdate := set.CreateStringSet(policies...) + var newPolicySet set.StringSet + newPolicyMapping := mp + if isAttach { + // new policies to attach => inputPolicies - existing (set difference) + policiesToUpdate = policiesToUpdate.Difference(existingPolicySet) + // validate that new policies to add are defined. + for _, p := range policiesToUpdate.ToSlice() { + if _, found := cache.iamPolicyDocsMap[p]; !found { + err = errNoSuchPolicy + return updatedAt, addedOrRemoved, effectivePolicies, err + } + } + newPolicySet = existingPolicySet.Union(policiesToUpdate) + } else { + // policies to detach => inputPolicies ∩ existing (intersection) + policiesToUpdate = policiesToUpdate.Intersection(existingPolicySet) + newPolicySet = existingPolicySet.Difference(policiesToUpdate) + } + // We return an error if the requested policy update will have no effect. + if policiesToUpdate.IsEmpty() { + err = errNoPolicyToAttachOrDetach + return updatedAt, addedOrRemoved, effectivePolicies, err + } + + newPolicies := newPolicySet.ToSlice() + newPolicyMapping.Policies = strings.Join(newPolicies, ",") + newPolicyMapping.UpdatedAt = UTCNow() + addedOrRemoved = policiesToUpdate.ToSlice() + + // In case of detach operation, it is possible that no policies are mapped - + // in this case, we delete the mapping from the store. + if len(newPolicies) == 0 { + if err = store.deleteMappedPolicy(ctx, name, userType, isGroup); err != nil && !errors.Is(err, errNoSuchPolicy) { + return updatedAt, addedOrRemoved, effectivePolicies, err + } + if !isGroup { + if userType == stsUser { + cache.iamSTSPolicyMap.Delete(name) + } else { + cache.iamUserPolicyMap.Delete(name) + } + } else { + cache.iamGroupPolicyMap.Delete(name) + } + } else { + if err = store.saveMappedPolicy(ctx, name, userType, isGroup, newPolicyMapping); err != nil { + return updatedAt, addedOrRemoved, effectivePolicies, err + } + if !isGroup { + if userType == stsUser { + cache.iamSTSPolicyMap.Store(name, newPolicyMapping) + } else { + cache.iamUserPolicyMap.Store(name, newPolicyMapping) + } + } else { + cache.iamGroupPolicyMap.Store(name, newPolicyMapping) + } + } + + cache.updatedAt = UTCNow() + return cache.updatedAt, addedOrRemoved, newPolicies, nil +} + +// PolicyDBSet - update the policy mapping for the given user or group in +// storage and in cache. We do not check for the existence of the user here +// since users can be virtual, such as for: +// - LDAP users +// - CommonName for STS accounts generated by AssumeRoleWithCertificate +func (store *IAMStoreSys) PolicyDBSet(ctx context.Context, name, policy string, userType IAMUserType, isGroup bool) (updatedAt time.Time, err error) { + if name == "" { + return updatedAt, errInvalidArgument + } + + cache := store.lock() + defer store.unlock() + + // Handle policy mapping removal. + if policy == "" { + if store.getUsersSysType() == LDAPUsersSysType { + // Add a fallback removal towards previous content that may come back + // as a ghost user due to lack of delete, this change occurred + // introduced in PR #11840 + store.deleteMappedPolicy(ctx, name, regUser, false) + } + err := store.deleteMappedPolicy(ctx, name, userType, isGroup) + if err != nil && !errors.Is(err, errNoSuchPolicy) { + return updatedAt, err + } + if !isGroup { + if userType == stsUser { + cache.iamSTSPolicyMap.Delete(name) + } else { + cache.iamUserPolicyMap.Delete(name) + } + } else { + cache.iamGroupPolicyMap.Delete(name) + } + cache.updatedAt = time.Now() + return cache.updatedAt, nil + } + + // Handle policy mapping set/update + mp := newMappedPolicy(policy) + for _, p := range mp.toSlice() { + if _, found := cache.iamPolicyDocsMap[p]; !found { + return updatedAt, errNoSuchPolicy + } + } + + if err := store.saveMappedPolicy(ctx, name, userType, isGroup, mp); err != nil { + return updatedAt, err + } + if !isGroup { + if userType == stsUser { + cache.iamSTSPolicyMap.Store(name, mp) + } else { + cache.iamUserPolicyMap.Store(name, mp) + } + } else { + cache.iamGroupPolicyMap.Store(name, mp) + } + cache.updatedAt = time.Now() + return mp.UpdatedAt, nil +} + +// PolicyNotificationHandler - loads given policy from storage. If not present, +// deletes from cache. This notification only reads from storage, and updates +// cache. When the notification is for a policy deletion, it updates the +// user-policy and group-policy maps as well. +func (store *IAMStoreSys) PolicyNotificationHandler(ctx context.Context, policy string) error { + if policy == "" { + return errInvalidArgument + } + + cache := store.lock() + defer store.unlock() + + err := store.loadPolicyDoc(ctx, policy, cache.iamPolicyDocsMap) + if errors.Is(err, errNoSuchPolicy) { + // policy was deleted, update cache. + delete(cache.iamPolicyDocsMap, policy) + + // update user policy map + cache.iamUserPolicyMap.Range(func(u string, mp MappedPolicy) bool { + pset := mp.policySet() + if !pset.Contains(policy) { + return true + } + if store.getUsersSysType() == MinIOUsersSysType { + _, ok := cache.iamUsersMap[u] + if !ok { + // happens when account is deleted or + // expired. + cache.iamUserPolicyMap.Delete(u) + return true + } + } + pset.Remove(policy) + cache.iamUserPolicyMap.Store(u, newMappedPolicy(strings.Join(pset.ToSlice(), ","))) + return true + }) + + // update group policy map + cache.iamGroupPolicyMap.Range(func(g string, mp MappedPolicy) bool { + pset := mp.policySet() + if !pset.Contains(policy) { + return true + } + pset.Remove(policy) + cache.iamGroupPolicyMap.Store(g, newMappedPolicy(strings.Join(pset.ToSlice(), ","))) + return true + }) + + cache.updatedAt = time.Now() + return nil + } + return err +} + +// DeletePolicy - deletes policy from storage and cache. When this called in +// response to a notification (i.e. isFromNotification = true), it skips the +// validation of policy usage and the attempt to delete in the backend as well +// (as this is already done by the notifying node). +func (store *IAMStoreSys) DeletePolicy(ctx context.Context, policy string, isFromNotification bool) error { + if policy == "" { + return errInvalidArgument + } + + cache := store.lock() + defer store.unlock() + + if !isFromNotification { + // Check if policy is mapped to any existing user or group. If so, we do not + // allow deletion of the policy. If the policy is mapped to an STS account, + // we do allow deletion. + users := []string{} + groups := []string{} + cache.iamUserPolicyMap.Range(func(u string, mp MappedPolicy) bool { + pset := mp.policySet() + if store.getUsersSysType() == MinIOUsersSysType { + if _, ok := cache.iamUsersMap[u]; !ok { + // This case can happen when a temporary account is + // deleted or expired - remove it from userPolicyMap. + cache.iamUserPolicyMap.Delete(u) + return true + } + } + if pset.Contains(policy) { + users = append(users, u) + } + return true + }) + cache.iamGroupPolicyMap.Range(func(g string, mp MappedPolicy) bool { + pset := mp.policySet() + if pset.Contains(policy) { + groups = append(groups, g) + } + return true + }) + if len(users) != 0 || len(groups) != 0 { + return errPolicyInUse + } + + err := store.deletePolicyDoc(ctx, policy) + if errors.Is(err, errNoSuchPolicy) { + // Ignore error if policy is already deleted. + err = nil + } + if err != nil { + return err + } + } + + delete(cache.iamPolicyDocsMap, policy) + cache.updatedAt = time.Now() + + return nil +} + +// GetPolicy - gets the policy definition. Allows specifying multiple comma +// separated policies - returns a combined policy. +func (store *IAMStoreSys) GetPolicy(name string) (policy.Policy, error) { + if name == "" { + return policy.Policy{}, errInvalidArgument + } + + cache := store.rlock() + defer store.runlock() + + policies := newMappedPolicy(name).toSlice() + var toMerge []policy.Policy + for _, policy := range policies { + if policy == "" { + continue + } + v, ok := cache.iamPolicyDocsMap[policy] + if !ok { + return v.Policy, errNoSuchPolicy + } + toMerge = append(toMerge, v.Policy) + } + if len(toMerge) == 0 { + return policy.Policy{}, errNoSuchPolicy + } + return policy.MergePolicies(toMerge...), nil +} + +// GetPolicyDoc - gets the policy doc which has the policy and some metadata. +// Exactly one policy must be specified here. +func (store *IAMStoreSys) GetPolicyDoc(name string) (r PolicyDoc, err error) { + name = strings.TrimSpace(name) + if name == "" { + return r, errInvalidArgument + } + + cache := store.rlock() + defer store.runlock() + + v, ok := cache.iamPolicyDocsMap[name] + if !ok { + return r, errNoSuchPolicy + } + return v, nil +} + +// SetPolicy - creates a policy with name. +func (store *IAMStoreSys) SetPolicy(ctx context.Context, name string, policy policy.Policy) (time.Time, error) { + if policy.IsEmpty() || name == "" { + return time.Time{}, errInvalidArgument + } + + cache := store.lock() + defer store.unlock() + + var ( + d PolicyDoc + ok bool + ) + if d, ok = cache.iamPolicyDocsMap[name]; ok { + d.update(policy) + } else { + d = newPolicyDoc(policy) + } + + if err := store.savePolicyDoc(ctx, name, d); err != nil { + return d.UpdateDate, err + } + + cache.iamPolicyDocsMap[name] = d + cache.updatedAt = time.Now() + + return d.UpdateDate, nil +} + +// ListPolicies - fetches all policies from storage and updates cache as well. +// If bucketName is non-empty, returns policies matching the bucket. +func (store *IAMStoreSys) ListPolicies(ctx context.Context, bucketName string) (map[string]policy.Policy, error) { + cache := store.lock() + defer store.unlock() + + m := map[string]PolicyDoc{} + err := store.loadPolicyDocs(ctx, m) + if err != nil { + return nil, err + } + + // Sets default canned policies + setDefaultCannedPolicies(m) + + cache.iamPolicyDocsMap = m + cache.updatedAt = time.Now() + + ret := map[string]policy.Policy{} + for k, v := range m { + if bucketName == "" || v.Policy.MatchResource(bucketName) { + ret[k] = v.Policy + } + } + + return ret, nil +} + +// ListPolicyDocs - fetches all policy docs from storage and updates cache as well. +// If bucketName is non-empty, returns policy docs matching the bucket. +func (store *IAMStoreSys) ListPolicyDocs(ctx context.Context, bucketName string) (map[string]PolicyDoc, error) { + cache := store.lock() + defer store.unlock() + + m := map[string]PolicyDoc{} + err := store.loadPolicyDocs(ctx, m) + if err != nil { + return nil, err + } + + // Sets default canned policies + setDefaultCannedPolicies(m) + + cache.iamPolicyDocsMap = m + cache.updatedAt = time.Now() + + ret := map[string]PolicyDoc{} + for k, v := range m { + if bucketName == "" || v.Policy.MatchResource(bucketName) { + ret[k] = v + } + } + + return ret, nil +} + +// fetches all policy docs from cache. +// If bucketName is non-empty, returns policy docs matching the bucket. +func (store *IAMStoreSys) listPolicyDocs(ctx context.Context, bucketName string) (map[string]PolicyDoc, error) { + cache := store.rlock() + defer store.runlock() + ret := map[string]PolicyDoc{} + for k, v := range cache.iamPolicyDocsMap { + if bucketName == "" || v.Policy.MatchResource(bucketName) { + ret[k] = v + } + } + return ret, nil +} + +// helper function - does not take locks. +func filterPolicies(cache *iamCache, policyName string, bucketName string) (string, policy.Policy) { + var policies []string + mp := newMappedPolicy(policyName) + var toMerge []policy.Policy + for _, policy := range mp.toSlice() { + if policy == "" { + continue + } + p, found := cache.iamPolicyDocsMap[policy] + if !found { + continue + } + if bucketName == "" || p.Policy.MatchResource(bucketName) { + policies = append(policies, policy) + toMerge = append(toMerge, p.Policy) + } + } + return strings.Join(policies, ","), policy.MergePolicies(toMerge...) +} + +// MergePolicies - accepts a comma separated list of policy names as a string +// and returns only policies that currently exist in MinIO. It includes hot loading +// of policies if not in the memory +func (store *IAMStoreSys) MergePolicies(policyName string) (string, policy.Policy) { + var policies []string + var missingPolicies []string + var toMerge []policy.Policy + + cache := store.rlock() + for _, policy := range newMappedPolicy(policyName).toSlice() { + if policy == "" { + continue + } + p, found := cache.iamPolicyDocsMap[policy] + if !found { + missingPolicies = append(missingPolicies, policy) + continue + } + policies = append(policies, policy) + toMerge = append(toMerge, p.Policy) + } + store.runlock() + + if len(missingPolicies) > 0 { + m := make(map[string]PolicyDoc) + for _, policy := range missingPolicies { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + _ = store.loadPolicyDoc(ctx, policy, m) + cancel() + } + + cache := store.lock() + maps.Copy(cache.iamPolicyDocsMap, m) + store.unlock() + + for policy, p := range m { + policies = append(policies, policy) + toMerge = append(toMerge, p.Policy) + } + } + + return strings.Join(policies, ","), policy.MergePolicies(toMerge...) +} + +// GetBucketUsers - returns users (not STS or service accounts) that have access +// to the bucket. User is included even if a group policy that grants access to +// the bucket is disabled. +func (store *IAMStoreSys) GetBucketUsers(bucket string) (map[string]madmin.UserInfo, error) { + if bucket == "" { + return nil, errInvalidArgument + } + + cache := store.rlock() + defer store.runlock() + + result := map[string]madmin.UserInfo{} + for k, v := range cache.iamUsersMap { + c := v.Credentials + if c.IsTemp() || c.IsServiceAccount() { + continue + } + var policies []string + mp, ok := cache.iamUserPolicyMap.Load(k) + if ok { + policies = append(policies, mp.Policies) + for _, group := range cache.iamUserGroupMemberships[k].ToSlice() { + if nmp, ok := cache.iamGroupPolicyMap.Load(group); ok { + policies = append(policies, nmp.Policies) + } + } + } + matchedPolicies, _ := filterPolicies(cache, strings.Join(policies, ","), bucket) + if len(matchedPolicies) > 0 { + result[k] = madmin.UserInfo{ + PolicyName: matchedPolicies, + Status: func() madmin.AccountStatus { + if c.IsValid() { + return madmin.AccountEnabled + } + return madmin.AccountDisabled + }(), + MemberOf: cache.iamUserGroupMemberships[k].ToSlice(), + } + } + } + + return result, nil +} + +// GetUsers - returns all users (not STS or service accounts). +func (store *IAMStoreSys) GetUsers() map[string]madmin.UserInfo { + cache := store.rlock() + defer store.runlock() + + result := map[string]madmin.UserInfo{} + for k, u := range cache.iamUsersMap { + v := u.Credentials + + if v.IsTemp() || v.IsServiceAccount() { + continue + } + pl, _ := cache.iamUserPolicyMap.Load(k) + result[k] = madmin.UserInfo{ + PolicyName: pl.Policies, + Status: func() madmin.AccountStatus { + if v.IsValid() { + return madmin.AccountEnabled + } + return madmin.AccountDisabled + }(), + MemberOf: cache.iamUserGroupMemberships[k].ToSlice(), + UpdatedAt: pl.UpdatedAt, + } + } + + return result +} + +// GetUsersWithMappedPolicies - safely returns the name of access keys with associated policies +func (store *IAMStoreSys) GetUsersWithMappedPolicies() map[string]string { + cache := store.rlock() + defer store.runlock() + + result := make(map[string]string) + cache.iamUserPolicyMap.Range(func(k string, v MappedPolicy) bool { + result[k] = v.Policies + return true + }) + cache.iamSTSPolicyMap.Range(func(k string, v MappedPolicy) bool { + result[k] = v.Policies + return true + }) + return result +} + +// GetUserInfo - get info on a user. +func (store *IAMStoreSys) GetUserInfo(name string) (u madmin.UserInfo, err error) { + if name == "" { + return u, errInvalidArgument + } + + cache := store.rlock() + defer store.runlock() + + if store.getUsersSysType() != MinIOUsersSysType { + // If the user has a mapped policy or is a member of a group, we + // return that info. Otherwise we return error. + var groups []string + for _, v := range cache.iamUsersMap { + if v.Credentials.ParentUser == name { + groups = v.Credentials.Groups + break + } + } + for _, v := range cache.iamSTSAccountsMap { + if v.Credentials.ParentUser == name { + groups = v.Credentials.Groups + break + } + } + mappedPolicy, ok := cache.iamUserPolicyMap.Load(name) + if !ok { + mappedPolicy, ok = cache.iamSTSPolicyMap.Load(name) + } + if !ok { + // Attempt to load parent user mapping for STS accounts + store.loadMappedPolicy(context.TODO(), name, stsUser, false, cache.iamSTSPolicyMap) + mappedPolicy, ok = cache.iamSTSPolicyMap.Load(name) + if !ok { + return u, errNoSuchUser + } + } + + return madmin.UserInfo{ + PolicyName: mappedPolicy.Policies, + MemberOf: groups, + UpdatedAt: mappedPolicy.UpdatedAt, + }, nil + } + + ui, found := cache.iamUsersMap[name] + if !found { + return u, errNoSuchUser + } + cred := ui.Credentials + if cred.IsTemp() || cred.IsServiceAccount() { + return u, errIAMActionNotAllowed + } + pl, _ := cache.iamUserPolicyMap.Load(name) + return madmin.UserInfo{ + PolicyName: pl.Policies, + Status: func() madmin.AccountStatus { + if cred.IsValid() { + return madmin.AccountEnabled + } + return madmin.AccountDisabled + }(), + MemberOf: cache.iamUserGroupMemberships[name].ToSlice(), + UpdatedAt: pl.UpdatedAt, + }, nil +} + +// PolicyMappingNotificationHandler - handles updating a policy mapping from storage. +func (store *IAMStoreSys) PolicyMappingNotificationHandler(ctx context.Context, userOrGroup string, isGroup bool, userType IAMUserType) error { + if userOrGroup == "" { + return errInvalidArgument + } + + cache := store.lock() + defer store.unlock() + + var m *xsync.MapOf[string, MappedPolicy] + switch { + case isGroup: + m = cache.iamGroupPolicyMap + case userType == stsUser: + m = cache.iamSTSPolicyMap + default: + m = cache.iamUserPolicyMap + } + err := store.loadMappedPolicy(ctx, userOrGroup, userType, isGroup, m) + if errors.Is(err, errNoSuchPolicy) { + // This means that the policy mapping was deleted, so we update + // the cache. + m.Delete(userOrGroup) + cache.updatedAt = time.Now() + + err = nil + } + return err +} + +// UserNotificationHandler - handles updating a user/STS account/service account +// from storage. +func (store *IAMStoreSys) UserNotificationHandler(ctx context.Context, accessKey string, userType IAMUserType) error { + if accessKey == "" { + return errInvalidArgument + } + + cache := store.lock() + defer store.unlock() + + var m map[string]UserIdentity + switch userType { + case stsUser: + m = cache.iamSTSAccountsMap + default: + m = cache.iamUsersMap + } + err := store.loadUser(ctx, accessKey, userType, m) + + if err == errNoSuchUser { + // User was deleted - we update the cache. + delete(m, accessKey) + + // Since cache was updated, we update the timestamp. + defer func() { + cache.updatedAt = time.Now() + }() + + // 1. Start with updating user-group memberships + if store.getUsersSysType() == MinIOUsersSysType { + memberOf := cache.iamUserGroupMemberships[accessKey].ToSlice() + for _, group := range memberOf { + _, removeErr := removeMembersFromGroup(ctx, store, cache, group, []string{accessKey}, true) + if removeErr == errNoSuchGroup { + removeErr = nil + } + if removeErr != nil { + return removeErr + } + } + } + + // 2. Remove any derived credentials from memory + if userType == regUser { + for k, u := range cache.iamUsersMap { + if u.Credentials.IsServiceAccount() && u.Credentials.ParentUser == accessKey { + delete(cache.iamUsersMap, k) + } + } + for k, u := range cache.iamSTSAccountsMap { + if u.Credentials.ParentUser == accessKey { + delete(cache.iamSTSAccountsMap, k) + } + } + } + + // 3. Delete any mapped policy + cache.iamUserPolicyMap.Delete(accessKey) + + return nil + } + + if err != nil { + return err + } + + // Since cache was updated, we update the timestamp. + defer func() { + cache.updatedAt = time.Now() + }() + + cred := m[accessKey].Credentials + switch userType { + case stsUser: + // For STS accounts a policy is mapped to the parent user (if a mapping exists). + err = store.loadMappedPolicy(ctx, cred.ParentUser, userType, false, cache.iamSTSPolicyMap) + case svcUser: + // For service accounts, the parent may be a regular (internal) IDP + // user or a "virtual" user (parent of an STS account). + // + // If parent is a regular user => policy mapping is done on that parent itself. + // + // If parent is "virtual" => policy mapping is done on the virtual + // parent and that virtual parent is an stsUser. + // + // To load the appropriate mapping, we check the parent user type. + _, parentIsRegularUser := cache.iamUsersMap[cred.ParentUser] + if parentIsRegularUser { + err = store.loadMappedPolicy(ctx, cred.ParentUser, regUser, false, cache.iamUserPolicyMap) + } else { + err = store.loadMappedPolicy(ctx, cred.ParentUser, stsUser, false, cache.iamSTSPolicyMap) + } + case regUser: + // For regular users, we load the mapped policy. + err = store.loadMappedPolicy(ctx, accessKey, userType, false, cache.iamUserPolicyMap) + default: + // This is just to ensure that we have covered all cases for new + // code in future. + panic("unknown user type") + } + // Ignore policy not mapped error + if err != nil && !errors.Is(err, errNoSuchPolicy) { + return err + } + + return nil +} + +// DeleteUser - deletes a user from storage and cache. This only used with +// long-term users and service accounts, not STS. +func (store *IAMStoreSys) DeleteUser(ctx context.Context, accessKey string, userType IAMUserType) error { + if accessKey == "" { + return errInvalidArgument + } + + cache := store.lock() + defer store.unlock() + + // first we remove the user from their groups. + if store.getUsersSysType() == MinIOUsersSysType && userType == regUser { + memberOf := cache.iamUserGroupMemberships[accessKey].ToSlice() + for _, group := range memberOf { + _, removeErr := removeMembersFromGroup(ctx, store, cache, group, []string{accessKey}, false) + if removeErr != nil { + return removeErr + } + } + } + + // Now we can remove the user from memory and IAM store + + // Delete any STS and service account derived from this credential + // first. + if userType == regUser { + for _, ui := range cache.iamUsersMap { + u := ui.Credentials + if u.ParentUser == accessKey { + switch { + case u.IsServiceAccount(): + _ = store.deleteUserIdentity(ctx, u.AccessKey, svcUser) + delete(cache.iamUsersMap, u.AccessKey) + case u.IsTemp(): + _ = store.deleteUserIdentity(ctx, u.AccessKey, stsUser) + delete(cache.iamSTSAccountsMap, u.AccessKey) + delete(cache.iamUsersMap, u.AccessKey) + } + if store.group != nil { + store.group.Forget(u.AccessKey) + } + } + } + } + + // It is ok to ignore deletion error on the mapped policy + store.deleteMappedPolicy(ctx, accessKey, userType, false) + cache.iamUserPolicyMap.Delete(accessKey) + + err := store.deleteUserIdentity(ctx, accessKey, userType) + if err == errNoSuchUser { + // ignore if user is already deleted. + err = nil + } + if userType == stsUser { + delete(cache.iamSTSAccountsMap, accessKey) + } + delete(cache.iamUsersMap, accessKey) + if store.group != nil { + store.group.Forget(accessKey) + } + + cache.updatedAt = time.Now() + + return err +} + +// SetTempUser - saves temporary (STS) credential to storage and cache. If a +// policy name is given, it is associated with the parent user specified in the +// credential. +func (store *IAMStoreSys) SetTempUser(ctx context.Context, accessKey string, cred auth.Credentials, policyName string) (time.Time, error) { + if accessKey == "" || !cred.IsTemp() || cred.IsExpired() || cred.ParentUser == "" { + return time.Time{}, errInvalidArgument + } + + ttl := int64(cred.Expiration.Sub(UTCNow()).Seconds()) + + cache := store.lock() + defer store.unlock() + + if policyName != "" { + mp := newMappedPolicy(policyName) + _, combinedPolicyStmt := filterPolicies(cache, mp.Policies, "") + + if combinedPolicyStmt.IsEmpty() { + return time.Time{}, fmt.Errorf("specified policy %s, not found %w", policyName, errNoSuchPolicy) + } + + err := store.saveMappedPolicy(ctx, cred.ParentUser, stsUser, false, mp, options{ttl: ttl}) + if err != nil { + return time.Time{}, err + } + + cache.iamSTSPolicyMap.Store(cred.ParentUser, mp) + } + + u := newUserIdentity(cred) + err := store.saveUserIdentity(ctx, accessKey, stsUser, u, options{ttl: ttl}) + if err != nil { + return time.Time{}, err + } + + cache.iamSTSAccountsMap[accessKey] = u + cache.updatedAt = time.Now() + + return u.UpdatedAt, nil +} + +// RevokeTokens - revokes all temporary credentials, or those with matching type, +// associated with the parent user. +func (store *IAMStoreSys) RevokeTokens(ctx context.Context, parentUser string, tokenRevokeType string) error { + if parentUser == "" { + return errInvalidArgument + } + + cache := store.lock() + defer store.unlock() + + secret, err := getTokenSigningKey() + if err != nil { + return err + } + + var revoked bool + for _, ui := range cache.iamSTSAccountsMap { + if ui.Credentials.ParentUser != parentUser { + continue + } + if tokenRevokeType != "" { + claims, err := getClaimsFromTokenWithSecret(ui.Credentials.SessionToken, secret) + if err != nil { + continue // skip if token is invalid + } + // skip if token type is given and does not match + if v, _ := claims.Lookup(tokenRevokeTypeClaim); v != tokenRevokeType { + continue + } + } + if err := store.deleteUserIdentity(ctx, ui.Credentials.AccessKey, stsUser); err != nil { + return err + } + delete(cache.iamSTSAccountsMap, ui.Credentials.AccessKey) + revoked = true + } + + if revoked { + cache.updatedAt = time.Now() + } + + return nil +} + +// DeleteUsers - given a set of users or access keys, deletes them along with +// any derived credentials (STS or service accounts) and any associated policy +// mappings. +func (store *IAMStoreSys) DeleteUsers(ctx context.Context, users []string) error { + cache := store.lock() + defer store.unlock() + + var deleted bool + usersToDelete := set.CreateStringSet(users...) + for user, ui := range cache.iamUsersMap { + userType := regUser + cred := ui.Credentials + + if cred.IsServiceAccount() { + userType = svcUser + } else if cred.IsTemp() { + userType = stsUser + } + + if usersToDelete.Contains(user) || usersToDelete.Contains(cred.ParentUser) { + // Delete this user account and its policy mapping + store.deleteMappedPolicy(ctx, user, userType, false) + cache.iamUserPolicyMap.Delete(user) + + // we are only logging errors, not handling them. + err := store.deleteUserIdentity(ctx, user, userType) + iamLogIf(GlobalContext, err) + if userType == stsUser { + delete(cache.iamSTSAccountsMap, user) + } + delete(cache.iamUsersMap, user) + if store.group != nil { + store.group.Forget(user) + } + + deleted = true + } + } + + if deleted { + cache.updatedAt = time.Now() + } + + return nil +} + +// ParentUserInfo contains extra info about a the parent user. +type ParentUserInfo struct { + subClaimValue string + roleArns set.StringSet +} + +// GetAllParentUsers - returns all distinct "parent-users" associated with STS +// or service credentials, mapped to all distinct roleARNs associated with the +// parent user. The dummy role ARN is associated with parent users from +// policy-claim based OpenID providers. The root credential as a parent +// user is not included in the result. +func (store *IAMStoreSys) GetAllParentUsers() map[string]ParentUserInfo { + cache := store.rlock() + defer store.runlock() + + return store.getParentUsers(cache) +} + +// assumes store is locked by caller. +func (store *IAMStoreSys) getParentUsers(cache *iamCache) map[string]ParentUserInfo { + res := map[string]ParentUserInfo{} + for _, ui := range cache.iamUsersMap { + cred := ui.Credentials + // Only consider service account or STS credentials with + // non-empty session tokens. + if (!cred.IsServiceAccount() && !cred.IsTemp()) || + cred.SessionToken == "" { + continue + } + + var ( + err error + claims *jwt.MapClaims + ) + + if cred.IsServiceAccount() { + claims, err = getClaimsFromTokenWithSecret(cred.SessionToken, cred.SecretKey) + } else if cred.IsTemp() { + var secretKey string + secretKey, err = getTokenSigningKey() + if err != nil { + continue + } + claims, err = getClaimsFromTokenWithSecret(cred.SessionToken, secretKey) + } + + if err != nil { + continue + } + if cred.ParentUser == "" || cred.ParentUser == globalActiveCred.AccessKey { + continue + } + + subClaimValue := cred.ParentUser + if v, ok := claims.Lookup(subClaim); ok { + subClaimValue = v + } + if v, ok := claims.Lookup(ldapActualUser); ok { + subClaimValue = v + } + + roleArn := openid.DummyRoleARN.String() + s, ok := claims.Lookup(roleArnClaim) + if ok { + roleArn = s + } + v, ok := res[cred.ParentUser] + if ok { + res[cred.ParentUser] = ParentUserInfo{ + subClaimValue: subClaimValue, + roleArns: v.roleArns.Union(set.CreateStringSet(roleArn)), + } + } else { + res[cred.ParentUser] = ParentUserInfo{ + subClaimValue: subClaimValue, + roleArns: set.CreateStringSet(roleArn), + } + } + } + + return res +} + +// GetAllSTSUserMappings - Loads all STS user policy mappings from storage and +// returns them. Also gets any STS users that do not have policy mappings but have +// Service Accounts or STS keys (This is useful if the user is part of a group) +func (store *IAMStoreSys) GetAllSTSUserMappings(userPredicate func(string) bool) (map[string]string, error) { + cache := store.rlock() + defer store.runlock() + + stsMap := make(map[string]string) + m := xsync.NewMapOf[string, MappedPolicy]() + if err := store.loadMappedPolicies(context.Background(), stsUser, false, m); err != nil { + return nil, err + } + + m.Range(func(user string, mappedPolicy MappedPolicy) bool { + if userPredicate != nil && !userPredicate(user) { + return true + } + stsMap[user] = mappedPolicy.Policies + return true + }) + + for user := range store.getParentUsers(cache) { + if _, ok := stsMap[user]; !ok { + if userPredicate != nil && !userPredicate(user) { + continue + } + stsMap[user] = "" + } + } + return stsMap, nil +} + +// Assumes store is locked by caller. If userMap is empty, returns all user mappings. +func (store *IAMStoreSys) listUserPolicyMappings(cache *iamCache, userMap map[string]set.StringSet, + userPredicate func(string) bool, decodeFunc func(string) string, +) []madmin.UserPolicyEntities { + stsMap := xsync.NewMapOf[string, MappedPolicy]() + resMap := make(map[string]madmin.UserPolicyEntities, len(userMap)) + + for user, groupSet := range userMap { + // Attempt to load parent user mapping for STS accounts + store.loadMappedPolicy(context.TODO(), user, stsUser, false, stsMap) + decodeUser := user + if decodeFunc != nil { + decodeUser = decodeFunc(user) + } + blankEntities := madmin.UserPolicyEntities{User: decodeUser} + if !groupSet.IsEmpty() { + blankEntities.MemberOfMappings = store.listGroupPolicyMappings(cache, groupSet, nil, decodeFunc) + } + resMap[user] = blankEntities + } + + var r []madmin.UserPolicyEntities + cache.iamUserPolicyMap.Range(func(user string, mappedPolicy MappedPolicy) bool { + if userPredicate != nil && !userPredicate(user) { + return true + } + + entitiesWithMemberOf, ok := resMap[user] + if !ok { + if len(userMap) > 0 { + return true + } + decodeUser := user + if decodeFunc != nil { + decodeUser = decodeFunc(user) + } + entitiesWithMemberOf = madmin.UserPolicyEntities{User: decodeUser} + } + + ps := mappedPolicy.toSlice() + sort.Strings(ps) + entitiesWithMemberOf.Policies = ps + resMap[user] = entitiesWithMemberOf + return true + }) + + stsMap.Range(func(user string, mappedPolicy MappedPolicy) bool { + if userPredicate != nil && !userPredicate(user) { + return true + } + + entitiesWithMemberOf := resMap[user] + + ps := mappedPolicy.toSlice() + sort.Strings(ps) + entitiesWithMemberOf.Policies = ps + resMap[user] = entitiesWithMemberOf + return true + }) + + for _, v := range resMap { + if v.Policies != nil || v.MemberOfMappings != nil { + r = append(r, v) + } + } + + sort.Slice(r, func(i, j int) bool { + return r[i].User < r[j].User + }) + + return r +} + +// Assumes store is locked by caller. If groups is empty, returns all group mappings. +func (store *IAMStoreSys) listGroupPolicyMappings(cache *iamCache, groupsSet set.StringSet, + groupPredicate func(string) bool, decodeFunc func(string) string, +) []madmin.GroupPolicyEntities { + var r []madmin.GroupPolicyEntities + + cache.iamGroupPolicyMap.Range(func(group string, mappedPolicy MappedPolicy) bool { + if groupPredicate != nil && !groupPredicate(group) { + return true + } + + if !groupsSet.IsEmpty() && !groupsSet.Contains(group) { + return true + } + + decodeGroup := group + if decodeFunc != nil { + decodeGroup = decodeFunc(group) + } + + ps := mappedPolicy.toSlice() + sort.Strings(ps) + r = append(r, madmin.GroupPolicyEntities{ + Group: decodeGroup, + Policies: ps, + }) + return true + }) + + sort.Slice(r, func(i, j int) bool { + return r[i].Group < r[j].Group + }) + + return r +} + +// Assumes store is locked by caller. If policies is empty, returns all policy mappings. +func (store *IAMStoreSys) listPolicyMappings(cache *iamCache, queryPolSet set.StringSet, + userPredicate, groupPredicate func(string) bool, decodeFunc func(string) string, +) []madmin.PolicyEntities { + policyToUsersMap := make(map[string]set.StringSet) + cache.iamUserPolicyMap.Range(func(user string, mappedPolicy MappedPolicy) bool { + if userPredicate != nil && !userPredicate(user) { + return true + } + + decodeUser := user + if decodeFunc != nil { + decodeUser = decodeFunc(user) + } + + commonPolicySet := mappedPolicy.policySet() + if !queryPolSet.IsEmpty() { + commonPolicySet = commonPolicySet.Intersection(queryPolSet) + } + for _, policy := range commonPolicySet.ToSlice() { + s, ok := policyToUsersMap[policy] + if !ok { + policyToUsersMap[policy] = set.CreateStringSet(decodeUser) + } else { + s.Add(decodeUser) + policyToUsersMap[policy] = s + } + } + return true + }) + + if iamOS, ok := store.IAMStorageAPI.(*IAMObjectStore); ok { + for item := range listIAMConfigItems(context.Background(), iamOS.objAPI, iamConfigPrefix+SlashSeparator+policyDBSTSUsersListKey) { + user := strings.TrimSuffix(item.Item, ".json") + if userPredicate != nil && !userPredicate(user) { + continue + } + + decodeUser := user + if decodeFunc != nil { + decodeUser = decodeFunc(user) + } + + var mappedPolicy MappedPolicy + store.loadIAMConfig(context.Background(), &mappedPolicy, getMappedPolicyPath(user, stsUser, false)) + + commonPolicySet := mappedPolicy.policySet() + if !queryPolSet.IsEmpty() { + commonPolicySet = commonPolicySet.Intersection(queryPolSet) + } + for _, policy := range commonPolicySet.ToSlice() { + s, ok := policyToUsersMap[policy] + if !ok { + policyToUsersMap[policy] = set.CreateStringSet(decodeUser) + } else { + s.Add(decodeUser) + policyToUsersMap[policy] = s + } + } + } + } + if iamOS, ok := store.IAMStorageAPI.(*IAMEtcdStore); ok { + m := xsync.NewMapOf[string, MappedPolicy]() + err := iamOS.loadMappedPolicies(context.Background(), stsUser, false, m) + if err == nil { + m.Range(func(user string, mappedPolicy MappedPolicy) bool { + if userPredicate != nil && !userPredicate(user) { + return true + } + + decodeUser := user + if decodeFunc != nil { + decodeUser = decodeFunc(user) + } + + commonPolicySet := mappedPolicy.policySet() + if !queryPolSet.IsEmpty() { + commonPolicySet = commonPolicySet.Intersection(queryPolSet) + } + for _, policy := range commonPolicySet.ToSlice() { + s, ok := policyToUsersMap[policy] + if !ok { + policyToUsersMap[policy] = set.CreateStringSet(decodeUser) + } else { + s.Add(decodeUser) + policyToUsersMap[policy] = s + } + } + return true + }) + } + } + + policyToGroupsMap := make(map[string]set.StringSet) + cache.iamGroupPolicyMap.Range(func(group string, mappedPolicy MappedPolicy) bool { + if groupPredicate != nil && !groupPredicate(group) { + return true + } + + decodeGroup := group + if decodeFunc != nil { + decodeGroup = decodeFunc(group) + } + + commonPolicySet := mappedPolicy.policySet() + if !queryPolSet.IsEmpty() { + commonPolicySet = commonPolicySet.Intersection(queryPolSet) + } + for _, policy := range commonPolicySet.ToSlice() { + s, ok := policyToGroupsMap[policy] + if !ok { + policyToGroupsMap[policy] = set.CreateStringSet(decodeGroup) + } else { + s.Add(decodeGroup) + policyToGroupsMap[policy] = s + } + } + return true + }) + + m := make(map[string]madmin.PolicyEntities, len(policyToGroupsMap)) + for policy, groups := range policyToGroupsMap { + s := groups.ToSlice() + sort.Strings(s) + m[policy] = madmin.PolicyEntities{ + Policy: policy, + Groups: s, + } + } + for policy, users := range policyToUsersMap { + s := users.ToSlice() + sort.Strings(s) + + // Update existing value in map + pe := m[policy] + pe.Policy = policy + pe.Users = s + m[policy] = pe + } + + policyEntities := make([]madmin.PolicyEntities, 0, len(m)) + for _, v := range m { + policyEntities = append(policyEntities, v) + } + + sort.Slice(policyEntities, func(i, j int) bool { + return policyEntities[i].Policy < policyEntities[j].Policy + }) + + return policyEntities +} + +// ListPolicyMappings - return users/groups mapped to policies. +func (store *IAMStoreSys) ListPolicyMappings(q cleanEntitiesQuery, + userPredicate, groupPredicate func(string) bool, decodeFunc func(string) string, +) madmin.PolicyEntitiesResult { + cache := store.rlock() + defer store.runlock() + + var result madmin.PolicyEntitiesResult + + isAllPoliciesQuery := len(q.Users) == 0 && len(q.Groups) == 0 && len(q.Policies) == 0 + + if len(q.Users) > 0 { + result.UserMappings = store.listUserPolicyMappings(cache, q.Users, userPredicate, decodeFunc) + } + if len(q.Groups) > 0 { + result.GroupMappings = store.listGroupPolicyMappings(cache, q.Groups, groupPredicate, decodeFunc) + } + if len(q.Policies) > 0 || isAllPoliciesQuery { + result.PolicyMappings = store.listPolicyMappings(cache, q.Policies, userPredicate, groupPredicate, decodeFunc) + } + return result +} + +// SetUserStatus - sets current user status. +func (store *IAMStoreSys) SetUserStatus(ctx context.Context, accessKey string, status madmin.AccountStatus) (updatedAt time.Time, err error) { + if accessKey != "" && status != madmin.AccountEnabled && status != madmin.AccountDisabled { + return updatedAt, errInvalidArgument + } + + cache := store.lock() + defer store.unlock() + + ui, ok := cache.iamUsersMap[accessKey] + if !ok { + return updatedAt, errNoSuchUser + } + cred := ui.Credentials + + if cred.IsTemp() || cred.IsServiceAccount() { + return updatedAt, errIAMActionNotAllowed + } + + uinfo := newUserIdentity(auth.Credentials{ + AccessKey: accessKey, + SecretKey: cred.SecretKey, + Status: func() string { + switch string(status) { + case string(madmin.AccountEnabled), string(auth.AccountOn): + return auth.AccountOn + } + return auth.AccountOff + }(), + }) + + if err := store.saveUserIdentity(ctx, accessKey, regUser, uinfo); err != nil { + return updatedAt, err + } + + if err := cache.updateUserWithClaims(accessKey, uinfo); err != nil { + return updatedAt, err + } + + return uinfo.UpdatedAt, nil +} + +// AddServiceAccount - add a new service account +func (store *IAMStoreSys) AddServiceAccount(ctx context.Context, cred auth.Credentials) (updatedAt time.Time, err error) { + cache := store.lock() + defer store.unlock() + + accessKey := cred.AccessKey + parentUser := cred.ParentUser + + // Found newly requested service account, to be an existing account - + // reject such operation (updates to the service account are handled in + // a different API). + if su, found := cache.iamUsersMap[accessKey]; found { + scred := su.Credentials + if scred.ParentUser != parentUser { + return updatedAt, fmt.Errorf("%w: the service account access key is taken by another user", errIAMServiceAccountNotAllowed) + } + return updatedAt, fmt.Errorf("%w: the service account access key already taken", errIAMServiceAccountNotAllowed) + } + + // Parent user must not be a service account. + if u, found := cache.iamUsersMap[parentUser]; found && u.Credentials.IsServiceAccount() { + return updatedAt, fmt.Errorf("%w: unable to create a service account for another service account", errIAMServiceAccountNotAllowed) + } + + u := newUserIdentity(cred) + err = store.saveUserIdentity(ctx, u.Credentials.AccessKey, svcUser, u) + if err != nil { + return updatedAt, err + } + + cache.updateUserWithClaims(u.Credentials.AccessKey, u) + + return u.UpdatedAt, nil +} + +// UpdateServiceAccount - updates a service account on storage. +func (store *IAMStoreSys) UpdateServiceAccount(ctx context.Context, accessKey string, opts updateServiceAccountOpts) (updatedAt time.Time, err error) { + cache := store.lock() + defer store.unlock() + + ui, ok := cache.iamUsersMap[accessKey] + if !ok || !ui.Credentials.IsServiceAccount() { + return updatedAt, errNoSuchServiceAccount + } + cr := ui.Credentials + currentSecretKey := cr.SecretKey + if opts.secretKey != "" { + if !auth.IsSecretKeyValid(opts.secretKey) { + return updatedAt, auth.ErrInvalidSecretKeyLength + } + cr.SecretKey = opts.secretKey + } + + if opts.name != "" { + cr.Name = opts.name + } + + if opts.description != "" { + cr.Description = opts.description + } + + if opts.expiration != nil { + expirationInUTC := opts.expiration.UTC() + if err := validateSvcExpirationInUTC(expirationInUTC); err != nil { + return updatedAt, err + } + cr.Expiration = expirationInUTC + } + + switch opts.status { + // The caller did not ask to update status account, do nothing + case "": + case string(madmin.AccountEnabled): + cr.Status = auth.AccountOn + case string(madmin.AccountDisabled): + cr.Status = auth.AccountOff + // Update account status + case auth.AccountOn, auth.AccountOff: + cr.Status = opts.status + default: + return updatedAt, errors.New("unknown account status value") + } + + m, err := getClaimsFromTokenWithSecret(cr.SessionToken, currentSecretKey) + if err != nil { + return updatedAt, fmt.Errorf("unable to get svc acc claims: %v", err) + } + + // Extracted session policy name string can be removed as its not useful + // at this point. + m.Delete(sessionPolicyNameExtracted) + + nosp := opts.sessionPolicy == nil || opts.sessionPolicy.Version == "" && len(opts.sessionPolicy.Statements) == 0 + + // sessionPolicy is nil and there is embedded policy attached we remove + // embedded policy at that point. + if _, ok := m.Lookup(policy.SessionPolicyName); ok && nosp { + m.Delete(policy.SessionPolicyName) + m.Set(iamPolicyClaimNameSA(), inheritedPolicyType) + } + + if opts.sessionPolicy != nil { // session policies is being updated + if err := opts.sessionPolicy.Validate(); err != nil { + return updatedAt, err + } + + if opts.sessionPolicy.Version != "" && len(opts.sessionPolicy.Statements) > 0 { + policyBuf, err := json.Marshal(opts.sessionPolicy) + if err != nil { + return updatedAt, err + } + + if len(policyBuf) > maxSVCSessionPolicySize { + return updatedAt, errSessionPolicyTooLarge + } + + // Overwrite session policy claims. + m.Set(policy.SessionPolicyName, base64.StdEncoding.EncodeToString(policyBuf)) + m.Set(iamPolicyClaimNameSA(), embeddedPolicyType) + } + } + + cr.SessionToken, err = auth.JWTSignWithAccessKey(accessKey, m.Map(), cr.SecretKey) + if err != nil { + return updatedAt, err + } + + u := newUserIdentity(cr) + if err := store.saveUserIdentity(ctx, u.Credentials.AccessKey, svcUser, u); err != nil { + return updatedAt, err + } + + if err := cache.updateUserWithClaims(u.Credentials.AccessKey, u); err != nil { + return updatedAt, err + } + + return u.UpdatedAt, nil +} + +// ListTempAccounts - lists only temporary accounts from the cache. +func (store *IAMStoreSys) ListTempAccounts(ctx context.Context, accessKey string) ([]UserIdentity, error) { + cache := store.rlock() + defer store.runlock() + + userExists := false + var tempAccounts []UserIdentity + for _, v := range cache.iamUsersMap { + isDerived := false + if v.Credentials.IsServiceAccount() || v.Credentials.IsTemp() { + isDerived = true + } + + if !isDerived && v.Credentials.AccessKey == accessKey { + userExists = true + } else if isDerived && v.Credentials.ParentUser == accessKey { + userExists = true + if v.Credentials.IsTemp() { + // Hide secret key & session key here + v.Credentials.SecretKey = "" + v.Credentials.SessionToken = "" + tempAccounts = append(tempAccounts, v) + } + } + } + + if !userExists { + return nil, errNoSuchUser + } + + return tempAccounts, nil +} + +// ListServiceAccounts - lists only service accounts from the cache. +func (store *IAMStoreSys) ListServiceAccounts(ctx context.Context, accessKey string) ([]auth.Credentials, error) { + cache := store.rlock() + defer store.runlock() + + var serviceAccounts []auth.Credentials + for _, u := range cache.iamUsersMap { + v := u.Credentials + if accessKey != "" && v.ParentUser == accessKey { + if v.IsServiceAccount() { + // Hide secret key & session key here + v.SecretKey = "" + v.SessionToken = "" + serviceAccounts = append(serviceAccounts, v) + } + } + } + + return serviceAccounts, nil +} + +// ListSTSAccounts - lists only STS accounts from the cache. +func (store *IAMStoreSys) ListSTSAccounts(ctx context.Context, accessKey string) ([]auth.Credentials, error) { + cache := store.rlock() + defer store.runlock() + + var stsAccounts []auth.Credentials + for _, u := range cache.iamSTSAccountsMap { + v := u.Credentials + if accessKey != "" && v.ParentUser == accessKey { + if v.IsTemp() { + // Hide secret key & session key here + v.SecretKey = "" + v.SessionToken = "" + stsAccounts = append(stsAccounts, v) + } + } + } + + return stsAccounts, nil +} + +// ListAccessKeys - lists all access keys (sts/service accounts) +func (store *IAMStoreSys) ListAccessKeys(ctx context.Context) ([]auth.Credentials, error) { + cache := store.rlock() + defer store.runlock() + + accessKeys := store.getSTSAndServiceAccounts(cache) + for i, accessKey := range accessKeys { + accessKeys[i].SecretKey = "" + if accessKey.IsTemp() { + secret, err := getTokenSigningKey() + if err != nil { + return nil, err + } + claims, err := getClaimsFromTokenWithSecret(accessKey.SessionToken, secret) + if err != nil { + continue // ignore invalid session tokens + } + accessKeys[i].Claims = claims.MapClaims + } + accessKeys[i].SessionToken = "" + } + + return accessKeys, nil +} + +// AddUser - adds/updates long term user account to storage. +func (store *IAMStoreSys) AddUser(ctx context.Context, accessKey string, ureq madmin.AddOrUpdateUserReq) (updatedAt time.Time, err error) { + cache := store.lock() + defer store.unlock() + + cache.updatedAt = time.Now() + + ui, ok := cache.iamUsersMap[accessKey] + + // It is not possible to update an STS account. + if ok && ui.Credentials.IsTemp() { + return updatedAt, errIAMActionNotAllowed + } + + u := newUserIdentity(auth.Credentials{ + AccessKey: accessKey, + SecretKey: ureq.SecretKey, + Status: func() string { + switch string(ureq.Status) { + case string(madmin.AccountEnabled), string(auth.AccountOn): + return auth.AccountOn + } + return auth.AccountOff + }(), + }) + + if err := store.saveUserIdentity(ctx, accessKey, regUser, u); err != nil { + return updatedAt, err + } + if err := cache.updateUserWithClaims(accessKey, u); err != nil { + return updatedAt, err + } + + return u.UpdatedAt, nil +} + +// UpdateUserSecretKey - sets user secret key to storage. +func (store *IAMStoreSys) UpdateUserSecretKey(ctx context.Context, accessKey, secretKey string) error { + cache := store.lock() + defer store.unlock() + + cache.updatedAt = time.Now() + + ui, ok := cache.iamUsersMap[accessKey] + if !ok { + return errNoSuchUser + } + cred := ui.Credentials + cred.SecretKey = secretKey + u := newUserIdentity(cred) + if err := store.saveUserIdentity(ctx, accessKey, regUser, u); err != nil { + return err + } + + return cache.updateUserWithClaims(accessKey, u) +} + +// GetSTSAndServiceAccounts - returns all STS and Service account credentials. +func (store *IAMStoreSys) GetSTSAndServiceAccounts() []auth.Credentials { + cache := store.rlock() + defer store.runlock() + + return store.getSTSAndServiceAccounts(cache) +} + +func (store *IAMStoreSys) getSTSAndServiceAccounts(cache *iamCache) []auth.Credentials { + var res []auth.Credentials + for _, u := range cache.iamUsersMap { + cred := u.Credentials + if cred.IsServiceAccount() { + res = append(res, cred) + } + } + for _, u := range cache.iamSTSAccountsMap { + res = append(res, u.Credentials) + } + + return res +} + +// UpdateUserIdentity - updates a user credential. +func (store *IAMStoreSys) UpdateUserIdentity(ctx context.Context, cred auth.Credentials) error { + cache := store.lock() + defer store.unlock() + + cache.updatedAt = time.Now() + + userType := regUser + if cred.IsServiceAccount() { + userType = svcUser + } else if cred.IsTemp() { + userType = stsUser + } + ui := newUserIdentity(cred) + // Overwrite the user identity here. As store should be + // atomic, it shouldn't cause any corruption. + if err := store.saveUserIdentity(ctx, cred.AccessKey, userType, ui); err != nil { + return err + } + + return cache.updateUserWithClaims(cred.AccessKey, ui) +} + +// LoadUser - attempts to load user info from storage and updates cache. +func (store *IAMStoreSys) LoadUser(ctx context.Context, accessKey string) error { + groupLoad := env.Get("_MINIO_IAM_GROUP_REFRESH", config.EnableOff) == config.EnableOn + + newCachePopulate := func() (val any, err error) { + newCache := newIamCache() + + // Check for service account first + store.loadUser(ctx, accessKey, svcUser, newCache.iamUsersMap) + + svc, found := newCache.iamUsersMap[accessKey] + if found { + // Load parent user and mapped policies. + if store.getUsersSysType() == MinIOUsersSysType { + err = store.loadUser(ctx, svc.Credentials.ParentUser, regUser, newCache.iamUsersMap) + // NOTE: we are not worried about loading errors from policies. + store.loadMappedPolicyWithRetry(ctx, svc.Credentials.ParentUser, regUser, false, newCache.iamUserPolicyMap, 3) + } else { + // In case of LDAP the parent user's policy mapping needs to be loaded into sts map + // NOTE: we are not worried about loading errors from policies. + store.loadMappedPolicyWithRetry(ctx, svc.Credentials.ParentUser, stsUser, false, newCache.iamSTSPolicyMap, 3) + } + } + + if !found { + err = store.loadUser(ctx, accessKey, regUser, newCache.iamUsersMap) + if _, found = newCache.iamUsersMap[accessKey]; found { + // NOTE: we are not worried about loading errors from policies. + store.loadMappedPolicyWithRetry(ctx, accessKey, regUser, false, newCache.iamUserPolicyMap, 3) + } + } + + // Check for STS account + var stsUserCred UserIdentity + if !found { + err = store.loadUser(ctx, accessKey, stsUser, newCache.iamSTSAccountsMap) + if stsUserCred, found = newCache.iamSTSAccountsMap[accessKey]; found { + // Load mapped policy + // NOTE: we are not worried about loading errors from policies. + store.loadMappedPolicyWithRetry(ctx, stsUserCred.Credentials.ParentUser, stsUser, false, newCache.iamSTSPolicyMap, 3) + } + } + + // Load any associated policy definitions + pols, _ := newCache.iamUserPolicyMap.Load(accessKey) + for _, policy := range pols.toSlice() { + if _, found = newCache.iamPolicyDocsMap[policy]; !found { + // NOTE: we are not worried about loading errors from policies. + store.loadPolicyDocWithRetry(ctx, policy, newCache.iamPolicyDocsMap, 3) + } + } + + pols, _ = newCache.iamSTSPolicyMap.Load(stsUserCred.Credentials.AccessKey) + for _, policy := range pols.toSlice() { + if _, found = newCache.iamPolicyDocsMap[policy]; !found { + // NOTE: we are not worried about loading errors from policies. + store.loadPolicyDocWithRetry(ctx, policy, newCache.iamPolicyDocsMap, 3) + } + } + + if groupLoad { + // NOTE: we are not worried about loading errors from groups. + store.updateGroups(ctx, newCache) + newCache.buildUserGroupMemberships() + } + + return newCache, err + } + + var ( + val any + err error + ) + if store.group != nil { + val, err, _ = store.group.Do(accessKey, newCachePopulate) + } else { + val, err = newCachePopulate() + } + + // Return error right away if any. + if err != nil { + if errors.Is(err, errNoSuchUser) || errors.Is(err, errConfigNotFound) { + return nil + } + return err + } + + newCache, ok := val.(*iamCache) + if !ok { + return nil + } + + cache := store.lock() + defer store.unlock() + + // We need to merge the new cache with the existing cache because the + // periodic IAM reload is partial. The periodic load here is to account. + newCache.iamGroupPolicyMap.Range(func(k string, v MappedPolicy) bool { + cache.iamGroupPolicyMap.Store(k, v) + return true + }) + + maps.Copy(cache.iamGroupsMap, newCache.iamGroupsMap) + + maps.Copy(cache.iamPolicyDocsMap, newCache.iamPolicyDocsMap) + + maps.Copy(cache.iamUserGroupMemberships, newCache.iamUserGroupMemberships) + + newCache.iamUserPolicyMap.Range(func(k string, v MappedPolicy) bool { + cache.iamUserPolicyMap.Store(k, v) + return true + }) + + maps.Copy(cache.iamUsersMap, newCache.iamUsersMap) + + maps.Copy(cache.iamSTSAccountsMap, newCache.iamSTSAccountsMap) + + newCache.iamSTSPolicyMap.Range(func(k string, v MappedPolicy) bool { + cache.iamSTSPolicyMap.Store(k, v) + return true + }) + + cache.updatedAt = time.Now() + + return nil +} + +func extractJWTClaims(u UserIdentity) (jwtClaims *jwt.MapClaims, err error) { + keys := make([]string, 0, 3) + + // Append credentials secret key itself + keys = append(keys, u.Credentials.SecretKey) + + // Use site-replication credentials if found + if globalSiteReplicationSys.isEnabled() { + secretKey, err := getTokenSigningKey() + if err != nil { + return nil, err + } + keys = append(keys, secretKey) + } + + // Iterate over all keys and return with the first successful claim extraction + for _, key := range keys { + jwtClaims, err = getClaimsFromTokenWithSecret(u.Credentials.SessionToken, key) + if err == nil { + break + } + } + return jwtClaims, err +} + +func validateSvcExpirationInUTC(expirationInUTC time.Time) error { + if expirationInUTC.IsZero() || expirationInUTC.Equal(timeSentinel) { + // Service accounts might not have expiration in older releases. + return nil + } + + currentTime := time.Now().UTC() + minExpiration := currentTime.Add(minServiceAccountExpiry) + maxExpiration := currentTime.Add(maxServiceAccountExpiry) + if expirationInUTC.Before(minExpiration) || expirationInUTC.After(maxExpiration) { + return errInvalidSvcAcctExpiration + } + + return nil +} diff --git a/cmd/iam.go b/cmd/iam.go new file mode 100644 index 00000000..18cdc148 --- /dev/null +++ b/cmd/iam.go @@ -0,0 +1,2556 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "maps" + "math/rand" + "path" + "sort" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/arn" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/color" + "github.com/minio/minio/internal/config" + xldap "github.com/minio/minio/internal/config/identity/ldap" + "github.com/minio/minio/internal/config/identity/openid" + idplugin "github.com/minio/minio/internal/config/identity/plugin" + xtls "github.com/minio/minio/internal/config/identity/tls" + "github.com/minio/minio/internal/config/policy/opa" + polplugin "github.com/minio/minio/internal/config/policy/plugin" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/jwt" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/env" + "github.com/minio/pkg/v3/ldap" + "github.com/minio/pkg/v3/policy" + etcd "go.etcd.io/etcd/client/v3" + "golang.org/x/sync/singleflight" +) + +// UsersSysType - defines the type of users and groups system that is +// active on the server. +type UsersSysType string + +// Types of users configured in the server. +const ( + // This mode uses the internal users system in MinIO. + MinIOUsersSysType UsersSysType = "MinIOUsersSys" + + // This mode uses users and groups from a configured LDAP + // server. + LDAPUsersSysType UsersSysType = "LDAPUsersSys" +) + +const ( + statusEnabled = "enabled" + statusDisabled = "disabled" +) + +const ( + embeddedPolicyType = "embedded-policy" + inheritedPolicyType = "inherited-policy" +) + +const ( + maxSVCSessionPolicySize = 4096 +) + +// IAMSys - config system. +type IAMSys struct { + // Need to keep them here to keep alignment - ref: https://golang.org/pkg/sync/atomic/#pkg-note-BUG + // metrics + LastRefreshTimeUnixNano uint64 + LastRefreshDurationMilliseconds uint64 + TotalRefreshSuccesses uint64 + TotalRefreshFailures uint64 + + sync.Mutex + + iamRefreshInterval time.Duration + + LDAPConfig xldap.Config // only valid if usersSysType is LDAPUsers + OpenIDConfig openid.Config // only valid if OpenID is configured + STSTLSConfig xtls.Config // only valid if STS TLS is configured + + usersSysType UsersSysType + + rolesMap map[arn.ARN]string + + // Persistence layer for IAM subsystem + store *IAMStoreSys + + // configLoaded will be closed and remain so after first load. + configLoaded chan struct{} +} + +// IAMUserType represents a user type inside MinIO server +type IAMUserType int + +const ( + unknownIAMUserType IAMUserType = iota - 1 + regUser + stsUser + svcUser +) + +// LoadGroup - loads a specific group from storage, and updates the +// memberships cache. If the specified group does not exist in +// storage, it is removed from in-memory maps as well - this +// simplifies the implementation for group removal. This is called +// only via IAM notifications. +func (sys *IAMSys) LoadGroup(ctx context.Context, objAPI ObjectLayer, group string) error { + if !sys.Initialized() { + return errServerNotInitialized + } + + return sys.store.GroupNotificationHandler(ctx, group) +} + +// LoadPolicy - reloads a specific canned policy from backend disks or etcd. +func (sys *IAMSys) LoadPolicy(ctx context.Context, objAPI ObjectLayer, policyName string) error { + if !sys.Initialized() { + return errServerNotInitialized + } + + return sys.store.PolicyNotificationHandler(ctx, policyName) +} + +// LoadPolicyMapping - loads the mapped policy for a user or group +// from storage into server memory. +func (sys *IAMSys) LoadPolicyMapping(ctx context.Context, objAPI ObjectLayer, userOrGroup string, userType IAMUserType, isGroup bool) error { + if !sys.Initialized() { + return errServerNotInitialized + } + + return sys.store.PolicyMappingNotificationHandler(ctx, userOrGroup, isGroup, userType) +} + +// LoadUser - reloads a specific user from backend disks or etcd. +func (sys *IAMSys) LoadUser(ctx context.Context, objAPI ObjectLayer, accessKey string, userType IAMUserType) error { + if !sys.Initialized() { + return errServerNotInitialized + } + + return sys.store.UserNotificationHandler(ctx, accessKey, userType) +} + +// LoadServiceAccount - reloads a specific service account from backend disks or etcd. +func (sys *IAMSys) LoadServiceAccount(ctx context.Context, accessKey string) error { + if !sys.Initialized() { + return errServerNotInitialized + } + + return sys.store.UserNotificationHandler(ctx, accessKey, svcUser) +} + +// initStore initializes IAM stores +func (sys *IAMSys) initStore(objAPI ObjectLayer, etcdClient *etcd.Client) { + if sys.LDAPConfig.Enabled() { + sys.SetUsersSysType(LDAPUsersSysType) + } + + if etcdClient == nil { + var ( + group *singleflight.Group + policy *singleflight.Group + ) + if env.Get("_MINIO_IAM_SINGLE_FLIGHT", config.EnableOn) == config.EnableOn { + group = &singleflight.Group{} + policy = &singleflight.Group{} + } + sys.store = &IAMStoreSys{ + IAMStorageAPI: newIAMObjectStore(objAPI, sys.usersSysType), + group: group, + policy: policy, + } + } else { + sys.store = &IAMStoreSys{IAMStorageAPI: newIAMEtcdStore(etcdClient, sys.usersSysType)} + } +} + +// Initialized checks if IAM is initialized +func (sys *IAMSys) Initialized() bool { + if sys == nil { + return false + } + sys.Lock() + defer sys.Unlock() + return sys.store != nil +} + +// Load - loads all credentials, policies and policy mappings. +func (sys *IAMSys) Load(ctx context.Context, firstTime bool) error { + loadStartTime := time.Now() + err := sys.store.LoadIAMCache(ctx, firstTime) + if err != nil { + atomic.AddUint64(&sys.TotalRefreshFailures, 1) + return err + } + loadDuration := time.Since(loadStartTime) + + atomic.StoreUint64(&sys.LastRefreshDurationMilliseconds, uint64(loadDuration.Milliseconds())) + atomic.StoreUint64(&sys.LastRefreshTimeUnixNano, uint64(loadStartTime.Add(loadDuration).UnixNano())) + atomic.AddUint64(&sys.TotalRefreshSuccesses, 1) + + if !globalSiteReplicatorCred.IsValid() { + sa, _, err := sys.getServiceAccount(ctx, siteReplicatorSvcAcc) + if err == nil { + globalSiteReplicatorCred.Set(sa.Credentials.SecretKey) + } + } + + if firstTime { + bootstrapTraceMsg(fmt.Sprintf("globalIAMSys.Load(): (duration: %s)", loadDuration)) + if globalIsDistErasure { + logger.Info("IAM load(startup) finished. (duration: %s)", loadDuration) + } + } + + select { + case <-sys.configLoaded: + default: + close(sys.configLoaded) + } + return nil +} + +// Init - initializes config system by reading entries from config/iam +func (sys *IAMSys) Init(ctx context.Context, objAPI ObjectLayer, etcdClient *etcd.Client, iamRefreshInterval time.Duration) { + bootstrapTraceMsg("IAM initialization started") + globalServerConfigMu.RLock() + s := globalServerConfig + globalServerConfigMu.RUnlock() + + sys.Lock() + sys.iamRefreshInterval = iamRefreshInterval + sys.Unlock() + + r := rand.New(rand.NewSource(time.Now().UnixNano())) + + var ( + openidInit bool + ldapInit bool + authNInit bool + authZInit bool + ) + + stsTLSConfig, err := xtls.Lookup(s[config.IdentityTLSSubSys][config.Default]) + if err != nil { + iamLogIf(ctx, fmt.Errorf("Unable to initialize X.509/TLS STS API: %w", err), logger.WarningKind) + } else { + if stsTLSConfig.InsecureSkipVerify { + iamLogIf(ctx, fmt.Errorf("Enabling %s is not recommended in a production environment", xtls.EnvIdentityTLSSkipVerify), logger.WarningKind) + } + sys.Lock() + sys.STSTLSConfig = stsTLSConfig + sys.Unlock() + } + + for { + if !openidInit { + openidConfig, err := openid.LookupConfig(s, + xhttp.WithUserAgent(NewHTTPTransport(), func() string { + return getUserAgent(getMinioMode()) + }), xhttp.DrainBody, globalSite.Region()) + if err != nil { + iamLogIf(ctx, fmt.Errorf("Unable to initialize OpenID: %w", err), logger.WarningKind) + } else { + openidInit = true + sys.Lock() + sys.OpenIDConfig = openidConfig + sys.Unlock() + } + } + + if !ldapInit { + // Initialize if LDAP is enabled + ldapConfig, err := xldap.Lookup(s, globalRootCAs) + if err != nil { + iamLogIf(ctx, fmt.Errorf("Unable to load LDAP configuration (LDAP configuration will be disabled!): %w", err), logger.WarningKind) + } else { + ldapInit = true + sys.Lock() + sys.LDAPConfig = ldapConfig + sys.Unlock() + } + } + + if !authNInit { + authNPluginCfg, err := idplugin.LookupConfig(s[config.IdentityPluginSubSys][config.Default], + NewHTTPTransport(), xhttp.DrainBody, globalSite.Region()) + if err != nil { + iamLogIf(ctx, fmt.Errorf("Unable to initialize AuthNPlugin: %w", err), logger.WarningKind) + } else { + authNInit = true + setGlobalAuthNPlugin(idplugin.New(GlobalContext, authNPluginCfg)) + } + } + + if !authZInit { + authZPluginCfg, err := polplugin.LookupConfig(s, GetDefaultConnSettings(), xhttp.DrainBody) + if err != nil { + iamLogIf(ctx, fmt.Errorf("Unable to initialize AuthZPlugin: %w", err), logger.WarningKind) + } else { + authZInit = true + } + if authZPluginCfg.URL == nil { + opaCfg, err := opa.LookupConfig(s[config.PolicyOPASubSys][config.Default], + NewHTTPTransport(), xhttp.DrainBody) + if err != nil { + iamLogIf(ctx, fmt.Errorf("Unable to initialize AuthZPlugin from legacy OPA config: %w", err)) + } else { + authZPluginCfg.URL = opaCfg.URL + authZPluginCfg.AuthToken = opaCfg.AuthToken + authZPluginCfg.Transport = opaCfg.Transport + authZPluginCfg.CloseRespFn = opaCfg.CloseRespFn + authZInit = true + } + } + if authZInit { + setGlobalAuthZPlugin(polplugin.New(authZPluginCfg)) + } + } + + if !openidInit || !ldapInit || !authNInit || !authZInit { + retryInterval := time.Duration(r.Float64() * float64(3*time.Second)) + if !openidInit { + logger.Info("Waiting for OpenID to be initialized.. (retrying in %s)", retryInterval) + } + if !ldapInit { + logger.Info("Waiting for LDAP to be initialized.. (retrying in %s)", retryInterval) + } + if !authNInit { + logger.Info("Waiting for AuthN to be initialized.. (retrying in %s)", retryInterval) + } + if !authZInit { + logger.Info("Waiting for AuthZ to be initialized.. (retrying in %s)", retryInterval) + } + time.Sleep(retryInterval) + continue + } + + break + } + + // Initialize IAM store + sys.Lock() + + sys.initStore(objAPI, etcdClient) + + // Initialize RoleARNs + sys.rolesMap = make(map[arn.ARN]string) + + // From OpenID + maps.Copy(sys.rolesMap, sys.OpenIDConfig.GetRoleInfo()) + + // From AuthN plugin if enabled. + if authn := newGlobalAuthNPluginFn(); authn != nil { + maps.Copy(sys.rolesMap, authn.GetRoleInfo()) + } + + sys.printIAMRoles() + sys.Unlock() + + retryCtx, cancel := context.WithCancel(ctx) + + // Indicate to our routine to exit cleanly upon return. + defer cancel() + + // Migrate storage format if needed. + for { + // Migrate IAM configuration, if necessary. + if err := saveIAMFormat(retryCtx, sys.store); err != nil { + if configRetriableErrors(err) { + retryInterval := time.Duration(r.Float64() * float64(time.Second)) + logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. possible cause (%v) (retrying in %s)", err, retryInterval) + time.Sleep(retryInterval) + continue + } + iamLogIf(ctx, fmt.Errorf("IAM sub-system is partially initialized, unable to write the IAM format: %w", err), logger.WarningKind) + return + } + + break + } + + cache := sys.store.lock() + setDefaultCannedPolicies(cache.iamPolicyDocsMap) + sys.store.unlock() + + // Load IAM data from storage. + for { + if err := sys.Load(retryCtx, true); err != nil { + if configRetriableErrors(err) { + retryInterval := time.Duration(r.Float64() * float64(time.Second)) + logger.Info("Waiting for all MinIO IAM sub-system to be initialized.. possible cause (%v) (retrying in %s)", err, retryInterval) + time.Sleep(retryInterval) + continue + } + if err != nil { + iamLogIf(ctx, fmt.Errorf("Unable to initialize IAM sub-system, some users may not be available: %w", err), logger.WarningKind) + } + } + break + } + + refreshInterval := sys.iamRefreshInterval + go sys.periodicRoutines(ctx, refreshInterval) + + bootstrapTraceMsg("finishing IAM loading") +} + +const maxDurationSecondsForLog = 5 + +func (sys *IAMSys) periodicRoutines(ctx context.Context, baseInterval time.Duration) { + // Watch for IAM config changes for iamStorageWatcher. + watcher, isWatcher := sys.store.IAMStorageAPI.(iamStorageWatcher) + if isWatcher { + go func() { + ch := watcher.watch(ctx, iamConfigPrefix) + for event := range ch { + if err := sys.loadWatchedEvent(ctx, event); err != nil { + // we simply log errors + iamLogIf(ctx, fmt.Errorf("Failure in loading watch event: %v", err), logger.WarningKind) + } + } + }() + } + + r := rand.New(rand.NewSource(time.Now().UnixNano())) + + // Calculate the waitInterval between periodic refreshes so that each server + // independently picks a (uniformly distributed) random time in an interval + // of size = baseInterval. + // + // For example: + // + // - if baseInterval=10s, then 5s <= waitInterval() < 15s + // + // - if baseInterval=10m, then 5m <= waitInterval() < 15m + waitInterval := func() time.Duration { + // Calculate a random value such that 0 <= value < baseInterval + randAmt := time.Duration(r.Float64() * float64(baseInterval)) + return baseInterval/2 + randAmt + } + + timer := time.NewTimer(waitInterval()) + defer timer.Stop() + + lastPurgeHour := -1 + for { + select { + case <-timer.C: + // Load all IAM items (except STS creds) periodically. + refreshStart := time.Now() + if err := sys.Load(ctx, false); err != nil { + iamLogIf(ctx, fmt.Errorf("Failure in periodic refresh for IAM (duration: %s): %v", time.Since(refreshStart), err), logger.WarningKind) + } else { + took := time.Since(refreshStart).Seconds() + if took > maxDurationSecondsForLog { + // Log if we took a lot of time to load. + logger.Info("IAM refresh took (duration: %.2fs)", took) + } + } + + // Run purge routines once in each hour. + if refreshStart.Hour() != lastPurgeHour { + lastPurgeHour = refreshStart.Hour() + // Poll and remove accounts for those users who were removed + // from LDAP/OpenID. + if sys.LDAPConfig.Enabled() { + sys.purgeExpiredCredentialsForLDAP(ctx) + sys.updateGroupMembershipsForLDAP(ctx) + } + if sys.OpenIDConfig.ProviderEnabled() { + sys.purgeExpiredCredentialsForExternalSSO(ctx) + } + } + + timer.Reset(waitInterval()) + case <-ctx.Done(): + return + } + } +} + +// Prints IAM role ARNs. +func (sys *IAMSys) printIAMRoles() { + if len(sys.rolesMap) == 0 { + return + } + var arns []string + for arn := range sys.rolesMap { + arns = append(arns, arn.String()) + } + sort.Strings(arns) + msgs := make([]string, 0, len(arns)) + for _, arn := range arns { + msgs = append(msgs, color.Bold(arn)) + } + + logger.Info(fmt.Sprintf("%s %s", color.Blue("IAM Roles:"), strings.Join(msgs, " "))) +} + +// HasWatcher - returns if the IAM system has a watcher to be notified of +// changes. +func (sys *IAMSys) HasWatcher() bool { + return sys.store.HasWatcher() +} + +func (sys *IAMSys) loadWatchedEvent(ctx context.Context, event iamWatchEvent) (err error) { + usersPrefix := strings.HasPrefix(event.keyPath, iamConfigUsersPrefix) + groupsPrefix := strings.HasPrefix(event.keyPath, iamConfigGroupsPrefix) + stsPrefix := strings.HasPrefix(event.keyPath, iamConfigSTSPrefix) + svcPrefix := strings.HasPrefix(event.keyPath, iamConfigServiceAccountsPrefix) + policyPrefix := strings.HasPrefix(event.keyPath, iamConfigPoliciesPrefix) + policyDBUsersPrefix := strings.HasPrefix(event.keyPath, iamConfigPolicyDBUsersPrefix) + policyDBSTSUsersPrefix := strings.HasPrefix(event.keyPath, iamConfigPolicyDBSTSUsersPrefix) + policyDBGroupsPrefix := strings.HasPrefix(event.keyPath, iamConfigPolicyDBGroupsPrefix) + + ctx, cancel := context.WithTimeout(ctx, defaultContextTimeout) + defer cancel() + + switch { + case usersPrefix: + accessKey := path.Dir(strings.TrimPrefix(event.keyPath, iamConfigUsersPrefix)) + err = sys.store.UserNotificationHandler(ctx, accessKey, regUser) + case stsPrefix: + accessKey := path.Dir(strings.TrimPrefix(event.keyPath, iamConfigSTSPrefix)) + err = sys.store.UserNotificationHandler(ctx, accessKey, stsUser) + case svcPrefix: + accessKey := path.Dir(strings.TrimPrefix(event.keyPath, iamConfigServiceAccountsPrefix)) + err = sys.store.UserNotificationHandler(ctx, accessKey, svcUser) + case groupsPrefix: + group := path.Dir(strings.TrimPrefix(event.keyPath, iamConfigGroupsPrefix)) + err = sys.store.GroupNotificationHandler(ctx, group) + case policyPrefix: + policyName := path.Dir(strings.TrimPrefix(event.keyPath, iamConfigPoliciesPrefix)) + err = sys.store.PolicyNotificationHandler(ctx, policyName) + case policyDBUsersPrefix: + policyMapFile := strings.TrimPrefix(event.keyPath, iamConfigPolicyDBUsersPrefix) + user := strings.TrimSuffix(policyMapFile, ".json") + err = sys.store.PolicyMappingNotificationHandler(ctx, user, false, regUser) + case policyDBSTSUsersPrefix: + policyMapFile := strings.TrimPrefix(event.keyPath, iamConfigPolicyDBSTSUsersPrefix) + user := strings.TrimSuffix(policyMapFile, ".json") + err = sys.store.PolicyMappingNotificationHandler(ctx, user, false, stsUser) + case policyDBGroupsPrefix: + policyMapFile := strings.TrimPrefix(event.keyPath, iamConfigPolicyDBGroupsPrefix) + user := strings.TrimSuffix(policyMapFile, ".json") + err = sys.store.PolicyMappingNotificationHandler(ctx, user, true, regUser) + } + return err +} + +// HasRolePolicy - returns if a role policy is configured for IAM. +func (sys *IAMSys) HasRolePolicy() bool { + return len(sys.rolesMap) > 0 +} + +// GetRolePolicy - returns policies associated with a role ARN. +func (sys *IAMSys) GetRolePolicy(arnStr string) (arn.ARN, string, error) { + roleArn, err := arn.Parse(arnStr) + if err != nil { + return arn.ARN{}, "", fmt.Errorf("RoleARN parse err: %v", err) + } + rolePolicy, ok := sys.rolesMap[roleArn] + if !ok { + return arn.ARN{}, "", fmt.Errorf("RoleARN %s is not defined.", arnStr) + } + return roleArn, rolePolicy, nil +} + +// DeletePolicy - deletes a canned policy from backend. `notifyPeers` is true +// whenever this is called via the API. It is false when called via a +// notification from another peer. This is to avoid infinite loops. +func (sys *IAMSys) DeletePolicy(ctx context.Context, policyName string, notifyPeers bool) error { + if !sys.Initialized() { + return errServerNotInitialized + } + + for _, v := range policy.DefaultPolicies { + if v.Name == policyName { + if err := checkConfig(ctx, globalObjectAPI, getPolicyDocPath(policyName)); err != nil && err == errConfigNotFound { + return fmt.Errorf("inbuilt policy `%s` not allowed to be deleted", policyName) + } + } + } + + err := sys.store.DeletePolicy(ctx, policyName, !notifyPeers) + if err != nil { + return err + } + + if !notifyPeers || sys.HasWatcher() { + return nil + } + + // Notify all other MinIO peers to delete policy + for _, nerr := range globalNotificationSys.DeletePolicy(ctx, policyName) { + if nerr.Err != nil { + logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String()) + iamLogIf(ctx, nerr.Err) + } + } + + return nil +} + +// InfoPolicy - returns the policy definition with some metadata. +func (sys *IAMSys) InfoPolicy(policyName string) (*madmin.PolicyInfo, error) { + if !sys.Initialized() { + return nil, errServerNotInitialized + } + + d, err := sys.store.GetPolicyDoc(policyName) + if err != nil { + return nil, err + } + + pdata, err := json.Marshal(d.Policy) + if err != nil { + return nil, err + } + + return &madmin.PolicyInfo{ + PolicyName: policyName, + Policy: pdata, + CreateDate: d.CreateDate, + UpdateDate: d.UpdateDate, + }, nil +} + +// ListPolicies - lists all canned policies. +func (sys *IAMSys) ListPolicies(ctx context.Context, bucketName string) (map[string]policy.Policy, error) { + if !sys.Initialized() { + return nil, errServerNotInitialized + } + + return sys.store.ListPolicies(ctx, bucketName) +} + +// ListPolicyDocs - lists all canned policy docs. +func (sys *IAMSys) ListPolicyDocs(ctx context.Context, bucketName string) (map[string]PolicyDoc, error) { + if !sys.Initialized() { + return nil, errServerNotInitialized + } + + return sys.store.ListPolicyDocs(ctx, bucketName) +} + +// SetPolicy - sets a new named policy. +func (sys *IAMSys) SetPolicy(ctx context.Context, policyName string, p policy.Policy) (time.Time, error) { + if !sys.Initialized() { + return time.Time{}, errServerNotInitialized + } + + updatedAt, err := sys.store.SetPolicy(ctx, policyName, p) + if err != nil { + return updatedAt, err + } + + if !sys.HasWatcher() { + // Notify all other MinIO peers to reload policy + for _, nerr := range globalNotificationSys.LoadPolicy(ctx, policyName) { + if nerr.Err != nil { + logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String()) + iamLogIf(ctx, nerr.Err) + } + } + } + return updatedAt, nil +} + +// RevokeTokens - revokes all STS tokens, or those of specified type, for a user +// If `tokenRevokeType` is empty, all tokens are revoked. +func (sys *IAMSys) RevokeTokens(ctx context.Context, accessKey, tokenRevokeType string) error { + if !sys.Initialized() { + return errServerNotInitialized + } + + return sys.store.RevokeTokens(ctx, accessKey, tokenRevokeType) +} + +// DeleteUser - delete user (only for long-term users not STS users). +func (sys *IAMSys) DeleteUser(ctx context.Context, accessKey string, notifyPeers bool) error { + if !sys.Initialized() { + return errServerNotInitialized + } + + if err := sys.store.DeleteUser(ctx, accessKey, regUser); err != nil { + return err + } + + // Notify all other MinIO peers to delete user. + if notifyPeers && !sys.HasWatcher() { + for _, nerr := range globalNotificationSys.DeleteUser(ctx, accessKey) { + if nerr.Err != nil { + logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String()) + iamLogIf(ctx, nerr.Err) + } + } + } + + return nil +} + +// CurrentPolicies - returns comma separated policy string, from +// an input policy after validating if there are any current +// policies which exist on MinIO corresponding to the input. +func (sys *IAMSys) CurrentPolicies(policyName string) string { + if !sys.Initialized() { + return "" + } + + policies, _ := sys.store.MergePolicies(policyName) + return policies +} + +func (sys *IAMSys) notifyForUser(ctx context.Context, accessKey string, isTemp bool) { + // Notify all other MinIO peers to reload user. + if !sys.HasWatcher() { + for _, nerr := range globalNotificationSys.LoadUser(ctx, accessKey, isTemp) { + if nerr.Err != nil { + logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String()) + iamLogIf(ctx, nerr.Err) + } + } + } +} + +// SetTempUser - set temporary user credentials, these credentials have an +// expiry. The permissions for these STS credentials is determined in one of the +// following ways: +// +// - RoleARN - if a role-arn is specified in the request, the STS credential's +// policy is the role's policy. +// +// - inherited from parent - this is the case for AssumeRole API, where the +// parent user is an actual real user with their own (permanent) credentials and +// policy association. +// +// - inherited from "virtual" parent - this is the case for AssumeRoleWithLDAP +// where the parent user is the DN of the actual LDAP user. The parent user +// itself cannot login, but the policy associated with them determines the base +// policy for the STS credential. The policy mapping can be updated by the +// administrator. +// +// - from `Subject.CommonName` field from the STS request for +// AssumeRoleWithCertificate. In this case, the policy for the STS credential +// has the same name as the value of this field. +// +// - from special JWT claim from STS request for AssumeRoleWithOIDC API (when +// not using RoleARN). The claim value can be a string or a list and refers to +// the names of access policies. +// +// For all except the RoleARN case, the implementation is the same - the policy +// for the STS credential is associated with a parent user. For the +// AssumeRoleWithCertificate case, the "virtual" parent user is the value of the +// `Subject.CommonName` field. For the OIDC (without RoleARN) case the "virtual" +// parent is derived as a concatenation of the `sub` and `iss` fields. The +// policies applicable to the STS credential are associated with this "virtual" +// parent. +// +// When a policyName is given to this function, the policy association is +// created and stored in the IAM store. Thus, it should NOT be given for the +// role-arn case (because the role-to-policy mapping is separately stored +// elsewhere), the AssumeRole case (because the parent user is real and their +// policy is associated via policy-set API) and the AssumeRoleWithLDAP case +// (because the policy association is made via policy-set API). +func (sys *IAMSys) SetTempUser(ctx context.Context, accessKey string, cred auth.Credentials, policyName string) (time.Time, error) { + if !sys.Initialized() { + return time.Time{}, errServerNotInitialized + } + + if newGlobalAuthZPluginFn() != nil { + // If OPA is set, we do not need to set a policy mapping. + policyName = "" + } + + updatedAt, err := sys.store.SetTempUser(ctx, accessKey, cred, policyName) + if err != nil { + return time.Time{}, err + } + + sys.notifyForUser(ctx, cred.AccessKey, true) + + return updatedAt, nil +} + +// ListBucketUsers - list all users who can access this 'bucket' +func (sys *IAMSys) ListBucketUsers(ctx context.Context, bucket string) (map[string]madmin.UserInfo, error) { + if !sys.Initialized() { + return nil, errServerNotInitialized + } + + select { + case <-sys.configLoaded: + return sys.store.GetBucketUsers(bucket) + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// ListUsers - list all users. +func (sys *IAMSys) ListUsers(ctx context.Context) (map[string]madmin.UserInfo, error) { + if !sys.Initialized() { + return nil, errServerNotInitialized + } + select { + case <-sys.configLoaded: + return sys.store.GetUsers(), nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// ListLDAPUsers - list LDAP users which has +func (sys *IAMSys) ListLDAPUsers(ctx context.Context) (map[string]madmin.UserInfo, error) { + if !sys.Initialized() { + return nil, errServerNotInitialized + } + + if sys.usersSysType != LDAPUsersSysType { + return nil, errIAMActionNotAllowed + } + + select { + case <-sys.configLoaded: + stsMap, err := sys.store.GetAllSTSUserMappings(sys.LDAPConfig.IsLDAPUserDN) + if err != nil { + return nil, err + } + ldapUsers := make(map[string]madmin.UserInfo, len(stsMap)) + for user, policy := range stsMap { + ldapUsers[user] = madmin.UserInfo{ + PolicyName: policy, + Status: statusEnabled, + } + } + return ldapUsers, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +type cleanEntitiesQuery struct { + Users map[string]set.StringSet + Groups set.StringSet + Policies set.StringSet +} + +// createCleanEntitiesQuery - maps users to their groups and normalizes user or group DNs if ldap. +func (sys *IAMSys) createCleanEntitiesQuery(q madmin.PolicyEntitiesQuery, ldap bool) cleanEntitiesQuery { + cleanQ := cleanEntitiesQuery{ + Users: make(map[string]set.StringSet), + Groups: set.CreateStringSet(q.Groups...), + Policies: set.CreateStringSet(q.Policy...), + } + + if ldap { + // Validate and normalize users, then fetch and normalize their groups + // Also include unvalidated users for backward compatibility. + for _, user := range q.Users { + lookupRes, actualGroups, _ := sys.LDAPConfig.GetValidatedDNWithGroups(user) + if lookupRes != nil { + groupSet := set.CreateStringSet(actualGroups...) + + // duplicates can be overwritten, fetched groups should be identical. + cleanQ.Users[lookupRes.NormDN] = groupSet + } + // Search for non-normalized DN as well for backward compatibility. + if _, ok := cleanQ.Users[user]; !ok { + cleanQ.Users[user] = nil + } + } + + // Validate and normalize groups. + for _, group := range q.Groups { + lookupRes, underDN, _ := sys.LDAPConfig.GetValidatedGroupDN(nil, group) + if lookupRes != nil && underDN { + cleanQ.Groups.Add(lookupRes.NormDN) + } + } + } else { + for _, user := range q.Users { + info, err := sys.store.GetUserInfo(user) + var groupSet set.StringSet + if err == nil { + groupSet = set.CreateStringSet(info.MemberOf...) + } + cleanQ.Users[user] = groupSet + } + } + return cleanQ +} + +// QueryLDAPPolicyEntities - queries policy associations for LDAP users/groups/policies. +func (sys *IAMSys) QueryLDAPPolicyEntities(ctx context.Context, q madmin.PolicyEntitiesQuery) (*madmin.PolicyEntitiesResult, error) { + if !sys.Initialized() { + return nil, errServerNotInitialized + } + + if !sys.LDAPConfig.Enabled() { + return nil, errIAMActionNotAllowed + } + + select { + case <-sys.configLoaded: + cleanQuery := sys.createCleanEntitiesQuery(q, true) + pe := sys.store.ListPolicyMappings(cleanQuery, sys.LDAPConfig.IsLDAPUserDN, sys.LDAPConfig.IsLDAPGroupDN, sys.LDAPConfig.DecodeDN) + pe.Timestamp = UTCNow() + return &pe, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// IsTempUser - returns if given key is a temporary user and parent user. +func (sys *IAMSys) IsTempUser(name string) (bool, string, error) { + if !sys.Initialized() { + return false, "", errServerNotInitialized + } + + u, found := sys.store.GetUser(name) + if !found { + return false, "", errNoSuchUser + } + cred := u.Credentials + if cred.IsTemp() { + return true, cred.ParentUser, nil + } + + return false, "", nil +} + +// IsServiceAccount - returns if given key is a service account +func (sys *IAMSys) IsServiceAccount(name string) (bool, string, error) { + if !sys.Initialized() { + return false, "", errServerNotInitialized + } + + u, found := sys.store.GetUser(name) + if !found { + return false, "", errNoSuchUser + } + cred := u.Credentials + if cred.IsServiceAccount() { + return true, cred.ParentUser, nil + } + + return false, "", nil +} + +// GetUserInfo - get info on a user. +func (sys *IAMSys) GetUserInfo(ctx context.Context, name string) (u madmin.UserInfo, err error) { + if !sys.Initialized() { + return u, errServerNotInitialized + } + + loadUserCalled := false + select { + case <-sys.configLoaded: + default: + sys.store.LoadUser(ctx, name) + loadUserCalled = true + } + + userInfo, err := sys.store.GetUserInfo(name) + if err == errNoSuchUser && !loadUserCalled { + sys.store.LoadUser(ctx, name) + userInfo, err = sys.store.GetUserInfo(name) + } + return userInfo, err +} + +// QueryPolicyEntities - queries policy associations for builtin users/groups/policies. +func (sys *IAMSys) QueryPolicyEntities(ctx context.Context, q madmin.PolicyEntitiesQuery) (*madmin.PolicyEntitiesResult, error) { + if !sys.Initialized() { + return nil, errServerNotInitialized + } + + select { + case <-sys.configLoaded: + cleanQuery := sys.createCleanEntitiesQuery(q, false) + var userPredicate, groupPredicate func(string) bool + if sys.LDAPConfig.Enabled() { + userPredicate = func(s string) bool { + return !sys.LDAPConfig.IsLDAPUserDN(s) + } + groupPredicate = func(s string) bool { + return !sys.LDAPConfig.IsLDAPGroupDN(s) + } + } + pe := sys.store.ListPolicyMappings(cleanQuery, userPredicate, groupPredicate, nil) + pe.Timestamp = UTCNow() + return &pe, nil + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// SetUserStatus - sets current user status, supports disabled or enabled. +func (sys *IAMSys) SetUserStatus(ctx context.Context, accessKey string, status madmin.AccountStatus) (updatedAt time.Time, err error) { + if !sys.Initialized() { + return updatedAt, errServerNotInitialized + } + + if sys.usersSysType != MinIOUsersSysType { + return updatedAt, errIAMActionNotAllowed + } + + updatedAt, err = sys.store.SetUserStatus(ctx, accessKey, status) + if err != nil { + return updatedAt, err + } + + sys.notifyForUser(ctx, accessKey, false) + return updatedAt, nil +} + +func (sys *IAMSys) notifyForServiceAccount(ctx context.Context, accessKey string) { + // Notify all other Minio peers to reload the service account + if !sys.HasWatcher() { + for _, nerr := range globalNotificationSys.LoadServiceAccount(ctx, accessKey) { + if nerr.Err != nil { + logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String()) + iamLogIf(ctx, nerr.Err) + } + } + } +} + +type newServiceAccountOpts struct { + sessionPolicy *policy.Policy + accessKey string + secretKey string + name, description string + expiration *time.Time + allowSiteReplicatorAccount bool // allow creating internal service account for site-replication. + + claims map[string]any +} + +// NewServiceAccount - create a new service account +func (sys *IAMSys) NewServiceAccount(ctx context.Context, parentUser string, groups []string, opts newServiceAccountOpts) (auth.Credentials, time.Time, error) { + if !sys.Initialized() { + return auth.Credentials{}, time.Time{}, errServerNotInitialized + } + + if parentUser == "" { + return auth.Credentials{}, time.Time{}, errInvalidArgument + } + + if len(opts.accessKey) > 0 && len(opts.secretKey) == 0 { + return auth.Credentials{}, time.Time{}, auth.ErrNoSecretKeyWithAccessKey + } + if len(opts.secretKey) > 0 && len(opts.accessKey) == 0 { + return auth.Credentials{}, time.Time{}, auth.ErrNoAccessKeyWithSecretKey + } + + var policyBuf []byte + if opts.sessionPolicy != nil { + err := opts.sessionPolicy.Validate() + if err != nil { + return auth.Credentials{}, time.Time{}, err + } + policyBuf, err = json.Marshal(opts.sessionPolicy) + if err != nil { + return auth.Credentials{}, time.Time{}, err + } + if len(policyBuf) > maxSVCSessionPolicySize { + return auth.Credentials{}, time.Time{}, errSessionPolicyTooLarge + } + } + + // found newly requested service account, to be same as + // parentUser, reject such operations. + if parentUser == opts.accessKey { + return auth.Credentials{}, time.Time{}, errIAMActionNotAllowed + } + if siteReplicatorSvcAcc == opts.accessKey && !opts.allowSiteReplicatorAccount { + return auth.Credentials{}, time.Time{}, errIAMActionNotAllowed + } + m := make(map[string]any) + m[parentClaim] = parentUser + + if len(policyBuf) > 0 { + m[policy.SessionPolicyName] = base64.StdEncoding.EncodeToString(policyBuf) + m[iamPolicyClaimNameSA()] = embeddedPolicyType + } else { + m[iamPolicyClaimNameSA()] = inheritedPolicyType + } + + // Add all the necessary claims for the service account. + for k, v := range opts.claims { + _, ok := m[k] + if !ok { + m[k] = v + } + } + + var accessKey, secretKey string + var err error + if len(opts.accessKey) > 0 || len(opts.secretKey) > 0 { + accessKey, secretKey = opts.accessKey, opts.secretKey + } else { + accessKey, secretKey, err = auth.GenerateCredentials() + if err != nil { + return auth.Credentials{}, time.Time{}, err + } + } + cred, err := auth.CreateNewCredentialsWithMetadata(accessKey, secretKey, m, secretKey) + if err != nil { + return auth.Credentials{}, time.Time{}, err + } + cred.ParentUser = parentUser + cred.Groups = groups + cred.Status = string(auth.AccountOn) + cred.Name = opts.name + cred.Description = opts.description + + if opts.expiration != nil { + expirationInUTC := opts.expiration.UTC() + if err := validateSvcExpirationInUTC(expirationInUTC); err != nil { + return auth.Credentials{}, time.Time{}, err + } + cred.Expiration = expirationInUTC + } + + updatedAt, err := sys.store.AddServiceAccount(ctx, cred) + if err != nil { + return auth.Credentials{}, time.Time{}, err + } + + sys.notifyForServiceAccount(ctx, cred.AccessKey) + return cred, updatedAt, nil +} + +type updateServiceAccountOpts struct { + sessionPolicy *policy.Policy + secretKey string + status string + name, description string + expiration *time.Time +} + +// UpdateServiceAccount - edit a service account +func (sys *IAMSys) UpdateServiceAccount(ctx context.Context, accessKey string, opts updateServiceAccountOpts) (updatedAt time.Time, err error) { + if !sys.Initialized() { + return updatedAt, errServerNotInitialized + } + + updatedAt, err = sys.store.UpdateServiceAccount(ctx, accessKey, opts) + if err != nil { + return updatedAt, err + } + + sys.notifyForServiceAccount(ctx, accessKey) + return updatedAt, nil +} + +// ListServiceAccounts - lists all service accounts associated to a specific user +func (sys *IAMSys) ListServiceAccounts(ctx context.Context, accessKey string) ([]auth.Credentials, error) { + if !sys.Initialized() { + return nil, errServerNotInitialized + } + + select { + case <-sys.configLoaded: + return sys.store.ListServiceAccounts(ctx, accessKey) + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// ListTempAccounts - lists all temporary service accounts associated to a specific user +func (sys *IAMSys) ListTempAccounts(ctx context.Context, accessKey string) ([]UserIdentity, error) { + if !sys.Initialized() { + return nil, errServerNotInitialized + } + + select { + case <-sys.configLoaded: + return sys.store.ListTempAccounts(ctx, accessKey) + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// ListSTSAccounts - lists all STS accounts associated to a specific user +func (sys *IAMSys) ListSTSAccounts(ctx context.Context, accessKey string) ([]auth.Credentials, error) { + if !sys.Initialized() { + return nil, errServerNotInitialized + } + + select { + case <-sys.configLoaded: + return sys.store.ListSTSAccounts(ctx, accessKey) + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// ListAllAccessKeys - lists all access keys (sts/service accounts) +func (sys *IAMSys) ListAllAccessKeys(ctx context.Context) ([]auth.Credentials, error) { + if !sys.Initialized() { + return nil, errServerNotInitialized + } + + select { + case <-sys.configLoaded: + return sys.store.ListAccessKeys(ctx) + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// GetServiceAccount - wrapper method to get information about a service account +func (sys *IAMSys) GetServiceAccount(ctx context.Context, accessKey string) (auth.Credentials, *policy.Policy, error) { + sa, embeddedPolicy, err := sys.getServiceAccount(ctx, accessKey) + if err != nil { + return auth.Credentials{}, nil, err + } + // Hide secret & session keys + sa.Credentials.SecretKey = "" + sa.Credentials.SessionToken = "" + return sa.Credentials, embeddedPolicy, nil +} + +func (sys *IAMSys) getServiceAccount(ctx context.Context, accessKey string) (UserIdentity, *policy.Policy, error) { + sa, jwtClaims, err := sys.getAccountWithClaims(ctx, accessKey) + if err != nil { + if err == errNoSuchAccount { + return UserIdentity{}, nil, errNoSuchServiceAccount + } + return UserIdentity{}, nil, err + } + if !sa.Credentials.IsServiceAccount() { + return UserIdentity{}, nil, errNoSuchServiceAccount + } + + var embeddedPolicy *policy.Policy + + pt, ptok := jwtClaims.Lookup(iamPolicyClaimNameSA()) + sp, spok := jwtClaims.Lookup(policy.SessionPolicyName) + if ptok && spok && pt == embeddedPolicyType { + policyBytes, err := base64.StdEncoding.DecodeString(sp) + if err != nil { + return UserIdentity{}, nil, err + } + embeddedPolicy, err = policy.ParseConfig(bytes.NewReader(policyBytes)) + if err != nil { + return UserIdentity{}, nil, err + } + } + + return sa, embeddedPolicy, nil +} + +// GetTemporaryAccount - wrapper method to get information about a temporary account +func (sys *IAMSys) GetTemporaryAccount(ctx context.Context, accessKey string) (auth.Credentials, *policy.Policy, error) { + if !sys.Initialized() { + return auth.Credentials{}, nil, errServerNotInitialized + } + tmpAcc, embeddedPolicy, err := sys.getTempAccount(ctx, accessKey) + if err != nil { + if err == errNoSuchTempAccount { + sys.store.LoadUser(ctx, accessKey) + tmpAcc, embeddedPolicy, err = sys.getTempAccount(ctx, accessKey) + } + if err != nil { + return auth.Credentials{}, nil, err + } + } + // Hide secret & session keys + tmpAcc.Credentials.SecretKey = "" + tmpAcc.Credentials.SessionToken = "" + return tmpAcc.Credentials, embeddedPolicy, nil +} + +func (sys *IAMSys) getTempAccount(ctx context.Context, accessKey string) (UserIdentity, *policy.Policy, error) { + tmpAcc, claims, err := sys.getAccountWithClaims(ctx, accessKey) + if err != nil { + if err == errNoSuchAccount { + return UserIdentity{}, nil, errNoSuchTempAccount + } + return UserIdentity{}, nil, err + } + if !tmpAcc.Credentials.IsTemp() { + return UserIdentity{}, nil, errNoSuchTempAccount + } + + var embeddedPolicy *policy.Policy + + sp, spok := claims.Lookup(policy.SessionPolicyName) + if spok { + policyBytes, err := base64.StdEncoding.DecodeString(sp) + if err != nil { + return UserIdentity{}, nil, err + } + embeddedPolicy, err = policy.ParseConfig(bytes.NewReader(policyBytes)) + if err != nil { + return UserIdentity{}, nil, err + } + } + + return tmpAcc, embeddedPolicy, nil +} + +// getAccountWithClaims - gets information about an account with claims +func (sys *IAMSys) getAccountWithClaims(ctx context.Context, accessKey string) (UserIdentity, *jwt.MapClaims, error) { + if !sys.Initialized() { + return UserIdentity{}, nil, errServerNotInitialized + } + + acc, ok := sys.store.GetUser(accessKey) + if !ok { + return UserIdentity{}, nil, errNoSuchAccount + } + + jwtClaims, err := extractJWTClaims(acc) + if err != nil { + return UserIdentity{}, nil, err + } + + return acc, jwtClaims, nil +} + +// GetClaimsForSvcAcc - gets the claims associated with the service account. +func (sys *IAMSys) GetClaimsForSvcAcc(ctx context.Context, accessKey string) (map[string]any, error) { + if !sys.Initialized() { + return nil, errServerNotInitialized + } + + sa, ok := sys.store.GetUser(accessKey) + if !ok || !sa.Credentials.IsServiceAccount() { + return nil, errNoSuchServiceAccount + } + + jwtClaims, err := extractJWTClaims(sa) + if err != nil { + return nil, err + } + + return jwtClaims.Map(), nil +} + +// DeleteServiceAccount - delete a service account +func (sys *IAMSys) DeleteServiceAccount(ctx context.Context, accessKey string, notifyPeers bool) error { + if !sys.Initialized() { + return errServerNotInitialized + } + + sa, ok := sys.store.GetUser(accessKey) + if !ok || !sa.Credentials.IsServiceAccount() { + return nil + } + + if err := sys.store.DeleteUser(ctx, accessKey, svcUser); err != nil { + return err + } + + if notifyPeers && !sys.HasWatcher() { + for _, nerr := range globalNotificationSys.DeleteServiceAccount(ctx, accessKey) { + if nerr.Err != nil { + logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String()) + iamLogIf(ctx, nerr.Err) + } + } + } + + return nil +} + +// CreateUser - create new user credentials and policy, if user already exists +// they shall be rewritten with new inputs. +func (sys *IAMSys) CreateUser(ctx context.Context, accessKey string, ureq madmin.AddOrUpdateUserReq) (updatedAt time.Time, err error) { + if !sys.Initialized() { + return updatedAt, errServerNotInitialized + } + + if !auth.IsAccessKeyValid(accessKey) { + return updatedAt, auth.ErrInvalidAccessKeyLength + } + + if auth.ContainsReservedChars(accessKey) { + return updatedAt, auth.ErrContainsReservedChars + } + + if !auth.IsSecretKeyValid(ureq.SecretKey) { + return updatedAt, auth.ErrInvalidSecretKeyLength + } + + updatedAt, err = sys.store.AddUser(ctx, accessKey, ureq) + if err != nil { + return updatedAt, err + } + + sys.notifyForUser(ctx, accessKey, false) + return updatedAt, nil +} + +// SetUserSecretKey - sets user secret key +func (sys *IAMSys) SetUserSecretKey(ctx context.Context, accessKey string, secretKey string) error { + if !sys.Initialized() { + return errServerNotInitialized + } + + if sys.usersSysType != MinIOUsersSysType { + return errIAMActionNotAllowed + } + + if !auth.IsAccessKeyValid(accessKey) { + return auth.ErrInvalidAccessKeyLength + } + + if !auth.IsSecretKeyValid(secretKey) { + return auth.ErrInvalidSecretKeyLength + } + + return sys.store.UpdateUserSecretKey(ctx, accessKey, secretKey) +} + +// purgeExpiredCredentialsForExternalSSO - validates if local credentials are still valid +// by checking remote IDP if the relevant users are still active and present. +func (sys *IAMSys) purgeExpiredCredentialsForExternalSSO(ctx context.Context) { + parentUsersMap := sys.store.GetAllParentUsers() + var expiredUsers []string + for parentUser, puInfo := range parentUsersMap { + // There are multiple role ARNs for parent user only when there + // are multiple openid provider configurations with the same ID + // provider. We lookup the provider associated with some one of + // the roleARNs to check if the user still exists. If they don't + // we can safely remove credentials for this parent user + // associated with any of the provider configurations. + // + // If there is no roleARN mapped to the user, the user may be + // coming from a policy claim based openid provider. + roleArns := puInfo.roleArns.ToSlice() + var roleArn string + if len(roleArns) == 0 { + iamLogIf(GlobalContext, + fmt.Errorf("parentUser: %s had no roleArns mapped!", parentUser)) + continue + } + roleArn = roleArns[0] + u, err := sys.OpenIDConfig.LookupUser(roleArn, puInfo.subClaimValue) + if err != nil { + iamLogIf(GlobalContext, err) + continue + } + // If user is set to "disabled", we will remove them + // subsequently. + if !u.Enabled { + expiredUsers = append(expiredUsers, parentUser) + } + } + + // We ignore any errors + _ = sys.store.DeleteUsers(ctx, expiredUsers) +} + +// purgeExpiredCredentialsForLDAP - validates if local credentials are still +// valid by checking LDAP server if the relevant users are still present. +func (sys *IAMSys) purgeExpiredCredentialsForLDAP(ctx context.Context) { + parentUsers := sys.store.GetAllParentUsers() + var allDistNames []string + for parentUser, info := range parentUsers { + if !sys.LDAPConfig.IsLDAPUserDN(parentUser) { + continue + } + + if info.subClaimValue != "" { + // we need to ask LDAP about the actual user DN not normalized DN. + allDistNames = append(allDistNames, info.subClaimValue) + } else { + allDistNames = append(allDistNames, parentUser) + } + } + + expiredUsers, err := sys.LDAPConfig.GetNonEligibleUserDistNames(allDistNames) + if err != nil { + // Log and return on error - perhaps it'll work the next time. + iamLogIf(GlobalContext, err) + return + } + + // We ignore any errors + _ = sys.store.DeleteUsers(ctx, expiredUsers) +} + +// updateGroupMembershipsForLDAP - updates the list of groups associated with the credential. +func (sys *IAMSys) updateGroupMembershipsForLDAP(ctx context.Context) { + // 1. Collect all LDAP users with active creds. + allCreds := sys.store.GetSTSAndServiceAccounts() + // List of unique LDAP (parent) user DNs that have active creds + var parentUserActualDNList []string + // Map of LDAP user (internal representation) to list of active credential objects + parentUserToCredsMap := make(map[string][]auth.Credentials) + // DN to ldap username mapping for each LDAP user + actualDNToLDAPUsernameMap := make(map[string]string) + // External (actual) LDAP DN to internal normalized representation + actualDNToParentUserMap := make(map[string]string) + for _, cred := range allCreds { + // Expired credentials don't need parent user updates. + if cred.IsExpired() { + continue + } + + if !sys.LDAPConfig.IsLDAPUserDN(cred.ParentUser) { + continue + } + + // Check if this is the first time we are + // encountering this LDAP user. + if _, ok := parentUserToCredsMap[cred.ParentUser]; !ok { + // Try to find the ldapUsername for this + // parentUser by extracting JWT claims + var ( + jwtClaims *jwt.MapClaims + err error + ) + + if cred.SessionToken == "" { + continue + } + + if cred.IsServiceAccount() { + jwtClaims, err = auth.ExtractClaims(cred.SessionToken, cred.SecretKey) + if err != nil { + jwtClaims, err = auth.ExtractClaims(cred.SessionToken, globalActiveCred.SecretKey) + } + } else { + var secretKey string + secretKey, err = getTokenSigningKey() + if err != nil { + continue + } + jwtClaims, err = auth.ExtractClaims(cred.SessionToken, secretKey) + } + if err != nil { + // skip this cred - session token seems invalid + continue + } + + ldapUsername, okUserN := jwtClaims.Lookup(ldapUserN) + ldapActualDN, okDN := jwtClaims.Lookup(ldapActualUser) + if !okUserN || !okDN { + // skip this cred - we dont have the + // username info needed + continue + } + + // Collect each new cred.ParentUser into parentUsers + parentUserActualDNList = append(parentUserActualDNList, ldapActualDN) + + // Update the ldapUsernameMap + actualDNToLDAPUsernameMap[ldapActualDN] = ldapUsername + + // Update the actualDNToParentUserMap + actualDNToParentUserMap[ldapActualDN] = cred.ParentUser + } + parentUserToCredsMap[cred.ParentUser] = append(parentUserToCredsMap[cred.ParentUser], cred) + } + + // 2. Query LDAP server for groups of the LDAP users collected. + updatedGroups, err := sys.LDAPConfig.LookupGroupMemberships(parentUserActualDNList, actualDNToLDAPUsernameMap) + if err != nil { + // Log and return on error - perhaps it'll work the next time. + iamLogIf(GlobalContext, err) + return + } + + // 3. Update creds for those users whose groups are changed + for _, parentActualDN := range parentUserActualDNList { + currGroupsSet := updatedGroups[parentActualDN] + parentUser := actualDNToParentUserMap[parentActualDN] + currGroups := currGroupsSet.ToSlice() + for _, cred := range parentUserToCredsMap[parentUser] { + gSet := set.CreateStringSet(cred.Groups...) + if gSet.Equals(currGroupsSet) { + // No change to groups memberships for this + // credential. + continue + } + + // Expired credentials don't need group membership updates. + if cred.IsExpired() { + continue + } + + cred.Groups = currGroups + if err := sys.store.UpdateUserIdentity(ctx, cred); err != nil { + // Log and continue error - perhaps it'll work the next time. + iamLogIf(GlobalContext, err) + } + } + } +} + +// NormalizeLDAPAccessKeypairs - normalize the access key pairs (service +// accounts) for LDAP users. This normalizes the parent user and the group names +// whenever the parent user parses validly as a DN. +func (sys *IAMSys) NormalizeLDAPAccessKeypairs(ctx context.Context, accessKeyMap map[string]madmin.SRSvcAccCreate, +) (skippedAccessKeys []string, err error) { + conn, err := sys.LDAPConfig.LDAP.Connect() + if err != nil { + return skippedAccessKeys, err + } + defer conn.Close() + + // Bind to the lookup user account + if err = sys.LDAPConfig.LDAP.LookupBind(conn); err != nil { + return skippedAccessKeys, err + } + + var collectedErrors []error + updatedKeysMap := make(map[string]madmin.SRSvcAccCreate) + for ak, createReq := range accessKeyMap { + parent := createReq.Parent + groups := createReq.Groups + + _, err := ldap.NormalizeDN(parent) + if err != nil { + // not a valid DN, ignore. + continue + } + + hasDiff := false + + // For the parent value, we require that the parent exists in the LDAP + // server and is under a configured base DN. + validatedParent, isUnderBaseDN, err := sys.LDAPConfig.GetValidatedUserDN(conn, parent) + if err != nil { + collectedErrors = append(collectedErrors, fmt.Errorf("could not validate parent exists in LDAP directory: %w", err)) + continue + } + if validatedParent == nil || !isUnderBaseDN { + skippedAccessKeys = append(skippedAccessKeys, ak) + continue + } + + if validatedParent.NormDN != parent { + hasDiff = true + } + + var normalizedGroups []string + for _, group := range groups { + // For a group, we store the normalized DN even if it not under a + // configured base DN. + validatedGroup, _, err := sys.LDAPConfig.GetValidatedGroupDN(conn, group) + if err != nil { + collectedErrors = append(collectedErrors, fmt.Errorf("could not validate group exists in LDAP directory: %w", err)) + continue + } + if validatedGroup == nil { + // DN group was not found in the LDAP directory for access-key + continue + } + + if validatedGroup.NormDN != group { + hasDiff = true + } + normalizedGroups = append(normalizedGroups, validatedGroup.NormDN) + } + + if hasDiff { + updatedCreateReq := createReq + updatedCreateReq.Parent = validatedParent.NormDN + updatedCreateReq.Groups = normalizedGroups + + updatedKeysMap[ak] = updatedCreateReq + } + } + + // if there are any errors, return a collected error. + if len(collectedErrors) > 0 { + return skippedAccessKeys, fmt.Errorf("errors validating LDAP DN: %w", errors.Join(collectedErrors...)) + } + + // Replace the map values with the updated ones + maps.Copy(accessKeyMap, updatedKeysMap) + + return skippedAccessKeys, nil +} + +func (sys *IAMSys) getStoredLDAPPolicyMappingKeys(ctx context.Context, isGroup bool) set.StringSet { + entityKeysInStorage := set.NewStringSet() + cache := sys.store.rlock() + defer sys.store.runlock() + cachedPolicyMap := cache.iamSTSPolicyMap + if isGroup { + cachedPolicyMap = cache.iamGroupPolicyMap + } + cachedPolicyMap.Range(func(k string, v MappedPolicy) bool { + entityKeysInStorage.Add(k) + return true + }) + + return entityKeysInStorage +} + +// NormalizeLDAPMappingImport - validates the LDAP policy mappings. Keys in the +// given map may not correspond to LDAP DNs - these keys are ignored. +// +// For validated mappings, it updates the key in the given map to be in +// normalized form. +func (sys *IAMSys) NormalizeLDAPMappingImport(ctx context.Context, isGroup bool, + policyMap map[string]MappedPolicy, +) ([]string, error) { + conn, err := sys.LDAPConfig.LDAP.Connect() + if err != nil { + return []string{}, err + } + defer conn.Close() + + // Bind to the lookup user account + if err = sys.LDAPConfig.LDAP.LookupBind(conn); err != nil { + return []string{}, err + } + + // We map keys that correspond to LDAP DNs and validate that they exist in + // the LDAP server. + dnValidator := sys.LDAPConfig.GetValidatedUserDN + if isGroup { + dnValidator = sys.LDAPConfig.GetValidatedGroupDN + } + + // map of normalized DN keys to original keys. + normalizedDNKeysMap := make(map[string][]string) + var collectedErrors []error + var skipped []string + for k := range policyMap { + _, err := ldap.NormalizeDN(k) + if err != nil { + // not a valid DN, ignore. + continue + } + validatedDN, underBaseDN, err := dnValidator(conn, k) + if err != nil { + collectedErrors = append(collectedErrors, fmt.Errorf("could not validate `%s` exists in LDAP directory: %w", k, err)) + continue + } + if validatedDN == nil || !underBaseDN { + skipped = append(skipped, k) + continue + } + + if validatedDN.NormDN != k { + normalizedDNKeysMap[validatedDN.NormDN] = append(normalizedDNKeysMap[validatedDN.NormDN], k) + } + } + + // if there are any errors, return a collected error. + if len(collectedErrors) > 0 { + return []string{}, fmt.Errorf("errors validating LDAP DN: %w", errors.Join(collectedErrors...)) + } + + entityKeysInStorage := sys.getStoredLDAPPolicyMappingKeys(ctx, isGroup) + + for normKey, origKeys := range normalizedDNKeysMap { + if len(origKeys) > 1 { + // If there are multiple DN keys that normalize to the same value, + // check if the policy mappings are equal, if they are we don't need + // to return an error. + policiesDiffer := false + firstMappedPolicies := policyMap[origKeys[0]].policySet() + for i := 1; i < len(origKeys); i++ { + otherMappedPolicies := policyMap[origKeys[i]].policySet() + if !firstMappedPolicies.Equals(otherMappedPolicies) { + policiesDiffer = true + break + } + } + + if policiesDiffer { + return []string{}, fmt.Errorf("multiple DNs map to the same LDAP DN[%s]: %v; please remove DNs that are not needed", + normKey, origKeys) + } + + if len(origKeys[1:]) > 0 { + // Log that extra DN mappings will not be imported. + iamLogEvent(ctx, "import-ldap-normalize: extraneous DN mappings found for LDAP DN[%s]: %v will not be imported", origKeys[0], origKeys[1:]) + } + + // Policies mapped to the DN's are the same, so we remove the extra + // ones from the map. + for i := 1; i < len(origKeys); i++ { + delete(policyMap, origKeys[i]) + + // Remove the mapping from storage by setting the policy to "". + if entityKeysInStorage.Contains(origKeys[i]) { + // Ignore any deletion error. + _, delErr := sys.PolicyDBSet(ctx, origKeys[i], "", stsUser, isGroup) + if delErr != nil { + logErr := fmt.Errorf("failed to delete extraneous LDAP DN mapping for `%s`: %w", origKeys[i], delErr) + iamLogIf(ctx, logErr) + } + } + } + } + + // Replacing origKeys[0] with normKey in the policyMap + + // len(origKeys) is always > 0, so here len(origKeys) == 1 + mappingValue := policyMap[origKeys[0]] + delete(policyMap, origKeys[0]) + policyMap[normKey] = mappingValue + iamLogEvent(ctx, "import-ldap-normalize: normalized LDAP DN mapping from `%s` to `%s`", origKeys[0], normKey) + + // Remove the mapping from storage by setting the policy to "". + if entityKeysInStorage.Contains(origKeys[0]) { + // Ignore any deletion error. + _, delErr := sys.PolicyDBSet(ctx, origKeys[0], "", stsUser, isGroup) + if delErr != nil { + logErr := fmt.Errorf("failed to delete extraneous LDAP DN mapping for `%s`: %w", origKeys[0], delErr) + iamLogIf(ctx, logErr) + } + } + } + return skipped, nil +} + +// CheckKey validates the incoming accessKey +func (sys *IAMSys) CheckKey(ctx context.Context, accessKey string) (u UserIdentity, ok bool, err error) { + if !sys.Initialized() { + return u, false, nil + } + + if accessKey == globalActiveCred.AccessKey { + return newUserIdentity(globalActiveCred), true, nil + } + + loadUserCalled := false + select { + case <-sys.configLoaded: + default: + err = sys.store.LoadUser(ctx, accessKey) + loadUserCalled = true + } + + u, ok = sys.store.GetUser(accessKey) + if !ok && !loadUserCalled { + err = sys.store.LoadUser(ctx, accessKey) + loadUserCalled = true + + u, ok = sys.store.GetUser(accessKey) + } + + if !ok && loadUserCalled && err != nil { + iamLogOnceIf(ctx, err, accessKey) + + // return 503 to application + return u, false, errIAMNotInitialized + } + + return u, ok && u.Credentials.IsValid(), nil +} + +// GetUser - get user credentials +func (sys *IAMSys) GetUser(ctx context.Context, accessKey string) (u UserIdentity, ok bool) { + u, ok, _ = sys.CheckKey(ctx, accessKey) + return u, ok +} + +// Notify all other MinIO peers to load group. +func (sys *IAMSys) notifyForGroup(ctx context.Context, group string) { + if !sys.HasWatcher() { + for _, nerr := range globalNotificationSys.LoadGroup(ctx, group) { + if nerr.Err != nil { + logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String()) + iamLogIf(ctx, nerr.Err) + } + } + } +} + +// AddUsersToGroup - adds users to a group, creating the group if +// needed. No error if user(s) already are in the group. +func (sys *IAMSys) AddUsersToGroup(ctx context.Context, group string, members []string) (updatedAt time.Time, err error) { + if !sys.Initialized() { + return updatedAt, errServerNotInitialized + } + + if auth.ContainsReservedChars(group) { + return updatedAt, errGroupNameContainsReservedChars + } + + updatedAt, err = sys.store.AddUsersToGroup(ctx, group, members) + if err != nil { + return updatedAt, err + } + + sys.notifyForGroup(ctx, group) + return updatedAt, nil +} + +// RemoveUsersFromGroup - remove users from group. If no users are +// given, and the group is empty, deletes the group as well. +func (sys *IAMSys) RemoveUsersFromGroup(ctx context.Context, group string, members []string) (updatedAt time.Time, err error) { + if !sys.Initialized() { + return updatedAt, errServerNotInitialized + } + + if sys.usersSysType != MinIOUsersSysType { + return updatedAt, errIAMActionNotAllowed + } + + updatedAt, err = sys.store.RemoveUsersFromGroup(ctx, group, members) + if err != nil { + return updatedAt, err + } + + sys.notifyForGroup(ctx, group) + return updatedAt, nil +} + +// SetGroupStatus - enable/disabled a group +func (sys *IAMSys) SetGroupStatus(ctx context.Context, group string, enabled bool) (updatedAt time.Time, err error) { + if !sys.Initialized() { + return updatedAt, errServerNotInitialized + } + + if sys.usersSysType != MinIOUsersSysType { + return updatedAt, errIAMActionNotAllowed + } + + updatedAt, err = sys.store.SetGroupStatus(ctx, group, enabled) + if err != nil { + return updatedAt, err + } + + sys.notifyForGroup(ctx, group) + return updatedAt, nil +} + +// GetGroupDescription - builds up group description +func (sys *IAMSys) GetGroupDescription(group string) (gd madmin.GroupDesc, err error) { + if !sys.Initialized() { + return gd, errServerNotInitialized + } + + return sys.store.GetGroupDescription(group) +} + +// ListGroups - lists groups. +func (sys *IAMSys) ListGroups(ctx context.Context) (r []string, err error) { + if !sys.Initialized() { + return r, errServerNotInitialized + } + + select { + case <-sys.configLoaded: + return sys.store.ListGroups(ctx) + case <-ctx.Done(): + return nil, ctx.Err() + } +} + +// PolicyDBSet - sets a policy for a user or group in the PolicyDB. This does +// not validate if the user/group exists - that is the responsibility of the +// caller. +func (sys *IAMSys) PolicyDBSet(ctx context.Context, name, policy string, userType IAMUserType, isGroup bool) (updatedAt time.Time, err error) { + if !sys.Initialized() { + return updatedAt, errServerNotInitialized + } + + updatedAt, err = sys.store.PolicyDBSet(ctx, name, policy, userType, isGroup) + if err != nil { + return updatedAt, err + } + + // Notify all other MinIO peers to reload policy + if !sys.HasWatcher() { + for _, nerr := range globalNotificationSys.LoadPolicyMapping(ctx, name, userType, isGroup) { + if nerr.Err != nil { + logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String()) + iamLogIf(ctx, nerr.Err) + } + } + } + + return updatedAt, nil +} + +// PolicyDBUpdateBuiltin - adds or removes policies from a user or a group +// verified to be an internal IDP user. +func (sys *IAMSys) PolicyDBUpdateBuiltin(ctx context.Context, isAttach bool, + r madmin.PolicyAssociationReq, +) (updatedAt time.Time, addedOrRemoved, effectivePolicies []string, err error) { + if !sys.Initialized() { + err = errServerNotInitialized + return updatedAt, addedOrRemoved, effectivePolicies, err + } + + userOrGroup := r.User + var isGroup bool + if userOrGroup == "" { + isGroup = true + userOrGroup = r.Group + } + + if isGroup { + _, err = sys.GetGroupDescription(userOrGroup) + if err != nil { + return updatedAt, addedOrRemoved, effectivePolicies, err + } + } else { + var isTemp bool + isTemp, _, err = sys.IsTempUser(userOrGroup) + if err != nil && err != errNoSuchUser { + return updatedAt, addedOrRemoved, effectivePolicies, err + } + if isTemp { + err = errIAMActionNotAllowed + return updatedAt, addedOrRemoved, effectivePolicies, err + } + + // When the user is root credential you are not allowed to + // add policies for root user. + if userOrGroup == globalActiveCred.AccessKey { + err = errIAMActionNotAllowed + return updatedAt, addedOrRemoved, effectivePolicies, err + } + + // Validate that user exists. + var userExists bool + _, userExists = sys.GetUser(ctx, userOrGroup) + if !userExists { + err = errNoSuchUser + return updatedAt, addedOrRemoved, effectivePolicies, err + } + } + + updatedAt, addedOrRemoved, effectivePolicies, err = sys.store.PolicyDBUpdate(ctx, userOrGroup, isGroup, + regUser, r.Policies, isAttach) + if err != nil { + return updatedAt, addedOrRemoved, effectivePolicies, err + } + + // Notify all other MinIO peers to reload policy + if !sys.HasWatcher() { + for _, nerr := range globalNotificationSys.LoadPolicyMapping(ctx, userOrGroup, regUser, isGroup) { + if nerr.Err != nil { + logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String()) + iamLogIf(ctx, nerr.Err) + } + } + } + + replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemPolicyMapping, + PolicyMapping: &madmin.SRPolicyMapping{ + UserOrGroup: userOrGroup, + UserType: int(regUser), + IsGroup: isGroup, + Policy: strings.Join(effectivePolicies, ","), + }, + UpdatedAt: updatedAt, + })) + + return updatedAt, addedOrRemoved, effectivePolicies, err +} + +// PolicyDBUpdateLDAP - adds or removes policies from a user or a group verified +// to be in the LDAP directory. +func (sys *IAMSys) PolicyDBUpdateLDAP(ctx context.Context, isAttach bool, + r madmin.PolicyAssociationReq, +) (updatedAt time.Time, addedOrRemoved, effectivePolicies []string, err error) { + if !sys.Initialized() { + err = errServerNotInitialized + return updatedAt, addedOrRemoved, effectivePolicies, err + } + + var dn string + var dnResult *ldap.DNSearchResult + var isGroup bool + if r.User != "" { + dnResult, err = sys.LDAPConfig.GetValidatedDNForUsername(r.User) + if err != nil { + iamLogIf(ctx, err) + return updatedAt, addedOrRemoved, effectivePolicies, err + } + if dnResult == nil { + // dn not found - still attempt to detach if provided user is a DN. + if !isAttach && sys.LDAPConfig.IsLDAPUserDN(r.User) { + dn = sys.LDAPConfig.QuickNormalizeDN(r.User) + } else { + err = errNoSuchUser + return updatedAt, addedOrRemoved, effectivePolicies, err + } + } else { + dn = dnResult.NormDN + } + isGroup = false + } else { + var underBaseDN bool + if dnResult, underBaseDN, err = sys.LDAPConfig.GetValidatedGroupDN(nil, r.Group); err != nil { + iamLogIf(ctx, err) + return updatedAt, addedOrRemoved, effectivePolicies, err + } + if dnResult == nil || !underBaseDN { + if !isAttach { + dn = sys.LDAPConfig.QuickNormalizeDN(r.Group) + } else { + err = errNoSuchGroup + return updatedAt, addedOrRemoved, effectivePolicies, err + } + } else { + // We use the group DN returned by the LDAP server (this may not + // equal the input group name, but we assume it is canonical). + dn = dnResult.NormDN + } + isGroup = true + } + + // Backward compatibility in detaching non-normalized DNs. + if !isAttach { + var oldDN string + if isGroup { + oldDN = r.Group + } else { + oldDN = r.User + } + if oldDN != dn { + sys.store.PolicyDBUpdate(ctx, oldDN, isGroup, stsUser, r.Policies, isAttach) + } + } + + userType := stsUser + updatedAt, addedOrRemoved, effectivePolicies, err = sys.store.PolicyDBUpdate( + ctx, dn, isGroup, userType, r.Policies, isAttach) + if err != nil { + return updatedAt, addedOrRemoved, effectivePolicies, err + } + + // Notify all other MinIO peers to reload policy + if !sys.HasWatcher() { + for _, nerr := range globalNotificationSys.LoadPolicyMapping(ctx, dn, userType, isGroup) { + if nerr.Err != nil { + logger.GetReqInfo(ctx).SetTags("peerAddress", nerr.Host.String()) + iamLogIf(ctx, nerr.Err) + } + } + } + + replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemPolicyMapping, + PolicyMapping: &madmin.SRPolicyMapping{ + UserOrGroup: dn, + UserType: int(userType), + IsGroup: isGroup, + Policy: strings.Join(effectivePolicies, ","), + }, + UpdatedAt: updatedAt, + })) + + return updatedAt, addedOrRemoved, effectivePolicies, err +} + +// PolicyDBGet - gets policy set on a user or group. If a list of groups is +// given, policies associated with them are included as well. +func (sys *IAMSys) PolicyDBGet(name string, groups ...string) ([]string, error) { + if !sys.Initialized() { + return nil, errServerNotInitialized + } + + return sys.store.PolicyDBGet(name, groups...) +} + +const sessionPolicyNameExtracted = policy.SessionPolicyName + "-extracted" + +// IsAllowedServiceAccount - checks if the given service account is allowed to perform +// actions. The permission of the parent user is checked first +func (sys *IAMSys) IsAllowedServiceAccount(args policy.Args, parentUser string) bool { + // Verify if the parent claim matches the parentUser. + p, ok := args.Claims[parentClaim] + if ok { + parentInClaim, ok := p.(string) + if !ok { + // Reject malformed/malicious requests. + return false + } + // The parent claim in the session token should be equal + // to the parent detected in the backend + if parentInClaim != parentUser { + return false + } + } else { + // This is needed so a malicious user cannot + // use a leaked session key of another user + // to widen its privileges. + return false + } + + isOwnerDerived := parentUser == globalActiveCred.AccessKey + + var err error + var svcPolicies []string + roleArn := args.GetRoleArn() + + switch { + case isOwnerDerived: + // All actions are allowed by default and no policy evaluation is + // required. + + case roleArn != "": + arn, err := arn.Parse(roleArn) + if err != nil { + iamLogIf(GlobalContext, fmt.Errorf("error parsing role ARN %s: %v", roleArn, err)) + return false + } + svcPolicies = newMappedPolicy(sys.rolesMap[arn]).toSlice() + default: + // Check policy for parent user of service account. + svcPolicies, err = sys.PolicyDBGet(parentUser, args.Groups...) + if err != nil { + iamLogIf(GlobalContext, err) + return false + } + + // Finally, if there is no parent policy, check if a policy claim is + // present. + if len(svcPolicies) == 0 { + policySet, _ := policy.GetPoliciesFromClaims(args.Claims, iamPolicyClaimNameOpenID()) + svcPolicies = policySet.ToSlice() + } + } + + // Defensive code: Do not allow any operation if no policy is found. + if !isOwnerDerived && len(svcPolicies) == 0 { + return false + } + + var combinedPolicy policy.Policy + // Policies were found, evaluate all of them. + if !isOwnerDerived { + availablePoliciesStr, c := sys.store.MergePolicies(strings.Join(svcPolicies, ",")) + if availablePoliciesStr == "" { + return false + } + combinedPolicy = c + } + + parentArgs := args + parentArgs.AccountName = parentUser + + saPolicyClaim, ok := args.Claims[iamPolicyClaimNameSA()] + if !ok { + return false + } + + saPolicyClaimStr, ok := saPolicyClaim.(string) + if !ok { + // Sub policy if set, should be a string reject + // malformed/malicious requests. + return false + } + + if saPolicyClaimStr == inheritedPolicyType { + return isOwnerDerived || combinedPolicy.IsAllowed(parentArgs) + } + + // 3. If an inline session-policy is present, evaluate it. + hasSessionPolicy, isAllowedSP := isAllowedBySessionPolicyForServiceAccount(args) + if hasSessionPolicy { + return isAllowedSP && (isOwnerDerived || combinedPolicy.IsAllowed(parentArgs)) + } + + // Sub policy not set. Evaluate only the parent policies. + return (isOwnerDerived || combinedPolicy.IsAllowed(parentArgs)) +} + +// IsAllowedSTS is meant for STS based temporary credentials, +// which implements claims validation and verification other than +// applying policies. +func (sys *IAMSys) IsAllowedSTS(args policy.Args, parentUser string) bool { + // 1. Determine mapped policies + + isOwnerDerived := parentUser == globalActiveCred.AccessKey + var policies []string + roleArn := args.GetRoleArn() + + switch { + case isOwnerDerived: + // All actions are allowed by default and no policy evaluation is + // required. + + case roleArn != "": + // If a roleARN is present, the role policy is applied. + arn, err := arn.Parse(roleArn) + if err != nil { + iamLogIf(GlobalContext, fmt.Errorf("error parsing role ARN %s: %v", roleArn, err)) + return false + } + policies = newMappedPolicy(sys.rolesMap[arn]).toSlice() + + default: + // Otherwise, inherit parent user's policy + var err error + policies, err = sys.PolicyDBGet(parentUser, args.Groups...) + if err != nil { + iamLogIf(GlobalContext, fmt.Errorf("error fetching policies on %s: %v", parentUser, err)) + return false + } + + // Finally, if there is no parent policy, check if a policy claim is + // present in the session token. + if len(policies) == 0 { + // If there is no parent policy mapping, we fall back to + // using policy claim from JWT. + policySet, ok := args.GetPolicies(iamPolicyClaimNameOpenID()) + if !ok { + // When claims are set, it should have a policy claim field. + return false + } + policies = policySet.ToSlice() + } + } + + // Defensive code: Do not allow any operation if no policy is found in the session token + if !isOwnerDerived && len(policies) == 0 { + return false + } + + // 2. Combine the mapped policies into a single combined policy. + + var combinedPolicy policy.Policy + // Policies were found, evaluate all of them. + if !isOwnerDerived { + availablePoliciesStr, c := sys.store.MergePolicies(strings.Join(policies, ",")) + if availablePoliciesStr == "" { + // all policies presented in the claim should exist + iamLogIf(GlobalContext, fmt.Errorf("expected policy (%s) missing from the JWT claim %s, rejecting the request", policies, iamPolicyClaimNameOpenID())) + + return false + } + combinedPolicy = c + } + + // 3. If an inline session-policy is present, evaluate it. + + // Now check if we have a sessionPolicy. + hasSessionPolicy, isAllowedSP := isAllowedBySessionPolicy(args) + if hasSessionPolicy { + return isAllowedSP && (isOwnerDerived || combinedPolicy.IsAllowed(args)) + } + + // Sub policy not set, this is most common since subPolicy + // is optional, use the inherited policies. + return isOwnerDerived || combinedPolicy.IsAllowed(args) +} + +func isAllowedBySessionPolicyForServiceAccount(args policy.Args) (hasSessionPolicy bool, isAllowed bool) { + hasSessionPolicy = false + isAllowed = false + + // Now check if we have a sessionPolicy. + spolicy, ok := args.Claims[sessionPolicyNameExtracted] + if !ok { + return hasSessionPolicy, isAllowed + } + + hasSessionPolicy = true + + spolicyStr, ok := spolicy.(string) + if !ok { + // Sub policy if set, should be a string reject + // malformed/malicious requests. + return hasSessionPolicy, isAllowed + } + + // Check if policy is parseable. + subPolicy, err := policy.ParseConfig(bytes.NewReader([]byte(spolicyStr))) + if err != nil { + // Log any error in input session policy config. + iamLogIf(GlobalContext, err) + return hasSessionPolicy, isAllowed + } + + // SPECIAL CASE: For service accounts, any valid JSON is allowed as a + // policy, regardless of whether the number of statements is 0, this + // includes `null`, `{}` and `{"Statement": null}`. In fact, MinIO Console + // sends `null` when no policy is set and the intended behavior is that the + // service account should inherit parent policy. So when policy is empty in + // all fields we return hasSessionPolicy=false. + if subPolicy.Version == "" && subPolicy.Statements == nil && subPolicy.ID == "" { + hasSessionPolicy = false + return hasSessionPolicy, isAllowed + } + + // As the session policy exists, even if the parent is the root account, it + // must be restricted by it. So, we set `.IsOwner` to false here + // unconditionally. + // + // We also set `DenyOnly` arg to false here - this is an IMPORTANT corner + // case: DenyOnly is used only for allowing an account to do actions related + // to its own account (like create service accounts for itself, among + // others). However when a session policy is present, we need to validate + // that the action is actually allowed, rather than checking if the action + // is only disallowed. + sessionPolicyArgs := args + sessionPolicyArgs.IsOwner = false + sessionPolicyArgs.DenyOnly = false + + // Sub policy is set and valid. + return hasSessionPolicy, subPolicy.IsAllowed(sessionPolicyArgs) +} + +func isAllowedBySessionPolicy(args policy.Args) (hasSessionPolicy bool, isAllowed bool) { + hasSessionPolicy = false + isAllowed = false + + // Now check if we have a sessionPolicy. + spolicy, ok := args.Claims[sessionPolicyNameExtracted] + if !ok { + return hasSessionPolicy, isAllowed + } + + hasSessionPolicy = true + + spolicyStr, ok := spolicy.(string) + if !ok { + // Sub policy if set, should be a string reject + // malformed/malicious requests. + return hasSessionPolicy, isAllowed + } + + // Check if policy is parseable. + subPolicy, err := policy.ParseConfig(bytes.NewReader([]byte(spolicyStr))) + if err != nil { + // Log any error in input session policy config. + iamLogIf(GlobalContext, err) + return hasSessionPolicy, isAllowed + } + + // Policy without Version string value reject it. + if subPolicy.Version == "" { + return hasSessionPolicy, isAllowed + } + + // As the session policy exists, even if the parent is the root account, it + // must be restricted by it. So, we set `.IsOwner` to false here + // unconditionally. + // + // We also set `DenyOnly` arg to false here - this is an IMPORTANT corner + // case: DenyOnly is used only for allowing an account to do actions related + // to its own account (like create service accounts for itself, among + // others). However when a session policy is present, we need to validate + // that the action is actually allowed, rather than checking if the action + // is only disallowed. + sessionPolicyArgs := args + sessionPolicyArgs.IsOwner = false + sessionPolicyArgs.DenyOnly = false + + // Sub policy is set and valid. + return hasSessionPolicy, subPolicy.IsAllowed(sessionPolicyArgs) +} + +// GetCombinedPolicy returns a combined policy combining all policies +func (sys *IAMSys) GetCombinedPolicy(policies ...string) policy.Policy { + _, policy := sys.store.MergePolicies(strings.Join(policies, ",")) + return policy +} + +// doesPolicyAllow - checks if the given policy allows the passed action with given args. This is rarely needed. +// Notice there is no account name involved, so this is a dangerous function. +func (sys *IAMSys) doesPolicyAllow(policy string, args policy.Args) bool { + // Policies were found, evaluate all of them. + return sys.GetCombinedPolicy(policy).IsAllowed(args) +} + +// IsAllowed - checks given policy args is allowed to continue the Rest API. +func (sys *IAMSys) IsAllowed(args policy.Args) bool { + // If opa is configured, use OPA always. + if authz := newGlobalAuthZPluginFn(); authz != nil { + ok, err := authz.IsAllowed(args) + if err != nil { + authZLogIf(GlobalContext, err) + } + return ok + } + + // Policies don't apply to the owner. + if args.IsOwner { + return true + } + + // If the credential is temporary, perform STS related checks. + ok, parentUser, err := sys.IsTempUser(args.AccountName) + if err != nil { + return false + } + if ok { + return sys.IsAllowedSTS(args, parentUser) + } + + // If the credential is for a service account, perform related check + ok, parentUser, err = sys.IsServiceAccount(args.AccountName) + if err != nil { + return false + } + if ok { + return sys.IsAllowedServiceAccount(args, parentUser) + } + + // Continue with the assumption of a regular user + policies, err := sys.PolicyDBGet(args.AccountName, args.Groups...) + if err != nil { + return false + } + + if len(policies) == 0 { + // No policy found. + return false + } + + // Policies were found, evaluate all of them. + return sys.GetCombinedPolicy(policies...).IsAllowed(args) +} + +// SetUsersSysType - sets the users system type, regular or LDAP. +func (sys *IAMSys) SetUsersSysType(t UsersSysType) { + sys.usersSysType = t +} + +// GetUsersSysType - returns the users system type for this IAM +func (sys *IAMSys) GetUsersSysType() UsersSysType { + return sys.usersSysType +} + +// NewIAMSys - creates new config system object. +func NewIAMSys() *IAMSys { + return &IAMSys{ + usersSysType: MinIOUsersSysType, + configLoaded: make(chan struct{}), + } +} diff --git a/cmd/ilm-config.go b/cmd/ilm-config.go new file mode 100644 index 00000000..83e7e305 --- /dev/null +++ b/cmd/ilm-config.go @@ -0,0 +1,57 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "sync" + + "github.com/minio/minio/internal/config/ilm" +) + +var globalILMConfig = ilmConfig{ + cfg: ilm.Config{ + ExpirationWorkers: 100, + TransitionWorkers: 100, + }, +} + +type ilmConfig struct { + mu sync.RWMutex + cfg ilm.Config +} + +func (c *ilmConfig) getExpirationWorkers() int { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.cfg.ExpirationWorkers +} + +func (c *ilmConfig) getTransitionWorkers() int { + c.mu.RLock() + defer c.mu.RUnlock() + + return c.cfg.TransitionWorkers +} + +func (c *ilmConfig) update(cfg ilm.Config) { + c.mu.Lock() + defer c.mu.Unlock() + + c.cfg = cfg +} diff --git a/cmd/is-dir-empty_linux.go b/cmd/is-dir-empty_linux.go new file mode 100644 index 00000000..99945d67 --- /dev/null +++ b/cmd/is-dir-empty_linux.go @@ -0,0 +1,43 @@ +//go:build linux && !appengine +// +build linux,!appengine + +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "syscall" +) + +// Returns true if no error and there is no object or prefix inside this directory +func isDirEmpty(dirname string, legacy bool) bool { + if legacy { + // On filesystems such as btrfs, nfs this is not true, so fallback + // to performing readdir() instead. + entries, err := readDirN(dirname, 1) + if err != nil { + return false + } + return len(entries) == 0 + } + var stat syscall.Stat_t + if err := syscall.Stat(dirname, &stat); err != nil { + return false + } + return stat.Mode&syscall.S_IFMT == syscall.S_IFDIR && stat.Nlink == 2 +} diff --git a/cmd/is-dir-empty_other.go b/cmd/is-dir-empty_other.go new file mode 100644 index 00000000..ab7b2f7e --- /dev/null +++ b/cmd/is-dir-empty_other.go @@ -0,0 +1,30 @@ +//go:build !linux +// +build !linux + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +// isDirEmpty - returns true if there is no error and no object and prefix inside this directory +func isDirEmpty(dirname string, _ bool) bool { + entries, err := readDirN(dirname, 1) + if err != nil { + return false + } + return len(entries) == 0 +} diff --git a/cmd/jwt.go b/cmd/jwt.go new file mode 100644 index 00000000..c86b8f67 --- /dev/null +++ b/cmd/jwt.go @@ -0,0 +1,139 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "errors" + "maps" + "net/http" + "time" + + jwtgo "github.com/golang-jwt/jwt/v4" + jwtreq "github.com/golang-jwt/jwt/v4/request" + "github.com/minio/minio/internal/auth" + xjwt "github.com/minio/minio/internal/jwt" + "github.com/minio/pkg/v3/policy" +) + +const ( + jwtAlgorithm = "Bearer" + + // Default JWT token for web handlers is one day. + defaultJWTExpiry = 24 * time.Hour + + // Inter-node JWT token expiry is 100 years approx. + defaultInterNodeJWTExpiry = 100 * 365 * 24 * time.Hour +) + +var ( + errInvalidAccessKeyID = errors.New("The access key ID you provided does not exist in our records") + errAccessKeyDisabled = errors.New("The access key you provided is disabled") + errAuthentication = errors.New("Authentication failed, check your access credentials") + errNoAuthToken = errors.New("JWT token missing") + errSkewedAuthTime = errors.New("Skewed authentication date/time") + errMalformedAuth = errors.New("Malformed authentication input") +) + +func authenticateNode(accessKey, secretKey string) (string, error) { + claims := xjwt.NewStandardClaims() + claims.SetExpiry(UTCNow().Add(defaultInterNodeJWTExpiry)) + claims.SetAccessKey(accessKey) + + jwt := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, claims) + return jwt.SignedString([]byte(secretKey)) +} + +// Check if the request is authenticated. +// Returns nil if the request is authenticated. errNoAuthToken if token missing. +// Returns errAuthentication for all other errors. +func metricsRequestAuthenticate(req *http.Request) (*xjwt.MapClaims, []string, bool, error) { + token, err := jwtreq.AuthorizationHeaderExtractor.ExtractToken(req) + if err != nil { + if err == jwtreq.ErrNoTokenInRequest { + return nil, nil, false, errNoAuthToken + } + return nil, nil, false, err + } + claims := xjwt.NewMapClaims() + if err := xjwt.ParseWithClaims(token, claims, func(claims *xjwt.MapClaims) ([]byte, error) { + if claims.AccessKey != globalActiveCred.AccessKey { + u, ok := globalIAMSys.GetUser(req.Context(), claims.AccessKey) + if !ok { + // Credentials will be invalid but for disabled + // return a different error in such a scenario. + if u.Credentials.Status == auth.AccountOff { + return nil, errAccessKeyDisabled + } + return nil, errInvalidAccessKeyID + } + cred := u.Credentials + // Expired credentials return error. + if cred.IsTemp() && cred.IsExpired() { + return nil, errInvalidAccessKeyID + } + return []byte(cred.SecretKey), nil + } // this means claims.AccessKey == rootAccessKey + if !globalAPIConfig.permitRootAccess() { + // if root access is disabled, fail this request. + return nil, errAccessKeyDisabled + } + return []byte(globalActiveCred.SecretKey), nil + }); err != nil { + return claims, nil, false, errAuthentication + } + owner := true + var groups []string + if globalActiveCred.AccessKey != claims.AccessKey { + // Check if the access key is part of users credentials. + u, ok := globalIAMSys.GetUser(req.Context(), claims.AccessKey) + if !ok { + return nil, nil, false, errInvalidAccessKeyID + } + ucred := u.Credentials + // get embedded claims + eclaims, s3Err := checkClaimsFromToken(req, ucred) + if s3Err != ErrNone { + return nil, nil, false, errAuthentication + } + + maps.Copy(claims.MapClaims, eclaims) + + // if root access is disabled, disable all its service accounts and temporary credentials. + if ucred.ParentUser == globalActiveCred.AccessKey && !globalAPIConfig.permitRootAccess() { + return nil, nil, false, errAccessKeyDisabled + } + + // Now check if we have a sessionPolicy. + if _, ok = eclaims[policy.SessionPolicyName]; ok { + owner = false + } else { + owner = globalActiveCred.AccessKey == ucred.ParentUser + } + + groups = ucred.Groups + } + + return claims, groups, owner, nil +} + +// newCachedAuthToken returns the cached token. +func newCachedAuthToken() func() string { + return func() string { + return globalNodeAuthToken + } +} diff --git a/cmd/jwt_test.go b/cmd/jwt_test.go new file mode 100644 index 00000000..a45ce35c --- /dev/null +++ b/cmd/jwt_test.go @@ -0,0 +1,190 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "net/http" + "os" + "testing" + + jwtgo "github.com/golang-jwt/jwt/v4" + xjwt "github.com/minio/minio/internal/jwt" +) + +func getTokenString(accessKey, secretKey string) (string, error) { + claims := xjwt.NewMapClaims() + claims.SetExpiry(UTCNow().Add(defaultJWTExpiry)) + claims.SetAccessKey(accessKey) + token := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, claims) + return token.SignedString([]byte(secretKey)) +} + +// Tests web request authenticator. +func TestWebRequestAuthenticate(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + obj, fsDir, err := prepareFS(ctx) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(fsDir) + if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil { + t.Fatal(err) + } + + creds := globalActiveCred + token, err := getTokenString(creds.AccessKey, creds.SecretKey) + if err != nil { + t.Fatalf("unable get token %s", err) + } + testCases := []struct { + req *http.Request + expectedErr error + }{ + // Set valid authorization header. + { + req: &http.Request{ + Header: http.Header{ + "Authorization": []string{token}, + }, + }, + expectedErr: nil, + }, + // No authorization header. + { + req: &http.Request{ + Header: http.Header{}, + }, + expectedErr: errNoAuthToken, + }, + // Invalid authorization token. + { + req: &http.Request{ + Header: http.Header{ + "Authorization": []string{"invalid-token"}, + }, + }, + expectedErr: errAuthentication, + }, + } + + for i, testCase := range testCases { + _, _, _, gotErr := metricsRequestAuthenticate(testCase.req) + if testCase.expectedErr != gotErr { + t.Errorf("Test %d, expected err %s, got %s", i+1, testCase.expectedErr, gotErr) + } + } +} + +func BenchmarkParseJWTStandardClaims(b *testing.B) { + ctx, cancel := context.WithCancel(b.Context()) + defer cancel() + + obj, fsDir, err := prepareFS(ctx) + if err != nil { + b.Fatal(err) + } + defer os.RemoveAll(fsDir) + if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil { + b.Fatal(err) + } + + creds := globalActiveCred + token, err := authenticateNode(creds.AccessKey, creds.SecretKey) + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + err = xjwt.ParseWithStandardClaims(token, xjwt.NewStandardClaims(), []byte(creds.SecretKey)) + if err != nil { + b.Fatal(err) + } + } + }) +} + +func BenchmarkParseJWTMapClaims(b *testing.B) { + ctx, cancel := context.WithCancel(b.Context()) + defer cancel() + + obj, fsDir, err := prepareFS(ctx) + if err != nil { + b.Fatal(err) + } + defer os.RemoveAll(fsDir) + if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil { + b.Fatal(err) + } + + creds := globalActiveCred + token, err := authenticateNode(creds.AccessKey, creds.SecretKey) + if err != nil { + b.Fatal(err) + } + + b.ResetTimer() + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + err = xjwt.ParseWithClaims(token, xjwt.NewMapClaims(), func(*xjwt.MapClaims) ([]byte, error) { + return []byte(creds.SecretKey), nil + }) + if err != nil { + b.Fatal(err) + } + } + }) +} + +func BenchmarkAuthenticateNode(b *testing.B) { + ctx, cancel := context.WithCancel(b.Context()) + defer cancel() + + obj, fsDir, err := prepareFS(ctx) + if err != nil { + b.Fatal(err) + } + defer os.RemoveAll(fsDir) + if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil { + b.Fatal(err) + } + + creds := globalActiveCred + b.Run("uncached", func(b *testing.B) { + fn := authenticateNode + b.ResetTimer() + b.ReportAllocs() + for b.Loop() { + fn(creds.AccessKey, creds.SecretKey) + } + }) + b.Run("cached", func(b *testing.B) { + fn := newCachedAuthToken() + b.ResetTimer() + b.ReportAllocs() + for b.Loop() { + fn() + } + }) +} diff --git a/cmd/kms-handlers.go b/cmd/kms-handlers.go new file mode 100644 index 00000000..ce5017c1 --- /dev/null +++ b/cmd/kms-handlers.go @@ -0,0 +1,328 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "crypto/subtle" + "encoding/json" + "net/http" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/kms" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/policy" +) + +// KMSStatusHandler - GET /minio/kms/v1/status +func (a kmsAPIHandlers) KMSStatusHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "KMSStatus") + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSStatusAction) + if objectAPI == nil { + return + } + + if GlobalKMS == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL) + return + } + + stat, err := GlobalKMS.Status(ctx) + if err != nil { + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL) + return + } + resp, err := json.Marshal(stat) + if err != nil { + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL) + return + } + writeSuccessResponseJSON(w, resp) +} + +// KMSMetricsHandler - GET /minio/kms/v1/metrics +func (a kmsAPIHandlers) KMSMetricsHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "KMSMetrics") + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSMetricsAction) + if objectAPI == nil { + return + } + + if GlobalKMS == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL) + return + } + + metrics, err := GlobalKMS.Metrics(ctx) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + if res, err := json.Marshal(metrics); err != nil { + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL) + } else { + writeSuccessResponseJSON(w, res) + } +} + +// KMSAPIsHandler - GET /minio/kms/v1/apis +func (a kmsAPIHandlers) KMSAPIsHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "KMSAPIs") + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSAPIAction) + if objectAPI == nil { + return + } + + if GlobalKMS == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL) + return + } + + apis, err := GlobalKMS.APIs(ctx) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + if res, err := json.Marshal(apis); err != nil { + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL) + } else { + writeSuccessResponseJSON(w, res) + } +} + +type versionResponse struct { + Version string `json:"version"` +} + +// KMSVersionHandler - GET /minio/kms/v1/version +func (a kmsAPIHandlers) KMSVersionHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "KMSVersion") + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSVersionAction) + if objectAPI == nil { + return + } + + if GlobalKMS == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL) + return + } + + version, err := GlobalKMS.Version(ctx) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + res := &versionResponse{Version: version} + v, err := json.Marshal(res) + if err != nil { + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL) + return + } + writeSuccessResponseJSON(w, v) +} + +// KMSCreateKeyHandler - POST /minio/kms/v1/key/create?key-id= +func (a kmsAPIHandlers) KMSCreateKeyHandler(w http.ResponseWriter, r *http.Request) { + // If env variable MINIO_KMS_SECRET_KEY is populated, prevent creation of new keys + ctx := newContext(r, w, "KMSCreateKey") + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSCreateKeyAction) + if objectAPI == nil { + return + } + + if GlobalKMS == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL) + return + } + + keyID := r.Form.Get("key-id") + + // Ensure policy allows the user to create this key name + cred, owner, s3Err := validateAdminSignature(ctx, r, "") + if s3Err != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + if !checkKMSActionAllowed(r, owner, cred, policy.KMSCreateKeyAction, keyID) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + + if err := GlobalKMS.CreateKey(ctx, &kms.CreateKeyRequest{Name: keyID}); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + writeSuccessResponseHeadersOnly(w) +} + +// KMSListKeysHandler - GET /minio/kms/v1/key/list?pattern= +func (a kmsAPIHandlers) KMSListKeysHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "KMSListKeys") + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + // This only checks if the action (kms:ListKeys) is allowed, it does not check + // each key name against the policy's Resources. We check that below, once + // we have the list of key names from the KMS. + objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSListKeysAction) + if objectAPI == nil { + return + } + + if GlobalKMS == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL) + return + } + allKeys, _, err := GlobalKMS.ListKeys(ctx, &kms.ListRequest{ + Prefix: r.Form.Get("pattern"), + }) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Get the cred and owner for checking authz below. + cred, owner, s3Err := validateAdminSignature(ctx, r, "") + if s3Err != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + + // Now we have all the key names, for each of them, check whether the policy grants permission for + // the user to list it. Filter in place to leave only allowed keys. + n := 0 + for _, k := range allKeys { + if checkKMSActionAllowed(r, owner, cred, policy.KMSListKeysAction, k.Name) { + allKeys[n] = k + n++ + } + } + allKeys = allKeys[:n] + + if res, err := json.Marshal(allKeys); err != nil { + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL) + } else { + writeSuccessResponseJSON(w, res) + } +} + +// KMSKeyStatusHandler - GET /minio/kms/v1/key/status?key-id= +func (a kmsAPIHandlers) KMSKeyStatusHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "KMSKeyStatus") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + objectAPI, _ := validateAdminReq(ctx, w, r, policy.KMSKeyStatusAction) + if objectAPI == nil { + return + } + + if GlobalKMS == nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrKMSNotConfigured), r.URL) + return + } + + keyID := r.Form.Get("key-id") + if keyID == "" { + keyID = GlobalKMS.DefaultKey + } + response := madmin.KMSKeyStatus{ + KeyID: keyID, + } + + // Ensure policy allows the user to get this key's status + cred, owner, s3Err := validateAdminSignature(ctx, r, "") + if s3Err != ErrNone { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + if !checkKMSActionAllowed(r, owner, cred, policy.KMSKeyStatusAction, keyID) { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + + kmsContext := kms.Context{"MinIO admin API": "KMSKeyStatusHandler"} // Context for a test key operation + // 1. Generate a new key using the KMS. + key, err := GlobalKMS.GenerateKey(ctx, &kms.GenerateKeyRequest{Name: keyID, AssociatedData: kmsContext}) + if err != nil { + response.EncryptionErr = err.Error() + resp, err := json.Marshal(response) + if err != nil { + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL) + return + } + writeSuccessResponseJSON(w, resp) + return + } + + // 2. Verify that we can indeed decrypt the (encrypted) key + decryptedKey, err := GlobalKMS.Decrypt(ctx, &kms.DecryptRequest{ + Name: key.KeyID, + Ciphertext: key.Ciphertext, + AssociatedData: kmsContext, + }) + if err != nil { + response.DecryptionErr = err.Error() + resp, err := json.Marshal(response) + if err != nil { + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL) + return + } + writeSuccessResponseJSON(w, resp) + return + } + + // 3. Compare generated key with decrypted key + if subtle.ConstantTimeCompare(key.Plaintext, decryptedKey) != 1 { + response.DecryptionErr = "The generated and the decrypted data key do not match" + resp, err := json.Marshal(response) + if err != nil { + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL) + return + } + writeSuccessResponseJSON(w, resp) + return + } + + resp, err := json.Marshal(response) + if err != nil { + writeCustomErrorResponseJSON(ctx, w, errorCodes.ToAPIErr(ErrInternalError), err.Error(), r.URL) + return + } + writeSuccessResponseJSON(w, resp) +} + +// checkKMSActionAllowed checks for authorization for a specific action on a resource. +func checkKMSActionAllowed(r *http.Request, owner bool, cred auth.Credentials, action policy.KMSAction, resource string) bool { + return globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.Action(action), + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + Claims: cred.Claims, + BucketName: resource, // overloading BucketName as that's what the policy engine uses to assemble a Resource. + }) +} diff --git a/cmd/kms-handlers_test.go b/cmd/kms-handlers_test.go new file mode 100644 index 00000000..4eccab4c --- /dev/null +++ b/cmd/kms-handlers_test.go @@ -0,0 +1,847 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/kms" + "github.com/minio/pkg/v3/policy" +) + +const ( + // KMS API paths + // For example: /minio/kms/v1/key/list?pattern=* + kmsURL = kmsPathPrefix + kmsAPIVersionPrefix + kmsStatusPath = kmsURL + "/status" + kmsMetricsPath = kmsURL + "/metrics" + kmsAPIsPath = kmsURL + "/apis" + kmsVersionPath = kmsURL + "/version" + kmsKeyCreatePath = kmsURL + "/key/create" + kmsKeyListPath = kmsURL + "/key/list" + kmsKeyStatusPath = kmsURL + "/key/status" + + // Admin API paths + // For example: /minio/admin/v3/kms/status + adminURL = adminPathPrefix + adminAPIVersionPrefix + kmsAdminStatusPath = adminURL + "/kms/status" + kmsAdminKeyStatusPath = adminURL + "/kms/key/status" + kmsAdminKeyCreate = adminURL + "/kms/key/create" +) + +const ( + userAccessKey = "miniofakeuseraccesskey" + userSecretKey = "miniofakeusersecret" +) + +type kmsTestCase struct { + name string + method string + path string + query map[string]string + + // User credentials and policy for request + policy string + asRoot bool + + // Wanted in response. + wantStatusCode int + wantKeyNames []string + wantResp []string +} + +func TestKMSHandlersCreateKey(t *testing.T) { + adminTestBed, tearDown := setupKMSTest(t, true) + defer tearDown() + + tests := []kmsTestCase{ + // Create key test + { + name: "create key as user with no policy want forbidden", + method: http.MethodPost, + path: kmsKeyCreatePath, + query: map[string]string{"key-id": "new-test-key"}, + asRoot: false, + + wantStatusCode: http.StatusForbidden, + wantResp: []string{"AccessDenied"}, + }, + { + name: "create key as user with no resources specified want success", + method: http.MethodPost, + path: kmsKeyCreatePath, + query: map[string]string{"key-id": "new-test-key"}, + asRoot: false, + + policy: `{"Effect": "Allow", + "Action": ["kms:CreateKey"] }`, + + wantStatusCode: http.StatusOK, + }, + { + name: "create key as user set policy to allow want success", + method: http.MethodPost, + path: kmsKeyCreatePath, + query: map[string]string{"key-id": "second-new-test-key"}, + asRoot: false, + + policy: `{"Effect": "Allow", + "Action": ["kms:CreateKey"], + "Resource": ["arn:minio:kms:::second-new-test-*"] }`, + + wantStatusCode: http.StatusOK, + }, + { + name: "create key as user set policy to non matching resource want forbidden", + method: http.MethodPost, + path: kmsKeyCreatePath, + query: map[string]string{"key-id": "third-new-test-key"}, + asRoot: false, + + policy: `{"Effect": "Allow", + "Action": ["kms:CreateKey"], + "Resource": ["arn:minio:kms:::non-matching-key-name"] }`, + + wantStatusCode: http.StatusForbidden, + wantResp: []string{"AccessDenied"}, + }, + } + for testNum, test := range tests { + t.Run(fmt.Sprintf("%d %s", testNum+1, test.name), func(t *testing.T) { + execKMSTest(t, test, adminTestBed) + }) + } +} + +func TestKMSHandlersKeyStatus(t *testing.T) { + adminTestBed, tearDown := setupKMSTest(t, true) + defer tearDown() + + tests := []kmsTestCase{ + { + name: "create a first key root user", + method: http.MethodPost, + path: kmsKeyCreatePath, + query: map[string]string{"key-id": "abc-test-key"}, + asRoot: true, + + wantStatusCode: http.StatusOK, + }, + { + name: "key status as root want success", + method: http.MethodGet, + path: kmsKeyStatusPath, + query: map[string]string{"key-id": "abc-test-key"}, + asRoot: true, + + wantStatusCode: http.StatusOK, + wantResp: []string{"abc-test-key"}, + }, + { + name: "key status as user no policy want forbidden", + method: http.MethodGet, + path: kmsKeyStatusPath, + query: map[string]string{"key-id": "abc-test-key"}, + asRoot: false, + + wantStatusCode: http.StatusForbidden, + wantResp: []string{"AccessDenied"}, + }, + { + name: "key status as user legacy no resources specified want success", + method: http.MethodGet, + path: kmsKeyStatusPath, + query: map[string]string{"key-id": "abc-test-key"}, + asRoot: false, + + policy: `{"Effect": "Allow", + "Action": ["kms:KeyStatus"] }`, + + wantStatusCode: http.StatusOK, + wantResp: []string{"abc-test-key"}, + }, + { + name: "key status as user set policy to allow only one key", + method: http.MethodGet, + path: kmsKeyStatusPath, + query: map[string]string{"key-id": "abc-test-key"}, + asRoot: false, + + policy: `{"Effect": "Allow", + "Action": ["kms:KeyStatus"], + "Resource": ["arn:minio:kms:::abc-test-*"] }`, + + wantStatusCode: http.StatusOK, + wantResp: []string{"abc-test-key"}, + }, + { + name: "key status as user set policy to allow non-matching key", + method: http.MethodGet, + path: kmsKeyStatusPath, + query: map[string]string{"key-id": "abc-test-key"}, + asRoot: false, + + policy: `{"Effect": "Allow", + "Action": ["kms:KeyStatus"], + "Resource": ["arn:minio:kms:::xyz-test-key"] }`, + + wantStatusCode: http.StatusForbidden, + wantResp: []string{"AccessDenied"}, + }, + } + for testNum, test := range tests { + t.Run(fmt.Sprintf("%d %s", testNum+1, test.name), func(t *testing.T) { + execKMSTest(t, test, adminTestBed) + }) + } +} + +func TestKMSHandlersAPIs(t *testing.T) { + adminTestBed, tearDown := setupKMSTest(t, true) + defer tearDown() + + tests := []kmsTestCase{ + // Version test + { + name: "version as root want success", + method: http.MethodGet, + path: kmsVersionPath, + asRoot: true, + + wantStatusCode: http.StatusOK, + wantResp: []string{"version"}, + }, + { + name: "version as user with no policy want forbidden", + method: http.MethodGet, + path: kmsVersionPath, + asRoot: false, + + wantStatusCode: http.StatusForbidden, + wantResp: []string{"AccessDenied"}, + }, + { + name: "version as user with policy ignores resource want success", + method: http.MethodGet, + path: kmsVersionPath, + asRoot: false, + + policy: `{"Effect": "Allow", + "Action": ["kms:Version"], + "Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"] }`, + + wantStatusCode: http.StatusOK, + wantResp: []string{"version"}, + }, + + // APIs test + { + name: "apis as root want success", + method: http.MethodGet, + path: kmsAPIsPath, + asRoot: true, + + wantStatusCode: http.StatusOK, + wantResp: []string{"stub/path"}, + }, + { + name: "apis as user with no policy want forbidden", + method: http.MethodGet, + path: kmsAPIsPath, + asRoot: false, + + wantStatusCode: http.StatusForbidden, + wantResp: []string{"AccessDenied"}, + }, + { + name: "apis as user with policy ignores resource want success", + method: http.MethodGet, + path: kmsAPIsPath, + asRoot: false, + + policy: `{"Effect": "Allow", + "Action": ["kms:API"], + "Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"] }`, + + wantStatusCode: http.StatusOK, + wantResp: []string{"stub/path"}, + }, + + // Metrics test + { + name: "metrics as root want success", + method: http.MethodGet, + path: kmsMetricsPath, + asRoot: true, + + wantStatusCode: http.StatusOK, + wantResp: []string{"kms"}, + }, + { + name: "metrics as user with no policy want forbidden", + method: http.MethodGet, + path: kmsMetricsPath, + asRoot: false, + + wantStatusCode: http.StatusForbidden, + wantResp: []string{"AccessDenied"}, + }, + { + name: "metrics as user with policy ignores resource want success", + method: http.MethodGet, + path: kmsMetricsPath, + asRoot: false, + + policy: `{"Effect": "Allow", + "Action": ["kms:Metrics"], + "Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"] }`, + + wantStatusCode: http.StatusOK, + wantResp: []string{"kms"}, + }, + + // Status tests + { + name: "status as root want success", + method: http.MethodGet, + path: kmsStatusPath, + asRoot: true, + + wantStatusCode: http.StatusOK, + wantResp: []string{"MinIO builtin"}, + }, + { + name: "status as user with no policy want forbidden", + method: http.MethodGet, + path: kmsStatusPath, + asRoot: false, + + wantStatusCode: http.StatusForbidden, + wantResp: []string{"AccessDenied"}, + }, + { + name: "status as user with policy ignores resource want success", + method: http.MethodGet, + path: kmsStatusPath, + asRoot: false, + + policy: `{"Effect": "Allow", + "Action": ["kms:Status"], + "Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"]}`, + + wantStatusCode: http.StatusOK, + wantResp: []string{"MinIO builtin"}, + }, + } + for testNum, test := range tests { + t.Run(fmt.Sprintf("%d %s", testNum+1, test.name), func(t *testing.T) { + execKMSTest(t, test, adminTestBed) + }) + } +} + +func TestKMSHandlersListKeys(t *testing.T) { + adminTestBed, tearDown := setupKMSTest(t, true) + defer tearDown() + + tests := []kmsTestCase{ + { + name: "create a first key root user", + method: http.MethodPost, + path: kmsKeyCreatePath, + query: map[string]string{"key-id": "abc-test-key"}, + asRoot: true, + + wantStatusCode: http.StatusOK, + }, + { + name: "create a second key root user", + method: http.MethodPost, + path: kmsKeyCreatePath, + query: map[string]string{"key-id": "xyz-test-key"}, + asRoot: true, + + wantStatusCode: http.StatusOK, + }, + + // List keys tests + { + name: "list keys as root want all to be returned", + method: http.MethodGet, + path: kmsKeyListPath, + query: map[string]string{"pattern": "*"}, + asRoot: true, + + wantStatusCode: http.StatusOK, + wantKeyNames: []string{"default-test-key", "abc-test-key", "xyz-test-key"}, + }, + { + name: "list keys as user with no policy want forbidden", + method: http.MethodGet, + path: kmsKeyListPath, + query: map[string]string{"pattern": "*"}, + asRoot: false, + + wantStatusCode: http.StatusForbidden, + wantResp: []string{"AccessDenied"}, + }, + { + name: "list keys as user with no resources specified want success", + method: http.MethodGet, + path: kmsKeyListPath, + query: map[string]string{"pattern": "*"}, + asRoot: false, + + policy: `{"Effect": "Allow", + "Action": ["kms:ListKeys"] + }`, + + wantStatusCode: http.StatusOK, + wantKeyNames: []string{"default-test-key", "abc-test-key", "xyz-test-key"}, + }, + { + name: "list keys as user set policy resource to allow only one key", + method: http.MethodGet, + path: kmsKeyListPath, + query: map[string]string{"pattern": "*"}, + asRoot: false, + + policy: `{"Effect": "Allow", + "Action": ["kms:ListKeys"], + "Resource": ["arn:minio:kms:::abc*"]}`, + + wantStatusCode: http.StatusOK, + wantKeyNames: []string{"abc-test-key"}, + }, + { + name: "list keys as user set policy to allow only one key, use pattern that includes correct key", + method: http.MethodGet, + path: kmsKeyListPath, + query: map[string]string{"pattern": "abc*"}, + + policy: `{"Effect": "Allow", + "Action": ["kms:ListKeys"], + "Resource": ["arn:minio:kms:::abc*"]}`, + + wantStatusCode: http.StatusOK, + wantKeyNames: []string{"abc-test-key"}, + }, + { + name: "list keys as user set policy to allow only one key, use pattern that excludes correct key", + method: http.MethodGet, + path: kmsKeyListPath, + query: map[string]string{"pattern": "xyz*"}, + asRoot: false, + + policy: `{"Effect": "Allow", + "Action": ["kms:ListKeys"], + "Resource": ["arn:minio:kms:::abc*"]}`, + + wantStatusCode: http.StatusOK, + wantKeyNames: []string{}, + }, + { + name: "list keys as user set policy that has no matching key resources", + method: http.MethodGet, + path: kmsKeyListPath, + query: map[string]string{"pattern": "*"}, + asRoot: false, + + policy: `{"Effect": "Allow", + "Action": ["kms:ListKeys"], + "Resource": ["arn:minio:kms:::nonematch*"]}`, + + wantStatusCode: http.StatusOK, + wantKeyNames: []string{}, + }, + { + name: "list keys as user set policy that allows listing but denies specific keys", + method: http.MethodGet, + path: kmsKeyListPath, + query: map[string]string{"pattern": "*"}, + asRoot: false, + + // It looks like this should allow listing any key that isn't "default-test-key", however + // the policy engine matches all Deny statements first, without regard to Resources (for KMS). + // This is for backwards compatibility where historically KMS statements ignored Resources. + policy: `{ + "Effect": "Allow", + "Action": ["kms:ListKeys"] + },{ + "Effect": "Deny", + "Action": ["kms:ListKeys"], + "Resource": ["arn:minio:kms:::default-test-key"] + }`, + + wantStatusCode: http.StatusForbidden, + wantResp: []string{"AccessDenied"}, + }, + } + + for testNum, test := range tests { + t.Run(fmt.Sprintf("%d %s", testNum+1, test.name), func(t *testing.T) { + execKMSTest(t, test, adminTestBed) + }) + } +} + +func TestKMSHandlerAdminAPI(t *testing.T) { + adminTestBed, tearDown := setupKMSTest(t, true) + defer tearDown() + + tests := []kmsTestCase{ + // Create key tests + { + name: "create a key root user", + method: http.MethodPost, + path: kmsAdminKeyCreate, + query: map[string]string{"key-id": "abc-test-key"}, + asRoot: true, + + wantStatusCode: http.StatusOK, + }, + { + name: "create key as user with no policy want forbidden", + method: http.MethodPost, + path: kmsAdminKeyCreate, + query: map[string]string{"key-id": "new-test-key"}, + asRoot: false, + + wantStatusCode: http.StatusForbidden, + wantResp: []string{"AccessDenied"}, + }, + { + name: "create key as user with no resources specified want success", + method: http.MethodPost, + path: kmsAdminKeyCreate, + query: map[string]string{"key-id": "new-test-key"}, + asRoot: false, + + policy: `{"Effect": "Allow", + "Action": ["admin:KMSCreateKey"] }`, + + wantStatusCode: http.StatusOK, + }, + { + name: "create key as user set policy to non matching resource want success", + method: http.MethodPost, + path: kmsAdminKeyCreate, + query: map[string]string{"key-id": "third-new-test-key"}, + asRoot: false, + + // Admin actions ignore Resources + policy: `{"Effect": "Allow", + "Action": ["admin:KMSCreateKey"], + "Resource": ["arn:minio:kms:::this-is-disregarded"] }`, + + wantStatusCode: http.StatusOK, + }, + + // Status tests + { + name: "status as root want success", + method: http.MethodPost, + path: kmsAdminStatusPath, + asRoot: true, + + wantStatusCode: http.StatusOK, + wantResp: []string{"MinIO builtin"}, + }, + { + name: "status as user with no policy want forbidden", + method: http.MethodPost, + path: kmsAdminStatusPath, + asRoot: false, + + wantStatusCode: http.StatusForbidden, + wantResp: []string{"AccessDenied"}, + }, + { + name: "status as user with policy ignores resource want success", + method: http.MethodPost, + path: kmsAdminStatusPath, + asRoot: false, + + policy: `{"Effect": "Allow", + "Action": ["admin:KMSKeyStatus"], + "Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"] }`, + + wantStatusCode: http.StatusOK, + wantResp: []string{"MinIO builtin"}, + }, + + // Key status tests + { + name: "key status as root want success", + method: http.MethodGet, + path: kmsAdminKeyStatusPath, + asRoot: true, + + wantStatusCode: http.StatusOK, + wantResp: []string{"key-id"}, + }, + { + name: "key status as user with no policy want forbidden", + method: http.MethodGet, + path: kmsAdminKeyStatusPath, + asRoot: false, + + wantStatusCode: http.StatusForbidden, + wantResp: []string{"AccessDenied"}, + }, + { + name: "key status as user with policy ignores resource want success", + method: http.MethodGet, + path: kmsAdminKeyStatusPath, + asRoot: false, + + policy: `{"Effect": "Allow", + "Action": ["admin:KMSKeyStatus"], + "Resource": ["arn:minio:kms:::does-not-matter-it-is-ignored"] }`, + + wantStatusCode: http.StatusOK, + wantResp: []string{"key-id"}, + }, + } + + for testNum, test := range tests { + t.Run(fmt.Sprintf("%d %s", testNum+1, test.name), func(t *testing.T) { + execKMSTest(t, test, adminTestBed) + }) + } +} + +// execKMSTest runs a single test case for KMS handlers +func execKMSTest(t *testing.T, test kmsTestCase, adminTestBed *adminErasureTestBed) { + var accessKey, secretKey string + if test.asRoot { + accessKey, secretKey = globalActiveCred.AccessKey, globalActiveCred.SecretKey + } else { + setupKMSUser(t, userAccessKey, userSecretKey, test.policy) + accessKey = userAccessKey + secretKey = userSecretKey + } + + req := buildKMSRequest(t, test.method, test.path, accessKey, secretKey, test.query) + rec := httptest.NewRecorder() + adminTestBed.router.ServeHTTP(rec, req) + + t.Logf("HTTP req: %s, resp code: %d, resp body: %s", req.URL.String(), rec.Code, rec.Body.String()) + + // Check status code + if rec.Code != test.wantStatusCode { + t.Errorf("want status code %d, got %d", test.wantStatusCode, rec.Code) + } + + // Check returned key list is correct + if test.wantKeyNames != nil { + keys := []madmin.KMSKeyInfo{} + err := json.Unmarshal(rec.Body.Bytes(), &keys) + if err != nil { + t.Fatal(err) + } + if len(keys) != len(test.wantKeyNames) { + t.Fatalf("want %d keys, got %d", len(test.wantKeyNames), len(keys)) + } + + for i, want := range keys { + if want.CreatedBy != kms.StubCreatedBy { + t.Fatalf("want key created by %s, got %s", kms.StubCreatedBy, want.CreatedBy) + } + if want.CreatedAt != kms.StubCreatedAt { + t.Fatalf("want key created at %s, got %s", kms.StubCreatedAt, want.CreatedAt) + } + if test.wantKeyNames[i] != want.Name { + t.Fatalf("want key name %s, got %s", test.wantKeyNames[i], want.Name) + } + } + } + + // Check generic text in the response + if test.wantResp != nil { + for _, want := range test.wantResp { + if !strings.Contains(rec.Body.String(), want) { + t.Fatalf("want response to contain %s, got %s", want, rec.Body.String()) + } + } + } +} + +// TestKMSHandlerNotConfiguredOrInvalidCreds tests KMS handlers for situations where KMS is not configured +// or invalid credentials are provided. +func TestKMSHandlerNotConfiguredOrInvalidCreds(t *testing.T) { + adminTestBed, tearDown := setupKMSTest(t, false) + defer tearDown() + + tests := []struct { + name string + method string + path string + query map[string]string + }{ + { + name: "GET status", + method: http.MethodGet, + path: kmsStatusPath, + }, + { + name: "GET metrics", + method: http.MethodGet, + path: kmsMetricsPath, + }, + { + name: "GET apis", + method: http.MethodGet, + path: kmsAPIsPath, + }, + { + name: "GET version", + method: http.MethodGet, + path: kmsVersionPath, + }, + { + name: "POST key create", + method: http.MethodPost, + path: kmsKeyCreatePath, + query: map[string]string{"key-id": "master-key-id"}, + }, + { + name: "GET key list", + method: http.MethodGet, + path: kmsKeyListPath, + query: map[string]string{"pattern": "*"}, + }, + { + name: "GET key status", + method: http.MethodGet, + path: kmsKeyStatusPath, + query: map[string]string{"key-id": "master-key-id"}, + }, + } + + // Test when the GlobalKMS is not configured + for _, test := range tests { + t.Run(test.name+" not configured", func(t *testing.T) { + req := buildKMSRequest(t, test.method, test.path, "", "", test.query) + rec := httptest.NewRecorder() + adminTestBed.router.ServeHTTP(rec, req) + if rec.Code != http.StatusNotImplemented { + t.Errorf("want status code %d, got %d", http.StatusNotImplemented, rec.Code) + } + }) + } + + // Test when the GlobalKMS is configured but the credentials are invalid + GlobalKMS = kms.NewStub("default-test-key") + for _, test := range tests { + t.Run(test.name+" invalid credentials", func(t *testing.T) { + req := buildKMSRequest(t, test.method, test.path, userAccessKey, userSecretKey, test.query) + rec := httptest.NewRecorder() + adminTestBed.router.ServeHTTP(rec, req) + if rec.Code != http.StatusForbidden { + t.Errorf("want status code %d, got %d", http.StatusForbidden, rec.Code) + } + }) + } +} + +func setupKMSTest(t *testing.T, enableKMS bool) (*adminErasureTestBed, func()) { + adminTestBed, err := prepareAdminErasureTestBed(t.Context()) + if err != nil { + t.Fatal(err) + } + registerKMSRouter(adminTestBed.router) + + if enableKMS { + GlobalKMS = kms.NewStub("default-test-key") + } + + tearDown := func() { + adminTestBed.TearDown() + GlobalKMS = nil + } + return adminTestBed, tearDown +} + +func buildKMSRequest(t *testing.T, method, path, accessKey, secretKey string, query map[string]string) *http.Request { + if len(query) > 0 { + queryVal := url.Values{} + for k, v := range query { + queryVal.Add(k, v) + } + path = path + "?" + queryVal.Encode() + } + + if accessKey == "" && secretKey == "" { + accessKey = globalActiveCred.AccessKey + secretKey = globalActiveCred.SecretKey + } + + req, err := newTestSignedRequestV4(method, path, 0, nil, accessKey, secretKey, nil) + if err != nil { + t.Fatal(err) + } + return req +} + +// setupKMSUser is a test helper that creates a new user with the provided access key and secret key +// and applies the given policy to the user. +func setupKMSUser(t *testing.T, accessKey, secretKey, p string) { + ctx := t.Context() + createUserParams := madmin.AddOrUpdateUserReq{ + SecretKey: secretKey, + Status: madmin.AccountEnabled, + } + _, err := globalIAMSys.CreateUser(ctx, accessKey, createUserParams) + if err != nil { + t.Fatal(err) + } + + testKMSPolicyName := "testKMSPolicy" + if p != "" { + p = `{"Version":"2012-10-17","Statement":[` + p + `]}` + policyData, err := policy.ParseConfig(strings.NewReader(p)) + if err != nil { + t.Fatal(err) + } + _, err = globalIAMSys.SetPolicy(ctx, testKMSPolicyName, *policyData) + if err != nil { + t.Fatal(err) + } + _, err = globalIAMSys.PolicyDBSet(ctx, accessKey, testKMSPolicyName, regUser, false) + if err != nil { + t.Fatal(err) + } + } else { + err = globalIAMSys.DeletePolicy(ctx, testKMSPolicyName, false) + if err != nil { + t.Fatal(err) + } + _, err = globalIAMSys.PolicyDBSet(ctx, accessKey, "", regUser, false) + if err != nil { + t.Fatal(err) + } + } +} diff --git a/cmd/kms-router.go b/cmd/kms-router.go new file mode 100644 index 00000000..2428f4c1 --- /dev/null +++ b/cmd/kms-router.go @@ -0,0 +1,67 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "net/http" + + "github.com/klauspost/compress/gzhttp" + "github.com/klauspost/compress/gzip" + "github.com/minio/minio/internal/logger" + "github.com/minio/mux" +) + +const ( + kmsPathPrefix = minioReservedBucketPath + "/kms" + kmsAPIVersion = "v1" + kmsAPIVersionPrefix = SlashSeparator + kmsAPIVersion +) + +type kmsAPIHandlers struct{} + +// registerKMSRouter - Registers KMS APIs +func registerKMSRouter(router *mux.Router) { + kmsAPI := kmsAPIHandlers{} + kmsRouter := router.PathPrefix(kmsPathPrefix).Subrouter() + + KMSVersions := []string{ + kmsAPIVersionPrefix, + } + + gz, err := gzhttp.NewWrapper(gzhttp.MinSize(1000), gzhttp.CompressionLevel(gzip.BestSpeed)) + if err != nil { + // Static params, so this is very unlikely. + logger.Fatal(err, "Unable to initialize server") + } + + for _, version := range KMSVersions { + // KMS Status APIs + kmsRouter.Methods(http.MethodGet).Path(version + "/status").HandlerFunc(gz(httpTraceAll(kmsAPI.KMSStatusHandler))) + kmsRouter.Methods(http.MethodGet).Path(version + "/metrics").HandlerFunc(gz(httpTraceAll(kmsAPI.KMSMetricsHandler))) + kmsRouter.Methods(http.MethodGet).Path(version + "/apis").HandlerFunc(gz(httpTraceAll(kmsAPI.KMSAPIsHandler))) + kmsRouter.Methods(http.MethodGet).Path(version + "/version").HandlerFunc(gz(httpTraceAll(kmsAPI.KMSVersionHandler))) + // KMS Key APIs + kmsRouter.Methods(http.MethodPost).Path(version+"/key/create").HandlerFunc(gz(httpTraceAll(kmsAPI.KMSCreateKeyHandler))).Queries("key-id", "{key-id:.*}") + kmsRouter.Methods(http.MethodGet).Path(version+"/key/list").HandlerFunc(gz(httpTraceAll(kmsAPI.KMSListKeysHandler))).Queries("pattern", "{pattern:.*}") + kmsRouter.Methods(http.MethodGet).Path(version + "/key/status").HandlerFunc(gz(httpTraceAll(kmsAPI.KMSKeyStatusHandler))) + } + + // If none of the routes match add default error handler routes + kmsRouter.NotFoundHandler = httpTraceAll(errorResponseHandler) + kmsRouter.MethodNotAllowedHandler = httpTraceAll(methodNotAllowedHandler("KMS")) +} diff --git a/cmd/last-minute.go b/cmd/last-minute.go new file mode 100644 index 00000000..aff5965e --- /dev/null +++ b/cmd/last-minute.go @@ -0,0 +1,208 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//go:generate msgp -file=$GOFILE -unexported + +package cmd + +import ( + "time" + + "github.com/minio/madmin-go/v3" +) + +const ( + sizeLessThan1KiB = iota + sizeLessThan1MiB + sizeLessThan10MiB + sizeLessThan100MiB + sizeLessThan1GiB + sizeGreaterThan1GiB + // Add new entries here + + sizeLastElemMarker +) + +// sizeToTag converts a size to a tag. +func sizeToTag(size int64) int { + switch { + case size < 1024: + return sizeLessThan1KiB + case size < 1024*1024: + return sizeLessThan1MiB + case size < 10*1024*1024: + return sizeLessThan10MiB + case size < 100*1024*1024: + return sizeLessThan100MiB + case size < 1024*1024*1024: + return sizeLessThan1GiB + default: + return sizeGreaterThan1GiB + } +} + +func sizeTagToString(tag int) string { + switch tag { + case sizeLessThan1KiB: + return "LESS_THAN_1_KiB" + case sizeLessThan1MiB: + return "LESS_THAN_1_MiB" + case sizeLessThan10MiB: + return "LESS_THAN_10_MiB" + case sizeLessThan100MiB: + return "LESS_THAN_100_MiB" + case sizeLessThan1GiB: + return "LESS_THAN_1_GiB" + case sizeGreaterThan1GiB: + return "GREATER_THAN_1_GiB" + default: + return "unknown" + } +} + +// AccElem holds information for calculating an average value +type AccElem struct { + Total int64 + Size int64 + N int64 +} + +// Add a duration to a single element. +func (a *AccElem) add(dur time.Duration) { + if dur < 0 { + dur = 0 + } + a.Total += int64(dur) + a.N++ +} + +// Merge b into a. +func (a *AccElem) merge(b AccElem) { + a.N += b.N + a.Total += b.Total + a.Size += b.Size +} + +// Avg returns average time spent. +func (a AccElem) avg() time.Duration { + if a.N >= 1 && a.Total > 0 { + return time.Duration(a.Total / a.N) + } + return 0 +} + +// asTimedAction returns the element as a madmin.TimedAction. +func (a AccElem) asTimedAction() madmin.TimedAction { + return madmin.TimedAction{AccTime: uint64(a.Total), Count: uint64(a.N), Bytes: uint64(a.Size)} +} + +// lastMinuteLatency keeps track of last minute latency. +type lastMinuteLatency struct { + Totals [60]AccElem + LastSec int64 +} + +// Merge data of two lastMinuteLatency structure +func (l lastMinuteLatency) merge(o lastMinuteLatency) (merged lastMinuteLatency) { + if l.LastSec > o.LastSec { + o.forwardTo(l.LastSec) + merged.LastSec = l.LastSec + } else { + l.forwardTo(o.LastSec) + merged.LastSec = o.LastSec + } + + for i := range merged.Totals { + merged.Totals[i] = AccElem{ + Total: l.Totals[i].Total + o.Totals[i].Total, + N: l.Totals[i].N + o.Totals[i].N, + Size: l.Totals[i].Size + o.Totals[i].Size, + } + } + return merged +} + +// Add a new duration data +func (l *lastMinuteLatency) add(t time.Duration) { + sec := time.Now().Unix() + l.forwardTo(sec) + winIdx := sec % 60 + l.Totals[winIdx].add(t) + l.LastSec = sec +} + +// Add a new duration data +func (l *lastMinuteLatency) addAll(sec int64, a AccElem) { + l.forwardTo(sec) + winIdx := sec % 60 + l.Totals[winIdx].merge(a) + l.LastSec = sec +} + +// Merge all recorded latencies of last minute into one +func (l *lastMinuteLatency) getTotal() AccElem { + var res AccElem + sec := time.Now().Unix() + l.forwardTo(sec) + for _, elem := range l.Totals[:] { + res.merge(elem) + } + return res +} + +// forwardTo time t, clearing any entries in between. +func (l *lastMinuteLatency) forwardTo(t int64) { + if l.LastSec >= t { + return + } + if t-l.LastSec >= 60 { + l.Totals = [60]AccElem{} + return + } + for l.LastSec != t { + // Clear next element. + idx := (l.LastSec + 1) % 60 + l.Totals[idx] = AccElem{} + l.LastSec++ + } +} + +// LastMinuteHistogram keeps track of last minute sizes added. +type LastMinuteHistogram [sizeLastElemMarker]lastMinuteLatency + +// Merge safely merges two LastMinuteHistogram structures into one +func (l LastMinuteHistogram) Merge(o LastMinuteHistogram) (merged LastMinuteHistogram) { + for i := range l { + merged[i] = l[i].merge(o[i]) + } + return merged +} + +// Add latency t from object with the specified size. +func (l *LastMinuteHistogram) Add(size int64, t time.Duration) { + l[sizeToTag(size)].add(t) +} + +// GetAvgData will return the average for each bucket from the last time minute. +// The number of objects is also included. +func (l *LastMinuteHistogram) GetAvgData() [sizeLastElemMarker]AccElem { + var res [sizeLastElemMarker]AccElem + for i, elem := range l[:] { + res[i] = elem.getTotal() + } + return res +} diff --git a/cmd/last-minute_gen.go b/cmd/last-minute_gen.go new file mode 100644 index 00000000..4e0d6661 --- /dev/null +++ b/cmd/last-minute_gen.go @@ -0,0 +1,728 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *AccElem) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Total": + z.Total, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Total") + return + } + case "Size": + z.Size, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + case "N": + z.N, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "N") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z AccElem) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "Total" + err = en.Append(0x83, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c) + if err != nil { + return + } + err = en.WriteInt64(z.Total) + if err != nil { + err = msgp.WrapError(err, "Total") + return + } + // write "Size" + err = en.Append(0xa4, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.Size) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + // write "N" + err = en.Append(0xa1, 0x4e) + if err != nil { + return + } + err = en.WriteInt64(z.N) + if err != nil { + err = msgp.WrapError(err, "N") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z AccElem) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "Total" + o = append(o, 0x83, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c) + o = msgp.AppendInt64(o, z.Total) + // string "Size" + o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.Size) + // string "N" + o = append(o, 0xa1, 0x4e) + o = msgp.AppendInt64(o, z.N) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *AccElem) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Total": + z.Total, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Total") + return + } + case "Size": + z.Size, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + case "N": + z.N, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "N") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z AccElem) Msgsize() (s int) { + s = 1 + 6 + msgp.Int64Size + 5 + msgp.Int64Size + 2 + msgp.Int64Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *LastMinuteHistogram) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0001 uint32 + zb0001, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != uint32(sizeLastElemMarker) { + err = msgp.ArrayError{Wanted: uint32(sizeLastElemMarker), Got: zb0001} + return + } + for za0001 := range z { + var field []byte + _ = field + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, za0001) + return + } + for zb0002 > 0 { + zb0002-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, za0001) + return + } + switch msgp.UnsafeString(field) { + case "Totals": + var zb0003 uint32 + zb0003, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, za0001, "Totals") + return + } + if zb0003 != uint32(60) { + err = msgp.ArrayError{Wanted: uint32(60), Got: zb0003} + return + } + for za0002 := range z[za0001].Totals { + var zb0004 uint32 + zb0004, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, za0001, "Totals", za0002) + return + } + for zb0004 > 0 { + zb0004-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, za0001, "Totals", za0002) + return + } + switch msgp.UnsafeString(field) { + case "Total": + z[za0001].Totals[za0002].Total, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, za0001, "Totals", za0002, "Total") + return + } + case "Size": + z[za0001].Totals[za0002].Size, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, za0001, "Totals", za0002, "Size") + return + } + case "N": + z[za0001].Totals[za0002].N, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, za0001, "Totals", za0002, "N") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, za0001, "Totals", za0002) + return + } + } + } + } + case "LastSec": + z[za0001].LastSec, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, za0001, "LastSec") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, za0001) + return + } + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *LastMinuteHistogram) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteArrayHeader(uint32(sizeLastElemMarker)) + if err != nil { + err = msgp.WrapError(err) + return + } + for za0001 := range z { + // map header, size 2 + // write "Totals" + err = en.Append(0x82, 0xa6, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(60)) + if err != nil { + err = msgp.WrapError(err, za0001, "Totals") + return + } + for za0002 := range z[za0001].Totals { + // map header, size 3 + // write "Total" + err = en.Append(0x83, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c) + if err != nil { + return + } + err = en.WriteInt64(z[za0001].Totals[za0002].Total) + if err != nil { + err = msgp.WrapError(err, za0001, "Totals", za0002, "Total") + return + } + // write "Size" + err = en.Append(0xa4, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z[za0001].Totals[za0002].Size) + if err != nil { + err = msgp.WrapError(err, za0001, "Totals", za0002, "Size") + return + } + // write "N" + err = en.Append(0xa1, 0x4e) + if err != nil { + return + } + err = en.WriteInt64(z[za0001].Totals[za0002].N) + if err != nil { + err = msgp.WrapError(err, za0001, "Totals", za0002, "N") + return + } + } + // write "LastSec" + err = en.Append(0xa7, 0x4c, 0x61, 0x73, 0x74, 0x53, 0x65, 0x63) + if err != nil { + return + } + err = en.WriteInt64(z[za0001].LastSec) + if err != nil { + err = msgp.WrapError(err, za0001, "LastSec") + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *LastMinuteHistogram) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendArrayHeader(o, uint32(sizeLastElemMarker)) + for za0001 := range z { + // map header, size 2 + // string "Totals" + o = append(o, 0x82, 0xa6, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73) + o = msgp.AppendArrayHeader(o, uint32(60)) + for za0002 := range z[za0001].Totals { + // map header, size 3 + // string "Total" + o = append(o, 0x83, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c) + o = msgp.AppendInt64(o, z[za0001].Totals[za0002].Total) + // string "Size" + o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z[za0001].Totals[za0002].Size) + // string "N" + o = append(o, 0xa1, 0x4e) + o = msgp.AppendInt64(o, z[za0001].Totals[za0002].N) + } + // string "LastSec" + o = append(o, 0xa7, 0x4c, 0x61, 0x73, 0x74, 0x53, 0x65, 0x63) + o = msgp.AppendInt64(o, z[za0001].LastSec) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *LastMinuteHistogram) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0001 uint32 + zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != uint32(sizeLastElemMarker) { + err = msgp.ArrayError{Wanted: uint32(sizeLastElemMarker), Got: zb0001} + return + } + for za0001 := range z { + var field []byte + _ = field + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, za0001) + return + } + for zb0002 > 0 { + zb0002-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, za0001) + return + } + switch msgp.UnsafeString(field) { + case "Totals": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, za0001, "Totals") + return + } + if zb0003 != uint32(60) { + err = msgp.ArrayError{Wanted: uint32(60), Got: zb0003} + return + } + for za0002 := range z[za0001].Totals { + var zb0004 uint32 + zb0004, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, za0001, "Totals", za0002) + return + } + for zb0004 > 0 { + zb0004-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, za0001, "Totals", za0002) + return + } + switch msgp.UnsafeString(field) { + case "Total": + z[za0001].Totals[za0002].Total, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, za0001, "Totals", za0002, "Total") + return + } + case "Size": + z[za0001].Totals[za0002].Size, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, za0001, "Totals", za0002, "Size") + return + } + case "N": + z[za0001].Totals[za0002].N, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, za0001, "Totals", za0002, "N") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, za0001, "Totals", za0002) + return + } + } + } + } + case "LastSec": + z[za0001].LastSec, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, za0001, "LastSec") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, za0001) + return + } + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *LastMinuteHistogram) Msgsize() (s int) { + s = msgp.ArrayHeaderSize + (sizeLastElemMarker * (16 + (60 * (14 + msgp.Int64Size + msgp.Int64Size + msgp.Int64Size)) + msgp.Int64Size)) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *lastMinuteLatency) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Totals": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Totals") + return + } + if zb0002 != uint32(60) { + err = msgp.ArrayError{Wanted: uint32(60), Got: zb0002} + return + } + for za0001 := range z.Totals { + var zb0003 uint32 + zb0003, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Totals", za0001) + return + } + for zb0003 > 0 { + zb0003-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "Totals", za0001) + return + } + switch msgp.UnsafeString(field) { + case "Total": + z.Totals[za0001].Total, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Totals", za0001, "Total") + return + } + case "Size": + z.Totals[za0001].Size, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Totals", za0001, "Size") + return + } + case "N": + z.Totals[za0001].N, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Totals", za0001, "N") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "Totals", za0001) + return + } + } + } + } + case "LastSec": + z.LastSec, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "LastSec") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *lastMinuteLatency) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "Totals" + err = en.Append(0x82, 0xa6, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(60)) + if err != nil { + err = msgp.WrapError(err, "Totals") + return + } + for za0001 := range z.Totals { + // map header, size 3 + // write "Total" + err = en.Append(0x83, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c) + if err != nil { + return + } + err = en.WriteInt64(z.Totals[za0001].Total) + if err != nil { + err = msgp.WrapError(err, "Totals", za0001, "Total") + return + } + // write "Size" + err = en.Append(0xa4, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.Totals[za0001].Size) + if err != nil { + err = msgp.WrapError(err, "Totals", za0001, "Size") + return + } + // write "N" + err = en.Append(0xa1, 0x4e) + if err != nil { + return + } + err = en.WriteInt64(z.Totals[za0001].N) + if err != nil { + err = msgp.WrapError(err, "Totals", za0001, "N") + return + } + } + // write "LastSec" + err = en.Append(0xa7, 0x4c, 0x61, 0x73, 0x74, 0x53, 0x65, 0x63) + if err != nil { + return + } + err = en.WriteInt64(z.LastSec) + if err != nil { + err = msgp.WrapError(err, "LastSec") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *lastMinuteLatency) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "Totals" + o = append(o, 0x82, 0xa6, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x73) + o = msgp.AppendArrayHeader(o, uint32(60)) + for za0001 := range z.Totals { + // map header, size 3 + // string "Total" + o = append(o, 0x83, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c) + o = msgp.AppendInt64(o, z.Totals[za0001].Total) + // string "Size" + o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.Totals[za0001].Size) + // string "N" + o = append(o, 0xa1, 0x4e) + o = msgp.AppendInt64(o, z.Totals[za0001].N) + } + // string "LastSec" + o = append(o, 0xa7, 0x4c, 0x61, 0x73, 0x74, 0x53, 0x65, 0x63) + o = msgp.AppendInt64(o, z.LastSec) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *lastMinuteLatency) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Totals": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Totals") + return + } + if zb0002 != uint32(60) { + err = msgp.ArrayError{Wanted: uint32(60), Got: zb0002} + return + } + for za0001 := range z.Totals { + var zb0003 uint32 + zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Totals", za0001) + return + } + for zb0003 > 0 { + zb0003-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "Totals", za0001) + return + } + switch msgp.UnsafeString(field) { + case "Total": + z.Totals[za0001].Total, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Totals", za0001, "Total") + return + } + case "Size": + z.Totals[za0001].Size, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Totals", za0001, "Size") + return + } + case "N": + z.Totals[za0001].N, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Totals", za0001, "N") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "Totals", za0001) + return + } + } + } + } + case "LastSec": + z.LastSec, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "LastSec") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *lastMinuteLatency) Msgsize() (s int) { + s = 1 + 7 + msgp.ArrayHeaderSize + (60 * (14 + msgp.Int64Size + msgp.Int64Size + msgp.Int64Size)) + 8 + msgp.Int64Size + return +} diff --git a/cmd/last-minute_gen_test.go b/cmd/last-minute_gen_test.go new file mode 100644 index 00000000..de684551 --- /dev/null +++ b/cmd/last-minute_gen_test.go @@ -0,0 +1,349 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalAccElem(t *testing.T) { + v := AccElem{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgAccElem(b *testing.B) { + v := AccElem{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgAccElem(b *testing.B) { + v := AccElem{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalAccElem(b *testing.B) { + v := AccElem{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeAccElem(t *testing.T) { + v := AccElem{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeAccElem Msgsize() is inaccurate") + } + + vn := AccElem{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeAccElem(b *testing.B) { + v := AccElem{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeAccElem(b *testing.B) { + v := AccElem{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalLastMinuteHistogram(t *testing.T) { + v := LastMinuteHistogram{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgLastMinuteHistogram(b *testing.B) { + v := LastMinuteHistogram{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgLastMinuteHistogram(b *testing.B) { + v := LastMinuteHistogram{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalLastMinuteHistogram(b *testing.B) { + v := LastMinuteHistogram{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeLastMinuteHistogram(t *testing.T) { + v := LastMinuteHistogram{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeLastMinuteHistogram Msgsize() is inaccurate") + } + + vn := LastMinuteHistogram{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeLastMinuteHistogram(b *testing.B) { + v := LastMinuteHistogram{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeLastMinuteHistogram(b *testing.B) { + v := LastMinuteHistogram{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshallastMinuteLatency(t *testing.T) { + v := lastMinuteLatency{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsglastMinuteLatency(b *testing.B) { + v := lastMinuteLatency{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsglastMinuteLatency(b *testing.B) { + v := lastMinuteLatency{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshallastMinuteLatency(b *testing.B) { + v := lastMinuteLatency{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodelastMinuteLatency(t *testing.T) { + v := lastMinuteLatency{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodelastMinuteLatency Msgsize() is inaccurate") + } + + vn := lastMinuteLatency{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodelastMinuteLatency(b *testing.B) { + v := lastMinuteLatency{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodelastMinuteLatency(b *testing.B) { + v := lastMinuteLatency{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/lceventsrc_string.go b/cmd/lceventsrc_string.go new file mode 100644 index 00000000..dd121abb --- /dev/null +++ b/cmd/lceventsrc_string.go @@ -0,0 +1,33 @@ +// Code generated by "stringer -type lcEventSrc -trimprefix lcEventSrc_ bucket-lifecycle-audit.go"; DO NOT EDIT. + +package cmd + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[lcEventSrc_None-0] + _ = x[lcEventSrc_Heal-1] + _ = x[lcEventSrc_Scanner-2] + _ = x[lcEventSrc_Decom-3] + _ = x[lcEventSrc_Rebal-4] + _ = x[lcEventSrc_s3HeadObject-5] + _ = x[lcEventSrc_s3GetObject-6] + _ = x[lcEventSrc_s3ListObjects-7] + _ = x[lcEventSrc_s3PutObject-8] + _ = x[lcEventSrc_s3CopyObject-9] + _ = x[lcEventSrc_s3CompleteMultipartUpload-10] +} + +const _lcEventSrc_name = "NoneHealScannerDecomRebals3HeadObjects3GetObjects3ListObjectss3PutObjects3CopyObjects3CompleteMultipartUpload" + +var _lcEventSrc_index = [...]uint8{0, 4, 8, 15, 20, 25, 37, 48, 61, 72, 84, 109} + +func (i lcEventSrc) String() string { + if i >= lcEventSrc(len(_lcEventSrc_index)-1) { + return "lcEventSrc(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _lcEventSrc_name[_lcEventSrc_index[i]:_lcEventSrc_index[i+1]] +} diff --git a/cmd/leak-detect_test.go b/cmd/leak-detect_test.go new file mode 100644 index 00000000..95b98c4e --- /dev/null +++ b/cmd/leak-detect_test.go @@ -0,0 +1,163 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "runtime/debug" + "sort" + "strings" + "time" +) + +const ( + // deadline (in seconds) up to which the go routine leak detection has to be retried. + leakDetectDeadline = 5 + // pause time (in milliseconds) between each snapshot at the end of the go routine leak detection. + leakDetectPauseTimeMs = 50 +) + +// LeakDetect - type with methods for go routine leak detection. +type LeakDetect struct { + relevantRoutines map[string]bool +} + +// NewLeakDetect - Initialize a LeakDetector with the snapshot of relevant Go routines. +func NewLeakDetect() LeakDetect { + snapshot := LeakDetect{ + relevantRoutines: make(map[string]bool), + } + for _, g := range pickRelevantGoroutines() { + snapshot.relevantRoutines[g] = true + } + return snapshot +} + +// CompareCurrentSnapshot - Compares the initial relevant stack trace with the current one (during the time of invocation). +func (initialSnapShot LeakDetect) CompareCurrentSnapshot() []string { + var stackDiff []string + for _, g := range pickRelevantGoroutines() { + // Identify the Go routines those were not present in the initial snapshot. + // In other words a stack diff. + if !initialSnapShot.relevantRoutines[g] { + stackDiff = append(stackDiff, g) + } + } + return stackDiff +} + +// DetectLeak - Creates a snapshot of runtime stack and compares it with the initial stack snapshot. +func (initialSnapShot LeakDetect) DetectLeak(t TestErrHandler) { + if t.Failed() { + return + } + // Loop, waiting for goroutines to shut down. + // Wait up to 5 seconds, but finish as quickly as possible. + deadline := UTCNow().Add(leakDetectDeadline * time.Second) + for { + // get sack snapshot of relevant go routines. + leaked := initialSnapShot.CompareCurrentSnapshot() + // current stack snapshot matches the initial one, no leaks, return. + if len(leaked) == 0 { + return + } + // wait a test again will deadline. + if UTCNow().Before(deadline) { + time.Sleep(leakDetectPauseTimeMs * time.Millisecond) + continue + } + // after the deadline time report all the difference in the latest snapshot compared with the initial one. + for _, g := range leaked { + t.Errorf("Leaked goroutine: %v", g) + } + return + } +} + +// DetectTestLeak - snapshots the currently running goroutines and returns a +// function to be run at the end of tests to see whether any +// goroutines leaked. +// Usage: `defer DetectTestLeak(t)()` in beginning line of benchmarks or unit tests. +func DetectTestLeak(t TestErrHandler) func() { + initialStackSnapShot := NewLeakDetect() + return func() { + initialStackSnapShot.DetectLeak(t) + } +} + +// list of functions to be ignored from the stack trace. +// Leak detection is done when tests are run, should ignore the tests related functions, +// and other runtime functions while identifying leaks. +var ignoredStackFns = []string{ + "", + // Below are the stacks ignored by the upstream leaktest code. + "testing.Main(", + "testing.tRunner(", + "testing.tRunner(", + "runtime.goexit", + "created by runtime.gc", + // ignore the snapshot function. + // since the snapshot is taken here the entry will have the current function too. + "pickRelevantGoroutines", + "runtime.MHeap_Scavenger", + "signal.signal_recv", + "sigterm.handler", + "runtime_mcall", + "goroutine in C code", +} + +// Identify whether the stack trace entry is part of ignoredStackFn . +func isIgnoredStackFn(stack string) (ok bool) { + ok = true + for _, stackFn := range ignoredStackFns { + if !strings.Contains(stack, stackFn) { + ok = false + continue + } + break + } + return ok +} + +// pickRelevantGoroutines returns all goroutines we care about for the purpose +// of leak checking. It excludes testing or runtime ones. +func pickRelevantGoroutines() (gs []string) { + // get runtime stack buffer. + buf := debug.Stack() + // runtime stack of go routines will be listed with 2 blank spaces between each of them, so split on "\n\n" . + for g := range strings.SplitSeq(string(buf), "\n\n") { + // Again split on a new line, the first line of the second half contains the info about the go routine. + sl := strings.SplitN(g, "\n", 2) + if len(sl) != 2 { + continue + } + stack := strings.TrimSpace(sl[1]) + // ignore the testing go routine. + // since the tests will be invoking the leaktest it would contain the test go routine. + if strings.HasPrefix(stack, "testing.RunTests") { + continue + } + // Ignore the following go routines. + // testing and run time go routines should be ignored, only the application generated go routines should be taken into account. + if isIgnoredStackFn(stack) { + continue + } + gs = append(gs, g) + } + sort.Strings(gs) + return gs +} diff --git a/cmd/listen-notification-handlers.go b/cmd/listen-notification-handlers.go new file mode 100644 index 00000000..9f3210da --- /dev/null +++ b/cmd/listen-notification-handlers.go @@ -0,0 +1,221 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "encoding/json" + "net/http" + "strconv" + "time" + + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/grid" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/pubsub" + "github.com/minio/mux" + "github.com/minio/pkg/v3/policy" +) + +func (api objectAPIHandlers) ListenNotificationHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "ListenNotification") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + // Validate if bucket exists. + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + vars := mux.Vars(r) + bucketName := vars["bucket"] + + if bucketName == "" { + if s3Error := checkRequestAuthType(ctx, r, policy.ListenNotificationAction, bucketName, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + } else { + if s3Error := checkRequestAuthType(ctx, r, policy.ListenBucketNotificationAction, bucketName, ""); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + } + + values := r.Form + + var prefix string + if len(values[peerRESTListenPrefix]) > 1 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrFilterNamePrefix), r.URL) + return + } + + if len(values[peerRESTListenPrefix]) == 1 { + if err := event.ValidateFilterRuleValue(values[peerRESTListenPrefix][0]); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + prefix = values[peerRESTListenPrefix][0] + } + + var suffix string + if len(values[peerRESTListenSuffix]) > 1 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrFilterNameSuffix), r.URL) + return + } + + if len(values[peerRESTListenSuffix]) == 1 { + if err := event.ValidateFilterRuleValue(values[peerRESTListenSuffix][0]); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + suffix = values[peerRESTListenSuffix][0] + } + + pattern := event.NewPattern(prefix, suffix) + + var eventNames []event.Name + var mask pubsub.Mask + for _, s := range values[peerRESTListenEvents] { + eventName, err := event.ParseName(s) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + mask.MergeMaskable(eventName) + eventNames = append(eventNames, eventName) + } + + if bucketName != "" { + if _, err := objAPI.GetBucketInfo(ctx, bucketName, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + + rulesMap := event.NewRulesMap(eventNames, pattern, event.TargetID{ID: mustGetUUID()}) + + setEventStreamHeaders(w) + + // Listen Publisher and peer-listen-client uses nonblocking send and hence does not wait for slow receivers. + // Use buffered channel to take care of burst sends or slow w.Write() + mergeCh := make(chan []byte, globalAPIConfig.getRequestsPoolCapacity()*len(globalEndpoints.Hostnames())) + localCh := make(chan event.Event, globalAPIConfig.getRequestsPoolCapacity()) + + // Convert local messages to JSON and send to mergeCh + go func() { + buf := bytes.NewBuffer(grid.GetByteBuffer()[:0]) + enc := json.NewEncoder(buf) + tmpEvt := struct{ Records []event.Event }{[]event.Event{{}}} + for { + select { + case ev := <-localCh: + buf.Reset() + tmpEvt.Records[0] = ev + if err := enc.Encode(tmpEvt); err != nil { + bugLogIf(ctx, err, "event: Encode failed") + continue + } + mergeCh <- append(grid.GetByteBuffer()[:0], buf.Bytes()...) + case <-ctx.Done(): + grid.PutByteBuffer(buf.Bytes()) + return + } + } + }() + peers, _ := newPeerRestClients(globalEndpoints) + err := globalHTTPListen.Subscribe(mask, localCh, ctx.Done(), func(ev event.Event) bool { + if ev.S3.Bucket.Name != "" && bucketName != "" { + if ev.S3.Bucket.Name != bucketName { + return false + } + } + return rulesMap.MatchSimple(ev.EventName, ev.S3.Object.Key) + }) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if bucketName != "" { + values.Set(peerRESTListenBucket, bucketName) + } + for _, peer := range peers { + if peer == nil { + continue + } + peer.Listen(ctx, mergeCh, values) + } + + var ( + emptyEventTicker <-chan time.Time + keepAliveTicker <-chan time.Time + ) + + if p := values.Get("ping"); p != "" { + pingInterval, err := strconv.Atoi(p) + if err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidQueryParams), r.URL) + return + } + if pingInterval < 1 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidQueryParams), r.URL) + return + } + t := time.NewTicker(time.Duration(pingInterval) * time.Second) + defer t.Stop() + emptyEventTicker = t.C + } else { + // Deprecated Apr 2023 + t := time.NewTicker(500 * time.Millisecond) + defer t.Stop() + keepAliveTicker = t.C + } + + enc := json.NewEncoder(w) + for { + select { + case ev := <-mergeCh: + _, err := w.Write(ev) + if err != nil { + return + } + if len(mergeCh) == 0 { + // Flush if nothing is queued + xhttp.Flush(w) + } + grid.PutByteBuffer(ev) + case <-emptyEventTicker: + if err := enc.Encode(struct{ Records []event.Event }{}); err != nil { + return + } + xhttp.Flush(w) + case <-keepAliveTicker: + if _, err := w.Write([]byte(" ")); err != nil { + return + } + xhttp.Flush(w) + case <-ctx.Done(): + return + } + } +} diff --git a/cmd/local-locker.go b/cmd/local-locker.go new file mode 100644 index 00000000..3fd630b9 --- /dev/null +++ b/cmd/local-locker.go @@ -0,0 +1,452 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +//go:generate msgp -file=$GOFILE -unexported + +import ( + "context" + "fmt" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/minio/minio/internal/dsync" +) + +// Reject new lock requests immediately when this many are queued +// for the local lock mutex. +// We do not block unlocking or maintenance, but they add to the count. +// The limit is set to allow for bursty behavior, +// but prevent requests to overload the server completely. +// Rejected clients are expected to retry. +const lockMutexWaitLimit = 1000 + +// lockRequesterInfo stores various info from the client for each lock that is requested. +type lockRequesterInfo struct { + Name string // name of the resource lock was requested for + Writer bool // Bool whether write or read lock. + UID string // UID to uniquely identify request of client. + Timestamp int64 // Timestamp set at the time of initialization. + TimeLastRefresh int64 // Timestamp for last lock refresh. + Source string // Contains line, function and filename requesting the lock. + Group bool // indicates if it was a group lock. + Owner string // Owner represents the UUID of the owner who originally requested the lock. + Quorum int // Quorum represents the quorum required for this lock to be active. + idx int `msg:"-"` // index of the lock in the lockMap. +} + +// isWriteLock returns whether the lock is a write or read lock. +func isWriteLock(lri []lockRequesterInfo) bool { + return len(lri) == 1 && lri[0].Writer +} + +// localLocker implements Dsync.NetLocker +// +//msgp:ignore localLocker +type localLocker struct { + mutex sync.Mutex + waitMutex atomic.Int32 + lockMap map[string][]lockRequesterInfo + lockUID map[string]string // UUID -> resource map. + + // the following are updated on every cleanup defined in lockValidityDuration + readers atomic.Int32 + writers atomic.Int32 + lastCleanup atomic.Pointer[time.Time] + locksOverloaded atomic.Int64 +} + +// getMutex will lock the mutex. +// Call the returned function to unlock. +func (l *localLocker) getMutex() func() { + l.waitMutex.Add(1) + l.mutex.Lock() + l.waitMutex.Add(-1) + return l.mutex.Unlock +} + +func (l *localLocker) String() string { + return globalEndpoints.Localhost() +} + +func (l *localLocker) canTakeLock(resources ...string) bool { + for _, resource := range resources { + _, lockTaken := l.lockMap[resource] + if lockTaken { + return false + } + } + return true +} + +func (l *localLocker) Lock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) { + if len(args.Resources) > maxDeleteList { + return false, fmt.Errorf("internal error: localLocker.Lock called with more than %d resources", maxDeleteList) + } + + // If we have too many waiting, reject this at once. + if l.waitMutex.Load() > lockMutexWaitLimit { + l.locksOverloaded.Add(1) + return false, nil + } + // Wait for mutex + defer l.getMutex()() + if ctx.Err() != nil { + return false, ctx.Err() + } + if !l.canTakeLock(args.Resources...) { + // Not all locks can be taken on resources, + // reject it completely. + return false, nil + } + + // No locks held on the all resources, so claim write + // lock on all resources at once. + now := UTCNow() + for i, resource := range args.Resources { + l.lockMap[resource] = []lockRequesterInfo{ + { + Name: resource, + Writer: true, + Source: args.Source, + Owner: args.Owner, + UID: args.UID, + Timestamp: now.UnixNano(), + TimeLastRefresh: now.UnixNano(), + Group: len(args.Resources) > 1, + Quorum: *args.Quorum, + idx: i, + }, + } + l.lockUID[formatUUID(args.UID, i)] = resource + } + return true, nil +} + +func formatUUID(s string, idx int) string { + return concat(s, strconv.Itoa(idx)) +} + +func (l *localLocker) Unlock(_ context.Context, args dsync.LockArgs) (reply bool, err error) { + if len(args.Resources) > maxDeleteList { + return false, fmt.Errorf("internal error: localLocker.Unlock called with more than %d resources", maxDeleteList) + } + + defer l.getMutex()() + + for _, resource := range args.Resources { + lri, ok := l.lockMap[resource] + if ok && !isWriteLock(lri) { + // Unless it is a write lock reject it. + err = fmt.Errorf("unlock attempted on a read locked entity: %s", resource) + continue + } + if ok { + reply = l.removeEntry(resource, args, &lri) || reply + } + } + return reply, err +} + +// removeEntry based on the uid of the lock message, removes a single entry from the +// lockRequesterInfo array or the whole array from the map (in case of a write lock +// or last read lock) +// UID and optionally owner must match for entries to be deleted. +func (l *localLocker) removeEntry(name string, args dsync.LockArgs, lri *[]lockRequesterInfo) bool { + // Find correct entry to remove based on uid. + for index, entry := range *lri { + if entry.UID == args.UID && (args.Owner == "" || entry.Owner == args.Owner) { + if len(*lri) == 1 { + // Remove the write lock. + delete(l.lockMap, name) + } else { + // Remove the appropriate read lock. + *lri = append((*lri)[:index], (*lri)[index+1:]...) + l.lockMap[name] = *lri + } + delete(l.lockUID, formatUUID(args.UID, entry.idx)) + return true + } + } + + // None found return false, perhaps entry removed in previous run. + return false +} + +func (l *localLocker) RLock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) { + if len(args.Resources) != 1 { + return false, fmt.Errorf("internal error: localLocker.RLock called with more than one resource") + } + // If we have too many waiting, reject this at once. + if l.waitMutex.Load() > lockMutexWaitLimit { + l.locksOverloaded.Add(1) + return false, nil + } + + // Wait for mutex + defer l.getMutex()() + if ctx.Err() != nil { + return false, ctx.Err() + } + resource := args.Resources[0] + now := UTCNow() + lrInfo := lockRequesterInfo{ + Name: resource, + Writer: false, + Source: args.Source, + Owner: args.Owner, + UID: args.UID, + Timestamp: now.UnixNano(), + TimeLastRefresh: now.UnixNano(), + Quorum: *args.Quorum, + } + lri, ok := l.lockMap[resource] + if ok { + if reply = !isWriteLock(lri); reply { + // Unless there is a write lock + l.lockMap[resource] = append(l.lockMap[resource], lrInfo) + l.lockUID[formatUUID(args.UID, 0)] = resource + } + } else { + // No locks held on the given name, so claim (first) read lock + l.lockMap[resource] = []lockRequesterInfo{lrInfo} + l.lockUID[formatUUID(args.UID, 0)] = resource + reply = true + } + return reply, nil +} + +func (l *localLocker) RUnlock(_ context.Context, args dsync.LockArgs) (reply bool, err error) { + if len(args.Resources) > 1 { + return false, fmt.Errorf("internal error: localLocker.RUnlock called with more than one resource") + } + + defer l.getMutex()() + var lri []lockRequesterInfo + + resource := args.Resources[0] + if lri, reply = l.lockMap[resource]; !reply { + // No lock is held on the given name + return true, nil + } + if isWriteLock(lri) { + // A write-lock is held, cannot release a read lock + return false, fmt.Errorf("RUnlock attempted on a write locked entity: %s", resource) + } + l.removeEntry(resource, args, &lri) + return reply, nil +} + +type lockStats struct { + Total int + Writes int + Reads int + LockQueue int + LocksAbandoned int + LastCleanup *time.Time +} + +func (l *localLocker) stats() lockStats { + return lockStats{ + Total: len(l.lockMap), + Reads: int(l.readers.Load()), + Writes: int(l.writers.Load()), + LockQueue: int(l.waitMutex.Load()), + LastCleanup: l.lastCleanup.Load(), + } +} + +type localLockMap map[string][]lockRequesterInfo + +func (l *localLocker) DupLockMap() localLockMap { + defer l.getMutex()() + + lockCopy := make(map[string][]lockRequesterInfo, len(l.lockMap)) + for k, v := range l.lockMap { + if len(v) == 0 { + delete(l.lockMap, k) + continue + } + lockCopy[k] = append(make([]lockRequesterInfo, 0, len(v)), v...) + } + return lockCopy +} + +func (l *localLocker) Close() error { + return nil +} + +// IsOnline - local locker is always online. +func (l *localLocker) IsOnline() bool { + return true +} + +// IsLocal - local locker returns true. +func (l *localLocker) IsLocal() bool { + return true +} + +func (l *localLocker) ForceUnlock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) { + if ctx.Err() != nil { + return false, ctx.Err() + } + + defer l.getMutex()() + if ctx.Err() != nil { + return false, ctx.Err() + } + if len(args.UID) == 0 { + for _, resource := range args.Resources { + lris, ok := l.lockMap[resource] + if !ok { + continue + } + // Collect uids, so we don't mutate while we delete + uids := make([]string, 0, len(lris)) + for _, lri := range lris { + uids = append(uids, lri.UID) + } + + // Delete collected uids: + for _, uid := range uids { + lris, ok := l.lockMap[resource] + if !ok { + // Just to be safe, delete uuids. + for idx := range maxDeleteList { + mapID := formatUUID(uid, idx) + if _, ok := l.lockUID[mapID]; !ok { + break + } + delete(l.lockUID, mapID) + } + continue + } + l.removeEntry(resource, dsync.LockArgs{UID: uid}, &lris) + } + } + return true, nil + } + + idx := 0 + for { + mapID := formatUUID(args.UID, idx) + resource, ok := l.lockUID[mapID] + if !ok { + return idx > 0, nil + } + lris, ok := l.lockMap[resource] + if !ok { + // Unexpected inconsistency, delete. + delete(l.lockUID, mapID) + idx++ + continue + } + reply = true + l.removeEntry(resource, dsync.LockArgs{UID: args.UID}, &lris) + idx++ + } +} + +func (l *localLocker) Refresh(ctx context.Context, args dsync.LockArgs) (refreshed bool, err error) { + if ctx.Err() != nil { + return false, ctx.Err() + } + + defer l.getMutex()() + if ctx.Err() != nil { + return false, ctx.Err() + } + + // Check whether uid is still active. + resource, ok := l.lockUID[formatUUID(args.UID, 0)] + if !ok { + return false, nil + } + idx := 0 + for { + lris, ok := l.lockMap[resource] + if !ok { + // Inconsistent. Delete UID. + delete(l.lockUID, formatUUID(args.UID, idx)) + return idx > 0, nil + } + now := UTCNow() + for i := range lris { + if lris[i].UID == args.UID { + lris[i].TimeLastRefresh = now.UnixNano() + } + } + idx++ + resource, ok = l.lockUID[formatUUID(args.UID, idx)] + if !ok { + // No more resources for UID, but we did update at least one. + return true, nil + } + } +} + +// Similar to removeEntry but only removes an entry only if the lock entry exists in map. +// Caller must hold 'l.mutex' lock. +func (l *localLocker) expireOldLocks(interval time.Duration) { + defer l.getMutex()() + + var readers, writers int32 + for k, lris := range l.lockMap { + modified := false + for i := 0; i < len(lris); { + lri := &lris[i] + if time.Since(time.Unix(0, lri.TimeLastRefresh)) > interval { + delete(l.lockUID, formatUUID(lri.UID, lri.idx)) + if len(lris) == 1 { + // Remove the write lock. + delete(l.lockMap, lri.Name) + modified = false + break + } + modified = true + // Remove the appropriate lock. + lris = append(lris[:i], lris[i+1:]...) + // Check same i + } else { + if lri.Writer { + writers++ + } else { + readers++ + } + // Move to next + i++ + } + } + if modified { + l.lockMap[k] = lris + } + } + t := time.Now() + l.lastCleanup.Store(&t) + l.readers.Store(readers) + l.writers.Store(writers) +} + +func newLocker() *localLocker { + return &localLocker{ + lockMap: make(map[string][]lockRequesterInfo, 1000), + lockUID: make(map[string]string, 1000), + } +} diff --git a/cmd/local-locker_gen.go b/cmd/local-locker_gen.go new file mode 100644 index 00000000..41bb9254 --- /dev/null +++ b/cmd/local-locker_gen.go @@ -0,0 +1,736 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "time" + + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *localLockMap) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0004 uint32 + zb0004, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if (*z) == nil { + (*z) = make(localLockMap, zb0004) + } else if len((*z)) > 0 { + clear((*z)) + } + var field []byte + _ = field + for zb0004 > 0 { + zb0004-- + var zb0001 string + zb0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0002 []lockRequesterInfo + var zb0005 uint32 + zb0005, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, zb0001) + return + } + if cap(zb0002) >= int(zb0005) { + zb0002 = (zb0002)[:zb0005] + } else { + zb0002 = make([]lockRequesterInfo, zb0005) + } + for zb0003 := range zb0002 { + err = zb0002[zb0003].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, zb0001, zb0003) + return + } + } + (*z)[zb0001] = zb0002 + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z localLockMap) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteMapHeader(uint32(len(z))) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0006, zb0007 := range z { + err = en.WriteString(zb0006) + if err != nil { + err = msgp.WrapError(err) + return + } + err = en.WriteArrayHeader(uint32(len(zb0007))) + if err != nil { + err = msgp.WrapError(err, zb0006) + return + } + for zb0008 := range zb0007 { + err = zb0007[zb0008].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, zb0006, zb0008) + return + } + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z localLockMap) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendMapHeader(o, uint32(len(z))) + for zb0006, zb0007 := range z { + o = msgp.AppendString(o, zb0006) + o = msgp.AppendArrayHeader(o, uint32(len(zb0007))) + for zb0008 := range zb0007 { + o, err = zb0007[zb0008].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, zb0006, zb0008) + return + } + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *localLockMap) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0004 uint32 + zb0004, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if (*z) == nil { + (*z) = make(localLockMap, zb0004) + } else if len((*z)) > 0 { + clear((*z)) + } + var field []byte + _ = field + for zb0004 > 0 { + var zb0002 []lockRequesterInfo + zb0004-- + var zb0001 string + zb0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0005 uint32 + zb0005, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, zb0001) + return + } + if cap(zb0002) >= int(zb0005) { + zb0002 = (zb0002)[:zb0005] + } else { + zb0002 = make([]lockRequesterInfo, zb0005) + } + for zb0003 := range zb0002 { + bts, err = zb0002[zb0003].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, zb0001, zb0003) + return + } + } + (*z)[zb0001] = zb0002 + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z localLockMap) Msgsize() (s int) { + s = msgp.MapHeaderSize + if z != nil { + for zb0006, zb0007 := range z { + _ = zb0007 + s += msgp.StringPrefixSize + len(zb0006) + msgp.ArrayHeaderSize + for zb0008 := range zb0007 { + s += zb0007[zb0008].Msgsize() + } + } + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *lockRequesterInfo) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Name": + z.Name, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "Writer": + z.Writer, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Writer") + return + } + case "UID": + z.UID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "UID") + return + } + case "Timestamp": + z.Timestamp, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Timestamp") + return + } + case "TimeLastRefresh": + z.TimeLastRefresh, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "TimeLastRefresh") + return + } + case "Source": + z.Source, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Source") + return + } + case "Group": + z.Group, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Group") + return + } + case "Owner": + z.Owner, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Owner") + return + } + case "Quorum": + z.Quorum, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Quorum") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *lockRequesterInfo) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 9 + // write "Name" + err = en.Append(0x89, 0xa4, 0x4e, 0x61, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteString(z.Name) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + // write "Writer" + err = en.Append(0xa6, 0x57, 0x72, 0x69, 0x74, 0x65, 0x72) + if err != nil { + return + } + err = en.WriteBool(z.Writer) + if err != nil { + err = msgp.WrapError(err, "Writer") + return + } + // write "UID" + err = en.Append(0xa3, 0x55, 0x49, 0x44) + if err != nil { + return + } + err = en.WriteString(z.UID) + if err != nil { + err = msgp.WrapError(err, "UID") + return + } + // write "Timestamp" + err = en.Append(0xa9, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70) + if err != nil { + return + } + err = en.WriteInt64(z.Timestamp) + if err != nil { + err = msgp.WrapError(err, "Timestamp") + return + } + // write "TimeLastRefresh" + err = en.Append(0xaf, 0x54, 0x69, 0x6d, 0x65, 0x4c, 0x61, 0x73, 0x74, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68) + if err != nil { + return + } + err = en.WriteInt64(z.TimeLastRefresh) + if err != nil { + err = msgp.WrapError(err, "TimeLastRefresh") + return + } + // write "Source" + err = en.Append(0xa6, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65) + if err != nil { + return + } + err = en.WriteString(z.Source) + if err != nil { + err = msgp.WrapError(err, "Source") + return + } + // write "Group" + err = en.Append(0xa5, 0x47, 0x72, 0x6f, 0x75, 0x70) + if err != nil { + return + } + err = en.WriteBool(z.Group) + if err != nil { + err = msgp.WrapError(err, "Group") + return + } + // write "Owner" + err = en.Append(0xa5, 0x4f, 0x77, 0x6e, 0x65, 0x72) + if err != nil { + return + } + err = en.WriteString(z.Owner) + if err != nil { + err = msgp.WrapError(err, "Owner") + return + } + // write "Quorum" + err = en.Append(0xa6, 0x51, 0x75, 0x6f, 0x72, 0x75, 0x6d) + if err != nil { + return + } + err = en.WriteInt(z.Quorum) + if err != nil { + err = msgp.WrapError(err, "Quorum") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *lockRequesterInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 9 + // string "Name" + o = append(o, 0x89, 0xa4, 0x4e, 0x61, 0x6d, 0x65) + o = msgp.AppendString(o, z.Name) + // string "Writer" + o = append(o, 0xa6, 0x57, 0x72, 0x69, 0x74, 0x65, 0x72) + o = msgp.AppendBool(o, z.Writer) + // string "UID" + o = append(o, 0xa3, 0x55, 0x49, 0x44) + o = msgp.AppendString(o, z.UID) + // string "Timestamp" + o = append(o, 0xa9, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70) + o = msgp.AppendInt64(o, z.Timestamp) + // string "TimeLastRefresh" + o = append(o, 0xaf, 0x54, 0x69, 0x6d, 0x65, 0x4c, 0x61, 0x73, 0x74, 0x52, 0x65, 0x66, 0x72, 0x65, 0x73, 0x68) + o = msgp.AppendInt64(o, z.TimeLastRefresh) + // string "Source" + o = append(o, 0xa6, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65) + o = msgp.AppendString(o, z.Source) + // string "Group" + o = append(o, 0xa5, 0x47, 0x72, 0x6f, 0x75, 0x70) + o = msgp.AppendBool(o, z.Group) + // string "Owner" + o = append(o, 0xa5, 0x4f, 0x77, 0x6e, 0x65, 0x72) + o = msgp.AppendString(o, z.Owner) + // string "Quorum" + o = append(o, 0xa6, 0x51, 0x75, 0x6f, 0x72, 0x75, 0x6d) + o = msgp.AppendInt(o, z.Quorum) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *lockRequesterInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Name": + z.Name, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "Writer": + z.Writer, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Writer") + return + } + case "UID": + z.UID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UID") + return + } + case "Timestamp": + z.Timestamp, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Timestamp") + return + } + case "TimeLastRefresh": + z.TimeLastRefresh, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TimeLastRefresh") + return + } + case "Source": + z.Source, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Source") + return + } + case "Group": + z.Group, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Group") + return + } + case "Owner": + z.Owner, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Owner") + return + } + case "Quorum": + z.Quorum, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Quorum") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *lockRequesterInfo) Msgsize() (s int) { + s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 7 + msgp.BoolSize + 4 + msgp.StringPrefixSize + len(z.UID) + 10 + msgp.Int64Size + 16 + msgp.Int64Size + 7 + msgp.StringPrefixSize + len(z.Source) + 6 + msgp.BoolSize + 6 + msgp.StringPrefixSize + len(z.Owner) + 7 + msgp.IntSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *lockStats) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Total": + z.Total, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Total") + return + } + case "Writes": + z.Writes, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Writes") + return + } + case "Reads": + z.Reads, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Reads") + return + } + case "LockQueue": + z.LockQueue, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "LockQueue") + return + } + case "LocksAbandoned": + z.LocksAbandoned, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "LocksAbandoned") + return + } + case "LastCleanup": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "LastCleanup") + return + } + z.LastCleanup = nil + } else { + if z.LastCleanup == nil { + z.LastCleanup = new(time.Time) + } + *z.LastCleanup, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "LastCleanup") + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *lockStats) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 6 + // write "Total" + err = en.Append(0x86, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c) + if err != nil { + return + } + err = en.WriteInt(z.Total) + if err != nil { + err = msgp.WrapError(err, "Total") + return + } + // write "Writes" + err = en.Append(0xa6, 0x57, 0x72, 0x69, 0x74, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteInt(z.Writes) + if err != nil { + err = msgp.WrapError(err, "Writes") + return + } + // write "Reads" + err = en.Append(0xa5, 0x52, 0x65, 0x61, 0x64, 0x73) + if err != nil { + return + } + err = en.WriteInt(z.Reads) + if err != nil { + err = msgp.WrapError(err, "Reads") + return + } + // write "LockQueue" + err = en.Append(0xa9, 0x4c, 0x6f, 0x63, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65) + if err != nil { + return + } + err = en.WriteInt(z.LockQueue) + if err != nil { + err = msgp.WrapError(err, "LockQueue") + return + } + // write "LocksAbandoned" + err = en.Append(0xae, 0x4c, 0x6f, 0x63, 0x6b, 0x73, 0x41, 0x62, 0x61, 0x6e, 0x64, 0x6f, 0x6e, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteInt(z.LocksAbandoned) + if err != nil { + err = msgp.WrapError(err, "LocksAbandoned") + return + } + // write "LastCleanup" + err = en.Append(0xab, 0x4c, 0x61, 0x73, 0x74, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70) + if err != nil { + return + } + if z.LastCleanup == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = en.WriteTime(*z.LastCleanup) + if err != nil { + err = msgp.WrapError(err, "LastCleanup") + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *lockStats) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 6 + // string "Total" + o = append(o, 0x86, 0xa5, 0x54, 0x6f, 0x74, 0x61, 0x6c) + o = msgp.AppendInt(o, z.Total) + // string "Writes" + o = append(o, 0xa6, 0x57, 0x72, 0x69, 0x74, 0x65, 0x73) + o = msgp.AppendInt(o, z.Writes) + // string "Reads" + o = append(o, 0xa5, 0x52, 0x65, 0x61, 0x64, 0x73) + o = msgp.AppendInt(o, z.Reads) + // string "LockQueue" + o = append(o, 0xa9, 0x4c, 0x6f, 0x63, 0x6b, 0x51, 0x75, 0x65, 0x75, 0x65) + o = msgp.AppendInt(o, z.LockQueue) + // string "LocksAbandoned" + o = append(o, 0xae, 0x4c, 0x6f, 0x63, 0x6b, 0x73, 0x41, 0x62, 0x61, 0x6e, 0x64, 0x6f, 0x6e, 0x65, 0x64) + o = msgp.AppendInt(o, z.LocksAbandoned) + // string "LastCleanup" + o = append(o, 0xab, 0x4c, 0x61, 0x73, 0x74, 0x43, 0x6c, 0x65, 0x61, 0x6e, 0x75, 0x70) + if z.LastCleanup == nil { + o = msgp.AppendNil(o) + } else { + o = msgp.AppendTime(o, *z.LastCleanup) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *lockStats) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Total": + z.Total, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Total") + return + } + case "Writes": + z.Writes, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Writes") + return + } + case "Reads": + z.Reads, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Reads") + return + } + case "LockQueue": + z.LockQueue, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "LockQueue") + return + } + case "LocksAbandoned": + z.LocksAbandoned, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "LocksAbandoned") + return + } + case "LastCleanup": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.LastCleanup = nil + } else { + if z.LastCleanup == nil { + z.LastCleanup = new(time.Time) + } + *z.LastCleanup, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "LastCleanup") + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *lockStats) Msgsize() (s int) { + s = 1 + 6 + msgp.IntSize + 7 + msgp.IntSize + 6 + msgp.IntSize + 10 + msgp.IntSize + 15 + msgp.IntSize + 12 + if z.LastCleanup == nil { + s += msgp.NilSize + } else { + s += msgp.TimeSize + } + return +} diff --git a/cmd/local-locker_gen_test.go b/cmd/local-locker_gen_test.go new file mode 100644 index 00000000..fe0fee2d --- /dev/null +++ b/cmd/local-locker_gen_test.go @@ -0,0 +1,349 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshallocalLockMap(t *testing.T) { + v := localLockMap{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsglocalLockMap(b *testing.B) { + v := localLockMap{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsglocalLockMap(b *testing.B) { + v := localLockMap{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshallocalLockMap(b *testing.B) { + v := localLockMap{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodelocalLockMap(t *testing.T) { + v := localLockMap{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodelocalLockMap Msgsize() is inaccurate") + } + + vn := localLockMap{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodelocalLockMap(b *testing.B) { + v := localLockMap{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodelocalLockMap(b *testing.B) { + v := localLockMap{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshallockRequesterInfo(t *testing.T) { + v := lockRequesterInfo{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsglockRequesterInfo(b *testing.B) { + v := lockRequesterInfo{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsglockRequesterInfo(b *testing.B) { + v := lockRequesterInfo{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshallockRequesterInfo(b *testing.B) { + v := lockRequesterInfo{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodelockRequesterInfo(t *testing.T) { + v := lockRequesterInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodelockRequesterInfo Msgsize() is inaccurate") + } + + vn := lockRequesterInfo{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodelockRequesterInfo(b *testing.B) { + v := lockRequesterInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodelockRequesterInfo(b *testing.B) { + v := lockRequesterInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshallockStats(t *testing.T) { + v := lockStats{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsglockStats(b *testing.B) { + v := lockStats{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsglockStats(b *testing.B) { + v := lockStats{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshallockStats(b *testing.B) { + v := lockStats{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodelockStats(t *testing.T) { + v := lockStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodelockStats Msgsize() is inaccurate") + } + + vn := lockStats{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodelockStats(b *testing.B) { + v := lockStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodelockStats(b *testing.B) { + v := lockStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/local-locker_test.go b/cmd/local-locker_test.go new file mode 100644 index 00000000..674cf07b --- /dev/null +++ b/cmd/local-locker_test.go @@ -0,0 +1,442 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "encoding/hex" + "fmt" + "math/rand" + "testing" + "time" + + "github.com/google/uuid" + "github.com/minio/minio/internal/dsync" +) + +func TestLocalLockerExpire(t *testing.T) { + wResources := make([]string, 1000) + rResources := make([]string, 1000) + quorum := 0 + l := newLocker() + ctx := t.Context() + for i := range wResources { + arg := dsync.LockArgs{ + UID: mustGetUUID(), + Resources: []string{mustGetUUID()}, + Source: t.Name(), + Owner: "owner", + Quorum: &quorum, + } + ok, err := l.Lock(ctx, arg) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("did not get write lock") + } + wResources[i] = arg.Resources[0] + } + for i := range rResources { + name := mustGetUUID() + arg := dsync.LockArgs{ + UID: mustGetUUID(), + Resources: []string{name}, + Source: t.Name(), + Owner: "owner", + Quorum: &quorum, + } + ok, err := l.RLock(ctx, arg) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("did not get read lock") + } + // RLock twice + ok, err = l.RLock(ctx, arg) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("did not get write lock") + } + + rResources[i] = arg.Resources[0] + } + if len(l.lockMap) != len(rResources)+len(wResources) { + t.Fatalf("lockmap len, got %d, want %d + %d", len(l.lockMap), len(rResources), len(wResources)) + } + if len(l.lockUID) != len(rResources)+len(wResources) { + t.Fatalf("lockUID len, got %d, want %d + %d", len(l.lockUID), len(rResources), len(wResources)) + } + // Expire an hour from now, should keep all + l.expireOldLocks(time.Hour) + if len(l.lockMap) != len(rResources)+len(wResources) { + t.Fatalf("lockmap len, got %d, want %d + %d", len(l.lockMap), len(rResources), len(wResources)) + } + if len(l.lockUID) != len(rResources)+len(wResources) { + t.Fatalf("lockUID len, got %d, want %d + %d", len(l.lockUID), len(rResources), len(wResources)) + } + + // Expire a minute ago. + l.expireOldLocks(-time.Minute) + if len(l.lockMap) != 0 { + t.Fatalf("after cleanup should be empty, got %d", len(l.lockMap)) + } + if len(l.lockUID) != 0 { + t.Fatalf("lockUID len, got %d, want %d", len(l.lockUID), 0) + } +} + +func TestLocalLockerUnlock(t *testing.T) { + const n = 1000 + const m = 5 + wResources := make([][m]string, n) + rResources := make([]string, n) + wUIDs := make([]string, n) + rUIDs := make([]string, 0, n*2) + l := newLocker() + ctx := t.Context() + quorum := 0 + for i := range wResources { + names := [m]string{} + for j := range names { + names[j] = mustGetUUID() + } + uid := mustGetUUID() + arg := dsync.LockArgs{ + UID: uid, + Resources: names[:], + Source: t.Name(), + Owner: "owner", + Quorum: &quorum, + } + ok, err := l.Lock(ctx, arg) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("did not get write lock") + } + wResources[i] = names + wUIDs[i] = uid + } + for i := range rResources { + name := mustGetUUID() + uid := mustGetUUID() + arg := dsync.LockArgs{ + UID: uid, + Resources: []string{name}, + Source: t.Name(), + Owner: "owner", + Quorum: &quorum, + } + ok, err := l.RLock(ctx, arg) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("did not get write lock") + } + rUIDs = append(rUIDs, uid) + + // RLock twice, different uid + uid = mustGetUUID() + arg.UID = uid + ok, err = l.RLock(ctx, arg) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("did not get write lock") + } + rResources[i] = name + rUIDs = append(rUIDs, uid) + } + // Each Lock has m entries + if len(l.lockMap) != len(rResources)+len(wResources)*m { + t.Fatalf("lockmap len, got %d, want %d + %d", len(l.lockMap), len(rResources), len(wResources)*m) + } + // A UID is added for every resource + if len(l.lockUID) != len(rResources)*2+len(wResources)*m { + t.Fatalf("lockUID len, got %d, want %d + %d", len(l.lockUID), len(rResources)*2, len(wResources)*m) + } + // RUnlock once... + for i, name := range rResources { + arg := dsync.LockArgs{ + UID: rUIDs[i*2], + Resources: []string{name}, + Source: t.Name(), + Owner: "owner", + Quorum: &quorum, + } + ok, err := l.RUnlock(ctx, arg) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("did not get write lock") + } + } + + // Each Lock has m entries + if len(l.lockMap) != len(rResources)+len(wResources)*m { + t.Fatalf("lockmap len, got %d, want %d + %d", len(l.lockMap), len(rResources), len(wResources)*m) + } + // A UID is added for every resource. + // We removed len(rResources) read sources. + if len(l.lockUID) != len(rResources)+len(wResources)*m { + t.Fatalf("lockUID len, got %d, want %d + %d", len(l.lockUID), len(rResources), len(wResources)*m) + } + + // RUnlock again, different uids + for i, name := range rResources { + arg := dsync.LockArgs{ + UID: rUIDs[i*2+1], + Resources: []string{name}, + Source: "minio", + Owner: "owner", + Quorum: &quorum, + } + ok, err := l.RUnlock(ctx, arg) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("did not get write lock") + } + } + + // Each Lock has m entries + if len(l.lockMap) != 0+len(wResources)*m { + t.Fatalf("lockmap len, got %d, want %d + %d", len(l.lockMap), 0, len(wResources)*m) + } + // A UID is added for every resource. + // We removed Add Rlocked entries + if len(l.lockUID) != len(wResources)*m { + t.Fatalf("lockUID len, got %d, want %d + %d", len(l.lockUID), 0, len(wResources)*m) + } + + // Remove write locked + for i, names := range wResources { + arg := dsync.LockArgs{ + UID: wUIDs[i], + Resources: names[:], + Source: "minio", + Owner: "owner", + Quorum: &quorum, + } + ok, err := l.Unlock(ctx, arg) + if err != nil { + t.Fatal(err) + } + if !ok { + t.Fatal("did not get write lock") + } + } + + // All should be gone now... + // Each Lock has m entries + if len(l.lockMap) != 0 { + t.Fatalf("lockmap len, got %d, want %d + %d", len(l.lockMap), 0, 0) + } + if len(l.lockUID) != 0 { + t.Fatalf("lockUID len, got %d, want %d + %d", len(l.lockUID), 0, 0) + } +} + +func Test_localLocker_expireOldLocksExpire(t *testing.T) { + rng := rand.New(rand.NewSource(0)) + quorum := 0 + // Numbers of unique locks + for _, locks := range []int{100, 1000, 1e6} { + if testing.Short() && locks > 100 { + continue + } + t.Run(fmt.Sprintf("%d-locks", locks), func(t *testing.T) { + // Number of readers per lock... + for _, readers := range []int{1, 10, 100} { + if locks > 1000 && readers > 1 { + continue + } + if testing.Short() && readers > 10 { + continue + } + t.Run(fmt.Sprintf("%d-read", readers), func(t *testing.T) { + l := newLocker() + for range locks { + var tmp [16]byte + rng.Read(tmp[:]) + res := []string{hex.EncodeToString(tmp[:])} + + for range readers { + rng.Read(tmp[:]) + ok, err := l.RLock(t.Context(), dsync.LockArgs{ + UID: uuid.NewString(), + Resources: res, + Source: hex.EncodeToString(tmp[:8]), + Owner: hex.EncodeToString(tmp[8:]), + Quorum: &quorum, + }) + if !ok || err != nil { + t.Fatal("failed:", err, ok) + } + } + } + start := time.Now() + l.expireOldLocks(time.Hour) + t.Logf("Scan Took: %v. Left: %d/%d", time.Since(start).Round(time.Millisecond), len(l.lockUID), len(l.lockMap)) + if len(l.lockMap) != locks { + t.Fatalf("objects deleted, want %d != got %d", locks, len(l.lockMap)) + } + if len(l.lockUID) != locks*readers { + t.Fatalf("objects deleted, want %d != got %d", locks*readers, len(l.lockUID)) + } + + // Expire 50% + expired := time.Now().Add(-time.Hour * 2) + for _, v := range l.lockMap { + for i := range v { + if rng.Intn(2) == 0 { + v[i].TimeLastRefresh = expired.UnixNano() + } + } + } + start = time.Now() + l.expireOldLocks(time.Hour) + t.Logf("Expire 50%% took: %v. Left: %d/%d", time.Since(start).Round(time.Millisecond), len(l.lockUID), len(l.lockMap)) + + if len(l.lockUID) == locks*readers { + t.Fatalf("objects uids all remain, unlikely") + } + if len(l.lockMap) == 0 { + t.Fatalf("objects all deleted, 0 remains") + } + if len(l.lockUID) == 0 { + t.Fatalf("objects uids all deleted, 0 remains") + } + + start = time.Now() + l.expireOldLocks(-time.Minute) + t.Logf("Expire rest took: %v. Left: %d/%d", time.Since(start).Round(time.Millisecond), len(l.lockUID), len(l.lockMap)) + + if len(l.lockMap) != 0 { + t.Fatalf("objects not deleted, want %d != got %d", 0, len(l.lockMap)) + } + if len(l.lockUID) != 0 { + t.Fatalf("objects not deleted, want %d != got %d", 0, len(l.lockUID)) + } + }) + } + }) + } +} + +func Test_localLocker_RUnlock(t *testing.T) { + rng := rand.New(rand.NewSource(0)) + quorum := 0 + // Numbers of unique locks + for _, locks := range []int{1, 100, 1000, 1e6} { + if testing.Short() && locks > 100 { + continue + } + t.Run(fmt.Sprintf("%d-locks", locks), func(t *testing.T) { + // Number of readers per lock... + for _, readers := range []int{1, 10, 100} { + if locks > 1000 && readers > 1 { + continue + } + if testing.Short() && readers > 10 { + continue + } + t.Run(fmt.Sprintf("%d-read", readers), func(t *testing.T) { + l := newLocker() + for range locks { + var tmp [16]byte + rng.Read(tmp[:]) + res := []string{hex.EncodeToString(tmp[:])} + + for range readers { + rng.Read(tmp[:]) + ok, err := l.RLock(t.Context(), dsync.LockArgs{ + UID: uuid.NewString(), + Resources: res, + Source: hex.EncodeToString(tmp[:8]), + Owner: hex.EncodeToString(tmp[8:]), + Quorum: &quorum, + }) + if !ok || err != nil { + t.Fatal("failed:", err, ok) + } + } + } + + // Expire 50% + toUnLock := make([]dsync.LockArgs, 0, locks*readers) + for k, v := range l.lockMap { + for _, lock := range v { + if rng.Intn(2) == 0 { + toUnLock = append(toUnLock, dsync.LockArgs{Resources: []string{k}, UID: lock.UID}) + } + } + } + start := time.Now() + for _, lock := range toUnLock { + ok, err := l.ForceUnlock(t.Context(), lock) + if err != nil || !ok { + t.Fatal(err) + } + } + t.Logf("Expire 50%% took: %v. Left: %d/%d", time.Since(start).Round(time.Millisecond), len(l.lockUID), len(l.lockMap)) + + if len(l.lockUID) == locks*readers { + t.Fatalf("objects uids all remain, unlikely") + } + if len(l.lockMap) == 0 && locks > 10 { + t.Fatalf("objects all deleted, 0 remains") + } + if len(l.lockUID) != locks*readers-len(toUnLock) { + t.Fatalf("want %d objects uids all deleted, %d remains", len(l.lockUID), locks*readers-len(toUnLock)) + } + + toUnLock = toUnLock[:0] + for k, v := range l.lockMap { + for _, lock := range v { + toUnLock = append(toUnLock, dsync.LockArgs{Resources: []string{k}, UID: lock.UID, Owner: lock.Owner}) + } + } + start = time.Now() + for _, lock := range toUnLock { + ok, err := l.RUnlock(t.Context(), lock) + if err != nil || !ok { + t.Fatal(err) + } + } + t.Logf("Expire rest took: %v. Left: %d/%d", time.Since(start).Round(time.Millisecond), len(l.lockUID), len(l.lockMap)) + + if len(l.lockMap) != 0 { + t.Fatalf("objects not deleted, want %d != got %d", 0, len(l.lockMap)) + } + if len(l.lockUID) != 0 { + t.Fatalf("objects not deleted, want %d != got %d", 0, len(l.lockUID)) + } + }) + } + }) + } +} diff --git a/cmd/lock-rest-client.go b/cmd/lock-rest-client.go new file mode 100644 index 00000000..ff8e2c35 --- /dev/null +++ b/cmd/lock-rest-client.go @@ -0,0 +1,111 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + + "github.com/minio/minio/internal/dsync" + "github.com/minio/minio/internal/grid" +) + +// lockRESTClient is authenticable lock REST client +type lockRESTClient struct { + connection *grid.Connection +} + +// IsOnline - returns whether REST client failed to connect or not. +func (c *lockRESTClient) IsOnline() bool { + return c.connection.State() == grid.StateConnected +} + +// Not a local locker +func (c *lockRESTClient) IsLocal() bool { + return false +} + +// Close - marks the client as closed. +func (c *lockRESTClient) Close() error { + return nil +} + +// String - returns the remote host of the connection. +func (c *lockRESTClient) String() string { + return c.connection.Remote +} + +func (c *lockRESTClient) call(ctx context.Context, h *grid.SingleHandler[*dsync.LockArgs, *dsync.LockResp], args *dsync.LockArgs) (ok bool, err error) { + r, err := h.Call(ctx, c.connection, args) + if err != nil { + return false, err + } + defer h.PutResponse(r) + ok = r.Code == dsync.RespOK + switch r.Code { + case dsync.RespLockConflict, dsync.RespLockNotFound, dsync.RespOK: + // no error + case dsync.RespLockNotInitialized: + err = errLockNotInitialized + default: + err = errors.New(r.Err) + } + return ok, err +} + +// RLock calls read lock REST API. +func (c *lockRESTClient) RLock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) { + return c.call(ctx, lockRPCRLock, &args) +} + +// Lock calls lock REST API. +func (c *lockRESTClient) Lock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) { + return c.call(ctx, lockRPCLock, &args) +} + +// RUnlock calls read unlock REST API. +func (c *lockRESTClient) RUnlock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) { + return c.call(ctx, lockRPCRUnlock, &args) +} + +// Refresh calls Refresh REST API. +func (c *lockRESTClient) Refresh(ctx context.Context, args dsync.LockArgs) (reply bool, err error) { + return c.call(ctx, lockRPCRefresh, &args) +} + +// Unlock calls write unlock RPC. +func (c *lockRESTClient) Unlock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) { + return c.call(ctx, lockRPCUnlock, &args) +} + +// ForceUnlock calls force unlock handler to forcibly unlock an active lock. +func (c *lockRESTClient) ForceUnlock(ctx context.Context, args dsync.LockArgs) (reply bool, err error) { + return c.call(ctx, lockRPCForceUnlock, &args) +} + +func newLockAPI(endpoint Endpoint) dsync.NetLocker { + if endpoint.IsLocal { + return globalLockServer + } + return newlockRESTClient(endpoint) +} + +// Returns a lock rest client. +func newlockRESTClient(ep Endpoint) *lockRESTClient { + return &lockRESTClient{globalLockGrid.Load().Connection(ep.GridHost())} +} diff --git a/cmd/lock-rest-client_test.go b/cmd/lock-rest-client_test.go new file mode 100644 index 00000000..cdfadc6a --- /dev/null +++ b/cmd/lock-rest-client_test.go @@ -0,0 +1,72 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "testing" + + "github.com/minio/minio/internal/dsync" +) + +// Tests lock rpc client. +func TestLockRESTlient(t *testing.T) { + // These should not be connectable. + endpoint, err := NewEndpoint("http://localhost:9876") + if err != nil { + t.Fatalf("unexpected error %v", err) + } + endpointLocal, err := NewEndpoint("http://localhost:9012") + if err != nil { + t.Fatalf("unexpected error %v", err) + } + endpointLocal.IsLocal = true + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + err = initGlobalLockGrid(ctx, []PoolEndpoints{{Endpoints: Endpoints{endpoint, endpointLocal}}}) + if err != nil { + t.Fatal(err) + } + + lkClient := newlockRESTClient(endpoint) + if lkClient.IsOnline() { + t.Fatalf("unexpected result. connection was online") + } + + // Attempt all calls. + _, err = lkClient.RLock(t.Context(), dsync.LockArgs{}) + if err == nil { + t.Fatal("Expected for Rlock to fail") + } + + _, err = lkClient.Lock(t.Context(), dsync.LockArgs{}) + if err == nil { + t.Fatal("Expected for Lock to fail") + } + + _, err = lkClient.RUnlock(t.Context(), dsync.LockArgs{}) + if err == nil { + t.Fatal("Expected for RUnlock to fail") + } + + _, err = lkClient.Unlock(t.Context(), dsync.LockArgs{}) + if err == nil { + t.Fatal("Expected for Unlock to fail") + } +} diff --git a/cmd/lock-rest-server-common.go b/cmd/lock-rest-server-common.go new file mode 100644 index 00000000..ddaa6bb3 --- /dev/null +++ b/cmd/lock-rest-server-common.go @@ -0,0 +1,28 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "errors" +) + +var ( + errLockConflict = errors.New("lock conflict") + errLockNotInitialized = errors.New("lock not initialized") + errLockNotFound = errors.New("lock not found") +) diff --git a/cmd/lock-rest-server-common_test.go b/cmd/lock-rest-server-common_test.go new file mode 100644 index 00000000..21b7fdcd --- /dev/null +++ b/cmd/lock-rest-server-common_test.go @@ -0,0 +1,117 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "os" + "reflect" + "sync" + "testing" + + "github.com/minio/minio/internal/dsync" +) + +// Helper function to create a lock server for testing +func createLockTestServer(ctx context.Context, t *testing.T) (string, *lockRESTServer, string) { + obj, fsDir, err := prepareFS(ctx) + if err != nil { + t.Fatal(err) + } + if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil { + t.Fatalf("unable initialize config file, %s", err) + } + + locker := &lockRESTServer{ + ll: &localLocker{ + mutex: sync.Mutex{}, + lockMap: make(map[string][]lockRequesterInfo), + }, + } + creds := globalActiveCred + token, err := authenticateNode(creds.AccessKey, creds.SecretKey) + if err != nil { + t.Fatal(err) + } + return fsDir, locker, token +} + +// Test function to remove lock entries from map based on name & uid combination +func TestLockRpcServerRemoveEntry(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + testPath, locker, _ := createLockTestServer(ctx, t) + defer os.RemoveAll(testPath) + + lockRequesterInfo1 := lockRequesterInfo{ + Owner: "owner", + Writer: true, + UID: "0123-4567", + Timestamp: UTCNow().UnixNano(), + TimeLastRefresh: UTCNow().UnixNano(), + } + lockRequesterInfo2 := lockRequesterInfo{ + Owner: "owner", + Writer: true, + UID: "89ab-cdef", + Timestamp: UTCNow().UnixNano(), + TimeLastRefresh: UTCNow().UnixNano(), + } + + locker.ll.lockMap["name"] = []lockRequesterInfo{ + lockRequesterInfo1, + lockRequesterInfo2, + } + + lri := locker.ll.lockMap["name"] + + // test unknown uid + if locker.ll.removeEntry("name", dsync.LockArgs{ + Owner: "owner", + UID: "unknown-uid", + }, &lri) { + t.Errorf("Expected %#v, got %#v", false, true) + } + + if !locker.ll.removeEntry("name", dsync.LockArgs{ + Owner: "owner", + UID: "0123-4567", + }, &lri) { + t.Errorf("Expected %#v, got %#v", true, false) + } else { + gotLri := locker.ll.lockMap["name"] + expectedLri := []lockRequesterInfo{lockRequesterInfo2} + if !reflect.DeepEqual(expectedLri, gotLri) { + t.Errorf("Expected %#v, got %#v", expectedLri, gotLri) + } + } + + if !locker.ll.removeEntry("name", dsync.LockArgs{ + Owner: "owner", + UID: "89ab-cdef", + }, &lri) { + t.Errorf("Expected %#v, got %#v", true, false) + } else { + gotLri := locker.ll.lockMap["name"] + expectedLri := []lockRequesterInfo(nil) + if !reflect.DeepEqual(expectedLri, gotLri) { + t.Errorf("Expected %#v, got %#v", expectedLri, gotLri) + } + } +} diff --git a/cmd/lock-rest-server.go b/cmd/lock-rest-server.go new file mode 100644 index 00000000..79e3dfdb --- /dev/null +++ b/cmd/lock-rest-server.go @@ -0,0 +1,190 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "time" + + "github.com/minio/minio/internal/dsync" + "github.com/minio/minio/internal/grid" + "github.com/minio/minio/internal/logger" +) + +// To abstract a node over network. +type lockRESTServer struct { + ll *localLocker +} + +// RefreshHandler - refresh the current lock +func (l *lockRESTServer) RefreshHandler(args *dsync.LockArgs) (*dsync.LockResp, *grid.RemoteErr) { + // Add a timeout similar to what we expect upstream. + ctx, cancel := context.WithTimeout(context.Background(), dsync.DefaultTimeouts.RefreshCall) + defer cancel() + + resp := lockRPCRefresh.NewResponse() + refreshed, err := l.ll.Refresh(ctx, *args) + if err != nil { + return l.makeResp(resp, err) + } + if !refreshed { + return l.makeResp(resp, errLockNotFound) + } + return l.makeResp(resp, err) +} + +// LockHandler - Acquires a lock. +func (l *lockRESTServer) LockHandler(args *dsync.LockArgs) (*dsync.LockResp, *grid.RemoteErr) { + // Add a timeout similar to what we expect upstream. + ctx, cancel := context.WithTimeout(context.Background(), dsync.DefaultTimeouts.Acquire) + defer cancel() + resp := lockRPCLock.NewResponse() + success, err := l.ll.Lock(ctx, *args) + if err == nil && !success { + return l.makeResp(resp, errLockConflict) + } + return l.makeResp(resp, err) +} + +// UnlockHandler - releases the acquired lock. +func (l *lockRESTServer) UnlockHandler(args *dsync.LockArgs) (*dsync.LockResp, *grid.RemoteErr) { + resp := lockRPCUnlock.NewResponse() + _, err := l.ll.Unlock(context.Background(), *args) + // Ignore the Unlock() "reply" return value because if err == nil, "reply" is always true + // Consequently, if err != nil, reply is always false + return l.makeResp(resp, err) +} + +// RLockHandler - Acquires an RLock. +func (l *lockRESTServer) RLockHandler(args *dsync.LockArgs) (*dsync.LockResp, *grid.RemoteErr) { + // Add a timeout similar to what we expect upstream. + ctx, cancel := context.WithTimeout(context.Background(), dsync.DefaultTimeouts.Acquire) + defer cancel() + resp := lockRPCRLock.NewResponse() + success, err := l.ll.RLock(ctx, *args) + if err == nil && !success { + err = errLockConflict + } + return l.makeResp(resp, err) +} + +// RUnlockHandler - releases the acquired read lock. +func (l *lockRESTServer) RUnlockHandler(args *dsync.LockArgs) (*dsync.LockResp, *grid.RemoteErr) { + resp := lockRPCRUnlock.NewResponse() + + // Ignore the RUnlock() "reply" return value because if err == nil, "reply" is always true. + // Consequently, if err != nil, reply is always false + _, err := l.ll.RUnlock(context.Background(), *args) + return l.makeResp(resp, err) +} + +// ForceUnlockHandler - query expired lock status. +func (l *lockRESTServer) ForceUnlockHandler(args *dsync.LockArgs) (*dsync.LockResp, *grid.RemoteErr) { + resp := lockRPCForceUnlock.NewResponse() + + _, err := l.ll.ForceUnlock(context.Background(), *args) + return l.makeResp(resp, err) +} + +var ( + // Static lock handlers. + // All have the same signature. + lockRPCForceUnlock = newLockHandler(grid.HandlerLockForceUnlock) + lockRPCRefresh = newLockHandler(grid.HandlerLockRefresh) + lockRPCLock = newLockHandler(grid.HandlerLockLock) + lockRPCUnlock = newLockHandler(grid.HandlerLockUnlock) + lockRPCRLock = newLockHandler(grid.HandlerLockRLock) + lockRPCRUnlock = newLockHandler(grid.HandlerLockRUnlock) +) + +func newLockHandler(h grid.HandlerID) *grid.SingleHandler[*dsync.LockArgs, *dsync.LockResp] { + return grid.NewSingleHandler[*dsync.LockArgs, *dsync.LockResp](h, func() *dsync.LockArgs { + return &dsync.LockArgs{} + }, func() *dsync.LockResp { + return &dsync.LockResp{} + }) +} + +// registerLockRESTHandlers - register lock rest router. +func registerLockRESTHandlers(gm *grid.Manager) { + lockServer := &lockRESTServer{ + ll: newLocker(), + } + + logger.FatalIf(lockRPCForceUnlock.Register(gm, lockServer.ForceUnlockHandler), "unable to register handler") + logger.FatalIf(lockRPCRefresh.Register(gm, lockServer.RefreshHandler), "unable to register handler") + logger.FatalIf(lockRPCLock.Register(gm, lockServer.LockHandler), "unable to register handler") + logger.FatalIf(lockRPCUnlock.Register(gm, lockServer.UnlockHandler), "unable to register handler") + logger.FatalIf(lockRPCRLock.Register(gm, lockServer.RLockHandler), "unable to register handler") + logger.FatalIf(lockRPCRUnlock.Register(gm, lockServer.RUnlockHandler), "unable to register handler") + + globalLockServer = lockServer.ll + + go lockMaintenance(GlobalContext) +} + +func (l *lockRESTServer) makeResp(dst *dsync.LockResp, err error) (*dsync.LockResp, *grid.RemoteErr) { + *dst = dsync.LockResp{Code: dsync.RespOK} + switch err { + case nil: + case errLockNotInitialized: + dst.Code = dsync.RespLockNotInitialized + case errLockConflict: + dst.Code = dsync.RespLockConflict + case errLockNotFound: + dst.Code = dsync.RespLockNotFound + default: + dst.Code = dsync.RespErr + dst.Err = err.Error() + } + return dst, nil +} + +const ( + // Lock maintenance interval. + lockMaintenanceInterval = 1 * time.Minute + + // Lock validity duration + lockValidityDuration = 1 * time.Minute +) + +// lockMaintenance loops over all locks and discards locks +// that have not been refreshed for some time. +func lockMaintenance(ctx context.Context) { + if !globalIsDistErasure { + return + } + + // Initialize a new ticker with 1 minute between each ticks. + lkTimer := time.NewTimer(lockMaintenanceInterval) + // Stop the timer upon returning. + defer lkTimer.Stop() + + for { + // Verifies every minute for locks held more than 2 minutes. + select { + case <-ctx.Done(): + return + case <-lkTimer.C: + globalLockServer.expireOldLocks(lockValidityDuration) + + // Reset the timer for next cycle. + lkTimer.Reset(lockMaintenanceInterval) + } + } +} diff --git a/cmd/logging.go b/cmd/logging.go new file mode 100644 index 00000000..9c24ae22 --- /dev/null +++ b/cmd/logging.go @@ -0,0 +1,230 @@ +package cmd + +import ( + "context" + "errors" + + "github.com/minio/minio/internal/grid" + "github.com/minio/minio/internal/logger" +) + +func proxyLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "proxy", err, errKind...) +} + +func replLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "replication", err, errKind...) +} + +func replLogOnceIf(ctx context.Context, err error, id string, errKind ...any) { + logger.LogOnceIf(ctx, "replication", err, id, errKind...) +} + +func iamLogOnceIf(ctx context.Context, err error, id string, errKind ...any) { + logger.LogOnceIf(ctx, "iam", err, id, errKind...) +} + +func iamLogIf(ctx context.Context, err error, errKind ...any) { + if !errors.Is(err, grid.ErrDisconnected) { + logger.LogIf(ctx, "iam", err, errKind...) + } +} + +func iamLogEvent(ctx context.Context, msg string, args ...any) { + logger.Event(ctx, "iam", msg, args...) +} + +func rebalanceLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "rebalance", err, errKind...) +} + +func rebalanceLogEvent(ctx context.Context, msg string, args ...any) { + logger.Event(ctx, "rebalance", msg, args...) +} + +func adminLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "admin", err, errKind...) +} + +func authNLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "authN", err, errKind...) +} + +func authZLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "authZ", err, errKind...) +} + +func peersLogIf(ctx context.Context, err error, errKind ...any) { + if !errors.Is(err, grid.ErrDisconnected) { + logger.LogIf(ctx, "peers", err, errKind...) + } +} + +func peersLogAlwaysIf(ctx context.Context, err error, errKind ...any) { + if !errors.Is(err, grid.ErrDisconnected) { + logger.LogAlwaysIf(ctx, "peers", err, errKind...) + } +} + +func peersLogOnceIf(ctx context.Context, err error, id string, errKind ...any) { + if !errors.Is(err, grid.ErrDisconnected) { + logger.LogOnceIf(ctx, "peers", err, id, errKind...) + } +} + +func bugLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "internal", err, errKind...) +} + +func healingLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "healing", err, errKind...) +} + +func healingLogEvent(ctx context.Context, msg string, args ...any) { + logger.Event(ctx, "healing", msg, args...) +} + +func healingLogOnceIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "healing", err, errKind...) +} + +func batchLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "batch", err, errKind...) +} + +func batchLogOnceIf(ctx context.Context, err error, id string, errKind ...any) { + logger.LogOnceIf(ctx, "batch", err, id, errKind...) +} + +func bootLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "bootstrap", err, errKind...) +} + +func bootLogOnceIf(ctx context.Context, err error, id string, errKind ...any) { + logger.LogOnceIf(ctx, "bootstrap", err, id, errKind...) +} + +func dnsLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "dns", err, errKind...) +} + +func internalLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "internal", err, errKind...) +} + +func internalLogOnceIf(ctx context.Context, err error, id string, errKind ...any) { + logger.LogOnceIf(ctx, "internal", err, id, errKind...) +} + +func transitionLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "transition", err, errKind...) +} + +func configLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "config", err, errKind...) +} + +func configLogOnceIf(ctx context.Context, err error, id string, errKind ...any) { + logger.LogOnceIf(ctx, "config", err, id, errKind...) +} + +func configLogOnceConsoleIf(ctx context.Context, err error, id string, errKind ...any) { + logger.LogOnceConsoleIf(ctx, "config", err, id, errKind...) +} + +func scannerLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "scanner", err, errKind...) +} + +func scannerLogOnceIf(ctx context.Context, err error, id string, errKind ...any) { + logger.LogOnceIf(ctx, "scanner", err, id, errKind...) +} + +func ilmLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "ilm", err, errKind...) +} + +func ilmLogOnceIf(ctx context.Context, err error, id string, errKind ...any) { + logger.LogOnceIf(ctx, "ilm", err, id, errKind...) +} + +func encLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "encryption", err, errKind...) +} + +func encLogOnceIf(ctx context.Context, err error, id string, errKind ...any) { + logger.LogOnceIf(ctx, "encryption", err, id, errKind...) +} + +func storageLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "storage", err, errKind...) +} + +func storageLogAlwaysIf(ctx context.Context, err error, errKind ...any) { + logger.LogAlwaysIf(ctx, "storage", err, errKind...) +} + +func storageLogOnceIf(ctx context.Context, err error, id string, errKind ...any) { + logger.LogOnceIf(ctx, "storage", err, id, errKind...) +} + +func decomLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "decom", err, errKind...) +} + +func decomLogOnceIf(ctx context.Context, err error, id string, errKind ...any) { + logger.LogOnceIf(ctx, "decom", err, id, errKind...) +} + +func decomLogEvent(ctx context.Context, msg string, args ...any) { + logger.Event(ctx, "decom", msg, args...) +} + +func etcdLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "etcd", err, errKind...) +} + +func etcdLogOnceIf(ctx context.Context, err error, id string, errKind ...any) { + logger.LogOnceIf(ctx, "etcd", err, id, errKind...) +} + +func metricsLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "metrics", err, errKind...) +} + +func s3LogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "s3", err, errKind...) +} + +func sftpLogOnceIf(ctx context.Context, err error, id string, errKind ...any) { + logger.LogOnceIf(ctx, "sftp", err, id, errKind...) +} + +func shutdownLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "shutdown", err, errKind...) +} + +func stsLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "sts", err, errKind...) +} + +func tierLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "tier", err, errKind...) +} + +func kmsLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "kms", err, errKind...) +} + +// KMSLogger permits access to kms module specific logging +type KMSLogger struct{} + +// LogOnceIf is the implementation of LogOnceIf, accessible using the Logger interface +func (l KMSLogger) LogOnceIf(ctx context.Context, err error, id string, errKind ...any) { + logger.LogOnceIf(ctx, "kms", err, id, errKind...) +} + +// LogIf is the implementation of LogIf, accessible using the Logger interface +func (l KMSLogger) LogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "kms", err, errKind...) +} diff --git a/cmd/main.go b/cmd/main.go new file mode 100644 index 00000000..53249af4 --- /dev/null +++ b/cmd/main.go @@ -0,0 +1,228 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "runtime/debug" + "sort" + "strconv" + "strings" + "time" + + "github.com/minio/cli" + "github.com/minio/minio/internal/color" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/console" + "github.com/minio/pkg/v3/env" + "github.com/minio/pkg/v3/trie" + "github.com/minio/pkg/v3/words" +) + +// GlobalFlags - global flags for minio. +var GlobalFlags = []cli.Flag{ + // Deprecated flag, so its hidden now - existing deployments will keep working. + cli.StringFlag{ + Name: "config-dir, C", + Value: defaultConfigDir.Get(), + Usage: "[DEPRECATED] path to legacy configuration directory", + Hidden: true, + }, + cli.StringFlag{ + Name: "certs-dir, S", + Value: defaultCertsDir.Get(), + Usage: "path to certs directory", + }, + cli.BoolFlag{ + Name: "quiet", + Usage: "disable startup and info messages", + }, + cli.BoolFlag{ + Name: "anonymous", + Usage: "hide sensitive information from logging", + }, + cli.BoolFlag{ + Name: "json", + Usage: "output logs in JSON format", + }, + // Deprecated flag, so its hidden now, existing deployments will keep working. + cli.BoolFlag{ + Name: "compat", + Usage: "enable strict S3 compatibility by turning off certain performance optimizations", + Hidden: true, + }, + // This flag is hidden and to be used only during certain performance testing. + cli.BoolFlag{ + Name: "no-compat", + Usage: "disable strict S3 compatibility by turning on certain performance optimizations", + Hidden: true, + }, +} + +// Help template for minio. +var minioHelpTemplate = `NAME: + {{.Name}} - {{.Usage}} + +DESCRIPTION: + {{.Description}} + +USAGE: + {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}COMMAND{{if .VisibleFlags}}{{end}} [ARGS...] + +COMMANDS: + {{range .VisibleCommands}}{{join .Names ", "}}{{ "\t" }}{{.Usage}} + {{end}}{{if .VisibleFlags}} +FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}}{{end}} +VERSION: + {{.Version}} +` + +func newApp(name string) *cli.App { + // Collection of minio commands currently supported are. + commands := []cli.Command{} + + // Collection of minio commands currently supported in a trie tree. + commandsTree := trie.NewTrie() + + // registerCommand registers a cli command. + registerCommand := func(command cli.Command) { + // avoid registering commands which are not being built (via + // go:build tags) + if command.Name == "" { + return + } + commands = append(commands, command) + commandsTree.Insert(command.Name) + } + + findClosestCommands := func(command string) []string { + var closestCommands []string + closestCommands = append(closestCommands, commandsTree.PrefixMatch(command)...) + + sort.Strings(closestCommands) + // Suggest other close commands - allow missed, wrongly added and + // even transposed characters + for _, value := range commandsTree.Walk(commandsTree.Root()) { + if sort.SearchStrings(closestCommands, value) < len(closestCommands) { + continue + } + // 2 is arbitrary and represents the max + // allowed number of typed errors + if words.DamerauLevenshteinDistance(command, value) < 2 { + closestCommands = append(closestCommands, value) + } + } + + return closestCommands + } + + // Register all commands. + registerCommand(serverCmd) + registerCommand(fmtGenCmd) + + // Set up app. + cli.HelpFlag = cli.BoolFlag{ + Name: "help, h", + Usage: "show help", + } + cli.VersionPrinter = printMinIOVersion + + app := cli.NewApp() + app.Name = name + app.Author = "MinIO, Inc." + app.Version = ReleaseTag + app.Usage = "High Performance Object Storage" + app.Description = `Build high performance data infrastructure for machine learning, analytics and application data workloads with MinIO` + app.Flags = GlobalFlags + app.HideHelpCommand = true // Hide `help, h` command, we already have `minio --help`. + app.Commands = commands + app.CustomAppHelpTemplate = minioHelpTemplate + app.CommandNotFound = func(ctx *cli.Context, command string) { + console.Printf("‘%s’ is not a minio sub-command. See ‘minio --help’.\n", command) + closestCommands := findClosestCommands(command) + if len(closestCommands) > 0 { + console.Println() + console.Println("Did you mean one of these?") + for _, cmd := range closestCommands { + console.Printf("\t‘%s’\n", cmd) + } + } + + os.Exit(1) + } + + return app +} + +func startupBanner(banner io.Writer) { + CopyrightYear = strconv.Itoa(time.Now().Year()) + fmt.Fprintln(banner, color.Blue("Copyright:")+color.Bold(" 2015-%s MinIO, Inc.", CopyrightYear)) + fmt.Fprintln(banner, color.Blue("License:")+color.Bold(" "+MinioLicense)) + fmt.Fprintln(banner, color.Blue("Version:")+color.Bold(" %s (%s %s/%s)", ReleaseTag, runtime.Version(), runtime.GOOS, runtime.GOARCH)) +} + +func versionBanner(c *cli.Context) io.Reader { + banner := &strings.Builder{} + fmt.Fprintln(banner, color.Bold("%s version %s (commit-id=%s)", c.App.Name, c.App.Version, CommitID)) + fmt.Fprintln(banner, color.Blue("Runtime:")+color.Bold(" %s %s/%s", runtime.Version(), runtime.GOOS, runtime.GOARCH)) + fmt.Fprintln(banner, color.Blue("License:")+color.Bold(" GNU AGPLv3 - https://www.gnu.org/licenses/agpl-3.0.html")) + fmt.Fprintln(banner, color.Blue("Copyright:")+color.Bold(" 2015-%s MinIO, Inc.", CopyrightYear)) + return strings.NewReader(banner.String()) +} + +func printMinIOVersion(c *cli.Context) { + io.Copy(c.App.Writer, versionBanner(c)) +} + +var debugNoExit = env.Get("_MINIO_DEBUG_NO_EXIT", "") != "" + +// Main main for minio server. +func Main(args []string) { + // Set the minio app name. + appName := filepath.Base(args[0]) + + if debugNoExit { + freeze := func(_ int) { + // Infinite blocking op + <-make(chan struct{}) + } + + // Override the logger os.Exit() + logger.ExitFunc = freeze + + defer func() { + if err := recover(); err != nil { + fmt.Println("panic:", err) + fmt.Println("") + fmt.Println(string(debug.Stack())) + } + freeze(-1) + }() + } + + // Run the app - exit on error. + if err := newApp(appName).Run(args); err != nil { + os.Exit(1) //nolint:gocritic + } +} diff --git a/cmd/metacache-bucket.go b/cmd/metacache-bucket.go new file mode 100644 index 00000000..4df23d82 --- /dev/null +++ b/cmd/metacache-bucket.go @@ -0,0 +1,258 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "maps" + "runtime/debug" + "sort" + "sync" + "time" + + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/console" +) + +// a bucketMetacache keeps track of all caches generated +// for a bucket. +type bucketMetacache struct { + // Name of bucket + bucket string + + // caches indexed by id. + caches map[string]metacache + // cache ids indexed by root paths + cachesRoot map[string][]string `msg:"-"` + + // Internal state + mu sync.RWMutex `msg:"-"` + updated bool `msg:"-"` +} + +type deleteAllStorager interface { + deleteAll(ctx context.Context, bucket, prefix string) +} + +// newBucketMetacache creates a new bucketMetacache. +// Optionally remove all existing caches. +func newBucketMetacache(bucket string, cleanup bool) *bucketMetacache { + if cleanup { + // Recursively delete all caches. + objAPI := newObjectLayerFn() + if objAPI != nil { + ez, ok := objAPI.(deleteAllStorager) + if ok { + ctx := context.Background() + ez.deleteAll(ctx, minioMetaBucket, metacachePrefixForID(bucket, slashSeparator)) + } + } + } + return &bucketMetacache{ + bucket: bucket, + caches: make(map[string]metacache, 10), + cachesRoot: make(map[string][]string, 10), + } +} + +func (b *bucketMetacache) debugf(format string, data ...any) { + if serverDebugLog { + console.Debugf(format+"\n", data...) + } +} + +// findCache will attempt to find a matching cache for the provided options. +// If a cache with the same ID exists already it will be returned. +// If none can be found a new is created with the provided ID. +func (b *bucketMetacache) findCache(o listPathOptions) metacache { + if b == nil { + logger.Info("bucketMetacache.findCache: nil cache for bucket %s", o.Bucket) + return metacache{} + } + + if o.Bucket != b.bucket { + logger.Info("bucketMetacache.findCache: bucket %s does not match this bucket %s", o.Bucket, b.bucket) + debug.PrintStack() + return metacache{} + } + + // Grab a write lock, since we create one if we cannot find one. + b.mu.Lock() + defer b.mu.Unlock() + + // Check if exists already. + if c, ok := b.caches[o.ID]; ok { + c.lastHandout = time.Now() + b.caches[o.ID] = c + b.debugf("returning existing %v", o.ID) + return c + } + + if !o.Create { + return metacache{ + id: o.ID, + bucket: o.Bucket, + status: scanStateNone, + } + } + + // Create new and add. + best := o.newMetacache() + b.caches[o.ID] = best + b.cachesRoot[best.root] = append(b.cachesRoot[best.root], best.id) + b.updated = true + b.debugf("returning new cache %s, bucket: %v", best.id, best.bucket) + return best +} + +// cleanup removes redundant and outdated entries. +func (b *bucketMetacache) cleanup() { + // Entries to remove. + remove := make(map[string]struct{}) + + // Test on a copy + // cleanup is the only one deleting caches. + caches, _ := b.cloneCaches() + + for id, cache := range caches { + if !cache.worthKeeping() { + b.debugf("cache %s not worth keeping", id) + remove[id] = struct{}{} + continue + } + if cache.id != id { + logger.Info("cache ID mismatch %s != %s", id, cache.id) + remove[id] = struct{}{} + continue + } + if cache.bucket != b.bucket { + logger.Info("cache bucket mismatch %s != %s", b.bucket, cache.bucket) + remove[id] = struct{}{} + continue + } + } + + // If above limit, remove the caches with the oldest handout time. + if len(caches)-len(remove) > metacacheMaxEntries { + remainCaches := make([]metacache, 0, len(caches)-len(remove)) + for id, cache := range caches { + if _, ok := remove[id]; ok { + continue + } + remainCaches = append(remainCaches, cache) + } + if len(remainCaches) > metacacheMaxEntries { + // Sort oldest last... + sort.Slice(remainCaches, func(i, j int) bool { + return remainCaches[i].lastHandout.Before(remainCaches[j].lastHandout) + }) + // Keep first metacacheMaxEntries... + for _, cache := range remainCaches[metacacheMaxEntries:] { + if time.Since(cache.lastHandout) > metacacheMaxClientWait { + remove[cache.id] = struct{}{} + } + } + } + } + + for id := range remove { + b.deleteCache(id) + } +} + +// updateCacheEntry will update a cache. +// Returns the updated status. +func (b *bucketMetacache) updateCacheEntry(update metacache) (metacache, error) { + b.mu.Lock() + defer b.mu.Unlock() + existing, ok := b.caches[update.id] + if !ok { + return update, errFileNotFound + } + existing.update(update) + b.caches[update.id] = existing + b.updated = true + return existing, nil +} + +// cloneCaches will return a clone of all current caches. +func (b *bucketMetacache) cloneCaches() (map[string]metacache, map[string][]string) { + b.mu.RLock() + defer b.mu.RUnlock() + dst := make(map[string]metacache, len(b.caches)) + maps.Copy(dst, b.caches) + // Copy indexes + dst2 := make(map[string][]string, len(b.cachesRoot)) + for k, v := range b.cachesRoot { + tmp := make([]string, len(v)) + copy(tmp, v) + dst2[k] = tmp + } + + return dst, dst2 +} + +// deleteAll will delete all on disk data for ALL caches. +// Deletes are performed concurrently. +func (b *bucketMetacache) deleteAll() { + ctx := context.Background() + + objAPI := newObjectLayerFn() + if objAPI == nil { + return + } + + ez, ok := objAPI.(deleteAllStorager) + if !ok { + bugLogIf(ctx, errors.New("bucketMetacache: expected objAPI to be 'deleteAllStorager'")) + return + } + + b.mu.Lock() + defer b.mu.Unlock() + + b.updated = true + // Delete all. + ez.deleteAll(ctx, minioMetaBucket, metacachePrefixForID(b.bucket, slashSeparator)) + b.caches = make(map[string]metacache, 10) + b.cachesRoot = make(map[string][]string, 10) +} + +// deleteCache will delete a specific cache and all files related to it across the cluster. +func (b *bucketMetacache) deleteCache(id string) { + b.mu.Lock() + c, ok := b.caches[id] + if ok { + // Delete from root map. + list := b.cachesRoot[c.root] + for i, lid := range list { + if id == lid { + list = append(list[:i], list[i+1:]...) + break + } + } + b.cachesRoot[c.root] = list + delete(b.caches, id) + b.updated = true + } + b.mu.Unlock() + if ok { + c.delete(context.Background()) + } +} diff --git a/cmd/metacache-bucket_test.go b/cmd/metacache-bucket_test.go new file mode 100644 index 00000000..768d7853 --- /dev/null +++ b/cmd/metacache-bucket_test.go @@ -0,0 +1,68 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "testing" +) + +func Benchmark_bucketMetacache_findCache(b *testing.B) { + bm := newBucketMetacache("", false) + const elements = 50000 + const paths = 100 + if elements%paths != 0 { + b.Fatal("elements must be divisible by the number of paths") + } + var pathNames [paths]string + for i := range pathNames[:] { + pathNames[i] = fmt.Sprintf("prefix/%d", i) + } + for i := range elements { + bm.findCache(listPathOptions{ + ID: mustGetUUID(), + Bucket: "", + BaseDir: pathNames[i%paths], + Prefix: "", + FilterPrefix: "", + Marker: "", + Limit: 0, + AskDisks: "strict", + Recursive: false, + Separator: slashSeparator, + Create: true, + }) + } + b.ReportAllocs() + + for i := 0; b.Loop(); i++ { + bm.findCache(listPathOptions{ + ID: mustGetUUID(), + Bucket: "", + BaseDir: pathNames[i%paths], + Prefix: "", + FilterPrefix: "", + Marker: "", + Limit: 0, + AskDisks: "strict", + Recursive: false, + Separator: slashSeparator, + Create: true, + }) + } +} diff --git a/cmd/metacache-entries.go b/cmd/metacache-entries.go new file mode 100644 index 00000000..69c5e835 --- /dev/null +++ b/cmd/metacache-entries.go @@ -0,0 +1,976 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "os" + "path" + "sort" + "strings" + + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/pkg/v3/console" +) + +// metaCacheEntry is an object or a directory within an unknown bucket. +type metaCacheEntry struct { + // name is the full name of the object including prefixes + name string + // Metadata. If none is present it is not an object but only a prefix. + // Entries without metadata will only be present in non-recursive scans. + metadata []byte + + // cached contains the metadata if decoded. + cached *xlMetaV2 + + // Indicates the entry can be reused and only one reference to metadata is expected. + reusable bool +} + +// isDir returns if the entry is representing a prefix directory. +func (e metaCacheEntry) isDir() bool { + return len(e.metadata) == 0 && strings.HasSuffix(e.name, slashSeparator) +} + +// isObject returns if the entry is representing an object. +func (e metaCacheEntry) isObject() bool { + return len(e.metadata) > 0 +} + +// isObjectDir returns if the entry is representing an object/ +func (e metaCacheEntry) isObjectDir() bool { + return len(e.metadata) > 0 && strings.HasSuffix(e.name, slashSeparator) +} + +// hasPrefix returns whether an entry has a specific prefix +func (e metaCacheEntry) hasPrefix(s string) bool { + return strings.HasPrefix(e.name, s) +} + +// matches returns if the entries have the same versions. +// If strict is false we allow signatures to mismatch. +func (e *metaCacheEntry) matches(other *metaCacheEntry, strict bool) (prefer *metaCacheEntry, matches bool) { + if e == nil && other == nil { + return nil, true + } + if e == nil { + return other, false + } + if other == nil { + return e, false + } + + // Name should match... + if e.name != other.name { + if e.name < other.name { + return e, false + } + return other, false + } + + if other.isDir() || e.isDir() { + if e.isDir() { + return e, other.isDir() == e.isDir() + } + return other, other.isDir() == e.isDir() + } + eVers, eErr := e.xlmeta() + oVers, oErr := other.xlmeta() + if eErr != nil || oErr != nil { + return nil, false + } + + // check both fileInfo's have same number of versions, if not skip + // the `other` entry. + if len(eVers.versions) != len(oVers.versions) { + eTime := eVers.latestModtime() + oTime := oVers.latestModtime() + if !eTime.Equal(oTime) { + if eTime.After(oTime) { + return e, false + } + return other, false + } + // Tiebreak on version count. + if len(eVers.versions) > len(oVers.versions) { + return e, false + } + return other, false + } + + // Check if each version matches... + for i, eVer := range eVers.versions { + oVer := oVers.versions[i] + if eVer.header != oVer.header { + if eVer.header.hasEC() != oVer.header.hasEC() { + // One version has EC and the other doesn't - may have been written later. + // Compare without considering EC. + a, b := eVer.header, oVer.header + a.EcN, a.EcM = 0, 0 + b.EcN, b.EcM = 0, 0 + if a == b { + continue + } + } + if !strict && eVer.header.matchesNotStrict(oVer.header) { + if prefer == nil { + if eVer.header.sortsBefore(oVer.header) { + prefer = e + } else { + prefer = other + } + } + continue + } + if prefer != nil { + return prefer, false + } + + if eVer.header.sortsBefore(oVer.header) { + return e, false + } + return other, false + } + } + // If we match, return e + if prefer == nil { + prefer = e + } + return prefer, true +} + +// isInDir returns whether the entry is in the dir when considering the separator. +func (e metaCacheEntry) isInDir(dir, separator string) bool { + if len(dir) == 0 { + // Root + idx := strings.Index(e.name, separator) + return idx == -1 || idx == len(e.name)-len(separator) + } + ext := strings.TrimPrefix(e.name, dir) + if len(ext) != len(e.name) { + idx := strings.Index(ext, separator) + // If separator is not found or is last entry, ok. + return idx == -1 || idx == len(ext)-len(separator) + } + return false +} + +// isLatestDeletemarker returns whether the latest version is a delete marker. +// If metadata is NOT versioned false will always be returned. +// If v2 and UNABLE to load metadata true will be returned. +func (e *metaCacheEntry) isLatestDeletemarker() bool { + if e.cached != nil { + if len(e.cached.versions) == 0 { + return true + } + return e.cached.versions[0].header.Type == DeleteType + } + if !isXL2V1Format(e.metadata) { + return false + } + if meta, _, err := isIndexedMetaV2(e.metadata); meta != nil { + return meta.IsLatestDeleteMarker() + } else if err != nil { + return true + } + // Fall back... + xlMeta, err := e.xlmeta() + if err != nil || len(xlMeta.versions) == 0 { + return true + } + return xlMeta.versions[0].header.Type == DeleteType +} + +// isAllFreeVersions returns if all objects are free versions. +// If metadata is NOT versioned false will always be returned. +// If v2 and UNABLE to load metadata true will be returned. +func (e *metaCacheEntry) isAllFreeVersions() bool { + if e.cached != nil { + if len(e.cached.versions) == 0 { + return true + } + for _, v := range e.cached.versions { + if !v.header.FreeVersion() { + return false + } + } + return true + } + if !isXL2V1Format(e.metadata) { + return false + } + if meta, _, err := isIndexedMetaV2(e.metadata); meta != nil { + return meta.AllHidden(false) + } else if err != nil { + return true + } + // Fall back... + xlMeta, err := e.xlmeta() + if err != nil || len(xlMeta.versions) == 0 { + return true + } + // Check versions.. + for _, v := range e.cached.versions { + if !v.header.FreeVersion() { + return false + } + } + return true +} + +// fileInfo returns the decoded metadata. +// If entry is a directory it is returned as that. +// If versioned the latest version will be returned. +func (e *metaCacheEntry) fileInfo(bucket string) (FileInfo, error) { + if e.isDir() { + return FileInfo{ + Volume: bucket, + Name: e.name, + Mode: uint32(os.ModeDir), + }, nil + } + if e.cached != nil { + if len(e.cached.versions) == 0 { + // This special case is needed to handle xlMeta.versions == 0 + return FileInfo{ + Volume: bucket, + Name: e.name, + Deleted: true, + IsLatest: true, + ModTime: timeSentinel1970, + }, nil + } + return e.cached.ToFileInfo(bucket, e.name, "", false, true) + } + return getFileInfo(e.metadata, bucket, e.name, "", fileInfoOpts{}) +} + +// xlmeta returns the decoded metadata. +// This should not be called on directories. +func (e *metaCacheEntry) xlmeta() (*xlMetaV2, error) { + if e.isDir() { + return nil, errFileNotFound + } + if e.cached == nil { + if len(e.metadata) == 0 { + // only happens if the entry is not found. + return nil, errFileNotFound + } + var xl xlMetaV2 + err := xl.LoadOrConvert(e.metadata) + if err != nil { + return nil, err + } + e.cached = &xl + } + return e.cached, nil +} + +// fileInfoVersions returns the metadata as FileInfoVersions. +// If entry is a directory it is returned as that. +func (e *metaCacheEntry) fileInfoVersions(bucket string) (FileInfoVersions, error) { + if e.isDir() { + return FileInfoVersions{ + Volume: bucket, + Name: e.name, + Versions: []FileInfo{ + { + Volume: bucket, + Name: e.name, + Mode: uint32(os.ModeDir), + }, + }, + }, nil + } + // Too small gains to reuse cache here. + return getFileInfoVersions(e.metadata, bucket, e.name, true) +} + +// metaCacheEntries is a slice of metacache entries. +type metaCacheEntries []metaCacheEntry + +// less function for sorting. +func (m metaCacheEntries) less(i, j int) bool { + return m[i].name < m[j].name +} + +// sort entries by name. +// m is sorted and a sorted metadata object is returned. +// Changes to m will also be reflected in the returned object. +func (m metaCacheEntries) sort() metaCacheEntriesSorted { + if m.isSorted() { + return metaCacheEntriesSorted{o: m} + } + sort.Slice(m, m.less) + return metaCacheEntriesSorted{o: m} +} + +// isSorted returns whether the objects are sorted. +// This is usually orders of magnitude faster than actually sorting. +func (m metaCacheEntries) isSorted() bool { + return sort.SliceIsSorted(m, m.less) +} + +// shallowClone will create a shallow clone of the array objects, +// but object metadata will not be cloned. +func (m metaCacheEntries) shallowClone() metaCacheEntries { + dst := make(metaCacheEntries, len(m)) + copy(dst, m) + return dst +} + +type metadataResolutionParams struct { + dirQuorum int // Number if disks needed for a directory to 'exist'. + objQuorum int // Number of disks needed for an object to 'exist'. + + // An optimization request only an 'n' amount of versions from xl.meta + // to avoid resolving all versions to figure out the latest 'version' + // for ListObjects, ListObjectsV2 + requestedVersions int + + bucket string // Name of the bucket. Used for generating cached fileinfo. + strict bool // Versions must match exactly, including all metadata. + + // Reusable slice for resolution + candidates [][]xlMetaV2ShallowVersion +} + +// resolve multiple entries. +// entries are resolved by majority, then if tied by mod-time and versions. +// Names must match on all entries in m. +func (m metaCacheEntries) resolve(r *metadataResolutionParams) (selected *metaCacheEntry, ok bool) { + if len(m) == 0 { + return nil, false + } + + dirExists := 0 + if cap(r.candidates) < len(m) { + r.candidates = make([][]xlMetaV2ShallowVersion, 0, len(m)) + } + r.candidates = r.candidates[:0] + objsAgree := 0 + objsValid := 0 + for i := range m { + entry := &m[i] + // Empty entry + if entry.name == "" { + continue + } + + if entry.isDir() { + dirExists++ + selected = entry + continue + } + + // Get new entry metadata, + // shallow decode. + xl, err := entry.xlmeta() + if err != nil { + continue + } + objsValid++ + + // Add all valid to candidates. + r.candidates = append(r.candidates, xl.versions) + + // We select the first object we find as a candidate and see if all match that. + // This is to quickly identify if all agree. + if selected == nil { + selected = entry + objsAgree = 1 + continue + } + // Names match, check meta... + if prefer, ok := entry.matches(selected, r.strict); ok { + selected = prefer + objsAgree++ + continue + } + } + + // Return dir entries, if enough... + if selected != nil && selected.isDir() && dirExists >= r.dirQuorum { + return selected, true + } + + // If we would never be able to reach read quorum. + if objsValid < r.objQuorum { + return nil, false + } + + // If all objects agree. + if selected != nil && objsAgree == objsValid { + return selected, true + } + + // If cached is nil we shall skip the entry. + if selected.cached == nil { + return nil, false + } + + // Merge if we have disagreement. + // Create a new merged result. + selected = &metaCacheEntry{ + name: selected.name, + reusable: true, + cached: &xlMetaV2{metaV: selected.cached.metaV}, + } + selected.cached.versions = mergeXLV2Versions(r.objQuorum, r.strict, r.requestedVersions, r.candidates...) + if len(selected.cached.versions) == 0 { + return nil, false + } + + // Reserialize + var err error + selected.metadata, err = selected.cached.AppendTo(metaDataPoolGet()) + if err != nil { + bugLogIf(context.Background(), err) + return nil, false + } + return selected, true +} + +// firstFound returns the first found and the number of set entries. +func (m metaCacheEntries) firstFound() (first *metaCacheEntry, n int) { + for i, entry := range m { + if entry.name != "" { + n++ + if first == nil { + first = &m[i] + } + } + } + return first, n +} + +// names will return all names in order. +// Since this allocates it should not be used in critical functions. +func (m metaCacheEntries) names() []string { + res := make([]string, 0, len(m)) + for _, obj := range m { + res = append(res, obj.name) + } + return res +} + +// metaCacheEntriesSorted contains metacache entries that are sorted. +type metaCacheEntriesSorted struct { + o metaCacheEntries + // list id is not serialized + listID string + // Reuse buffers + reuse bool + // Contain the last skipped object after an ILM expiry evaluation + lastSkippedEntry string +} + +// shallowClone will create a shallow clone of the array objects, +// but object metadata will not be cloned. +func (m metaCacheEntriesSorted) shallowClone() metaCacheEntriesSorted { + // We have value receiver so we already have a copy. + m.o = m.o.shallowClone() + return m +} + +// fileInfoVersions converts the metadata to FileInfoVersions where possible. +// Metadata that cannot be decoded is skipped. +func (m *metaCacheEntriesSorted) fileInfoVersions(bucket, prefix, delimiter, afterV string) (versions []ObjectInfo) { + versions = make([]ObjectInfo, 0, m.len()) + prevPrefix := "" + vcfg, _ := globalBucketVersioningSys.Get(bucket) + + for _, entry := range m.o { + if entry.isObject() { + if delimiter != "" { + idx := strings.Index(strings.TrimPrefix(entry.name, prefix), delimiter) + if idx >= 0 { + idx = len(prefix) + idx + len(delimiter) + currPrefix := entry.name[:idx] + if currPrefix == prevPrefix { + continue + } + prevPrefix = currPrefix + versions = append(versions, ObjectInfo{ + IsDir: true, + Bucket: bucket, + Name: currPrefix, + }) + continue + } + } + + fiv, err := entry.fileInfoVersions(bucket) + if err != nil { + continue + } + + fiVersions := fiv.Versions + if afterV != "" { + vidMarkerIdx := fiv.findVersionIndex(afterV) + if vidMarkerIdx >= 0 { + fiVersions = fiVersions[vidMarkerIdx+1:] + } + afterV = "" + } + + for _, version := range fiVersions { + if !version.VersionPurgeStatus().Empty() { + continue + } + versioned := vcfg != nil && vcfg.Versioned(entry.name) + versions = append(versions, version.ToObjectInfo(bucket, entry.name, versioned)) + } + + continue + } + + if entry.isDir() { + if delimiter == "" { + continue + } + idx := strings.Index(strings.TrimPrefix(entry.name, prefix), delimiter) + if idx < 0 { + continue + } + idx = len(prefix) + idx + len(delimiter) + currPrefix := entry.name[:idx] + if currPrefix == prevPrefix { + continue + } + prevPrefix = currPrefix + versions = append(versions, ObjectInfo{ + IsDir: true, + Bucket: bucket, + Name: currPrefix, + }) + } + } + + return versions +} + +// fileInfos converts the metadata to ObjectInfo where possible. +// Metadata that cannot be decoded is skipped. +func (m *metaCacheEntriesSorted) fileInfos(bucket, prefix, delimiter string) (objects []ObjectInfo) { + objects = make([]ObjectInfo, 0, m.len()) + prevPrefix := "" + + vcfg, _ := globalBucketVersioningSys.Get(bucket) + + for _, entry := range m.o { + if entry.isObject() { + if delimiter != "" { + idx := strings.Index(strings.TrimPrefix(entry.name, prefix), delimiter) + if idx >= 0 { + idx = len(prefix) + idx + len(delimiter) + currPrefix := entry.name[:idx] + if currPrefix == prevPrefix { + continue + } + prevPrefix = currPrefix + objects = append(objects, ObjectInfo{ + IsDir: true, + Bucket: bucket, + Name: currPrefix, + }) + continue + } + } + + fi, err := entry.fileInfo(bucket) + if err == nil && fi.VersionPurgeStatus().Empty() { + versioned := vcfg != nil && vcfg.Versioned(entry.name) + objects = append(objects, fi.ToObjectInfo(bucket, entry.name, versioned)) + } + continue + } + if entry.isDir() { + if delimiter == "" { + continue + } + idx := strings.Index(strings.TrimPrefix(entry.name, prefix), delimiter) + if idx < 0 { + continue + } + idx = len(prefix) + idx + len(delimiter) + currPrefix := entry.name[:idx] + if currPrefix == prevPrefix { + continue + } + prevPrefix = currPrefix + objects = append(objects, ObjectInfo{ + IsDir: true, + Bucket: bucket, + Name: currPrefix, + }) + } + } + + return objects +} + +// forwardTo will truncate m so only entries that are s or after is in the list. +func (m *metaCacheEntriesSorted) forwardTo(s string) { + if s == "" { + return + } + idx := sort.Search(len(m.o), func(i int) bool { + return m.o[i].name >= s + }) + if m.reuse { + for i, entry := range m.o[:idx] { + metaDataPoolPut(entry.metadata) + m.o[i].metadata = nil + } + } + + m.o = m.o[idx:] +} + +// forwardPast will truncate m so only entries that are after s is in the list. +func (m *metaCacheEntriesSorted) forwardPast(s string) { + if s == "" { + return + } + idx := sort.Search(len(m.o), func(i int) bool { + return m.o[i].name > s + }) + if m.reuse { + for i, entry := range m.o[:idx] { + metaDataPoolPut(entry.metadata) + m.o[i].metadata = nil + } + } + m.o = m.o[idx:] +} + +// mergeEntryChannels will merge entries from in and return them sorted on out. +// To signify no more results are on an input channel, close it. +// The output channel will be closed when all inputs are emptied. +// If file names are equal, compareMeta is called to select which one to choose. +// The entry not chosen will be discarded. +// If the context is canceled the function will return the error, +// otherwise the function will return nil. +func mergeEntryChannels(ctx context.Context, in []chan metaCacheEntry, out chan<- metaCacheEntry, readQuorum int) error { + defer xioutil.SafeClose(out) + top := make([]*metaCacheEntry, len(in)) + nDone := 0 + ctxDone := ctx.Done() + + // Use simpler forwarder. + if len(in) == 1 { + for { + select { + case <-ctxDone: + return ctx.Err() + case v, ok := <-in[0]: + if !ok { + return nil + } + select { + case <-ctxDone: + return ctx.Err() + case out <- v: + } + } + } + } + + selectFrom := func(idx int) error { + select { + case <-ctxDone: + return ctx.Err() + case entry, ok := <-in[idx]: + if !ok { + top[idx] = nil + nDone++ + } else { + top[idx] = &entry + } + } + return nil + } + // Populate all... + for i := range in { + if err := selectFrom(i); err != nil { + return err + } + } + last := "" + var toMerge []int + + // Choose the best to return. + for { + if nDone == len(in) { + return nil + } + best := top[0] + bestIdx := 0 + toMerge = toMerge[:0] + for i, other := range top[1:] { + otherIdx := i + 1 + if other == nil { + continue + } + if best == nil { + best = other + bestIdx = otherIdx + continue + } + if path.Clean(best.name) == path.Clean(other.name) { + // We may be in a situation where we have a directory and an object with the same name. + // In that case we will drop the directory entry. + // This should however not be confused with an object with a trailing slash. + dirMatches := best.isDir() == other.isDir() + suffixMatches := strings.HasSuffix(best.name, slashSeparator) == strings.HasSuffix(other.name, slashSeparator) + + // Simple case. Both are same type with same suffix. + if dirMatches && suffixMatches { + toMerge = append(toMerge, otherIdx) + continue + } + + if !dirMatches { + // We have an object `name` or 'name/' and a directory `name/`. + if other.isDir() { + if serverDebugLog { + console.Debugln("mergeEntryChannels: discarding directory", other.name, "for object", best.name) + } + // Discard the directory. + if err := selectFrom(otherIdx); err != nil { + return err + } + continue + } + // Replace directory with object. + if serverDebugLog { + console.Debugln("mergeEntryChannels: discarding directory", best.name, "for object", other.name) + } + toMerge = toMerge[:0] + best = other + bestIdx = otherIdx + continue + } + // Leave it to be resolved. Names are different. + } + if best.name > other.name { + toMerge = toMerge[:0] + best = other + bestIdx = otherIdx + } + } + + // Merge any unmerged + if len(toMerge) > 0 { + versions := make([][]xlMetaV2ShallowVersion, 0, len(toMerge)+1) + xl, err := best.xlmeta() + if err == nil { + versions = append(versions, xl.versions) + } + for _, idx := range toMerge { + other := top[idx] + if other == nil { + continue + } + xl2, err := other.xlmeta() + if err != nil { + if err := selectFrom(idx); err != nil { + return err + } + continue + } + if xl == nil { + // Discard current "best" + if err := selectFrom(bestIdx); err != nil { + return err + } + bestIdx = idx + best = other + xl = xl2 + } else { + // Mark read, unless we added it as new "best". + if err := selectFrom(idx); err != nil { + return err + } + } + versions = append(versions, xl2.versions) + } + + if xl != nil && len(versions) > 0 { + // Merge all versions. 'strict' doesn't matter since we only need one. + xl.versions = mergeXLV2Versions(readQuorum, true, 0, versions...) + if meta, err := xl.AppendTo(metaDataPoolGet()); err == nil { + if best.reusable { + metaDataPoolPut(best.metadata) + } + best.metadata = meta + best.cached = xl + } + } + toMerge = toMerge[:0] + } + if best.name > last { + select { + case <-ctxDone: + return ctx.Err() + case out <- *best: + last = best.name + } + } else if serverDebugLog { + console.Debugln("mergeEntryChannels: discarding duplicate", best.name, "<=", last) + } + // Replace entry we just sent. + if err := selectFrom(bestIdx); err != nil { + return err + } + } +} + +// merge will merge other into m. +// If the same entries exists in both and metadata matches only one is added, +// otherwise the entry from m will be placed first. +// Operation time is expected to be O(n+m). +func (m *metaCacheEntriesSorted) merge(other metaCacheEntriesSorted, limit int) { + merged := make(metaCacheEntries, 0, m.len()+other.len()) + a := m.entries() + b := other.entries() + for len(a) > 0 && len(b) > 0 { + switch { + case a[0].name == b[0].name && bytes.Equal(a[0].metadata, b[0].metadata): + // Same, discard one. + merged = append(merged, a[0]) + a = a[1:] + b = b[1:] + case a[0].name < b[0].name: + merged = append(merged, a[0]) + a = a[1:] + default: + merged = append(merged, b[0]) + b = b[1:] + } + if limit > 0 && len(merged) >= limit { + break + } + } + // Append anything left. + if limit < 0 || len(merged) < limit { + merged = append(merged, a...) + merged = append(merged, b...) + } + m.o = merged +} + +// filterPrefix will filter m to only contain entries with the specified prefix. +func (m *metaCacheEntriesSorted) filterPrefix(s string) { + if s == "" { + return + } + m.forwardTo(s) + for i, o := range m.o { + if !o.hasPrefix(s) { + m.o = m.o[:i] + break + } + } +} + +// filterObjectsOnly will remove prefix directories. +// Order is preserved, but the underlying slice is modified. +func (m *metaCacheEntriesSorted) filterObjectsOnly() { + dst := m.o[:0] + for _, o := range m.o { + if !o.isDir() { + dst = append(dst, o) + } + } + m.o = dst +} + +// filterPrefixesOnly will remove objects. +// Order is preserved, but the underlying slice is modified. +func (m *metaCacheEntriesSorted) filterPrefixesOnly() { + dst := m.o[:0] + for _, o := range m.o { + if o.isDir() { + dst = append(dst, o) + } + } + m.o = dst +} + +// filterRecursiveEntries will keep entries only with the prefix that doesn't contain separator. +// This can be used to remove recursive listings. +// To return root elements only set prefix to an empty string. +// Order is preserved, but the underlying slice is modified. +func (m *metaCacheEntriesSorted) filterRecursiveEntries(prefix, separator string) { + dst := m.o[:0] + if prefix != "" { + m.forwardTo(prefix) + for _, o := range m.o { + ext := strings.TrimPrefix(o.name, prefix) + if len(ext) != len(o.name) { + if !strings.Contains(ext, separator) { + dst = append(dst, o) + } + } + } + } else { + // No prefix, simpler + for _, o := range m.o { + if !strings.Contains(o.name, separator) { + dst = append(dst, o) + } + } + } + m.o = dst +} + +// truncate the number of entries to maximum n. +func (m *metaCacheEntriesSorted) truncate(n int) { + if m == nil { + return + } + if len(m.o) > n { + if m.reuse { + for i, entry := range m.o[n:] { + metaDataPoolPut(entry.metadata) + m.o[n+i].metadata = nil + } + } + m.o = m.o[:n] + } +} + +// len returns the number of objects and prefix dirs in m. +func (m *metaCacheEntriesSorted) len() int { + if m == nil { + return 0 + } + return len(m.o) +} + +// entries returns the underlying objects as is currently represented. +func (m *metaCacheEntriesSorted) entries() metaCacheEntries { + if m == nil { + return nil + } + return m.o +} diff --git a/cmd/metacache-entries_test.go b/cmd/metacache-entries_test.go new file mode 100644 index 00000000..75af4d0f --- /dev/null +++ b/cmd/metacache-entries_test.go @@ -0,0 +1,659 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "math/rand" + "reflect" + "sort" + "testing" + "time" +) + +func Test_metaCacheEntries_sort(t *testing.T) { + entries := loadMetacacheSampleEntries(t) + + o := entries.entries() + if !o.isSorted() { + t.Fatal("Expected sorted objects") + } + + // Swap first and last + o[0], o[len(o)-1] = o[len(o)-1], o[0] + if o.isSorted() { + t.Fatal("Expected unsorted objects") + } + + sorted := o.sort() + if !o.isSorted() { + t.Fatal("Expected sorted o objects") + } + if !sorted.entries().isSorted() { + t.Fatal("Expected sorted wrapped objects") + } + want := loadMetacacheSampleNames + for i, got := range o { + if got.name != want[i] { + t.Errorf("entry %d, want %q, got %q", i, want[i], got.name) + } + } +} + +func Test_metaCacheEntries_forwardTo(t *testing.T) { + org := loadMetacacheSampleEntries(t) + entries := org + want := []string{"src/compress/zlib/reader_test.go", "src/compress/zlib/writer.go", "src/compress/zlib/writer_test.go"} + entries.forwardTo("src/compress/zlib/reader_test.go") + got := entries.entries().names() + if !reflect.DeepEqual(got, want) { + t.Errorf("got unexpected result: %#v", got) + } + + // Try with prefix + entries = org + entries.forwardTo("src/compress/zlib/reader_t") + got = entries.entries().names() + if !reflect.DeepEqual(got, want) { + t.Errorf("got unexpected result: %#v", got) + } +} + +func Test_metaCacheEntries_merge(t *testing.T) { + org := loadMetacacheSampleEntries(t) + a, b := org.shallowClone(), org.shallowClone() + be := b.entries() + for i := range be { + // Modify b so it isn't deduplicated. + be[i].metadata = []byte("something-else") + } + // Merge b into a + a.merge(b, -1) + //nolint:gocritic + want := append(loadMetacacheSampleNames, loadMetacacheSampleNames...) + sort.Strings(want) + got := a.entries().names() + if len(got) != len(want) { + t.Errorf("unexpected count, want %v, got %v", len(want), len(got)) + } + + for i, name := range got { + if want[i] != name { + t.Errorf("unexpected name, want %q, got %q", want[i], name) + } + } +} + +func Test_metaCacheEntries_filterObjects(t *testing.T) { + data := loadMetacacheSampleEntries(t) + data.filterObjectsOnly() + got := data.entries().names() + want := []string{"src/compress/bzip2/bit_reader.go", "src/compress/bzip2/bzip2.go", "src/compress/bzip2/bzip2_test.go", "src/compress/bzip2/huffman.go", "src/compress/bzip2/move_to_front.go", "src/compress/bzip2/testdata/Isaac.Newton-Opticks.txt.bz2", "src/compress/bzip2/testdata/e.txt.bz2", "src/compress/bzip2/testdata/fail-issue5747.bz2", "src/compress/bzip2/testdata/pass-random1.bin", "src/compress/bzip2/testdata/pass-random1.bz2", "src/compress/bzip2/testdata/pass-random2.bin", "src/compress/bzip2/testdata/pass-random2.bz2", "src/compress/bzip2/testdata/pass-sawtooth.bz2", "src/compress/bzip2/testdata/random.data.bz2", "src/compress/flate/deflate.go", "src/compress/flate/deflate_test.go", "src/compress/flate/deflatefast.go", "src/compress/flate/dict_decoder.go", "src/compress/flate/dict_decoder_test.go", "src/compress/flate/example_test.go", "src/compress/flate/flate_test.go", "src/compress/flate/huffman_bit_writer.go", "src/compress/flate/huffman_bit_writer_test.go", "src/compress/flate/huffman_code.go", "src/compress/flate/inflate.go", "src/compress/flate/inflate_test.go", "src/compress/flate/reader_test.go", "src/compress/flate/testdata/huffman-null-max.dyn.expect", "src/compress/flate/testdata/huffman-null-max.dyn.expect-noinput", "src/compress/flate/testdata/huffman-null-max.golden", "src/compress/flate/testdata/huffman-null-max.in", "src/compress/flate/testdata/huffman-null-max.wb.expect", "src/compress/flate/testdata/huffman-null-max.wb.expect-noinput", "src/compress/flate/testdata/huffman-pi.dyn.expect", "src/compress/flate/testdata/huffman-pi.dyn.expect-noinput", "src/compress/flate/testdata/huffman-pi.golden", "src/compress/flate/testdata/huffman-pi.in", "src/compress/flate/testdata/huffman-pi.wb.expect", "src/compress/flate/testdata/huffman-pi.wb.expect-noinput", "src/compress/flate/testdata/huffman-rand-1k.dyn.expect", "src/compress/flate/testdata/huffman-rand-1k.dyn.expect-noinput", "src/compress/flate/testdata/huffman-rand-1k.golden", "src/compress/flate/testdata/huffman-rand-1k.in", "src/compress/flate/testdata/huffman-rand-1k.wb.expect", "src/compress/flate/testdata/huffman-rand-1k.wb.expect-noinput", "src/compress/flate/testdata/huffman-rand-limit.dyn.expect", "src/compress/flate/testdata/huffman-rand-limit.dyn.expect-noinput", "src/compress/flate/testdata/huffman-rand-limit.golden", "src/compress/flate/testdata/huffman-rand-limit.in", "src/compress/flate/testdata/huffman-rand-limit.wb.expect", "src/compress/flate/testdata/huffman-rand-limit.wb.expect-noinput", "src/compress/flate/testdata/huffman-rand-max.golden", "src/compress/flate/testdata/huffman-rand-max.in", "src/compress/flate/testdata/huffman-shifts.dyn.expect", "src/compress/flate/testdata/huffman-shifts.dyn.expect-noinput", "src/compress/flate/testdata/huffman-shifts.golden", "src/compress/flate/testdata/huffman-shifts.in", "src/compress/flate/testdata/huffman-shifts.wb.expect", "src/compress/flate/testdata/huffman-shifts.wb.expect-noinput", "src/compress/flate/testdata/huffman-text-shift.dyn.expect", "src/compress/flate/testdata/huffman-text-shift.dyn.expect-noinput", "src/compress/flate/testdata/huffman-text-shift.golden", "src/compress/flate/testdata/huffman-text-shift.in", "src/compress/flate/testdata/huffman-text-shift.wb.expect", "src/compress/flate/testdata/huffman-text-shift.wb.expect-noinput", "src/compress/flate/testdata/huffman-text.dyn.expect", "src/compress/flate/testdata/huffman-text.dyn.expect-noinput", "src/compress/flate/testdata/huffman-text.golden", "src/compress/flate/testdata/huffman-text.in", "src/compress/flate/testdata/huffman-text.wb.expect", "src/compress/flate/testdata/huffman-text.wb.expect-noinput", "src/compress/flate/testdata/huffman-zero.dyn.expect", "src/compress/flate/testdata/huffman-zero.dyn.expect-noinput", "src/compress/flate/testdata/huffman-zero.golden", "src/compress/flate/testdata/huffman-zero.in", "src/compress/flate/testdata/huffman-zero.wb.expect", "src/compress/flate/testdata/huffman-zero.wb.expect-noinput", "src/compress/flate/testdata/null-long-match.dyn.expect-noinput", "src/compress/flate/testdata/null-long-match.wb.expect-noinput", "src/compress/flate/token.go", "src/compress/flate/writer_test.go", "src/compress/gzip/example_test.go", "src/compress/gzip/gunzip.go", "src/compress/gzip/gunzip_test.go", "src/compress/gzip/gzip.go", "src/compress/gzip/gzip_test.go", "src/compress/gzip/issue14937_test.go", "src/compress/gzip/testdata/issue6550.gz.base64", "src/compress/lzw/reader.go", "src/compress/lzw/reader_test.go", "src/compress/lzw/writer.go", "src/compress/lzw/writer_test.go", "src/compress/testdata/e.txt", "src/compress/testdata/gettysburg.txt", "src/compress/testdata/pi.txt", "src/compress/zlib/example_test.go", "src/compress/zlib/reader.go", "src/compress/zlib/reader_test.go", "src/compress/zlib/writer.go", "src/compress/zlib/writer_test.go"} + if !reflect.DeepEqual(want, got) { + t.Errorf("got unexpected result: %#v", got) + } +} + +func Test_metaCacheEntries_filterPrefixes(t *testing.T) { + data := loadMetacacheSampleEntries(t) + data.filterPrefixesOnly() + got := data.entries().names() + want := []string{"src/compress/bzip2/", "src/compress/bzip2/testdata/", "src/compress/flate/", "src/compress/flate/testdata/", "src/compress/gzip/", "src/compress/gzip/testdata/", "src/compress/lzw/", "src/compress/testdata/", "src/compress/zlib/"} + if !reflect.DeepEqual(want, got) { + t.Errorf("got unexpected result: %#v", got) + } +} + +func Test_metaCacheEntries_filterRecursive(t *testing.T) { + data := loadMetacacheSampleEntries(t) + data.filterRecursiveEntries("src/compress/bzip2/", slashSeparator) + got := data.entries().names() + want := []string{"src/compress/bzip2/", "src/compress/bzip2/bit_reader.go", "src/compress/bzip2/bzip2.go", "src/compress/bzip2/bzip2_test.go", "src/compress/bzip2/huffman.go", "src/compress/bzip2/move_to_front.go"} + if !reflect.DeepEqual(want, got) { + t.Errorf("got unexpected result: %#v", got) + } +} + +func Test_metaCacheEntries_filterRecursiveRoot(t *testing.T) { + data := loadMetacacheSampleEntries(t) + data.filterRecursiveEntries("", slashSeparator) + got := data.entries().names() + want := []string{} + if !reflect.DeepEqual(want, got) { + t.Errorf("got unexpected result: %#v", got) + } +} + +func Test_metaCacheEntries_filterRecursiveRootSep(t *testing.T) { + data := loadMetacacheSampleEntries(t) + // This will remove anything with "bzip2/" in the path since it is separator + data.filterRecursiveEntries("", "bzip2/") + got := data.entries().names() + want := []string{"src/compress/flate/", "src/compress/flate/deflate.go", "src/compress/flate/deflate_test.go", "src/compress/flate/deflatefast.go", "src/compress/flate/dict_decoder.go", "src/compress/flate/dict_decoder_test.go", "src/compress/flate/example_test.go", "src/compress/flate/flate_test.go", "src/compress/flate/huffman_bit_writer.go", "src/compress/flate/huffman_bit_writer_test.go", "src/compress/flate/huffman_code.go", "src/compress/flate/inflate.go", "src/compress/flate/inflate_test.go", "src/compress/flate/reader_test.go", "src/compress/flate/testdata/", "src/compress/flate/testdata/huffman-null-max.dyn.expect", "src/compress/flate/testdata/huffman-null-max.dyn.expect-noinput", "src/compress/flate/testdata/huffman-null-max.golden", "src/compress/flate/testdata/huffman-null-max.in", "src/compress/flate/testdata/huffman-null-max.wb.expect", "src/compress/flate/testdata/huffman-null-max.wb.expect-noinput", "src/compress/flate/testdata/huffman-pi.dyn.expect", "src/compress/flate/testdata/huffman-pi.dyn.expect-noinput", "src/compress/flate/testdata/huffman-pi.golden", "src/compress/flate/testdata/huffman-pi.in", "src/compress/flate/testdata/huffman-pi.wb.expect", "src/compress/flate/testdata/huffman-pi.wb.expect-noinput", "src/compress/flate/testdata/huffman-rand-1k.dyn.expect", "src/compress/flate/testdata/huffman-rand-1k.dyn.expect-noinput", "src/compress/flate/testdata/huffman-rand-1k.golden", "src/compress/flate/testdata/huffman-rand-1k.in", "src/compress/flate/testdata/huffman-rand-1k.wb.expect", "src/compress/flate/testdata/huffman-rand-1k.wb.expect-noinput", "src/compress/flate/testdata/huffman-rand-limit.dyn.expect", "src/compress/flate/testdata/huffman-rand-limit.dyn.expect-noinput", "src/compress/flate/testdata/huffman-rand-limit.golden", "src/compress/flate/testdata/huffman-rand-limit.in", "src/compress/flate/testdata/huffman-rand-limit.wb.expect", "src/compress/flate/testdata/huffman-rand-limit.wb.expect-noinput", "src/compress/flate/testdata/huffman-rand-max.golden", "src/compress/flate/testdata/huffman-rand-max.in", "src/compress/flate/testdata/huffman-shifts.dyn.expect", "src/compress/flate/testdata/huffman-shifts.dyn.expect-noinput", "src/compress/flate/testdata/huffman-shifts.golden", "src/compress/flate/testdata/huffman-shifts.in", "src/compress/flate/testdata/huffman-shifts.wb.expect", "src/compress/flate/testdata/huffman-shifts.wb.expect-noinput", "src/compress/flate/testdata/huffman-text-shift.dyn.expect", "src/compress/flate/testdata/huffman-text-shift.dyn.expect-noinput", "src/compress/flate/testdata/huffman-text-shift.golden", "src/compress/flate/testdata/huffman-text-shift.in", "src/compress/flate/testdata/huffman-text-shift.wb.expect", "src/compress/flate/testdata/huffman-text-shift.wb.expect-noinput", "src/compress/flate/testdata/huffman-text.dyn.expect", "src/compress/flate/testdata/huffman-text.dyn.expect-noinput", "src/compress/flate/testdata/huffman-text.golden", "src/compress/flate/testdata/huffman-text.in", "src/compress/flate/testdata/huffman-text.wb.expect", "src/compress/flate/testdata/huffman-text.wb.expect-noinput", "src/compress/flate/testdata/huffman-zero.dyn.expect", "src/compress/flate/testdata/huffman-zero.dyn.expect-noinput", "src/compress/flate/testdata/huffman-zero.golden", "src/compress/flate/testdata/huffman-zero.in", "src/compress/flate/testdata/huffman-zero.wb.expect", "src/compress/flate/testdata/huffman-zero.wb.expect-noinput", "src/compress/flate/testdata/null-long-match.dyn.expect-noinput", "src/compress/flate/testdata/null-long-match.wb.expect-noinput", "src/compress/flate/token.go", "src/compress/flate/writer_test.go", "src/compress/gzip/", "src/compress/gzip/example_test.go", "src/compress/gzip/gunzip.go", "src/compress/gzip/gunzip_test.go", "src/compress/gzip/gzip.go", "src/compress/gzip/gzip_test.go", "src/compress/gzip/issue14937_test.go", "src/compress/gzip/testdata/", "src/compress/gzip/testdata/issue6550.gz.base64", "src/compress/lzw/", "src/compress/lzw/reader.go", "src/compress/lzw/reader_test.go", "src/compress/lzw/writer.go", "src/compress/lzw/writer_test.go", "src/compress/testdata/", "src/compress/testdata/e.txt", "src/compress/testdata/gettysburg.txt", "src/compress/testdata/pi.txt", "src/compress/zlib/", "src/compress/zlib/example_test.go", "src/compress/zlib/reader.go", "src/compress/zlib/reader_test.go", "src/compress/zlib/writer.go", "src/compress/zlib/writer_test.go"} + if !reflect.DeepEqual(want, got) { + t.Errorf("got unexpected result: %#v", got) + } +} + +func Test_metaCacheEntries_filterPrefix(t *testing.T) { + data := loadMetacacheSampleEntries(t) + data.filterPrefix("src/compress/bzip2/") + got := data.entries().names() + want := []string{"src/compress/bzip2/", "src/compress/bzip2/bit_reader.go", "src/compress/bzip2/bzip2.go", "src/compress/bzip2/bzip2_test.go", "src/compress/bzip2/huffman.go", "src/compress/bzip2/move_to_front.go", "src/compress/bzip2/testdata/", "src/compress/bzip2/testdata/Isaac.Newton-Opticks.txt.bz2", "src/compress/bzip2/testdata/e.txt.bz2", "src/compress/bzip2/testdata/fail-issue5747.bz2", "src/compress/bzip2/testdata/pass-random1.bin", "src/compress/bzip2/testdata/pass-random1.bz2", "src/compress/bzip2/testdata/pass-random2.bin", "src/compress/bzip2/testdata/pass-random2.bz2", "src/compress/bzip2/testdata/pass-sawtooth.bz2", "src/compress/bzip2/testdata/random.data.bz2"} + if !reflect.DeepEqual(want, got) { + t.Errorf("got unexpected result: %#v", got) + } +} + +func Test_metaCacheEntry_isInDir(t *testing.T) { + tests := []struct { + testName string + entry string + dir string + sep string + want bool + }{ + { + testName: "basic-file", + entry: "src/file", + dir: "src/", + sep: slashSeparator, + want: true, + }, + { + testName: "basic-dir", + entry: "src/dir/", + dir: "src/", + sep: slashSeparator, + want: true, + }, + { + testName: "deeper-file", + entry: "src/dir/somewhere.ext", + dir: "src/", + sep: slashSeparator, + want: false, + }, + { + testName: "deeper-dir", + entry: "src/dir/somewhere/", + dir: "src/", + sep: slashSeparator, + want: false, + }, + { + testName: "root-dir", + entry: "doc/", + dir: "", + sep: slashSeparator, + want: true, + }, + { + testName: "root-file", + entry: "word.doc", + dir: "", + sep: slashSeparator, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.testName, func(t *testing.T) { + e := metaCacheEntry{ + name: tt.entry, + } + if got := e.isInDir(tt.dir, tt.sep); got != tt.want { + t.Errorf("isInDir() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_metaCacheEntries_resolve(t *testing.T) { + baseTime, err := time.Parse("2006/01/02", "2015/02/25") + if err != nil { + t.Fatal(err) + } + inputs := []xlMetaV2{ + 0: { + versions: []xlMetaV2ShallowVersion{ + {header: xlMetaV2VersionHeader{ + VersionID: [16]byte{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + ModTime: baseTime.Add(30 * time.Minute).UnixNano(), + Signature: [4]byte{1, 1, 1, 1}, + Type: ObjectType, + Flags: 0, + }}, + }, + }, + // Mismatches Modtime+Signature and older... + 1: { + versions: []xlMetaV2ShallowVersion{ + {header: xlMetaV2VersionHeader{ + VersionID: [16]byte{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + ModTime: baseTime.Add(15 * time.Minute).UnixNano(), + Signature: [4]byte{2, 1, 1, 1}, + Type: ObjectType, + Flags: 0, + }}, + }, + }, + // Has another version prior to the one we want. + 2: { + versions: []xlMetaV2ShallowVersion{ + {header: xlMetaV2VersionHeader{ + VersionID: [16]byte{2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + ModTime: baseTime.Add(60 * time.Minute).UnixNano(), + Signature: [4]byte{2, 1, 1, 1}, + Type: ObjectType, + Flags: 0, + }}, + {header: xlMetaV2VersionHeader{ + VersionID: [16]byte{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + ModTime: baseTime.Add(30 * time.Minute).UnixNano(), + Signature: [4]byte{1, 1, 1, 1}, + Type: ObjectType, + Flags: 0, + }}, + }, + }, + // Has a completely different version id + 3: { + versions: []xlMetaV2ShallowVersion{ + {header: xlMetaV2VersionHeader{ + VersionID: [16]byte{3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + ModTime: baseTime.Add(60 * time.Minute).UnixNano(), + Signature: [4]byte{1, 1, 1, 1}, + Type: ObjectType, + Flags: 0, + }}, + }, + }, + 4: { + versions: []xlMetaV2ShallowVersion{}, + }, + // Has a zero version id + 5: { + versions: []xlMetaV2ShallowVersion{ + {header: xlMetaV2VersionHeader{ + VersionID: [16]byte{}, + ModTime: baseTime.Add(60 * time.Minute).UnixNano(), + Signature: [4]byte{5, 1, 1, 1}, + Type: ObjectType, + Flags: 0, + }}, + }, + }, + // Zero version, modtime newer.. + 6: { + versions: []xlMetaV2ShallowVersion{ + {header: xlMetaV2VersionHeader{ + VersionID: [16]byte{}, + ModTime: baseTime.Add(90 * time.Minute).UnixNano(), + Signature: [4]byte{6, 1, 1, 1}, + Type: ObjectType, + Flags: 0, + }}, + }, + }, + 7: { + versions: []xlMetaV2ShallowVersion{ + {header: xlMetaV2VersionHeader{ + VersionID: [16]byte{}, + ModTime: baseTime.Add(90 * time.Minute).UnixNano(), + Signature: [4]byte{6, 1, 1, 1}, + Type: ObjectType, + Flags: 0, + }}, + {header: xlMetaV2VersionHeader{ + VersionID: [16]byte{2, 2, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + ModTime: baseTime.Add(60 * time.Minute).UnixNano(), + Signature: [4]byte{2, 1, 1, 1}, + Type: ObjectType, + Flags: 0, + }}, + {header: xlMetaV2VersionHeader{ + VersionID: [16]byte{3, 3, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + ModTime: baseTime.Add(60 * time.Minute).UnixNano(), + Signature: [4]byte{1, 1, 1, 1}, + Type: ObjectType, + Flags: 0, + }}, + + {header: xlMetaV2VersionHeader{ + VersionID: [16]byte{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + ModTime: baseTime.Add(30 * time.Minute).UnixNano(), + Signature: [4]byte{1, 1, 1, 1}, + Type: ObjectType, + Flags: 0, + }}, + }, + }, + // Delete marker. + 8: { + versions: []xlMetaV2ShallowVersion{ + {header: xlMetaV2VersionHeader{ + VersionID: [16]byte{7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7}, + ModTime: baseTime.Add(90 * time.Minute).UnixNano(), + Signature: [4]byte{6, 1, 1, 1}, + Type: DeleteType, + Flags: 0, + }}, + }, + }, + // Delete marker and version from 1 + 9: { + versions: []xlMetaV2ShallowVersion{ + {header: xlMetaV2VersionHeader{ + VersionID: [16]byte{7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7, 7}, + ModTime: baseTime.Add(90 * time.Minute).UnixNano(), + Signature: [4]byte{6, 1, 1, 1}, + Type: DeleteType, + Flags: 0, + }}, + {header: xlMetaV2VersionHeader{ + VersionID: [16]byte{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1}, + ModTime: baseTime.Add(15 * time.Minute).UnixNano(), + Signature: [4]byte{2, 1, 1, 1}, + Type: ObjectType, + Flags: 0, + }}, + }, + }, + } + inputSerialized := make([]metaCacheEntry, len(inputs)) + for i, xl := range inputs { + xl.sortByModTime() + var err error + entry := metaCacheEntry{ + name: "testobject", + } + entry.metadata, err = xl.AppendTo(nil) + if err != nil { + t.Fatal(err) + } + inputSerialized[i] = entry + } + + tests := []struct { + name string + m metaCacheEntries + r metadataResolutionParams + wantSelected *metaCacheEntry + wantOk bool + }{ + { + name: "consistent", + m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[0]}, + r: metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: false}, + wantSelected: &inputSerialized[0], + wantOk: true, + }, + { + name: "consistent-strict", + m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[0]}, + r: metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: false}, + wantSelected: &inputSerialized[0], + wantOk: true, + }, + { + name: "one zero, below quorum", + m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], metaCacheEntry{}}, + r: metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: false}, + wantSelected: nil, + wantOk: false, + }, + { + name: "one zero, below quorum, strict", + m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], metaCacheEntry{}}, + r: metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: true}, + wantSelected: nil, + wantOk: false, + }, + { + name: "one zero, at quorum", + m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], metaCacheEntry{}}, + r: metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: false}, + wantSelected: &inputSerialized[0], + wantOk: true, + }, + { + name: "one zero, at quorum, strict", + m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], metaCacheEntry{}}, + r: metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: true}, + wantSelected: &inputSerialized[0], + wantOk: true, + }, + { + name: "modtime, signature mismatch", + m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[1]}, + r: metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: false}, + wantSelected: &inputSerialized[0], + wantOk: true, + }, + { + name: "modtime,signature mismatch, strict", + m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[1]}, + r: metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: true}, + wantSelected: nil, + wantOk: false, + }, + { + name: "modtime, signature mismatch, at quorum", + m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[1]}, + r: metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: false}, + wantSelected: &inputSerialized[0], + wantOk: true, + }, + { + name: "modtime,signature mismatch, at quorum, strict", + m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[1]}, + r: metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: true}, + wantSelected: &inputSerialized[0], + wantOk: true, + }, + { + name: "additional version", + m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[2]}, + r: metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: false}, + wantSelected: &inputSerialized[0], + wantOk: true, + }, + { + // Since we have the same version in all inputs, that is strictly ok. + name: "additional version, strict", + m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[2]}, + r: metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: true}, + wantSelected: &inputSerialized[0], + wantOk: true, + }, + { + // Since we have the same version in all inputs, that is strictly ok. + name: "additional version, quorum one", + m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[2]}, + r: metadataResolutionParams{dirQuorum: 1, objQuorum: 1, strict: true}, + // We get the both versions, since we only request quorum 1 + wantSelected: &inputSerialized[2], + wantOk: true, + }, + { + name: "additional version, quorum two", + m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[0], inputSerialized[2]}, + r: metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: true}, + wantSelected: &inputSerialized[0], + wantOk: true, + }, + { + name: "2 additional versions, quorum two", + m: metaCacheEntries{inputSerialized[0], inputSerialized[0], inputSerialized[2], inputSerialized[2]}, + r: metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: true}, + wantSelected: &inputSerialized[2], + wantOk: true, + }, + { + // inputSerialized[1] have older versions of the second in inputSerialized[2] + name: "modtimemismatch", + m: metaCacheEntries{inputSerialized[1], inputSerialized[1], inputSerialized[2], inputSerialized[2]}, + r: metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: false}, + wantSelected: &inputSerialized[2], + wantOk: true, + }, + { + // inputSerialized[1] have older versions of the second in inputSerialized[2] + name: "modtimemismatch,strict", + m: metaCacheEntries{inputSerialized[1], inputSerialized[1], inputSerialized[2], inputSerialized[2]}, + r: metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: true}, + wantSelected: &inputSerialized[2], + wantOk: true, + }, + { + // inputSerialized[1] have older versions of the second in inputSerialized[2], but + // since it is not strict, we should get it that one (with latest modtime) + name: "modtimemismatch,not strict", + m: metaCacheEntries{inputSerialized[1], inputSerialized[1], inputSerialized[2], inputSerialized[2]}, + r: metadataResolutionParams{dirQuorum: 4, objQuorum: 4, strict: false}, + wantSelected: &inputSerialized[0], + wantOk: true, + }, + { + name: "one-q1", + m: metaCacheEntries{inputSerialized[0], inputSerialized[4], inputSerialized[4], inputSerialized[4]}, + r: metadataResolutionParams{dirQuorum: 1, objQuorum: 1, strict: false}, + wantSelected: &inputSerialized[0], + wantOk: true, + }, + { + name: "one-q1-strict", + m: metaCacheEntries{inputSerialized[0], inputSerialized[4], inputSerialized[4], inputSerialized[4]}, + r: metadataResolutionParams{dirQuorum: 1, objQuorum: 1, strict: true}, + wantSelected: &inputSerialized[0], + wantOk: true, + }, + { + name: "one-q2", + m: metaCacheEntries{inputSerialized[0], inputSerialized[4], inputSerialized[4], inputSerialized[4]}, + r: metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: false}, + wantSelected: nil, + wantOk: false, + }, + { + name: "one-q2-strict", + m: metaCacheEntries{inputSerialized[0], inputSerialized[4], inputSerialized[4], inputSerialized[4]}, + r: metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: true}, + wantSelected: nil, + wantOk: false, + }, + { + name: "two-diff-q2", + m: metaCacheEntries{inputSerialized[0], inputSerialized[3], inputSerialized[4], inputSerialized[4]}, + r: metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: false}, + wantSelected: nil, + wantOk: false, + }, + { + name: "zeroid", + m: metaCacheEntries{inputSerialized[5], inputSerialized[5], inputSerialized[6], inputSerialized[6]}, + r: metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: false}, + wantSelected: &inputSerialized[6], + wantOk: true, + }, + { + // When ID is zero, do not allow non-strict matches to reach quorum. + name: "zeroid-belowq", + m: metaCacheEntries{inputSerialized[5], inputSerialized[5], inputSerialized[6], inputSerialized[6]}, + r: metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: false}, + wantSelected: nil, + wantOk: false, + }, + { + name: "merge4", + m: metaCacheEntries{inputSerialized[2], inputSerialized[3], inputSerialized[5], inputSerialized[6]}, + r: metadataResolutionParams{dirQuorum: 1, objQuorum: 1, strict: false}, + wantSelected: &inputSerialized[7], + wantOk: true, + }, + { + name: "deletemarker", + m: metaCacheEntries{inputSerialized[8], inputSerialized[4], inputSerialized[4], inputSerialized[4]}, + r: metadataResolutionParams{dirQuorum: 1, objQuorum: 1, strict: false}, + wantSelected: &inputSerialized[8], + wantOk: true, + }, + { + name: "deletemarker-nonq", + m: metaCacheEntries{inputSerialized[8], inputSerialized[8], inputSerialized[4], inputSerialized[4]}, + r: metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: false}, + wantSelected: nil, + wantOk: false, + }, + { + name: "deletemarker-nonq", + m: metaCacheEntries{inputSerialized[8], inputSerialized[8], inputSerialized[8], inputSerialized[1]}, + r: metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: false}, + wantSelected: &inputSerialized[8], + wantOk: true, + }, + { + name: "deletemarker-mixed", + m: metaCacheEntries{inputSerialized[8], inputSerialized[8], inputSerialized[1], inputSerialized[1]}, + r: metadataResolutionParams{dirQuorum: 2, objQuorum: 2, strict: false}, + wantSelected: &inputSerialized[9], + wantOk: true, + }, + { + name: "deletemarker-q3", + m: metaCacheEntries{inputSerialized[8], inputSerialized[9], inputSerialized[9], inputSerialized[1]}, + r: metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: false}, + wantSelected: &inputSerialized[9], + wantOk: true, + }, + { + name: "deletemarker-q3-strict", + m: metaCacheEntries{inputSerialized[8], inputSerialized[9], inputSerialized[9], inputSerialized[1]}, + r: metadataResolutionParams{dirQuorum: 3, objQuorum: 3, strict: true}, + wantSelected: &inputSerialized[9], + wantOk: true, + }, + } + + for testID, tt := range tests { + rng := rand.New(rand.NewSource(0)) + // Run for a number of times, shuffling the input to ensure that output is consistent. + for i := range 10 { + t.Run(fmt.Sprintf("test-%d-%s-run-%d", testID, tt.name, i), func(t *testing.T) { + if i > 0 { + rng.Shuffle(len(tt.m), func(i, j int) { + tt.m[i], tt.m[j] = tt.m[j], tt.m[i] + }) + } + gotSelected, gotOk := tt.m.resolve(&tt.r) + if gotOk != tt.wantOk { + t.Errorf("resolve() gotOk = %v, want %v", gotOk, tt.wantOk) + } + if gotSelected != nil { + gotSelected.cached = nil + gotSelected.reusable = false + } + if !reflect.DeepEqual(gotSelected, tt.wantSelected) { + wantM, _ := tt.wantSelected.xlmeta() + gotM, _ := gotSelected.xlmeta() + t.Errorf("resolve() gotSelected = \n%#v, want \n%#v", *gotM, *wantM) + } + }) + } + } +} diff --git a/cmd/metacache-manager.go b/cmd/metacache-manager.go new file mode 100644 index 00000000..d23a8aa1 --- /dev/null +++ b/cmd/metacache-manager.go @@ -0,0 +1,210 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "fmt" + "runtime/debug" + "sync" + "time" + + "github.com/minio/minio/internal/logger" +) + +// localMetacacheMgr is the *local* manager for this peer. +// It should never be used directly since buckets are +// distributed deterministically. +// Therefore no cluster locks are required. +var localMetacacheMgr = &metacacheManager{ + buckets: make(map[string]*bucketMetacache), + trash: make(map[string]metacache), +} + +type metacacheManager struct { + mu sync.RWMutex + init sync.Once + buckets map[string]*bucketMetacache + trash map[string]metacache // Recently deleted lists. +} + +const metacacheMaxEntries = 5000 + +// initManager will start async saving the cache. +func (m *metacacheManager) initManager() { + // Add a transient bucket. + // Start saver when object layer is ready. + go func() { + objAPI := newObjectLayerFn() + for objAPI == nil { + time.Sleep(time.Second) + objAPI = newObjectLayerFn() + } + + t := time.NewTicker(time.Minute) + defer t.Stop() + + var exit bool + for !exit { + select { + case <-t.C: + case <-GlobalContext.Done(): + exit = true + } + m.mu.RLock() + for _, v := range m.buckets { + if !exit { + v.cleanup() + } + } + m.mu.RUnlock() + m.mu.Lock() + for k, v := range m.trash { + if time.Since(v.lastUpdate) > metacacheMaxRunningAge { + v.delete(context.Background()) + delete(m.trash, k) + } + } + m.mu.Unlock() + } + }() +} + +// updateCacheEntry will update non-transient state. +func (m *metacacheManager) updateCacheEntry(update metacache) (metacache, error) { + m.mu.RLock() + if meta, ok := m.trash[update.id]; ok { + m.mu.RUnlock() + return meta, nil + } + + b, ok := m.buckets[update.bucket] + m.mu.RUnlock() + if ok { + return b.updateCacheEntry(update) + } + + // We should have either a trashed bucket or this + return metacache{}, errVolumeNotFound +} + +// getBucket will get a bucket metacache or load it from disk if needed. +func (m *metacacheManager) getBucket(ctx context.Context, bucket string) *bucketMetacache { + m.init.Do(m.initManager) + + // Return a transient bucket for invalid or system buckets. + m.mu.RLock() + b, ok := m.buckets[bucket] + if ok { + m.mu.RUnlock() + if b.bucket != bucket { + logger.Info("getBucket: cached bucket %s does not match this bucket %s", b.bucket, bucket) + debug.PrintStack() + } + return b + } + + m.mu.RUnlock() + m.mu.Lock() + defer m.mu.Unlock() + // See if someone else fetched it while we waited for the lock. + b, ok = m.buckets[bucket] + if ok { + if b.bucket != bucket { + logger.Info("getBucket: newly cached bucket %s does not match this bucket %s", b.bucket, bucket) + debug.PrintStack() + } + return b + } + + // New bucket. If we fail return the transient bucket. + b = newBucketMetacache(bucket, true) + m.buckets[bucket] = b + return b +} + +// deleteBucketCache will delete the bucket cache if it exists. +func (m *metacacheManager) deleteBucketCache(bucket string) { + m.init.Do(m.initManager) + m.mu.Lock() + b, ok := m.buckets[bucket] + if !ok { + m.mu.Unlock() + return + } + delete(m.buckets, bucket) + m.mu.Unlock() + + // Since deletes may take some time we try to do it without + // holding lock to m all the time. + b.mu.Lock() + defer b.mu.Unlock() + for k, v := range b.caches { + if time.Since(v.lastUpdate) > metacacheMaxRunningAge { + v.delete(context.Background()) + continue + } + v.error = "Bucket deleted" + v.status = scanStateError + m.mu.Lock() + m.trash[k] = v + m.mu.Unlock() + } +} + +// deleteAll will delete all caches. +func (m *metacacheManager) deleteAll() { + m.init.Do(m.initManager) + m.mu.Lock() + defer m.mu.Unlock() + for bucket, b := range m.buckets { + b.deleteAll() + delete(m.buckets, bucket) + } +} + +// checkMetacacheState should be used if data is not updating. +// Should only be called if a failure occurred. +func (o listPathOptions) checkMetacacheState(ctx context.Context, rpc *peerRESTClient) error { + // We operate on a copy... + o.Create = false + c, err := rpc.GetMetacacheListing(ctx, o) + if err != nil { + return err + } + cache := *c + + if cache.status == scanStateNone || cache.fileNotFound { + return errFileNotFound + } + if cache.status == scanStateSuccess || cache.status == scanStateStarted { + if time.Since(cache.lastUpdate) > metacacheMaxRunningAge { + // We got a stale entry, mark error on handling server. + err := fmt.Errorf("timeout: list %s not updated", cache.id) + cache.error = err.Error() + cache.status = scanStateError + rpc.UpdateMetacacheListing(ctx, cache) + return err + } + return nil + } + if cache.error != "" { + return fmt.Errorf("async cache listing failed with: %s", cache.error) + } + return nil +} diff --git a/cmd/metacache-marker.go b/cmd/metacache-marker.go new file mode 100644 index 00000000..4548af2b --- /dev/null +++ b/cmd/metacache-marker.go @@ -0,0 +1,90 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "fmt" + "strconv" + "strings" +) + +// markerTagVersion is the marker version. +// Should not need to be updated unless a fundamental change is made to the marker format. +const markerTagVersion = "v2" + +// parseMarker will parse a marker possibly encoded with encodeMarker +func (o *listPathOptions) parseMarker() { + s := o.Marker + if !strings.Contains(s, "[minio_cache:"+markerTagVersion) { + return + } + start := strings.LastIndex(s, "[") + o.Marker = s[:start] + end := strings.LastIndex(s, "]") + tag := strings.Trim(s[start:end], "[]") + tags := strings.SplitSeq(tag, ",") + for tag := range tags { + kv := strings.Split(tag, ":") + if len(kv) < 2 { + continue + } + switch kv[0] { + case "minio_cache": + if kv[1] != markerTagVersion { + continue + } + case "id": + o.ID = kv[1] + case "return": + o.ID = mustGetUUID() + o.Create = true + case "p": // pool + v, err := strconv.ParseInt(kv[1], 10, 64) + if err != nil { + o.ID = mustGetUUID() + o.Create = true + continue + } + o.pool = int(v) + case "s": // set + v, err := strconv.ParseInt(kv[1], 10, 64) + if err != nil { + o.ID = mustGetUUID() + o.Create = true + continue + } + o.set = int(v) + default: + // Ignore unknown + } + } +} + +// encodeMarker will encode a uuid and return it as a marker. +// uuid cannot contain '[', ':' or ','. +func (o listPathOptions) encodeMarker(marker string) string { + if o.ID == "" { + // Mark as returning listing... + return fmt.Sprintf("%s[minio_cache:%s,return:]", marker, markerTagVersion) + } + if strings.ContainsAny(o.ID, "[:,") { + internalLogIf(context.Background(), fmt.Errorf("encodeMarker: uuid %s contained invalid characters", o.ID)) + } + return fmt.Sprintf("%s[minio_cache:%s,id:%s,p:%d,s:%d]", marker, markerTagVersion, o.ID, o.pool, o.set) +} diff --git a/cmd/metacache-server-pool.go b/cmd/metacache-server-pool.go new file mode 100644 index 00000000..bcdfed8f --- /dev/null +++ b/cmd/metacache-server-pool.go @@ -0,0 +1,456 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "os" + pathutil "path" + "strings" + "sync" + "time" + + "github.com/minio/minio/internal/grid" + xioutil "github.com/minio/minio/internal/ioutil" +) + +func renameAllBucketMetacache(epPath string) error { + // Rename all previous `.minio.sys/buckets//.metacache` to + // to `.minio.sys/tmp/` for deletion. + return readDirFn(pathJoin(epPath, minioMetaBucket, bucketMetaPrefix), func(name string, typ os.FileMode) error { + if typ == os.ModeDir { + tmpMetacacheOld := pathutil.Join(epPath, minioMetaTmpDeletedBucket, mustGetUUID()) + if err := renameAll(pathJoin(epPath, minioMetaBucket, metacachePrefixForID(name, slashSeparator)), + tmpMetacacheOld, epPath); err != nil && err != errFileNotFound { + return fmt.Errorf("unable to rename (%s -> %s) %w", + pathJoin(epPath, minioMetaBucket+metacachePrefixForID(minioMetaBucket, slashSeparator)), + tmpMetacacheOld, + osErrToFileErr(err)) + } + } + return nil + }) +} + +// listPath will return the requested entries. +// If no more entries are in the listing io.EOF is returned, +// otherwise nil or an unexpected error is returned. +// The listPathOptions given will be checked and modified internally. +// Required important fields are Bucket, Prefix, Separator. +// Other important fields are Limit, Marker. +// List ID always derived from the Marker. +func (z *erasureServerPools) listPath(ctx context.Context, o *listPathOptions) (entries metaCacheEntriesSorted, err error) { + if err := checkListObjsArgs(ctx, o.Bucket, o.Prefix, o.Marker); err != nil { + return entries, err + } + + // Marker points to before the prefix, just ignore it. + if o.Marker < o.Prefix { + o.Marker = "" + } + + // Marker is set validate pre-condition. + if o.Marker != "" && o.Prefix != "" { + // Marker not common with prefix is not implemented. Send an empty response + if !HasPrefix(o.Marker, o.Prefix) { + return entries, io.EOF + } + } + + // With max keys of zero we have reached eof, return right here. + if o.Limit == 0 { + return entries, io.EOF + } + + // For delimiter and prefix as '/' we do not list anything at all + // along // with the prefix. On a flat namespace with 'prefix' + // as '/' we don't have any entries, since all the keys are + // of form 'keyName/...' + if strings.HasPrefix(o.Prefix, SlashSeparator) { + return entries, io.EOF + } + + // If delimiter is slashSeparator we must return directories of + // the non-recursive scan unless explicitly requested. + o.IncludeDirectories = o.Separator == slashSeparator + if (o.Separator == slashSeparator || o.Separator == "") && !o.Recursive { + o.Recursive = o.Separator != slashSeparator + o.Separator = slashSeparator + } else { + // Default is recursive, if delimiter is set then list non recursive. + o.Recursive = true + } + + // Decode and get the optional list id from the marker. + o.parseMarker() + if o.BaseDir == "" { + o.BaseDir = baseDirFromPrefix(o.Prefix) + } + o.Transient = o.Transient || isReservedOrInvalidBucket(o.Bucket, false) + o.SetFilter() + if o.Transient { + o.Create = false + } + + // We have 2 cases: + // 1) Cold listing, just list. + // 2) Returning, but with no id. Start async listing. + // 3) Returning, with ID, stream from list. + // + // If we don't have a list id we must ask the server if it has a cache or create a new. + if o.ID != "" && !o.Transient { + // Create or ping with handout... + rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix)) + var c *metacache + if rpc == nil { + resp := localMetacacheMgr.getBucket(ctx, o.Bucket).findCache(*o) + c = &resp + } else { + rctx, cancel := context.WithTimeout(ctx, 5*time.Second) + c, err = rpc.GetMetacacheListing(rctx, *o) + cancel() + } + if err != nil { + if errors.Is(err, context.Canceled) { + // Context is canceled, return at once. + // request canceled, no entries to return + return entries, io.EOF + } + if !IsErr(err, context.DeadlineExceeded, grid.ErrDisconnected) { + // Report error once per bucket, but continue listing.x + storageLogOnceIf(ctx, err, "GetMetacacheListing:"+o.Bucket) + } + o.Transient = true + o.Create = false + o.ID = mustGetUUID() + } else { + if c.fileNotFound { + // No cache found, no entries found. + return entries, io.EOF + } + if c.status == scanStateError || c.status == scanStateNone { + o.ID = "" + o.Create = false + o.debugln("scan status", c.status, " - waiting a roundtrip to create") + } else { + // Continue listing + o.ID = c.id + go c.keepAlive(ctx, rpc) + } + } + } + + if o.ID != "" && !o.Transient { + // We have an existing list ID, continue streaming. + if o.Create { + o.debugln("Creating", o) + entries, err = z.listAndSave(ctx, o) + if err == nil || err == io.EOF { + return entries, err + } + entries.truncate(0) + } else { + if o.pool < len(z.serverPools) && o.set < len(z.serverPools[o.pool].sets) { + o.debugln("Resuming", o) + entries, err = z.serverPools[o.pool].sets[o.set].streamMetadataParts(ctx, *o) + entries.reuse = true // We read from stream and are not sharing results. + if err == nil { + return entries, nil + } + } else { + err = fmt.Errorf("invalid pool/set") + o.pool, o.set = 0, 0 + } + } + if IsErr(err, []error{ + nil, + context.Canceled, + context.DeadlineExceeded, + // io.EOF is expected and should be returned but no need to log it. + io.EOF, + }...) { + // Expected good errors we don't need to return error. + return entries, err + } + entries.truncate(0) + go func() { + rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix)) + if rpc != nil { + ctx, cancel := context.WithTimeout(GlobalContext, 5*time.Second) + defer cancel() + c, err := rpc.GetMetacacheListing(ctx, *o) + if err == nil { + c.error = "no longer used" + c.status = scanStateError + rpc.UpdateMetacacheListing(ctx, *c) + } + } + }() + o.ID = "" + } + + if contextCanceled(ctx) { + return entries, ctx.Err() + } + // Do listing in-place. + // Create output for our results. + // Create filter for results. + o.debugln("Raw List", o) + filterCh := make(chan metaCacheEntry, o.Limit) + listCtx, cancelList := context.WithCancel(ctx) + filteredResults := o.gatherResults(listCtx, filterCh) + var wg sync.WaitGroup + wg.Add(1) + var listErr error + + go func(o listPathOptions) { + defer wg.Done() + if o.Lifecycle == nil { + // No filtering ahead, ask drives to stop + // listing exactly at a specific limit. + o.StopDiskAtLimit = true + } + listErr = z.listMerged(listCtx, o, filterCh) + o.debugln("listMerged returned with", listErr) + }(*o) + + entries, err = filteredResults() + cancelList() + wg.Wait() + if listErr != nil && !errors.Is(listErr, context.Canceled) { + return entries, listErr + } + entries.reuse = true + truncated := entries.len() > o.Limit || err == nil + entries.truncate(o.Limit) + if !o.Transient && truncated { + if o.ID == "" { + entries.listID = mustGetUUID() + } else { + entries.listID = o.ID + } + } + if !truncated { + return entries, io.EOF + } + return entries, nil +} + +// listMerged will list across all sets and return a merged results stream. +// The result channel is closed when no more results are expected. +func (z *erasureServerPools) listMerged(ctx context.Context, o listPathOptions, results chan<- metaCacheEntry) error { + var mu sync.Mutex + var wg sync.WaitGroup + var errs []error + allAtEOF := true + var inputs []chan metaCacheEntry + mu.Lock() + // Ask all sets and merge entries. + listCtx, cancelList := context.WithCancel(ctx) + defer cancelList() + for _, pool := range z.serverPools { + for _, set := range pool.sets { + wg.Add(1) + innerResults := make(chan metaCacheEntry, 100) + inputs = append(inputs, innerResults) + go func(i int, set *erasureObjects) { + defer wg.Done() + err := set.listPath(listCtx, o, innerResults) + mu.Lock() + defer mu.Unlock() + if err == nil { + allAtEOF = false + } + errs[i] = err + }(len(errs), set) + errs = append(errs, nil) + } + } + mu.Unlock() + + // Gather results to a single channel. + // Quorum is one since we are merging across sets. + err := mergeEntryChannels(ctx, inputs, results, 1) + + cancelList() + wg.Wait() + + // we should return 'errs' from per disk + if isAllNotFound(errs) { + if isAllVolumeNotFound(errs) { + return errVolumeNotFound + } + return nil + } + + if err != nil { + return err + } + + if contextCanceled(ctx) { + return ctx.Err() + } + + for _, err := range errs { + if errors.Is(err, io.EOF) { + continue + } + if err == nil || contextCanceled(ctx) || errors.Is(err, context.Canceled) { + allAtEOF = false + continue + } + storageLogIf(ctx, err) + return err + } + if allAtEOF { + return io.EOF + } + return nil +} + +// triggerExpiryAndRepl applies lifecycle and replication actions on the listing +// It returns true if the listing is non-versioned and the given object is expired. +func triggerExpiryAndRepl(ctx context.Context, o listPathOptions, obj metaCacheEntry) (skip bool) { + versioned := o.Versioning != nil && o.Versioning.Versioned(obj.name) + + // skip latest object from listing only for regular + // listObjects calls, versioned based listing cannot + // filter out between versions 'obj' cannot be truncated + // in such a manner, so look for skipping an object only + // for regular ListObjects() call only. + if !o.Versioned && !o.V1 { + fi, err := obj.fileInfo(o.Bucket) + if err != nil { + return skip + } + objInfo := fi.ToObjectInfo(o.Bucket, obj.name, versioned) + if o.Lifecycle != nil { + act := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, o.Replication.Config, objInfo).Action + skip = act.Delete() && !act.DeleteRestored() + } + } + + fiv, err := obj.fileInfoVersions(o.Bucket) + if err != nil { + return skip + } + + // Expire all versions if needed, if not attempt to queue for replication. + for _, version := range fiv.Versions { + objInfo := version.ToObjectInfo(o.Bucket, obj.name, versioned) + + if o.Lifecycle != nil { + evt := evalActionFromLifecycle(ctx, *o.Lifecycle, o.Retention, o.Replication.Config, objInfo) + if evt.Action.Delete() { + globalExpiryState.enqueueByDays(objInfo, evt, lcEventSrc_s3ListObjects) + if !evt.Action.DeleteRestored() { + continue + } // queue version for replication upon expired restored copies if needed. + } + } + + queueReplicationHeal(ctx, o.Bucket, objInfo, o.Replication, 0) + } + return skip +} + +func (z *erasureServerPools) listAndSave(ctx context.Context, o *listPathOptions) (entries metaCacheEntriesSorted, err error) { + // Use ID as the object name... + o.pool = z.getAvailablePoolIdx(ctx, minioMetaBucket, o.ID, 10<<20) + if o.pool < 0 { + // No space or similar, don't persist the listing. + o.pool = 0 + o.Create = false + o.ID = "" + o.Transient = true + return entries, errDiskFull + } + o.set = z.serverPools[o.pool].getHashedSetIndex(o.ID) + saver := z.serverPools[o.pool].sets[o.set] + + // Disconnect from call above, but cancel on exit. + listCtx, cancel := context.WithCancel(GlobalContext) + saveCh := make(chan metaCacheEntry, metacacheBlockSize) + inCh := make(chan metaCacheEntry, metacacheBlockSize) + outCh := make(chan metaCacheEntry, o.Limit) + + filteredResults := o.gatherResults(ctx, outCh) + + mc := o.newMetacache() + meta := metaCacheRPC{meta: &mc, cancel: cancel, rpc: globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix)), o: *o} + + // Save listing... + go func() { + if err := saver.saveMetaCacheStream(listCtx, &meta, saveCh); err != nil { + meta.setErr(err.Error()) + } + cancel() + }() + + // Do listing... + go func(o listPathOptions) { + err := z.listMerged(listCtx, o, inCh) + if err != nil { + meta.setErr(err.Error()) + } + o.debugln("listAndSave: listing", o.ID, "finished with ", err) + }(*o) + + // Keep track of when we return since we no longer have to send entries to output. + var funcReturned bool + var funcReturnedMu sync.Mutex + defer func() { + funcReturnedMu.Lock() + funcReturned = true + funcReturnedMu.Unlock() + }() + // Write listing to results and saver. + go func() { + var returned bool + for entry := range inCh { + if o.shouldSkip(ctx, entry) { + continue + } + if !returned { + funcReturnedMu.Lock() + returned = funcReturned + funcReturnedMu.Unlock() + outCh <- entry + if returned { + xioutil.SafeClose(outCh) + } + } + entry.reusable = returned + saveCh <- entry + } + if !returned { + xioutil.SafeClose(outCh) + } + xioutil.SafeClose(saveCh) + }() + + entries, err = filteredResults() + if err == nil { + // Check if listing recorded an error. + err = meta.getErr() + } + return entries, err +} diff --git a/cmd/metacache-set.go b/cmd/metacache-set.go new file mode 100644 index 00000000..c43d18d7 --- /dev/null +++ b/cmd/metacache-set.go @@ -0,0 +1,1204 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/gob" + "encoding/json" + "errors" + "fmt" + "io" + "maps" + "math/rand" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + jsoniter "github.com/json-iterator/go" + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/versioning" + "github.com/minio/minio/internal/color" + "github.com/minio/minio/internal/hash" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/pkg/v3/console" +) + +//go:generate msgp -file $GOFILE -unexported + +type listPathOptions struct { + // ID of the listing. + // This will be used to persist the list. + ID string + + // Bucket of the listing. + Bucket string + + // Directory inside the bucket. + // When unset listPath will set this based on Prefix + BaseDir string + + // Scan/return only content with prefix. + Prefix string + + // FilterPrefix will return only results with this prefix when scanning. + // Should never contain a slash. + // Prefix should still be set. + FilterPrefix string + + // Marker to resume listing. + // The response will be the first entry >= this object name. + Marker string + + // Limit the number of results. + Limit int + + // The number of disks to ask. + AskDisks string + + // InclDeleted will keep all entries where latest version is a delete marker. + InclDeleted bool + + // Scan recursively. + // If false only main directory will be scanned. + // Should always be true if Separator is n SlashSeparator. + Recursive bool + + // Separator to use. + Separator string + + // Create indicates that the lister should not attempt to load an existing cache. + Create bool + + // Include pure directories. + IncludeDirectories bool + + // Transient is set if the cache is transient due to an error or being a reserved bucket. + // This means the cache metadata will not be persisted on disk. + // A transient result will never be returned from the cache so knowing the list id is required. + Transient bool + + // Versioned is this a ListObjectVersions call. + Versioned bool + // V1 listing type + V1 bool + + // Versioning config is used for if the path + // has versioning enabled. + Versioning *versioning.Versioning `msg:"-"` + + // Lifecycle performs filtering based on lifecycle. + // This will filter out objects if the most recent version should be deleted by lifecycle. + // Is not transferred across request calls. + Lifecycle *lifecycle.Lifecycle `msg:"-"` + + // Retention configuration, needed to be passed along with lifecycle if set. + Retention lock.Retention `msg:"-"` + + // Replication configuration + Replication replicationConfig `msg:"-"` + + // StopDiskAtLimit will stop listing on each disk when limit number off objects has been returned. + StopDiskAtLimit bool + + // pool and set of where the cache is located. + pool, set int +} + +func init() { + gob.Register(listPathOptions{}) +} + +func (o *listPathOptions) setBucketMeta(ctx context.Context) { + lc, _ := globalLifecycleSys.Get(o.Bucket) + vc, _ := globalBucketVersioningSys.Get(o.Bucket) + + // Check if bucket is object locked. + rcfg, _ := globalBucketObjectLockSys.Get(o.Bucket) + replCfg, _, _ := globalBucketMetadataSys.GetReplicationConfig(ctx, o.Bucket) + tgts, _ := globalBucketTargetSys.ListBucketTargets(ctx, o.Bucket) + o.Lifecycle = lc + o.Versioning = vc + o.Replication = replicationConfig{ + Config: replCfg, + remotes: tgts, + } + o.Retention = rcfg +} + +// newMetacache constructs a new metacache from the options. +func (o listPathOptions) newMetacache() metacache { + return metacache{ + id: o.ID, + bucket: o.Bucket, + root: o.BaseDir, + recursive: o.Recursive, + status: scanStateStarted, + error: "", + started: UTCNow(), + lastHandout: UTCNow(), + lastUpdate: UTCNow(), + ended: time.Time{}, + dataVersion: metacacheStreamVersion, + filter: o.FilterPrefix, + } +} + +func (o *listPathOptions) debugf(format string, data ...any) { + if serverDebugLog { + console.Debugf(format+"\n", data...) + } +} + +func (o *listPathOptions) debugln(data ...any) { + if serverDebugLog { + console.Debugln(data...) + } +} + +func (o *listPathOptions) shouldSkip(ctx context.Context, entry metaCacheEntry) (yes bool) { + if !o.IncludeDirectories && (entry.isDir() || (!o.Versioned && entry.isObjectDir() && entry.isLatestDeletemarker())) { + return true + } + if o.Marker != "" && entry.name < o.Marker { + return true + } + if !strings.HasPrefix(entry.name, o.Prefix) { + return true + } + if o.Separator != "" && entry.isDir() && !strings.Contains(strings.TrimPrefix(entry.name, o.Prefix), o.Separator) { + return true + } + if !o.Recursive && !entry.isInDir(o.Prefix, o.Separator) { + return true + } + if !o.InclDeleted && entry.isObject() && entry.isLatestDeletemarker() && !entry.isObjectDir() { + return true + } + if o.Lifecycle != nil || o.Replication.Config != nil { + return triggerExpiryAndRepl(ctx, *o, entry) + } + return false +} + +// gatherResults will collect all results on the input channel and filter results according +// to the options or to the current bucket ILM expiry rules. +// Caller should close the channel when done. +// The returned function will return the results once there is enough or input is closed, +// or the context is canceled. +func (o *listPathOptions) gatherResults(ctx context.Context, in <-chan metaCacheEntry) func() (metaCacheEntriesSorted, error) { + resultsDone := make(chan metaCacheEntriesSorted) + // Copy so we can mutate + resCh := resultsDone + var done atomic.Bool + resErr := io.EOF + + go func() { + var results metaCacheEntriesSorted + var returned bool + for entry := range in { + if returned { + // past limit + continue + } + returned = done.Load() + if returned { + resCh = nil + continue + } + if yes := o.shouldSkip(ctx, entry); yes { + // when we have not enough results, record the skipped entry + if o.Limit > 0 && results.len() < o.Limit { + results.lastSkippedEntry = entry.name + } + continue + } + if o.Limit > 0 && results.len() >= o.Limit { + // We have enough and we have more. + // Do not return io.EOF + if resCh != nil { + resErr = nil + select { + case resCh <- results: + case <-ctx.Done(): + } + resCh = nil + returned = true + } + continue + } + results.o = append(results.o, entry) + } + if resCh != nil { + resErr = io.EOF + select { + case <-ctx.Done(): + // Nobody wants it. + case resCh <- results: + } + } + }() + return func() (metaCacheEntriesSorted, error) { + select { + case <-ctx.Done(): + done.Store(true) + return metaCacheEntriesSorted{}, ctx.Err() + case r := <-resultsDone: + return r, resErr + } + } +} + +// findFirstPart will find the part with 0 being the first that corresponds to the marker in the options. +// io.ErrUnexpectedEOF is returned if the place containing the marker hasn't been scanned yet. +// io.EOF indicates the marker is beyond the end of the stream and does not exist. +func (o *listPathOptions) findFirstPart(fi FileInfo) (int, error) { + search := o.Marker + if search == "" { + search = o.Prefix + } + if search == "" { + return 0, nil + } + o.debugln("searching for ", search) + var tmp metacacheBlock + json := jsoniter.ConfigCompatibleWithStandardLibrary + i := 0 + for { + partKey := fmt.Sprintf("%s-metacache-part-%d", ReservedMetadataPrefixLower, i) + v, ok := fi.Metadata[partKey] + if !ok { + o.debugln("no match in metadata, waiting") + return -1, io.ErrUnexpectedEOF + } + err := json.Unmarshal([]byte(v), &tmp) + if !ok { + bugLogIf(context.Background(), err) + return -1, err + } + if tmp.First == "" && tmp.Last == "" && tmp.EOS { + return 0, errFileNotFound + } + if tmp.First >= search { + o.debugln("First >= search", v) + return i, nil + } + if tmp.Last >= search { + o.debugln("Last >= search", v) + return i, nil + } + if tmp.EOS { + o.debugln("no match, at EOS", v) + return -3, io.EOF + } + o.debugln("First ", tmp.First, "<", search, " search", i) + i++ + } +} + +// updateMetacacheListing will update the metacache listing. +func (o *listPathOptions) updateMetacacheListing(m metacache, rpc *peerRESTClient) (metacache, error) { + if rpc == nil { + return localMetacacheMgr.updateCacheEntry(m) + } + return rpc.UpdateMetacacheListing(context.Background(), m) +} + +func getMetacacheBlockInfo(fi FileInfo, block int) (*metacacheBlock, error) { + var tmp metacacheBlock + partKey := fmt.Sprintf("%s-metacache-part-%d", ReservedMetadataPrefixLower, block) + v, ok := fi.Metadata[partKey] + if !ok { + return nil, io.ErrUnexpectedEOF + } + return &tmp, json.Unmarshal([]byte(v), &tmp) +} + +const metacachePrefix = ".metacache" + +func metacachePrefixForID(bucket, id string) string { + return pathJoin(bucketMetaPrefix, bucket, metacachePrefix, id) +} + +// objectPath returns the object path of the cache. +func (o *listPathOptions) objectPath(block int) string { + return pathJoin(metacachePrefixForID(o.Bucket, o.ID), "block-"+strconv.Itoa(block)+".s2") +} + +func (o *listPathOptions) SetFilter() { + switch { + case metacacheSharePrefix: + return + case o.Prefix == o.BaseDir: + // No additional prefix + return + } + // Remove basedir. + o.FilterPrefix = strings.TrimPrefix(o.Prefix, o.BaseDir) + // Remove leading and trailing slashes. + o.FilterPrefix = strings.Trim(o.FilterPrefix, slashSeparator) + + if strings.Contains(o.FilterPrefix, slashSeparator) { + // Sanity check, should not happen. + o.FilterPrefix = "" + } +} + +// filter will apply the options and return the number of objects requested by the limit. +// Will return io.EOF if there are no more entries with the same filter. +// The last entry can be used as a marker to resume the listing. +func (r *metacacheReader) filter(o listPathOptions) (entries metaCacheEntriesSorted, err error) { + // Forward to prefix, if any + err = r.forwardTo(o.Prefix) + if err != nil { + return entries, err + } + if o.Marker != "" { + err = r.forwardTo(o.Marker) + if err != nil { + return entries, err + } + } + o.debugln("forwarded to ", o.Prefix, "marker:", o.Marker, "sep:", o.Separator) + + // Filter + if !o.Recursive { + entries.o = make(metaCacheEntries, 0, o.Limit) + pastPrefix := false + err := r.readFn(func(entry metaCacheEntry) bool { + if o.Prefix != "" && !strings.HasPrefix(entry.name, o.Prefix) { + // We are past the prefix, don't continue. + pastPrefix = true + return false + } + if !o.IncludeDirectories && (entry.isDir() || (!o.Versioned && entry.isObjectDir() && entry.isLatestDeletemarker())) { + return true + } + if !entry.isInDir(o.Prefix, o.Separator) { + return true + } + if !o.InclDeleted && entry.isObject() && entry.isLatestDeletemarker() && !entry.isObjectDir() { + return true + } + if !o.InclDeleted && entry.isAllFreeVersions() { + return true + } + entries.o = append(entries.o, entry) + return entries.len() < o.Limit + }) + if (err != nil && errors.Is(err, io.EOF)) || pastPrefix || r.nextEOF() { + return entries, io.EOF + } + return entries, err + } + + // We should not need to filter more. + return r.readN(o.Limit, o.InclDeleted, o.IncludeDirectories, o.Versioned, o.Prefix) +} + +func (er *erasureObjects) streamMetadataParts(ctx context.Context, o listPathOptions) (entries metaCacheEntriesSorted, err error) { + retries := 0 + rpc := globalNotificationSys.restClientFromHash(pathJoin(o.Bucket, o.Prefix)) + + const ( + retryDelay = 50 * time.Millisecond + retryDelay250 = 250 * time.Millisecond + ) + + for { + if contextCanceled(ctx) { + return entries, ctx.Err() + } + + // If many failures, check the cache state. + if retries > 10 { + err := o.checkMetacacheState(ctx, rpc) + if err != nil { + return entries, fmt.Errorf("remote listing canceled: %w", err) + } + retries = 1 + } + + // All operations are performed without locks, so we must be careful and allow for failures. + // Read metadata associated with the object from a disk. + if retries > 0 { + for _, disk := range er.getDisks() { + if disk == nil { + continue + } + if !disk.IsOnline() { + continue + } + _, err := disk.ReadVersion(ctx, "", minioMetaBucket, + o.objectPath(0), "", ReadOptions{}) + if err != nil { + time.Sleep(retryDelay250) + retries++ + continue + } + break + } + } + retryWait := func() { + retries++ + if retries == 1 { + time.Sleep(retryDelay) + } else { + time.Sleep(retryDelay250) + } + } + // Load first part metadata... + // Read metadata associated with the object from all disks. + fi, metaArr, onlineDisks, err := er.getObjectFileInfo(ctx, minioMetaBucket, o.objectPath(0), ObjectOptions{}, true) + if err != nil { + switch toObjectErr(err, minioMetaBucket, o.objectPath(0)).(type) { + case ObjectNotFound, InsufficientReadQuorum: + retryWait() + continue + } + // Allow one fast retry for other errors. + if retries > 0 { + return entries, fmt.Errorf("reading first part metadata: %v", err) + } + retryWait() + continue + } + + partN, err := o.findFirstPart(fi) + switch { + case err == nil: + case errors.Is(err, io.ErrUnexpectedEOF): + if retries == 10 { + err := o.checkMetacacheState(ctx, rpc) + if err != nil { + return entries, fmt.Errorf("remote listing canceled: %w", err) + } + retries = -1 + } + retryWait() + continue + case errors.Is(err, io.EOF): + return entries, io.EOF + } + + // We got a stream to start at. + loadedPart := 0 + for { + if contextCanceled(ctx) { + return entries, ctx.Err() + } + + if partN != loadedPart { + if retries > 10 { + err := o.checkMetacacheState(ctx, rpc) + if err != nil { + return entries, fmt.Errorf("waiting for next part %d: %w", partN, err) + } + retries = 1 + } + + if retries > 0 { + // Load from one disk only + for _, disk := range er.getDisks() { + if disk == nil { + continue + } + if !disk.IsOnline() { + continue + } + _, err := disk.ReadVersion(ctx, "", minioMetaBucket, + o.objectPath(partN), "", ReadOptions{}) + if err != nil { + time.Sleep(retryDelay250) + retries++ + continue + } + break + } + } + + // Load partN metadata... + fi, metaArr, onlineDisks, err = er.getObjectFileInfo(ctx, minioMetaBucket, o.objectPath(partN), ObjectOptions{}, true) + if err != nil { + time.Sleep(retryDelay250) + retries++ + continue + } + loadedPart = partN + bi, err := getMetacacheBlockInfo(fi, partN) + internalLogIf(ctx, err) + if err == nil { + if bi.pastPrefix(o.Prefix) { + return entries, io.EOF + } + } + } + + pr, pw := io.Pipe() + go func() { + werr := er.getObjectWithFileInfo(ctx, minioMetaBucket, o.objectPath(partN), 0, + fi.Size, pw, fi, metaArr, onlineDisks) + pw.CloseWithError(werr) + }() + + tmp := newMetacacheReader(pr) + e, err := tmp.filter(o) + pr.CloseWithError(err) + tmp.Close() + entries.o = append(entries.o, e.o...) + if o.Limit > 0 && entries.len() > o.Limit { + entries.truncate(o.Limit) + return entries, nil + } + if err == nil { + // We stopped within the listing, we are done for now... + return entries, nil + } + if err != nil && !errors.Is(err, io.EOF) { + switch toObjectErr(err, minioMetaBucket, o.objectPath(partN)).(type) { + case ObjectNotFound: + retries++ + time.Sleep(retryDelay250) + continue + case InsufficientReadQuorum: + retries++ + loadedPart = -1 + time.Sleep(retryDelay250) + continue + default: + internalLogIf(ctx, err) + return entries, err + } + } + + // We finished at the end of the block. + // And should not expect any more results. + bi, err := getMetacacheBlockInfo(fi, partN) + internalLogIf(ctx, err) + if err != nil || bi.EOS { + // We are done and there are no more parts. + return entries, io.EOF + } + if bi.endedPrefix(o.Prefix) { + // Nothing more for prefix. + return entries, io.EOF + } + partN++ + retries = 0 + } + } +} + +// getListQuorum interprets list quorum values and returns appropriate +// acceptable quorum expected for list operations +func getListQuorum(quorum string, driveCount int) int { + switch quorum { + case "disk": + return 1 + case "reduced": + return 2 + case "optimal": + return (driveCount + 1) / 2 + case "auto": + return -1 + } + // defaults to 'strict' + return driveCount +} + +func calcCommonWritesDeletes(infos []DiskInfo, readQuorum int) (commonWrite, commonDelete uint64) { + deletes := make([]uint64, len(infos)) + writes := make([]uint64, len(infos)) + for index, di := range infos { + deletes[index] = di.Metrics.TotalDeletes + writes[index] = di.Metrics.TotalWrites + } + + filter := func(list []uint64) (commonCount uint64) { + maxCnt := 0 + signatureMap := map[uint64]int{} + for _, v := range list { + signatureMap[v]++ + } + for ops, count := range signatureMap { + if maxCnt < count && commonCount < ops { + maxCnt = count + commonCount = ops + } + } + if maxCnt < readQuorum { + return 0 + } + return commonCount + } + + commonWrite = filter(writes) + commonDelete = filter(deletes) + return commonWrite, commonDelete +} + +func calcCommonCounter(infos []DiskInfo, readQuorum int) (commonCount uint64) { + filter := func() (commonCount uint64) { + maxCnt := 0 + signatureMap := map[uint64]int{} + for _, info := range infos { + if info.Error != "" { + continue + } + mutations := info.Metrics.TotalDeletes + info.Metrics.TotalWrites + signatureMap[mutations]++ + } + for ops, count := range signatureMap { + if maxCnt < count && commonCount < ops { + maxCnt = count + commonCount = ops + } + } + if maxCnt < readQuorum { + return 0 + } + return commonCount + } + + return filter() +} + +func getQuorumDiskInfos(disks []StorageAPI, infos []DiskInfo, readQuorum int) (newDisks []StorageAPI, newInfos []DiskInfo) { + commonMutations := calcCommonCounter(infos, readQuorum) + for i, info := range infos { + mutations := info.Metrics.TotalDeletes + info.Metrics.TotalWrites + if mutations >= commonMutations { + newDisks = append(newDisks, disks[i]) + newInfos = append(newInfos, infos[i]) + } + } + + return newDisks, newInfos +} + +func getQuorumDisks(disks []StorageAPI, infos []DiskInfo, readQuorum int) (newDisks []StorageAPI) { + newDisks, _ = getQuorumDiskInfos(disks, infos, readQuorum) + return newDisks +} + +// Will return io.EOF if continuing would not yield more results. +func (er *erasureObjects) listPath(ctx context.Context, o listPathOptions, results chan<- metaCacheEntry) (err error) { + defer xioutil.SafeClose(results) + o.debugf(color.Green("listPath:")+" with options: %#v", o) + + // get prioritized non-healing disks for listing + disks, infos, _ := er.getOnlineDisksWithHealingAndInfo(true) + askDisks := getListQuorum(o.AskDisks, er.setDriveCount) + if askDisks == -1 { + newDisks := getQuorumDisks(disks, infos, (len(disks)+1)/2) + if newDisks != nil { + // If we found disks signature in quorum, we proceed to list + // from a single drive, shuffling of the drives is subsequently. + disks = newDisks + askDisks = 1 + } else { + // If we did not find suitable disks, perform strict quorum listing + // as no disk agrees on quorum anymore. + askDisks = getListQuorum("strict", er.setDriveCount) + } + } + + var fallbackDisks []StorageAPI + + // Special case: ask all disks if the drive count is 4 + if er.setDriveCount == 4 || askDisks > len(disks) { + askDisks = len(disks) // use all available drives + } + + // However many we ask, versions must exist on ~50% + listingQuorum := (askDisks + 1) / 2 + + if askDisks > 0 && len(disks) > askDisks { + rand.Shuffle(len(disks), func(i, j int) { + disks[i], disks[j] = disks[j], disks[i] + }) + fallbackDisks = disks[askDisks:] + disks = disks[:askDisks] + } + + // How to resolve results. + resolver := metadataResolutionParams{ + dirQuorum: listingQuorum, + objQuorum: listingQuorum, + bucket: o.Bucket, + } + + // Maximum versions requested for "latest" object + // resolution on versioned buckets, this is to be only + // used when o.Versioned is false + if !o.Versioned { + resolver.requestedVersions = 1 + } + var limit int + if o.Limit > 0 && o.StopDiskAtLimit { + // Over-read by 4 + 1 for every 16 in limit to give some space for resolver, + // allow for truncating the list and know if we have more results. + limit = o.Limit + 4 + (o.Limit / 16) + } + ctxDone := ctx.Done() + return listPathRaw(ctx, listPathRawOptions{ + disks: disks, + fallbackDisks: fallbackDisks, + bucket: o.Bucket, + path: o.BaseDir, + recursive: o.Recursive, + filterPrefix: o.FilterPrefix, + minDisks: listingQuorum, + forwardTo: o.Marker, + perDiskLimit: limit, + agreed: func(entry metaCacheEntry) { + select { + case <-ctxDone: + case results <- entry: + } + }, + partial: func(entries metaCacheEntries, errs []error) { + // Results Disagree :-( + entry, ok := entries.resolve(&resolver) + if ok { + select { + case <-ctxDone: + case results <- *entry: + } + } + }, + }) +} + +//msgp:ignore metaCacheRPC +type metaCacheRPC struct { + o listPathOptions + mu sync.Mutex + meta *metacache + rpc *peerRESTClient + cancel context.CancelFunc +} + +func (m *metaCacheRPC) setErr(err string) { + m.mu.Lock() + defer m.mu.Unlock() + meta := *m.meta + if meta.status != scanStateError { + meta.error = err + meta.status = scanStateError + } else { + // An error is already set. + return + } + meta, _ = m.o.updateMetacacheListing(meta, m.rpc) + *m.meta = meta +} + +// getErr will return an error if the listing failed. +// The error is not type safe. +func (m *metaCacheRPC) getErr() error { + m.mu.Lock() + defer m.mu.Unlock() + if m.meta.status == scanStateError { + return errors.New(m.meta.error) + } + return nil +} + +func (er *erasureObjects) saveMetaCacheStream(ctx context.Context, mc *metaCacheRPC, entries <-chan metaCacheEntry) (err error) { + o := mc.o + o.debugf(color.Green("saveMetaCacheStream:")+" with options: %#v", o) + + metaMu := &mc.mu + rpc := mc.rpc + cancel := mc.cancel + defer func() { + o.debugln(color.Green("saveMetaCacheStream:")+"err:", err) + if err != nil && !errors.Is(err, io.EOF) { + go mc.setErr(err.Error()) + cancel() + } + }() + + defer cancel() + // Save continuous updates + go func() { + var err error + ticker := time.NewTicker(10 * time.Second) + defer ticker.Stop() + var exit bool + for !exit { + select { + case <-ticker.C: + case <-ctx.Done(): + exit = true + } + metaMu.Lock() + meta := *mc.meta + meta, err = o.updateMetacacheListing(meta, rpc) + if err == nil && time.Since(meta.lastHandout) > metacacheMaxClientWait { + cancel() + exit = true + meta.status = scanStateError + meta.error = fmt.Sprintf("listing canceled since time since last handout was %v ago", time.Since(meta.lastHandout).Round(time.Second)) + o.debugln(color.Green("saveMetaCacheStream: ") + meta.error) + meta, err = o.updateMetacacheListing(meta, rpc) + } + if err == nil { + *mc.meta = meta + if meta.status == scanStateError { + cancel() + exit = true + } + } + metaMu.Unlock() + } + }() + + const retryDelay = 200 * time.Millisecond + const maxTries = 5 + + // Keep destination... + // Write results to disk. + bw := newMetacacheBlockWriter(entries, func(b *metacacheBlock) error { + // if the block is 0 bytes and its a first block skip it. + // skip only this for Transient caches. + if len(b.data) == 0 && b.n == 0 && o.Transient { + return nil + } + o.debugln(color.Green("saveMetaCacheStream:")+" saving block", b.n, "to", o.objectPath(b.n)) + r, err := hash.NewReader(ctx, bytes.NewReader(b.data), int64(len(b.data)), "", "", int64(len(b.data))) + bugLogIf(ctx, err) + custom := b.headerKV() + _, err = er.putMetacacheObject(ctx, o.objectPath(b.n), NewPutObjReader(r), ObjectOptions{ + UserDefined: custom, + }) + if err != nil { + mc.setErr(err.Error()) + cancel() + return err + } + if b.n == 0 { + return nil + } + // Update block 0 metadata. + var retries int + for { + meta := b.headerKV() + fi := FileInfo{ + Metadata: make(map[string]string, len(meta)), + } + maps.Copy(fi.Metadata, meta) + err := er.updateObjectMetaWithOpts(ctx, minioMetaBucket, o.objectPath(0), fi, er.getDisks(), UpdateMetadataOpts{NoPersistence: true}) + if err == nil { + break + } + switch err.(type) { + case ObjectNotFound: + return err + case StorageErr: + return err + case InsufficientReadQuorum: + default: + internalLogIf(ctx, err) + } + if retries >= maxTries { + return err + } + retries++ + time.Sleep(retryDelay) + } + return nil + }) + + // Blocks while consuming entries or an error occurs. + err = bw.Close() + if err != nil { + mc.setErr(err.Error()) + } + metaMu.Lock() + defer metaMu.Unlock() + if mc.meta.error != "" { + return err + } + // Save success + mc.meta.status = scanStateSuccess + meta, err := o.updateMetacacheListing(*mc.meta, rpc) + if err == nil { + *mc.meta = meta + } + return nil +} + +//msgp:ignore listPathRawOptions +type listPathRawOptions struct { + disks []StorageAPI + fallbackDisks []StorageAPI + bucket, path string + recursive bool + + // Only return results with this prefix. + filterPrefix string + + // Forward to this prefix before returning results. + forwardTo string + + // Minimum number of good disks to continue. + // An error will be returned if this many disks returned an error. + minDisks int + reportNotFound bool + + // perDiskLimit will limit each disk to return n objects. + // If <= 0 all results will be returned until canceled. + perDiskLimit int + + // Callbacks with results: + // If set to nil, it will not be called. + + // agreed is called if all disks agreed. + agreed func(entry metaCacheEntry) + + // partial will be called when there is disagreement between disks. + // if disk did not return any result, but also haven't errored + // the entry will be empty and errs will + partial func(entries metaCacheEntries, errs []error) + + // finished will be called when all streams have finished and + // more than one disk returned an error. + // Will not be called if everything operates as expected. + finished func(errs []error) +} + +// listPathRaw will list a path on the provided drives. +// See listPathRawOptions on how results are delivered. +// Directories are always returned. +// Cache will be bypassed. +// Context cancellation will be respected but may take a while to effectuate. +func listPathRaw(ctx context.Context, opts listPathRawOptions) (err error) { + disks := opts.disks + if len(disks) == 0 { + return fmt.Errorf("listPathRaw: 0 drives provided") + } + + // Cancel upstream if we finish before we expect. + ctx, cancel := context.WithCancel(ctx) + defer cancel() + + // Keep track of fallback disks + var fdMu sync.Mutex + fds := opts.fallbackDisks + fallback := func(err error) StorageAPI { + if _, ok := err.(StorageErr); ok { + // Attempt to grab a fallback disk + fdMu.Lock() + defer fdMu.Unlock() + if len(fds) == 0 { + return nil + } + fdsCopy := fds + for _, fd := range fdsCopy { + // Grab a fallback disk + fds = fds[1:] + if fd != nil && fd.IsOnline() { + return fd + } + } + } + // Either no more disks for fallback or + // not a storage error. + return nil + } + readers := make([]*metacacheReader, len(disks)) + defer func() { + for _, r := range readers { + r.Close() + } + }() + for i := range disks { + r, w := io.Pipe() + // Make sure we close the pipe so blocked writes doesn't stay around. + defer r.CloseWithError(context.Canceled) + + readers[i] = newMetacacheReader(r) + d := disks[i] + + // Send request to each disk. + go func() { + var werr error + if d == nil { + werr = errDiskNotFound + } else { + werr = d.WalkDir(ctx, WalkDirOptions{ + Limit: opts.perDiskLimit, + Bucket: opts.bucket, + BaseDir: opts.path, + Recursive: opts.recursive, + ReportNotFound: opts.reportNotFound, + FilterPrefix: opts.filterPrefix, + ForwardTo: opts.forwardTo, + }, w) + } + + // fallback only when set. + for { + fd := fallback(werr) + if fd == nil { + break + } + // This fallback is only set when + // askDisks is less than total + // number of disks per set. + werr = fd.WalkDir(ctx, WalkDirOptions{ + Limit: opts.perDiskLimit, + Bucket: opts.bucket, + BaseDir: opts.path, + Recursive: opts.recursive, + ReportNotFound: opts.reportNotFound, + FilterPrefix: opts.filterPrefix, + ForwardTo: opts.forwardTo, + }, w) + if werr == nil { + break + } + } + w.CloseWithError(werr) + }() + } + + topEntries := make(metaCacheEntries, len(readers)) + errs := make([]error, len(readers)) + for { + // Get the top entry from each + var current metaCacheEntry + var atEOF, fnf, vnf, hasErr, agree int + for i := range topEntries { + topEntries[i] = metaCacheEntry{} + } + if contextCanceled(ctx) { + return ctx.Err() + } + for i, r := range readers { + if errs[i] != nil { + hasErr++ + continue + } + entry, err := r.peek() + switch err { + case io.EOF: + atEOF++ + continue + case nil: + default: + switch err.Error() { + case errFileNotFound.Error(): + atEOF++ + fnf++ + continue + case errVolumeNotFound.Error(): + atEOF++ + fnf++ + vnf++ + continue + } + hasErr++ + errs[i] = fmt.Errorf("drive: %s returned err: %v", disks[i], err) + continue + } + // If no current, add it. + if current.name == "" { + topEntries[i] = entry + current = entry + agree++ + continue + } + // If exact match, we agree. + if _, ok := current.matches(&entry, true); ok { + topEntries[i] = entry + agree++ + continue + } + // If only the name matches we didn't agree, but add it for resolution. + if entry.name == current.name { + topEntries[i] = entry + continue + } + // We got different entries + if entry.name > current.name { + continue + } + // We got a new, better current. + // Clear existing entries. + for i := range topEntries[:i] { + topEntries[i] = metaCacheEntry{} + } + agree = 1 + current = entry + topEntries[i] = entry + } + + // Since minDisks is set to quorum, we return if we have enough. + if vnf > 0 && vnf >= len(readers)-opts.minDisks { + return errVolumeNotFound + } + // Since minDisks is set to quorum, we return if we have enough. + if fnf > 0 && fnf >= len(readers)-opts.minDisks { + return errFileNotFound + } + + // Stop if we exceed number of bad disks. + if hasErr > 0 && hasErr+fnf > len(disks)-opts.minDisks { + if opts.finished != nil { + opts.finished(errs) + } + return errors.Join(errs...) + } + + // Break if all at EOF or error. + if atEOF+hasErr == len(readers) { + if hasErr > 0 && opts.finished != nil { + opts.finished(errs) + } + break + } + + if agree == len(readers) { + // Everybody agreed + for _, r := range readers { + r.skip(1) + } + if opts.agreed != nil { + opts.agreed(current) + } + continue + } + if opts.partial != nil { + opts.partial(topEntries, errs) + } + // Skip the inputs we used. + for i, r := range readers { + if topEntries[i].name != "" { + r.skip(1) + } + } + } + return nil +} diff --git a/cmd/metacache-set_gen.go b/cmd/metacache-set_gen.go new file mode 100644 index 00000000..8b29bfca --- /dev/null +++ b/cmd/metacache-set_gen.go @@ -0,0 +1,560 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *listPathOptions) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ID": + z.ID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + case "Bucket": + z.Bucket, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "BaseDir": + z.BaseDir, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "BaseDir") + return + } + case "Prefix": + z.Prefix, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + case "FilterPrefix": + z.FilterPrefix, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "FilterPrefix") + return + } + case "Marker": + z.Marker, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Marker") + return + } + case "Limit": + z.Limit, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Limit") + return + } + case "AskDisks": + z.AskDisks, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "AskDisks") + return + } + case "InclDeleted": + z.InclDeleted, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "InclDeleted") + return + } + case "Recursive": + z.Recursive, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Recursive") + return + } + case "Separator": + z.Separator, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Separator") + return + } + case "Create": + z.Create, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Create") + return + } + case "IncludeDirectories": + z.IncludeDirectories, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "IncludeDirectories") + return + } + case "Transient": + z.Transient, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Transient") + return + } + case "Versioned": + z.Versioned, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Versioned") + return + } + case "V1": + z.V1, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "V1") + return + } + case "StopDiskAtLimit": + z.StopDiskAtLimit, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "StopDiskAtLimit") + return + } + case "pool": + z.pool, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "pool") + return + } + case "set": + z.set, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "set") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *listPathOptions) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 19 + // write "ID" + err = en.Append(0xde, 0x0, 0x13, 0xa2, 0x49, 0x44) + if err != nil { + return + } + err = en.WriteString(z.ID) + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + // write "Bucket" + err = en.Append(0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Bucket) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + // write "BaseDir" + err = en.Append(0xa7, 0x42, 0x61, 0x73, 0x65, 0x44, 0x69, 0x72) + if err != nil { + return + } + err = en.WriteString(z.BaseDir) + if err != nil { + err = msgp.WrapError(err, "BaseDir") + return + } + // write "Prefix" + err = en.Append(0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78) + if err != nil { + return + } + err = en.WriteString(z.Prefix) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + // write "FilterPrefix" + err = en.Append(0xac, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78) + if err != nil { + return + } + err = en.WriteString(z.FilterPrefix) + if err != nil { + err = msgp.WrapError(err, "FilterPrefix") + return + } + // write "Marker" + err = en.Append(0xa6, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72) + if err != nil { + return + } + err = en.WriteString(z.Marker) + if err != nil { + err = msgp.WrapError(err, "Marker") + return + } + // write "Limit" + err = en.Append(0xa5, 0x4c, 0x69, 0x6d, 0x69, 0x74) + if err != nil { + return + } + err = en.WriteInt(z.Limit) + if err != nil { + err = msgp.WrapError(err, "Limit") + return + } + // write "AskDisks" + err = en.Append(0xa8, 0x41, 0x73, 0x6b, 0x44, 0x69, 0x73, 0x6b, 0x73) + if err != nil { + return + } + err = en.WriteString(z.AskDisks) + if err != nil { + err = msgp.WrapError(err, "AskDisks") + return + } + // write "InclDeleted" + err = en.Append(0xab, 0x49, 0x6e, 0x63, 0x6c, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteBool(z.InclDeleted) + if err != nil { + err = msgp.WrapError(err, "InclDeleted") + return + } + // write "Recursive" + err = en.Append(0xa9, 0x52, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65) + if err != nil { + return + } + err = en.WriteBool(z.Recursive) + if err != nil { + err = msgp.WrapError(err, "Recursive") + return + } + // write "Separator" + err = en.Append(0xa9, 0x53, 0x65, 0x70, 0x61, 0x72, 0x61, 0x74, 0x6f, 0x72) + if err != nil { + return + } + err = en.WriteString(z.Separator) + if err != nil { + err = msgp.WrapError(err, "Separator") + return + } + // write "Create" + err = en.Append(0xa6, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65) + if err != nil { + return + } + err = en.WriteBool(z.Create) + if err != nil { + err = msgp.WrapError(err, "Create") + return + } + // write "IncludeDirectories" + err = en.Append(0xb2, 0x49, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteBool(z.IncludeDirectories) + if err != nil { + err = msgp.WrapError(err, "IncludeDirectories") + return + } + // write "Transient" + err = en.Append(0xa9, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x69, 0x65, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteBool(z.Transient) + if err != nil { + err = msgp.WrapError(err, "Transient") + return + } + // write "Versioned" + err = en.Append(0xa9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteBool(z.Versioned) + if err != nil { + err = msgp.WrapError(err, "Versioned") + return + } + // write "V1" + err = en.Append(0xa2, 0x56, 0x31) + if err != nil { + return + } + err = en.WriteBool(z.V1) + if err != nil { + err = msgp.WrapError(err, "V1") + return + } + // write "StopDiskAtLimit" + err = en.Append(0xaf, 0x53, 0x74, 0x6f, 0x70, 0x44, 0x69, 0x73, 0x6b, 0x41, 0x74, 0x4c, 0x69, 0x6d, 0x69, 0x74) + if err != nil { + return + } + err = en.WriteBool(z.StopDiskAtLimit) + if err != nil { + err = msgp.WrapError(err, "StopDiskAtLimit") + return + } + // write "pool" + err = en.Append(0xa4, 0x70, 0x6f, 0x6f, 0x6c) + if err != nil { + return + } + err = en.WriteInt(z.pool) + if err != nil { + err = msgp.WrapError(err, "pool") + return + } + // write "set" + err = en.Append(0xa3, 0x73, 0x65, 0x74) + if err != nil { + return + } + err = en.WriteInt(z.set) + if err != nil { + err = msgp.WrapError(err, "set") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *listPathOptions) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 19 + // string "ID" + o = append(o, 0xde, 0x0, 0x13, 0xa2, 0x49, 0x44) + o = msgp.AppendString(o, z.ID) + // string "Bucket" + o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + o = msgp.AppendString(o, z.Bucket) + // string "BaseDir" + o = append(o, 0xa7, 0x42, 0x61, 0x73, 0x65, 0x44, 0x69, 0x72) + o = msgp.AppendString(o, z.BaseDir) + // string "Prefix" + o = append(o, 0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78) + o = msgp.AppendString(o, z.Prefix) + // string "FilterPrefix" + o = append(o, 0xac, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78) + o = msgp.AppendString(o, z.FilterPrefix) + // string "Marker" + o = append(o, 0xa6, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72) + o = msgp.AppendString(o, z.Marker) + // string "Limit" + o = append(o, 0xa5, 0x4c, 0x69, 0x6d, 0x69, 0x74) + o = msgp.AppendInt(o, z.Limit) + // string "AskDisks" + o = append(o, 0xa8, 0x41, 0x73, 0x6b, 0x44, 0x69, 0x73, 0x6b, 0x73) + o = msgp.AppendString(o, z.AskDisks) + // string "InclDeleted" + o = append(o, 0xab, 0x49, 0x6e, 0x63, 0x6c, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64) + o = msgp.AppendBool(o, z.InclDeleted) + // string "Recursive" + o = append(o, 0xa9, 0x52, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65) + o = msgp.AppendBool(o, z.Recursive) + // string "Separator" + o = append(o, 0xa9, 0x53, 0x65, 0x70, 0x61, 0x72, 0x61, 0x74, 0x6f, 0x72) + o = msgp.AppendString(o, z.Separator) + // string "Create" + o = append(o, 0xa6, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65) + o = msgp.AppendBool(o, z.Create) + // string "IncludeDirectories" + o = append(o, 0xb2, 0x49, 0x6e, 0x63, 0x6c, 0x75, 0x64, 0x65, 0x44, 0x69, 0x72, 0x65, 0x63, 0x74, 0x6f, 0x72, 0x69, 0x65, 0x73) + o = msgp.AppendBool(o, z.IncludeDirectories) + // string "Transient" + o = append(o, 0xa9, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x69, 0x65, 0x6e, 0x74) + o = msgp.AppendBool(o, z.Transient) + // string "Versioned" + o = append(o, 0xa9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x65, 0x64) + o = msgp.AppendBool(o, z.Versioned) + // string "V1" + o = append(o, 0xa2, 0x56, 0x31) + o = msgp.AppendBool(o, z.V1) + // string "StopDiskAtLimit" + o = append(o, 0xaf, 0x53, 0x74, 0x6f, 0x70, 0x44, 0x69, 0x73, 0x6b, 0x41, 0x74, 0x4c, 0x69, 0x6d, 0x69, 0x74) + o = msgp.AppendBool(o, z.StopDiskAtLimit) + // string "pool" + o = append(o, 0xa4, 0x70, 0x6f, 0x6f, 0x6c) + o = msgp.AppendInt(o, z.pool) + // string "set" + o = append(o, 0xa3, 0x73, 0x65, 0x74) + o = msgp.AppendInt(o, z.set) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *listPathOptions) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ID": + z.ID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + case "Bucket": + z.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "BaseDir": + z.BaseDir, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "BaseDir") + return + } + case "Prefix": + z.Prefix, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + case "FilterPrefix": + z.FilterPrefix, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "FilterPrefix") + return + } + case "Marker": + z.Marker, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Marker") + return + } + case "Limit": + z.Limit, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Limit") + return + } + case "AskDisks": + z.AskDisks, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AskDisks") + return + } + case "InclDeleted": + z.InclDeleted, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "InclDeleted") + return + } + case "Recursive": + z.Recursive, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Recursive") + return + } + case "Separator": + z.Separator, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Separator") + return + } + case "Create": + z.Create, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Create") + return + } + case "IncludeDirectories": + z.IncludeDirectories, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "IncludeDirectories") + return + } + case "Transient": + z.Transient, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Transient") + return + } + case "Versioned": + z.Versioned, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Versioned") + return + } + case "V1": + z.V1, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "V1") + return + } + case "StopDiskAtLimit": + z.StopDiskAtLimit, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "StopDiskAtLimit") + return + } + case "pool": + z.pool, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "pool") + return + } + case "set": + z.set, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "set") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *listPathOptions) Msgsize() (s int) { + s = 3 + 3 + msgp.StringPrefixSize + len(z.ID) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 8 + msgp.StringPrefixSize + len(z.BaseDir) + 7 + msgp.StringPrefixSize + len(z.Prefix) + 13 + msgp.StringPrefixSize + len(z.FilterPrefix) + 7 + msgp.StringPrefixSize + len(z.Marker) + 6 + msgp.IntSize + 9 + msgp.StringPrefixSize + len(z.AskDisks) + 12 + msgp.BoolSize + 10 + msgp.BoolSize + 10 + msgp.StringPrefixSize + len(z.Separator) + 7 + msgp.BoolSize + 19 + msgp.BoolSize + 10 + msgp.BoolSize + 10 + msgp.BoolSize + 3 + msgp.BoolSize + 16 + msgp.BoolSize + 5 + msgp.IntSize + 4 + msgp.IntSize + return +} diff --git a/cmd/metacache-set_gen_test.go b/cmd/metacache-set_gen_test.go new file mode 100644 index 00000000..bee9830b --- /dev/null +++ b/cmd/metacache-set_gen_test.go @@ -0,0 +1,123 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshallistPathOptions(t *testing.T) { + v := listPathOptions{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsglistPathOptions(b *testing.B) { + v := listPathOptions{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsglistPathOptions(b *testing.B) { + v := listPathOptions{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshallistPathOptions(b *testing.B) { + v := listPathOptions{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodelistPathOptions(t *testing.T) { + v := listPathOptions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodelistPathOptions Msgsize() is inaccurate") + } + + vn := listPathOptions{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodelistPathOptions(b *testing.B) { + v := listPathOptions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodelistPathOptions(b *testing.B) { + v := listPathOptions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/metacache-stream.go b/cmd/metacache-stream.go new file mode 100644 index 00000000..cf61895f --- /dev/null +++ b/cmd/metacache-stream.go @@ -0,0 +1,871 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + "sync" + + jsoniter "github.com/json-iterator/go" + "github.com/klauspost/compress/s2" + "github.com/minio/minio/internal/bpool" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/tinylib/msgp/msgp" + "github.com/valyala/bytebufferpool" +) + +// metadata stream format: +// +// The stream is s2 compressed. +// https://github.com/klauspost/compress/tree/master/s2#s2-compression +// This ensures integrity and reduces the size typically by at least 50%. +// +// All stream elements are msgpack encoded. +// +// 1 Integer, metacacheStreamVersion of the writer. +// This can be used for managing breaking changes. +// +// For each element: +// 1. Bool. If false at end of stream. +// 2. String. Name of object. Directories contains a trailing slash. +// 3. Binary. Blob of metadata. Length 0 on directories. +// ... Next element. +// +// Streams can be assumed to be sorted in ascending order. +// If the stream ends before a false boolean it can be assumed it was truncated. + +const metacacheStreamVersion = 2 + +// metacacheWriter provides a serializer of metacache objects. +type metacacheWriter struct { + streamErr error + mw *msgp.Writer + creator func() error + closer func() error + blockSize int + streamWg sync.WaitGroup + reuseBlocks bool +} + +// newMetacacheWriter will create a serializer that will write objects in given order to the output. +// Provide a block size that affects latency. If 0 a default of 128KiB will be used. +// Block size can be up to 4MiB. +func newMetacacheWriter(out io.Writer, blockSize int) *metacacheWriter { + if blockSize < 8<<10 { + blockSize = 128 << 10 + } + w := metacacheWriter{ + mw: nil, + blockSize: blockSize, + } + w.creator = func() error { + s2w := s2.NewWriter(out, s2.WriterBlockSize(blockSize), s2.WriterConcurrency(2)) + w.mw = msgp.NewWriter(s2w) + w.creator = nil + if err := w.mw.WriteByte(metacacheStreamVersion); err != nil { + return err + } + + w.closer = func() (err error) { + defer func() { + cerr := s2w.Close() + if err == nil && cerr != nil { + err = cerr + } + }() + if w.streamErr != nil { + return w.streamErr + } + if err = w.mw.WriteBool(false); err != nil { + return err + } + return w.mw.Flush() + } + return nil + } + return &w +} + +// write one or more objects to the stream in order. +// It is favorable to send as many objects as possible in a single write, +// but no more than math.MaxUint32 +func (w *metacacheWriter) write(objs ...metaCacheEntry) error { + if w == nil { + return errors.New("metacacheWriter: nil writer") + } + if len(objs) == 0 { + return nil + } + if w.creator != nil { + err := w.creator() + w.creator = nil + if err != nil { + return fmt.Errorf("metacacheWriter: unable to create writer: %w", err) + } + if w.mw == nil { + return errors.New("metacacheWriter: writer not initialized") + } + } + for _, o := range objs { + if len(o.name) == 0 { + return errors.New("metacacheWriter: no name provided") + } + // Indicate EOS + err := w.mw.WriteBool(true) + if err != nil { + return err + } + err = w.mw.WriteString(o.name) + if err != nil { + return err + } + err = w.mw.WriteBytes(o.metadata) + if err != nil { + return err + } + if w.reuseBlocks || o.reusable { + metaDataPoolPut(o.metadata) + } + } + + return nil +} + +// stream entries to the output. +// The returned channel should be closed when done. +// Any error is reported when closing the metacacheWriter. +func (w *metacacheWriter) stream() (chan<- metaCacheEntry, error) { + if w.creator != nil { + err := w.creator() + w.creator = nil + if err != nil { + return nil, fmt.Errorf("metacacheWriter: unable to create writer: %w", err) + } + if w.mw == nil { + return nil, errors.New("metacacheWriter: writer not initialized") + } + } + objs := make(chan metaCacheEntry, 100) + w.streamErr = nil + w.streamWg.Add(1) + go func() { + defer w.streamWg.Done() + for o := range objs { + if len(o.name) == 0 || w.streamErr != nil { + continue + } + // Indicate EOS + err := w.mw.WriteBool(true) + if err != nil { + w.streamErr = err + continue + } + err = w.mw.WriteString(o.name) + if err != nil { + w.streamErr = err + continue + } + err = w.mw.WriteBytes(o.metadata) + if w.reuseBlocks || o.reusable { + metaDataPoolPut(o.metadata) + } + if err != nil { + w.streamErr = err + continue + } + } + }() + + return objs, nil +} + +// Close and release resources. +func (w *metacacheWriter) Close() error { + if w == nil || w.closer == nil { + return nil + } + w.streamWg.Wait() + err := w.closer() + w.closer = nil + return err +} + +// Reset and start writing to new writer. +// Close must have been called before this. +func (w *metacacheWriter) Reset(out io.Writer) { + w.streamErr = nil + w.creator = func() error { + s2w := s2.NewWriter(out, s2.WriterBlockSize(w.blockSize), s2.WriterConcurrency(2)) + w.mw = msgp.NewWriter(s2w) + w.creator = nil + if err := w.mw.WriteByte(metacacheStreamVersion); err != nil { + return err + } + + w.closer = func() error { + if w.streamErr != nil { + return w.streamErr + } + if err := w.mw.WriteBool(false); err != nil { + return err + } + if err := w.mw.Flush(); err != nil { + return err + } + return s2w.Close() + } + return nil + } +} + +var s2DecPool = bpool.Pool[*s2.Reader]{New: func() *s2.Reader { + // Default alloc block for network transfer. + return s2.NewReader(nil, s2.ReaderAllocBlock(16<<10)) +}} + +// metacacheReader allows reading a cache stream. +type metacacheReader struct { + mr *msgp.Reader + current metaCacheEntry + err error // stateful error + closer func() + creator func() error +} + +// newMetacacheReader creates a new cache reader. +// Nothing will be read from the stream yet. +func newMetacacheReader(r io.Reader) *metacacheReader { + dec := s2DecPool.Get() + dec.Reset(r) + mr := msgpNewReader(dec) + return &metacacheReader{ + mr: mr, + closer: func() { + dec.Reset(nil) + s2DecPool.Put(dec) + readMsgpReaderPoolPut(mr) + }, + creator: func() error { + v, err := mr.ReadByte() + if err != nil { + return err + } + switch v { + case 1, 2: + default: + return fmt.Errorf("metacacheReader: Unknown version: %d", v) + } + return nil + }, + } +} + +func (r *metacacheReader) checkInit() { + if r.creator == nil || r.err != nil { + return + } + r.err = r.creator() + r.creator = nil +} + +// peek will return the name of the next object. +// Will return io.EOF if there are no more objects. +// Should be used sparingly. +func (r *metacacheReader) peek() (metaCacheEntry, error) { + r.checkInit() + if r.err != nil { + return metaCacheEntry{}, r.err + } + if r.current.name != "" { + return r.current, nil + } + if more, err := r.mr.ReadBool(); !more { + switch err { + case nil: + r.err = io.EOF + return metaCacheEntry{}, io.EOF + case io.EOF: + r.err = io.ErrUnexpectedEOF + return metaCacheEntry{}, io.ErrUnexpectedEOF + } + r.err = err + return metaCacheEntry{}, err + } + + var err error + if r.current.name, err = r.mr.ReadString(); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + r.err = err + return metaCacheEntry{}, err + } + r.current.metadata, err = r.mr.ReadBytes(r.current.metadata[:0]) + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + r.err = err + return r.current, err +} + +// next will read one entry from the stream. +// Generally not recommended for fast operation. +func (r *metacacheReader) next() (metaCacheEntry, error) { + r.checkInit() + if r.err != nil { + return metaCacheEntry{}, r.err + } + var m metaCacheEntry + var err error + if r.current.name != "" { + m.name = r.current.name + m.metadata = r.current.metadata + r.current.name = "" + r.current.metadata = nil + return m, nil + } + if more, err := r.mr.ReadBool(); !more { + switch err { + case nil: + r.err = io.EOF + return m, io.EOF + case io.EOF: + r.err = io.ErrUnexpectedEOF + return m, io.ErrUnexpectedEOF + } + r.err = err + return m, err + } + if m.name, err = r.mr.ReadString(); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + r.err = err + return m, err + } + m.metadata, err = r.mr.ReadBytes(metaDataPoolGet()) + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + if len(m.metadata) == 0 && cap(m.metadata) >= metaDataReadDefault { + metaDataPoolPut(m.metadata) + m.metadata = nil + } + r.err = err + return m, err +} + +// next will read one entry from the stream. +// Generally not recommended for fast operation. +func (r *metacacheReader) nextEOF() bool { + r.checkInit() + if r.err != nil { + return r.err == io.EOF + } + if r.current.name != "" { + return false + } + _, err := r.peek() + if err != nil { + r.err = err + return r.err == io.EOF + } + return false +} + +// forwardTo will forward to the first entry that is >= s. +// Will return io.EOF if end of stream is reached without finding any. +func (r *metacacheReader) forwardTo(s string) error { + r.checkInit() + if r.err != nil { + return r.err + } + + if s == "" { + return nil + } + if r.current.name != "" { + if r.current.name >= s { + return nil + } + r.current.name = "" + r.current.metadata = nil + } + // temporary name buffer. + tmp := make([]byte, 0, 256) + for { + if more, err := r.mr.ReadBool(); !more { + switch err { + case nil: + r.err = io.EOF + return io.EOF + case io.EOF: + r.err = io.ErrUnexpectedEOF + return io.ErrUnexpectedEOF + } + r.err = err + return err + } + // Read name without allocating more than 1 buffer. + sz, err := r.mr.ReadStringHeader() + if err != nil { + r.err = err + return err + } + if cap(tmp) < int(sz) { + tmp = make([]byte, 0, sz+256) + } + tmp = tmp[:sz] + _, err = r.mr.R.ReadFull(tmp) + if err != nil { + r.err = err + return err + } + if string(tmp) >= s { + r.current.name = string(tmp) + r.current.metadata, r.err = r.mr.ReadBytes(nil) + return r.err + } + // Skip metadata + err = r.mr.Skip() + if err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + r.err = err + return err + } + } +} + +// readN will return all the requested number of entries in order +// or all if n < 0. +// Will return io.EOF if end of stream is reached. +// If requesting 0 objects nil error will always be returned regardless of at end of stream. +// Use peek to determine if at end of stream. +func (r *metacacheReader) readN(n int, inclDeleted, inclDirs, inclVersions bool, prefix string) (metaCacheEntriesSorted, error) { + r.checkInit() + if n == 0 { + return metaCacheEntriesSorted{}, nil + } + if r.err != nil { + return metaCacheEntriesSorted{}, r.err + } + + var res metaCacheEntries + if n > 0 { + res = make(metaCacheEntries, 0, n) + } + if prefix != "" { + if err := r.forwardTo(prefix); err != nil { + return metaCacheEntriesSorted{}, err + } + } + next, err := r.peek() + if err != nil { + return metaCacheEntriesSorted{}, err + } + if !next.hasPrefix(prefix) { + return metaCacheEntriesSorted{}, io.EOF + } + + if r.current.name != "" { + if (inclDeleted || !r.current.isLatestDeletemarker()) && r.current.hasPrefix(prefix) && (inclDirs || r.current.isObject()) { + res = append(res, r.current) + } + r.current.name = "" + r.current.metadata = nil + } + + for n < 0 || len(res) < n { + if more, err := r.mr.ReadBool(); !more { + switch err { + case nil: + r.err = io.EOF + return metaCacheEntriesSorted{o: res}, io.EOF + case io.EOF: + r.err = io.ErrUnexpectedEOF + return metaCacheEntriesSorted{o: res}, io.ErrUnexpectedEOF + } + r.err = err + return metaCacheEntriesSorted{o: res}, err + } + var err error + var meta metaCacheEntry + if meta.name, err = r.mr.ReadString(); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + r.err = err + return metaCacheEntriesSorted{o: res}, err + } + if !meta.hasPrefix(prefix) { + r.mr.R.Skip(1) + return metaCacheEntriesSorted{o: res}, io.EOF + } + if meta.metadata, err = r.mr.ReadBytes(metaDataPoolGet()); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + r.err = err + return metaCacheEntriesSorted{o: res}, err + } + if len(meta.metadata) == 0 { + metaDataPoolPut(meta.metadata) + meta.metadata = nil + } + if !inclDirs && (meta.isDir() || (!inclVersions && meta.isObjectDir() && meta.isLatestDeletemarker())) { + continue + } + if !inclDeleted && meta.isLatestDeletemarker() && meta.isObject() && !meta.isObjectDir() { + continue + } + res = append(res, meta) + } + return metaCacheEntriesSorted{o: res}, nil +} + +// readAll will return all remaining objects on the dst channel and close it when done. +// The context allows the operation to be canceled. +func (r *metacacheReader) readAll(ctx context.Context, dst chan<- metaCacheEntry) error { + r.checkInit() + if r.err != nil { + return r.err + } + defer xioutil.SafeClose(dst) + if r.current.name != "" { + select { + case <-ctx.Done(): + r.err = ctx.Err() + return ctx.Err() + case dst <- r.current: + } + r.current.name = "" + r.current.metadata = nil + } + for { + if more, err := r.mr.ReadBool(); !more { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + r.err = err + return err + } + + var err error + var meta metaCacheEntry + if meta.name, err = r.mr.ReadString(); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + r.err = err + return err + } + if meta.metadata, err = r.mr.ReadBytes(metaDataPoolGet()); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + r.err = err + return err + } + if len(meta.metadata) == 0 { + metaDataPoolPut(meta.metadata) + meta.metadata = nil + } + select { + case <-ctx.Done(): + r.err = ctx.Err() + return ctx.Err() + case dst <- meta: + } + } +} + +// readFn will return all remaining objects +// and provide a callback for each entry read in order +// as long as true is returned on the callback. +func (r *metacacheReader) readFn(fn func(entry metaCacheEntry) bool) error { + r.checkInit() + if r.err != nil { + return r.err + } + if r.current.name != "" { + fn(r.current) + r.current.name = "" + r.current.metadata = nil + } + for { + if more, err := r.mr.ReadBool(); !more { + switch err { + case io.EOF: + r.err = io.ErrUnexpectedEOF + return io.ErrUnexpectedEOF + case nil: + r.err = io.EOF + return io.EOF + } + return err + } + + var err error + var meta metaCacheEntry + if meta.name, err = r.mr.ReadString(); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + r.err = err + return err + } + if meta.metadata, err = r.mr.ReadBytes(nil); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + r.err = err + return err + } + // Send it! + if !fn(meta) { + return nil + } + } +} + +// readNames will return all the requested number of names in order +// or all if n < 0. +// Will return io.EOF if end of stream is reached. +func (r *metacacheReader) readNames(n int) ([]string, error) { + r.checkInit() + if r.err != nil { + return nil, r.err + } + if n == 0 { + return nil, nil + } + var res []string + if n > 0 { + res = make([]string, 0, n) + } + if r.current.name != "" { + res = append(res, r.current.name) + r.current.name = "" + r.current.metadata = nil + } + for n < 0 || len(res) < n { + if more, err := r.mr.ReadBool(); !more { + switch err { + case nil: + r.err = io.EOF + return res, io.EOF + case io.EOF: + r.err = io.ErrUnexpectedEOF + return res, io.ErrUnexpectedEOF + } + return res, err + } + + var err error + var name string + if name, err = r.mr.ReadString(); err != nil { + r.err = err + return res, err + } + if err = r.mr.Skip(); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + r.err = err + return res, err + } + res = append(res, name) + } + return res, nil +} + +// skip n entries on the input stream. +// If there are less entries left io.EOF is returned. +func (r *metacacheReader) skip(n int) error { + r.checkInit() + if r.err != nil { + return r.err + } + if n <= 0 { + return nil + } + if r.current.name != "" { + n-- + r.current.name = "" + r.current.metadata = nil + } + for n > 0 { + if more, err := r.mr.ReadBool(); !more { + switch err { + case nil: + r.err = io.EOF + return io.EOF + case io.EOF: + r.err = io.ErrUnexpectedEOF + return io.ErrUnexpectedEOF + } + return err + } + + if err := r.mr.Skip(); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + r.err = err + return err + } + if err := r.mr.Skip(); err != nil { + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + r.err = err + return err + } + n-- + } + return nil +} + +// Close and release resources. +func (r *metacacheReader) Close() error { + if r == nil || r.closer == nil { + return nil + } + r.closer() + r.closer = nil + r.creator = nil + return nil +} + +// metacacheBlockWriter collects blocks and provides a callback to store them. +type metacacheBlockWriter struct { + wg sync.WaitGroup + streamErr error + blockEntries int +} + +// newMetacacheBlockWriter provides a streaming block writer. +// Each block is the size of the capacity of the input channel. +// The caller should close to indicate the stream has ended. +func newMetacacheBlockWriter(in <-chan metaCacheEntry, nextBlock func(b *metacacheBlock) error) *metacacheBlockWriter { + w := metacacheBlockWriter{blockEntries: cap(in)} + w.wg.Add(1) + go func() { + defer w.wg.Done() + var current metacacheBlock + var n int + + buf := bytebufferpool.Get() + defer func() { + buf.Reset() + bytebufferpool.Put(buf) + }() + + block := newMetacacheWriter(buf, 1<<20) + defer block.Close() + finishBlock := func() { + if err := block.Close(); err != nil { + w.streamErr = err + return + } + current.data = buf.Bytes() + w.streamErr = nextBlock(¤t) + // Prepare for next + current.n++ + buf.Reset() + block.Reset(buf) + current.First = "" + } + for o := range in { + if len(o.name) == 0 || w.streamErr != nil { + continue + } + if current.First == "" { + current.First = o.name + } + + if n >= w.blockEntries-1 { + finishBlock() + n = 0 + } + n++ + + w.streamErr = block.write(o) + if w.streamErr != nil { + continue + } + current.Last = o.name + } + if n > 0 || current.n == 0 { + current.EOS = true + finishBlock() + } + }() + return &w +} + +// Close the stream. +// The incoming channel must be closed before calling this. +// Returns the first error the occurred during the writing if any. +func (w *metacacheBlockWriter) Close() error { + w.wg.Wait() + return w.streamErr +} + +type metacacheBlock struct { + data []byte + n int + First string `json:"f"` + Last string `json:"l"` + EOS bool `json:"eos,omitempty"` +} + +func (b metacacheBlock) headerKV() map[string]string { + json := jsoniter.ConfigCompatibleWithStandardLibrary + v, err := json.Marshal(b) + if err != nil { + bugLogIf(context.Background(), err) // Unlikely + return nil + } + return map[string]string{fmt.Sprintf("%s-metacache-part-%d", ReservedMetadataPrefixLower, b.n): string(v)} +} + +// pastPrefix returns true if the given prefix is before start of the block. +func (b metacacheBlock) pastPrefix(prefix string) bool { + if prefix == "" || strings.HasPrefix(b.First, prefix) { + return false + } + // We have checked if prefix matches, so we can do direct compare. + return b.First > prefix +} + +// endedPrefix returns true if the given prefix ends within the block. +func (b metacacheBlock) endedPrefix(prefix string) bool { + if prefix == "" || strings.HasPrefix(b.Last, prefix) { + return false + } + + // We have checked if prefix matches, so we can do direct compare. + return b.Last > prefix +} diff --git a/cmd/metacache-stream_test.go b/cmd/metacache-stream_test.go new file mode 100644 index 00000000..36286f68 --- /dev/null +++ b/cmd/metacache-stream_test.go @@ -0,0 +1,427 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "io" + "os" + "reflect" + "sync" + "testing" +) + +var loadMetacacheSampleNames = []string{"src/compress/bzip2/", "src/compress/bzip2/bit_reader.go", "src/compress/bzip2/bzip2.go", "src/compress/bzip2/bzip2_test.go", "src/compress/bzip2/huffman.go", "src/compress/bzip2/move_to_front.go", "src/compress/bzip2/testdata/", "src/compress/bzip2/testdata/Isaac.Newton-Opticks.txt.bz2", "src/compress/bzip2/testdata/e.txt.bz2", "src/compress/bzip2/testdata/fail-issue5747.bz2", "src/compress/bzip2/testdata/pass-random1.bin", "src/compress/bzip2/testdata/pass-random1.bz2", "src/compress/bzip2/testdata/pass-random2.bin", "src/compress/bzip2/testdata/pass-random2.bz2", "src/compress/bzip2/testdata/pass-sawtooth.bz2", "src/compress/bzip2/testdata/random.data.bz2", "src/compress/flate/", "src/compress/flate/deflate.go", "src/compress/flate/deflate_test.go", "src/compress/flate/deflatefast.go", "src/compress/flate/dict_decoder.go", "src/compress/flate/dict_decoder_test.go", "src/compress/flate/example_test.go", "src/compress/flate/flate_test.go", "src/compress/flate/huffman_bit_writer.go", "src/compress/flate/huffman_bit_writer_test.go", "src/compress/flate/huffman_code.go", "src/compress/flate/inflate.go", "src/compress/flate/inflate_test.go", "src/compress/flate/reader_test.go", "src/compress/flate/testdata/", "src/compress/flate/testdata/huffman-null-max.dyn.expect", "src/compress/flate/testdata/huffman-null-max.dyn.expect-noinput", "src/compress/flate/testdata/huffman-null-max.golden", "src/compress/flate/testdata/huffman-null-max.in", "src/compress/flate/testdata/huffman-null-max.wb.expect", "src/compress/flate/testdata/huffman-null-max.wb.expect-noinput", "src/compress/flate/testdata/huffman-pi.dyn.expect", "src/compress/flate/testdata/huffman-pi.dyn.expect-noinput", "src/compress/flate/testdata/huffman-pi.golden", "src/compress/flate/testdata/huffman-pi.in", "src/compress/flate/testdata/huffman-pi.wb.expect", "src/compress/flate/testdata/huffman-pi.wb.expect-noinput", "src/compress/flate/testdata/huffman-rand-1k.dyn.expect", "src/compress/flate/testdata/huffman-rand-1k.dyn.expect-noinput", "src/compress/flate/testdata/huffman-rand-1k.golden", "src/compress/flate/testdata/huffman-rand-1k.in", "src/compress/flate/testdata/huffman-rand-1k.wb.expect", "src/compress/flate/testdata/huffman-rand-1k.wb.expect-noinput", "src/compress/flate/testdata/huffman-rand-limit.dyn.expect", "src/compress/flate/testdata/huffman-rand-limit.dyn.expect-noinput", "src/compress/flate/testdata/huffman-rand-limit.golden", "src/compress/flate/testdata/huffman-rand-limit.in", "src/compress/flate/testdata/huffman-rand-limit.wb.expect", "src/compress/flate/testdata/huffman-rand-limit.wb.expect-noinput", "src/compress/flate/testdata/huffman-rand-max.golden", "src/compress/flate/testdata/huffman-rand-max.in", "src/compress/flate/testdata/huffman-shifts.dyn.expect", "src/compress/flate/testdata/huffman-shifts.dyn.expect-noinput", "src/compress/flate/testdata/huffman-shifts.golden", "src/compress/flate/testdata/huffman-shifts.in", "src/compress/flate/testdata/huffman-shifts.wb.expect", "src/compress/flate/testdata/huffman-shifts.wb.expect-noinput", "src/compress/flate/testdata/huffman-text-shift.dyn.expect", "src/compress/flate/testdata/huffman-text-shift.dyn.expect-noinput", "src/compress/flate/testdata/huffman-text-shift.golden", "src/compress/flate/testdata/huffman-text-shift.in", "src/compress/flate/testdata/huffman-text-shift.wb.expect", "src/compress/flate/testdata/huffman-text-shift.wb.expect-noinput", "src/compress/flate/testdata/huffman-text.dyn.expect", "src/compress/flate/testdata/huffman-text.dyn.expect-noinput", "src/compress/flate/testdata/huffman-text.golden", "src/compress/flate/testdata/huffman-text.in", "src/compress/flate/testdata/huffman-text.wb.expect", "src/compress/flate/testdata/huffman-text.wb.expect-noinput", "src/compress/flate/testdata/huffman-zero.dyn.expect", "src/compress/flate/testdata/huffman-zero.dyn.expect-noinput", "src/compress/flate/testdata/huffman-zero.golden", "src/compress/flate/testdata/huffman-zero.in", "src/compress/flate/testdata/huffman-zero.wb.expect", "src/compress/flate/testdata/huffman-zero.wb.expect-noinput", "src/compress/flate/testdata/null-long-match.dyn.expect-noinput", "src/compress/flate/testdata/null-long-match.wb.expect-noinput", "src/compress/flate/token.go", "src/compress/flate/writer_test.go", "src/compress/gzip/", "src/compress/gzip/example_test.go", "src/compress/gzip/gunzip.go", "src/compress/gzip/gunzip_test.go", "src/compress/gzip/gzip.go", "src/compress/gzip/gzip_test.go", "src/compress/gzip/issue14937_test.go", "src/compress/gzip/testdata/", "src/compress/gzip/testdata/issue6550.gz.base64", "src/compress/lzw/", "src/compress/lzw/reader.go", "src/compress/lzw/reader_test.go", "src/compress/lzw/writer.go", "src/compress/lzw/writer_test.go", "src/compress/testdata/", "src/compress/testdata/e.txt", "src/compress/testdata/gettysburg.txt", "src/compress/testdata/pi.txt", "src/compress/zlib/", "src/compress/zlib/example_test.go", "src/compress/zlib/reader.go", "src/compress/zlib/reader_test.go", "src/compress/zlib/writer.go", "src/compress/zlib/writer_test.go"} + +func loadMetacacheSample(t testing.TB) *metacacheReader { + b, err := os.ReadFile("testdata/metacache.s2") + if err != nil { + t.Fatal(err) + } + return newMetacacheReader(bytes.NewReader(b)) +} + +func loadMetacacheSampleEntries(t testing.TB) metaCacheEntriesSorted { + r := loadMetacacheSample(t) + defer r.Close() + entries, err := r.readN(-1, false, true, false, "") + if err != io.EOF { + t.Fatal(err) + } + + return entries +} + +func Test_metacacheReader_readNames(t *testing.T) { + r := loadMetacacheSample(t) + defer r.Close() + names, err := r.readNames(-1) + if err != io.EOF { + t.Fatal(err) + } + want := loadMetacacheSampleNames + if !reflect.DeepEqual(names, want) { + t.Errorf("got unexpected result: %#v", names) + } +} + +func Test_metacacheReader_readN(t *testing.T) { + r := loadMetacacheSample(t) + defer r.Close() + entries, err := r.readN(-1, false, true, false, "") + if err != io.EOF { + t.Fatal(err, entries.len()) + } + want := loadMetacacheSampleNames + for i, entry := range entries.entries() { + if entry.name != want[i] { + t.Errorf("entry %d, want %q, got %q", i, want[i], entry.name) + } + } + if entries.len() != len(want) { + t.Fatal("unexpected length:", entries.len(), "want:", len(want)) + } + + want = want[:0] + entries, err = r.readN(0, false, true, false, "") + if err != nil { + t.Fatal(err, entries.len()) + } + if entries.len() != len(want) { + t.Fatal("unexpected length:", entries.len(), "want:", len(want)) + } + + // Reload. + r = loadMetacacheSample(t) + defer r.Close() + entries, err = r.readN(0, false, true, false, "") + if err != nil { + t.Fatal(err, entries.len()) + } + if entries.len() != len(want) { + t.Fatal("unexpected length:", entries.len(), "want:", len(want)) + } + + entries, err = r.readN(5, false, true, false, "") + if err != nil { + t.Fatal(err, entries.len()) + } + want = loadMetacacheSampleNames[:5] + if entries.len() != len(want) { + t.Fatal("unexpected length:", entries.len(), "want:", len(want)) + } + + for i, entry := range entries.entries() { + if entry.name != want[i] { + t.Errorf("entry %d, want %q, got %q", i, want[i], entry.name) + } + } +} + +func Test_metacacheReader_readNDirs(t *testing.T) { + r := loadMetacacheSample(t) + defer r.Close() + entries, err := r.readN(-1, false, true, false, "") + if err != io.EOF { + t.Fatal(err, entries.len()) + } + want := loadMetacacheSampleNames + var noDirs []string + for i, entry := range entries.entries() { + if entry.name != want[i] { + t.Errorf("entry %d, want %q, got %q", i, want[i], entry.name) + } + if !entry.isDir() { + noDirs = append(noDirs, entry.name) + } + } + if entries.len() != len(want) { + t.Fatal("unexpected length:", entries.len(), "want:", len(want)) + } + + want = noDirs + r = loadMetacacheSample(t) + defer r.Close() + entries, err = r.readN(-1, false, false, false, "") + if err != io.EOF { + t.Fatal(err, entries.len()) + } + for i, entry := range entries.entries() { + if entry.name != want[i] { + t.Errorf("entry %d, want %q, got %q", i, want[i], entry.name) + } + } + if entries.len() != len(want) { + t.Fatal("unexpected length:", entries.len(), "want:", len(want)) + } + + want = want[:0] + entries, err = r.readN(0, false, false, false, "") + if err != nil { + t.Fatal(err, entries.len()) + } + if entries.len() != len(want) { + t.Fatal("unexpected length:", entries.len(), "want:", len(want)) + } + + // Reload. + r = loadMetacacheSample(t) + defer r.Close() + entries, err = r.readN(0, false, false, false, "") + if err != nil { + t.Fatal(err, entries.len()) + } + if entries.len() != len(want) { + t.Fatal("unexpected length:", entries.len(), "want:", len(want)) + } + + entries, err = r.readN(5, false, false, false, "") + if err != nil { + t.Fatal(err, entries.len()) + } + want = noDirs[:5] + if entries.len() != len(want) { + t.Fatal("unexpected length:", entries.len(), "want:", len(want)) + } + + for i, entry := range entries.entries() { + if entry.name != want[i] { + t.Errorf("entry %d, want %q, got %q", i, want[i], entry.name) + } + } +} + +func Test_metacacheReader_readNPrefix(t *testing.T) { + r := loadMetacacheSample(t) + defer r.Close() + entries, err := r.readN(-1, false, true, false, "src/compress/bzip2/") + if err != io.EOF { + t.Fatal(err, entries.len()) + } + want := loadMetacacheSampleNames[:16] + for i, entry := range entries.entries() { + if entry.name != want[i] { + t.Errorf("entry %d, want %q, got %q", i, want[i], entry.name) + } + } + if entries.len() != len(want) { + t.Fatal("unexpected length:", entries.len(), "want:", len(want)) + } + + r = loadMetacacheSample(t) + defer r.Close() + entries, err = r.readN(-1, false, true, false, "src/nonexist") + if err != io.EOF { + t.Fatal(err, entries.len()) + } + want = loadMetacacheSampleNames[:0] + if entries.len() != len(want) { + t.Fatal("unexpected length:", entries.len(), "want:", len(want)) + } + for i, entry := range entries.entries() { + if entry.name != want[i] { + t.Errorf("entry %d, want %q, got %q", i, want[i], entry.name) + } + } + + r = loadMetacacheSample(t) + defer r.Close() + entries, err = r.readN(-1, false, true, false, "src/a") + if err != io.EOF { + t.Fatal(err, entries.len()) + } + want = loadMetacacheSampleNames[:0] + if entries.len() != len(want) { + t.Fatal("unexpected length:", entries.len(), "want:", len(want)) + } + for i, entry := range entries.entries() { + if entry.name != want[i] { + t.Errorf("entry %d, want %q, got %q", i, want[i], entry.name) + } + } + + r = loadMetacacheSample(t) + defer r.Close() + entries, err = r.readN(-1, false, true, false, "src/compress/zlib/e") + if err != io.EOF { + t.Fatal(err, entries.len()) + } + want = []string{"src/compress/zlib/example_test.go"} + if entries.len() != len(want) { + t.Fatal("unexpected length:", entries.len(), "want:", len(want)) + } + for i, entry := range entries.entries() { + if entry.name != want[i] { + t.Errorf("entry %d, want %q, got %q", i, want[i], entry.name) + } + } +} + +func Test_metacacheReader_readFn(t *testing.T) { + r := loadMetacacheSample(t) + defer r.Close() + i := 0 + err := r.readFn(func(entry metaCacheEntry) bool { + want := loadMetacacheSampleNames[i] + if entry.name != want { + t.Errorf("entry %d, want %q, got %q", i, want, entry.name) + } + i++ + return true + }) + if err != io.EOF { + t.Fatal(err) + } +} + +func Test_metacacheReader_readAll(t *testing.T) { + r := loadMetacacheSample(t) + defer r.Close() + var readErr error + objs := make(chan metaCacheEntry, 1) + var wg sync.WaitGroup + wg.Add(1) + go func() { + readErr = r.readAll(t.Context(), objs) + wg.Done() + }() + want := loadMetacacheSampleNames + i := 0 + for entry := range objs { + if entry.name != want[i] { + t.Errorf("entry %d, want %q, got %q", i, want[i], entry.name) + } + i++ + } + wg.Wait() + if readErr != nil { + t.Fatal(readErr) + } +} + +func Test_metacacheReader_forwardTo(t *testing.T) { + r := loadMetacacheSample(t) + defer r.Close() + err := r.forwardTo("src/compress/zlib/reader_test.go") + if err != nil { + t.Fatal(err) + } + names, err := r.readNames(-1) + if err != io.EOF { + t.Fatal(err) + } + want := []string{"src/compress/zlib/reader_test.go", "src/compress/zlib/writer.go", "src/compress/zlib/writer_test.go"} + if !reflect.DeepEqual(names, want) { + t.Errorf("got unexpected result: %#v", names) + } + + // Try with prefix + r = loadMetacacheSample(t) + err = r.forwardTo("src/compress/zlib/reader_t") + if err != nil { + t.Fatal(err) + } + names, err = r.readNames(-1) + if err != io.EOF { + t.Fatal(err) + } + if !reflect.DeepEqual(names, want) { + t.Errorf("got unexpected result: %#v", names) + } +} + +func Test_metacacheReader_next(t *testing.T) { + r := loadMetacacheSample(t) + defer r.Close() + for i, want := range loadMetacacheSampleNames { + gotObj, err := r.next() + if err != nil { + t.Fatal(err) + } + if gotObj.name != want { + t.Errorf("entry %d, want %q, got %q", i, want, gotObj.name) + } + } +} + +func Test_metacacheReader_peek(t *testing.T) { + r := loadMetacacheSample(t) + defer r.Close() + for i, want := range loadMetacacheSampleNames { + got, err := r.peek() + if err == io.EOF { + break + } + if err != nil { + t.Fatal(err) + } + if got.name != want { + t.Errorf("entry %d, want %q, got %q", i, want, got.name) + } + gotObj, err := r.next() + if err != nil { + t.Fatal(err) + } + if gotObj.name != want { + t.Errorf("entry %d, want %q, got %q", i, want, gotObj.name) + } + } +} + +func Test_newMetacacheStream(t *testing.T) { + r := loadMetacacheSample(t) + var buf bytes.Buffer + w := newMetacacheWriter(&buf, 1<<20) + defer w.Close() + err := r.readFn(func(object metaCacheEntry) bool { + err := w.write(object) + if err != nil { + t.Fatal(err) + } + return true + }) + r.Close() + if err != io.EOF { + t.Fatal(err) + } + err = w.Close() + if err != nil { + t.Fatal(err) + } + + r = newMetacacheReader(&buf) + defer r.Close() + names, err := r.readNames(-1) + if err != io.EOF { + t.Fatal(err) + } + want := loadMetacacheSampleNames + if !reflect.DeepEqual(names, want) { + t.Errorf("got unexpected result: %#v", names) + } +} + +func Test_metacacheReader_skip(t *testing.T) { + r := loadMetacacheSample(t) + defer r.Close() + names, err := r.readNames(5) + if err != nil { + t.Fatal(err) + } + want := loadMetacacheSampleNames[:5] + if !reflect.DeepEqual(names, want) { + t.Errorf("got unexpected result: %#v", names) + } + err = r.skip(5) + if err != nil { + t.Fatal(err) + } + names, err = r.readNames(5) + if err != nil { + t.Fatal(err) + } + want = loadMetacacheSampleNames[10:15] + if !reflect.DeepEqual(names, want) { + t.Errorf("got unexpected result: %#v", names) + } + + err = r.skip(len(loadMetacacheSampleNames)) + if err != io.EOF { + t.Fatal(err) + } +} diff --git a/cmd/metacache-walk.go b/cmd/metacache-walk.go new file mode 100644 index 00000000..fa949d9a --- /dev/null +++ b/cmd/metacache-walk.go @@ -0,0 +1,457 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "io" + "sort" + "strings" + + "github.com/minio/minio/internal/grid" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/valyala/bytebufferpool" +) + +//go:generate msgp -file $GOFILE + +// WalkDirOptions provides options for WalkDir operations. +type WalkDirOptions struct { + // Bucket to scanner + Bucket string + + // Directory inside the bucket. + BaseDir string + + // Do a full recursive scan. + Recursive bool + + // ReportNotFound will return errFileNotFound if all disks reports the BaseDir cannot be found. + ReportNotFound bool + + // FilterPrefix will only return results with given prefix within folder. + // Should never contain a slash. + FilterPrefix string + + // ForwardTo will forward to the given object path. + ForwardTo string + + // Limit the number of returned objects if > 0. + Limit int + + // DiskID contains the disk ID of the disk. + // Leave empty to not check disk ID. + DiskID string +} + +// supported FS for Nlink optimization in readdir. +const ( + xfs = "XFS" + ext4 = "EXT4" +) + +// WalkDir will traverse a directory and return all entries found. +// On success a sorted meta cache stream will be returned. +// Metadata has data stripped, if any. +// The function tries to quit as fast as the context is canceled to avoid further drive IO +func (s *xlStorage) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writer) (err error) { + legacyFS := s.fsType != xfs && s.fsType != ext4 + + s.RLock() + legacy := s.formatLegacy + s.RUnlock() + + // Verify if volume is valid and it exists. + volumeDir, err := s.getVolDir(opts.Bucket) + if err != nil { + return err + } + + if !skipAccessChecks(opts.Bucket) { + // Stat a volume entry. + if err = Access(volumeDir); err != nil { + return convertAccessError(err, errVolumeAccessDenied) + } + } + + // Use a small block size to start sending quickly + w := newMetacacheWriter(wr, 16<<10) + w.reuseBlocks = true // We are not sharing results, so reuse buffers. + defer w.Close() + out, err := w.stream() + if err != nil { + return err + } + defer xioutil.SafeClose(out) + var objsReturned int + + objReturned := func(metadata []byte) { + if opts.Limit <= 0 { + return + } + if m, _, _ := isIndexedMetaV2(metadata); m != nil && !m.AllHidden(true) { + objsReturned++ + } + } + send := func(entry metaCacheEntry) error { + objReturned(entry.metadata) + select { + case <-ctx.Done(): + return ctx.Err() + case out <- entry: + } + return nil + } + + // Fast exit track to check if we are listing an object with + // a trailing slash, this will avoid to list the object content. + if HasSuffix(opts.BaseDir, SlashSeparator) { + metadata, err := s.readMetadata(ctx, pathJoin(volumeDir, + opts.BaseDir[:len(opts.BaseDir)-1]+globalDirSuffix, + xlStorageFormatFile)) + diskHealthCheckOK(ctx, err) + if err == nil { + // if baseDir is already a directory object, consider it + // as part of the list call, this is AWS S3 specific + // behavior. + if err := send(metaCacheEntry{ + name: opts.BaseDir, + metadata: metadata, + }); err != nil { + return err + } + } else { + st, sterr := Lstat(pathJoin(volumeDir, opts.BaseDir, xlStorageFormatFile)) + if sterr == nil && st.Mode().IsRegular() { + return errFileNotFound + } + } + } + + prefix := opts.FilterPrefix + var scanDir func(path string) error + + scanDir = func(current string) error { + if contextCanceled(ctx) { + return ctx.Err() + } + if opts.Limit > 0 && objsReturned >= opts.Limit { + return nil + } + + // Skip forward, if requested... + sb := bytebufferpool.Get() + defer func() { + sb.Reset() + bytebufferpool.Put(sb) + }() + + forward := "" + if len(opts.ForwardTo) > 0 && strings.HasPrefix(opts.ForwardTo, current) { + forward = strings.TrimPrefix(opts.ForwardTo, current) + // Trim further directories and trailing slash. + if idx := strings.IndexByte(forward, '/'); idx > 0 { + forward = forward[:idx] + } + } + + if s.walkMu != nil { + s.walkMu.Lock() + } + entries, err := s.ListDir(ctx, "", opts.Bucket, current, -1) + if s.walkMu != nil { + s.walkMu.Unlock() + } + if err != nil { + // Folder could have gone away in-between + if err != errVolumeNotFound && err != errFileNotFound { + internalLogOnceIf(ctx, err, "metacache-walk-scan-dir") + } + if opts.ReportNotFound && err == errFileNotFound && current == opts.BaseDir { + err = errFileNotFound + } else { + err = nil + } + diskHealthCheckOK(ctx, err) + return err + } + diskHealthCheckOK(ctx, err) + if len(entries) == 0 { + return nil + } + dirObjects := make(map[string]struct{}) + + // Avoid a bunch of cleanup when joining. + current = strings.Trim(current, SlashSeparator) + for i, entry := range entries { + if contextCanceled(ctx) { + return ctx.Err() + } + if opts.Limit > 0 && objsReturned >= opts.Limit { + return nil + } + if len(prefix) > 0 && !strings.HasPrefix(entry, prefix) { + // Do not retain the file, since it doesn't + // match the prefix. + entries[i] = "" + continue + } + if len(forward) > 0 && entry < forward { + // Do not retain the file, since its + // lexially smaller than 'forward' + entries[i] = "" + continue + } + if hasSuffixByte(entry, SlashSeparatorChar) { + if strings.HasSuffix(entry, globalDirSuffixWithSlash) { + // Add without extension so it is sorted correctly. + entry = strings.TrimSuffix(entry, globalDirSuffixWithSlash) + slashSeparator + dirObjects[entry] = struct{}{} + entries[i] = entry + continue + } + // Trim slash, since we don't know if this is folder or object. + entries[i] = entries[i][:len(entry)-1] + continue + } + // Do not retain the file. + entries[i] = "" + + if contextCanceled(ctx) { + return ctx.Err() + } + // If root was an object return it as such. + if HasSuffix(entry, xlStorageFormatFile) { + var meta metaCacheEntry + if s.walkReadMu != nil { + s.walkReadMu.Lock() + } + meta.metadata, err = s.readMetadata(ctx, pathJoinBuf(sb, volumeDir, current, entry)) + if s.walkReadMu != nil { + s.walkReadMu.Unlock() + } + diskHealthCheckOK(ctx, err) + if err != nil { + // It is totally possible that xl.meta was overwritten + // while being concurrently listed at the same time in + // such scenarios the 'xl.meta' might get truncated + if !IsErrIgnored(err, io.EOF, io.ErrUnexpectedEOF) { + internalLogOnceIf(ctx, err, "metacache-walk-read-metadata") + } + continue + } + meta.name = strings.TrimSuffix(entry, xlStorageFormatFile) + meta.name = strings.TrimSuffix(meta.name, SlashSeparator) + meta.name = pathJoinBuf(sb, current, meta.name) + meta.name = decodeDirObject(meta.name) + + return send(meta) + } + // Check legacy. + if HasSuffix(entry, xlStorageFormatFileV1) && legacy { + var meta metaCacheEntry + meta.metadata, err = xioutil.ReadFile(pathJoinBuf(sb, volumeDir, current, entry)) + diskHealthCheckOK(ctx, err) + if err != nil { + if !IsErrIgnored(err, io.EOF, io.ErrUnexpectedEOF) { + internalLogIf(ctx, err) + } + continue + } + meta.name = strings.TrimSuffix(entry, xlStorageFormatFileV1) + meta.name = strings.TrimSuffix(meta.name, SlashSeparator) + meta.name = pathJoinBuf(sb, current, meta.name) + + return send(meta) + } + // Skip all other files. + } + + // Process in sort order. + sort.Strings(entries) + dirStack := make([]string, 0, 5) + prefix = "" // Remove prefix after first level as we have already filtered the list. + if len(forward) > 0 { + // Conservative forwarding. Entries may be either objects or prefixes. + for i, entry := range entries { + if entry >= forward || strings.HasPrefix(forward, entry) { + entries = entries[i:] + break + } + } + } + + for _, entry := range entries { + if contextCanceled(ctx) { + return ctx.Err() + } + if opts.Limit > 0 && objsReturned >= opts.Limit { + return nil + } + if entry == "" { + continue + } + meta := metaCacheEntry{name: pathJoinBuf(sb, current, entry)} + + // If directory entry on stack before this, pop it now. + for len(dirStack) > 0 && dirStack[len(dirStack)-1] < meta.name { + pop := dirStack[len(dirStack)-1] + select { + case <-ctx.Done(): + return ctx.Err() + case out <- metaCacheEntry{name: pop}: + } + if opts.Recursive { + // Scan folder we found. Should be in correct sort order where we are. + err := scanDir(pop) + if err != nil { + if errors.Is(err, context.Canceled) { + return err + } + internalLogIf(ctx, err) + } + } + dirStack = dirStack[:len(dirStack)-1] + } + + // All objects will be returned as directories, there has been no object check yet. + // Check it by attempting to read metadata. + _, isDirObj := dirObjects[entry] + if isDirObj { + meta.name = meta.name[:len(meta.name)-1] + globalDirSuffixWithSlash + } + + if s.walkReadMu != nil { + s.walkReadMu.Lock() + } + meta.metadata, err = s.readMetadata(ctx, pathJoinBuf(sb, volumeDir, meta.name, xlStorageFormatFile)) + if s.walkReadMu != nil { + s.walkReadMu.Unlock() + } + diskHealthCheckOK(ctx, err) + switch { + case err == nil: + // It was an object + if isDirObj { + meta.name = strings.TrimSuffix(meta.name, globalDirSuffixWithSlash) + slashSeparator + } + if err := send(meta); err != nil { + return err + } + case osIsNotExist(err), isSysErrIsDir(err): + if legacy { + meta.metadata, err = xioutil.ReadFile(pathJoinBuf(sb, volumeDir, meta.name, xlStorageFormatFileV1)) + diskHealthCheckOK(ctx, err) + if err == nil { + // It was an object + if err := send(meta); err != nil { + return err + } + continue + } + } + + // NOT an object, append to stack (with slash) + // If dirObject, but no metadata (which is unexpected) we skip it. + if !isDirObj { + if !isDirEmpty(pathJoinBuf(sb, volumeDir, meta.name), legacyFS) { + dirStack = append(dirStack, meta.name+slashSeparator) + } + } + case isSysErrNotDir(err): + // skip + } + } + + // If directory entry left on stack, pop it now. + for len(dirStack) > 0 { + if opts.Limit > 0 && objsReturned >= opts.Limit { + return nil + } + if contextCanceled(ctx) { + return ctx.Err() + } + pop := dirStack[len(dirStack)-1] + select { + case <-ctx.Done(): + return ctx.Err() + case out <- metaCacheEntry{name: pop}: + } + if opts.Recursive { + // Scan folder we found. Should be in correct sort order where we are. + internalLogIf(ctx, scanDir(pop)) + } + dirStack = dirStack[:len(dirStack)-1] + } + return nil + } + + // Stream output. + return scanDir(opts.BaseDir) +} + +func (p *xlStorageDiskIDCheck) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writer) (err error) { + if err := p.checkID(opts.DiskID); err != nil { + return err + } + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricWalkDir, opts.Bucket, opts.BaseDir) + if err != nil { + return err + } + defer done(0, &err) + + return p.storage.WalkDir(ctx, opts, wr) +} + +// WalkDir will traverse a directory and return all entries found. +// On success a meta cache stream will be returned, that should be closed when done. +func (client *storageRESTClient) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writer) error { + // Ensure remote has the same disk ID. + opts.DiskID = *client.diskID.Load() + b, err := opts.MarshalMsg(grid.GetByteBuffer()[:0]) + if err != nil { + return toStorageErr(err) + } + + st, err := client.gridConn.NewStream(ctx, grid.HandlerWalkDir, b) + if err != nil { + return toStorageErr(err) + } + return toStorageErr(st.Results(func(in []byte) error { + _, err := wr.Write(in) + return err + })) +} + +// WalkDirHandler - remote caller to list files and folders in a requested directory path. +func (s *storageRESTServer) WalkDirHandler(ctx context.Context, payload []byte, _ <-chan []byte, out chan<- []byte) (gerr *grid.RemoteErr) { + var opts WalkDirOptions + _, err := opts.UnmarshalMsg(payload) + if err != nil { + return grid.NewRemoteErr(err) + } + + if !s.checkID(opts.DiskID) { + return grid.NewRemoteErr(errDiskNotFound) + } + + ctx, cancel := context.WithCancel(ctx) + defer cancel() + return grid.NewRemoteErr(s.getStorage().WalkDir(ctx, opts, grid.WriterToChannel(ctx, out))) +} diff --git a/cmd/metacache-walk_gen.go b/cmd/metacache-walk_gen.go new file mode 100644 index 00000000..6f0624f9 --- /dev/null +++ b/cmd/metacache-walk_gen.go @@ -0,0 +1,285 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *WalkDirOptions) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Bucket": + z.Bucket, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "BaseDir": + z.BaseDir, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "BaseDir") + return + } + case "Recursive": + z.Recursive, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Recursive") + return + } + case "ReportNotFound": + z.ReportNotFound, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "ReportNotFound") + return + } + case "FilterPrefix": + z.FilterPrefix, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "FilterPrefix") + return + } + case "ForwardTo": + z.ForwardTo, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "ForwardTo") + return + } + case "Limit": + z.Limit, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Limit") + return + } + case "DiskID": + z.DiskID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *WalkDirOptions) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 8 + // write "Bucket" + err = en.Append(0x88, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Bucket) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + // write "BaseDir" + err = en.Append(0xa7, 0x42, 0x61, 0x73, 0x65, 0x44, 0x69, 0x72) + if err != nil { + return + } + err = en.WriteString(z.BaseDir) + if err != nil { + err = msgp.WrapError(err, "BaseDir") + return + } + // write "Recursive" + err = en.Append(0xa9, 0x52, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65) + if err != nil { + return + } + err = en.WriteBool(z.Recursive) + if err != nil { + err = msgp.WrapError(err, "Recursive") + return + } + // write "ReportNotFound" + err = en.Append(0xae, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x4e, 0x6f, 0x74, 0x46, 0x6f, 0x75, 0x6e, 0x64) + if err != nil { + return + } + err = en.WriteBool(z.ReportNotFound) + if err != nil { + err = msgp.WrapError(err, "ReportNotFound") + return + } + // write "FilterPrefix" + err = en.Append(0xac, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78) + if err != nil { + return + } + err = en.WriteString(z.FilterPrefix) + if err != nil { + err = msgp.WrapError(err, "FilterPrefix") + return + } + // write "ForwardTo" + err = en.Append(0xa9, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x54, 0x6f) + if err != nil { + return + } + err = en.WriteString(z.ForwardTo) + if err != nil { + err = msgp.WrapError(err, "ForwardTo") + return + } + // write "Limit" + err = en.Append(0xa5, 0x4c, 0x69, 0x6d, 0x69, 0x74) + if err != nil { + return + } + err = en.WriteInt(z.Limit) + if err != nil { + err = msgp.WrapError(err, "Limit") + return + } + // write "DiskID" + err = en.Append(0xa6, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x44) + if err != nil { + return + } + err = en.WriteString(z.DiskID) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *WalkDirOptions) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 8 + // string "Bucket" + o = append(o, 0x88, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + o = msgp.AppendString(o, z.Bucket) + // string "BaseDir" + o = append(o, 0xa7, 0x42, 0x61, 0x73, 0x65, 0x44, 0x69, 0x72) + o = msgp.AppendString(o, z.BaseDir) + // string "Recursive" + o = append(o, 0xa9, 0x52, 0x65, 0x63, 0x75, 0x72, 0x73, 0x69, 0x76, 0x65) + o = msgp.AppendBool(o, z.Recursive) + // string "ReportNotFound" + o = append(o, 0xae, 0x52, 0x65, 0x70, 0x6f, 0x72, 0x74, 0x4e, 0x6f, 0x74, 0x46, 0x6f, 0x75, 0x6e, 0x64) + o = msgp.AppendBool(o, z.ReportNotFound) + // string "FilterPrefix" + o = append(o, 0xac, 0x46, 0x69, 0x6c, 0x74, 0x65, 0x72, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78) + o = msgp.AppendString(o, z.FilterPrefix) + // string "ForwardTo" + o = append(o, 0xa9, 0x46, 0x6f, 0x72, 0x77, 0x61, 0x72, 0x64, 0x54, 0x6f) + o = msgp.AppendString(o, z.ForwardTo) + // string "Limit" + o = append(o, 0xa5, 0x4c, 0x69, 0x6d, 0x69, 0x74) + o = msgp.AppendInt(o, z.Limit) + // string "DiskID" + o = append(o, 0xa6, 0x44, 0x69, 0x73, 0x6b, 0x49, 0x44) + o = msgp.AppendString(o, z.DiskID) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *WalkDirOptions) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Bucket": + z.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "BaseDir": + z.BaseDir, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "BaseDir") + return + } + case "Recursive": + z.Recursive, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Recursive") + return + } + case "ReportNotFound": + z.ReportNotFound, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReportNotFound") + return + } + case "FilterPrefix": + z.FilterPrefix, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "FilterPrefix") + return + } + case "ForwardTo": + z.ForwardTo, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ForwardTo") + return + } + case "Limit": + z.Limit, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Limit") + return + } + case "DiskID": + z.DiskID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *WalkDirOptions) Msgsize() (s int) { + s = 1 + 7 + msgp.StringPrefixSize + len(z.Bucket) + 8 + msgp.StringPrefixSize + len(z.BaseDir) + 10 + msgp.BoolSize + 15 + msgp.BoolSize + 13 + msgp.StringPrefixSize + len(z.FilterPrefix) + 10 + msgp.StringPrefixSize + len(z.ForwardTo) + 6 + msgp.IntSize + 7 + msgp.StringPrefixSize + len(z.DiskID) + return +} diff --git a/cmd/metacache-walk_gen_test.go b/cmd/metacache-walk_gen_test.go new file mode 100644 index 00000000..16edd846 --- /dev/null +++ b/cmd/metacache-walk_gen_test.go @@ -0,0 +1,123 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalWalkDirOptions(t *testing.T) { + v := WalkDirOptions{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgWalkDirOptions(b *testing.B) { + v := WalkDirOptions{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgWalkDirOptions(b *testing.B) { + v := WalkDirOptions{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalWalkDirOptions(b *testing.B) { + v := WalkDirOptions{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeWalkDirOptions(t *testing.T) { + v := WalkDirOptions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeWalkDirOptions Msgsize() is inaccurate") + } + + vn := WalkDirOptions{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeWalkDirOptions(b *testing.B) { + v := WalkDirOptions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeWalkDirOptions(b *testing.B) { + v := WalkDirOptions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/metacache.go b/cmd/metacache.go new file mode 100644 index 00000000..7f35d391 --- /dev/null +++ b/cmd/metacache.go @@ -0,0 +1,200 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "path" + "strings" + "time" + + "github.com/minio/pkg/v3/console" +) + +type scanStatus uint8 + +const ( + scanStateNone scanStatus = iota + scanStateStarted + scanStateSuccess + scanStateError + + // Time in which the initiator of a scan must have reported back. + metacacheMaxRunningAge = time.Minute + + // Max time between client calls before dropping an async cache listing. + metacacheMaxClientWait = 3 * time.Minute + + // metacacheBlockSize is the number of file/directory entries to have in each block. + metacacheBlockSize = 5000 + + // metacacheSharePrefix controls whether prefixes on dirty paths are always shared. + // This will make `test/a` and `test/b` share listings if they are concurrent. + // Enabling this will make cache sharing more likely and cause less IO, + // but may cause additional latency to some calls. + metacacheSharePrefix = false +) + +//go:generate msgp -file $GOFILE -unexported + +// metacache contains a tracked cache entry. +type metacache struct { + // do not re-arrange the struct this struct has been ordered to use less + // space - if you do so please run https://github.com/orijtech/structslop + // and verify if your changes are optimal. + ended time.Time `msg:"end"` + started time.Time `msg:"st"` + lastHandout time.Time `msg:"lh"` + lastUpdate time.Time `msg:"u"` + bucket string `msg:"b"` + filter string `msg:"flt"` + id string `msg:"id"` + error string `msg:"err"` + root string `msg:"root"` + fileNotFound bool `msg:"fnf"` + status scanStatus `msg:"stat"` + recursive bool `msg:"rec"` + dataVersion uint8 `msg:"v"` +} + +func (m *metacache) finished() bool { + return !m.ended.IsZero() +} + +// worthKeeping indicates if the cache by itself is worth keeping. +func (m *metacache) worthKeeping() bool { + if m == nil { + return false + } + cache := m + switch { + case !cache.finished() && time.Since(cache.lastUpdate) > metacacheMaxRunningAge: + // Not finished and update for metacacheMaxRunningAge, discard it. + return false + case cache.finished() && time.Since(cache.lastHandout) > 5*metacacheMaxClientWait: + // Keep for 15 minutes after we last saw the client. + // Since the cache is finished keeping it a bit longer doesn't hurt us. + return false + case cache.status == scanStateError || cache.status == scanStateNone: + // Remove failed listings after 5 minutes. + return time.Since(cache.lastUpdate) > 5*time.Minute + } + return true +} + +// keepAlive will continuously update lastHandout until ctx is canceled. +func (m metacache) keepAlive(ctx context.Context, rpc *peerRESTClient) { + // we intentionally operate on a copy of m, so we can update without locks. + t := time.NewTicker(metacacheMaxClientWait / 10) + defer t.Stop() + for { + select { + case <-ctx.Done(): + // Request is done, stop updating. + return + case <-t.C: + m.lastHandout = time.Now() + + if m2, err := rpc.UpdateMetacacheListing(ctx, m); err == nil { + if m2.status != scanStateStarted { + if serverDebugLog { + console.Debugln("returning", m.id, "due to scan state", m2.status, time.Now().Format(time.RFC3339)) + } + return + } + m = m2 + if serverDebugLog { + console.Debugln("refreshed", m.id, time.Now().Format(time.RFC3339)) + } + } else if serverDebugLog { + console.Debugln("error refreshing", m.id, time.Now().Format(time.RFC3339)) + } + } + } +} + +// baseDirFromPrefix will return the base directory given an object path. +// For example an object with name prefix/folder/object.ext will return `prefix/folder/`. +func baseDirFromPrefix(prefix string) string { + b := path.Dir(prefix) + if b == "." || b == "./" || b == "/" { + b = "" + } + if !strings.Contains(prefix, slashSeparator) { + b = "" + } + if len(b) > 0 && !strings.HasSuffix(b, slashSeparator) { + b += slashSeparator + } + return b +} + +// update cache with new status. +// The updates are conditional so multiple callers can update with different states. +func (m *metacache) update(update metacache) { + now := UTCNow() + m.lastUpdate = now + + if update.lastHandout.After(m.lastHandout) { + m.lastHandout = update.lastUpdate + if m.lastHandout.After(now) { + m.lastHandout = now + } + } + if m.status == scanStateStarted && update.status == scanStateSuccess { + m.ended = now + } + + if m.status == scanStateStarted && update.status != scanStateStarted { + m.status = update.status + } + + if m.status == scanStateStarted && time.Since(m.lastHandout) > metacacheMaxClientWait { + // Drop if client hasn't been seen for 3 minutes. + m.status = scanStateError + m.error = "client not seen" + } + + if m.error == "" && update.error != "" { + m.error = update.error + m.status = scanStateError + m.ended = now + } + m.fileNotFound = m.fileNotFound || update.fileNotFound +} + +// delete all cache data on disks. +func (m *metacache) delete(ctx context.Context) { + if m.bucket == "" || m.id == "" { + bugLogIf(ctx, fmt.Errorf("metacache.delete: bucket (%s) or id (%s) empty", m.bucket, m.id)) + return + } + objAPI := newObjectLayerFn() + if objAPI == nil { + internalLogIf(ctx, errors.New("metacache.delete: no object layer")) + return + } + ez, ok := objAPI.(deleteAllStorager) + if !ok { + bugLogIf(ctx, errors.New("metacache.delete: expected objAPI to be 'deleteAllStorager'")) + return + } + ez.deleteAll(ctx, minioMetaBucket, metacachePrefixForID(m.bucket, m.id)) +} diff --git a/cmd/metacache_gen.go b/cmd/metacache_gen.go new file mode 100644 index 00000000..9ca1bca8 --- /dev/null +++ b/cmd/metacache_gen.go @@ -0,0 +1,470 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *metacache) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "end": + z.ended, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "ended") + return + } + case "st": + z.started, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "started") + return + } + case "lh": + z.lastHandout, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "lastHandout") + return + } + case "u": + z.lastUpdate, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "lastUpdate") + return + } + case "b": + z.bucket, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "bucket") + return + } + case "flt": + z.filter, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "filter") + return + } + case "id": + z.id, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "id") + return + } + case "err": + z.error, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "error") + return + } + case "root": + z.root, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "root") + return + } + case "fnf": + z.fileNotFound, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "fileNotFound") + return + } + case "stat": + { + var zb0002 uint8 + zb0002, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err, "status") + return + } + z.status = scanStatus(zb0002) + } + case "rec": + z.recursive, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "recursive") + return + } + case "v": + z.dataVersion, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err, "dataVersion") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *metacache) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 13 + // write "end" + err = en.Append(0x8d, 0xa3, 0x65, 0x6e, 0x64) + if err != nil { + return + } + err = en.WriteTime(z.ended) + if err != nil { + err = msgp.WrapError(err, "ended") + return + } + // write "st" + err = en.Append(0xa2, 0x73, 0x74) + if err != nil { + return + } + err = en.WriteTime(z.started) + if err != nil { + err = msgp.WrapError(err, "started") + return + } + // write "lh" + err = en.Append(0xa2, 0x6c, 0x68) + if err != nil { + return + } + err = en.WriteTime(z.lastHandout) + if err != nil { + err = msgp.WrapError(err, "lastHandout") + return + } + // write "u" + err = en.Append(0xa1, 0x75) + if err != nil { + return + } + err = en.WriteTime(z.lastUpdate) + if err != nil { + err = msgp.WrapError(err, "lastUpdate") + return + } + // write "b" + err = en.Append(0xa1, 0x62) + if err != nil { + return + } + err = en.WriteString(z.bucket) + if err != nil { + err = msgp.WrapError(err, "bucket") + return + } + // write "flt" + err = en.Append(0xa3, 0x66, 0x6c, 0x74) + if err != nil { + return + } + err = en.WriteString(z.filter) + if err != nil { + err = msgp.WrapError(err, "filter") + return + } + // write "id" + err = en.Append(0xa2, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteString(z.id) + if err != nil { + err = msgp.WrapError(err, "id") + return + } + // write "err" + err = en.Append(0xa3, 0x65, 0x72, 0x72) + if err != nil { + return + } + err = en.WriteString(z.error) + if err != nil { + err = msgp.WrapError(err, "error") + return + } + // write "root" + err = en.Append(0xa4, 0x72, 0x6f, 0x6f, 0x74) + if err != nil { + return + } + err = en.WriteString(z.root) + if err != nil { + err = msgp.WrapError(err, "root") + return + } + // write "fnf" + err = en.Append(0xa3, 0x66, 0x6e, 0x66) + if err != nil { + return + } + err = en.WriteBool(z.fileNotFound) + if err != nil { + err = msgp.WrapError(err, "fileNotFound") + return + } + // write "stat" + err = en.Append(0xa4, 0x73, 0x74, 0x61, 0x74) + if err != nil { + return + } + err = en.WriteUint8(uint8(z.status)) + if err != nil { + err = msgp.WrapError(err, "status") + return + } + // write "rec" + err = en.Append(0xa3, 0x72, 0x65, 0x63) + if err != nil { + return + } + err = en.WriteBool(z.recursive) + if err != nil { + err = msgp.WrapError(err, "recursive") + return + } + // write "v" + err = en.Append(0xa1, 0x76) + if err != nil { + return + } + err = en.WriteUint8(z.dataVersion) + if err != nil { + err = msgp.WrapError(err, "dataVersion") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *metacache) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 13 + // string "end" + o = append(o, 0x8d, 0xa3, 0x65, 0x6e, 0x64) + o = msgp.AppendTime(o, z.ended) + // string "st" + o = append(o, 0xa2, 0x73, 0x74) + o = msgp.AppendTime(o, z.started) + // string "lh" + o = append(o, 0xa2, 0x6c, 0x68) + o = msgp.AppendTime(o, z.lastHandout) + // string "u" + o = append(o, 0xa1, 0x75) + o = msgp.AppendTime(o, z.lastUpdate) + // string "b" + o = append(o, 0xa1, 0x62) + o = msgp.AppendString(o, z.bucket) + // string "flt" + o = append(o, 0xa3, 0x66, 0x6c, 0x74) + o = msgp.AppendString(o, z.filter) + // string "id" + o = append(o, 0xa2, 0x69, 0x64) + o = msgp.AppendString(o, z.id) + // string "err" + o = append(o, 0xa3, 0x65, 0x72, 0x72) + o = msgp.AppendString(o, z.error) + // string "root" + o = append(o, 0xa4, 0x72, 0x6f, 0x6f, 0x74) + o = msgp.AppendString(o, z.root) + // string "fnf" + o = append(o, 0xa3, 0x66, 0x6e, 0x66) + o = msgp.AppendBool(o, z.fileNotFound) + // string "stat" + o = append(o, 0xa4, 0x73, 0x74, 0x61, 0x74) + o = msgp.AppendUint8(o, uint8(z.status)) + // string "rec" + o = append(o, 0xa3, 0x72, 0x65, 0x63) + o = msgp.AppendBool(o, z.recursive) + // string "v" + o = append(o, 0xa1, 0x76) + o = msgp.AppendUint8(o, z.dataVersion) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *metacache) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "end": + z.ended, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ended") + return + } + case "st": + z.started, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "started") + return + } + case "lh": + z.lastHandout, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "lastHandout") + return + } + case "u": + z.lastUpdate, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "lastUpdate") + return + } + case "b": + z.bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "bucket") + return + } + case "flt": + z.filter, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "filter") + return + } + case "id": + z.id, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "id") + return + } + case "err": + z.error, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "error") + return + } + case "root": + z.root, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "root") + return + } + case "fnf": + z.fileNotFound, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "fileNotFound") + return + } + case "stat": + { + var zb0002 uint8 + zb0002, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "status") + return + } + z.status = scanStatus(zb0002) + } + case "rec": + z.recursive, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "recursive") + return + } + case "v": + z.dataVersion, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "dataVersion") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *metacache) Msgsize() (s int) { + s = 1 + 4 + msgp.TimeSize + 3 + msgp.TimeSize + 3 + msgp.TimeSize + 2 + msgp.TimeSize + 2 + msgp.StringPrefixSize + len(z.bucket) + 4 + msgp.StringPrefixSize + len(z.filter) + 3 + msgp.StringPrefixSize + len(z.id) + 4 + msgp.StringPrefixSize + len(z.error) + 5 + msgp.StringPrefixSize + len(z.root) + 4 + msgp.BoolSize + 5 + msgp.Uint8Size + 4 + msgp.BoolSize + 2 + msgp.Uint8Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *scanStatus) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 uint8 + zb0001, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = scanStatus(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z scanStatus) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteUint8(uint8(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z scanStatus) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendUint8(o, uint8(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *scanStatus) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 uint8 + zb0001, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = scanStatus(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z scanStatus) Msgsize() (s int) { + s = msgp.Uint8Size + return +} diff --git a/cmd/metacache_gen_test.go b/cmd/metacache_gen_test.go new file mode 100644 index 00000000..017d4cc3 --- /dev/null +++ b/cmd/metacache_gen_test.go @@ -0,0 +1,123 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalmetacache(t *testing.T) { + v := metacache{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgmetacache(b *testing.B) { + v := metacache{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgmetacache(b *testing.B) { + v := metacache{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalmetacache(b *testing.B) { + v := metacache{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodemetacache(t *testing.T) { + v := metacache{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodemetacache Msgsize() is inaccurate") + } + + vn := metacache{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodemetacache(b *testing.B) { + v := metacache{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodemetacache(b *testing.B) { + v := metacache{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/metacache_test.go b/cmd/metacache_test.go new file mode 100644 index 00000000..84860d85 --- /dev/null +++ b/cmd/metacache_test.go @@ -0,0 +1,246 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "testing" + "time" +) + +var metaCacheTestsetTimestamp = time.Now() + +var metaCacheTestset = []metacache{ + 0: { + id: "case-1-normal", + bucket: "bucket", + root: "folder/prefix", + recursive: false, + status: scanStateSuccess, + fileNotFound: false, + error: "", + started: metaCacheTestsetTimestamp, + ended: metaCacheTestsetTimestamp.Add(time.Minute), + lastUpdate: metaCacheTestsetTimestamp.Add(time.Minute), + lastHandout: metaCacheTestsetTimestamp, + dataVersion: metacacheStreamVersion, + }, + 1: { + id: "case-2-recursive", + bucket: "bucket", + root: "folder/prefix", + recursive: true, + status: scanStateSuccess, + fileNotFound: false, + error: "", + started: metaCacheTestsetTimestamp, + ended: metaCacheTestsetTimestamp.Add(time.Minute), + lastUpdate: metaCacheTestsetTimestamp.Add(time.Minute), + lastHandout: metaCacheTestsetTimestamp, + dataVersion: metacacheStreamVersion, + }, + 2: { + id: "case-3-older", + bucket: "bucket", + root: "folder/prefix", + recursive: false, + status: scanStateSuccess, + fileNotFound: true, + error: "", + started: metaCacheTestsetTimestamp.Add(-time.Minute), + ended: metaCacheTestsetTimestamp, + lastUpdate: metaCacheTestsetTimestamp, + lastHandout: metaCacheTestsetTimestamp, + dataVersion: metacacheStreamVersion, + }, + 3: { + id: "case-4-error", + bucket: "bucket", + root: "folder/prefix", + recursive: false, + status: scanStateError, + fileNotFound: false, + error: "an error lol", + started: metaCacheTestsetTimestamp.Add(-20 * time.Minute), + ended: metaCacheTestsetTimestamp.Add(-20 * time.Minute), + lastUpdate: metaCacheTestsetTimestamp.Add(-20 * time.Minute), + lastHandout: metaCacheTestsetTimestamp.Add(-20 * time.Minute), + dataVersion: metacacheStreamVersion, + }, + 4: { + id: "case-5-noupdate", + bucket: "bucket", + root: "folder/prefix", + recursive: false, + status: scanStateStarted, + fileNotFound: false, + error: "", + started: metaCacheTestsetTimestamp.Add(-time.Minute), + ended: time.Time{}, + lastUpdate: metaCacheTestsetTimestamp.Add(-time.Minute), + lastHandout: metaCacheTestsetTimestamp, + dataVersion: metacacheStreamVersion, + }, + 5: { + id: "case-6-404notfound", + bucket: "bucket", + root: "folder/notfound", + recursive: true, + status: scanStateSuccess, + fileNotFound: true, + error: "", + started: metaCacheTestsetTimestamp, + ended: metaCacheTestsetTimestamp.Add(time.Minute), + lastUpdate: metaCacheTestsetTimestamp.Add(time.Minute), + lastHandout: metaCacheTestsetTimestamp, + dataVersion: metacacheStreamVersion, + }, + 6: { + id: "case-7-oldcycle", + bucket: "bucket", + root: "folder/prefix", + recursive: true, + status: scanStateSuccess, + fileNotFound: false, + error: "", + started: metaCacheTestsetTimestamp.Add(-10 * time.Minute), + ended: metaCacheTestsetTimestamp.Add(-8 * time.Minute), + lastUpdate: metaCacheTestsetTimestamp.Add(-8 * time.Minute), + lastHandout: metaCacheTestsetTimestamp, + dataVersion: metacacheStreamVersion, + }, + 7: { + id: "case-8-running", + bucket: "bucket", + root: "folder/running", + recursive: false, + status: scanStateStarted, + fileNotFound: false, + error: "", + started: metaCacheTestsetTimestamp.Add(-1 * time.Minute), + ended: time.Time{}, + lastUpdate: metaCacheTestsetTimestamp.Add(-1 * time.Minute), + lastHandout: metaCacheTestsetTimestamp, + dataVersion: metacacheStreamVersion, + }, + 8: { + id: "case-8-finished-a-week-ago", + bucket: "bucket", + root: "folder/finished", + recursive: false, + status: scanStateSuccess, + fileNotFound: false, + error: "", + started: metaCacheTestsetTimestamp.Add(-7 * 24 * time.Hour), + ended: metaCacheTestsetTimestamp.Add(-7 * 24 * time.Hour), + lastUpdate: metaCacheTestsetTimestamp.Add(-7 * 24 * time.Hour), + lastHandout: metaCacheTestsetTimestamp.Add(-7 * 24 * time.Hour), + dataVersion: metacacheStreamVersion, + }, +} + +func Test_baseDirFromPrefix(t *testing.T) { + tests := []struct { + name string + prefix string + want string + }{ + { + name: "root", + prefix: "object.ext", + want: "", + }, + { + name: "rootdotslash", + prefix: "./object.ext", + want: "", + }, + { + name: "rootslash", + prefix: "/", + want: "", + }, + { + name: "folder", + prefix: "prefix/", + want: "prefix/", + }, + { + name: "folderobj", + prefix: "prefix/obj.ext", + want: "prefix/", + }, + { + name: "folderfolderobj", + prefix: "prefix/prefix2/obj.ext", + want: "prefix/prefix2/", + }, + { + name: "folderfolder", + prefix: "prefix/prefix2/", + want: "prefix/prefix2/", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := baseDirFromPrefix(tt.prefix); got != tt.want { + t.Errorf("baseDirFromPrefix() = %v, want %v", got, tt.want) + } + }) + } +} + +func Test_metacache_finished(t *testing.T) { + wantResults := []bool{0: true, 1: true, 2: true, 3: true, 4: false, 5: true, 6: true, 7: false, 8: true} + + for i, tt := range metaCacheTestset { + t.Run(tt.id, func(t *testing.T) { + var want bool + if i >= len(wantResults) { + t.Logf("no expected result for test #%d", i) + } else { + want = wantResults[i] + } + + got := tt.finished() + if got != want { + t.Errorf("#%d: want %v, got %v", i, want, got) + } + }) + } +} + +func Test_metacache_worthKeeping(t *testing.T) { + // TODO: Update... + wantResults := []bool{0: true, 1: true, 2: true, 3: false, 4: false, 5: true, 6: true, 7: false, 8: false} + + for i, tt := range metaCacheTestset { + t.Run(tt.id, func(t *testing.T) { + var want bool + if i >= len(wantResults) { + t.Logf("no expected result for test #%d", i) + } else { + want = wantResults[i] + } + + got := tt.worthKeeping() + if got != want { + t.Errorf("#%d: want %v, got %v", i, want, got) + } + }) + } +} diff --git a/cmd/metrics-realtime.go b/cmd/metrics-realtime.go new file mode 100644 index 00000000..4a1d5573 --- /dev/null +++ b/cmd/metrics-realtime.go @@ -0,0 +1,231 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/disk" + "github.com/minio/minio/internal/net" + c "github.com/shirou/gopsutil/v3/cpu" + "github.com/shirou/gopsutil/v3/load" +) + +type collectMetricsOpts struct { + hosts map[string]struct{} + disks map[string]struct{} + jobID string + depID string +} + +func collectLocalMetrics(types madmin.MetricType, opts collectMetricsOpts) (m madmin.RealtimeMetrics) { + if types == madmin.MetricsNone { + return m + } + + byHostName := globalMinioAddr + if len(opts.hosts) > 0 { + server := getLocalServerProperty(globalEndpoints, &http.Request{ + Host: globalLocalNodeName, + }, false) + if _, ok := opts.hosts[server.Endpoint]; ok { + byHostName = server.Endpoint + } else { + return m + } + } + + if strings.HasPrefix(byHostName, ":") && !strings.HasPrefix(globalLocalNodeName, ":") { + byHostName = globalLocalNodeName + } + + if types.Contains(madmin.MetricsDisk) { + m.ByDisk = make(map[string]madmin.DiskMetric) + aggr := madmin.DiskMetric{ + CollectedAt: time.Now(), + } + for name, disk := range collectLocalDisksMetrics(opts.disks) { + m.ByDisk[name] = disk + aggr.Merge(&disk) + } + m.Aggregated.Disk = &aggr + } + + if types.Contains(madmin.MetricsScanner) { + metrics := globalScannerMetrics.report() + m.Aggregated.Scanner = &metrics + } + if types.Contains(madmin.MetricsOS) { + metrics := globalOSMetrics.report() + m.Aggregated.OS = &metrics + } + if types.Contains(madmin.MetricsBatchJobs) { + m.Aggregated.BatchJobs = globalBatchJobsMetrics.report(opts.jobID) + } + if types.Contains(madmin.MetricsSiteResync) { + m.Aggregated.SiteResync = globalSiteResyncMetrics.report(opts.depID) + } + if types.Contains(madmin.MetricNet) { + m.Aggregated.Net = &madmin.NetMetrics{ + CollectedAt: UTCNow(), + InterfaceName: globalInternodeInterface, + } + netStats, err := net.GetInterfaceNetStats(globalInternodeInterface) + if err != nil { + m.Errors = append(m.Errors, fmt.Sprintf("%s: %v (nicstats)", byHostName, err.Error())) + } else { + m.Aggregated.Net.NetStats = netStats + } + } + if types.Contains(madmin.MetricsMem) { + m.Aggregated.Mem = &madmin.MemMetrics{ + CollectedAt: UTCNow(), + } + m.Aggregated.Mem.Info = madmin.GetMemInfo(GlobalContext, byHostName) + } + if types.Contains(madmin.MetricsCPU) { + m.Aggregated.CPU = &madmin.CPUMetrics{ + CollectedAt: UTCNow(), + } + cm, err := c.Times(false) + if err != nil { + m.Errors = append(m.Errors, fmt.Sprintf("%s: %v (cpuTimes)", byHostName, err.Error())) + } else { + // not collecting per-cpu stats, so there will be only one element + if len(cm) == 1 { + m.Aggregated.CPU.TimesStat = &cm[0] + } else { + m.Errors = append(m.Errors, fmt.Sprintf("%s: Expected one CPU stat, got %d", byHostName, len(cm))) + } + } + cpuCount, err := c.Counts(true) + if err != nil { + m.Errors = append(m.Errors, fmt.Sprintf("%s: %v (cpuCount)", byHostName, err.Error())) + } else { + m.Aggregated.CPU.CPUCount = cpuCount + } + + loadStat, err := load.Avg() + if err != nil { + m.Errors = append(m.Errors, fmt.Sprintf("%s: %v (loadStat)", byHostName, err.Error())) + } else { + m.Aggregated.CPU.LoadStat = loadStat + } + } + if types.Contains(madmin.MetricsRPC) { + gr := globalGrid.Load() + if gr == nil { + m.Errors = append(m.Errors, fmt.Sprintf("%s: Grid not initialized", byHostName)) + } else { + stats := gr.ConnStats() + m.Aggregated.RPC = &stats + } + } + // Add types... + + // ByHost is a shallow reference, so careful about sharing. + m.ByHost = map[string]madmin.Metrics{byHostName: m.Aggregated} + m.Hosts = append(m.Hosts, byHostName) + + return m +} + +func collectLocalDisksMetrics(disks map[string]struct{}) map[string]madmin.DiskMetric { + objLayer := newObjectLayerFn() + if objLayer == nil { + return nil + } + + metrics := make(map[string]madmin.DiskMetric) + storageInfo := objLayer.LocalStorageInfo(GlobalContext, true) + for _, d := range storageInfo.Disks { + if len(disks) != 0 { + _, ok := disks[d.Endpoint] + if !ok { + continue + } + } + + if d.State != madmin.DriveStateOk && d.State != madmin.DriveStateUnformatted { + metrics[d.Endpoint] = madmin.DiskMetric{NDisks: 1, Offline: 1} + continue + } + + var dm madmin.DiskMetric + dm.NDisks = 1 + if d.Healing { + dm.Healing++ + } + if d.Metrics != nil { + dm.LifeTimeOps = make(map[string]uint64, len(d.Metrics.APICalls)) + for k, v := range d.Metrics.APICalls { + if v != 0 { + dm.LifeTimeOps[k] = v + } + } + dm.LastMinute.Operations = make(map[string]madmin.TimedAction, len(d.Metrics.APICalls)) + for k, v := range d.Metrics.LastMinute { + if v.Count != 0 { + dm.LastMinute.Operations[k] = v + } + } + } + + st, err := disk.GetDriveStats(d.Major, d.Minor) + if err == nil { + dm.IOStats = madmin.DiskIOStats{ + ReadIOs: st.ReadIOs, + ReadMerges: st.ReadMerges, + ReadSectors: st.ReadSectors, + ReadTicks: st.ReadTicks, + WriteIOs: st.WriteIOs, + WriteMerges: st.WriteMerges, + WriteSectors: st.WriteSectors, + WriteTicks: st.WriteTicks, + CurrentIOs: st.CurrentIOs, + TotalTicks: st.TotalTicks, + ReqTicks: st.ReqTicks, + DiscardIOs: st.DiscardIOs, + DiscardMerges: st.DiscardMerges, + DiscardSectors: st.DiscardSectors, + DiscardTicks: st.DiscardTicks, + FlushIOs: st.FlushIOs, + FlushTicks: st.FlushTicks, + } + } + + metrics[d.Endpoint] = dm + } + return metrics +} + +func collectRemoteMetrics(ctx context.Context, types madmin.MetricType, opts collectMetricsOpts) (m madmin.RealtimeMetrics) { + if !globalIsDistErasure { + return m + } + all := globalNotificationSys.GetMetrics(ctx, types, opts) + for _, remote := range all { + m.Merge(&remote) + } + return m +} diff --git a/cmd/metrics-resource.go b/cmd/metrics-resource.go new file mode 100644 index 00000000..94c91873 --- /dev/null +++ b/cmd/metrics-resource.go @@ -0,0 +1,493 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "fmt" + "math" + "net/http" + "sync" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/prometheus/client_golang/prometheus" +) + +const ( + resourceMetricsCollectionInterval = time.Minute + resourceMetricsCacheInterval = time.Minute + + // drive stats + totalInodes MetricName = "total_inodes" + readsPerSec MetricName = "reads_per_sec" + writesPerSec MetricName = "writes_per_sec" + readsKBPerSec MetricName = "reads_kb_per_sec" + writesKBPerSec MetricName = "writes_kb_per_sec" + readsAwait MetricName = "reads_await" + writesAwait MetricName = "writes_await" + percUtil MetricName = "perc_util" + usedInodes MetricName = "used_inodes" + + // network stats + interfaceRxBytes MetricName = "rx_bytes" + interfaceRxErrors MetricName = "rx_errors" + interfaceTxBytes MetricName = "tx_bytes" + interfaceTxErrors MetricName = "tx_errors" + + // cpu stats + cpuUser MetricName = "user" + cpuSystem MetricName = "system" + cpuIOWait MetricName = "iowait" + cpuIdle MetricName = "idle" + cpuNice MetricName = "nice" + cpuSteal MetricName = "steal" + cpuLoad1 MetricName = "load1" + cpuLoad5 MetricName = "load5" + cpuLoad15 MetricName = "load15" + cpuLoad1Perc MetricName = "load1_perc" + cpuLoad5Perc MetricName = "load5_perc" + cpuLoad15Perc MetricName = "load15_perc" +) + +var ( + resourceCollector *minioResourceCollector + // resourceMetricsMap is a map of subsystem to its metrics + resourceMetricsMap map[MetricSubsystem]ResourceMetrics + resourceMetricsMapMu sync.RWMutex + // resourceMetricsHelpMap maps metric name to its help string + resourceMetricsHelpMap map[MetricName]string + resourceMetricsGroups []*MetricsGroupV2 + // initial values for drives (at the time of server startup) + // used for calculating avg values for drive metrics + latestDriveStats map[string]madmin.DiskIOStats + latestDriveStatsMu sync.RWMutex + lastDriveStatsRefresh time.Time +) + +// PeerResourceMetrics represents the resource metrics +// retrieved from a peer, along with errors if any +type PeerResourceMetrics struct { + Metrics map[MetricSubsystem]ResourceMetrics + Errors []string +} + +// ResourceMetrics is a map of unique key identifying +// a resource metric (e.g. reads_per_sec_{node}_{drive}) +// to its data +type ResourceMetrics map[string]ResourceMetric + +// ResourceMetric represents a single resource metric +// The metrics are collected from all servers periodically +// and stored in the resource metrics map. +// It also maintains the count of number of times this metric +// was collected since the server started, and the sum, +// average and max values across the same. +type ResourceMetric struct { + Name MetricName + Labels map[string]string + + // value captured in current cycle + Current float64 + + // Used when system provides cumulative (since uptime) values + // helps in calculating the current value by comparing the new + // cumulative value with previous one + Cumulative float64 + + Max float64 + Avg float64 + Sum float64 + Count uint64 +} + +func init() { + interval := fmt.Sprintf("%ds", int(resourceMetricsCollectionInterval.Seconds())) + resourceMetricsHelpMap = map[MetricName]string{ + interfaceRxBytes: "Bytes received on the interface in " + interval, + interfaceRxErrors: "Receive errors in " + interval, + interfaceTxBytes: "Bytes transmitted in " + interval, + interfaceTxErrors: "Transmit errors in " + interval, + total: "Total memory on the node", + memUsed: "Used memory on the node", + memUsedPerc: "Used memory percentage on the node", + memFree: "Free memory on the node", + memShared: "Shared memory on the node", + memBuffers: "Buffers memory on the node", + memCache: "Cache memory on the node", + memAvailable: "Available memory on the node", + readsPerSec: "Reads per second on a drive", + writesPerSec: "Writes per second on a drive", + readsKBPerSec: "Kilobytes read per second on a drive", + writesKBPerSec: "Kilobytes written per second on a drive", + readsAwait: "Average time for read requests to be served on a drive", + writesAwait: "Average time for write requests to be served on a drive", + percUtil: "Percentage of time the disk was busy", + usedBytes: "Used bytes on a drive", + totalBytes: "Total bytes on a drive", + usedInodes: "Total inodes used on a drive", + totalInodes: "Total inodes on a drive", + cpuUser: "CPU user time", + cpuSystem: "CPU system time", + cpuIdle: "CPU idle time", + cpuIOWait: "CPU ioWait time", + cpuSteal: "CPU steal time", + cpuNice: "CPU nice time", + cpuLoad1: "CPU load average 1min", + cpuLoad5: "CPU load average 5min", + cpuLoad15: "CPU load average 15min", + cpuLoad1Perc: "CPU load average 1min (percentage)", + cpuLoad5Perc: "CPU load average 5min (percentage)", + cpuLoad15Perc: "CPU load average 15min (percentage)", + } + resourceMetricsGroups = []*MetricsGroupV2{ + getResourceMetrics(), + } + + resourceCollector = newMinioResourceCollector(resourceMetricsGroups) +} + +func getResourceKey(name MetricName, labels map[string]string) string { + // labels are used to uniquely identify a metric + // e.g. reads_per_sec_{drive} inside the map + sfx := "" + for _, v := range labels { + if len(sfx) > 0 { + sfx += "_" + } + sfx += v + } + + return string(name) + "_" + sfx +} + +func updateResourceMetrics(subSys MetricSubsystem, name MetricName, val float64, labels map[string]string, isCumulative bool) { + resourceMetricsMapMu.Lock() + defer resourceMetricsMapMu.Unlock() + subsysMetrics, found := resourceMetricsMap[subSys] + if !found { + subsysMetrics = ResourceMetrics{} + } + + key := getResourceKey(name, labels) + metric, found := subsysMetrics[key] + if !found { + metric = ResourceMetric{ + Name: name, + Labels: labels, + } + } + + if isCumulative { + metric.Current = val - metric.Cumulative + metric.Cumulative = val + } else { + metric.Current = val + } + + if metric.Current > metric.Max { + metric.Max = val + } + + metric.Sum += metric.Current + metric.Count++ + + metric.Avg = metric.Sum / float64(metric.Count) + subsysMetrics[key] = metric + + resourceMetricsMap[subSys] = subsysMetrics +} + +// updateDriveIOStats - Updates the drive IO stats by calculating the difference between the current and latest updated values. +func updateDriveIOStats(currentStats madmin.DiskIOStats, latestStats madmin.DiskIOStats, labels map[string]string) { + sectorSize := uint64(512) + kib := float64(1 << 10) + diffInSeconds := time.Now().UTC().Sub(lastDriveStatsRefresh).Seconds() + if diffInSeconds == 0 { + // too soon to update the stats + return + } + diffStats := getDiffStats(latestStats, currentStats) + + updateResourceMetrics(driveSubsystem, readsPerSec, float64(diffStats.ReadIOs)/diffInSeconds, labels, false) + readKib := float64(diffStats.ReadSectors*sectorSize) / kib + updateResourceMetrics(driveSubsystem, readsKBPerSec, readKib/diffInSeconds, labels, false) + + updateResourceMetrics(driveSubsystem, writesPerSec, float64(diffStats.WriteIOs)/diffInSeconds, labels, false) + writeKib := float64(diffStats.WriteSectors*sectorSize) / kib + updateResourceMetrics(driveSubsystem, writesKBPerSec, writeKib/diffInSeconds, labels, false) + + rdAwait := 0.0 + if diffStats.ReadIOs > 0 { + rdAwait = float64(diffStats.ReadTicks) / float64(diffStats.ReadIOs) + } + updateResourceMetrics(driveSubsystem, readsAwait, rdAwait, labels, false) + + wrAwait := 0.0 + if diffStats.WriteIOs > 0 { + wrAwait = float64(diffStats.WriteTicks) / float64(diffStats.WriteIOs) + } + updateResourceMetrics(driveSubsystem, writesAwait, wrAwait, labels, false) + updateResourceMetrics(driveSubsystem, percUtil, float64(diffStats.TotalTicks)/(diffInSeconds*10), labels, false) +} + +func collectDriveMetrics(m madmin.RealtimeMetrics) { + latestDriveStatsMu.Lock() + for d, dm := range m.ByDisk { + labels := map[string]string{"drive": d} + latestStats, ok := latestDriveStats[d] + if !ok { + latestDriveStats[d] = dm.IOStats + continue + } + updateDriveIOStats(dm.IOStats, latestStats, labels) + latestDriveStats[d] = dm.IOStats + } + lastDriveStatsRefresh = time.Now().UTC() + latestDriveStatsMu.Unlock() + + globalLocalDrivesMu.RLock() + localDrives := cloneDrives(globalLocalDrivesMap) + globalLocalDrivesMu.RUnlock() + + for _, d := range localDrives { + di, err := d.DiskInfo(GlobalContext, DiskInfoOptions{}) + labels := map[string]string{"drive": di.Endpoint} + if err == nil { + updateResourceMetrics(driveSubsystem, usedBytes, float64(di.Used), labels, false) + updateResourceMetrics(driveSubsystem, totalBytes, float64(di.Total), labels, false) + updateResourceMetrics(driveSubsystem, usedInodes, float64(di.UsedInodes), labels, false) + updateResourceMetrics(driveSubsystem, totalInodes, float64(di.FreeInodes+di.UsedInodes), labels, false) + } + } +} + +func collectLocalResourceMetrics() { + types := madmin.MetricsDisk | madmin.MetricNet | madmin.MetricsMem | madmin.MetricsCPU + + m := collectLocalMetrics(types, collectMetricsOpts{}) + for _, hm := range m.ByHost { + if hm.Net != nil && len(hm.Net.NetStats.Name) > 0 { + stats := hm.Net.NetStats + labels := map[string]string{"interface": stats.Name} + updateResourceMetrics(interfaceSubsystem, interfaceRxBytes, float64(stats.RxBytes), labels, true) + updateResourceMetrics(interfaceSubsystem, interfaceRxErrors, float64(stats.RxErrors), labels, true) + updateResourceMetrics(interfaceSubsystem, interfaceTxBytes, float64(stats.TxBytes), labels, true) + updateResourceMetrics(interfaceSubsystem, interfaceTxErrors, float64(stats.TxErrors), labels, true) + } + if hm.Mem != nil && len(hm.Mem.Info.Addr) > 0 { + labels := map[string]string{} + stats := hm.Mem.Info + updateResourceMetrics(memSubsystem, total, float64(stats.Total), labels, false) + updateResourceMetrics(memSubsystem, memUsed, float64(stats.Used), labels, false) + perc := math.Round(float64(stats.Used*100*100)/float64(stats.Total)) / 100 + updateResourceMetrics(memSubsystem, memUsedPerc, perc, labels, false) + updateResourceMetrics(memSubsystem, memFree, float64(stats.Free), labels, false) + updateResourceMetrics(memSubsystem, memShared, float64(stats.Shared), labels, false) + updateResourceMetrics(memSubsystem, memBuffers, float64(stats.Buffers), labels, false) + updateResourceMetrics(memSubsystem, memAvailable, float64(stats.Available), labels, false) + updateResourceMetrics(memSubsystem, memCache, float64(stats.Cache), labels, false) + } + if hm.CPU != nil { + labels := map[string]string{} + ts := hm.CPU.TimesStat + if ts != nil { + tot := ts.User + ts.System + ts.Idle + ts.Iowait + ts.Nice + ts.Steal + cpuUserVal := math.Round(ts.User/tot*100*100) / 100 + updateResourceMetrics(cpuSubsystem, cpuUser, cpuUserVal, labels, false) + cpuSystemVal := math.Round(ts.System/tot*100*100) / 100 + updateResourceMetrics(cpuSubsystem, cpuSystem, cpuSystemVal, labels, false) + cpuIdleVal := math.Round(ts.Idle/tot*100*100) / 100 + updateResourceMetrics(cpuSubsystem, cpuIdle, cpuIdleVal, labels, false) + cpuIOWaitVal := math.Round(ts.Iowait/tot*100*100) / 100 + updateResourceMetrics(cpuSubsystem, cpuIOWait, cpuIOWaitVal, labels, false) + cpuNiceVal := math.Round(ts.Nice/tot*100*100) / 100 + updateResourceMetrics(cpuSubsystem, cpuNice, cpuNiceVal, labels, false) + cpuStealVal := math.Round(ts.Steal/tot*100*100) / 100 + updateResourceMetrics(cpuSubsystem, cpuSteal, cpuStealVal, labels, false) + } + ls := hm.CPU.LoadStat + if ls != nil { + updateResourceMetrics(cpuSubsystem, cpuLoad1, ls.Load1, labels, false) + updateResourceMetrics(cpuSubsystem, cpuLoad5, ls.Load5, labels, false) + updateResourceMetrics(cpuSubsystem, cpuLoad15, ls.Load15, labels, false) + if hm.CPU.CPUCount > 0 { + perc := math.Round(ls.Load1*100*100/float64(hm.CPU.CPUCount)) / 100 + updateResourceMetrics(cpuSubsystem, cpuLoad1Perc, perc, labels, false) + perc = math.Round(ls.Load5*100*100/float64(hm.CPU.CPUCount)) / 100 + updateResourceMetrics(cpuSubsystem, cpuLoad5Perc, perc, labels, false) + perc = math.Round(ls.Load15*100*100/float64(hm.CPU.CPUCount)) / 100 + updateResourceMetrics(cpuSubsystem, cpuLoad15Perc, perc, labels, false) + } + } + } + break // only one host expected + } + + collectDriveMetrics(m) +} + +func initLatestValues() { + m := collectLocalMetrics(madmin.MetricsDisk, collectMetricsOpts{}) + latestDriveStatsMu.Lock() + latestDriveStats = map[string]madmin.DiskIOStats{} + for d, dm := range m.ByDisk { + latestDriveStats[d] = dm.IOStats + } + lastDriveStatsRefresh = time.Now().UTC() + latestDriveStatsMu.Unlock() +} + +// startResourceMetricsCollection - starts the job for collecting resource metrics +func startResourceMetricsCollection() { + initLatestValues() + + resourceMetricsMapMu.Lock() + resourceMetricsMap = map[MetricSubsystem]ResourceMetrics{} + resourceMetricsMapMu.Unlock() + metricsTimer := time.NewTimer(resourceMetricsCollectionInterval) + defer metricsTimer.Stop() + + collectLocalResourceMetrics() + + for { + select { + case <-GlobalContext.Done(): + return + case <-metricsTimer.C: + collectLocalResourceMetrics() + + // Reset the timer for next cycle. + metricsTimer.Reset(resourceMetricsCollectionInterval) + } + } +} + +// minioResourceCollector is the Collector for resource metrics +type minioResourceCollector struct { + metricsGroups []*MetricsGroupV2 + desc *prometheus.Desc +} + +// Describe sends the super-set of all possible descriptors of metrics +func (c *minioResourceCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.desc +} + +// Collect is called by the Prometheus registry when collecting metrics. +func (c *minioResourceCollector) Collect(out chan<- prometheus.Metric) { + var wg sync.WaitGroup + publish := func(in <-chan MetricV2) { + defer wg.Done() + for metric := range in { + labels, values := getOrderedLabelValueArrays(metric.VariableLabels) + collectMetric(metric, labels, values, "resource", out) + } + } + + // Call peer api to fetch metrics + wg.Add(2) + go publish(ReportMetrics(GlobalContext, c.metricsGroups)) + go publish(globalNotificationSys.GetResourceMetrics(GlobalContext)) + wg.Wait() +} + +// newMinioResourceCollector describes the collector +// and returns reference of minio resource Collector +// It creates the Prometheus Description which is used +// to define Metric and help string +func newMinioResourceCollector(metricsGroups []*MetricsGroupV2) *minioResourceCollector { + return &minioResourceCollector{ + metricsGroups: metricsGroups, + desc: prometheus.NewDesc("minio_resource_stats", "Resource statistics exposed by MinIO server", nil, nil), + } +} + +func prepareResourceMetrics(rm ResourceMetric, subSys MetricSubsystem, requireAvgMax bool) []MetricV2 { + help := resourceMetricsHelpMap[rm.Name] + name := rm.Name + metrics := make([]MetricV2, 0, 3) + metrics = append(metrics, MetricV2{ + Description: getResourceMetricDescription(subSys, name, help), + Value: rm.Current, + VariableLabels: cloneMSS(rm.Labels), + }) + + if requireAvgMax { + avgName := MetricName(fmt.Sprintf("%s_avg", name)) + avgHelp := fmt.Sprintf("%s (avg)", help) + metrics = append(metrics, MetricV2{ + Description: getResourceMetricDescription(subSys, avgName, avgHelp), + Value: math.Round(rm.Avg*100) / 100, + VariableLabels: cloneMSS(rm.Labels), + }) + + maxName := MetricName(fmt.Sprintf("%s_max", name)) + maxHelp := fmt.Sprintf("%s (max)", help) + metrics = append(metrics, MetricV2{ + Description: getResourceMetricDescription(subSys, maxName, maxHelp), + Value: rm.Max, + VariableLabels: cloneMSS(rm.Labels), + }) + } + + return metrics +} + +func getResourceMetricDescription(subSys MetricSubsystem, name MetricName, help string) MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: subSys, + Name: name, + Help: help, + Type: gaugeMetric, + } +} + +func getResourceMetrics() *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: resourceMetricsCacheInterval, + } + mg.RegisterRead(func(ctx context.Context) []MetricV2 { + metrics := []MetricV2{} + + subSystems := []MetricSubsystem{interfaceSubsystem, memSubsystem, driveSubsystem, cpuSubsystem} + resourceMetricsMapMu.RLock() + defer resourceMetricsMapMu.RUnlock() + for _, subSys := range subSystems { + stats, found := resourceMetricsMap[subSys] + if found { + requireAvgMax := true + if subSys == driveSubsystem { + requireAvgMax = false + } + for _, m := range stats { + metrics = append(metrics, prepareResourceMetrics(m, subSys, requireAvgMax)...) + } + } + } + + return metrics + }) + return mg +} + +// metricsResourceHandler is the prometheus handler for resource metrics +func metricsResourceHandler() http.Handler { + return metricsHTTPHandler(resourceCollector, "handler.MetricsResource") +} diff --git a/cmd/metrics-router.go b/cmd/metrics-router.go new file mode 100644 index 00000000..f8b85c25 --- /dev/null +++ b/cmd/metrics-router.go @@ -0,0 +1,75 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "net/http" + "strings" + + "github.com/minio/mux" + "github.com/minio/pkg/v3/env" +) + +const ( + prometheusMetricsPathLegacy = "/prometheus/metrics" + prometheusMetricsV2ClusterPath = "/v2/metrics/cluster" + prometheusMetricsV2BucketPath = "/v2/metrics/bucket" + prometheusMetricsV2NodePath = "/v2/metrics/node" + prometheusMetricsV2ResourcePath = "/v2/metrics/resource" + + // Metrics v3 endpoints + metricsV3Path = "/metrics/v3" +) + +// Standard env prometheus auth type +const ( + EnvPrometheusAuthType = "MINIO_PROMETHEUS_AUTH_TYPE" + EnvPrometheusOpenMetrics = "MINIO_PROMETHEUS_OPEN_METRICS" +) + +type prometheusAuthType string + +const ( + prometheusJWT prometheusAuthType = "jwt" + prometheusPublic prometheusAuthType = "public" +) + +// registerMetricsRouter - add handler functions for metrics. +func registerMetricsRouter(router *mux.Router) { + // metrics router + metricsRouter := router.NewRoute().PathPrefix(minioReservedBucketPath).Subrouter() + authType := prometheusAuthType(strings.ToLower(env.Get(EnvPrometheusAuthType, string(prometheusJWT)))) + + auth := AuthMiddleware + if authType == prometheusPublic { + auth = NoAuthMiddleware + } + + metricsRouter.Handle(prometheusMetricsPathLegacy, auth(metricsHandler())) + metricsRouter.Handle(prometheusMetricsV2ClusterPath, auth(metricsServerHandler())) + metricsRouter.Handle(prometheusMetricsV2BucketPath, auth(metricsBucketHandler())) + metricsRouter.Handle(prometheusMetricsV2NodePath, auth(metricsNodeHandler())) + metricsRouter.Handle(prometheusMetricsV2ResourcePath, auth(metricsResourceHandler())) + + // Metrics v3 + metricsV3Server := newMetricsV3Server(auth) + + // Register metrics v3 handler. It also accepts an optional query + // parameter `?list` - see handler for details. + metricsRouter.Methods(http.MethodGet).Path(metricsV3Path + "{pathComps:.*}").Handler(metricsV3Server) +} diff --git a/cmd/metrics-v2.go b/cmd/metrics-v2.go new file mode 100644 index 00000000..e38750a5 --- /dev/null +++ b/cmd/metrics-v2.go @@ -0,0 +1,4435 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "fmt" + "maps" + "math" + "net/http" + "runtime" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/minio/kms-go/kes" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/cachevalue" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/mcontext" + "github.com/minio/minio/internal/rest" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + "github.com/prometheus/procfs" +) + +//go:generate msgp -file=$GOFILE -unexported -io=false + +var ( + nodeCollector *minioNodeCollector + clusterCollector *minioClusterCollector + bucketCollector *minioBucketCollector + peerMetricsGroups []*MetricsGroupV2 + bucketPeerMetricsGroups []*MetricsGroupV2 +) + +// v2MetricsMaxBuckets enforces a bucket count limit on metrics for v2 calls. +// If people hit this limit, they should move to v3, as certain calls explode with high bucket count. +const v2MetricsMaxBuckets = 100 + +func init() { + clusterMetricsGroups := []*MetricsGroupV2{ + getNodeHealthMetrics(MetricsGroupOpts{dependGlobalNotificationSys: true}), + getClusterStorageMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}), + getClusterTierMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}), + getClusterUsageMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}), + getKMSMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true, dependGlobalKMS: true}), + getClusterHealthMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}), + getIAMNodeMetrics(MetricsGroupOpts{dependGlobalAuthNPlugin: true, dependGlobalIAMSys: true}), + getReplicationSiteMetrics(MetricsGroupOpts{dependGlobalSiteReplicationSys: true}), + getBatchJobsMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}), + } + + peerMetricsGroups = []*MetricsGroupV2{ + getGoMetrics(), + getHTTPMetrics(MetricsGroupOpts{}), + getNotificationMetrics(MetricsGroupOpts{dependGlobalLambdaTargetList: true}), + getMinioProcMetrics(), + getMinioVersionMetrics(), + getNetworkMetrics(), + getS3TTFBMetric(), + getILMNodeMetrics(), + getScannerNodeMetrics(), + getIAMNodeMetrics(MetricsGroupOpts{dependGlobalAuthNPlugin: true, dependGlobalIAMSys: true}), + getKMSNodeMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true, dependGlobalKMS: true}), + getMinioHealingMetrics(MetricsGroupOpts{dependGlobalBackgroundHealState: true}), + getWebhookMetrics(), + getTierMetrics(), + } + + allMetricsGroups := func() (allMetrics []*MetricsGroupV2) { + allMetrics = append(allMetrics, clusterMetricsGroups...) + allMetrics = append(allMetrics, peerMetricsGroups...) + return allMetrics + }() + + nodeGroups := []*MetricsGroupV2{ + getNodeHealthMetrics(MetricsGroupOpts{dependGlobalNotificationSys: true}), + getHTTPMetrics(MetricsGroupOpts{}), + getNetworkMetrics(), + getMinioVersionMetrics(), + getS3TTFBMetric(), + getTierMetrics(), + getNotificationMetrics(MetricsGroupOpts{dependGlobalLambdaTargetList: true}), + getDistLockMetrics(MetricsGroupOpts{dependGlobalIsDistErasure: true, dependGlobalLockServer: true}), + getIAMNodeMetrics(MetricsGroupOpts{dependGlobalAuthNPlugin: true, dependGlobalIAMSys: true}), + getLocalStorageMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}), + getReplicationNodeMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true, dependBucketTargetSys: true}), + } + + bucketMetricsGroups := []*MetricsGroupV2{ + getBucketUsageMetrics(MetricsGroupOpts{dependGlobalObjectAPI: true}), + getHTTPMetrics(MetricsGroupOpts{bucketOnly: true}), + getBucketTTFBMetric(), + } + + bucketPeerMetricsGroups = []*MetricsGroupV2{ + getHTTPMetrics(MetricsGroupOpts{bucketOnly: true}), + getBucketTTFBMetric(), + } + + nodeCollector = newMinioCollectorNode(nodeGroups) + clusterCollector = newMinioClusterCollector(allMetricsGroups) + bucketCollector = newMinioBucketCollector(bucketMetricsGroups) +} + +// MetricNamespace is top level grouping of metrics to create the metric name. +type MetricNamespace string + +// MetricSubsystem is the sub grouping for metrics within a namespace. +type MetricSubsystem string + +const ( + bucketMetricNamespace MetricNamespace = "minio_bucket" + clusterMetricNamespace MetricNamespace = "minio_cluster" + healMetricNamespace MetricNamespace = "minio_heal" + interNodeMetricNamespace MetricNamespace = "minio_inter_node" + nodeMetricNamespace MetricNamespace = "minio_node" + minioMetricNamespace MetricNamespace = "minio" + s3MetricNamespace MetricNamespace = "minio_s3" +) + +const ( + cacheSubsystem MetricSubsystem = "cache" + capacityRawSubsystem MetricSubsystem = "capacity_raw" + capacityUsableSubsystem MetricSubsystem = "capacity_usable" + driveSubsystem MetricSubsystem = "drive" + interfaceSubsystem MetricSubsystem = "if" + memSubsystem MetricSubsystem = "mem" + cpuSubsystem MetricSubsystem = "cpu_avg" + storageClassSubsystem MetricSubsystem = "storage_class" + fileDescriptorSubsystem MetricSubsystem = "file_descriptor" + goRoutines MetricSubsystem = "go_routine" + ioSubsystem MetricSubsystem = "io" + nodesSubsystem MetricSubsystem = "nodes" + objectsSubsystem MetricSubsystem = "objects" + bucketsSubsystem MetricSubsystem = "bucket" + processSubsystem MetricSubsystem = "process" + replicationSubsystem MetricSubsystem = "replication" + requestsSubsystem MetricSubsystem = "requests" + requestsRejectedSubsystem MetricSubsystem = "requests_rejected" + timeSubsystem MetricSubsystem = "time" + ttfbSubsystem MetricSubsystem = "requests_ttfb" + trafficSubsystem MetricSubsystem = "traffic" + softwareSubsystem MetricSubsystem = "software" + sysCallSubsystem MetricSubsystem = "syscall" + usageSubsystem MetricSubsystem = "usage" + quotaSubsystem MetricSubsystem = "quota" + ilmSubsystem MetricSubsystem = "ilm" + tierSubsystem MetricSubsystem = "tier" + scannerSubsystem MetricSubsystem = "scanner" + iamSubsystem MetricSubsystem = "iam" + kmsSubsystem MetricSubsystem = "kms" + notifySubsystem MetricSubsystem = "notify" + lambdaSubsystem MetricSubsystem = "lambda" + auditSubsystem MetricSubsystem = "audit" + webhookSubsystem MetricSubsystem = "webhook" +) + +// MetricName are the individual names for the metric. +type MetricName string + +const ( + authTotal MetricName = "auth_total" + canceledTotal MetricName = "canceled_total" + errorsTotal MetricName = "errors_total" + headerTotal MetricName = "header_total" + healTotal MetricName = "heal_total" + hitsTotal MetricName = "hits_total" + inflightTotal MetricName = "inflight_total" + invalidTotal MetricName = "invalid_total" + limitTotal MetricName = "limit_total" + missedTotal MetricName = "missed_total" + waitingTotal MetricName = "waiting_total" + incomingTotal MetricName = "incoming_total" + objectTotal MetricName = "object_total" + versionTotal MetricName = "version_total" + deleteMarkerTotal MetricName = "deletemarker_total" + offlineTotal MetricName = "offline_total" + onlineTotal MetricName = "online_total" + openTotal MetricName = "open_total" + readTotal MetricName = "read_total" + timestampTotal MetricName = "timestamp_total" + writeTotal MetricName = "write_total" + total MetricName = "total" + freeInodes MetricName = "free_inodes" + + lastMinFailedCount MetricName = "last_minute_failed_count" + lastMinFailedBytes MetricName = "last_minute_failed_bytes" + lastHourFailedCount MetricName = "last_hour_failed_count" + lastHourFailedBytes MetricName = "last_hour_failed_bytes" + totalFailedCount MetricName = "total_failed_count" + totalFailedBytes MetricName = "total_failed_bytes" + + currActiveWorkers MetricName = "current_active_workers" + avgActiveWorkers MetricName = "average_active_workers" + maxActiveWorkers MetricName = "max_active_workers" + recentBacklogCount MetricName = "recent_backlog_count" + currInQueueCount MetricName = "last_minute_queued_count" + currInQueueBytes MetricName = "last_minute_queued_bytes" + receivedCount MetricName = "received_count" + sentCount MetricName = "sent_count" + currTransferRate MetricName = "current_transfer_rate" + avgTransferRate MetricName = "average_transfer_rate" + maxTransferRate MetricName = "max_transfer_rate" + credentialErrors MetricName = "credential_errors" + + currLinkLatency MetricName = "current_link_latency_ms" + avgLinkLatency MetricName = "average_link_latency_ms" + maxLinkLatency MetricName = "max_link_latency_ms" + + linkOnline MetricName = "link_online" + linkOfflineDuration MetricName = "link_offline_duration_seconds" + linkDowntimeTotalDuration MetricName = "link_downtime_duration_seconds" + + avgInQueueCount MetricName = "average_queued_count" + avgInQueueBytes MetricName = "average_queued_bytes" + maxInQueueCount MetricName = "max_queued_count" + maxInQueueBytes MetricName = "max_queued_bytes" + proxiedGetRequestsTotal MetricName = "proxied_get_requests_total" + proxiedHeadRequestsTotal MetricName = "proxied_head_requests_total" + proxiedPutTaggingRequestsTotal MetricName = "proxied_put_tagging_requests_total" + proxiedGetTaggingRequestsTotal MetricName = "proxied_get_tagging_requests_total" + proxiedDeleteTaggingRequestsTotal MetricName = "proxied_delete_tagging_requests_total" + proxiedGetRequestsFailures MetricName = "proxied_get_requests_failures" + proxiedHeadRequestsFailures MetricName = "proxied_head_requests_failures" + proxiedPutTaggingRequestFailures MetricName = "proxied_put_tagging_requests_failures" + proxiedGetTaggingRequestFailures MetricName = "proxied_get_tagging_requests_failures" + proxiedDeleteTaggingRequestFailures MetricName = "proxied_delete_tagging_requests_failures" + + freeBytes MetricName = "free_bytes" + readBytes MetricName = "read_bytes" + rcharBytes MetricName = "rchar_bytes" + receivedBytes MetricName = "received_bytes" + latencyMilliSec MetricName = "latency_ms" + sentBytes MetricName = "sent_bytes" + totalBytes MetricName = "total_bytes" + usedBytes MetricName = "used_bytes" + writeBytes MetricName = "write_bytes" + wcharBytes MetricName = "wchar_bytes" + + latencyMicroSec MetricName = "latency_us" + latencyNanoSec MetricName = "latency_ns" + + commitInfo MetricName = "commit_info" + usageInfo MetricName = "usage_info" + versionInfo MetricName = "version_info" + + sizeDistribution = "size_distribution" + versionDistribution = "version_distribution" + ttfbDistribution = "seconds_distribution" + ttlbDistribution = "ttlb_seconds_distribution" + + lastActivityTime = "last_activity_nano_seconds" + startTime = "starttime_seconds" + upTime = "uptime_seconds" + memory = "resident_memory_bytes" + vmemory = "virtual_memory_bytes" + cpu = "cpu_total_seconds" + + expiryMissedTasks MetricName = "expiry_missed_tasks" + expiryMissedFreeVersions MetricName = "expiry_missed_freeversions" + expiryMissedTierJournalTasks MetricName = "expiry_missed_tierjournal_tasks" + expiryNumWorkers MetricName = "expiry_num_workers" + transitionMissedTasks MetricName = "transition_missed_immediate_tasks" + + transitionedBytes MetricName = "transitioned_bytes" + transitionedObjects MetricName = "transitioned_objects" + transitionedVersions MetricName = "transitioned_versions" + + tierRequestsSuccess MetricName = "requests_success" + tierRequestsFailure MetricName = "requests_failure" + + kmsOnline = "online" + kmsRequestsSuccess = "request_success" + kmsRequestsError = "request_error" + kmsRequestsFail = "request_failure" + kmsUptime = "uptime" + + webhookOnline = "online" +) + +const ( + serverName = "server" +) + +// MetricTypeV2 for the types of metrics supported +type MetricTypeV2 string + +const ( + gaugeMetric = "gaugeMetric" + counterMetric = "counterMetric" + histogramMetric = "histogramMetric" +) + +// MetricDescription describes the metric +type MetricDescription struct { + Namespace MetricNamespace `json:"MetricNamespace"` + Subsystem MetricSubsystem `json:"Subsystem"` + Name MetricName `json:"MetricName"` + Help string `json:"Help"` + Type MetricTypeV2 `json:"Type"` +} + +// MetricV2 captures the details for a metric +type MetricV2 struct { + Description MetricDescription `json:"Description"` + StaticLabels map[string]string `json:"StaticLabels"` + Value float64 `json:"Value"` + VariableLabels map[string]string `json:"VariableLabels"` + HistogramBucketLabel string `json:"HistogramBucketLabel"` + Histogram map[string]uint64 `json:"Histogram"` +} + +// MetricsGroupV2 are a group of metrics that are initialized together. +type MetricsGroupV2 struct { + metricsCache *cachevalue.Cache[[]MetricV2] `msg:"-"` + cacheInterval time.Duration + metricsGroupOpts MetricsGroupOpts +} + +// MetricsGroupOpts are a group of metrics opts to be used to initialize the metrics group. +type MetricsGroupOpts struct { + dependGlobalObjectAPI bool + dependGlobalAuthNPlugin bool + dependGlobalSiteReplicationSys bool + dependGlobalNotificationSys bool + dependGlobalKMS bool + bucketOnly bool + dependGlobalLambdaTargetList bool + dependGlobalIAMSys bool + dependGlobalLockServer bool + dependGlobalIsDistErasure bool + dependGlobalBackgroundHealState bool + dependBucketTargetSys bool +} + +// RegisterRead register the metrics populator function to be used +// to populate new values upon cache invalidation. +func (g *MetricsGroupV2) RegisterRead(read func(context.Context) []MetricV2) { + g.metricsCache = cachevalue.NewFromFunc(g.cacheInterval, + cachevalue.Opts{ReturnLastGood: true}, + func(ctx context.Context) ([]MetricV2, error) { + if g.metricsGroupOpts.dependGlobalObjectAPI { + objLayer := newObjectLayerFn() + // Service not initialized yet + if objLayer == nil { + return []MetricV2{}, nil + } + } + if g.metricsGroupOpts.dependGlobalAuthNPlugin { + if globalAuthNPlugin == nil { + return []MetricV2{}, nil + } + } + if g.metricsGroupOpts.dependGlobalSiteReplicationSys { + if !globalSiteReplicationSys.isEnabled() { + return []MetricV2{}, nil + } + } + if g.metricsGroupOpts.dependGlobalNotificationSys { + if globalNotificationSys == nil { + return []MetricV2{}, nil + } + } + if g.metricsGroupOpts.dependGlobalKMS { + if GlobalKMS == nil { + return []MetricV2{}, nil + } + } + if g.metricsGroupOpts.dependGlobalLambdaTargetList { + if globalLambdaTargetList == nil { + return []MetricV2{}, nil + } + } + if g.metricsGroupOpts.dependGlobalIAMSys { + if globalIAMSys == nil { + return []MetricV2{}, nil + } + } + if g.metricsGroupOpts.dependGlobalLockServer { + if globalLockServer == nil { + return []MetricV2{}, nil + } + } + if g.metricsGroupOpts.dependGlobalIsDistErasure { + if !globalIsDistErasure { + return []MetricV2{}, nil + } + } + if g.metricsGroupOpts.dependGlobalBackgroundHealState { + if globalBackgroundHealState == nil { + return []MetricV2{}, nil + } + } + if g.metricsGroupOpts.dependBucketTargetSys { + if globalBucketTargetSys == nil { + return []MetricV2{}, nil + } + } + return read(GlobalContext), nil + }, + ) +} + +func (m *MetricV2) clone() MetricV2 { + metric := MetricV2{ + Description: m.Description, + Value: m.Value, + HistogramBucketLabel: m.HistogramBucketLabel, + StaticLabels: make(map[string]string, len(m.StaticLabels)), + VariableLabels: make(map[string]string, len(m.VariableLabels)), + Histogram: make(map[string]uint64, len(m.Histogram)), + } + maps.Copy(metric.StaticLabels, m.StaticLabels) + maps.Copy(metric.VariableLabels, m.VariableLabels) + maps.Copy(metric.Histogram, m.Histogram) + return metric +} + +// Get - returns cached value always upton the configured TTL, +// once the TTL expires "read()" registered function is called +// to return the new values and updated. +func (g *MetricsGroupV2) Get() (metrics []MetricV2) { + m, _ := g.metricsCache.Get() + if len(m) == 0 { + return []MetricV2{} + } + + metrics = make([]MetricV2, 0, len(m)) + for i := range m { + metrics = append(metrics, m[i].clone()) + } + return metrics +} + +func getClusterBucketsTotalMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: bucketsSubsystem, + Name: total, + Help: "Total number of buckets in the cluster", + Type: gaugeMetric, + } +} + +func getClusterCapacityTotalBytesMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: capacityRawSubsystem, + Name: totalBytes, + Help: "Total capacity online in the cluster", + Type: gaugeMetric, + } +} + +func getClusterCapacityFreeBytesMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: capacityRawSubsystem, + Name: freeBytes, + Help: "Total free capacity online in the cluster", + Type: gaugeMetric, + } +} + +func getClusterCapacityUsageBytesMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: capacityUsableSubsystem, + Name: totalBytes, + Help: "Total usable capacity online in the cluster", + Type: gaugeMetric, + } +} + +func getClusterCapacityUsageFreeBytesMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: capacityUsableSubsystem, + Name: freeBytes, + Help: "Total free usable capacity online in the cluster", + Type: gaugeMetric, + } +} + +func getNodeDriveAPILatencyMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: driveSubsystem, + Name: latencyMicroSec, + Help: "Average last minute latency in µs for drive API storage operations", + Type: gaugeMetric, + } +} + +func getNodeDriveUsedBytesMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: driveSubsystem, + Name: usedBytes, + Help: "Total storage used on a drive", + Type: gaugeMetric, + } +} + +func getNodeDriveTimeoutErrorsMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: driveSubsystem, + Name: "errors_timeout", + Help: "Total number of drive timeout errors since server uptime", + Type: counterMetric, + } +} + +func getNodeDriveIOErrorsMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: driveSubsystem, + Name: "errors_ioerror", + Help: "Total number of drive I/O errors since server uptime", + Type: counterMetric, + } +} + +func getNodeDriveAvailabilityErrorsMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: driveSubsystem, + Name: "errors_availability", + Help: "Total number of drive I/O errors, timeouts since server uptime", + Type: counterMetric, + } +} + +func getNodeDriveWaitingIOMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: driveSubsystem, + Name: "io_waiting", + Help: "Total number I/O operations waiting on drive", + Type: counterMetric, + } +} + +func getNodeDriveFreeBytesMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: driveSubsystem, + Name: freeBytes, + Help: "Total storage available on a drive", + Type: gaugeMetric, + } +} + +func getClusterDrivesOfflineTotalMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: driveSubsystem, + Name: offlineTotal, + Help: "Total drives offline in this cluster", + Type: gaugeMetric, + } +} + +func getClusterDrivesOnlineTotalMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: driveSubsystem, + Name: onlineTotal, + Help: "Total drives online in this cluster", + Type: gaugeMetric, + } +} + +func getClusterDrivesTotalMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: driveSubsystem, + Name: total, + Help: "Total drives in this cluster", + Type: gaugeMetric, + } +} + +func getNodeDrivesOfflineTotalMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: driveSubsystem, + Name: offlineTotal, + Help: "Total drives offline in this node", + Type: gaugeMetric, + } +} + +func getNodeDrivesOnlineTotalMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: driveSubsystem, + Name: onlineTotal, + Help: "Total drives online in this node", + Type: gaugeMetric, + } +} + +func getNodeDrivesTotalMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: driveSubsystem, + Name: total, + Help: "Total drives in this node", + Type: gaugeMetric, + } +} + +func getNodeStandardParityMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: storageClassSubsystem, + Name: "standard_parity", + Help: "standard storage class parity", + Type: gaugeMetric, + } +} + +func getNodeRRSParityMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: storageClassSubsystem, + Name: "rrs_parity", + Help: "reduced redundancy storage class parity", + Type: gaugeMetric, + } +} + +func getNodeDrivesFreeInodesMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: driveSubsystem, + Name: freeInodes, + Help: "Free inodes on a drive", + Type: gaugeMetric, + } +} + +func getNodeDriveTotalBytesMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: driveSubsystem, + Name: totalBytes, + Help: "Total storage on a drive", + Type: gaugeMetric, + } +} + +func getUsageLastScanActivityMD() MetricDescription { + return MetricDescription{ + Namespace: minioMetricNamespace, + Subsystem: usageSubsystem, + Name: lastActivityTime, + Help: "Time elapsed (in nano seconds) since last scan activity", + Type: gaugeMetric, + } +} + +func getBucketUsageLastScanActivityMD() MetricDescription { + return MetricDescription{ + Namespace: bucketMetricNamespace, + Subsystem: usageSubsystem, + Name: lastActivityTime, + Help: "Time elapsed (in nano seconds) since last scan activity", + Type: gaugeMetric, + } +} + +func getBucketUsageQuotaTotalBytesMD() MetricDescription { + return MetricDescription{ + Namespace: bucketMetricNamespace, + Subsystem: quotaSubsystem, + Name: totalBytes, + Help: "Total bucket quota size in bytes", + Type: gaugeMetric, + } +} + +func getBucketTrafficReceivedBytes() MetricDescription { + return MetricDescription{ + Namespace: bucketMetricNamespace, + Subsystem: trafficSubsystem, + Name: receivedBytes, + Help: "Total number of S3 bytes received for this bucket", + Type: gaugeMetric, + } +} + +func getBucketTrafficSentBytes() MetricDescription { + return MetricDescription{ + Namespace: bucketMetricNamespace, + Subsystem: trafficSubsystem, + Name: sentBytes, + Help: "Total number of S3 bytes sent for this bucket", + Type: gaugeMetric, + } +} + +func getBucketUsageTotalBytesMD() MetricDescription { + return MetricDescription{ + Namespace: bucketMetricNamespace, + Subsystem: usageSubsystem, + Name: totalBytes, + Help: "Total bucket size in bytes", + Type: gaugeMetric, + } +} + +func getClusterUsageTotalBytesMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: usageSubsystem, + Name: totalBytes, + Help: "Total cluster usage in bytes", + Type: gaugeMetric, + } +} + +func getClusterUsageObjectsTotalMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: usageSubsystem, + Name: objectTotal, + Help: "Total number of objects in a cluster", + Type: gaugeMetric, + } +} + +func getClusterUsageVersionsTotalMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: usageSubsystem, + Name: versionTotal, + Help: "Total number of versions (includes delete marker) in a cluster", + Type: gaugeMetric, + } +} + +func getClusterUsageDeleteMarkersTotalMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: usageSubsystem, + Name: deleteMarkerTotal, + Help: "Total number of delete markers in a cluster", + Type: gaugeMetric, + } +} + +func getBucketUsageObjectsTotalMD() MetricDescription { + return MetricDescription{ + Namespace: bucketMetricNamespace, + Subsystem: usageSubsystem, + Name: objectTotal, + Help: "Total number of objects", + Type: gaugeMetric, + } +} + +func getBucketUsageVersionsTotalMD() MetricDescription { + return MetricDescription{ + Namespace: bucketMetricNamespace, + Subsystem: usageSubsystem, + Name: versionTotal, + Help: "Total number of versions (includes delete marker)", + Type: gaugeMetric, + } +} + +func getBucketUsageDeleteMarkersTotalMD() MetricDescription { + return MetricDescription{ + Namespace: bucketMetricNamespace, + Subsystem: usageSubsystem, + Name: deleteMarkerTotal, + Help: "Total number of delete markers", + Type: gaugeMetric, + } +} + +func getClusterObjectDistributionMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: objectsSubsystem, + Name: sizeDistribution, + Help: "Distribution of object sizes across a cluster", + Type: histogramMetric, + } +} + +func getClusterObjectVersionsMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: objectsSubsystem, + Name: versionDistribution, + Help: "Distribution of object versions across a cluster", + Type: histogramMetric, + } +} + +func getClusterRepLinkLatencyCurrMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: replicationSubsystem, + Name: currLinkLatency, + Help: "Replication current link latency in milliseconds", + Type: gaugeMetric, + } +} + +func getClusterRepLinkOnlineMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: replicationSubsystem, + Name: linkOnline, + Help: "Reports whether replication link is online (1) or offline(0)", + Type: gaugeMetric, + } +} + +func getClusterRepLinkCurrOfflineDurationMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: replicationSubsystem, + Name: linkOfflineDuration, + Help: "Duration of replication link being offline in seconds since last offline event", + Type: gaugeMetric, + } +} + +func getClusterRepLinkTotalOfflineDurationMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: replicationSubsystem, + Name: linkDowntimeTotalDuration, + Help: "Total downtime of replication link in seconds since server uptime", + Type: gaugeMetric, + } +} + +func getBucketRepLatencyMD() MetricDescription { + return MetricDescription{ + Namespace: bucketMetricNamespace, + Subsystem: replicationSubsystem, + Name: latencyMilliSec, + Help: "Replication latency in milliseconds", + Type: histogramMetric, + } +} + +func getRepFailedBytesLastMinuteMD(namespace MetricNamespace) MetricDescription { + return MetricDescription{ + Namespace: namespace, + Subsystem: replicationSubsystem, + Name: lastMinFailedBytes, + Help: "Total number of bytes failed at least once to replicate in the last full minute", + Type: gaugeMetric, + } +} + +func getRepFailedOperationsLastMinuteMD(namespace MetricNamespace) MetricDescription { + return MetricDescription{ + Namespace: namespace, + Subsystem: replicationSubsystem, + Name: lastMinFailedCount, + Help: "Total number of objects which failed replication in the last full minute", + Type: gaugeMetric, + } +} + +func getRepFailedBytesLastHourMD(namespace MetricNamespace) MetricDescription { + return MetricDescription{ + Namespace: namespace, + Subsystem: replicationSubsystem, + Name: lastHourFailedBytes, + Help: "Total number of bytes failed at least once to replicate in the last hour", + Type: gaugeMetric, + } +} + +func getRepFailedOperationsLastHourMD(namespace MetricNamespace) MetricDescription { + return MetricDescription{ + Namespace: namespace, + Subsystem: replicationSubsystem, + Name: lastHourFailedCount, + Help: "Total number of objects which failed replication in the last hour", + Type: gaugeMetric, + } +} + +func getRepFailedBytesTotalMD(namespace MetricNamespace) MetricDescription { + return MetricDescription{ + Namespace: namespace, + Subsystem: replicationSubsystem, + Name: totalFailedBytes, + Help: "Total number of bytes failed at least once to replicate since server uptime", + Type: counterMetric, + } +} + +func getRepFailedOperationsTotalMD(namespace MetricNamespace) MetricDescription { + return MetricDescription{ + Namespace: namespace, + Subsystem: replicationSubsystem, + Name: totalFailedCount, + Help: "Total number of objects which failed replication since server uptime", + Type: counterMetric, + } +} + +func getRepSentBytesMD(namespace MetricNamespace) MetricDescription { + return MetricDescription{ + Namespace: namespace, + Subsystem: replicationSubsystem, + Name: sentBytes, + Help: "Total number of bytes replicated to the target", + Type: counterMetric, + } +} + +func getRepSentOperationsMD(namespace MetricNamespace) MetricDescription { + return MetricDescription{ + Namespace: namespace, + Subsystem: replicationSubsystem, + Name: sentCount, + Help: "Total number of objects replicated to the target", + Type: gaugeMetric, + } +} + +func getRepReceivedBytesMD(namespace MetricNamespace) MetricDescription { + helpText := "Total number of bytes replicated to this bucket from another source bucket" + if namespace == clusterMetricNamespace { + helpText = "Total number of bytes replicated to this cluster from site replication peer" + } + return MetricDescription{ + Namespace: namespace, + Subsystem: replicationSubsystem, + Name: receivedBytes, + Help: helpText, + Type: counterMetric, + } +} + +func getRepReceivedOperationsMD(namespace MetricNamespace) MetricDescription { + help := "Total number of objects received by this cluster" + if namespace == bucketMetricNamespace { + help = "Total number of objects received by this bucket from another source bucket" + } + return MetricDescription{ + Namespace: namespace, + Subsystem: replicationSubsystem, + Name: receivedCount, + Help: help, + Type: gaugeMetric, + } +} + +func getClusterReplMRFFailedOperationsMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: replicationSubsystem, + Name: recentBacklogCount, + Help: "Total number of objects seen in replication backlog in the last 5 minutes", + Type: gaugeMetric, + } +} + +func getClusterRepCredentialErrorsMD(namespace MetricNamespace) MetricDescription { + return MetricDescription{ + Namespace: namespace, + Subsystem: replicationSubsystem, + Name: credentialErrors, + Help: "Total number of replication credential errors since server uptime", + Type: counterMetric, + } +} + +func getClusterReplCurrQueuedOperationsMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: replicationSubsystem, + Name: currInQueueCount, + Help: "Total number of objects queued for replication in the last full minute", + Type: gaugeMetric, + } +} + +func getClusterReplCurrQueuedBytesMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: replicationSubsystem, + Name: currInQueueBytes, + Help: "Total number of bytes queued for replication in the last full minute", + Type: gaugeMetric, + } +} + +func getClusterReplActiveWorkersCountMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: replicationSubsystem, + Name: currActiveWorkers, + Help: "Total number of active replication workers", + Type: gaugeMetric, + } +} + +func getClusterReplAvgActiveWorkersCountMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: replicationSubsystem, + Name: avgActiveWorkers, + Help: "Average number of active replication workers", + Type: gaugeMetric, + } +} + +func getClusterReplMaxActiveWorkersCountMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: replicationSubsystem, + Name: maxActiveWorkers, + Help: "Maximum number of active replication workers seen since server uptime", + Type: gaugeMetric, + } +} + +func getClusterReplCurrentTransferRateMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: replicationSubsystem, + Name: currTransferRate, + Help: "Current replication transfer rate in bytes/sec", + Type: gaugeMetric, + } +} + +func getClusterRepLinkLatencyMaxMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: replicationSubsystem, + Name: maxLinkLatency, + Help: "Maximum replication link latency in milliseconds seen since server uptime", + Type: gaugeMetric, + } +} + +func getClusterRepLinkLatencyAvgMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: replicationSubsystem, + Name: avgLinkLatency, + Help: "Average replication link latency in milliseconds", + Type: gaugeMetric, + } +} + +func getClusterReplAvgQueuedOperationsMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: replicationSubsystem, + Name: avgInQueueCount, + Help: "Average number of objects queued for replication since server uptime", + Type: gaugeMetric, + } +} + +func getClusterReplAvgQueuedBytesMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: replicationSubsystem, + Name: avgInQueueBytes, + Help: "Average number of bytes queued for replication since server uptime", + Type: gaugeMetric, + } +} + +func getClusterReplMaxQueuedOperationsMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: replicationSubsystem, + Name: maxInQueueCount, + Help: "Maximum number of objects queued for replication since server uptime", + Type: gaugeMetric, + } +} + +func getClusterReplMaxQueuedBytesMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: replicationSubsystem, + Name: maxInQueueBytes, + Help: "Maximum number of bytes queued for replication since server uptime", + Type: gaugeMetric, + } +} + +func getClusterReplAvgTransferRateMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: replicationSubsystem, + Name: avgTransferRate, + Help: "Average replication transfer rate in bytes/sec", + Type: gaugeMetric, + } +} + +func getClusterReplMaxTransferRateMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: replicationSubsystem, + Name: maxTransferRate, + Help: "Maximum replication transfer rate in bytes/sec seen since server uptime", + Type: gaugeMetric, + } +} + +func getClusterReplProxiedGetOperationsMD(ns MetricNamespace) MetricDescription { + return MetricDescription{ + Namespace: ns, + Subsystem: replicationSubsystem, + Name: proxiedGetRequestsTotal, + Help: "Number of GET requests proxied to replication target", + Type: counterMetric, + } +} + +func getClusterReplProxiedHeadOperationsMD(ns MetricNamespace) MetricDescription { + return MetricDescription{ + Namespace: ns, + Subsystem: replicationSubsystem, + Name: proxiedHeadRequestsTotal, + Help: "Number of HEAD requests proxied to replication target", + Type: counterMetric, + } +} + +func getClusterReplProxiedPutTaggingOperationsMD(ns MetricNamespace) MetricDescription { + return MetricDescription{ + Namespace: ns, + Subsystem: replicationSubsystem, + Name: proxiedPutTaggingRequestsTotal, + Help: "Number of PUT tagging requests proxied to replication target", + Type: counterMetric, + } +} + +func getClusterReplProxiedGetTaggingOperationsMD(ns MetricNamespace) MetricDescription { + return MetricDescription{ + Namespace: ns, + Subsystem: replicationSubsystem, + Name: proxiedGetTaggingRequestsTotal, + Help: "Number of GET tagging requests proxied to replication target", + Type: counterMetric, + } +} + +func getClusterReplProxiedRmvTaggingOperationsMD(ns MetricNamespace) MetricDescription { + return MetricDescription{ + Namespace: ns, + Subsystem: replicationSubsystem, + Name: proxiedDeleteTaggingRequestsTotal, + Help: "Number of DELETE tagging requests proxied to replication target", + Type: counterMetric, + } +} + +func getClusterReplProxiedGetFailedOperationsMD(ns MetricNamespace) MetricDescription { + return MetricDescription{ + Namespace: ns, + Subsystem: replicationSubsystem, + Name: proxiedGetRequestsFailures, + Help: "Number of failures in GET requests proxied to replication target", + Type: counterMetric, + } +} + +func getClusterReplProxiedHeadFailedOperationsMD(ns MetricNamespace) MetricDescription { + return MetricDescription{ + Namespace: ns, + Subsystem: replicationSubsystem, + Name: proxiedHeadRequestsFailures, + Help: "Number of failures in HEAD requests proxied to replication target", + Type: counterMetric, + } +} + +func getClusterReplProxiedPutTaggingFailedOperationsMD(ns MetricNamespace) MetricDescription { + return MetricDescription{ + Namespace: ns, + Subsystem: replicationSubsystem, + Name: proxiedPutTaggingRequestFailures, + Help: "Number of failures in PUT tagging proxy requests to replication target", + Type: counterMetric, + } +} + +func getClusterReplProxiedGetTaggingFailedOperationsMD(ns MetricNamespace) MetricDescription { + return MetricDescription{ + Namespace: ns, + Subsystem: replicationSubsystem, + Name: proxiedGetTaggingRequestFailures, + Help: "Number of failures in GET tagging proxy requests to replication target", + Type: counterMetric, + } +} + +func getClusterReplProxiedRmvTaggingFailedOperationsMD(ns MetricNamespace) MetricDescription { + return MetricDescription{ + Namespace: ns, + Subsystem: replicationSubsystem, + Name: proxiedDeleteTaggingRequestFailures, + Help: "Number of failures in DELETE tagging proxy requests to replication target", + Type: counterMetric, + } +} + +func getBucketObjectDistributionMD() MetricDescription { + return MetricDescription{ + Namespace: bucketMetricNamespace, + Subsystem: objectsSubsystem, + Name: sizeDistribution, + Help: "Distribution of object sizes in the bucket, includes label for the bucket name", + Type: histogramMetric, + } +} + +func getBucketObjectVersionsMD() MetricDescription { + return MetricDescription{ + Namespace: bucketMetricNamespace, + Subsystem: objectsSubsystem, + Name: versionDistribution, + Help: "Distribution of object sizes in the bucket, includes label for the bucket name", + Type: histogramMetric, + } +} + +func getInternodeFailedRequests() MetricDescription { + return MetricDescription{ + Namespace: interNodeMetricNamespace, + Subsystem: trafficSubsystem, + Name: errorsTotal, + Help: "Total number of failed internode calls", + Type: counterMetric, + } +} + +func getInternodeTCPDialTimeout() MetricDescription { + return MetricDescription{ + Namespace: interNodeMetricNamespace, + Subsystem: trafficSubsystem, + Name: "dial_errors", + Help: "Total number of internode TCP dial timeouts and errors", + Type: counterMetric, + } +} + +func getInternodeTCPAvgDuration() MetricDescription { + return MetricDescription{ + Namespace: interNodeMetricNamespace, + Subsystem: trafficSubsystem, + Name: "dial_avg_time", + Help: "Average time of internodes TCP dial calls", + Type: gaugeMetric, + } +} + +func getInterNodeSentBytesMD() MetricDescription { + return MetricDescription{ + Namespace: interNodeMetricNamespace, + Subsystem: trafficSubsystem, + Name: sentBytes, + Help: "Total number of bytes sent to the other peer nodes", + Type: counterMetric, + } +} + +func getInterNodeReceivedBytesMD() MetricDescription { + return MetricDescription{ + Namespace: interNodeMetricNamespace, + Subsystem: trafficSubsystem, + Name: receivedBytes, + Help: "Total number of bytes received from other peer nodes", + Type: counterMetric, + } +} + +func getS3SentBytesMD() MetricDescription { + return MetricDescription{ + Namespace: s3MetricNamespace, + Subsystem: trafficSubsystem, + Name: sentBytes, + Help: "Total number of s3 bytes sent", + Type: counterMetric, + } +} + +func getS3ReceivedBytesMD() MetricDescription { + return MetricDescription{ + Namespace: s3MetricNamespace, + Subsystem: trafficSubsystem, + Name: receivedBytes, + Help: "Total number of s3 bytes received", + Type: counterMetric, + } +} + +func getS3RequestsInFlightMD() MetricDescription { + return MetricDescription{ + Namespace: s3MetricNamespace, + Subsystem: requestsSubsystem, + Name: inflightTotal, + Help: "Total number of S3 requests currently in flight", + Type: gaugeMetric, + } +} + +func getS3RequestsInQueueMD() MetricDescription { + return MetricDescription{ + Namespace: s3MetricNamespace, + Subsystem: requestsSubsystem, + Name: waitingTotal, + Help: "Total number of S3 requests in the waiting queue", + Type: gaugeMetric, + } +} + +func getIncomingS3RequestsMD() MetricDescription { + return MetricDescription{ + Namespace: s3MetricNamespace, + Subsystem: requestsSubsystem, + Name: incomingTotal, + Help: "Total number of incoming S3 requests", + Type: gaugeMetric, + } +} + +func getS3RequestsTotalMD() MetricDescription { + return MetricDescription{ + Namespace: s3MetricNamespace, + Subsystem: requestsSubsystem, + Name: total, + Help: "Total number of S3 requests", + Type: counterMetric, + } +} + +func getS3RequestsErrorsMD() MetricDescription { + return MetricDescription{ + Namespace: s3MetricNamespace, + Subsystem: requestsSubsystem, + Name: errorsTotal, + Help: "Total number of S3 requests with (4xx and 5xx) errors", + Type: counterMetric, + } +} + +func getS3Requests4xxErrorsMD() MetricDescription { + return MetricDescription{ + Namespace: s3MetricNamespace, + Subsystem: requestsSubsystem, + Name: "4xx_" + errorsTotal, + Help: "Total number of S3 requests with (4xx) errors", + Type: counterMetric, + } +} + +func getS3Requests5xxErrorsMD() MetricDescription { + return MetricDescription{ + Namespace: s3MetricNamespace, + Subsystem: requestsSubsystem, + Name: "5xx_" + errorsTotal, + Help: "Total number of S3 requests with (5xx) errors", + Type: counterMetric, + } +} + +func getS3RequestsCanceledMD() MetricDescription { + return MetricDescription{ + Namespace: s3MetricNamespace, + Subsystem: requestsSubsystem, + Name: canceledTotal, + Help: "Total number of S3 requests that were canceled by the client", + Type: counterMetric, + } +} + +func getS3RejectedAuthRequestsTotalMD() MetricDescription { + return MetricDescription{ + Namespace: s3MetricNamespace, + Subsystem: requestsRejectedSubsystem, + Name: authTotal, + Help: "Total number of S3 requests rejected for auth failure", + Type: counterMetric, + } +} + +func getS3RejectedHeaderRequestsTotalMD() MetricDescription { + return MetricDescription{ + Namespace: s3MetricNamespace, + Subsystem: requestsRejectedSubsystem, + Name: headerTotal, + Help: "Total number of S3 requests rejected for invalid header", + Type: counterMetric, + } +} + +func getS3RejectedTimestampRequestsTotalMD() MetricDescription { + return MetricDescription{ + Namespace: s3MetricNamespace, + Subsystem: requestsRejectedSubsystem, + Name: timestampTotal, + Help: "Total number of S3 requests rejected for invalid timestamp", + Type: counterMetric, + } +} + +func getS3RejectedInvalidRequestsTotalMD() MetricDescription { + return MetricDescription{ + Namespace: s3MetricNamespace, + Subsystem: requestsRejectedSubsystem, + Name: invalidTotal, + Help: "Total number of invalid S3 requests", + Type: counterMetric, + } +} + +func getHealObjectsTotalMD() MetricDescription { + return MetricDescription{ + Namespace: healMetricNamespace, + Subsystem: objectsSubsystem, + Name: total, + Help: "Objects scanned since server uptime", + Type: counterMetric, + } +} + +func getHealObjectsHealTotalMD() MetricDescription { + return MetricDescription{ + Namespace: healMetricNamespace, + Subsystem: objectsSubsystem, + Name: healTotal, + Help: "Objects healed since server uptime", + Type: counterMetric, + } +} + +func getHealObjectsFailTotalMD() MetricDescription { + return MetricDescription{ + Namespace: healMetricNamespace, + Subsystem: objectsSubsystem, + Name: errorsTotal, + Help: "Objects with healing failed since server uptime", + Type: counterMetric, + } +} + +func getHealLastActivityTimeMD() MetricDescription { + return MetricDescription{ + Namespace: healMetricNamespace, + Subsystem: timeSubsystem, + Name: lastActivityTime, + Help: "Time elapsed (in nano seconds) since last self healing activity", + Type: gaugeMetric, + } +} + +func getNodeOnlineTotalMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: nodesSubsystem, + Name: onlineTotal, + Help: "Total number of MinIO nodes online", + Type: gaugeMetric, + } +} + +func getNodeOfflineTotalMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: nodesSubsystem, + Name: offlineTotal, + Help: "Total number of MinIO nodes offline", + Type: gaugeMetric, + } +} + +func getMinIOVersionMD() MetricDescription { + return MetricDescription{ + Namespace: minioMetricNamespace, + Subsystem: softwareSubsystem, + Name: versionInfo, + Help: "MinIO Release tag for the server", + Type: gaugeMetric, + } +} + +func getMinIOCommitMD() MetricDescription { + return MetricDescription{ + Namespace: minioMetricNamespace, + Subsystem: softwareSubsystem, + Name: commitInfo, + Help: "Git commit hash for the MinIO release", + Type: gaugeMetric, + } +} + +func getS3TTFBDistributionMD() MetricDescription { + return MetricDescription{ + Namespace: s3MetricNamespace, + Subsystem: ttfbSubsystem, + Name: ttfbDistribution, + Help: "Distribution of time to first byte across API calls", + Type: gaugeMetric, + } +} + +func getBucketTTFBDistributionMD() MetricDescription { + return MetricDescription{ + Namespace: bucketMetricNamespace, + Subsystem: ttfbSubsystem, + Name: ttfbDistribution, + Help: "Distribution of time to first byte across API calls per bucket", + Type: gaugeMetric, + } +} + +func getMinioFDOpenMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: fileDescriptorSubsystem, + Name: openTotal, + Help: "Total number of open file descriptors by the MinIO Server process", + Type: gaugeMetric, + } +} + +func getMinioFDLimitMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: fileDescriptorSubsystem, + Name: limitTotal, + Help: "Limit on total number of open file descriptors for the MinIO Server process", + Type: gaugeMetric, + } +} + +func getMinioProcessIOWriteBytesMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: ioSubsystem, + Name: writeBytes, + Help: "Total bytes written by the process to the underlying storage system, /proc/[pid]/io write_bytes", + Type: counterMetric, + } +} + +func getMinioProcessIOReadBytesMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: ioSubsystem, + Name: readBytes, + Help: "Total bytes read by the process from the underlying storage system, /proc/[pid]/io read_bytes", + Type: counterMetric, + } +} + +func getMinioProcessIOWriteCachedBytesMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: ioSubsystem, + Name: wcharBytes, + Help: "Total bytes written by the process to the underlying storage system including page cache, /proc/[pid]/io wchar", + Type: counterMetric, + } +} + +func getMinioProcessIOReadCachedBytesMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: ioSubsystem, + Name: rcharBytes, + Help: "Total bytes read by the process from the underlying storage system including cache, /proc/[pid]/io rchar", + Type: counterMetric, + } +} + +func getMinIOProcessSysCallRMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: sysCallSubsystem, + Name: readTotal, + Help: "Total read SysCalls to the kernel. /proc/[pid]/io syscr", + Type: counterMetric, + } +} + +func getMinIOProcessSysCallWMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: sysCallSubsystem, + Name: writeTotal, + Help: "Total write SysCalls to the kernel. /proc/[pid]/io syscw", + Type: counterMetric, + } +} + +func getMinIOGORoutineCountMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: goRoutines, + Name: total, + Help: "Total number of go routines running", + Type: gaugeMetric, + } +} + +func getMinIOProcessStartTimeMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: processSubsystem, + Name: startTime, + Help: "Start time for MinIO process per node, time in seconds since Unix epoc", + Type: gaugeMetric, + } +} + +func getMinIOProcessUptimeMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: processSubsystem, + Name: upTime, + Help: "Uptime for MinIO process per node in seconds", + Type: gaugeMetric, + } +} + +func getMinIOProcessResidentMemory() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: processSubsystem, + Name: memory, + Help: "Resident memory size in bytes", + Type: gaugeMetric, + } +} + +func getMinIOProcessVirtualMemory() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: processSubsystem, + Name: memory, + Help: "Virtual memory size in bytes", + Type: gaugeMetric, + } +} + +func getMinIOProcessCPUTime() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: processSubsystem, + Name: cpu, + Help: "Total user and system CPU time spent in seconds", + Type: counterMetric, + } +} + +func getMinioProcMetrics() *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 10 * time.Second, + } + mg.RegisterRead(func(ctx context.Context) (metrics []MetricV2) { + if runtime.GOOS == globalWindowsOSName || runtime.GOOS == globalMacOSName { + return nil + } + + p, err := procfs.Self() + if err != nil { + internalLogOnceIf(ctx, err, string(nodeMetricNamespace)) + return metrics + } + + openFDs, _ := p.FileDescriptorsLen() + l, _ := p.Limits() + io, _ := p.IO() + stat, _ := p.Stat() + startTime, _ := stat.StartTime() + + metrics = make([]MetricV2, 0, 20) + + if openFDs > 0 { + metrics = append(metrics, + MetricV2{ + Description: getMinioFDOpenMD(), + Value: float64(openFDs), + }, + ) + } + + if l.OpenFiles > 0 { + metrics = append(metrics, + MetricV2{ + Description: getMinioFDLimitMD(), + Value: float64(l.OpenFiles), + }) + } + + if io.SyscR > 0 { + metrics = append(metrics, + MetricV2{ + Description: getMinIOProcessSysCallRMD(), + Value: float64(io.SyscR), + }) + } + + if io.SyscW > 0 { + metrics = append(metrics, + MetricV2{ + Description: getMinIOProcessSysCallWMD(), + Value: float64(io.SyscW), + }) + } + + if io.ReadBytes > 0 { + metrics = append(metrics, + MetricV2{ + Description: getMinioProcessIOReadBytesMD(), + Value: float64(io.ReadBytes), + }) + } + + if io.WriteBytes > 0 { + metrics = append(metrics, + MetricV2{ + Description: getMinioProcessIOWriteBytesMD(), + Value: float64(io.WriteBytes), + }) + } + + if io.RChar > 0 { + metrics = append(metrics, + MetricV2{ + Description: getMinioProcessIOReadCachedBytesMD(), + Value: float64(io.RChar), + }) + } + + if io.WChar > 0 { + metrics = append(metrics, + MetricV2{ + Description: getMinioProcessIOWriteCachedBytesMD(), + Value: float64(io.WChar), + }) + } + + if startTime > 0 { + metrics = append(metrics, + MetricV2{ + Description: getMinIOProcessStartTimeMD(), + Value: startTime, + }) + } + + if !globalBootTime.IsZero() { + metrics = append(metrics, + MetricV2{ + Description: getMinIOProcessUptimeMD(), + Value: time.Since(globalBootTime).Seconds(), + }) + } + + if stat.ResidentMemory() > 0 { + metrics = append(metrics, + MetricV2{ + Description: getMinIOProcessResidentMemory(), + Value: float64(stat.ResidentMemory()), + }) + } + + if stat.VirtualMemory() > 0 { + metrics = append(metrics, + MetricV2{ + Description: getMinIOProcessVirtualMemory(), + Value: float64(stat.VirtualMemory()), + }) + } + + if stat.CPUTime() > 0 { + metrics = append(metrics, + MetricV2{ + Description: getMinIOProcessCPUTime(), + Value: stat.CPUTime(), + }) + } + return metrics + }) + return mg +} + +func getGoMetrics() *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 10 * time.Second, + } + mg.RegisterRead(func(ctx context.Context) (metrics []MetricV2) { + metrics = append(metrics, MetricV2{ + Description: getMinIOGORoutineCountMD(), + Value: float64(runtime.NumGoroutine()), + }) + return metrics + }) + return mg +} + +// getHistogramMetrics fetches histogram metrics and returns it in a []Metric +// Note: Typically used in MetricGroup.RegisterRead +// +// The toLowerAPILabels parameter is added for compatibility, +// if set, it lowercases the `api` label values. +func getHistogramMetrics(hist *prometheus.HistogramVec, desc MetricDescription, toLowerAPILabels, limitBuckets bool) []MetricV2 { + ch := make(chan prometheus.Metric) + go func() { + defer xioutil.SafeClose(ch) + // Collects prometheus metrics from hist and sends it over ch + hist.Collect(ch) + }() + + // Converts metrics received into internal []Metric type + var metrics []MetricV2 + buckets := make(map[string][]MetricV2, v2MetricsMaxBuckets) + for promMetric := range ch { + dtoMetric := &dto.Metric{} + err := promMetric.Write(dtoMetric) + if err != nil { + // Log error and continue to receive other metric + // values + bugLogIf(GlobalContext, err) + continue + } + + h := dtoMetric.GetHistogram() + for _, b := range h.Bucket { + labels := make(map[string]string) + for _, lp := range dtoMetric.GetLabel() { + if *lp.Name == "api" && toLowerAPILabels { + labels[*lp.Name] = strings.ToLower(*lp.Value) + } else { + labels[*lp.Name] = *lp.Value + } + } + labels["le"] = fmt.Sprintf("%.3f", *b.UpperBound) + metric := MetricV2{ + Description: desc, + VariableLabels: labels, + Value: float64(b.GetCumulativeCount()), + } + if limitBuckets && labels["bucket"] != "" { + buckets[labels["bucket"]] = append(buckets[labels["bucket"]], metric) + } else { + metrics = append(metrics, metric) + } + } + // add metrics with +Inf label + labels1 := make(map[string]string) + for _, lp := range dtoMetric.GetLabel() { + if *lp.Name == "api" && toLowerAPILabels { + labels1[*lp.Name] = strings.ToLower(*lp.Value) + } else { + labels1[*lp.Name] = *lp.Value + } + } + labels1["le"] = fmt.Sprintf("%.3f", math.Inf(+1)) + + metric := MetricV2{ + Description: desc, + VariableLabels: labels1, + Value: float64(dtoMetric.Histogram.GetSampleCount()), + } + if limitBuckets && labels1["bucket"] != "" { + buckets[labels1["bucket"]] = append(buckets[labels1["bucket"]], metric) + } else { + metrics = append(metrics, metric) + } + } + + // Limit bucket metrics... + if limitBuckets { + bucketNames := mapKeysSorted(buckets) + bucketNames = bucketNames[:min(len(buckets), v2MetricsMaxBuckets)] + for _, b := range bucketNames { + metrics = append(metrics, buckets[b]...) + } + } + return metrics +} + +func getBucketTTFBMetric() *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 10 * time.Second, + } + mg.RegisterRead(func(ctx context.Context) []MetricV2 { + return getHistogramMetrics(bucketHTTPRequestsDuration, + getBucketTTFBDistributionMD(), true, true) + }) + return mg +} + +func getS3TTFBMetric() *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 10 * time.Second, + } + mg.RegisterRead(func(ctx context.Context) []MetricV2 { + return getHistogramMetrics(httpRequestsDuration, + getS3TTFBDistributionMD(), true, true) + }) + return mg +} + +func getTierMetrics() *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 10 * time.Second, + } + mg.RegisterRead(func(ctx context.Context) []MetricV2 { + return globalTierMetrics.Report() + }) + return mg +} + +func getTransitionPendingTasksMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: ilmSubsystem, + Name: transitionPendingTasks, + Help: "Number of pending ILM transition tasks in the queue", + Type: gaugeMetric, + } +} + +func getTransitionActiveTasksMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: ilmSubsystem, + Name: transitionActiveTasks, + Help: "Number of active ILM transition tasks", + Type: gaugeMetric, + } +} + +func getTransitionMissedTasksMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: ilmSubsystem, + Name: transitionMissedTasks, + Help: "Number of missed immediate ILM transition tasks", + Type: gaugeMetric, + } +} + +func getExpiryPendingTasksMD() MetricDescription { + return MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: ilmSubsystem, + Name: expiryPendingTasks, + Help: "Number of pending ILM expiry tasks in the queue", + Type: gaugeMetric, + } +} + +func getBucketS3RequestsInFlightMD() MetricDescription { + return MetricDescription{ + Namespace: bucketMetricNamespace, + Subsystem: requestsSubsystem, + Name: inflightTotal, + Help: "Total number of S3 requests currently in flight on a bucket", + Type: gaugeMetric, + } +} + +func getBucketS3RequestsTotalMD() MetricDescription { + return MetricDescription{ + Namespace: bucketMetricNamespace, + Subsystem: requestsSubsystem, + Name: total, + Help: "Total number of S3 requests on a bucket", + Type: counterMetric, + } +} + +func getBucketS3Requests4xxErrorsMD() MetricDescription { + return MetricDescription{ + Namespace: bucketMetricNamespace, + Subsystem: requestsSubsystem, + Name: "4xx_" + errorsTotal, + Help: "Total number of S3 requests with (4xx) errors on a bucket", + Type: counterMetric, + } +} + +func getBucketS3Requests5xxErrorsMD() MetricDescription { + return MetricDescription{ + Namespace: bucketMetricNamespace, + Subsystem: requestsSubsystem, + Name: "5xx_" + errorsTotal, + Help: "Total number of S3 requests with (5xx) errors on a bucket", + Type: counterMetric, + } +} + +func getBucketS3RequestsCanceledMD() MetricDescription { + return MetricDescription{ + Namespace: bucketMetricNamespace, + Subsystem: requestsSubsystem, + Name: canceledTotal, + Help: "Total number of S3 requests that were canceled from the client while processing on a bucket", + Type: counterMetric, + } +} + +func getILMNodeMetrics() *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 10 * time.Second, + } + mg.RegisterRead(func(_ context.Context) []MetricV2 { + expPendingTasks := MetricV2{ + Description: getExpiryPendingTasksMD(), + } + expMissedTasks := MetricV2{ + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: ilmSubsystem, + Name: expiryMissedTasks, + Help: "Number of object version expiry missed due to busy system", + Type: counterMetric, + }, + } + expMissedFreeVersions := MetricV2{ + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: ilmSubsystem, + Name: expiryMissedFreeVersions, + Help: "Number of free versions expiry missed due to busy system", + Type: counterMetric, + }, + } + expMissedTierJournalTasks := MetricV2{ + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: ilmSubsystem, + Name: expiryMissedTierJournalTasks, + Help: "Number of tier journal entries cleanup missed due to busy system", + Type: counterMetric, + }, + } + expNumWorkers := MetricV2{ + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: ilmSubsystem, + Name: expiryNumWorkers, + Help: "Number of workers expiring object versions currently", + Type: gaugeMetric, + }, + } + trPendingTasks := MetricV2{ + Description: getTransitionPendingTasksMD(), + } + trActiveTasks := MetricV2{ + Description: getTransitionActiveTasksMD(), + } + trMissedTasks := MetricV2{ + Description: getTransitionMissedTasksMD(), + } + if globalExpiryState != nil { + expPendingTasks.Value = float64(globalExpiryState.PendingTasks()) + expMissedTasks.Value = float64(globalExpiryState.stats.MissedTasks()) + expMissedFreeVersions.Value = float64(globalExpiryState.stats.MissedFreeVersTasks()) + expMissedTierJournalTasks.Value = float64(globalExpiryState.stats.MissedTierJournalTasks()) + expNumWorkers.Value = float64(globalExpiryState.stats.NumWorkers()) + } + if globalTransitionState != nil { + trPendingTasks.Value = float64(globalTransitionState.PendingTasks()) + trActiveTasks.Value = float64(globalTransitionState.ActiveTasks()) + trMissedTasks.Value = float64(globalTransitionState.MissedImmediateTasks()) + } + return []MetricV2{ + expPendingTasks, + expMissedTasks, + expMissedFreeVersions, + expMissedTierJournalTasks, + expNumWorkers, + trPendingTasks, + trActiveTasks, + trMissedTasks, + } + }) + return mg +} + +func getScannerNodeMetrics() *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 10 * time.Second, + } + mg.RegisterRead(func(_ context.Context) []MetricV2 { + metrics := []MetricV2{ + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: scannerSubsystem, + Name: "objects_scanned", + Help: "Total number of unique objects scanned since server uptime", + Type: counterMetric, + }, + Value: float64(globalScannerMetrics.lifetime(scannerMetricScanObject)), + }, + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: scannerSubsystem, + Name: "versions_scanned", + Help: "Total number of object versions scanned since server uptime", + Type: counterMetric, + }, + Value: float64(globalScannerMetrics.lifetime(scannerMetricApplyVersion)), + }, + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: scannerSubsystem, + Name: "directories_scanned", + Help: "Total number of directories scanned since server uptime", + Type: counterMetric, + }, + Value: float64(globalScannerMetrics.lifetime(scannerMetricScanFolder)), + }, + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: scannerSubsystem, + Name: "bucket_scans_started", + Help: "Total number of bucket scans started since server uptime", + Type: counterMetric, + }, + Value: float64(globalScannerMetrics.lifetime(scannerMetricScanBucketDrive) + uint64(globalScannerMetrics.activeDrives())), + }, + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: scannerSubsystem, + Name: "bucket_scans_finished", + Help: "Total number of bucket scans finished since server uptime", + Type: counterMetric, + }, + Value: float64(globalScannerMetrics.lifetime(scannerMetricScanBucketDrive)), + }, + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: ilmSubsystem, + Name: "versions_scanned", + Help: "Total number of object versions checked for ilm actions since server uptime", + Type: counterMetric, + }, + Value: float64(globalScannerMetrics.lifetime(scannerMetricILM)), + }, + } + for i := range globalScannerMetrics.actions { + action := lifecycle.Action(i) + v := globalScannerMetrics.lifetimeActions(action) + if v == 0 { + continue + } + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: ilmSubsystem, + Name: MetricName("action_count_" + toSnake(action.String())), + Help: "Total action outcome of lifecycle checks since server uptime", + Type: counterMetric, + }, + Value: float64(v), + }) + } + return metrics + }) + return mg +} + +func getIAMNodeMetrics(opts MetricsGroupOpts) *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 10 * time.Second, + metricsGroupOpts: opts, + } + mg.RegisterRead(func(_ context.Context) (metrics []MetricV2) { + lastSyncTime := atomic.LoadUint64(&globalIAMSys.LastRefreshTimeUnixNano) + var sinceLastSyncMillis uint64 + if lastSyncTime != 0 { + sinceLastSyncMillis = (uint64(time.Now().UnixNano()) - lastSyncTime) / uint64(time.Millisecond) + } + + pluginAuthNMetrics := globalAuthNPlugin.Metrics() + metrics = []MetricV2{ + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: iamSubsystem, + Name: "last_sync_duration_millis", + Help: "Last successful IAM data sync duration in milliseconds", + Type: gaugeMetric, + }, + Value: float64(atomic.LoadUint64(&globalIAMSys.LastRefreshDurationMilliseconds)), + }, + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: iamSubsystem, + Name: "since_last_sync_millis", + Help: "Time (in milliseconds) since last successful IAM data sync", + Type: gaugeMetric, + }, + Value: float64(sinceLastSyncMillis), + }, + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: iamSubsystem, + Name: "sync_successes", + Help: "Number of successful IAM data syncs since server uptime", + Type: counterMetric, + }, + Value: float64(atomic.LoadUint64(&globalIAMSys.TotalRefreshSuccesses)), + }, + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: iamSubsystem, + Name: "sync_failures", + Help: "Number of failed IAM data syncs since server uptime", + Type: counterMetric, + }, + Value: float64(atomic.LoadUint64(&globalIAMSys.TotalRefreshFailures)), + }, + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: iamSubsystem, + Name: "plugin_authn_service_last_succ_seconds", + Help: "When plugin authentication is configured, returns time (in seconds) since the last successful request to the service", + Type: gaugeMetric, + }, + Value: pluginAuthNMetrics.LastReachableSecs, + }, + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: iamSubsystem, + Name: "plugin_authn_service_last_fail_seconds", + Help: "When plugin authentication is configured, returns time (in seconds) since the last failed request to the service", + Type: gaugeMetric, + }, + Value: pluginAuthNMetrics.LastUnreachableSecs, + }, + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: iamSubsystem, + Name: "plugin_authn_service_total_requests_minute", + Help: "When plugin authentication is configured, returns total requests count in the last full minute", + Type: gaugeMetric, + }, + Value: float64(pluginAuthNMetrics.TotalRequests), + }, + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: iamSubsystem, + Name: "plugin_authn_service_failed_requests_minute", + Help: "When plugin authentication is configured, returns failed requests count in the last full minute", + Type: gaugeMetric, + }, + Value: float64(pluginAuthNMetrics.FailedRequests), + }, + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: iamSubsystem, + Name: "plugin_authn_service_succ_avg_rtt_ms_minute", + Help: "When plugin authentication is configured, returns average round-trip-time of successful requests in the last full minute", + Type: gaugeMetric, + }, + Value: pluginAuthNMetrics.AvgSuccRTTMs, + }, + { + Description: MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: iamSubsystem, + Name: "plugin_authn_service_succ_max_rtt_ms_minute", + Help: "When plugin authentication is configured, returns maximum round-trip-time of successful requests in the last full minute", + Type: gaugeMetric, + }, + Value: pluginAuthNMetrics.MaxSuccRTTMs, + }, + } + + return metrics + }) + return mg +} + +// replication metrics for each node - published to the cluster endpoint with nodename as label +func getReplicationNodeMetrics(opts MetricsGroupOpts) *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 1 * time.Minute, + metricsGroupOpts: opts, + } + const ( + Online = 1 + Offline = 0 + ) + + mg.RegisterRead(func(_ context.Context) []MetricV2 { + var ml []MetricV2 + // common operational metrics for bucket replication and site replication - published + // at cluster level + if rStats := globalReplicationStats.Load(); rStats != nil { + qs := rStats.getNodeQueueStatsSummary() + activeWorkersCount := MetricV2{ + Description: getClusterReplActiveWorkersCountMD(), + } + avgActiveWorkersCount := MetricV2{ + Description: getClusterReplAvgActiveWorkersCountMD(), + } + maxActiveWorkersCount := MetricV2{ + Description: getClusterReplMaxActiveWorkersCountMD(), + } + currInQueueCount := MetricV2{ + Description: getClusterReplCurrQueuedOperationsMD(), + } + currInQueueBytes := MetricV2{ + Description: getClusterReplCurrQueuedBytesMD(), + } + + currTransferRate := MetricV2{ + Description: getClusterReplCurrentTransferRateMD(), + } + avgQueueCount := MetricV2{ + Description: getClusterReplAvgQueuedOperationsMD(), + } + avgQueueBytes := MetricV2{ + Description: getClusterReplAvgQueuedBytesMD(), + } + maxQueueCount := MetricV2{ + Description: getClusterReplMaxQueuedOperationsMD(), + } + maxQueueBytes := MetricV2{ + Description: getClusterReplMaxQueuedBytesMD(), + } + avgTransferRate := MetricV2{ + Description: getClusterReplAvgTransferRateMD(), + } + maxTransferRate := MetricV2{ + Description: getClusterReplMaxTransferRateMD(), + } + mrfCount := MetricV2{ + Description: getClusterReplMRFFailedOperationsMD(), + Value: float64(qs.MRFStats.LastFailedCount), + } + + if qs.QStats.Avg.Count > 0 || qs.QStats.Curr.Count > 0 { + qt := qs.QStats + currInQueueBytes.Value = qt.Curr.Bytes + currInQueueCount.Value = qt.Curr.Count + avgQueueBytes.Value = qt.Avg.Bytes + avgQueueCount.Value = qt.Avg.Count + maxQueueBytes.Value = qt.Max.Bytes + maxQueueCount.Value = qt.Max.Count + } + activeWorkersCount.Value = float64(qs.ActiveWorkers.Curr) + avgActiveWorkersCount.Value = float64(qs.ActiveWorkers.Avg) + maxActiveWorkersCount.Value = float64(qs.ActiveWorkers.Max) + + if len(qs.XferStats) > 0 { + tots := qs.XferStats[Total] + currTransferRate.Value = tots.Curr + avgTransferRate.Value = tots.Avg + maxTransferRate.Value = tots.Peak + } + ml = []MetricV2{ + activeWorkersCount, + avgActiveWorkersCount, + maxActiveWorkersCount, + currInQueueCount, + currInQueueBytes, + avgQueueCount, + avgQueueBytes, + maxQueueCount, + maxQueueBytes, + currTransferRate, + avgTransferRate, + maxTransferRate, + mrfCount, + } + } + for ep, health := range globalBucketTargetSys.healthStats() { + // link latency current + m := MetricV2{ + Description: getClusterRepLinkLatencyCurrMD(), + VariableLabels: map[string]string{ + "endpoint": ep, + }, + } + m.Value = float64(health.latency.curr / time.Millisecond) + ml = append(ml, m) + + // link latency average + m = MetricV2{ + Description: getClusterRepLinkLatencyAvgMD(), + VariableLabels: map[string]string{ + "endpoint": ep, + }, + } + m.Value = float64(health.latency.avg / time.Millisecond) + ml = append(ml, m) + + // link latency max + m = MetricV2{ + Description: getClusterRepLinkLatencyMaxMD(), + VariableLabels: map[string]string{ + "endpoint": ep, + }, + } + m.Value = float64(health.latency.peak / time.Millisecond) + ml = append(ml, m) + + linkOnline := MetricV2{ + Description: getClusterRepLinkOnlineMD(), + VariableLabels: map[string]string{ + "endpoint": ep, + }, + } + online := Offline + if health.Online { + online = Online + } + linkOnline.Value = float64(online) + ml = append(ml, linkOnline) + offlineDuration := MetricV2{ + Description: getClusterRepLinkCurrOfflineDurationMD(), + VariableLabels: map[string]string{ + "endpoint": ep, + }, + } + currDowntime := time.Duration(0) + if !health.Online && !health.lastOnline.IsZero() { + currDowntime = UTCNow().Sub(health.lastOnline) + } + offlineDuration.Value = float64(currDowntime / time.Second) + ml = append(ml, offlineDuration) + + downtimeDuration := MetricV2{ + Description: getClusterRepLinkTotalOfflineDurationMD(), + VariableLabels: map[string]string{ + "endpoint": ep, + }, + } + dwntime := max(health.offlineDuration, currDowntime) + downtimeDuration.Value = float64(dwntime / time.Second) + ml = append(ml, downtimeDuration) + } + return ml + }) + return mg +} + +// replication metrics for site replication +func getReplicationSiteMetrics(opts MetricsGroupOpts) *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 1 * time.Minute, + metricsGroupOpts: opts, + } + mg.RegisterRead(func(_ context.Context) []MetricV2 { + ml := []MetricV2{} + + // metrics pertinent to site replication - overall roll up. + if globalSiteReplicationSys.isEnabled() { + m, err := globalSiteReplicationSys.getSiteMetrics(GlobalContext) + if err != nil { + metricsLogIf(GlobalContext, err) + return ml + } + ml = append(ml, MetricV2{ + Description: getRepReceivedBytesMD(clusterMetricNamespace), + Value: float64(m.ReplicaSize), + }) + ml = append(ml, MetricV2{ + Description: getRepReceivedOperationsMD(clusterMetricNamespace), + Value: float64(m.ReplicaCount), + }) + + for _, stat := range m.Metrics { + ml = append(ml, MetricV2{ + Description: getRepFailedBytesLastMinuteMD(clusterMetricNamespace), + Value: float64(stat.Failed.LastMinute.Bytes), + VariableLabels: map[string]string{"endpoint": stat.Endpoint}, + }) + ml = append(ml, MetricV2{ + Description: getRepFailedOperationsLastMinuteMD(clusterMetricNamespace), + Value: stat.Failed.LastMinute.Count, + VariableLabels: map[string]string{"endpoint": stat.Endpoint}, + }) + ml = append(ml, MetricV2{ + Description: getRepFailedBytesLastHourMD(clusterMetricNamespace), + Value: float64(stat.Failed.LastHour.Bytes), + VariableLabels: map[string]string{"endpoint": stat.Endpoint}, + }) + ml = append(ml, MetricV2{ + Description: getRepFailedOperationsLastHourMD(clusterMetricNamespace), + Value: stat.Failed.LastHour.Count, + VariableLabels: map[string]string{"endpoint": stat.Endpoint}, + }) + ml = append(ml, MetricV2{ + Description: getRepFailedBytesTotalMD(clusterMetricNamespace), + Value: float64(stat.Failed.Totals.Bytes), + VariableLabels: map[string]string{"endpoint": stat.Endpoint}, + }) + ml = append(ml, MetricV2{ + Description: getRepFailedOperationsTotalMD(clusterMetricNamespace), + Value: stat.Failed.Totals.Count, + VariableLabels: map[string]string{"endpoint": stat.Endpoint}, + }) + + ml = append(ml, MetricV2{ + Description: getRepSentBytesMD(clusterMetricNamespace), + Value: float64(stat.ReplicatedSize), + VariableLabels: map[string]string{"endpoint": stat.Endpoint}, + }) + ml = append(ml, MetricV2{ + Description: getRepSentOperationsMD(clusterMetricNamespace), + Value: float64(stat.ReplicatedCount), + VariableLabels: map[string]string{"endpoint": stat.Endpoint}, + }) + + if c, ok := stat.Failed.ErrCounts["AccessDenied"]; ok { + ml = append(ml, MetricV2{ + Description: getClusterRepCredentialErrorsMD(clusterMetricNamespace), + Value: float64(c), + VariableLabels: map[string]string{"endpoint": stat.Endpoint}, + }) + } + } + ml = append(ml, MetricV2{ + Description: getClusterReplProxiedGetOperationsMD(clusterMetricNamespace), + Value: float64(m.Proxied.GetTotal), + }) + ml = append(ml, MetricV2{ + Description: getClusterReplProxiedHeadOperationsMD(clusterMetricNamespace), + Value: float64(m.Proxied.HeadTotal), + }) + ml = append(ml, MetricV2{ + Description: getClusterReplProxiedPutTaggingOperationsMD(clusterMetricNamespace), + Value: float64(m.Proxied.PutTagTotal), + }) + ml = append(ml, MetricV2{ + Description: getClusterReplProxiedGetTaggingOperationsMD(clusterMetricNamespace), + Value: float64(m.Proxied.GetTagTotal), + }) + ml = append(ml, MetricV2{ + Description: getClusterReplProxiedRmvTaggingOperationsMD(clusterMetricNamespace), + Value: float64(m.Proxied.RmvTagTotal), + }) + ml = append(ml, MetricV2{ + Description: getClusterReplProxiedGetFailedOperationsMD(clusterMetricNamespace), + Value: float64(m.Proxied.GetFailedTotal), + }) + ml = append(ml, MetricV2{ + Description: getClusterReplProxiedHeadFailedOperationsMD(clusterMetricNamespace), + Value: float64(m.Proxied.HeadFailedTotal), + }) + ml = append(ml, MetricV2{ + Description: getClusterReplProxiedPutTaggingFailedOperationsMD(clusterMetricNamespace), + Value: float64(m.Proxied.PutTagFailedTotal), + }) + ml = append(ml, MetricV2{ + Description: getClusterReplProxiedGetTaggingFailedOperationsMD(clusterMetricNamespace), + Value: float64(m.Proxied.GetTagFailedTotal), + }) + ml = append(ml, MetricV2{ + Description: getClusterReplProxiedRmvTaggingFailedOperationsMD(clusterMetricNamespace), + Value: float64(m.Proxied.RmvTagFailedTotal), + }) + } + + return ml + }) + return mg +} + +func getMinioVersionMetrics() *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 10 * time.Second, + } + mg.RegisterRead(func(_ context.Context) (metrics []MetricV2) { + metrics = append(metrics, MetricV2{ + Description: getMinIOCommitMD(), + VariableLabels: map[string]string{"commit": CommitID}, + }) + metrics = append(metrics, MetricV2{ + Description: getMinIOVersionMD(), + VariableLabels: map[string]string{"version": Version}, + }) + return metrics + }) + return mg +} + +func getNodeHealthMetrics(opts MetricsGroupOpts) *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 1 * time.Minute, + metricsGroupOpts: opts, + } + mg.RegisterRead(func(_ context.Context) (metrics []MetricV2) { + metrics = make([]MetricV2, 0, 16) + nodesUp, nodesDown := globalNotificationSys.GetPeerOnlineCount() + metrics = append(metrics, MetricV2{ + Description: getNodeOnlineTotalMD(), + Value: float64(nodesUp), + }) + metrics = append(metrics, MetricV2{ + Description: getNodeOfflineTotalMD(), + Value: float64(nodesDown), + }) + return metrics + }) + return mg +} + +func getMinioHealingMetrics(opts MetricsGroupOpts) *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 10 * time.Second, + metricsGroupOpts: opts, + } + mg.RegisterRead(func(_ context.Context) (metrics []MetricV2) { + bgSeq, exists := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID) + if !exists { + return metrics + } + + if bgSeq.lastHealActivity.IsZero() { + return metrics + } + + metrics = make([]MetricV2, 0, 5) + metrics = append(metrics, MetricV2{ + Description: getHealLastActivityTimeMD(), + Value: float64(time.Since(bgSeq.lastHealActivity)), + }) + metrics = append(metrics, getObjectsScanned(bgSeq)...) + metrics = append(metrics, getHealedItems(bgSeq)...) + metrics = append(metrics, getFailedItems(bgSeq)...) + return metrics + }) + return mg +} + +func getFailedItems(seq *healSequence) (m []MetricV2) { + items := seq.getHealFailedItemsMap() + m = make([]MetricV2, 0, len(items)) + for k, v := range items { + m = append(m, MetricV2{ + Description: getHealObjectsFailTotalMD(), + VariableLabels: map[string]string{"type": string(k)}, + Value: float64(v), + }) + } + return m +} + +func getHealedItems(seq *healSequence) (m []MetricV2) { + items := seq.getHealedItemsMap() + m = make([]MetricV2, 0, len(items)) + for k, v := range items { + m = append(m, MetricV2{ + Description: getHealObjectsHealTotalMD(), + VariableLabels: map[string]string{"type": string(k)}, + Value: float64(v), + }) + } + return m +} + +func getObjectsScanned(seq *healSequence) (m []MetricV2) { + items := seq.getScannedItemsMap() + m = make([]MetricV2, 0, len(items)) + for k, v := range items { + m = append(m, MetricV2{ + Description: getHealObjectsTotalMD(), + VariableLabels: map[string]string{"type": string(k)}, + Value: float64(v), + }) + } + return m +} + +func getDistLockMetrics(opts MetricsGroupOpts) *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 1 * time.Second, + metricsGroupOpts: opts, + } + mg.RegisterRead(func(ctx context.Context) []MetricV2 { + if !globalIsDistErasure { + return []MetricV2{} + } + + st := globalLockServer.stats() + + metrics := make([]MetricV2, 0, 3) + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: minioNamespace, + Subsystem: "locks", + Name: "total", + Help: "Number of current locks on this peer", + Type: gaugeMetric, + }, + Value: float64(st.Total), + }) + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: minioNamespace, + Subsystem: "locks", + Name: "write_total", + Help: "Number of current WRITE locks on this peer", + Type: gaugeMetric, + }, + Value: float64(st.Writes), + }) + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: minioNamespace, + Subsystem: "locks", + Name: "read_total", + Help: "Number of current READ locks on this peer", + Type: gaugeMetric, + }, + Value: float64(st.Reads), + }) + return metrics + }) + return mg +} + +func getNotificationMetrics(opts MetricsGroupOpts) *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 10 * time.Second, + metricsGroupOpts: opts, + } + mg.RegisterRead(func(ctx context.Context) []MetricV2 { + metrics := make([]MetricV2, 0, 3) + + if globalEventNotifier != nil { + nstats := globalEventNotifier.targetList.Stats() + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: minioNamespace, + Subsystem: notifySubsystem, + Name: "current_send_in_progress", + Help: "Number of concurrent async Send calls active to all targets (deprecated, please use 'minio_notify_target_current_send_in_progress' instead)", + Type: gaugeMetric, + }, + Value: float64(nstats.CurrentSendCalls), + }) + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: minioNamespace, + Subsystem: notifySubsystem, + Name: "events_skipped_total", + Help: "Events that were skipped to be sent to the targets due to the in-memory queue being full", + Type: counterMetric, + }, + Value: float64(nstats.EventsSkipped), + }) + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: minioNamespace, + Subsystem: notifySubsystem, + Name: "events_errors_total", + Help: "Events that were failed to be sent to the targets (deprecated, please use 'minio_notify_target_failed_events' instead)", + Type: counterMetric, + }, + Value: float64(nstats.EventsErrorsTotal), + }) + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: minioNamespace, + Subsystem: notifySubsystem, + Name: "events_sent_total", + Help: "Total number of events sent to the targets (deprecated, please use 'minio_notify_target_total_events' instead)", + Type: counterMetric, + }, + Value: float64(nstats.TotalEvents), + }) + for id, st := range nstats.TargetStats { + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: minioNamespace, + Subsystem: notifySubsystem, + Name: "target_total_events", + Help: "Total number of events sent (or) queued to the target", + Type: counterMetric, + }, + VariableLabels: map[string]string{"target_id": id.ID, "target_name": id.Name}, + Value: float64(st.TotalEvents), + }) + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: minioNamespace, + Subsystem: notifySubsystem, + Name: "target_failed_events", + Help: "Number of events failed to be sent (or) queued to the target", + Type: counterMetric, + }, + VariableLabels: map[string]string{"target_id": id.ID, "target_name": id.Name}, + Value: float64(st.FailedEvents), + }) + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: minioNamespace, + Subsystem: notifySubsystem, + Name: "target_current_send_in_progress", + Help: "Number of concurrent async Send calls active to the target", + Type: gaugeMetric, + }, + VariableLabels: map[string]string{"target_id": id.ID, "target_name": id.Name}, + Value: float64(st.CurrentSendCalls), + }) + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: minioNamespace, + Subsystem: notifySubsystem, + Name: "target_queue_length", + Help: "Number of events currently staged in the queue_dir configured for the target", + Type: gaugeMetric, + }, + VariableLabels: map[string]string{"target_id": id.ID, "target_name": id.Name}, + Value: float64(st.CurrentQueue), + }) + } + } + + lstats := globalLambdaTargetList.Stats() + for _, st := range lstats.TargetStats { + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: minioNamespace, + Subsystem: lambdaSubsystem, + Name: "active_requests", + Help: "Number of in progress requests", + }, + VariableLabels: map[string]string{"target_id": st.ID.ID, "target_name": st.ID.Name}, + Value: float64(st.ActiveRequests), + }) + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: minioNamespace, + Subsystem: lambdaSubsystem, + Name: "total_requests", + Help: "Total number of requests sent since start", + Type: counterMetric, + }, + VariableLabels: map[string]string{"target_id": st.ID.ID, "target_name": st.ID.Name}, + Value: float64(st.TotalRequests), + }) + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: minioNamespace, + Subsystem: lambdaSubsystem, + Name: "failed_requests", + Help: "Total number of requests that failed to send since start", + Type: counterMetric, + }, + VariableLabels: map[string]string{"target_id": st.ID.ID, "target_name": st.ID.Name}, + Value: float64(st.FailedRequests), + }) + } + + // Audit and system: + audit := logger.CurrentStats() + for id, st := range audit { + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: minioNamespace, + Subsystem: auditSubsystem, + Name: "target_queue_length", + Help: "Number of unsent messages in queue for target", + Type: gaugeMetric, + }, + VariableLabels: map[string]string{"target_id": id}, + Value: float64(st.QueueLength), + }) + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: minioNamespace, + Subsystem: auditSubsystem, + Name: "total_messages", + Help: "Total number of messages sent since start", + Type: counterMetric, + }, + VariableLabels: map[string]string{"target_id": id}, + Value: float64(st.TotalMessages), + }) + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: minioNamespace, + Subsystem: auditSubsystem, + Name: "failed_messages", + Help: "Total number of messages that failed to send since start", + Type: counterMetric, + }, + VariableLabels: map[string]string{"target_id": id}, + Value: float64(st.FailedMessages), + }) + } + return metrics + }) + return mg +} + +func getHTTPMetrics(opts MetricsGroupOpts) *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 10 * time.Second, + metricsGroupOpts: opts, + } + mg.RegisterRead(func(ctx context.Context) (metrics []MetricV2) { + if !mg.metricsGroupOpts.bucketOnly { + httpStats := globalHTTPStats.toServerHTTPStats(true) + metrics = make([]MetricV2, 0, 3+ + len(httpStats.CurrentS3Requests.APIStats)+ + len(httpStats.TotalS3Requests.APIStats)+ + len(httpStats.TotalS3Errors.APIStats)+ + len(httpStats.TotalS35xxErrors.APIStats)+ + len(httpStats.TotalS34xxErrors.APIStats)) + metrics = append(metrics, MetricV2{ + Description: getS3RejectedAuthRequestsTotalMD(), + Value: float64(httpStats.TotalS3RejectedAuth), + }) + metrics = append(metrics, MetricV2{ + Description: getS3RejectedTimestampRequestsTotalMD(), + Value: float64(httpStats.TotalS3RejectedTime), + }) + metrics = append(metrics, MetricV2{ + Description: getS3RejectedHeaderRequestsTotalMD(), + Value: float64(httpStats.TotalS3RejectedHeader), + }) + metrics = append(metrics, MetricV2{ + Description: getS3RejectedInvalidRequestsTotalMD(), + Value: float64(httpStats.TotalS3RejectedInvalid), + }) + metrics = append(metrics, MetricV2{ + Description: getS3RequestsInQueueMD(), + Value: float64(httpStats.S3RequestsInQueue), + }) + metrics = append(metrics, MetricV2{ + Description: getIncomingS3RequestsMD(), + Value: float64(httpStats.S3RequestsIncoming), + }) + + for api, value := range httpStats.CurrentS3Requests.APIStats { + metrics = append(metrics, MetricV2{ + Description: getS3RequestsInFlightMD(), + Value: float64(value), + VariableLabels: map[string]string{"api": api}, + }) + } + for api, value := range httpStats.TotalS3Requests.APIStats { + metrics = append(metrics, MetricV2{ + Description: getS3RequestsTotalMD(), + Value: float64(value), + VariableLabels: map[string]string{"api": api}, + }) + } + for api, value := range httpStats.TotalS3Errors.APIStats { + metrics = append(metrics, MetricV2{ + Description: getS3RequestsErrorsMD(), + Value: float64(value), + VariableLabels: map[string]string{"api": api}, + }) + } + for api, value := range httpStats.TotalS35xxErrors.APIStats { + metrics = append(metrics, MetricV2{ + Description: getS3Requests5xxErrorsMD(), + Value: float64(value), + VariableLabels: map[string]string{"api": api}, + }) + } + for api, value := range httpStats.TotalS34xxErrors.APIStats { + metrics = append(metrics, MetricV2{ + Description: getS3Requests4xxErrorsMD(), + Value: float64(value), + VariableLabels: map[string]string{"api": api}, + }) + } + for api, value := range httpStats.TotalS3Canceled.APIStats { + metrics = append(metrics, MetricV2{ + Description: getS3RequestsCanceledMD(), + Value: float64(value), + VariableLabels: map[string]string{"api": api}, + }) + } + return metrics + } + + // If we have too many, limit them + bConnStats := globalBucketConnStats.getS3InOutBytes() + buckets := mapKeysSorted(bConnStats) + buckets = buckets[:min(v2MetricsMaxBuckets, len(buckets))] + + for _, bucket := range buckets { + inOut := bConnStats[bucket] + recvBytes := inOut.In + if recvBytes > 0 { + metrics = append(metrics, MetricV2{ + Description: getBucketTrafficReceivedBytes(), + Value: float64(recvBytes), + VariableLabels: map[string]string{"bucket": bucket}, + }) + } + sentBytes := inOut.Out + if sentBytes > 0 { + metrics = append(metrics, MetricV2{ + Description: getBucketTrafficSentBytes(), + Value: float64(sentBytes), + VariableLabels: map[string]string{"bucket": bucket}, + }) + } + + httpStats := globalBucketHTTPStats.load(bucket) + for k, v := range httpStats.currentS3Requests.Load(true) { + metrics = append(metrics, MetricV2{ + Description: getBucketS3RequestsInFlightMD(), + Value: float64(v), + VariableLabels: map[string]string{"bucket": bucket, "api": k}, + }) + } + + for k, v := range httpStats.totalS3Requests.Load(true) { + metrics = append(metrics, MetricV2{ + Description: getBucketS3RequestsTotalMD(), + Value: float64(v), + VariableLabels: map[string]string{"bucket": bucket, "api": k}, + }) + } + + for k, v := range httpStats.totalS3Canceled.Load(true) { + metrics = append(metrics, MetricV2{ + Description: getBucketS3RequestsCanceledMD(), + Value: float64(v), + VariableLabels: map[string]string{"bucket": bucket, "api": k}, + }) + } + + for k, v := range httpStats.totalS34xxErrors.Load(true) { + metrics = append(metrics, MetricV2{ + Description: getBucketS3Requests4xxErrorsMD(), + Value: float64(v), + VariableLabels: map[string]string{"bucket": bucket, "api": k}, + }) + } + + for k, v := range httpStats.totalS35xxErrors.Load(true) { + metrics = append(metrics, MetricV2{ + Description: getBucketS3Requests5xxErrorsMD(), + Value: float64(v), + VariableLabels: map[string]string{"bucket": bucket, "api": k}, + }) + } + } + + return metrics + }) + return mg +} + +func getNetworkMetrics() *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 10 * time.Second, + } + mg.RegisterRead(func(ctx context.Context) (metrics []MetricV2) { + metrics = make([]MetricV2, 0, 10) + connStats := globalConnStats.toServerConnStats() + rpcStats := rest.GetRPCStats() + if globalIsDistErasure { + metrics = append(metrics, MetricV2{ + Description: getInternodeFailedRequests(), + Value: float64(rpcStats.Errs), + }) + metrics = append(metrics, MetricV2{ + Description: getInternodeTCPDialTimeout(), + Value: float64(rpcStats.DialErrs), + }) + metrics = append(metrics, MetricV2{ + Description: getInternodeTCPAvgDuration(), + Value: float64(rpcStats.DialAvgDuration), + }) + metrics = append(metrics, MetricV2{ + Description: getInterNodeSentBytesMD(), + Value: float64(connStats.internodeOutputBytes), + }) + metrics = append(metrics, MetricV2{ + Description: getInterNodeReceivedBytesMD(), + Value: float64(connStats.internodeInputBytes), + }) + } + metrics = append(metrics, MetricV2{ + Description: getS3SentBytesMD(), + Value: float64(connStats.s3OutputBytes), + }) + metrics = append(metrics, MetricV2{ + Description: getS3ReceivedBytesMD(), + Value: float64(connStats.s3InputBytes), + }) + return metrics + }) + return mg +} + +func getClusterUsageMetrics(opts MetricsGroupOpts) *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 1 * time.Minute, + metricsGroupOpts: opts, + } + mg.RegisterRead(func(ctx context.Context) (metrics []MetricV2) { + objLayer := newObjectLayerFn() + if objLayer == nil { + return metrics + } + + metrics = make([]MetricV2, 0, 50) + dataUsageInfo, err := loadDataUsageFromBackend(ctx, objLayer) + if err != nil { + metricsLogIf(ctx, err) + return metrics + } + + // data usage has not captured any data yet. + if dataUsageInfo.LastUpdate.IsZero() { + return metrics + } + + metrics = append(metrics, MetricV2{ + Description: getUsageLastScanActivityMD(), + Value: float64(time.Since(dataUsageInfo.LastUpdate)), + }) + + var ( + clusterSize uint64 + clusterBuckets uint64 + clusterObjectsCount uint64 + clusterVersionsCount uint64 + clusterDeleteMarkersCount uint64 + ) + + clusterObjectSizesHistogram := map[string]uint64{} + clusterVersionsHistogram := map[string]uint64{} + for _, usage := range dataUsageInfo.BucketsUsage { + clusterBuckets++ + clusterSize += usage.Size + clusterObjectsCount += usage.ObjectsCount + clusterVersionsCount += usage.VersionsCount + clusterDeleteMarkersCount += usage.DeleteMarkersCount + for k, v := range usage.ObjectSizesHistogram { + v1, ok := clusterObjectSizesHistogram[k] + if !ok { + clusterObjectSizesHistogram[k] = v + } else { + v1 += v + clusterObjectSizesHistogram[k] = v1 + } + } + for k, v := range usage.ObjectVersionsHistogram { + v1, ok := clusterVersionsHistogram[k] + if !ok { + clusterVersionsHistogram[k] = v + } else { + v1 += v + clusterVersionsHistogram[k] = v1 + } + } + } + + metrics = append(metrics, MetricV2{ + Description: getClusterUsageTotalBytesMD(), + Value: float64(clusterSize), + }) + + metrics = append(metrics, MetricV2{ + Description: getClusterUsageObjectsTotalMD(), + Value: float64(clusterObjectsCount), + }) + + metrics = append(metrics, MetricV2{ + Description: getClusterUsageVersionsTotalMD(), + Value: float64(clusterVersionsCount), + }) + + metrics = append(metrics, MetricV2{ + Description: getClusterUsageDeleteMarkersTotalMD(), + Value: float64(clusterDeleteMarkersCount), + }) + + metrics = append(metrics, MetricV2{ + Description: getClusterObjectDistributionMD(), + Histogram: clusterObjectSizesHistogram, + HistogramBucketLabel: "range", + }) + + metrics = append(metrics, MetricV2{ + Description: getClusterObjectVersionsMD(), + Histogram: clusterVersionsHistogram, + HistogramBucketLabel: "range", + }) + + metrics = append(metrics, MetricV2{ + Description: getClusterBucketsTotalMD(), + Value: float64(clusterBuckets), + }) + + return metrics + }) + return mg +} + +func getBucketUsageMetrics(opts MetricsGroupOpts) *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 1 * time.Minute, + metricsGroupOpts: opts, + } + mg.RegisterRead(func(ctx context.Context) (metrics []MetricV2) { + objLayer := newObjectLayerFn() + + metrics = make([]MetricV2, 0, 50) + dataUsageInfo, err := loadDataUsageFromBackend(ctx, objLayer) + if err != nil { + metricsLogIf(ctx, err) + return metrics + } + + // data usage has not captured any data yet. + if dataUsageInfo.LastUpdate.IsZero() { + return metrics + } + + metrics = append(metrics, MetricV2{ + Description: getBucketUsageLastScanActivityMD(), + Value: float64(time.Since(dataUsageInfo.LastUpdate)), + }) + + var bucketReplStats map[string]BucketStats + if !globalSiteReplicationSys.isEnabled() { + bucketReplStats = globalReplicationStats.Load().getAllLatest(dataUsageInfo.BucketsUsage) + } + buckets := mapKeysSorted(dataUsageInfo.BucketsUsage) + if len(buckets) > v2MetricsMaxBuckets { + buckets = buckets[:v2MetricsMaxBuckets] + } + for _, bucket := range buckets { + usage := dataUsageInfo.BucketsUsage[bucket] + quota, _ := globalBucketQuotaSys.Get(ctx, bucket) + + metrics = append(metrics, MetricV2{ + Description: getBucketUsageTotalBytesMD(), + Value: float64(usage.Size), + VariableLabels: map[string]string{"bucket": bucket}, + }) + + metrics = append(metrics, MetricV2{ + Description: getBucketUsageObjectsTotalMD(), + Value: float64(usage.ObjectsCount), + VariableLabels: map[string]string{"bucket": bucket}, + }) + + metrics = append(metrics, MetricV2{ + Description: getBucketUsageVersionsTotalMD(), + Value: float64(usage.VersionsCount), + VariableLabels: map[string]string{"bucket": bucket}, + }) + + metrics = append(metrics, MetricV2{ + Description: getBucketUsageDeleteMarkersTotalMD(), + Value: float64(usage.DeleteMarkersCount), + VariableLabels: map[string]string{"bucket": bucket}, + }) + + if quota != nil && quota.Quota > 0 { + metrics = append(metrics, MetricV2{ + Description: getBucketUsageQuotaTotalBytesMD(), + Value: float64(quota.Quota), + VariableLabels: map[string]string{"bucket": bucket}, + }) + } + if !globalSiteReplicationSys.isEnabled() { + var stats BucketReplicationStats + s, ok := bucketReplStats[bucket] + if ok { + stats = s.ReplicationStats + metrics = append(metrics, MetricV2{ + Description: getRepReceivedBytesMD(bucketMetricNamespace), + Value: float64(stats.ReplicaSize), + VariableLabels: map[string]string{"bucket": bucket}, + }) + metrics = append(metrics, MetricV2{ + Description: getRepReceivedOperationsMD(bucketMetricNamespace), + Value: float64(stats.ReplicaCount), + VariableLabels: map[string]string{"bucket": bucket}, + }) + metrics = append(metrics, MetricV2{ + Description: getClusterReplProxiedGetOperationsMD(bucketMetricNamespace), + Value: float64(s.ProxyStats.GetTotal), + VariableLabels: map[string]string{"bucket": bucket}, + }) + metrics = append(metrics, MetricV2{ + Description: getClusterReplProxiedHeadOperationsMD(bucketMetricNamespace), + Value: float64(s.ProxyStats.HeadTotal), + VariableLabels: map[string]string{"bucket": bucket}, + }) + metrics = append(metrics, MetricV2{ + Description: getClusterReplProxiedPutTaggingOperationsMD(bucketMetricNamespace), + Value: float64(s.ProxyStats.PutTagTotal), + VariableLabels: map[string]string{"bucket": bucket}, + }) + metrics = append(metrics, MetricV2{ + Description: getClusterReplProxiedGetTaggingOperationsMD(bucketMetricNamespace), + Value: float64(s.ProxyStats.GetTagTotal), + VariableLabels: map[string]string{"bucket": bucket}, + }) + metrics = append(metrics, MetricV2{ + Description: getClusterReplProxiedRmvTaggingOperationsMD(bucketMetricNamespace), + Value: float64(s.ProxyStats.RmvTagTotal), + VariableLabels: map[string]string{"bucket": bucket}, + }) + metrics = append(metrics, MetricV2{ + Description: getClusterReplProxiedGetFailedOperationsMD(bucketMetricNamespace), + Value: float64(s.ProxyStats.GetFailedTotal), + }) + metrics = append(metrics, MetricV2{ + Description: getClusterReplProxiedHeadFailedOperationsMD(bucketMetricNamespace), + Value: float64(s.ProxyStats.HeadFailedTotal), + }) + metrics = append(metrics, MetricV2{ + Description: getClusterReplProxiedPutTaggingFailedOperationsMD(bucketMetricNamespace), + Value: float64(s.ProxyStats.PutTagFailedTotal), + }) + metrics = append(metrics, MetricV2{ + Description: getClusterReplProxiedGetTaggingFailedOperationsMD(bucketMetricNamespace), + Value: float64(s.ProxyStats.GetTagFailedTotal), + }) + metrics = append(metrics, MetricV2{ + Description: getClusterReplProxiedRmvTaggingFailedOperationsMD(bucketMetricNamespace), + Value: float64(s.ProxyStats.RmvTagFailedTotal), + }) + } + if stats.hasReplicationUsage() { + for arn, stat := range stats.Stats { + metrics = append(metrics, MetricV2{ + Description: getRepFailedBytesLastMinuteMD(bucketMetricNamespace), + Value: float64(stat.Failed.LastMinute.Bytes), + VariableLabels: map[string]string{"bucket": bucket, "targetArn": arn}, + }) + metrics = append(metrics, MetricV2{ + Description: getRepFailedOperationsLastMinuteMD(bucketMetricNamespace), + Value: stat.Failed.LastMinute.Count, + VariableLabels: map[string]string{"bucket": bucket, "targetArn": arn}, + }) + metrics = append(metrics, MetricV2{ + Description: getRepFailedBytesLastHourMD(bucketMetricNamespace), + Value: float64(stat.Failed.LastHour.Bytes), + VariableLabels: map[string]string{"bucket": bucket, "targetArn": arn}, + }) + metrics = append(metrics, MetricV2{ + Description: getRepFailedOperationsLastHourMD(bucketMetricNamespace), + Value: stat.Failed.LastHour.Count, + VariableLabels: map[string]string{"bucket": bucket, "targetArn": arn}, + }) + metrics = append(metrics, MetricV2{ + Description: getRepFailedBytesTotalMD(bucketMetricNamespace), + Value: float64(stat.Failed.Totals.Bytes), + VariableLabels: map[string]string{"bucket": bucket, "targetArn": arn}, + }) + metrics = append(metrics, MetricV2{ + Description: getRepFailedOperationsTotalMD(bucketMetricNamespace), + Value: stat.Failed.Totals.Count, + VariableLabels: map[string]string{"bucket": bucket, "targetArn": arn}, + }) + metrics = append(metrics, MetricV2{ + Description: getRepSentBytesMD(bucketMetricNamespace), + Value: float64(stat.ReplicatedSize), + VariableLabels: map[string]string{"bucket": bucket, "targetArn": arn}, + }) + metrics = append(metrics, MetricV2{ + Description: getRepSentOperationsMD(bucketMetricNamespace), + Value: float64(stat.ReplicatedCount), + VariableLabels: map[string]string{"bucket": bucket, "targetArn": arn}, + }) + metrics = append(metrics, MetricV2{ + Description: getBucketRepLatencyMD(), + HistogramBucketLabel: "range", + Histogram: stat.Latency.getUploadLatency(), + VariableLabels: map[string]string{"bucket": bucket, "operation": "upload", "targetArn": arn}, + }) + if c, ok := stat.Failed.ErrCounts["AccessDenied"]; ok { + metrics = append(metrics, MetricV2{ + Description: getClusterRepCredentialErrorsMD(bucketMetricNamespace), + Value: float64(c), + VariableLabels: map[string]string{"bucket": bucket, "targetArn": arn}, + }) + } + } + } + } + metrics = append(metrics, MetricV2{ + Description: getBucketObjectDistributionMD(), + Histogram: usage.ObjectSizesHistogram, + HistogramBucketLabel: "range", + VariableLabels: map[string]string{"bucket": bucket}, + }) + + metrics = append(metrics, MetricV2{ + Description: getBucketObjectVersionsMD(), + Histogram: usage.ObjectVersionsHistogram, + HistogramBucketLabel: "range", + VariableLabels: map[string]string{"bucket": bucket}, + }) + } + return metrics + }) + return mg +} + +func getClusterTransitionedBytesMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: ilmSubsystem, + Name: transitionedBytes, + Help: "Total bytes transitioned to a tier", + Type: gaugeMetric, + } +} + +func getClusterTransitionedObjectsMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: ilmSubsystem, + Name: transitionedObjects, + Help: "Total number of objects transitioned to a tier", + Type: gaugeMetric, + } +} + +func getClusterTransitionedVersionsMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: ilmSubsystem, + Name: transitionedVersions, + Help: "Total number of versions transitioned to a tier", + Type: gaugeMetric, + } +} + +func getClusterTierMetrics(opts MetricsGroupOpts) *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 1 * time.Minute, + metricsGroupOpts: opts, + } + mg.RegisterRead(func(ctx context.Context) (metrics []MetricV2) { + objLayer := newObjectLayerFn() + + if globalTierConfigMgr.Empty() { + return metrics + } + + dui, err := loadDataUsageFromBackend(ctx, objLayer) + if err != nil { + metricsLogIf(ctx, err) + return metrics + } + // data usage has not captured any tier stats yet. + if dui.TierStats == nil { + return metrics + } + + return dui.tierMetrics() + }) + return mg +} + +func getLocalStorageMetrics(opts MetricsGroupOpts) *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 1 * time.Minute, + metricsGroupOpts: opts, + } + mg.RegisterRead(func(ctx context.Context) (metrics []MetricV2) { + objLayer := newObjectLayerFn() + + metrics = make([]MetricV2, 0, 50) + storageInfo := objLayer.LocalStorageInfo(ctx, true) + onlineDrives, offlineDrives := getOnlineOfflineDisksStats(storageInfo.Disks) + totalDrives := onlineDrives.Merge(offlineDrives) + + for _, disk := range storageInfo.Disks { + metrics = append(metrics, MetricV2{ + Description: getNodeDriveUsedBytesMD(), + Value: float64(disk.UsedSpace), + VariableLabels: map[string]string{"drive": disk.DrivePath}, + }) + + metrics = append(metrics, MetricV2{ + Description: getNodeDriveFreeBytesMD(), + Value: float64(disk.AvailableSpace), + VariableLabels: map[string]string{"drive": disk.DrivePath}, + }) + + metrics = append(metrics, MetricV2{ + Description: getNodeDriveTotalBytesMD(), + Value: float64(disk.TotalSpace), + VariableLabels: map[string]string{"drive": disk.DrivePath}, + }) + + metrics = append(metrics, MetricV2{ + Description: getNodeDrivesFreeInodesMD(), + Value: float64(disk.FreeInodes), + VariableLabels: map[string]string{"drive": disk.DrivePath}, + }) + + if disk.Metrics != nil { + metrics = append(metrics, MetricV2{ + Description: getNodeDriveTimeoutErrorsMD(), + Value: float64(disk.Metrics.TotalErrorsTimeout), + VariableLabels: map[string]string{"drive": disk.DrivePath}, + }) + + metrics = append(metrics, MetricV2{ + Description: getNodeDriveIOErrorsMD(), + Value: float64(disk.Metrics.TotalErrorsAvailability - disk.Metrics.TotalErrorsTimeout), + VariableLabels: map[string]string{"drive": disk.DrivePath}, + }) + + metrics = append(metrics, MetricV2{ + Description: getNodeDriveAvailabilityErrorsMD(), + Value: float64(disk.Metrics.TotalErrorsAvailability), + VariableLabels: map[string]string{"drive": disk.DrivePath}, + }) + + metrics = append(metrics, MetricV2{ + Description: getNodeDriveWaitingIOMD(), + Value: float64(disk.Metrics.TotalWaiting), + VariableLabels: map[string]string{"drive": disk.DrivePath}, + }) + + for apiName, latency := range disk.Metrics.LastMinute { + metrics = append(metrics, MetricV2{ + Description: getNodeDriveAPILatencyMD(), + Value: float64(latency.Avg().Microseconds()), + VariableLabels: map[string]string{"drive": disk.DrivePath, "api": "storage." + apiName}, + }) + } + } + } + + metrics = append(metrics, MetricV2{ + Description: getNodeDrivesOfflineTotalMD(), + Value: float64(offlineDrives.Sum()), + }) + + metrics = append(metrics, MetricV2{ + Description: getNodeDrivesOnlineTotalMD(), + Value: float64(onlineDrives.Sum()), + }) + + metrics = append(metrics, MetricV2{ + Description: getNodeDrivesTotalMD(), + Value: float64(totalDrives.Sum()), + }) + + metrics = append(metrics, MetricV2{ + Description: getNodeStandardParityMD(), + Value: float64(storageInfo.Backend.StandardSCParity), + }) + + metrics = append(metrics, MetricV2{ + Description: getNodeRRSParityMD(), + Value: float64(storageInfo.Backend.RRSCParity), + }) + + return metrics + }) + return mg +} + +func getClusterWriteQuorumMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: "write", + Name: "quorum", + Help: "Maximum write quorum across all pools and sets", + Type: gaugeMetric, + } +} + +func getClusterHealthStatusMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: "health", + Name: "status", + Help: "Get current cluster health status", + Type: gaugeMetric, + } +} + +func getClusterErasureSetHealthStatusMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: "health", + Name: "erasure_set_status", + Help: "Get current health status for this erasure set", + Type: gaugeMetric, + } +} + +func getClusterErasureSetReadQuorumMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: "health", + Name: "erasure_set_read_quorum", + Help: "Get the read quorum for this erasure set", + Type: gaugeMetric, + } +} + +func getClusterErasureSetWriteQuorumMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: "health", + Name: "erasure_set_write_quorum", + Help: "Get the write quorum for this erasure set", + Type: gaugeMetric, + } +} + +func getClusterErasureSetOnlineDrivesMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: "health", + Name: "erasure_set_online_drives", + Help: "Get the count of the online drives in this erasure set", + Type: gaugeMetric, + } +} + +func getClusterErasureSetHealingDrivesMD() MetricDescription { + return MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: "health", + Name: "erasure_set_healing_drives", + Help: "Get the count of healing drives of this erasure set", + Type: gaugeMetric, + } +} + +func getClusterHealthMetrics(opts MetricsGroupOpts) *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 10 * time.Second, + metricsGroupOpts: opts, + } + mg.RegisterRead(func(ctx context.Context) (metrics []MetricV2) { + objLayer := newObjectLayerFn() + + opts := HealthOptions{} + result := objLayer.Health(ctx, opts) + + metrics = make([]MetricV2, 0, 2+4*len(result.ESHealth)) + + metrics = append(metrics, MetricV2{ + Description: getClusterWriteQuorumMD(), + Value: float64(result.WriteQuorum), + }) + + health := 1 + if !result.Healthy { + health = 0 + } + + metrics = append(metrics, MetricV2{ + Description: getClusterHealthStatusMD(), + Value: float64(health), + }) + + for _, h := range result.ESHealth { + labels := map[string]string{ + "pool": strconv.Itoa(h.PoolID), + "set": strconv.Itoa(h.SetID), + } + metrics = append(metrics, MetricV2{ + Description: getClusterErasureSetReadQuorumMD(), + VariableLabels: labels, + Value: float64(h.ReadQuorum), + }) + metrics = append(metrics, MetricV2{ + Description: getClusterErasureSetWriteQuorumMD(), + VariableLabels: labels, + Value: float64(h.WriteQuorum), + }) + metrics = append(metrics, MetricV2{ + Description: getClusterErasureSetOnlineDrivesMD(), + VariableLabels: labels, + Value: float64(h.HealthyDrives), + }) + metrics = append(metrics, MetricV2{ + Description: getClusterErasureSetHealingDrivesMD(), + VariableLabels: labels, + Value: float64(h.HealingDrives), + }) + + health := 1 + if !h.Healthy { + health = 0 + } + + metrics = append(metrics, MetricV2{ + Description: getClusterErasureSetHealthStatusMD(), + VariableLabels: labels, + Value: float64(health), + }) + } + + return metrics + }) + + return mg +} + +func getBatchJobsMetrics(opts MetricsGroupOpts) *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 10 * time.Second, + metricsGroupOpts: opts, + } + + mg.RegisterRead(func(ctx context.Context) (metrics []MetricV2) { + var m madmin.RealtimeMetrics + mLocal := collectLocalMetrics(madmin.MetricsBatchJobs, collectMetricsOpts{}) + m.Merge(&mLocal) + + mRemote := collectRemoteMetrics(ctx, madmin.MetricsBatchJobs, collectMetricsOpts{}) + m.Merge(&mRemote) + + if m.Aggregated.BatchJobs == nil { + return metrics + } + + for _, mj := range m.Aggregated.BatchJobs.Jobs { + jtype := toSnake(mj.JobType) + var objects, objectsFailed float64 + var bucket string + switch madmin.BatchJobType(mj.JobType) { + case madmin.BatchJobReplicate: + objects = float64(mj.Replicate.Objects) + objectsFailed = float64(mj.Replicate.ObjectsFailed) + bucket = mj.Replicate.Bucket + case madmin.BatchJobKeyRotate: + objects = float64(mj.KeyRotate.Objects) + objectsFailed = float64(mj.KeyRotate.ObjectsFailed) + bucket = mj.KeyRotate.Bucket + case madmin.BatchJobExpire: + objects = float64(mj.Expired.Objects) + objectsFailed = float64(mj.Expired.ObjectsFailed) + bucket = mj.Expired.Bucket + } + metrics = append(metrics, + MetricV2{ + Description: MetricDescription{ + Namespace: bucketMetricNamespace, + Subsystem: "batch", + Name: MetricName(jtype + "_objects"), + Help: "Get successfully completed batch job " + jtype + "objects", + Type: counterMetric, + }, + Value: objects, + VariableLabels: map[string]string{"bucket": bucket, "jobId": mj.JobID}, + }, + MetricV2{ + Description: MetricDescription{ + Namespace: bucketMetricNamespace, + Subsystem: "batch", + Name: MetricName(jtype + "_objects_failed"), + Help: "Get failed batch job " + jtype + "objects", + Type: counterMetric, + }, + Value: objectsFailed, + VariableLabels: map[string]string{"bucket": bucket, "jobId": mj.JobID}, + }, + ) + } + return metrics + }) + return mg +} + +func getClusterStorageMetrics(opts MetricsGroupOpts) *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 1 * time.Minute, + metricsGroupOpts: opts, + } + mg.RegisterRead(func(ctx context.Context) (metrics []MetricV2) { + objLayer := newObjectLayerFn() + + // Fetch disk space info, ignore errors + metrics = make([]MetricV2, 0, 10) + storageInfo := objLayer.StorageInfo(ctx, true) + onlineDrives, offlineDrives := getOnlineOfflineDisksStats(storageInfo.Disks) + totalDrives := onlineDrives.Merge(offlineDrives) + + metrics = append(metrics, MetricV2{ + Description: getClusterCapacityTotalBytesMD(), + Value: float64(GetTotalCapacity(storageInfo.Disks)), + }) + + metrics = append(metrics, MetricV2{ + Description: getClusterCapacityFreeBytesMD(), + Value: float64(GetTotalCapacityFree(storageInfo.Disks)), + }) + + metrics = append(metrics, MetricV2{ + Description: getClusterCapacityUsageBytesMD(), + Value: float64(GetTotalUsableCapacity(storageInfo.Disks, storageInfo)), + }) + + metrics = append(metrics, MetricV2{ + Description: getClusterCapacityUsageFreeBytesMD(), + Value: float64(GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo)), + }) + + metrics = append(metrics, MetricV2{ + Description: getClusterDrivesOfflineTotalMD(), + Value: float64(offlineDrives.Sum()), + }) + + metrics = append(metrics, MetricV2{ + Description: getClusterDrivesOnlineTotalMD(), + Value: float64(onlineDrives.Sum()), + }) + + metrics = append(metrics, MetricV2{ + Description: getClusterDrivesTotalMD(), + Value: float64(totalDrives.Sum()), + }) + return metrics + }) + return mg +} + +func getKMSNodeMetrics(opts MetricsGroupOpts) *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 10 * time.Second, + metricsGroupOpts: opts, + } + + mg.RegisterRead(func(ctx context.Context) (metrics []MetricV2) { + const ( + Online = 1 + Offline = 0 + ) + desc := MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: kmsSubsystem, + Name: kmsOnline, + Help: "Reports whether the KMS is online (1) or offline (0)", + Type: gaugeMetric, + } + _, err := GlobalKMS.Metrics(ctx) + if _, ok := kes.IsConnError(err); ok { + return []MetricV2{{ + Description: desc, + Value: float64(Offline), + }} + } + return []MetricV2{{ + Description: desc, + Value: float64(Online), + }} + }) + return mg +} + +func getWebhookMetrics() *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 10 * time.Second, + } + mg.RegisterRead(func(ctx context.Context) []MetricV2 { + tgts := append(logger.SystemTargets(), logger.AuditTargets()...) + metrics := make([]MetricV2, 0, len(tgts)*4) + for _, t := range tgts { + isOnline := 0 + if t.IsOnline(ctx) { + isOnline = 1 + } + labels := map[string]string{ + "name": t.String(), + "endpoint": t.Endpoint(), + } + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: webhookSubsystem, + Name: webhookOnline, + Help: "Is the webhook online?", + Type: gaugeMetric, + }, + VariableLabels: labels, + Value: float64(isOnline), + }) + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: webhookSubsystem, + Name: webhookQueueLength, + Help: "Webhook queue length", + Type: gaugeMetric, + }, + VariableLabels: labels, + Value: float64(t.Stats().QueueLength), + }) + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: webhookSubsystem, + Name: webhookTotalMessages, + Help: "Total number of messages sent to this target", + Type: counterMetric, + }, + VariableLabels: labels, + Value: float64(t.Stats().TotalMessages), + }) + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: webhookSubsystem, + Name: webhookFailedMessages, + Help: "Number of messages that failed to send", + Type: counterMetric, + }, + VariableLabels: labels, + Value: float64(t.Stats().FailedMessages), + }) + } + + return metrics + }) + return mg +} + +func getKMSMetrics(opts MetricsGroupOpts) *MetricsGroupV2 { + mg := &MetricsGroupV2{ + cacheInterval: 10 * time.Second, + metricsGroupOpts: opts, + } + + mg.RegisterRead(func(ctx context.Context) []MetricV2 { + metrics := make([]MetricV2, 0, 4) + metric, err := GlobalKMS.Metrics(ctx) + if err != nil { + return metrics + } + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: kmsSubsystem, + Name: kmsRequestsSuccess, + Help: "Number of KMS requests that succeeded", + Type: counterMetric, + }, + Value: float64(metric.ReqOK), + }) + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: kmsSubsystem, + Name: kmsRequestsError, + Help: "Number of KMS requests that failed due to some error. (HTTP 4xx status code)", + Type: counterMetric, + }, + Value: float64(metric.ReqErr), + }) + metrics = append(metrics, MetricV2{ + Description: MetricDescription{ + Namespace: clusterMetricNamespace, + Subsystem: kmsSubsystem, + Name: kmsRequestsFail, + Help: "Number of KMS requests that failed due to some internal failure. (HTTP 5xx status code)", + Type: counterMetric, + }, + Value: float64(metric.ReqFail), + }) + return metrics + }) + return mg +} + +func collectMetric(metric MetricV2, labels []string, values []string, metricName string, out chan<- prometheus.Metric) { + if metric.Description.Type == histogramMetric { + if metric.Histogram == nil { + return + } + for k, v := range metric.Histogram { + pmetric, err := prometheus.NewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(string(metric.Description.Namespace), + string(metric.Description.Subsystem), + string(metric.Description.Name)), + metric.Description.Help, + append(labels, metric.HistogramBucketLabel), + metric.StaticLabels, + ), + prometheus.GaugeValue, + float64(v), + append(values, k)...) + if err != nil { + // Enable for debugging + if serverDebugLog { + bugLogIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v+%v", err, values, metric.Histogram)) + } + } else { + out <- pmetric + } + } + return + } + metricType := prometheus.GaugeValue + if metric.Description.Type == counterMetric { + metricType = prometheus.CounterValue + } + pmetric, err := prometheus.NewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(string(metric.Description.Namespace), + string(metric.Description.Subsystem), + string(metric.Description.Name)), + metric.Description.Help, + labels, + metric.StaticLabels, + ), + metricType, + metric.Value, + values...) + if err != nil { + // Enable for debugging + if serverDebugLog { + bugLogIf(GlobalContext, fmt.Errorf("unable to validate prometheus metric (%w) %v", err, values)) + } + } else { + out <- pmetric + } +} + +//msgp:ignore minioBucketCollector +type minioBucketCollector struct { + metricsGroups []*MetricsGroupV2 + desc *prometheus.Desc +} + +func newMinioBucketCollector(metricsGroups []*MetricsGroupV2) *minioBucketCollector { + return &minioBucketCollector{ + metricsGroups: metricsGroups, + desc: prometheus.NewDesc("minio_bucket_stats", "Statistics exposed by MinIO server cluster wide per bucket", nil, nil), + } +} + +// Describe sends the super-set of all possible descriptors of metrics +func (c *minioBucketCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.desc +} + +// Collect is called by the Prometheus registry when collecting metrics. +func (c *minioBucketCollector) Collect(out chan<- prometheus.Metric) { + var wg sync.WaitGroup + publish := func(in <-chan MetricV2) { + defer wg.Done() + for metric := range in { + labels, values := getOrderedLabelValueArrays(metric.VariableLabels) + collectMetric(metric, labels, values, "bucket", out) + } + } + + // Call peer api to fetch metrics + wg.Add(2) + go publish(ReportMetrics(GlobalContext, c.metricsGroups)) + go publish(globalNotificationSys.GetBucketMetrics(GlobalContext)) + wg.Wait() +} + +//msgp:ignore minioClusterCollector +type minioClusterCollector struct { + metricsGroups []*MetricsGroupV2 + desc *prometheus.Desc +} + +func newMinioClusterCollector(metricsGroups []*MetricsGroupV2) *minioClusterCollector { + return &minioClusterCollector{ + metricsGroups: metricsGroups, + desc: prometheus.NewDesc("minio_stats", "Statistics exposed by MinIO server per cluster", nil, nil), + } +} + +// Describe sends the super-set of all possible descriptors of metrics +func (c *minioClusterCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.desc +} + +// Collect is called by the Prometheus registry when collecting metrics. +func (c *minioClusterCollector) Collect(out chan<- prometheus.Metric) { + var wg sync.WaitGroup + publish := func(in <-chan MetricV2) { + defer wg.Done() + for metric := range in { + labels, values := getOrderedLabelValueArrays(metric.VariableLabels) + collectMetric(metric, labels, values, "cluster", out) + } + } + + // Call peer api to fetch metrics + wg.Add(2) + go publish(ReportMetrics(GlobalContext, c.metricsGroups)) + go publish(globalNotificationSys.GetClusterMetrics(GlobalContext)) + wg.Wait() +} + +// ReportMetrics reports serialized metrics to the channel passed for the metrics generated. +func ReportMetrics(ctx context.Context, metricsGroups []*MetricsGroupV2) <-chan MetricV2 { + ch := make(chan MetricV2) + go func() { + defer xioutil.SafeClose(ch) + populateAndPublish(metricsGroups, func(m MetricV2) bool { + if m.VariableLabels == nil { + m.VariableLabels = make(map[string]string) + } + m.VariableLabels[serverName] = globalLocalNodeName + for { + select { + case ch <- m: + return true + case <-ctx.Done(): + return false + } + } + }) + }() + return ch +} + +// minioNodeCollector is the Custom Collector +// +//msgp:ignore minioNodeCollector +type minioNodeCollector struct { + metricsGroups []*MetricsGroupV2 + desc *prometheus.Desc +} + +// Describe sends the super-set of all possible descriptors of metrics +func (c *minioNodeCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.desc +} + +// populateAndPublish populates and then publishes the metrics generated by the generator function. +func populateAndPublish(metricsGroups []*MetricsGroupV2, publish func(m MetricV2) bool) { + for _, mg := range metricsGroups { + if mg == nil { + continue + } + for _, metric := range mg.Get() { + if !publish(metric) { + return + } + } + } +} + +// Collect is called by the Prometheus registry when collecting metrics. +func (c *minioNodeCollector) Collect(ch chan<- prometheus.Metric) { + // Expose MinIO's version information + minioVersionInfo.WithLabelValues(Version, CommitID).Set(1.0) + + populateAndPublish(c.metricsGroups, func(metric MetricV2) bool { + labels, values := getOrderedLabelValueArrays(metric.VariableLabels) + values = append(values, globalLocalNodeName) + labels = append(labels, serverName) + + if metric.Description.Type == histogramMetric { + if metric.Histogram == nil { + return true + } + for k, v := range metric.Histogram { + labels = append(labels, metric.HistogramBucketLabel) + values = append(values, k) + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(string(metric.Description.Namespace), + string(metric.Description.Subsystem), + string(metric.Description.Name)), + metric.Description.Help, + labels, + metric.StaticLabels, + ), + prometheus.GaugeValue, + float64(v), + values...) + } + return true + } + + metricType := prometheus.GaugeValue + if metric.Description.Type == counterMetric { + metricType = prometheus.CounterValue + } + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(string(metric.Description.Namespace), + string(metric.Description.Subsystem), + string(metric.Description.Name)), + metric.Description.Help, + labels, + metric.StaticLabels, + ), + metricType, + metric.Value, + values...) + return true + }) +} + +func getOrderedLabelValueArrays(labelsWithValue map[string]string) (labels, values []string) { + labels = make([]string, 0, len(labelsWithValue)) + values = make([]string, 0, len(labelsWithValue)) + for l, v := range labelsWithValue { + labels = append(labels, l) + values = append(values, v) + } + return labels, values +} + +// newMinioCollectorNode describes the collector +// and returns reference of minioCollector for version 2 +// It creates the Prometheus Description which is used +// to define Metric and help string +func newMinioCollectorNode(metricsGroups []*MetricsGroupV2) *minioNodeCollector { + return &minioNodeCollector{ + metricsGroups: metricsGroups, + desc: prometheus.NewDesc("minio_stats", "Statistics exposed by MinIO server per node", nil, nil), + } +} + +func metricsHTTPHandler(c prometheus.Collector, funcName string) http.Handler { + registry := prometheus.NewRegistry() + + // Report all other metrics + logger.CriticalIf(GlobalContext, registry.Register(c)) + + // DefaultGatherers include golang metrics and process metrics. + gatherers := prometheus.Gatherers{ + registry, + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt) + if ok { + tc.FuncName = funcName + tc.ResponseRecorder.LogErrBody = true + } + + mfs, err := gatherers.Gather() + if err != nil && len(mfs) == 0 { + writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), err), r.URL) + return + } + + contentType := expfmt.Negotiate(r.Header) + w.Header().Set("Content-Type", string(contentType)) + + enc := expfmt.NewEncoder(w, contentType) + for _, mf := range mfs { + if err := enc.Encode(mf); err != nil { + // client may disconnect for any reasons + // we do not have to log this. + return + } + } + if closer, ok := enc.(expfmt.Closer); ok { + closer.Close() + } + }) +} + +func metricsBucketHandler() http.Handler { + return metricsHTTPHandler(bucketCollector, "handler.MetricsBucket") +} + +func metricsServerHandler() http.Handler { + registry := prometheus.NewRegistry() + + // Report all other metrics + logger.CriticalIf(GlobalContext, registry.Register(clusterCollector)) + + // DefaultGatherers include golang metrics and process metrics. + gatherers := prometheus.Gatherers{ + registry, + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt) + if ok { + tc.FuncName = "handler.MetricsCluster" + tc.ResponseRecorder.LogErrBody = true + } + + mfs, err := gatherers.Gather() + if err != nil && len(mfs) == 0 { + writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), err), r.URL) + return + } + + contentType := expfmt.Negotiate(r.Header) + w.Header().Set("Content-Type", string(contentType)) + + enc := expfmt.NewEncoder(w, contentType) + for _, mf := range mfs { + if err := enc.Encode(mf); err != nil { + // client may disconnect for any reasons + // we do not have to log this. + return + } + } + if closer, ok := enc.(expfmt.Closer); ok { + closer.Close() + } + }) +} + +func metricsNodeHandler() http.Handler { + registry := prometheus.NewRegistry() + + logger.CriticalIf(GlobalContext, registry.Register(nodeCollector)) + if err := registry.Register(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{ + Namespace: minioNamespace, + ReportErrors: true, + })); err != nil { + logger.CriticalIf(GlobalContext, err) + } + if err := registry.Register(prometheus.NewGoCollector()); err != nil { + logger.CriticalIf(GlobalContext, err) + } + gatherers := prometheus.Gatherers{ + registry, + } + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt) + if ok { + tc.FuncName = "handler.MetricsNode" + tc.ResponseRecorder.LogErrBody = true + } + + mfs, err := gatherers.Gather() + if err != nil { + if len(mfs) == 0 { + writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), err), r.URL) + return + } + } + + contentType := expfmt.Negotiate(r.Header) + w.Header().Set("Content-Type", string(contentType)) + + enc := expfmt.NewEncoder(w, contentType) + for _, mf := range mfs { + if err := enc.Encode(mf); err != nil { + metricsLogIf(r.Context(), err) + return + } + } + if closer, ok := enc.(expfmt.Closer); ok { + closer.Close() + } + }) +} + +func toSnake(camel string) (snake string) { + var b strings.Builder + l := len(camel) + for i, v := range camel { + // A is 65, a is 97 + if v >= 'a' { + b.WriteRune(v) + continue + } + // v is capital letter here + // disregard first letter + // add underscore if last letter is capital letter + // add underscore when previous letter is lowercase + // add underscore when next letter is lowercase + if (i != 0 || i == l-1) && ((i > 0 && rune(camel[i-1]) >= 'a') || + (i < l-1 && rune(camel[i+1]) >= 'a')) { + b.WriteRune('_') + } + b.WriteRune(v + 'a' - 'A') + } + return b.String() +} diff --git a/cmd/metrics-v2_gen.go b/cmd/metrics-v2_gen.go new file mode 100644 index 00000000..9b13c4b8 --- /dev/null +++ b/cmd/metrics-v2_gen.go @@ -0,0 +1,638 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// MarshalMsg implements msgp.Marshaler +func (z *MetricDescription) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 5 + // string "Namespace" + o = append(o, 0x85, 0xa9, 0x4e, 0x61, 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65) + o = msgp.AppendString(o, string(z.Namespace)) + // string "Subsystem" + o = append(o, 0xa9, 0x53, 0x75, 0x62, 0x73, 0x79, 0x73, 0x74, 0x65, 0x6d) + o = msgp.AppendString(o, string(z.Subsystem)) + // string "Name" + o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65) + o = msgp.AppendString(o, string(z.Name)) + // string "Help" + o = append(o, 0xa4, 0x48, 0x65, 0x6c, 0x70) + o = msgp.AppendString(o, z.Help) + // string "Type" + o = append(o, 0xa4, 0x54, 0x79, 0x70, 0x65) + o = msgp.AppendString(o, string(z.Type)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *MetricDescription) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Namespace": + { + var zb0002 string + zb0002, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Namespace") + return + } + z.Namespace = MetricNamespace(zb0002) + } + case "Subsystem": + { + var zb0003 string + zb0003, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Subsystem") + return + } + z.Subsystem = MetricSubsystem(zb0003) + } + case "Name": + { + var zb0004 string + zb0004, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + z.Name = MetricName(zb0004) + } + case "Help": + z.Help, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Help") + return + } + case "Type": + { + var zb0005 string + zb0005, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + z.Type = MetricTypeV2(zb0005) + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *MetricDescription) Msgsize() (s int) { + s = 1 + 10 + msgp.StringPrefixSize + len(string(z.Namespace)) + 10 + msgp.StringPrefixSize + len(string(z.Subsystem)) + 5 + msgp.StringPrefixSize + len(string(z.Name)) + 5 + msgp.StringPrefixSize + len(z.Help) + 5 + msgp.StringPrefixSize + len(string(z.Type)) + return +} + +// MarshalMsg implements msgp.Marshaler +func (z MetricName) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendString(o, string(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *MetricName) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 string + zb0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = MetricName(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z MetricName) Msgsize() (s int) { + s = msgp.StringPrefixSize + len(string(z)) + return +} + +// MarshalMsg implements msgp.Marshaler +func (z MetricNamespace) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendString(o, string(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *MetricNamespace) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 string + zb0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = MetricNamespace(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z MetricNamespace) Msgsize() (s int) { + s = msgp.StringPrefixSize + len(string(z)) + return +} + +// MarshalMsg implements msgp.Marshaler +func (z MetricSubsystem) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendString(o, string(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *MetricSubsystem) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 string + zb0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = MetricSubsystem(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z MetricSubsystem) Msgsize() (s int) { + s = msgp.StringPrefixSize + len(string(z)) + return +} + +// MarshalMsg implements msgp.Marshaler +func (z MetricTypeV2) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendString(o, string(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *MetricTypeV2) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 string + zb0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = MetricTypeV2(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z MetricTypeV2) Msgsize() (s int) { + s = msgp.StringPrefixSize + len(string(z)) + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *MetricV2) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 6 + // string "Description" + o = append(o, 0x86, 0xab, 0x44, 0x65, 0x73, 0x63, 0x72, 0x69, 0x70, 0x74, 0x69, 0x6f, 0x6e) + o, err = z.Description.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Description") + return + } + // string "StaticLabels" + o = append(o, 0xac, 0x53, 0x74, 0x61, 0x74, 0x69, 0x63, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.StaticLabels))) + for za0001, za0002 := range z.StaticLabels { + o = msgp.AppendString(o, za0001) + o = msgp.AppendString(o, za0002) + } + // string "Value" + o = append(o, 0xa5, 0x56, 0x61, 0x6c, 0x75, 0x65) + o = msgp.AppendFloat64(o, z.Value) + // string "VariableLabels" + o = append(o, 0xae, 0x56, 0x61, 0x72, 0x69, 0x61, 0x62, 0x6c, 0x65, 0x4c, 0x61, 0x62, 0x65, 0x6c, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.VariableLabels))) + for za0003, za0004 := range z.VariableLabels { + o = msgp.AppendString(o, za0003) + o = msgp.AppendString(o, za0004) + } + // string "HistogramBucketLabel" + o = append(o, 0xb4, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x4c, 0x61, 0x62, 0x65, 0x6c) + o = msgp.AppendString(o, z.HistogramBucketLabel) + // string "Histogram" + o = append(o, 0xa9, 0x48, 0x69, 0x73, 0x74, 0x6f, 0x67, 0x72, 0x61, 0x6d) + o = msgp.AppendMapHeader(o, uint32(len(z.Histogram))) + for za0005, za0006 := range z.Histogram { + o = msgp.AppendString(o, za0005) + o = msgp.AppendUint64(o, za0006) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *MetricV2) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Description": + bts, err = z.Description.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Description") + return + } + case "StaticLabels": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "StaticLabels") + return + } + if z.StaticLabels == nil { + z.StaticLabels = make(map[string]string, zb0002) + } else if len(z.StaticLabels) > 0 { + clear(z.StaticLabels) + } + for zb0002 > 0 { + var za0002 string + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "StaticLabels") + return + } + za0002, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "StaticLabels", za0001) + return + } + z.StaticLabels[za0001] = za0002 + } + case "Value": + z.Value, bts, err = msgp.ReadFloat64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Value") + return + } + case "VariableLabels": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "VariableLabels") + return + } + if z.VariableLabels == nil { + z.VariableLabels = make(map[string]string, zb0003) + } else if len(z.VariableLabels) > 0 { + clear(z.VariableLabels) + } + for zb0003 > 0 { + var za0004 string + zb0003-- + var za0003 string + za0003, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "VariableLabels") + return + } + za0004, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "VariableLabels", za0003) + return + } + z.VariableLabels[za0003] = za0004 + } + case "HistogramBucketLabel": + z.HistogramBucketLabel, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "HistogramBucketLabel") + return + } + case "Histogram": + var zb0004 uint32 + zb0004, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Histogram") + return + } + if z.Histogram == nil { + z.Histogram = make(map[string]uint64, zb0004) + } else if len(z.Histogram) > 0 { + clear(z.Histogram) + } + for zb0004 > 0 { + var za0006 uint64 + zb0004-- + var za0005 string + za0005, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Histogram") + return + } + za0006, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Histogram", za0005) + return + } + z.Histogram[za0005] = za0006 + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *MetricV2) Msgsize() (s int) { + s = 1 + 12 + z.Description.Msgsize() + 13 + msgp.MapHeaderSize + if z.StaticLabels != nil { + for za0001, za0002 := range z.StaticLabels { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + msgp.StringPrefixSize + len(za0002) + } + } + s += 6 + msgp.Float64Size + 15 + msgp.MapHeaderSize + if z.VariableLabels != nil { + for za0003, za0004 := range z.VariableLabels { + _ = za0004 + s += msgp.StringPrefixSize + len(za0003) + msgp.StringPrefixSize + len(za0004) + } + } + s += 21 + msgp.StringPrefixSize + len(z.HistogramBucketLabel) + 10 + msgp.MapHeaderSize + if z.Histogram != nil { + for za0005, za0006 := range z.Histogram { + _ = za0006 + s += msgp.StringPrefixSize + len(za0005) + msgp.Uint64Size + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *MetricsGroupOpts) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 12 + // string "dependGlobalObjectAPI" + o = append(o, 0x8c, 0xb5, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x41, 0x50, 0x49) + o = msgp.AppendBool(o, z.dependGlobalObjectAPI) + // string "dependGlobalAuthNPlugin" + o = append(o, 0xb7, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x41, 0x75, 0x74, 0x68, 0x4e, 0x50, 0x6c, 0x75, 0x67, 0x69, 0x6e) + o = msgp.AppendBool(o, z.dependGlobalAuthNPlugin) + // string "dependGlobalSiteReplicationSys" + o = append(o, 0xbe, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x53, 0x69, 0x74, 0x65, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x79, 0x73) + o = msgp.AppendBool(o, z.dependGlobalSiteReplicationSys) + // string "dependGlobalNotificationSys" + o = append(o, 0xbb, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x4e, 0x6f, 0x74, 0x69, 0x66, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x79, 0x73) + o = msgp.AppendBool(o, z.dependGlobalNotificationSys) + // string "dependGlobalKMS" + o = append(o, 0xaf, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x4b, 0x4d, 0x53) + o = msgp.AppendBool(o, z.dependGlobalKMS) + // string "bucketOnly" + o = append(o, 0xaa, 0x62, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x4f, 0x6e, 0x6c, 0x79) + o = msgp.AppendBool(o, z.bucketOnly) + // string "dependGlobalLambdaTargetList" + o = append(o, 0xbc, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x4c, 0x61, 0x6d, 0x62, 0x64, 0x61, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x4c, 0x69, 0x73, 0x74) + o = msgp.AppendBool(o, z.dependGlobalLambdaTargetList) + // string "dependGlobalIAMSys" + o = append(o, 0xb2, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x49, 0x41, 0x4d, 0x53, 0x79, 0x73) + o = msgp.AppendBool(o, z.dependGlobalIAMSys) + // string "dependGlobalLockServer" + o = append(o, 0xb6, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x4c, 0x6f, 0x63, 0x6b, 0x53, 0x65, 0x72, 0x76, 0x65, 0x72) + o = msgp.AppendBool(o, z.dependGlobalLockServer) + // string "dependGlobalIsDistErasure" + o = append(o, 0xb9, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x49, 0x73, 0x44, 0x69, 0x73, 0x74, 0x45, 0x72, 0x61, 0x73, 0x75, 0x72, 0x65) + o = msgp.AppendBool(o, z.dependGlobalIsDistErasure) + // string "dependGlobalBackgroundHealState" + o = append(o, 0xbf, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x47, 0x6c, 0x6f, 0x62, 0x61, 0x6c, 0x42, 0x61, 0x63, 0x6b, 0x67, 0x72, 0x6f, 0x75, 0x6e, 0x64, 0x48, 0x65, 0x61, 0x6c, 0x53, 0x74, 0x61, 0x74, 0x65) + o = msgp.AppendBool(o, z.dependGlobalBackgroundHealState) + // string "dependBucketTargetSys" + o = append(o, 0xb5, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x79, 0x73) + o = msgp.AppendBool(o, z.dependBucketTargetSys) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *MetricsGroupOpts) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "dependGlobalObjectAPI": + z.dependGlobalObjectAPI, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "dependGlobalObjectAPI") + return + } + case "dependGlobalAuthNPlugin": + z.dependGlobalAuthNPlugin, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "dependGlobalAuthNPlugin") + return + } + case "dependGlobalSiteReplicationSys": + z.dependGlobalSiteReplicationSys, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "dependGlobalSiteReplicationSys") + return + } + case "dependGlobalNotificationSys": + z.dependGlobalNotificationSys, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "dependGlobalNotificationSys") + return + } + case "dependGlobalKMS": + z.dependGlobalKMS, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "dependGlobalKMS") + return + } + case "bucketOnly": + z.bucketOnly, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "bucketOnly") + return + } + case "dependGlobalLambdaTargetList": + z.dependGlobalLambdaTargetList, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "dependGlobalLambdaTargetList") + return + } + case "dependGlobalIAMSys": + z.dependGlobalIAMSys, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "dependGlobalIAMSys") + return + } + case "dependGlobalLockServer": + z.dependGlobalLockServer, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "dependGlobalLockServer") + return + } + case "dependGlobalIsDistErasure": + z.dependGlobalIsDistErasure, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "dependGlobalIsDistErasure") + return + } + case "dependGlobalBackgroundHealState": + z.dependGlobalBackgroundHealState, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "dependGlobalBackgroundHealState") + return + } + case "dependBucketTargetSys": + z.dependBucketTargetSys, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "dependBucketTargetSys") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *MetricsGroupOpts) Msgsize() (s int) { + s = 1 + 22 + msgp.BoolSize + 24 + msgp.BoolSize + 31 + msgp.BoolSize + 28 + msgp.BoolSize + 16 + msgp.BoolSize + 11 + msgp.BoolSize + 29 + msgp.BoolSize + 19 + msgp.BoolSize + 23 + msgp.BoolSize + 26 + msgp.BoolSize + 32 + msgp.BoolSize + 22 + msgp.BoolSize + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *MetricsGroupV2) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "cacheInterval" + o = append(o, 0x82, 0xad, 0x63, 0x61, 0x63, 0x68, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x76, 0x61, 0x6c) + o = msgp.AppendDuration(o, z.cacheInterval) + // string "metricsGroupOpts" + o = append(o, 0xb0, 0x6d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73, 0x47, 0x72, 0x6f, 0x75, 0x70, 0x4f, 0x70, 0x74, 0x73) + o, err = z.metricsGroupOpts.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "metricsGroupOpts") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *MetricsGroupV2) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "cacheInterval": + z.cacheInterval, bts, err = msgp.ReadDurationBytes(bts) + if err != nil { + err = msgp.WrapError(err, "cacheInterval") + return + } + case "metricsGroupOpts": + bts, err = z.metricsGroupOpts.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "metricsGroupOpts") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *MetricsGroupV2) Msgsize() (s int) { + s = 1 + 14 + msgp.DurationSize + 17 + z.metricsGroupOpts.Msgsize() + return +} diff --git a/cmd/metrics-v2_gen_test.go b/cmd/metrics-v2_gen_test.go new file mode 100644 index 00000000..026c16d7 --- /dev/null +++ b/cmd/metrics-v2_gen_test.go @@ -0,0 +1,241 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalMetricDescription(t *testing.T) { + v := MetricDescription{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgMetricDescription(b *testing.B) { + v := MetricDescription{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgMetricDescription(b *testing.B) { + v := MetricDescription{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalMetricDescription(b *testing.B) { + v := MetricDescription{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalMetricV2(t *testing.T) { + v := MetricV2{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgMetricV2(b *testing.B) { + v := MetricV2{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgMetricV2(b *testing.B) { + v := MetricV2{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalMetricV2(b *testing.B) { + v := MetricV2{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalMetricsGroupOpts(t *testing.T) { + v := MetricsGroupOpts{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgMetricsGroupOpts(b *testing.B) { + v := MetricsGroupOpts{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgMetricsGroupOpts(b *testing.B) { + v := MetricsGroupOpts{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalMetricsGroupOpts(b *testing.B) { + v := MetricsGroupOpts{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalMetricsGroupV2(t *testing.T) { + v := MetricsGroupV2{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgMetricsGroupV2(b *testing.B) { + v := MetricsGroupV2{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgMetricsGroupV2(b *testing.B) { + v := MetricsGroupV2{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalMetricsGroupV2(b *testing.B) { + v := MetricsGroupV2{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/metrics-v2_test.go b/cmd/metrics-v2_test.go new file mode 100644 index 00000000..a3994fd0 --- /dev/null +++ b/cmd/metrics-v2_test.go @@ -0,0 +1,215 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "slices" + "strings" + "testing" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +func TestGetHistogramMetrics_BucketCount(t *testing.T) { + histBuckets := []float64{0.05, 0.1, 0.25, 0.5, 0.75} + labels := []string{"GetObject", "PutObject", "CopyObject", "CompleteMultipartUpload"} + ttfbHist := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "s3_ttfb_seconds", + Help: "Time taken by requests served by current MinIO server instance", + Buckets: histBuckets, + }, + []string{"api"}, + ) + observations := []struct { + val float64 + label string + }{ + { + val: 0.02, + label: labels[0], + }, + { + val: 0.07, + label: labels[1], + }, + { + val: 0.11, + label: labels[1], + }, + { + val: 0.19, + label: labels[1], + }, + { + val: 0.31, + label: labels[1], + }, + { + val: 0.61, + label: labels[3], + }, + { + val: 0.79, + label: labels[2], + }, + } + ticker := time.NewTicker(1 * time.Millisecond) + defer ticker.Stop() + for _, obs := range observations { + // Send observations once every 1ms, to simulate delay between + // observations. This is to test the channel based + // synchronization used internally. + <-ticker.C + ttfbHist.With(prometheus.Labels{"api": obs.label}).Observe(obs.val) + } + + metrics := getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD(), false, false) + // additional labels for +Inf for all histogram metrics + if expPoints := len(labels) * (len(histBuckets) + 1); expPoints != len(metrics) { + t.Fatalf("Expected %v data points when toLowerAPILabels=false but got %v", expPoints, len(metrics)) + } + + metrics = getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD(), true, false) + // additional labels for +Inf for all histogram metrics + if expPoints := len(labels) * (len(histBuckets) + 1); expPoints != len(metrics) { + t.Fatalf("Expected %v data points when toLowerAPILabels=true but got %v", expPoints, len(metrics)) + } +} + +func TestGetHistogramMetrics_Values(t *testing.T) { + histBuckets := []float64{0.50, 5.00} + labels := []string{"PutObject", "CopyObject"} + ttfbHist := prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "s3_ttfb_seconds", + Help: "Time taken by requests served by current MinIO server instance", + Buckets: histBuckets, + }, + []string{"api"}, + ) + observations := []struct { + val float64 + label string + }{ + { + val: 0.02, + label: labels[0], + }, + { + val: 0.19, + label: labels[1], + }, + { + val: 0.31, + label: labels[1], + }, + { + val: 0.61, + label: labels[0], + }, + { + val: 6.79, + label: labels[1], + }, + } + ticker := time.NewTicker(1 * time.Millisecond) + defer ticker.Stop() + for _, obs := range observations { + // Send observations once every 1ms, to simulate delay between + // observations. This is to test the channel based + // synchronization used internally. + <-ticker.C + ttfbHist.With(prometheus.Labels{"api": obs.label}).Observe(obs.val) + } + + // Accumulate regular-cased API label metrics for 'PutObject' for deeper verification + metrics := getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD(), false, false) + capitalPutObjects := make([]MetricV2, 0, len(histBuckets)+1) + for _, metric := range metrics { + if value := metric.VariableLabels["api"]; value == "PutObject" { + capitalPutObjects = append(capitalPutObjects, metric) + } + } + if expMetricsPerAPI := len(histBuckets) + 1; expMetricsPerAPI != len(capitalPutObjects) { + t.Fatalf("Expected %d api=PutObject metrics but got %d", expMetricsPerAPI, len(capitalPutObjects)) + } + + // Deterministic ordering + slices.SortFunc(capitalPutObjects, func(a MetricV2, b MetricV2) int { + le1 := a.VariableLabels["le"] + le2 := a.VariableLabels["le"] + return strings.Compare(le1, le2) + }) + if le := capitalPutObjects[0].VariableLabels["le"]; le != "0.500" { + t.Errorf("Expected le='0.050' api=PutObject metrics but got '%v'", le) + } + if value := capitalPutObjects[0].Value; value != 1 { + t.Errorf("Expected le='0.050' api=PutObject value to be 1 but got '%v'", value) + } + if le := capitalPutObjects[1].VariableLabels["le"]; le != "5.000" { + t.Errorf("Expected le='5.000' api=PutObject metrics but got '%v'", le) + } + if value := capitalPutObjects[1].Value; value != 2 { + t.Errorf("Expected le='5.000' api=PutObject value to be 2 but got '%v'", value) + } + if le := capitalPutObjects[2].VariableLabels["le"]; le != "+Inf" { + t.Errorf("Expected le='+Inf' api=PutObject metrics but got '%v'", le) + } + if value := capitalPutObjects[2].Value; value != 2 { + t.Errorf("Expected le='+Inf' api=PutObject value to be 2 but got '%v'", value) + } + + // Accumulate lower-cased API label metrics for 'copyobject' for deeper verification + metrics = getHistogramMetrics(ttfbHist, getBucketTTFBDistributionMD(), true, false) + lowerCopyObjects := make([]MetricV2, 0, len(histBuckets)+1) + for _, metric := range metrics { + if value := metric.VariableLabels["api"]; value == "copyobject" { + lowerCopyObjects = append(lowerCopyObjects, metric) + } + } + if expMetricsPerAPI := len(histBuckets) + 1; expMetricsPerAPI != len(lowerCopyObjects) { + t.Fatalf("Expected %d api=copyobject metrics but got %d", expMetricsPerAPI, len(lowerCopyObjects)) + } + + // Deterministic ordering + slices.SortFunc(lowerCopyObjects, func(a MetricV2, b MetricV2) int { + le1 := a.VariableLabels["le"] + le2 := a.VariableLabels["le"] + return strings.Compare(le1, le2) + }) + if le := lowerCopyObjects[0].VariableLabels["le"]; le != "0.500" { + t.Errorf("Expected le='0.050' api=copyobject metrics but got '%v'", le) + } + if value := lowerCopyObjects[0].Value; value != 2 { + t.Errorf("Expected le='0.050' api=copyobject value to be 2 but got '%v'", value) + } + if le := lowerCopyObjects[1].VariableLabels["le"]; le != "5.000" { + t.Errorf("Expected le='5.000' api=copyobject metrics but got '%v'", le) + } + if value := lowerCopyObjects[1].Value; value != 2 { + t.Errorf("Expected le='5.000' api=copyobject value to be 2 but got '%v'", value) + } + if le := lowerCopyObjects[2].VariableLabels["le"]; le != "+Inf" { + t.Errorf("Expected le='+Inf' api=copyobject metrics but got '%v'", le) + } + if value := lowerCopyObjects[2].Value; value != 3 { + t.Errorf("Expected le='+Inf' api=copyobject value to be 3 but got '%v'", value) + } +} diff --git a/cmd/metrics-v3-api.go b/cmd/metrics-v3-api.go new file mode 100644 index 00000000..07265f03 --- /dev/null +++ b/cmd/metrics-v3-api.go @@ -0,0 +1,224 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + + "github.com/minio/minio-go/v7/pkg/set" +) + +const ( + apiRejectedAuthTotal MetricName = "rejected_auth_total" + apiRejectedHeaderTotal MetricName = "rejected_header_total" + apiRejectedTimestampTotal MetricName = "rejected_timestamp_total" + apiRejectedInvalidTotal MetricName = "rejected_invalid_total" + + apiRequestsWaitingTotal MetricName = "waiting_total" + apiRequestsIncomingTotal MetricName = "incoming_total" + + apiRequestsInFlightTotal MetricName = "inflight_total" + apiRequestsTotal MetricName = "total" + apiRequestsErrorsTotal MetricName = "errors_total" + apiRequests5xxErrorsTotal MetricName = "5xx_errors_total" + apiRequests4xxErrorsTotal MetricName = "4xx_errors_total" + apiRequestsCanceledTotal MetricName = "canceled_total" + + apiRequestsTTFBSecondsDistribution MetricName = "ttfb_seconds_distribution" + + apiTrafficSentBytes MetricName = "traffic_sent_bytes" + apiTrafficRecvBytes MetricName = "traffic_received_bytes" +) + +var ( + apiRejectedAuthTotalMD = NewCounterMD(apiRejectedAuthTotal, + "Total number of requests rejected for auth failure", "type") + apiRejectedHeaderTotalMD = NewCounterMD(apiRejectedHeaderTotal, + "Total number of requests rejected for invalid header", "type") + apiRejectedTimestampTotalMD = NewCounterMD(apiRejectedTimestampTotal, + "Total number of requests rejected for invalid timestamp", "type") + apiRejectedInvalidTotalMD = NewCounterMD(apiRejectedInvalidTotal, + "Total number of invalid requests", "type") + + apiRequestsWaitingTotalMD = NewGaugeMD(apiRequestsWaitingTotal, + "Total number of requests in the waiting queue", "type") + apiRequestsIncomingTotalMD = NewGaugeMD(apiRequestsIncomingTotal, + "Total number of incoming requests", "type") + + apiRequestsInFlightTotalMD = NewGaugeMD(apiRequestsInFlightTotal, + "Total number of requests currently in flight", "name", "type") + apiRequestsTotalMD = NewCounterMD(apiRequestsTotal, + "Total number of requests", "name", "type") + apiRequestsErrorsTotalMD = NewCounterMD(apiRequestsErrorsTotal, + "Total number of requests with (4xx and 5xx) errors", "name", "type") + apiRequests5xxErrorsTotalMD = NewCounterMD(apiRequests5xxErrorsTotal, + "Total number of requests with 5xx errors", "name", "type") + apiRequests4xxErrorsTotalMD = NewCounterMD(apiRequests4xxErrorsTotal, + "Total number of requests with 4xx errors", "name", "type") + apiRequestsCanceledTotalMD = NewCounterMD(apiRequestsCanceledTotal, + "Total number of requests canceled by the client", "name", "type") + + apiRequestsTTFBSecondsDistributionMD = NewCounterMD(apiRequestsTTFBSecondsDistribution, + "Distribution of time to first byte across API calls", "name", "type", "le") + + apiTrafficSentBytesMD = NewCounterMD(apiTrafficSentBytes, + "Total number of bytes sent", "type") + apiTrafficRecvBytesMD = NewCounterMD(apiTrafficRecvBytes, + "Total number of bytes received", "type") +) + +// loadAPIRequestsHTTPMetrics - reads S3 HTTP metrics. +// +// This is a `MetricsLoaderFn`. +// +// This includes node level S3 HTTP metrics. +// +// This function currently ignores `opts`. +func loadAPIRequestsHTTPMetrics(ctx context.Context, m MetricValues, _ *metricsCache) error { + // Collect node level S3 HTTP metrics. + httpStats := globalHTTPStats.toServerHTTPStats(false) + + // Currently we only collect S3 API related stats, so we set the "type" + // label to "s3". + + m.Set(apiRejectedAuthTotal, float64(httpStats.TotalS3RejectedAuth), "type", "s3") + m.Set(apiRejectedTimestampTotal, float64(httpStats.TotalS3RejectedTime), "type", "s3") + m.Set(apiRejectedHeaderTotal, float64(httpStats.TotalS3RejectedHeader), "type", "s3") + m.Set(apiRejectedInvalidTotal, float64(httpStats.TotalS3RejectedInvalid), "type", "s3") + m.Set(apiRequestsWaitingTotal, float64(httpStats.S3RequestsInQueue), "type", "s3") + m.Set(apiRequestsIncomingTotal, float64(httpStats.S3RequestsIncoming), "type", "s3") + + for name, value := range httpStats.CurrentS3Requests.APIStats { + m.Set(apiRequestsInFlightTotal, float64(value), "name", name, "type", "s3") + } + for name, value := range httpStats.TotalS3Requests.APIStats { + m.Set(apiRequestsTotal, float64(value), "name", name, "type", "s3") + } + for name, value := range httpStats.TotalS3Errors.APIStats { + m.Set(apiRequestsErrorsTotal, float64(value), "name", name, "type", "s3") + } + for name, value := range httpStats.TotalS35xxErrors.APIStats { + m.Set(apiRequests5xxErrorsTotal, float64(value), "name", name, "type", "s3") + } + for name, value := range httpStats.TotalS34xxErrors.APIStats { + m.Set(apiRequests4xxErrorsTotal, float64(value), "name", name, "type", "s3") + } + for name, value := range httpStats.TotalS3Canceled.APIStats { + m.Set(apiRequestsCanceledTotal, float64(value), "name", name, "type", "s3") + } + return nil +} + +// loadAPIRequestsTTFBMetrics - loads S3 TTFB metrics. +// +// This is a `MetricsLoaderFn`. +func loadAPIRequestsTTFBMetrics(ctx context.Context, m MetricValues, _ *metricsCache) error { + renameLabels := map[string]string{"api": "name"} + labelsFilter := map[string]set.StringSet{} + m.SetHistogram(apiRequestsTTFBSecondsDistribution, httpRequestsDuration, labelsFilter, renameLabels, nil, + "type", "s3") + return nil +} + +// loadAPIRequestsNetworkMetrics - loads S3 network metrics. +// +// This is a `MetricsLoaderFn`. +func loadAPIRequestsNetworkMetrics(ctx context.Context, m MetricValues, _ *metricsCache) error { + connStats := globalConnStats.toServerConnStats() + m.Set(apiTrafficSentBytes, float64(connStats.s3OutputBytes), "type", "s3") + m.Set(apiTrafficRecvBytes, float64(connStats.s3InputBytes), "type", "s3") + return nil +} + +// Metric Descriptions for bucket level S3 metrics. +var ( + bucketAPITrafficSentBytesMD = NewCounterMD(apiTrafficSentBytes, + "Total number of bytes received for a bucket", "bucket", "type") + bucketAPITrafficRecvBytesMD = NewCounterMD(apiTrafficRecvBytes, + "Total number of bytes sent for a bucket", "bucket", "type") + + bucketAPIRequestsInFlightMD = NewGaugeMD(apiRequestsInFlightTotal, + "Total number of requests currently in flight for a bucket", "bucket", "name", "type") + bucketAPIRequestsTotalMD = NewCounterMD(apiRequestsTotal, + "Total number of requests for a bucket", "bucket", "name", "type") + bucketAPIRequestsCanceledMD = NewCounterMD(apiRequestsCanceledTotal, + "Total number of requests canceled by the client for a bucket", "bucket", "name", "type") + bucketAPIRequests4xxErrorsMD = NewCounterMD(apiRequests4xxErrorsTotal, + "Total number of requests with 4xx errors for a bucket", "bucket", "name", "type") + bucketAPIRequests5xxErrorsMD = NewCounterMD(apiRequests5xxErrorsTotal, + "Total number of requests with 5xx errors for a bucket", "bucket", "name", "type") + + bucketAPIRequestsTTFBSecondsDistributionMD = NewCounterMD(apiRequestsTTFBSecondsDistribution, + "Distribution of time to first byte across API calls for a bucket", + "bucket", "name", "le", "type") +) + +// loadBucketAPIHTTPMetrics - loads bucket level S3 HTTP metrics. +// +// This is a `MetricsLoaderFn`. +// +// This includes bucket level S3 HTTP metrics and S3 network in/out metrics. +func loadBucketAPIHTTPMetrics(ctx context.Context, m MetricValues, _ *metricsCache, buckets []string) error { + if len(buckets) == 0 { + return nil + } + for bucket, inOut := range globalBucketConnStats.getBucketS3InOutBytes(buckets) { + recvBytes := inOut.In + if recvBytes > 0 { + m.Set(apiTrafficSentBytes, float64(recvBytes), "bucket", bucket, "type", "s3") + } + sentBytes := inOut.Out + if sentBytes > 0 { + m.Set(apiTrafficRecvBytes, float64(sentBytes), "bucket", bucket, "type", "s3") + } + + httpStats := globalBucketHTTPStats.load(bucket) + for k, v := range httpStats.currentS3Requests.Load(false) { + m.Set(apiRequestsInFlightTotal, float64(v), "bucket", bucket, "name", k, "type", "s3") + } + + for k, v := range httpStats.totalS3Requests.Load(false) { + m.Set(apiRequestsTotal, float64(v), "bucket", bucket, "name", k, "type", "s3") + } + + for k, v := range httpStats.totalS3Canceled.Load(false) { + m.Set(apiRequestsCanceledTotal, float64(v), "bucket", bucket, "name", k, "type", "s3") + } + + for k, v := range httpStats.totalS34xxErrors.Load(false) { + m.Set(apiRequests4xxErrorsTotal, float64(v), "bucket", bucket, "name", k, "type", "s3") + } + + for k, v := range httpStats.totalS35xxErrors.Load(false) { + m.Set(apiRequests5xxErrorsTotal, float64(v), "bucket", bucket, "name", k, "type", "s3") + } + } + + return nil +} + +// loadBucketAPITTFBMetrics - loads bucket S3 TTFB metrics. +// +// This is a `MetricsLoaderFn`. +func loadBucketAPITTFBMetrics(ctx context.Context, m MetricValues, _ *metricsCache, buckets []string) error { + renameLabels := map[string]string{"api": "name"} + labelsFilter := map[string]set.StringSet{} + m.SetHistogram(apiRequestsTTFBSecondsDistribution, bucketHTTPRequestsDuration, labelsFilter, renameLabels, + buckets, "type", "s3") + return nil +} diff --git a/cmd/metrics-v3-audit.go b/cmd/metrics-v3-audit.go new file mode 100644 index 00000000..a9f89796 --- /dev/null +++ b/cmd/metrics-v3-audit.go @@ -0,0 +1,57 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + + "github.com/minio/minio/internal/logger" +) + +const ( + auditFailedMessages = "failed_messages" + auditTargetQueueLength = "target_queue_length" + auditTotalMessages = "total_messages" + targetID = "target_id" +) + +var ( + auditFailedMessagesMD = NewCounterMD(auditFailedMessages, + "Total number of messages that failed to send since start", + targetID) + auditTargetQueueLengthMD = NewGaugeMD(auditTargetQueueLength, + "Number of unsent messages in queue for target", + targetID) + auditTotalMessagesMD = NewCounterMD(auditTotalMessages, + "Total number of messages sent since start", + targetID) +) + +// loadAuditMetrics - `MetricsLoaderFn` for audit +// such as failed messages and total messages. +func loadAuditMetrics(_ context.Context, m MetricValues, c *metricsCache) error { + audit := logger.CurrentStats() + for id, st := range audit { + labels := []string{targetID, id} + m.Set(auditFailedMessages, float64(st.FailedMessages), labels...) + m.Set(auditTargetQueueLength, float64(st.QueueLength), labels...) + m.Set(auditTotalMessages, float64(st.TotalMessages), labels...) + } + + return nil +} diff --git a/cmd/metrics-v3-bucket-replication.go b/cmd/metrics-v3-bucket-replication.go new file mode 100644 index 00000000..5f16d32e --- /dev/null +++ b/cmd/metrics-v3-bucket-replication.go @@ -0,0 +1,155 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" +) + +const ( + bucketReplLastHrFailedBytes = "last_hour_failed_bytes" + bucketReplLastHrFailedCount = "last_hour_failed_count" + bucketReplLastMinFailedBytes = "last_minute_failed_bytes" + bucketReplLastMinFailedCount = "last_minute_failed_count" + bucketReplLatencyMs = "latency_ms" + bucketReplProxiedDeleteTaggingRequestsTotal = "proxied_delete_tagging_requests_total" + bucketReplProxiedGetRequestsFailures = "proxied_get_requests_failures" + bucketReplProxiedGetRequestsTotal = "proxied_get_requests_total" + bucketReplProxiedGetTaggingRequestsFailures = "proxied_get_tagging_requests_failures" + bucketReplProxiedGetTaggingRequestsTotal = "proxied_get_tagging_requests_total" + bucketReplProxiedHeadRequestsFailures = "proxied_head_requests_failures" + bucketReplProxiedHeadRequestsTotal = "proxied_head_requests_total" + bucketReplProxiedPutTaggingRequestsFailures = "proxied_put_tagging_requests_failures" + bucketReplProxiedPutTaggingRequestsTotal = "proxied_put_tagging_requests_total" + bucketReplSentBytes = "sent_bytes" + bucketReplSentCount = "sent_count" + bucketReplTotalFailedBytes = "total_failed_bytes" + bucketReplTotalFailedCount = "total_failed_count" + bucketReplProxiedDeleteTaggingRequestsFailures = "proxied_delete_tagging_requests_failures" + bucketL = "bucket" + operationL = "operation" + targetArnL = "targetArn" +) + +var ( + bucketReplLastHrFailedBytesMD = NewGaugeMD(bucketReplLastHrFailedBytes, + "Total number of bytes failed at least once to replicate in the last hour on a bucket", + bucketL, targetArnL) + bucketReplLastHrFailedCountMD = NewGaugeMD(bucketReplLastHrFailedCount, + "Total number of objects which failed replication in the last hour on a bucket", + bucketL, targetArnL) + bucketReplLastMinFailedBytesMD = NewGaugeMD(bucketReplLastMinFailedBytes, + "Total number of bytes failed at least once to replicate in the last full minute on a bucket", + bucketL, targetArnL) + bucketReplLastMinFailedCountMD = NewGaugeMD(bucketReplLastMinFailedCount, + "Total number of objects which failed replication in the last full minute on a bucket", + bucketL, targetArnL) + bucketReplLatencyMsMD = NewGaugeMD(bucketReplLatencyMs, + "Replication latency on a bucket in milliseconds", + bucketL, operationL, rangeL, targetArnL) + bucketReplProxiedDeleteTaggingRequestsTotalMD = NewCounterMD(bucketReplProxiedDeleteTaggingRequestsTotal, + "Number of DELETE tagging requests proxied to replication target", + bucketL, targetArnL) + bucketReplProxiedGetRequestsFailuresMD = NewCounterMD(bucketReplProxiedGetRequestsFailures, + "Number of failures in GET requests proxied to replication target", + bucketL, targetArnL) + bucketReplProxiedGetRequestsTotalMD = NewCounterMD(bucketReplProxiedGetRequestsTotal, + "Number of GET requests proxied to replication target", + bucketL, targetArnL) + bucketReplProxiedGetTaggingRequestsFailuresMD = NewCounterMD(bucketReplProxiedGetTaggingRequestsFailures, + "Number of failures in GET tagging requests proxied to replication target", + bucketL, targetArnL) + bucketReplProxiedGetTaggingRequestsTotalMD = NewCounterMD(bucketReplProxiedGetTaggingRequestsTotal, + "Number of GET tagging requests proxied to replication target", + bucketL, targetArnL) + bucketReplProxiedHeadRequestsFailuresMD = NewCounterMD(bucketReplProxiedHeadRequestsFailures, + "Number of failures in HEAD requests proxied to replication target", + bucketL, targetArnL) + bucketReplProxiedHeadRequestsTotalMD = NewCounterMD(bucketReplProxiedHeadRequestsTotal, + "Number of HEAD requests proxied to replication target", + bucketL, targetArnL) + bucketReplProxiedPutTaggingRequestsFailuresMD = NewCounterMD(bucketReplProxiedPutTaggingRequestsFailures, + "Number of failures in PUT tagging requests proxied to replication target", + bucketL, targetArnL) + bucketReplProxiedPutTaggingRequestsTotalMD = NewCounterMD(bucketReplProxiedPutTaggingRequestsTotal, + "Number of PUT tagging requests proxied to replication target", + bucketL, targetArnL) + bucketReplSentBytesMD = NewCounterMD(bucketReplSentBytes, + "Total number of bytes replicated to the target", + bucketL, targetArnL) + bucketReplSentCountMD = NewCounterMD(bucketReplSentCount, + "Total number of objects replicated to the target", + bucketL, targetArnL) + bucketReplTotalFailedBytesMD = NewCounterMD(bucketReplTotalFailedBytes, + "Total number of bytes failed at least once to replicate since server start", + bucketL, targetArnL) + bucketReplTotalFailedCountMD = NewCounterMD(bucketReplTotalFailedCount, + "Total number of objects which failed replication since server start", + bucketL, targetArnL) + bucketReplProxiedDeleteTaggingRequestsFailuresMD = NewCounterMD(bucketReplProxiedDeleteTaggingRequestsFailures, + "Number of failures in DELETE tagging requests proxied to replication target", + bucketL, targetArnL) +) + +// loadBucketReplicationMetrics - `BucketMetricsLoaderFn` for bucket replication metrics +// such as latency and sent bytes. +func loadBucketReplicationMetrics(ctx context.Context, m MetricValues, c *metricsCache, buckets []string) error { + if globalSiteReplicationSys.isEnabled() { + return nil + } + + dataUsageInfo, err := c.dataUsageInfo.Get() + if err != nil { + metricsLogIf(ctx, err) + return nil + } + + bucketReplStats := globalReplicationStats.Load().getAllLatest(dataUsageInfo.BucketsUsage) + for _, bucket := range buckets { + if s, ok := bucketReplStats[bucket]; ok { + stats := s.ReplicationStats + if stats.hasReplicationUsage() { + for arn, stat := range stats.Stats { + labels := []string{bucketL, bucket, targetArnL, arn} + m.Set(bucketReplLastHrFailedBytes, float64(stat.Failed.LastHour.Bytes), labels...) + m.Set(bucketReplLastHrFailedCount, float64(stat.Failed.LastHour.Count), labels...) + m.Set(bucketReplLastMinFailedBytes, float64(stat.Failed.LastMinute.Bytes), labels...) + m.Set(bucketReplLastMinFailedCount, float64(stat.Failed.LastMinute.Count), labels...) + m.Set(bucketReplProxiedDeleteTaggingRequestsTotal, float64(s.ProxyStats.RmvTagTotal), labels...) + m.Set(bucketReplProxiedGetRequestsFailures, float64(s.ProxyStats.GetFailedTotal), labels...) + m.Set(bucketReplProxiedGetRequestsTotal, float64(s.ProxyStats.GetTotal), labels...) + m.Set(bucketReplProxiedGetTaggingRequestsFailures, float64(s.ProxyStats.GetTagFailedTotal), labels...) + m.Set(bucketReplProxiedGetTaggingRequestsTotal, float64(s.ProxyStats.GetTagTotal), labels...) + m.Set(bucketReplProxiedHeadRequestsFailures, float64(s.ProxyStats.HeadFailedTotal), labels...) + m.Set(bucketReplProxiedHeadRequestsTotal, float64(s.ProxyStats.HeadTotal), labels...) + m.Set(bucketReplProxiedPutTaggingRequestsFailures, float64(s.ProxyStats.PutTagFailedTotal), labels...) + m.Set(bucketReplProxiedPutTaggingRequestsTotal, float64(s.ProxyStats.PutTagTotal), labels...) + m.Set(bucketReplSentCount, float64(stat.ReplicatedCount), labels...) + m.Set(bucketReplTotalFailedBytes, float64(stat.Failed.Totals.Bytes), labels...) + m.Set(bucketReplTotalFailedCount, float64(stat.Failed.Totals.Count), labels...) + m.Set(bucketReplProxiedDeleteTaggingRequestsFailures, float64(s.ProxyStats.RmvTagFailedTotal), labels...) + m.Set(bucketReplSentBytes, float64(stat.ReplicatedSize), labels...) + + SetHistogramValues(m, bucketReplLatencyMs, stat.Latency.getUploadLatency(), bucketL, bucket, operationL, "upload", targetArnL, arn) + } + } + } + } + + return nil +} diff --git a/cmd/metrics-v3-cache.go b/cmd/metrics-v3-cache.go new file mode 100644 index 00000000..e94eeef7 --- /dev/null +++ b/cmd/metrics-v3-cache.go @@ -0,0 +1,277 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "sync" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/cachevalue" +) + +// metricsCache - cache for metrics. +// +// When serving metrics, this cache is passed to the MetricsLoaderFn. +// +// This cache is used for metrics that would result in network/storage calls. +type metricsCache struct { + dataUsageInfo *cachevalue.Cache[DataUsageInfo] + esetHealthResult *cachevalue.Cache[HealthResult] + driveMetrics *cachevalue.Cache[storageMetrics] + memoryMetrics *cachevalue.Cache[madmin.MemInfo] + cpuMetrics *cachevalue.Cache[madmin.CPUMetrics] + clusterDriveMetrics *cachevalue.Cache[storageMetrics] + nodesUpDown *cachevalue.Cache[nodesOnline] +} + +func newMetricsCache() *metricsCache { + return &metricsCache{ + dataUsageInfo: newDataUsageInfoCache(), + esetHealthResult: newESetHealthResultCache(), + driveMetrics: newDriveMetricsCache(), + memoryMetrics: newMemoryMetricsCache(), + cpuMetrics: newCPUMetricsCache(), + clusterDriveMetrics: newClusterStorageInfoCache(), + nodesUpDown: newNodesUpDownCache(), + } +} + +type nodesOnline struct { + Online, Offline int +} + +func newNodesUpDownCache() *cachevalue.Cache[nodesOnline] { + loadNodesUpDown := func(ctx context.Context) (v nodesOnline, err error) { + v.Online, v.Offline = globalNotificationSys.GetPeerOnlineCount() + return v, err + } + return cachevalue.NewFromFunc(1*time.Minute, + cachevalue.Opts{ReturnLastGood: true}, + loadNodesUpDown) +} + +type driveIOStatMetrics struct { + readsPerSec float64 + readsKBPerSec float64 + readsAwait float64 + writesPerSec float64 + writesKBPerSec float64 + writesAwait float64 + percUtil float64 +} + +// storageMetrics - cached storage metrics. +type storageMetrics struct { + storageInfo madmin.StorageInfo + ioStats map[string]driveIOStatMetrics + onlineDrives, offlineDrives, totalDrives int +} + +func newDataUsageInfoCache() *cachevalue.Cache[DataUsageInfo] { + loadDataUsage := func(ctx context.Context) (u DataUsageInfo, err error) { + objLayer := newObjectLayerFn() + if objLayer == nil { + return u, err + } + + // Collect cluster level object metrics. + u, err = loadDataUsageFromBackend(GlobalContext, objLayer) + return u, err + } + return cachevalue.NewFromFunc(1*time.Minute, + cachevalue.Opts{ReturnLastGood: true}, + loadDataUsage) +} + +func newESetHealthResultCache() *cachevalue.Cache[HealthResult] { + loadHealth := func(ctx context.Context) (r HealthResult, err error) { + objLayer := newObjectLayerFn() + if objLayer == nil { + return r, err + } + + r = objLayer.Health(GlobalContext, HealthOptions{}) + return r, err + } + return cachevalue.NewFromFunc(1*time.Minute, + cachevalue.Opts{ReturnLastGood: true}, + loadHealth, + ) +} + +func getDiffStats(initialStats, currentStats madmin.DiskIOStats) madmin.DiskIOStats { + return madmin.DiskIOStats{ + ReadIOs: currentStats.ReadIOs - initialStats.ReadIOs, + WriteIOs: currentStats.WriteIOs - initialStats.WriteIOs, + ReadSectors: currentStats.ReadSectors - initialStats.ReadSectors, + WriteSectors: currentStats.WriteSectors - initialStats.WriteSectors, + ReadTicks: currentStats.ReadTicks - initialStats.ReadTicks, + WriteTicks: currentStats.WriteTicks - initialStats.WriteTicks, + TotalTicks: currentStats.TotalTicks - initialStats.TotalTicks, + } +} + +func getDriveIOStatMetrics(ioStats madmin.DiskIOStats, duration time.Duration) (m driveIOStatMetrics) { + durationSecs := duration.Seconds() + + m.readsPerSec = float64(ioStats.ReadIOs) / durationSecs + m.readsKBPerSec = float64(ioStats.ReadSectors) * float64(sectorSize) / kib / durationSecs + if ioStats.ReadIOs > 0 { + m.readsAwait = float64(ioStats.ReadTicks) / float64(ioStats.ReadIOs) + } + + m.writesPerSec = float64(ioStats.WriteIOs) / durationSecs + m.writesKBPerSec = float64(ioStats.WriteSectors) * float64(sectorSize) / kib / durationSecs + if ioStats.WriteIOs > 0 { + m.writesAwait = float64(ioStats.WriteTicks) / float64(ioStats.WriteIOs) + } + + // TotalTicks is in milliseconds + m.percUtil = float64(ioStats.TotalTicks) * 100 / (durationSecs * 1000) + + return m +} + +func newDriveMetricsCache() *cachevalue.Cache[storageMetrics] { + var ( + // prevDriveIOStats is used to calculate "per second" + // values for IOStat related disk metrics e.g. reads/sec. + prevDriveIOStats map[string]madmin.DiskIOStats + prevDriveIOStatsMu sync.RWMutex + prevDriveIOStatsRefreshedAt time.Time + ) + + loadDriveMetrics := func(ctx context.Context) (v storageMetrics, err error) { + objLayer := newObjectLayerFn() + if objLayer == nil { + return v, err + } + + storageInfo := objLayer.LocalStorageInfo(GlobalContext, true) + onlineDrives, offlineDrives := getOnlineOfflineDisksStats(storageInfo.Disks) + totalDrives := onlineDrives.Merge(offlineDrives) + + v = storageMetrics{ + storageInfo: storageInfo, + onlineDrives: onlineDrives.Sum(), + offlineDrives: offlineDrives.Sum(), + totalDrives: totalDrives.Sum(), + ioStats: map[string]driveIOStatMetrics{}, + } + + currentStats := getCurrentDriveIOStats() + now := time.Now().UTC() + + prevDriveIOStatsMu.Lock() + if prevDriveIOStats != nil { + duration := now.Sub(prevDriveIOStatsRefreshedAt) + if duration.Seconds() > 1 { + for d, cs := range currentStats { + if ps, found := prevDriveIOStats[d]; found { + v.ioStats[d] = getDriveIOStatMetrics(getDiffStats(ps, cs), duration) + } + } + } + } + + prevDriveIOStats = currentStats + prevDriveIOStatsRefreshedAt = now + prevDriveIOStatsMu.Unlock() + + return v, err + } + + return cachevalue.NewFromFunc(1*time.Minute, + cachevalue.Opts{ReturnLastGood: true}, + loadDriveMetrics) +} + +func newCPUMetricsCache() *cachevalue.Cache[madmin.CPUMetrics] { + loadCPUMetrics := func(ctx context.Context) (v madmin.CPUMetrics, err error) { + types := madmin.MetricsCPU + + m := collectLocalMetrics(types, collectMetricsOpts{ + hosts: map[string]struct{}{ + globalLocalNodeName: {}, + }, + }) + + for _, hm := range m.ByHost { + if hm.CPU != nil { + v = *hm.CPU + break + } + } + + return v, err + } + + return cachevalue.NewFromFunc(1*time.Minute, + cachevalue.Opts{ReturnLastGood: true}, + loadCPUMetrics) +} + +func newMemoryMetricsCache() *cachevalue.Cache[madmin.MemInfo] { + loadMemoryMetrics := func(ctx context.Context) (v madmin.MemInfo, err error) { + types := madmin.MetricsMem + + m := collectLocalMetrics(types, collectMetricsOpts{ + hosts: map[string]struct{}{ + globalLocalNodeName: {}, + }, + }) + + for _, hm := range m.ByHost { + if hm.Mem != nil && len(hm.Mem.Info.Addr) > 0 { + v = hm.Mem.Info + break + } + } + + return v, err + } + + return cachevalue.NewFromFunc(1*time.Minute, + cachevalue.Opts{ReturnLastGood: true}, + loadMemoryMetrics) +} + +func newClusterStorageInfoCache() *cachevalue.Cache[storageMetrics] { + loadStorageInfo := func(ctx context.Context) (v storageMetrics, err error) { + objLayer := newObjectLayerFn() + if objLayer == nil { + return storageMetrics{}, nil + } + storageInfo := objLayer.StorageInfo(GlobalContext, true) + onlineDrives, offlineDrives := getOnlineOfflineDisksStats(storageInfo.Disks) + totalDrives := onlineDrives.Merge(offlineDrives) + v = storageMetrics{ + storageInfo: storageInfo, + onlineDrives: onlineDrives.Sum(), + offlineDrives: offlineDrives.Sum(), + totalDrives: totalDrives.Sum(), + } + return v, err + } + return cachevalue.NewFromFunc(1*time.Minute, + cachevalue.Opts{ReturnLastGood: true}, + loadStorageInfo, + ) +} diff --git a/cmd/metrics-v3-cluster-config.go b/cmd/metrics-v3-cluster-config.go new file mode 100644 index 00000000..9d96c6bb --- /dev/null +++ b/cmd/metrics-v3-cluster-config.go @@ -0,0 +1,46 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import "context" + +const ( + configRRSParity = "rrs_parity" + configStandardParity = "standard_parity" +) + +var ( + configRRSParityMD = NewGaugeMD(configRRSParity, + "Reduced redundancy storage class parity") + configStandardParityMD = NewGaugeMD(configStandardParity, + "Standard storage class parity") +) + +// loadClusterConfigMetrics - `MetricsLoaderFn` for cluster config +// such as standard and RRS parity. +func loadClusterConfigMetrics(ctx context.Context, m MetricValues, c *metricsCache) error { + clusterDriveMetrics, err := c.clusterDriveMetrics.Get() + if err != nil { + metricsLogIf(ctx, err) + } else { + m.Set(configStandardParity, float64(clusterDriveMetrics.storageInfo.Backend.StandardSCParity)) + m.Set(configRRSParity, float64(clusterDriveMetrics.storageInfo.Backend.RRSCParity)) + } + + return nil +} diff --git a/cmd/metrics-v3-cluster-erasure-set.go b/cmd/metrics-v3-cluster-erasure-set.go new file mode 100644 index 00000000..04824c62 --- /dev/null +++ b/cmd/metrics-v3-cluster-erasure-set.go @@ -0,0 +1,117 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "strconv" +) + +const ( + erasureSetOverallWriteQuorum = "overall_write_quorum" + erasureSetOverallHealth = "overall_health" + erasureSetReadQuorum = "read_quorum" + erasureSetWriteQuorum = "write_quorum" + erasureSetOnlineDrivesCount = "online_drives_count" + erasureSetHealingDrivesCount = "healing_drives_count" + erasureSetHealth = "health" + erasureSetReadTolerance = "read_tolerance" + erasureSetWriteTolerance = "write_tolerance" + erasureSetReadHealth = "read_health" + erasureSetWriteHealth = "write_health" +) + +const ( + poolIDL = "pool_id" + setIDL = "set_id" +) + +var ( + erasureSetOverallWriteQuorumMD = NewGaugeMD(erasureSetOverallWriteQuorum, + "Overall write quorum across pools and sets") + erasureSetOverallHealthMD = NewGaugeMD(erasureSetOverallHealth, + "Overall health across pools and sets (1=healthy, 0=unhealthy)") + erasureSetReadQuorumMD = NewGaugeMD(erasureSetReadQuorum, + "Read quorum for the erasure set in a pool", poolIDL, setIDL) + erasureSetWriteQuorumMD = NewGaugeMD(erasureSetWriteQuorum, + "Write quorum for the erasure set in a pool", poolIDL, setIDL) + erasureSetOnlineDrivesCountMD = NewGaugeMD(erasureSetOnlineDrivesCount, + "Count of online drives in the erasure set in a pool", poolIDL, setIDL) + erasureSetHealingDrivesCountMD = NewGaugeMD(erasureSetHealingDrivesCount, + "Count of healing drives in the erasure set in a pool", poolIDL, setIDL) + erasureSetHealthMD = NewGaugeMD(erasureSetHealth, + "Health of the erasure set in a pool (1=healthy, 0=unhealthy)", + poolIDL, setIDL) + erasureSetReadToleranceMD = NewGaugeMD(erasureSetReadTolerance, + "No of drive failures that can be tolerated without disrupting read operations", + poolIDL, setIDL) + erasureSetWriteToleranceMD = NewGaugeMD(erasureSetWriteTolerance, + "No of drive failures that can be tolerated without disrupting write operations", + poolIDL, setIDL) + erasureSetReadHealthMD = NewGaugeMD(erasureSetReadHealth, + "Health of the erasure set in a pool for read operations (1=healthy, 0=unhealthy)", + poolIDL, setIDL) + erasureSetWriteHealthMD = NewGaugeMD(erasureSetWriteHealth, + "Health of the erasure set in a pool for write operations (1=healthy, 0=unhealthy)", + poolIDL, setIDL) +) + +func b2f(v bool) float64 { + if v { + return 1 + } + return 0 +} + +// loadClusterErasureSetMetrics - `MetricsLoaderFn` for cluster storage erasure +// set metrics. +func loadClusterErasureSetMetrics(ctx context.Context, m MetricValues, c *metricsCache) error { + result, _ := c.esetHealthResult.Get() + + m.Set(erasureSetOverallWriteQuorum, float64(result.WriteQuorum)) + m.Set(erasureSetOverallHealth, b2f(result.Healthy)) + + for _, h := range result.ESHealth { + poolLV := strconv.Itoa(h.PoolID) + setLV := strconv.Itoa(h.SetID) + labels := []string{poolIDL, poolLV, setIDL, setLV} + m.Set(erasureSetReadQuorum, float64(h.ReadQuorum), labels...) + m.Set(erasureSetWriteQuorum, float64(h.WriteQuorum), labels...) + m.Set(erasureSetOnlineDrivesCount, float64(h.HealthyDrives), labels...) + m.Set(erasureSetHealingDrivesCount, float64(h.HealingDrives), labels...) + m.Set(erasureSetHealth, b2f(h.Healthy), labels...) + + readHealthy := true + readTolerance := float64(h.HealthyDrives - h.ReadQuorum) + if readTolerance < 0 { + readHealthy = false + } + m.Set(erasureSetReadTolerance, readTolerance, labels...) + m.Set(erasureSetReadHealth, b2f(readHealthy), labels...) + + writeHealthy := true + writeTolerance := float64(h.HealthyDrives + h.HealingDrives - h.WriteQuorum) + if writeTolerance < 0 { + writeHealthy = false + } + m.Set(erasureSetWriteTolerance, writeTolerance, labels...) + m.Set(erasureSetWriteHealth, b2f(writeHealthy), labels...) + } + + return nil +} diff --git a/cmd/metrics-v3-cluster-health.go b/cmd/metrics-v3-cluster-health.go new file mode 100644 index 00000000..8636fc96 --- /dev/null +++ b/cmd/metrics-v3-cluster-health.go @@ -0,0 +1,109 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import "context" + +const ( + healthDrivesOfflineCount = "drives_offline_count" + healthDrivesOnlineCount = "drives_online_count" + healthDrivesCount = "drives_count" +) + +var ( + healthDrivesOfflineCountMD = NewGaugeMD(healthDrivesOfflineCount, + "Count of offline drives in the cluster") + healthDrivesOnlineCountMD = NewGaugeMD(healthDrivesOnlineCount, + "Count of online drives in the cluster") + healthDrivesCountMD = NewGaugeMD(healthDrivesCount, + "Count of all drives in the cluster") +) + +// loadClusterHealthDriveMetrics - `MetricsLoaderFn` for cluster storage drive metrics +// such as online, offline and total drives. +func loadClusterHealthDriveMetrics(ctx context.Context, m MetricValues, + c *metricsCache, +) error { + clusterDriveMetrics, _ := c.clusterDriveMetrics.Get() + + m.Set(healthDrivesOfflineCount, float64(clusterDriveMetrics.offlineDrives)) + m.Set(healthDrivesOnlineCount, float64(clusterDriveMetrics.onlineDrives)) + m.Set(healthDrivesCount, float64(clusterDriveMetrics.totalDrives)) + + return nil +} + +const ( + healthNodesOfflineCount = "nodes_offline_count" + healthNodesOnlineCount = "nodes_online_count" +) + +var ( + healthNodesOfflineCountMD = NewGaugeMD(healthNodesOfflineCount, + "Count of offline nodes in the cluster") + healthNodesOnlineCountMD = NewGaugeMD(healthNodesOnlineCount, + "Count of online nodes in the cluster") +) + +// loadClusterHealthNodeMetrics - `MetricsLoaderFn` for cluster health node +// metrics. +func loadClusterHealthNodeMetrics(ctx context.Context, m MetricValues, + c *metricsCache, +) error { + nodesUpDown, _ := c.nodesUpDown.Get() + + m.Set(healthNodesOfflineCount, float64(nodesUpDown.Offline)) + m.Set(healthNodesOnlineCount, float64(nodesUpDown.Online)) + + return nil +} + +const ( + healthCapacityRawTotalBytes = "capacity_raw_total_bytes" + healthCapacityRawFreeBytes = "capacity_raw_free_bytes" + healthCapacityUsableTotalBytes = "capacity_usable_total_bytes" + healthCapacityUsableFreeBytes = "capacity_usable_free_bytes" +) + +var ( + healthCapacityRawTotalBytesMD = NewGaugeMD(healthCapacityRawTotalBytes, + "Total cluster raw storage capacity in bytes") + healthCapacityRawFreeBytesMD = NewGaugeMD(healthCapacityRawFreeBytes, + "Total cluster raw storage free in bytes") + healthCapacityUsableTotalBytesMD = NewGaugeMD(healthCapacityUsableTotalBytes, + "Total cluster usable storage capacity in bytes") + healthCapacityUsableFreeBytesMD = NewGaugeMD(healthCapacityUsableFreeBytes, + "Total cluster usable storage free in bytes") +) + +// loadClusterHealthCapacityMetrics - `MetricsLoaderFn` for cluster storage +// capacity metrics. +func loadClusterHealthCapacityMetrics(ctx context.Context, m MetricValues, + c *metricsCache, +) error { + clusterDriveMetrics, _ := c.clusterDriveMetrics.Get() + + storageInfo := clusterDriveMetrics.storageInfo + + m.Set(healthCapacityRawTotalBytes, float64(GetTotalCapacity(storageInfo.Disks))) + m.Set(healthCapacityRawFreeBytes, float64(GetTotalCapacityFree(storageInfo.Disks))) + m.Set(healthCapacityUsableTotalBytes, float64(GetTotalUsableCapacity(storageInfo.Disks, storageInfo))) + m.Set(healthCapacityUsableFreeBytes, float64(GetTotalUsableCapacityFree(storageInfo.Disks, storageInfo))) + + return nil +} diff --git a/cmd/metrics-v3-cluster-iam.go b/cmd/metrics-v3-cluster-iam.go new file mode 100644 index 00000000..6689fe55 --- /dev/null +++ b/cmd/metrics-v3-cluster-iam.go @@ -0,0 +1,69 @@ +// Copyright (c) 2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "sync/atomic" + "time" +) + +const ( + lastSyncDurationMillis = "last_sync_duration_millis" + pluginAuthnServiceFailedRequestsMinute = "plugin_authn_service_failed_requests_minute" + pluginAuthnServiceLastFailSeconds = "plugin_authn_service_last_fail_seconds" + pluginAuthnServiceLastSuccSeconds = "plugin_authn_service_last_succ_seconds" + pluginAuthnServiceSuccAvgRttMsMinute = "plugin_authn_service_succ_avg_rtt_ms_minute" + pluginAuthnServiceSuccMaxRttMsMinute = "plugin_authn_service_succ_max_rtt_ms_minute" + pluginAuthnServiceTotalRequestsMinute = "plugin_authn_service_total_requests_minute" + sinceLastSyncMillis = "since_last_sync_millis" + syncFailures = "sync_failures" + syncSuccesses = "sync_successes" +) + +var ( + lastSyncDurationMillisMD = NewCounterMD(lastSyncDurationMillis, "Last successful IAM data sync duration in milliseconds") + pluginAuthnServiceFailedRequestsMinuteMD = NewCounterMD(pluginAuthnServiceFailedRequestsMinute, "When plugin authentication is configured, returns failed requests count in the last full minute") + pluginAuthnServiceLastFailSecondsMD = NewCounterMD(pluginAuthnServiceLastFailSeconds, "When plugin authentication is configured, returns time (in seconds) since the last failed request to the service") + pluginAuthnServiceLastSuccSecondsMD = NewCounterMD(pluginAuthnServiceLastSuccSeconds, "When plugin authentication is configured, returns time (in seconds) since the last successful request to the service") + pluginAuthnServiceSuccAvgRttMsMinuteMD = NewCounterMD(pluginAuthnServiceSuccAvgRttMsMinute, "When plugin authentication is configured, returns average round-trip-time of successful requests in the last full minute") + pluginAuthnServiceSuccMaxRttMsMinuteMD = NewCounterMD(pluginAuthnServiceSuccMaxRttMsMinute, "When plugin authentication is configured, returns maximum round-trip-time of successful requests in the last full minute") + pluginAuthnServiceTotalRequestsMinuteMD = NewCounterMD(pluginAuthnServiceTotalRequestsMinute, "When plugin authentication is configured, returns total requests count in the last full minute") + sinceLastSyncMillisMD = NewCounterMD(sinceLastSyncMillis, "Time (in milliseconds) since last successful IAM data sync.") + syncFailuresMD = NewCounterMD(syncFailures, "Number of failed IAM data syncs since server start.") + syncSuccessesMD = NewCounterMD(syncSuccesses, "Number of successful IAM data syncs since server start.") +) + +// loadClusterIAMMetrics - `MetricsLoaderFn` for cluster IAM metrics. +func loadClusterIAMMetrics(_ context.Context, m MetricValues, _ *metricsCache) error { + m.Set(lastSyncDurationMillis, float64(atomic.LoadUint64(&globalIAMSys.LastRefreshDurationMilliseconds))) + pluginAuthNMetrics := globalAuthNPlugin.Metrics() + m.Set(pluginAuthnServiceFailedRequestsMinute, float64(pluginAuthNMetrics.FailedRequests)) + m.Set(pluginAuthnServiceLastFailSeconds, pluginAuthNMetrics.LastUnreachableSecs) + m.Set(pluginAuthnServiceLastSuccSeconds, pluginAuthNMetrics.LastReachableSecs) + m.Set(pluginAuthnServiceSuccAvgRttMsMinute, pluginAuthNMetrics.AvgSuccRTTMs) + m.Set(pluginAuthnServiceSuccMaxRttMsMinute, pluginAuthNMetrics.MaxSuccRTTMs) + m.Set(pluginAuthnServiceTotalRequestsMinute, float64(pluginAuthNMetrics.TotalRequests)) + lastSyncTime := atomic.LoadUint64(&globalIAMSys.LastRefreshTimeUnixNano) + if lastSyncTime != 0 { + m.Set(sinceLastSyncMillis, float64((uint64(time.Now().UnixNano())-lastSyncTime)/uint64(time.Millisecond))) + } + m.Set(syncFailures, float64(atomic.LoadUint64(&globalIAMSys.TotalRefreshFailures))) + m.Set(syncSuccesses, float64(atomic.LoadUint64(&globalIAMSys.TotalRefreshSuccesses))) + return nil +} diff --git a/cmd/metrics-v3-cluster-notification.go b/cmd/metrics-v3-cluster-notification.go new file mode 100644 index 00000000..f4d76b62 --- /dev/null +++ b/cmd/metrics-v3-cluster-notification.go @@ -0,0 +1,51 @@ +// Copyright (c) 2024 MinIO, Inc. +// +// # This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" +) + +const ( + notificationCurrentSendInProgress = "current_send_in_progress" + notificationEventsErrorsTotal = "events_errors_total" + notificationEventsSentTotal = "events_sent_total" + notificationEventsSkippedTotal = "events_skipped_total" +) + +var ( + notificationCurrentSendInProgressMD = NewCounterMD(notificationCurrentSendInProgress, "Number of concurrent async Send calls active to all targets") + notificationEventsErrorsTotalMD = NewCounterMD(notificationEventsErrorsTotal, "Events that were failed to be sent to the targets") + notificationEventsSentTotalMD = NewCounterMD(notificationEventsSentTotal, "Total number of events sent to the targets") + notificationEventsSkippedTotalMD = NewCounterMD(notificationEventsSkippedTotal, "Events that were skipped to be sent to the targets due to the in-memory queue being full") +) + +// loadClusterNotificationMetrics - `MetricsLoaderFn` for cluster notification metrics. +func loadClusterNotificationMetrics(_ context.Context, m MetricValues, _ *metricsCache) error { + if globalEventNotifier == nil { + return nil + } + + nstats := globalEventNotifier.targetList.Stats() + m.Set(notificationCurrentSendInProgress, float64(nstats.CurrentSendCalls)) + m.Set(notificationEventsErrorsTotal, float64(nstats.EventsErrorsTotal)) + m.Set(notificationEventsSentTotal, float64(nstats.TotalEvents)) + m.Set(notificationEventsSkippedTotal, float64(nstats.EventsSkipped)) + + return nil +} diff --git a/cmd/metrics-v3-cluster-usage.go b/cmd/metrics-v3-cluster-usage.go new file mode 100644 index 00000000..38dc0aef --- /dev/null +++ b/cmd/metrics-v3-cluster-usage.go @@ -0,0 +1,182 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "time" +) + +const ( + usageSinceLastUpdateSeconds = "since_last_update_seconds" + usageTotalBytes = "total_bytes" + usageObjectsCount = "count" + usageVersionsCount = "versions_count" + usageDeleteMarkersCount = "delete_markers_count" + usageBucketsCount = "buckets_count" + usageSizeDistribution = "size_distribution" + usageVersionCountDistribution = "version_count_distribution" +) + +var ( + usageSinceLastUpdateSecondsMD = NewGaugeMD(usageSinceLastUpdateSeconds, + "Time since last update of usage metrics in seconds") + usageTotalBytesMD = NewGaugeMD(usageTotalBytes, + "Total cluster usage in bytes") + usageObjectsCountMD = NewGaugeMD(usageObjectsCount, + "Total cluster objects count") + usageVersionsCountMD = NewGaugeMD(usageVersionsCount, + "Total cluster object versions (including delete markers) count") + usageDeleteMarkersCountMD = NewGaugeMD(usageDeleteMarkersCount, + "Total cluster delete markers count") + usageBucketsCountMD = NewGaugeMD(usageBucketsCount, + "Total cluster buckets count") + usageObjectsDistributionMD = NewGaugeMD(usageSizeDistribution, + "Cluster object size distribution", "range") + usageVersionsDistributionMD = NewGaugeMD(usageVersionCountDistribution, + "Cluster object version count distribution", "range") +) + +// loadClusterUsageObjectMetrics - reads cluster usage metrics. +// +// This is a `MetricsLoaderFn`. +func loadClusterUsageObjectMetrics(ctx context.Context, m MetricValues, c *metricsCache) error { + dataUsageInfo, err := c.dataUsageInfo.Get() + if err != nil { + metricsLogIf(ctx, err) + return nil + } + + // data usage has not captured any data yet. + if dataUsageInfo.LastUpdate.IsZero() { + return nil + } + + var ( + clusterSize uint64 + clusterBuckets uint64 + clusterObjectsCount uint64 + clusterVersionsCount uint64 + clusterDeleteMarkersCount uint64 + ) + + clusterObjectSizesHistogram := map[string]uint64{} + clusterVersionsHistogram := map[string]uint64{} + for _, usage := range dataUsageInfo.BucketsUsage { + clusterBuckets++ + clusterSize += usage.Size + clusterObjectsCount += usage.ObjectsCount + clusterVersionsCount += usage.VersionsCount + clusterDeleteMarkersCount += usage.DeleteMarkersCount + for k, v := range usage.ObjectSizesHistogram { + clusterObjectSizesHistogram[k] += v + } + for k, v := range usage.ObjectVersionsHistogram { + clusterVersionsHistogram[k] += v + } + } + + m.Set(usageSinceLastUpdateSeconds, time.Since(dataUsageInfo.LastUpdate).Seconds()) + m.Set(usageTotalBytes, float64(clusterSize)) + m.Set(usageObjectsCount, float64(clusterObjectsCount)) + m.Set(usageVersionsCount, float64(clusterVersionsCount)) + m.Set(usageDeleteMarkersCount, float64(clusterDeleteMarkersCount)) + m.Set(usageBucketsCount, float64(clusterBuckets)) + for k, v := range clusterObjectSizesHistogram { + m.Set(usageSizeDistribution, float64(v), "range", k) + } + for k, v := range clusterVersionsHistogram { + m.Set(usageVersionCountDistribution, float64(v), "range", k) + } + + return nil +} + +const ( + usageBucketQuotaTotalBytes = "quota_total_bytes" + + usageBucketTotalBytes = "total_bytes" + usageBucketObjectsCount = "objects_count" + usageBucketVersionsCount = "versions_count" + usageBucketDeleteMarkersCount = "delete_markers_count" + usageBucketObjectSizeDistribution = "object_size_distribution" + usageBucketObjectVersionCountDistribution = "object_version_count_distribution" +) + +var ( + usageBucketTotalBytesMD = NewGaugeMD(usageBucketTotalBytes, + "Total bucket size in bytes", "bucket") + usageBucketObjectsTotalMD = NewGaugeMD(usageBucketObjectsCount, + "Total objects count in bucket", "bucket") + usageBucketVersionsCountMD = NewGaugeMD(usageBucketVersionsCount, + "Total object versions (including delete markers) count in bucket", "bucket") + usageBucketDeleteMarkersCountMD = NewGaugeMD(usageBucketDeleteMarkersCount, + "Total delete markers count in bucket", "bucket") + + usageBucketQuotaTotalBytesMD = NewGaugeMD(usageBucketQuotaTotalBytes, + "Total bucket quota in bytes", "bucket") + + usageBucketObjectSizeDistributionMD = NewGaugeMD(usageBucketObjectSizeDistribution, + "Bucket object size distribution", "range", "bucket") + usageBucketObjectVersionCountDistributionMD = NewGaugeMD( + usageBucketObjectVersionCountDistribution, + "Bucket object version count distribution", "range", "bucket") +) + +// loadClusterUsageBucketMetrics - `MetricsLoaderFn` to load bucket usage metrics. +func loadClusterUsageBucketMetrics(ctx context.Context, m MetricValues, c *metricsCache) error { + dataUsageInfo, err := c.dataUsageInfo.Get() + if err != nil { + metricsLogIf(ctx, err) + return nil + } + + // data usage has not been captured yet. + if dataUsageInfo.LastUpdate.IsZero() { + return nil + } + + m.Set(usageSinceLastUpdateSeconds, float64(time.Since(dataUsageInfo.LastUpdate))) + + for bucket, usage := range dataUsageInfo.BucketsUsage { + quota, err := globalBucketQuotaSys.Get(ctx, bucket) + if err != nil { + // Log and continue if we are unable to retrieve metrics for this + // bucket. + metricsLogIf(ctx, err) + continue + } + + m.Set(usageBucketTotalBytes, float64(usage.Size), "bucket", bucket) + m.Set(usageBucketObjectsCount, float64(usage.ObjectsCount), "bucket", bucket) + m.Set(usageBucketVersionsCount, float64(usage.VersionsCount), "bucket", bucket) + m.Set(usageBucketDeleteMarkersCount, float64(usage.DeleteMarkersCount), "bucket", bucket) + + if quota != nil && quota.Quota > 0 { + m.Set(usageBucketQuotaTotalBytes, float64(quota.Quota), "bucket", bucket) + } + + for k, v := range usage.ObjectSizesHistogram { + m.Set(usageBucketObjectSizeDistribution, float64(v), "range", k, "bucket", bucket) + } + for k, v := range usage.ObjectVersionsHistogram { + m.Set(usageBucketObjectVersionCountDistribution, float64(v), "range", k, "bucket", bucket) + } + } + return nil +} diff --git a/cmd/metrics-v3-handler.go b/cmd/metrics-v3-handler.go new file mode 100644 index 00000000..7f07f58e --- /dev/null +++ b/cmd/metrics-v3-handler.go @@ -0,0 +1,251 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "encoding/json" + "fmt" + "net/http" + "slices" + "strings" + "sync" + + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/mcontext" + "github.com/minio/mux" + "github.com/minio/pkg/v3/env" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" +) + +type promLogger struct{} + +func (p promLogger) Println(v ...any) { + metricsLogIf(GlobalContext, fmt.Errorf("metrics handler error: %v", v)) +} + +type metricsV3Server struct { + registry *prometheus.Registry + opts promhttp.HandlerOpts + auth func(http.Handler) http.Handler + + metricsData *metricsV3Collection +} + +var ( + globalMetricsV3CollectorPaths []collectorPath + globalMetricsV3Once sync.Once +) + +func newMetricsV3Server(auth func(h http.Handler) http.Handler) *metricsV3Server { + registry := prometheus.NewRegistry() + metricGroups := newMetricGroups(registry) + globalMetricsV3Once.Do(func() { + globalMetricsV3CollectorPaths = metricGroups.collectorPaths + }) + return &metricsV3Server{ + registry: registry, + opts: promhttp.HandlerOpts{ + ErrorLog: promLogger{}, + ErrorHandling: promhttp.ContinueOnError, + Registry: registry, + MaxRequestsInFlight: 2, + EnableOpenMetrics: env.Get(EnvPrometheusOpenMetrics, config.EnableOff) == config.EnableOn, + ProcessStartTime: globalBootTime, + }, + auth: auth, + metricsData: metricGroups, + } +} + +// metricDisplay - contains info on a metric for display purposes. +type metricDisplay struct { + Name string `json:"name"` + Help string `json:"help"` + Type string `json:"type"` + Labels []string `json:"labels"` +} + +func (md metricDisplay) String() string { + return fmt.Sprintf("Name: %s\nType: %s\nHelp: %s\nLabels: {%s}\n", md.Name, md.Type, md.Help, strings.Join(md.Labels, ",")) +} + +func (md metricDisplay) TableRow() string { + labels := strings.Join(md.Labels, ",") + if labels == "" { + labels = "" + } else { + labels = "`" + labels + "`" + } + return fmt.Sprintf("| `%s` | `%s` | %s | %s |\n", md.Name, md.Type, md.Help, labels) +} + +// listMetrics - returns a handler that lists all the metrics that could be +// returned for the requested path. +// +// FIXME: It currently only lists `minio_` prefixed metrics. +func (h *metricsV3Server) listMetrics(path string) http.Handler { + // First collect all matching MetricsGroup's + matchingMG := make(map[collectorPath]*MetricsGroup) + for _, collPath := range h.metricsData.collectorPaths { + if collPath.isDescendantOf(path) { + if v, ok := h.metricsData.mgMap[collPath]; ok { + matchingMG[collPath] = v + } else if v, ok := h.metricsData.bucketMGMap[collPath]; ok { + matchingMG[collPath] = v + } + } + } + + if len(matchingMG) == 0 { + return nil + } + + var metrics []metricDisplay + for _, collectorPath := range h.metricsData.collectorPaths { + if mg, ok := matchingMG[collectorPath]; ok { + var commonLabels []string + for k := range mg.ExtraLabels { + commonLabels = append(commonLabels, k) + } + for _, d := range mg.Descriptors { + labels := slices.Clone(d.VariableLabels) + labels = append(labels, commonLabels...) + metric := metricDisplay{ + Name: mg.MetricFQN(d.Name), + Help: d.Help, + Type: d.Type.String(), + Labels: labels, + } + metrics = append(metrics, metric) + } + } + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + contentType := r.Header.Get("Content-Type") + if contentType == "application/json" { + w.Header().Set("Content-Type", "application/json") + jsonEncoder := json.NewEncoder(w) + jsonEncoder.Encode(metrics) + return + } + + // If not JSON, return plain text. We format it as a markdown table for + // readability. + w.Header().Set("Content-Type", "text/plain") + var b strings.Builder + b.WriteString("| Name | Type | Help | Labels |\n") + b.WriteString("| ---- | ---- | ---- | ------ |\n") + for _, metric := range metrics { + b.WriteString(metric.TableRow()) + } + w.Write([]byte(b.String())) + }) +} + +func (h *metricsV3Server) handle(path string, isListingRequest bool, buckets []string) http.Handler { + var notFoundHandler http.Handler = http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "Metrics Resource Not found", http.StatusNotFound) + }) + + // Require that metrics path has one component at least. + if path == "/" { + return notFoundHandler + } + + if isListingRequest { + handler := h.listMetrics(path) + if handler == nil { + return notFoundHandler + } + return handler + } + + // In each of the following cases, we check if the collect path is a + // descendant of `path`, and if so, we add the corresponding gatherer to + // the list of gatherers. This way, /api/a will return all metrics returned + // by /api/a/b and /api/a/c (and any other matching descendant collector + // paths). + + var gatherers []prometheus.Gatherer + for _, collectorPath := range h.metricsData.collectorPaths { + if collectorPath.isDescendantOf(path) { + gatherer := h.metricsData.mgGatherers[collectorPath] + + // For Bucket metrics we need to set the buckets argument inside the + // metric group, so that it will affect collection. If no buckets + // are provided, we will not return bucket metrics. + if bmg, ok := h.metricsData.bucketMGMap[collectorPath]; ok { + if len(buckets) == 0 { + continue + } + unLocker := bmg.LockAndSetBuckets(buckets) + defer unLocker() + } + gatherers = append(gatherers, gatherer) + } + } + + if len(gatherers) == 0 { + return notFoundHandler + } + + return promhttp.HandlerFor(prometheus.Gatherers(gatherers), h.opts) +} + +// ServeHTTP - implements http.Handler interface. +// +// When the `list` query parameter is provided (its value is ignored), the +// server lists all metrics that could be returned for the requested path. +// +// The (repeatable) `buckets` query parameter is a list of bucket names (or it +// could be a comma separated value) to return metrics with a bucket label. +// Bucket metrics will be returned only for the provided buckets. If no buckets +// parameter is provided, no bucket metrics are returned. +func (h *metricsV3Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + pathComponents := mux.Vars(r)["pathComps"] + isListingRequest := r.Form.Has("list") + + var buckets []string + if strings.HasPrefix(pathComponents, "/bucket/") { + // bucket specific metrics, extract the bucket name from the path. + // it's the last part of the path. e.g. /bucket/api/ + bucketIdx := strings.LastIndex(pathComponents, "/") + buckets = append(buckets, pathComponents[bucketIdx+1:]) + // remove bucket from pathComponents as it is dynamic and + // hence not included in the collector path. + pathComponents = pathComponents[:bucketIdx] + } + + innerHandler := h.handle(pathComponents, isListingRequest, buckets) + + // Add tracing to the prom. handler + tracedHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt) + if ok { + tc.FuncName = "handler.MetricsV3" + tc.ResponseRecorder.LogErrBody = true + } + + innerHandler.ServeHTTP(w, r) + }) + + // Add authentication + h.auth(tracedHandler).ServeHTTP(w, r) +} diff --git a/cmd/metrics-v3-ilm.go b/cmd/metrics-v3-ilm.go new file mode 100644 index 00000000..604beaae --- /dev/null +++ b/cmd/metrics-v3-ilm.go @@ -0,0 +1,53 @@ +// Copyright (c) 2024 MinIO, Inc. +// +// # This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" +) + +const ( + expiryPendingTasks = "expiry_pending_tasks" + transitionActiveTasks = "transition_active_tasks" + transitionPendingTasks = "transition_pending_tasks" + transitionMissedImmediateTasks = "transition_missed_immediate_tasks" + versionsScanned = "versions_scanned" +) + +var ( + ilmExpiryPendingTasksMD = NewGaugeMD(expiryPendingTasks, "Number of pending ILM expiry tasks in the queue") + ilmTransitionActiveTasksMD = NewGaugeMD(transitionActiveTasks, "Number of active ILM transition tasks") + ilmTransitionPendingTasksMD = NewGaugeMD(transitionPendingTasks, "Number of pending ILM transition tasks in the queue") + ilmTransitionMissedImmediateTasksMD = NewCounterMD(transitionMissedImmediateTasks, "Number of missed immediate ILM transition tasks") + ilmVersionsScannedMD = NewCounterMD(versionsScanned, "Total number of object versions checked for ILM actions since server start") +) + +// loadILMMetrics - `MetricsLoaderFn` for ILM metrics. +func loadILMMetrics(_ context.Context, m MetricValues, _ *metricsCache) error { + if globalExpiryState != nil { + m.Set(expiryPendingTasks, float64(globalExpiryState.PendingTasks())) + } + if globalTransitionState != nil { + m.Set(transitionActiveTasks, float64(globalTransitionState.ActiveTasks())) + m.Set(transitionPendingTasks, float64(globalTransitionState.PendingTasks())) + m.Set(transitionMissedImmediateTasks, float64(globalTransitionState.MissedImmediateTasks())) + } + m.Set(versionsScanned, float64(globalScannerMetrics.lifetime(scannerMetricILM))) + + return nil +} diff --git a/cmd/metrics-v3-logger-webhook.go b/cmd/metrics-v3-logger-webhook.go new file mode 100644 index 00000000..aa85de94 --- /dev/null +++ b/cmd/metrics-v3-logger-webhook.go @@ -0,0 +1,59 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + + "github.com/minio/minio/internal/logger" +) + +const ( + webhookQueueLength = "queue_length" + webhookTotalMessages = "total_messages" + webhookFailedMessages = "failed_messages" + nameL = "name" + endpointL = "endpoint" +) + +var ( + allWebhookLabels = []string{nameL, endpointL} + webhookFailedMessagesMD = NewCounterMD(webhookFailedMessages, + "Number of messages that failed to send", + allWebhookLabels...) + webhookQueueLengthMD = NewGaugeMD(webhookQueueLength, + "Webhook queue length", + allWebhookLabels...) + webhookTotalMessagesMD = NewCounterMD(webhookTotalMessages, + "Total number of messages sent to this target", + allWebhookLabels...) +) + +// loadLoggerWebhookMetrics - `MetricsLoaderFn` for logger webhook +// such as failed messages and total messages. +func loadLoggerWebhookMetrics(ctx context.Context, m MetricValues, c *metricsCache) error { + tgts := append(logger.SystemTargets(), logger.AuditTargets()...) + for _, t := range tgts { + labels := []string{nameL, t.String(), endpointL, t.Endpoint()} + m.Set(webhookFailedMessages, float64(t.Stats().FailedMessages), labels...) + m.Set(webhookQueueLength, float64(t.Stats().QueueLength), labels...) + m.Set(webhookTotalMessages, float64(t.Stats().TotalMessages), labels...) + } + + return nil +} diff --git a/cmd/metrics-v3-replication.go b/cmd/metrics-v3-replication.go new file mode 100644 index 00000000..44a8e87a --- /dev/null +++ b/cmd/metrics-v3-replication.go @@ -0,0 +1,101 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" +) + +const ( + replicationAverageActiveWorkers = "average_active_workers" + replicationAverageQueuedBytes = "average_queued_bytes" + replicationAverageQueuedCount = "average_queued_count" + replicationAverageDataTransferRate = "average_data_transfer_rate" + replicationCurrentActiveWorkers = "current_active_workers" + replicationCurrentDataTransferRate = "current_data_transfer_rate" + replicationLastMinuteQueuedBytes = "last_minute_queued_bytes" + replicationLastMinuteQueuedCount = "last_minute_queued_count" + replicationMaxActiveWorkers = "max_active_workers" + replicationMaxQueuedBytes = "max_queued_bytes" + replicationMaxQueuedCount = "max_queued_count" + replicationMaxDataTransferRate = "max_data_transfer_rate" + replicationRecentBacklogCount = "recent_backlog_count" +) + +var ( + replicationAverageActiveWorkersMD = NewGaugeMD(replicationAverageActiveWorkers, + "Average number of active replication workers") + replicationAverageQueuedBytesMD = NewGaugeMD(replicationAverageQueuedBytes, + "Average number of bytes queued for replication since server start") + replicationAverageQueuedCountMD = NewGaugeMD(replicationAverageQueuedCount, + "Average number of objects queued for replication since server start") + replicationAverageDataTransferRateMD = NewGaugeMD(replicationAverageDataTransferRate, + "Average replication data transfer rate in bytes/sec") + replicationCurrentActiveWorkersMD = NewGaugeMD(replicationCurrentActiveWorkers, + "Total number of active replication workers") + replicationCurrentDataTransferRateMD = NewGaugeMD(replicationCurrentDataTransferRate, + "Current replication data transfer rate in bytes/sec") + replicationLastMinuteQueuedBytesMD = NewGaugeMD(replicationLastMinuteQueuedBytes, + "Number of bytes queued for replication in the last full minute") + replicationLastMinuteQueuedCountMD = NewGaugeMD(replicationLastMinuteQueuedCount, + "Number of objects queued for replication in the last full minute") + replicationMaxActiveWorkersMD = NewGaugeMD(replicationMaxActiveWorkers, + "Maximum number of active replication workers seen since server start") + replicationMaxQueuedBytesMD = NewGaugeMD(replicationMaxQueuedBytes, + "Maximum number of bytes queued for replication since server start") + replicationMaxQueuedCountMD = NewGaugeMD(replicationMaxQueuedCount, + "Maximum number of objects queued for replication since server start") + replicationMaxDataTransferRateMD = NewGaugeMD(replicationMaxDataTransferRate, + "Maximum replication data transfer rate in bytes/sec seen since server start") + replicationRecentBacklogCountMD = NewGaugeMD(replicationRecentBacklogCount, + "Total number of objects seen in replication backlog in the last 5 minutes") +) + +// loadClusterReplicationMetrics - `MetricsLoaderFn` for cluster replication metrics +// such as transfer rate and objects queued. +func loadClusterReplicationMetrics(ctx context.Context, m MetricValues, c *metricsCache) error { + st := globalReplicationStats.Load() + if st == nil { + return nil + } + + qs := st.getNodeQueueStatsSummary() + + qt := qs.QStats + m.Set(replicationAverageQueuedBytes, float64(qt.Avg.Bytes)) + m.Set(replicationAverageQueuedCount, float64(qt.Avg.Count)) + m.Set(replicationMaxQueuedBytes, float64(qt.Max.Bytes)) + m.Set(replicationMaxQueuedCount, float64(qt.Max.Count)) + m.Set(replicationLastMinuteQueuedBytes, float64(qt.Curr.Bytes)) + m.Set(replicationLastMinuteQueuedCount, float64(qt.Curr.Count)) + + qa := qs.ActiveWorkers + m.Set(replicationAverageActiveWorkers, float64(qa.Avg)) + m.Set(replicationCurrentActiveWorkers, float64(qa.Curr)) + m.Set(replicationMaxActiveWorkers, float64(qa.Max)) + + if len(qs.XferStats) > 0 { + tots := qs.XferStats[Total] + m.Set(replicationAverageDataTransferRate, tots.Avg) + m.Set(replicationCurrentDataTransferRate, tots.Curr) + m.Set(replicationMaxDataTransferRate, tots.Peak) + } + m.Set(replicationRecentBacklogCount, float64(qs.MRFStats.LastFailedCount)) + + return nil +} diff --git a/cmd/metrics-v3-scanner.go b/cmd/metrics-v3-scanner.go new file mode 100644 index 00000000..8f661e23 --- /dev/null +++ b/cmd/metrics-v3-scanner.go @@ -0,0 +1,66 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "time" +) + +const ( + scannerBucketScansFinished = "bucket_scans_finished" + scannerBucketScansStarted = "bucket_scans_started" + scannerDirectoriesScanned = "directories_scanned" + scannerObjectsScanned = "objects_scanned" + scannerVersionsScanned = "versions_scanned" + scannerLastActivitySeconds = "last_activity_seconds" +) + +var ( + scannerBucketScansFinishedMD = NewCounterMD(scannerBucketScansFinished, + "Total number of bucket scans finished since server start") + scannerBucketScansStartedMD = NewCounterMD(scannerBucketScansStarted, + "Total number of bucket scans started since server start") + scannerDirectoriesScannedMD = NewCounterMD(scannerDirectoriesScanned, + "Total number of directories scanned since server start") + scannerObjectsScannedMD = NewCounterMD(scannerObjectsScanned, + "Total number of unique objects scanned since server start") + scannerVersionsScannedMD = NewCounterMD(scannerVersionsScanned, + "Total number of object versions scanned since server start") + scannerLastActivitySecondsMD = NewGaugeMD(scannerLastActivitySeconds, + "Time elapsed (in seconds) since last scan activity.") +) + +// loadClusterScannerMetrics - `MetricsLoaderFn` for cluster webhook +// such as failed objects and directories scanned. +func loadClusterScannerMetrics(ctx context.Context, m MetricValues, c *metricsCache) error { + m.Set(scannerBucketScansFinished, float64(globalScannerMetrics.lifetime(scannerMetricScanBucketDrive))) + m.Set(scannerBucketScansStarted, float64(globalScannerMetrics.lifetime(scannerMetricScanBucketDrive)+uint64(globalScannerMetrics.activeDrives()))) + m.Set(scannerDirectoriesScanned, float64(globalScannerMetrics.lifetime(scannerMetricScanFolder))) + m.Set(scannerObjectsScanned, float64(globalScannerMetrics.lifetime(scannerMetricScanObject))) + m.Set(scannerVersionsScanned, float64(globalScannerMetrics.lifetime(scannerMetricApplyVersion))) + + dui, err := c.dataUsageInfo.Get() + if err != nil { + metricsLogIf(ctx, err) + } else { + m.Set(scannerLastActivitySeconds, time.Since(dui.LastUpdate).Seconds()) + } + + return nil +} diff --git a/cmd/metrics-v3-system-cpu.go b/cmd/metrics-v3-system-cpu.go new file mode 100644 index 00000000..cb31b835 --- /dev/null +++ b/cmd/metrics-v3-system-cpu.go @@ -0,0 +1,84 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// # This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "math" +) + +const ( + sysCPUAvgIdle = "avg_idle" + sysCPUAvgIOWait = "avg_iowait" + sysCPULoad = "load" + sysCPULoadPerc = "load_perc" + sysCPUNice = "nice" + sysCPUSteal = "steal" + sysCPUSystem = "system" + sysCPUUser = "user" +) + +var ( + sysCPUAvgIdleMD = NewGaugeMD(sysCPUAvgIdle, "Average CPU idle time") + sysCPUAvgIOWaitMD = NewGaugeMD(sysCPUAvgIOWait, "Average CPU IOWait time") + sysCPULoadMD = NewGaugeMD(sysCPULoad, "CPU load average 1min") + sysCPULoadPercMD = NewGaugeMD(sysCPULoadPerc, "CPU load average 1min (percentage)") + sysCPUNiceMD = NewGaugeMD(sysCPUNice, "CPU nice time") + sysCPUStealMD = NewGaugeMD(sysCPUSteal, "CPU steal time") + sysCPUSystemMD = NewGaugeMD(sysCPUSystem, "CPU system time") + sysCPUUserMD = NewGaugeMD(sysCPUUser, "CPU user time") +) + +// loadCPUMetrics - `MetricsLoaderFn` for system CPU metrics. +func loadCPUMetrics(ctx context.Context, m MetricValues, c *metricsCache) error { + cpuMetrics, _ := c.cpuMetrics.Get() + + if cpuMetrics.LoadStat != nil { + m.Set(sysCPULoad, cpuMetrics.LoadStat.Load1) + perc := cpuMetrics.LoadStat.Load1 * 100 / float64(cpuMetrics.CPUCount) + m.Set(sysCPULoadPerc, math.Round(perc*100)/100) + } + + ts := cpuMetrics.TimesStat + if ts != nil { + tot := ts.User + ts.System + ts.Idle + ts.Iowait + ts.Nice + ts.Steal + cpuUserVal := math.Round(ts.User/tot*100*100) / 100 + m.Set(sysCPUUser, cpuUserVal) + cpuSystemVal := math.Round(ts.System/tot*100*100) / 100 + m.Set(sysCPUSystem, cpuSystemVal) + cpuNiceVal := math.Round(ts.Nice/tot*100*100) / 100 + m.Set(sysCPUNice, cpuNiceVal) + cpuStealVal := math.Round(ts.Steal/tot*100*100) / 100 + m.Set(sysCPUSteal, cpuStealVal) + } + + // metrics-resource.go runs a job to collect resource metrics including their Avg values and + // stores them in resourceMetricsMap. We can use it to get the Avg values of CPU idle and IOWait. + cpuResourceMetrics, found := resourceMetricsMap[cpuSubsystem] + if found { + if cpuIdleMetric, ok := cpuResourceMetrics[getResourceKey(cpuIdle, nil)]; ok { + avgVal := math.Round(cpuIdleMetric.Avg*100) / 100 + m.Set(sysCPUAvgIdle, avgVal) + } + if cpuIOWaitMetric, ok := cpuResourceMetrics[getResourceKey(cpuIOWait, nil)]; ok { + avgVal := math.Round(cpuIOWaitMetric.Avg*100) / 100 + m.Set(sysCPUAvgIOWait, avgVal) + } + } + return nil +} diff --git a/cmd/metrics-v3-system-drive.go b/cmd/metrics-v3-system-drive.go new file mode 100644 index 00000000..d25a623c --- /dev/null +++ b/cmd/metrics-v3-system-drive.go @@ -0,0 +1,234 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "strconv" + + "github.com/minio/madmin-go/v3" +) + +// label constants +const ( + driveL = "drive" + poolIndexL = "pool_index" + setIndexL = "set_index" + driveIndexL = "drive_index" + + apiL = "api" + + sectorSize = uint64(512) + kib = float64(1 << 10) + + driveHealthOffline = float64(0) + driveHealthOnline = float64(1) + driveHealthHealing = float64(2) +) + +var allDriveLabels = []string{driveL, poolIndexL, setIndexL, driveIndexL} + +const ( + driveUsedBytes = "used_bytes" + driveFreeBytes = "free_bytes" + driveTotalBytes = "total_bytes" + driveUsedInodes = "used_inodes" + driveFreeInodes = "free_inodes" + driveTotalInodes = "total_inodes" + driveTimeoutErrorsTotal = "timeout_errors_total" + driveIOErrorsTotal = "io_errors_total" + driveAvailabilityErrorsTotal = "availability_errors_total" + driveWaitingIO = "waiting_io" + driveAPILatencyMicros = "api_latency_micros" + driveHealth = "health" + + driveOfflineCount = "offline_count" + driveOnlineCount = "online_count" + driveCount = "count" + + // iostat related + driveReadsPerSec = "reads_per_sec" + driveReadsKBPerSec = "reads_kb_per_sec" + driveReadsAwait = "reads_await" + driveWritesPerSec = "writes_per_sec" + driveWritesKBPerSec = "writes_kb_per_sec" + driveWritesAwait = "writes_await" + drivePercUtil = "perc_util" +) + +var ( + driveUsedBytesMD = NewGaugeMD(driveUsedBytes, + "Total storage used on a drive in bytes", allDriveLabels...) + driveFreeBytesMD = NewGaugeMD(driveFreeBytes, + "Total storage free on a drive in bytes", allDriveLabels...) + driveTotalBytesMD = NewGaugeMD(driveTotalBytes, + "Total storage available on a drive in bytes", allDriveLabels...) + driveUsedInodesMD = NewGaugeMD(driveUsedInodes, + "Total used inodes on a drive", allDriveLabels...) + driveFreeInodesMD = NewGaugeMD(driveFreeInodes, + "Total free inodes on a drive", allDriveLabels...) + driveTotalInodesMD = NewGaugeMD(driveTotalInodes, + "Total inodes available on a drive", allDriveLabels...) + driveTimeoutErrorsMD = NewCounterMD(driveTimeoutErrorsTotal, + "Total timeout errors on a drive", allDriveLabels...) + driveIOErrorsMD = NewCounterMD(driveIOErrorsTotal, + "Total I/O errors on a drive", allDriveLabels...) + driveAvailabilityErrorsMD = NewCounterMD(driveAvailabilityErrorsTotal, + "Total availability errors (I/O errors, timeouts) on a drive", + allDriveLabels...) + driveWaitingIOMD = NewGaugeMD(driveWaitingIO, + "Total waiting I/O operations on a drive", allDriveLabels...) + driveAPILatencyMD = NewGaugeMD(driveAPILatencyMicros, + "Average last minute latency in µs for drive API storage operations", + append(allDriveLabels, apiL)...) + driveHealthMD = NewGaugeMD(driveHealth, + "Drive health (0 = offline, 1 = healthy, 2 = healing)", allDriveLabels...) + + driveOfflineCountMD = NewGaugeMD(driveOfflineCount, + "Count of offline drives") + driveOnlineCountMD = NewGaugeMD(driveOnlineCount, + "Count of online drives") + driveCountMD = NewGaugeMD(driveCount, + "Count of all drives") + + // iostat related + driveReadsPerSecMD = NewGaugeMD(driveReadsPerSec, + "Reads per second on a drive", + allDriveLabels...) + driveReadsKBPerSecMD = NewGaugeMD(driveReadsKBPerSec, + "Kilobytes read per second on a drive", + allDriveLabels...) + driveReadsAwaitMD = NewGaugeMD(driveReadsAwait, + "Average time for read requests served on a drive", + allDriveLabels...) + driveWritesPerSecMD = NewGaugeMD(driveWritesPerSec, + "Writes per second on a drive", + allDriveLabels...) + driveWritesKBPerSecMD = NewGaugeMD(driveWritesKBPerSec, + "Kilobytes written per second on a drive", + allDriveLabels...) + driveWritesAwaitMD = NewGaugeMD(driveWritesAwait, + "Average time for write requests served on a drive", + allDriveLabels...) + drivePercUtilMD = NewGaugeMD(drivePercUtil, + "Percentage of time the disk was busy", + allDriveLabels...) +) + +func getCurrentDriveIOStats() map[string]madmin.DiskIOStats { + types := madmin.MetricsDisk + driveRealtimeMetrics := collectLocalMetrics(types, collectMetricsOpts{ + hosts: map[string]struct{}{ + globalLocalNodeName: {}, + }, + }) + + stats := map[string]madmin.DiskIOStats{} + for d, m := range driveRealtimeMetrics.ByDisk { + stats[d] = m.IOStats + } + return stats +} + +func (m *MetricValues) setDriveBasicMetrics(drive madmin.Disk, labels []string) { + m.Set(driveUsedBytes, float64(drive.UsedSpace), labels...) + m.Set(driveFreeBytes, float64(drive.AvailableSpace), labels...) + m.Set(driveTotalBytes, float64(drive.TotalSpace), labels...) + m.Set(driveUsedInodes, float64(drive.UsedInodes), labels...) + m.Set(driveFreeInodes, float64(drive.FreeInodes), labels...) + m.Set(driveTotalInodes, float64(drive.UsedInodes+drive.FreeInodes), labels...) + + var health float64 + switch drive.Healing { + case true: + health = driveHealthHealing + case false: + if drive.State == "ok" { + health = driveHealthOnline + } else { + health = driveHealthOffline + } + } + m.Set(driveHealth, health, labels...) +} + +func (m *MetricValues) setDriveAPIMetrics(disk madmin.Disk, labels []string) { + if disk.Metrics == nil { + return + } + + m.Set(driveTimeoutErrorsTotal, float64(disk.Metrics.TotalErrorsTimeout), labels...) + m.Set(driveIOErrorsTotal, float64(disk.Metrics.TotalErrorsAvailability-disk.Metrics.TotalErrorsTimeout), labels...) + m.Set(driveAvailabilityErrorsTotal, float64(disk.Metrics.TotalErrorsAvailability), labels...) + m.Set(driveWaitingIO, float64(disk.Metrics.TotalWaiting), labels...) + + // Append the api label for the drive API latencies. + labels = append(labels, "api", "") + lastIdx := len(labels) - 1 + for apiName, latency := range disk.Metrics.LastMinute { + labels[lastIdx] = "storage." + apiName + m.Set(driveAPILatencyMicros, float64(latency.Avg().Microseconds()), + labels...) + } +} + +func (m *MetricValues) setDriveIOStatMetrics(ioStats driveIOStatMetrics, labels []string) { + m.Set(driveReadsPerSec, ioStats.readsPerSec, labels...) + m.Set(driveReadsKBPerSec, ioStats.readsKBPerSec, labels...) + if ioStats.readsPerSec > 0 { + m.Set(driveReadsAwait, ioStats.readsAwait, labels...) + } + + m.Set(driveWritesPerSec, ioStats.writesPerSec, labels...) + m.Set(driveWritesKBPerSec, ioStats.writesKBPerSec, labels...) + if ioStats.writesPerSec > 0 { + m.Set(driveWritesAwait, ioStats.writesAwait, labels...) + } + + m.Set(drivePercUtil, ioStats.percUtil, labels...) +} + +// loadDriveMetrics - `MetricsLoaderFn` for node drive metrics. +func loadDriveMetrics(ctx context.Context, m MetricValues, c *metricsCache) error { + driveMetrics, err := c.driveMetrics.Get() + if err != nil { + metricsLogIf(ctx, err) + return nil + } + + for _, disk := range driveMetrics.storageInfo.Disks { + labels := []string{ + driveL, disk.DrivePath, + poolIndexL, strconv.Itoa(disk.PoolIndex), + setIndexL, strconv.Itoa(disk.SetIndex), + driveIndexL, strconv.Itoa(disk.DiskIndex), + } + + m.setDriveBasicMetrics(disk, labels) + if dm, found := driveMetrics.ioStats[disk.DrivePath]; found { + m.setDriveIOStatMetrics(dm, labels) + } + m.setDriveAPIMetrics(disk, labels) + } + + m.Set(driveOfflineCount, float64(driveMetrics.offlineDrives)) + m.Set(driveOnlineCount, float64(driveMetrics.onlineDrives)) + m.Set(driveCount, float64(driveMetrics.totalDrives)) + + return nil +} diff --git a/cmd/metrics-v3-system-memory.go b/cmd/metrics-v3-system-memory.go new file mode 100644 index 00000000..f304631b --- /dev/null +++ b/cmd/metrics-v3-system-memory.go @@ -0,0 +1,65 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// # This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" +) + +const ( + memTotal = "total" + memUsed = "used" + memFree = "free" + memBuffers = "buffers" + memCache = "cache" + memUsedPerc = "used_perc" + memShared = "shared" + memAvailable = "available" +) + +var ( + memTotalMD = NewGaugeMD(memTotal, "Total memory on the node") + memUsedMD = NewGaugeMD(memUsed, "Used memory on the node") + memUsedPercMD = NewGaugeMD(memUsedPerc, "Used memory percentage on the node") + memFreeMD = NewGaugeMD(memFree, "Free memory on the node") + memBuffersMD = NewGaugeMD(memBuffers, "Buffers memory on the node") + memCacheMD = NewGaugeMD(memCache, "Cache memory on the node") + memSharedMD = NewGaugeMD(memShared, "Shared memory on the node") + memAvailableMD = NewGaugeMD(memAvailable, "Available memory on the node") +) + +// loadMemoryMetrics - `MetricsLoaderFn` for node memory metrics. +func loadMemoryMetrics(ctx context.Context, m MetricValues, c *metricsCache) error { + memMetrics, err := c.memoryMetrics.Get() + if err != nil { + metricsLogIf(ctx, err) + return err + } + + m.Set(memTotal, float64(memMetrics.Total)) + m.Set(memUsed, float64(memMetrics.Used)) + usedPerc := float64(memMetrics.Used) * 100 / float64(memMetrics.Total) + m.Set(memUsedPerc, usedPerc) + m.Set(memFree, float64(memMetrics.Free)) + m.Set(memBuffers, float64(memMetrics.Buffers)) + m.Set(memCache, float64(memMetrics.Cache)) + m.Set(memShared, float64(memMetrics.Shared)) + m.Set(memAvailable, float64(memMetrics.Available)) + + return nil +} diff --git a/cmd/metrics-v3-system-network.go b/cmd/metrics-v3-system-network.go new file mode 100644 index 00000000..e0328afc --- /dev/null +++ b/cmd/metrics-v3-system-network.go @@ -0,0 +1,61 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + + "github.com/minio/minio/internal/rest" +) + +const ( + internodeErrorsTotal MetricName = "errors_total" + internodeDialErrorsTotal MetricName = "dial_errors_total" + internodeDialAvgTimeNanos MetricName = "dial_avg_time_nanos" + internodeSentBytesTotal MetricName = "sent_bytes_total" + internodeRecvBytesTotal MetricName = "recv_bytes_total" +) + +var ( + internodeErrorsTotalMD = NewCounterMD(internodeErrorsTotal, + "Total number of failed internode calls") + internodeDialedErrorsTotalMD = NewCounterMD(internodeDialErrorsTotal, + "Total number of internode TCP dial timeouts and errors") + internodeDialAvgTimeNanosMD = NewGaugeMD(internodeDialAvgTimeNanos, + "Average dial time of internode TCP calls in nanoseconds") + internodeSentBytesTotalMD = NewCounterMD(internodeSentBytesTotal, + "Total number of bytes sent to other peer nodes") + internodeRecvBytesTotalMD = NewCounterMD(internodeRecvBytesTotal, + "Total number of bytes received from other peer nodes") +) + +// loadNetworkInternodeMetrics - reads internode network metrics. +// +// This is a `MetricsLoaderFn`. +func loadNetworkInternodeMetrics(ctx context.Context, m MetricValues, _ *metricsCache) error { + connStats := globalConnStats.toServerConnStats() + rpcStats := rest.GetRPCStats() + if globalIsDistErasure { + m.Set(internodeErrorsTotal, float64(rpcStats.Errs)) + m.Set(internodeDialErrorsTotal, float64(rpcStats.DialErrs)) + m.Set(internodeDialAvgTimeNanos, float64(rpcStats.DialAvgDuration)) + m.Set(internodeSentBytesTotal, float64(connStats.internodeOutputBytes)) + m.Set(internodeRecvBytesTotal, float64(connStats.internodeInputBytes)) + } + return nil +} diff --git a/cmd/metrics-v3-system-process.go b/cmd/metrics-v3-system-process.go new file mode 100644 index 00000000..01dbba88 --- /dev/null +++ b/cmd/metrics-v3-system-process.go @@ -0,0 +1,174 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// # This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "runtime" + "time" + + "github.com/prometheus/procfs" +) + +const ( + processLocksReadTotal = "locks_read_total" + processLocksWriteTotal = "locks_write_total" + processCPUTotalSeconds = "cpu_total_seconds" + processGoRoutineTotal = "go_routine_total" + processIORCharBytes = "io_rchar_bytes" + processIOReadBytes = "io_read_bytes" + processIOWCharBytes = "io_wchar_bytes" + processIOWriteBytes = "io_write_bytes" + processStartTimeSeconds = "start_time_seconds" + processUptimeSeconds = "uptime_seconds" + processFileDescriptorLimitTotal = "file_descriptor_limit_total" + processFileDescriptorOpenTotal = "file_descriptor_open_total" + processSyscallReadTotal = "syscall_read_total" + processSyscallWriteTotal = "syscall_write_total" + processResidentMemoryBytes = "resident_memory_bytes" + processVirtualMemoryBytes = "virtual_memory_bytes" + processVirtualMemoryMaxBytes = "virtual_memory_max_bytes" +) + +var ( + processLocksReadTotalMD = NewGaugeMD(processLocksReadTotal, "Number of current READ locks on this peer") + processLocksWriteTotalMD = NewGaugeMD(processLocksWriteTotal, "Number of current WRITE locks on this peer") + processCPUTotalSecondsMD = NewCounterMD(processCPUTotalSeconds, "Total user and system CPU time spent in seconds") + processGoRoutineTotalMD = NewGaugeMD(processGoRoutineTotal, "Total number of go routines running") + processIORCharBytesMD = NewCounterMD(processIORCharBytes, "Total bytes read by the process from the underlying storage system including cache, /proc/[pid]/io rchar") + processIOReadBytesMD = NewCounterMD(processIOReadBytes, "Total bytes read by the process from the underlying storage system, /proc/[pid]/io read_bytes") + processIOWCharBytesMD = NewCounterMD(processIOWCharBytes, "Total bytes written by the process to the underlying storage system including page cache, /proc/[pid]/io wchar") + processIOWriteBytesMD = NewCounterMD(processIOWriteBytes, "Total bytes written by the process to the underlying storage system, /proc/[pid]/io write_bytes") + processStarttimeSecondsMD = NewGaugeMD(processStartTimeSeconds, "Start time for MinIO process in seconds since Unix epoc") + processUptimeSecondsMD = NewGaugeMD(processUptimeSeconds, "Uptime for MinIO process in seconds") + processFileDescriptorLimitTotalMD = NewGaugeMD(processFileDescriptorLimitTotal, "Limit on total number of open file descriptors for the MinIO Server process") + processFileDescriptorOpenTotalMD = NewGaugeMD(processFileDescriptorOpenTotal, "Total number of open file descriptors by the MinIO Server process") + processSyscallReadTotalMD = NewCounterMD(processSyscallReadTotal, "Total read SysCalls to the kernel. /proc/[pid]/io syscr") + processSyscallWriteTotalMD = NewCounterMD(processSyscallWriteTotal, "Total write SysCalls to the kernel. /proc/[pid]/io syscw") + processResidentMemoryBytesMD = NewGaugeMD(processResidentMemoryBytes, "Resident memory size in bytes") + processVirtualMemoryBytesMD = NewGaugeMD(processVirtualMemoryBytes, "Virtual memory size in bytes") + processVirtualMemoryMaxBytesMD = NewGaugeMD(processVirtualMemoryMaxBytes, "Maximum virtual memory size in bytes") +) + +func loadProcStatMetrics(ctx context.Context, stat procfs.ProcStat, m MetricValues) { + if stat.CPUTime() > 0 { + m.Set(processCPUTotalSeconds, float64(stat.CPUTime())) + } + + if stat.ResidentMemory() > 0 { + m.Set(processResidentMemoryBytes, float64(stat.ResidentMemory())) + } + + if stat.VirtualMemory() > 0 { + m.Set(processVirtualMemoryBytes, float64(stat.VirtualMemory())) + } + + startTime, err := stat.StartTime() + if err != nil { + metricsLogIf(ctx, err) + } else if startTime > 0 { + m.Set(processStartTimeSeconds, float64(startTime)) + } +} + +func loadProcIOMetrics(ctx context.Context, io procfs.ProcIO, m MetricValues) { + if io.RChar > 0 { + m.Set(processIORCharBytes, float64(io.RChar)) + } + + if io.ReadBytes > 0 { + m.Set(processIOReadBytes, float64(io.ReadBytes)) + } + + if io.WChar > 0 { + m.Set(processIOWCharBytes, float64(io.WChar)) + } + + if io.WriteBytes > 0 { + m.Set(processIOWriteBytes, float64(io.WriteBytes)) + } + + if io.SyscR > 0 { + m.Set(processSyscallReadTotal, float64(io.SyscR)) + } + + if io.SyscW > 0 { + m.Set(processSyscallWriteTotal, float64(io.SyscW)) + } +} + +func loadProcFSMetrics(ctx context.Context, p procfs.Proc, m MetricValues) { + stat, err := p.Stat() + if err != nil { + metricsLogIf(ctx, err) + } else { + loadProcStatMetrics(ctx, stat, m) + } + + io, err := p.IO() + if err != nil { + metricsLogIf(ctx, err) + } else { + loadProcIOMetrics(ctx, io, m) + } + + l, err := p.Limits() + if err != nil { + metricsLogIf(ctx, err) + } else { + if l.OpenFiles > 0 { + m.Set(processFileDescriptorLimitTotal, float64(l.OpenFiles)) + } + + if l.AddressSpace > 0 { + m.Set(processVirtualMemoryMaxBytes, float64(l.AddressSpace)) + } + } + + openFDs, err := p.FileDescriptorsLen() + if err != nil { + metricsLogIf(ctx, err) + } else if openFDs > 0 { + m.Set(processFileDescriptorOpenTotal, float64(openFDs)) + } +} + +// loadProcessMetrics - `MetricsLoaderFn` for process metrics +func loadProcessMetrics(ctx context.Context, m MetricValues, c *metricsCache) error { + m.Set(processGoRoutineTotal, float64(runtime.NumGoroutine())) + + if !globalBootTime.IsZero() { + m.Set(processUptimeSeconds, time.Since(globalBootTime).Seconds()) + } + + if runtime.GOOS != globalWindowsOSName && runtime.GOOS != globalMacOSName { + p, err := procfs.Self() + if err != nil { + metricsLogIf(ctx, err) + } else { + loadProcFSMetrics(ctx, p, m) + } + } + + if globalIsDistErasure && globalLockServer != nil { + st := globalLockServer.stats() + m.Set(processLocksReadTotal, float64(st.Reads)) + m.Set(processLocksWriteTotal, float64(st.Writes)) + } + return nil +} diff --git a/cmd/metrics-v3-types.go b/cmd/metrics-v3-types.go new file mode 100644 index 00000000..92004c96 --- /dev/null +++ b/cmd/metrics-v3-types.go @@ -0,0 +1,515 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "fmt" + "slices" + "strings" + "sync" + + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/logger" + "github.com/pkg/errors" + "github.com/prometheus/client_golang/prometheus" +) + +type collectorPath string + +// metricPrefix converts a collector path to a metric name prefix. The path is +// converted to snake-case (by replaced '/' and '-' with '_') and prefixed with +// `minio_`. +func (cp collectorPath) metricPrefix() string { + s := strings.TrimPrefix(string(cp), SlashSeparator) + s = strings.ReplaceAll(s, SlashSeparator, "_") + s = strings.ReplaceAll(s, "-", "_") + return "minio_" + s +} + +// isDescendantOf returns true if it is a descendant of (or the same as) +// `ancestor`. +// +// For example: +// +// /a, /a/b, /a/b/c are all descendants of /a. +// /abc or /abd/a are not descendants of /ab. +func (cp collectorPath) isDescendantOf(arg string) bool { + descendant := string(cp) + if descendant == arg { + return true + } + if len(arg) >= len(descendant) { + return false + } + if !strings.HasSuffix(arg, SlashSeparator) { + arg += SlashSeparator + } + return strings.HasPrefix(descendant, arg) +} + +// MetricType - represents the type of a metric. +type MetricType int + +const ( + // CounterMT - represents a counter metric. + CounterMT MetricType = iota + // GaugeMT - represents a gauge metric. + GaugeMT + // HistogramMT - represents a histogram metric. + HistogramMT +) + +// rangeL - represents a range label. +const rangeL = "range" + +func (mt MetricType) String() string { + switch mt { + case CounterMT: + return "counter" + case GaugeMT: + return "gauge" + case HistogramMT: + return "histogram" + default: + return "*unknown*" + } +} + +func (mt MetricType) toProm() prometheus.ValueType { + switch mt { + case CounterMT: + return prometheus.CounterValue + case GaugeMT: + return prometheus.GaugeValue + case HistogramMT: + return prometheus.CounterValue + default: + panic(fmt.Sprintf("unknown metric type: %d", mt)) + } +} + +// MetricDescriptor - represents a metric descriptor. +type MetricDescriptor struct { + Name MetricName + Type MetricType + Help string + VariableLabels []string + + // managed values follow: + labelSet map[string]struct{} +} + +func (md *MetricDescriptor) getLabelSet() map[string]struct{} { + if md.labelSet != nil { + return md.labelSet + } + md.labelSet = make(map[string]struct{}, len(md.VariableLabels)) + for _, label := range md.VariableLabels { + md.labelSet[label] = struct{}{} + } + return md.labelSet +} + +func (md *MetricDescriptor) toPromName(namePrefix string) string { + return prometheus.BuildFQName(namePrefix, "", string(md.Name)) +} + +func (md *MetricDescriptor) toPromDesc(namePrefix string, extraLabels map[string]string) *prometheus.Desc { + return prometheus.NewDesc( + md.toPromName(namePrefix), + md.Help, + md.VariableLabels, extraLabels, + ) +} + +// NewCounterMD - creates a new counter metric descriptor. +func NewCounterMD(name MetricName, help string, labels ...string) MetricDescriptor { + return MetricDescriptor{ + Name: name, + Type: CounterMT, + Help: help, + VariableLabels: labels, + } +} + +// NewGaugeMD - creates a new gauge metric descriptor. +func NewGaugeMD(name MetricName, help string, labels ...string) MetricDescriptor { + return MetricDescriptor{ + Name: name, + Type: GaugeMT, + Help: help, + VariableLabels: labels, + } +} + +type metricValue struct { + Labels map[string]string + Value float64 +} + +// MetricValues - type to set metric values retrieved while loading metrics. A +// value of this type is passed to the `MetricsLoaderFn`. +type MetricValues struct { + values map[MetricName][]metricValue + descriptors map[MetricName]MetricDescriptor +} + +func newMetricValues(d map[MetricName]MetricDescriptor) MetricValues { + return MetricValues{ + values: make(map[MetricName][]metricValue, len(d)), + descriptors: d, + } +} + +// ToPromMetrics - converts the internal metric values to Prometheus +// adding the given name prefix. The extraLabels are added to each metric as +// constant labels. +func (m *MetricValues) ToPromMetrics(namePrefix string, extraLabels map[string]string, +) []prometheus.Metric { + metrics := make([]prometheus.Metric, 0, len(m.values)) + for metricName, mv := range m.values { + desc := m.descriptors[metricName] + promDesc := desc.toPromDesc(namePrefix, extraLabels) + for _, v := range mv { + // labelValues is in the same order as the variable labels in the + // descriptor. + labelValues := make([]string, 0, len(v.Labels)) + for _, k := range desc.VariableLabels { + labelValues = append(labelValues, v.Labels[k]) + } + metrics = append(metrics, + prometheus.MustNewConstMetric(promDesc, desc.Type.toProm(), v.Value, + labelValues...)) + } + } + return metrics +} + +// Set - sets a metric value along with any provided labels. It is used only +// with Gauge and Counter metrics. +// +// If the MetricName given here is not present in the `MetricsGroup`'s +// descriptors, this function panics. +// +// Panics if `labels` is not a list of ordered label name and label value pairs +// or if all labels for the metric are not provided. +func (m *MetricValues) Set(name MetricName, value float64, labels ...string) { + desc, ok := m.descriptors[name] + if !ok { + panic(fmt.Sprintf("metric has no description: %s", name)) + } + + if len(labels)%2 != 0 { + panic("labels must be a list of ordered key-value pairs") + } + + validLabels := desc.getLabelSet() + labelMap := make(map[string]string, len(labels)/2) + for i := 0; i < len(labels); i += 2 { + if _, ok := validLabels[labels[i]]; !ok { + panic(fmt.Sprintf("invalid label: %s (metric: %s)", labels[i], name)) + } + labelMap[labels[i]] = labels[i+1] + } + + if len(labels)/2 != len(validLabels) { + panic("not all labels were given values") + } + + v, ok := m.values[name] + if !ok { + v = make([]metricValue, 0, 1) + } + // If valid non zero value set the metrics + if value > 0 { + m.values[name] = append(v, metricValue{ + Labels: labelMap, + Value: value, + }) + } +} + +// SetHistogram - sets values for the given MetricName using the provided +// histogram. +// +// `filterByLabels` is a map of label names to list of allowed label values to +// filter by. Note that this filtering happens before any renaming of labels. +// +// `renameLabels` is a map of label names to rename. The keys are the original +// label names and the values are the new label names. +// +// `bucketFilter` is a list of bucket values to filter. If this is non-empty, +// only metrics for the given buckets are added. +// +// `extraLabels` are additional labels to add to each metric. They are ordered +// label name and value pairs. +func (m *MetricValues) SetHistogram(name MetricName, hist *prometheus.HistogramVec, + filterByLabels map[string]set.StringSet, renameLabels map[string]string, bucketFilter []string, + extraLabels ...string, +) { + if _, ok := m.descriptors[name]; !ok { + panic(fmt.Sprintf("metric has no description: %s", name)) + } + dummyDesc := MetricDescription{} + metricsV2 := getHistogramMetrics(hist, dummyDesc, false, false) +mainLoop: + for _, metric := range metricsV2 { + for label, allowedValues := range filterByLabels { + if !allowedValues.Contains(metric.VariableLabels[label]) { + continue mainLoop + } + } + + // If a bucket filter is provided, only add metrics for the given + // buckets. + if len(bucketFilter) > 0 && !slices.Contains(bucketFilter, metric.VariableLabels["bucket"]) { + continue + } + + labels := make([]string, 0, len(metric.VariableLabels)*2) + for k, v := range metric.VariableLabels { + if newLabel, ok := renameLabels[k]; ok { + labels = append(labels, newLabel, v) + } else { + labels = append(labels, k, v) + } + } + labels = append(labels, extraLabels...) + // If valid non zero value set the metrics + if metric.Value > 0 { + m.Set(name, metric.Value, labels...) + } + } +} + +// SetHistogramValues - sets values for the given MetricName using the provided map of +// range to value. +func SetHistogramValues[V uint64 | int64 | float64](m MetricValues, name MetricName, values map[string]V, labels ...string) { + for rng, val := range values { + m.Set(name, float64(val), append(labels, rangeL, rng)...) + } +} + +// MetricsLoaderFn - represents a function to load metrics from the +// metricsCache. +// +// Note that returning an error here will cause the Metrics handler to return a +// 500 Internal Server Error. +type MetricsLoaderFn func(context.Context, MetricValues, *metricsCache) error + +// JoinLoaders - joins multiple loaders into a single loader. The returned +// loader will call each of the given loaders in order. If any of the loaders +// return an error, the returned loader will return that error. +func JoinLoaders(loaders ...MetricsLoaderFn) MetricsLoaderFn { + return func(ctx context.Context, m MetricValues, c *metricsCache) error { + for _, loader := range loaders { + if err := loader(ctx, m, c); err != nil { + return err + } + } + return nil + } +} + +// BucketMetricsLoaderFn - represents a function to load metrics from the +// metricsCache and the system for a given list of buckets. +// +// Note that returning an error here will cause the Metrics handler to return a +// 500 Internal Server Error. +type BucketMetricsLoaderFn func(context.Context, MetricValues, *metricsCache, []string) error + +// JoinBucketLoaders - joins multiple bucket loaders into a single loader, +// similar to `JoinLoaders`. +func JoinBucketLoaders(loaders ...BucketMetricsLoaderFn) BucketMetricsLoaderFn { + return func(ctx context.Context, m MetricValues, c *metricsCache, b []string) error { + for _, loader := range loaders { + if err := loader(ctx, m, c, b); err != nil { + return err + } + } + return nil + } +} + +// MetricsGroup - represents a group of metrics. It includes a `MetricsLoaderFn` +// function that provides a way to load the metrics from the system. The metrics +// are cached and refreshed after a given timeout. +// +// For metrics with a `bucket` dimension, a list of buckets argument is required +// to collect the metrics. +// +// It implements the prometheus.Collector interface for metric groups without a +// bucket dimension. For metric groups with a bucket dimension, use the +// `GetBucketCollector` method to get a `BucketCollector` that implements the +// prometheus.Collector interface. +type MetricsGroup struct { + // Path (relative to the Metrics v3 base endpoint) at which this group of + // metrics is served. This value is converted into a metric name prefix + // using `.metricPrefix()` and is added to each metric returned. + CollectorPath collectorPath + // List of all metric descriptors that could be returned by the loader. + Descriptors []MetricDescriptor + // (Optional) Extra (constant) label KV pairs to be added to each metric in + // the group. + ExtraLabels map[string]string + + // Loader functions to load metrics. Only one of these will be set. Metrics + // returned by these functions must be present in the `Descriptors` list. + loader MetricsLoaderFn + bucketLoader BucketMetricsLoaderFn + + // Cache for all metrics groups. Set via `.SetCache` method. + cache *metricsCache + + // managed values follow: + + // map of metric descriptors by metric name. + descriptorMap map[MetricName]MetricDescriptor + + // For bucket metrics, the list of buckets is stored here. It is used in the + // Collect() call. This is protected by the `bucketsLock`. + bucketsLock sync.Mutex + buckets []string +} + +// NewMetricsGroup creates a new MetricsGroup. To create a metrics group for +// metrics with a `bucket` dimension (label), use `NewBucketMetricsGroup`. +// +// The `loader` function loads metrics from the cache and the system. +func NewMetricsGroup(path collectorPath, descriptors []MetricDescriptor, + loader MetricsLoaderFn, +) *MetricsGroup { + mg := &MetricsGroup{ + CollectorPath: path, + Descriptors: descriptors, + loader: loader, + } + mg.validate() + return mg +} + +// NewBucketMetricsGroup creates a new MetricsGroup for metrics with a `bucket` +// dimension (label). +// +// The `loader` function loads metrics from the cache and the system for a given +// list of buckets. +func NewBucketMetricsGroup(path collectorPath, descriptors []MetricDescriptor, + loader BucketMetricsLoaderFn, +) *MetricsGroup { + mg := &MetricsGroup{ + CollectorPath: path, + Descriptors: descriptors, + bucketLoader: loader, + } + mg.validate() + return mg +} + +// AddExtraLabels - adds extra (constant) label KV pairs to the metrics group. +// This is a helper to initialize the `ExtraLabels` field. The argument is a +// list of ordered label name and value pairs. +func (mg *MetricsGroup) AddExtraLabels(labels ...string) { + if len(labels)%2 != 0 { + panic("Labels must be an ordered list of name value pairs") + } + if mg.ExtraLabels == nil { + mg.ExtraLabels = make(map[string]string, len(labels)) + } + for i := 0; i < len(labels); i += 2 { + mg.ExtraLabels[labels[i]] = labels[i+1] + } +} + +// IsBucketMetricsGroup - returns true if the given MetricsGroup is a bucket +// metrics group. +func (mg *MetricsGroup) IsBucketMetricsGroup() bool { + return mg.bucketLoader != nil +} + +// Describe - implements prometheus.Collector interface. +func (mg *MetricsGroup) Describe(ch chan<- *prometheus.Desc) { + for _, desc := range mg.Descriptors { + ch <- desc.toPromDesc(mg.CollectorPath.metricPrefix(), mg.ExtraLabels) + } +} + +// Collect - implements prometheus.Collector interface. +func (mg *MetricsGroup) Collect(ch chan<- prometheus.Metric) { + metricValues := newMetricValues(mg.descriptorMap) + + var err error + if mg.IsBucketMetricsGroup() { + err = mg.bucketLoader(GlobalContext, metricValues, mg.cache, mg.buckets) + } else { + err = mg.loader(GlobalContext, metricValues, mg.cache) + } + + // There is no way to handle errors here, so we panic the current goroutine + // and the Metrics API handler returns a 500 HTTP status code. This should + // normally not happen, and usually indicates a bug. + logger.CriticalIf(GlobalContext, errors.Wrap(err, "failed to get metrics")) + + promMetrics := metricValues.ToPromMetrics(mg.CollectorPath.metricPrefix(), + mg.ExtraLabels) + for _, metric := range promMetrics { + ch <- metric + } +} + +// LockAndSetBuckets - locks the buckets and sets the given buckets. It returns +// a function to unlock the buckets. +func (mg *MetricsGroup) LockAndSetBuckets(buckets []string) func() { + mg.bucketsLock.Lock() + mg.buckets = buckets + return func() { + mg.bucketsLock.Unlock() + } +} + +// MetricFQN - returns the fully qualified name for the given metric name. +func (mg *MetricsGroup) MetricFQN(name MetricName) string { + v, ok := mg.descriptorMap[name] + if !ok { + // This should never happen. + return "" + } + return v.toPromName(mg.CollectorPath.metricPrefix()) +} + +func (mg *MetricsGroup) validate() { + if len(mg.Descriptors) == 0 { + panic("Descriptors must be set") + } + + // For bools A and B, A XOR B <=> A != B. + isExactlyOneSet := (mg.loader == nil) != (mg.bucketLoader == nil) + if !isExactlyOneSet { + panic("Exactly one Loader function must be set") + } + + mg.descriptorMap = make(map[MetricName]MetricDescriptor, len(mg.Descriptors)) + for _, desc := range mg.Descriptors { + mg.descriptorMap[desc.Name] = desc + } +} + +// SetCache is a helper to initialize MetricsGroup. It sets the cache object. +func (mg *MetricsGroup) SetCache(c *metricsCache) { + mg.cache = c +} diff --git a/cmd/metrics-v3.go b/cmd/metrics-v3.go new file mode 100644 index 00000000..93749258 --- /dev/null +++ b/cmd/metrics-v3.go @@ -0,0 +1,487 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "slices" + "strings" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" +) + +// Collector paths. +// +// These are paths under the top-level /minio/metrics/v3 metrics endpoint. Each +// of these paths returns a set of V3 metrics. +// +// Per-bucket metrics endpoints always start with /bucket and the bucket name is +// appended to the path. e.g. if the collector path is /bucket/api, the endpoint +// for the bucket "mybucket" would be /minio/metrics/v3/bucket/api/mybucket +const ( + apiRequestsCollectorPath collectorPath = "/api/requests" + + bucketAPICollectorPath collectorPath = "/bucket/api" + bucketReplicationCollectorPath collectorPath = "/bucket/replication" + + systemNetworkInternodeCollectorPath collectorPath = "/system/network/internode" + systemDriveCollectorPath collectorPath = "/system/drive" + systemMemoryCollectorPath collectorPath = "/system/memory" + systemCPUCollectorPath collectorPath = "/system/cpu" + systemProcessCollectorPath collectorPath = "/system/process" + + debugGoCollectorPath collectorPath = "/debug/go" + + clusterHealthCollectorPath collectorPath = "/cluster/health" + clusterUsageObjectsCollectorPath collectorPath = "/cluster/usage/objects" + clusterUsageBucketsCollectorPath collectorPath = "/cluster/usage/buckets" + clusterErasureSetCollectorPath collectorPath = "/cluster/erasure-set" + clusterIAMCollectorPath collectorPath = "/cluster/iam" + clusterConfigCollectorPath collectorPath = "/cluster/config" + + ilmCollectorPath collectorPath = "/ilm" + auditCollectorPath collectorPath = "/audit" + loggerWebhookCollectorPath collectorPath = "/logger/webhook" + replicationCollectorPath collectorPath = "/replication" + notificationCollectorPath collectorPath = "/notification" + scannerCollectorPath collectorPath = "/scanner" +) + +const ( + clusterBasePath = "/cluster" +) + +type metricsV3Collection struct { + mgMap map[collectorPath]*MetricsGroup + bucketMGMap map[collectorPath]*MetricsGroup + + // Gatherers for non-bucket MetricsGroup's + mgGatherers map[collectorPath]prometheus.Gatherer + + collectorPaths []collectorPath +} + +func newMetricGroups(r *prometheus.Registry) *metricsV3Collection { + // Create all metric groups. + apiRequestsMG := NewMetricsGroup(apiRequestsCollectorPath, + []MetricDescriptor{ + apiRejectedAuthTotalMD, + apiRejectedHeaderTotalMD, + apiRejectedTimestampTotalMD, + apiRejectedInvalidTotalMD, + + apiRequestsWaitingTotalMD, + apiRequestsIncomingTotalMD, + apiRequestsInFlightTotalMD, + apiRequestsTotalMD, + apiRequestsErrorsTotalMD, + apiRequests5xxErrorsTotalMD, + apiRequests4xxErrorsTotalMD, + apiRequestsCanceledTotalMD, + + apiRequestsTTFBSecondsDistributionMD, + + apiTrafficSentBytesMD, + apiTrafficRecvBytesMD, + }, + JoinLoaders(loadAPIRequestsHTTPMetrics, loadAPIRequestsTTFBMetrics, + loadAPIRequestsNetworkMetrics), + ) + + bucketAPIMG := NewBucketMetricsGroup(bucketAPICollectorPath, + []MetricDescriptor{ + bucketAPITrafficRecvBytesMD, + bucketAPITrafficSentBytesMD, + + bucketAPIRequestsInFlightMD, + bucketAPIRequestsTotalMD, + bucketAPIRequestsCanceledMD, + bucketAPIRequests4xxErrorsMD, + bucketAPIRequests5xxErrorsMD, + + bucketAPIRequestsTTFBSecondsDistributionMD, + }, + JoinBucketLoaders(loadBucketAPIHTTPMetrics, loadBucketAPITTFBMetrics), + ) + + bucketReplicationMG := NewBucketMetricsGroup(bucketReplicationCollectorPath, + []MetricDescriptor{ + bucketReplLastHrFailedBytesMD, + bucketReplLastHrFailedCountMD, + bucketReplLastMinFailedBytesMD, + bucketReplLastMinFailedCountMD, + bucketReplLatencyMsMD, + bucketReplProxiedDeleteTaggingRequestsTotalMD, + bucketReplProxiedGetRequestsFailuresMD, + bucketReplProxiedGetRequestsTotalMD, + bucketReplProxiedGetTaggingRequestsFailuresMD, + bucketReplProxiedGetTaggingRequestsTotalMD, + bucketReplProxiedHeadRequestsFailuresMD, + bucketReplProxiedHeadRequestsTotalMD, + bucketReplProxiedPutTaggingRequestsFailuresMD, + bucketReplProxiedPutTaggingRequestsTotalMD, + bucketReplSentBytesMD, + bucketReplSentCountMD, + bucketReplTotalFailedBytesMD, + bucketReplTotalFailedCountMD, + bucketReplProxiedDeleteTaggingRequestsFailuresMD, + }, + loadBucketReplicationMetrics, + ) + + systemNetworkInternodeMG := NewMetricsGroup(systemNetworkInternodeCollectorPath, + []MetricDescriptor{ + internodeErrorsTotalMD, + internodeDialedErrorsTotalMD, + internodeDialAvgTimeNanosMD, + internodeSentBytesTotalMD, + internodeRecvBytesTotalMD, + }, + loadNetworkInternodeMetrics, + ) + + systemMemoryMG := NewMetricsGroup(systemMemoryCollectorPath, + []MetricDescriptor{ + memTotalMD, + memUsedMD, + memFreeMD, + memAvailableMD, + memBuffersMD, + memCacheMD, + memSharedMD, + memUsedPercMD, + }, + loadMemoryMetrics, + ) + + systemCPUMG := NewMetricsGroup(systemCPUCollectorPath, + []MetricDescriptor{ + sysCPUAvgIdleMD, + sysCPUAvgIOWaitMD, + sysCPULoadMD, + sysCPULoadPercMD, + sysCPUNiceMD, + sysCPUStealMD, + sysCPUSystemMD, + sysCPUUserMD, + }, + loadCPUMetrics, + ) + + systemProcessMG := NewMetricsGroup(systemProcessCollectorPath, + []MetricDescriptor{ + processLocksReadTotalMD, + processLocksWriteTotalMD, + processCPUTotalSecondsMD, + processGoRoutineTotalMD, + processIORCharBytesMD, + processIOReadBytesMD, + processIOWCharBytesMD, + processIOWriteBytesMD, + processStarttimeSecondsMD, + processUptimeSecondsMD, + processFileDescriptorLimitTotalMD, + processFileDescriptorOpenTotalMD, + processSyscallReadTotalMD, + processSyscallWriteTotalMD, + processResidentMemoryBytesMD, + processVirtualMemoryBytesMD, + processVirtualMemoryMaxBytesMD, + }, + loadProcessMetrics, + ) + + systemDriveMG := NewMetricsGroup(systemDriveCollectorPath, + []MetricDescriptor{ + driveUsedBytesMD, + driveFreeBytesMD, + driveTotalBytesMD, + driveUsedInodesMD, + driveFreeInodesMD, + driveTotalInodesMD, + driveTimeoutErrorsMD, + driveIOErrorsMD, + driveAvailabilityErrorsMD, + driveWaitingIOMD, + driveAPILatencyMD, + driveHealthMD, + + driveOfflineCountMD, + driveOnlineCountMD, + driveCountMD, + + // iostat related + driveReadsPerSecMD, + driveReadsKBPerSecMD, + driveReadsAwaitMD, + driveWritesPerSecMD, + driveWritesKBPerSecMD, + driveWritesAwaitMD, + drivePercUtilMD, + }, + loadDriveMetrics, + ) + + clusterHealthMG := NewMetricsGroup(clusterHealthCollectorPath, + []MetricDescriptor{ + healthDrivesOfflineCountMD, + healthDrivesOnlineCountMD, + healthDrivesCountMD, + + healthNodesOfflineCountMD, + healthNodesOnlineCountMD, + + healthCapacityRawTotalBytesMD, + healthCapacityRawFreeBytesMD, + healthCapacityUsableTotalBytesMD, + healthCapacityUsableFreeBytesMD, + }, + JoinLoaders(loadClusterHealthDriveMetrics, + loadClusterHealthNodeMetrics, + loadClusterHealthCapacityMetrics), + ) + + clusterUsageObjectsMG := NewMetricsGroup(clusterUsageObjectsCollectorPath, + []MetricDescriptor{ + usageSinceLastUpdateSecondsMD, + usageTotalBytesMD, + usageObjectsCountMD, + usageVersionsCountMD, + usageDeleteMarkersCountMD, + usageBucketsCountMD, + usageObjectsDistributionMD, + usageVersionsDistributionMD, + }, + loadClusterUsageObjectMetrics, + ) + + clusterUsageBucketsMG := NewMetricsGroup(clusterUsageBucketsCollectorPath, + []MetricDescriptor{ + usageSinceLastUpdateSecondsMD, + usageBucketTotalBytesMD, + usageBucketObjectsTotalMD, + usageBucketVersionsCountMD, + usageBucketDeleteMarkersCountMD, + usageBucketQuotaTotalBytesMD, + usageBucketObjectSizeDistributionMD, + usageBucketObjectVersionCountDistributionMD, + }, + loadClusterUsageBucketMetrics, + ) + + clusterErasureSetMG := NewMetricsGroup(clusterErasureSetCollectorPath, + []MetricDescriptor{ + erasureSetOverallWriteQuorumMD, + erasureSetOverallHealthMD, + erasureSetReadQuorumMD, + erasureSetWriteQuorumMD, + erasureSetOnlineDrivesCountMD, + erasureSetHealingDrivesCountMD, + erasureSetHealthMD, + erasureSetReadToleranceMD, + erasureSetWriteToleranceMD, + erasureSetReadHealthMD, + erasureSetWriteHealthMD, + }, + loadClusterErasureSetMetrics, + ) + + clusterNotificationMG := NewMetricsGroup(notificationCollectorPath, + []MetricDescriptor{ + notificationCurrentSendInProgressMD, + notificationEventsErrorsTotalMD, + notificationEventsSentTotalMD, + notificationEventsSkippedTotalMD, + }, + loadClusterNotificationMetrics, + ) + + clusterIAMMG := NewMetricsGroup(clusterIAMCollectorPath, + []MetricDescriptor{ + lastSyncDurationMillisMD, + pluginAuthnServiceFailedRequestsMinuteMD, + pluginAuthnServiceLastFailSecondsMD, + pluginAuthnServiceLastSuccSecondsMD, + pluginAuthnServiceSuccAvgRttMsMinuteMD, + pluginAuthnServiceSuccMaxRttMsMinuteMD, + pluginAuthnServiceTotalRequestsMinuteMD, + sinceLastSyncMillisMD, + syncFailuresMD, + syncSuccessesMD, + }, + loadClusterIAMMetrics, + ) + + clusterReplicationMG := NewMetricsGroup(replicationCollectorPath, + []MetricDescriptor{ + replicationAverageActiveWorkersMD, + replicationAverageQueuedBytesMD, + replicationAverageQueuedCountMD, + replicationAverageDataTransferRateMD, + replicationCurrentActiveWorkersMD, + replicationCurrentDataTransferRateMD, + replicationLastMinuteQueuedBytesMD, + replicationLastMinuteQueuedCountMD, + replicationMaxActiveWorkersMD, + replicationMaxQueuedBytesMD, + replicationMaxQueuedCountMD, + replicationMaxDataTransferRateMD, + replicationRecentBacklogCountMD, + }, + loadClusterReplicationMetrics, + ) + + clusterConfigMG := NewMetricsGroup(clusterConfigCollectorPath, + []MetricDescriptor{ + configRRSParityMD, + configStandardParityMD, + }, + loadClusterConfigMetrics, + ) + + scannerMG := NewMetricsGroup(scannerCollectorPath, + []MetricDescriptor{ + scannerBucketScansFinishedMD, + scannerBucketScansStartedMD, + scannerDirectoriesScannedMD, + scannerObjectsScannedMD, + scannerVersionsScannedMD, + scannerLastActivitySecondsMD, + }, + loadClusterScannerMetrics, + ) + + loggerWebhookMG := NewMetricsGroup(loggerWebhookCollectorPath, + []MetricDescriptor{ + webhookFailedMessagesMD, + webhookQueueLengthMD, + webhookTotalMessagesMD, + }, + loadLoggerWebhookMetrics, + ) + + auditMG := NewMetricsGroup(auditCollectorPath, + []MetricDescriptor{ + auditFailedMessagesMD, + auditTargetQueueLengthMD, + auditTotalMessagesMD, + }, + loadAuditMetrics, + ) + + ilmMG := NewMetricsGroup(ilmCollectorPath, + []MetricDescriptor{ + ilmExpiryPendingTasksMD, + ilmTransitionActiveTasksMD, + ilmTransitionPendingTasksMD, + ilmTransitionMissedImmediateTasksMD, + ilmVersionsScannedMD, + }, + loadILMMetrics, + ) + + allMetricGroups := []*MetricsGroup{ + apiRequestsMG, + bucketAPIMG, + bucketReplicationMG, + + systemNetworkInternodeMG, + systemDriveMG, + systemMemoryMG, + systemCPUMG, + systemProcessMG, + + clusterHealthMG, + clusterUsageObjectsMG, + clusterUsageBucketsMG, + clusterErasureSetMG, + clusterNotificationMG, + clusterIAMMG, + clusterReplicationMG, + clusterConfigMG, + + ilmMG, + scannerMG, + auditMG, + loggerWebhookMG, + } + + // Bucket metrics are special, they always include the bucket label. These + // metrics required a list of buckets to be passed to the loader, and the list + // of buckets is not known until the request is made. So we keep a separate + // map for bucket metrics and handle them specially. + + // Add the serverName and poolIndex labels to all non-cluster metrics. + // + // Also create metric group maps and set the cache. + metricsCache := newMetricsCache() + mgMap := make(map[collectorPath]*MetricsGroup) + bucketMGMap := make(map[collectorPath]*MetricsGroup) + for _, mg := range allMetricGroups { + if !strings.HasPrefix(string(mg.CollectorPath), clusterBasePath) { + mg.AddExtraLabels( + serverName, globalLocalNodeName, + // poolIndex, strconv.Itoa(globalLocalPoolIdx), + ) + } + mg.SetCache(metricsCache) + if mg.IsBucketMetricsGroup() { + bucketMGMap[mg.CollectorPath] = mg + } else { + mgMap[mg.CollectorPath] = mg + } + } + + // Prepare to register the collectors. Other than `MetricGroup` collectors, + // we also have standard collectors like `GoCollector`. + + // Create all Non-`MetricGroup` collectors here. + collectors := map[collectorPath]prometheus.Collector{ + debugGoCollectorPath: collectors.NewGoCollector(), + } + + // Add all `MetricGroup` collectors to the map. + for _, mg := range allMetricGroups { + collectors[mg.CollectorPath] = mg + } + + // Helper function to register a collector and return a gatherer for it. + mustRegister := func(c ...prometheus.Collector) prometheus.Gatherer { + subRegistry := prometheus.NewRegistry() + for _, col := range c { + subRegistry.MustRegister(col) + } + r.MustRegister(subRegistry) + return subRegistry + } + + // Register all collectors and create gatherers for them. + gatherers := make(map[collectorPath]prometheus.Gatherer, len(collectors)) + collectorPaths := make([]collectorPath, 0, len(collectors)) + for path, collector := range collectors { + gatherers[path] = mustRegister(collector) + collectorPaths = append(collectorPaths, path) + } + slices.Sort(collectorPaths) + return &metricsV3Collection{ + mgMap: mgMap, + bucketMGMap: bucketMGMap, + mgGatherers: gatherers, + collectorPaths: collectorPaths, + } +} diff --git a/cmd/metrics.go b/cmd/metrics.go new file mode 100644 index 00000000..0548ff2b --- /dev/null +++ b/cmd/metrics.go @@ -0,0 +1,573 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "net/http" + "time" + + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/mcontext" + "github.com/minio/pkg/v3/policy" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/common/expfmt" +) + +var ( + httpRequestsDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "s3_ttfb_seconds", + Help: "Time taken by requests served by current MinIO server instance", + Buckets: []float64{.05, .1, .25, .5, 1, 2.5, 5, 10}, + }, + []string{"api"}, + ) + bucketHTTPRequestsDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "s3_ttfb_seconds", + Help: "Time taken by requests served by current MinIO server instance per bucket", + Buckets: []float64{.05, .1, .25, .5, 1, 2.5, 5, 10}, + }, + []string{"api", "bucket"}, + ) + minioVersionInfo = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "minio", + Name: "version_info", + Help: "Version of current MinIO server instance", + }, + []string{ + // current version + "version", + // commit-id of the current version + "commit", + }, + ) +) + +const ( + healMetricsNamespace = "self_heal" + cacheNamespace = "cache" + s3Namespace = "s3" + bucketNamespace = "bucket" + minioNamespace = "minio" + diskNamespace = "disk" + interNodeNamespace = "internode" +) + +func init() { + prometheus.MustRegister(httpRequestsDuration) + prometheus.MustRegister(newMinioCollector()) + prometheus.MustRegister(minioVersionInfo) +} + +// newMinioCollector describes the collector +// and returns reference of minioCollector +// It creates the Prometheus Description which is used +// to define metric and help string +func newMinioCollector() *minioCollector { + return &minioCollector{ + desc: prometheus.NewDesc("minio_stats", "Statistics exposed by MinIO server", nil, nil), + } +} + +// minioCollector is the Custom Collector +type minioCollector struct { + desc *prometheus.Desc +} + +// Describe sends the super-set of all possible descriptors of metrics +func (c *minioCollector) Describe(ch chan<- *prometheus.Desc) { + ch <- c.desc +} + +// Collect is called by the Prometheus registry when collecting metrics. +func (c *minioCollector) Collect(ch chan<- prometheus.Metric) { + // Expose MinIO's version information + minioVersionInfo.WithLabelValues(Version, CommitID).Set(1.0) + + storageMetricsPrometheus(ch) + nodeHealthMetricsPrometheus(ch) + bucketUsageMetricsPrometheus(ch) + networkMetricsPrometheus(ch) + httpMetricsPrometheus(ch) + healingMetricsPrometheus(ch) +} + +func nodeHealthMetricsPrometheus(ch chan<- prometheus.Metric) { + nodesUp, nodesDown := globalNotificationSys.GetPeerOnlineCount() + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(minioNamespace, "nodes", "online"), + "Total number of MinIO nodes online", + nil, nil), + prometheus.GaugeValue, + float64(nodesUp), + ) + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(minioNamespace, "nodes", "offline"), + "Total number of MinIO nodes offline", + nil, nil), + prometheus.GaugeValue, + float64(nodesDown), + ) +} + +// collects healing specific metrics for MinIO instance in Prometheus specific format +// and sends to given channel +func healingMetricsPrometheus(ch chan<- prometheus.Metric) { + bgSeq, exists := globalBackgroundHealState.getHealSequenceByToken(bgHealingUUID) + if !exists { + return + } + + var dur time.Duration + if !bgSeq.lastHealActivity.IsZero() { + dur = time.Since(bgSeq.lastHealActivity) + } + + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(healMetricsNamespace, "time", "since_last_activity"), + "Time elapsed (in nano seconds) since last self healing activity. This is set to -1 until initial self heal activity", + nil, nil), + prometheus.GaugeValue, + float64(dur), + ) + for k, v := range bgSeq.getScannedItemsMap() { + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(healMetricsNamespace, "objects", "scanned"), + "Objects scanned since uptime", + []string{"type"}, nil), + prometheus.CounterValue, + float64(v), string(k), + ) + } + for k, v := range bgSeq.getHealedItemsMap() { + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(healMetricsNamespace, "objects", "healed"), + "Objects healed since uptime", + []string{"type"}, nil), + prometheus.CounterValue, + float64(v), string(k), + ) + } + for k, v := range bgSeq.getHealFailedItemsMap() { + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(healMetricsNamespace, "objects", "heal_failed"), + "Objects for which healing failed since uptime", + []string{"type"}, nil), + prometheus.CounterValue, + float64(v), string(k), + ) + } +} + +// collects http metrics for MinIO server in Prometheus specific format +// and sends to given channel +func httpMetricsPrometheus(ch chan<- prometheus.Metric) { + httpStats := globalHTTPStats.toServerHTTPStats(true) + + for api, value := range httpStats.CurrentS3Requests.APIStats { + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(s3Namespace, "requests", "current"), + "Total number of running s3 requests in current MinIO server instance", + []string{"api"}, nil), + prometheus.CounterValue, + float64(value), + api, + ) + } + + for api, value := range httpStats.TotalS3Requests.APIStats { + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(s3Namespace, "requests", "total"), + "Total number of s3 requests in current MinIO server instance", + []string{"api"}, nil), + prometheus.CounterValue, + float64(value), + api, + ) + } + + for api, value := range httpStats.TotalS3Errors.APIStats { + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(s3Namespace, "errors", "total"), + "Total number of s3 errors in current MinIO server instance", + []string{"api"}, nil), + prometheus.CounterValue, + float64(value), + api, + ) + } + + for api, value := range httpStats.TotalS3Canceled.APIStats { + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(s3Namespace, "canceled", "total"), + "Total number of client canceled s3 request in current MinIO server instance", + []string{"api"}, nil), + prometheus.CounterValue, + float64(value), + api, + ) + } +} + +// collects network metrics for MinIO server in Prometheus specific format +// and sends to given channel +func networkMetricsPrometheus(ch chan<- prometheus.Metric) { + connStats := globalConnStats.toServerConnStats() + + // Network Sent/Received Bytes (internode) + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(interNodeNamespace, "tx", "bytes_total"), + "Total number of bytes sent to the other peer nodes by current MinIO server instance", + nil, nil), + prometheus.CounterValue, + float64(connStats.internodeOutputBytes), + ) + + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(interNodeNamespace, "rx", "bytes_total"), + "Total number of internode bytes received by current MinIO server instance", + nil, nil), + prometheus.CounterValue, + float64(connStats.internodeInputBytes), + ) + + // Network Sent/Received Bytes (Outbound) + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(s3Namespace, "tx", "bytes_total"), + "Total number of s3 bytes sent by current MinIO server instance", + nil, nil), + prometheus.CounterValue, + float64(connStats.s3OutputBytes), + ) + + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(s3Namespace, "rx", "bytes_total"), + "Total number of s3 bytes received by current MinIO server instance", + nil, nil), + prometheus.CounterValue, + float64(connStats.s3InputBytes), + ) +} + +// Populates prometheus with bucket usage metrics, this metrics +// is only enabled if scanner is enabled. +func bucketUsageMetricsPrometheus(ch chan<- prometheus.Metric) { + objLayer := newObjectLayerFn() + // Service not initialized yet + if objLayer == nil { + return + } + + dataUsageInfo, err := loadDataUsageFromBackend(GlobalContext, objLayer) + if err != nil { + return + } + // data usage has not captured any data yet. + if dataUsageInfo.LastUpdate.IsZero() { + return + } + + for bucket, usageInfo := range dataUsageInfo.BucketsUsage { + stat := globalReplicationStats.Load().getLatestReplicationStats(bucket) + // Total space used by bucket + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(bucketNamespace, "usage", "size"), + "Total bucket size", + []string{"bucket"}, nil), + prometheus.GaugeValue, + float64(usageInfo.Size), + bucket, + ) + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(bucketNamespace, "objects", "count"), + "Total number of objects in a bucket", + []string{"bucket"}, nil), + prometheus.GaugeValue, + float64(usageInfo.ObjectsCount), + bucket, + ) + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName("bucket", "replication", "successful_size"), + "Total capacity replicated to destination", + []string{"bucket"}, nil), + prometheus.GaugeValue, + float64(stat.ReplicationStats.ReplicatedSize), + bucket, + ) + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName("bucket", "replication", "received_size"), + "Total capacity replicated to this instance", + []string{"bucket"}, nil), + prometheus.GaugeValue, + float64(stat.ReplicationStats.ReplicaSize), + bucket, + ) + + for k, v := range usageInfo.ObjectSizesHistogram { + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(bucketNamespace, "objects", "histogram"), + "Total number of objects of different sizes in a bucket", + []string{"bucket", "object_size"}, nil), + prometheus.GaugeValue, + float64(v), + bucket, + k, + ) + } + for k, v := range usageInfo.ObjectVersionsHistogram { + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(bucketNamespace, "objects", "histogram"), + "Total number of versions of objects in a bucket", + []string{"bucket", "object_versions"}, nil), + prometheus.GaugeValue, + float64(v), + bucket, + k, + ) + } + } +} + +// collects storage metrics for MinIO server in Prometheus specific format +// and sends to given channel +func storageMetricsPrometheus(ch chan<- prometheus.Metric) { + objLayer := newObjectLayerFn() + // Service not initialized yet + if objLayer == nil { + return + } + + server := getLocalServerProperty(globalEndpoints, &http.Request{ + Host: globalLocalNodeName, + }, true) + + onlineDisks, offlineDisks := getOnlineOfflineDisksStats(server.Disks) + totalDisks := offlineDisks.Merge(onlineDisks) + + // Report total capacity + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(minioNamespace, "capacity_raw", "total"), + "Total capacity online in the cluster", + nil, nil), + prometheus.GaugeValue, + float64(GetTotalCapacity(server.Disks)), + ) + + // Report total capacity free + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(minioNamespace, "capacity_raw_free", "total"), + "Total free capacity online in the cluster", + nil, nil), + prometheus.GaugeValue, + float64(GetTotalCapacityFree(server.Disks)), + ) + + sinfo := objLayer.StorageInfo(GlobalContext, true) + + // Report total usable capacity + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(minioNamespace, "capacity_usable", "total"), + "Total usable capacity online in the cluster", + nil, nil), + prometheus.GaugeValue, + float64(GetTotalUsableCapacity(server.Disks, sinfo)), + ) + + // Report total usable capacity free + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(minioNamespace, "capacity_usable_free", "total"), + "Total free usable capacity online in the cluster", + nil, nil), + prometheus.GaugeValue, + float64(GetTotalUsableCapacityFree(server.Disks, sinfo)), + ) + + // MinIO Offline Disks per node + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(minioNamespace, "disks", "offline"), + "Total number of offline drives in current MinIO server instance", + nil, nil), + prometheus.GaugeValue, + float64(offlineDisks.Sum()), + ) + + // MinIO Total Disks per node + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(minioNamespace, "drives", "total"), + "Total number of drives for current MinIO server instance", + nil, nil), + prometheus.GaugeValue, + float64(totalDisks.Sum()), + ) + + for _, disk := range server.Disks { + // Total disk usage by the disk + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(diskNamespace, "storage", "used"), + "Total disk storage used on the drive", + []string{"disk"}, nil), + prometheus.GaugeValue, + float64(disk.UsedSpace), + disk.DrivePath, + ) + + // Total available space in the disk + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(diskNamespace, "storage", "available"), + "Total available space left on the drive", + []string{"disk"}, nil), + prometheus.GaugeValue, + float64(disk.AvailableSpace), + disk.DrivePath, + ) + + // Total storage space of the disk + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc( + prometheus.BuildFQName(diskNamespace, "storage", "total"), + "Total space on the drive", + []string{"disk"}, nil), + prometheus.GaugeValue, + float64(disk.TotalSpace), + disk.DrivePath, + ) + } +} + +func metricsHandler() http.Handler { + registry := prometheus.NewRegistry() + + logger.CriticalIf(GlobalContext, registry.Register(minioVersionInfo)) + + logger.CriticalIf(GlobalContext, registry.Register(newMinioCollector())) + + gatherers := prometheus.Gatherers{ + prometheus.DefaultGatherer, + registry, + } + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt) + if ok { + tc.FuncName = "handler.MetricsLegacy" + tc.ResponseRecorder.LogErrBody = true + } + + mfs, err := gatherers.Gather() + if err != nil { + if len(mfs) == 0 { + writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), err), r.URL) + return + } + } + + contentType := expfmt.Negotiate(r.Header) + w.Header().Set("Content-Type", string(contentType)) + + enc := expfmt.NewEncoder(w, contentType) + for _, mf := range mfs { + if err := enc.Encode(mf); err != nil { + // client may disconnect for any reasons + // we do not have to log this. + return + } + } + if closer, ok := enc.(expfmt.Closer); ok { + closer.Close() + } + }) +} + +// NoAuthMiddleware no auth middle ware. +func NoAuthMiddleware(h http.Handler) http.Handler { + return h +} + +// AuthMiddleware checks if the bearer token is valid and authorized. +func AuthMiddleware(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + tc, ok := r.Context().Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt) + + claims, groups, owner, authErr := metricsRequestAuthenticate(r) + if authErr != nil || (claims != nil && !claims.VerifyIssuer("prometheus", true)) { + if ok { + tc.FuncName = "handler.MetricsAuth" + tc.ResponseRecorder.LogErrBody = true + } + + writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), errAuthentication), r.URL) + return + } + + cred := auth.Credentials{ + AccessKey: claims.AccessKey, + Claims: claims.Map(), + Groups: groups, + } + + // For authenticated users apply IAM policy. + if !globalIAMSys.IsAllowed(policy.Args{ + AccountName: cred.AccessKey, + Groups: cred.Groups, + Action: policy.PrometheusAdminAction, + ConditionValues: getConditionValues(r, "", cred), + IsOwner: owner, + Claims: cred.Claims, + }) { + if ok { + tc.FuncName = "handler.MetricsAuth" + tc.ResponseRecorder.LogErrBody = true + } + + writeErrorResponseJSON(r.Context(), w, toAdminAPIErr(r.Context(), errAuthentication), r.URL) + return + } + h.ServeHTTP(w, r) + }) +} diff --git a/cmd/mrf.go b/cmd/mrf.go new file mode 100644 index 00000000..4d002c27 --- /dev/null +++ b/cmd/mrf.go @@ -0,0 +1,281 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//go:generate msgp -file=$GOFILE + +package cmd + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "sync" + "sync/atomic" + "time" + + "github.com/google/uuid" + "github.com/minio/madmin-go/v3" + "github.com/minio/pkg/v3/wildcard" + "github.com/tinylib/msgp/msgp" +) + +const ( + mrfOpsQueueSize = 100000 +) + +const ( + healDir = ".heal" + healMRFDir = bucketMetaPrefix + SlashSeparator + healDir + SlashSeparator + "mrf" + healMRFMetaFormat = 1 + healMRFMetaVersionV1 = 1 +) + +// PartialOperation is a successful upload/delete of an object +// but not written in all disks (having quorum) +type PartialOperation struct { + Bucket string + Object string + VersionID string + Versions []byte + SetIndex, PoolIndex int + Queued time.Time + BitrotScan bool +} + +// mrfState sncapsulates all the information +// related to the global background MRF. +type mrfState struct { + opCh chan PartialOperation + + closed int32 + closing int32 + wg sync.WaitGroup +} + +func newMRFState() mrfState { + return mrfState{ + opCh: make(chan PartialOperation, mrfOpsQueueSize), + } +} + +// Add a partial S3 operation (put/delete) when one or more disks are offline. +func (m *mrfState) addPartialOp(op PartialOperation) { + if m == nil { + return + } + + if atomic.LoadInt32(&m.closed) == 1 { + return + } + + m.wg.Add(1) + defer m.wg.Done() + + if atomic.LoadInt32(&m.closing) == 1 { + return + } + + select { + case m.opCh <- op: + default: + } +} + +// Do not accept new MRF operations anymore and start to save +// the current heal status in one available disk +func (m *mrfState) shutdown() { + atomic.StoreInt32(&m.closing, 1) + m.wg.Wait() + close(m.opCh) + atomic.StoreInt32(&m.closed, 1) + + if len(m.opCh) > 0 { + healingLogEvent(context.Background(), "Saving MRF healing data (%d entries)", len(m.opCh)) + } + + newReader := func() io.ReadCloser { + r, w := io.Pipe() + go func() { + // Initialize MRF meta header. + var data [4]byte + binary.LittleEndian.PutUint16(data[0:2], healMRFMetaFormat) + binary.LittleEndian.PutUint16(data[2:4], healMRFMetaVersionV1) + mw := msgp.NewWriter(w) + n, err := mw.Write(data[:]) + if err != nil { + w.CloseWithError(err) + return + } + if n != len(data) { + w.CloseWithError(io.ErrShortWrite) + return + } + for item := range m.opCh { + err = item.EncodeMsg(mw) + if err != nil { + break + } + } + mw.Flush() + w.CloseWithError(err) + }() + return r + } + + globalLocalDrivesMu.RLock() + localDrives := cloneDrives(globalLocalDrivesMap) + globalLocalDrivesMu.RUnlock() + + for _, localDrive := range localDrives { + r := newReader() + err := localDrive.CreateFile(context.Background(), "", minioMetaBucket, pathJoin(healMRFDir, "list.bin"), -1, r) + r.Close() + if err == nil { + break + } + } +} + +func (m *mrfState) startMRFPersistence() { + loadMRF := func(rc io.ReadCloser, opCh chan PartialOperation) error { + defer rc.Close() + var data [4]byte + n, err := rc.Read(data[:]) + if err != nil { + return err + } + if n != len(data) { + return errors.New("heal mrf: no data") + } + // Read resync meta header + switch binary.LittleEndian.Uint16(data[0:2]) { + case healMRFMetaFormat: + default: + return fmt.Errorf("heal mrf: unknown format: %d", binary.LittleEndian.Uint16(data[0:2])) + } + switch binary.LittleEndian.Uint16(data[2:4]) { + case healMRFMetaVersionV1: + default: + return fmt.Errorf("heal mrf: unknown version: %d", binary.LittleEndian.Uint16(data[2:4])) + } + + mr := msgp.NewReader(rc) + for { + op := PartialOperation{} + err = op.DecodeMsg(mr) + if err != nil { + break + } + opCh <- op + } + + return nil + } + + globalLocalDrivesMu.RLock() + localDrives := cloneDrives(globalLocalDrivesMap) + globalLocalDrivesMu.RUnlock() + + for _, localDrive := range localDrives { + if localDrive == nil { + continue + } + rc, err := localDrive.ReadFileStream(context.Background(), minioMetaBucket, pathJoin(healMRFDir, "list.bin"), 0, -1) + if err != nil { + continue + } + err = loadMRF(rc, m.opCh) + if err != nil { + continue + } + // finally delete the file after processing mrf entries + localDrive.Delete(GlobalContext, minioMetaBucket, pathJoin(healMRFDir, "list.bin"), DeleteOptions{}) + break + } +} + +var healSleeper = newDynamicSleeper(5, time.Second, false) + +// healRoutine listens to new disks reconnection events and +// issues healing requests for queued objects belonging to the +// corresponding erasure set +func (m *mrfState) healRoutine(z *erasureServerPools) { + for { + select { + case <-GlobalContext.Done(): + return + case u, ok := <-m.opCh: + if !ok { + return + } + + // We might land at .metacache, .trash, .multipart + // no need to heal them skip, only when bucket + // is '.minio.sys' + if u.Bucket == minioMetaBucket { + // No MRF needed for temporary objects + if wildcard.Match("buckets/*/.metacache/*", u.Object) { + continue + } + if wildcard.Match("tmp/*", u.Object) { + continue + } + if wildcard.Match("multipart/*", u.Object) { + continue + } + if wildcard.Match("tmp-old/*", u.Object) { + continue + } + } + + now := time.Now() + if now.Sub(u.Queued) < time.Second { + // let recently failed networks to reconnect + // making MRF wait for 1s before retrying, + // i.e 4 reconnect attempts. + time.Sleep(time.Second) + } + + // wait on timer per heal + wait := healSleeper.Timer(context.Background()) + + scan := madmin.HealNormalScan + if u.BitrotScan { + scan = madmin.HealDeepScan + } + + if u.Object == "" { + healBucket(u.Bucket, scan) + } else { + if len(u.Versions) > 0 { + vers := len(u.Versions) / 16 + if vers > 0 { + for i := range vers { + healObject(u.Bucket, u.Object, uuid.UUID(u.Versions[16*i:]).String(), scan) + } + } + } else { + healObject(u.Bucket, u.Object, u.VersionID, scan) + } + } + + wait() + } + } +} diff --git a/cmd/mrf_gen.go b/cmd/mrf_gen.go new file mode 100644 index 00000000..8c0467b5 --- /dev/null +++ b/cmd/mrf_gen.go @@ -0,0 +1,285 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *PartialOperation) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Bucket": + z.Bucket, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "Object": + z.Object, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + case "VersionID": + z.VersionID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + case "Versions": + z.Versions, err = dc.ReadBytes(z.Versions) + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + case "SetIndex": + z.SetIndex, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "SetIndex") + return + } + case "PoolIndex": + z.PoolIndex, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "PoolIndex") + return + } + case "Queued": + z.Queued, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "Queued") + return + } + case "BitrotScan": + z.BitrotScan, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "BitrotScan") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *PartialOperation) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 8 + // write "Bucket" + err = en.Append(0x88, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Bucket) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + // write "Object" + err = en.Append(0xa6, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Object) + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + // write "VersionID" + err = en.Append(0xa9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x44) + if err != nil { + return + } + err = en.WriteString(z.VersionID) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + // write "Versions" + err = en.Append(0xa8, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73) + if err != nil { + return + } + err = en.WriteBytes(z.Versions) + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + // write "SetIndex" + err = en.Append(0xa8, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78) + if err != nil { + return + } + err = en.WriteInt(z.SetIndex) + if err != nil { + err = msgp.WrapError(err, "SetIndex") + return + } + // write "PoolIndex" + err = en.Append(0xa9, 0x50, 0x6f, 0x6f, 0x6c, 0x49, 0x6e, 0x64, 0x65, 0x78) + if err != nil { + return + } + err = en.WriteInt(z.PoolIndex) + if err != nil { + err = msgp.WrapError(err, "PoolIndex") + return + } + // write "Queued" + err = en.Append(0xa6, 0x51, 0x75, 0x65, 0x75, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteTime(z.Queued) + if err != nil { + err = msgp.WrapError(err, "Queued") + return + } + // write "BitrotScan" + err = en.Append(0xaa, 0x42, 0x69, 0x74, 0x72, 0x6f, 0x74, 0x53, 0x63, 0x61, 0x6e) + if err != nil { + return + } + err = en.WriteBool(z.BitrotScan) + if err != nil { + err = msgp.WrapError(err, "BitrotScan") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *PartialOperation) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 8 + // string "Bucket" + o = append(o, 0x88, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + o = msgp.AppendString(o, z.Bucket) + // string "Object" + o = append(o, 0xa6, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74) + o = msgp.AppendString(o, z.Object) + // string "VersionID" + o = append(o, 0xa9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x44) + o = msgp.AppendString(o, z.VersionID) + // string "Versions" + o = append(o, 0xa8, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73) + o = msgp.AppendBytes(o, z.Versions) + // string "SetIndex" + o = append(o, 0xa8, 0x53, 0x65, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78) + o = msgp.AppendInt(o, z.SetIndex) + // string "PoolIndex" + o = append(o, 0xa9, 0x50, 0x6f, 0x6f, 0x6c, 0x49, 0x6e, 0x64, 0x65, 0x78) + o = msgp.AppendInt(o, z.PoolIndex) + // string "Queued" + o = append(o, 0xa6, 0x51, 0x75, 0x65, 0x75, 0x65, 0x64) + o = msgp.AppendTime(o, z.Queued) + // string "BitrotScan" + o = append(o, 0xaa, 0x42, 0x69, 0x74, 0x72, 0x6f, 0x74, 0x53, 0x63, 0x61, 0x6e) + o = msgp.AppendBool(o, z.BitrotScan) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *PartialOperation) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Bucket": + z.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "Object": + z.Object, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + case "VersionID": + z.VersionID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + case "Versions": + z.Versions, bts, err = msgp.ReadBytesBytes(bts, z.Versions) + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + case "SetIndex": + z.SetIndex, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SetIndex") + return + } + case "PoolIndex": + z.PoolIndex, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "PoolIndex") + return + } + case "Queued": + z.Queued, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Queued") + return + } + case "BitrotScan": + z.BitrotScan, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "BitrotScan") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *PartialOperation) Msgsize() (s int) { + s = 1 + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Object) + 10 + msgp.StringPrefixSize + len(z.VersionID) + 9 + msgp.BytesPrefixSize + len(z.Versions) + 9 + msgp.IntSize + 10 + msgp.IntSize + 7 + msgp.TimeSize + 11 + msgp.BoolSize + return +} diff --git a/cmd/mrf_gen_test.go b/cmd/mrf_gen_test.go new file mode 100644 index 00000000..b8e039f8 --- /dev/null +++ b/cmd/mrf_gen_test.go @@ -0,0 +1,123 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalPartialOperation(t *testing.T) { + v := PartialOperation{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgPartialOperation(b *testing.B) { + v := PartialOperation{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgPartialOperation(b *testing.B) { + v := PartialOperation{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalPartialOperation(b *testing.B) { + v := PartialOperation{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodePartialOperation(t *testing.T) { + v := PartialOperation{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodePartialOperation Msgsize() is inaccurate") + } + + vn := PartialOperation{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodePartialOperation(b *testing.B) { + v := PartialOperation{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodePartialOperation(b *testing.B) { + v := PartialOperation{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/namespace-lock.go b/cmd/namespace-lock.go new file mode 100644 index 00000000..0cfe44de --- /dev/null +++ b/cmd/namespace-lock.go @@ -0,0 +1,329 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + pathutil "path" + "runtime" + "sort" + "strings" + "sync" + "time" + + "github.com/minio/minio/internal/dsync" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/lsync" +) + +// local lock servers +var globalLockServer *localLocker + +// RWLocker - locker interface to introduce GetRLock, RUnlock. +type RWLocker interface { + GetLock(ctx context.Context, timeout *dynamicTimeout) (lkCtx LockContext, timedOutErr error) + Unlock(lkCtx LockContext) + GetRLock(ctx context.Context, timeout *dynamicTimeout) (lkCtx LockContext, timedOutErr error) + RUnlock(lkCtx LockContext) +} + +// LockContext lock context holds the lock backed context and canceler for the context. +type LockContext struct { + ctx context.Context + cancel context.CancelFunc +} + +// Context returns lock context +func (l LockContext) Context() context.Context { + return l.ctx +} + +// Cancel function calls cancel() function +func (l LockContext) Cancel() { + if l.cancel != nil { + l.cancel() + } +} + +// newNSLock - return a new name space lock map. +func newNSLock(isDistErasure bool) *nsLockMap { + nsMutex := nsLockMap{ + isDistErasure: isDistErasure, + } + if isDistErasure { + return &nsMutex + } + nsMutex.lockMap = make(map[string]*nsLock) + return &nsMutex +} + +// nsLock - provides primitives for locking critical namespace regions. +type nsLock struct { + ref int32 + *lsync.LRWMutex +} + +// nsLockMap - namespace lock map, provides primitives to Lock, +// Unlock, RLock and RUnlock. +type nsLockMap struct { + // Indicates if namespace is part of a distributed setup. + isDistErasure bool + lockMap map[string]*nsLock + lockMapMutex sync.Mutex +} + +// Lock the namespace resource. +func (n *nsLockMap) lock(ctx context.Context, volume string, path string, lockSource, opsID string, readLock bool, timeout time.Duration) (locked bool) { + resource := pathJoin(volume, path) + + n.lockMapMutex.Lock() + nsLk, found := n.lockMap[resource] + if !found { + nsLk = &nsLock{ + LRWMutex: lsync.NewLRWMutex(), + } + // Add a count to indicate that a parallel unlock doesn't clear this entry. + } + nsLk.ref++ + n.lockMap[resource] = nsLk + n.lockMapMutex.Unlock() + + // Locking here will block (until timeout). + if readLock { + locked = nsLk.GetRLock(ctx, opsID, lockSource, timeout) + } else { + locked = nsLk.GetLock(ctx, opsID, lockSource, timeout) + } + + if !locked { // We failed to get the lock + // Decrement ref count since we failed to get the lock + n.lockMapMutex.Lock() + n.lockMap[resource].ref-- + if n.lockMap[resource].ref < 0 { + logger.CriticalIf(GlobalContext, errors.New("resource reference count was lower than 0")) + } + if n.lockMap[resource].ref == 0 { + // Remove from the map if there are no more references. + delete(n.lockMap, resource) + } + n.lockMapMutex.Unlock() + } + + return locked +} + +// Unlock the namespace resource. +func (n *nsLockMap) unlock(volume string, path string, readLock bool) { + resource := pathJoin(volume, path) + + n.lockMapMutex.Lock() + defer n.lockMapMutex.Unlock() + if _, found := n.lockMap[resource]; !found { + return + } + if readLock { + n.lockMap[resource].RUnlock() + } else { + n.lockMap[resource].Unlock() + } + n.lockMap[resource].ref-- + if n.lockMap[resource].ref < 0 { + logger.CriticalIf(GlobalContext, errors.New("resource reference count was lower than 0")) + } + if n.lockMap[resource].ref == 0 { + // Remove from the map if there are no more references. + delete(n.lockMap, resource) + } +} + +// dsync's distributed lock instance. +type distLockInstance struct { + rwMutex *dsync.DRWMutex + opsID string +} + +// Lock - block until write lock is taken or timeout has occurred. +func (di *distLockInstance) GetLock(ctx context.Context, timeout *dynamicTimeout) (LockContext, error) { + lockSource := getSource(2) + start := UTCNow() + + newCtx, cancel := context.WithCancel(ctx) + if !di.rwMutex.GetLock(newCtx, cancel, di.opsID, lockSource, dsync.Options{ + Timeout: timeout.Timeout(), + RetryInterval: timeout.RetryInterval(), + }) { + timeout.LogFailure() + defer cancel() + if err := newCtx.Err(); err == context.Canceled { + return LockContext{ctx: ctx, cancel: func() {}}, err + } + return LockContext{ctx: ctx, cancel: func() {}}, OperationTimedOut{} + } + timeout.LogSuccess(UTCNow().Sub(start)) + return LockContext{ctx: newCtx, cancel: cancel}, nil +} + +// Unlock - block until write lock is released. +func (di *distLockInstance) Unlock(lc LockContext) { + if lc.cancel != nil { + lc.cancel() + } + di.rwMutex.Unlock(context.Background()) +} + +// RLock - block until read lock is taken or timeout has occurred. +func (di *distLockInstance) GetRLock(ctx context.Context, timeout *dynamicTimeout) (LockContext, error) { + lockSource := getSource(2) + start := UTCNow() + + newCtx, cancel := context.WithCancel(ctx) + if !di.rwMutex.GetRLock(ctx, cancel, di.opsID, lockSource, dsync.Options{ + Timeout: timeout.Timeout(), + RetryInterval: timeout.RetryInterval(), + }) { + timeout.LogFailure() + defer cancel() + if errors.Is(newCtx.Err(), context.Canceled) { + return LockContext{ctx: ctx, cancel: func() {}}, newCtx.Err() + } + return LockContext{ctx: ctx, cancel: func() {}}, OperationTimedOut{} + } + timeout.LogSuccess(UTCNow().Sub(start)) + return LockContext{ctx: newCtx, cancel: cancel}, nil +} + +// RUnlock - block until read lock is released. +func (di *distLockInstance) RUnlock(lc LockContext) { + if lc.cancel != nil { + lc.cancel() + } + di.rwMutex.RUnlock(lc.ctx) +} + +// localLockInstance - frontend/top-level interface for namespace locks. +type localLockInstance struct { + ns *nsLockMap + volume string + paths []string + opsID string +} + +// NewNSLock - returns a lock instance for a given volume and +// path. The returned lockInstance object encapsulates the nsLockMap, +// volume, path and operation ID. +func (n *nsLockMap) NewNSLock(lockers func() ([]dsync.NetLocker, string), volume string, paths ...string) RWLocker { + sort.Strings(paths) + opsID := mustGetUUID() + if n.isDistErasure { + drwmutex := dsync.NewDRWMutex(&dsync.Dsync{ + GetLockers: lockers, + Timeouts: dsync.DefaultTimeouts, + }, pathsJoinPrefix(volume, paths...)...) + return &distLockInstance{drwmutex, opsID} + } + return &localLockInstance{n, volume, paths, opsID} +} + +// Lock - block until write lock is taken or timeout has occurred. +func (li *localLockInstance) GetLock(ctx context.Context, timeout *dynamicTimeout) (_ LockContext, timedOutErr error) { + lockSource := getSource(2) + start := UTCNow() + const readLock = false + success := make([]int, len(li.paths)) + for i, path := range li.paths { + if !li.ns.lock(ctx, li.volume, path, lockSource, li.opsID, readLock, timeout.Timeout()) { + timeout.LogFailure() + for si, sint := range success { + if sint == 1 { + li.ns.unlock(li.volume, li.paths[si], readLock) + } + } + if errors.Is(ctx.Err(), context.Canceled) { + return LockContext{}, ctx.Err() + } + return LockContext{}, OperationTimedOut{} + } + success[i] = 1 + } + timeout.LogSuccess(UTCNow().Sub(start)) + return LockContext{ctx: ctx, cancel: func() {}}, nil +} + +// Unlock - block until write lock is released. +func (li *localLockInstance) Unlock(lc LockContext) { + if lc.cancel != nil { + lc.cancel() + } + const readLock = false + for _, path := range li.paths { + li.ns.unlock(li.volume, path, readLock) + } +} + +// RLock - block until read lock is taken or timeout has occurred. +func (li *localLockInstance) GetRLock(ctx context.Context, timeout *dynamicTimeout) (_ LockContext, timedOutErr error) { + lockSource := getSource(2) + start := UTCNow() + const readLock = true + success := make([]int, len(li.paths)) + for i, path := range li.paths { + if !li.ns.lock(ctx, li.volume, path, lockSource, li.opsID, readLock, timeout.Timeout()) { + timeout.LogFailure() + for si, sint := range success { + if sint == 1 { + li.ns.unlock(li.volume, li.paths[si], readLock) + } + } + if err := ctx.Err(); err == context.Canceled { + return LockContext{}, err + } + return LockContext{}, OperationTimedOut{} + } + success[i] = 1 + } + timeout.LogSuccess(UTCNow().Sub(start)) + return LockContext{ctx: ctx, cancel: func() {}}, nil +} + +// RUnlock - block until read lock is released. +func (li *localLockInstance) RUnlock(lc LockContext) { + if lc.cancel != nil { + lc.cancel() + } + const readLock = true + for _, path := range li.paths { + li.ns.unlock(li.volume, path, readLock) + } +} + +func getSource(n int) string { + var funcName string + pc, filename, lineNum, ok := runtime.Caller(n) + if ok { + filename = pathutil.Base(filename) + funcName = strings.TrimPrefix(runtime.FuncForPC(pc).Name(), + "github.com/minio/minio/cmd.") + } else { + filename = "" + lineNum = 0 + } + + return fmt.Sprintf("[%s:%d:%s()]", filename, lineNum, funcName) +} diff --git a/cmd/namespace-lock_test.go b/cmd/namespace-lock_test.go new file mode 100644 index 00000000..3f44e5f9 --- /dev/null +++ b/cmd/namespace-lock_test.go @@ -0,0 +1,102 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "runtime" + "testing" + "time" +) + +// WARNING: +// +// Expected source line number is hard coded, 35, in the +// following test. Adding new code before this test or changing its +// position will cause the line number to change and the test to FAIL +// Tests getSource(). +func TestGetSource(t *testing.T) { + currentSource := func() string { return getSource(2) } + gotSource := currentSource() + // Hard coded line number, 34, in the "expectedSource" value + expectedSource := "[namespace-lock_test.go:34:TestGetSource()]" + if gotSource != expectedSource { + t.Errorf("expected : %s, got : %s", expectedSource, gotSource) + } +} + +// Test lock race +func TestNSLockRace(t *testing.T) { + t.Skip("long test skip it") + + ctx := t.Context() + + for i := range 10000 { + nsLk := newNSLock(false) + + // lk1; ref=1 + if !nsLk.lock(ctx, "volume", "path", "source", "opsID", false, time.Second) { + t.Fatal("failed to acquire lock") + } + + // lk2 + lk2ch := make(chan struct{}) + go func() { + defer close(lk2ch) + nsLk.lock(ctx, "volume", "path", "source", "opsID", false, 1*time.Millisecond) + }() + time.Sleep(1 * time.Millisecond) // wait for goroutine to advance; ref=2 + + // Unlock the 1st lock; ref=1 after this line + nsLk.unlock("volume", "path", false) + + // Taking another lockMapMutex here allows queuing up additional lockers. This should + // not be required but makes reproduction much easier. + nsLk.lockMapMutex.Lock() + + // lk3 blocks. + lk3ch := make(chan bool) + go func() { + lk3ch <- nsLk.lock(ctx, "volume", "path", "source", "opsID", false, 0) + }() + + // lk4, blocks. + lk4ch := make(chan bool) + go func() { + lk4ch <- nsLk.lock(ctx, "volume", "path", "source", "opsID", false, 0) + }() + runtime.Gosched() + + // unlock the manual lock + nsLk.lockMapMutex.Unlock() + + // To trigger the race: + // 1) lk3 or lk4 need to advance and increment the ref on the existing resource, + // successfully acquiring the lock. + // 2) lk2 then needs to advance and remove the resource from lockMap. + // 3) lk3 or lk4 (whichever didn't execute in step 1) then executes and creates + // a new entry in lockMap and acquires a lock for the same resource. + + <-lk2ch + lk3ok := <-lk3ch + lk4ok := <-lk4ch + + if lk3ok && lk4ok { + t.Fatalf("multiple locks acquired; iteration=%d, lk3=%t, lk4=%t", i, lk3ok, lk4ok) + } + } +} diff --git a/cmd/naughty-disk_test.go b/cmd/naughty-disk_test.go new file mode 100644 index 00000000..33162253 --- /dev/null +++ b/cmd/naughty-disk_test.go @@ -0,0 +1,340 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "io" + "sync" + "time" + + "github.com/minio/madmin-go/v3" +) + +// naughtyDisk wraps a POSIX disk and returns programmed errors +// specified by the developer. The purpose is to simulate errors +// that are hard to simulate in practice like DiskNotFound. +// Programmed errors are stored in errors field. +type naughtyDisk struct { + // The real disk + disk StorageAPI + // Programmed errors: API call number => error to return + errors map[int]error + // The error to return when no error value is programmed + defaultErr error + // The current API call number + callNR int + // Data protection + mu sync.Mutex +} + +func newNaughtyDisk(d StorageAPI, errs map[int]error, defaultErr error) *naughtyDisk { + return &naughtyDisk{disk: d, errors: errs, defaultErr: defaultErr} +} + +func (d *naughtyDisk) String() string { + return d.disk.String() +} + +func (d *naughtyDisk) IsOnline() bool { + if err := d.calcError(); err != nil { + return err == errDiskNotFound + } + return d.disk.IsOnline() +} + +func (d *naughtyDisk) LastConn() time.Time { + return d.disk.LastConn() +} + +func (d *naughtyDisk) IsLocal() bool { + return d.disk.IsLocal() +} + +func (d *naughtyDisk) Endpoint() Endpoint { + return d.disk.Endpoint() +} + +func (d *naughtyDisk) Hostname() string { + return d.disk.Hostname() +} + +func (d *naughtyDisk) Healing() *healingTracker { + return d.disk.Healing() +} + +func (d *naughtyDisk) Close() (err error) { + if err = d.calcError(); err != nil { + return err + } + return d.disk.Close() +} + +func (d *naughtyDisk) calcError() (err error) { + d.mu.Lock() + defer d.mu.Unlock() + d.callNR++ + if err, ok := d.errors[d.callNR]; ok { + return err + } + if d.defaultErr != nil { + return d.defaultErr + } + return nil +} + +func (d *naughtyDisk) GetDiskLoc() (poolIdx, setIdx, diskIdx int) { + return -1, -1, -1 +} + +func (d *naughtyDisk) GetDiskID() (string, error) { + return d.disk.GetDiskID() +} + +func (d *naughtyDisk) SetDiskID(id string) { + d.disk.SetDiskID(id) +} + +func (d *naughtyDisk) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry, scanMode madmin.HealScanMode, weSleep func() bool) (info dataUsageCache, err error) { + if err := d.calcError(); err != nil { + return info, err + } + return d.disk.NSScanner(ctx, cache, updates, scanMode, weSleep) +} + +func (d *naughtyDisk) DiskInfo(ctx context.Context, opts DiskInfoOptions) (info DiskInfo, err error) { + if err := d.calcError(); err != nil { + return info, err + } + return d.disk.DiskInfo(ctx, opts) +} + +func (d *naughtyDisk) MakeVolBulk(ctx context.Context, volumes ...string) (err error) { + if err := d.calcError(); err != nil { + return err + } + return d.disk.MakeVolBulk(ctx, volumes...) +} + +func (d *naughtyDisk) MakeVol(ctx context.Context, volume string) (err error) { + if err := d.calcError(); err != nil { + return err + } + return d.disk.MakeVol(ctx, volume) +} + +func (d *naughtyDisk) ListVols(ctx context.Context) (vols []VolInfo, err error) { + if err := d.calcError(); err != nil { + return nil, err + } + return d.disk.ListVols(ctx) +} + +func (d *naughtyDisk) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) { + if err := d.calcError(); err != nil { + return VolInfo{}, err + } + return d.disk.StatVol(ctx, volume) +} + +func (d *naughtyDisk) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) { + if err := d.calcError(); err != nil { + return err + } + return d.disk.DeleteVol(ctx, volume, forceDelete) +} + +func (d *naughtyDisk) WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writer) error { + if err := d.calcError(); err != nil { + return err + } + return d.disk.WalkDir(ctx, opts, wr) +} + +func (d *naughtyDisk) ListDir(ctx context.Context, origvolume, volume, dirPath string, count int) (entries []string, err error) { + if err := d.calcError(); err != nil { + return []string{}, err + } + return d.disk.ListDir(ctx, origvolume, volume, dirPath, count) +} + +func (d *naughtyDisk) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) { + if err := d.calcError(); err != nil { + return 0, err + } + return d.disk.ReadFile(ctx, volume, path, offset, buf, verifier) +} + +func (d *naughtyDisk) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) { + if err := d.calcError(); err != nil { + return nil, err + } + return d.disk.ReadFileStream(ctx, volume, path, offset, length) +} + +func (d *naughtyDisk) CreateFile(ctx context.Context, origvolume, volume, path string, size int64, reader io.Reader) error { + if err := d.calcError(); err != nil { + return err + } + return d.disk.CreateFile(ctx, origvolume, volume, path, size, reader) +} + +func (d *naughtyDisk) AppendFile(ctx context.Context, volume string, path string, buf []byte) error { + if err := d.calcError(); err != nil { + return err + } + return d.disk.AppendFile(ctx, volume, path, buf) +} + +func (d *naughtyDisk) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) { + if err := d.calcError(); err != nil { + return RenameDataResp{}, err + } + return d.disk.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts) +} + +func (d *naughtyDisk) RenamePart(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string, meta []byte, skipParent string) error { + if err := d.calcError(); err != nil { + return err + } + return d.disk.RenamePart(ctx, srcVolume, srcPath, dstVolume, dstPath, meta, skipParent) +} + +func (d *naughtyDisk) ReadParts(ctx context.Context, bucket string, partMetaPaths ...string) ([]*ObjectPartInfo, error) { + if err := d.calcError(); err != nil { + return nil, err + } + return d.disk.ReadParts(ctx, bucket, partMetaPaths...) +} + +func (d *naughtyDisk) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error { + if err := d.calcError(); err != nil { + return err + } + return d.disk.RenameFile(ctx, srcVolume, srcPath, dstVolume, dstPath) +} + +func (d *naughtyDisk) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (*CheckPartsResp, error) { + if err := d.calcError(); err != nil { + return nil, err + } + return d.disk.CheckParts(ctx, volume, path, fi) +} + +func (d *naughtyDisk) DeleteBulk(ctx context.Context, volume string, paths ...string) (err error) { + if err := d.calcError(); err != nil { + return err + } + return d.disk.DeleteBulk(ctx, volume, paths...) +} + +func (d *naughtyDisk) Delete(ctx context.Context, volume string, path string, deleteOpts DeleteOptions) (err error) { + if err := d.calcError(); err != nil { + return err + } + return d.disk.Delete(ctx, volume, path, deleteOpts) +} + +func (d *naughtyDisk) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions, opts DeleteOptions) []error { + if err := d.calcError(); err != nil { + errs := make([]error, len(versions)) + for i := range errs { + errs[i] = err + } + return errs + } + return d.disk.DeleteVersions(ctx, volume, versions, opts) +} + +func (d *naughtyDisk) WriteMetadata(ctx context.Context, origvolume, volume, path string, fi FileInfo) (err error) { + if err := d.calcError(); err != nil { + return err + } + return d.disk.WriteMetadata(ctx, origvolume, volume, path, fi) +} + +func (d *naughtyDisk) UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo, opts UpdateMetadataOpts) (err error) { + if err := d.calcError(); err != nil { + return err + } + return d.disk.UpdateMetadata(ctx, volume, path, fi, opts) +} + +func (d *naughtyDisk) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool, opts DeleteOptions) (err error) { + if err := d.calcError(); err != nil { + return err + } + return d.disk.DeleteVersion(ctx, volume, path, fi, forceDelMarker, opts) +} + +func (d *naughtyDisk) ReadVersion(ctx context.Context, origvolume, volume, path, versionID string, opts ReadOptions) (fi FileInfo, err error) { + if err := d.calcError(); err != nil { + return FileInfo{}, err + } + return d.disk.ReadVersion(ctx, origvolume, volume, path, versionID, opts) +} + +func (d *naughtyDisk) WriteAll(ctx context.Context, volume string, path string, b []byte) (err error) { + if err := d.calcError(); err != nil { + return err + } + return d.disk.WriteAll(ctx, volume, path, b) +} + +func (d *naughtyDisk) ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) { + if err := d.calcError(); err != nil { + return nil, err + } + return d.disk.ReadAll(ctx, volume, path) +} + +func (d *naughtyDisk) ReadXL(ctx context.Context, volume string, path string, readData bool) (rf RawFileInfo, err error) { + if err := d.calcError(); err != nil { + return rf, err + } + return d.disk.ReadXL(ctx, volume, path, readData) +} + +func (d *naughtyDisk) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error) { + if err := d.calcError(); err != nil { + return nil, err + } + return d.disk.VerifyFile(ctx, volume, path, fi) +} + +func (d *naughtyDisk) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) { + if err := d.calcError(); err != nil { + return stat, err + } + return d.disk.StatInfoFile(ctx, volume, path, glob) +} + +func (d *naughtyDisk) ReadMultiple(ctx context.Context, req ReadMultipleReq, resp chan<- ReadMultipleResp) error { + if err := d.calcError(); err != nil { + close(resp) + return err + } + return d.disk.ReadMultiple(ctx, req, resp) +} + +func (d *naughtyDisk) CleanAbandonedData(ctx context.Context, volume string, path string) error { + if err := d.calcError(); err != nil { + return err + } + return d.disk.CleanAbandonedData(ctx, volume, path) +} diff --git a/cmd/net.go b/cmd/net.go new file mode 100644 index 00000000..f0462851 --- /dev/null +++ b/cmd/net.go @@ -0,0 +1,379 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "errors" + "net" + "net/url" + "runtime" + "sort" + "strings" + + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/logger" + xnet "github.com/minio/pkg/v3/net" +) + +var ( + // IPv4 addresses of localhost. + localIP4 = mustGetLocalIP4() + + // IPv6 addresses of localhost. + localIP6 = mustGetLocalIP6() + + // List of all local loopback addresses. + localLoopbacks = mustGetLocalLoopbacks() +) + +// mustSplitHostPort is a wrapper to net.SplitHostPort() where error is assumed to be a fatal. +func mustSplitHostPort(hostPort string) (host, port string) { + xh, err := xnet.ParseHost(hostPort) + if err != nil { + logger.FatalIf(err, "Unable to split host port %s", hostPort) + } + return xh.Name, xh.Port.String() +} + +// mustGetLocalIPs returns IPs of local interface +func mustGetLocalIPs() (ipList []net.IP) { + ifs, err := net.Interfaces() + logger.FatalIf(err, "Unable to get IP addresses of this host") + + for _, interf := range ifs { + addrs, err := interf.Addrs() + if err != nil { + continue + } + if runtime.GOOS == "windows" && interf.Flags&net.FlagUp == 0 { + continue + } + + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + + ipList = append(ipList, ip) + } + } + + return ipList +} + +func mustGetLocalLoopbacks() (ipList set.StringSet) { + ipList = set.NewStringSet() + for _, ip := range mustGetLocalIPs() { + if ip != nil && ip.IsLoopback() { + ipList.Add(ip.String()) + } + } + return ipList +} + +// mustGetLocalIP4 returns IPv4 addresses of localhost. It panics on error. +func mustGetLocalIP4() (ipList set.StringSet) { + ipList = set.NewStringSet() + for _, ip := range mustGetLocalIPs() { + if ip.To4() != nil { + ipList.Add(ip.String()) + } + } + return ipList +} + +// mustGetLocalIP6 returns IPv6 addresses of localhost. It panics on error. +func mustGetLocalIP6() (ipList set.StringSet) { + ipList = set.NewStringSet() + for _, ip := range mustGetLocalIPs() { + if ip.To4() == nil { + ipList.Add(ip.String()) + } + } + return ipList +} + +// getHostIP returns IP address of given host. +func getHostIP(host string) (ipList set.StringSet, err error) { + addrs, err := globalDNSCache.LookupHost(GlobalContext, host) + if err != nil { + return ipList, err + } + + ipList = set.NewStringSet() + for _, addr := range addrs { + ipList.Add(addr) + } + + return ipList, err +} + +// sortIPs - sort ips based on higher octets. +// The logic to sort by last octet is implemented to +// prefer CIDRs with higher octets, this in-turn skips the +// localhost/loopback address to be not preferred as the +// first ip on the list. Subsequently this list helps us print +// a user friendly message with appropriate values. +func sortIPs(ipList []string) []string { + if len(ipList) == 1 { + return ipList + } + + var ipV4s []net.IP + var nonIPs []string + for _, ip := range ipList { + nip := net.ParseIP(ip) + if nip != nil { + ipV4s = append(ipV4s, nip) + } else { + nonIPs = append(nonIPs, ip) + } + } + + sort.Slice(ipV4s, func(i, j int) bool { + // This case is needed when all ips in the list + // have same last octets, Following just ensures that + // 127.0.0.1 is moved to the end of the list. + if ipV4s[i].IsLoopback() { + return false + } + if ipV4s[j].IsLoopback() { + return true + } + return []byte(ipV4s[i].To4())[3] > []byte(ipV4s[j].To4())[3] + }) + + var ips []string + for _, ip := range ipV4s { + ips = append(ips, ip.String()) + } + + return append(nonIPs, ips...) +} + +func getConsoleEndpoints() (consoleEndpoints []string) { + if globalBrowserRedirectURL != nil { + return []string{globalBrowserRedirectURL.String()} + } + var ipList []string + if globalMinioConsoleHost == "" { + ipList = sortIPs(localIP4.ToSlice()) + ipList = append(ipList, localIP6.ToSlice()...) + } else { + ipList = []string{globalMinioConsoleHost} + } + + consoleEndpoints = make([]string, 0, len(ipList)) + for _, ip := range ipList { + consoleEndpoints = append(consoleEndpoints, getURLScheme(globalIsTLS)+"://"+net.JoinHostPort(ip, globalMinioConsolePort)) + } + + return consoleEndpoints +} + +func getAPIEndpoints() (apiEndpoints []string) { + if globalMinioEndpoint != "" { + return []string{globalMinioEndpoint} + } + var ipList []string + if globalMinioHost == "" { + ipList = sortIPs(localIP4.ToSlice()) + ipList = append(ipList, localIP6.ToSlice()...) + } else { + ipList = []string{globalMinioHost} + } + + apiEndpoints = make([]string, 0, len(ipList)) + for _, ip := range ipList { + apiEndpoints = append(apiEndpoints, getURLScheme(globalIsTLS)+"://"+net.JoinHostPort(ip, globalMinioPort)) + } + + return apiEndpoints +} + +// isHostIP - helper for validating if the provided arg is an ip address. +func isHostIP(ipAddress string) bool { + host, _, err := net.SplitHostPort(ipAddress) + if err != nil { + host = ipAddress + } + // Strip off IPv6 zone information. + if i := strings.Index(host, "%"); i > -1 { + host = host[:i] + } + return net.ParseIP(host) != nil +} + +// extractHostPort - extracts host/port from many address formats +// such as, ":9000", "localhost:9000", "http://localhost:9000/" +func extractHostPort(hostAddr string) (string, string, error) { + var addr, scheme string + + if hostAddr == "" { + return "", "", errors.New("unable to process empty address") + } + + // Simplify the work of url.Parse() and always send a url with + if !strings.HasPrefix(hostAddr, "http://") && !strings.HasPrefix(hostAddr, "https://") { + hostAddr = "//" + hostAddr + } + + // Parse address to extract host and scheme field + u, err := url.Parse(hostAddr) + if err != nil { + return "", "", err + } + + addr = u.Host + scheme = u.Scheme + + // Use the given parameter again if url.Parse() + // didn't return any useful result. + if addr == "" { + addr = hostAddr + scheme = "http" + } + + // At this point, addr can be one of the following form: + // ":9000" + // "localhost:9000" + // "localhost" <- in this case, we check for scheme + + host, port, err := net.SplitHostPort(addr) + if err != nil { + if !strings.Contains(err.Error(), "missing port in address") { + return "", "", err + } + + host = addr + + switch scheme { + case "https": + port = "443" + case "http": + port = "80" + default: + return "", "", errors.New("unable to guess port from scheme") + } + } + + return host, port, nil +} + +// isLocalHost - checks if the given parameter +// correspond to one of the local IP of the +// current machine +func isLocalHost(host string, port string, localPort string) (bool, error) { + hostIPs, err := getHostIP(host) + if err != nil { + return false, err + } + + nonInterIPV4s := mustGetLocalIP4().Intersection(hostIPs) + if nonInterIPV4s.IsEmpty() { + hostIPs = hostIPs.ApplyFunc(func(ip string) string { + if net.ParseIP(ip).IsLoopback() { + // Any loopback IP which is not 127.0.0.1 + // convert it to check for intersections. + return "127.0.0.1" + } + return ip + }) + nonInterIPV4s = mustGetLocalIP4().Intersection(hostIPs) + } + nonInterIPV6s := mustGetLocalIP6().Intersection(hostIPs) + + // If intersection of two IP sets is not empty, then the host is localhost. + isLocalv4 := !nonInterIPV4s.IsEmpty() + isLocalv6 := !nonInterIPV6s.IsEmpty() + if port != "" { + return (isLocalv4 || isLocalv6) && (port == localPort), nil + } + return isLocalv4 || isLocalv6, nil +} + +// sameLocalAddrs - returns true if two addresses, even with different +// formats, point to the same machine, e.g: +// +// ':9000' and 'http://localhost:9000/' will return true +func sameLocalAddrs(addr1, addr2 string) (bool, error) { + // Extract host & port from given parameters + host1, port1, err := extractHostPort(addr1) + if err != nil { + return false, err + } + host2, port2, err := extractHostPort(addr2) + if err != nil { + return false, err + } + + var addr1Local, addr2Local bool + + if host1 == "" { + // If empty host means it is localhost + addr1Local = true + } else if addr1Local, err = isLocalHost(host1, port1, port1); err != nil { + // Host not empty, check if it is local + return false, err + } + + if host2 == "" { + // If empty host means it is localhost + addr2Local = true + } else if addr2Local, err = isLocalHost(host2, port2, port2); err != nil { + // Host not empty, check if it is local + return false, err + } + + // If both of addresses point to the same machine, check if + // have the same port + if addr1Local && addr2Local { + if port1 == port2 { + return true, nil + } + } + return false, nil +} + +// CheckLocalServerAddr - checks if serverAddr is valid and local host. +func CheckLocalServerAddr(serverAddr string) error { + host, err := xnet.ParseHost(serverAddr) + if err != nil { + return config.ErrInvalidAddressFlag(err) + } + + // 0.0.0.0 is a wildcard address and refers to local network + // addresses. I.e, 0.0.0.0:9000 like ":9000" refers to port + // 9000 on localhost. + if host.Name != "" && host.Name != net.IPv4zero.String() && host.Name != net.IPv6zero.String() { + localHost, err := isLocalHost(host.Name, host.Port.String(), host.Port.String()) + if err != nil { + return err + } + if !localHost { + return config.ErrInvalidAddressFlag(nil).Msg("host in server address should be this server") + } + } + + return nil +} diff --git a/cmd/net_test.go b/cmd/net_test.go new file mode 100644 index 00000000..d482cad2 --- /dev/null +++ b/cmd/net_test.go @@ -0,0 +1,317 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "errors" + "fmt" + "reflect" + "testing" + + "github.com/minio/minio-go/v7/pkg/set" +) + +func TestMustSplitHostPort(t *testing.T) { + testCases := []struct { + hostPort string + expectedHost string + expectedPort string + }{ + {":54321", "", "54321"}, + {"server:54321", "server", "54321"}, + {":0", "", "0"}, + {"server:https", "server", "443"}, + {"server:http", "server", "80"}, + } + + for _, testCase := range testCases { + host, port := mustSplitHostPort(testCase.hostPort) + if testCase.expectedHost != host { + t.Fatalf("host: expected = %v, got = %v", testCase.expectedHost, host) + } + + if testCase.expectedPort != port { + t.Fatalf("port: expected = %v, got = %v", testCase.expectedPort, port) + } + } +} + +func TestSortIPs(t *testing.T) { + testCases := []struct { + ipList []string + sortedIPList []string + }{ + // Default case of two ips one with higher octet moves + // to the beginning of the list. + { + ipList: []string{"127.0.0.1", "10.0.0.13"}, + sortedIPList: []string{"10.0.0.13", "127.0.0.1"}, + }, + // With multiple types of octet, chooses a higher octet. + { + ipList: []string{"127.0.0.1", "172.0.21.1", "192.168.1.106"}, + sortedIPList: []string{"192.168.1.106", "172.0.21.1", "127.0.0.1"}, + }, + // With different ip along with localhost. + { + ipList: []string{"127.0.0.1", "192.168.1.106"}, + sortedIPList: []string{"192.168.1.106", "127.0.0.1"}, + }, + // With a list of only one element nothing to sort. + { + ipList: []string{"hostname"}, + sortedIPList: []string{"hostname"}, + }, + // With a list of only one element nothing to sort. + { + ipList: []string{"127.0.0.1"}, + sortedIPList: []string{"127.0.0.1"}, + }, + // Non parsable ip is assumed to be hostname and gets preserved + // as the left most elements, regardless of IP based sorting. + { + ipList: []string{"hostname", "127.0.0.1", "192.168.1.106"}, + sortedIPList: []string{"hostname", "192.168.1.106", "127.0.0.1"}, + }, + // Non parsable ip is assumed to be hostname, with a mixed input of ip and hostname. + // gets preserved and moved into left most elements, regardless of + // IP based sorting. + { + ipList: []string{"hostname1", "10.0.0.13", "hostname2", "127.0.0.1", "192.168.1.106"}, + sortedIPList: []string{"hostname1", "hostname2", "192.168.1.106", "10.0.0.13", "127.0.0.1"}, + }, + // With same higher octets, preferentially move the localhost. + { + ipList: []string{"127.0.0.1", "10.0.0.1", "192.168.0.1"}, + sortedIPList: []string{"10.0.0.1", "192.168.0.1", "127.0.0.1"}, + }, + } + for i, testCase := range testCases { + gotIPList := sortIPs(testCase.ipList) + if !reflect.DeepEqual(testCase.sortedIPList, gotIPList) { + t.Errorf("Test %d: Expected %s, got %s", i+1, testCase.sortedIPList, gotIPList) + } + } +} + +func TestMustGetLocalIP4(t *testing.T) { + testCases := []struct { + expectedIPList set.StringSet + }{ + {set.CreateStringSet("127.0.0.1")}, + } + + for _, testCase := range testCases { + ipList := mustGetLocalIP4() + if testCase.expectedIPList != nil && testCase.expectedIPList.Intersection(ipList).IsEmpty() { + t.Fatalf("host: expected = %v, got = %v", testCase.expectedIPList, ipList) + } + } +} + +func TestGetHostIP(t *testing.T) { + testCases := []struct { + host string + expectedIPList set.StringSet + expectedErr error + }{ + {"localhost", set.CreateStringSet("127.0.0.1"), nil}, + } + + for _, testCase := range testCases { + ipList, err := getHostIP(testCase.host) + switch { + case testCase.expectedErr == nil: + if err != nil { + t.Fatalf("error: expected = , got = %v", err) + } + case err == nil: + t.Fatalf("error: expected = %v, got = ", testCase.expectedErr) + case testCase.expectedErr.Error() != err.Error(): + t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err) + } + + if testCase.expectedIPList != nil { + var found bool + for _, ip := range ipList.ToSlice() { + if testCase.expectedIPList.Contains(ip) { + found = true + } + } + if !found { + t.Fatalf("host: expected = %v, got = %v", testCase.expectedIPList, ipList) + } + } + } +} + +// Tests finalize api endpoints. +func TestGetAPIEndpoints(t *testing.T) { + host, port := globalMinioHost, globalMinioPort + defer func() { + globalMinioHost, globalMinioPort = host, port + }() + testCases := []struct { + host, port string + expectedResult string + }{ + {"", "80", "http://127.0.0.1:80"}, + {"127.0.0.1", "80", "http://127.0.0.1:80"}, + {"localhost", "80", "http://localhost:80"}, + } + + for i, testCase := range testCases { + globalMinioHost, globalMinioPort = testCase.host, testCase.port + apiEndpoints := getAPIEndpoints() + apiEndpointSet := set.CreateStringSet(apiEndpoints...) + if !apiEndpointSet.Contains(testCase.expectedResult) { + t.Fatalf("test %d: expected: Found, got: Not Found", i+1) + } + } +} + +func TestCheckLocalServerAddr(t *testing.T) { + testCases := []struct { + serverAddr string + expectedErr error + }{ + {":54321", nil}, + {"localhost:54321", nil}, + {"0.0.0.0:9000", nil}, + {":0", nil}, + {"localhost", nil}, + {"", fmt.Errorf("invalid argument")}, + {"example.org:54321", fmt.Errorf("host in server address should be this server")}, + {":-10", fmt.Errorf("port must be between 0 to 65535")}, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + err := CheckLocalServerAddr(testCase.serverAddr) + switch { + case testCase.expectedErr == nil: + if err != nil { + t.Errorf("error: expected = , got = %v", err) + } + case err == nil: + t.Errorf("error: expected = %v, got = ", testCase.expectedErr) + case testCase.expectedErr.Error() != err.Error(): + t.Errorf("error: expected = %v, got = %v", testCase.expectedErr, err) + } + }) + } +} + +func TestExtractHostPort(t *testing.T) { + testCases := []struct { + addr string + host string + port string + expectedErr error + }{ + {"", "", "", errors.New("unable to process empty address")}, + {"localhost:9000", "localhost", "9000", nil}, + {"http://:9000/", "", "9000", nil}, + {"http://8.8.8.8:9000/", "8.8.8.8", "9000", nil}, + {"https://facebook.com:9000/", "facebook.com", "9000", nil}, + } + + for i, testCase := range testCases { + host, port, err := extractHostPort(testCase.addr) + if testCase.expectedErr == nil && err != nil { + t.Fatalf("Test %d: should succeed but failed with err: %v", i+1, err) + } + if testCase.expectedErr != nil && err == nil { + t.Fatalf("Test %d:, should fail but succeeded.", i+1) + } + if err == nil { + if host != testCase.host { + t.Fatalf("Test %d: expected: %v, found: %v", i+1, testCase.host, host) + } + if port != testCase.port { + t.Fatalf("Test %d: expected: %v, found: %v", i+1, testCase.port, port) + } + } + if testCase.expectedErr != nil && err != nil { + if testCase.expectedErr.Error() != err.Error() { + t.Fatalf("Test %d: failed with different error, expected: '%v', found:'%v'.", i+1, testCase.expectedErr, err) + } + } + } +} + +func TestSameLocalAddrs(t *testing.T) { + testCases := []struct { + addr1 string + addr2 string + sameAddr bool + expectedErr error + }{ + {"", "", false, errors.New("unable to process empty address")}, + {":9000", ":9000", true, nil}, + {"localhost:9000", ":9000", true, nil}, + {"localhost:9000", "http://localhost:9000", true, nil}, + {"http://localhost:9000", ":9000", true, nil}, + {"http://localhost:9000", "http://localhost:9000", true, nil}, + {"http://8.8.8.8:9000", "http://localhost:9000", false, nil}, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + sameAddr, err := sameLocalAddrs(testCase.addr1, testCase.addr2) + if testCase.expectedErr != nil && err == nil { + t.Errorf("should fail but succeeded") + } + if testCase.expectedErr == nil && err != nil { + t.Errorf("should succeed but failed with %v", err) + } + if err == nil { + if sameAddr != testCase.sameAddr { + t.Errorf("expected: %v, found: %v", testCase.sameAddr, sameAddr) + } + } else { + if err.Error() != testCase.expectedErr.Error() { + t.Errorf("failed with different error, expected: '%v', found:'%v'.", + testCase.expectedErr, err) + } + } + }) + } +} + +func TestIsHostIP(t *testing.T) { + testCases := []struct { + args string + expectedResult bool + }{ + {"localhost", false}, + {"localhost:9000", false}, + {"example.com", false}, + {"http://192.168.1.0", false}, + {"http://192.168.1.0:9000", false}, + {"192.168.1.0", true}, + {"[2001:3984:3989::20%eth0]:9000", true}, + } + + for _, testCase := range testCases { + ret := isHostIP(testCase.args) + if testCase.expectedResult != ret { + t.Fatalf("expected: %v , got: %v", testCase.expectedResult, ret) + } + } +} diff --git a/cmd/notification-summary.go b/cmd/notification-summary.go new file mode 100644 index 00000000..28d07c7e --- /dev/null +++ b/cmd/notification-summary.go @@ -0,0 +1,70 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "github.com/minio/madmin-go/v3" +) + +// GetTotalCapacity gets the total capacity in the cluster. +func GetTotalCapacity(diskInfo []madmin.Disk) (capacity uint64) { + for _, disk := range diskInfo { + capacity += disk.TotalSpace + } + return capacity +} + +// GetTotalUsableCapacity gets the total usable capacity in the cluster. +func GetTotalUsableCapacity(diskInfo []madmin.Disk, s StorageInfo) (capacity uint64) { + for _, disk := range diskInfo { + // Ignore invalid. + if disk.PoolIndex < 0 || len(s.Backend.StandardSCData) <= disk.PoolIndex { + // https://github.com/minio/minio/issues/16500 + continue + } + // Ignore parity disks + if disk.DiskIndex < s.Backend.StandardSCData[disk.PoolIndex] { + capacity += disk.TotalSpace + } + } + return capacity +} + +// GetTotalCapacityFree gets the total capacity free in the cluster. +func GetTotalCapacityFree(diskInfo []madmin.Disk) (capacity uint64) { + for _, d := range diskInfo { + capacity += d.AvailableSpace + } + return capacity +} + +// GetTotalUsableCapacityFree gets the total usable capacity free in the cluster. +func GetTotalUsableCapacityFree(diskInfo []madmin.Disk, s StorageInfo) (capacity uint64) { + for _, disk := range diskInfo { + // Ignore invalid. + if disk.PoolIndex < 0 || len(s.Backend.StandardSCData) <= disk.PoolIndex { + // https://github.com/minio/minio/issues/16500 + continue + } + // Ignore parity disks + if disk.DiskIndex < s.Backend.StandardSCData[disk.PoolIndex] { + capacity += disk.AvailableSpace + } + } + return capacity +} diff --git a/cmd/notification.go b/cmd/notification.go new file mode 100644 index 00000000..152856ac --- /dev/null +++ b/cmd/notification.go @@ -0,0 +1,1632 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "math/rand" + "net/http" + "net/url" + "runtime" + "sync" + "time" + + "github.com/cespare/xxhash/v2" + "github.com/klauspost/compress/zip" + "github.com/minio/madmin-go/v3" + xioutil "github.com/minio/minio/internal/ioutil" + xnet "github.com/minio/pkg/v3/net" + "github.com/minio/pkg/v3/sync/errgroup" + "github.com/minio/pkg/v3/workers" + + "github.com/minio/minio/internal/bucket/bandwidth" + "github.com/minio/minio/internal/logger" +) + +// This file contains peer related notifications. For sending notifications to +// external systems, see event-notification.go + +// NotificationSys - notification system. +type NotificationSys struct { + peerClients []*peerRESTClient // Excludes self + allPeerClients []*peerRESTClient // Includes nil client for self +} + +// NotificationPeerErr returns error associated for a remote peer. +type NotificationPeerErr struct { + Host xnet.Host // Remote host on which the rpc call was initiated + Err error // Error returned by the remote peer for an rpc call +} + +// A NotificationGroup is a collection of goroutines working on subtasks that are part of +// the same overall task. +// +// A zero NotificationGroup is valid and does not cancel on error. +type NotificationGroup struct { + workers *workers.Workers + errs []NotificationPeerErr + retryCount int +} + +// WithNPeers returns a new NotificationGroup with length of errs slice upto nerrs, +// upon Wait() errors are returned collected from all tasks. +func WithNPeers(nerrs int) *NotificationGroup { + if nerrs <= 0 { + nerrs = 1 + } + wk, _ := workers.New(nerrs) + return &NotificationGroup{errs: make([]NotificationPeerErr, nerrs), workers: wk, retryCount: 3} +} + +// WithNPeersThrottled returns a new NotificationGroup with length of errs slice upto nerrs, +// upon Wait() errors are returned collected from all tasks, optionally allows for X workers +// only "per" parallel task. +func WithNPeersThrottled(nerrs, wks int) *NotificationGroup { + if nerrs <= 0 { + nerrs = 1 + } + if wks > nerrs { + wks = nerrs + } + wk, _ := workers.New(wks) + return &NotificationGroup{errs: make([]NotificationPeerErr, nerrs), workers: wk, retryCount: 3} +} + +// WithRetries sets the retry count for all function calls from the Go method. +func (g *NotificationGroup) WithRetries(retryCount int) *NotificationGroup { + if g != nil { + g.retryCount = retryCount + } + return g +} + +// Wait blocks until all function calls from the Go method have returned, then +// returns the slice of errors from all function calls. +func (g *NotificationGroup) Wait() []NotificationPeerErr { + g.workers.Wait() + return g.errs +} + +// Go calls the given function in a new goroutine. +// +// The first call to return a non-nil error will be +// collected in errs slice and returned by Wait(). +func (g *NotificationGroup) Go(ctx context.Context, f func() error, index int, addr xnet.Host) { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + + g.workers.Take() + + go func() { + defer g.workers.Give() + + g.errs[index] = NotificationPeerErr{ + Host: addr, + } + + retryCount := g.retryCount + for i := range retryCount { + g.errs[index].Err = nil + if err := f(); err != nil { + g.errs[index].Err = err + + if contextCanceled(ctx) { + // context already canceled no retries. + retryCount = 0 + } + + // Last iteration log the error. + if i == retryCount-1 { + reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", addr.String()) + ctx := logger.SetReqInfo(ctx, reqInfo) + peersLogOnceIf(ctx, err, addr.String()) + } + + // Wait for a minimum of 100ms and dynamically increase this based on number of attempts. + if i < retryCount-1 { + time.Sleep(100*time.Millisecond + time.Duration(r.Float64()*float64(time.Second))) + continue + } + } + break + } + }() +} + +// DeletePolicy - deletes policy across all peers. +func (sys *NotificationSys) DeletePolicy(ctx context.Context, policyName string) []NotificationPeerErr { + ng := WithNPeers(len(sys.peerClients)).WithRetries(1) + for idx, client := range sys.peerClients { + ng.Go(ctx, func() error { + if client == nil { + return errPeerNotReachable + } + return client.DeletePolicy(ctx, policyName) + }, idx, *client.host) + } + return ng.Wait() +} + +// LoadPolicy - reloads a specific modified policy across all peers +func (sys *NotificationSys) LoadPolicy(ctx context.Context, policyName string) []NotificationPeerErr { + ng := WithNPeers(len(sys.peerClients)).WithRetries(1) + for idx, client := range sys.peerClients { + ng.Go(ctx, func() error { + if client == nil { + return errPeerNotReachable + } + return client.LoadPolicy(ctx, policyName) + }, idx, *client.host) + } + return ng.Wait() +} + +// LoadPolicyMapping - reloads a policy mapping across all peers +func (sys *NotificationSys) LoadPolicyMapping(ctx context.Context, userOrGroup string, userType IAMUserType, isGroup bool) []NotificationPeerErr { + ng := WithNPeers(len(sys.peerClients)).WithRetries(1) + for idx, client := range sys.peerClients { + ng.Go(ctx, func() error { + if client == nil { + return errPeerNotReachable + } + return client.LoadPolicyMapping(ctx, userOrGroup, userType, isGroup) + }, idx, *client.host) + } + return ng.Wait() +} + +// DeleteUser - deletes a specific user across all peers +func (sys *NotificationSys) DeleteUser(ctx context.Context, accessKey string) []NotificationPeerErr { + ng := WithNPeers(len(sys.peerClients)).WithRetries(1) + for idx, client := range sys.peerClients { + ng.Go(ctx, func() error { + if client == nil { + return errPeerNotReachable + } + return client.DeleteUser(ctx, accessKey) + }, idx, *client.host) + } + return ng.Wait() +} + +// LoadUser - reloads a specific user across all peers +func (sys *NotificationSys) LoadUser(ctx context.Context, accessKey string, temp bool) []NotificationPeerErr { + ng := WithNPeers(len(sys.peerClients)).WithRetries(1) + for idx, client := range sys.peerClients { + ng.Go(ctx, func() error { + if client == nil { + return errPeerNotReachable + } + return client.LoadUser(ctx, accessKey, temp) + }, idx, *client.host) + } + return ng.Wait() +} + +// LoadGroup - loads a specific group on all peers. +func (sys *NotificationSys) LoadGroup(ctx context.Context, group string) []NotificationPeerErr { + ng := WithNPeers(len(sys.peerClients)).WithRetries(1) + for idx, client := range sys.peerClients { + ng.Go(ctx, func() error { + if client == nil { + return errPeerNotReachable + } + return client.LoadGroup(ctx, group) + }, idx, *client.host) + } + return ng.Wait() +} + +// DeleteServiceAccount - deletes a specific service account across all peers +func (sys *NotificationSys) DeleteServiceAccount(ctx context.Context, accessKey string) []NotificationPeerErr { + ng := WithNPeers(len(sys.peerClients)).WithRetries(1) + for idx, client := range sys.peerClients { + ng.Go(ctx, func() error { + if client == nil { + return errPeerNotReachable + } + return client.DeleteServiceAccount(ctx, accessKey) + }, idx, *client.host) + } + return ng.Wait() +} + +// LoadServiceAccount - reloads a specific service account across all peers +func (sys *NotificationSys) LoadServiceAccount(ctx context.Context, accessKey string) []NotificationPeerErr { + ng := WithNPeers(len(sys.peerClients)).WithRetries(1) + for idx, client := range sys.peerClients { + ng.Go(ctx, func() error { + if client == nil { + return errPeerNotReachable + } + return client.LoadServiceAccount(ctx, accessKey) + }, idx, *client.host) + } + return ng.Wait() +} + +// BackgroundHealStatus - returns background heal status of all peers +func (sys *NotificationSys) BackgroundHealStatus(ctx context.Context) ([]madmin.BgHealState, []NotificationPeerErr) { + ng := WithNPeers(len(sys.peerClients)) + states := make([]madmin.BgHealState, len(sys.peerClients)) + for idx, client := range sys.peerClients { + client := client + ng.Go(ctx, func() error { + if client == nil { + return errPeerNotReachable + } + st, err := client.BackgroundHealStatus(ctx) + if err != nil { + return err + } + states[idx] = st + return nil + }, idx, *client.host) + } + + return states, ng.Wait() +} + +// StartProfiling - start profiling on remote peers, by initiating a remote RPC. +func (sys *NotificationSys) StartProfiling(ctx context.Context, profiler string) []NotificationPeerErr { + ng := WithNPeers(len(sys.peerClients)) + for idx, client := range sys.peerClients { + if client == nil { + continue + } + client := client + ng.Go(ctx, func() error { + return client.StartProfiling(ctx, profiler) + }, idx, *client.host) + } + return ng.Wait() +} + +// DownloadProfilingData - download profiling data from all remote peers. +func (sys *NotificationSys) DownloadProfilingData(ctx context.Context, writer io.Writer) (profilingDataFound bool) { + // Initialize a zip writer which will provide a zipped content + // of profiling data of all nodes + zipWriter := zip.NewWriter(writer) + defer zipWriter.Close() + + // Start by embedding cluster info. + if b := getClusterMetaInfo(ctx); len(b) > 0 { + internalLogIf(ctx, embedFileInZip(zipWriter, "cluster.info", b, 0o600)) + } + + // Profiles can be quite big, so we limit to max 16 concurrent downloads. + ng := WithNPeersThrottled(len(sys.peerClients), 16) + var writeMu sync.Mutex + for i, client := range sys.peerClients { + if client == nil { + continue + } + ng.Go(ctx, func() error { + // Give 15 seconds to each remote call. + // Errors are logged but not returned. + ctx, cancel := context.WithTimeout(ctx, 15*time.Second) + defer cancel() + data, err := client.DownloadProfileData(ctx) + if err != nil { + reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String()) + ctx := logger.SetReqInfo(ctx, reqInfo) + peersLogOnceIf(ctx, err, client.host.String()) + return nil + } + + for typ, data := range data { + // zip writer only handles one concurrent write + writeMu.Lock() + profilingDataFound = true + err := embedFileInZip(zipWriter, fmt.Sprintf("profile-%s-%s", client.host.String(), typ), data, 0o600) + writeMu.Unlock() + if err != nil { + reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", client.host.String()) + ctx := logger.SetReqInfo(ctx, reqInfo) + peersLogOnceIf(ctx, err, client.host.String()) + } + } + return nil + }, i, *client.host) + } + ng.Wait() + if ctx.Err() != nil { + return false + } + + // Local host + thisAddr, err := xnet.ParseHost(globalLocalNodeName) + if err != nil { + bugLogIf(ctx, err) + return profilingDataFound + } + + data, err := getProfileData() + if err != nil { + reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", thisAddr.String()) + ctx := logger.SetReqInfo(ctx, reqInfo) + bugLogIf(ctx, err) + return profilingDataFound + } + + profilingDataFound = true + + // Send profiling data to zip as file + for typ, data := range data { + err := embedFileInZip(zipWriter, fmt.Sprintf("profile-%s-%s", thisAddr, typ), data, 0o600) + internalLogIf(ctx, err) + } + + return profilingDataFound +} + +// VerifyBinary - asks remote peers to verify the checksum +func (sys *NotificationSys) VerifyBinary(ctx context.Context, u *url.URL, sha256Sum []byte, releaseInfo string, bin []byte) []NotificationPeerErr { + // FIXME: network calls made in this manner such as one goroutine per node, + // can easily eat into the internode bandwidth. This function would be mostly + // TX saturating, however there are situations where a RX might also saturate. + // To avoid these problems we must split the work at scale. With 1000 node + // setup becoming a reality we must try to shard the work properly such as + // pick 10 nodes that precisely can send those 100 requests the first node + // in the 10 node shard would coordinate between other 9 shards to get the + // rest of the `99*9` requests. + // + // This essentially splits the workload properly and also allows for network + // utilization to be optimal, instead of blindly throttling the way we are + // doing below. However the changes that are needed here are a bit involved, + // further discussion advised. Remove this comment and remove the worker model + // for this function in future. + maxWorkers := runtime.GOMAXPROCS(0) / 2 + ng := WithNPeersThrottled(len(sys.peerClients), maxWorkers) + for idx, client := range sys.peerClients { + if client == nil { + continue + } + client := client + ng.Go(ctx, func() error { + return client.VerifyBinary(ctx, u, sha256Sum, releaseInfo, bytes.NewReader(bin)) + }, idx, *client.host) + } + return ng.Wait() +} + +// CommitBinary - asks remote peers to overwrite the old binary with the new one +func (sys *NotificationSys) CommitBinary(ctx context.Context) []NotificationPeerErr { + ng := WithNPeers(len(sys.peerClients)) + for idx, client := range sys.peerClients { + if client == nil { + continue + } + client := client + ng.Go(ctx, func() error { + return client.CommitBinary(ctx) + }, idx, *client.host) + } + return ng.Wait() +} + +// SignalConfigReload reloads requested sub-system on a remote peer dynamically. +func (sys *NotificationSys) SignalConfigReload(subSys string) []NotificationPeerErr { + ng := WithNPeers(len(sys.peerClients)) + for idx, client := range sys.peerClients { + if client == nil { + continue + } + client := client + ng.Go(GlobalContext, func() error { + return client.SignalService(serviceReloadDynamic, subSys, false, nil) + }, idx, *client.host) + } + return ng.Wait() +} + +// SignalService - calls signal service RPC call on all peers. +func (sys *NotificationSys) SignalService(sig serviceSignal) []NotificationPeerErr { + ng := WithNPeers(len(sys.peerClients)) + for idx, client := range sys.peerClients { + if client == nil { + continue + } + client := client + ng.Go(GlobalContext, func() error { + // force == true preserves the current behavior + return client.SignalService(sig, "", false, nil) + }, idx, *client.host) + } + return ng.Wait() +} + +// SignalServiceV2 - calls signal service RPC call on all peers with v2 API +func (sys *NotificationSys) SignalServiceV2(sig serviceSignal, dryRun bool, execAt *time.Time) []NotificationPeerErr { + ng := WithNPeers(len(sys.peerClients)) + for idx, client := range sys.peerClients { + if client == nil { + continue + } + client := client + ng.Go(GlobalContext, func() error { + return client.SignalService(sig, "", dryRun, execAt) + }, idx, *client.host) + } + return ng.Wait() +} + +var errPeerNotReachable = errors.New("peer is not reachable") + +// GetLocks - makes GetLocks RPC call on all peers. +func (sys *NotificationSys) GetLocks(ctx context.Context, r *http.Request) []*PeerLocks { + locksResp := make([]*PeerLocks, len(sys.peerClients)) + g := errgroup.WithNErrs(len(sys.peerClients)) + for index, client := range sys.peerClients { + client := client + g.Go(func() error { + if client == nil { + return errPeerNotReachable + } + serverLocksResp, err := sys.peerClients[index].GetLocks(ctx) + if err != nil { + return err + } + locksResp[index] = &PeerLocks{ + Addr: sys.peerClients[index].host.String(), + Locks: serverLocksResp, + } + return nil + }, index) + } + for index, err := range g.Wait() { + reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", + sys.peerClients[index].host.String()) + ctx := logger.SetReqInfo(ctx, reqInfo) + peersLogOnceIf(ctx, err, sys.peerClients[index].host.String()) + } + locksResp = append(locksResp, &PeerLocks{ + Addr: getHostName(r), + Locks: globalLockServer.DupLockMap(), + }) + return locksResp +} + +// LoadBucketMetadata - calls LoadBucketMetadata call on all peers +func (sys *NotificationSys) LoadBucketMetadata(ctx context.Context, bucketName string) { + ng := WithNPeers(len(sys.peerClients)) + for idx, client := range sys.peerClients { + if client == nil { + continue + } + client := client + ng.Go(ctx, func() error { + return client.LoadBucketMetadata(ctx, bucketName) + }, idx, *client.host) + } + for _, nErr := range ng.Wait() { + reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String()) + if nErr.Err != nil { + peersLogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String()) + } + } +} + +// DeleteBucketMetadata - calls DeleteBucketMetadata call on all peers +func (sys *NotificationSys) DeleteBucketMetadata(ctx context.Context, bucketName string) { + globalReplicationStats.Load().Delete(bucketName) + globalBucketMetadataSys.Remove(bucketName) + globalBucketTargetSys.Delete(bucketName) + globalEventNotifier.RemoveNotification(bucketName) + globalBucketConnStats.delete(bucketName) + globalBucketHTTPStats.delete(bucketName) + if localMetacacheMgr != nil { + localMetacacheMgr.deleteBucketCache(bucketName) + } + + ng := WithNPeers(len(sys.peerClients)) + for idx, client := range sys.peerClients { + if client == nil { + continue + } + client := client + ng.Go(ctx, func() error { + return client.DeleteBucketMetadata(ctx, bucketName) + }, idx, *client.host) + } + for _, nErr := range ng.Wait() { + reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String()) + if nErr.Err != nil { + peersLogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String()) + } + } +} + +// GetClusterAllBucketStats - returns bucket stats for all buckets from all remote peers. +func (sys *NotificationSys) GetClusterAllBucketStats(ctx context.Context) []BucketStatsMap { + ng := WithNPeers(len(sys.peerClients)).WithRetries(1) + replicationStats := make([]BucketStatsMap, len(sys.peerClients)) + for index, client := range sys.peerClients { + client := client + ng.Go(ctx, func() error { + if client == nil { + return errPeerNotReachable + } + bsMap, err := client.GetAllBucketStats(ctx) + if err != nil { + return err + } + replicationStats[index] = bsMap + return nil + }, index, *client.host) + } + for _, nErr := range ng.Wait() { + reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String()) + if nErr.Err != nil { + peersLogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String()) + } + } + + replicationStatsList := globalReplicationStats.Load().GetAll() + bucketStatsMap := BucketStatsMap{ + Stats: make(map[string]BucketStats, len(replicationStatsList)), + Timestamp: UTCNow(), + } + for k, replicationStats := range replicationStatsList { + bucketStatsMap.Stats[k] = BucketStats{ + ReplicationStats: replicationStats, + ProxyStats: globalReplicationStats.Load().getProxyStats(k), + } + } + + replicationStats = append(replicationStats, bucketStatsMap) + return replicationStats +} + +// GetClusterBucketStats - calls GetClusterBucketStats call on all peers for a cluster statistics view. +func (sys *NotificationSys) GetClusterBucketStats(ctx context.Context, bucketName string) []BucketStats { + ng := WithNPeers(len(sys.peerClients)).WithRetries(1) + bucketStats := make([]BucketStats, len(sys.peerClients)) + for index, client := range sys.peerClients { + client := client + ng.Go(ctx, func() error { + if client == nil { + return errPeerNotReachable + } + bs, err := client.GetBucketStats(ctx, bucketName) + if err != nil { + return err + } + bucketStats[index] = bs + return nil + }, index, *client.host) + } + for _, nErr := range ng.Wait() { + reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String()) + if nErr.Err != nil { + peersLogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String()) + } + } + if st := globalReplicationStats.Load(); st != nil { + bucketStats = append(bucketStats, BucketStats{ + ReplicationStats: st.Get(bucketName), + QueueStats: ReplicationQueueStats{Nodes: []ReplQNodeStats{st.getNodeQueueStats(bucketName)}}, + ProxyStats: st.getProxyStats(bucketName), + }) + } + return bucketStats +} + +// GetClusterSiteMetrics - calls GetClusterSiteMetrics call on all peers for a cluster statistics view. +func (sys *NotificationSys) GetClusterSiteMetrics(ctx context.Context) []SRMetricsSummary { + ng := WithNPeers(len(sys.peerClients)).WithRetries(1) + siteStats := make([]SRMetricsSummary, len(sys.peerClients)) + for index, client := range sys.peerClients { + client := client + ng.Go(ctx, func() error { + if client == nil { + return errPeerNotReachable + } + sm, err := client.GetSRMetrics(ctx) + if err != nil { + return err + } + siteStats[index] = sm + return nil + }, index, *client.host) + } + for _, nErr := range ng.Wait() { + reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String()) + if nErr.Err != nil { + peersLogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String()) + } + } + siteStats = append(siteStats, globalReplicationStats.Load().getSRMetricsForNode()) + return siteStats +} + +// ReloadPoolMeta reloads on disk updates on pool metadata +func (sys *NotificationSys) ReloadPoolMeta(ctx context.Context) { + ng := WithNPeers(len(sys.peerClients)) + for idx, client := range sys.peerClients { + if client == nil { + continue + } + client := client + ng.Go(ctx, func() error { + return client.ReloadPoolMeta(ctx) + }, idx, *client.host) + } + for _, nErr := range ng.Wait() { + reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String()) + if nErr.Err != nil { + peersLogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String()) + } + } +} + +// DeleteUploadID notifies all the MinIO nodes to remove the +// given uploadID from cache +func (sys *NotificationSys) DeleteUploadID(ctx context.Context, uploadID string) { + ng := WithNPeers(len(sys.peerClients)) + for idx, client := range sys.peerClients { + if client == nil { + continue + } + client := client + ng.Go(ctx, func() error { + return client.DeleteUploadID(ctx, uploadID) + }, idx, *client.host) + } + for _, nErr := range ng.Wait() { + reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String()) + if nErr.Err != nil { + peersLogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String()) + } + } +} + +// StopRebalance notifies all MinIO nodes to signal any ongoing rebalance +// goroutine to stop. +func (sys *NotificationSys) StopRebalance(ctx context.Context) { + objAPI := newObjectLayerFn() + if objAPI == nil { + internalLogIf(ctx, errServerNotInitialized) + return + } + + ng := WithNPeers(len(sys.peerClients)) + for idx, client := range sys.peerClients { + if client == nil { + continue + } + client := client + ng.Go(ctx, func() error { + return client.StopRebalance(ctx) + }, idx, *client.host) + } + for _, nErr := range ng.Wait() { + reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String()) + if nErr.Err != nil { + peersLogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String()) + } + } + + if pools, ok := objAPI.(*erasureServerPools); ok { + pools.StopRebalance() + } +} + +// LoadRebalanceMeta notifies all peers to load rebalance.bin from object layer. +// Note: Only peers participating in rebalance operation, namely the first node +// in each pool will load rebalance.bin. +func (sys *NotificationSys) LoadRebalanceMeta(ctx context.Context, startRebalance bool) { + ng := WithNPeers(len(sys.peerClients)) + for idx, client := range sys.peerClients { + if client == nil { + continue + } + client := client + ng.Go(ctx, func() error { + return client.LoadRebalanceMeta(ctx, startRebalance) + }, idx, *client.host) + } + for _, nErr := range ng.Wait() { + reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String()) + if nErr.Err != nil { + peersLogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String()) + } + } +} + +// LoadTransitionTierConfig notifies remote peers to load their remote tier +// configs from config store. +func (sys *NotificationSys) LoadTransitionTierConfig(ctx context.Context) { + ng := WithNPeers(len(sys.peerClients)) + for idx, client := range sys.peerClients { + if client == nil { + continue + } + client := client + ng.Go(ctx, func() error { + return client.LoadTransitionTierConfig(ctx) + }, idx, *client.host) + } + for _, nErr := range ng.Wait() { + reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", nErr.Host.String()) + if nErr.Err != nil { + peersLogOnceIf(logger.SetReqInfo(ctx, reqInfo), nErr.Err, nErr.Host.String()) + } + } +} + +// GetCPUs - Get all CPU information. +func (sys *NotificationSys) GetCPUs(ctx context.Context) []madmin.CPUs { + reply := make([]madmin.CPUs, len(sys.peerClients)) + + g := errgroup.WithNErrs(len(sys.peerClients)) + for index, client := range sys.peerClients { + if client == nil { + continue + } + index := index + g.Go(func() error { + var err error + reply[index], err = sys.peerClients[index].GetCPUs(ctx) + return err + }, index) + } + + for index, err := range g.Wait() { + if err != nil { + sys.addNodeErr(&reply[index], sys.peerClients[index], err) + } + } + return reply +} + +// GetNetInfo - Network information +func (sys *NotificationSys) GetNetInfo(ctx context.Context) []madmin.NetInfo { + reply := make([]madmin.NetInfo, len(sys.peerClients)) + + g := errgroup.WithNErrs(len(sys.peerClients)) + for index, client := range sys.peerClients { + if client == nil { + continue + } + index := index + g.Go(func() error { + var err error + reply[index], err = sys.peerClients[index].GetNetInfo(ctx) + return err + }, index) + } + + for index, err := range g.Wait() { + if err != nil { + sys.addNodeErr(&reply[index], sys.peerClients[index], err) + } + } + return reply +} + +// GetPartitions - Disk partition information +func (sys *NotificationSys) GetPartitions(ctx context.Context) []madmin.Partitions { + reply := make([]madmin.Partitions, len(sys.peerClients)) + + g := errgroup.WithNErrs(len(sys.peerClients)) + for index, client := range sys.peerClients { + if client == nil { + continue + } + index := index + g.Go(func() error { + var err error + reply[index], err = sys.peerClients[index].GetPartitions(ctx) + return err + }, index) + } + + for index, err := range g.Wait() { + if err != nil { + sys.addNodeErr(&reply[index], sys.peerClients[index], err) + } + } + return reply +} + +// GetOSInfo - Get operating system's information +func (sys *NotificationSys) GetOSInfo(ctx context.Context) []madmin.OSInfo { + reply := make([]madmin.OSInfo, len(sys.peerClients)) + + g := errgroup.WithNErrs(len(sys.peerClients)) + for index, client := range sys.peerClients { + if client == nil { + continue + } + index := index + g.Go(func() error { + var err error + reply[index], err = sys.peerClients[index].GetOSInfo(ctx) + return err + }, index) + } + + for index, err := range g.Wait() { + if err != nil { + sys.addNodeErr(&reply[index], sys.peerClients[index], err) + } + } + return reply +} + +// GetMetrics - Get metrics from all peers. +func (sys *NotificationSys) GetMetrics(ctx context.Context, t madmin.MetricType, opts collectMetricsOpts) []madmin.RealtimeMetrics { + reply := make([]madmin.RealtimeMetrics, len(sys.peerClients)) + + g := errgroup.WithNErrs(len(sys.peerClients)) + for index, client := range sys.peerClients { + if client == nil { + continue + } + host := client.host.String() + if len(opts.hosts) > 0 { + if _, ok := opts.hosts[host]; !ok { + continue + } + } + + index := index + g.Go(func() error { + var err error + reply[index], err = sys.peerClients[index].GetMetrics(ctx, t, opts) + return err + }, index) + } + + for index, err := range g.Wait() { + if err != nil { + reply[index].Errors = []string{fmt.Sprintf("%s: %s (rpc)", sys.peerClients[index].String(), err.Error())} + } + } + return reply +} + +// GetResourceMetrics - gets the resource metrics from all nodes excluding self. +func (sys *NotificationSys) GetResourceMetrics(ctx context.Context) <-chan MetricV2 { + if sys == nil { + return nil + } + g := errgroup.WithNErrs(len(sys.peerClients)) + peerChannels := make([]<-chan MetricV2, len(sys.peerClients)) + for index := range sys.peerClients { + g.Go(func() error { + if sys.peerClients[index] == nil { + return errPeerNotReachable + } + var err error + peerChannels[index], err = sys.peerClients[index].GetResourceMetrics(ctx) + return err + }, index) + } + return sys.collectPeerMetrics(ctx, peerChannels, g) +} + +// GetSysConfig - Get information about system config +// (only the config that are of concern to minio) +func (sys *NotificationSys) GetSysConfig(ctx context.Context) []madmin.SysConfig { + reply := make([]madmin.SysConfig, len(sys.peerClients)) + + g := errgroup.WithNErrs(len(sys.peerClients)) + for index, client := range sys.peerClients { + if client == nil { + continue + } + index := index + g.Go(func() error { + var err error + reply[index], err = sys.peerClients[index].GetSysConfig(ctx) + return err + }, index) + } + + for index, err := range g.Wait() { + if err != nil { + sys.addNodeErr(&reply[index], sys.peerClients[index], err) + } + } + return reply +} + +// GetSysServices - Get information about system services +// (only the services that are of concern to minio) +func (sys *NotificationSys) GetSysServices(ctx context.Context) []madmin.SysServices { + reply := make([]madmin.SysServices, len(sys.peerClients)) + + g := errgroup.WithNErrs(len(sys.peerClients)) + for index, client := range sys.peerClients { + if client == nil { + continue + } + index := index + g.Go(func() error { + var err error + reply[index], err = sys.peerClients[index].GetSELinuxInfo(ctx) + return err + }, index) + } + + for index, err := range g.Wait() { + if err != nil { + sys.addNodeErr(&reply[index], sys.peerClients[index], err) + } + } + return reply +} + +func (sys *NotificationSys) addNodeErr(nodeInfo madmin.NodeInfo, peerClient *peerRESTClient, err error) { + addr := peerClient.host.String() + reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", addr) + ctx := logger.SetReqInfo(GlobalContext, reqInfo) + peersLogOnceIf(ctx, err, "add-node-err-"+addr) + nodeInfo.SetAddr(addr) + nodeInfo.SetError(err.Error()) +} + +// GetSysErrors - Memory information +func (sys *NotificationSys) GetSysErrors(ctx context.Context) []madmin.SysErrors { + reply := make([]madmin.SysErrors, len(sys.peerClients)) + + g := errgroup.WithNErrs(len(sys.peerClients)) + for index, client := range sys.peerClients { + if client == nil { + continue + } + index := index + g.Go(func() error { + var err error + reply[index], err = sys.peerClients[index].GetSysErrors(ctx) + return err + }, index) + } + + for index, err := range g.Wait() { + if err != nil { + sys.addNodeErr(&reply[index], sys.peerClients[index], err) + } + } + return reply +} + +// GetMemInfo - Memory information +func (sys *NotificationSys) GetMemInfo(ctx context.Context) []madmin.MemInfo { + reply := make([]madmin.MemInfo, len(sys.peerClients)) + + g := errgroup.WithNErrs(len(sys.peerClients)) + for index, client := range sys.peerClients { + if client == nil { + continue + } + index := index + g.Go(func() error { + var err error + reply[index], err = sys.peerClients[index].GetMemInfo(ctx) + return err + }, index) + } + + for index, err := range g.Wait() { + if err != nil { + sys.addNodeErr(&reply[index], sys.peerClients[index], err) + } + } + return reply +} + +// GetProcInfo - Process information +func (sys *NotificationSys) GetProcInfo(ctx context.Context) []madmin.ProcInfo { + reply := make([]madmin.ProcInfo, len(sys.peerClients)) + + g := errgroup.WithNErrs(len(sys.peerClients)) + for index, client := range sys.peerClients { + if client == nil { + continue + } + index := index + g.Go(func() error { + var err error + reply[index], err = sys.peerClients[index].GetProcInfo(ctx) + return err + }, index) + } + + for index, err := range g.Wait() { + if err != nil { + sys.addNodeErr(&reply[index], sys.peerClients[index], err) + } + } + return reply +} + +// Construct a list of offline disks information for a given node. +// If offlineHost is empty, do it for the local disks. +func getOfflineDisks(offlineHost string, endpoints EndpointServerPools) []madmin.Disk { + var offlineDisks []madmin.Disk + for _, pool := range endpoints { + for _, ep := range pool.Endpoints { + if offlineHost == "" && ep.IsLocal || offlineHost == ep.Host { + offlineDisks = append(offlineDisks, madmin.Disk{ + Endpoint: ep.String(), + State: string(madmin.ItemOffline), + PoolIndex: ep.PoolIdx, + SetIndex: ep.SetIdx, + DiskIndex: ep.DiskIdx, + }) + } + } + } + return offlineDisks +} + +// StorageInfo returns disk information across all peers +func (sys *NotificationSys) StorageInfo(ctx context.Context, objLayer ObjectLayer, metrics bool) StorageInfo { + var storageInfo StorageInfo + replies := make([]StorageInfo, len(sys.peerClients)) + + var wg sync.WaitGroup + for i, client := range sys.peerClients { + if client == nil { + continue + } + wg.Add(1) + go func(client *peerRESTClient, idx int) { + defer wg.Done() + info, err := client.LocalStorageInfo(ctx, metrics) + if err != nil { + info.Disks = getOfflineDisks(client.host.String(), globalEndpoints) + } + replies[idx] = info + }(client, i) + } + wg.Wait() + + // Add local to this server. + replies = append(replies, objLayer.LocalStorageInfo(ctx, metrics)) + + storageInfo.Backend = objLayer.BackendInfo() + for _, sinfo := range replies { + storageInfo.Disks = append(storageInfo.Disks, sinfo.Disks...) + } + + return storageInfo +} + +// ServerInfo - calls ServerInfo RPC call on all peers. +func (sys *NotificationSys) ServerInfo(ctx context.Context, metrics bool) []madmin.ServerProperties { + reply := make([]madmin.ServerProperties, len(sys.peerClients)) + var wg sync.WaitGroup + for i, client := range sys.peerClients { + if client == nil { + continue + } + wg.Add(1) + go func(client *peerRESTClient, idx int) { + defer wg.Done() + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + info, err := client.ServerInfo(ctx, metrics) + if err != nil { + info.Endpoint = client.host.String() + info.State = string(madmin.ItemOffline) + info.Disks = getOfflineDisks(info.Endpoint, globalEndpoints) + } + reply[idx] = info + }(client, i) + } + wg.Wait() + + return reply +} + +// restClientFromHash will return a deterministic peerRESTClient based on s. +// Will return nil if client is local. +func (sys *NotificationSys) restClientFromHash(s string) (client *peerRESTClient) { + if len(sys.peerClients) == 0 { + return nil + } + peerClients := sys.allPeerClients + if len(peerClients) == 0 { + return nil + } + idx := xxhash.Sum64String(s) % uint64(len(peerClients)) + return peerClients[idx] +} + +// GetPeerOnlineCount gets the count of online and offline nodes. +func (sys *NotificationSys) GetPeerOnlineCount() (nodesOnline, nodesOffline int) { + nodesOnline = 1 // Self is always online. + nodesOffline = 0 + nodesOnlineIndex := make([]bool, len(sys.peerClients)) + var wg sync.WaitGroup + for idx, client := range sys.peerClients { + if client == nil { + continue + } + wg.Add(1) + go func(idx int, client *peerRESTClient) { + defer wg.Done() + nodesOnlineIndex[idx] = client.restClient.HealthCheckFn() + }(idx, client) + } + wg.Wait() + + for _, online := range nodesOnlineIndex { + if online { + nodesOnline++ + } else { + nodesOffline++ + } + } + return nodesOnline, nodesOffline +} + +// NewNotificationSys - creates new notification system object. +func NewNotificationSys(endpoints EndpointServerPools) *NotificationSys { + remote, all := newPeerRestClients(endpoints) + return &NotificationSys{ + peerClients: remote, + allPeerClients: all, + } +} + +// GetBandwidthReports - gets the bandwidth report from all nodes including self. +func (sys *NotificationSys) GetBandwidthReports(ctx context.Context, buckets ...string) bandwidth.BucketBandwidthReport { + reports := make([]*bandwidth.BucketBandwidthReport, len(sys.peerClients)) + g := errgroup.WithNErrs(len(sys.peerClients)) + for index := range sys.peerClients { + if sys.peerClients[index] == nil { + continue + } + index := index + g.Go(func() error { + var err error + reports[index], err = sys.peerClients[index].MonitorBandwidth(ctx, buckets) + return err + }, index) + } + + for index, err := range g.Wait() { + reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", + sys.peerClients[index].host.String()) + ctx := logger.SetReqInfo(ctx, reqInfo) + peersLogOnceIf(ctx, err, sys.peerClients[index].host.String()) + } + reports = append(reports, globalBucketMonitor.GetReport(bandwidth.SelectBuckets(buckets...))) + consolidatedReport := bandwidth.BucketBandwidthReport{ + BucketStats: make(map[bandwidth.BucketOptions]bandwidth.Details), + } + for _, report := range reports { + if report == nil || report.BucketStats == nil { + continue + } + for opts := range report.BucketStats { + d, ok := consolidatedReport.BucketStats[opts] + if !ok { + d = bandwidth.Details{ + LimitInBytesPerSecond: report.BucketStats[opts].LimitInBytesPerSecond, + } + } + dt, ok := report.BucketStats[opts] + if ok { + d.CurrentBandwidthInBytesPerSecond += dt.CurrentBandwidthInBytesPerSecond + } + consolidatedReport.BucketStats[opts] = d + } + } + return consolidatedReport +} + +func (sys *NotificationSys) collectPeerMetrics(ctx context.Context, peerChannels []<-chan MetricV2, g *errgroup.Group) <-chan MetricV2 { + ch := make(chan MetricV2) + var wg sync.WaitGroup + for index, err := range g.Wait() { + if err != nil { + if sys.peerClients[index] != nil { + reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", + sys.peerClients[index].host.String()) + peersLogOnceIf(logger.SetReqInfo(ctx, reqInfo), err, sys.peerClients[index].host.String()) + } else { + peersLogOnceIf(ctx, err, "peer-offline") + } + continue + } + wg.Add(1) + go func(ctx context.Context, peerChannel <-chan MetricV2, wg *sync.WaitGroup) { + defer wg.Done() + for { + select { + case m, ok := <-peerChannel: + if !ok { + return + } + select { + case ch <- m: + case <-ctx.Done(): + return + } + case <-ctx.Done(): + return + } + } + }(ctx, peerChannels[index], &wg) + } + go func(wg *sync.WaitGroup, ch chan MetricV2) { + wg.Wait() + xioutil.SafeClose(ch) + }(&wg, ch) + return ch +} + +// GetBucketMetrics - gets the cluster level bucket metrics from all nodes excluding self. +func (sys *NotificationSys) GetBucketMetrics(ctx context.Context) <-chan MetricV2 { + if sys == nil { + return nil + } + g := errgroup.WithNErrs(len(sys.peerClients)) + peerChannels := make([]<-chan MetricV2, len(sys.peerClients)) + for index := range sys.peerClients { + g.Go(func() error { + if sys.peerClients[index] == nil { + return errPeerNotReachable + } + var err error + peerChannels[index], err = sys.peerClients[index].GetPeerBucketMetrics(ctx) + return err + }, index) + } + return sys.collectPeerMetrics(ctx, peerChannels, g) +} + +// GetClusterMetrics - gets the cluster metrics from all nodes excluding self. +func (sys *NotificationSys) GetClusterMetrics(ctx context.Context) <-chan MetricV2 { + if sys == nil { + return nil + } + g := errgroup.WithNErrs(len(sys.peerClients)) + peerChannels := make([]<-chan MetricV2, len(sys.peerClients)) + for index := range sys.peerClients { + g.Go(func() error { + if sys.peerClients[index] == nil { + return errPeerNotReachable + } + var err error + peerChannels[index], err = sys.peerClients[index].GetPeerMetrics(ctx) + return err + }, index) + } + return sys.collectPeerMetrics(ctx, peerChannels, g) +} + +// ServiceFreeze freezes all S3 API calls when 'freeze' is true, +// 'freeze' is 'false' would resume all S3 API calls again. +// NOTE: once a tenant is frozen either two things needs to +// happen before resuming normal operations. +// - Server needs to be restarted 'mc admin service restart' +// - 'freeze' should be set to 'false' for this call +// to resume normal operations. +func (sys *NotificationSys) ServiceFreeze(ctx context.Context, freeze bool) []NotificationPeerErr { + serviceSig := serviceUnFreeze + if freeze { + serviceSig = serviceFreeze + } + ng := WithNPeers(len(sys.peerClients)) + for idx, client := range sys.peerClients { + if client == nil { + continue + } + client := client + ng.Go(GlobalContext, func() error { + return client.SignalService(serviceSig, "", false, nil) + }, idx, *client.host) + } + nerrs := ng.Wait() + if freeze { + freezeServices() + } else { + unfreezeServices() + } + return nerrs +} + +// Netperf - perform mesh style network throughput test +func (sys *NotificationSys) Netperf(ctx context.Context, duration time.Duration) []madmin.NetperfNodeResult { + length := len(sys.allPeerClients) + if length == 0 { + // For single node erasure setup. + return nil + } + results := make([]madmin.NetperfNodeResult, length) + + scheme := "http" + if globalIsTLS { + scheme = "https" + } + + var wg sync.WaitGroup + for index := range sys.peerClients { + if sys.peerClients[index] == nil { + continue + } + wg.Add(1) + go func(index int) { + defer wg.Done() + r, err := sys.peerClients[index].Netperf(ctx, duration) + u := &url.URL{ + Scheme: scheme, + Host: sys.peerClients[index].host.String(), + } + if err != nil { + results[index].Error = err.Error() + } else { + results[index] = r + } + results[index].Endpoint = u.String() + }(index) + } + + wg.Add(1) + go func() { + defer wg.Done() + r := netperf(ctx, duration) + u := &url.URL{ + Scheme: scheme, + Host: globalLocalNodeName, + } + results[len(results)-1] = r + results[len(results)-1].Endpoint = u.String() + }() + wg.Wait() + + return results +} + +// SpeedTest run GET/PUT tests at input concurrency for requested object size, +// optionally you can extend the tests longer with time.Duration. +func (sys *NotificationSys) SpeedTest(ctx context.Context, sopts speedTestOpts) []SpeedTestResult { + length := len(sys.allPeerClients) + if length == 0 { + // For single node erasure setup. + length = 1 + } + results := make([]SpeedTestResult, length) + + scheme := "http" + if globalIsTLS { + scheme = "https" + } + + var wg sync.WaitGroup + for index := range sys.peerClients { + if sys.peerClients[index] == nil { + continue + } + wg.Add(1) + go func(index int) { + defer wg.Done() + r, err := sys.peerClients[index].SpeedTest(ctx, sopts) + u := &url.URL{ + Scheme: scheme, + Host: sys.peerClients[index].host.String(), + } + if err != nil { + results[index].Error = err.Error() + } else { + results[index] = r + } + results[index].Endpoint = u.String() + }(index) + } + + wg.Add(1) + go func() { + defer wg.Done() + r, err := selfSpeedTest(ctx, sopts) + u := &url.URL{ + Scheme: scheme, + Host: globalLocalNodeName, + } + if err != nil { + results[len(results)-1].Error = err.Error() + } else { + results[len(results)-1] = r + } + results[len(results)-1].Endpoint = u.String() + }() + wg.Wait() + + return results +} + +// DriveSpeedTest - Drive performance information +func (sys *NotificationSys) DriveSpeedTest(ctx context.Context, opts madmin.DriveSpeedTestOpts) chan madmin.DriveSpeedTestResult { + ch := make(chan madmin.DriveSpeedTestResult) + var wg sync.WaitGroup + for _, client := range sys.peerClients { + if client == nil { + continue + } + wg.Add(1) + go func(client *peerRESTClient) { + defer wg.Done() + resp, err := client.DriveSpeedTest(ctx, opts) + if err != nil { + resp.Error = err.Error() + } + + select { + case <-ctx.Done(): + case ch <- resp: + } + + reqInfo := (&logger.ReqInfo{}).AppendTags("remotePeer", client.host.String()) + ctx := logger.SetReqInfo(GlobalContext, reqInfo) + peersLogOnceIf(ctx, err, client.host.String()) + }(client) + } + + wg.Add(1) + go func() { + defer wg.Done() + select { + case <-ctx.Done(): + case ch <- driveSpeedTest(ctx, opts): + } + }() + + go func(wg *sync.WaitGroup, ch chan madmin.DriveSpeedTestResult) { + wg.Wait() + xioutil.SafeClose(ch) + }(&wg, ch) + + return ch +} + +// ReloadSiteReplicationConfig - tells all peer minio nodes to reload the +// site-replication configuration. +func (sys *NotificationSys) ReloadSiteReplicationConfig(ctx context.Context) []error { + errs := make([]error, len(sys.allPeerClients)) + var wg sync.WaitGroup + for index := range sys.peerClients { + if sys.peerClients[index] == nil { + continue + } + wg.Add(1) + go func(index int) { + defer wg.Done() + errs[index] = sys.peerClients[index].ReloadSiteReplicationConfig(ctx) + }(index) + } + + wg.Wait() + return errs +} + +// GetLastDayTierStats fetches per-tier stats of the last 24hrs from all peers +func (sys *NotificationSys) GetLastDayTierStats(ctx context.Context) DailyAllTierStats { + errs := make([]error, len(sys.allPeerClients)) + lastDayStats := make([]DailyAllTierStats, len(sys.allPeerClients)) + var wg sync.WaitGroup + for index := range sys.peerClients { + if sys.peerClients[index] == nil { + continue + } + wg.Add(1) + go func(index int) { + defer wg.Done() + lastDayStats[index], errs[index] = sys.peerClients[index].GetLastDayTierStats(ctx) + }(index) + } + + wg.Wait() + merged := globalTransitionState.getDailyAllTierStats() + for i, stat := range lastDayStats { + if errs[i] != nil { + peersLogOnceIf(ctx, fmt.Errorf("failed to fetch last day tier stats: %w", errs[i]), sys.peerClients[i].host.String()) + continue + } + merged.merge(stat) + } + return merged +} + +// GetReplicationMRF - Get replication MRF from all peers. +func (sys *NotificationSys) GetReplicationMRF(ctx context.Context, bucket, node string) (mrfCh chan madmin.ReplicationMRF, err error) { + g := errgroup.WithNErrs(len(sys.peerClients)) + peerChannels := make([]<-chan madmin.ReplicationMRF, len(sys.peerClients)) + for index, client := range sys.peerClients { + if client == nil { + continue + } + host := client.host.String() + if host != node && node != "all" { + continue + } + index := index + g.Go(func() error { + var err error + peerChannels[index], err = sys.peerClients[index].GetReplicationMRF(ctx, bucket) + return err + }, index) + } + mrfCh = make(chan madmin.ReplicationMRF, 4000) + var wg sync.WaitGroup + + for index, err := range g.Wait() { + if err != nil { + if sys.peerClients[index] != nil { + reqInfo := (&logger.ReqInfo{}).AppendTags("peerAddress", + sys.peerClients[index].host.String()) + peersLogOnceIf(logger.SetReqInfo(ctx, reqInfo), err, sys.peerClients[index].host.String()) + } else { + peersLogOnceIf(ctx, err, "peer-offline") + } + continue + } + wg.Add(1) + go func(ctx context.Context, peerChannel <-chan madmin.ReplicationMRF, wg *sync.WaitGroup) { + defer wg.Done() + for { + select { + case m, ok := <-peerChannel: + if !ok { + return + } + select { + case <-ctx.Done(): + return + case mrfCh <- m: + } + case <-ctx.Done(): + return + } + } + }(ctx, peerChannels[index], &wg) + } + wg.Add(1) + go func(ch chan madmin.ReplicationMRF) error { + defer wg.Done() + if node != "all" && node != globalLocalNodeName { + return nil + } + mCh, err := globalReplicationPool.Get().getMRF(ctx, bucket) + if err != nil { + return err + } + for e := range mCh { + select { + case <-ctx.Done(): + return err + case mrfCh <- e: + } + } + return nil + }(mrfCh) + go func(wg *sync.WaitGroup) { + wg.Wait() + xioutil.SafeClose(mrfCh) + }(&wg) + return mrfCh, nil +} diff --git a/cmd/object-api-common.go b/cmd/object-api-common.go new file mode 100644 index 00000000..a4fe6a97 --- /dev/null +++ b/cmd/object-api-common.go @@ -0,0 +1,71 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "sync" + + "github.com/dustin/go-humanize" +) + +const ( + // Block size used for all internal operations version 1. + + // TLDR.. + // Not used anymore xl.meta captures the right blockSize + // so blockSizeV2 should be used for all future purposes. + // this value is kept here to calculate the max API + // requests based on RAM size for existing content. + blockSizeV1 = 10 * humanize.MiByte + + // Block size used in erasure coding version 2. + blockSizeV2 = 1 * humanize.MiByte + + // Buckets meta prefix. + bucketMetaPrefix = "buckets" + + // Deleted Buckets prefix. + deletedBucketsPrefix = ".deleted" + + // ETag (hex encoded md5sum) of empty string. + emptyETag = "d41d8cd98f00b204e9800998ecf8427e" +) + +// Global object layer mutex, used for safely updating object layer. +var globalObjLayerMutex sync.RWMutex + +// Global object layer, only accessed by globalObjectAPI. +var globalObjectAPI ObjectLayer + +type storageOpts struct { + cleanUp bool + healthCheck bool +} + +// Depending on the disk type network or local, initialize storage API. +func newStorageAPI(endpoint Endpoint, opts storageOpts) (storage StorageAPI, err error) { + if endpoint.IsLocal { + storage, err := newXLStorage(endpoint, opts.cleanUp) + if err != nil { + return nil, err + } + return newXLStorageDiskIDCheck(storage, opts.healthCheck), nil + } + + return newStorageRESTClient(endpoint, opts.healthCheck, globalGrid.Load()) +} diff --git a/cmd/object-api-datatypes.go b/cmd/object-api-datatypes.go new file mode 100644 index 00000000..d00e2be1 --- /dev/null +++ b/cmd/object-api-datatypes.go @@ -0,0 +1,682 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "io" + "maps" + "math" + "net/http" + "time" + + "github.com/dustin/go-humanize" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/hash" +) + +//go:generate msgp -file $GOFILE -io=false -tests=false -unexported=false + +// BackendType - represents different backend types. +type BackendType int + +// Enum for different backend types. +const ( + Unknown = BackendType(madmin.Unknown) + // Filesystem backend. + BackendFS = BackendType(madmin.FS) + // Multi disk BackendErasure (single, distributed) backend. + BackendErasure = BackendType(madmin.Erasure) + // Add your own backend. +) + +// StorageInfo - represents total capacity of underlying storage. +type StorageInfo = madmin.StorageInfo + +// objectHistogramInterval is an interval that will be +// used to report the histogram of objects data sizes +type objectHistogramInterval struct { + name string + start, end int64 +} + +const ( + // dataUsageBucketLenV1 must be length of ObjectsHistogramIntervalsV1 + dataUsageBucketLenV1 = 7 + // dataUsageBucketLen must be length of ObjectsHistogramIntervals + dataUsageBucketLen = 11 + dataUsageVersionLen = 7 +) + +// ObjectsHistogramIntervalsV1 is the list of all intervals +// of object sizes to be included in objects histogram(V1). +var ObjectsHistogramIntervalsV1 = [dataUsageBucketLenV1]objectHistogramInterval{ + {"LESS_THAN_1024_B", 0, humanize.KiByte - 1}, + {"BETWEEN_1024B_AND_1_MB", humanize.KiByte, humanize.MiByte - 1}, + {"BETWEEN_1_MB_AND_10_MB", humanize.MiByte, humanize.MiByte*10 - 1}, + {"BETWEEN_10_MB_AND_64_MB", humanize.MiByte * 10, humanize.MiByte*64 - 1}, + {"BETWEEN_64_MB_AND_128_MB", humanize.MiByte * 64, humanize.MiByte*128 - 1}, + {"BETWEEN_128_MB_AND_512_MB", humanize.MiByte * 128, humanize.MiByte*512 - 1}, + {"GREATER_THAN_512_MB", humanize.MiByte * 512, math.MaxInt64}, +} + +// ObjectsHistogramIntervals is the list of all intervals +// of object sizes to be included in objects histogram. +// Note: this histogram expands 1024B-1MB to incl. 1024B-64KB, 64KB-256KB, 256KB-512KB and 512KB-1MiB +var ObjectsHistogramIntervals = [dataUsageBucketLen]objectHistogramInterval{ + {"LESS_THAN_1024_B", 0, humanize.KiByte - 1}, + {"BETWEEN_1024_B_AND_64_KB", humanize.KiByte, 64*humanize.KiByte - 1}, // not exported, for support use only + {"BETWEEN_64_KB_AND_256_KB", 64 * humanize.KiByte, 256*humanize.KiByte - 1}, // not exported, for support use only + {"BETWEEN_256_KB_AND_512_KB", 256 * humanize.KiByte, 512*humanize.KiByte - 1}, // not exported, for support use only + {"BETWEEN_512_KB_AND_1_MB", 512 * humanize.KiByte, humanize.MiByte - 1}, // not exported, for support use only + {"BETWEEN_1024B_AND_1_MB", humanize.KiByte, humanize.MiByte - 1}, + {"BETWEEN_1_MB_AND_10_MB", humanize.MiByte, humanize.MiByte*10 - 1}, + {"BETWEEN_10_MB_AND_64_MB", humanize.MiByte * 10, humanize.MiByte*64 - 1}, + {"BETWEEN_64_MB_AND_128_MB", humanize.MiByte * 64, humanize.MiByte*128 - 1}, + {"BETWEEN_128_MB_AND_512_MB", humanize.MiByte * 128, humanize.MiByte*512 - 1}, + {"GREATER_THAN_512_MB", humanize.MiByte * 512, math.MaxInt64}, +} + +// ObjectsVersionCountIntervals is the list of all intervals +// of object version count to be included in objects histogram. +var ObjectsVersionCountIntervals = [dataUsageVersionLen]objectHistogramInterval{ + {"UNVERSIONED", 0, 0}, + {"SINGLE_VERSION", 1, 1}, + {"BETWEEN_2_AND_10", 2, 9}, + {"BETWEEN_10_AND_100", 10, 99}, + {"BETWEEN_100_AND_1000", 100, 999}, + {"BETWEEN_1000_AND_10000", 1000, 9999}, + {"GREATER_THAN_10000", 10000, math.MaxInt64}, +} + +// BucketInfo - represents bucket metadata. +type BucketInfo struct { + // Name of the bucket. + Name string + + // Date and time when the bucket was created. + Created time.Time + Deleted time.Time + + // Bucket features enabled + Versioning, ObjectLocking bool +} + +// ObjectInfo - represents object metadata. +type ObjectInfo struct { + // Name of the bucket. + Bucket string + + // Name of the object. + Name string + + // Date and time when the object was last modified. + ModTime time.Time + + // Total object size. + Size int64 + + // Actual size is the real size of the object uploaded by client. + ActualSize *int64 + + // IsDir indicates if the object is prefix. + IsDir bool + + // Hex encoded unique entity tag of the object. + ETag string + + // Version ID of this object. + VersionID string + + // IsLatest indicates if this is the latest current version + // latest can be true for delete marker or a version. + IsLatest bool + + // DeleteMarker indicates if the versionId corresponds + // to a delete marker on an object. + DeleteMarker bool + + // Transitioned object information + TransitionedObject TransitionedObject + + // RestoreExpires indicates date a restored object expires + RestoreExpires time.Time + + // RestoreOngoing indicates if a restore is in progress + RestoreOngoing bool + + // A standard MIME type describing the format of the object. + ContentType string + + // Specifies what content encodings have been applied to the object and thus + // what decoding mechanisms must be applied to obtain the object referenced + // by the Content-Type header field. + ContentEncoding string + + // Date and time at which the object is no longer able to be cached + Expires time.Time + + // Cache-Control - Specifies caching behavior along the request/reply chain + CacheControl string + + // Specify object storage class + StorageClass string + + ReplicationStatusInternal string + ReplicationStatus replication.StatusType + // User-Defined metadata + UserDefined map[string]string + + // User-Defined object tags + UserTags string + + // List of individual parts, maximum size of upto 10,000 + Parts []ObjectPartInfo `json:"-"` + + // Implements writer and reader used by CopyObject API + Writer io.WriteCloser `json:"-" msg:"-"` + Reader *hash.Reader `json:"-" msg:"-"` + PutObjReader *PutObjReader `json:"-" msg:"-"` + + metadataOnly bool + versionOnly bool // adds a new version, only used by CopyObject + keyRotation bool + + // Date and time when the object was last accessed. + AccTime time.Time + + Legacy bool // indicates object on disk is in legacy data format + + // internal representation of version purge status + VersionPurgeStatusInternal string + VersionPurgeStatus VersionPurgeStatusType + + replicationDecision string // internal representation of replication decision for use by DeleteObject handler + // The total count of all versions of this object + NumVersions int + // The modtime of the successor object version if any + SuccessorModTime time.Time + + // Checksums added on upload. + // Encoded, maybe encrypted. + Checksum []byte + + // Inlined + Inlined bool + + DataBlocks int + ParityBlocks int +} + +// ExpiresStr returns a stringified version of Expires header in http.TimeFormat +func (o ObjectInfo) ExpiresStr() string { + var expires string + if !o.Expires.IsZero() { + expires = o.Expires.UTC().Format(http.TimeFormat) + } + return expires +} + +// ArchiveInfo returns any saved zip archive meta information. +// It will be decrypted if needed. +func (o *ObjectInfo) ArchiveInfo(h http.Header) []byte { + if len(o.UserDefined) == 0 { + return nil + } + z, ok := o.UserDefined[archiveInfoMetadataKey] + if !ok { + return nil + } + data := []byte(z) + if v, ok := o.UserDefined[archiveTypeMetadataKey]; ok && v == archiveTypeEnc { + decrypted, err := o.metadataDecrypter(h)(archiveTypeEnc, data) + if err != nil { + encLogIf(GlobalContext, err) + return nil + } + data = decrypted + } + return data +} + +// Clone - Returns a cloned copy of current objectInfo +func (o *ObjectInfo) Clone() (cinfo ObjectInfo) { + cinfo = ObjectInfo{ + Bucket: o.Bucket, + Name: o.Name, + ModTime: o.ModTime, + Size: o.Size, + IsDir: o.IsDir, + ETag: o.ETag, + VersionID: o.VersionID, + IsLatest: o.IsLatest, + DeleteMarker: o.DeleteMarker, + TransitionedObject: o.TransitionedObject, + RestoreExpires: o.RestoreExpires, + RestoreOngoing: o.RestoreOngoing, + ContentType: o.ContentType, + ContentEncoding: o.ContentEncoding, + Expires: o.Expires, + StorageClass: o.StorageClass, + ReplicationStatus: o.ReplicationStatus, + UserTags: o.UserTags, + Parts: o.Parts, + Writer: o.Writer, + Reader: o.Reader, + PutObjReader: o.PutObjReader, + metadataOnly: o.metadataOnly, + versionOnly: o.versionOnly, + keyRotation: o.keyRotation, + AccTime: o.AccTime, + Legacy: o.Legacy, + VersionPurgeStatus: o.VersionPurgeStatus, + NumVersions: o.NumVersions, + SuccessorModTime: o.SuccessorModTime, + ReplicationStatusInternal: o.ReplicationStatusInternal, + VersionPurgeStatusInternal: o.VersionPurgeStatusInternal, + } + cinfo.UserDefined = make(map[string]string, len(o.UserDefined)) + maps.Copy(cinfo.UserDefined, o.UserDefined) + return cinfo +} + +func (o ObjectInfo) tierStats() tierStats { + ts := tierStats{ + TotalSize: uint64(o.Size), + NumVersions: 1, + } + // the current version of an object is accounted towards objects count + if o.IsLatest { + ts.NumObjects = 1 + } + return ts +} + +// ToObjectInfo converts a replication object info to a partial ObjectInfo +// do not rely on this function to give you correct ObjectInfo, this +// function is merely and optimization. +func (ri ReplicateObjectInfo) ToObjectInfo() ObjectInfo { + return ObjectInfo{ + Name: ri.Name, + Bucket: ri.Bucket, + VersionID: ri.VersionID, + ModTime: ri.ModTime, + UserTags: ri.UserTags, + Size: ri.Size, + ActualSize: &ri.ActualSize, + ReplicationStatus: ri.ReplicationStatus, + ReplicationStatusInternal: ri.ReplicationStatusInternal, + VersionPurgeStatus: ri.VersionPurgeStatus, + VersionPurgeStatusInternal: ri.VersionPurgeStatusInternal, + DeleteMarker: true, + UserDefined: map[string]string{}, + Checksum: ri.Checksum, + } +} + +// ReplicateObjectInfo represents object info to be replicated +type ReplicateObjectInfo struct { + Name string + Bucket string + VersionID string + ETag string + Size int64 + ActualSize int64 + ModTime time.Time + UserTags string + SSEC bool + ReplicationStatus replication.StatusType + ReplicationStatusInternal string + VersionPurgeStatusInternal string + VersionPurgeStatus VersionPurgeStatusType + ReplicationState ReplicationState + DeleteMarker bool + + OpType replication.Type + EventType string + RetryCount uint32 + ResetID string + Dsc ReplicateDecision + ExistingObjResync ResyncDecision + TargetArn string + TargetStatuses map[string]replication.StatusType + TargetPurgeStatuses map[string]VersionPurgeStatusType + ReplicationTimestamp time.Time + Checksum []byte +} + +// MultipartInfo captures metadata information about the uploadId +// this data structure is used primarily for some internal purposes +// for verifying upload type such as was the upload +// - encrypted +// - compressed +type MultipartInfo struct { + // Name of the bucket. + Bucket string + + // Name of the object. + Object string + + // Upload ID identifying the multipart upload whose parts are being listed. + UploadID string + + // Date and time at which the multipart upload was initiated. + Initiated time.Time + + // Any metadata set during InitMultipartUpload, including encryption headers. + UserDefined map[string]string +} + +// ListPartsInfo - represents list of all parts. +type ListPartsInfo struct { + // Name of the bucket. + Bucket string + + // Name of the object. + Object string + + // Upload ID identifying the multipart upload whose parts are being listed. + UploadID string + + // The class of storage used to store the object. + StorageClass string + + // Part number after which listing begins. + PartNumberMarker int + + // When a list is truncated, this element specifies the last part in the list, + // as well as the value to use for the part-number-marker request parameter + // in a subsequent request. + NextPartNumberMarker int + + // Maximum number of parts that were allowed in the response. + MaxParts int + + // Indicates whether the returned list of parts is truncated. + IsTruncated bool + + // List of all parts. + Parts []PartInfo + + // Any metadata set during InitMultipartUpload, including encryption headers. + UserDefined map[string]string + + // ChecksumAlgorithm if set + ChecksumAlgorithm string + + // ChecksumType if set + ChecksumType string +} + +// Lookup - returns if uploadID is valid +func (lm ListMultipartsInfo) Lookup(uploadID string) bool { + for _, upload := range lm.Uploads { + if upload.UploadID == uploadID { + return true + } + } + return false +} + +// ListMultipartsInfo - represents bucket resources for incomplete multipart uploads. +type ListMultipartsInfo struct { + // Together with upload-id-marker, this parameter specifies the multipart upload + // after which listing should begin. + KeyMarker string + + // Together with key-marker, specifies the multipart upload after which listing + // should begin. If key-marker is not specified, the upload-id-marker parameter + // is ignored. + UploadIDMarker string + + // When a list is truncated, this element specifies the value that should be + // used for the key-marker request parameter in a subsequent request. + NextKeyMarker string + + // When a list is truncated, this element specifies the value that should be + // used for the upload-id-marker request parameter in a subsequent request. + NextUploadIDMarker string + + // Maximum number of multipart uploads that could have been included in the + // response. + MaxUploads int + + // Indicates whether the returned list of multipart uploads is truncated. A + // value of true indicates that the list was truncated. The list can be truncated + // if the number of multipart uploads exceeds the limit allowed or specified + // by max uploads. + IsTruncated bool + + // List of all pending uploads. + Uploads []MultipartInfo + + // When a prefix is provided in the request, The result contains only keys + // starting with the specified prefix. + Prefix string + + // A character used to truncate the object prefixes. + // NOTE: only supported delimiter is '/'. + Delimiter string + + // CommonPrefixes contains all (if there are any) keys between Prefix and the + // next occurrence of the string specified by delimiter. + CommonPrefixes []string + + EncodingType string // Not supported yet. +} + +// TransitionedObject transitioned object tier and status. +type TransitionedObject struct { + Name string + VersionID string + Tier string + FreeVersion bool + Status string +} + +// DeletedObjectInfo - container for list objects versions deleted objects. +type DeletedObjectInfo struct { + // Name of the bucket. + Bucket string + + // Name of the object. + Name string + + // Date and time when the object was last modified. + ModTime time.Time + + // Version ID of this object. + VersionID string + + // Indicates the deleted marker is latest + IsLatest bool +} + +// ListObjectVersionsInfo - container for list objects versions. +type ListObjectVersionsInfo struct { + // Indicates whether the returned list objects response is truncated. A + // value of true indicates that the list was truncated. The list can be truncated + // if the number of objects exceeds the limit allowed or specified + // by max keys. + IsTruncated bool + + // When response is truncated (the IsTruncated element value in the response is true), + // you can use the key name in this field as marker in the subsequent + // request to get next set of objects. + // + // NOTE: AWS S3 returns NextMarker only if you have delimiter request parameter specified, + // MinIO always returns NextMarker. + NextMarker string + + // NextVersionIDMarker may be set of IsTruncated is true + NextVersionIDMarker string + + // List of objects info for this request. + Objects []ObjectInfo + + // List of prefixes for this request. + Prefixes []string +} + +// ListObjectsInfo - container for list objects. +type ListObjectsInfo struct { + // Indicates whether the returned list objects response is truncated. A + // value of true indicates that the list was truncated. The list can be truncated + // if the number of objects exceeds the limit allowed or specified + // by max keys. + IsTruncated bool + + // When response is truncated (the IsTruncated element value in the response is true), + // you can use the key name in this field as marker in the subsequent + // request to get next set of objects. + // + // NOTE: AWS S3 returns NextMarker only if you have delimiter request parameter specified, + // MinIO always returns NextMarker. + NextMarker string + + // List of objects info for this request. + Objects []ObjectInfo + + // List of prefixes for this request. + Prefixes []string +} + +// ListObjectsV2Info - container for list objects version 2. +type ListObjectsV2Info struct { + // Indicates whether the returned list objects response is truncated. A + // value of true indicates that the list was truncated. The list can be truncated + // if the number of objects exceeds the limit allowed or specified + // by max keys. + IsTruncated bool + + // When response is truncated (the IsTruncated element value in the response + // is true), you can use the key name in this field as marker in the subsequent + // request to get next set of objects. + // + // NOTE: This element is returned only if you have delimiter request parameter + // specified. + ContinuationToken string + NextContinuationToken string + + // List of objects info for this request. + Objects []ObjectInfo + + // List of prefixes for this request. + Prefixes []string +} + +// PartInfo - represents individual part metadata. +type PartInfo struct { + // Part number that identifies the part. This is a positive integer between + // 1 and 10,000. + PartNumber int + + // Date and time at which the part was uploaded. + LastModified time.Time + + // Entity tag returned when the part was initially uploaded. + ETag string + + // Size in bytes of the part. + Size int64 + + // Real size of the object uploaded by client. + ActualSize int64 + + // Checksum values + ChecksumCRC32 string + ChecksumCRC32C string + ChecksumSHA1 string + ChecksumSHA256 string + ChecksumCRC64NVME string +} + +// CompletePart - represents the part that was completed, this is sent by the client +// during CompleteMultipartUpload request. +type CompletePart struct { + // Part number identifying the part. This is a positive integer between 1 and + // 10,000 + PartNumber int + + // Entity tag returned when the part was uploaded. + ETag string + + Size int64 + + // Checksum values. Optional. + ChecksumCRC32 string + ChecksumCRC32C string + ChecksumSHA1 string + ChecksumSHA256 string + ChecksumCRC64NVME string +} + +// CompleteMultipartUpload - represents list of parts which are completed, this is sent by the +// client during CompleteMultipartUpload request. +type CompleteMultipartUpload struct { + Parts []CompletePart `xml:"Part"` +} + +// NewMultipartUploadResult contains information about a newly created multipart upload. +type NewMultipartUploadResult struct { + UploadID string + ChecksumAlgo string + ChecksumType string +} + +type getObjectAttributesResponse struct { + ETag string `xml:",omitempty"` + Checksum *objectAttributesChecksum `xml:",omitempty"` + ObjectParts *objectAttributesParts `xml:",omitempty"` + StorageClass string `xml:",omitempty"` + ObjectSize int64 `xml:",omitempty"` +} + +type objectAttributesChecksum struct { + ChecksumCRC32 string `xml:",omitempty"` + ChecksumCRC32C string `xml:",omitempty"` + ChecksumSHA1 string `xml:",omitempty"` + ChecksumSHA256 string `xml:",omitempty"` + ChecksumCRC64NVME string `xml:",omitempty"` + ChecksumType string `xml:",omitempty"` +} + +type objectAttributesParts struct { + IsTruncated bool + MaxParts int + NextPartNumberMarker int + PartNumberMarker int + PartsCount int + Parts []*objectAttributesPart `xml:"Part"` +} + +type objectAttributesPart struct { + PartNumber int + Size int64 + ChecksumCRC32 string `xml:",omitempty"` + ChecksumCRC32C string `xml:",omitempty"` + ChecksumSHA1 string `xml:",omitempty"` + ChecksumSHA256 string `xml:",omitempty"` + ChecksumCRC64NVME string `xml:",omitempty"` +} + +type objectAttributesErrorResponse struct { + ArgumentValue *string `xml:"ArgumentValue,omitempty"` + ArgumentName *string `xml:"ArgumentName"` + APIErrorResponse +} diff --git a/cmd/object-api-datatypes_gen.go b/cmd/object-api-datatypes_gen.go new file mode 100644 index 00000000..b15bb3ab --- /dev/null +++ b/cmd/object-api-datatypes_gen.go @@ -0,0 +1,2396 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/minio/minio/internal/bucket/replication" + "github.com/tinylib/msgp/msgp" +) + +// MarshalMsg implements msgp.Marshaler +func (z BackendType) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendInt(o, int(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BackendType) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 int + zb0001, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = BackendType(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z BackendType) Msgsize() (s int) { + s = msgp.IntSize + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *BucketInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 5 + // string "Name" + o = append(o, 0x85, 0xa4, 0x4e, 0x61, 0x6d, 0x65) + o = msgp.AppendString(o, z.Name) + // string "Created" + o = append(o, 0xa7, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64) + o = msgp.AppendTime(o, z.Created) + // string "Deleted" + o = append(o, 0xa7, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64) + o = msgp.AppendTime(o, z.Deleted) + // string "Versioning" + o = append(o, 0xaa, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x69, 0x6e, 0x67) + o = msgp.AppendBool(o, z.Versioning) + // string "ObjectLocking" + o = append(o, 0xad, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x4c, 0x6f, 0x63, 0x6b, 0x69, 0x6e, 0x67) + o = msgp.AppendBool(o, z.ObjectLocking) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BucketInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Name": + z.Name, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "Created": + z.Created, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Created") + return + } + case "Deleted": + z.Deleted, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Deleted") + return + } + case "Versioning": + z.Versioning, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Versioning") + return + } + case "ObjectLocking": + z.ObjectLocking, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjectLocking") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *BucketInfo) Msgsize() (s int) { + s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 8 + msgp.TimeSize + 11 + msgp.BoolSize + 14 + msgp.BoolSize + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *CompleteMultipartUpload) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "Parts" + o = append(o, 0x81, 0xa5, 0x50, 0x61, 0x72, 0x74, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Parts))) + for za0001 := range z.Parts { + o, err = z.Parts[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Parts", za0001) + return + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *CompleteMultipartUpload) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Parts": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Parts") + return + } + if cap(z.Parts) >= int(zb0002) { + z.Parts = (z.Parts)[:zb0002] + } else { + z.Parts = make([]CompletePart, zb0002) + } + for za0001 := range z.Parts { + bts, err = z.Parts[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Parts", za0001) + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *CompleteMultipartUpload) Msgsize() (s int) { + s = 1 + 6 + msgp.ArrayHeaderSize + for za0001 := range z.Parts { + s += z.Parts[za0001].Msgsize() + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *CompletePart) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 8 + // string "PartNumber" + o = append(o, 0x88, 0xaa, 0x50, 0x61, 0x72, 0x74, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72) + o = msgp.AppendInt(o, z.PartNumber) + // string "ETag" + o = append(o, 0xa4, 0x45, 0x54, 0x61, 0x67) + o = msgp.AppendString(o, z.ETag) + // string "Size" + o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.Size) + // string "ChecksumCRC32" + o = append(o, 0xad, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x43, 0x52, 0x43, 0x33, 0x32) + o = msgp.AppendString(o, z.ChecksumCRC32) + // string "ChecksumCRC32C" + o = append(o, 0xae, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x43, 0x52, 0x43, 0x33, 0x32, 0x43) + o = msgp.AppendString(o, z.ChecksumCRC32C) + // string "ChecksumSHA1" + o = append(o, 0xac, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x53, 0x48, 0x41, 0x31) + o = msgp.AppendString(o, z.ChecksumSHA1) + // string "ChecksumSHA256" + o = append(o, 0xae, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x53, 0x48, 0x41, 0x32, 0x35, 0x36) + o = msgp.AppendString(o, z.ChecksumSHA256) + // string "ChecksumCRC64NVME" + o = append(o, 0xb1, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x43, 0x52, 0x43, 0x36, 0x34, 0x4e, 0x56, 0x4d, 0x45) + o = msgp.AppendString(o, z.ChecksumCRC64NVME) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *CompletePart) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "PartNumber": + z.PartNumber, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "PartNumber") + return + } + case "ETag": + z.ETag, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ETag") + return + } + case "Size": + z.Size, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + case "ChecksumCRC32": + z.ChecksumCRC32, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ChecksumCRC32") + return + } + case "ChecksumCRC32C": + z.ChecksumCRC32C, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ChecksumCRC32C") + return + } + case "ChecksumSHA1": + z.ChecksumSHA1, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ChecksumSHA1") + return + } + case "ChecksumSHA256": + z.ChecksumSHA256, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ChecksumSHA256") + return + } + case "ChecksumCRC64NVME": + z.ChecksumCRC64NVME, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ChecksumCRC64NVME") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *CompletePart) Msgsize() (s int) { + s = 1 + 11 + msgp.IntSize + 5 + msgp.StringPrefixSize + len(z.ETag) + 5 + msgp.Int64Size + 14 + msgp.StringPrefixSize + len(z.ChecksumCRC32) + 15 + msgp.StringPrefixSize + len(z.ChecksumCRC32C) + 13 + msgp.StringPrefixSize + len(z.ChecksumSHA1) + 15 + msgp.StringPrefixSize + len(z.ChecksumSHA256) + 18 + msgp.StringPrefixSize + len(z.ChecksumCRC64NVME) + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *DeletedObjectInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 5 + // string "Bucket" + o = append(o, 0x85, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + o = msgp.AppendString(o, z.Bucket) + // string "Name" + o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65) + o = msgp.AppendString(o, z.Name) + // string "ModTime" + o = append(o, 0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65) + o = msgp.AppendTime(o, z.ModTime) + // string "VersionID" + o = append(o, 0xa9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x44) + o = msgp.AppendString(o, z.VersionID) + // string "IsLatest" + o = append(o, 0xa8, 0x49, 0x73, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74) + o = msgp.AppendBool(o, z.IsLatest) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *DeletedObjectInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Bucket": + z.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "Name": + z.Name, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "ModTime": + z.ModTime, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + case "VersionID": + z.VersionID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + case "IsLatest": + z.IsLatest, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "IsLatest") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *DeletedObjectInfo) Msgsize() (s int) { + s = 1 + 7 + msgp.StringPrefixSize + len(z.Bucket) + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 10 + msgp.StringPrefixSize + len(z.VersionID) + 9 + msgp.BoolSize + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ListMultipartsInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 11 + // string "KeyMarker" + o = append(o, 0x8b, 0xa9, 0x4b, 0x65, 0x79, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72) + o = msgp.AppendString(o, z.KeyMarker) + // string "UploadIDMarker" + o = append(o, 0xae, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x44, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72) + o = msgp.AppendString(o, z.UploadIDMarker) + // string "NextKeyMarker" + o = append(o, 0xad, 0x4e, 0x65, 0x78, 0x74, 0x4b, 0x65, 0x79, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72) + o = msgp.AppendString(o, z.NextKeyMarker) + // string "NextUploadIDMarker" + o = append(o, 0xb2, 0x4e, 0x65, 0x78, 0x74, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x44, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72) + o = msgp.AppendString(o, z.NextUploadIDMarker) + // string "MaxUploads" + o = append(o, 0xaa, 0x4d, 0x61, 0x78, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x73) + o = msgp.AppendInt(o, z.MaxUploads) + // string "IsTruncated" + o = append(o, 0xab, 0x49, 0x73, 0x54, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64) + o = msgp.AppendBool(o, z.IsTruncated) + // string "Uploads" + o = append(o, 0xa7, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Uploads))) + for za0001 := range z.Uploads { + o, err = z.Uploads[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Uploads", za0001) + return + } + } + // string "Prefix" + o = append(o, 0xa6, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78) + o = msgp.AppendString(o, z.Prefix) + // string "Delimiter" + o = append(o, 0xa9, 0x44, 0x65, 0x6c, 0x69, 0x6d, 0x69, 0x74, 0x65, 0x72) + o = msgp.AppendString(o, z.Delimiter) + // string "CommonPrefixes" + o = append(o, 0xae, 0x43, 0x6f, 0x6d, 0x6d, 0x6f, 0x6e, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.CommonPrefixes))) + for za0002 := range z.CommonPrefixes { + o = msgp.AppendString(o, z.CommonPrefixes[za0002]) + } + // string "EncodingType" + o = append(o, 0xac, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67, 0x54, 0x79, 0x70, 0x65) + o = msgp.AppendString(o, z.EncodingType) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ListMultipartsInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "KeyMarker": + z.KeyMarker, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "KeyMarker") + return + } + case "UploadIDMarker": + z.UploadIDMarker, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UploadIDMarker") + return + } + case "NextKeyMarker": + z.NextKeyMarker, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "NextKeyMarker") + return + } + case "NextUploadIDMarker": + z.NextUploadIDMarker, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "NextUploadIDMarker") + return + } + case "MaxUploads": + z.MaxUploads, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "MaxUploads") + return + } + case "IsTruncated": + z.IsTruncated, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "IsTruncated") + return + } + case "Uploads": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Uploads") + return + } + if cap(z.Uploads) >= int(zb0002) { + z.Uploads = (z.Uploads)[:zb0002] + } else { + z.Uploads = make([]MultipartInfo, zb0002) + } + for za0001 := range z.Uploads { + bts, err = z.Uploads[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Uploads", za0001) + return + } + } + case "Prefix": + z.Prefix, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + case "Delimiter": + z.Delimiter, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Delimiter") + return + } + case "CommonPrefixes": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "CommonPrefixes") + return + } + if cap(z.CommonPrefixes) >= int(zb0003) { + z.CommonPrefixes = (z.CommonPrefixes)[:zb0003] + } else { + z.CommonPrefixes = make([]string, zb0003) + } + for za0002 := range z.CommonPrefixes { + z.CommonPrefixes[za0002], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "CommonPrefixes", za0002) + return + } + } + case "EncodingType": + z.EncodingType, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "EncodingType") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ListMultipartsInfo) Msgsize() (s int) { + s = 1 + 10 + msgp.StringPrefixSize + len(z.KeyMarker) + 15 + msgp.StringPrefixSize + len(z.UploadIDMarker) + 14 + msgp.StringPrefixSize + len(z.NextKeyMarker) + 19 + msgp.StringPrefixSize + len(z.NextUploadIDMarker) + 11 + msgp.IntSize + 12 + msgp.BoolSize + 8 + msgp.ArrayHeaderSize + for za0001 := range z.Uploads { + s += z.Uploads[za0001].Msgsize() + } + s += 7 + msgp.StringPrefixSize + len(z.Prefix) + 10 + msgp.StringPrefixSize + len(z.Delimiter) + 15 + msgp.ArrayHeaderSize + for za0002 := range z.CommonPrefixes { + s += msgp.StringPrefixSize + len(z.CommonPrefixes[za0002]) + } + s += 13 + msgp.StringPrefixSize + len(z.EncodingType) + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ListObjectVersionsInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 5 + // string "IsTruncated" + o = append(o, 0x85, 0xab, 0x49, 0x73, 0x54, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64) + o = msgp.AppendBool(o, z.IsTruncated) + // string "NextMarker" + o = append(o, 0xaa, 0x4e, 0x65, 0x78, 0x74, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72) + o = msgp.AppendString(o, z.NextMarker) + // string "NextVersionIDMarker" + o = append(o, 0xb3, 0x4e, 0x65, 0x78, 0x74, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x44, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72) + o = msgp.AppendString(o, z.NextVersionIDMarker) + // string "Objects" + o = append(o, 0xa7, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Objects))) + for za0001 := range z.Objects { + o, err = z.Objects[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Objects", za0001) + return + } + } + // string "Prefixes" + o = append(o, 0xa8, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Prefixes))) + for za0002 := range z.Prefixes { + o = msgp.AppendString(o, z.Prefixes[za0002]) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ListObjectVersionsInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "IsTruncated": + z.IsTruncated, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "IsTruncated") + return + } + case "NextMarker": + z.NextMarker, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "NextMarker") + return + } + case "NextVersionIDMarker": + z.NextVersionIDMarker, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "NextVersionIDMarker") + return + } + case "Objects": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + if cap(z.Objects) >= int(zb0002) { + z.Objects = (z.Objects)[:zb0002] + } else { + z.Objects = make([]ObjectInfo, zb0002) + } + for za0001 := range z.Objects { + bts, err = z.Objects[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Objects", za0001) + return + } + } + case "Prefixes": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Prefixes") + return + } + if cap(z.Prefixes) >= int(zb0003) { + z.Prefixes = (z.Prefixes)[:zb0003] + } else { + z.Prefixes = make([]string, zb0003) + } + for za0002 := range z.Prefixes { + z.Prefixes[za0002], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Prefixes", za0002) + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ListObjectVersionsInfo) Msgsize() (s int) { + s = 1 + 12 + msgp.BoolSize + 11 + msgp.StringPrefixSize + len(z.NextMarker) + 20 + msgp.StringPrefixSize + len(z.NextVersionIDMarker) + 8 + msgp.ArrayHeaderSize + for za0001 := range z.Objects { + s += z.Objects[za0001].Msgsize() + } + s += 9 + msgp.ArrayHeaderSize + for za0002 := range z.Prefixes { + s += msgp.StringPrefixSize + len(z.Prefixes[za0002]) + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ListObjectsInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 4 + // string "IsTruncated" + o = append(o, 0x84, 0xab, 0x49, 0x73, 0x54, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64) + o = msgp.AppendBool(o, z.IsTruncated) + // string "NextMarker" + o = append(o, 0xaa, 0x4e, 0x65, 0x78, 0x74, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72) + o = msgp.AppendString(o, z.NextMarker) + // string "Objects" + o = append(o, 0xa7, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Objects))) + for za0001 := range z.Objects { + o, err = z.Objects[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Objects", za0001) + return + } + } + // string "Prefixes" + o = append(o, 0xa8, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Prefixes))) + for za0002 := range z.Prefixes { + o = msgp.AppendString(o, z.Prefixes[za0002]) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ListObjectsInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "IsTruncated": + z.IsTruncated, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "IsTruncated") + return + } + case "NextMarker": + z.NextMarker, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "NextMarker") + return + } + case "Objects": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + if cap(z.Objects) >= int(zb0002) { + z.Objects = (z.Objects)[:zb0002] + } else { + z.Objects = make([]ObjectInfo, zb0002) + } + for za0001 := range z.Objects { + bts, err = z.Objects[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Objects", za0001) + return + } + } + case "Prefixes": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Prefixes") + return + } + if cap(z.Prefixes) >= int(zb0003) { + z.Prefixes = (z.Prefixes)[:zb0003] + } else { + z.Prefixes = make([]string, zb0003) + } + for za0002 := range z.Prefixes { + z.Prefixes[za0002], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Prefixes", za0002) + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ListObjectsInfo) Msgsize() (s int) { + s = 1 + 12 + msgp.BoolSize + 11 + msgp.StringPrefixSize + len(z.NextMarker) + 8 + msgp.ArrayHeaderSize + for za0001 := range z.Objects { + s += z.Objects[za0001].Msgsize() + } + s += 9 + msgp.ArrayHeaderSize + for za0002 := range z.Prefixes { + s += msgp.StringPrefixSize + len(z.Prefixes[za0002]) + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ListObjectsV2Info) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 5 + // string "IsTruncated" + o = append(o, 0x85, 0xab, 0x49, 0x73, 0x54, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64) + o = msgp.AppendBool(o, z.IsTruncated) + // string "ContinuationToken" + o = append(o, 0xb1, 0x43, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e) + o = msgp.AppendString(o, z.ContinuationToken) + // string "NextContinuationToken" + o = append(o, 0xb5, 0x4e, 0x65, 0x78, 0x74, 0x43, 0x6f, 0x6e, 0x74, 0x69, 0x6e, 0x75, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x6f, 0x6b, 0x65, 0x6e) + o = msgp.AppendString(o, z.NextContinuationToken) + // string "Objects" + o = append(o, 0xa7, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Objects))) + for za0001 := range z.Objects { + o, err = z.Objects[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Objects", za0001) + return + } + } + // string "Prefixes" + o = append(o, 0xa8, 0x50, 0x72, 0x65, 0x66, 0x69, 0x78, 0x65, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Prefixes))) + for za0002 := range z.Prefixes { + o = msgp.AppendString(o, z.Prefixes[za0002]) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ListObjectsV2Info) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "IsTruncated": + z.IsTruncated, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "IsTruncated") + return + } + case "ContinuationToken": + z.ContinuationToken, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ContinuationToken") + return + } + case "NextContinuationToken": + z.NextContinuationToken, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "NextContinuationToken") + return + } + case "Objects": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Objects") + return + } + if cap(z.Objects) >= int(zb0002) { + z.Objects = (z.Objects)[:zb0002] + } else { + z.Objects = make([]ObjectInfo, zb0002) + } + for za0001 := range z.Objects { + bts, err = z.Objects[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Objects", za0001) + return + } + } + case "Prefixes": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Prefixes") + return + } + if cap(z.Prefixes) >= int(zb0003) { + z.Prefixes = (z.Prefixes)[:zb0003] + } else { + z.Prefixes = make([]string, zb0003) + } + for za0002 := range z.Prefixes { + z.Prefixes[za0002], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Prefixes", za0002) + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ListObjectsV2Info) Msgsize() (s int) { + s = 1 + 12 + msgp.BoolSize + 18 + msgp.StringPrefixSize + len(z.ContinuationToken) + 22 + msgp.StringPrefixSize + len(z.NextContinuationToken) + 8 + msgp.ArrayHeaderSize + for za0001 := range z.Objects { + s += z.Objects[za0001].Msgsize() + } + s += 9 + msgp.ArrayHeaderSize + for za0002 := range z.Prefixes { + s += msgp.StringPrefixSize + len(z.Prefixes[za0002]) + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ListPartsInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 12 + // string "Bucket" + o = append(o, 0x8c, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + o = msgp.AppendString(o, z.Bucket) + // string "Object" + o = append(o, 0xa6, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74) + o = msgp.AppendString(o, z.Object) + // string "UploadID" + o = append(o, 0xa8, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x44) + o = msgp.AppendString(o, z.UploadID) + // string "StorageClass" + o = append(o, 0xac, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73) + o = msgp.AppendString(o, z.StorageClass) + // string "PartNumberMarker" + o = append(o, 0xb0, 0x50, 0x61, 0x72, 0x74, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72) + o = msgp.AppendInt(o, z.PartNumberMarker) + // string "NextPartNumberMarker" + o = append(o, 0xb4, 0x4e, 0x65, 0x78, 0x74, 0x50, 0x61, 0x72, 0x74, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72) + o = msgp.AppendInt(o, z.NextPartNumberMarker) + // string "MaxParts" + o = append(o, 0xa8, 0x4d, 0x61, 0x78, 0x50, 0x61, 0x72, 0x74, 0x73) + o = msgp.AppendInt(o, z.MaxParts) + // string "IsTruncated" + o = append(o, 0xab, 0x49, 0x73, 0x54, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64) + o = msgp.AppendBool(o, z.IsTruncated) + // string "Parts" + o = append(o, 0xa5, 0x50, 0x61, 0x72, 0x74, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Parts))) + for za0001 := range z.Parts { + o, err = z.Parts[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Parts", za0001) + return + } + } + // string "UserDefined" + o = append(o, 0xab, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64) + o = msgp.AppendMapHeader(o, uint32(len(z.UserDefined))) + for za0002, za0003 := range z.UserDefined { + o = msgp.AppendString(o, za0002) + o = msgp.AppendString(o, za0003) + } + // string "ChecksumAlgorithm" + o = append(o, 0xb1, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d) + o = msgp.AppendString(o, z.ChecksumAlgorithm) + // string "ChecksumType" + o = append(o, 0xac, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x54, 0x79, 0x70, 0x65) + o = msgp.AppendString(o, z.ChecksumType) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ListPartsInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Bucket": + z.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "Object": + z.Object, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + case "UploadID": + z.UploadID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UploadID") + return + } + case "StorageClass": + z.StorageClass, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "StorageClass") + return + } + case "PartNumberMarker": + z.PartNumberMarker, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "PartNumberMarker") + return + } + case "NextPartNumberMarker": + z.NextPartNumberMarker, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "NextPartNumberMarker") + return + } + case "MaxParts": + z.MaxParts, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "MaxParts") + return + } + case "IsTruncated": + z.IsTruncated, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "IsTruncated") + return + } + case "Parts": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Parts") + return + } + if cap(z.Parts) >= int(zb0002) { + z.Parts = (z.Parts)[:zb0002] + } else { + z.Parts = make([]PartInfo, zb0002) + } + for za0001 := range z.Parts { + bts, err = z.Parts[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Parts", za0001) + return + } + } + case "UserDefined": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UserDefined") + return + } + if z.UserDefined == nil { + z.UserDefined = make(map[string]string, zb0003) + } else if len(z.UserDefined) > 0 { + clear(z.UserDefined) + } + for zb0003 > 0 { + var za0003 string + zb0003-- + var za0002 string + za0002, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UserDefined") + return + } + za0003, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UserDefined", za0002) + return + } + z.UserDefined[za0002] = za0003 + } + case "ChecksumAlgorithm": + z.ChecksumAlgorithm, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ChecksumAlgorithm") + return + } + case "ChecksumType": + z.ChecksumType, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ChecksumType") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ListPartsInfo) Msgsize() (s int) { + s = 1 + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Object) + 9 + msgp.StringPrefixSize + len(z.UploadID) + 13 + msgp.StringPrefixSize + len(z.StorageClass) + 17 + msgp.IntSize + 21 + msgp.IntSize + 9 + msgp.IntSize + 12 + msgp.BoolSize + 6 + msgp.ArrayHeaderSize + for za0001 := range z.Parts { + s += z.Parts[za0001].Msgsize() + } + s += 12 + msgp.MapHeaderSize + if z.UserDefined != nil { + for za0002, za0003 := range z.UserDefined { + _ = za0003 + s += msgp.StringPrefixSize + len(za0002) + msgp.StringPrefixSize + len(za0003) + } + } + s += 18 + msgp.StringPrefixSize + len(z.ChecksumAlgorithm) + 13 + msgp.StringPrefixSize + len(z.ChecksumType) + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *MultipartInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 5 + // string "Bucket" + o = append(o, 0x85, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + o = msgp.AppendString(o, z.Bucket) + // string "Object" + o = append(o, 0xa6, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74) + o = msgp.AppendString(o, z.Object) + // string "UploadID" + o = append(o, 0xa8, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x44) + o = msgp.AppendString(o, z.UploadID) + // string "Initiated" + o = append(o, 0xa9, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x74, 0x65, 0x64) + o = msgp.AppendTime(o, z.Initiated) + // string "UserDefined" + o = append(o, 0xab, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64) + o = msgp.AppendMapHeader(o, uint32(len(z.UserDefined))) + for za0001, za0002 := range z.UserDefined { + o = msgp.AppendString(o, za0001) + o = msgp.AppendString(o, za0002) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *MultipartInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Bucket": + z.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "Object": + z.Object, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Object") + return + } + case "UploadID": + z.UploadID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UploadID") + return + } + case "Initiated": + z.Initiated, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Initiated") + return + } + case "UserDefined": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UserDefined") + return + } + if z.UserDefined == nil { + z.UserDefined = make(map[string]string, zb0002) + } else if len(z.UserDefined) > 0 { + clear(z.UserDefined) + } + for zb0002 > 0 { + var za0002 string + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UserDefined") + return + } + za0002, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UserDefined", za0001) + return + } + z.UserDefined[za0001] = za0002 + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *MultipartInfo) Msgsize() (s int) { + s = 1 + 7 + msgp.StringPrefixSize + len(z.Bucket) + 7 + msgp.StringPrefixSize + len(z.Object) + 9 + msgp.StringPrefixSize + len(z.UploadID) + 10 + msgp.TimeSize + 12 + msgp.MapHeaderSize + if z.UserDefined != nil { + for za0001, za0002 := range z.UserDefined { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + msgp.StringPrefixSize + len(za0002) + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z NewMultipartUploadResult) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "UploadID" + o = append(o, 0x83, 0xa8, 0x55, 0x70, 0x6c, 0x6f, 0x61, 0x64, 0x49, 0x44) + o = msgp.AppendString(o, z.UploadID) + // string "ChecksumAlgo" + o = append(o, 0xac, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x41, 0x6c, 0x67, 0x6f) + o = msgp.AppendString(o, z.ChecksumAlgo) + // string "ChecksumType" + o = append(o, 0xac, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x54, 0x79, 0x70, 0x65) + o = msgp.AppendString(o, z.ChecksumType) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *NewMultipartUploadResult) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "UploadID": + z.UploadID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UploadID") + return + } + case "ChecksumAlgo": + z.ChecksumAlgo, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ChecksumAlgo") + return + } + case "ChecksumType": + z.ChecksumType, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ChecksumType") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z NewMultipartUploadResult) Msgsize() (s int) { + s = 1 + 9 + msgp.StringPrefixSize + len(z.UploadID) + 13 + msgp.StringPrefixSize + len(z.ChecksumAlgo) + 13 + msgp.StringPrefixSize + len(z.ChecksumType) + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ObjectInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 33 + // string "Bucket" + o = append(o, 0xde, 0x0, 0x21, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + o = msgp.AppendString(o, z.Bucket) + // string "Name" + o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65) + o = msgp.AppendString(o, z.Name) + // string "ModTime" + o = append(o, 0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65) + o = msgp.AppendTime(o, z.ModTime) + // string "Size" + o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.Size) + // string "ActualSize" + o = append(o, 0xaa, 0x41, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x53, 0x69, 0x7a, 0x65) + if z.ActualSize == nil { + o = msgp.AppendNil(o) + } else { + o = msgp.AppendInt64(o, *z.ActualSize) + } + // string "IsDir" + o = append(o, 0xa5, 0x49, 0x73, 0x44, 0x69, 0x72) + o = msgp.AppendBool(o, z.IsDir) + // string "ETag" + o = append(o, 0xa4, 0x45, 0x54, 0x61, 0x67) + o = msgp.AppendString(o, z.ETag) + // string "VersionID" + o = append(o, 0xa9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x44) + o = msgp.AppendString(o, z.VersionID) + // string "IsLatest" + o = append(o, 0xa8, 0x49, 0x73, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74) + o = msgp.AppendBool(o, z.IsLatest) + // string "DeleteMarker" + o = append(o, 0xac, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72) + o = msgp.AppendBool(o, z.DeleteMarker) + // string "TransitionedObject" + o = append(o, 0xb2, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x69, 0x74, 0x69, 0x6f, 0x6e, 0x65, 0x64, 0x4f, 0x62, 0x6a, 0x65, 0x63, 0x74) + o, err = z.TransitionedObject.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "TransitionedObject") + return + } + // string "RestoreExpires" + o = append(o, 0xae, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73) + o = msgp.AppendTime(o, z.RestoreExpires) + // string "RestoreOngoing" + o = append(o, 0xae, 0x52, 0x65, 0x73, 0x74, 0x6f, 0x72, 0x65, 0x4f, 0x6e, 0x67, 0x6f, 0x69, 0x6e, 0x67) + o = msgp.AppendBool(o, z.RestoreOngoing) + // string "ContentType" + o = append(o, 0xab, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65) + o = msgp.AppendString(o, z.ContentType) + // string "ContentEncoding" + o = append(o, 0xaf, 0x43, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x45, 0x6e, 0x63, 0x6f, 0x64, 0x69, 0x6e, 0x67) + o = msgp.AppendString(o, z.ContentEncoding) + // string "Expires" + o = append(o, 0xa7, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65, 0x73) + o = msgp.AppendTime(o, z.Expires) + // string "CacheControl" + o = append(o, 0xac, 0x43, 0x61, 0x63, 0x68, 0x65, 0x43, 0x6f, 0x6e, 0x74, 0x72, 0x6f, 0x6c) + o = msgp.AppendString(o, z.CacheControl) + // string "StorageClass" + o = append(o, 0xac, 0x53, 0x74, 0x6f, 0x72, 0x61, 0x67, 0x65, 0x43, 0x6c, 0x61, 0x73, 0x73) + o = msgp.AppendString(o, z.StorageClass) + // string "ReplicationStatusInternal" + o = append(o, 0xb9, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c) + o = msgp.AppendString(o, z.ReplicationStatusInternal) + // string "ReplicationStatus" + o = append(o, 0xb1, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73) + o, err = z.ReplicationStatus.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "ReplicationStatus") + return + } + // string "UserDefined" + o = append(o, 0xab, 0x55, 0x73, 0x65, 0x72, 0x44, 0x65, 0x66, 0x69, 0x6e, 0x65, 0x64) + o = msgp.AppendMapHeader(o, uint32(len(z.UserDefined))) + for za0001, za0002 := range z.UserDefined { + o = msgp.AppendString(o, za0001) + o = msgp.AppendString(o, za0002) + } + // string "UserTags" + o = append(o, 0xa8, 0x55, 0x73, 0x65, 0x72, 0x54, 0x61, 0x67, 0x73) + o = msgp.AppendString(o, z.UserTags) + // string "Parts" + o = append(o, 0xa5, 0x50, 0x61, 0x72, 0x74, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Parts))) + for za0003 := range z.Parts { + o, err = z.Parts[za0003].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Parts", za0003) + return + } + } + // string "AccTime" + o = append(o, 0xa7, 0x41, 0x63, 0x63, 0x54, 0x69, 0x6d, 0x65) + o = msgp.AppendTime(o, z.AccTime) + // string "Legacy" + o = append(o, 0xa6, 0x4c, 0x65, 0x67, 0x61, 0x63, 0x79) + o = msgp.AppendBool(o, z.Legacy) + // string "VersionPurgeStatusInternal" + o = append(o, 0xba, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x50, 0x75, 0x72, 0x67, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c) + o = msgp.AppendString(o, z.VersionPurgeStatusInternal) + // string "VersionPurgeStatus" + o = append(o, 0xb2, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x50, 0x75, 0x72, 0x67, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73) + o, err = z.VersionPurgeStatus.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "VersionPurgeStatus") + return + } + // string "NumVersions" + o = append(o, 0xab, 0x4e, 0x75, 0x6d, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73) + o = msgp.AppendInt(o, z.NumVersions) + // string "SuccessorModTime" + o = append(o, 0xb0, 0x53, 0x75, 0x63, 0x63, 0x65, 0x73, 0x73, 0x6f, 0x72, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65) + o = msgp.AppendTime(o, z.SuccessorModTime) + // string "Checksum" + o = append(o, 0xa8, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d) + o = msgp.AppendBytes(o, z.Checksum) + // string "Inlined" + o = append(o, 0xa7, 0x49, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x64) + o = msgp.AppendBool(o, z.Inlined) + // string "DataBlocks" + o = append(o, 0xaa, 0x44, 0x61, 0x74, 0x61, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x73) + o = msgp.AppendInt(o, z.DataBlocks) + // string "ParityBlocks" + o = append(o, 0xac, 0x50, 0x61, 0x72, 0x69, 0x74, 0x79, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x73) + o = msgp.AppendInt(o, z.ParityBlocks) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ObjectInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Bucket": + z.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "Name": + z.Name, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "ModTime": + z.ModTime, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + case "Size": + z.Size, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + case "ActualSize": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.ActualSize = nil + } else { + if z.ActualSize == nil { + z.ActualSize = new(int64) + } + *z.ActualSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ActualSize") + return + } + } + case "IsDir": + z.IsDir, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "IsDir") + return + } + case "ETag": + z.ETag, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ETag") + return + } + case "VersionID": + z.VersionID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + case "IsLatest": + z.IsLatest, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "IsLatest") + return + } + case "DeleteMarker": + z.DeleteMarker, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DeleteMarker") + return + } + case "TransitionedObject": + bts, err = z.TransitionedObject.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "TransitionedObject") + return + } + case "RestoreExpires": + z.RestoreExpires, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "RestoreExpires") + return + } + case "RestoreOngoing": + z.RestoreOngoing, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "RestoreOngoing") + return + } + case "ContentType": + z.ContentType, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ContentType") + return + } + case "ContentEncoding": + z.ContentEncoding, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ContentEncoding") + return + } + case "Expires": + z.Expires, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Expires") + return + } + case "CacheControl": + z.CacheControl, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "CacheControl") + return + } + case "StorageClass": + z.StorageClass, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "StorageClass") + return + } + case "ReplicationStatusInternal": + z.ReplicationStatusInternal, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicationStatusInternal") + return + } + case "ReplicationStatus": + bts, err = z.ReplicationStatus.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicationStatus") + return + } + case "UserDefined": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UserDefined") + return + } + if z.UserDefined == nil { + z.UserDefined = make(map[string]string, zb0002) + } else if len(z.UserDefined) > 0 { + clear(z.UserDefined) + } + for zb0002 > 0 { + var za0002 string + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UserDefined") + return + } + za0002, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UserDefined", za0001) + return + } + z.UserDefined[za0001] = za0002 + } + case "UserTags": + z.UserTags, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UserTags") + return + } + case "Parts": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Parts") + return + } + if cap(z.Parts) >= int(zb0003) { + z.Parts = (z.Parts)[:zb0003] + } else { + z.Parts = make([]ObjectPartInfo, zb0003) + } + for za0003 := range z.Parts { + bts, err = z.Parts[za0003].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Parts", za0003) + return + } + } + case "AccTime": + z.AccTime, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AccTime") + return + } + case "Legacy": + z.Legacy, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Legacy") + return + } + case "VersionPurgeStatusInternal": + z.VersionPurgeStatusInternal, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "VersionPurgeStatusInternal") + return + } + case "VersionPurgeStatus": + bts, err = z.VersionPurgeStatus.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "VersionPurgeStatus") + return + } + case "NumVersions": + z.NumVersions, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "NumVersions") + return + } + case "SuccessorModTime": + z.SuccessorModTime, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SuccessorModTime") + return + } + case "Checksum": + z.Checksum, bts, err = msgp.ReadBytesBytes(bts, z.Checksum) + if err != nil { + err = msgp.WrapError(err, "Checksum") + return + } + case "Inlined": + z.Inlined, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Inlined") + return + } + case "DataBlocks": + z.DataBlocks, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DataBlocks") + return + } + case "ParityBlocks": + z.ParityBlocks, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ParityBlocks") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ObjectInfo) Msgsize() (s int) { + s = 3 + 7 + msgp.StringPrefixSize + len(z.Bucket) + 5 + msgp.StringPrefixSize + len(z.Name) + 8 + msgp.TimeSize + 5 + msgp.Int64Size + 11 + if z.ActualSize == nil { + s += msgp.NilSize + } else { + s += msgp.Int64Size + } + s += 6 + msgp.BoolSize + 5 + msgp.StringPrefixSize + len(z.ETag) + 10 + msgp.StringPrefixSize + len(z.VersionID) + 9 + msgp.BoolSize + 13 + msgp.BoolSize + 19 + z.TransitionedObject.Msgsize() + 15 + msgp.TimeSize + 15 + msgp.BoolSize + 12 + msgp.StringPrefixSize + len(z.ContentType) + 16 + msgp.StringPrefixSize + len(z.ContentEncoding) + 8 + msgp.TimeSize + 13 + msgp.StringPrefixSize + len(z.CacheControl) + 13 + msgp.StringPrefixSize + len(z.StorageClass) + 26 + msgp.StringPrefixSize + len(z.ReplicationStatusInternal) + 18 + z.ReplicationStatus.Msgsize() + 12 + msgp.MapHeaderSize + if z.UserDefined != nil { + for za0001, za0002 := range z.UserDefined { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + msgp.StringPrefixSize + len(za0002) + } + } + s += 9 + msgp.StringPrefixSize + len(z.UserTags) + 6 + msgp.ArrayHeaderSize + for za0003 := range z.Parts { + s += z.Parts[za0003].Msgsize() + } + s += 8 + msgp.TimeSize + 7 + msgp.BoolSize + 27 + msgp.StringPrefixSize + len(z.VersionPurgeStatusInternal) + 19 + z.VersionPurgeStatus.Msgsize() + 12 + msgp.IntSize + 17 + msgp.TimeSize + 9 + msgp.BytesPrefixSize + len(z.Checksum) + 8 + msgp.BoolSize + 11 + msgp.IntSize + 13 + msgp.IntSize + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *PartInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 10 + // string "PartNumber" + o = append(o, 0x8a, 0xaa, 0x50, 0x61, 0x72, 0x74, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72) + o = msgp.AppendInt(o, z.PartNumber) + // string "LastModified" + o = append(o, 0xac, 0x4c, 0x61, 0x73, 0x74, 0x4d, 0x6f, 0x64, 0x69, 0x66, 0x69, 0x65, 0x64) + o = msgp.AppendTime(o, z.LastModified) + // string "ETag" + o = append(o, 0xa4, 0x45, 0x54, 0x61, 0x67) + o = msgp.AppendString(o, z.ETag) + // string "Size" + o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.Size) + // string "ActualSize" + o = append(o, 0xaa, 0x41, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.ActualSize) + // string "ChecksumCRC32" + o = append(o, 0xad, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x43, 0x52, 0x43, 0x33, 0x32) + o = msgp.AppendString(o, z.ChecksumCRC32) + // string "ChecksumCRC32C" + o = append(o, 0xae, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x43, 0x52, 0x43, 0x33, 0x32, 0x43) + o = msgp.AppendString(o, z.ChecksumCRC32C) + // string "ChecksumSHA1" + o = append(o, 0xac, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x53, 0x48, 0x41, 0x31) + o = msgp.AppendString(o, z.ChecksumSHA1) + // string "ChecksumSHA256" + o = append(o, 0xae, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x53, 0x48, 0x41, 0x32, 0x35, 0x36) + o = msgp.AppendString(o, z.ChecksumSHA256) + // string "ChecksumCRC64NVME" + o = append(o, 0xb1, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x43, 0x52, 0x43, 0x36, 0x34, 0x4e, 0x56, 0x4d, 0x45) + o = msgp.AppendString(o, z.ChecksumCRC64NVME) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *PartInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "PartNumber": + z.PartNumber, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "PartNumber") + return + } + case "LastModified": + z.LastModified, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "LastModified") + return + } + case "ETag": + z.ETag, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ETag") + return + } + case "Size": + z.Size, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + case "ActualSize": + z.ActualSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ActualSize") + return + } + case "ChecksumCRC32": + z.ChecksumCRC32, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ChecksumCRC32") + return + } + case "ChecksumCRC32C": + z.ChecksumCRC32C, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ChecksumCRC32C") + return + } + case "ChecksumSHA1": + z.ChecksumSHA1, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ChecksumSHA1") + return + } + case "ChecksumSHA256": + z.ChecksumSHA256, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ChecksumSHA256") + return + } + case "ChecksumCRC64NVME": + z.ChecksumCRC64NVME, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ChecksumCRC64NVME") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *PartInfo) Msgsize() (s int) { + s = 1 + 11 + msgp.IntSize + 13 + msgp.TimeSize + 5 + msgp.StringPrefixSize + len(z.ETag) + 5 + msgp.Int64Size + 11 + msgp.Int64Size + 14 + msgp.StringPrefixSize + len(z.ChecksumCRC32) + 15 + msgp.StringPrefixSize + len(z.ChecksumCRC32C) + 13 + msgp.StringPrefixSize + len(z.ChecksumSHA1) + 15 + msgp.StringPrefixSize + len(z.ChecksumSHA256) + 18 + msgp.StringPrefixSize + len(z.ChecksumCRC64NVME) + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ReplicateObjectInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 26 + // string "Name" + o = append(o, 0xde, 0x0, 0x1a, 0xa4, 0x4e, 0x61, 0x6d, 0x65) + o = msgp.AppendString(o, z.Name) + // string "Bucket" + o = append(o, 0xa6, 0x42, 0x75, 0x63, 0x6b, 0x65, 0x74) + o = msgp.AppendString(o, z.Bucket) + // string "VersionID" + o = append(o, 0xa9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x44) + o = msgp.AppendString(o, z.VersionID) + // string "ETag" + o = append(o, 0xa4, 0x45, 0x54, 0x61, 0x67) + o = msgp.AppendString(o, z.ETag) + // string "Size" + o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.Size) + // string "ActualSize" + o = append(o, 0xaa, 0x41, 0x63, 0x74, 0x75, 0x61, 0x6c, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.ActualSize) + // string "ModTime" + o = append(o, 0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65) + o = msgp.AppendTime(o, z.ModTime) + // string "UserTags" + o = append(o, 0xa8, 0x55, 0x73, 0x65, 0x72, 0x54, 0x61, 0x67, 0x73) + o = msgp.AppendString(o, z.UserTags) + // string "SSEC" + o = append(o, 0xa4, 0x53, 0x53, 0x45, 0x43) + o = msgp.AppendBool(o, z.SSEC) + // string "ReplicationStatus" + o = append(o, 0xb1, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73) + o, err = z.ReplicationStatus.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "ReplicationStatus") + return + } + // string "ReplicationStatusInternal" + o = append(o, 0xb9, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c) + o = msgp.AppendString(o, z.ReplicationStatusInternal) + // string "VersionPurgeStatusInternal" + o = append(o, 0xba, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x50, 0x75, 0x72, 0x67, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x49, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c) + o = msgp.AppendString(o, z.VersionPurgeStatusInternal) + // string "VersionPurgeStatus" + o = append(o, 0xb2, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x50, 0x75, 0x72, 0x67, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73) + o, err = z.VersionPurgeStatus.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "VersionPurgeStatus") + return + } + // string "ReplicationState" + o = append(o, 0xb0, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x53, 0x74, 0x61, 0x74, 0x65) + o, err = z.ReplicationState.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "ReplicationState") + return + } + // string "DeleteMarker" + o = append(o, 0xac, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72) + o = msgp.AppendBool(o, z.DeleteMarker) + // string "OpType" + o = append(o, 0xa6, 0x4f, 0x70, 0x54, 0x79, 0x70, 0x65) + o, err = z.OpType.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "OpType") + return + } + // string "EventType" + o = append(o, 0xa9, 0x45, 0x76, 0x65, 0x6e, 0x74, 0x54, 0x79, 0x70, 0x65) + o = msgp.AppendString(o, z.EventType) + // string "RetryCount" + o = append(o, 0xaa, 0x52, 0x65, 0x74, 0x72, 0x79, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendUint32(o, z.RetryCount) + // string "ResetID" + o = append(o, 0xa7, 0x52, 0x65, 0x73, 0x65, 0x74, 0x49, 0x44) + o = msgp.AppendString(o, z.ResetID) + // string "Dsc" + o = append(o, 0xa3, 0x44, 0x73, 0x63) + o, err = z.Dsc.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Dsc") + return + } + // string "ExistingObjResync" + o = append(o, 0xb1, 0x45, 0x78, 0x69, 0x73, 0x74, 0x69, 0x6e, 0x67, 0x4f, 0x62, 0x6a, 0x52, 0x65, 0x73, 0x79, 0x6e, 0x63) + o, err = z.ExistingObjResync.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "ExistingObjResync") + return + } + // string "TargetArn" + o = append(o, 0xa9, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x41, 0x72, 0x6e) + o = msgp.AppendString(o, z.TargetArn) + // string "TargetStatuses" + o = append(o, 0xae, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.TargetStatuses))) + for za0001, za0002 := range z.TargetStatuses { + o = msgp.AppendString(o, za0001) + o, err = za0002.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "TargetStatuses", za0001) + return + } + } + // string "TargetPurgeStatuses" + o = append(o, 0xb3, 0x54, 0x61, 0x72, 0x67, 0x65, 0x74, 0x50, 0x75, 0x72, 0x67, 0x65, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73, 0x65, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.TargetPurgeStatuses))) + for za0003, za0004 := range z.TargetPurgeStatuses { + o = msgp.AppendString(o, za0003) + o, err = za0004.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "TargetPurgeStatuses", za0003) + return + } + } + // string "ReplicationTimestamp" + o = append(o, 0xb4, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x54, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70) + o = msgp.AppendTime(o, z.ReplicationTimestamp) + // string "Checksum" + o = append(o, 0xa8, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d) + o = msgp.AppendBytes(o, z.Checksum) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ReplicateObjectInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Name": + z.Name, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "Bucket": + z.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "VersionID": + z.VersionID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + case "ETag": + z.ETag, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ETag") + return + } + case "Size": + z.Size, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + case "ActualSize": + z.ActualSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ActualSize") + return + } + case "ModTime": + z.ModTime, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + case "UserTags": + z.UserTags, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UserTags") + return + } + case "SSEC": + z.SSEC, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SSEC") + return + } + case "ReplicationStatus": + bts, err = z.ReplicationStatus.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicationStatus") + return + } + case "ReplicationStatusInternal": + z.ReplicationStatusInternal, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicationStatusInternal") + return + } + case "VersionPurgeStatusInternal": + z.VersionPurgeStatusInternal, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "VersionPurgeStatusInternal") + return + } + case "VersionPurgeStatus": + bts, err = z.VersionPurgeStatus.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "VersionPurgeStatus") + return + } + case "ReplicationState": + bts, err = z.ReplicationState.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicationState") + return + } + case "DeleteMarker": + z.DeleteMarker, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DeleteMarker") + return + } + case "OpType": + bts, err = z.OpType.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "OpType") + return + } + case "EventType": + z.EventType, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "EventType") + return + } + case "RetryCount": + z.RetryCount, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "RetryCount") + return + } + case "ResetID": + z.ResetID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ResetID") + return + } + case "Dsc": + bts, err = z.Dsc.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Dsc") + return + } + case "ExistingObjResync": + bts, err = z.ExistingObjResync.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ExistingObjResync") + return + } + case "TargetArn": + z.TargetArn, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "TargetArn") + return + } + case "TargetStatuses": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "TargetStatuses") + return + } + if z.TargetStatuses == nil { + z.TargetStatuses = make(map[string]replication.StatusType, zb0002) + } else if len(z.TargetStatuses) > 0 { + clear(z.TargetStatuses) + } + for zb0002 > 0 { + var za0002 replication.StatusType + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "TargetStatuses") + return + } + bts, err = za0002.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "TargetStatuses", za0001) + return + } + z.TargetStatuses[za0001] = za0002 + } + case "TargetPurgeStatuses": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "TargetPurgeStatuses") + return + } + if z.TargetPurgeStatuses == nil { + z.TargetPurgeStatuses = make(map[string]VersionPurgeStatusType, zb0003) + } else if len(z.TargetPurgeStatuses) > 0 { + clear(z.TargetPurgeStatuses) + } + for zb0003 > 0 { + var za0004 VersionPurgeStatusType + zb0003-- + var za0003 string + za0003, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "TargetPurgeStatuses") + return + } + bts, err = za0004.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "TargetPurgeStatuses", za0003) + return + } + z.TargetPurgeStatuses[za0003] = za0004 + } + case "ReplicationTimestamp": + z.ReplicationTimestamp, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicationTimestamp") + return + } + case "Checksum": + z.Checksum, bts, err = msgp.ReadBytesBytes(bts, z.Checksum) + if err != nil { + err = msgp.WrapError(err, "Checksum") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ReplicateObjectInfo) Msgsize() (s int) { + s = 3 + 5 + msgp.StringPrefixSize + len(z.Name) + 7 + msgp.StringPrefixSize + len(z.Bucket) + 10 + msgp.StringPrefixSize + len(z.VersionID) + 5 + msgp.StringPrefixSize + len(z.ETag) + 5 + msgp.Int64Size + 11 + msgp.Int64Size + 8 + msgp.TimeSize + 9 + msgp.StringPrefixSize + len(z.UserTags) + 5 + msgp.BoolSize + 18 + z.ReplicationStatus.Msgsize() + 26 + msgp.StringPrefixSize + len(z.ReplicationStatusInternal) + 27 + msgp.StringPrefixSize + len(z.VersionPurgeStatusInternal) + 19 + z.VersionPurgeStatus.Msgsize() + 17 + z.ReplicationState.Msgsize() + 13 + msgp.BoolSize + 7 + z.OpType.Msgsize() + 10 + msgp.StringPrefixSize + len(z.EventType) + 11 + msgp.Uint32Size + 8 + msgp.StringPrefixSize + len(z.ResetID) + 4 + z.Dsc.Msgsize() + 18 + z.ExistingObjResync.Msgsize() + 10 + msgp.StringPrefixSize + len(z.TargetArn) + 15 + msgp.MapHeaderSize + if z.TargetStatuses != nil { + for za0001, za0002 := range z.TargetStatuses { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize() + } + } + s += 20 + msgp.MapHeaderSize + if z.TargetPurgeStatuses != nil { + for za0003, za0004 := range z.TargetPurgeStatuses { + _ = za0004 + s += msgp.StringPrefixSize + len(za0003) + za0004.Msgsize() + } + } + s += 21 + msgp.TimeSize + 9 + msgp.BytesPrefixSize + len(z.Checksum) + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *TransitionedObject) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 5 + // string "Name" + o = append(o, 0x85, 0xa4, 0x4e, 0x61, 0x6d, 0x65) + o = msgp.AppendString(o, z.Name) + // string "VersionID" + o = append(o, 0xa9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x44) + o = msgp.AppendString(o, z.VersionID) + // string "Tier" + o = append(o, 0xa4, 0x54, 0x69, 0x65, 0x72) + o = msgp.AppendString(o, z.Tier) + // string "FreeVersion" + o = append(o, 0xab, 0x46, 0x72, 0x65, 0x65, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e) + o = msgp.AppendBool(o, z.FreeVersion) + // string "Status" + o = append(o, 0xa6, 0x53, 0x74, 0x61, 0x74, 0x75, 0x73) + o = msgp.AppendString(o, z.Status) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *TransitionedObject) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Name": + z.Name, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "VersionID": + z.VersionID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + case "Tier": + z.Tier, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tier") + return + } + case "FreeVersion": + z.FreeVersion, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "FreeVersion") + return + } + case "Status": + z.Status, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Status") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *TransitionedObject) Msgsize() (s int) { + s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 10 + msgp.StringPrefixSize + len(z.VersionID) + 5 + msgp.StringPrefixSize + len(z.Tier) + 12 + msgp.BoolSize + 7 + msgp.StringPrefixSize + len(z.Status) + return +} diff --git a/cmd/object-api-deleteobject_test.go b/cmd/object-api-deleteobject_test.go new file mode 100644 index 00000000..b7e888e4 --- /dev/null +++ b/cmd/object-api-deleteobject_test.go @@ -0,0 +1,126 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "crypto/md5" + "encoding/hex" + "strings" + "testing" +) + +// Wrapper for calling DeleteObject tests for both Erasure multiple disks and single node setup. +func TestDeleteObject(t *testing.T) { + ExecObjectLayerTest(t, testDeleteObject) +} + +// Unit test for DeleteObject in general. +func testDeleteObject(obj ObjectLayer, instanceType string, t TestErrHandler) { + type objectUpload struct { + name string + content string + } + + testCases := []struct { + bucketName string + objectToUploads []objectUpload + pathToDelete string + objectsAfterDelete []string + }{ + // Test 1: removes an object and checks it is the only object + // that has been deleted. + { + "bucket1", + []objectUpload{{"object0", "content"}, {"object1", "content"}}, + "object0", + []string{"object1"}, + }, + // Test 2: remove an object inside a directory and checks it is deleted + // with its parent since this former becomes empty + { + "bucket2", + []objectUpload{{"object0", "content"}, {"dir/object1", "content"}}, + "dir/object1", + []string{"object0"}, + }, + // Test 3: remove an object inside a directory and checks if it is deleted + // but other sibling object in the same directory still exists + { + "bucket3", + []objectUpload{{"dir/object1", "content"}, {"dir/object2", "content"}}, + "dir/object1", + []string{"dir/object2"}, + }, + // Test 4: remove a non empty directory and checks it has no effect + { + "bucket4", + []objectUpload{{"object0", "content"}, {"dir/object1", "content"}}, + "dir/", + []string{"dir/object1", "object0"}, + }, + // Test 5: Remove an empty directory and checks it is really removed + { + "bucket5", + []objectUpload{{"object0", "content"}, {"dir/", ""}}, + "dir/", + []string{"object0"}, + }, + } + + for i, testCase := range testCases { + err := obj.MakeBucket(context.Background(), testCase.bucketName, MakeBucketOptions{}) + if err != nil { + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + for _, object := range testCase.objectToUploads { + md5Bytes := md5.Sum([]byte(object.content)) + oi, err := obj.PutObject(context.Background(), testCase.bucketName, object.name, mustGetPutObjReader(t, strings.NewReader(object.content), + int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{}) + if err != nil { + t.Log(oi) + t.Fatalf("%s : %s", instanceType, err.Error()) + } + } + + oi, err := obj.DeleteObject(context.Background(), testCase.bucketName, testCase.pathToDelete, ObjectOptions{}) + if err != nil && !isErrObjectNotFound(err) { + t.Log(oi) + t.Errorf("Test %d: %s: Expected to pass, but failed with: %s", i+1, instanceType, err) + continue + } + + result, err := obj.ListObjects(context.Background(), testCase.bucketName, "", "", "", 1000) + if err != nil { + t.Errorf("Test %d: %s: Expected to pass, but failed with: %s", i+1, instanceType, err.Error()) + continue + } + + if len(result.Objects) != len(testCase.objectsAfterDelete) { + t.Errorf("Test %d: %s: mismatch number of objects after delete, expected = %v, found = %v", i+1, instanceType, testCase.objectsAfterDelete, result.Objects) + continue + } + + for idx := range result.Objects { + if result.Objects[idx].Name != testCase.objectsAfterDelete[idx] { + t.Errorf("Test %d: %s: Unexpected object found after delete, found = `%v`", i+1, instanceType, result.Objects[idx].Name) + } + } + } +} diff --git a/cmd/object-api-errors.go b/cmd/object-api-errors.go new file mode 100644 index 00000000..ef372393 --- /dev/null +++ b/cmd/object-api-errors.go @@ -0,0 +1,807 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "io" +) + +// Converts underlying storage error. Convenience function written to +// handle all cases where we have known types of errors returned by +// underlying storage layer. +func toObjectErr(oerr error, params ...string) error { + if oerr == nil { + return nil + } + + // Unwarp the error first + err := unwrapAll(oerr) + + if err == context.Canceled { + return context.Canceled + } + + switch err.Error() { + case errVolumeNotFound.Error(): + apiErr := BucketNotFound{} + if len(params) >= 1 { + apiErr.Bucket = params[0] + } + return apiErr + case errVolumeNotEmpty.Error(): + apiErr := BucketNotEmpty{} + if len(params) >= 1 { + apiErr.Bucket = params[0] + } + return apiErr + case errVolumeExists.Error(): + apiErr := BucketExists{} + if len(params) >= 1 { + apiErr.Bucket = params[0] + } + return apiErr + case errDiskFull.Error(): + return StorageFull{} + case errTooManyOpenFiles.Error(): + return SlowDown{} + case errFileAccessDenied.Error(): + apiErr := PrefixAccessDenied{} + if len(params) >= 1 { + apiErr.Bucket = params[0] + } + if len(params) >= 2 { + apiErr.Object = decodeDirObject(params[1]) + } + return apiErr + case errIsNotRegular.Error(): + apiErr := ObjectExistsAsDirectory{} + if len(params) >= 1 { + apiErr.Bucket = params[0] + } + if len(params) >= 2 { + apiErr.Object = decodeDirObject(params[1]) + } + return apiErr + case errFileVersionNotFound.Error(): + apiErr := VersionNotFound{} + if len(params) >= 1 { + apiErr.Bucket = params[0] + } + if len(params) >= 2 { + apiErr.Object = decodeDirObject(params[1]) + } + if len(params) >= 3 { + apiErr.VersionID = params[2] + } + return apiErr + case errMethodNotAllowed.Error(): + apiErr := MethodNotAllowed{} + if len(params) >= 1 { + apiErr.Bucket = params[0] + } + if len(params) >= 2 { + apiErr.Object = decodeDirObject(params[1]) + } + return apiErr + case errFileNotFound.Error(): + apiErr := ObjectNotFound{} + if len(params) >= 1 { + apiErr.Bucket = params[0] + } + if len(params) >= 2 { + apiErr.Object = decodeDirObject(params[1]) + } + return apiErr + case errUploadIDNotFound.Error(): + apiErr := InvalidUploadID{} + if len(params) >= 1 { + apiErr.Bucket = params[0] + } + if len(params) >= 2 { + apiErr.Object = decodeDirObject(params[1]) + } + if len(params) >= 3 { + apiErr.UploadID = params[2] + } + return apiErr + case errFileNameTooLong.Error(): + apiErr := ObjectNameInvalid{} + if len(params) >= 1 { + apiErr.Bucket = params[0] + } + if len(params) >= 2 { + apiErr.Object = decodeDirObject(params[1]) + } + return apiErr + case errDataTooLarge.Error(): + apiErr := ObjectTooLarge{} + if len(params) >= 1 { + apiErr.Bucket = params[0] + } + if len(params) >= 2 { + apiErr.Object = decodeDirObject(params[1]) + } + return apiErr + case errDataTooSmall.Error(): + apiErr := ObjectTooSmall{} + if len(params) >= 1 { + apiErr.Bucket = params[0] + } + if len(params) >= 2 { + apiErr.Object = decodeDirObject(params[1]) + } + return apiErr + case errErasureReadQuorum.Error(): + apiErr := InsufficientReadQuorum{} + if len(params) >= 1 { + apiErr.Bucket = params[0] + } + if len(params) >= 2 { + apiErr.Object = decodeDirObject(params[1]) + } + if v, ok := oerr.(InsufficientReadQuorum); ok { + apiErr.Type = v.Type + } + return apiErr + case errErasureWriteQuorum.Error(): + apiErr := InsufficientWriteQuorum{} + if len(params) >= 1 { + apiErr.Bucket = params[0] + } + if len(params) >= 2 { + apiErr.Object = decodeDirObject(params[1]) + } + return apiErr + case io.ErrUnexpectedEOF.Error(), io.ErrShortWrite.Error(), context.Canceled.Error(), context.DeadlineExceeded.Error(): + apiErr := IncompleteBody{} + if len(params) >= 1 { + apiErr.Bucket = params[0] + } + if len(params) >= 2 { + apiErr.Object = decodeDirObject(params[1]) + } + return apiErr + } + return err +} + +// SignatureDoesNotMatch - when content md5 does not match with what was sent from client. +type SignatureDoesNotMatch struct{} + +func (e SignatureDoesNotMatch) Error() string { + return "The request signature we calculated does not match the signature you provided. Check your key and signing method." +} + +// StorageFull storage ran out of space. +type StorageFull struct{} + +func (e StorageFull) Error() string { + return "Storage reached its minimum free drive threshold." +} + +// SlowDown too many file descriptors open or backend busy . +type SlowDown struct{} + +func (e SlowDown) Error() string { + return "Please reduce your request rate" +} + +// RQErrType reason for read quorum error. +type RQErrType int + +const ( + // RQInsufficientOnlineDrives - not enough online drives. + RQInsufficientOnlineDrives RQErrType = 1 << iota + // RQInconsistentMeta - inconsistent metadata. + RQInconsistentMeta +) + +func (t RQErrType) String() string { + switch t { + case RQInsufficientOnlineDrives: + return "InsufficientOnlineDrives" + case RQInconsistentMeta: + return "InconsistentMeta" + default: + return "Unknown" + } +} + +// InsufficientReadQuorum storage cannot satisfy quorum for read operation. +type InsufficientReadQuorum struct { + Bucket string + Object string + Err error + Type RQErrType +} + +func (e InsufficientReadQuorum) Error() string { + return "Storage resources are insufficient for the read operation " + e.Bucket + "/" + e.Object +} + +// Unwrap the error. +func (e InsufficientReadQuorum) Unwrap() error { + return errErasureReadQuorum +} + +// InsufficientWriteQuorum storage cannot satisfy quorum for write operation. +type InsufficientWriteQuorum GenericError + +func (e InsufficientWriteQuorum) Error() string { + return "Storage resources are insufficient for the write operation " + e.Bucket + "/" + e.Object +} + +// Unwrap the error. +func (e InsufficientWriteQuorum) Unwrap() error { + return errErasureWriteQuorum +} + +// GenericError - generic object layer error. +type GenericError struct { + Bucket string + Object string + VersionID string + Err error +} + +// Unwrap the error to its underlying error. +func (e GenericError) Unwrap() error { + return e.Err +} + +// InvalidArgument incorrect input argument +type InvalidArgument GenericError + +func (e InvalidArgument) Error() string { + if e.Err != nil { + return "Invalid arguments provided for " + e.Bucket + "/" + e.Object + ": (" + e.Err.Error() + ")" + } + return "Invalid arguments provided for " + e.Bucket + "/" + e.Object +} + +// BucketNotFound bucket does not exist. +type BucketNotFound GenericError + +func (e BucketNotFound) Error() string { + return "Bucket not found: " + e.Bucket +} + +// BucketAlreadyExists the requested bucket name is not available. +type BucketAlreadyExists GenericError + +func (e BucketAlreadyExists) Error() string { + return "The requested bucket name is not available. The bucket namespace is shared by all users of the system. Please select a different name and try again." +} + +// BucketAlreadyOwnedByYou already owned by you. +type BucketAlreadyOwnedByYou GenericError + +func (e BucketAlreadyOwnedByYou) Error() string { + return "Bucket already owned by you: " + e.Bucket +} + +// BucketNotEmpty bucket is not empty. +type BucketNotEmpty GenericError + +func (e BucketNotEmpty) Error() string { + return "Bucket not empty: " + e.Bucket +} + +// InvalidVersionID invalid version id +type InvalidVersionID GenericError + +func (e InvalidVersionID) Error() string { + return "Invalid version id: " + e.Bucket + "/" + e.Object + "(" + e.VersionID + ")" +} + +// VersionNotFound version does not exist. +type VersionNotFound GenericError + +func (e VersionNotFound) Error() string { + return "Version not found: " + e.Bucket + "/" + e.Object + "(" + e.VersionID + ")" +} + +// ObjectNotFound object does not exist. +type ObjectNotFound GenericError + +func (e ObjectNotFound) Error() string { + return "Object not found: " + e.Bucket + "/" + e.Object +} + +// MethodNotAllowed on the object +type MethodNotAllowed GenericError + +func (e MethodNotAllowed) Error() string { + return "Method not allowed: " + e.Bucket + "/" + e.Object +} + +// ObjectLocked object is currently WORM protected. +type ObjectLocked GenericError + +func (e ObjectLocked) Error() string { + return "Object is WORM protected and cannot be overwritten: " + e.Bucket + "/" + e.Object + "(" + e.VersionID + ")" +} + +// ObjectAlreadyExists object already exists. +type ObjectAlreadyExists GenericError + +func (e ObjectAlreadyExists) Error() string { + return "Object: " + e.Bucket + "/" + e.Object + " already exists" +} + +// ObjectExistsAsDirectory object already exists as a directory. +type ObjectExistsAsDirectory GenericError + +func (e ObjectExistsAsDirectory) Error() string { + return "Object exists on : " + e.Bucket + " as directory " + e.Object +} + +// PrefixAccessDenied object access is denied. +type PrefixAccessDenied GenericError + +func (e PrefixAccessDenied) Error() string { + return "Prefix access is denied: " + e.Bucket + SlashSeparator + e.Object +} + +// BucketExists bucket exists. +type BucketExists GenericError + +func (e BucketExists) Error() string { + return "Bucket exists: " + e.Bucket +} + +// InvalidUploadIDKeyCombination - invalid upload id and key marker combination. +type InvalidUploadIDKeyCombination struct { + UploadIDMarker, KeyMarker string +} + +func (e InvalidUploadIDKeyCombination) Error() string { + return fmt.Sprintf("Invalid combination of uploadID marker '%s' and marker '%s'", e.UploadIDMarker, e.KeyMarker) +} + +// BucketPolicyNotFound - no bucket policy found. +type BucketPolicyNotFound GenericError + +func (e BucketPolicyNotFound) Error() string { + return "No bucket policy configuration found for bucket: " + e.Bucket +} + +// BucketLifecycleNotFound - no bucket lifecycle found. +type BucketLifecycleNotFound GenericError + +func (e BucketLifecycleNotFound) Error() string { + return "No bucket lifecycle configuration found for bucket : " + e.Bucket +} + +// BucketSSEConfigNotFound - no bucket encryption found +type BucketSSEConfigNotFound GenericError + +func (e BucketSSEConfigNotFound) Error() string { + return "No bucket encryption configuration found for bucket: " + e.Bucket +} + +// BucketTaggingNotFound - no bucket tags found +type BucketTaggingNotFound GenericError + +func (e BucketTaggingNotFound) Error() string { + return "No bucket tags found for bucket: " + e.Bucket +} + +// BucketObjectLockConfigNotFound - no bucket object lock config found +type BucketObjectLockConfigNotFound GenericError + +func (e BucketObjectLockConfigNotFound) Error() string { + return "No bucket object lock configuration found for bucket: " + e.Bucket +} + +// BucketQuotaConfigNotFound - no bucket quota config found. +type BucketQuotaConfigNotFound GenericError + +func (e BucketQuotaConfigNotFound) Error() string { + return "No quota config found for bucket : " + e.Bucket +} + +// BucketQuotaExceeded - bucket quota exceeded. +type BucketQuotaExceeded GenericError + +func (e BucketQuotaExceeded) Error() string { + return "Bucket quota exceeded for bucket: " + e.Bucket +} + +// BucketReplicationConfigNotFound - no bucket replication config found +type BucketReplicationConfigNotFound GenericError + +func (e BucketReplicationConfigNotFound) Error() string { + return "The replication configuration was not found: " + e.Bucket +} + +// BucketRemoteDestinationNotFound bucket does not exist. +type BucketRemoteDestinationNotFound GenericError + +func (e BucketRemoteDestinationNotFound) Error() string { + return "Destination bucket does not exist: " + e.Bucket +} + +// BucketRemoteTargetNotFound remote target does not exist. +type BucketRemoteTargetNotFound GenericError + +func (e BucketRemoteTargetNotFound) Error() string { + return "Remote target not found: " + e.Bucket +} + +// RemoteTargetConnectionErr remote target connection failure. +type RemoteTargetConnectionErr struct { + Err error + Bucket string + Endpoint string + AccessKey string +} + +func (e RemoteTargetConnectionErr) Error() string { + if e.Bucket != "" { + return fmt.Sprintf("Remote service endpoint offline, target bucket: %s or remote service credentials: %s invalid \n\t%s", e.Bucket, e.AccessKey, e.Err.Error()) + } + return fmt.Sprintf("Remote service endpoint %s not available\n\t%s", e.Endpoint, e.Err.Error()) +} + +// BucketRemoteIdenticalToSource remote already exists for this target type. +type BucketRemoteIdenticalToSource struct { + GenericError + Endpoint string +} + +func (e BucketRemoteIdenticalToSource) Error() string { + return fmt.Sprintf("Remote service endpoint %s is self referential to current cluster", e.Endpoint) +} + +// BucketRemoteAlreadyExists remote already exists for this target type. +type BucketRemoteAlreadyExists GenericError + +func (e BucketRemoteAlreadyExists) Error() string { + return "Remote already exists for this bucket: " + e.Bucket +} + +// BucketRemoteLabelInUse remote already exists for this target label. +type BucketRemoteLabelInUse GenericError + +func (e BucketRemoteLabelInUse) Error() string { + return "Remote with this label already exists for this bucket: " + e.Bucket +} + +// BucketRemoteArnTypeInvalid arn type for remote is not valid. +type BucketRemoteArnTypeInvalid GenericError + +func (e BucketRemoteArnTypeInvalid) Error() string { + return "Remote ARN type not valid: " + e.Bucket +} + +// BucketRemoteArnInvalid arn needs to be specified. +type BucketRemoteArnInvalid GenericError + +func (e BucketRemoteArnInvalid) Error() string { + return "Remote ARN has invalid format: " + e.Bucket +} + +// BucketRemoteRemoveDisallowed when replication configuration exists +type BucketRemoteRemoveDisallowed GenericError + +func (e BucketRemoteRemoveDisallowed) Error() string { + return "Replication configuration exists with this ARN:" + e.Bucket +} + +// BucketRemoteTargetNotVersioned remote target does not have versioning enabled. +type BucketRemoteTargetNotVersioned GenericError + +func (e BucketRemoteTargetNotVersioned) Error() string { + return "Remote target does not have versioning enabled: " + e.Bucket +} + +// BucketReplicationSourceNotVersioned replication source does not have versioning enabled. +type BucketReplicationSourceNotVersioned GenericError + +func (e BucketReplicationSourceNotVersioned) Error() string { + return "Replication source does not have versioning enabled: " + e.Bucket +} + +// TransitionStorageClassNotFound remote tier not configured. +type TransitionStorageClassNotFound GenericError + +func (e TransitionStorageClassNotFound) Error() string { + return "Transition storage class not found " +} + +// InvalidObjectState restore-object doesn't apply for the current state of the object. +type InvalidObjectState GenericError + +func (e InvalidObjectState) Error() string { + return "The operation is not valid for the current state of the object " + e.Bucket + "/" + e.Object + "(" + e.VersionID + ")" +} + +// Bucket related errors. + +// BucketNameInvalid - bucketname provided is invalid. +type BucketNameInvalid GenericError + +// Error returns string an error formatted as the given text. +func (e BucketNameInvalid) Error() string { + return "Bucket name invalid: " + e.Bucket +} + +// Object related errors. + +// ObjectNameInvalid - object name provided is invalid. +type ObjectNameInvalid GenericError + +// ObjectNameTooLong - object name too long. +type ObjectNameTooLong GenericError + +// ObjectNamePrefixAsSlash - object name has a slash as prefix. +type ObjectNamePrefixAsSlash GenericError + +// Error returns string an error formatted as the given text. +func (e ObjectNameInvalid) Error() string { + return "Object name invalid: " + e.Bucket + "/" + e.Object +} + +// Error returns string an error formatted as the given text. +func (e ObjectNameTooLong) Error() string { + return "Object name too long: " + e.Bucket + "/" + e.Object +} + +// Error returns string an error formatted as the given text. +func (e ObjectNamePrefixAsSlash) Error() string { + return "Object name contains forward slash as prefix: " + e.Bucket + "/" + e.Object +} + +// AllAccessDisabled All access to this object has been disabled +type AllAccessDisabled GenericError + +// Error returns string an error formatted as the given text. +func (e AllAccessDisabled) Error() string { + return "All access to this object has been disabled" +} + +// IncompleteBody You did not provide the number of bytes specified by the Content-Length HTTP header. +type IncompleteBody GenericError + +// Error returns string an error formatted as the given text. +func (e IncompleteBody) Error() string { + return e.Bucket + "/" + e.Object + " has incomplete body" +} + +// InvalidRange - invalid range typed error. +type InvalidRange struct { + OffsetBegin int64 + OffsetEnd int64 + ResourceSize int64 +} + +func (e InvalidRange) Error() string { + return fmt.Sprintf("The requested range 'bytes=%d-%d' is not satisfiable", e.OffsetBegin, e.OffsetEnd) +} + +// ObjectTooLarge error returned when the size of the object > max object size allowed (5G) per request. +type ObjectTooLarge GenericError + +func (e ObjectTooLarge) Error() string { + return "size of the object greater than what is allowed(5G)" +} + +// ObjectTooSmall error returned when the size of the object < what is expected. +type ObjectTooSmall GenericError + +func (e ObjectTooSmall) Error() string { + return "size of the object less than what is expected" +} + +// OperationTimedOut - a timeout occurred. +type OperationTimedOut struct{} + +func (e OperationTimedOut) Error() string { + return "Operation timed out" +} + +// Multipart related errors. + +// MalformedUploadID malformed upload id. +type MalformedUploadID struct { + UploadID string +} + +func (e MalformedUploadID) Error() string { + return "Malformed upload id " + e.UploadID +} + +// InvalidUploadID invalid upload id. +type InvalidUploadID struct { + Bucket string + Object string + UploadID string +} + +func (e InvalidUploadID) Error() string { + return "Invalid upload id " + e.UploadID +} + +// InvalidPart One or more of the specified parts could not be found +type InvalidPart struct { + PartNumber int + ExpETag string + GotETag string +} + +func (e InvalidPart) Error() string { + return fmt.Sprintf("Specified part could not be found. PartNumber %d, Expected %s, got %s", + e.PartNumber, e.ExpETag, e.GotETag) +} + +// PartTooSmall - error if part size is less than 5MB. +type PartTooSmall struct { + PartSize int64 + PartNumber int + PartETag string +} + +func (e PartTooSmall) Error() string { + return fmt.Sprintf("Part size for %d should be at least 5MB", e.PartNumber) +} + +// PartTooBig returned if size of part is bigger than the allowed limit. +type PartTooBig struct{} + +func (e PartTooBig) Error() string { + return "Part size bigger than the allowed limit" +} + +// InvalidETag error returned when the etag has changed on disk +type InvalidETag struct{} + +func (e InvalidETag) Error() string { + return "etag of the object has changed" +} + +// BackendDown is returned for network errors +type BackendDown struct { + Err string +} + +func (e BackendDown) Error() string { + return e.Err +} + +// NotImplemented If a feature is not implemented +type NotImplemented struct { + Message string +} + +func (e NotImplemented) Error() string { + return e.Message +} + +// UnsupportedMetadata - unsupported metadata +type UnsupportedMetadata struct{} + +func (e UnsupportedMetadata) Error() string { + return "Unsupported headers in Metadata" +} + +// isErrBucketNotFound - Check if error type is BucketNotFound. +func isErrBucketNotFound(err error) bool { + if errors.Is(err, errVolumeNotFound) { + return true + } + + var bkNotFound BucketNotFound + return errors.As(err, &bkNotFound) +} + +// isErrReadQuorum check if the error type is InsufficientReadQuorum +func isErrReadQuorum(err error) bool { + var rquorum InsufficientReadQuorum + return errors.As(err, &rquorum) +} + +// isErrWriteQuorum check if the error type is InsufficientWriteQuorum +func isErrWriteQuorum(err error) bool { + var rquorum InsufficientWriteQuorum + return errors.As(err, &rquorum) +} + +// isErrObjectNotFound - Check if error type is ObjectNotFound. +func isErrObjectNotFound(err error) bool { + if errors.Is(err, errFileNotFound) { + return true + } + + var objNotFound ObjectNotFound + return errors.As(err, &objNotFound) +} + +// isErrVersionNotFound - Check if error type is VersionNotFound. +func isErrVersionNotFound(err error) bool { + if errors.Is(err, errFileVersionNotFound) { + return true + } + + var versionNotFound VersionNotFound + return errors.As(err, &versionNotFound) +} + +// isErrSignatureDoesNotMatch - Check if error type is SignatureDoesNotMatch. +func isErrSignatureDoesNotMatch(err error) bool { + var signatureDoesNotMatch SignatureDoesNotMatch + return errors.As(err, &signatureDoesNotMatch) +} + +// PreConditionFailed - Check if copy precondition failed +type PreConditionFailed struct{} + +func (e PreConditionFailed) Error() string { + return "At least one of the pre-conditions you specified did not hold" +} + +func isErrPreconditionFailed(err error) bool { + _, ok := err.(PreConditionFailed) + return ok +} + +// isErrMethodNotAllowed - Check if error type is MethodNotAllowed. +func isErrMethodNotAllowed(err error) bool { + var methodNotAllowed MethodNotAllowed + return errors.As(err, &methodNotAllowed) +} + +func isErrInvalidRange(err error) bool { + if errors.Is(err, errInvalidRange) { + return true + } + _, ok := err.(InvalidRange) + return ok +} + +// ReplicationPermissionCheck - Check if error type is ReplicationPermissionCheck. +type ReplicationPermissionCheck struct{} + +func (e ReplicationPermissionCheck) Error() string { + return "Replication permission validation requests cannot be completed" +} + +func isReplicationPermissionCheck(err error) bool { + _, ok := err.(ReplicationPermissionCheck) + return ok +} + +// DataMovementOverwriteErr - captures the error when a data movement activity +// like rebalance incorrectly tries to overwrite an object. +type DataMovementOverwriteErr GenericError + +func (de DataMovementOverwriteErr) Error() string { + objInfoStr := fmt.Sprintf("bucket=%s object=%s", de.Bucket, de.Object) + if de.VersionID != "" { + objInfoStr = fmt.Sprintf("%s version-id=%s", objInfoStr, de.VersionID) + } + return fmt.Sprintf("invalid data movement operation, source and destination pool are the same for %s", objInfoStr) +} + +func isDataMovementOverWriteErr(err error) bool { + var de DataMovementOverwriteErr + return errors.As(err, &de) +} diff --git a/cmd/object-api-getobjectinfo_test.go b/cmd/object-api-getobjectinfo_test.go new file mode 100644 index 00000000..6bee4b56 --- /dev/null +++ b/cmd/object-api-getobjectinfo_test.go @@ -0,0 +1,115 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "testing" +) + +// Wrapper for calling GetObjectInfo tests for both Erasure multiple disks and single node setup. +func TestGetObjectInfo(t *testing.T) { + ExecObjectLayerTest(t, testGetObjectInfo) +} + +// Testing GetObjectInfo(). +func testGetObjectInfo(obj ObjectLayer, instanceType string, t TestErrHandler) { + // This bucket is used for testing getObjectInfo operations. + err := obj.MakeBucket(context.Background(), "test-getobjectinfo", MakeBucketOptions{}) + if err != nil { + t.Fatalf("%s : %s", instanceType, err.Error()) + } + opts := ObjectOptions{} + _, err = obj.PutObject(context.Background(), "test-getobjectinfo", "Asia/asiapics.jpg", mustGetPutObjReader(t, bytes.NewBufferString("asiapics"), int64(len("asiapics")), "", ""), opts) + if err != nil { + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + // Put an empty directory + _, err = obj.PutObject(context.Background(), "test-getobjectinfo", "Asia/empty-dir/", mustGetPutObjReader(t, bytes.NewBufferString(""), int64(len("")), "", ""), opts) + if err != nil { + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + resultCases := []ObjectInfo{ + // ObjectInfo -1. + // ObjectName set to a existing object in the test case (Test case 14). + {Bucket: "test-getobjectinfo", Name: "Asia/asiapics.jpg", ContentType: "image/jpeg", IsDir: false}, + {Bucket: "test-getobjectinfo", Name: "Asia/empty-dir/", ContentType: "application/octet-stream", IsDir: true}, + } + testCases := []struct { + bucketName string + objectName string + + // Expected output of GetObjectInfo. + result ObjectInfo + err error + // Flag indicating whether the test is expected to pass or not. + shouldPass bool + }{ + // Test cases with invalid bucket names ( Test number 1-4 ). + {".test", "", ObjectInfo{}, BucketNameInvalid{Bucket: ".test"}, false}, + {"---", "", ObjectInfo{}, BucketNameInvalid{Bucket: "---"}, false}, + {"ad", "", ObjectInfo{}, BucketNameInvalid{Bucket: "ad"}, false}, + // Test cases with valid but non-existing bucket names (Test number 5-6). + {"abcdefgh", "abc", ObjectInfo{}, BucketNotFound{Bucket: "abcdefgh"}, false}, + {"ijklmnop", "efg", ObjectInfo{}, BucketNotFound{Bucket: "ijklmnop"}, false}, + // Test cases with valid but non-existing bucket names and invalid object name (Test number 7-8). + {"test-getobjectinfo", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "test-getobjectinfo", Object: ""}, false}, + {"test-getobjectinfo", "", ObjectInfo{}, ObjectNameInvalid{Bucket: "test-getobjectinfo", Object: ""}, false}, + // Test cases with non-existing object name with existing bucket (Test number 9-11). + {"test-getobjectinfo", "Africa", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Africa"}, false}, + {"test-getobjectinfo", "Antartica", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Antartica"}, false}, + {"test-getobjectinfo", "Asia/myfile", ObjectInfo{}, ObjectNotFound{Bucket: "test-getobjectinfo", Object: "Asia/myfile"}, false}, + // Valid case with existing object (Test number 12). + {"test-getobjectinfo", "Asia/asiapics.jpg", resultCases[0], nil, true}, + {"test-getobjectinfo", "Asia/empty-dir/", resultCases[1], nil, true}, + } + for i, testCase := range testCases { + result, err := obj.GetObjectInfo(context.Background(), testCase.bucketName, testCase.objectName, opts) + if err != nil && testCase.shouldPass { + t.Errorf("Test %d: %s: Expected to pass, but failed with: %s", i+1, instanceType, err.Error()) + } + if err == nil && !testCase.shouldPass { + t.Errorf("Test %d: %s: Expected to fail with \"%s\", but passed instead", i+1, instanceType, testCase.err.Error()) + } + // Failed as expected, but does it fail for the expected reason. + if err != nil && !testCase.shouldPass { + if testCase.err.Error() != err.Error() { + t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.err.Error(), err.Error()) + } + } + + // Test passes as expected, but the output values are verified for correctness here. + if err == nil && testCase.shouldPass { + if testCase.result.Bucket != result.Bucket { + t.Fatalf("Test %d: %s: Expected Bucket name to be '%s', but found '%s' instead", i+1, instanceType, testCase.result.Bucket, result.Bucket) + } + if testCase.result.Name != result.Name { + t.Errorf("Test %d: %s: Expected Object name to be %s, but instead found it to be %s", i+1, instanceType, testCase.result.Name, result.Name) + } + if testCase.result.ContentType != result.ContentType { + t.Errorf("Test %d: %s: Expected Content Type of the object to be %v, but instead found it to be %v", i+1, instanceType, testCase.result.ContentType, result.ContentType) + } + if testCase.result.IsDir != result.IsDir { + t.Errorf("Test %d: %s: Expected IsDir flag of the object to be %v, but instead found it to be %v", i+1, instanceType, testCase.result.IsDir, result.IsDir) + } + } + } +} diff --git a/cmd/object-api-input-checks.go b/cmd/object-api-input-checks.go new file mode 100644 index 00000000..9c8b213e --- /dev/null +++ b/cmd/object-api-input-checks.go @@ -0,0 +1,178 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/base64" + "runtime" + "strings" + + "github.com/minio/minio-go/v7/pkg/s3utils" +) + +// Checks on CopyObject arguments, bucket and object. +func checkCopyObjArgs(ctx context.Context, bucket, object string) error { + return checkBucketAndObjectNames(ctx, bucket, object) +} + +// Checks on GetObject arguments, bucket and object. +func checkGetObjArgs(ctx context.Context, bucket, object string) error { + return checkBucketAndObjectNames(ctx, bucket, object) +} + +// Checks on DeleteObject arguments, bucket and object. +func checkDelObjArgs(ctx context.Context, bucket, object string) error { + return checkBucketAndObjectNames(ctx, bucket, object) +} + +// Checks bucket and object name validity, returns nil if both are valid. +func checkBucketAndObjectNames(ctx context.Context, bucket, object string) error { + // Verify if bucket is valid. + if !isMinioMetaBucketName(bucket) && s3utils.CheckValidBucketNameStrict(bucket) != nil { + return BucketNameInvalid{Bucket: bucket} + } + // Verify if object is valid. + if len(object) == 0 { + return ObjectNameInvalid{Bucket: bucket, Object: object} + } + if !IsValidObjectPrefix(object) { + return ObjectNameInvalid{Bucket: bucket, Object: object} + } + if runtime.GOOS == globalWindowsOSName && strings.Contains(object, "\\") { + // Objects cannot be contain \ in Windows and is listed as `Characters to Avoid`. + return ObjectNameInvalid{Bucket: bucket, Object: object} + } + return nil +} + +// Checks for all ListObjects arguments validity. +func checkListObjsArgs(ctx context.Context, bucket, prefix, marker string) error { + // Verify if bucket is valid. + if !isMinioMetaBucketName(bucket) && s3utils.CheckValidBucketNameStrict(bucket) != nil { + return BucketNameInvalid{Bucket: bucket} + } + + // Validates object prefix validity after bucket exists. + if !IsValidObjectPrefix(prefix) { + return ObjectNameInvalid{ + Bucket: bucket, + Object: prefix, + } + } + return nil +} + +// Checks for all ListMultipartUploads arguments validity. +func checkListMultipartArgs(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string) error { + if err := checkListObjsArgs(ctx, bucket, prefix, keyMarker); err != nil { + return err + } + if uploadIDMarker != "" { + if HasSuffix(keyMarker, SlashSeparator) { + return InvalidUploadIDKeyCombination{ + UploadIDMarker: uploadIDMarker, + KeyMarker: keyMarker, + } + } + _, err := base64.RawURLEncoding.DecodeString(uploadIDMarker) + if err != nil { + return MalformedUploadID{ + UploadID: uploadIDMarker, + } + } + } + return nil +} + +// Checks for NewMultipartUpload arguments validity, also validates if bucket exists. +func checkNewMultipartArgs(ctx context.Context, bucket, object string) error { + return checkObjectArgs(ctx, bucket, object) +} + +func checkMultipartObjectArgs(ctx context.Context, bucket, object, uploadID string) error { + _, err := base64.RawURLEncoding.DecodeString(uploadID) + if err != nil { + return MalformedUploadID{ + UploadID: uploadID, + } + } + return checkObjectArgs(ctx, bucket, object) +} + +// Checks for PutObjectPart arguments validity, also validates if bucket exists. +func checkPutObjectPartArgs(ctx context.Context, bucket, object, uploadID string) error { + return checkMultipartObjectArgs(ctx, bucket, object, uploadID) +} + +// Checks for ListParts arguments validity, also validates if bucket exists. +func checkListPartsArgs(ctx context.Context, bucket, object, uploadID string) error { + return checkMultipartObjectArgs(ctx, bucket, object, uploadID) +} + +// Checks for CompleteMultipartUpload arguments validity, also validates if bucket exists. +func checkCompleteMultipartArgs(ctx context.Context, bucket, object, uploadID string) error { + return checkMultipartObjectArgs(ctx, bucket, object, uploadID) +} + +// Checks for AbortMultipartUpload arguments validity, also validates if bucket exists. +func checkAbortMultipartArgs(ctx context.Context, bucket, object, uploadID string) error { + return checkMultipartObjectArgs(ctx, bucket, object, uploadID) +} + +// Checks Object arguments validity. +func checkObjectArgs(ctx context.Context, bucket, object string) error { + // Verify if bucket is valid. + if !isMinioMetaBucketName(bucket) && s3utils.CheckValidBucketNameStrict(bucket) != nil { + return BucketNameInvalid{Bucket: bucket} + } + + if err := checkObjectNameForLengthAndSlash(bucket, object); err != nil { + return err + } + + // Validates object name validity after bucket exists. + if !IsValidObjectName(object) { + return ObjectNameInvalid{ + Bucket: bucket, + Object: object, + } + } + + return nil +} + +// Checks for PutObject arguments validity. +func checkPutObjectArgs(ctx context.Context, bucket, object string) error { + // Verify if bucket is valid. + if !isMinioMetaBucketName(bucket) && s3utils.CheckValidBucketNameStrict(bucket) != nil { + return BucketNameInvalid{Bucket: bucket} + } + + if err := checkObjectNameForLengthAndSlash(bucket, object); err != nil { + return err + } + if len(object) == 0 || + !IsValidObjectPrefix(object) { + return ObjectNameInvalid{ + Bucket: bucket, + Object: object, + } + } + return nil +} diff --git a/cmd/object-api-interface.go b/cmd/object-api-interface.go new file mode 100644 index 00000000..06c73741 --- /dev/null +++ b/cmd/object-api-interface.go @@ -0,0 +1,338 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "io" + "net/http" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7/pkg/encrypt" + "github.com/minio/minio-go/v7/pkg/tags" + "github.com/minio/minio/internal/hash" + + "github.com/minio/minio/internal/bucket/replication" + xioutil "github.com/minio/minio/internal/ioutil" +) + +//go:generate msgp -file $GOFILE -io=false -tests=false -unexported=false + +//msgp:ignore ObjectOptions TransitionOptions DeleteBucketOptions + +// CheckPreconditionFn returns true if precondition check failed. +type CheckPreconditionFn func(o ObjectInfo) bool + +// EvalMetadataFn validates input objInfo and GetObjectInfo error and returns an updated metadata and replication decision if any +type EvalMetadataFn func(o *ObjectInfo, gerr error) (ReplicateDecision, error) + +// EvalRetentionBypassFn validates input objInfo and GetObjectInfo error and returns an error if retention bypass is not allowed. +type EvalRetentionBypassFn func(o ObjectInfo, gerr error) error + +// GetObjectInfoFn is the signature of GetObjectInfo function. +type GetObjectInfoFn func(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) + +// WalkVersionsSortOrder represents the sort order in which versions of an +// object should be returned by ObjectLayer.Walk method +type WalkVersionsSortOrder uint8 + +const ( + // WalkVersionsSortAsc - Sort in ascending order of ModTime + WalkVersionsSortAsc WalkVersionsSortOrder = iota + // WalkVersionsSortDesc - Sort in descending order of ModTime + WalkVersionsSortDesc +) + +// ObjectOptions represents object options for ObjectLayer object operations +type ObjectOptions struct { + ServerSideEncryption encrypt.ServerSide + VersionSuspended bool // indicates if the bucket was previously versioned but is currently suspended. + Versioned bool // indicates if the bucket is versioned + VersionID string // Specifies the versionID which needs to be overwritten or read + MTime time.Time // Is only set in POST/PUT operations + Expires time.Time // Is only used in POST/PUT operations + + DeleteMarker bool // Is only set in DELETE operations for delete marker replication + CheckDMReplicationReady bool // Is delete marker ready to be replicated - set only during HEAD + Tagging bool // Is only in GET/HEAD operations to return tagging metadata along with regular metadata and body. + + UserDefined map[string]string // only set in case of POST/PUT operations + ObjectAttributes map[string]struct{} // Attribute tags defined by the users for the GetObjectAttributes request + MaxParts int // used in GetObjectAttributes. Signals how many parts we should return + PartNumberMarker int // used in GetObjectAttributes. Signals the part number after which results should be returned + PartNumber int // only useful in case of GetObject/HeadObject + CheckPrecondFn CheckPreconditionFn // only set during GetObject/HeadObject/CopyObjectPart preconditional valuation + EvalMetadataFn EvalMetadataFn // only set for retention settings, meant to be used only when updating metadata in-place. + DeleteReplication ReplicationState // Represents internal replication state needed for Delete replication + Transition TransitionOptions + Expiration ExpirationOptions + LifecycleAuditEvent lcAuditEvent + + WantChecksum *hash.Checksum // x-amz-checksum-XXX checksum sent to PutObject/ CompleteMultipartUpload. + + WantServerSideChecksumType hash.ChecksumType // if set, we compute a server-side checksum of this type + + NoDecryption bool // indicates if the stream must be decrypted. + PreserveETag string // preserves this etag during a PUT call. + NoLock bool // indicates to lower layers if the caller is expecting to hold locks. + HasIfMatch bool // indicates if the request has If-Match header + ProxyRequest bool // only set for GET/HEAD in active-active replication scenario + ProxyHeaderSet bool // only set for GET/HEAD in active-active replication scenario + ReplicationRequest bool // true only if replication request + ReplicationSourceTaggingTimestamp time.Time // set if MinIOSourceTaggingTimestamp received + ReplicationSourceLegalholdTimestamp time.Time // set if MinIOSourceObjectLegalholdTimestamp received + ReplicationSourceRetentionTimestamp time.Time // set if MinIOSourceObjectRetentionTimestamp received + DeletePrefix bool // set true to enforce a prefix deletion, only application for DeleteObject API, + DeletePrefixObject bool // set true when object's erasure set is resolvable by object name (using getHashedSetIndex) + + Speedtest bool // object call specifically meant for SpeedTest code, set to 'true' when invoked by SpeedtestHandler. + + // Use the maximum parity (N/2), used when saving server configuration files + MaxParity bool + + // Provides a per object encryption function, allowing metadata encryption. + EncryptFn objectMetaEncryptFn + + // SkipDecommissioned set to 'true' if the call requires skipping the pool being decommissioned. + // mainly set for certain WRITE operations. + SkipDecommissioned bool + // SkipRebalancing should be set to 'true' if the call should skip pools + // participating in a rebalance operation. Typically set for 'write' operations. + SkipRebalancing bool + + SrcPoolIdx int // set by PutObject/CompleteMultipart operations due to rebalance; used to prevent rebalance src, dst pools to be the same + + DataMovement bool // indicates an going decommisionning or rebalacing + + PrefixEnabledFn func(prefix string) bool // function which returns true if versioning is enabled on prefix + + // IndexCB will return any index created but the compression. + // Object must have been read at this point. + IndexCB func() []byte + + // InclFreeVersions indicates that free versions need to be included + // when looking up a version by fi.VersionID + InclFreeVersions bool + // SkipFreeVersion skips adding a free version when a tiered version is + // being 'replaced' + // Note: Used only when a tiered object is being expired. + SkipFreeVersion bool + + MetadataChg bool // is true if it is a metadata update operation. + EvalRetentionBypassFn EvalRetentionBypassFn // only set for enforcing retention bypass on DeleteObject. + + FastGetObjInfo bool // Only for S3 Head/Get Object calls for now + NoAuditLog bool // Only set for decom, rebalance, to avoid double audits. +} + +// WalkOptions provides filtering, marker and other Walk() specific options. +type WalkOptions struct { + Filter func(info FileInfo) bool // return WalkFilter returns 'true/false' + Marker string // set to skip until this object + LatestOnly bool // returns only latest versions for all matching objects + AskDisks string // dictates how many disks are being listed + VersionsSort WalkVersionsSortOrder // sort order for versions of the same object; default: Ascending order in ModTime + Limit int // maximum number of items, 0 means no limit +} + +// ExpirationOptions represents object options for object expiration at objectLayer. +type ExpirationOptions struct { + Expire bool +} + +// TransitionOptions represents object options for transition ObjectLayer operation +type TransitionOptions struct { + Status string + Tier string + ETag string + RestoreRequest *RestoreObjectRequest + RestoreExpiry time.Time + ExpireRestored bool +} + +// MakeBucketOptions represents bucket options for ObjectLayer bucket operations +type MakeBucketOptions struct { + LockEnabled bool + VersioningEnabled bool + ForceCreate bool // Create buckets even if they are already created. + CreatedAt time.Time // only for site replication + NoLock bool // does not lock the make bucket call if set to 'true' +} + +// DeleteBucketOptions provides options for DeleteBucket calls. +type DeleteBucketOptions struct { + NoLock bool // does not lock the delete bucket call if set to 'true' + NoRecreate bool // do not recreate bucket on delete failures + Force bool // Force deletion + SRDeleteOp SRBucketDeleteOp // only when site replication is enabled +} + +// BucketOptions provides options for ListBuckets and GetBucketInfo call. +type BucketOptions struct { + Deleted bool // true only when site replication is enabled + Cached bool // true only when we are requesting a cached response instead of hitting the disk for example ListBuckets() call. + NoMetadata bool +} + +// SetReplicaStatus sets replica status and timestamp for delete operations in ObjectOptions +func (o *ObjectOptions) SetReplicaStatus(st replication.StatusType) { + o.DeleteReplication.ReplicaStatus = st + o.DeleteReplication.ReplicaTimeStamp = UTCNow() +} + +// DeleteMarkerReplicationStatus - returns replication status of delete marker from DeleteReplication state in ObjectOptions +func (o *ObjectOptions) DeleteMarkerReplicationStatus() replication.StatusType { + return o.DeleteReplication.CompositeReplicationStatus() +} + +// VersionPurgeStatus - returns version purge status from DeleteReplication state in ObjectOptions +func (o *ObjectOptions) VersionPurgeStatus() VersionPurgeStatusType { + return o.DeleteReplication.CompositeVersionPurgeStatus() +} + +// SetDeleteReplicationState sets the delete replication options. +func (o *ObjectOptions) SetDeleteReplicationState(dsc ReplicateDecision, vID string) { + o.DeleteReplication = ReplicationState{ + ReplicateDecisionStr: dsc.String(), + } + switch o.VersionID { + case "": + o.DeleteReplication.ReplicationStatusInternal = dsc.PendingStatus() + o.DeleteReplication.Targets = replicationStatusesMap(o.DeleteReplication.ReplicationStatusInternal) + default: + o.DeleteReplication.VersionPurgeStatusInternal = dsc.PendingStatus() + o.DeleteReplication.PurgeTargets = versionPurgeStatusesMap(o.DeleteReplication.VersionPurgeStatusInternal) + } +} + +// PutReplicationState gets ReplicationState for PUT operation from ObjectOptions +func (o *ObjectOptions) PutReplicationState() (r ReplicationState) { + rstatus, ok := o.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] + if !ok { + return r + } + r.ReplicationStatusInternal = rstatus + r.Targets = replicationStatusesMap(rstatus) + return r +} + +// SetEvalMetadataFn sets the metadata evaluation function +func (o *ObjectOptions) SetEvalMetadataFn(f EvalMetadataFn) { + o.EvalMetadataFn = f +} + +// SetEvalRetentionBypassFn sets the retention bypass function +func (o *ObjectOptions) SetEvalRetentionBypassFn(f EvalRetentionBypassFn) { + o.EvalRetentionBypassFn = f +} + +// ObjectLayer implements primitives for object API layer. +type ObjectLayer interface { + // Locking operations on object. + NewNSLock(bucket string, objects ...string) RWLocker + + // Storage operations. + Shutdown(context.Context) error + NSScanner(ctx context.Context, updates chan<- DataUsageInfo, wantCycle uint32, scanMode madmin.HealScanMode) error + BackendInfo() madmin.BackendInfo + Legacy() bool // Only returns true for deployments which use CRCMOD as its object distribution algorithm. + StorageInfo(ctx context.Context, metrics bool) StorageInfo + LocalStorageInfo(ctx context.Context, metrics bool) StorageInfo + + // Bucket operations. + MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error + GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (bucketInfo BucketInfo, err error) + ListBuckets(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error) + DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error + ListObjects(ctx context.Context, bucket, prefix, marker, delimiter string, maxKeys int) (result ListObjectsInfo, err error) + ListObjectsV2(ctx context.Context, bucket, prefix, continuationToken, delimiter string, maxKeys int, fetchOwner bool, startAfter string) (result ListObjectsV2Info, err error) + ListObjectVersions(ctx context.Context, bucket, prefix, marker, versionMarker, delimiter string, maxKeys int) (result ListObjectVersionsInfo, err error) + // Walk lists all objects including versions, delete markers. + Walk(ctx context.Context, bucket, prefix string, results chan<- itemOrErr[ObjectInfo], opts WalkOptions) error + + // Object operations. + + // GetObjectNInfo returns a GetObjectReader that satisfies the + // ReadCloser interface. The Close method runs any cleanup + // functions, so it must always be called after reading till EOF + // + // IMPORTANTLY, when implementations return err != nil, this + // function MUST NOT return a non-nil ReadCloser. + GetObjectNInfo(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, h http.Header, opts ObjectOptions) (reader *GetObjectReader, err error) + GetObjectInfo(ctx context.Context, bucket, object string, opts ObjectOptions) (objInfo ObjectInfo, err error) + PutObject(ctx context.Context, bucket, object string, data *PutObjReader, opts ObjectOptions) (objInfo ObjectInfo, err error) + CopyObject(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (objInfo ObjectInfo, err error) + DeleteObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) + DeleteObjects(ctx context.Context, bucket string, objects []ObjectToDelete, opts ObjectOptions) ([]DeletedObject, []error) + TransitionObject(ctx context.Context, bucket, object string, opts ObjectOptions) error + RestoreTransitionedObject(ctx context.Context, bucket, object string, opts ObjectOptions) error + + // Multipart operations. + ListMultipartUploads(ctx context.Context, bucket, prefix, keyMarker, uploadIDMarker, delimiter string, maxUploads int) (result ListMultipartsInfo, err error) + NewMultipartUpload(ctx context.Context, bucket, object string, opts ObjectOptions) (result *NewMultipartUploadResult, err error) + CopyObjectPart(ctx context.Context, srcBucket, srcObject, destBucket, destObject string, uploadID string, partID int, + startOffset int64, length int64, srcInfo ObjectInfo, srcOpts, dstOpts ObjectOptions) (info PartInfo, err error) + PutObjectPart(ctx context.Context, bucket, object, uploadID string, partID int, data *PutObjReader, opts ObjectOptions) (info PartInfo, err error) + GetMultipartInfo(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) (info MultipartInfo, err error) + ListObjectParts(ctx context.Context, bucket, object, uploadID string, partNumberMarker int, maxParts int, opts ObjectOptions) (result ListPartsInfo, err error) + AbortMultipartUpload(ctx context.Context, bucket, object, uploadID string, opts ObjectOptions) error + CompleteMultipartUpload(ctx context.Context, bucket, object, uploadID string, uploadedParts []CompletePart, opts ObjectOptions) (objInfo ObjectInfo, err error) + + GetDisks(poolIdx, setIdx int) ([]StorageAPI, error) // return the disks belonging to pool and set. + SetDriveCounts() []int // list of erasure stripe size for each pool in order. + + // Healing operations. + HealFormat(ctx context.Context, dryRun bool) (madmin.HealResultItem, error) + HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, error) + HealObject(ctx context.Context, bucket, object, versionID string, opts madmin.HealOpts) (madmin.HealResultItem, error) + HealObjects(ctx context.Context, bucket, prefix string, opts madmin.HealOpts, fn HealObjectFn) error + CheckAbandonedParts(ctx context.Context, bucket, object string, opts madmin.HealOpts) error + + // Returns health of the backend + Health(ctx context.Context, opts HealthOptions) HealthResult + + // Metadata operations + PutObjectMetadata(context.Context, string, string, ObjectOptions) (ObjectInfo, error) + DecomTieredObject(context.Context, string, string, FileInfo, ObjectOptions) error + + // ObjectTagging operations + PutObjectTags(context.Context, string, string, string, ObjectOptions) (ObjectInfo, error) + GetObjectTags(context.Context, string, string, ObjectOptions) (*tags.Tags, error) + DeleteObjectTags(context.Context, string, string, ObjectOptions) (ObjectInfo, error) +} + +// GetObject - TODO(aead): This function just acts as an adapter for GetObject tests and benchmarks +// since the GetObject method of the ObjectLayer interface has been removed. Once, the +// tests are adjusted to use GetObjectNInfo this function can be removed. +func GetObject(ctx context.Context, api ObjectLayer, bucket, object string, startOffset int64, length int64, writer io.Writer, etag string, opts ObjectOptions) (err error) { + var header http.Header + if etag != "" { + header.Set("ETag", etag) + } + Range := &HTTPRangeSpec{Start: startOffset, End: startOffset + length} + + reader, err := api.GetObjectNInfo(ctx, bucket, object, Range, header, opts) + if err != nil { + return err + } + defer reader.Close() + + _, err = xioutil.Copy(writer, reader) + return err +} diff --git a/cmd/object-api-interface_gen.go b/cmd/object-api-interface_gen.go new file mode 100644 index 00000000..919cdb17 --- /dev/null +++ b/cmd/object-api-interface_gen.go @@ -0,0 +1,337 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// MarshalMsg implements msgp.Marshaler +func (z BucketOptions) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "Deleted" + o = append(o, 0x83, 0xa7, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x64) + o = msgp.AppendBool(o, z.Deleted) + // string "Cached" + o = append(o, 0xa6, 0x43, 0x61, 0x63, 0x68, 0x65, 0x64) + o = msgp.AppendBool(o, z.Cached) + // string "NoMetadata" + o = append(o, 0xaa, 0x4e, 0x6f, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61) + o = msgp.AppendBool(o, z.NoMetadata) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BucketOptions) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Deleted": + z.Deleted, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Deleted") + return + } + case "Cached": + z.Cached, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Cached") + return + } + case "NoMetadata": + z.NoMetadata, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "NoMetadata") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z BucketOptions) Msgsize() (s int) { + s = 1 + 8 + msgp.BoolSize + 7 + msgp.BoolSize + 11 + msgp.BoolSize + return +} + +// MarshalMsg implements msgp.Marshaler +func (z ExpirationOptions) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "Expire" + o = append(o, 0x81, 0xa6, 0x45, 0x78, 0x70, 0x69, 0x72, 0x65) + o = msgp.AppendBool(o, z.Expire) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ExpirationOptions) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Expire": + z.Expire, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Expire") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z ExpirationOptions) Msgsize() (s int) { + s = 1 + 7 + msgp.BoolSize + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *MakeBucketOptions) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 5 + // string "LockEnabled" + o = append(o, 0x85, 0xab, 0x4c, 0x6f, 0x63, 0x6b, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64) + o = msgp.AppendBool(o, z.LockEnabled) + // string "VersioningEnabled" + o = append(o, 0xb1, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x69, 0x6e, 0x67, 0x45, 0x6e, 0x61, 0x62, 0x6c, 0x65, 0x64) + o = msgp.AppendBool(o, z.VersioningEnabled) + // string "ForceCreate" + o = append(o, 0xab, 0x46, 0x6f, 0x72, 0x63, 0x65, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65) + o = msgp.AppendBool(o, z.ForceCreate) + // string "CreatedAt" + o = append(o, 0xa9, 0x43, 0x72, 0x65, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + o = msgp.AppendTime(o, z.CreatedAt) + // string "NoLock" + o = append(o, 0xa6, 0x4e, 0x6f, 0x4c, 0x6f, 0x63, 0x6b) + o = msgp.AppendBool(o, z.NoLock) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *MakeBucketOptions) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "LockEnabled": + z.LockEnabled, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "LockEnabled") + return + } + case "VersioningEnabled": + z.VersioningEnabled, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "VersioningEnabled") + return + } + case "ForceCreate": + z.ForceCreate, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ForceCreate") + return + } + case "CreatedAt": + z.CreatedAt, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "CreatedAt") + return + } + case "NoLock": + z.NoLock, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "NoLock") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *MakeBucketOptions) Msgsize() (s int) { + s = 1 + 12 + msgp.BoolSize + 18 + msgp.BoolSize + 12 + msgp.BoolSize + 10 + msgp.TimeSize + 7 + msgp.BoolSize + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *WalkOptions) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 5 + // string "Marker" + o = append(o, 0x85, 0xa6, 0x4d, 0x61, 0x72, 0x6b, 0x65, 0x72) + o = msgp.AppendString(o, z.Marker) + // string "LatestOnly" + o = append(o, 0xaa, 0x4c, 0x61, 0x74, 0x65, 0x73, 0x74, 0x4f, 0x6e, 0x6c, 0x79) + o = msgp.AppendBool(o, z.LatestOnly) + // string "AskDisks" + o = append(o, 0xa8, 0x41, 0x73, 0x6b, 0x44, 0x69, 0x73, 0x6b, 0x73) + o = msgp.AppendString(o, z.AskDisks) + // string "VersionsSort" + o = append(o, 0xac, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x73, 0x53, 0x6f, 0x72, 0x74) + o = msgp.AppendUint8(o, uint8(z.VersionsSort)) + // string "Limit" + o = append(o, 0xa5, 0x4c, 0x69, 0x6d, 0x69, 0x74) + o = msgp.AppendInt(o, z.Limit) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *WalkOptions) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Marker": + z.Marker, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Marker") + return + } + case "LatestOnly": + z.LatestOnly, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "LatestOnly") + return + } + case "AskDisks": + z.AskDisks, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AskDisks") + return + } + case "VersionsSort": + { + var zb0002 uint8 + zb0002, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "VersionsSort") + return + } + z.VersionsSort = WalkVersionsSortOrder(zb0002) + } + case "Limit": + z.Limit, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Limit") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *WalkOptions) Msgsize() (s int) { + s = 1 + 7 + msgp.StringPrefixSize + len(z.Marker) + 11 + msgp.BoolSize + 9 + msgp.StringPrefixSize + len(z.AskDisks) + 13 + msgp.Uint8Size + 6 + msgp.IntSize + return +} + +// MarshalMsg implements msgp.Marshaler +func (z WalkVersionsSortOrder) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendUint8(o, uint8(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *WalkVersionsSortOrder) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 uint8 + zb0001, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = WalkVersionsSortOrder(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z WalkVersionsSortOrder) Msgsize() (s int) { + s = msgp.Uint8Size + return +} diff --git a/cmd/object-api-listobjects_test.go b/cmd/object-api-listobjects_test.go new file mode 100644 index 00000000..58bfc4f8 --- /dev/null +++ b/cmd/object-api-listobjects_test.go @@ -0,0 +1,2046 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/hex" + "fmt" + "strconv" + "strings" + "testing" + "time" + + "github.com/minio/minio/internal/bucket/lifecycle" +) + +func TestListObjectsVersionedFolders(t *testing.T) { + ExecObjectLayerTest(t, testListObjectsVersionedFolders) +} + +func testListObjectsVersionedFolders(obj ObjectLayer, instanceType string, t1 TestErrHandler) { + t, _ := t1.(*testing.T) + testBuckets := []string{ + // This bucket is used for testing ListObject operations. + "test-bucket-folders", + // This bucket has file delete marker. + "test-bucket-files", + } + for _, bucket := range testBuckets { + err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{ + VersioningEnabled: true, + }) + if err != nil { + t.Fatalf("%s : %s", instanceType, err.Error()) + } + } + + var err error + testObjects := []struct { + parentBucket string + name string + content string + meta map[string]string + addDeleteMarker bool + }{ + {testBuckets[0], "unique/folder/", "", nil, true}, + {testBuckets[0], "unique/folder/1.txt", "content", nil, false}, + {testBuckets[1], "unique/folder/1.txt", "content", nil, true}, + } + for _, object := range testObjects { + md5Bytes := md5.Sum([]byte(object.content)) + _, err = obj.PutObject(context.Background(), object.parentBucket, object.name, mustGetPutObjReader(t, bytes.NewBufferString(object.content), + int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{ + Versioned: globalBucketVersioningSys.PrefixEnabled(object.parentBucket, object.name), + UserDefined: object.meta, + }) + if err != nil { + t.Fatalf("%s : %s", instanceType, err.Error()) + } + if object.addDeleteMarker { + oi, err := obj.DeleteObject(context.Background(), object.parentBucket, object.name, ObjectOptions{ + Versioned: globalBucketVersioningSys.PrefixEnabled(object.parentBucket, object.name), + }) + if err != nil { + t.Fatalf("%s : %s", instanceType, err.Error()) + } + if oi.DeleteMarker != object.addDeleteMarker { + t.Fatalf("Expected, marker %t : got %t", object.addDeleteMarker, oi.DeleteMarker) + } + } + } + + // Formulating the result data set to be expected from ListObjects call inside the tests, + // This will be used in testCases and used for asserting the correctness of ListObjects output in the tests. + + resultCases := []ListObjectsInfo{ + { + IsTruncated: false, + Prefixes: []string{"unique/folder/"}, + }, + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "unique/folder/1.txt"}, + }, + }, + { + IsTruncated: false, + Objects: []ObjectInfo{}, + }, + } + + resultCasesV := []ListObjectVersionsInfo{ + { + IsTruncated: false, + Prefixes: []string{"unique/folder/"}, + }, + { + IsTruncated: false, + Objects: []ObjectInfo{ + { + Name: "unique/folder/", + DeleteMarker: true, + }, + { + Name: "unique/folder/", + DeleteMarker: false, + }, + { + Name: "unique/folder/1.txt", + DeleteMarker: false, + }, + }, + }, + } + + testCases := []struct { + // Inputs to ListObjects. + bucketName string + prefix string + marker string + delimiter string + maxKeys int + versioned bool + // Expected output of ListObjects. + resultL ListObjectsInfo + resultV ListObjectVersionsInfo + err error + // Flag indicating whether the test is expected to pass or not. + shouldPass bool + }{ + {testBuckets[0], "unique/", "", "/", 1000, false, resultCases[0], ListObjectVersionsInfo{}, nil, true}, + {testBuckets[0], "unique/folder", "", "/", 1000, false, resultCases[0], ListObjectVersionsInfo{}, nil, true}, + {testBuckets[0], "unique/", "", "", 1000, false, resultCases[1], ListObjectVersionsInfo{}, nil, true}, + {testBuckets[1], "unique/", "", "/", 1000, false, resultCases[0], ListObjectVersionsInfo{}, nil, true}, + {testBuckets[1], "unique/folder/", "", "/", 1000, false, resultCases[2], ListObjectVersionsInfo{}, nil, true}, + {testBuckets[0], "unique/", "", "/", 1000, true, ListObjectsInfo{}, resultCasesV[0], nil, true}, + {testBuckets[0], "unique/", "", "", 1000, true, ListObjectsInfo{}, resultCasesV[1], nil, true}, + } + + for i, testCase := range testCases { + t.Run(fmt.Sprintf("%s-Test%d", instanceType, i+1), func(t *testing.T) { + var err error + var resultL ListObjectsInfo + var resultV ListObjectVersionsInfo + if testCase.versioned { + t.Log("ListObjectVersions, bucket:", testCase.bucketName, "prefix:", + testCase.prefix, "marker:", testCase.marker, "delimiter:", + testCase.delimiter, "maxkeys:", testCase.maxKeys) + + resultV, err = obj.ListObjectVersions(t.Context(), testCase.bucketName, + testCase.prefix, testCase.marker, "", testCase.delimiter, testCase.maxKeys) + } else { + t.Log("ListObjects, bucket:", testCase.bucketName, "prefix:", + testCase.prefix, "marker:", testCase.marker, "delimiter:", + testCase.delimiter, "maxkeys:", testCase.maxKeys) + + resultL, err = obj.ListObjects(t.Context(), testCase.bucketName, + testCase.prefix, testCase.marker, testCase.delimiter, testCase.maxKeys) + } + if err != nil && testCase.shouldPass { + t.Errorf("Test %d: %s: Expected to pass, but failed with: %s", i+1, instanceType, err.Error()) + } + if err == nil && !testCase.shouldPass { + t.Errorf("Test %d: %s: Expected to fail with \"%s\", but passed instead", i+1, instanceType, testCase.err.Error()) + } + // Failed as expected, but does it fail for the expected reason. + if err != nil && !testCase.shouldPass { + if !strings.Contains(err.Error(), testCase.err.Error()) { + t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.err.Error(), err.Error()) + } + } + // Since there are cases for which ListObjects fails, this is + // necessary. Test passes as expected, but the output values + // are verified for correctness here. + if err == nil && testCase.shouldPass { + // The length of the expected ListObjectsResult.Objects + // should match in both expected result from test cases + // and in the output. On failure calling t.Fatalf, + // otherwise it may lead to index out of range error in + // assertion following this. + if !testCase.versioned { + if len(testCase.resultL.Objects) != len(resultL.Objects) { + t.Logf("want: %v", objInfoNames(testCase.resultL.Objects)) + t.Logf("got: %v", objInfoNames(resultL.Objects)) + t.Errorf("Test %d: %s: Expected number of object in the result to be '%d', but found '%d' objects instead", i+1, instanceType, len(testCase.resultL.Objects), len(resultL.Objects)) + } + for j := 0; j < len(testCase.resultL.Objects); j++ { + if j >= len(resultL.Objects) { + t.Errorf("Test %d: %s: Expected object name to be \"%s\", but not nothing instead", i+1, instanceType, testCase.resultL.Objects[j].Name) + continue + } + if testCase.resultL.Objects[j].Name != resultL.Objects[j].Name { + t.Errorf("Test %d: %s: Expected object name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.resultL.Objects[j].Name, resultL.Objects[j].Name) + } + } + + if len(testCase.resultL.Prefixes) != len(resultL.Prefixes) { + t.Logf("want: %v", testCase.resultL.Prefixes) + t.Logf("got: %v", resultL.Prefixes) + t.Errorf("Test %d: %s: Expected number of prefixes in the result to be '%d', but found '%d' prefixes instead", i+1, instanceType, len(testCase.resultL.Prefixes), len(resultL.Prefixes)) + } + for j := 0; j < len(testCase.resultL.Prefixes); j++ { + if j >= len(resultL.Prefixes) { + t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found no result", i+1, instanceType, testCase.resultL.Prefixes[j]) + continue + } + if testCase.resultL.Prefixes[j] != resultL.Prefixes[j] { + t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.resultL.Prefixes[j], resultL.Prefixes[j]) + } + } + + if testCase.resultL.IsTruncated != resultL.IsTruncated { + // Allow an extra continuation token. + if !resultL.IsTruncated || len(resultL.Objects) == 0 { + t.Errorf("Test %d: %s: Expected IsTruncated flag to be %v, but instead found it to be %v", i+1, instanceType, testCase.resultL.IsTruncated, resultL.IsTruncated) + } + } + + if testCase.resultL.IsTruncated && resultL.NextMarker == "" { + t.Errorf("Test %d: %s: Expected NextMarker to contain a string since listing is truncated, but instead found it to be empty", i+1, instanceType) + } + + if !testCase.resultL.IsTruncated && resultL.NextMarker != "" { + if !resultL.IsTruncated || len(resultL.Objects) == 0 { + t.Errorf("Test %d: %s: Expected NextMarker to be empty since listing is not truncated, but instead found `%v`", i+1, instanceType, resultL.NextMarker) + } + } + } else { + if len(testCase.resultV.Objects) != len(resultV.Objects) { + t.Logf("want: %v", objInfoNames(testCase.resultV.Objects)) + t.Logf("got: %v", objInfoNames(resultV.Objects)) + t.Errorf("Test %d: %s: Expected number of object in the result to be '%d', but found '%d' objects instead", i+1, instanceType, len(testCase.resultV.Objects), len(resultV.Objects)) + } + for j := 0; j < len(testCase.resultV.Objects); j++ { + if j >= len(resultV.Objects) { + t.Errorf("Test %d: %s: Expected object name to be \"%s\", but not nothing instead", i+1, instanceType, testCase.resultV.Objects[j].Name) + continue + } + if testCase.resultV.Objects[j].Name != resultV.Objects[j].Name { + t.Errorf("Test %d: %s: Expected object name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.resultV.Objects[j].Name, resultV.Objects[j].Name) + } + } + + if len(testCase.resultV.Prefixes) != len(resultV.Prefixes) { + t.Logf("want: %v", testCase.resultV.Prefixes) + t.Logf("got: %v", resultV.Prefixes) + t.Errorf("Test %d: %s: Expected number of prefixes in the result to be '%d', but found '%d' prefixes instead", i+1, instanceType, len(testCase.resultV.Prefixes), len(resultV.Prefixes)) + } + for j := 0; j < len(testCase.resultV.Prefixes); j++ { + if j >= len(resultV.Prefixes) { + t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found no result", i+1, instanceType, testCase.resultV.Prefixes[j]) + continue + } + if testCase.resultV.Prefixes[j] != resultV.Prefixes[j] { + t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.resultV.Prefixes[j], resultV.Prefixes[j]) + } + } + + if testCase.resultV.IsTruncated != resultV.IsTruncated { + // Allow an extra continuation token. + if !resultV.IsTruncated || len(resultV.Objects) == 0 { + t.Errorf("Test %d: %s: Expected IsTruncated flag to be %v, but instead found it to be %v", i+1, instanceType, testCase.resultV.IsTruncated, resultV.IsTruncated) + } + } + + if testCase.resultV.IsTruncated && resultV.NextMarker == "" { + t.Errorf("Test %d: %s: Expected NextMarker to contain a string since listing is truncated, but instead found it to be empty", i+1, instanceType) + } + + if !testCase.resultV.IsTruncated && resultV.NextMarker != "" { + if !resultV.IsTruncated || len(resultV.Objects) == 0 { + t.Errorf("Test %d: %s: Expected NextMarker to be empty since listing is not truncated, but instead found `%v`", i+1, instanceType, resultV.NextMarker) + } + } + } + } + }) + } +} + +// Wrapper for calling ListObjectsOnVersionedBuckets tests for both +// Erasure multiple disks and single node setup. +func TestListObjectsOnVersionedBuckets(t *testing.T) { + ExecObjectLayerTest(t, testListObjectsOnVersionedBuckets) +} + +// Wrapper for calling ListObjects tests for both Erasure multiple +// disks and single node setup. +func TestListObjects(t *testing.T) { + ExecObjectLayerTest(t, testListObjects) +} + +// Unit test for ListObjects on VersionedBucket. +func testListObjectsOnVersionedBuckets(obj ObjectLayer, instanceType string, t1 TestErrHandler) { + _testListObjects(obj, instanceType, t1, true) +} + +// Unit test for ListObjects. +func testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler) { + _testListObjects(obj, instanceType, t1, false) +} + +func _testListObjects(obj ObjectLayer, instanceType string, t1 TestErrHandler, versioned bool) { + t, _ := t1.(*testing.T) + testBuckets := []string{ + // This bucket is used for testing ListObject operations. + 0: "test-bucket-list-object", + // This bucket will be tested with empty directories + 1: "test-bucket-empty-dir", + // Will not store any objects in this bucket, + // Its to test ListObjects on an empty bucket. + 2: "empty-bucket", + // Listing the case where the marker > last object. + 3: "test-bucket-single-object", + // Listing uncommon delimiter. + 4: "test-bucket-delimiter", + // Listing prefixes > maxKeys + 5: "test-bucket-max-keys-prefixes", + // Listing custom delimiters + 6: "test-bucket-custom-delimiter", + } + for _, bucket := range testBuckets { + err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{ + VersioningEnabled: versioned, + }) + if err != nil { + t.Fatalf("%s : %s", instanceType, err.Error()) + } + } + + var err error + testObjects := []struct { + parentBucket string + name string + content string + meta map[string]string + }{ + {testBuckets[0], "Asia-maps.png", "asis-maps", map[string]string{"content-type": "image/png"}}, + {testBuckets[0], "Asia/India/India-summer-photos-1", "contentstring", nil}, + {testBuckets[0], "Asia/India/Karnataka/Bangalore/Koramangala/pics", "contentstring", nil}, + {testBuckets[0], "newPrefix0", "newPrefix0", nil}, + {testBuckets[0], "newPrefix1", "newPrefix1", nil}, + {testBuckets[0], "newzen/zen/recurse/again/again/again/pics", "recurse", nil}, + {testBuckets[0], "obj0", "obj0", nil}, + {testBuckets[0], "obj1", "obj1", nil}, + {testBuckets[0], "obj2", "obj2", nil}, + {testBuckets[1], "obj1", "obj1", nil}, + {testBuckets[1], "obj2", "obj2", nil}, + {testBuckets[1], "temporary/0/", "", nil}, + {testBuckets[3], "A/B", "contentstring", nil}, + {testBuckets[4], "file1/receipt.json", "content", nil}, + {testBuckets[4], "file1/guidSplunk-aaaa/file", "content", nil}, + {testBuckets[5], "dir/day_id=2017-10-10/issue", "content", nil}, + {testBuckets[5], "dir/day_id=2017-10-11/issue", "content", nil}, + {testBuckets[5], "foo/201910/1122", "content", nil}, + {testBuckets[5], "foo/201910/1112", "content", nil}, + {testBuckets[5], "foo/201910/2112", "content", nil}, + {testBuckets[5], "foo/201910_txt", "content", nil}, + {testBuckets[5], "201910/foo/bar/xl.meta/1.txt", "content", nil}, + {testBuckets[6], "aaa", "content", nil}, + {testBuckets[6], "bbb_aaa", "content", nil}, + {testBuckets[6], "bbb_aaa", "content", nil}, + {testBuckets[6], "ccc", "content", nil}, + } + for _, object := range testObjects { + md5Bytes := md5.Sum([]byte(object.content)) + _, err = obj.PutObject(context.Background(), object.parentBucket, object.name, + mustGetPutObjReader(t, bytes.NewBufferString(object.content), + int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{ + Versioned: globalBucketVersioningSys.PrefixEnabled(object.parentBucket, object.name), + UserDefined: object.meta, + }) + if err != nil { + t.Fatalf("%s : %s", instanceType, err.Error()) + } + } + + // Formulating the result data set to be expected from ListObjects call inside the tests, + // This will be used in testCases and used for asserting the correctness of ListObjects output in the tests. + + resultCases := []ListObjectsInfo{ + // ListObjectsResult-0. + // Testing for listing all objects in the bucket, (testCase 20,21,22). + 0: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "Asia-maps.png"}, + {Name: "Asia/India/India-summer-photos-1"}, + {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, + {Name: "newPrefix0"}, + {Name: "newPrefix1"}, + {Name: "newzen/zen/recurse/again/again/again/pics"}, + {Name: "obj0"}, + {Name: "obj1"}, + {Name: "obj2"}, + }, + }, + // ListObjectsResult-1. + // Used for asserting the truncated case, (testCase 23). + 1: { + IsTruncated: true, + Objects: []ObjectInfo{ + {Name: "Asia-maps.png"}, + {Name: "Asia/India/India-summer-photos-1"}, + {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, + {Name: "newPrefix0"}, + {Name: "newPrefix1"}, + }, + }, + // ListObjectsResult-2. + // (TestCase 24). + 2: { + IsTruncated: true, + Objects: []ObjectInfo{ + {Name: "Asia-maps.png"}, + {Name: "Asia/India/India-summer-photos-1"}, + {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, + {Name: "newPrefix0"}, + }, + }, + // ListObjectsResult-3. + // (TestCase 25). + 3: { + IsTruncated: true, + Objects: []ObjectInfo{ + {Name: "Asia-maps.png"}, + {Name: "Asia/India/India-summer-photos-1"}, + {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, + }, + }, + // ListObjectsResult-4. + // Again used for truncated case. + // (TestCase 26). + 4: { + IsTruncated: true, + Objects: []ObjectInfo{ + {Name: "Asia-maps.png"}, + }, + }, + // ListObjectsResult-5. + // Used for Asserting prefixes. + // Used for test case with prefix "new", (testCase 27-29). + 5: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "newPrefix0"}, + {Name: "newPrefix1"}, + {Name: "newzen/zen/recurse/again/again/again/pics"}, + }, + }, + // ListObjectsResult-6. + // Used for Asserting prefixes. + // Used for test case with prefix = "obj", (testCase 30). + 6: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "obj0"}, + {Name: "obj1"}, + {Name: "obj2"}, + }, + }, + // ListObjectsResult-7. + // Used for Asserting prefixes and truncation. + // Used for test case with prefix = "new" and maxKeys = 1, (testCase 31). + 7: { + IsTruncated: true, + Objects: []ObjectInfo{ + {Name: "newPrefix0"}, + }, + }, + // ListObjectsResult-8. + // Used for Asserting prefixes. + // Used for test case with prefix = "obj" and maxKeys = 2, (testCase 32). + 8: { + IsTruncated: true, + Objects: []ObjectInfo{ + {Name: "obj0"}, + {Name: "obj1"}, + }, + }, + // ListObjectsResult-9. + // Used for asserting the case with marker, but without prefix. + // marker is set to "newPrefix0" in the testCase, (testCase 33). + 9: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "newPrefix1"}, + {Name: "newzen/zen/recurse/again/again/again/pics"}, + {Name: "obj0"}, + {Name: "obj1"}, + {Name: "obj2"}, + }, + }, + // ListObjectsResult-10. + // marker is set to "newPrefix1" in the testCase, (testCase 34). + 10: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "newzen/zen/recurse/again/again/again/pics"}, + {Name: "obj0"}, + {Name: "obj1"}, + {Name: "obj2"}, + }, + }, + // ListObjectsResult-11. + // marker is set to "obj0" in the testCase, (testCase 35). + 11: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "obj1"}, + {Name: "obj2"}, + }, + }, + // ListObjectsResult-12. + // Marker is set to "obj1" in the testCase, (testCase 36). + 12: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "obj2"}, + }, + }, + // ListObjectsResult-13. + // Marker is set to "man" in the testCase, (testCase37). + 13: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "newPrefix0"}, + {Name: "newPrefix1"}, + {Name: "newzen/zen/recurse/again/again/again/pics"}, + {Name: "obj0"}, + {Name: "obj1"}, + {Name: "obj2"}, + }, + }, + // ListObjectsResult-14. + // Marker is set to "Abc" in the testCase, (testCase 39). + 14: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "Asia-maps.png"}, + {Name: "Asia/India/India-summer-photos-1"}, + {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, + {Name: "newPrefix0"}, + {Name: "newPrefix1"}, + {Name: "newzen/zen/recurse/again/again/again/pics"}, + {Name: "obj0"}, + {Name: "obj1"}, + {Name: "obj2"}, + }, + }, + // ListObjectsResult-15. + // Marker is set to "Asia/India/India-summer-photos-1" in the testCase, (testCase 40). + 15: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, + {Name: "newPrefix0"}, + {Name: "newPrefix1"}, + {Name: "newzen/zen/recurse/again/again/again/pics"}, + {Name: "obj0"}, + {Name: "obj1"}, + {Name: "obj2"}, + }, + }, + // ListObjectsResult-16. + // Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase, (testCase 41). + 16: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "newPrefix0"}, + {Name: "newPrefix1"}, + {Name: "newzen/zen/recurse/again/again/again/pics"}, + {Name: "obj0"}, + {Name: "obj1"}, + {Name: "obj2"}, + }, + }, + // ListObjectsResult-17. + // Used for asserting the case with marker, without prefix but with truncation. + // Marker = "newPrefix0" & maxKeys = 3 in the testCase, (testCase42). + // Output truncated to 3 values. + 17: { + IsTruncated: true, + Objects: []ObjectInfo{ + {Name: "newPrefix1"}, + {Name: "newzen/zen/recurse/again/again/again/pics"}, + {Name: "obj0"}, + }, + }, + // ListObjectsResult-18. + // Marker = "newPrefix1" & maxkeys = 1 in the testCase, (testCase43). + // Output truncated to 1 value. + 18: { + IsTruncated: true, + Objects: []ObjectInfo{ + {Name: "newzen/zen/recurse/again/again/again/pics"}, + }, + }, + // ListObjectsResult-19. + // Marker = "obj0" & maxKeys = 1 in the testCase, (testCase44). + // Output truncated to 1 value. + 19: { + IsTruncated: true, + Objects: []ObjectInfo{ + {Name: "obj1"}, + }, + }, + // ListObjectsResult-20. + // Marker = "obj0" & prefix = "obj" in the testCase, (testCase 45). + 20: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "obj1"}, + {Name: "obj2"}, + }, + }, + // ListObjectsResult-21. + // Marker = "obj1" & prefix = "obj" in the testCase, (testCase 46). + 21: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "obj2"}, + }, + }, + // ListObjectsResult-22. + // Marker = "newPrefix0" & prefix = "new" in the testCase,, (testCase 47). + 22: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "newPrefix1"}, + {Name: "newzen/zen/recurse/again/again/again/pics"}, + }, + }, + // ListObjectsResult-23. + // Prefix is set to "Asia/India/" in the testCase, and delimiter is not set (testCase 55). + 23: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "Asia/India/India-summer-photos-1"}, + {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, + }, + }, + + // ListObjectsResult-24. + // Prefix is set to "Asia" in the testCase, and delimiter is not set (testCase 56). + 24: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "Asia-maps.png"}, + {Name: "Asia/India/India-summer-photos-1"}, + {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, + }, + }, + + // ListObjectsResult-25. + // Prefix is set to "Asia" in the testCase, and delimiter is set (testCase 57). + 25: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "Asia-maps.png"}, + }, + Prefixes: []string{"Asia/"}, + }, + // ListObjectsResult-26. + // prefix = "new" and delimiter is set in the testCase.(testCase 58). + 26: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "newPrefix0"}, + {Name: "newPrefix1"}, + }, + Prefixes: []string{"newzen/"}, + }, + // ListObjectsResult-27. + // Prefix is set to "Asia/India/" in the testCase, and delimiter is set to forward slash '/' (testCase 59). + 27: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "Asia/India/India-summer-photos-1"}, + }, + Prefixes: []string{"Asia/India/Karnataka/"}, + }, + // ListObjectsResult-28. + // Marker is set to "Asia/India/India-summer-photos-1" and delimiter set in the testCase, (testCase 60). + 28: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "newPrefix0"}, + {Name: "newPrefix1"}, + {Name: "obj0"}, + {Name: "obj1"}, + {Name: "obj2"}, + }, + Prefixes: []string{"newzen/"}, + }, + // ListObjectsResult-29. + // Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase and delimiter set, (testCase 61). + 29: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "newPrefix0"}, + {Name: "newPrefix1"}, + {Name: "obj0"}, + {Name: "obj1"}, + {Name: "obj2"}, + }, + Prefixes: []string{"newzen/"}, + }, + // ListObjectsResult-30. + // Prefix and Delimiter is set to '/', (testCase 62). + 30: { + IsTruncated: false, + Objects: []ObjectInfo{}, + }, + // ListObjectsResult-31 Empty directory, recursive listing + 31: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "obj1"}, + {Name: "obj2"}, + {Name: "temporary/0/"}, + }, + }, + // ListObjectsResult-32 Empty directory, non recursive listing + 32: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "obj1"}, + {Name: "obj2"}, + }, + Prefixes: []string{"temporary/"}, + }, + // ListObjectsResult-33 Listing empty directory only + 33: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "temporary/0/"}, + }, + }, + // ListObjectsResult-34: + // * Listing with marker > last object should return empty + // * Listing an object with a trailing slash and '/' delimiter + 34: { + IsTruncated: false, + Objects: []ObjectInfo{}, + }, + // ListObjectsResult-35 list with custom uncommon delimiter + 35: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "file1/receipt.json"}, + }, + Prefixes: []string{"file1/guidSplunk"}, + }, + // ListObjectsResult-36 list with nextmarker prefix and maxKeys set to 1. + 36: { + IsTruncated: true, + Prefixes: []string{"dir/day_id=2017-10-10/"}, + }, + // ListObjectsResult-37 list with prefix match 2 levels deep + 37: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "foo/201910/1112"}, + {Name: "foo/201910/1122"}, + }, + }, + // ListObjectsResult-38 list with prefix match 1 level deep + 38: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "foo/201910/1112"}, + {Name: "foo/201910/1122"}, + {Name: "foo/201910/2112"}, + {Name: "foo/201910_txt"}, + }, + }, + // ListObjectsResult-39 list with prefix match 1 level deep + 39: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "201910/foo/bar/xl.meta/1.txt"}, + }, + }, + // ListObjectsResult-40 + 40: { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "aaa"}, + {Name: "ccc"}, + }, + Prefixes: []string{"bbb_"}, + }, + } + + testCases := []struct { + // Inputs to ListObjects. + bucketName string + prefix string + marker string + delimiter string + maxKeys int32 + // Expected output of ListObjects. + result ListObjectsInfo + err error + // Flag indicating whether the test is expected to pass or not. + shouldPass bool + }{ + // Test cases with invalid bucket names ( Test number 1-4 ). + {".test", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: ".test"}, false}, + {"Test", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "Test"}, false}, + {"---", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "---"}, false}, + {"ad", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "ad"}, false}, + // Valid bucket names, but they do not exist (6-8). + {"volatile-bucket-1", "", "", "", 1000, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false}, + {"volatile-bucket-2", "", "", "", 1000, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false}, + {"volatile-bucket-3", "", "", "", 1000, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false}, + // If marker is *after* the last possible object from the prefix it should return an empty list. + {"test-bucket-list-object", "Asia", "europe-object", "", 0, ListObjectsInfo{}, nil, true}, + // If the marker is *before* the first possible object from the prefix it should return the first object. + {"test-bucket-list-object", "Asia", "A", "", 1, resultCases[4], nil, true}, + // Setting a non-existing directory to be prefix (12-13). + {"empty-bucket", "europe/france/", "", "", 1, ListObjectsInfo{}, nil, true}, + {"empty-bucket", "africa/tunisia/", "", "", 1, ListObjectsInfo{}, nil, true}, + // Testing on empty bucket, that is, bucket without any objects in it (14). + {"empty-bucket", "", "", "", 0, ListObjectsInfo{}, nil, true}, + // Setting maxKeys to negative value (15-16). + {"empty-bucket", "", "", "", -1, ListObjectsInfo{}, nil, true}, + {"empty-bucket", "", "", "", 1, ListObjectsInfo{}, nil, true}, + // Setting maxKeys to a very large value (17). + {"empty-bucket", "", "", "", 111100000, ListObjectsInfo{}, nil, true}, + // Testing for all 10 objects in the bucket (18). + {"test-bucket-list-object", "", "", "", 10, resultCases[0], nil, true}, + // Testing for negative value of maxKey, this should set maxKeys to listObjectsLimit (19). + {"test-bucket-list-object", "", "", "", -1, resultCases[0], nil, true}, + // Testing for very large value of maxKey, this should set maxKeys to listObjectsLimit (20). + {"test-bucket-list-object", "", "", "", 1234567890, resultCases[0], nil, true}, + // Testing for truncated value (21-24). + {"test-bucket-list-object", "", "", "", 5, resultCases[1], nil, true}, + {"test-bucket-list-object", "", "", "", 4, resultCases[2], nil, true}, + {"test-bucket-list-object", "", "", "", 3, resultCases[3], nil, true}, + {"test-bucket-list-object", "", "", "", 1, resultCases[4], nil, true}, + // Testing with prefix (25-28). + {"test-bucket-list-object", "new", "", "", 3, resultCases[5], nil, true}, + {"test-bucket-list-object", "new", "", "", 4, resultCases[5], nil, true}, + {"test-bucket-list-object", "new", "", "", 5, resultCases[5], nil, true}, + {"test-bucket-list-object", "obj", "", "", 3, resultCases[6], nil, true}, + {"test-bucket-list-object", "/obj", "", "", 0, ListObjectsInfo{}, nil, true}, + // Testing with prefix and truncation (29-30). + {"test-bucket-list-object", "new", "", "", 1, resultCases[7], nil, true}, + {"test-bucket-list-object", "obj", "", "", 2, resultCases[8], nil, true}, + // Testing with marker, but without prefix and truncation (31-35). + {"test-bucket-list-object", "", "newPrefix0", "", 6, resultCases[9], nil, true}, + {"test-bucket-list-object", "", "newPrefix1", "", 5, resultCases[10], nil, true}, + {"test-bucket-list-object", "", "obj0", "", 4, resultCases[11], nil, true}, + {"test-bucket-list-object", "", "obj1", "", 2, resultCases[12], nil, true}, + {"test-bucket-list-object", "", "man", "", 11, resultCases[13], nil, true}, + // Marker being set to a value which is greater than and all object names when sorted (36). + // Expected to send an empty response in this case. + {"test-bucket-list-object", "", "zen", "", 10, ListObjectsInfo{}, nil, true}, + // Marker being set to a value which is lesser than and all object names when sorted (37). + // Expected to send all the objects in the bucket in this case. + {"test-bucket-list-object", "", "Abc", "", 10, resultCases[14], nil, true}, + // Marker is to a hierarchical value (38-39). + {"test-bucket-list-object", "", "Asia/India/India-summer-photos-1", "", 10, resultCases[15], nil, true}, + {"test-bucket-list-object", "", "Asia/India/Karnataka/Bangalore/Koramangala/pics", "", 10, resultCases[16], nil, true}, + // Testing with marker and truncation, but no prefix (40-42). + {"test-bucket-list-object", "", "newPrefix0", "", 3, resultCases[17], nil, true}, + {"test-bucket-list-object", "", "newPrefix1", "", 1, resultCases[18], nil, true}, + {"test-bucket-list-object", "", "obj0", "", 1, resultCases[19], nil, true}, + // Testing with both marker and prefix, but without truncation (43-45). + // The valid combination of marker and prefix should satisfy strings.HasPrefix(marker, prefix). + {"test-bucket-list-object", "obj", "obj0", "", 2, resultCases[20], nil, true}, + {"test-bucket-list-object", "obj", "obj1", "", 1, resultCases[21], nil, true}, + {"test-bucket-list-object", "new", "newPrefix0", "", 2, resultCases[22], nil, true}, + // Testing with maxKeys set to 0 (46-52). + // The parameters have to valid. + {"test-bucket-list-object", "", "obj1", "", 0, ListObjectsInfo{}, nil, true}, + {"test-bucket-list-object", "", "obj0", "", 0, ListObjectsInfo{}, nil, true}, + {"test-bucket-list-object", "new", "", "", 0, ListObjectsInfo{}, nil, true}, + {"test-bucket-list-object", "obj", "", "", 0, ListObjectsInfo{}, nil, true}, + {"test-bucket-list-object", "obj", "obj0", "", 0, ListObjectsInfo{}, nil, true}, + {"test-bucket-list-object", "obj", "obj1", "", 0, ListObjectsInfo{}, nil, true}, + {"test-bucket-list-object", "new", "newPrefix0", "", 0, ListObjectsInfo{}, nil, true}, + // Tests on hierarchical key names as prefix. + // Without delimteter the code should recurse into the prefix Dir. + // Tests with prefix, but without delimiter (53-54). + {"test-bucket-list-object", "Asia/India/", "", "", 10, resultCases[23], nil, true}, + {"test-bucket-list-object", "Asia", "", "", 10, resultCases[24], nil, true}, + // Tests with prefix and delimiter (55-57). + // With delimiter the code should not recurse into the sub-directories of prefix Dir. + {"test-bucket-list-object", "Asia", "", SlashSeparator, 10, resultCases[25], nil, true}, + {"test-bucket-list-object", "new", "", SlashSeparator, 10, resultCases[26], nil, true}, + {"test-bucket-list-object", "Asia/India/", "", SlashSeparator, 10, resultCases[27], nil, true}, + // Test with marker set as hierarchical value and with delimiter. (58-59) + {"test-bucket-list-object", "", "Asia/India/India-summer-photos-1", SlashSeparator, 10, resultCases[28], nil, true}, + {"test-bucket-list-object", "", "Asia/India/Karnataka/Bangalore/Koramangala/pics", SlashSeparator, 10, resultCases[29], nil, true}, + // Test with prefix and delimiter set to '/'. (60) + {"test-bucket-list-object", SlashSeparator, "", SlashSeparator, 10, resultCases[30], nil, true}, + // Test with invalid prefix (61) + {"test-bucket-list-object", "\\", "", SlashSeparator, 10, ListObjectsInfo{}, nil, true}, + // Test listing an empty directory in recursive mode (62) + {"test-bucket-empty-dir", "", "", "", 10, resultCases[31], nil, true}, + // Test listing an empty directory in a non recursive mode (63) + {"test-bucket-empty-dir", "", "", SlashSeparator, 10, resultCases[32], nil, true}, + // Test listing a directory which contains an empty directory (64) + {"test-bucket-empty-dir", "", "temporary/", "", 10, resultCases[33], nil, true}, + // Test listing with marker > last object such that response should be empty (65) + {"test-bucket-single-object", "", "A/C", "", 1000, resultCases[34], nil, true}, + // Test listing an object with a trailing slash and a slash delimiter (66) + {"test-bucket-list-object", "Asia-maps.png/", "", "/", 1000, resultCases[34], nil, true}, + // Test listing an object with uncommon delimiter + {testBuckets[4], "", "", "guidSplunk", 1000, resultCases[35], nil, true}, + // Test listing an object with uncommon delimiter and matching prefix + {testBuckets[4], "file1/", "", "guidSplunk", 1000, resultCases[35], nil, true}, + // Test listing at prefix with expected prefix markers + {testBuckets[5], "dir/", "", SlashSeparator, 1, resultCases[36], nil, true}, + // Test listing with prefix match + {testBuckets[5], "foo/201910/11", "", "", 1000, resultCases[37], nil, true}, + {testBuckets[5], "foo/201910", "", "", 1000, resultCases[38], nil, true}, + // Test listing with prefix match with 'xl.meta' + {testBuckets[5], "201910/foo/bar", "", "", 1000, resultCases[39], nil, true}, + // Test listing with custom prefix + {testBuckets[6], "", "", "_", 1000, resultCases[40], nil, true}, + } + + for i, testCase := range testCases { + t.Run(fmt.Sprintf("%s-Test%d", instanceType, i+1), func(t *testing.T) { + t.Log("ListObjects, bucket:", testCase.bucketName, "prefix:", testCase.prefix, "marker:", testCase.marker, "delimiter:", testCase.delimiter, "maxkeys:", testCase.maxKeys) + result, err := obj.ListObjects(t.Context(), testCase.bucketName, + testCase.prefix, testCase.marker, testCase.delimiter, int(testCase.maxKeys)) + if err != nil && testCase.shouldPass { + t.Errorf("Test %d: %s: Expected to pass, but failed with: %s", i+1, instanceType, err.Error()) + } + if err == nil && !testCase.shouldPass { + t.Errorf("Test %d: %s: Expected to fail with \"%s\", but passed instead", i+1, instanceType, testCase.err.Error()) + } + // Failed as expected, but does it fail for the expected reason. + if err != nil && !testCase.shouldPass { + if !strings.Contains(err.Error(), testCase.err.Error()) { + t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.err.Error(), err.Error()) + } + } + // Since there are cases for which ListObjects fails, this is + // necessary. Test passes as expected, but the output values + // are verified for correctness here. + if err == nil && testCase.shouldPass { + // The length of the expected ListObjectsResult.Objects + // should match in both expected result from test cases + // and in the output. On failure calling t.Fatalf, + // otherwise it may lead to index out of range error in + // assertion following this. + if len(testCase.result.Objects) != len(result.Objects) { + t.Logf("want: %v", objInfoNames(testCase.result.Objects)) + t.Logf("got: %v", objInfoNames(result.Objects)) + t.Errorf("Test %d: %s: Expected number of object in the result to be '%d', but found '%d' objects instead", i+1, instanceType, len(testCase.result.Objects), len(result.Objects)) + } + for j := 0; j < len(testCase.result.Objects); j++ { + if j >= len(result.Objects) { + t.Errorf("Test %d: %s: Expected object name to be \"%s\", but not nothing instead", i+1, instanceType, testCase.result.Objects[j].Name) + continue + } + if testCase.result.Objects[j].Name != result.Objects[j].Name { + t.Errorf("Test %d: %s: Expected object name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Objects[j].Name, result.Objects[j].Name) + } + } + + if len(testCase.result.Prefixes) != len(result.Prefixes) { + t.Logf("want: %v", testCase.result.Prefixes) + t.Logf("got: %v", result.Prefixes) + t.Errorf("Test %d: %s: Expected number of prefixes in the result to be '%d', but found '%d' prefixes instead", i+1, instanceType, len(testCase.result.Prefixes), len(result.Prefixes)) + } + for j := 0; j < len(testCase.result.Prefixes); j++ { + if j >= len(result.Prefixes) { + t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found no result", i+1, instanceType, testCase.result.Prefixes[j]) + continue + } + if testCase.result.Prefixes[j] != result.Prefixes[j] { + t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Prefixes[j], result.Prefixes[j]) + } + } + + if testCase.result.IsTruncated != result.IsTruncated { + // Allow an extra continuation token. + if !result.IsTruncated || len(result.Objects) == 0 { + t.Errorf("Test %d: %s: Expected IsTruncated flag to be %v, but instead found it to be %v", i+1, instanceType, testCase.result.IsTruncated, result.IsTruncated) + } + } + + if testCase.result.IsTruncated && result.NextMarker == "" { + t.Errorf("Test %d: %s: Expected NextMarker to contain a string since listing is truncated, but instead found it to be empty", i+1, instanceType) + } + + if !testCase.result.IsTruncated && result.NextMarker != "" { + if !result.IsTruncated || len(result.Objects) == 0 { + t.Errorf("Test %d: %s: Expected NextMarker to be empty since listing is not truncated, but instead found `%v`", i+1, instanceType, result.NextMarker) + } + } + } + }) + } +} + +func objInfoNames(o []ObjectInfo) []string { + res := make([]string, len(o)) + for i := range o { + res[i] = o[i].Name + } + return res +} + +func TestDeleteObjectVersionMarker(t *testing.T) { + ExecObjectLayerTest(t, testDeleteObjectVersion) +} + +func testDeleteObjectVersion(obj ObjectLayer, instanceType string, t1 TestErrHandler) { + t, _ := t1.(*testing.T) + + testBuckets := []string{ + "bucket-suspended-version", + "bucket-suspended-version-id", + } + for _, bucket := range testBuckets { + err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{ + VersioningEnabled: true, + }) + if err != nil { + t.Fatalf("%s : %s", instanceType, err) + } + meta, err := loadBucketMetadata(context.Background(), obj, bucket) + if err != nil { + t.Fatalf("%s : %s", instanceType, err) + } + meta.VersioningConfigXML = []byte(`Suspended`) + if err := meta.Save(context.Background(), obj); err != nil { + t.Fatalf("%s : %s", instanceType, err) + } + globalBucketMetadataSys.Set(bucket, meta) + globalNotificationSys.LoadBucketMetadata(context.Background(), bucket) + } + + testObjects := []struct { + parentBucket string + name string + content string + meta map[string]string + versionID string + expectDelMarker bool + }{ + {testBuckets[0], "delete-file", "contentstring", nil, "", true}, + {testBuckets[1], "delete-file", "contentstring", nil, "null", false}, + } + for _, object := range testObjects { + md5Bytes := md5.Sum([]byte(object.content)) + _, err := obj.PutObject(context.Background(), object.parentBucket, object.name, + mustGetPutObjReader(t, bytes.NewBufferString(object.content), + int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{ + Versioned: globalBucketVersioningSys.PrefixEnabled(object.parentBucket, object.name), + VersionSuspended: globalBucketVersioningSys.PrefixSuspended(object.parentBucket, object.name), + UserDefined: object.meta, + }) + if err != nil { + t.Fatalf("%s : %s", instanceType, err) + } + obj, err := obj.DeleteObject(context.Background(), object.parentBucket, object.name, ObjectOptions{ + Versioned: globalBucketVersioningSys.PrefixEnabled(object.parentBucket, object.name), + VersionSuspended: globalBucketVersioningSys.PrefixSuspended(object.parentBucket, object.name), + VersionID: object.versionID, + }) + if err != nil { + if object.versionID != "" { + if !isErrVersionNotFound(err) { + t.Fatalf("%s : %s", instanceType, err) + } + } else { + if !isErrObjectNotFound(err) { + t.Fatalf("%s : %s", instanceType, err) + } + } + } + if obj.DeleteMarker != object.expectDelMarker { + t.Fatalf("%s : expected deleted marker %t, found %t", instanceType, object.expectDelMarker, obj.DeleteMarker) + } + } +} + +// Wrapper for calling ListObjectVersions tests for both Erasure multiple disks and single node setup. +func TestListObjectVersions(t *testing.T) { + ExecObjectLayerTest(t, testListObjectVersions) +} + +// Unit test for ListObjectVersions +func testListObjectVersions(obj ObjectLayer, instanceType string, t1 TestErrHandler) { + t, _ := t1.(*testing.T) + testBuckets := []string{ + // This bucket is used for testing ListObject operations. + "test-bucket-list-object", + // This bucket will be tested with empty directories + "test-bucket-empty-dir", + // Will not store any objects in this bucket, + // Its to test ListObjects on an empty bucket. + "empty-bucket", + // Listing the case where the marker > last object. + "test-bucket-single-object", + // Listing uncommon delimiter. + "test-bucket-delimiter", + // Listing prefixes > maxKeys + "test-bucket-max-keys-prefixes", + } + for _, bucket := range testBuckets { + err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{VersioningEnabled: true}) + if err != nil { + t.Fatalf("%s : %s", instanceType, err.Error()) + } + } + + var err error + testObjects := []struct { + parentBucket string + name string + content string + meta map[string]string + }{ + {testBuckets[0], "Asia-maps.png", "asis-maps", map[string]string{"content-type": "image/png"}}, + {testBuckets[0], "Asia/India/India-summer-photos-1", "contentstring", nil}, + {testBuckets[0], "Asia/India/Karnataka/Bangalore/Koramangala/pics", "contentstring", nil}, + {testBuckets[0], "newPrefix0", "newPrefix0", nil}, + {testBuckets[0], "newPrefix1", "newPrefix1", nil}, + {testBuckets[0], "newzen/zen/recurse/again/again/again/pics", "recurse", nil}, + {testBuckets[0], "obj0", "obj0", nil}, + {testBuckets[0], "obj1", "obj1", nil}, + {testBuckets[0], "obj2", "obj2", nil}, + {testBuckets[1], "obj1", "obj1", nil}, + {testBuckets[1], "obj2", "obj2", nil}, + {testBuckets[1], "temporary/0/", "", nil}, + {testBuckets[3], "A/B", "contentstring", nil}, + {testBuckets[4], "file1/receipt.json", "content", nil}, + {testBuckets[4], "file1/guidSplunk-aaaa/file", "content", nil}, + {testBuckets[5], "dir/day_id=2017-10-10/issue", "content", nil}, + {testBuckets[5], "dir/day_id=2017-10-11/issue", "content", nil}, + } + + for _, object := range testObjects { + md5Bytes := md5.Sum([]byte(object.content)) + _, err = obj.PutObject(context.Background(), object.parentBucket, object.name, mustGetPutObjReader(t, bytes.NewBufferString(object.content), + int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{UserDefined: object.meta}) + if err != nil { + t.Fatalf("%s : %s", instanceType, err.Error()) + } + } + + // Formulating the result data set to be expected from ListObjects call inside the tests, + // This will be used in testCases and used for asserting the correctness of ListObjects output in the tests. + + resultCases := []ListObjectsInfo{ + // ListObjectsResult-0. + // Testing for listing all objects in the bucket, (testCase 20,21,22). + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "Asia-maps.png"}, + {Name: "Asia/India/India-summer-photos-1"}, + {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, + {Name: "newPrefix0"}, + {Name: "newPrefix1"}, + {Name: "newzen/zen/recurse/again/again/again/pics"}, + {Name: "obj0"}, + {Name: "obj1"}, + {Name: "obj2"}, + }, + }, + // ListObjectsResult-1. + // Used for asserting the truncated case, (testCase 23). + { + IsTruncated: true, + Objects: []ObjectInfo{ + {Name: "Asia-maps.png"}, + {Name: "Asia/India/India-summer-photos-1"}, + {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, + {Name: "newPrefix0"}, + {Name: "newPrefix1"}, + }, + }, + // ListObjectsResult-2. + // (TestCase 24). + { + IsTruncated: true, + Objects: []ObjectInfo{ + {Name: "Asia-maps.png"}, + {Name: "Asia/India/India-summer-photos-1"}, + {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, + {Name: "newPrefix0"}, + }, + }, + // ListObjectsResult-3. + // (TestCase 25). + { + IsTruncated: true, + Objects: []ObjectInfo{ + {Name: "Asia-maps.png"}, + {Name: "Asia/India/India-summer-photos-1"}, + {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, + }, + }, + // ListObjectsResult-4. + // Again used for truncated case. + // (TestCase 26). + { + IsTruncated: true, + Objects: []ObjectInfo{ + {Name: "Asia-maps.png"}, + }, + }, + // ListObjectsResult-5. + // Used for Asserting prefixes. + // Used for test case with prefix "new", (testCase 27-29). + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "newPrefix0"}, + {Name: "newPrefix1"}, + {Name: "newzen/zen/recurse/again/again/again/pics"}, + }, + }, + // ListObjectsResult-6. + // Used for Asserting prefixes. + // Used for test case with prefix = "obj", (testCase 30). + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "obj0"}, + {Name: "obj1"}, + {Name: "obj2"}, + }, + }, + // ListObjectsResult-7. + // Used for Asserting prefixes and truncation. + // Used for test case with prefix = "new" and maxKeys = 1, (testCase 31). + { + IsTruncated: true, + Objects: []ObjectInfo{ + {Name: "newPrefix0"}, + }, + }, + // ListObjectsResult-8. + // Used for Asserting prefixes. + // Used for test case with prefix = "obj" and maxKeys = 2, (testCase 32). + { + IsTruncated: true, + Objects: []ObjectInfo{ + {Name: "obj0"}, + {Name: "obj1"}, + }, + }, + // ListObjectsResult-9. + // Used for asserting the case with marker, but without prefix. + // marker is set to "newPrefix0" in the testCase, (testCase 33). + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "newPrefix1"}, + {Name: "newzen/zen/recurse/again/again/again/pics"}, + {Name: "obj0"}, + {Name: "obj1"}, + {Name: "obj2"}, + }, + }, + // ListObjectsResult-10. + // marker is set to "newPrefix1" in the testCase, (testCase 34). + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "newzen/zen/recurse/again/again/again/pics"}, + {Name: "obj0"}, + {Name: "obj1"}, + {Name: "obj2"}, + }, + }, + // ListObjectsResult-11. + // marker is set to "obj0" in the testCase, (testCase 35). + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "obj1"}, + {Name: "obj2"}, + }, + }, + // ListObjectsResult-12. + // Marker is set to "obj1" in the testCase, (testCase 36). + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "obj2"}, + }, + }, + // ListObjectsResult-13. + // Marker is set to "man" in the testCase, (testCase37). + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "newPrefix0"}, + {Name: "newPrefix1"}, + {Name: "newzen/zen/recurse/again/again/again/pics"}, + {Name: "obj0"}, + {Name: "obj1"}, + {Name: "obj2"}, + }, + }, + // ListObjectsResult-14. + // Marker is set to "Abc" in the testCase, (testCase 39). + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "Asia-maps.png"}, + {Name: "Asia/India/India-summer-photos-1"}, + {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, + {Name: "newPrefix0"}, + {Name: "newPrefix1"}, + {Name: "newzen/zen/recurse/again/again/again/pics"}, + {Name: "obj0"}, + {Name: "obj1"}, + {Name: "obj2"}, + }, + }, + // ListObjectsResult-15. + // Marker is set to "Asia/India/India-summer-photos-1" in the testCase, (testCase 40). + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, + {Name: "newPrefix0"}, + {Name: "newPrefix1"}, + {Name: "newzen/zen/recurse/again/again/again/pics"}, + {Name: "obj0"}, + {Name: "obj1"}, + {Name: "obj2"}, + }, + }, + // ListObjectsResult-16. + // Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase, (testCase 41). + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "newPrefix0"}, + {Name: "newPrefix1"}, + {Name: "newzen/zen/recurse/again/again/again/pics"}, + {Name: "obj0"}, + {Name: "obj1"}, + {Name: "obj2"}, + }, + }, + // ListObjectsResult-17. + // Used for asserting the case with marker, without prefix but with truncation. + // Marker = "newPrefix0" & maxKeys = 3 in the testCase, (testCase42). + // Output truncated to 3 values. + { + IsTruncated: true, + Objects: []ObjectInfo{ + {Name: "newPrefix1"}, + {Name: "newzen/zen/recurse/again/again/again/pics"}, + {Name: "obj0"}, + }, + }, + // ListObjectsResult-18. + // Marker = "newPrefix1" & maxkeys = 1 in the testCase, (testCase43). + // Output truncated to 1 value. + { + IsTruncated: true, + Objects: []ObjectInfo{ + {Name: "newzen/zen/recurse/again/again/again/pics"}, + }, + }, + // ListObjectsResult-19. + // Marker = "obj0" & maxKeys = 1 in the testCase, (testCase44). + // Output truncated to 1 value. + { + IsTruncated: true, + Objects: []ObjectInfo{ + {Name: "obj1"}, + }, + }, + // ListObjectsResult-20. + // Marker = "obj0" & prefix = "obj" in the testCase, (testCase 45). + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "obj1"}, + {Name: "obj2"}, + }, + }, + // ListObjectsResult-21. + // Marker = "obj1" & prefix = "obj" in the testCase, (testCase 46). + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "obj2"}, + }, + }, + // ListObjectsResult-22. + // Marker = "newPrefix0" & prefix = "new" in the testCase,, (testCase 47). + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "newPrefix1"}, + {Name: "newzen/zen/recurse/again/again/again/pics"}, + }, + }, + // ListObjectsResult-23. + // Prefix is set to "Asia/India/" in the testCase, and delimiter is not set (testCase 55). + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "Asia/India/India-summer-photos-1"}, + {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, + }, + }, + + // ListObjectsResult-24. + // Prefix is set to "Asia" in the testCase, and delimiter is not set (testCase 56). + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "Asia-maps.png"}, + {Name: "Asia/India/India-summer-photos-1"}, + {Name: "Asia/India/Karnataka/Bangalore/Koramangala/pics"}, + }, + }, + + // ListObjectsResult-25. + // Prefix is set to "Asia" in the testCase, and delimiter is set (testCase 57). + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "Asia-maps.png"}, + }, + Prefixes: []string{"Asia/"}, + }, + // ListObjectsResult-26. + // prefix = "new" and delimiter is set in the testCase.(testCase 58). + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "newPrefix0"}, + {Name: "newPrefix1"}, + }, + Prefixes: []string{"newzen/"}, + }, + // ListObjectsResult-27. + // Prefix is set to "Asia/India/" in the testCase, and delimiter is set to forward slash '/' (testCase 59). + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "Asia/India/India-summer-photos-1"}, + }, + Prefixes: []string{"Asia/India/Karnataka/"}, + }, + // ListObjectsResult-28. + // Marker is set to "Asia/India/India-summer-photos-1" and delimiter set in the testCase, (testCase 60). + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "newPrefix0"}, + {Name: "newPrefix1"}, + {Name: "obj0"}, + {Name: "obj1"}, + {Name: "obj2"}, + }, + Prefixes: []string{"newzen/"}, + }, + // ListObjectsResult-29. + // Marker is set to "Asia/India/Karnataka/Bangalore/Koramangala/pics" in the testCase and delimiter set, (testCase 61). + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "newPrefix0"}, + {Name: "newPrefix1"}, + {Name: "obj0"}, + {Name: "obj1"}, + {Name: "obj2"}, + }, + Prefixes: []string{"newzen/"}, + }, + // ListObjectsResult-30. + // Prefix and Delimiter is set to '/', (testCase 62). + { + IsTruncated: false, + Objects: []ObjectInfo{}, + }, + // ListObjectsResult-31 Empty directory, recursive listing + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "obj1"}, + {Name: "obj2"}, + {Name: "temporary/0/"}, + }, + }, + // ListObjectsResult-32 Empty directory, non recursive listing + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "obj1"}, + {Name: "obj2"}, + }, + Prefixes: []string{"temporary/"}, + }, + // ListObjectsResult-33 Listing empty directory only + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "temporary/0/"}, + }, + }, + // ListObjectsResult-34: + // * Listing with marker > last object should return empty + // * Listing an object with a trailing slash and '/' delimiter + { + IsTruncated: false, + Objects: []ObjectInfo{}, + }, + // ListObjectsResult-35 list with custom uncommon delimiter + { + IsTruncated: false, + Objects: []ObjectInfo{ + {Name: "file1/receipt.json"}, + }, + Prefixes: []string{"file1/guidSplunk"}, + }, + // ListObjectsResult-36 list with nextmarker prefix and maxKeys set to 1. + { + IsTruncated: true, + Prefixes: []string{"dir/day_id=2017-10-10/"}, + }, + } + + testCases := []struct { + // Inputs to ListObjects. + bucketName string + prefix string + marker string + delimiter string + maxKeys int32 + // Expected output of ListObjects. + result ListObjectsInfo + err error + // Flag indicating whether the test is expected to pass or not. + shouldPass bool + }{ + // Test cases with invalid bucket names ( Test number 1-4). + {".test", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: ".test"}, false}, + {"Test", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "Test"}, false}, + {"---", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "---"}, false}, + {"ad", "", "", "", 0, ListObjectsInfo{}, BucketNameInvalid{Bucket: "ad"}, false}, + // Valid bucket names, but they do not exist (6-8). + {"volatile-bucket-1", "", "", "", 1000, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false}, + {"volatile-bucket-2", "", "", "", 1000, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false}, + {"volatile-bucket-3", "", "", "", 1000, ListObjectsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false}, + // If marker is *after* the last possible object from the prefix it should return an empty list. + {"test-bucket-list-object", "Asia", "europe-object", "", 0, ListObjectsInfo{}, nil, true}, + // Setting a non-existing directory to be prefix (10-11). + {"empty-bucket", "europe/france/", "", "", 1, ListObjectsInfo{}, nil, true}, + {"empty-bucket", "africa/tunisia/", "", "", 1, ListObjectsInfo{}, nil, true}, + // Testing on empty bucket, that is, bucket without any objects in it (12). + {"empty-bucket", "", "", "", 0, ListObjectsInfo{}, nil, true}, + // Setting maxKeys to negative value (13-14). + {"empty-bucket", "", "", "", -1, ListObjectsInfo{}, nil, true}, + {"empty-bucket", "", "", "", 1, ListObjectsInfo{}, nil, true}, + // Setting maxKeys to a very large value (15). + {"empty-bucket", "", "", "", 111100000, ListObjectsInfo{}, nil, true}, + // Testing for all 10 objects in the bucket (16). + {"test-bucket-list-object", "", "", "", 10, resultCases[0], nil, true}, + // Testing for negative value of maxKey, this should set maxKeys to listObjectsLimit (17). + {"test-bucket-list-object", "", "", "", -1, resultCases[0], nil, true}, + // Testing for very large value of maxKey, this should set maxKeys to listObjectsLimit (18). + {"test-bucket-list-object", "", "", "", 1234567890, resultCases[0], nil, true}, + // Testing for truncated value (19-22). + {"test-bucket-list-object", "", "", "", 5, resultCases[1], nil, true}, + {"test-bucket-list-object", "", "", "", 4, resultCases[2], nil, true}, + {"test-bucket-list-object", "", "", "", 3, resultCases[3], nil, true}, + {"test-bucket-list-object", "", "", "", 1, resultCases[4], nil, true}, + // Testing with prefix (23-26). + {"test-bucket-list-object", "new", "", "", 3, resultCases[5], nil, true}, + {"test-bucket-list-object", "new", "", "", 4, resultCases[5], nil, true}, + {"test-bucket-list-object", "new", "", "", 5, resultCases[5], nil, true}, + {"test-bucket-list-object", "obj", "", "", 3, resultCases[6], nil, true}, + // Testing with prefix and truncation (27-28). + {"test-bucket-list-object", "new", "", "", 1, resultCases[7], nil, true}, + {"test-bucket-list-object", "obj", "", "", 2, resultCases[8], nil, true}, + // Testing with marker, but without prefix and truncation (29-33). + {"test-bucket-list-object", "", "newPrefix0", "", 6, resultCases[9], nil, true}, + {"test-bucket-list-object", "", "newPrefix1", "", 5, resultCases[10], nil, true}, + {"test-bucket-list-object", "", "obj0", "", 4, resultCases[11], nil, true}, + {"test-bucket-list-object", "", "obj1", "", 2, resultCases[12], nil, true}, + {"test-bucket-list-object", "", "man", "", 11, resultCases[13], nil, true}, + // Marker being set to a value which is greater than and all object names when sorted (34). + // Expected to send an empty response in this case. + {"test-bucket-list-object", "", "zen", "", 10, ListObjectsInfo{}, nil, true}, + // Marker being set to a value which is lesser than and all object names when sorted (35). + // Expected to send all the objects in the bucket in this case. + {"test-bucket-list-object", "", "Abc", "", 10, resultCases[14], nil, true}, + // Marker is to a hierarchical value (36-37). + {"test-bucket-list-object", "", "Asia/India/India-summer-photos-1", "", 10, resultCases[15], nil, true}, + {"test-bucket-list-object", "", "Asia/India/Karnataka/Bangalore/Koramangala/pics", "", 10, resultCases[16], nil, true}, + // Testing with marker and truncation, but no prefix (38-40). + {"test-bucket-list-object", "", "newPrefix0", "", 3, resultCases[17], nil, true}, + {"test-bucket-list-object", "", "newPrefix1", "", 1, resultCases[18], nil, true}, + {"test-bucket-list-object", "", "obj0", "", 1, resultCases[19], nil, true}, + // Testing with both marker and prefix, but without truncation (41-43). + // The valid combination of marker and prefix should satisfy strings.HasPrefix(marker, prefix). + {"test-bucket-list-object", "obj", "obj0", "", 2, resultCases[20], nil, true}, + {"test-bucket-list-object", "obj", "obj1", "", 1, resultCases[21], nil, true}, + {"test-bucket-list-object", "new", "newPrefix0", "", 2, resultCases[22], nil, true}, + // Testing with maxKeys set to 0 (44-50). + // The parameters have to valid. + {"test-bucket-list-object", "", "obj1", "", 0, ListObjectsInfo{}, nil, true}, + {"test-bucket-list-object", "", "obj0", "", 0, ListObjectsInfo{}, nil, true}, + {"test-bucket-list-object", "new", "", "", 0, ListObjectsInfo{}, nil, true}, + {"test-bucket-list-object", "obj", "", "", 0, ListObjectsInfo{}, nil, true}, + {"test-bucket-list-object", "obj", "obj0", "", 0, ListObjectsInfo{}, nil, true}, + {"test-bucket-list-object", "obj", "obj1", "", 0, ListObjectsInfo{}, nil, true}, + {"test-bucket-list-object", "new", "newPrefix0", "", 0, ListObjectsInfo{}, nil, true}, + // Tests on hierarchical key names as prefix. + // Without delimteter the code should recurse into the prefix Dir. + // Tests with prefix, but without delimiter (51-52). + {"test-bucket-list-object", "Asia/India/", "", "", 10, resultCases[23], nil, true}, + {"test-bucket-list-object", "Asia", "", "", 10, resultCases[24], nil, true}, + // Tests with prefix and delimiter (53-55). + // With delimiter the code should not recurse into the sub-directories of prefix Dir. + {"test-bucket-list-object", "Asia", "", SlashSeparator, 10, resultCases[25], nil, true}, + {"test-bucket-list-object", "new", "", SlashSeparator, 10, resultCases[26], nil, true}, + {"test-bucket-list-object", "Asia/India/", "", SlashSeparator, 10, resultCases[27], nil, true}, + // Test with marker set as hierarchical value and with delimiter. (56-57) + {"test-bucket-list-object", "", "Asia/India/India-summer-photos-1", SlashSeparator, 10, resultCases[28], nil, true}, + {"test-bucket-list-object", "", "Asia/India/Karnataka/Bangalore/Koramangala/pics", SlashSeparator, 10, resultCases[29], nil, true}, + // Test with prefix and delimiter set to '/'. (58) + {"test-bucket-list-object", SlashSeparator, "", SlashSeparator, 10, resultCases[30], nil, true}, + // Test with invalid prefix (59) + {"test-bucket-list-object", "\\", "", SlashSeparator, 10, ListObjectsInfo{}, nil, true}, + // Test listing an empty directory in recursive mode (60) + {"test-bucket-empty-dir", "", "", "", 10, resultCases[31], nil, true}, + // Test listing an empty directory in a non recursive mode (61) + {"test-bucket-empty-dir", "", "", SlashSeparator, 10, resultCases[32], nil, true}, + // Test listing a directory which contains an empty directory (62) + {"test-bucket-empty-dir", "", "temporary/", "", 10, resultCases[33], nil, true}, + // Test listing with marker > last object such that response should be empty (63) + {"test-bucket-single-object", "", "A/C", "", 1000, resultCases[34], nil, true}, + // Test listing an object with a trailing slash and a slash delimiter (64) + {"test-bucket-list-object", "Asia-maps.png/", "", "/", 1000, resultCases[34], nil, true}, + // Test listing an object with uncommon delimiter + {testBuckets[4], "", "", "guidSplunk", 1000, resultCases[35], nil, true}, + // Test listing an object with uncommon delimiter and matching prefix + {testBuckets[4], "file1/", "", "guidSplunk", 1000, resultCases[35], nil, true}, + // Test listing at prefix with expected prefix markers + {testBuckets[5], "dir/", "", SlashSeparator, 1, resultCases[36], nil, true}, + {"test-bucket-list-object", "Asia", "A", "", 1, resultCases[4], nil, true}, + } + + for i, testCase := range testCases { + t.Run(fmt.Sprintf("%s-Test%d", instanceType, i+1), func(t *testing.T) { + result, err := obj.ListObjectVersions(t.Context(), testCase.bucketName, + testCase.prefix, testCase.marker, "", testCase.delimiter, int(testCase.maxKeys)) + if err != nil && testCase.shouldPass { + t.Errorf("%s: Expected to pass, but failed with: %s", instanceType, err.Error()) + } + if err == nil && !testCase.shouldPass { + t.Errorf("%s: Expected to fail with \"%s\", but passed instead", instanceType, testCase.err.Error()) + } + // Failed as expected, but does it fail for the expected reason. + if err != nil && !testCase.shouldPass { + if !strings.Contains(err.Error(), testCase.err.Error()) { + t.Errorf("%s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", instanceType, testCase.err.Error(), err.Error()) + } + } + // Since there are cases for which ListObjects fails, this is + // necessary. Test passes as expected, but the output values + // are verified for correctness here. + if err == nil && testCase.shouldPass { + // The length of the expected ListObjectsResult.Objects + // should match in both expected result from test cases + // and in the output. On failure calling t.Fatalf, + // otherwise it may lead to index out of range error in + // assertion following this. + if len(testCase.result.Objects) != len(result.Objects) { + t.Fatalf("%s: Expected number of object in the result to be '%d', but found '%d' objects instead", instanceType, len(testCase.result.Objects), len(result.Objects)) + } + for j := 0; j < len(testCase.result.Objects); j++ { + if testCase.result.Objects[j].Name != result.Objects[j].Name { + t.Errorf("%s: Expected object name to be \"%s\", but found \"%s\" instead", instanceType, testCase.result.Objects[j].Name, result.Objects[j].Name) + } + } + + if len(testCase.result.Prefixes) != len(result.Prefixes) { + t.Log(testCase, testCase.result.Prefixes, result.Prefixes) + t.Fatalf("%s: Expected number of prefixes in the result to be '%d', but found '%d' prefixes instead", instanceType, len(testCase.result.Prefixes), len(result.Prefixes)) + } + for j := 0; j < len(testCase.result.Prefixes); j++ { + if testCase.result.Prefixes[j] != result.Prefixes[j] { + t.Errorf("%s: Expected prefix name to be \"%s\", but found \"%s\" instead", instanceType, testCase.result.Prefixes[j], result.Prefixes[j]) + } + } + + if testCase.result.IsTruncated != result.IsTruncated { + // Allow an extra continuation token. + if !result.IsTruncated || len(result.Objects) == 0 { + t.Errorf("%s: Expected IsTruncated flag to be %v, but instead found it to be %v", instanceType, testCase.result.IsTruncated, result.IsTruncated) + } + } + + if testCase.result.IsTruncated && result.NextMarker == "" { + t.Errorf("%s: Expected NextMarker to contain a string since listing is truncated, but instead found it to be empty", instanceType) + } + + if !testCase.result.IsTruncated && result.NextMarker != "" { + if !result.IsTruncated || len(result.Objects) == 0 { + t.Errorf("%s: Expected NextMarker to be empty since listing is not truncated, but instead found `%v`", instanceType, result.NextMarker) + } + } + } + }) + } +} + +// Wrapper for calling ListObjects continuation tests for both Erasure multiple disks and single node setup. +func TestListObjectsContinuation(t *testing.T) { + ExecObjectLayerTest(t, testListObjectsContinuation) +} + +// Unit test for ListObjects in general. +func testListObjectsContinuation(obj ObjectLayer, instanceType string, t1 TestErrHandler) { + t, _ := t1.(*testing.T) + testBuckets := []string{ + // This bucket is used for testing ListObject operations. + "test-bucket-list-object-continuation-1", + "test-bucket-list-object-continuation-2", + } + for _, bucket := range testBuckets { + err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{}) + if err != nil { + t.Fatalf("%s : %s", instanceType, err.Error()) + } + } + + var err error + testObjects := []struct { + parentBucket string + name string + content string + meta map[string]string + }{ + {testBuckets[0], "a/1.txt", "contentstring", nil}, + {testBuckets[0], "a-1.txt", "contentstring", nil}, + {testBuckets[0], "a.txt", "contentstring", nil}, + {testBuckets[0], "apache2-doc/1.txt", "contentstring", nil}, + {testBuckets[0], "apache2/1.txt", "contentstring", nil}, + {testBuckets[0], "apache2/-sub/2.txt", "contentstring", nil}, + {testBuckets[1], "azerty/1.txt", "contentstring", nil}, + {testBuckets[1], "apache2-doc/1.txt", "contentstring", nil}, + {testBuckets[1], "apache2/1.txt", "contentstring", nil}, + } + for _, object := range testObjects { + md5Bytes := md5.Sum([]byte(object.content)) + _, err = obj.PutObject(context.Background(), object.parentBucket, object.name, mustGetPutObjReader(t, bytes.NewBufferString(object.content), + int64(len(object.content)), hex.EncodeToString(md5Bytes[:]), ""), ObjectOptions{UserDefined: object.meta}) + if err != nil { + t.Fatalf("%s : %s", instanceType, err.Error()) + } + } + + // Formulating the result data set to be expected from ListObjects call inside the tests, + // This will be used in testCases and used for asserting the correctness of ListObjects output in the tests. + resultCases := []ListObjectsInfo{ + { + Objects: []ObjectInfo{ + {Name: "a-1.txt"}, + {Name: "a.txt"}, + {Name: "a/1.txt"}, + {Name: "apache2-doc/1.txt"}, + {Name: "apache2/-sub/2.txt"}, + {Name: "apache2/1.txt"}, + }, + }, + { + Objects: []ObjectInfo{ + {Name: "apache2-doc/1.txt"}, + {Name: "apache2/1.txt"}, + }, + }, + { + Prefixes: []string{"apache2-doc/", "apache2/", "azerty/"}, + }, + } + + testCases := []struct { + // Inputs to ListObjects. + bucketName string + prefix string + delimiter string + page int + // Expected output of ListObjects. + result ListObjectsInfo + }{ + {testBuckets[0], "", "", 1, resultCases[0]}, + {testBuckets[0], "a", "", 1, resultCases[0]}, + {testBuckets[1], "apache", "", 1, resultCases[1]}, + {testBuckets[1], "", "/", 1, resultCases[2]}, + } + + for i, testCase := range testCases { + t.Run(fmt.Sprintf("%s-Test%d", instanceType, i+1), func(t *testing.T) { + var foundObjects []ObjectInfo + var foundPrefixes []string + marker := "" + for { + result, err := obj.ListObjects(t.Context(), testCase.bucketName, + testCase.prefix, marker, testCase.delimiter, testCase.page) + if err != nil { + t.Fatalf("Test %d: %s: Expected to pass, but failed with: %s", i+1, instanceType, err.Error()) + } + foundObjects = append(foundObjects, result.Objects...) + foundPrefixes = append(foundPrefixes, result.Prefixes...) + if !result.IsTruncated { + break + } + marker = result.NextMarker + if len(result.Objects) > 0 { + // Discard marker, so it cannot resume listing. + marker = result.Objects[len(result.Objects)-1].Name + } + } + + if len(testCase.result.Objects) != len(foundObjects) { + t.Logf("want: %v", objInfoNames(testCase.result.Objects)) + t.Logf("got: %v", objInfoNames(foundObjects)) + t.Errorf("Test %d: %s: Expected number of objects in the result to be '%d', but found '%d' objects instead", + i+1, instanceType, len(testCase.result.Objects), len(foundObjects)) + } + for j := 0; j < len(testCase.result.Objects); j++ { + if j >= len(foundObjects) { + t.Errorf("Test %d: %s: Expected object name to be \"%s\", but not nothing instead", i+1, instanceType, testCase.result.Objects[j].Name) + continue + } + if testCase.result.Objects[j].Name != foundObjects[j].Name { + t.Errorf("Test %d: %s: Expected object name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Objects[j].Name, foundObjects[j].Name) + } + } + + if len(testCase.result.Prefixes) != len(foundPrefixes) { + t.Logf("want: %v", testCase.result.Prefixes) + t.Logf("got: %v", foundPrefixes) + t.Errorf("Test %d: %s: Expected number of prefixes in the result to be '%d', but found '%d' prefixes instead", + i+1, instanceType, len(testCase.result.Prefixes), len(foundPrefixes)) + } + for j := 0; j < len(testCase.result.Prefixes); j++ { + if j >= len(foundPrefixes) { + t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found no result", i+1, instanceType, testCase.result.Prefixes[j]) + continue + } + if testCase.result.Prefixes[j] != foundPrefixes[j] { + t.Errorf("Test %d: %s: Expected prefix name to be \"%s\", but found \"%s\" instead", i+1, instanceType, testCase.result.Prefixes[j], foundPrefixes[j]) + } + } + }) + } +} + +// Initialize FS backend for the benchmark. +func initFSObjectsB(disk string, t *testing.B) (obj ObjectLayer) { + obj, _, err := initObjectLayer(context.Background(), mustGetPoolEndpoints(0, disk)) + if err != nil { + t.Fatal(err) + } + + newTestConfig(globalMinioDefaultRegion, obj) + + initAllSubsystems(GlobalContext) + return obj +} + +// BenchmarkListObjects - Run ListObject Repeatedly and benchmark. +func BenchmarkListObjects(b *testing.B) { + // Make a temporary directory to use as the obj. + directory := b.TempDir() + + // Create the obj. + obj := initFSObjectsB(directory, b) + + bucket := "ls-benchmark-bucket" + // Create a bucket. + err := obj.MakeBucket(b.Context(), bucket, MakeBucketOptions{}) + if err != nil { + b.Fatal(err) + } + + // Insert objects to be listed and benchmarked later. + for i := range 20000 { + key := "obj" + strconv.Itoa(i) + _, err = obj.PutObject(b.Context(), bucket, key, mustGetPutObjReader(b, bytes.NewBufferString(key), int64(len(key)), "", ""), ObjectOptions{}) + if err != nil { + b.Fatal(err) + } + } + + // List the buckets over and over and over. + for b.Loop() { + _, err = obj.ListObjects(b.Context(), bucket, "", "obj9000", "", -1) + if err != nil { + b.Fatal(err) + } + } +} + +func TestListObjectsWithILM(t *testing.T) { + ExecObjectLayerTest(t, testListObjectsWithILM) +} + +func testListObjectsWithILM(obj ObjectLayer, instanceType string, t1 TestErrHandler) { + // Prepare lifecycle expiration workers + es := newExpiryState(t1.Context(), obj, 0) + globalExpiryState = es + + t, _ := t1.(*testing.T) + + objContent := "test-content" + objMd5 := md5.Sum([]byte(objContent)) + + uploads := []struct { + bucket string + expired int + notExpired int + }{ + {"test-list-ilm-nothing-expired", 0, 6}, + {"test-list-ilm-all-expired", 6, 0}, + {"test-list-ilm-all-half-expired", 3, 3}, + } + + oneWeekAgo := time.Now().Add(-7 * 24 * time.Hour) + + lifecycleBytes := []byte(` + + + Enabled + + 1 + + + +`) + + lifecycleConfig, err := lifecycle.ParseLifecycleConfig(bytes.NewReader(lifecycleBytes)) + if err != nil { + t.Fatal(err) + } + + for i, upload := range uploads { + err := obj.MakeBucket(context.Background(), upload.bucket, MakeBucketOptions{}) + if err != nil { + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + metadata, err := globalBucketMetadataSys.Get(upload.bucket) + if err != nil { + t.Fatal(err) + } + metadata.lifecycleConfig = lifecycleConfig + globalBucketMetadataSys.Set(upload.bucket, metadata) + defer globalBucketMetadataSys.Remove(upload.bucket) + + // Upload objects which modtime as one week ago, supposed to be expired by ILM + for range upload.expired { + _, err := obj.PutObject(context.Background(), upload.bucket, randString(32), + mustGetPutObjReader(t, + bytes.NewBufferString(objContent), + int64(len(objContent)), + hex.EncodeToString(objMd5[:]), + ""), + ObjectOptions{MTime: oneWeekAgo}, + ) + if err != nil { + t.Fatal(err) + } + } + + // Upload objects which current time as modtime, not expired by ILM + for range upload.notExpired { + _, err := obj.PutObject(context.Background(), upload.bucket, randString(32), + mustGetPutObjReader(t, + bytes.NewBufferString(objContent), + int64(len(objContent)), + hex.EncodeToString(objMd5[:]), + ""), + ObjectOptions{}, + ) + if err != nil { + t.Fatal(err) + } + } + + for _, maxKeys := range []int{1, 10, 49} { + // Test ListObjects V2 + totalObjs, didRuns := 0, 0 + marker := "" + for { + didRuns++ + if didRuns > 1000 { + t.Fatal("too many runs") + return + } + result, err := obj.ListObjectsV2(context.Background(), upload.bucket, "", marker, "", maxKeys, false, "") + if err != nil { + t.Fatalf("Test %d: %s: Expected to pass, but failed with: %s", i, instanceType, err.Error()) + } + totalObjs += len(result.Objects) + if !result.IsTruncated { + break + } + if marker != "" && marker == result.NextContinuationToken { + t.Fatalf("infinite loop marker: %s", result.NextContinuationToken) + } + marker = result.NextContinuationToken + } + + if totalObjs != upload.notExpired { + t.Fatalf("Test %d: %s: max-keys=%d, %d objects are expected to be seen, but %d found instead (didRuns=%d)", + i+1, instanceType, maxKeys, upload.notExpired, totalObjs, didRuns) + } + } + } +} diff --git a/cmd/object-api-multipart_test.go b/cmd/object-api-multipart_test.go new file mode 100644 index 00000000..d888df71 --- /dev/null +++ b/cmd/object-api-multipart_test.go @@ -0,0 +1,2238 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "fmt" + "math/rand" + "reflect" + "runtime" + "strings" + "testing" + + "github.com/dustin/go-humanize" + "github.com/minio/minio/internal/config/storageclass" + "github.com/minio/minio/internal/hash" + "github.com/minio/minio/internal/ioutil" +) + +// Wrapper for calling NewMultipartUpload tests for both Erasure multiple disks and single node setup. +func TestObjectNewMultipartUpload(t *testing.T) { + if runtime.GOOS == globalWindowsOSName { + t.Skip() + } + ExecObjectLayerTest(t, testObjectNewMultipartUpload) +} + +// Tests validate creation of new multipart upload instance. +func testObjectNewMultipartUpload(obj ObjectLayer, instanceType string, t TestErrHandler) { + bucket := "minio-bucket" + object := "minio-object" + opts := ObjectOptions{} + _, err := obj.NewMultipartUpload(context.Background(), "--", object, opts) + if err == nil { + t.Fatalf("%s: Expected to fail since bucket name is invalid.", instanceType) + } + + errMsg := "Bucket not found: minio-bucket" + // operation expected to fail since the bucket on which NewMultipartUpload is being initiated doesn't exist. + _, err = obj.NewMultipartUpload(context.Background(), bucket, object, opts) + if err == nil { + t.Fatalf("%s: Expected to fail since the NewMultipartUpload is initialized on a non-existent bucket.", instanceType) + } + if errMsg != err.Error() { + t.Errorf("%s, Expected to fail with Error \"%s\", but instead found \"%s\".", instanceType, errMsg, err.Error()) + } + + // Create bucket before initiating NewMultipartUpload. + err = obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{}) + if err != nil { + // failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + res, err := obj.NewMultipartUpload(context.Background(), bucket, "\\", opts) + if err != nil { + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + err = obj.AbortMultipartUpload(context.Background(), bucket, "\\", res.UploadID, opts) + if err != nil { + switch err.(type) { + case InvalidUploadID: + t.Fatalf("%s: New Multipart upload failed to create uuid file.", instanceType) + default: + t.Fatal(err.Error()) + } + } +} + +// Wrapper for calling AbortMultipartUpload tests for both Erasure multiple disks and single node setup. +func TestObjectAbortMultipartUpload(t *testing.T) { + ExecObjectLayerTest(t, testObjectAbortMultipartUpload) +} + +// Tests validate creation of abort multipart upload instance. +func testObjectAbortMultipartUpload(obj ObjectLayer, instanceType string, t TestErrHandler) { + bucket := "minio-bucket" + object := "minio-object" + opts := ObjectOptions{} + // Create bucket before initiating NewMultipartUpload. + err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{}) + if err != nil { + // failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + res, err := obj.NewMultipartUpload(context.Background(), bucket, object, opts) + if err != nil { + t.Fatalf("%s : %s", instanceType, err.Error()) + } + uploadID := res.UploadID + + abortTestCases := []struct { + bucketName string + objName string + uploadID string + expectedErrType error + }{ + {"--", object, uploadID, BucketNameInvalid{}}, + {"foo", object, uploadID, BucketNotFound{}}, + {bucket, object, "foo-foo", InvalidUploadID{}}, + {bucket, object, uploadID, nil}, + } + + if runtime.GOOS != globalWindowsOSName { + abortTestCases = append(abortTestCases, struct { + bucketName string + objName string + uploadID string + expectedErrType error + }{bucket, "\\", uploadID, InvalidUploadID{}}) + } + + // Iterating over creatPartCases to generate multipart chunks. + for i, testCase := range abortTestCases { + err = obj.AbortMultipartUpload(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, opts) + if testCase.expectedErrType == nil && err != nil { + t.Errorf("Test %d, unexpected err is received: %v, expected:%v\n", i+1, err, testCase.expectedErrType) + } + if testCase.expectedErrType != nil && !isSameType(err, testCase.expectedErrType) { + t.Errorf("Test %d, unexpected err is received: %v, expected:%v\n", i+1, err, testCase.expectedErrType) + } + } +} + +// Wrapper for calling isUploadIDExists tests for both Erasure multiple disks and single node setup. +func TestObjectAPIIsUploadIDExists(t *testing.T) { + ExecObjectLayerTest(t, testObjectAPIIsUploadIDExists) +} + +// Tests validates the validator for existence of uploadID. +func testObjectAPIIsUploadIDExists(obj ObjectLayer, instanceType string, t TestErrHandler) { + bucket := "minio-bucket" + object := "minio-object" + + // Create bucket before initiating NewMultipartUpload. + err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{}) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + _, err = obj.NewMultipartUpload(context.Background(), bucket, object, ObjectOptions{}) + if err != nil { + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + opts := ObjectOptions{} + err = obj.AbortMultipartUpload(context.Background(), bucket, object, "abc", opts) + switch err.(type) { + case InvalidUploadID: + default: + t.Fatalf("%s: Expected uploadIDPath to exist.", instanceType) + } +} + +// Wrapper for calling PutObjectPart tests for both Erasure multiple disks and single node setup. +func TestObjectAPIPutObjectPart(t *testing.T) { + ExecExtendedObjectLayerTest(t, testObjectAPIPutObjectPart) +} + +// Tests validate correctness of PutObjectPart. +func testObjectAPIPutObjectPart(obj ObjectLayer, instanceType string, t TestErrHandler) { + // Generating cases for which the PutObjectPart fails. + bucket := "minio-bucket" + object := "minio-object" + opts := ObjectOptions{} + // Create bucket before initiating NewMultipartUpload. + err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{}) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + // Initiate Multipart Upload on the above created bucket. + res, err := obj.NewMultipartUpload(context.Background(), bucket, object, opts) + if err != nil { + // Failed to create NewMultipartUpload, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + err = obj.MakeBucket(context.Background(), "abc", MakeBucketOptions{}) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + resN, err := obj.NewMultipartUpload(context.Background(), "abc", "def", opts) + if err != nil { + // Failed to create NewMultipartUpload, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + uploadID := res.UploadID + // Creating a dummy bucket for tests. + err = obj.MakeBucket(context.Background(), "unused-bucket", MakeBucketOptions{}) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + obj.DeleteBucket(context.Background(), "abc", DeleteBucketOptions{}) + + // Collection of non-exhaustive PutObjectPart test cases, valid errors + // and success responses. + testCases := []struct { + bucketName string + objName string + uploadID string + PartID int + inputReaderData string + inputMd5 string + inputSHA256 string + inputDataSize int64 + // flag indicating whether the test should pass. + shouldPass bool + // expected error output. + expectedMd5 string + expectedError error + }{ + // Test case 1-4. + // Cases with invalid bucket name. + {bucketName: ".test", objName: "obj", PartID: 1, expectedError: fmt.Errorf("%s", "Bucket name invalid: .test")}, + {bucketName: "------", objName: "obj", PartID: 1, expectedError: fmt.Errorf("%s", "Bucket name invalid: ------")}, + { + bucketName: "$this-is-not-valid-too", objName: "obj", PartID: 1, + expectedError: fmt.Errorf("%s", "Bucket name invalid: $this-is-not-valid-too"), + }, + {bucketName: "a", objName: "obj", PartID: 1, expectedError: fmt.Errorf("%s", "Bucket name invalid: a")}, + // Test case - 5. + // Case with invalid object names. + {bucketName: bucket, PartID: 1, expectedError: fmt.Errorf("%s", "Object name invalid: minio-bucket/")}, + // Test case - 6. + // Valid object and bucket names but non-existent bucket. + {bucketName: "abc", objName: "def", uploadID: resN.UploadID, PartID: 1, expectedError: fmt.Errorf("%s", "Bucket not found: abc")}, + // Test Case - 7. + // Existing bucket, but using a bucket on which NewMultipartUpload is not Initiated. + {bucketName: "unused-bucket", objName: "def", uploadID: "xyz", PartID: 1, expectedError: fmt.Errorf("%s", "Invalid upload id xyz")}, + // Test Case - 8. + // Existing bucket, object name different from which NewMultipartUpload is constructed from. + // Expecting "Invalid upload id". + {bucketName: bucket, objName: "def", uploadID: "xyz", PartID: 1, expectedError: fmt.Errorf("%s", "Invalid upload id xyz")}, + // Test Case - 9. + // Existing bucket, bucket and object name are the ones from which NewMultipartUpload is constructed from. + // But the uploadID is invalid. + // Expecting "Invalid upload id". + {bucketName: bucket, objName: object, uploadID: "xyz", PartID: 1, expectedError: fmt.Errorf("%s", "Invalid upload id xyz")}, + // Test Case - 10. + // Case with valid UploadID, existing bucket name. + // But using the bucket name from which NewMultipartUpload is not constructed from. + {bucketName: "unused-bucket", objName: object, uploadID: uploadID, PartID: 1, expectedError: fmt.Errorf("%s", "Invalid upload id "+uploadID)}, + // Test Case - 11. + // Case with valid UploadID, existing bucket name. + // But using the object name from which NewMultipartUpload is not constructed from. + {bucketName: bucket, objName: "none-object", uploadID: uploadID, PartID: 1, expectedError: fmt.Errorf("%s", "Invalid upload id "+uploadID)}, + // Test case - 12. + // Input to replicate Md5 mismatch. + { + bucketName: bucket, objName: object, uploadID: uploadID, PartID: 1, inputMd5: "d41d8cd98f00b204e9800998ecf8427f", + expectedError: hash.BadDigest{ExpectedMD5: "d41d8cd98f00b204e9800998ecf8427f", CalculatedMD5: "d41d8cd98f00b204e9800998ecf8427e"}, + }, + // Test case - 13. + // When incorrect sha256 is provided. + { + bucketName: bucket, objName: object, uploadID: uploadID, PartID: 1, inputSHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b854", + expectedError: hash.SHA256Mismatch{ + ExpectedSHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b854", + CalculatedSHA256: "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855", + }, + }, + // Test case - 14. + // Input with size more than the size of actual data inside the reader. + { + bucketName: bucket, objName: object, uploadID: uploadID, PartID: 1, inputReaderData: "abcd", inputMd5: "e2fc714c4727ee9395f324cd2e7f3335", inputDataSize: int64(len("abcd") + 1), + expectedError: hash.BadDigest{ExpectedMD5: "e2fc714c4727ee9395f324cd2e7f3335", CalculatedMD5: "e2fc714c4727ee9395f324cd2e7f331f"}, + }, + // Test case - 15. + // Input with size less than the size of actual data inside the reader. + { + bucketName: bucket, objName: object, uploadID: uploadID, PartID: 1, inputReaderData: "abcd", inputMd5: "900150983cd24fb0d6963f7d28e17f73", inputDataSize: int64(len("abcd") - 1), + expectedError: ioutil.ErrOverread, + }, + + // Test case - 16-19. + // Validating for success cases. + {bucketName: bucket, objName: object, uploadID: uploadID, PartID: 1, inputReaderData: "abcd", inputMd5: "e2fc714c4727ee9395f324cd2e7f331f", inputSHA256: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", inputDataSize: int64(len("abcd")), shouldPass: true}, + {bucketName: bucket, objName: object, uploadID: uploadID, PartID: 2, inputReaderData: "efgh", inputMd5: "1f7690ebdd9b4caf8fab49ca1757bf27", inputSHA256: "e5e088a0b66163a0a26a5e053d2a4496dc16ab6e0e3dd1adf2d16aa84a078c9d", inputDataSize: int64(len("efgh")), shouldPass: true}, + {bucketName: bucket, objName: object, uploadID: uploadID, PartID: 3, inputReaderData: "ijkl", inputMd5: "09a0877d04abf8759f99adec02baf579", inputSHA256: "005c19658919186b85618c5870463eec8d9b8c1a9d00208a5352891ba5bbe086", inputDataSize: int64(len("abcd")), shouldPass: true}, + {bucketName: bucket, objName: object, uploadID: uploadID, PartID: 4, inputReaderData: "mnop", inputMd5: "e132e96a5ddad6da8b07bba6f6131fef", inputSHA256: "f1afc31479522d6cff1ed068f93998f05a8cd3b22f5c37d7f307084f62d1d270", inputDataSize: int64(len("abcd")), shouldPass: true}, + } + + // Validate all the test cases. + for i, testCase := range testCases { + actualInfo, actualErr := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.inputDataSize, testCase.inputMd5, testCase.inputSHA256), opts) + // All are test cases above are expected to fail. + if actualErr != nil && testCase.shouldPass { + t.Errorf("Test %d: %s: Expected to pass, but failed with: %s.", i+1, instanceType, actualErr.Error()) + } + if actualErr == nil && !testCase.shouldPass { + t.Errorf("Test %d: %s: Expected to fail with \"%s\", but passed instead.", i+1, instanceType, testCase.expectedError.Error()) + } + // Failed as expected, but does it fail for the expected reason. + if actualErr != nil && !testCase.shouldPass { + if testCase.expectedError.Error() != actualErr.Error() { + t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1, instanceType, testCase.expectedError.Error(), actualErr.Error()) + } + } + // Test passes as expected, but the output values are verified for correctness here. + if actualErr == nil && testCase.shouldPass { + // Asserting whether the md5 output is correct. + if testCase.inputMd5 != actualInfo.ETag { + t.Errorf("Test %d: %s: Calculated Md5 different from the actual one %s.", i+1, instanceType, actualInfo.ETag) + } + } + } +} + +// Wrapper for calling TestListMultipartUploads tests for both Erasure multiple disks and single node setup. +func TestListMultipartUploads(t *testing.T) { + ExecExtendedObjectLayerTest(t, testListMultipartUploads) +} + +// testListMultipartUploads - Tests validate listing of multipart uploads. +func testListMultipartUploads(obj ObjectLayer, instanceType string, t TestErrHandler) { + bucketNames := []string{"minio-bucket", "minio-2-bucket", "minio-3-bucket"} + objectNames := []string{"minio-object-1.txt", "minio-object.txt", "neymar-1.jpeg", "neymar.jpeg", "parrot-1.png", "parrot.png"} + uploadIDs := []string{} + opts := ObjectOptions{} + // bucketnames[0]. + // objectNames[0]. + // uploadIds [0]. + // Create bucket before initiating NewMultipartUpload. + err := obj.MakeBucket(context.Background(), bucketNames[0], MakeBucketOptions{}) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + // Initiate Multipart Upload on the above created bucket. + res, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], opts) + if err != nil { + // Failed to create NewMultipartUpload, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + uploadIDs = append(uploadIDs, res.UploadID) + + // bucketnames[1]. + // objectNames[0]. + // uploadIds [1-3]. + // Bucket to test for multiple upload Id's for a given object. + err = obj.MakeBucket(context.Background(), bucketNames[1], MakeBucketOptions{}) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + for range 3 { + // Initiate Multipart Upload on bucketNames[1] for the same object 3 times. + // Used to test the listing for the case of multiple uploadID's for a given object. + res, err = obj.NewMultipartUpload(context.Background(), bucketNames[1], objectNames[0], opts) + if err != nil { + // Failed to create NewMultipartUpload, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + uploadIDs = append(uploadIDs, res.UploadID) + } + + // Bucket to test for multiple objects, each with unique UUID. + // bucketnames[2]. + // objectNames[0-2]. + // uploadIds [4-9]. + err = obj.MakeBucket(context.Background(), bucketNames[2], MakeBucketOptions{}) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + // Initiate Multipart Upload on bucketNames[2]. + // Used to test the listing for the case of multiple objects for a given bucket. + for i := range 6 { + res, err = obj.NewMultipartUpload(context.Background(), bucketNames[2], objectNames[i], opts) + if err != nil { + // Failed to create NewMultipartUpload, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + // uploadIds [4-9]. + uploadIDs = append(uploadIDs, res.UploadID) + } + // Create multipart parts. + // Need parts to be uploaded before MultipartLists can be called and tested. + createPartCases := []struct { + bucketName string + objName string + uploadID string + PartID int + inputReaderData string + inputMd5 string + inputDataSize int64 + expectedMd5 string + }{ + // Case 1-4. + // Creating sequence of parts for same uploadID. + // Used to ensure that the ListMultipartResult produces one output for the four parts uploaded below for the given upload ID. + {bucketNames[0], objectNames[0], uploadIDs[0], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"}, + {bucketNames[0], objectNames[0], uploadIDs[0], 2, "efgh", "1f7690ebdd9b4caf8fab49ca1757bf27", int64(len("efgh")), "1f7690ebdd9b4caf8fab49ca1757bf27"}, + {bucketNames[0], objectNames[0], uploadIDs[0], 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("abcd")), "09a0877d04abf8759f99adec02baf579"}, + {bucketNames[0], objectNames[0], uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd")), "e132e96a5ddad6da8b07bba6f6131fef"}, + // Cases 5-7. + // Create parts with 3 uploadID's for the same object. + // Testing for listing of all the uploadID's for given object. + // Insertion with 3 different uploadID's are done for same bucket and object. + {bucketNames[1], objectNames[0], uploadIDs[1], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"}, + {bucketNames[1], objectNames[0], uploadIDs[2], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"}, + {bucketNames[1], objectNames[0], uploadIDs[3], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"}, + // Case 8-13. + // Generating parts for different objects. + {bucketNames[2], objectNames[0], uploadIDs[4], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"}, + {bucketNames[2], objectNames[1], uploadIDs[5], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"}, + {bucketNames[2], objectNames[2], uploadIDs[6], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"}, + {bucketNames[2], objectNames[3], uploadIDs[7], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"}, + {bucketNames[2], objectNames[4], uploadIDs[8], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"}, + {bucketNames[2], objectNames[5], uploadIDs[9], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"}, + } + sha256sum := "" + // Iterating over creatPartCases to generate multipart chunks. + for _, testCase := range createPartCases { + _, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.inputDataSize, testCase.inputMd5, sha256sum), opts) + if err != nil { + t.Fatalf("%s : %s", instanceType, err.Error()) + } + } + + // Expected Results set for asserting ListObjectMultipart test. + listMultipartResults := []ListMultipartsInfo{ + // listMultipartResults - 1. + // Used to check that the result produces only one output for the 4 parts uploaded in cases 1-4 of createPartCases above. + // ListMultipartUploads doesn't list the parts. + { + MaxUploads: 100, + Uploads: []MultipartInfo{ + { + Object: objectNames[0], + UploadID: uploadIDs[0], + }, + }, + }, + // listMultipartResults - 2. + // Used to check that the result produces if keyMarker is set to the only available object. + // `KeyMarker` is set. + // ListMultipartUploads doesn't list the parts. + { + MaxUploads: 100, + KeyMarker: "minio-object-1.txt", + }, + // listMultipartResults - 3. + // `KeyMarker` is set, no MultipartInfo expected. + // ListMultipartUploads doesn't list the parts. + // `Maxupload` value is asserted. + { + MaxUploads: 100, + KeyMarker: "orange", + }, + // listMultipartResults - 4. + // `KeyMarker` is set, no MultipartInfo expected. + // Maxupload value is asserted. + { + MaxUploads: 1, + KeyMarker: "orange", + }, + // listMultipartResults - 5. + // `KeyMarker` is set. It contains part of the objectname as `KeyPrefix`. + // Expecting the result to contain one MultipartInfo entry and Istruncated to be false. + { + MaxUploads: 10, + KeyMarker: "min", + IsTruncated: false, + Uploads: []MultipartInfo{ + { + Object: objectNames[0], + UploadID: uploadIDs[0], + }, + }, + }, + // listMultipartResults - 6. + // `KeyMarker` is set. It contains part of the objectname as `KeyPrefix`. + // `MaxUploads` is set equal to the number of meta data entries in the result, the result contains only one entry. + // Expecting the result to contain one MultipartInfo entry and IsTruncated to be false. + { + MaxUploads: 1, + KeyMarker: "min", + IsTruncated: false, + Uploads: []MultipartInfo{ + { + Object: objectNames[0], + UploadID: uploadIDs[0], + }, + }, + }, + // listMultipartResults - 7. + // `KeyMarker` is set. It contains part of the objectname as `KeyPrefix`. + // Testing for the case with `MaxUploads` set to 0. + // Expecting the result to contain no MultipartInfo entry since `MaxUploads` is set to 0. + // Expecting `IsTruncated` to be true. + { + MaxUploads: 0, + KeyMarker: "min", + IsTruncated: true, + }, + // listMultipartResults - 8. + // `KeyMarker` is set. It contains part of the objectname as KeyPrefix. + // Testing for the case with `MaxUploads` set to 0. + // Expecting the result to contain no MultipartInfo entry since `MaxUploads` is set to 0. + // Expecting `isTruncated` to be true. + { + MaxUploads: 0, + KeyMarker: "min", + IsTruncated: true, + }, + // listMultipartResults - 9. + // `KeyMarker` is set. It contains part of the objectname as KeyPrefix. + // `KeyMarker` is set equal to the object name in the result. + // Expecting the result to contain one MultipartInfo entry and IsTruncated to be false. + { + MaxUploads: 2, + KeyMarker: "minio-object", + IsTruncated: false, + Uploads: []MultipartInfo{ + { + Object: objectNames[0], + UploadID: uploadIDs[0], + }, + }, + }, + // listMultipartResults - 10. + // Prefix is set. It is set equal to the object name. + // MaxUploads is set more than number of meta data entries in the result. + // Expecting the result to contain one MultipartInfo entry and IsTruncated to be false. + { + MaxUploads: 2, + Prefix: "minio-object-1.txt", + IsTruncated: false, + Uploads: []MultipartInfo{ + { + Object: objectNames[0], + UploadID: uploadIDs[0], + }, + }, + }, + // listMultipartResults - 11. + // Setting `Prefix` to contain the object name as its prefix. + // MaxUploads is set more than number of meta data entries in the result. + // Expecting the result to contain one MultipartInfo entry and IsTruncated to be false. + { + MaxUploads: 2, + Prefix: "min", + IsTruncated: false, + Uploads: []MultipartInfo{ + { + Object: objectNames[0], + UploadID: uploadIDs[0], + }, + }, + }, + // listMultipartResults - 12. + // Setting `Prefix` to contain the object name as its prefix. + // MaxUploads is set equal to number of meta data entries in the result. + // Expecting the result to contain one MultipartInfo entry and IsTruncated to be false. + { + MaxUploads: 1, + Prefix: "min", + IsTruncated: false, + Uploads: []MultipartInfo{ + { + Object: objectNames[0], + UploadID: uploadIDs[0], + }, + }, + }, + // listMultipartResults - 13. + // `Prefix` is set. It doesn't contain object name as its preifx. + // MaxUploads is set more than number of meta data entries in the result. + // Expecting no `Uploads` metadata. + { + MaxUploads: 2, + Prefix: "orange", + IsTruncated: false, + }, + // listMultipartResults - 14. + // `Prefix` is set. It doesn't contain object name as its preifx. + // MaxUploads is set more than number of meta data entries in the result. + // Expecting the result to contain 0 uploads and isTruncated to false. + { + MaxUploads: 2, + Prefix: "Asia", + IsTruncated: false, + }, + // listMultipartResults - 15. + // Setting `Delimiter`. + // MaxUploads is set more than number of meta data entries in the result. + // Expecting the result to contain one MultipartInfo entry and IsTruncated to be false. + { + MaxUploads: 2, + Delimiter: SlashSeparator, + Prefix: "", + IsTruncated: false, + Uploads: []MultipartInfo{ + { + Object: objectNames[0], + UploadID: uploadIDs[0], + }, + }, + }, + // listMultipartResults - 16. + // Testing for listing of 3 uploadID's for a given object. + // Will be used to list on bucketNames[1]. + { + MaxUploads: 100, + Uploads: []MultipartInfo{ + { + Object: objectNames[0], + UploadID: uploadIDs[1], + }, + { + Object: objectNames[0], + UploadID: uploadIDs[2], + }, + { + Object: objectNames[0], + UploadID: uploadIDs[3], + }, + }, + }, + // listMultipartResults - 17. + // Testing for listing of 3 uploadID's (uploadIDs[1-3]) for a given object with uploadID Marker set. + // uploadIDs[1] is set as UploadMarker, Expecting it to be skipped in the result. + // uploadIDs[2] and uploadIDs[3] are expected to be in the result. + // Istruncted is expected to be false. + // Will be used to list on bucketNames[1]. + { + MaxUploads: 100, + KeyMarker: "minio-object-1.txt", + UploadIDMarker: uploadIDs[1], + IsTruncated: false, + Uploads: []MultipartInfo{ + { + Object: objectNames[0], + UploadID: uploadIDs[2], + }, + { + Object: objectNames[0], + UploadID: uploadIDs[3], + }, + }, + }, + // listMultipartResults - 18. + // Testing for listing of 3 uploadID's (uploadIDs[1-3]) for a given object with uploadID Marker set. + // uploadIDs[2] is set as UploadMarker, Expecting it to be skipped in the result. + // Only uploadIDs[3] are expected to be in the result. + // Istruncted is expected to be false. + // Will be used to list on bucketNames[1]. + { + MaxUploads: 100, + KeyMarker: "minio-object-1.txt", + UploadIDMarker: uploadIDs[2], + IsTruncated: false, + Uploads: []MultipartInfo{ + { + Object: objectNames[0], + UploadID: uploadIDs[3], + }, + }, + }, + // listMultipartResults - 19. + // Testing for listing of 3 uploadID's for a given object, setting maxKeys to be 2. + // There are 3 MultipartInfo in the result (uploadIDs[1-3]), it should be truncated to 2. + // Since there is only single object for bucketNames[1], the NextKeyMarker is set to its name. + // The last entry in the result, uploadIDs[2], that is should be set as NextUploadIDMarker. + // Will be used to list on bucketNames[1]. + { + MaxUploads: 2, + IsTruncated: true, + NextKeyMarker: objectNames[0], + NextUploadIDMarker: uploadIDs[2], + Uploads: []MultipartInfo{ + { + Object: objectNames[0], + UploadID: uploadIDs[1], + }, + { + Object: objectNames[0], + UploadID: uploadIDs[2], + }, + }, + }, + // listMultipartResults - 20. + // Testing for listing of 3 uploadID's for a given object, setting maxKeys to be 1. + // There are 3 MultipartInfo in the result (uploadIDs[1-3]), it should be truncated to 1. + // The last entry in the result, uploadIDs[1], that is should be set as NextUploadIDMarker. + // Will be used to list on bucketNames[1]. + { + MaxUploads: 1, + IsTruncated: true, + NextKeyMarker: objectNames[0], + NextUploadIDMarker: uploadIDs[1], + Uploads: []MultipartInfo{ + { + Object: objectNames[0], + UploadID: uploadIDs[1], + }, + }, + }, + // listMultipartResults - 21. + // Testing for listing of 3 uploadID's for a given object, setting maxKeys to be 3. + // There are 3 MultipartInfo in the result (uploadIDs[1-3]), hence no truncation is expected. + // Since all the MultipartInfo is listed, expecting no values for NextUploadIDMarker and NextKeyMarker. + // Will be used to list on bucketNames[1]. + { + MaxUploads: 3, + IsTruncated: false, + Uploads: []MultipartInfo{ + { + Object: objectNames[0], + UploadID: uploadIDs[1], + }, + { + Object: objectNames[0], + UploadID: uploadIDs[2], + }, + { + Object: objectNames[0], + UploadID: uploadIDs[3], + }, + }, + }, + // listMultipartResults - 22. + // Testing for listing of 3 uploadID's for a given object, setting `prefix` to be "min". + // Will be used to list on bucketNames[1]. + { + MaxUploads: 10, + IsTruncated: false, + Prefix: "min", + Uploads: []MultipartInfo{ + { + Object: objectNames[0], + UploadID: uploadIDs[1], + }, + { + Object: objectNames[0], + UploadID: uploadIDs[2], + }, + { + Object: objectNames[0], + UploadID: uploadIDs[3], + }, + }, + }, + // listMultipartResults - 23. + // Testing for listing of 3 uploadID's for a given object + // setting `prefix` to be "orange". + // Will be used to list on bucketNames[1]. + { + MaxUploads: 10, + IsTruncated: false, + Prefix: "orange", + }, + // listMultipartResults - 24. + // Testing for listing of 3 uploadID's for a given object. + // setting `prefix` to be "Asia". + // Will be used to list on bucketNames[1]. + { + MaxUploads: 10, + IsTruncated: false, + Prefix: "Asia", + }, + // listMultipartResults - 25. + // Testing for listing of 3 uploadID's for a given object. + // setting `prefix` and uploadIDMarker. + // Will be used to list on bucketNames[1]. + { + MaxUploads: 10, + KeyMarker: "minio-object-1.txt", + IsTruncated: false, + Prefix: "min", + UploadIDMarker: uploadIDs[1], + Uploads: []MultipartInfo{ + { + Object: objectNames[0], + UploadID: uploadIDs[2], + }, + { + Object: objectNames[0], + UploadID: uploadIDs[3], + }, + }, + }, + + // Operations on bucket 2. + // listMultipartResults - 26. + // checking listing everything. + { + MaxUploads: 100, + IsTruncated: false, + + Uploads: []MultipartInfo{ + { + Object: objectNames[0], + UploadID: uploadIDs[4], + }, + { + Object: objectNames[1], + UploadID: uploadIDs[5], + }, + { + Object: objectNames[2], + UploadID: uploadIDs[6], + }, + { + Object: objectNames[3], + UploadID: uploadIDs[7], + }, + { + Object: objectNames[4], + UploadID: uploadIDs[8], + }, + { + Object: objectNames[5], + UploadID: uploadIDs[9], + }, + }, + }, + // listMultipartResults - 27. + // listing with `prefix` "min". + { + MaxUploads: 100, + IsTruncated: false, + Prefix: "min", + Uploads: []MultipartInfo{ + { + Object: objectNames[0], + UploadID: uploadIDs[4], + }, + { + Object: objectNames[1], + UploadID: uploadIDs[5], + }, + }, + }, + // listMultipartResults - 28. + // listing with `prefix` "ney". + { + MaxUploads: 100, + IsTruncated: false, + Prefix: "ney", + Uploads: []MultipartInfo{ + { + Object: objectNames[2], + UploadID: uploadIDs[6], + }, + { + Object: objectNames[3], + UploadID: uploadIDs[7], + }, + }, + }, + // listMultipartResults - 29. + // listing with `prefix` "parrot". + { + MaxUploads: 100, + IsTruncated: false, + Prefix: "parrot", + Uploads: []MultipartInfo{ + { + Object: objectNames[4], + UploadID: uploadIDs[8], + }, + { + Object: objectNames[5], + UploadID: uploadIDs[9], + }, + }, + }, + // listMultipartResults - 30. + // listing with `prefix` "neymar.jpeg". + // prefix set to object name. + { + MaxUploads: 100, + IsTruncated: false, + Prefix: "neymar.jpeg", + Uploads: []MultipartInfo{ + { + Object: objectNames[3], + UploadID: uploadIDs[7], + }, + }, + }, + + // listMultipartResults - 31. + // checking listing with marker set to 3. + // `NextUploadIDMarker` is expected to be set on last uploadID in the result. + // `NextKeyMarker` is expected to be set on the last object key in the list. + { + MaxUploads: 3, + IsTruncated: true, + NextUploadIDMarker: uploadIDs[6], + NextKeyMarker: objectNames[2], + Uploads: []MultipartInfo{ + { + Object: objectNames[0], + UploadID: uploadIDs[4], + }, + { + Object: objectNames[1], + UploadID: uploadIDs[5], + }, + { + Object: objectNames[2], + UploadID: uploadIDs[6], + }, + }, + }, + // listMultipartResults - 32. + // checking listing with marker set to no of objects in the list. + // `NextUploadIDMarker` is expected to be empty since all results are listed. + // `NextKeyMarker` is expected to be empty since all results are listed. + { + MaxUploads: 6, + IsTruncated: false, + Uploads: []MultipartInfo{ + { + Object: objectNames[0], + UploadID: uploadIDs[4], + }, + { + Object: objectNames[1], + UploadID: uploadIDs[5], + }, + { + Object: objectNames[2], + UploadID: uploadIDs[6], + }, + { + Object: objectNames[3], + UploadID: uploadIDs[7], + }, + { + Object: objectNames[4], + UploadID: uploadIDs[8], + }, + { + Object: objectNames[5], + UploadID: uploadIDs[9], + }, + }, + }, + // listMultipartResults - 33. + // checking listing with `UploadIDMarker` set. + { + MaxUploads: 10, + IsTruncated: false, + UploadIDMarker: uploadIDs[6], + Uploads: []MultipartInfo{ + { + Object: objectNames[3], + UploadID: uploadIDs[7], + }, + { + Object: objectNames[4], + UploadID: uploadIDs[8], + }, + { + Object: objectNames[5], + UploadID: uploadIDs[9], + }, + }, + }, + // listMultipartResults - 34. + // checking listing with `KeyMarker` set. + { + MaxUploads: 10, + IsTruncated: false, + KeyMarker: objectNames[3], + Uploads: []MultipartInfo{ + { + Object: objectNames[4], + UploadID: uploadIDs[8], + }, + { + Object: objectNames[5], + UploadID: uploadIDs[9], + }, + }, + }, + // listMultipartResults - 35. + // Checking listing with `Prefix` and `KeyMarker`. + // No upload MultipartInfo in the result expected since KeyMarker is set to last Key in the result. + { + MaxUploads: 10, + IsTruncated: false, + Prefix: "minio-object", + KeyMarker: objectNames[1], + }, + // listMultipartResults - 36. + // checking listing with `Prefix` and `UploadIDMarker` set. + { + MaxUploads: 10, + IsTruncated: false, + Prefix: "minio", + UploadIDMarker: uploadIDs[4], + Uploads: []MultipartInfo{ + { + Object: objectNames[1], + UploadID: uploadIDs[5], + }, + }, + }, + // listMultipartResults - 37. + // Checking listing with `KeyMarker` and `UploadIDMarker` set. + { + MaxUploads: 10, + IsTruncated: false, + KeyMarker: "minio-object.txt", + UploadIDMarker: uploadIDs[5], + }, + } + + // Collection of non-exhaustive ListMultipartUploads test cases, valid errors + // and success responses. + testCases := []struct { + // Inputs to ListMultipartUploads. + bucket string + prefix string + keyMarker string + uploadIDMarker string + delimiter string + maxUploads int + // Expected output of ListMultipartUploads. + expectedResult ListMultipartsInfo + expectedErr error + // Flag indicating whether the test is expected to pass or not. + shouldPass bool + }{ + // Test cases with invalid bucket names ( Test number 1-4 ). + {".test", "", "", "", "", 0, ListMultipartsInfo{}, BucketNameInvalid{Bucket: ".test"}, false}, + {"Test", "", "", "", "", 0, ListMultipartsInfo{}, BucketNameInvalid{Bucket: "Test"}, false}, + {"---", "", "", "", "", 0, ListMultipartsInfo{}, BucketNameInvalid{Bucket: "---"}, false}, + {"ad", "", "", "", "", 0, ListMultipartsInfo{}, BucketNameInvalid{Bucket: "ad"}, false}, + // Valid bucket names, but they do not exist (Test number 5-7). + {"volatile-bucket-1", "", "", "", "", 0, ListMultipartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false}, + {"volatile-bucket-2", "", "", "", "", 0, ListMultipartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false}, + {"volatile-bucket-3", "", "", "", "", 0, ListMultipartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false}, + // Valid, existing bucket, delimiter not supported, returns empty values (Test number 8-9). + {bucketNames[0], "", "", "", "*", 0, ListMultipartsInfo{Delimiter: "*"}, nil, true}, + {bucketNames[0], "", "", "", "-", 0, ListMultipartsInfo{Delimiter: "-"}, nil, true}, + // If marker is *after* the last possible object from the prefix it should return an empty list. + { + bucketNames[0], "Asia", "europe-object", "", "", 0, + ListMultipartsInfo{KeyMarker: "europe-object", Prefix: "Asia", IsTruncated: false}, + nil, true, + }, + // Setting an invalid combination of uploadIDMarker and Marker (Test number 11-12). + { + bucketNames[0], "asia", "asia/europe/", "abc", "", 0, + ListMultipartsInfo{}, + fmt.Errorf("Invalid combination of uploadID marker '%s' and marker '%s'", "abc", "asia/europe/"), false, + }, + { + // Contains a base64 padding character + bucketNames[0], "asia", "asia/europe", "abc=", "", 0, + ListMultipartsInfo{}, + fmt.Errorf("Malformed upload id %s", "abc="), false, + }, + + // Setting up valid case of ListMultiPartUploads. + // Test case with multiple parts for a single uploadID (Test number 13). + {bucketNames[0], "", "", "", "", 100, listMultipartResults[0], nil, true}, + // Test with a KeyMarker (Test number 14-17). + {bucketNames[0], "", "minio-object-1.txt", "", "", 100, listMultipartResults[1], nil, true}, + {bucketNames[0], "", "orange", "", "", 100, listMultipartResults[2], nil, true}, + {bucketNames[0], "", "orange", "", "", 1, listMultipartResults[3], nil, true}, + {bucketNames[0], "", "min", "", "", 10, listMultipartResults[4], nil, true}, + // Test case with keyMarker set equal to number of parts in the result. (Test number 18). + {bucketNames[0], "", "min", "", "", 1, listMultipartResults[5], nil, true}, + // Test case with keyMarker set to 0. (Test number 19). + {bucketNames[0], "", "min", "", "", 0, listMultipartResults[6], nil, true}, + // Test case with keyMarker less than 0. (Test number 20). + // {bucketNames[0], "", "min", "", "", -1, listMultipartResults[7], nil, true}, + // The result contains only one entry. The KeyPrefix is set to the object name in the result. + // Expecting the result to skip the KeyPrefix entry in the result (Test number 21). + {bucketNames[0], "", "minio-object", "", "", 2, listMultipartResults[8], nil, true}, + // Test case containing prefix values. + // Setting prefix to be equal to object name.(Test number 22). + {bucketNames[0], "minio-object-1.txt", "", "", "", 2, listMultipartResults[9], nil, true}, + // Setting `prefix` to contain the object name as its prefix (Test number 23). + {bucketNames[0], "min", "", "", "", 2, listMultipartResults[10], nil, true}, + // Setting `prefix` to contain the object name as its prefix (Test number 24). + {bucketNames[0], "min", "", "", "", 1, listMultipartResults[11], nil, true}, + // Setting `prefix` to not to contain the object name as its prefix (Test number 25-26). + {bucketNames[0], "orange", "", "", "", 2, listMultipartResults[12], nil, true}, + {bucketNames[0], "Asia", "", "", "", 2, listMultipartResults[13], nil, true}, + // setting delimiter (Test number 27). + {bucketNames[0], "", "", "", SlashSeparator, 2, listMultipartResults[14], nil, true}, + // Test case with multiple uploadID listing for given object (Test number 28). + {bucketNames[1], "", "", "", "", 100, listMultipartResults[15], nil, true}, + // Test case with multiple uploadID listing for given object, but uploadID marker set. + // Testing whether the marker entry is skipped (Test number 29-30). + {bucketNames[1], "", "minio-object-1.txt", uploadIDs[1], "", 100, listMultipartResults[16], nil, true}, + {bucketNames[1], "", "minio-object-1.txt", uploadIDs[2], "", 100, listMultipartResults[17], nil, true}, + // Test cases with multiple uploadID listing for a given object (Test number 31-32). + // MaxKeys set to values lesser than the number of entries in the MultipartInfo. + // IsTruncated is expected to be true. + {bucketNames[1], "", "", "", "", 2, listMultipartResults[18], nil, true}, + {bucketNames[1], "", "", "", "", 1, listMultipartResults[19], nil, true}, + // MaxKeys set to the value which is equal to no of entries in the MultipartInfo (Test number 33). + // In case of bucketNames[1], there are 3 entries. + // Since all available entries are listed, IsTruncated is expected to be false + // and NextMarkers are expected to empty. + {bucketNames[1], "", "", "", "", 3, listMultipartResults[20], nil, true}, + // Adding prefix (Test number 34-36). + {bucketNames[1], "min", "", "", "", 10, listMultipartResults[21], nil, true}, + {bucketNames[1], "orange", "", "", "", 10, listMultipartResults[22], nil, true}, + {bucketNames[1], "Asia", "", "", "", 10, listMultipartResults[23], nil, true}, + // Test case with `Prefix` and `UploadIDMarker` (Test number 37). + {bucketNames[1], "min", "minio-object-1.txt", uploadIDs[1], "", 10, listMultipartResults[24], nil, true}, + // Test case for bucket with multiple objects in it. + // Bucket used : `bucketNames[2]`. + // Objects used: `objectNames[1-5]`. + // UploadId's used: uploadIds[4-8]. + // (Test number 39). + {bucketNames[2], "", "", "", "", 100, listMultipartResults[25], nil, true}, + // Test cases with prefixes. + // Testing listing with prefix set to "min" (Test number 40) . + {bucketNames[2], "min", "", "", "", 100, listMultipartResults[26], nil, true}, + // Testing listing with prefix set to "ney" (Test number 41). + {bucketNames[2], "ney", "", "", "", 100, listMultipartResults[27], nil, true}, + // Testing listing with prefix set to "par" (Test number 42). + {bucketNames[2], "parrot", "", "", "", 100, listMultipartResults[28], nil, true}, + // Testing listing with prefix set to object name "neymar.jpeg" (Test number 43). + {bucketNames[2], "neymar.jpeg", "", "", "", 100, listMultipartResults[29], nil, true}, + // Testing listing with `MaxUploads` set to 3 (Test number 44). + {bucketNames[2], "", "", "", "", 3, listMultipartResults[30], nil, true}, + // In case of bucketNames[2], there are 6 entries (Test number 45). + // Since all available entries are listed, IsTruncated is expected to be false + // and NextMarkers are expected to empty. + {bucketNames[2], "", "", "", "", 6, listMultipartResults[31], nil, true}, + // Test case with `KeyMarker` (Test number 47). + {bucketNames[2], "", objectNames[3], "", "", 10, listMultipartResults[33], nil, true}, + // Test case with `prefix` and `KeyMarker` (Test number 48). + {bucketNames[2], "minio-object", objectNames[1], "", "", 10, listMultipartResults[34], nil, true}, + } + + for i, testCase := range testCases { + // fmt.Println(i+1, testCase) // uncomment to peek into the test cases. + actualResult, actualErr := obj.ListMultipartUploads(context.Background(), testCase.bucket, testCase.prefix, testCase.keyMarker, testCase.uploadIDMarker, testCase.delimiter, testCase.maxUploads) + if actualErr != nil && testCase.shouldPass { + t.Errorf("Test %d: %s: Expected to pass, but failed with: %s", i+1, instanceType, actualErr.Error()) + } + if actualErr == nil && !testCase.shouldPass { + t.Errorf("Test %d: %s: Expected to fail with \"%s\", but passed instead", i+1, instanceType, testCase.expectedErr.Error()) + } + // Failed as expected, but does it fail for the expected reason. + if actualErr != nil && !testCase.shouldPass { + if !strings.Contains(actualErr.Error(), testCase.expectedErr.Error()) { + t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.expectedErr.Error(), actualErr.Error()) + } + } + // Passes as expected, but asserting the results. + if actualErr == nil && testCase.shouldPass { + expectedResult := testCase.expectedResult + // Asserting the MaxUploads. + if actualResult.MaxUploads != expectedResult.MaxUploads { + t.Errorf("Test %d: %s: Expected the MaxUploads to be %d, but instead found it to be %d", i+1, instanceType, expectedResult.MaxUploads, actualResult.MaxUploads) + } + // Asserting Prefix. + if actualResult.Prefix != expectedResult.Prefix { + t.Errorf("Test %d: %s: Expected Prefix to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Prefix, actualResult.Prefix) + } + // Asserting Delimiter. + if actualResult.Delimiter != expectedResult.Delimiter { + t.Errorf("Test %d: %s: Expected Delimiter to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Delimiter, actualResult.Delimiter) + } + // Asserting the keyMarker. + if actualResult.KeyMarker != expectedResult.KeyMarker { + t.Errorf("Test %d: %s: Expected keyMarker to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.KeyMarker, actualResult.KeyMarker) + } + } + } +} + +// Wrapper for calling TestListObjectPartsStale tests for both Erasure multiple disks and single node setup. +func TestListObjectPartsStale(t *testing.T) { + ExecObjectLayerDiskAlteredTest(t, testListObjectPartsStale) +} + +// testListObjectPartsStale - Tests validate listing of object parts when parts are stale +func testListObjectPartsStale(obj ObjectLayer, instanceType string, disks []string, t *testing.T) { + bucketNames := []string{"minio-bucket", "minio-2-bucket"} + objectNames := []string{"minio-object-1.txt"} + uploadIDs := []string{} + + globalStorageClass.Update(storageclass.Config{ + RRS: storageclass.StorageClass{ + Parity: 2, + }, + Standard: storageclass.StorageClass{ + Parity: 4, + }, + }) + + // bucketnames[0]. + // objectNames[0]. + // uploadIds [0]. + // Create bucket before initiating NewMultipartUpload. + err := obj.MakeBucket(context.Background(), bucketNames[0], MakeBucketOptions{}) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + opts := ObjectOptions{} + // Initiate Multipart Upload on the above created bucket. + res, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], opts) + if err != nil { + // Failed to create NewMultipartUpload, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + z := obj.(*erasureServerPools) + er := z.serverPools[0].sets[0] + + uploadIDs = append(uploadIDs, res.UploadID) + + // Create multipart parts. + // Need parts to be uploaded before MultipartLists can be called and tested. + createPartCases := []struct { + bucketName string + objName string + uploadID string + PartID int + inputReaderData string + inputMd5 string + inputDataSize int64 + expectedMd5 string + }{ + // Case 1-4. + // Creating sequence of parts for same uploadID. + // Used to ensure that the ListMultipartResult produces one output for the four parts uploaded below for the given upload ID. + {bucketNames[0], objectNames[0], uploadIDs[0], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"}, + {bucketNames[0], objectNames[0], uploadIDs[0], 2, "efgh", "1f7690ebdd9b4caf8fab49ca1757bf27", int64(len("efgh")), "1f7690ebdd9b4caf8fab49ca1757bf27"}, + {bucketNames[0], objectNames[0], uploadIDs[0], 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("abcd")), "09a0877d04abf8759f99adec02baf579"}, + {bucketNames[0], objectNames[0], uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd")), "e132e96a5ddad6da8b07bba6f6131fef"}, + } + sha256sum := "" + // Iterating over creatPartCases to generate multipart chunks. + for _, testCase := range createPartCases { + _, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.inputDataSize, testCase.inputMd5, sha256sum), opts) + if err != nil { + t.Fatalf("%s : %s", instanceType, err.Error()) + } + } + + erasureDisks := er.getDisks() + uploadIDPath := er.getUploadIDDir(bucketNames[0], objectNames[0], uploadIDs[0]) + dataDirs, err := erasureDisks[0].ListDir(context.Background(), minioMetaMultipartBucket, minioMetaMultipartBucket, uploadIDPath, -1) + if err != nil { + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + var dataDir string + for _, folder := range dataDirs { + if strings.HasSuffix(folder, SlashSeparator) { + dataDir = folder + break + } + } + + toDel := (len(erasureDisks) / 2) + 1 + for _, disk := range erasureDisks[:toDel] { + disk.DeleteBulk(context.Background(), minioMetaMultipartBucket, []string{pathJoin(uploadIDPath, dataDir, "part.2")}...) + } + + partInfos := []ListPartsInfo{ + // partinfos - 0. + { + Bucket: bucketNames[0], + Object: objectNames[0], + MaxParts: 10, + UploadID: uploadIDs[0], + Parts: []PartInfo{ + { + PartNumber: 1, + Size: 4, + ETag: "e2fc714c4727ee9395f324cd2e7f331f", + }, + { + PartNumber: 3, + Size: 4, + ETag: "09a0877d04abf8759f99adec02baf579", + }, + { + PartNumber: 4, + Size: 4, + ETag: "e132e96a5ddad6da8b07bba6f6131fef", + }, + }, + }, + // partinfos - 1. + { + Bucket: bucketNames[0], + Object: objectNames[0], + MaxParts: 3, + UploadID: uploadIDs[0], + Parts: []PartInfo{ + { + PartNumber: 1, + Size: 4, + ETag: "e2fc714c4727ee9395f324cd2e7f331f", + }, + { + PartNumber: 3, + Size: 4, + ETag: "09a0877d04abf8759f99adec02baf579", + }, + { + PartNumber: 4, + Size: 4, + ETag: "e132e96a5ddad6da8b07bba6f6131fef", + }, + }, + }, + // partinfos - 2. + { + Bucket: bucketNames[0], + Object: objectNames[0], + MaxParts: 2, + NextPartNumberMarker: 3, + IsTruncated: true, + UploadID: uploadIDs[0], + Parts: []PartInfo{ + { + PartNumber: 1, + Size: 4, + ETag: "e2fc714c4727ee9395f324cd2e7f331f", + }, + { + PartNumber: 3, + Size: 4, + ETag: "09a0877d04abf8759f99adec02baf579", + }, + }, + }, + // partinfos - 3. + { + Bucket: bucketNames[0], + Object: objectNames[0], + MaxParts: 2, + IsTruncated: false, + UploadID: uploadIDs[0], + PartNumberMarker: 3, + Parts: []PartInfo{ + { + PartNumber: 4, + Size: 4, + ETag: "e132e96a5ddad6da8b07bba6f6131fef", + }, + }, + }, + // partinfos - 4. + { + Bucket: bucketNames[0], + Object: objectNames[0], + MaxParts: 2, + IsTruncated: false, + UploadID: uploadIDs[0], + PartNumberMarker: 4, + }, + // partinfos - 5. + { + Bucket: bucketNames[0], + Object: objectNames[0], + MaxParts: 2, + IsTruncated: false, + UploadID: uploadIDs[0], + PartNumberMarker: 100, + }, + } + + // Collection of non-exhaustive ListObjectParts test cases, valid errors + // and success responses. + testCases := []struct { + bucket string + object string + uploadID string + partNumberMarker int + maxParts int + // Expected output of ListPartsInfo. + expectedResult ListPartsInfo + expectedErr error + // Flag indicating whether the test is expected to pass or not. + shouldPass bool + }{ + // Test cases with invalid bucket names (Test number 1-4). + {".test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: ".test"}, false}, + {"Test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "Test"}, false}, + {"---", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "---"}, false}, + {"ad", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "ad"}, false}, + // Test cases for listing uploadID with single part. + // Valid bucket names, but they do not exist (Test number 5-7). + {"volatile-bucket-1", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false}, + {"volatile-bucket-2", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false}, + {"volatile-bucket-3", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false}, + // Test case for Asserting for invalid objectName (Test number 8). + {bucketNames[0], "", "", 0, 0, ListPartsInfo{}, ObjectNameInvalid{Bucket: bucketNames[0]}, false}, + // Asserting for Invalid UploadID (Test number 9). + {bucketNames[0], objectNames[0], "abc", 0, 0, ListPartsInfo{}, InvalidUploadID{UploadID: "abc"}, false}, + // Test case for uploadID with multiple parts (Test number 12). + {bucketNames[0], objectNames[0], uploadIDs[0], 0, 10, partInfos[0], nil, true}, + // Test case with maxParts set to less than number of parts (Test number 13). + {bucketNames[0], objectNames[0], uploadIDs[0], 0, 3, partInfos[1], nil, true}, + // Test case with partNumberMarker set (Test number 14). + {bucketNames[0], objectNames[0], uploadIDs[0], 0, 2, partInfos[2], nil, true}, + // Test case with partNumberMarker set (Test number 15). + {bucketNames[0], objectNames[0], uploadIDs[0], 3, 2, partInfos[3], nil, true}, + // Test case with partNumberMarker set (Test number 16). + {bucketNames[0], objectNames[0], uploadIDs[0], 4, 2, partInfos[4], nil, true}, + // Test case with partNumberMarker set (Test number 17). + {bucketNames[0], objectNames[0], uploadIDs[0], 100, 2, partInfos[5], nil, true}, + } + + for i, testCase := range testCases { + actualResult, actualErr := obj.ListObjectParts(context.Background(), testCase.bucket, testCase.object, testCase.uploadID, testCase.partNumberMarker, testCase.maxParts, ObjectOptions{}) + if actualErr != nil && testCase.shouldPass { + t.Errorf("Test %d: %s: Expected to pass, but failed with: %s", i+1, instanceType, actualErr.Error()) + } + if actualErr == nil && !testCase.shouldPass { + t.Errorf("Test %d: %s: Expected to fail with \"%s\", but passed instead", i+1, instanceType, testCase.expectedErr.Error()) + } + // Failed as expected, but does it fail for the expected reason. + if actualErr != nil && !testCase.shouldPass { + if !strings.Contains(actualErr.Error(), testCase.expectedErr.Error()) { + t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.expectedErr, actualErr) + } + } + // Passes as expected, but asserting the results. + if actualErr == nil && testCase.shouldPass { + expectedResult := testCase.expectedResult + // Asserting the MaxParts. + if actualResult.MaxParts != expectedResult.MaxParts { + t.Errorf("Test %d: %s: Expected the MaxParts to be %d, but instead found it to be %d", i+1, instanceType, expectedResult.MaxParts, actualResult.MaxParts) + } + // Asserting Object Name. + if actualResult.Object != expectedResult.Object { + t.Errorf("Test %d: %s: Expected Object name to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Object, actualResult.Object) + } + // Asserting UploadID. + if actualResult.UploadID != expectedResult.UploadID { + t.Errorf("Test %d: %s: Expected UploadID to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.UploadID, actualResult.UploadID) + } + // Asserting NextPartNumberMarker. + if actualResult.NextPartNumberMarker != expectedResult.NextPartNumberMarker { + t.Errorf("Test %d: %s: Expected NextPartNumberMarker to be \"%d\", but instead found it to be \"%d\"", i+1, instanceType, expectedResult.NextPartNumberMarker, actualResult.NextPartNumberMarker) + } + // Asserting PartNumberMarker. + if actualResult.PartNumberMarker != expectedResult.PartNumberMarker { + t.Errorf("Test %d: %s: Expected PartNumberMarker to be \"%d\", but instead found it to be \"%d\"", i+1, instanceType, expectedResult.PartNumberMarker, actualResult.PartNumberMarker) + } + // Asserting the BucketName. + if actualResult.Bucket != expectedResult.Bucket { + t.Errorf("Test %d: %s: Expected Bucket to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Bucket, actualResult.Bucket) + } + // Asserting IsTruncated. + if actualResult.IsTruncated != testCase.expectedResult.IsTruncated { + t.Errorf("Test %d: %s: Expected IsTruncated to be \"%v\", but found it to \"%v\"", i+1, instanceType, expectedResult.IsTruncated, actualResult.IsTruncated) + } + // Asserting the number of Parts. + if len(expectedResult.Parts) != len(actualResult.Parts) { + t.Errorf("Test %d: %s: Expected the result to contain info of %d Parts, but found %d instead", i+1, instanceType, len(expectedResult.Parts), len(actualResult.Parts)) + } else { + // Iterating over the partInfos and asserting the fields. + for j, actualMetaData := range actualResult.Parts { + // Asserting the PartNumber in the PartInfo. + if actualMetaData.PartNumber != expectedResult.Parts[j].PartNumber { + t.Errorf("Test %d: %s: Part %d: Expected PartNumber to be \"%d\", but instead found \"%d\"", i+1, instanceType, j+1, expectedResult.Parts[j].PartNumber, actualMetaData.PartNumber) + } + // Asserting the Size in the PartInfo. + if actualMetaData.Size != expectedResult.Parts[j].Size { + t.Errorf("Test %d: %s: Part %d: Expected Part Size to be \"%d\", but instead found \"%d\"", i+1, instanceType, j+1, expectedResult.Parts[j].Size, actualMetaData.Size) + } + // Asserting the ETag in the PartInfo. + if actualMetaData.ETag != expectedResult.Parts[j].ETag { + t.Errorf("Test %d: %s: Part %d: Expected Etag to be \"%s\", but instead found \"%s\"", i+1, instanceType, j+1, expectedResult.Parts[j].ETag, actualMetaData.ETag) + } + } + } + } + } +} + +// Wrapper for calling TestListObjectPartsDiskNotFound tests for both Erasure multiple disks and single node setup. +func TestListObjectPartsDiskNotFound(t *testing.T) { + ExecObjectLayerDiskAlteredTest(t, testListObjectPartsDiskNotFound) +} + +// testListObjectParts - Tests validate listing of object parts when disks go offline. +func testListObjectPartsDiskNotFound(obj ObjectLayer, instanceType string, disks []string, t *testing.T) { + bucketNames := []string{"minio-bucket", "minio-2-bucket"} + objectNames := []string{"minio-object-1.txt"} + uploadIDs := []string{} + + globalStorageClass.Update(storageclass.Config{ + RRS: storageclass.StorageClass{ + Parity: 2, + }, + Standard: storageclass.StorageClass{ + Parity: 4, + }, + }) + + // bucketnames[0]. + // objectNames[0]. + // uploadIds [0]. + // Create bucket before initiating NewMultipartUpload. + err := obj.MakeBucket(context.Background(), bucketNames[0], MakeBucketOptions{}) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + opts := ObjectOptions{} + // Initiate Multipart Upload on the above created bucket. + res, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], opts) + if err != nil { + // Failed to create NewMultipartUpload, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + z := obj.(*erasureServerPools) + er := z.serverPools[0].sets[0] + + erasureDisks := er.getDisks() + ridx := rand.Intn(len(erasureDisks)) + + z.serverPools[0].erasureDisksMu.Lock() + er.getDisks = func() []StorageAPI { + erasureDisks[ridx] = newNaughtyDisk(erasureDisks[ridx], nil, errFaultyDisk) + return erasureDisks + } + z.serverPools[0].erasureDisksMu.Unlock() + + uploadIDs = append(uploadIDs, res.UploadID) + + // Create multipart parts. + // Need parts to be uploaded before MultipartLists can be called and tested. + createPartCases := []struct { + bucketName string + objName string + uploadID string + PartID int + inputReaderData string + inputMd5 string + inputDataSize int64 + expectedMd5 string + }{ + // Case 1-4. + // Creating sequence of parts for same uploadID. + // Used to ensure that the ListMultipartResult produces one output for the four parts uploaded below for the given upload ID. + {bucketNames[0], objectNames[0], uploadIDs[0], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"}, + {bucketNames[0], objectNames[0], uploadIDs[0], 2, "efgh", "1f7690ebdd9b4caf8fab49ca1757bf27", int64(len("efgh")), "1f7690ebdd9b4caf8fab49ca1757bf27"}, + {bucketNames[0], objectNames[0], uploadIDs[0], 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("abcd")), "09a0877d04abf8759f99adec02baf579"}, + {bucketNames[0], objectNames[0], uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd")), "e132e96a5ddad6da8b07bba6f6131fef"}, + } + sha256sum := "" + // Iterating over creatPartCases to generate multipart chunks. + for _, testCase := range createPartCases { + _, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.inputDataSize, testCase.inputMd5, sha256sum), opts) + if err != nil { + t.Fatalf("%s : %s", instanceType, err.Error()) + } + } + + partInfos := []ListPartsInfo{ + // partinfos - 0. + { + Bucket: bucketNames[0], + Object: objectNames[0], + MaxParts: 10, + UploadID: uploadIDs[0], + Parts: []PartInfo{ + { + PartNumber: 1, + Size: 4, + ETag: "e2fc714c4727ee9395f324cd2e7f331f", + }, + { + PartNumber: 2, + Size: 4, + ETag: "1f7690ebdd9b4caf8fab49ca1757bf27", + }, + { + PartNumber: 3, + Size: 4, + ETag: "09a0877d04abf8759f99adec02baf579", + }, + { + PartNumber: 4, + Size: 4, + ETag: "e132e96a5ddad6da8b07bba6f6131fef", + }, + }, + }, + // partinfos - 1. + { + Bucket: bucketNames[0], + Object: objectNames[0], + MaxParts: 3, + NextPartNumberMarker: 3, + IsTruncated: true, + UploadID: uploadIDs[0], + Parts: []PartInfo{ + { + PartNumber: 1, + Size: 4, + ETag: "e2fc714c4727ee9395f324cd2e7f331f", + }, + { + PartNumber: 2, + Size: 4, + ETag: "1f7690ebdd9b4caf8fab49ca1757bf27", + }, + { + PartNumber: 3, + Size: 4, + ETag: "09a0877d04abf8759f99adec02baf579", + }, + }, + }, + // partinfos - 2. + { + Bucket: bucketNames[0], + Object: objectNames[0], + MaxParts: 2, + IsTruncated: false, + UploadID: uploadIDs[0], + PartNumberMarker: 3, + Parts: []PartInfo{ + { + PartNumber: 4, + Size: 4, + ETag: "e132e96a5ddad6da8b07bba6f6131fef", + }, + }, + }, + } + + // Collection of non-exhaustive ListObjectParts test cases, valid errors + // and success responses. + testCases := []struct { + bucket string + object string + uploadID string + partNumberMarker int + maxParts int + // Expected output of ListPartsInfo. + expectedResult ListPartsInfo + expectedErr error + // Flag indicating whether the test is expected to pass or not. + shouldPass bool + }{ + // Test cases with invalid bucket names (Test number 1-4). + {".test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: ".test"}, false}, + {"Test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "Test"}, false}, + {"---", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "---"}, false}, + {"ad", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "ad"}, false}, + // Test cases for listing uploadID with single part. + // Valid bucket names, but they do not exist (Test number 5-7). + {"volatile-bucket-1", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false}, + {"volatile-bucket-2", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false}, + {"volatile-bucket-3", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false}, + // Test case for Asserting for invalid objectName (Test number 8). + {bucketNames[0], "", "", 0, 0, ListPartsInfo{}, ObjectNameInvalid{Bucket: bucketNames[0]}, false}, + // Asserting for Invalid UploadID (Test number 9). + {bucketNames[0], objectNames[0], "abc", 0, 0, ListPartsInfo{}, InvalidUploadID{UploadID: "abc"}, false}, + // Test case for uploadID with multiple parts (Test number 12). + {bucketNames[0], objectNames[0], uploadIDs[0], 0, 10, partInfos[0], nil, true}, + // Test case with maxParts set to less than number of parts (Test number 13). + {bucketNames[0], objectNames[0], uploadIDs[0], 0, 3, partInfos[1], nil, true}, + // Test case with partNumberMarker set (Test number 14)-. + {bucketNames[0], objectNames[0], uploadIDs[0], 3, 2, partInfos[2], nil, true}, + } + + for i, testCase := range testCases { + actualResult, actualErr := obj.ListObjectParts(context.Background(), testCase.bucket, testCase.object, testCase.uploadID, testCase.partNumberMarker, testCase.maxParts, ObjectOptions{}) + if actualErr != nil && testCase.shouldPass { + t.Errorf("Test %d: %s: Expected to pass, but failed with: %s", i+1, instanceType, actualErr.Error()) + } + if actualErr == nil && !testCase.shouldPass { + t.Errorf("Test %d: %s: Expected to fail with \"%s\", but passed instead", i+1, instanceType, testCase.expectedErr.Error()) + } + // Failed as expected, but does it fail for the expected reason. + if actualErr != nil && !testCase.shouldPass { + if !strings.Contains(actualErr.Error(), testCase.expectedErr.Error()) { + t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.expectedErr, actualErr) + } + } + // Passes as expected, but asserting the results. + if actualErr == nil && testCase.shouldPass { + expectedResult := testCase.expectedResult + // Asserting the MaxParts. + if actualResult.MaxParts != expectedResult.MaxParts { + t.Errorf("Test %d: %s: Expected the MaxParts to be %d, but instead found it to be %d", i+1, instanceType, expectedResult.MaxParts, actualResult.MaxParts) + } + // Asserting Object Name. + if actualResult.Object != expectedResult.Object { + t.Errorf("Test %d: %s: Expected Object name to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Object, actualResult.Object) + } + // Asserting UploadID. + if actualResult.UploadID != expectedResult.UploadID { + t.Errorf("Test %d: %s: Expected UploadID to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.UploadID, actualResult.UploadID) + } + // Asserting NextPartNumberMarker. + if actualResult.NextPartNumberMarker != expectedResult.NextPartNumberMarker { + t.Errorf("Test %d: %s: Expected NextPartNumberMarker to be \"%d\", but instead found it to be \"%d\"", i+1, instanceType, expectedResult.NextPartNumberMarker, actualResult.NextPartNumberMarker) + } + // Asserting PartNumberMarker. + if actualResult.PartNumberMarker != expectedResult.PartNumberMarker { + t.Errorf("Test %d: %s: Expected PartNumberMarker to be \"%d\", but instead found it to be \"%d\"", i+1, instanceType, expectedResult.PartNumberMarker, actualResult.PartNumberMarker) + } + // Asserting the BucketName. + if actualResult.Bucket != expectedResult.Bucket { + t.Errorf("Test %d: %s: Expected Bucket to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Bucket, actualResult.Bucket) + } + // Asserting IsTruncated. + if actualResult.IsTruncated != testCase.expectedResult.IsTruncated { + t.Errorf("Test %d: %s: Expected IsTruncated to be \"%v\", but found it to \"%v\"", i+1, instanceType, expectedResult.IsTruncated, actualResult.IsTruncated) + } + // Asserting the number of Parts. + if len(expectedResult.Parts) != len(actualResult.Parts) { + t.Errorf("Test %d: %s: Expected the result to contain info of %d Parts, but found %d instead", i+1, instanceType, len(expectedResult.Parts), len(actualResult.Parts)) + } else { + // Iterating over the partInfos and asserting the fields. + for j, actualMetaData := range actualResult.Parts { + // Asserting the PartNumber in the PartInfo. + if actualMetaData.PartNumber != expectedResult.Parts[j].PartNumber { + t.Errorf("Test %d: %s: Part %d: Expected PartNumber to be \"%d\", but instead found \"%d\"", i+1, instanceType, j+1, expectedResult.Parts[j].PartNumber, actualMetaData.PartNumber) + } + // Asserting the Size in the PartInfo. + if actualMetaData.Size != expectedResult.Parts[j].Size { + t.Errorf("Test %d: %s: Part %d: Expected Part Size to be \"%d\", but instead found \"%d\"", i+1, instanceType, j+1, expectedResult.Parts[j].Size, actualMetaData.Size) + } + // Asserting the ETag in the PartInfo. + if actualMetaData.ETag != expectedResult.Parts[j].ETag { + t.Errorf("Test %d: %s: Part %d: Expected Etag to be \"%s\", but instead found \"%s\"", i+1, instanceType, j+1, expectedResult.Parts[j].ETag, actualMetaData.ETag) + } + } + } + } + } +} + +// Wrapper for calling TestListObjectParts tests for both Erasure multiple disks and single node setup. +func TestListObjectParts(t *testing.T) { + ExecObjectLayerTest(t, testListObjectParts) +} + +// testListObjectParts - test validate listing of object parts. +func testListObjectParts(obj ObjectLayer, instanceType string, t TestErrHandler) { + bucketNames := []string{"minio-bucket", "minio-2-bucket"} + objectNames := []string{"minio-object-1.txt"} + uploadIDs := []string{} + opts := ObjectOptions{} + // bucketnames[0]. + // objectNames[0]. + // uploadIds [0]. + // Create bucket before initiating NewMultipartUpload. + err := obj.MakeBucket(context.Background(), bucketNames[0], MakeBucketOptions{}) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + // Initiate Multipart Upload on the above created bucket. + res, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], opts) + if err != nil { + // Failed to create NewMultipartUpload, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + uploadIDs = append(uploadIDs, res.UploadID) + + // Create multipart parts. + // Need parts to be uploaded before MultipartLists can be called and tested. + createPartCases := []struct { + bucketName string + objName string + uploadID string + PartID int + inputReaderData string + inputMd5 string + inputDataSize int64 + expectedMd5 string + }{ + // Case 1-4. + // Creating sequence of parts for same uploadID. + // Used to ensure that the ListMultipartResult produces one output for the four parts uploaded below for the given upload ID. + {bucketNames[0], objectNames[0], uploadIDs[0], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd")), "e2fc714c4727ee9395f324cd2e7f331f"}, + {bucketNames[0], objectNames[0], uploadIDs[0], 2, "efgh", "1f7690ebdd9b4caf8fab49ca1757bf27", int64(len("efgh")), "1f7690ebdd9b4caf8fab49ca1757bf27"}, + {bucketNames[0], objectNames[0], uploadIDs[0], 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("abcd")), "09a0877d04abf8759f99adec02baf579"}, + {bucketNames[0], objectNames[0], uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd")), "e132e96a5ddad6da8b07bba6f6131fef"}, + } + sha256sum := "" + // Iterating over creatPartCases to generate multipart chunks. + for _, testCase := range createPartCases { + _, err := obj.PutObjectPart(context.Background(), testCase.bucketName, testCase.objName, testCase.uploadID, testCase.PartID, mustGetPutObjReader(t, bytes.NewBufferString(testCase.inputReaderData), testCase.inputDataSize, testCase.inputMd5, sha256sum), opts) + if err != nil { + t.Fatalf("%s : %s", instanceType, err.Error()) + } + } + + partInfos := []ListPartsInfo{ + // partinfos - 0. + { + Bucket: bucketNames[0], + Object: objectNames[0], + MaxParts: 10, + UploadID: uploadIDs[0], + Parts: []PartInfo{ + { + PartNumber: 1, + Size: 4, + ETag: "e2fc714c4727ee9395f324cd2e7f331f", + }, + { + PartNumber: 2, + Size: 4, + ETag: "1f7690ebdd9b4caf8fab49ca1757bf27", + }, + { + PartNumber: 3, + Size: 4, + ETag: "09a0877d04abf8759f99adec02baf579", + }, + { + PartNumber: 4, + Size: 4, + ETag: "e132e96a5ddad6da8b07bba6f6131fef", + }, + }, + }, + // partinfos - 1. + { + Bucket: bucketNames[0], + Object: objectNames[0], + MaxParts: 3, + NextPartNumberMarker: 3, + IsTruncated: true, + UploadID: uploadIDs[0], + Parts: []PartInfo{ + { + PartNumber: 1, + Size: 4, + ETag: "e2fc714c4727ee9395f324cd2e7f331f", + }, + { + PartNumber: 2, + Size: 4, + ETag: "1f7690ebdd9b4caf8fab49ca1757bf27", + }, + { + PartNumber: 3, + Size: 4, + ETag: "09a0877d04abf8759f99adec02baf579", + }, + }, + }, + // partinfos - 2. + { + Bucket: bucketNames[0], + Object: objectNames[0], + MaxParts: 2, + IsTruncated: false, + UploadID: uploadIDs[0], + PartNumberMarker: 3, + Parts: []PartInfo{ + { + PartNumber: 4, + Size: 4, + ETag: "e132e96a5ddad6da8b07bba6f6131fef", + }, + }, + }, + } + + // Collection of non-exhaustive ListObjectParts test cases, valid errors + // and success responses. + testCases := []struct { + bucket string + object string + uploadID string + partNumberMarker int + maxParts int + // Expected output of ListPartsInfo. + expectedResult ListPartsInfo + expectedErr error + // Flag indicating whether the test is expected to pass or not. + shouldPass bool + }{ + // Test cases with invalid bucket names (Test number 1-4). + {".test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: ".test"}, false}, + {"Test", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "Test"}, false}, + {"---", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "---"}, false}, + {"ad", "", "", 0, 0, ListPartsInfo{}, BucketNameInvalid{Bucket: "ad"}, false}, + // Test cases for listing uploadID with single part. + // Valid bucket names, but they do not exist (Test number 5-7). + {"volatile-bucket-1", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-1"}, false}, + {"volatile-bucket-2", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-2"}, false}, + {"volatile-bucket-3", "test1", "", 0, 0, ListPartsInfo{}, BucketNotFound{Bucket: "volatile-bucket-3"}, false}, + // Test case for Asserting for invalid objectName (Test number 8). + {bucketNames[0], "", "", 0, 0, ListPartsInfo{}, ObjectNameInvalid{Bucket: bucketNames[0]}, false}, + // Asserting for Invalid UploadID (Test number 9). + {bucketNames[0], objectNames[0], "abc", 0, 0, ListPartsInfo{}, InvalidUploadID{UploadID: "abc"}, false}, + // Test case for uploadID with multiple parts (Test number 12). + {bucketNames[0], objectNames[0], uploadIDs[0], 0, 10, partInfos[0], nil, true}, + // Test case with maxParts set to less than number of parts (Test number 13). + {bucketNames[0], objectNames[0], uploadIDs[0], 0, 3, partInfos[1], nil, true}, + // Test case with partNumberMarker set (Test number 14)-. + {bucketNames[0], objectNames[0], uploadIDs[0], 3, 2, partInfos[2], nil, true}, + } + + for i, testCase := range testCases { + actualResult, actualErr := obj.ListObjectParts(context.Background(), testCase.bucket, testCase.object, testCase.uploadID, testCase.partNumberMarker, testCase.maxParts, opts) + if actualErr != nil && testCase.shouldPass { + t.Errorf("Test %d: %s: Expected to pass, but failed with: %s", i+1, instanceType, actualErr.Error()) + } + if actualErr == nil && !testCase.shouldPass { + t.Errorf("Test %d: %s: Expected to fail with \"%s\", but passed instead", i+1, instanceType, testCase.expectedErr.Error()) + } + // Failed as expected, but does it fail for the expected reason. + if actualErr != nil && !testCase.shouldPass { + if !strings.Contains(actualErr.Error(), testCase.expectedErr.Error()) { + t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead", i+1, instanceType, testCase.expectedErr.Error(), actualErr.Error()) + } + } + // Passes as expected, but asserting the results. + if actualErr == nil && testCase.shouldPass { + expectedResult := testCase.expectedResult + // Asserting the MaxParts. + if actualResult.MaxParts != expectedResult.MaxParts { + t.Errorf("Test %d: %s: Expected the MaxParts to be %d, but instead found it to be %d", i+1, instanceType, expectedResult.MaxParts, actualResult.MaxParts) + } + // Asserting Object Name. + if actualResult.Object != expectedResult.Object { + t.Errorf("Test %d: %s: Expected Object name to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Object, actualResult.Object) + } + // Asserting UploadID. + if actualResult.UploadID != expectedResult.UploadID { + t.Errorf("Test %d: %s: Expected UploadID to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.UploadID, actualResult.UploadID) + } + // Asserting PartNumberMarker. + if actualResult.PartNumberMarker != expectedResult.PartNumberMarker { + t.Errorf("Test %d: %s: Expected PartNumberMarker to be \"%d\", but instead found it to be \"%d\"", i+1, instanceType, expectedResult.PartNumberMarker, actualResult.PartNumberMarker) + } + // Asserting the BucketName. + if actualResult.Bucket != expectedResult.Bucket { + t.Errorf("Test %d: %s: Expected Bucket to be \"%s\", but instead found it to be \"%s\"", i+1, instanceType, expectedResult.Bucket, actualResult.Bucket) + } + + // Asserting IsTruncated. + if actualResult.IsTruncated != testCase.expectedResult.IsTruncated { + t.Errorf("Test %d: %s: Expected IsTruncated to be \"%v\", but found it to \"%v\"", i+1, instanceType, expectedResult.IsTruncated, actualResult.IsTruncated) + continue + } + // Asserting NextPartNumberMarker. + if actualResult.NextPartNumberMarker != expectedResult.NextPartNumberMarker { + t.Errorf("Test %d: %s: Expected NextPartNumberMarker to be \"%d\", but instead found it to be \"%d\"", i+1, instanceType, expectedResult.NextPartNumberMarker, actualResult.NextPartNumberMarker) + continue + } + // Asserting the number of Parts. + if len(expectedResult.Parts) != len(actualResult.Parts) { + t.Errorf("Test %d: %s: Expected the result to contain info of %d Parts, but found %d instead", i+1, instanceType, len(expectedResult.Parts), len(actualResult.Parts)) + continue + } + // Iterating over the partInfos and asserting the fields. + for j, actualMetaData := range actualResult.Parts { + // Asserting the PartNumber in the PartInfo. + if actualMetaData.PartNumber != expectedResult.Parts[j].PartNumber { + t.Errorf("Test %d: %s: Part %d: Expected PartNumber to be \"%d\", but instead found \"%d\"", i+1, instanceType, j+1, expectedResult.Parts[j].PartNumber, actualMetaData.PartNumber) + } + // Asserting the Size in the PartInfo. + if actualMetaData.Size != expectedResult.Parts[j].Size { + t.Errorf("Test %d: %s: Part %d: Expected Part Size to be \"%d\", but instead found \"%d\"", i+1, instanceType, j+1, expectedResult.Parts[j].Size, actualMetaData.Size) + } + // Asserting the ETag in the PartInfo. + if actualMetaData.ETag != expectedResult.Parts[j].ETag { + t.Errorf("Test %d: %s: Part %d: Expected Etag to be \"%s\", but instead found \"%s\"", i+1, instanceType, j+1, expectedResult.Parts[j].ETag, actualMetaData.ETag) + } + } + } + } +} + +// Test for validating complete Multipart upload. +func TestObjectCompleteMultipartUpload(t *testing.T) { + ExecExtendedObjectLayerTest(t, testObjectCompleteMultipartUpload) +} + +// Tests validate CompleteMultipart functionality. +func testObjectCompleteMultipartUpload(obj ObjectLayer, instanceType string, t TestErrHandler) { + var err error + bucketNames := []string{"minio-bucket", "minio-2-bucket"} + objectNames := []string{"minio-object-1.txt"} + uploadIDs := []string{} + + // bucketnames[0]. + // objectNames[0]. + // uploadIds [0]. + // Create bucket before initiating NewMultipartUpload. + err = obj.MakeBucket(context.Background(), bucketNames[0], MakeBucketOptions{}) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err) + } + // Initiate Multipart Upload on the above created bucket. + res, err := obj.NewMultipartUpload(context.Background(), bucketNames[0], objectNames[0], ObjectOptions{UserDefined: map[string]string{"X-Amz-Meta-Id": "id"}}) + if err != nil { + // Failed to create NewMultipartUpload, abort. + t.Fatalf("%s : %s", instanceType, err) + } + + uploadIDs = append(uploadIDs, res.UploadID) + // Parts with size greater than 5 MiB. + // Generating a 6MiB byte array. + validPart := bytes.Repeat([]byte("abcdef"), 1*humanize.MiByte) + validPartMD5 := getMD5Hash(validPart) + // Create multipart parts. + // Need parts to be uploaded before CompleteMultiPartUpload can be called tested. + parts := []struct { + bucketName string + objName string + uploadID string + PartID int + inputReaderData string + inputMd5 string + inputDataSize int64 + }{ + // Case 1-4. + // Creating sequence of parts for same uploadID. + {bucketNames[0], objectNames[0], uploadIDs[0], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd"))}, + {bucketNames[0], objectNames[0], uploadIDs[0], 2, "efgh", "1f7690ebdd9b4caf8fab49ca1757bf27", int64(len("efgh"))}, + {bucketNames[0], objectNames[0], uploadIDs[0], 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("abcd"))}, + {bucketNames[0], objectNames[0], uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd"))}, + // Part with size larger than 5Mb. + {bucketNames[0], objectNames[0], uploadIDs[0], 5, string(validPart), validPartMD5, int64(len(validPart))}, + {bucketNames[0], objectNames[0], uploadIDs[0], 6, string(validPart), validPartMD5, int64(len(validPart))}, + {bucketNames[0], objectNames[0], uploadIDs[0], 7, string(validPart), validPartMD5, int64(len(validPart))}, + } + sha256sum := "" + var opts ObjectOptions + // Iterating over creatPartCases to generate multipart chunks. + for _, part := range parts { + _, err = obj.PutObjectPart(context.Background(), part.bucketName, part.objName, part.uploadID, part.PartID, mustGetPutObjReader(t, bytes.NewBufferString(part.inputReaderData), part.inputDataSize, part.inputMd5, sha256sum), opts) + if err != nil { + t.Fatalf("%s : %s", instanceType, err) + } + } + // Parts to be sent as input for CompleteMultipartUpload. + inputParts := []struct { + parts []CompletePart + }{ + // inputParts - 0. + // Case for replicating ETag mismatch. + { + []CompletePart{ + {ETag: "abcd", PartNumber: 1}, + }, + }, + // inputParts - 1. + // should error out with part too small. + { + []CompletePart{ + {ETag: "e2fc714c4727ee9395f324cd2e7f331f", PartNumber: 1}, + {ETag: "1f7690ebdd9b4caf8fab49ca1757bf27", PartNumber: 2}, + }, + }, + // inputParts - 2. + // Case with invalid Part number. + { + []CompletePart{ + {ETag: "e2fc714c4727ee9395f324cd2e7f331f", PartNumber: 10}, + }, + }, + // inputParts - 3. + // Case with valid part. + // Part size greater than 5MB. + { + []CompletePart{ + {ETag: fmt.Sprintf("\"\"\"\"\"%s\"\"\"", validPartMD5), PartNumber: 5}, + }, + }, + // inputParts - 4. + // Used to verify that the other remaining parts are deleted after + // a successful call to CompleteMultipartUpload. + { + []CompletePart{ + {ETag: validPartMD5, PartNumber: 6}, + }, + }, + } + s3MD5 := getCompleteMultipartMD5(inputParts[3].parts) + + // Test cases with sample input values for CompleteMultipartUpload. + testCases := []struct { + bucket string + object string + uploadID string + parts []CompletePart + // Expected output of CompleteMultipartUpload. + expectedS3MD5 string + expectedErr error + // Flag indicating whether the test is expected to pass or not. + shouldPass bool + }{ + // Test cases with invalid bucket names (Test number 1-4). + {".test", "", "", []CompletePart{}, "", BucketNameInvalid{Bucket: ".test"}, false}, + {"Test", "", "", []CompletePart{}, "", BucketNameInvalid{Bucket: "Test"}, false}, + {"---", "", "", []CompletePart{}, "", BucketNameInvalid{Bucket: "---"}, false}, + {"ad", "", "", []CompletePart{}, "", BucketNameInvalid{Bucket: "ad"}, false}, + // Test cases for listing uploadID with single part. + // Valid bucket names, but they do not exist (Test number 5-7). + {"volatile-bucket-1", "test1", "", []CompletePart{}, "", BucketNotFound{Bucket: "volatile-bucket-1"}, false}, + {"volatile-bucket-2", "test1", "", []CompletePart{}, "", BucketNotFound{Bucket: "volatile-bucket-2"}, false}, + {"volatile-bucket-3", "test1", "", []CompletePart{}, "", BucketNotFound{Bucket: "volatile-bucket-3"}, false}, + // Test case for Asserting for invalid objectName (Test number 8). + {bucketNames[0], "", "", []CompletePart{}, "", ObjectNameInvalid{Bucket: bucketNames[0]}, false}, + // Asserting for Invalid UploadID (Test number 9). + {bucketNames[0], objectNames[0], "abc", []CompletePart{}, "", InvalidUploadID{UploadID: "abc"}, false}, + // Test case with invalid Part Etag (Test number 10-11). + {bucketNames[0], objectNames[0], uploadIDs[0], []CompletePart{{ETag: "abc"}}, "", InvalidPart{}, false}, + {bucketNames[0], objectNames[0], uploadIDs[0], []CompletePart{{ETag: "abcz"}}, "", InvalidPart{}, false}, + // Part number 0 doesn't exist, expecting InvalidPart error (Test number 12). + {bucketNames[0], objectNames[0], uploadIDs[0], []CompletePart{{ETag: "abcd", PartNumber: 0}}, "", InvalidPart{}, false}, + // // Upload and PartNumber exists, But a deliberate ETag mismatch is introduced (Test number 13). + {bucketNames[0], objectNames[0], uploadIDs[0], inputParts[0].parts, "", InvalidPart{}, false}, + // Test case with non existent object name (Test number 14). + {bucketNames[0], "my-object", uploadIDs[0], []CompletePart{{ETag: "abcd", PartNumber: 1}}, "", InvalidUploadID{UploadID: uploadIDs[0]}, false}, + // Testing for Part being too small (Test number 15). + {bucketNames[0], objectNames[0], uploadIDs[0], inputParts[1].parts, "", PartTooSmall{PartNumber: 1}, false}, + // TestCase with invalid Part Number (Test number 16). + // Should error with Invalid Part . + {bucketNames[0], objectNames[0], uploadIDs[0], inputParts[2].parts, "", InvalidPart{}, false}, + // Test case with unsorted parts (Test number 17). + {bucketNames[0], objectNames[0], uploadIDs[0], inputParts[3].parts, s3MD5, nil, true}, + // The other parts will be flushed after a successful CompletePart (Test number 18). + // the case above successfully completes CompleteMultipartUpload, the remaining Parts will be flushed. + // Expecting to fail with Invalid UploadID. + {bucketNames[0], objectNames[0], uploadIDs[0], inputParts[4].parts, "", InvalidUploadID{UploadID: uploadIDs[0]}, false}, + } + + for _, testCase := range testCases { + t.(*testing.T).Run("", func(t *testing.T) { + opts = ObjectOptions{} + actualResult, actualErr := obj.CompleteMultipartUpload(t.Context(), testCase.bucket, testCase.object, testCase.uploadID, testCase.parts, ObjectOptions{}) + if actualErr != nil && testCase.shouldPass { + t.Errorf("%s: Expected to pass, but failed with: %s", instanceType, actualErr) + } + if actualErr == nil && !testCase.shouldPass { + t.Errorf("%s: Expected to fail with \"%s\", but passed instead", instanceType, testCase.expectedErr) + } + // Failed as expected, but does it fail for the expected reason. + if actualErr != nil && !testCase.shouldPass { + if reflect.TypeOf(actualErr) != reflect.TypeOf(testCase.expectedErr) { + t.Errorf("%s: Expected to fail with error \"%s\", but instead failed with error \"%s\"", instanceType, testCase.expectedErr, actualErr) + } + } + // Passes as expected, but asserting the results. + if actualErr == nil && testCase.shouldPass { + // Asserting IsTruncated. + if actualResult.ETag != testCase.expectedS3MD5 { + t.Errorf("%s: Expected the result to be \"%v\", but found it to \"%v\"", instanceType, testCase.expectedS3MD5, actualResult) + } + } + }) + } +} + +// Benchmarks for ObjectLayer.PutObjectPart(). +// The intent is to benchmark PutObjectPart for various sizes ranging from few bytes to 100MB. +// Also each of these Benchmarks are run both Erasure and FS backends. + +// BenchmarkPutObjectPart5MbFS - Benchmark FS.PutObjectPart() for object size of 5MB. +func BenchmarkPutObjectPart5MbFS(b *testing.B) { + benchmarkPutObjectPart(b, "FS", 5*humanize.MiByte) +} + +// BenchmarkPutObjectPart5MbErasure - Benchmark Erasure.PutObjectPart() for object size of 5MB. +func BenchmarkPutObjectPart5MbErasure(b *testing.B) { + benchmarkPutObjectPart(b, "Erasure", 5*humanize.MiByte) +} + +// BenchmarkPutObjectPart10MbFS - Benchmark FS.PutObjectPart() for object size of 10MB. +func BenchmarkPutObjectPart10MbFS(b *testing.B) { + benchmarkPutObjectPart(b, "FS", 10*humanize.MiByte) +} + +// BenchmarkPutObjectPart10MbErasure - Benchmark Erasure.PutObjectPart() for object size of 10MB. +func BenchmarkPutObjectPart10MbErasure(b *testing.B) { + benchmarkPutObjectPart(b, "Erasure", 10*humanize.MiByte) +} + +// BenchmarkPutObjectPart25MbFS - Benchmark FS.PutObjectPart() for object size of 25MB. +func BenchmarkPutObjectPart25MbFS(b *testing.B) { + benchmarkPutObjectPart(b, "FS", 25*humanize.MiByte) +} + +// BenchmarkPutObjectPart25MbErasure - Benchmark Erasure.PutObjectPart() for object size of 25MB. +func BenchmarkPutObjectPart25MbErasure(b *testing.B) { + benchmarkPutObjectPart(b, "Erasure", 25*humanize.MiByte) +} + +// BenchmarkPutObjectPart50MbFS - Benchmark FS.PutObjectPart() for object size of 50MB. +func BenchmarkPutObjectPart50MbFS(b *testing.B) { + benchmarkPutObjectPart(b, "FS", 50*humanize.MiByte) +} + +// BenchmarkPutObjectPart50MbErasure - Benchmark Erasure.PutObjectPart() for object size of 50MB. +func BenchmarkPutObjectPart50MbErasure(b *testing.B) { + benchmarkPutObjectPart(b, "Erasure", 50*humanize.MiByte) +} diff --git a/cmd/object-api-options.go b/cmd/object-api-options.go new file mode 100644 index 00000000..6482f200 --- /dev/null +++ b/cmd/object-api-options.go @@ -0,0 +1,496 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "github.com/minio/minio-go/v7/pkg/encrypt" + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" +) + +func getDefaultOpts(header http.Header, copySource bool, metadata map[string]string) (opts ObjectOptions, err error) { + var clientKey [32]byte + var sse encrypt.ServerSide + + opts = ObjectOptions{UserDefined: metadata} + if v, ok := header[xhttp.MinIOSourceProxyRequest]; ok { + opts.ProxyHeaderSet = true + opts.ProxyRequest = strings.Join(v, "") == "true" + } + if _, ok := header[xhttp.MinIOSourceReplicationRequest]; ok { + opts.ReplicationRequest = true + } + opts.Speedtest = header.Get(globalObjectPerfUserMetadata) != "" + + if copySource { + if crypto.SSECopy.IsRequested(header) { + clientKey, err = crypto.SSECopy.ParseHTTP(header) + if err != nil { + return opts, err + } + if sse, err = encrypt.NewSSEC(clientKey[:]); err != nil { + return opts, err + } + opts.ServerSideEncryption = encrypt.SSECopy(sse) + return opts, err + } + return opts, err + } + + if crypto.SSEC.IsRequested(header) { + clientKey, err = crypto.SSEC.ParseHTTP(header) + if err != nil { + return opts, err + } + if sse, err = encrypt.NewSSEC(clientKey[:]); err != nil { + return opts, err + } + opts.ServerSideEncryption = sse + return opts, err + } + if crypto.S3.IsRequested(header) || (metadata != nil && crypto.S3.IsEncrypted(metadata)) { + opts.ServerSideEncryption = encrypt.NewSSE() + } + + return opts, err +} + +// get ObjectOptions for GET calls from encryption headers +func getOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) { + var opts ObjectOptions + + var partNumber int + var err error + if pn := r.Form.Get(xhttp.PartNumber); pn != "" { + partNumber, err = strconv.Atoi(pn) + if err != nil { + return opts, err + } + if isMaxPartID(partNumber) { + return opts, errInvalidMaxParts + } + if partNumber <= 0 { + return opts, errInvalidArgument + } + } + + vid := strings.TrimSpace(r.Form.Get(xhttp.VersionID)) + if vid != "" && vid != nullVersionID { + if _, err := uuid.Parse(vid); err != nil { + return opts, InvalidVersionID{ + Bucket: bucket, + Object: object, + VersionID: vid, + } + } + } + + // default case of passing encryption headers to backend + opts, err = getDefaultOpts(r.Header, false, nil) + if err != nil { + return opts, err + } + opts.PartNumber = partNumber + opts.VersionID = vid + + delMarker, err := parseBoolHeader(bucket, object, r.Header, xhttp.MinIOSourceDeleteMarker) + if err != nil { + return opts, err + } + opts.DeleteMarker = delMarker + + replReadyCheck, err := parseBoolHeader(bucket, object, r.Header, xhttp.MinIOCheckDMReplicationReady) + if err != nil { + return opts, err + } + opts.CheckDMReplicationReady = replReadyCheck + + opts.Tagging = r.Header.Get(xhttp.AmzTagDirective) == accessDirective + opts.Versioned = globalBucketVersioningSys.PrefixEnabled(bucket, object) + opts.VersionSuspended = globalBucketVersioningSys.PrefixSuspended(bucket, object) + return opts, nil +} + +func getAndValidateAttributesOpts(ctx context.Context, w http.ResponseWriter, r *http.Request, bucket, object string) (opts ObjectOptions, valid bool) { + var argumentName string + var argumentValue string + var apiErr APIError + var err error + valid = true + + defer func() { + if valid { + return + } + + errResp := objectAttributesErrorResponse{ + ArgumentName: &argumentName, + ArgumentValue: &argumentValue, + APIErrorResponse: getAPIErrorResponse( + ctx, + apiErr, + r.URL.Path, + w.Header().Get(xhttp.AmzRequestID), + w.Header().Get(xhttp.AmzRequestHostID), + ), + } + + writeResponse(w, apiErr.HTTPStatusCode, encodeResponse(errResp), mimeXML) + }() + + opts, err = getOpts(ctx, r, bucket, object) + if err != nil { + switch vErr := err.(type) { + case InvalidVersionID: + apiErr = toAPIError(ctx, vErr) + argumentName = strings.ToLower("versionId") + argumentValue = vErr.VersionID + default: + apiErr = toAPIError(ctx, vErr) + } + valid = false + return opts, valid + } + + opts.MaxParts, err = parseIntHeader(bucket, object, r.Header, xhttp.AmzMaxParts) + if err != nil { + apiErr = toAPIError(ctx, err) + argumentName = strings.ToLower(xhttp.AmzMaxParts) + valid = false + return opts, valid + } + + if opts.MaxParts == 0 { + opts.MaxParts = maxPartsList + } + + opts.PartNumberMarker, err = parseIntHeader(bucket, object, r.Header, xhttp.AmzPartNumberMarker) + if err != nil { + apiErr = toAPIError(ctx, err) + argumentName = strings.ToLower(xhttp.AmzPartNumberMarker) + valid = false + return opts, valid + } + + opts.ObjectAttributes = parseObjectAttributes(r.Header) + if len(opts.ObjectAttributes) < 1 { + apiErr = errorCodes.ToAPIErr(ErrInvalidAttributeName) + argumentName = strings.ToLower(xhttp.AmzObjectAttributes) + valid = false + return opts, valid + } + + for tag := range opts.ObjectAttributes { + switch tag { + case xhttp.ETag: + case xhttp.Checksum: + case xhttp.StorageClass: + case xhttp.ObjectSize: + case xhttp.ObjectParts: + default: + apiErr = errorCodes.ToAPIErr(ErrInvalidAttributeName) + argumentName = strings.ToLower(xhttp.AmzObjectAttributes) + argumentValue = tag + valid = false + return opts, valid + } + } + + return opts, valid +} + +func parseObjectAttributes(h http.Header) (attributes map[string]struct{}) { + attributes = make(map[string]struct{}) + for _, headerVal := range h.Values(xhttp.AmzObjectAttributes) { + for v := range strings.SplitSeq(strings.TrimSpace(headerVal), ",") { + if v != "" { + attributes[v] = struct{}{} + } + } + } + + return attributes +} + +func parseIntHeader(bucket, object string, h http.Header, headerName string) (value int, err error) { + stringInt := strings.TrimSpace(h.Get(headerName)) + if stringInt == "" { + return value, err + } + value, err = strconv.Atoi(stringInt) + if err != nil { + return 0, InvalidArgument{ + Bucket: bucket, + Object: object, + Err: fmt.Errorf("Unable to parse %s, value should be an integer", headerName), + } + } + return value, err +} + +func parseBoolHeader(bucket, object string, h http.Header, headerName string) (bool, error) { + value := strings.TrimSpace(h.Get(headerName)) + if value != "" { + switch value { + case "true": + return true, nil + case "false": + default: + return false, InvalidArgument{ + Bucket: bucket, + Object: object, + Err: fmt.Errorf("Unable to parse %s, value should be either 'true' or 'false'", headerName), + } + } + } + return false, nil +} + +func delOpts(ctx context.Context, r *http.Request, bucket, object string) (opts ObjectOptions, err error) { + opts, err = getOpts(ctx, r, bucket, object) + if err != nil { + return opts, err + } + + deletePrefix := false + if d := r.Header.Get(xhttp.MinIOForceDelete); d != "" { + if b, err := strconv.ParseBool(d); err == nil { + deletePrefix = b + } else { + return opts, err + } + } + + opts.DeletePrefix = deletePrefix + opts.Versioned = globalBucketVersioningSys.PrefixEnabled(bucket, object) + // Objects matching prefixes should not leave delete markers, + // dramatically reduces namespace pollution while keeping the + // benefits of replication, make sure to apply version suspension + // only at bucket level instead. + opts.VersionSuspended = globalBucketVersioningSys.Suspended(bucket) + // For directory objects, delete `null` version permanently. + if isDirObject(object) && opts.VersionID == "" { + opts.VersionID = nullVersionID + } + + delMarker, err := parseBoolHeader(bucket, object, r.Header, xhttp.MinIOSourceDeleteMarker) + if err != nil { + return opts, err + } + opts.DeleteMarker = delMarker + + mtime := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceMTime)) + if mtime != "" { + opts.MTime, err = time.Parse(time.RFC3339Nano, mtime) + if err != nil { + return opts, InvalidArgument{ + Bucket: bucket, + Object: object, + Err: fmt.Errorf("Unable to parse %s, failed with %w", xhttp.MinIOSourceMTime, err), + } + } + } + return opts, nil +} + +// get ObjectOptions for PUT calls from encryption headers and metadata +func putOptsFromReq(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) { + return putOpts(ctx, bucket, object, r.Form.Get(xhttp.VersionID), r.Header, metadata) +} + +func putOpts(ctx context.Context, bucket, object, vid string, hdrs http.Header, metadata map[string]string) (opts ObjectOptions, err error) { + versioned := globalBucketVersioningSys.PrefixEnabled(bucket, object) + versionSuspended := globalBucketVersioningSys.PrefixSuspended(bucket, object) + + vid = strings.TrimSpace(vid) + if vid != "" && vid != nullVersionID { + _, err := uuid.Parse(vid) + if err != nil { + return opts, InvalidVersionID{ + Bucket: bucket, + Object: object, + VersionID: vid, + } + } + if !versioned { + return opts, InvalidArgument{ + Bucket: bucket, + Object: object, + Err: fmt.Errorf("VersionID specified %s, but versioning not enabled on bucket=%s", opts.VersionID, bucket), + } + } + } + opts, err = putOptsFromHeaders(ctx, hdrs, metadata) + if err != nil { + return opts, InvalidArgument{ + Bucket: bucket, + Object: object, + Err: err, + } + } + + opts.VersionID = vid + opts.Versioned = versioned + opts.VersionSuspended = versionSuspended + + // For directory objects skip creating new versions. + if isDirObject(object) && vid == "" { + opts.VersionID = nullVersionID + } + + return opts, nil +} + +func putOptsFromHeaders(ctx context.Context, hdr http.Header, metadata map[string]string) (opts ObjectOptions, err error) { + mtimeStr := strings.TrimSpace(hdr.Get(xhttp.MinIOSourceMTime)) + var mtime time.Time + if mtimeStr != "" { + mtime, err = time.Parse(time.RFC3339Nano, mtimeStr) + if err != nil { + return opts, fmt.Errorf("Unable to parse %s, failed with %w", xhttp.MinIOSourceMTime, err) + } + } + retaintimeStr := strings.TrimSpace(hdr.Get(xhttp.MinIOSourceObjectRetentionTimestamp)) + var retaintimestmp time.Time + if retaintimeStr != "" { + retaintimestmp, err = time.Parse(time.RFC3339, retaintimeStr) + if err != nil { + return opts, fmt.Errorf("Unable to parse %s, failed with %w", xhttp.MinIOSourceObjectRetentionTimestamp, err) + } + } + + lholdtimeStr := strings.TrimSpace(hdr.Get(xhttp.MinIOSourceObjectLegalHoldTimestamp)) + var lholdtimestmp time.Time + if lholdtimeStr != "" { + lholdtimestmp, err = time.Parse(time.RFC3339, lholdtimeStr) + if err != nil { + return opts, fmt.Errorf("Unable to parse %s, failed with %w", xhttp.MinIOSourceObjectLegalHoldTimestamp, err) + } + } + tagtimeStr := strings.TrimSpace(hdr.Get(xhttp.MinIOSourceTaggingTimestamp)) + var taggingtimestmp time.Time + if tagtimeStr != "" { + taggingtimestmp, err = time.Parse(time.RFC3339, tagtimeStr) + if err != nil { + return opts, fmt.Errorf("Unable to parse %s, failed with %w", xhttp.MinIOSourceTaggingTimestamp, err) + } + } + + if metadata == nil { + metadata = make(map[string]string) + } + + etag := strings.TrimSpace(hdr.Get(xhttp.MinIOSourceETag)) + if crypto.S3KMS.IsRequested(hdr) { + keyID, context, err := crypto.S3KMS.ParseHTTP(hdr) + if err != nil { + return ObjectOptions{}, err + } + sseKms, err := encrypt.NewSSEKMS(keyID, context) + if err != nil { + return ObjectOptions{}, err + } + op := ObjectOptions{ + ServerSideEncryption: sseKms, + UserDefined: metadata, + MTime: mtime, + PreserveETag: etag, + } + return op, nil + } + // default case of passing encryption headers and UserDefined metadata to backend + opts, err = getDefaultOpts(hdr, false, metadata) + if err != nil { + return opts, err + } + + opts.MTime = mtime + opts.ReplicationSourceLegalholdTimestamp = lholdtimestmp + opts.ReplicationSourceRetentionTimestamp = retaintimestmp + opts.ReplicationSourceTaggingTimestamp = taggingtimestmp + opts.PreserveETag = etag + + return opts, nil +} + +// get ObjectOptions for Copy calls with encryption headers provided on the target side and source side metadata +func copyDstOpts(ctx context.Context, r *http.Request, bucket, object string, metadata map[string]string) (opts ObjectOptions, err error) { + return putOptsFromReq(ctx, r, bucket, object, metadata) +} + +// get ObjectOptions for Copy calls with encryption headers provided on the source side +func copySrcOpts(ctx context.Context, r *http.Request, bucket, object string) (ObjectOptions, error) { + var opts ObjectOptions + + // default case of passing encryption headers to backend + opts, err := getDefaultOpts(r.Header, false, nil) + if err != nil { + return opts, err + } + return opts, nil +} + +// get ObjectOptions for CompleteMultipart calls +func completeMultipartOpts(ctx context.Context, r *http.Request, bucket, object string) (opts ObjectOptions, err error) { + mtimeStr := strings.TrimSpace(r.Header.Get(xhttp.MinIOSourceMTime)) + var mtime time.Time + if mtimeStr != "" { + mtime, err = time.Parse(time.RFC3339Nano, mtimeStr) + if err != nil { + return opts, InvalidArgument{ + Bucket: bucket, + Object: object, + Err: fmt.Errorf("Unable to parse %s, failed with %w", xhttp.MinIOSourceMTime, err), + } + } + } + + opts.WantChecksum, err = hash.GetContentChecksum(r.Header) + if err != nil { + return opts, err + } + opts.MTime = mtime + opts.UserDefined = make(map[string]string) + // Transfer SSEC key in opts.EncryptFn + if crypto.SSEC.IsRequested(r.Header) { + key, err := ParseSSECustomerRequest(r) + if err == nil { + // Set EncryptFn to return SSEC key + opts.EncryptFn = func(baseKey string, data []byte) []byte { + return key + } + } + } + if _, ok := r.Header[xhttp.MinIOSourceReplicationRequest]; ok { + opts.ReplicationRequest = true + opts.UserDefined[ReservedMetadataPrefix+"Actual-Object-Size"] = r.Header.Get(xhttp.MinIOReplicationActualObjectSize) + } + if r.Header.Get(ReplicationSsecChecksumHeader) != "" { + opts.UserDefined[ReplicationSsecChecksumHeader] = r.Header.Get(ReplicationSsecChecksumHeader) + } + return opts, nil +} diff --git a/cmd/object-api-options_test.go b/cmd/object-api-options_test.go new file mode 100644 index 00000000..661372cd --- /dev/null +++ b/cmd/object-api-options_test.go @@ -0,0 +1,78 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "net/http" + "net/http/httptest" + "reflect" + "testing" + + xhttp "github.com/minio/minio/internal/http" +) + +// TestGetAndValidateAttributesOpts is currently minimal and covers a subset of getAndValidateAttributesOpts(), +// it is intended to be expanded when the function is worked on in the future. +func TestGetAndValidateAttributesOpts(t *testing.T) { + globalBucketVersioningSys = &BucketVersioningSys{} + bucket := minioMetaBucket + ctx := t.Context() + testCases := []struct { + name string + headers http.Header + wantObjectAttrs map[string]struct{} + }{ + { + name: "empty header", + headers: http.Header{}, + wantObjectAttrs: map[string]struct{}{}, + }, + { + name: "single header line", + headers: http.Header{ + xhttp.AmzObjectAttributes: []string{"test1,test2"}, + }, + wantObjectAttrs: map[string]struct{}{ + "test1": {}, "test2": {}, + }, + }, + { + name: "multiple header lines with some duplicates", + headers: http.Header{ + xhttp.AmzObjectAttributes: []string{"test1,test2", "test3,test4", "test4,test3"}, + }, + wantObjectAttrs: map[string]struct{}{ + "test1": {}, "test2": {}, "test3": {}, "test4": {}, + }, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + rec := httptest.NewRecorder() + req := httptest.NewRequest("GET", "/test", nil) + req.Header = testCase.headers + + opts, _ := getAndValidateAttributesOpts(ctx, rec, req, bucket, "testobject") + + if !reflect.DeepEqual(opts.ObjectAttributes, testCase.wantObjectAttrs) { + t.Errorf("want opts %v, got %v", testCase.wantObjectAttrs, opts.ObjectAttributes) + } + }) + } +} diff --git a/cmd/object-api-putobject_test.go b/cmd/object-api-putobject_test.go new file mode 100644 index 00000000..0c449320 --- /dev/null +++ b/cmd/object-api-putobject_test.go @@ -0,0 +1,614 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/hex" + "errors" + "os" + "path" + "testing" + + "github.com/dustin/go-humanize" + "github.com/minio/minio/internal/hash" + "github.com/minio/minio/internal/ioutil" +) + +func md5Header(data []byte) map[string]string { + return map[string]string{"etag": getMD5Hash(data)} +} + +// Wrapper for calling PutObject tests for both Erasure multiple disks and single node setup. +func TestObjectAPIPutObjectSingle(t *testing.T) { + ExecExtendedObjectLayerTest(t, testObjectAPIPutObject) +} + +// Tests validate correctness of PutObject. +func testObjectAPIPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) { + // Generating cases for which the PutObject fails. + bucket := "minio-bucket" + object := "minio-object" + + // Create bucket. + err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{}) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + // Creating a dummy bucket for tests. + err = obj.MakeBucket(context.Background(), "unused-bucket", MakeBucketOptions{}) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + var ( + nilBytes []byte + data = []byte("hello") + fiveMBBytes = bytes.Repeat([]byte("a"), 5*humanize.MiByte) + ) + invalidMD5 := getMD5Hash([]byte("meh")) + invalidMD5Header := md5Header([]byte("meh")) + + testCases := []struct { + bucketName string + objName string + inputData []byte + inputMeta map[string]string + inputSHA256 string + inputDataSize int64 + // expected error output. + expectedMd5 string + expectedError error + }{ + // Cases with invalid bucket name. + 0: {bucketName: ".test", objName: "obj", inputData: []byte(""), expectedError: BucketNameInvalid{Bucket: ".test"}}, + 1: {bucketName: "------", objName: "obj", inputData: []byte(""), expectedError: BucketNameInvalid{Bucket: "------"}}, + 2: { + bucketName: "$this-is-not-valid-too", objName: "obj", inputData: []byte(""), + expectedError: BucketNameInvalid{Bucket: "$this-is-not-valid-too"}, + }, + 3: {bucketName: "a", objName: "obj", inputData: []byte(""), expectedError: BucketNameInvalid{Bucket: "a"}}, + + // Case with invalid object names. + 4: {bucketName: bucket, inputData: []byte(""), expectedError: ObjectNameInvalid{Bucket: bucket, Object: ""}}, + + // Valid object and bucket names but non-existent bucket. + 5: {bucketName: "abc", objName: "def", inputData: []byte(""), expectedError: BucketNotFound{Bucket: "abc"}}, + + // Input to replicate Md5 mismatch. + 6: { + bucketName: bucket, objName: object, inputData: []byte(""), + inputMeta: map[string]string{"etag": "d41d8cd98f00b204e9800998ecf8427f"}, + expectedError: hash.BadDigest{ExpectedMD5: "d41d8cd98f00b204e9800998ecf8427f", CalculatedMD5: "d41d8cd98f00b204e9800998ecf8427e"}, + }, + + // With incorrect sha256. + 7: { + bucketName: bucket, objName: object, inputData: []byte("abcd"), + inputMeta: map[string]string{"etag": "e2fc714c4727ee9395f324cd2e7f331f"}, + inputSHA256: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031580", inputDataSize: int64(len("abcd")), + expectedError: hash.SHA256Mismatch{ + ExpectedSHA256: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031580", + CalculatedSHA256: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", + }, + }, + + // Input with size more than the size of actual data inside the reader. + 8: { + bucketName: bucket, objName: object, inputData: []byte("abcd"), + inputMeta: map[string]string{"etag": "e2fc714c4727ee9395f324cd2e7f331e"}, inputDataSize: int64(len("abcd") + 1), + expectedError: hash.BadDigest{ExpectedMD5: "e2fc714c4727ee9395f324cd2e7f331e", CalculatedMD5: "e2fc714c4727ee9395f324cd2e7f331f"}, + }, + + // Input with size less than the size of actual data inside the reader. + 9: { + bucketName: bucket, objName: object, inputData: []byte("abcd"), + inputMeta: map[string]string{"etag": "900150983cd24fb0d6963f7d28e17f73"}, inputDataSize: int64(len("abcd") - 1), + expectedError: ioutil.ErrOverread, + }, + + // Validating for success cases. + 10: {bucketName: bucket, objName: object, inputData: []byte("abcd"), inputMeta: map[string]string{"etag": "e2fc714c4727ee9395f324cd2e7f331f"}, inputDataSize: int64(len("abcd"))}, + 11: {bucketName: bucket, objName: object, inputData: []byte("efgh"), inputMeta: map[string]string{"etag": "1f7690ebdd9b4caf8fab49ca1757bf27"}, inputDataSize: int64(len("efgh"))}, + 12: {bucketName: bucket, objName: object, inputData: []byte("ijkl"), inputMeta: map[string]string{"etag": "09a0877d04abf8759f99adec02baf579"}, inputDataSize: int64(len("ijkl"))}, + 13: {bucketName: bucket, objName: object, inputData: []byte("mnop"), inputMeta: map[string]string{"etag": "e132e96a5ddad6da8b07bba6f6131fef"}, inputDataSize: int64(len("mnop"))}, + + // With no metadata + 14: {bucketName: bucket, objName: object, inputData: data, inputDataSize: int64(len(data)), expectedMd5: getMD5Hash(data)}, + 15: {bucketName: bucket, objName: object, inputData: nilBytes, inputDataSize: int64(len(nilBytes)), expectedMd5: getMD5Hash(nilBytes)}, + 16: {bucketName: bucket, objName: object, inputData: fiveMBBytes, inputDataSize: int64(len(fiveMBBytes)), expectedMd5: getMD5Hash(fiveMBBytes)}, + + // With arbitrary metadata + 17: {bucketName: bucket, objName: object, inputData: data, inputMeta: map[string]string{"answer": "42"}, inputDataSize: int64(len(data)), expectedMd5: getMD5Hash(data)}, + 18: {bucketName: bucket, objName: object, inputData: nilBytes, inputMeta: map[string]string{"answer": "42"}, inputDataSize: int64(len(nilBytes)), expectedMd5: getMD5Hash(nilBytes)}, + 19: {bucketName: bucket, objName: object, inputData: fiveMBBytes, inputMeta: map[string]string{"answer": "42"}, inputDataSize: int64(len(fiveMBBytes)), expectedMd5: getMD5Hash(fiveMBBytes)}, + + // With valid md5sum and sha256. + 20: {bucketName: bucket, objName: object, inputData: data, inputMeta: md5Header(data), inputSHA256: getSHA256Hash(data), inputDataSize: int64(len(data)), expectedMd5: getMD5Hash(data)}, + 21: {bucketName: bucket, objName: object, inputData: nilBytes, inputMeta: md5Header(nilBytes), inputSHA256: getSHA256Hash(nilBytes), inputDataSize: int64(len(nilBytes)), expectedMd5: getMD5Hash(nilBytes)}, + 22: {bucketName: bucket, objName: object, inputData: fiveMBBytes, inputMeta: md5Header(fiveMBBytes), inputSHA256: getSHA256Hash(fiveMBBytes), inputDataSize: int64(len(fiveMBBytes)), expectedMd5: getMD5Hash(fiveMBBytes)}, + + // data with invalid md5sum in header + 23: { + bucketName: bucket, objName: object, inputData: data, inputMeta: invalidMD5Header, inputDataSize: int64(len(data)), expectedMd5: getMD5Hash(data), + expectedError: hash.BadDigest{ExpectedMD5: invalidMD5, CalculatedMD5: getMD5Hash(data)}, + }, + 24: { + bucketName: bucket, objName: object, inputData: nilBytes, inputMeta: invalidMD5Header, inputDataSize: int64(len(nilBytes)), expectedMd5: getMD5Hash(nilBytes), + expectedError: hash.BadDigest{ExpectedMD5: invalidMD5, CalculatedMD5: getMD5Hash(nilBytes)}, + }, + 25: { + bucketName: bucket, objName: object, inputData: fiveMBBytes, inputMeta: invalidMD5Header, inputDataSize: int64(len(fiveMBBytes)), expectedMd5: getMD5Hash(fiveMBBytes), + expectedError: hash.BadDigest{ExpectedMD5: invalidMD5, CalculatedMD5: getMD5Hash(fiveMBBytes)}, + }, + + // data with size different from the actual number of bytes available in the reader + 26: {bucketName: bucket, objName: object, inputData: data, inputDataSize: int64(len(data) - 1), expectedMd5: getMD5Hash(data[:len(data)-1]), expectedError: ioutil.ErrOverread}, + 27: {bucketName: bucket, objName: object, inputData: nilBytes, inputDataSize: int64(len(nilBytes) + 1), expectedMd5: getMD5Hash(nilBytes), expectedError: IncompleteBody{Bucket: bucket, Object: object}}, + 28: {bucketName: bucket, objName: object, inputData: fiveMBBytes, expectedMd5: getMD5Hash(fiveMBBytes), expectedError: ioutil.ErrOverread}, + + // valid data with X-Amz-Meta- meta + 29: {bucketName: bucket, objName: object, inputData: data, inputMeta: map[string]string{"X-Amz-Meta-AppID": "a42"}, inputDataSize: int64(len(data)), expectedMd5: getMD5Hash(data)}, + + // Put an empty object with a trailing slash + 30: {bucketName: bucket, objName: "emptydir/", inputData: []byte{}, expectedMd5: getMD5Hash([]byte{})}, + // Put an object inside the empty directory + 31: {bucketName: bucket, objName: "emptydir/" + object, inputData: data, inputDataSize: int64(len(data)), expectedMd5: getMD5Hash(data)}, + // Put the empty object with a trailing slash again (refer to Test case 30), this needs to succeed + 32: {bucketName: bucket, objName: "emptydir/", inputData: []byte{}, expectedMd5: getMD5Hash([]byte{})}, + + // With invalid crc32. + 33: { + bucketName: bucket, objName: object, inputData: []byte("abcd"), + inputMeta: map[string]string{"etag": "e2fc714c4727ee9395f324cd2e7f331f", "x-amz-checksum-crc32": "abcd"}, + inputDataSize: int64(len("abcd")), + }, + } + for i, testCase := range testCases { + in := mustGetPutObjReader(t, bytes.NewReader(testCase.inputData), testCase.inputDataSize, testCase.inputMeta["etag"], testCase.inputSHA256) + objInfo, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, in, ObjectOptions{UserDefined: testCase.inputMeta}) + if actualErr != nil && testCase.expectedError == nil { + t.Errorf("Test %d: %s: Expected to pass, but failed with: error %s.", i, instanceType, actualErr.Error()) + continue + } + if actualErr == nil && testCase.expectedError != nil { + t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but passed instead.", i, instanceType, testCase.expectedError.Error()) + continue + } + // Failed as expected, but does it fail for the expected reason. + if actualErr != nil && actualErr != testCase.expectedError { + t.Errorf("Test %d: %s: Expected to fail with error \"%v\", but instead failed with error \"%v\" instead.", i, instanceType, testCase.expectedError, actualErr) + continue + } + // Test passes as expected, but the output values are verified for correctness here. + if actualErr == nil { + // Asserting whether the md5 output is correct. + if expectedMD5, ok := testCase.inputMeta["etag"]; ok && expectedMD5 != objInfo.ETag { + t.Errorf("Test %d: %s: Calculated Md5 different from the actual one %s.", i, instanceType, objInfo.ETag) + continue + } + } + } +} + +// Wrapper for calling PutObject tests for both Erasure multiple disks case +// when quorum is not available. +func TestObjectAPIPutObjectDiskNotFound(t *testing.T) { + ExecObjectLayerDiskAlteredTest(t, testObjectAPIPutObjectDiskNotFound) +} + +// Tests validate correctness of PutObject. +func testObjectAPIPutObjectDiskNotFound(obj ObjectLayer, instanceType string, disks []string, t *testing.T) { + // Generating cases for which the PutObject fails. + bucket := "minio-bucket" + object := "minio-object" + + // Create bucket. + err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{}) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + // Creating a dummy bucket for tests. + err = obj.MakeBucket(context.Background(), "unused-bucket", MakeBucketOptions{}) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + // Take 4 disks down, one more we loose quorum on 16 disk node. + for _, disk := range disks[:4] { + os.RemoveAll(disk) + } + + testCases := []struct { + bucketName string + objName string + inputData []byte + inputMeta map[string]string + inputDataSize int64 + // flag indicating whether the test should pass. + shouldPass bool + // expected error output. + expectedMd5 string + expectedError error + }{ + // Validating for success cases. + {bucket, object, []byte("abcd"), map[string]string{"etag": "e2fc714c4727ee9395f324cd2e7f331f"}, int64(len("abcd")), true, "", nil}, + {bucket, object, []byte("efgh"), map[string]string{"etag": "1f7690ebdd9b4caf8fab49ca1757bf27"}, int64(len("efgh")), true, "", nil}, + {bucket, object, []byte("ijkl"), map[string]string{"etag": "09a0877d04abf8759f99adec02baf579"}, int64(len("ijkl")), true, "", nil}, + {bucket, object, []byte("mnop"), map[string]string{"etag": "e132e96a5ddad6da8b07bba6f6131fef"}, int64(len("mnop")), true, "", nil}, + } + + sha256sum := "" + for i, testCase := range testCases { + objInfo, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetPutObjReader(t, bytes.NewReader(testCase.inputData), testCase.inputDataSize, testCase.inputMeta["etag"], sha256sum), ObjectOptions{UserDefined: testCase.inputMeta}) + if actualErr != nil && testCase.shouldPass { + t.Errorf("Test %d: %s: Expected to pass, but failed with: %s.", i+1, instanceType, actualErr.Error()) + } + if actualErr == nil && !testCase.shouldPass { + t.Errorf("Test %d: %s: Expected to fail with \"%s\", but passed instead.", i+1, instanceType, testCase.expectedError.Error()) + } + // Failed as expected, but does it fail for the expected reason. + if actualErr != nil && !testCase.shouldPass { + if testCase.expectedError.Error() != actualErr.Error() { + t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", i+1, + instanceType, testCase.expectedError.Error(), actualErr.Error()) + } + } + // Test passes as expected, but the output values are verified for correctness here. + if actualErr == nil && testCase.shouldPass { + // Asserting whether the md5 output is correct. + if testCase.inputMeta["etag"] != objInfo.ETag { + t.Errorf("Test %d: %s: Calculated Md5 different from the actual one %s.", i+1, instanceType, objInfo.ETag) + } + } + } + + // This causes quorum failure verify. + os.RemoveAll(disks[len(disks)-1]) + + // Validate the last test. + testCase := struct { + bucketName string + objName string + inputData []byte + inputMeta map[string]string + inputDataSize int64 + // flag indicating whether the test should pass. + shouldPass bool + // expected error output. + expectedMd5 string + expectedError error + }{ + bucket, + object, + []byte("mnop"), + map[string]string{"etag": "e132e96a5ddad6da8b07bba6f6131fef"}, + int64(len("mnop")), + false, + "", + errErasureWriteQuorum, + } + + _, actualErr := obj.PutObject(context.Background(), testCase.bucketName, testCase.objName, mustGetPutObjReader(t, bytes.NewReader(testCase.inputData), testCase.inputDataSize, testCase.inputMeta["etag"], sha256sum), ObjectOptions{UserDefined: testCase.inputMeta}) + if actualErr != nil && testCase.shouldPass { + t.Errorf("Test %d: %s: Expected to pass, but failed with: %s.", len(testCases)+1, instanceType, actualErr.Error()) + } + // Failed as expected, but does it fail for the expected reason. + if actualErr != nil && !testCase.shouldPass { + if !errors.Is(actualErr, testCase.expectedError) { + t.Errorf("Test %d: %s: Expected to fail with error \"%s\", but instead failed with error \"%s\" instead.", len(testCases)+1, instanceType, testCase.expectedError.Error(), actualErr.Error()) + } + } +} + +// Wrapper for calling PutObject tests for both Erasure multiple disks and single node setup. +func TestObjectAPIPutObjectStaleFiles(t *testing.T) { + ExecObjectLayerStaleFilesTest(t, testObjectAPIPutObjectStaleFiles) +} + +// Tests validate correctness of PutObject. +func testObjectAPIPutObjectStaleFiles(obj ObjectLayer, instanceType string, disks []string, t *testing.T) { + // Generating cases for which the PutObject fails. + bucket := "minio-bucket" + object := "minio-object" + + // Create bucket. + err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{}) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + data := []byte("hello, world") + // Create object. + _, err = obj.PutObject(context.Background(), bucket, object, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), "", ""), ObjectOptions{}) + if err != nil { + // Failed to create object, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + for _, disk := range disks { + tmpMetaDir := path.Join(disk, minioMetaTmpBucket) + files, err := os.ReadDir(tmpMetaDir) + if err != nil { + t.Fatal(err) + } + var found bool + for _, fi := range files { + if fi.Name() == ".trash" { + continue + } + found = true + } + if found { + t.Fatalf("%s: expected: empty, got: non-empty %#v", minioMetaTmpBucket, files) + } + } +} + +// Wrapper for calling Multipart PutObject tests for both Erasure multiple disks and single node setup. +func TestObjectAPIMultipartPutObjectStaleFiles(t *testing.T) { + ExecObjectLayerStaleFilesTest(t, testObjectAPIMultipartPutObjectStaleFiles) +} + +// Tests validate correctness of PutObject. +func testObjectAPIMultipartPutObjectStaleFiles(obj ObjectLayer, instanceType string, disks []string, t *testing.T) { + // Generating cases for which the PutObject fails. + bucket := "minio-bucket" + object := "minio-object" + + // Create bucket. + err := obj.MakeBucket(context.Background(), bucket, MakeBucketOptions{}) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + opts := ObjectOptions{} + // Initiate Multipart Upload on the above created bucket. + res, err := obj.NewMultipartUpload(context.Background(), bucket, object, opts) + if err != nil { + // Failed to create NewMultipartUpload, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + uploadID := res.UploadID + + // Upload part1. + fiveMBBytes := bytes.Repeat([]byte("a"), 5*humanize.MiByte) + md5Writer := md5.New() + md5Writer.Write(fiveMBBytes) + etag1 := hex.EncodeToString(md5Writer.Sum(nil)) + sha256sum := "" + _, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader(fiveMBBytes), int64(len(fiveMBBytes)), etag1, sha256sum), opts) + if err != nil { + // Failed to upload object part, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + // Upload part2. + data := []byte("hello, world") + md5Writer = md5.New() + md5Writer.Write(data) + etag2 := hex.EncodeToString(md5Writer.Sum(nil)) + _, err = obj.PutObjectPart(context.Background(), bucket, object, uploadID, 2, mustGetPutObjReader(t, bytes.NewReader(data), int64(len(data)), etag2, sha256sum), opts) + if err != nil { + // Failed to upload object part, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + // Complete multipart. + parts := []CompletePart{ + {ETag: etag1, PartNumber: 1}, + {ETag: etag2, PartNumber: 2}, + } + _, err = obj.CompleteMultipartUpload(context.Background(), bucket, object, uploadID, parts, ObjectOptions{}) + if err != nil { + // Failed to complete multipart upload, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + for _, disk := range disks { + tmpMetaDir := path.Join(disk, minioMetaTmpBucket) + files, err := os.ReadDir(tmpMetaDir) + if err != nil { + // It's OK to have non-existing tmpMetaDir. + if osIsNotExist(err) { + continue + } + + // Print the error + t.Errorf("%s", err) + } + + var found bool + for _, fi := range files { + if fi.Name() == ".trash" { + continue + } + found = true + break + } + + if found { + t.Fatalf("%s: expected: empty, got: non-empty. content: %#v", tmpMetaDir, files) + } + } +} + +// Benchmarks for ObjectLayer.PutObject(). +// The intent is to benchmark PutObject for various sizes ranging from few bytes to 100MB. +// Also each of these Benchmarks are run both Erasure and FS backends. + +// BenchmarkPutObjectVerySmallFS - Benchmark FS.PutObject() for object size of 10 bytes. +func BenchmarkPutObjectVerySmallFS(b *testing.B) { + benchmarkPutObject(b, "FS", 10) +} + +// BenchmarkPutObjectVerySmallErasure - Benchmark Erasure.PutObject() for object size of 10 bytes. +func BenchmarkPutObjectVerySmallErasure(b *testing.B) { + benchmarkPutObject(b, "Erasure", 10) +} + +// BenchmarkPutObject10KbFS - Benchmark FS.PutObject() for object size of 10KB. +func BenchmarkPutObject10KbFS(b *testing.B) { + benchmarkPutObject(b, "FS", 10*humanize.KiByte) +} + +// BenchmarkPutObject10KbErasure - Benchmark Erasure.PutObject() for object size of 10KB. +func BenchmarkPutObject10KbErasure(b *testing.B) { + benchmarkPutObject(b, "Erasure", 10*humanize.KiByte) +} + +// BenchmarkPutObject100KbFS - Benchmark FS.PutObject() for object size of 100KB. +func BenchmarkPutObject100KbFS(b *testing.B) { + benchmarkPutObject(b, "FS", 100*humanize.KiByte) +} + +// BenchmarkPutObject100KbErasure - Benchmark Erasure.PutObject() for object size of 100KB. +func BenchmarkPutObject100KbErasure(b *testing.B) { + benchmarkPutObject(b, "Erasure", 100*humanize.KiByte) +} + +// BenchmarkPutObject1MbFS - Benchmark FS.PutObject() for object size of 1MB. +func BenchmarkPutObject1MbFS(b *testing.B) { + benchmarkPutObject(b, "FS", 1*humanize.MiByte) +} + +// BenchmarkPutObject1MbErasure - Benchmark Erasure.PutObject() for object size of 1MB. +func BenchmarkPutObject1MbErasure(b *testing.B) { + benchmarkPutObject(b, "Erasure", 1*humanize.MiByte) +} + +// BenchmarkPutObject5MbFS - Benchmark FS.PutObject() for object size of 5MB. +func BenchmarkPutObject5MbFS(b *testing.B) { + benchmarkPutObject(b, "FS", 5*humanize.MiByte) +} + +// BenchmarkPutObject5MbErasure - Benchmark Erasure.PutObject() for object size of 5MB. +func BenchmarkPutObject5MbErasure(b *testing.B) { + benchmarkPutObject(b, "Erasure", 5*humanize.MiByte) +} + +// BenchmarkPutObject10MbFS - Benchmark FS.PutObject() for object size of 10MB. +func BenchmarkPutObject10MbFS(b *testing.B) { + benchmarkPutObject(b, "FS", 10*humanize.MiByte) +} + +// BenchmarkPutObject10MbErasure - Benchmark Erasure.PutObject() for object size of 10MB. +func BenchmarkPutObject10MbErasure(b *testing.B) { + benchmarkPutObject(b, "Erasure", 10*humanize.MiByte) +} + +// BenchmarkPutObject25MbFS - Benchmark FS.PutObject() for object size of 25MB. +func BenchmarkPutObject25MbFS(b *testing.B) { + benchmarkPutObject(b, "FS", 25*humanize.MiByte) +} + +// BenchmarkPutObject25MbErasure - Benchmark Erasure.PutObject() for object size of 25MB. +func BenchmarkPutObject25MbErasure(b *testing.B) { + benchmarkPutObject(b, "Erasure", 25*humanize.MiByte) +} + +// BenchmarkPutObject50MbFS - Benchmark FS.PutObject() for object size of 50MB. +func BenchmarkPutObject50MbFS(b *testing.B) { + benchmarkPutObject(b, "FS", 50*humanize.MiByte) +} + +// BenchmarkPutObject50MbErasure - Benchmark Erasure.PutObject() for object size of 50MB. +func BenchmarkPutObject50MbErasure(b *testing.B) { + benchmarkPutObject(b, "Erasure", 50*humanize.MiByte) +} + +// parallel benchmarks for ObjectLayer.PutObject() . + +// BenchmarkParallelPutObjectVerySmallFS - BenchmarkParallel FS.PutObject() for object size of 10 bytes. +func BenchmarkParallelPutObjectVerySmallFS(b *testing.B) { + benchmarkPutObjectParallel(b, "FS", 10) +} + +// BenchmarkParallelPutObjectVerySmallErasure - BenchmarkParallel Erasure.PutObject() for object size of 10 bytes. +func BenchmarkParallelPutObjectVerySmallErasure(b *testing.B) { + benchmarkPutObjectParallel(b, "Erasure", 10) +} + +// BenchmarkParallelPutObject10KbFS - BenchmarkParallel FS.PutObject() for object size of 10KB. +func BenchmarkParallelPutObject10KbFS(b *testing.B) { + benchmarkPutObjectParallel(b, "FS", 10*humanize.KiByte) +} + +// BenchmarkParallelPutObject10KbErasure - BenchmarkParallel Erasure.PutObject() for object size of 10KB. +func BenchmarkParallelPutObject10KbErasure(b *testing.B) { + benchmarkPutObjectParallel(b, "Erasure", 10*humanize.KiByte) +} + +// BenchmarkParallelPutObject100KbFS - BenchmarkParallel FS.PutObject() for object size of 100KB. +func BenchmarkParallelPutObject100KbFS(b *testing.B) { + benchmarkPutObjectParallel(b, "FS", 100*humanize.KiByte) +} + +// BenchmarkParallelPutObject100KbErasure - BenchmarkParallel Erasure.PutObject() for object size of 100KB. +func BenchmarkParallelPutObject100KbErasure(b *testing.B) { + benchmarkPutObjectParallel(b, "Erasure", 100*humanize.KiByte) +} + +// BenchmarkParallelPutObject1MbFS - BenchmarkParallel FS.PutObject() for object size of 1MB. +func BenchmarkParallelPutObject1MbFS(b *testing.B) { + benchmarkPutObjectParallel(b, "FS", 1*humanize.MiByte) +} + +// BenchmarkParallelPutObject1MbErasure - BenchmarkParallel Erasure.PutObject() for object size of 1MB. +func BenchmarkParallelPutObject1MbErasure(b *testing.B) { + benchmarkPutObjectParallel(b, "Erasure", 1*humanize.MiByte) +} + +// BenchmarkParallelPutObject5MbFS - BenchmarkParallel FS.PutObject() for object size of 5MB. +func BenchmarkParallelPutObject5MbFS(b *testing.B) { + benchmarkPutObjectParallel(b, "FS", 5*humanize.MiByte) +} + +// BenchmarkParallelPutObject5MbErasure - BenchmarkParallel Erasure.PutObject() for object size of 5MB. +func BenchmarkParallelPutObject5MbErasure(b *testing.B) { + benchmarkPutObjectParallel(b, "Erasure", 5*humanize.MiByte) +} + +// BenchmarkParallelPutObject10MbFS - BenchmarkParallel FS.PutObject() for object size of 10MB. +func BenchmarkParallelPutObject10MbFS(b *testing.B) { + benchmarkPutObjectParallel(b, "FS", 10*humanize.MiByte) +} + +// BenchmarkParallelPutObject10MbErasure - BenchmarkParallel Erasure.PutObject() for object size of 10MB. +func BenchmarkParallelPutObject10MbErasure(b *testing.B) { + benchmarkPutObjectParallel(b, "Erasure", 10*humanize.MiByte) +} + +// BenchmarkParallelPutObject25MbFS - BenchmarkParallel FS.PutObject() for object size of 25MB. +func BenchmarkParallelPutObject25MbFS(b *testing.B) { + benchmarkPutObjectParallel(b, "FS", 25*humanize.MiByte) +} + +// BenchmarkParallelPutObject25MbErasure - BenchmarkParallel Erasure.PutObject() for object size of 25MB. +func BenchmarkParallelPutObject25MbErasure(b *testing.B) { + benchmarkPutObjectParallel(b, "Erasure", 25*humanize.MiByte) +} diff --git a/cmd/object-api-utils.go b/cmd/object-api-utils.go new file mode 100644 index 00000000..5d791ce4 --- /dev/null +++ b/cmd/object-api-utils.go @@ -0,0 +1,1306 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/hex" + "errors" + "fmt" + "io" + "math/rand" + "net" + "net/http" + "path" + "runtime" + "slices" + "strconv" + "strings" + "sync" + "time" + "unicode/utf8" + "unsafe" + + "github.com/google/uuid" + "github.com/klauspost/compress/s2" + "github.com/klauspost/readahead" + "github.com/minio/minio-go/v7/pkg/s3utils" + "github.com/minio/minio/internal/config/compress" + "github.com/minio/minio/internal/config/dns" + "github.com/minio/minio/internal/config/storageclass" + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/trie" + "github.com/minio/pkg/v3/wildcard" + "github.com/valyala/bytebufferpool" +) + +const ( + // MinIO meta bucket. + minioMetaBucket = ".minio.sys" + // Multipart meta prefix. + mpartMetaPrefix = "multipart" + // MinIO Multipart meta prefix. + minioMetaMultipartBucket = minioMetaBucket + SlashSeparator + mpartMetaPrefix + // MinIO tmp meta prefix. + minioMetaTmpBucket = minioMetaBucket + "/tmp" + // MinIO tmp meta prefix for deleted objects. + minioMetaTmpDeletedBucket = minioMetaTmpBucket + "/.trash" + + // DNS separator (period), used for bucket name validation. + dnsDelimiter = "." + // On compressed files bigger than this; + compReadAheadSize = 100 << 20 + // Read this many buffers ahead. + compReadAheadBuffers = 5 + // Size of each buffer. + compReadAheadBufSize = 1 << 20 + // Pad Encrypted+Compressed files to a multiple of this. + compPadEncrypted = 256 + // Disable compressed file indices below this size + compMinIndexSize = 8 << 20 +) + +// getkeyeparator - returns the separator to be used for +// persisting on drive. +// +// - ":" is used on non-windows platforms +// - "_" is used on windows platforms +func getKeySeparator() string { + if runtime.GOOS == globalWindowsOSName { + return "_" + } + return ":" +} + +// isMinioBucket returns true if given bucket is a MinIO internal +// bucket and false otherwise. +func isMinioMetaBucketName(bucket string) bool { + return strings.HasPrefix(bucket, minioMetaBucket) +} + +// IsValidBucketName verifies that a bucket name is in accordance with +// Amazon's requirements (i.e. DNS naming conventions). It must be 3-63 +// characters long, and it must be a sequence of one or more labels +// separated by periods. Each label can contain lowercase ascii +// letters, decimal digits and hyphens, but must not begin or end with +// a hyphen. See: +// http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html +func IsValidBucketName(bucket string) bool { + // Special case when bucket is equal to one of the meta buckets. + if isMinioMetaBucketName(bucket) { + return true + } + if len(bucket) < 3 || len(bucket) > 63 { + return false + } + + // Split on dot and check each piece conforms to rules. + allNumbers := true + pieces := strings.Split(bucket, dnsDelimiter) + for _, piece := range pieces { + if len(piece) == 0 || piece[0] == '-' || + piece[len(piece)-1] == '-' { + // Current piece has 0-length or starts or + // ends with a hyphen. + return false + } + // Now only need to check if each piece is a valid + // 'label' in AWS terminology and if the bucket looks + // like an IP address. + isNotNumber := false + for i := range len(piece) { + switch { + case (piece[i] >= 'a' && piece[i] <= 'z' || + piece[i] == '-'): + // Found a non-digit character, so + // this piece is not a number. + isNotNumber = true + case piece[i] >= '0' && piece[i] <= '9': + // Nothing to do. + default: + // Found invalid character. + return false + } + } + allNumbers = allNumbers && !isNotNumber + } + // Does the bucket name look like an IP address? + return len(pieces) != 4 || !allNumbers +} + +// IsValidObjectName verifies an object name in accordance with Amazon's +// requirements. It cannot exceed 1024 characters and must be a valid UTF8 +// string. +// +// See: +// http://docs.aws.amazon.com/AmazonS3/latest/dev/UsingMetadata.html +// +// You should avoid the following characters in a key name because of +// significant special handling for consistency across all +// applications. +// +// Rejects strings with following characters. +// +// - Backslash ("\") +// +// additionally minio does not support object names with trailing SlashSeparator. +func IsValidObjectName(object string) bool { + if len(object) == 0 { + return false + } + if HasSuffix(object, SlashSeparator) { + return false + } + return IsValidObjectPrefix(object) +} + +// IsValidObjectPrefix verifies whether the prefix is a valid object name. +// Its valid to have a empty prefix. +func IsValidObjectPrefix(object string) bool { + if hasBadPathComponent(object) { + return false + } + if !utf8.ValidString(object) { + return false + } + if strings.Contains(object, `//`) { + return false + } + // This is valid for AWS S3 but it will never + // work with file systems, we will reject here + // to return object name invalid rather than + // a cryptic error from the file system. + return !strings.ContainsRune(object, 0) +} + +// checkObjectNameForLengthAndSlash -check for the validity of object name length and prefis as slash +func checkObjectNameForLengthAndSlash(bucket, object string) error { + // Check for the length of object name + if len(object) > 1024 { + return ObjectNameTooLong{ + Bucket: bucket, + Object: object, + } + } + // Check for slash as prefix in object name + if HasPrefix(object, SlashSeparator) { + return ObjectNamePrefixAsSlash{ + Bucket: bucket, + Object: object, + } + } + if runtime.GOOS == globalWindowsOSName { + // Explicitly disallowed characters on windows. + // Avoids most problematic names. + if strings.ContainsAny(object, `\:*?"|<>`) { + return ObjectNameInvalid{ + Bucket: bucket, + Object: object, + } + } + } + return nil +} + +// SlashSeparator - slash separator. +const SlashSeparator = "/" + +// SlashSeparatorChar - slash separator. +const SlashSeparatorChar = '/' + +// retainSlash - retains slash from a path. +func retainSlash(s string) string { + if s == "" { + return s + } + return strings.TrimSuffix(s, SlashSeparator) + SlashSeparator +} + +// pathsJoinPrefix - like pathJoin retains trailing SlashSeparator +// for all elements, prepends them with 'prefix' respectively. +func pathsJoinPrefix(prefix string, elem ...string) (paths []string) { + paths = make([]string, len(elem)) + for i, e := range elem { + paths[i] = pathJoin(prefix, e) + } + return paths +} + +// string concat alternative to s1 + s2 with low overhead. +func concat(ss ...string) string { + length := len(ss) + if length == 0 { + return "" + } + // create & allocate the memory in advance. + n := 0 + for i := range length { + n += len(ss[i]) + } + b := make([]byte, 0, n) + for i := range length { + b = append(b, ss[i]...) + } + return unsafe.String(unsafe.SliceData(b), n) +} + +// pathJoin - like path.Join() but retains trailing SlashSeparator of the last element +func pathJoin(elem ...string) string { + sb := bytebufferpool.Get() + defer func() { + sb.Reset() + bytebufferpool.Put(sb) + }() + + return pathJoinBuf(sb, elem...) +} + +// pathJoinBuf - like path.Join() but retains trailing SlashSeparator of the last element. +// Provide a string builder to reduce allocation. +func pathJoinBuf(dst *bytebufferpool.ByteBuffer, elem ...string) string { + trailingSlash := len(elem) > 0 && hasSuffixByte(elem[len(elem)-1], SlashSeparatorChar) + dst.Reset() + added := 0 + for _, e := range elem { + if added > 0 || e != "" { + if added > 0 { + dst.WriteByte(SlashSeparatorChar) + } + dst.WriteString(e) + added += len(e) + } + } + + if pathNeedsClean(dst.Bytes()) { + s := path.Clean(dst.String()) + if trailingSlash { + return s + SlashSeparator + } + return s + } + if trailingSlash { + dst.WriteByte(SlashSeparatorChar) + } + return dst.String() +} + +// hasSuffixByte returns true if the last byte of s is 'suffix' +func hasSuffixByte(s string, suffix byte) bool { + return len(s) > 0 && s[len(s)-1] == suffix +} + +// pathNeedsClean returns whether path.Clean may change the path. +// Will detect all cases that will be cleaned, +// but may produce false positives on non-trivial paths. +func pathNeedsClean(path []byte) bool { + if len(path) == 0 { + return true + } + + rooted := path[0] == '/' + n := len(path) + + r, w := 0, 0 + if rooted { + r, w = 1, 1 + } + + for r < n { + switch { + case path[r] > 127: + // Non ascii. + return true + case path[r] == '/': + // multiple / elements + return true + case path[r] == '.' && (r+1 == n || path[r+1] == '/'): + // . element - assume it has to be cleaned. + return true + case path[r] == '.' && path[r+1] == '.' && (r+2 == n || path[r+2] == '/'): + // .. element: remove to last / - assume it has to be cleaned. + return true + default: + // real path element. + // add slash if needed + if rooted && w != 1 || !rooted && w != 0 { + w++ + } + // copy element + for ; r < n && path[r] != '/'; r++ { + w++ + } + // allow one slash, not at end + if r < n-1 && path[r] == '/' { + r++ + } + } + } + + // Turn empty string into "." + if w == 0 { + return true + } + + return false +} + +// mustGetUUID - get a random UUID. +func mustGetUUID() string { + u, err := uuid.NewRandom() + if err != nil { + logger.CriticalIf(GlobalContext, err) + } + + return u.String() +} + +// mustGetUUIDBytes - get a random UUID as 16 bytes unencoded. +func mustGetUUIDBytes() []byte { + u, err := uuid.NewRandom() + if err != nil { + logger.CriticalIf(GlobalContext, err) + } + return u[:] +} + +// Create an s3 compatible MD5sum for complete multipart transaction. +func getCompleteMultipartMD5(parts []CompletePart) string { + var finalMD5Bytes []byte + for _, part := range parts { + md5Bytes, err := hex.DecodeString(canonicalizeETag(part.ETag)) + if err != nil { + finalMD5Bytes = append(finalMD5Bytes, []byte(part.ETag)...) + } else { + finalMD5Bytes = append(finalMD5Bytes, md5Bytes...) + } + } + s3MD5 := fmt.Sprintf("%s-%d", getMD5Hash(finalMD5Bytes), len(parts)) + return s3MD5 +} + +// Clean unwanted fields from metadata +func cleanMetadata(metadata map[string]string) map[string]string { + // Remove STANDARD StorageClass + metadata = removeStandardStorageClass(metadata) + // Clean meta etag keys 'md5Sum', 'etag', "expires", "x-amz-tagging". + return cleanMetadataKeys(metadata, "md5Sum", "etag", "expires", xhttp.AmzObjectTagging, "last-modified", VersionPurgeStatusKey) +} + +// Filter X-Amz-Storage-Class field only if it is set to STANDARD. +// This is done since AWS S3 doesn't return STANDARD Storage class as response header. +func removeStandardStorageClass(metadata map[string]string) map[string]string { + if metadata[xhttp.AmzStorageClass] == storageclass.STANDARD { + delete(metadata, xhttp.AmzStorageClass) + } + return metadata +} + +// cleanMetadataKeys takes keyNames to be filtered +// and returns a new map with all the entries with keyNames removed. +func cleanMetadataKeys(metadata map[string]string, keyNames ...string) map[string]string { + newMeta := make(map[string]string, len(metadata)) + for k, v := range metadata { + if slices.Contains(keyNames, k) { + continue + } + newMeta[k] = v + } + return newMeta +} + +// Extracts etag value from the metadata. +func extractETag(metadata map[string]string) string { + etag, ok := metadata["etag"] + if !ok { + // md5Sum tag is kept for backward compatibility. + etag = metadata["md5Sum"] + } + // Success. + return etag +} + +// HasPrefix - Prefix matcher string matches prefix in a platform specific way. +// For example on windows since its case insensitive we are supposed +// to do case insensitive checks. +func HasPrefix(s string, prefix string) bool { + if runtime.GOOS == globalWindowsOSName { + return stringsHasPrefixFold(s, prefix) + } + return strings.HasPrefix(s, prefix) +} + +// HasSuffix - Suffix matcher string matches suffix in a platform specific way. +// For example on windows since its case insensitive we are supposed +// to do case insensitive checks. +func HasSuffix(s string, suffix string) bool { + if runtime.GOOS == globalWindowsOSName { + return strings.HasSuffix(strings.ToLower(s), strings.ToLower(suffix)) + } + return strings.HasSuffix(s, suffix) +} + +// Validates if two strings are equal. +func isStringEqual(s1 string, s2 string) bool { + if runtime.GOOS == globalWindowsOSName { + return strings.EqualFold(s1, s2) + } + return s1 == s2 +} + +// Ignores all reserved bucket names or invalid bucket names. +func isReservedOrInvalidBucket(bucketEntry string, strict bool) bool { + if bucketEntry == "" { + return true + } + + bucketEntry = strings.TrimSuffix(bucketEntry, SlashSeparator) + if strict { + if err := s3utils.CheckValidBucketNameStrict(bucketEntry); err != nil { + return true + } + } else { + if err := s3utils.CheckValidBucketName(bucketEntry); err != nil { + return true + } + } + return isMinioMetaBucket(bucketEntry) || isMinioReservedBucket(bucketEntry) +} + +// Returns true if input bucket is a reserved minio meta bucket '.minio.sys'. +func isMinioMetaBucket(bucketName string) bool { + return bucketName == minioMetaBucket +} + +// Returns true if input bucket is a reserved minio bucket 'minio'. +func isMinioReservedBucket(bucketName string) bool { + return bucketName == minioReservedBucket +} + +// returns a slice of hosts by reading a slice of DNS records +func getHostsSlice(records []dns.SrvRecord) []string { + hosts := make([]string, len(records)) + for i, r := range records { + hosts[i] = net.JoinHostPort(r.Host, string(r.Port)) + } + return hosts +} + +// returns an online host (and corresponding port) from a slice of DNS records +func getHostFromSrv(records []dns.SrvRecord) (host string) { + hosts := getHostsSlice(records) + rng := rand.New(rand.NewSource(time.Now().UTC().UnixNano())) + var d net.Dialer + var retry int + for retry < len(hosts) { + ctx, cancel := context.WithTimeout(GlobalContext, 300*time.Millisecond) + + host = hosts[rng.Intn(len(hosts))] + conn, err := d.DialContext(ctx, "tcp", host) + cancel() + if err != nil { + retry++ + continue + } + conn.Close() + break + } + + return host +} + +// IsCompressed returns true if the object is marked as compressed. +func (o *ObjectInfo) IsCompressed() bool { + _, ok := o.UserDefined[ReservedMetadataPrefix+"compression"] + return ok +} + +// IsCompressedOK returns whether the object is compressed and can be decompressed. +func (o *ObjectInfo) IsCompressedOK() (bool, error) { + scheme, ok := o.UserDefined[ReservedMetadataPrefix+"compression"] + if !ok { + return false, nil + } + switch scheme { + case compressionAlgorithmV1, compressionAlgorithmV2: + return true, nil + } + return true, fmt.Errorf("unknown compression scheme: %s", scheme) +} + +// GetActualSize - returns the actual size of the stored object +func (o ObjectInfo) GetActualSize() (int64, error) { + if o.ActualSize != nil { + return *o.ActualSize, nil + } + if o.IsCompressed() { + sizeStr := o.UserDefined[ReservedMetadataPrefix+"actual-size"] + if sizeStr != "" { + size, err := strconv.ParseInt(sizeStr, 10, 64) + if err != nil { + return -1, errInvalidDecompressedSize + } + return size, nil + } + var actualSize int64 + for _, part := range o.Parts { + actualSize += part.ActualSize + } + if (actualSize == 0) && (actualSize != o.Size) { + return -1, errInvalidDecompressedSize + } + return actualSize, nil + } + if _, ok := crypto.IsEncrypted(o.UserDefined); ok { + sizeStr := o.UserDefined[ReservedMetadataPrefix+"actual-size"] + if sizeStr != "" { + size, err := strconv.ParseInt(sizeStr, 10, 64) + if err != nil { + return -1, errObjectTampered + } + return size, nil + } + actualSize, err := o.DecryptedSize() + if err != nil { + return -1, err + } + if (actualSize == 0) && (actualSize != o.Size) { + return -1, errObjectTampered + } + return actualSize, nil + } + return o.Size, nil +} + +// Disabling compression for encrypted enabled requests. +// Using compression and encryption together enables room for side channel attacks. +// Eliminate non-compressible objects by extensions/content-types. +func isCompressible(header http.Header, object string) bool { + globalCompressConfigMu.Lock() + cfg := globalCompressConfig + globalCompressConfigMu.Unlock() + + return !excludeForCompression(header, object, cfg) +} + +// Eliminate the non-compressible objects. +func excludeForCompression(header http.Header, object string, cfg compress.Config) bool { + objStr := object + contentType := header.Get(xhttp.ContentType) + if !cfg.Enabled { + return true + } + + if crypto.Requested(header) && !cfg.AllowEncrypted { + return true + } + + // We strictly disable compression for standard extensions/content-types (`compressed`). + if hasStringSuffixInSlice(objStr, standardExcludeCompressExtensions) || hasPattern(standardExcludeCompressContentTypes, contentType) { + return true + } + + // Filter compression includes. + if len(cfg.Extensions) == 0 && len(cfg.MimeTypes) == 0 { + // Nothing to filter, include everything. + return false + } + + if len(cfg.Extensions) > 0 && hasStringSuffixInSlice(objStr, cfg.Extensions) { + // Matched an extension to compress, do not exclude. + return false + } + + if len(cfg.MimeTypes) > 0 && hasPattern(cfg.MimeTypes, contentType) { + // Matched an MIME type to compress, do not exclude. + return false + } + + // Did not match any inclusion filters, exclude from compression. + return true +} + +// Utility which returns if a string is present in the list. +// Comparison is case insensitive. Explicit short-circuit if +// the list contains the wildcard "*". +func hasStringSuffixInSlice(str string, list []string) bool { + str = strings.ToLower(str) + for _, v := range list { + if v == "*" { + return true + } + + if strings.HasSuffix(str, strings.ToLower(v)) { + return true + } + } + return false +} + +// Returns true if any of the given wildcard patterns match the matchStr. +func hasPattern(patterns []string, matchStr string) bool { + for _, pattern := range patterns { + if ok := wildcard.MatchSimple(pattern, matchStr); ok { + return true + } + } + return false +} + +// Returns the part file name which matches the partNumber and etag. +func getPartFile(entriesTrie *trie.Trie, partNumber int, etag string) (partFile string) { + for _, match := range entriesTrie.PrefixMatch(fmt.Sprintf("%.5d.%s.", partNumber, etag)) { + partFile = match + break + } + return partFile +} + +func partNumberToRangeSpec(oi ObjectInfo, partNumber int) *HTTPRangeSpec { + if oi.Size == 0 || len(oi.Parts) == 0 { + return nil + } + + var start int64 + end := int64(-1) + for i := 0; i < len(oi.Parts) && i < partNumber; i++ { + start = end + 1 + end = start + oi.Parts[i].ActualSize - 1 + } + + return &HTTPRangeSpec{Start: start, End: end} +} + +// Returns the compressed offset which should be skipped. +// If encrypted offsets are adjusted for encrypted block headers/trailers. +// Since de-compression is after decryption encryption overhead is only added to compressedOffset. +func getCompressedOffsets(oi ObjectInfo, offset int64, decrypt func([]byte) ([]byte, error)) (compressedOffset int64, partSkip int64, firstPart int, decryptSkip int64, seqNum uint32) { + var skipLength int64 + var cumulativeActualSize int64 + var firstPartIdx int + for i, part := range oi.Parts { + cumulativeActualSize += part.ActualSize + if cumulativeActualSize <= offset { + compressedOffset += part.Size + } else { + firstPartIdx = i + skipLength = cumulativeActualSize - part.ActualSize + break + } + } + partSkip = offset - skipLength + + // Load index and skip more if feasible. + if partSkip > 0 && len(oi.Parts) > firstPartIdx && len(oi.Parts[firstPartIdx].Index) > 0 { + _, isEncrypted := crypto.IsEncrypted(oi.UserDefined) + if isEncrypted { + dec, err := decrypt(oi.Parts[firstPartIdx].Index) + if err == nil { + // Load Index + var idx s2.Index + _, err := idx.Load(s2.RestoreIndexHeaders(dec)) + + // Find compressed/uncompressed offsets of our partskip + compOff, uCompOff, err2 := idx.Find(partSkip) + + if err == nil && err2 == nil && compOff > 0 { + // Encrypted. + const sseDAREEncPackageBlockSize = SSEDAREPackageBlockSize + SSEDAREPackageMetaSize + // Number of full blocks in skipped area + seqNum = uint32(compOff / SSEDAREPackageBlockSize) + // Skip this many inside a decrypted block to get to compression block start + decryptSkip = compOff % SSEDAREPackageBlockSize + // Skip this number of full blocks. + skipEnc := compOff / SSEDAREPackageBlockSize + skipEnc *= sseDAREEncPackageBlockSize + compressedOffset += skipEnc + // Skip this number of uncompressed bytes. + partSkip -= uCompOff + } + } + } else { + // Not encrypted + var idx s2.Index + _, err := idx.Load(s2.RestoreIndexHeaders(oi.Parts[firstPartIdx].Index)) + + // Find compressed/uncompressed offsets of our partskip + compOff, uCompOff, err2 := idx.Find(partSkip) + + if err == nil && err2 == nil && compOff > 0 { + compressedOffset += compOff + partSkip -= uCompOff + } + } + } + + return compressedOffset, partSkip, firstPartIdx, decryptSkip, seqNum +} + +// GetObjectReader is a type that wraps a reader with a lock to +// provide a ReadCloser interface that unlocks on Close() +type GetObjectReader struct { + io.Reader + ObjInfo ObjectInfo + cleanUpFns []func() + once sync.Once +} + +// WithCleanupFuncs sets additional cleanup functions to be called when closing +// the GetObjectReader. +func (g *GetObjectReader) WithCleanupFuncs(fns ...func()) *GetObjectReader { + g.cleanUpFns = append(g.cleanUpFns, fns...) + return g +} + +// NewGetObjectReaderFromReader sets up a GetObjectReader with a given +// reader. This ignores any object properties. +func NewGetObjectReaderFromReader(r io.Reader, oi ObjectInfo, opts ObjectOptions, cleanupFns ...func()) (*GetObjectReader, error) { + if opts.CheckPrecondFn != nil && opts.CheckPrecondFn(oi) { + // Call the cleanup funcs + for i := len(cleanupFns) - 1; i >= 0; i-- { + cleanupFns[i]() + } + return nil, PreConditionFailed{} + } + return &GetObjectReader{ + ObjInfo: oi, + Reader: r, + cleanUpFns: cleanupFns, + }, nil +} + +// ObjReaderFn is a function type that takes a reader and returns +// GetObjectReader and an error. Request headers are passed to provide +// encryption parameters. cleanupFns allow cleanup funcs to be +// registered for calling after usage of the reader. +type ObjReaderFn func(inputReader io.Reader, h http.Header, cleanupFns ...func()) (r *GetObjectReader, err error) + +// NewGetObjectReader creates a new GetObjectReader. The cleanUpFns +// are called on Close() in FIFO order as passed in ObjReadFn(). NOTE: It is +// assumed that clean up functions do not panic (otherwise, they may +// not all run!). +func NewGetObjectReader(rs *HTTPRangeSpec, oi ObjectInfo, opts ObjectOptions, h http.Header) ( + fn ObjReaderFn, off, length int64, err error, +) { + if opts.CheckPrecondFn != nil && opts.CheckPrecondFn(oi) { + return nil, 0, 0, PreConditionFailed{} + } + + if rs == nil && opts.PartNumber > 0 { + rs = partNumberToRangeSpec(oi, opts.PartNumber) + } + + _, isEncrypted := crypto.IsEncrypted(oi.UserDefined) + isCompressed, err := oi.IsCompressedOK() + if err != nil { + return nil, 0, 0, err + } + + // if object is encrypted and it is a restore request or if NoDecryption + // was requested, fetch content without decrypting. + if opts.Transition.RestoreRequest != nil || opts.NoDecryption { + isEncrypted = false + isCompressed = false + } + + // Calculate range to read (different for encrypted/compressed objects) + switch { + case isCompressed: + var firstPart int + if opts.PartNumber > 0 { + // firstPart is an index to Parts slice, + // make sure that PartNumber uses the + // index value properly. + firstPart = opts.PartNumber - 1 + } + + // If compressed, we start from the beginning of the part. + // Read the decompressed size from the meta.json. + actualSize, err := oi.GetActualSize() + if err != nil { + return nil, 0, 0, err + } + var decryptSkip int64 + var seqNum uint32 + + off, length = int64(0), oi.Size + decOff, decLength := int64(0), actualSize + if rs != nil { + off, length, err = rs.GetOffsetLength(actualSize) + if err != nil { + return nil, 0, 0, err + } + decrypt := func(b []byte) ([]byte, error) { + return b, nil + } + if isEncrypted { + decrypt = func(b []byte) ([]byte, error) { + return oi.compressionIndexDecrypt(b, h) + } + } + // In case of range based queries on multiparts, the offset and length are reduced. + off, decOff, firstPart, decryptSkip, seqNum = getCompressedOffsets(oi, off, decrypt) + decLength = length + length = oi.Size - off + // For negative length we read everything. + if decLength < 0 { + decLength = actualSize - decOff + } + + // Reply back invalid range if the input offset and length fall out of range. + if decOff > actualSize || decOff+decLength > actualSize { + return nil, 0, 0, errInvalidRange + } + } + fn = func(inputReader io.Reader, h http.Header, cFns ...func()) (r *GetObjectReader, err error) { + if isEncrypted { + copySource := h.Get(xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm) != "" + // Attach decrypter on inputReader + inputReader, err = DecryptBlocksRequestR(inputReader, h, seqNum, firstPart, oi, copySource) + if err != nil { + // Call the cleanup funcs + for i := len(cFns) - 1; i >= 0; i-- { + cFns[i]() + } + return nil, err + } + if decryptSkip > 0 { + inputReader = xioutil.NewSkipReader(inputReader, decryptSkip) + } + oi.Size = decLength + } + // Decompression reader. + var dopts []s2.ReaderOption + if off > 0 || decOff > 0 { + // We are not starting at the beginning, so ignore stream identifiers. + dopts = append(dopts, s2.ReaderIgnoreStreamIdentifier()) + } + s2Reader := s2.NewReader(inputReader, dopts...) + // Apply the skipLen and limit on the decompressed stream. + if decOff > 0 { + if err = s2Reader.Skip(decOff); err != nil { + // Call the cleanup funcs + for i := len(cFns) - 1; i >= 0; i-- { + cFns[i]() + } + return nil, err + } + } + + decReader := io.LimitReader(s2Reader, decLength) + if decLength > compReadAheadSize { + rah, err := readahead.NewReaderSize(decReader, compReadAheadBuffers, compReadAheadBufSize) + if err == nil { + decReader = rah + cFns = append([]func(){func() { + rah.Close() + }}, cFns...) + } + } + oi.Size = decLength + + // Assemble the GetObjectReader + r = &GetObjectReader{ + ObjInfo: oi, + Reader: decReader, + cleanUpFns: cFns, + } + return r, nil + } + + case isEncrypted: + var seqNumber uint32 + var partStart int + var skipLen int64 + + off, length, skipLen, seqNumber, partStart, err = oi.GetDecryptedRange(rs) + if err != nil { + return nil, 0, 0, err + } + var decSize int64 + decSize, err = oi.DecryptedSize() + if err != nil { + return nil, 0, 0, err + } + var decRangeLength int64 + decRangeLength, err = rs.GetLength(decSize) + if err != nil { + return nil, 0, 0, err + } + + // We define a closure that performs decryption given + // a reader that returns the desired range of + // encrypted bytes. The header parameter is used to + // provide encryption parameters. + fn = func(inputReader io.Reader, h http.Header, cFns ...func()) (r *GetObjectReader, err error) { + copySource := h.Get(xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm) != "" + + // Attach decrypter on inputReader + var decReader io.Reader + decReader, err = DecryptBlocksRequestR(inputReader, h, seqNumber, partStart, oi, copySource) + if err != nil { + // Call the cleanup funcs + for i := len(cFns) - 1; i >= 0; i-- { + cFns[i]() + } + return nil, err + } + + oi.ETag = getDecryptedETag(h, oi, false) + + // Apply the skipLen and limit on the + // decrypted stream + decReader = io.LimitReader(xioutil.NewSkipReader(decReader, skipLen), decRangeLength) + + // Assemble the GetObjectReader + r = &GetObjectReader{ + ObjInfo: oi, + Reader: decReader, + cleanUpFns: cFns, + } + return r, nil + } + + default: + off, length, err = rs.GetOffsetLength(oi.Size) + if err != nil { + return nil, 0, 0, err + } + fn = func(inputReader io.Reader, _ http.Header, cFns ...func()) (r *GetObjectReader, err error) { + r = &GetObjectReader{ + ObjInfo: oi, + Reader: inputReader, + cleanUpFns: cFns, + } + return r, nil + } + } + return fn, off, length, nil +} + +// Close - calls the cleanup actions in reverse order +func (g *GetObjectReader) Close() error { + if g == nil { + return nil + } + // sync.Once is used here to ensure that Close() is + // idempotent. + g.once.Do(func() { + for i := len(g.cleanUpFns) - 1; i >= 0; i-- { + g.cleanUpFns[i]() + } + }) + return nil +} + +// compressionIndexEncrypter returns a function that will read data from input, +// encrypt it using the provided key and return the result. +func compressionIndexEncrypter(key crypto.ObjectKey, input func() []byte) func() []byte { + var data []byte + var fetched bool + return func() []byte { + if !fetched { + data = input() + fetched = true + } + return metadataEncrypter(key)("compression-index", data) + } +} + +// compressionIndexDecrypt reverses compressionIndexEncrypter. +func (o *ObjectInfo) compressionIndexDecrypt(input []byte, h http.Header) ([]byte, error) { + return o.metadataDecrypter(h)("compression-index", input) +} + +// SealMD5CurrFn seals md5sum with object encryption key and returns sealed +// md5sum +type SealMD5CurrFn func([]byte) []byte + +// PutObjReader is a type that wraps sio.EncryptReader and +// underlying hash.Reader in a struct +type PutObjReader struct { + *hash.Reader // actual data stream + rawReader *hash.Reader // original data stream + sealMD5Fn SealMD5CurrFn +} + +// Size returns the absolute number of bytes the Reader +// will return during reading. It returns -1 for unlimited +// data. +func (p *PutObjReader) Size() int64 { + return p.Reader.Size() +} + +// MD5CurrentHexString returns the current MD5Sum or encrypted MD5Sum +// as a hex encoded string +func (p *PutObjReader) MD5CurrentHexString() string { + md5sumCurr := p.rawReader.MD5Current() + var appendHyphen bool + // md5sumcurr is not empty in two scenarios + // - server is running in strict compatibility mode + // - client set Content-Md5 during PUT operation + if len(md5sumCurr) == 0 { + // md5sumCurr is only empty when we are running + // in non-compatibility mode. + md5sumCurr = make([]byte, 16) + rand.Read(md5sumCurr) + appendHyphen = true + } + if p.sealMD5Fn != nil { + md5sumCurr = p.sealMD5Fn(md5sumCurr) + } + if appendHyphen { + // Make sure to return etag string upto 32 length, for SSE + // requests ETag might be longer and the code decrypting the + // ETag ignores ETag in multipart ETag form i.e -N + return hex.EncodeToString(md5sumCurr)[:32] + "-1" + } + return hex.EncodeToString(md5sumCurr) +} + +// WithEncryption sets up encrypted reader and the sealing for content md5sum +// using objEncKey. Unsealed md5sum is computed from the rawReader setup when +// NewPutObjReader was called. It returns an error if called on an uninitialized +// PutObjReader. +func (p *PutObjReader) WithEncryption(encReader *hash.Reader, objEncKey *crypto.ObjectKey) (*PutObjReader, error) { + if p.Reader == nil { + return nil, errors.New("put-object reader uninitialized") + } + p.Reader = encReader + p.sealMD5Fn = sealETagFn(*objEncKey) + return p, nil +} + +// NewPutObjReader returns a new PutObjReader. It uses given hash.Reader's +// MD5Current method to construct md5sum when requested downstream. +func NewPutObjReader(rawReader *hash.Reader) *PutObjReader { + return &PutObjReader{Reader: rawReader, rawReader: rawReader} +} + +// RawServerSideChecksumResult returns the ServerSideChecksumResult from the +// underlying rawReader, since the PutObjReader might be encrypted data and +// thus any checksum from that would be incorrect. +func (p *PutObjReader) RawServerSideChecksumResult() *hash.Checksum { + if p.rawReader != nil { + return p.rawReader.ServerSideChecksumResult + } + return nil +} + +func sealETag(encKey crypto.ObjectKey, md5CurrSum []byte) []byte { + var emptyKey [32]byte + if bytes.Equal(encKey[:], emptyKey[:]) { + return md5CurrSum + } + return encKey.SealETag(md5CurrSum) +} + +func sealETagFn(key crypto.ObjectKey) SealMD5CurrFn { + fn := func(md5sumcurr []byte) []byte { + return sealETag(key, md5sumcurr) + } + return fn +} + +// compressOpts are the options for writing compressed data. +var compressOpts []s2.WriterOption + +func init() { + if runtime.GOARCH == "amd64" { + // On amd64 we have assembly and can use stronger compression. + compressOpts = append(compressOpts, s2.WriterBetterCompression()) + } +} + +// newS2CompressReader will read data from r, compress it and return the compressed data as a Reader. +// Use Close to ensure resources are released on incomplete streams. +// +// input 'on' is always recommended such that this function works +// properly, because we do not wish to create an object even if +// client closed the stream prematurely. +func newS2CompressReader(r io.Reader, on int64, encrypted bool) (rc io.ReadCloser, idx func() []byte) { + pr, pw := io.Pipe() + // Copy input to compressor + opts := compressOpts + if encrypted { + // The values used for padding are not a security concern, + // but we choose pseudo-random numbers instead of just zeros. + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + opts = append([]s2.WriterOption{s2.WriterPadding(compPadEncrypted), s2.WriterPaddingSrc(rng)}, compressOpts...) + } + comp := s2.NewWriter(pw, opts...) + indexCh := make(chan []byte, 1) + go func() { + defer xioutil.SafeClose(indexCh) + cn, err := io.Copy(comp, r) + if err != nil { + comp.Close() + pw.CloseWithError(err) + return + } + if on > 0 && on != cn { + // if client didn't sent all data + // from the client verify here. + comp.Close() + pw.CloseWithError(IncompleteBody{}) + return + } + // Close the stream. + // If more than compMinIndexSize was written, generate index. + if cn > compMinIndexSize { + idx, err := comp.CloseIndex() + idx = s2.RemoveIndexHeaders(idx) + indexCh <- idx + pw.CloseWithError(err) + return + } + pw.CloseWithError(comp.Close()) + }() + var gotIdx []byte + return pr, func() []byte { + if gotIdx != nil { + return gotIdx + } + // Will get index or nil if closed. + gotIdx = <-indexCh + return gotIdx + } +} + +// compressSelfTest performs a self-test to ensure that compression +// algorithms completes a roundtrip. If any algorithm +// produces an incorrect checksum it fails with a hard error. +// +// compressSelfTest tries to catch any issue in the compression implementation +// early instead of silently corrupting data. +func compressSelfTest() { + // 4 MB block. + // Approx runtime ~30ms + data := make([]byte, 4<<20) + rng := rand.New(rand.NewSource(0)) + for i := range data { + // Generate compressible stream... + data[i] = byte(rng.Int63() & 3) + } + failOnErr := func(err error) { + if err != nil { + logger.Fatal(errSelfTestFailure, "compress: error on self-test: %v", err) + } + } + const skip = 2<<20 + 511 + r, _ := newS2CompressReader(bytes.NewBuffer(data), int64(len(data)), true) + b, err := io.ReadAll(r) + failOnErr(err) + failOnErr(r.Close()) + // Decompression reader. + s2Reader := s2.NewReader(bytes.NewBuffer(b)) + // Apply the skipLen on the decompressed stream. + failOnErr(s2Reader.Skip(skip)) + got, err := io.ReadAll(s2Reader) + failOnErr(err) + if !bytes.Equal(got, data[skip:]) { + logger.Fatal(errSelfTestFailure, "compress: self-test roundtrip mismatch.") + } +} + +// getDiskInfos returns the disk information for the provided disks. +// If a disk is nil or an error is returned the result will be nil as well. +func getDiskInfos(ctx context.Context, disks ...StorageAPI) []*DiskInfo { + res := make([]*DiskInfo, len(disks)) + opts := DiskInfoOptions{} + for i, disk := range disks { + if disk == nil { + continue + } + if di, err := disk.DiskInfo(ctx, opts); err == nil { + res[i] = &di + } + } + return res +} + +// hasSpaceFor returns whether the disks in `di` have space for and object of a given size. +func hasSpaceFor(di []*DiskInfo, size int64) (bool, error) { + // We multiply the size by 2 to account for erasure coding. + size *= 2 + if size < 0 { + // If no size, assume diskAssumeUnknownSize. + size = diskAssumeUnknownSize + } + + var available uint64 + var total uint64 + var nDisks int + for _, disk := range di { + if disk == nil || disk.Total == 0 { + // Disk offline, no inodes or something else is wrong. + continue + } + nDisks++ + total += disk.Total + available += disk.Total - disk.Used + } + + if nDisks < len(di)/2 || nDisks <= 0 { + var errs []error + for index, disk := range di { + switch { + case disk == nil: + errs = append(errs, fmt.Errorf("disk[%d]: offline", index)) + case disk.Error != "": + errs = append(errs, fmt.Errorf("disk %s: %s", disk.Endpoint, disk.Error)) + case disk.Total == 0: + errs = append(errs, fmt.Errorf("disk %s: total is zero", disk.Endpoint)) + } + } + // Log disk errors. + peersLogIf(context.Background(), errors.Join(errs...)) + return false, fmt.Errorf("not enough online disks to calculate the available space, need %d, found %d", (len(di)/2)+1, nDisks) + } + + // Check we have enough on each disk, ignoring diskFillFraction. + perDisk := size / int64(nDisks) + for _, disk := range di { + if disk == nil || disk.Total == 0 { + continue + } + if !globalIsErasureSD && disk.FreeInodes < diskMinInodes && disk.UsedInodes > 0 { + // We have an inode count, but not enough inodes. + return false, nil + } + if int64(disk.Free) <= perDisk { + return false, nil + } + } + + // Make sure we can fit "size" on to the disk without getting above the diskFillFraction + if available < uint64(size) { + return false, nil + } + + // How much will be left after adding the file. + available -= uint64(size) + + // wantLeft is how much space there at least must be left. + wantLeft := uint64(float64(total) * (1.0 - diskFillFraction)) + return available > wantLeft, nil +} diff --git a/cmd/object-api-utils_test.go b/cmd/object-api-utils_test.go new file mode 100644 index 00000000..53992bf8 --- /dev/null +++ b/cmd/object-api-utils_test.go @@ -0,0 +1,883 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/hex" + "fmt" + "io" + "math/rand" + "net/http" + "net/http/httptest" + "path" + "reflect" + "runtime" + "strconv" + "testing" + + "github.com/klauspost/compress/s2" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/config/compress" + "github.com/minio/minio/internal/crypto" + "github.com/minio/pkg/v3/trie" +) + +func pathJoinOld(elem ...string) string { + trailingSlash := "" + if len(elem) > 0 { + if hasSuffixByte(elem[len(elem)-1], SlashSeparatorChar) { + trailingSlash = SlashSeparator + } + } + return path.Join(elem...) + trailingSlash +} + +func concatNaive(ss ...string) string { + rs := ss[0] + for i := 1; i < len(ss); i++ { + rs += ss[i] + } + return rs +} + +func benchmark(b *testing.B, data []string) { + b.Run("concat naive", func(b *testing.B) { + b.ResetTimer() + b.ReportAllocs() + for b.Loop() { + concatNaive(data...) + } + }) + b.Run("concat fast", func(b *testing.B) { + b.ResetTimer() + b.ReportAllocs() + for b.Loop() { + concat(data...) + } + }) +} + +func BenchmarkConcatImplementation(b *testing.B) { + data := make([]string, 2) + rng := rand.New(rand.NewSource(0)) + for i := range 2 { + var tmp [16]byte + rng.Read(tmp[:]) + data[i] = hex.EncodeToString(tmp[:]) + } + b.ResetTimer() + benchmark(b, data) +} + +func BenchmarkPathJoinOld(b *testing.B) { + b.Run("PathJoin", func(b *testing.B) { + b.ResetTimer() + b.ReportAllocs() + + for b.Loop() { + pathJoinOld("volume", "path/path/path") + } + }) +} + +func BenchmarkPathJoin(b *testing.B) { + b.Run("PathJoin", func(b *testing.B) { + b.ResetTimer() + b.ReportAllocs() + + for b.Loop() { + pathJoin("volume", "path/path/path") + } + }) +} + +// Wrapper +func TestPathTraversalExploit(t *testing.T) { + if runtime.GOOS != globalWindowsOSName { + t.Skip() + } + defer DetectTestLeak(t)() + ExecExtendedObjectLayerAPITest(t, testPathTraversalExploit, []string{"PutObject"}) +} + +// testPathTraversal exploit test, exploits path traversal on windows +// with following object names "\\../.minio.sys/config/iam/${username}/identity.json" +// #16852 +func testPathTraversalExploit(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + if err := newTestConfig(globalMinioDefaultRegion, obj); err != nil { + t.Fatalf("Initializing config.json failed") + } + + objectName := `\../.minio.sys/config/hello.txt` + + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + // construct HTTP request for Get Object end point. + req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, objectName), + int64(5), bytes.NewReader([]byte("hello")), credentials.AccessKey, credentials.SecretKey, map[string]string{}) + if err != nil { + t.Fatalf("failed to create HTTP request for Put Object: %v", err) + } + + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + + ctx, cancel := context.WithCancel(GlobalContext) + defer cancel() + + // Now check if we actually wrote to backend (regardless of the response + // returned by the server). + z := obj.(*erasureServerPools) + xl := z.serverPools[0].sets[0] + erasureDisks := xl.getDisks() + parts, errs := readAllFileInfo(ctx, erasureDisks, "", bucketName, objectName, "", false, false) + for i := range parts { + if errs[i] == nil { + if parts[i].Name == objectName { + t.Errorf("path traversal allowed to allow writing to minioMetaBucket: %s", instanceType) + } + } + } +} + +// Tests validate bucket name. +func TestIsValidBucketName(t *testing.T) { + testCases := []struct { + bucketName string + shouldPass bool + }{ + // cases which should pass the test. + // passing in valid bucket names. + {"lol", true}, + {"1-this-is-valid", true}, + {"1-this-too-is-valid-1", true}, + {"this.works.too.1", true}, + {"1234567", true}, + {"123", true}, + {"s3-eu-west-1.amazonaws.com", true}, + {"ideas-are-more-powerful-than-guns", true}, + {"testbucket", true}, + {"1bucket", true}, + {"bucket1", true}, + {"a.b", true}, + {"ab.a.bc", true}, + // cases for which test should fail. + // passing invalid bucket names. + {"------", false}, + {"my..bucket", false}, + {"192.168.1.1", false}, + {"$this-is-not-valid-too", false}, + {"contains-$-dollar", false}, + {"contains-^-caret", false}, + {"contains-$-dollar", false}, + {"contains-$-dollar", false}, + {"......", false}, + {"", false}, + {"a", false}, + {"ab", false}, + {".starts-with-a-dot", false}, + {"ends-with-a-dot.", false}, + {"ends-with-a-dash-", false}, + {"-starts-with-a-dash", false}, + {"THIS-BEGINS-WITH-UPPERCASe", false}, + {"tHIS-ENDS-WITH-UPPERCASE", false}, + {"ThisBeginsAndEndsWithUpperCasE", false}, + {"una ñina", false}, + {"dash-.may-not-appear-next-to-dot", false}, + {"dash.-may-not-appear-next-to-dot", false}, + {"dash-.-may-not-appear-next-to-dot", false}, + {"lalalallalallalalalallalallalala-thestring-size-is-greater-than-63", false}, + } + + for i, testCase := range testCases { + isValidBucketName := IsValidBucketName(testCase.bucketName) + if testCase.shouldPass && !isValidBucketName { + t.Errorf("Test case %d: Expected \"%s\" to be a valid bucket name", i+1, testCase.bucketName) + } + if !testCase.shouldPass && isValidBucketName { + t.Errorf("Test case %d: Expected bucket name \"%s\" to be invalid", i+1, testCase.bucketName) + } + } +} + +// Tests for validate object name. +func TestIsValidObjectName(t *testing.T) { + testCases := []struct { + objectName string + shouldPass bool + }{ + // cases which should pass the test. + // passing in valid object name. + {"object", true}, + {"The Shining Script .pdf", true}, + {"Cost Benefit Analysis (2009-2010).pptx", true}, + {"117Gn8rfHL2ACARPAhaFd0AGzic9pUbIA/5OCn5A", true}, + {"SHØRT", true}, + {"f*le", true}, + {"contains-^-caret", true}, + {"contains-|-pipe", true}, + {"contains-`-tick", true}, + {"..test", true}, + {".. test", true}, + {". test", true}, + {".test", true}, + {"There are far too many object names, and far too few bucket names!", true}, + {"!\"#$%&'()*+,-.／:;<=>?@[\\]^_`{|}~/!\"#$%&'()*+,-.／:;<=>?@[\\]^_`{|}~)", true}, + {"!\"#$%&'()*+,-.／:;<=>?@[\\]^_`{|}~", true}, + {"␀␁␂␃␄␅␆␇␈␉␊␋␌␍␎␏␐␑␒␓␔␕␖␗␘␙␚␛␜␝␞␟␡", true}, + {"trailing VT␋/trailing VT␋", true}, + {"␋leading VT/␋leading VT", true}, + {"~leading tilde", true}, + {"\rleading CR", true}, + {"\nleading LF", true}, + {"\tleading HT", true}, + {"trailing CR\r", true}, + {"trailing LF\n", true}, + {"trailing HT\t", true}, + // cases for which test should fail. + // passing invalid object names. + {"", false}, + {"a/b/c/", false}, + {"../../etc", false}, + {"../../", false}, + {"/../../etc", false}, + {" ../etc", false}, + {"./././", false}, + {"./etc", false}, + {`contains//double/forwardslash`, false}, + {`//contains/double-forwardslash-prefix`, false}, + {string([]byte{0xff, 0xfe, 0xfd}), false}, + } + + for i, testCase := range testCases { + isValidObjectName := IsValidObjectName(testCase.objectName) + if testCase.shouldPass && !isValidObjectName { + t.Errorf("Test case %d: Expected \"%s\" to be a valid object name", i+1, testCase.objectName) + } + if !testCase.shouldPass && isValidObjectName { + t.Errorf("Test case %d: Expected object name \"%s\" to be invalid", i+1, testCase.objectName) + } + } +} + +// Tests getCompleteMultipartMD5 +func TestGetCompleteMultipartMD5(t *testing.T) { + testCases := []struct { + parts []CompletePart + expectedResult string + expectedErr string + }{ + // Wrong MD5 hash string, returns md5um of hash + {[]CompletePart{{ETag: "wrong-md5-hash-string"}}, "0deb8cb07527b4b2669c861cb9653607-1", ""}, + + // Single CompletePart with valid MD5 hash string. + {[]CompletePart{{ETag: "cf1f738a5924e645913c984e0fe3d708"}}, "10dc1617fbcf0bd0858048cb96e6bd77-1", ""}, + + // Multiple CompletePart with valid MD5 hash string. + {[]CompletePart{{ETag: "cf1f738a5924e645913c984e0fe3d708"}, {ETag: "9ccbc9a80eee7fb6fdd22441db2aedbd"}}, "0239a86b5266bb624f0ac60ba2aed6c8-2", ""}, + } + + for i, test := range testCases { + result := getCompleteMultipartMD5(test.parts) + if result != test.expectedResult { + t.Fatalf("test %d failed: expected: result=%v, got=%v", i+1, test.expectedResult, result) + } + } +} + +// TestIsMinioBucketName - Tests isMinioBucketName helper function. +func TestIsMinioMetaBucketName(t *testing.T) { + testCases := []struct { + bucket string + result bool + }{ + // MinIO meta bucket. + { + bucket: minioMetaBucket, + result: true, + }, + // MinIO meta bucket. + { + bucket: minioMetaMultipartBucket, + result: true, + }, + // MinIO meta bucket. + { + bucket: minioMetaTmpBucket, + result: true, + }, + // Normal bucket + { + bucket: "mybucket", + result: false, + }, + } + + for i, test := range testCases { + actual := isMinioMetaBucketName(test.bucket) + if actual != test.result { + t.Errorf("Test %d - expected %v but received %v", + i+1, test.result, actual) + } + } +} + +// Tests RemoveStandardStorageClass method. Expectation is metadata map +// should be cleared of x-amz-storage-class, if it is set to STANDARD +func TestRemoveStandardStorageClass(t *testing.T) { + tests := []struct { + name string + metadata map[string]string + want map[string]string + }{ + { + name: "1", + metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "STANDARD"}, + want: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86"}, + }, + { + name: "2", + metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY"}, + want: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY"}, + }, + { + name: "3", + metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86"}, + want: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86"}, + }, + } + for _, tt := range tests { + if got := removeStandardStorageClass(tt.metadata); !reflect.DeepEqual(got, tt.want) { + t.Errorf("Test %s failed, expected %v, got %v", tt.name, tt.want, got) + } + } +} + +// Tests CleanMetadata method. Expectation is metadata map +// should be cleared of etag, md5Sum and x-amz-storage-class, if it is set to STANDARD +func TestCleanMetadata(t *testing.T) { + tests := []struct { + name string + metadata map[string]string + want map[string]string + }{ + { + name: "1", + metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "STANDARD"}, + want: map[string]string{"content-type": "application/octet-stream"}, + }, + { + name: "2", + metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY"}, + want: map[string]string{"content-type": "application/octet-stream", "x-amz-storage-class": "REDUCED_REDUNDANCY"}, + }, + { + name: "3", + metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "md5Sum": "abcde"}, + want: map[string]string{"content-type": "application/octet-stream"}, + }, + } + for _, tt := range tests { + if got := cleanMetadata(tt.metadata); !reflect.DeepEqual(got, tt.want) { + t.Errorf("Test %s failed, expected %v, got %v", tt.name, tt.want, got) + } + } +} + +// Tests CleanMetadataKeys method. Expectation is metadata map +// should be cleared of keys passed to CleanMetadataKeys method +func TestCleanMetadataKeys(t *testing.T) { + tests := []struct { + name string + metadata map[string]string + keys []string + want map[string]string + }{ + { + name: "1", + metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "STANDARD", "md5": "abcde"}, + keys: []string{"etag", "md5"}, + want: map[string]string{"content-type": "application/octet-stream", "x-amz-storage-class": "STANDARD"}, + }, + { + name: "2", + metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "x-amz-storage-class": "REDUCED_REDUNDANCY", "md5sum": "abcde"}, + keys: []string{"etag", "md5sum"}, + want: map[string]string{"content-type": "application/octet-stream", "x-amz-storage-class": "REDUCED_REDUNDANCY"}, + }, + { + name: "3", + metadata: map[string]string{"content-type": "application/octet-stream", "etag": "de75a98baf2c6aef435b57dd0fc33c86", "xyz": "abcde"}, + keys: []string{"etag", "xyz"}, + want: map[string]string{"content-type": "application/octet-stream"}, + }, + } + for _, tt := range tests { + if got := cleanMetadataKeys(tt.metadata, tt.keys...); !reflect.DeepEqual(got, tt.want) { + t.Errorf("Test %s failed, expected %v, got %v", tt.name, tt.want, got) + } + } +} + +// Tests isCompressed method +func TestIsCompressed(t *testing.T) { + testCases := []struct { + objInfo ObjectInfo + result bool + err bool + }{ + 0: { + objInfo: ObjectInfo{ + UserDefined: map[string]string{ + "X-Minio-Internal-compression": compressionAlgorithmV1, + "content-type": "application/octet-stream", + "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2", + }, + }, + result: true, + }, + 1: { + objInfo: ObjectInfo{ + UserDefined: map[string]string{ + "X-Minio-Internal-compression": compressionAlgorithmV2, + "content-type": "application/octet-stream", + "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2", + }, + }, + result: true, + }, + 2: { + objInfo: ObjectInfo{ + UserDefined: map[string]string{ + "X-Minio-Internal-compression": "unknown/compression/type", + "content-type": "application/octet-stream", + "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2", + }, + }, + result: true, + err: true, + }, + 3: { + objInfo: ObjectInfo{ + UserDefined: map[string]string{ + "X-Minio-Internal-compression": compressionAlgorithmV2, + "content-type": "application/octet-stream", + "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2", + crypto.MetaIV: "yes", + }, + }, + result: true, + err: false, + }, + 4: { + objInfo: ObjectInfo{ + UserDefined: map[string]string{ + "X-Minio-Internal-XYZ": "klauspost/compress/s2", + "content-type": "application/octet-stream", + "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2", + }, + }, + result: false, + }, + 5: { + objInfo: ObjectInfo{ + UserDefined: map[string]string{ + "content-type": "application/octet-stream", + "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2", + }, + }, + result: false, + }, + } + for i, test := range testCases { + t.Run(strconv.Itoa(i), func(t *testing.T) { + got := test.objInfo.IsCompressed() + if got != test.result { + t.Errorf("IsCompressed: Expected %v but received %v", + test.result, got) + } + got, gErr := test.objInfo.IsCompressedOK() + if got != test.result { + t.Errorf("IsCompressedOK: Expected %v but received %v", + test.result, got) + } + if gErr != nil != test.err { + t.Errorf("IsCompressedOK: want error: %t, got error: %v", test.err, gErr) + } + }) + } +} + +// Tests excludeForCompression. +func TestExcludeForCompression(t *testing.T) { + testCases := []struct { + object string + header http.Header + result bool + }{ + { + object: "object.txt", + header: http.Header{ + "Content-Type": []string{"application/zip"}, + }, + result: true, + }, + { + object: "object.zip", + header: http.Header{ + "Content-Type": []string{"application/XYZ"}, + }, + result: true, + }, + { + object: "object.json", + header: http.Header{ + "Content-Type": []string{"application/json"}, + }, + result: false, + }, + { + object: "object.txt", + header: http.Header{ + "Content-Type": []string{"text/plain"}, + }, + result: false, + }, + { + object: "object", + header: http.Header{ + "Content-Type": []string{"text/something"}, + }, + result: false, + }, + } + for i, test := range testCases { + got := excludeForCompression(test.header, test.object, compress.Config{ + Enabled: true, + }) + if got != test.result { + t.Errorf("Test %d - expected %v but received %v", + i+1, test.result, got) + } + } +} + +func BenchmarkGetPartFileWithTrie(b *testing.B) { + b.ResetTimer() + + entriesTrie := trie.NewTrie() + for i := 1; i <= 10000; i++ { + entriesTrie.Insert(fmt.Sprintf("%.5d.8a034f82cb9cb31140d87d3ce2a9ede3.67108864", i)) + } + + for i := 1; i <= 10000; i++ { + partFile := getPartFile(entriesTrie, i, "8a034f82cb9cb31140d87d3ce2a9ede3") + if partFile == "" { + b.Fatal("partFile returned is empty") + } + } + + b.ReportAllocs() +} + +func TestGetActualSize(t *testing.T) { + testCases := []struct { + objInfo ObjectInfo + result int64 + }{ + { + objInfo: ObjectInfo{ + UserDefined: map[string]string{ + "X-Minio-Internal-compression": "klauspost/compress/s2", + "content-type": "application/octet-stream", + "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2", + }, + Parts: []ObjectPartInfo{ + { + Size: 39235668, + ActualSize: 67108864, + }, + { + Size: 19177372, + ActualSize: 32891137, + }, + }, + Size: 100000001, + }, + result: 100000001, + }, + { + objInfo: ObjectInfo{ + UserDefined: map[string]string{ + "X-Minio-Internal-compression": "klauspost/compress/s2", + "X-Minio-Internal-actual-size": "841", + "content-type": "application/octet-stream", + "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2", + }, + Parts: []ObjectPartInfo{}, + Size: 841, + }, + result: 841, + }, + { + objInfo: ObjectInfo{ + UserDefined: map[string]string{ + "X-Minio-Internal-compression": "klauspost/compress/s2", + "content-type": "application/octet-stream", + "etag": "b3ff3ef3789147152fbfbc50efba4bfd-2", + }, + Parts: []ObjectPartInfo{}, + Size: 100, + }, + result: -1, + }, + } + for i, test := range testCases { + got, _ := test.objInfo.GetActualSize() + if got != test.result { + t.Errorf("Test %d - expected %d but received %d", + i+1, test.result, got) + } + } +} + +func TestGetCompressedOffsets(t *testing.T) { + testCases := []struct { + objInfo ObjectInfo + offset int64 + startOffset int64 + snappyStartOffset int64 + firstPart int + }{ + 0: { + objInfo: ObjectInfo{ + Parts: []ObjectPartInfo{ + { + Size: 39235668, + ActualSize: 67108864, + }, + { + Size: 19177372, + ActualSize: 32891137, + }, + }, + }, + offset: 79109865, + startOffset: 39235668, + snappyStartOffset: 12001001, + firstPart: 1, + }, + 1: { + objInfo: ObjectInfo{ + Parts: []ObjectPartInfo{ + { + Size: 39235668, + ActualSize: 67108864, + }, + { + Size: 19177372, + ActualSize: 32891137, + }, + }, + }, + offset: 19109865, + startOffset: 0, + snappyStartOffset: 19109865, + }, + 2: { + objInfo: ObjectInfo{ + Parts: []ObjectPartInfo{ + { + Size: 39235668, + ActualSize: 67108864, + }, + { + Size: 19177372, + ActualSize: 32891137, + }, + }, + }, + offset: 0, + startOffset: 0, + snappyStartOffset: 0, + }, + } + for i, test := range testCases { + startOffset, snappyStartOffset, firstPart, _, _ := getCompressedOffsets(test.objInfo, test.offset, nil) + if startOffset != test.startOffset { + t.Errorf("Test %d - expected startOffset %d but received %d", + i, test.startOffset, startOffset) + } + if snappyStartOffset != test.snappyStartOffset { + t.Errorf("Test %d - expected snappyOffset %d but received %d", + i, test.snappyStartOffset, snappyStartOffset) + } + if firstPart != test.firstPart { + t.Errorf("Test %d - expected firstPart %d but received %d", + i, test.firstPart, firstPart) + } + } +} + +func TestS2CompressReader(t *testing.T) { + tests := []struct { + name string + data []byte + wantIdx bool + }{ + {name: "empty", data: nil}, + {name: "small", data: []byte("hello, world!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!")}, + {name: "large", data: bytes.Repeat([]byte("hello, world"), 1000000), wantIdx: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + buf := make([]byte, 100) // make small buffer to ensure multiple reads are required for large case + + r, idxCB := newS2CompressReader(bytes.NewReader(tt.data), int64(len(tt.data)), false) + defer r.Close() + + var rdrBuf bytes.Buffer + _, err := io.CopyBuffer(&rdrBuf, r, buf) + if err != nil { + t.Fatal(err) + } + r.Close() + idx := idxCB() + if !tt.wantIdx && len(idx) > 0 { + t.Errorf("index returned above threshold") + } + if tt.wantIdx { + if idx == nil { + t.Errorf("no index returned") + } + var index s2.Index + _, err = index.Load(s2.RestoreIndexHeaders(idx)) + if err != nil { + t.Errorf("error loading index: %v", err) + } + t.Log("size:", len(idx)) + t.Log(string(index.JSON())) + if index.TotalUncompressed != int64(len(tt.data)) { + t.Errorf("Expected size %d, got %d", len(tt.data), index.TotalUncompressed) + } + } + var stdBuf bytes.Buffer + w := s2.NewWriter(&stdBuf) + _, err = io.CopyBuffer(w, bytes.NewReader(tt.data), buf) + if err != nil { + t.Fatal(err) + } + err = w.Close() + if err != nil { + t.Fatal(err) + } + + var ( + got = rdrBuf.Bytes() + want = stdBuf.Bytes() + ) + if !bytes.Equal(got, want) { + t.Errorf("encoded data does not match\n\t%q\n\t%q", got, want) + } + + var decBuf bytes.Buffer + decRdr := s2.NewReader(&rdrBuf) + _, err = io.Copy(&decBuf, decRdr) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(tt.data, decBuf.Bytes()) { + t.Errorf("roundtrip failed\n\t%q\n\t%q", tt.data, decBuf.Bytes()) + } + }) + } +} + +func Test_pathNeedsClean(t *testing.T) { + type pathTest struct { + path, result string + } + + cleantests := []pathTest{ + // Already clean + {"", "."}, + {"abc", "abc"}, + {"abc/def", "abc/def"}, + {"a/b/c", "a/b/c"}, + {".", "."}, + {"..", ".."}, + {"../..", "../.."}, + {"../../abc", "../../abc"}, + {"/abc", "/abc"}, + {"/abc/def", "/abc/def"}, + {"/", "/"}, + + // Remove trailing slash + {"abc/", "abc"}, + {"abc/def/", "abc/def"}, + {"a/b/c/", "a/b/c"}, + {"./", "."}, + {"../", ".."}, + {"../../", "../.."}, + {"/abc/", "/abc"}, + + // Remove doubled slash + {"abc//def//ghi", "abc/def/ghi"}, + {"//abc", "/abc"}, + {"///abc", "/abc"}, + {"//abc//", "/abc"}, + {"abc//", "abc"}, + + // Remove . elements + {"abc/./def", "abc/def"}, + {"/./abc/def", "/abc/def"}, + {"abc/.", "abc"}, + + // Remove .. elements + {"abc/def/ghi/../jkl", "abc/def/jkl"}, + {"abc/def/../ghi/../jkl", "abc/jkl"}, + {"abc/def/..", "abc"}, + {"abc/def/../..", "."}, + {"/abc/def/../..", "/"}, + {"abc/def/../../..", ".."}, + {"/abc/def/../../..", "/"}, + {"abc/def/../../../ghi/jkl/../../../mno", "../../mno"}, + + // Combinations + {"abc/./../def", "def"}, + {"abc//./../def", "def"}, + {"abc/../../././../def", "../../def"}, + } + for _, test := range cleantests { + want := test.path != test.result + got := pathNeedsClean([]byte(test.path)) + if !got { + t.Logf("no clean: %q", test.path) + } + if want && !got { + t.Errorf("input: %q, want %v, got %v", test.path, want, got) + } + } +} diff --git a/cmd/object-handlers-common.go b/cmd/object-handlers-common.go new file mode 100644 index 00000000..a6febc12 --- /dev/null +++ b/cmd/object-handlers-common.go @@ -0,0 +1,428 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "fmt" + "net/http" + "regexp" + "strconv" + "strings" + "time" + + "github.com/minio/minio/internal/amztime" + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" +) + +var etagRegex = regexp.MustCompile("\"*?([^\"]*?)\"*?$") + +// Validates the preconditions for CopyObjectPart, returns true if CopyObjectPart +// operation should not proceed. Preconditions supported are: +// +// x-amz-copy-source-if-modified-since +// x-amz-copy-source-if-unmodified-since +// x-amz-copy-source-if-match +// x-amz-copy-source-if-none-match +func checkCopyObjectPartPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Request, objInfo ObjectInfo) bool { + return checkCopyObjectPreconditions(ctx, w, r, objInfo) +} + +// Validates the preconditions for CopyObject, returns true if CopyObject operation should not proceed. +// Preconditions supported are: +// +// x-amz-copy-source-if-modified-since +// x-amz-copy-source-if-unmodified-since +// x-amz-copy-source-if-match +// x-amz-copy-source-if-none-match +func checkCopyObjectPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Request, objInfo ObjectInfo) bool { + // Return false for methods other than GET and HEAD. + if r.Method != http.MethodPut { + return false + } + // If the object doesn't have a modtime (IsZero), or the modtime + // is obviously garbage (Unix time == 0), then ignore modtimes + // and don't process the If-Modified-Since header. + if objInfo.ModTime.IsZero() || objInfo.ModTime.Equal(time.Unix(0, 0)) { + return false + } + + // Headers to be set of object content is not going to be written to the client. + writeHeaders := func() { + // set common headers + setCommonHeaders(w) + + // set object-related metadata headers + w.Header().Set(xhttp.LastModified, objInfo.ModTime.UTC().Format(http.TimeFormat)) + + if objInfo.ETag != "" { + w.Header()[xhttp.ETag] = []string{"\"" + objInfo.ETag + "\""} + } + } + // x-amz-copy-source-if-modified-since: Return the object only if it has been modified + // since the specified time otherwise return 412 (precondition failed). + ifModifiedSinceHeader := r.Header.Get(xhttp.AmzCopySourceIfModifiedSince) + if ifModifiedSinceHeader != "" { + if givenTime, err := amztime.ParseHeader(ifModifiedSinceHeader); err == nil { + if !ifModifiedSince(objInfo.ModTime, givenTime) { + // If the object is not modified since the specified time. + writeHeaders() + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPreconditionFailed), r.URL) + return true + } + } + } + + // x-amz-copy-source-if-unmodified-since : Return the object only if it has not been + // modified since the specified time, otherwise return a 412 (precondition failed). + ifUnmodifiedSinceHeader := r.Header.Get(xhttp.AmzCopySourceIfUnmodifiedSince) + if ifUnmodifiedSinceHeader != "" { + if givenTime, err := amztime.ParseHeader(ifUnmodifiedSinceHeader); err == nil { + if ifModifiedSince(objInfo.ModTime, givenTime) { + // If the object is modified since the specified time. + writeHeaders() + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPreconditionFailed), r.URL) + return true + } + } + } + + // x-amz-copy-source-if-match : Return the object only if its entity tag (ETag) is the + // same as the one specified; otherwise return a 412 (precondition failed). + ifMatchETagHeader := r.Header.Get(xhttp.AmzCopySourceIfMatch) + if ifMatchETagHeader != "" { + if !isETagEqual(objInfo.ETag, ifMatchETagHeader) { + // If the object ETag does not match with the specified ETag. + writeHeaders() + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPreconditionFailed), r.URL) + return true + } + } + + // If-None-Match : Return the object only if its entity tag (ETag) is different from the + // one specified otherwise, return a 304 (not modified). + ifNoneMatchETagHeader := r.Header.Get(xhttp.AmzCopySourceIfNoneMatch) + if ifNoneMatchETagHeader != "" { + if isETagEqual(objInfo.ETag, ifNoneMatchETagHeader) { + // If the object ETag matches with the specified ETag. + writeHeaders() + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPreconditionFailed), r.URL) + return true + } + } + // Object content should be written to http.ResponseWriter + return false +} + +// Validates the preconditions. Returns true if PUT operation should not proceed. +// Preconditions supported are: +// +// x-minio-source-mtime +// x-minio-source-etag +func checkPreconditionsPUT(ctx context.Context, w http.ResponseWriter, r *http.Request, objInfo ObjectInfo, opts ObjectOptions) bool { + // Return false for methods other than PUT. + if r.Method != http.MethodPut && r.Method != http.MethodPost { + return false + } + // If the object doesn't have a modtime (IsZero), or the modtime + // is obviously garbage (Unix time == 0), then ignore modtimes + // and don't process the If-Modified-Since header. + if objInfo.ModTime.IsZero() || objInfo.ModTime.Equal(time.Unix(0, 0)) { + return false + } + + // If top level is a delete marker proceed to upload. + if objInfo.DeleteMarker { + return false + } + + // Headers to be set of object content is not going to be written to the client. + writeHeaders := func() { + // set common headers + setCommonHeaders(w) + + // set object-related metadata headers + w.Header().Set(xhttp.LastModified, objInfo.ModTime.UTC().Format(http.TimeFormat)) + + if objInfo.ETag != "" { + w.Header()[xhttp.ETag] = []string{"\"" + objInfo.ETag + "\""} + } + } + + // If-Match : Return the object only if its entity tag (ETag) is the same as the one specified; + // otherwise return a 412 (precondition failed). + ifMatchETagHeader := r.Header.Get(xhttp.IfMatch) + if ifMatchETagHeader != "" { + if !isETagEqual(objInfo.ETag, ifMatchETagHeader) { + // If the object ETag does not match with the specified ETag. + writeHeaders() + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPreconditionFailed), r.URL) + return true + } + } + + // If-None-Match : Return the object only if its entity tag (ETag) is different from the + // one specified otherwise, return a 304 (not modified). + ifNoneMatchETagHeader := r.Header.Get(xhttp.IfNoneMatch) + if ifNoneMatchETagHeader != "" { + if isETagEqual(objInfo.ETag, ifNoneMatchETagHeader) { + // If the object ETag matches with the specified ETag. + writeHeaders() + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPreconditionFailed), r.URL) + return true + } + } + + etagMatch := opts.PreserveETag != "" && isETagEqual(objInfo.ETag, opts.PreserveETag) + vidMatch := opts.VersionID != "" && opts.VersionID == objInfo.VersionID + if etagMatch && vidMatch { + writeHeaders() + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPreconditionFailed), r.URL) + return true + } + + // Object content should be persisted. + return false +} + +// Headers to be set of object content is not going to be written to the client. +func writeHeadersPrecondition(w http.ResponseWriter, objInfo ObjectInfo) { + // set common headers + setCommonHeaders(w) + + // set object-related metadata headers + w.Header().Set(xhttp.LastModified, objInfo.ModTime.UTC().Format(http.TimeFormat)) + + if objInfo.ETag != "" { + w.Header()[xhttp.ETag] = []string{"\"" + objInfo.ETag + "\""} + } + + if objInfo.VersionID != "" { + w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID} + } + + if !objInfo.Expires.IsZero() { + w.Header().Set(xhttp.Expires, objInfo.Expires.UTC().Format(http.TimeFormat)) + } + + if objInfo.CacheControl != "" { + w.Header().Set(xhttp.CacheControl, objInfo.CacheControl) + } +} + +// Validates the preconditions. Returns true if GET/HEAD operation should not proceed. +// Preconditions supported are: +// +// If-Modified-Since +// If-Unmodified-Since +// If-Match +// If-None-Match +func checkPreconditions(ctx context.Context, w http.ResponseWriter, r *http.Request, objInfo ObjectInfo, opts ObjectOptions) bool { + // Return false for methods other than GET and HEAD. + if r.Method != http.MethodGet && r.Method != http.MethodHead { + return false + } + // If the object doesn't have a modtime (IsZero), or the modtime + // is obviously garbage (Unix time == 0), then ignore modtimes + // and don't process the If-Modified-Since header. + if objInfo.ModTime.IsZero() || objInfo.ModTime.Equal(time.Unix(0, 0)) { + return false + } + + // Check if the part number is correct. + if opts.PartNumber > 1 { + partFound := false + for _, pi := range objInfo.Parts { + if pi.Number == opts.PartNumber { + partFound = true + break + } + } + if !partFound { + // According to S3 we don't need to set any object information here. + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidPartNumber), r.URL) + return true + } + } + + // If-None-Match : Return the object only if its entity tag (ETag) is different from the + // one specified otherwise, return a 304 (not modified). + ifNoneMatchETagHeader := r.Header.Get(xhttp.IfNoneMatch) + if ifNoneMatchETagHeader != "" { + if isETagEqual(objInfo.ETag, ifNoneMatchETagHeader) { + // Do not care If-Modified-Since, Because: + // 1. If If-Modified-Since condition evaluates to true. + // If both of the If-None-Match and If-Modified-Since headers are present in the request as follows: + // If-None-Match condition evaluates to false , and; + // If-Modified-Since condition evaluates to true ; + // Then Amazon S3 returns the 304 Not Modified response code. + // 2. If If-Modified-Since condition evaluates to false, The following `ifModifiedSinceHeader` judgment will also return 304 + + // If the object ETag matches with the specified ETag. + writeHeadersPrecondition(w, objInfo) + w.WriteHeader(http.StatusNotModified) + return true + } + } + + // If-Modified-Since : Return the object only if it has been modified since the specified time, + // otherwise return a 304 (not modified). + ifModifiedSinceHeader := r.Header.Get(xhttp.IfModifiedSince) + if ifModifiedSinceHeader != "" { + if givenTime, err := amztime.ParseHeader(ifModifiedSinceHeader); err == nil { + if !ifModifiedSince(objInfo.ModTime, givenTime) { + // If the object is not modified since the specified time. + writeHeadersPrecondition(w, objInfo) + w.WriteHeader(http.StatusNotModified) + return true + } + } + } + + // If-Match : Return the object only if its entity tag (ETag) is the same as the one specified; + // otherwise return a 412 (precondition failed). + ifMatchETagHeader := r.Header.Get(xhttp.IfMatch) + if ifMatchETagHeader != "" { + if !isETagEqual(objInfo.ETag, ifMatchETagHeader) { + // If the object ETag does not match with the specified ETag. + writeHeadersPrecondition(w, objInfo) + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPreconditionFailed), r.URL) + return true + } + } + + // If-Unmodified-Since : Return the object only if it has not been modified since the specified + // time, otherwise return a 412 (precondition failed). + ifUnmodifiedSinceHeader := r.Header.Get(xhttp.IfUnmodifiedSince) + if ifUnmodifiedSinceHeader != "" && ifMatchETagHeader == "" { + if givenTime, err := amztime.ParseHeader(ifUnmodifiedSinceHeader); err == nil { + if ifModifiedSince(objInfo.ModTime, givenTime) { + // If the object is modified since the specified time. + writeHeadersPrecondition(w, objInfo) + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrPreconditionFailed), r.URL) + return true + } + } + } + + // Object content should be written to http.ResponseWriter + return false +} + +// returns true if object was modified after givenTime. +func ifModifiedSince(objTime time.Time, givenTime time.Time) bool { + // The Date-Modified header truncates sub-second precision, so + // use mtime < t+1s instead of mtime <= t to check for unmodified. + return !objTime.Before(givenTime.Add(1 * time.Second)) +} + +// canonicalizeETag returns ETag with leading and trailing double-quotes removed, +// if any present +func canonicalizeETag(etag string) string { + return etagRegex.ReplaceAllString(etag, "$1") +} + +// isETagEqual return true if the canonical representations of two ETag strings +// are equal, false otherwise +func isETagEqual(left, right string) bool { + if strings.TrimSpace(right) == "*" { + return true + } + return canonicalizeETag(left) == canonicalizeETag(right) +} + +// setPutObjHeaders sets all the necessary headers returned back +// upon a success Put/Copy/CompleteMultipart/Delete requests +// to activate delete only headers set delete as true +func setPutObjHeaders(w http.ResponseWriter, objInfo ObjectInfo, del bool, h http.Header) { + // We must not use the http.Header().Set method here because some (broken) + // clients expect the ETag header key to be literally "ETag" - not "Etag" (case-sensitive). + // Therefore, we have to set the ETag directly as map entry. + if objInfo.ETag != "" && !del { + w.Header()[xhttp.ETag] = []string{`"` + objInfo.ETag + `"`} + } + + // Set the relevant version ID as part of the response header. + if objInfo.VersionID != "" && objInfo.VersionID != nullVersionID { + w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID} + // If version is a deleted marker, set this header as well + if objInfo.DeleteMarker && del { // only returned during delete object + w.Header()[xhttp.AmzDeleteMarker] = []string{strconv.FormatBool(objInfo.DeleteMarker)} + } + } + + if objInfo.Bucket != "" && objInfo.Name != "" { + if lc, err := globalLifecycleSys.Get(objInfo.Bucket); err == nil && !del { + lc.SetPredictionHeaders(w, objInfo.ToLifecycleOpts()) + } + } + cs, _ := objInfo.decryptChecksums(0, h) + hash.AddChecksumHeader(w, cs) +} + +func deleteObjectVersions(ctx context.Context, o ObjectLayer, bucket string, toDel []ObjectToDelete, lcEvent []lifecycle.Event) { + for remaining := toDel; len(remaining) > 0; toDel = remaining { + if len(toDel) > maxDeleteList { + remaining = toDel[maxDeleteList:] + toDel = toDel[:maxDeleteList] + } else { + remaining = nil + } + vc, _ := globalBucketVersioningSys.Get(bucket) + deletedObjs, errs := o.DeleteObjects(ctx, bucket, toDel, ObjectOptions{ + PrefixEnabledFn: vc.PrefixEnabled, + VersionSuspended: vc.Suspended(), + }) + + for i, dobj := range deletedObjs { + oi := ObjectInfo{ + Bucket: bucket, + Name: dobj.ObjectName, + VersionID: dobj.VersionID, + } + traceFn := globalLifecycleSys.trace(oi) + tags := newLifecycleAuditEvent(lcEventSrc_Scanner, lcEvent[i]).Tags() + + // Send audit for the lifecycle delete operation + auditLogLifecycle( + ctx, + oi, + ILMExpiry, tags, traceFn) + + evArgs := eventArgs{ + EventName: event.ObjectRemovedDelete, + BucketName: bucket, + Object: ObjectInfo{ + Name: dobj.ObjectName, + VersionID: dobj.VersionID, + }, + UserAgent: "Internal: [ILM-Expiry]", + Host: globalLocalNodeName, + } + if errs[i] != nil { + evArgs.RespElements = map[string]string{ + "error": fmt.Sprintf("failed to delete %s(%s), with error %v", dobj.ObjectName, dobj.VersionID, errs[i]), + } + } + sendEvent(evArgs) + } + } +} diff --git a/cmd/object-handlers-common_test.go b/cmd/object-handlers-common_test.go new file mode 100644 index 00000000..80555a1a --- /dev/null +++ b/cmd/object-handlers-common_test.go @@ -0,0 +1,181 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "net/http" + "net/http/httptest" + "testing" + "time" + + xhttp "github.com/minio/minio/internal/http" +) + +// Tests - canonicalizeETag() +func TestCanonicalizeETag(t *testing.T) { + testCases := []struct { + etag string + canonicalizedETag string + }{ + { + etag: "\"\"\"", + canonicalizedETag: "", + }, + { + etag: "\"\"\"abc\"", + canonicalizedETag: "abc", + }, + { + etag: "abcd", + canonicalizedETag: "abcd", + }, + { + etag: "abcd\"\"", + canonicalizedETag: "abcd", + }, + } + for _, test := range testCases { + etag := canonicalizeETag(test.etag) + if test.canonicalizedETag != etag { + t.Fatalf("Expected %s , got %s", test.canonicalizedETag, etag) + } + } +} + +// Tests - CheckPreconditions() +func TestCheckPreconditions(t *testing.T) { + objModTime := time.Date(2024, time.August, 26, 0o2, 0o1, 0o1, 0, time.UTC) + objInfo := ObjectInfo{ETag: "aa", ModTime: objModTime} + testCases := []struct { + name string + ifMatch string + ifNoneMatch string + ifModifiedSince string + ifUnmodifiedSince string + objInfo ObjectInfo + expectedFlag bool + expectedCode int + }{ + // If-None-Match(false) and If-Modified-Since(true) + { + name: "If-None-Match1", + ifNoneMatch: "aa", + ifModifiedSince: "Sun, 26 Aug 2024 02:01:00 GMT", + objInfo: objInfo, + expectedFlag: true, + expectedCode: 304, + }, + // If-Modified-Since(false) + { + name: "If-None-Match2", + ifNoneMatch: "aaa", + ifModifiedSince: "Sun, 26 Aug 2024 02:01:01 GMT", + objInfo: objInfo, + expectedFlag: true, + expectedCode: 304, + }, + { + name: "If-None-Match3", + ifNoneMatch: "aaa", + ifModifiedSince: "Sun, 26 Aug 2024 02:01:02 GMT", + objInfo: objInfo, + expectedFlag: true, + expectedCode: 304, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodHead, "/bucket/a", bytes.NewReader([]byte{})) + request.Header.Set(xhttp.IfNoneMatch, tc.ifNoneMatch) + request.Header.Set(xhttp.IfModifiedSince, tc.ifModifiedSince) + request.Header.Set(xhttp.IfMatch, tc.ifMatch) + request.Header.Set(xhttp.IfUnmodifiedSince, tc.ifUnmodifiedSince) + actualFlag := checkPreconditions(t.Context(), recorder, request, tc.objInfo, ObjectOptions{}) + if tc.expectedFlag != actualFlag { + t.Errorf("test: %s, got flag: %v, want: %v", tc.name, actualFlag, tc.expectedFlag) + } + if tc.expectedCode != recorder.Code { + t.Errorf("test: %s, got code: %d, want: %d", tc.name, recorder.Code, tc.expectedCode) + } + }) + } + testCases = []struct { + name string + ifMatch string + ifNoneMatch string + ifModifiedSince string + ifUnmodifiedSince string + objInfo ObjectInfo + expectedFlag bool + expectedCode int + }{ + // If-Match(true) and If-Unmodified-Since(false) + { + name: "If-Match1", + ifMatch: "aa", + ifUnmodifiedSince: "Sun, 26 Aug 2024 02:01:00 GMT", + objInfo: objInfo, + expectedFlag: false, + expectedCode: 200, + }, + // If-Unmodified-Since(true) + { + name: "If-Match2", + ifMatch: "aa", + ifUnmodifiedSince: "Sun, 26 Aug 2024 02:01:01 GMT", + objInfo: objInfo, + expectedFlag: false, + expectedCode: 200, + }, + { + name: "If-Match3", + ifMatch: "aa", + ifUnmodifiedSince: "Sun, 26 Aug 2024 02:01:02 GMT", + objInfo: objInfo, + expectedFlag: false, + expectedCode: 200, + }, + // If-Match(true) + { + name: "If-Match4", + ifMatch: "aa", + objInfo: objInfo, + expectedFlag: false, + expectedCode: 200, + }, + } + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + recorder := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodHead, "/bucket/a", bytes.NewReader([]byte{})) + request.Header.Set(xhttp.IfNoneMatch, tc.ifNoneMatch) + request.Header.Set(xhttp.IfModifiedSince, tc.ifModifiedSince) + request.Header.Set(xhttp.IfMatch, tc.ifMatch) + request.Header.Set(xhttp.IfUnmodifiedSince, tc.ifUnmodifiedSince) + actualFlag := checkPreconditions(t.Context(), recorder, request, tc.objInfo, ObjectOptions{}) + if tc.expectedFlag != actualFlag { + t.Errorf("test: %s, got flag: %v, want: %v", tc.name, actualFlag, tc.expectedFlag) + } + if tc.expectedCode != recorder.Code { + t.Errorf("test: %s, got code: %d, want: %d", tc.name, recorder.Code, tc.expectedCode) + } + }) + } +} diff --git a/cmd/object-handlers.go b/cmd/object-handlers.go new file mode 100644 index 00000000..6542c465 --- /dev/null +++ b/cmd/object-handlers.go @@ -0,0 +1,3585 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "archive/tar" + "context" + "encoding/hex" + "encoding/xml" + "errors" + "fmt" + "io" + "maps" + "net/http" + "net/http/httptest" + "net/textproto" + "net/url" + "os" + "sort" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/google/uuid" + "github.com/klauspost/compress/gzhttp" + miniogo "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio-go/v7/pkg/encrypt" + "github.com/minio/minio-go/v7/pkg/tags" + "github.com/minio/minio/internal/amztime" + "github.com/minio/minio/internal/auth" + sse "github.com/minio/minio/internal/bucket/encryption" + "github.com/minio/minio/internal/bucket/lifecycle" + objectlock "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/config/dns" + "github.com/minio/minio/internal/config/storageclass" + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/etag" + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/handlers" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/kms" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/s3select" + "github.com/minio/mux" + "github.com/minio/pkg/v3/policy" +) + +// supportedHeadGetReqParams - supported request parameters for GET and HEAD presigned request. +var supportedHeadGetReqParams = map[string]string{ + "response-expires": xhttp.Expires, + "response-content-type": xhttp.ContentType, + "response-cache-control": xhttp.CacheControl, + "response-content-encoding": xhttp.ContentEncoding, + "response-content-language": xhttp.ContentLanguage, + "response-content-disposition": xhttp.ContentDisposition, +} + +const ( + compressionAlgorithmV1 = "golang/snappy/LZ77" + compressionAlgorithmV2 = "klauspost/compress/s2" + + // When an upload exceeds encryptBufferThreshold ... + encryptBufferThreshold = 1 << 20 + // add an input buffer of this size. + encryptBufferSize = 1 << 20 + + // minCompressibleSize is the minimum size at which we enable compression. + minCompressibleSize = 4096 +) + +// setHeadGetRespHeaders - set any requested parameters as response headers. +func setHeadGetRespHeaders(w http.ResponseWriter, reqParams url.Values) { + for k, v := range reqParams { + if header, ok := supportedHeadGetReqParams[strings.ToLower(k)]; ok { + w.Header()[header] = []string{strings.Join(v, ",")} + } + } +} + +// SelectObjectContentHandler - GET Object?select +// ---------- +// This implementation of the GET operation retrieves object content based +// on an SQL expression. In the request, along with the sql expression, you must +// also specify a data serialization format (JSON, CSV) of the object. +func (api objectAPIHandlers) SelectObjectContentHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "SelectObject") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + // Fetch object stat info. + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if crypto.S3.IsRequested(r.Header) || crypto.S3KMS.IsRequested(r.Header) { // If SSE-S3 or SSE-KMS present -> AWS fails with undefined error + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL) + return + } + + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + opts, err := getOpts(ctx, r, bucket, object) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + getObjectInfo := objectAPI.GetObjectInfo + + // Check for auth type to return S3 compatible error. + // type to return the correct error (NoSuchKey vs AccessDenied) + if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, bucket, object); s3Error != ErrNone { + if getRequestAuthType(r) == authTypeAnonymous { + // As per "Permission" section in + // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html + // If the object you request does not exist, + // the error Amazon S3 returns depends on + // whether you also have the s3:ListBucket + // permission. + // * If you have the s3:ListBucket permission + // on the bucket, Amazon S3 will return an + // HTTP status code 404 ("no such key") + // error. + // * if you don’t have the s3:ListBucket + // permission, Amazon S3 will return an HTTP + // status code 403 ("access denied") error.` + if globalPolicySys.IsAllowed(policy.BucketPolicyArgs{ + Action: policy.ListBucketAction, + BucketName: bucket, + ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials), + IsOwner: false, + }) { + _, err = getObjectInfo(ctx, bucket, object, opts) + if toAPIError(ctx, err).Code == "NoSuchKey" { + s3Error = ErrNoSuchKey + } + } + } + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Get request range. + rangeHeader := r.Header.Get(xhttp.Range) + if rangeHeader != "" { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrUnsupportedRangeHeader), r.URL) + return + } + + if r.ContentLength <= 0 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEmptyRequestBody), r.URL) + return + } + + // Take read lock on object, here so subsequent lower-level + // calls do not need to. + lock := objectAPI.NewNSLock(bucket, object) + lkctx, err := lock.GetRLock(ctx, globalOperationTimeout) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + ctx = lkctx.Context() + defer lock.RUnlock(lkctx) + + getObjectNInfo := objectAPI.GetObjectNInfo + + gopts := opts + gopts.NoLock = true // We already have a lock, we can live with it. + objInfo, err := getObjectInfo(ctx, bucket, object, gopts) + if err != nil { + // Versioning enabled quite possibly object is deleted might be delete-marker + // if present set the headers, no idea why AWS S3 sets these headers. + if objInfo.VersionID != "" && objInfo.DeleteMarker { + w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID} + w.Header()[xhttp.AmzDeleteMarker] = []string{strconv.FormatBool(objInfo.DeleteMarker)} + } + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // filter object lock metadata if permission does not permit + getRetPerms := checkRequestAuthType(ctx, r, policy.GetObjectRetentionAction, bucket, object) + legalHoldPerms := checkRequestAuthType(ctx, r, policy.GetObjectLegalHoldAction, bucket, object) + + // filter object lock metadata if permission does not permit + objInfo.UserDefined = objectlock.FilterObjectLockMetadata(objInfo.UserDefined, getRetPerms != ErrNone, legalHoldPerms != ErrNone) + + if _, err = DecryptObjectInfo(&objInfo, r); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + actualSize, err := objInfo.GetActualSize() + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + objectRSC := s3select.NewObjectReadSeekCloser( + func(offset int64) (io.ReadCloser, error) { + rs := &HTTPRangeSpec{ + IsSuffixLength: false, + Start: offset, + End: -1, + } + opts.NoLock = true + return getObjectNInfo(ctx, bucket, object, rs, r.Header, opts) + }, + actualSize, + ) + defer objectRSC.Close() + s3Select, err := s3select.NewS3Select(r.Body) + if err != nil { + if serr, ok := err.(s3select.SelectError); ok { + encodedErrorResponse := encodeResponse(APIErrorResponse{ + Code: serr.ErrorCode(), + Message: serr.ErrorMessage(), + BucketName: bucket, + Key: object, + Resource: r.URL.Path, + RequestID: w.Header().Get(xhttp.AmzRequestID), + HostID: globalDeploymentID(), + }) + writeResponse(w, serr.HTTPStatusCode(), encodedErrorResponse, mimeXML) + } else { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + } + return + } + defer s3Select.Close() + + if err = s3Select.Open(objectRSC); err != nil { + if serr, ok := err.(s3select.SelectError); ok { + encodedErrorResponse := encodeResponse(APIErrorResponse{ + Code: serr.ErrorCode(), + Message: serr.ErrorMessage(), + BucketName: bucket, + Key: object, + Resource: r.URL.Path, + RequestID: w.Header().Get(xhttp.AmzRequestID), + HostID: globalDeploymentID(), + }) + writeResponse(w, serr.HTTPStatusCode(), encodedErrorResponse, mimeXML) + } else { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + } + return + } + + // Set encryption response headers + switch kind, _ := crypto.IsEncrypted(objInfo.UserDefined); kind { + case crypto.S3: + w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES) + case crypto.S3KMS: + w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS) + w.Header().Set(xhttp.AmzServerSideEncryptionKmsID, objInfo.KMSKeyID()) + if kmsCtx, ok := objInfo.UserDefined[crypto.MetaContext]; ok { + w.Header().Set(xhttp.AmzServerSideEncryptionKmsContext, kmsCtx) + } + case crypto.SSEC: + // Validate the SSE-C Key set in the header. + if _, err = crypto.SSEC.UnsealObjectKey(r.Header, objInfo.UserDefined, bucket, object); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + w.Header().Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerAlgorithm)) + w.Header().Set(xhttp.AmzServerSideEncryptionCustomerKeyMD5, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5)) + } + + s3Select.Evaluate(w) + + // Notify object accessed via a GET request. + sendEvent(eventArgs{ + EventName: event.ObjectAccessedGet, + BucketName: bucket, + Object: objInfo, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) +} + +func (api objectAPIHandlers) getObjectHandler(ctx context.Context, objectAPI ObjectLayer, bucket, object string, w http.ResponseWriter, r *http.Request) { + if crypto.S3.IsRequested(r.Header) || crypto.S3KMS.IsRequested(r.Header) { // If SSE-S3 or SSE-KMS present -> AWS fails with undefined error + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL) + return + } + + opts, err := getOpts(ctx, r, bucket, object) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Check for auth type to return S3 compatible error. + // type to return the correct error (NoSuchKey vs AccessDenied) + if s3Error := authenticateRequest(ctx, r, policy.GetObjectAction); s3Error != ErrNone { + if getRequestAuthType(r) == authTypeAnonymous { + // As per "Permission" section in + // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html + // If the object you request does not exist, + // the error Amazon S3 returns depends on + // whether you also have the s3:ListBucket + // permission. + // * If you have the s3:ListBucket permission + // on the bucket, Amazon S3 will return an + // HTTP status code 404 ("no such key") + // error. + // * if you don’t have the s3:ListBucket + // permission, Amazon S3 will return an HTTP + // status code 403 ("access denied") error.` + if globalPolicySys.IsAllowed(policy.BucketPolicyArgs{ + Action: policy.ListBucketAction, + BucketName: bucket, + ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials), + IsOwner: false, + }) { + getObjectInfo := objectAPI.GetObjectInfo + + _, err = getObjectInfo(ctx, bucket, object, opts) + if toAPIError(ctx, err).Code == "NoSuchKey" { + s3Error = ErrNoSuchKey + } + } + } + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + getObjectNInfo := objectAPI.GetObjectNInfo + + // Get request range. + var rs *HTTPRangeSpec + var rangeErr error + rangeHeader := r.Header.Get(xhttp.Range) + if rangeHeader != "" { + // Both 'Range' and 'partNumber' cannot be specified at the same time + if opts.PartNumber > 0 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRangePartNumber), r.URL) + return + } + + rs, rangeErr = parseRequestRangeSpec(rangeHeader) + // Handle only errInvalidRange. Ignore other + // parse error and treat it as regular Get + // request like Amazon S3. + if errors.Is(rangeErr, errInvalidRange) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRange), r.URL) + return + } + } + + // Validate pre-conditions if any. + opts.CheckPrecondFn = func(oi ObjectInfo) bool { + if _, err := DecryptObjectInfo(&oi, r); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return true + } + + if oi.UserTags != "" { + r.Header.Set(xhttp.AmzObjectTagging, oi.UserTags) + } + + if s3Error := authorizeRequest(ctx, r, policy.GetObjectAction); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return true + } + + return checkPreconditions(ctx, w, r, oi, opts) + } + + opts.FastGetObjInfo = true + + var proxy proxyResult + gr, err := getObjectNInfo(ctx, bucket, object, rs, r.Header, opts) + if err != nil { + var ( + reader *GetObjectReader + perr error + ) + + if (isErrObjectNotFound(err) || isErrVersionNotFound(err) || isErrReadQuorum(err)) && (gr == nil || !gr.ObjInfo.DeleteMarker) { + proxytgts := getProxyTargets(ctx, bucket, object, opts) + if !proxytgts.Empty() { + globalReplicationStats.Load().incProxy(bucket, getObjectAPI, false) + // proxy to replication target if active-active replication is in place. + reader, proxy, perr = proxyGetToReplicationTarget(ctx, bucket, object, rs, r.Header, opts, proxytgts) + if perr != nil { + globalReplicationStats.Load().incProxy(bucket, getObjectAPI, true) + proxyGetErr := ErrorRespToObjectError(perr, bucket, object) + if !isErrBucketNotFound(proxyGetErr) && !isErrObjectNotFound(proxyGetErr) && !isErrVersionNotFound(proxyGetErr) && + !isErrPreconditionFailed(proxyGetErr) && !isErrInvalidRange(proxyGetErr) { + replLogIf(ctx, fmt.Errorf("Proxying request (replication) failed for %s/%s(%s) - %w", bucket, object, opts.VersionID, perr)) + } + } + if reader != nil && proxy.Proxy && perr == nil { + gr = reader + } + } + } + if reader == nil || !proxy.Proxy { + // validate if the request indeed was authorized, if it wasn't we need to return "ErrAccessDenied" + // instead of any namespace related error. + if s3Error := authorizeRequest(ctx, r, policy.GetObjectAction); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + if isErrPreconditionFailed(err) { + return + } + if proxy.Err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, proxy.Err), r.URL) + return + } + if gr != nil { + if !gr.ObjInfo.VersionPurgeStatus.Empty() { + // Shows the replication status of a permanent delete of a version + w.Header()[xhttp.MinIODeleteReplicationStatus] = []string{string(gr.ObjInfo.VersionPurgeStatus)} + } + if !gr.ObjInfo.ReplicationStatus.Empty() && gr.ObjInfo.DeleteMarker { + w.Header()[xhttp.MinIODeleteMarkerReplicationStatus] = []string{string(gr.ObjInfo.ReplicationStatus)} + } + + // Versioning enabled quite possibly object is deleted might be delete-marker + // if present set the headers, no idea why AWS S3 sets these headers. + if gr.ObjInfo.VersionID != "" && gr.ObjInfo.DeleteMarker { + w.Header()[xhttp.AmzVersionID] = []string{gr.ObjInfo.VersionID} + w.Header()[xhttp.AmzDeleteMarker] = []string{strconv.FormatBool(gr.ObjInfo.DeleteMarker)} + } + QueueReplicationHeal(ctx, bucket, gr.ObjInfo, 0) + } + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + defer gr.Close() + + objInfo := gr.ObjInfo + + if !proxy.Proxy { // apply lifecycle rules only for local requests + // Automatically remove the object/version if an expiry lifecycle rule can be applied + if lc, err := globalLifecycleSys.Get(bucket); err == nil { + rcfg, err := globalBucketObjectLockSys.Get(bucket) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + replcfg, err := getReplicationConfig(ctx, bucket) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + event := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, objInfo) + if event.Action.Delete() { + // apply whatever the expiry rule is. + applyExpiryRule(event, lcEventSrc_s3GetObject, objInfo) + if !event.Action.DeleteRestored() { + // If the ILM action is not on restored object return error. + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNoSuchKey), r.URL) + return + } + } + } + + QueueReplicationHeal(ctx, bucket, gr.ObjInfo, 0) + } + + // filter object lock metadata if permission does not permit + getRetPerms := checkRequestAuthType(ctx, r, policy.GetObjectRetentionAction, bucket, object) + legalHoldPerms := checkRequestAuthType(ctx, r, policy.GetObjectLegalHoldAction, bucket, object) + + // filter object lock metadata if permission does not permit + objInfo.UserDefined = objectlock.FilterObjectLockMetadata(objInfo.UserDefined, getRetPerms != ErrNone, legalHoldPerms != ErrNone) + + // Set encryption response headers + if kind, isEncrypted := crypto.IsEncrypted(objInfo.UserDefined); isEncrypted { + switch kind { + case crypto.S3: + w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES) + case crypto.S3KMS: + w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS) + w.Header().Set(xhttp.AmzServerSideEncryptionKmsID, objInfo.KMSKeyID()) + if kmsCtx, ok := objInfo.UserDefined[crypto.MetaContext]; ok { + w.Header().Set(xhttp.AmzServerSideEncryptionKmsContext, kmsCtx) + } + case crypto.SSEC: + w.Header().Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerAlgorithm)) + w.Header().Set(xhttp.AmzServerSideEncryptionCustomerKeyMD5, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5)) + } + objInfo.ETag = getDecryptedETag(r.Header, objInfo, false) + } + + if r.Header.Get(xhttp.AmzChecksumMode) == "ENABLED" && rs == nil { + // AWS S3 silently drops checksums on range requests. + cs, _ := objInfo.decryptChecksums(opts.PartNumber, r.Header) + hash.AddChecksumHeader(w, cs) + } + + if err = setObjectHeaders(ctx, w, objInfo, rs, opts); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Set Parts Count Header + if opts.PartNumber > 0 && len(objInfo.Parts) > 0 { + setPartsCountHeaders(w, objInfo) + } + + setHeadGetRespHeaders(w, r.Form) + + var iw io.Writer = w + + statusCodeWritten := false + httpWriter := xioutil.WriteOnClose(iw) + if rs != nil || opts.PartNumber > 0 { + statusCodeWritten = true + w.WriteHeader(http.StatusPartialContent) + } + + // Write object content to response body + if _, err = xioutil.Copy(httpWriter, gr); err != nil { + if !httpWriter.HasWritten() && !statusCodeWritten { + // write error response only if no data or headers has been written to client yet + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + return + } + + if err = httpWriter.Close(); err != nil { + if !httpWriter.HasWritten() && !statusCodeWritten { // write error response only if no data or headers has been written to client yet + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + return + } + + // Notify object accessed via a GET request. + sendEvent(eventArgs{ + EventName: event.ObjectAccessedGet, + BucketName: bucket, + Object: objInfo, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) +} + +// GetObjectAttributes ... +func (api objectAPIHandlers) getObjectAttributesHandler(ctx context.Context, objectAPI ObjectLayer, bucket, object string, w http.ResponseWriter, r *http.Request) { + opts, valid := getAndValidateAttributesOpts(ctx, w, r, bucket, object) + if !valid { + return + } + + var s3Error APIErrorCode + if opts.VersionID != "" { + s3Error = checkRequestAuthType(ctx, r, policy.GetObjectVersionAttributesAction, bucket, object) + if s3Error == ErrNone { + s3Error = checkRequestAuthType(ctx, r, policy.GetObjectVersionAction, bucket, object) + } + } else { + s3Error = checkRequestAuthType(ctx, r, policy.GetObjectAttributesAction, bucket, object) + if s3Error == ErrNone { + s3Error = checkRequestAuthType(ctx, r, policy.GetObjectAction, bucket, object) + } + } + + if s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + objInfo, err := objectAPI.GetObjectInfo(ctx, bucket, object, opts) + if err != nil { + s3Error = checkRequestAuthType(ctx, r, policy.ListBucketAction, bucket, object) + if s3Error == ErrNone { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrAccessDenied), r.URL) + return + } + + if _, err = DecryptObjectInfo(&objInfo, r); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if checkPreconditions(ctx, w, r, objInfo, opts) { + return + } + + OA := new(getObjectAttributesResponse) + + if opts.Versioned { + w.Header().Set(xhttp.AmzVersionID, objInfo.VersionID) + } + + lastModified := objInfo.ModTime.UTC().Format(http.TimeFormat) + w.Header().Set(xhttp.LastModified, lastModified) + w.Header().Del(xhttp.ContentType) + + if _, ok := opts.ObjectAttributes[xhttp.Checksum]; ok { + chkSums, _ := objInfo.decryptChecksums(0, r.Header) + // AWS does not appear to append part number on this API call. + if len(chkSums) > 0 { + OA.Checksum = &objectAttributesChecksum{ + ChecksumCRC32: strings.Split(chkSums["CRC32"], "-")[0], + ChecksumCRC32C: strings.Split(chkSums["CRC32C"], "-")[0], + ChecksumSHA1: strings.Split(chkSums["SHA1"], "-")[0], + ChecksumSHA256: strings.Split(chkSums["SHA256"], "-")[0], + ChecksumCRC64NVME: strings.Split(chkSums["CRC64NVME"], "-")[0], + ChecksumType: chkSums[xhttp.AmzChecksumType], + } + } + } + + if _, ok := opts.ObjectAttributes[xhttp.ETag]; ok { + OA.ETag = objInfo.ETag + } + + if _, ok := opts.ObjectAttributes[xhttp.ObjectSize]; ok { + OA.ObjectSize, _ = objInfo.GetActualSize() + } + + if _, ok := opts.ObjectAttributes[xhttp.StorageClass]; ok { + OA.StorageClass = filterStorageClass(ctx, objInfo.StorageClass) + } + + objInfo.decryptPartsChecksums(r.Header) + + if _, ok := opts.ObjectAttributes[xhttp.ObjectParts]; ok { + OA.ObjectParts = new(objectAttributesParts) + OA.ObjectParts.PartNumberMarker = opts.PartNumberMarker + + OA.ObjectParts.MaxParts = opts.MaxParts + partsLength := len(objInfo.Parts) + OA.ObjectParts.PartsCount = partsLength + + if opts.MaxParts > -1 { + for i, v := range objInfo.Parts { + if v.Number <= opts.PartNumberMarker { + continue + } + + if len(OA.ObjectParts.Parts) == opts.MaxParts { + break + } + + OA.ObjectParts.NextPartNumberMarker = v.Number + OA.ObjectParts.Parts = append(OA.ObjectParts.Parts, &objectAttributesPart{ + ChecksumSHA1: objInfo.Parts[i].Checksums["SHA1"], + ChecksumSHA256: objInfo.Parts[i].Checksums["SHA256"], + ChecksumCRC32: objInfo.Parts[i].Checksums["CRC32"], + ChecksumCRC32C: objInfo.Parts[i].Checksums["CRC32C"], + ChecksumCRC64NVME: objInfo.Parts[i].Checksums["CRC64NVME"], + PartNumber: objInfo.Parts[i].Number, + Size: objInfo.Parts[i].Size, + }) + } + } + + if OA.ObjectParts.NextPartNumberMarker != partsLength { + OA.ObjectParts.IsTruncated = true + } + } + + writeSuccessResponseXML(w, encodeResponse(OA)) + + sendEvent(eventArgs{ + EventName: event.ObjectAccessedAttributes, + BucketName: bucket, + Object: objInfo, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) +} + +// GetObjectHandler - GET Object +// ---------- +// This implementation of the GET operation retrieves object. To use GET, +// you must have READ access to the object. +func (api objectAPIHandlers) GetObjectHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetObject") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if !globalAPIConfig.shouldGzipObjects() { + w.Header().Set(gzhttp.HeaderNoCompression, "true") + } + + if r.Header.Get(xMinIOExtract) == "true" && strings.Contains(object, archivePattern) { + api.getObjectInArchiveFileHandler(ctx, objectAPI, bucket, object, w, r) + } else { + api.getObjectHandler(ctx, objectAPI, bucket, object, w, r) + } +} + +func (api objectAPIHandlers) headObjectHandler(ctx context.Context, objectAPI ObjectLayer, bucket, object string, w http.ResponseWriter, r *http.Request) { + if crypto.S3.IsRequested(r.Header) || crypto.S3KMS.IsRequested(r.Header) { // If SSE-S3 or SSE-KMS present -> AWS fails with undefined error + writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrBadRequest)) + return + } + + getObjectInfo := objectAPI.GetObjectInfo + + opts, err := getOpts(ctx, r, bucket, object) + if err != nil { + writeErrorResponseHeadersOnly(w, toAPIError(ctx, err)) + return + } + + // Check for auth type to return S3 compatible error. + // type to return the correct error (NoSuchKey vs AccessDenied) + if s3Error := authenticateRequest(ctx, r, policy.GetObjectAction); s3Error != ErrNone { + if getRequestAuthType(r) == authTypeAnonymous { + // As per "Permission" section in + // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html + // If the object you request does not exist, + // the error Amazon S3 returns depends on + // whether you also have the s3:ListBucket + // permission. + // * If you have the s3:ListBucket permission + // on the bucket, Amazon S3 will return an + // HTTP status code 404 ("no such key") + // error. + // * if you don’t have the s3:ListBucket + // permission, Amazon S3 will return an HTTP + // status code 403 ("access denied") error.` + if globalPolicySys.IsAllowed(policy.BucketPolicyArgs{ + Action: policy.ListBucketAction, + BucketName: bucket, + ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials), + IsOwner: false, + }) { + getObjectInfo := objectAPI.GetObjectInfo + + _, err = getObjectInfo(ctx, bucket, object, opts) + if toAPIError(ctx, err).Code == "NoSuchKey" { + s3Error = ErrNoSuchKey + } + } + } + writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error)) + return + } + + // Get request range. + var rs *HTTPRangeSpec + rangeHeader := r.Header.Get(xhttp.Range) + if rangeHeader != "" { + rs, _ = parseRequestRangeSpec(rangeHeader) + } + + if rangeHeader != "" { + // Both 'Range' and 'partNumber' cannot be specified at the same time + if opts.PartNumber > 0 { + writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrInvalidRangePartNumber)) + return + } + + if rs, err = parseRequestRangeSpec(rangeHeader); err != nil { + // Handle only errInvalidRange. Ignore other + // parse error and treat it as regular Get + // request like Amazon S3. + if errors.Is(err, errInvalidRange) { + writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrInvalidRange)) + return + } + } + } + + opts.FastGetObjInfo = true + + objInfo, err := getObjectInfo(ctx, bucket, object, opts) + var proxy proxyResult + if err != nil && !objInfo.DeleteMarker && (isErrObjectNotFound(err) || isErrVersionNotFound(err) || isErrReadQuorum(err)) { + // proxy HEAD to replication target if active-active replication configured on bucket + proxytgts := getProxyTargets(ctx, bucket, object, opts) + if !proxytgts.Empty() { + globalReplicationStats.Load().incProxy(bucket, headObjectAPI, false) + var oi ObjectInfo + oi, proxy = proxyHeadToReplicationTarget(ctx, bucket, object, rs, opts, proxytgts) + if proxy.Proxy { + objInfo = oi + } + if proxy.Err != nil { + globalReplicationStats.Load().incProxy(bucket, headObjectAPI, true) + writeErrorResponseHeadersOnly(w, toAPIError(ctx, proxy.Err)) + return + } + } + } + + if objInfo.UserTags != "" { + // Set this such that authorization policies can be applied on the object tags. + r.Header.Set(xhttp.AmzObjectTagging, objInfo.UserTags) + } + + if s3Error := authorizeRequest(ctx, r, policy.GetObjectAction); s3Error != ErrNone { + writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(s3Error)) + return + } + + if err != nil && !proxy.Proxy { + switch { + case !objInfo.VersionPurgeStatus.Empty(): + w.Header()[xhttp.MinIODeleteReplicationStatus] = []string{string(objInfo.VersionPurgeStatus)} + case !objInfo.ReplicationStatus.Empty() && objInfo.DeleteMarker: + w.Header()[xhttp.MinIODeleteMarkerReplicationStatus] = []string{string(objInfo.ReplicationStatus)} + } + // Versioning enabled quite possibly object is deleted might be delete-marker + // if present set the headers, no idea why AWS S3 sets these headers. + if objInfo.VersionID != "" && objInfo.DeleteMarker { + w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID} + w.Header()[xhttp.AmzDeleteMarker] = []string{strconv.FormatBool(objInfo.DeleteMarker)} + } + + QueueReplicationHeal(ctx, bucket, objInfo, 0) + // do an additional verification whether object exists when object is deletemarker and request + // is from replication + if opts.CheckDMReplicationReady { + topts := opts + topts.VersionID = "" + goi, gerr := getObjectInfo(ctx, bucket, object, topts) + if gerr == nil || goi.VersionID != "" { // object layer returned more info because object is deleted + w.Header().Set(xhttp.MinIOTargetReplicationReady, "true") + } + } + + writeErrorResponseHeadersOnly(w, toAPIError(ctx, err)) + return + } + + if !proxy.Proxy { // apply lifecycle rules only locally not for proxied requests + // Automatically remove the object/version if an expiry lifecycle rule can be applied + if lc, err := globalLifecycleSys.Get(bucket); err == nil { + rcfg, err := globalBucketObjectLockSys.Get(bucket) + if err != nil { + writeErrorResponseHeadersOnly(w, toAPIError(ctx, err)) + return + } + replcfg, err := getReplicationConfig(ctx, bucket) + if err != nil { + writeErrorResponseHeadersOnly(w, toAPIError(ctx, err)) + return + } + event := evalActionFromLifecycle(ctx, *lc, rcfg, replcfg, objInfo) + if event.Action.Delete() { + // apply whatever the expiry rule is. + applyExpiryRule(event, lcEventSrc_s3HeadObject, objInfo) + if !event.Action.DeleteRestored() { + // If the ILM action is not on restored object return error. + writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey)) + return + } + } + } + QueueReplicationHeal(ctx, bucket, objInfo, 0) + } + + // filter object lock metadata if permission does not permit + getRetPerms := checkRequestAuthType(ctx, r, policy.GetObjectRetentionAction, bucket, object) + legalHoldPerms := checkRequestAuthType(ctx, r, policy.GetObjectLegalHoldAction, bucket, object) + + // filter object lock metadata if permission does not permit + objInfo.UserDefined = objectlock.FilterObjectLockMetadata(objInfo.UserDefined, getRetPerms != ErrNone, legalHoldPerms != ErrNone) + + if _, err = DecryptObjectInfo(&objInfo, r); err != nil { + writeErrorResponseHeadersOnly(w, toAPIError(ctx, err)) + return + } + + // Validate pre-conditions if any. + if checkPreconditions(ctx, w, r, objInfo, opts) { + return + } + + // Set encryption response headers + switch kind, _ := crypto.IsEncrypted(objInfo.UserDefined); kind { + case crypto.S3: + w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES) + case crypto.S3KMS: + w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS) + w.Header().Set(xhttp.AmzServerSideEncryptionKmsID, objInfo.KMSKeyID()) + if kmsCtx, ok := objInfo.UserDefined[crypto.MetaContext]; ok { + w.Header().Set(xhttp.AmzServerSideEncryptionKmsContext, kmsCtx) + } + case crypto.SSEC: + // Validate the SSE-C Key set in the header. + if _, err = crypto.SSEC.UnsealObjectKey(r.Header, objInfo.UserDefined, bucket, object); err != nil { + writeErrorResponseHeadersOnly(w, toAPIError(ctx, err)) + return + } + w.Header().Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerAlgorithm)) + w.Header().Set(xhttp.AmzServerSideEncryptionCustomerKeyMD5, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5)) + } + + if r.Header.Get(xhttp.AmzChecksumMode) == "ENABLED" && rs == nil { + // AWS S3 silently drops checksums on range requests. + cs, _ := objInfo.decryptChecksums(opts.PartNumber, r.Header) + hash.AddChecksumHeader(w, cs) + } + + // Set standard object headers. + if err = setObjectHeaders(ctx, w, objInfo, rs, opts); err != nil { + writeErrorResponseHeadersOnly(w, toAPIError(ctx, err)) + return + } + + // Set Parts Count Header + if opts.PartNumber > 0 && len(objInfo.Parts) > 0 { + setPartsCountHeaders(w, objInfo) + } + + // Set any additional requested response headers. + setHeadGetRespHeaders(w, r.Form) + + // Successful response. + if rs != nil || opts.PartNumber > 0 { + w.WriteHeader(http.StatusPartialContent) + } else { + w.WriteHeader(http.StatusOK) + } + + // Notify object accessed via a HEAD request. + sendEvent(eventArgs{ + EventName: event.ObjectAccessedHead, + BucketName: bucket, + Object: objInfo, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) +} + +// GetObjectAttributesHandles - GET Object +// ----------- +// This operation retrieves metadata and part metadata from an object without returning the object itself. +func (api objectAPIHandlers) GetObjectAttributesHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetObjectAttributes") + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrServerNotInitialized)) + return + } + + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + api.getObjectAttributesHandler(ctx, objectAPI, bucket, object, w, r) +} + +// HeadObjectHandler - HEAD Object +// ----------- +// The HEAD operation retrieves metadata from an object without returning the object itself. +func (api objectAPIHandlers) HeadObjectHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "HeadObject") + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrServerNotInitialized)) + return + } + + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if r.Header.Get(xMinIOExtract) == "true" && strings.Contains(object, archivePattern) { + api.headObjectInArchiveFileHandler(ctx, objectAPI, bucket, object, w, r) + } else { + api.headObjectHandler(ctx, objectAPI, bucket, object, w, r) + } +} + +// Extract metadata relevant for an CopyObject operation based on conditional +// header values specified in X-Amz-Metadata-Directive. +func getCpObjMetadataFromHeader(ctx context.Context, r *http.Request, userMeta map[string]string) (map[string]string, error) { + // Make a copy of the supplied metadata to avoid + // to change the original one. + defaultMeta := make(map[string]string, len(userMeta)) + for k, v := range userMeta { + // skip tier metadata when copying metadata from source object + switch k { + case metaTierName, metaTierStatus, metaTierObjName, metaTierVersionID: + continue + } + defaultMeta[k] = v + } + + // remove SSE Headers from source info + crypto.RemoveSSEHeaders(defaultMeta) + + // Storage class is special, it can be replaced regardless of the + // metadata directive, if set should be preserved and replaced + // to the destination metadata. + sc := r.Header.Get(xhttp.AmzStorageClass) + if sc == "" { + sc = r.Form.Get(xhttp.AmzStorageClass) + } + + // if x-amz-metadata-directive says REPLACE then + // we extract metadata from the input headers. + if isDirectiveReplace(r.Header.Get(xhttp.AmzMetadataDirective)) { + emetadata, err := extractMetadataFromReq(ctx, r) + if err != nil { + return nil, err + } + if sc != "" { + emetadata[xhttp.AmzStorageClass] = sc + } + return emetadata, nil + } + + if sc != "" { + defaultMeta[xhttp.AmzStorageClass] = sc + } + + // if x-amz-metadata-directive says COPY then we + // return the default metadata. + if isDirectiveCopy(r.Header.Get(xhttp.AmzMetadataDirective)) { + return defaultMeta, nil + } + + // Copy is default behavior if not x-amz-metadata-directive is set. + return defaultMeta, nil +} + +// getRemoteInstanceTransport contains a roundtripper for external (not peers) servers +var remoteInstanceTransport atomic.Value + +func setRemoteInstanceTransport(tr http.RoundTripper) { + remoteInstanceTransport.Store(tr) +} + +func getRemoteInstanceTransport() http.RoundTripper { + rt, ok := remoteInstanceTransport.Load().(http.RoundTripper) + if ok { + return rt + } + return nil +} + +// Returns a minio-go Client configured to access remote host described by destDNSRecord +// Applicable only in a federated deployment +var getRemoteInstanceClient = func(r *http.Request, host string) (*miniogo.Core, error) { + cred := getReqAccessCred(r, globalSite.Region()) + // In a federated deployment, all the instances share config files + // and hence expected to have same credentials. + core, err := miniogo.NewCore(host, &miniogo.Options{ + Creds: credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, ""), + Secure: globalIsTLS, + Transport: getRemoteInstanceTransport(), + }) + if err != nil { + return nil, err + } + core.SetAppInfo("minio-federated", ReleaseTag) + return core, nil +} + +// Check if the destination bucket is on a remote site, this code only gets executed +// when federation is enabled, ie when globalDNSConfig is non 'nil'. +// +// This function is similar to isRemoteCallRequired but specifically for COPY object API +// if destination and source are same we do not need to check for destination bucket +// to exist locally. +func isRemoteCopyRequired(ctx context.Context, srcBucket, dstBucket string, objAPI ObjectLayer) bool { + if srcBucket == dstBucket { + return false + } + return isRemoteCallRequired(ctx, dstBucket, objAPI) +} + +// Check if the bucket is on a remote site, this code only gets executed when federation is enabled. +func isRemoteCallRequired(ctx context.Context, bucket string, objAPI ObjectLayer) bool { + if globalDNSConfig == nil { + return false + } + if globalBucketFederation { + _, err := objAPI.GetBucketInfo(ctx, bucket, BucketOptions{}) + return err == toObjectErr(errVolumeNotFound, bucket) + } + return false +} + +// CopyObjectHandler - Copy Object +// ---------- +// This implementation of the PUT operation adds an object to a bucket +// while reading the object from another source. +// Notice: The S3 client can send secret keys in headers for encryption related jobs, +// the handler should ensure to remove these keys before sending them to the object layer. +// Currently these keys are: +// - X-Amz-Server-Side-Encryption-Customer-Key +// - X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key +func (api objectAPIHandlers) CopyObjectHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "CopyObject") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + vars := mux.Vars(r) + dstBucket := vars["bucket"] + dstObject, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectAction, dstBucket, dstObject); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Read escaped copy source path to check for parameters. + cpSrcPath := r.Header.Get(xhttp.AmzCopySource) + var vid string + if u, err := url.Parse(cpSrcPath); err == nil { + vid = strings.TrimSpace(u.Query().Get(xhttp.VersionID)) + // Note that url.Parse does the unescaping + cpSrcPath = u.Path + } + + srcBucket, srcObject := path2BucketObject(cpSrcPath) + // If source object is empty or bucket is empty, reply back invalid copy source. + if srcObject == "" || srcBucket == "" { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopySource), r.URL) + return + } + + // Sanitize the source object name similar to NewMultipart and PutObject API + srcObject = trimLeadingSlash(srcObject) + + if vid != "" && vid != nullVersionID { + _, err := uuid.Parse(vid) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, VersionNotFound{ + Bucket: srcBucket, + Object: srcObject, + VersionID: vid, + }), r.URL) + return + } + } + + if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, srcBucket, srcObject); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Check if metadata directive is valid. + if !isDirectiveValid(r.Header.Get(xhttp.AmzMetadataDirective)) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidMetadataDirective), r.URL) + return + } + + // check if tag directive is valid + if !isDirectiveValid(r.Header.Get(xhttp.AmzTagDirective)) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidTagDirective), r.URL) + return + } + + // Validate storage class metadata if present + dstSc := r.Header.Get(xhttp.AmzStorageClass) + if dstSc != "" && !storageclass.IsValid(dstSc) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidStorageClass), r.URL) + return + } + + // Check if bucket encryption is enabled + sseConfig, _ := globalBucketSSEConfigSys.Get(dstBucket) + sseConfig.Apply(r.Header, sse.ApplyOptions{ + AutoEncrypt: globalAutoEncryption, + }) + + var srcOpts, dstOpts ObjectOptions + srcOpts, err = copySrcOpts(ctx, r, srcBucket, srcObject) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + srcOpts.VersionID = vid + + // convert copy src encryption options for GET calls + getOpts := ObjectOptions{ + VersionID: srcOpts.VersionID, + Versioned: srcOpts.Versioned, + VersionSuspended: srcOpts.VersionSuspended, + ReplicationRequest: r.Header.Get(xhttp.MinIOSourceReplicationRequest) == "true", + } + getSSE := encrypt.SSE(srcOpts.ServerSideEncryption) + if getSSE != srcOpts.ServerSideEncryption { + getOpts.ServerSideEncryption = getSSE + } + + dstOpts, err = copyDstOpts(ctx, r, dstBucket, dstObject, nil) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + cpSrcDstSame := isStringEqual(pathJoin(srcBucket, srcObject), pathJoin(dstBucket, dstObject)) + + getObjectNInfo := objectAPI.GetObjectNInfo + + checkCopyPrecondFn := func(o ObjectInfo) bool { + if _, err := DecryptObjectInfo(&o, r); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return true + } + return checkCopyObjectPreconditions(ctx, w, r, o) + } + getOpts.CheckPrecondFn = checkCopyPrecondFn + if cpSrcDstSame { + getOpts.NoLock = true + } + + var rs *HTTPRangeSpec + gr, err := getObjectNInfo(ctx, srcBucket, srcObject, rs, r.Header, getOpts) + if err != nil { + if isErrPreconditionFailed(err) { + return + } + + // Versioning enabled quite possibly object is deleted might be delete-marker + // if present set the headers, no idea why AWS S3 sets these headers. + if gr != nil && gr.ObjInfo.VersionID != "" && gr.ObjInfo.DeleteMarker { + w.Header()[xhttp.AmzVersionID] = []string{gr.ObjInfo.VersionID} + w.Header()[xhttp.AmzDeleteMarker] = []string{strconv.FormatBool(gr.ObjInfo.DeleteMarker)} + } + + // Update context bucket & object names for correct S3 XML error response + reqInfo := logger.GetReqInfo(ctx) + reqInfo.BucketName = srcBucket + reqInfo.ObjectName = srcObject + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + defer gr.Close() + srcInfo := gr.ObjInfo + + // maximum Upload size for object in a single CopyObject operation. + if isMaxObjectSize(srcInfo.Size) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL) + return + } + + // We have to copy metadata only if source and destination are same. + // this changes for encryption which can be observed below. + if cpSrcDstSame { + srcInfo.metadataOnly = true + } + + var chStorageClass bool + if dstSc != "" && dstSc != srcInfo.StorageClass { + chStorageClass = true + srcInfo.metadataOnly = false + } // no changes in storage-class expected so its a metadataonly operation. + + var reader io.Reader = gr + + // Set the actual size to the compressed/decrypted size if encrypted. + actualSize, err := srcInfo.GetActualSize() + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + length := actualSize + + if !cpSrcDstSame { + if err := enforceBucketQuotaHard(ctx, dstBucket, actualSize); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + + var compressMetadata map[string]string + // No need to compress for remote etcd calls + // Pass the decompressed stream to such calls. + isDstCompressed := isCompressible(r.Header, dstObject) && + length > minCompressibleSize && + !isRemoteCopyRequired(ctx, srcBucket, dstBucket, objectAPI) + if isDstCompressed { + compressMetadata = make(map[string]string, 2) + // Preserving the compression metadata. + compressMetadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2 + compressMetadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(actualSize, 10) + + reader = etag.NewReader(ctx, reader, nil, nil) + wantEncryption := crypto.Requested(r.Header) + s2c, cb := newS2CompressReader(reader, actualSize, wantEncryption) + dstOpts.IndexCB = cb + defer s2c.Close() + reader = etag.Wrap(s2c, reader) + length = -1 + } else { + delete(srcInfo.UserDefined, ReservedMetadataPrefix+"compression") + delete(srcInfo.UserDefined, ReservedMetadataPrefix+"actual-size") + reader = gr + } + + srcInfo.Reader, err = hash.NewReader(ctx, reader, length, "", "", actualSize) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + pReader := NewPutObjReader(srcInfo.Reader) + + // Handle encryption + encMetadata := make(map[string]string) + // Encryption parameters not applicable for this object. + if _, ok := crypto.IsEncrypted(srcInfo.UserDefined); !ok && crypto.SSECopy.IsRequested(r.Header) { + writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL) + return + } + // Encryption parameters not present for this object. + if crypto.SSEC.IsEncrypted(srcInfo.UserDefined) && !crypto.SSECopy.IsRequested(r.Header) && r.Header.Get(xhttp.MinIOSourceReplicationRequest) != "true" { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidSSECustomerAlgorithm), r.URL) + return + } + + var oldKey, newKey []byte + var newKeyID string + var kmsCtx kms.Context + var objEncKey crypto.ObjectKey + sseCopyKMS := crypto.S3KMS.IsEncrypted(srcInfo.UserDefined) + sseCopyS3 := crypto.S3.IsEncrypted(srcInfo.UserDefined) + sseCopyC := crypto.SSEC.IsEncrypted(srcInfo.UserDefined) && crypto.SSECopy.IsRequested(r.Header) + sseC := crypto.SSEC.IsRequested(r.Header) + sseS3 := crypto.S3.IsRequested(r.Header) + sseKMS := crypto.S3KMS.IsRequested(r.Header) + + isSourceEncrypted := sseCopyC || sseCopyS3 || sseCopyKMS + isTargetEncrypted := sseC || sseS3 || sseKMS + + if sseC { + newKey, err = ParseSSECustomerRequest(r) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + if crypto.S3KMS.IsRequested(r.Header) { + newKeyID, kmsCtx, err = crypto.S3KMS.ParseHTTP(r.Header) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + + // If src == dst and either + // - the object is encrypted using SSE-C and two different SSE-C keys are present + // - the object is encrypted using SSE-S3 and the SSE-S3 header is present + // - the object storage class is not changing + // then execute a key rotation. + if cpSrcDstSame && (sseCopyC && sseC) && !chStorageClass { + oldKey, err = ParseSSECopyCustomerRequest(r.Header, srcInfo.UserDefined) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + for k, v := range srcInfo.UserDefined { + if stringsHasPrefixFold(k, ReservedMetadataPrefixLower) { + encMetadata[k] = v + } + } + + if err = rotateKey(ctx, oldKey, newKeyID, newKey, srcBucket, srcObject, encMetadata, kmsCtx); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Since we are rotating the keys, make sure to update the metadata. + srcInfo.metadataOnly = true + srcInfo.keyRotation = true + } else { + if isSourceEncrypted || isTargetEncrypted { + // We are not only copying just metadata instead + // we are creating a new object at this point, even + // if source and destination are same objects. + if !srcInfo.keyRotation { + srcInfo.metadataOnly = false + } + } + + // Calculate the size of the target object + var targetSize int64 + + switch { + case isDstCompressed: + targetSize = -1 + case !isSourceEncrypted && !isTargetEncrypted: + targetSize, _ = srcInfo.GetActualSize() + case isSourceEncrypted && isTargetEncrypted: + objInfo := ObjectInfo{Size: actualSize} + targetSize = objInfo.EncryptedSize() + case !isSourceEncrypted && isTargetEncrypted: + targetSize = srcInfo.EncryptedSize() + case isSourceEncrypted && !isTargetEncrypted: + targetSize, _ = srcInfo.DecryptedSize() + } + + // Client can request that a different type of checksum is computed server-side for the + // destination object using the x-amz-checksum-algorithm header. + headerChecksumType := hash.NewChecksumHeader(r.Header) + if headerChecksumType.IsSet() { + dstOpts.WantServerSideChecksumType = headerChecksumType.Base() + srcInfo.Reader.AddServerSideChecksumHasher(headerChecksumType) + dstOpts.WantChecksum = nil + } else { + // Check the source object for checksum. + // If Checksum is not encrypted, decryptChecksum will be a no-op and return + // the already unencrypted value. + srcChecksumDecrypted, err := srcInfo.decryptChecksum(r.Header) + if err != nil { + encLogOnceIf(GlobalContext, + fmt.Errorf("Unable to decryptChecksum for object: %s/%s, error: %w", srcBucket, srcObject, err), + "copy-object-decrypt-checksums-"+srcBucket+srcObject) + } + + // The source object has a checksum set, we need the destination to have one too. + if srcChecksumDecrypted != nil { + dstOpts.WantChecksum = hash.ChecksumFromBytes(srcChecksumDecrypted) + + // When an object is being copied from a source that is multipart, the destination will + // no longer be multipart, and thus the checksum becomes full-object instead. Since + // the CopyObject API does not require that the caller send us this final checksum, we need + // to compute it server-side, with the same type as the source object. + if dstOpts.WantChecksum != nil && dstOpts.WantChecksum.Type.IsMultipartComposite() { + dstOpts.WantServerSideChecksumType = dstOpts.WantChecksum.Type.Base() + srcInfo.Reader.AddServerSideChecksumHasher(dstOpts.WantServerSideChecksumType) + dstOpts.WantChecksum = nil + } + } else { + // S3: All copied objects without checksums and specified destination checksum algorithms + // automatically gain a CRC-64NVME checksum algorithm. + dstOpts.WantServerSideChecksumType = hash.ChecksumCRC64NVME + srcInfo.Reader.AddServerSideChecksumHasher(dstOpts.WantServerSideChecksumType) + dstOpts.WantChecksum = nil + } + } + + if isTargetEncrypted { + var encReader io.Reader + kind, _ := crypto.IsRequested(r.Header) + encReader, objEncKey, err = newEncryptReader(ctx, srcInfo.Reader, kind, newKeyID, newKey, dstBucket, dstObject, encMetadata, kmsCtx) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + reader = etag.Wrap(encReader, srcInfo.Reader) + } + + if isSourceEncrypted { + // Remove all source encrypted related metadata to + // avoid copying them in target object. + crypto.RemoveInternalEntries(srcInfo.UserDefined) + } + + // do not try to verify encrypted content + srcInfo.Reader, err = hash.NewReader(ctx, reader, targetSize, "", "", actualSize) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if isTargetEncrypted { + pReader, err = pReader.WithEncryption(srcInfo.Reader, &objEncKey) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if dstOpts.IndexCB != nil { + dstOpts.IndexCB = compressionIndexEncrypter(objEncKey, dstOpts.IndexCB) + } + dstOpts.EncryptFn = metadataEncrypter(objEncKey) + } + } + + srcInfo.PutObjReader = pReader + + srcInfo.UserDefined, err = getCpObjMetadataFromHeader(ctx, r, srcInfo.UserDefined) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + objTags := srcInfo.UserTags + // If x-amz-tagging-directive header is REPLACE, get passed tags. + if isDirectiveReplace(r.Header.Get(xhttp.AmzTagDirective)) { + objTags = r.Header.Get(xhttp.AmzObjectTagging) + if _, err := tags.ParseObjectTags(objTags); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + + if objTags != "" { + lastTaggingTimestamp := srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] + if dstOpts.ReplicationRequest { + srcTimestamp := dstOpts.ReplicationSourceTaggingTimestamp + if !srcTimestamp.IsZero() { + ondiskTimestamp, err := time.Parse(time.RFC3339Nano, lastTaggingTimestamp) + // update tagging metadata only if replica timestamp is newer than what's on disk + if err != nil || (err == nil && !ondiskTimestamp.After(srcTimestamp)) { + srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = srcTimestamp.UTC().Format(time.RFC3339Nano) + srcInfo.UserDefined[xhttp.AmzObjectTagging] = objTags + } + } + } else { + srcInfo.UserDefined[xhttp.AmzObjectTagging] = objTags + srcInfo.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano) + } + } + + srcInfo.UserDefined = filterReplicationStatusMetadata(srcInfo.UserDefined) + srcInfo.UserDefined = objectlock.FilterObjectLockMetadata(srcInfo.UserDefined, true, true) + retPerms := isPutActionAllowed(ctx, getRequestAuthType(r), dstBucket, dstObject, r, policy.PutObjectRetentionAction) + holdPerms := isPutActionAllowed(ctx, getRequestAuthType(r), dstBucket, dstObject, r, policy.PutObjectLegalHoldAction) + getObjectInfo := objectAPI.GetObjectInfo + + // apply default bucket configuration/governance headers for dest side. + retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, dstBucket, dstObject, getObjectInfo, retPerms, holdPerms) + if s3Err == ErrNone && retentionMode.Valid() { + lastretentionTimestamp := srcInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] + if dstOpts.ReplicationRequest { + srcTimestamp := dstOpts.ReplicationSourceRetentionTimestamp + if !srcTimestamp.IsZero() { + ondiskTimestamp, err := time.Parse(time.RFC3339Nano, lastretentionTimestamp) + // update retention metadata only if replica timestamp is newer than what's on disk + if err != nil || (err == nil && ondiskTimestamp.Before(srcTimestamp)) { + srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode) + srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC()) + srcInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = srcTimestamp.UTC().Format(time.RFC3339Nano) + } + } + } else { + srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode) + srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC()) + srcInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = UTCNow().Format(time.RFC3339Nano) + } + } + + if s3Err == ErrNone && legalHold.Status.Valid() { + lastLegalHoldTimestamp := srcInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] + if dstOpts.ReplicationRequest { + srcTimestamp := dstOpts.ReplicationSourceLegalholdTimestamp + if !srcTimestamp.IsZero() { + ondiskTimestamp, err := time.Parse(time.RFC3339Nano, lastLegalHoldTimestamp) + // update legalhold metadata only if replica timestamp is newer than what's on disk + if err != nil || (err == nil && ondiskTimestamp.Before(srcTimestamp)) { + srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status) + srcInfo.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = srcTimestamp.Format(time.RFC3339Nano) + } + } + } else { + srcInfo.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status) + } + } + if s3Err != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + if rs := r.Header.Get(xhttp.AmzBucketReplicationStatus); rs != "" { + srcInfo.UserDefined[ReservedMetadataPrefixLower+ReplicaStatus] = replication.Replica.String() + srcInfo.UserDefined[ReservedMetadataPrefixLower+ReplicaTimestamp] = UTCNow().Format(time.RFC3339Nano) + srcInfo.UserDefined[xhttp.AmzBucketReplicationStatus] = rs + } + + op := replication.ObjectReplicationType + if srcInfo.metadataOnly { + op = replication.MetadataReplicationType + } + if dsc := mustReplicate(ctx, dstBucket, dstObject, srcInfo.getMustReplicateOptions(op, dstOpts)); dsc.ReplicateAny() { + srcInfo.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus() + srcInfo.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano) + } + // Store the preserved compression metadata. + maps.Copy(srcInfo.UserDefined, compressMetadata) + + // We need to preserve the encryption headers set in EncryptRequest, + // so we do not want to override them, copy them instead. + maps.Copy(srcInfo.UserDefined, encMetadata) + + // Ensure that metadata does not contain sensitive information + crypto.RemoveSensitiveEntries(srcInfo.UserDefined) + + // If we see legacy source, metadataOnly we have to overwrite the content. + if srcInfo.Legacy { + srcInfo.metadataOnly = false + } + + // Check if x-amz-metadata-directive or x-amz-tagging-directive was not set to REPLACE and source, + // destination are same objects. Apply this restriction also when + // metadataOnly is true indicating that we are not overwriting the object. + // if encryption is enabled we do not need explicit "REPLACE" metadata to + // be enabled as well - this is to allow for key-rotation. + if !isDirectiveReplace(r.Header.Get(xhttp.AmzMetadataDirective)) && !isDirectiveReplace(r.Header.Get(xhttp.AmzTagDirective)) && + srcInfo.metadataOnly && srcOpts.VersionID == "" && + !crypto.Requested(r.Header) && + !crypto.IsSourceEncrypted(srcInfo.UserDefined) { + // If x-amz-metadata-directive is not set to REPLACE then we need + // to error out if source and destination are same. + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopyDest), r.URL) + return + } + + // After we've checked for an invalid copy (above), if a server-side checksum type + // is requested, we need to read the source to recompute the checksum. + if dstOpts.WantServerSideChecksumType.IsSet() { + srcInfo.metadataOnly = false + } + + // Federation only. + remoteCallRequired := isRemoteCopyRequired(ctx, srcBucket, dstBucket, objectAPI) + + var objInfo ObjectInfo + var os *objSweeper + if remoteCallRequired { + var dstRecords []dns.SrvRecord + dstRecords, err = globalDNSConfig.Get(dstBucket) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Send PutObject request to appropriate instance (in federated deployment) + core, rerr := getRemoteInstanceClient(r, getHostFromSrv(dstRecords)) + if rerr != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, rerr), r.URL) + return + } + tag, err := tags.ParseObjectTags(objTags) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + // Remove the metadata for remote calls. + delete(srcInfo.UserDefined, ReservedMetadataPrefix+"compression") + delete(srcInfo.UserDefined, ReservedMetadataPrefix+"actual-size") + opts := miniogo.PutObjectOptions{ + UserMetadata: srcInfo.UserDefined, + ServerSideEncryption: dstOpts.ServerSideEncryption, + UserTags: tag.ToMap(), + } + remoteObjInfo, rerr := core.PutObject(ctx, dstBucket, dstObject, srcInfo.Reader, + srcInfo.Size, "", "", opts) + if rerr != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, rerr), r.URL) + return + } + objInfo.UserDefined = cloneMSS(opts.UserMetadata) + objInfo.ETag = remoteObjInfo.ETag + objInfo.ModTime = remoteObjInfo.LastModified + } else { + os = newObjSweeper(dstBucket, dstObject).WithVersioning(dstOpts.Versioned, dstOpts.VersionSuspended) + // Get appropriate object info to identify the remote object to delete + if !srcInfo.metadataOnly { + goiOpts := os.GetOpts() + if !globalTierConfigMgr.Empty() { + if goi, gerr := getObjectInfo(ctx, dstBucket, dstObject, goiOpts); gerr == nil { + os.SetTransitionState(goi.TransitionedObject) + } + } + } + + copyObjectFn := objectAPI.CopyObject + + // Copy source object to destination, if source and destination + // object is same then only metadata is updated. + objInfo, err = copyObjectFn(ctx, srcBucket, srcObject, dstBucket, dstObject, srcInfo, srcOpts, dstOpts) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + + origETag := objInfo.ETag + objInfo.ETag = getDecryptedETag(r.Header, objInfo, false) + response := generateCopyObjectResponse(objInfo.ETag, objInfo.ModTime) + encodedSuccessResponse := encodeResponse(response) + + if dsc := mustReplicate(ctx, dstBucket, dstObject, objInfo.getMustReplicateOptions(replication.ObjectReplicationType, dstOpts)); dsc.ReplicateAny() { + scheduleReplication(ctx, objInfo, objectAPI, dsc, replication.ObjectReplicationType) + } + + setPutObjHeaders(w, objInfo, false, r.Header) + // We must not use the http.Header().Set method here because some (broken) + // clients expect the x-amz-copy-source-version-id header key to be literally + // "x-amz-copy-source-version-id"- not in canonicalized form, preserve it. + if srcOpts.VersionID != "" { + w.Header()[strings.ToLower(xhttp.AmzCopySourceVersionID)] = []string{srcOpts.VersionID} + } + + // Write success response. + writeSuccessResponseXML(w, encodedSuccessResponse) + + // Notify object created event. + sendEvent(eventArgs{ + EventName: event.ObjectCreatedCopy, + BucketName: dstBucket, + Object: objInfo, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) + + if !remoteCallRequired && !globalTierConfigMgr.Empty() { + // Schedule object for immediate transition if eligible. + objInfo.ETag = origETag + enqueueTransitionImmediate(objInfo, lcEventSrc_s3CopyObject) + // Remove the transitioned object whose object version is being overwritten. + os.Sweep() + } +} + +// PutObjectHandler - PUT Object +// ---------- +// This implementation of the PUT operation adds an object to a bucket. +// Notice: The S3 client can send secret keys in headers for encryption related jobs, +// the handler should ensure to remove these keys before sending them to the object layer. +// Currently these keys are: +// - X-Amz-Server-Side-Encryption-Customer-Key +// - X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key +func (api objectAPIHandlers) PutObjectHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PutObject") + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // X-Amz-Copy-Source shouldn't be set for this call. + if _, ok := r.Header[xhttp.AmzCopySource]; ok { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopySource), r.URL) + return + } + + // Validate storage class metadata if present + if sc := r.Header.Get(xhttp.AmzStorageClass); sc != "" { + if !storageclass.IsValid(sc) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidStorageClass), r.URL) + return + } + } + + clientETag, err := etag.FromContentMD5(r.Header) + if err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDigest), r.URL) + return + } + + // if Content-Length is unknown/missing, deny the request + size := r.ContentLength + rAuthType := getRequestAuthType(r) + switch rAuthType { + // Check signature types that must have content length + case authTypeStreamingSigned, authTypeStreamingSignedTrailer, authTypeStreamingUnsignedTrailer: + if sizeStr, ok := r.Header[xhttp.AmzDecodedContentLength]; ok { + if sizeStr[0] == "" { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL) + return + } + size, err = strconv.ParseInt(sizeStr[0], 10, 64) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + } + if size == -1 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL) + return + } + + // maximum Upload size for objects in a single operation + if isMaxObjectSize(size) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL) + return + } + + metadata, err := extractMetadataFromReq(ctx, r) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if objTags := r.Header.Get(xhttp.AmzObjectTagging); objTags != "" { + if _, err := tags.ParseObjectTags(objTags); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + metadata[xhttp.AmzObjectTagging] = objTags + } + + var ( + md5hex = clientETag.String() + sha256hex = "" + rd io.Reader = r.Body + s3Err APIErrorCode + putObject = objectAPI.PutObject + ) + + // Check if put is allowed + if s3Err = isPutActionAllowed(ctx, rAuthType, bucket, object, r, policy.PutObjectAction); s3Err != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + + switch rAuthType { + case authTypeStreamingSigned, authTypeStreamingSignedTrailer: + // Initialize stream signature verifier. + rd, s3Err = newSignV4ChunkedReader(r, rAuthType == authTypeStreamingSignedTrailer) + if s3Err != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + case authTypeStreamingUnsignedTrailer: + // Initialize stream chunked reader with optional trailers. + rd, s3Err = newUnsignedV4ChunkedReader(r, true, r.Header.Get(xhttp.Authorization) != "") + if s3Err != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + case authTypeSignedV2, authTypePresignedV2: + s3Err = isReqAuthenticatedV2(r) + if s3Err != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + + case authTypePresigned, authTypeSigned: + if s3Err = reqSignatureV4Verify(r, globalSite.Region(), serviceS3); s3Err != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + if !skipContentSha256Cksum(r) { + sha256hex = getContentSha256Cksum(r, serviceS3) + } + } + + if _, ok := r.Header[xhttp.MinIOSourceReplicationCheck]; ok { + // requests to just validate replication settings and permissions are not allowed to write data + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationPermissionCheckError), r.URL) + return + } + + if err := enforceBucketQuotaHard(ctx, bucket, size); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String() { + if s3Err = isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.ReplicateObjectAction); s3Err != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + metadata[ReservedMetadataPrefixLower+ReplicaStatus] = replication.Replica.String() + metadata[ReservedMetadataPrefixLower+ReplicaTimestamp] = UTCNow().Format(time.RFC3339Nano) + defer globalReplicationStats.Load().UpdateReplicaStat(bucket, size) + } + + // Check if bucket encryption is enabled + sseConfig, _ := globalBucketSSEConfigSys.Get(bucket) + sseConfig.Apply(r.Header, sse.ApplyOptions{ + AutoEncrypt: globalAutoEncryption, + }) + + var reader io.Reader + reader = rd + + var opts ObjectOptions + opts, err = putOptsFromReq(ctx, r, bucket, object, metadata) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + actualSize := size + var idxCb func() []byte + if isCompressible(r.Header, object) && size > minCompressibleSize { + // Storing the compression metadata. + metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2 + metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(size, 10) + + actualReader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, actualSize) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if err = actualReader.AddChecksum(r, false); err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL) + return + } + opts.WantChecksum = actualReader.Checksum() + + // Set compression metrics. + var s2c io.ReadCloser + wantEncryption := crypto.Requested(r.Header) + s2c, idxCb = newS2CompressReader(actualReader, actualSize, wantEncryption) + defer s2c.Close() + + reader = etag.Wrap(s2c, actualReader) + size = -1 // Since compressed size is un-predictable. + md5hex = "" // Do not try to verify the content. + sha256hex = "" + } + + var forceMD5 []byte + // Optimization: If SSE-KMS and SSE-C did not request Content-Md5. Use uuid as etag. Optionally enable this also + // for server that is started with `--no-compat`. + if !etag.ContentMD5Requested(r.Header) && (crypto.S3KMS.IsRequested(r.Header) || crypto.SSEC.IsRequested(r.Header) || !globalServerCtxt.StrictS3Compat) { + forceMD5 = mustGetUUIDBytes() + } + hashReader, err := hash.NewReaderWithOpts(ctx, reader, hash.Options{ + Size: size, + MD5Hex: md5hex, + SHA256Hex: sha256hex, + ActualSize: actualSize, + DisableMD5: false, + ForceMD5: forceMD5, + }) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if size >= 0 { + if err := hashReader.AddChecksum(r, false); err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL) + return + } + opts.WantChecksum = hashReader.Checksum() + } + + rawReader := hashReader + pReader := NewPutObjReader(rawReader) + opts.IndexCB = idxCb + + if r.Header.Get(xhttp.IfMatch) != "" { + opts.HasIfMatch = true + } + if opts.PreserveETag != "" || + r.Header.Get(xhttp.IfMatch) != "" || + r.Header.Get(xhttp.IfNoneMatch) != "" { + opts.CheckPrecondFn = func(oi ObjectInfo) bool { + if _, err := DecryptObjectInfo(&oi, r); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return true + } + return checkPreconditionsPUT(ctx, w, r, oi, opts) + } + } + + retPerms := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.PutObjectRetentionAction) + holdPerms := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.PutObjectLegalHoldAction) + + getObjectInfo := objectAPI.GetObjectInfo + + retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms) + if s3Err == ErrNone && retentionMode.Valid() { + metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode) + metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC()) + } + if s3Err == ErrNone && legalHold.Status.Valid() { + metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status) + } + if s3Err != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + if dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(metadata, "", "", replication.ObjectReplicationType, opts)); dsc.ReplicateAny() { + metadata[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano) + metadata[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus() + } + var objectEncryptionKey crypto.ObjectKey + if crypto.Requested(r.Header) { + if crypto.SSECopy.IsRequested(r.Header) { + writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL) + return + } + + if crypto.SSEC.IsRequested(r.Header) && crypto.S3.IsRequested(r.Header) { + writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionMethod), r.URL) + return + } + + if crypto.SSEC.IsRequested(r.Header) && crypto.S3KMS.IsRequested(r.Header) { + writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionMethod), r.URL) + return + } + + reader, objectEncryptionKey, err = EncryptRequest(hashReader, r, bucket, object, metadata) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + wantSize := int64(-1) + if size >= 0 { + info := ObjectInfo{Size: size} + wantSize = info.EncryptedSize() + } + + // do not try to verify encrypted content + hashReader, err = hash.NewReader(ctx, etag.Wrap(reader, hashReader), wantSize, "", "", actualSize) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if opts.IndexCB != nil { + opts.IndexCB = compressionIndexEncrypter(objectEncryptionKey, opts.IndexCB) + } + opts.EncryptFn = metadataEncrypter(objectEncryptionKey) + } + + // Ensure that metadata does not contain sensitive information + crypto.RemoveSensitiveEntries(metadata) + + os := newObjSweeper(bucket, object).WithVersioning(opts.Versioned, opts.VersionSuspended) + if !globalTierConfigMgr.Empty() { + // Get appropriate object info to identify the remote object to delete + goiOpts := os.GetOpts() + if goi, gerr := getObjectInfo(ctx, bucket, object, goiOpts); gerr == nil { + os.SetTransitionState(goi.TransitionedObject) + } + } + + // Create the object.. + objInfo, err := putObject(ctx, bucket, object, pReader, opts) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if r.Header.Get(xMinIOExtract) == "true" && HasSuffix(object, archiveExt) { + opts := ObjectOptions{VersionID: objInfo.VersionID, MTime: objInfo.ModTime} + if _, err := updateObjectMetadataWithZipInfo(ctx, objectAPI, bucket, object, opts); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + + origETag := objInfo.ETag + if kind, encrypted := crypto.IsEncrypted(objInfo.UserDefined); encrypted { + switch kind { + case crypto.S3: + w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES) + objInfo.ETag, _ = DecryptETag(objectEncryptionKey, ObjectInfo{ETag: objInfo.ETag}) + case crypto.S3KMS: + w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS) + w.Header().Set(xhttp.AmzServerSideEncryptionKmsID, objInfo.KMSKeyID()) + if kmsCtx, ok := objInfo.UserDefined[crypto.MetaContext]; ok { + w.Header().Set(xhttp.AmzServerSideEncryptionKmsContext, kmsCtx) + } + if len(objInfo.ETag) >= 32 && strings.Count(objInfo.ETag, "-") != 1 { + objInfo.ETag = objInfo.ETag[len(objInfo.ETag)-32:] + } + case crypto.SSEC: + w.Header().Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerAlgorithm)) + w.Header().Set(xhttp.AmzServerSideEncryptionCustomerKeyMD5, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5)) + + if len(objInfo.ETag) >= 32 && strings.Count(objInfo.ETag, "-") != 1 { + objInfo.ETag = objInfo.ETag[len(objInfo.ETag)-32:] + } + } + } + if dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(metadata, "", "", replication.ObjectReplicationType, opts)); dsc.ReplicateAny() { + scheduleReplication(ctx, objInfo, objectAPI, dsc, replication.ObjectReplicationType) + } + + setPutObjHeaders(w, objInfo, false, r.Header) + + // Notify object created event. + evt := eventArgs{ + EventName: event.ObjectCreatedPut, + BucketName: bucket, + Object: objInfo, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + } + sendEvent(evt) + if objInfo.NumVersions > int(scannerExcessObjectVersions.Load()) { + evt.EventName = event.ObjectManyVersions + sendEvent(evt) + + auditLogInternal(context.Background(), AuditLogOptions{ + Event: "scanner:manyversions", + APIName: "PutObject", + Bucket: objInfo.Bucket, + Object: objInfo.Name, + VersionID: objInfo.VersionID, + Status: http.StatusText(http.StatusOK), + }) + } + + // Do not send checksums in events to avoid leaks. + hash.TransferChecksumHeader(w, r) + writeSuccessResponseHeadersOnly(w) + + // Remove the transitioned object whose object version is being overwritten. + if !globalTierConfigMgr.Empty() { + // Schedule object for immediate transition if eligible. + objInfo.ETag = origETag + enqueueTransitionImmediate(objInfo, lcEventSrc_s3PutObject) + os.Sweep() + } +} + +// PutObjectExtractHandler - PUT Object extract is an extended API +// based off from AWS Snowball feature to auto extract compressed +// stream will be extracted in the same directory it is stored in +// and the folder structures will be built out accordingly. +func (api objectAPIHandlers) PutObjectExtractHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PutObjectExtract") + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if crypto.S3KMS.IsRequested(r.Header) { // SSE-KMS is not supported + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // X-Amz-Copy-Source shouldn't be set for this call. + if _, ok := r.Header[xhttp.AmzCopySource]; ok { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopySource), r.URL) + return + } + + // Validate storage class metadata if present + sc := r.Header.Get(xhttp.AmzStorageClass) + if sc != "" { + if !storageclass.IsValid(sc) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidStorageClass), r.URL) + return + } + } + + clientETag, err := etag.FromContentMD5(r.Header) + if err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDigest), r.URL) + return + } + + // if Content-Length is unknown/missing, deny the request + size := r.ContentLength + rAuthType := getRequestAuthType(r) + if rAuthType == authTypeStreamingSigned || rAuthType == authTypeStreamingSignedTrailer { + if sizeStr, ok := r.Header[xhttp.AmzDecodedContentLength]; ok { + if sizeStr[0] == "" { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL) + return + } + size, err = strconv.ParseInt(sizeStr[0], 10, 64) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + } + + if size == -1 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL) + return + } + + // maximum Upload size for objects in a single operation + if isMaxObjectSize(size) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL) + return + } + + var ( + md5hex = clientETag.String() + sha256hex = "" + reader io.Reader = r.Body + s3Err APIErrorCode + putObject = objectAPI.PutObject + ) + + var opts untarOptions + opts.ignoreDirs = strings.EqualFold(r.Header.Get(xhttp.MinIOSnowballIgnoreDirs), "true") + opts.ignoreErrs = strings.EqualFold(r.Header.Get(xhttp.MinIOSnowballIgnoreErrors), "true") + opts.prefixAll = r.Header.Get(xhttp.MinIOSnowballPrefix) + if opts.prefixAll != "" { + opts.prefixAll = trimLeadingSlash(pathJoin(opts.prefixAll, slashSeparator)) + } + // Check if put is allow for specified prefix. + if s3Err = isPutActionAllowed(ctx, rAuthType, bucket, opts.prefixAll, r, policy.PutObjectAction); s3Err != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + + switch rAuthType { + case authTypeStreamingSigned, authTypeStreamingSignedTrailer: + // Initialize stream signature verifier. + reader, s3Err = newSignV4ChunkedReader(r, rAuthType == authTypeStreamingSignedTrailer) + if s3Err != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + case authTypeSignedV2, authTypePresignedV2: + s3Err = isReqAuthenticatedV2(r) + if s3Err != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + + case authTypePresigned, authTypeSigned: + if s3Err = reqSignatureV4Verify(r, globalSite.Region(), serviceS3); s3Err != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + if !skipContentSha256Cksum(r) { + sha256hex = getContentSha256Cksum(r, serviceS3) + } + } + + hreader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, size) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if err = hreader.AddChecksum(r, false); err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL) + return + } + + if err := enforceBucketQuotaHard(ctx, bucket, size); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Check if bucket encryption is enabled + sseConfig, _ := globalBucketSSEConfigSys.Get(bucket) + sseConfig.Apply(r.Header, sse.ApplyOptions{ + AutoEncrypt: globalAutoEncryption, + }) + + retPerms := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.PutObjectRetentionAction) + holdPerms := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.PutObjectLegalHoldAction) + + getObjectInfo := objectAPI.GetObjectInfo + + // These are static for all objects extracted. + reqParams := extractReqParams(r) + respElements := map[string]string{ + "requestId": w.Header().Get(xhttp.AmzRequestID), + "nodeId": w.Header().Get(xhttp.AmzRequestHostID), + } + if sc == "" { + sc = storageclass.STANDARD + } + + putObjectTar := func(reader io.Reader, info os.FileInfo, object string) error { + size := info.Size() + if s3Err = isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.PutObjectAction); s3Err != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return errors.New(errorCodes.ToAPIErr(s3Err).Code) + } + metadata := map[string]string{ + xhttp.AmzStorageClass: sc, // save same storage-class as incoming stream. + } + + actualSize := size + var idxCb func() []byte + if isCompressible(r.Header, object) && size > minCompressibleSize { + // Storing the compression metadata. + metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2 + metadata[ReservedMetadataPrefix+"actual-size"] = strconv.FormatInt(size, 10) + + actualReader, err := hash.NewReader(ctx, reader, size, "", "", actualSize) + if err != nil { + return err + } + + // Set compression metrics. + wantEncryption := crypto.Requested(r.Header) + s2c, cb := newS2CompressReader(actualReader, actualSize, wantEncryption) + defer s2c.Close() + idxCb = cb + reader = etag.Wrap(s2c, actualReader) + size = -1 // Since compressed size is un-predictable. + } + + hashReader, err := hash.NewReader(ctx, reader, size, "", "", actualSize) + if err != nil { + return err + } + + rawReader := hashReader + pReader := NewPutObjReader(rawReader) + + if r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String() { + if s3Err = isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.ReplicateObjectAction); s3Err != ErrNone { + return errors.New(errorCodes.ToAPIErr(s3Err).Code) + } + metadata[ReservedMetadataPrefixLower+ReplicaStatus] = replication.Replica.String() + metadata[ReservedMetadataPrefixLower+ReplicaTimestamp] = UTCNow().Format(time.RFC3339Nano) + } + + var ( + versionID string + hdrs http.Header + ) + + if tarHdrs, ok := info.Sys().(*tar.Header); ok && len(tarHdrs.PAXRecords) > 0 { + versionID = tarHdrs.PAXRecords["minio.versionId"] + hdrs = make(http.Header) + for k, v := range tarHdrs.PAXRecords { + if k == "minio.versionId" { + continue + } + if after, ok0 := strings.CutPrefix(k, "minio.metadata."); ok0 { + k = after + hdrs.Set(k, v) + } + } + m, err := extractMetadata(ctx, textproto.MIMEHeader(hdrs)) + if err != nil { + return err + } + maps.Copy(metadata, m) + } else { + versionID = r.Form.Get(xhttp.VersionID) + hdrs = r.Header + } + + opts, err := putOpts(ctx, bucket, object, versionID, hdrs, metadata) + if err != nil { + return err + } + + opts.MTime = info.ModTime() + if opts.MTime.Unix() <= 0 { + opts.MTime = UTCNow() + } + opts.IndexCB = idxCb + + retentionMode, retentionDate, legalHold, s3err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms) + if s3err == ErrNone && retentionMode.Valid() { + metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode) + metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC()) + } + + if s3err == ErrNone && legalHold.Status.Valid() { + metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status) + } + + if s3err != ErrNone { + s3Err = s3err + return ObjectLocked{} + } + + if dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(metadata, "", "", replication.ObjectReplicationType, opts)); dsc.ReplicateAny() { + metadata[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano) + metadata[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus() + } + + var objectEncryptionKey crypto.ObjectKey + if crypto.Requested(r.Header) { + if crypto.SSECopy.IsRequested(r.Header) { + return errInvalidEncryptionParameters + } + + reader, objectEncryptionKey, err = EncryptRequest(hashReader, r, bucket, object, metadata) + if err != nil { + return err + } + + wantSize := int64(-1) + if size >= 0 { + info := ObjectInfo{Size: size} + wantSize = info.EncryptedSize() + } + + // do not try to verify encrypted content + hashReader, err = hash.NewReader(ctx, etag.Wrap(reader, hashReader), wantSize, "", "", actualSize) + if err != nil { + return err + } + + pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey) + if err != nil { + return err + } + } + if opts.IndexCB != nil { + opts.IndexCB = compressionIndexEncrypter(objectEncryptionKey, opts.IndexCB) + } + + // Ensure that metadata does not contain sensitive information + crypto.RemoveSensitiveEntries(metadata) + + os := newObjSweeper(bucket, object).WithVersioning(opts.Versioned, opts.VersionSuspended) + if !globalTierConfigMgr.Empty() { + // Get appropriate object info to identify the remote object to delete + goiOpts := os.GetOpts() + if goi, gerr := getObjectInfo(ctx, bucket, object, goiOpts); gerr == nil { + os.SetTransitionState(goi.TransitionedObject) + } + } + + // Create the object.. + objInfo, err := putObject(ctx, bucket, object, pReader, opts) + if err != nil { + return err + } + + origETag := objInfo.ETag + objInfo.ETag = getDecryptedETag(r.Header, objInfo, false) + + if dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(metadata, "", "", replication.ObjectReplicationType, opts)); dsc.ReplicateAny() { + scheduleReplication(ctx, objInfo, objectAPI, dsc, replication.ObjectReplicationType) + } + + // Notify object created event. + evt := eventArgs{ + EventName: event.ObjectCreatedPut, + BucketName: bucket, + Object: objInfo, + ReqParams: reqParams, + RespElements: respElements, + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + } + sendEvent(evt) + + // Remove the transitioned object whose object version is being overwritten. + if !globalTierConfigMgr.Empty() { + objInfo.ETag = origETag + // Schedule object for immediate transition if eligible. + enqueueTransitionImmediate(objInfo, lcEventSrc_s3PutObject) + os.Sweep() + } + + return nil + } + + if err = untar(ctx, hreader, putObjectTar, opts); err != nil { + apiErr := errorCodes.ToAPIErr(s3Err) + // If not set, convert or use BadRequest + if s3Err == ErrNone { + apiErr = toAPIError(ctx, err) + if apiErr.Code == "InternalError" { + // Convert generic internal errors to bad requests. + apiErr = APIError{ + Code: "BadRequest", + Description: err.Error(), + HTTPStatusCode: http.StatusBadRequest, + } + } + } + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + + w.Header()[xhttp.ETag] = []string{`"` + hex.EncodeToString(hreader.MD5Current()) + `"`} + hash.TransferChecksumHeader(w, r) + writeSuccessResponseHeadersOnly(w) +} + +// Delete objectAPIHandlers + +// DeleteObjectHandler - delete an object +func (api objectAPIHandlers) DeleteObjectHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "DeleteObject") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.DeleteObjectAction, bucket, object); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + if _, ok := r.Header[xhttp.MinIOSourceReplicationCheck]; ok { + // requests to just validate replication settings and permissions are not allowed to delete data + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrReplicationPermissionCheckError), r.URL) + return + } + + replica := r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String() + if replica { + if s3Error := checkRequestAuthType(ctx, r, policy.ReplicateDeleteAction, bucket, object); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + } + + if globalDNSConfig != nil { + _, err := globalDNSConfig.Get(bucket) + if err != nil && err != dns.ErrNotImplemented { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + + opts, err := delOpts(ctx, r, bucket, object) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + rcfg, _ := globalBucketObjectLockSys.Get(bucket) + if rcfg.LockEnabled && opts.DeletePrefix { + apiErr := toAPIError(ctx, errInvalidArgument) + apiErr.Description = "force-delete is forbidden on Object Locking enabled buckets" + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + + os := newObjSweeper(bucket, object).WithVersion(opts.VersionID).WithVersioning(opts.Versioned, opts.VersionSuspended) + + opts.SetEvalMetadataFn(func(oi *ObjectInfo, gerr error) (dsc ReplicateDecision, err error) { + if replica { // no need to check replication on receiver + return dsc, nil + } + dsc = checkReplicateDelete(ctx, bucket, ObjectToDelete{ + ObjectV: ObjectV{ + ObjectName: object, + VersionID: opts.VersionID, + }, + }, *oi, opts, gerr) + // Mutations of objects on versioning suspended buckets + // affect its null version. Through opts below we select + // the null version's remote object to delete if + // transitioned. + if gerr == nil { + os.SetTransitionState(oi.TransitionedObject) + } + return dsc, nil + }) + + vID := opts.VersionID + if replica { + opts.SetReplicaStatus(replication.Replica) + if opts.VersionPurgeStatus().Empty() { + // opts.VersionID holds delete marker version ID to replicate and not yet present on disk + vID = "" + } + } + opts.SetEvalRetentionBypassFn(func(goi ObjectInfo, gerr error) (err error) { + err = nil + if vID != "" { + err := enforceRetentionBypassForDelete(ctx, r, bucket, ObjectToDelete{ + ObjectV: ObjectV{ + ObjectName: object, + VersionID: vID, + }, + }, goi, gerr) + if err != nil && !isErrObjectNotFound(err) { + return err + } + } + return err + }) + + deleteObject := objectAPI.DeleteObject + + // http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectDELETE.html + objInfo, err := deleteObject(ctx, bucket, object, opts) + if err != nil { + if _, ok := err.(BucketNotFound); ok { + // When bucket doesn't exist specially handle it. + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if isErrObjectNotFound(err) || isErrVersionNotFound(err) { + // Send an event when the object is not found + objInfo.Name = object + objInfo.VersionID = opts.VersionID + sendEvent(eventArgs{ + EventName: event.ObjectRemovedNoOP, + BucketName: bucket, + Object: objInfo, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) + writeSuccessNoContent(w) + return + } + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if objInfo.Name == "" { + writeSuccessNoContent(w) + return + } + + setPutObjHeaders(w, objInfo, true, r.Header) + writeSuccessNoContent(w) + + eventName := event.ObjectRemovedDelete + if objInfo.DeleteMarker { + eventName = event.ObjectRemovedDeleteMarkerCreated + } + + // Notify object deleted event. + sendEvent(eventArgs{ + EventName: eventName, + BucketName: bucket, + Object: objInfo, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) + + if objInfo.ReplicationStatus == replication.Pending || objInfo.VersionPurgeStatus == replication.VersionPurgePending { + dmVersionID := "" + versionID := "" + if objInfo.DeleteMarker { + dmVersionID = objInfo.VersionID + } else { + versionID = objInfo.VersionID + } + dobj := DeletedObjectReplicationInfo{ + DeletedObject: DeletedObject{ + ObjectName: object, + VersionID: versionID, + DeleteMarkerVersionID: dmVersionID, + DeleteMarkerMTime: DeleteMarkerMTime{objInfo.ModTime}, + DeleteMarker: objInfo.DeleteMarker, + ReplicationState: objInfo.ReplicationState(), + }, + Bucket: bucket, + EventType: ReplicateIncomingDelete, + } + scheduleReplicationDelete(ctx, dobj, objectAPI) + } + + // Remove the transitioned object whose object version is being overwritten. + if !globalTierConfigMgr.Empty() { + os.Sweep() + } +} + +// PutObjectLegalHoldHandler - set legal hold configuration to object, +func (api objectAPIHandlers) PutObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PutObjectLegalHold") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + // Check permissions to perform this legal hold operation + if s3Err := checkRequestAuthType(ctx, r, policy.PutObjectLegalHoldAction, bucket, object); s3Err != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + + if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if !validateLengthAndChecksum(r) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL) + return + } + + if rcfg, _ := globalBucketObjectLockSys.Get(bucket); !rcfg.LockEnabled { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketObjectLockConfiguration), r.URL) + return + } + + legalHold, err := objectlock.ParseObjectLegalHold(r.Body) + if err != nil { + apiErr := errorCodes.ToAPIErr(ErrMalformedXML) + apiErr.Description = err.Error() + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + + opts, err := getOpts(ctx, r, bucket, object) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + popts := ObjectOptions{ + MTime: opts.MTime, + VersionID: opts.VersionID, + EvalMetadataFn: func(oi *ObjectInfo, gerr error) (ReplicateDecision, error) { + oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = strings.ToUpper(string(legalHold.Status)) + oi.UserDefined[ReservedMetadataPrefixLower+ObjectLockLegalHoldTimestamp] = UTCNow().Format(time.RFC3339Nano) + + dsc := mustReplicate(ctx, bucket, object, oi.getMustReplicateOptions(replication.MetadataReplicationType, opts)) + if dsc.ReplicateAny() { + oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano) + oi.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus() + } + return dsc, nil + }, + } + + objInfo, err := objectAPI.PutObjectMetadata(ctx, bucket, object, popts) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + dsc := mustReplicate(ctx, bucket, object, objInfo.getMustReplicateOptions(replication.MetadataReplicationType, opts)) + if dsc.ReplicateAny() { + scheduleReplication(ctx, objInfo, objectAPI, dsc, replication.MetadataReplicationType) + } + + writeSuccessResponseHeadersOnly(w) + + // Notify object event. + sendEvent(eventArgs{ + EventName: event.ObjectCreatedPutLegalHold, + BucketName: bucket, + Object: objInfo, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) +} + +// GetObjectLegalHoldHandler - get legal hold configuration to object, +func (api objectAPIHandlers) GetObjectLegalHoldHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetObjectLegalHold") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectLegalHoldAction, bucket, object); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + getObjectInfo := objectAPI.GetObjectInfo + + if rcfg, _ := globalBucketObjectLockSys.Get(bucket); !rcfg.LockEnabled { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketObjectLockConfiguration), r.URL) + return + } + + opts, err := getOpts(ctx, r, bucket, object) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + objInfo, err := getObjectInfo(ctx, bucket, object, opts) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + legalHold := objectlock.GetObjectLegalHoldMeta(objInfo.UserDefined) + if legalHold.IsEmpty() { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNoSuchObjectLockConfiguration), r.URL) + return + } + + writeSuccessResponseXML(w, encodeResponse(legalHold)) + // Notify object legal hold accessed via a GET request. + sendEvent(eventArgs{ + EventName: event.ObjectAccessedGetLegalHold, + BucketName: bucket, + Object: objInfo, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) +} + +// PutObjectRetentionHandler - set object hold configuration to object, +func (api objectAPIHandlers) PutObjectRetentionHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PutObjectRetention") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + // Check permissions to perform this object retention operation + if s3Error := authenticateRequest(ctx, r, policy.PutObjectRetentionAction); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + if _, err := objectAPI.GetBucketInfo(ctx, bucket, BucketOptions{}); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if !validateLengthAndChecksum(r) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentMD5), r.URL) + return + } + + if rcfg, _ := globalBucketObjectLockSys.Get(bucket); !rcfg.LockEnabled { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketObjectLockConfiguration), r.URL) + return + } + + objRetention, err := objectlock.ParseObjectRetention(r.Body) + if err != nil { + apiErr := errorCodes.ToAPIErr(ErrMalformedXML) + apiErr.Description = err.Error() + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + + reqInfo := logger.GetReqInfo(ctx) + reqInfo.SetTags("retention", objRetention.String()) + + opts, err := getOpts(ctx, r, bucket, object) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + popts := ObjectOptions{ + MTime: opts.MTime, + VersionID: opts.VersionID, + EvalMetadataFn: func(oi *ObjectInfo, gerr error) (dsc ReplicateDecision, err error) { + if err := enforceRetentionBypassForPut(ctx, r, *oi, objRetention, reqInfo.Cred, reqInfo.Owner); err != nil { + return dsc, err + } + if objRetention.Mode.Valid() { + oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = string(objRetention.Mode) + oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(objRetention.RetainUntilDate.UTC()) + } else { + oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockMode)] = "" + oi.UserDefined[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = "" + } + oi.UserDefined[ReservedMetadataPrefixLower+ObjectLockRetentionTimestamp] = UTCNow().Format(time.RFC3339Nano) + dsc = mustReplicate(ctx, bucket, object, oi.getMustReplicateOptions(replication.MetadataReplicationType, opts)) + if dsc.ReplicateAny() { + oi.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano) + oi.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus() + } + return dsc, nil + }, + } + + objInfo, err := objectAPI.PutObjectMetadata(ctx, bucket, object, popts) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + dsc := mustReplicate(ctx, bucket, object, objInfo.getMustReplicateOptions(replication.MetadataReplicationType, opts)) + if dsc.ReplicateAny() { + scheduleReplication(ctx, objInfo, objectAPI, dsc, replication.MetadataReplicationType) + } + + writeSuccessResponseHeadersOnly(w) + + // Notify object event. + sendEvent(eventArgs{ + EventName: event.ObjectCreatedPutRetention, + BucketName: bucket, + Object: objInfo, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) +} + +// GetObjectRetentionHandler - get object retention configuration of object, +func (api objectAPIHandlers) GetObjectRetentionHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetObjectRetention") + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectRetentionAction, bucket, object); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + getObjectInfo := objectAPI.GetObjectInfo + + if rcfg, _ := globalBucketObjectLockSys.Get(bucket); !rcfg.LockEnabled { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidBucketObjectLockConfiguration), r.URL) + return + } + + opts, err := getOpts(ctx, r, bucket, object) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + objInfo, err := getObjectInfo(ctx, bucket, object, opts) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + retention := objectlock.GetObjectRetentionMeta(objInfo.UserDefined) + if !retention.Mode.Valid() { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNoSuchObjectLockConfiguration), r.URL) + return + } + + writeSuccessResponseXML(w, encodeResponse(retention)) + // Notify object retention accessed via a GET request. + sendEvent(eventArgs{ + EventName: event.ObjectAccessedGetRetention, + BucketName: bucket, + Object: objInfo, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) +} + +// ObjectTagSet key value tags +type ObjectTagSet struct { + Tags []tags.Tag `xml:"Tag"` +} + +type objectTagging struct { + XMLName xml.Name `xml:"Tagging"` + TagSet *ObjectTagSet `xml:"TagSet"` +} + +// GetObjectTaggingHandler - GET object tagging +func (api objectAPIHandlers) GetObjectTaggingHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetObjectTagging") + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if s3Error := authenticateRequest(ctx, r, policy.GetObjectTaggingAction); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + opts, err := getOpts(ctx, r, bucket, object) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + ot, err := objAPI.GetObjectTags(ctx, bucket, object, opts) + if err != nil { + // if object/version is not found locally, but exists on peer site - proxy + // the tagging request to peer site. The response to client will + // return tags from peer site. + if isErrObjectNotFound(err) || isErrVersionNotFound(err) { + proxytgts := getProxyTargets(ctx, bucket, object, opts) + if !proxytgts.Empty() { + globalReplicationStats.Load().incProxy(bucket, getObjectTaggingAPI, false) + // proxy to replication target if site replication is in place. + tags, gerr := proxyGetTaggingToRepTarget(ctx, bucket, object, opts, proxytgts) + if gerr.Err != nil || tags == nil { + globalReplicationStats.Load().incProxy(bucket, getObjectTaggingAPI, true) + writeErrorResponse(ctx, w, toAPIError(ctx, gerr.Err), r.URL) + return + } // overlay tags from peer site. + ot = tags + w.Header()[xhttp.MinIOTaggingProxied] = []string{"true"} // indicate that the request was proxied. + } else { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } else { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + + // Set this such that authorization policies can be applied on the object tags. + if tags := ot.String(); tags != "" { + r.Header.Set(xhttp.AmzObjectTagging, tags) + } + + if s3Error := authorizeRequest(ctx, r, policy.GetObjectTaggingAction); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + if opts.VersionID != "" && opts.VersionID != nullVersionID { + w.Header()[xhttp.AmzVersionID] = []string{opts.VersionID} + } + + otags := &objectTagging{ + TagSet: &ObjectTagSet{}, + } + + var list []tags.Tag + for k, v := range ot.ToMap() { + list = append(list, tags.Tag{ + Key: k, + Value: v, + }) + } + // Always return in sorted order for tags. + sort.Slice(list, func(i, j int) bool { + return list[i].Key < list[j].Key + }) + otags.TagSet.Tags = list + + writeSuccessResponseXML(w, encodeResponse(otags)) +} + +// PutObjectTaggingHandler - PUT object tagging +func (api objectAPIHandlers) PutObjectTaggingHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PutObjectTagging") + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + // Tags XML will not be bigger than 1MiB in size, fail if its bigger. + tags, err := tags.ParseObjectXML(io.LimitReader(r.Body, 1<<20)) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Set this such that authorization policies can be applied on the object tags. + r.Header.Set(xhttp.AmzObjectTagging, tags.String()) + + // Allow putObjectTagging if policy action is set + if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectTaggingAction, bucket, object); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + opts, err := getOpts(ctx, r, bucket, object) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + objInfo, err := objAPI.GetObjectInfo(ctx, bucket, object, opts) + if err != nil { + // if object is not found locally, but exists on peer site - proxy + // the tagging request to peer site. The response to client will + // be 200 with extra header indicating that the request was proxied. + if isErrObjectNotFound(err) || isErrVersionNotFound(err) { + proxytgts := getProxyTargets(ctx, bucket, object, opts) + if !proxytgts.Empty() { + globalReplicationStats.Load().incProxy(bucket, putObjectTaggingAPI, false) + // proxy to replication target if site replication is in place. + perr := proxyTaggingToRepTarget(ctx, bucket, object, tags, opts, proxytgts) + if perr.Err != nil { + globalReplicationStats.Load().incProxy(bucket, putObjectTaggingAPI, true) + writeErrorResponse(ctx, w, toAPIError(ctx, perr.Err), r.URL) + return + } + w.Header()[xhttp.MinIOTaggingProxied] = []string{"true"} + writeSuccessResponseHeadersOnly(w) + // when tagging is proxied, the object version is not available to return + // as header in the response, or ObjectInfo in the notification event. + sendEvent(eventArgs{ + EventName: event.ObjectCreatedPutTagging, + BucketName: bucket, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) + return + } + } + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + tagsStr := tags.String() + + dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(objInfo.UserDefined, tagsStr, objInfo.ReplicationStatus, replication.MetadataReplicationType, opts)) + if dsc.ReplicateAny() { + opts.UserDefined = make(map[string]string) + opts.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano) + opts.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus() + opts.UserDefined[ReservedMetadataPrefixLower+TaggingTimestamp] = UTCNow().Format(time.RFC3339Nano) + } + + // Put object tags + objInfo, err = objAPI.PutObjectTags(ctx, bucket, object, tagsStr, opts) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if dsc.ReplicateAny() { + scheduleReplication(ctx, objInfo, objAPI, dsc, replication.MetadataReplicationType) + } + + if objInfo.VersionID != "" && objInfo.VersionID != nullVersionID { + w.Header()[xhttp.AmzVersionID] = []string{objInfo.VersionID} + } + + writeSuccessResponseHeadersOnly(w) + + sendEvent(eventArgs{ + EventName: event.ObjectCreatedPutTagging, + BucketName: bucket, + Object: objInfo, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) +} + +// DeleteObjectTaggingHandler - DELETE object tagging +func (api objectAPIHandlers) DeleteObjectTaggingHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "DeleteObjectTagging") + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objAPI := api.ObjectAPI() + if objAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + opts, err := getOpts(ctx, r, bucket, object) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + oi, err := objAPI.GetObjectInfo(ctx, bucket, object, opts) + if err != nil { + // if object is not found locally, but exists on peer site - proxy + // the tagging request to peer site. The response to client will + // be 200 OK with extra header indicating that the request was proxied. + if isErrObjectNotFound(err) || isErrVersionNotFound(err) { + proxytgts := getProxyTargets(ctx, bucket, object, opts) + if !proxytgts.Empty() { + globalReplicationStats.Load().incProxy(bucket, removeObjectTaggingAPI, false) + // proxy to replication target if active-active replication is in place. + perr := proxyTaggingToRepTarget(ctx, bucket, object, nil, opts, proxytgts) + if perr.Err != nil { + globalReplicationStats.Load().incProxy(bucket, removeObjectTaggingAPI, true) + writeErrorResponse(ctx, w, toAPIError(ctx, perr.Err), r.URL) + return + } + // when delete tagging is proxied, the object version/tags are not available to return + // as header in the response, nor ObjectInfo in the notification event. + w.Header()[xhttp.MinIOTaggingProxied] = []string{"true"} + writeSuccessNoContent(w) + sendEvent(eventArgs{ + EventName: event.ObjectCreatedDeleteTagging, + BucketName: bucket, + Object: oi, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) + return + } + } + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if userTags := oi.UserTags; userTags != "" { + // Set this such that authorization policies can be applied on the object tags. + r.Header.Set(xhttp.AmzObjectTagging, oi.UserTags) + } + + // Allow deleteObjectTagging if policy action is set + if s3Error := checkRequestAuthType(ctx, r, policy.DeleteObjectTaggingAction, bucket, object); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + dsc := mustReplicate(ctx, bucket, object, oi.getMustReplicateOptions(replication.MetadataReplicationType, opts)) + if dsc.ReplicateAny() { + opts.UserDefined = make(map[string]string) + opts.UserDefined[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano) + opts.UserDefined[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus() + } + + oi, err = objAPI.DeleteObjectTags(ctx, bucket, object, opts) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if dsc.ReplicateAny() { + scheduleReplication(ctx, oi, objAPI, dsc, replication.MetadataReplicationType) + } + + if oi.VersionID != "" && oi.VersionID != nullVersionID { + w.Header()[xhttp.AmzVersionID] = []string{oi.VersionID} + } + writeSuccessNoContent(w) + + sendEvent(eventArgs{ + EventName: event.ObjectCreatedDeleteTagging, + BucketName: bucket, + Object: oi, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) +} + +// RestoreObjectHandler - POST restore object handler. +// ---------- +func (api objectAPIHandlers) PostRestoreObjectHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PostRestoreObject") + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Fetch object stat info. + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + getObjectInfo := objectAPI.GetObjectInfo + + // Check for auth type to return S3 compatible error. + if s3Error := checkRequestAuthType(ctx, r, policy.RestoreObjectAction, bucket, object); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + if r.ContentLength <= 0 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEmptyRequestBody), r.URL) + return + } + opts, err := postRestoreOpts(ctx, r, bucket, object) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + objInfo, err := getObjectInfo(ctx, bucket, object, opts) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if objInfo.TransitionedObject.Status != lifecycle.TransitionComplete { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidObjectState), r.URL) + return + } + + rreq, err := parseRestoreRequest(io.LimitReader(r.Body, r.ContentLength)) + if err != nil { + apiErr := errorCodes.ToAPIErr(ErrMalformedXML) + apiErr.Description = err.Error() + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + // validate the request + if err := rreq.validate(ctx, objectAPI); err != nil { + apiErr := errorCodes.ToAPIErr(ErrMalformedXML) + apiErr.Description = err.Error() + writeErrorResponse(ctx, w, apiErr, r.URL) + return + } + statusCode := http.StatusOK + alreadyRestored := false + if err == nil { + if objInfo.RestoreOngoing && rreq.Type != SelectRestoreRequest { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrObjectRestoreAlreadyInProgress), r.URL) + return + } + if !objInfo.RestoreOngoing && !objInfo.RestoreExpires.IsZero() { + statusCode = http.StatusAccepted + alreadyRestored = true + } + } + // set or upgrade restore expiry + restoreExpiry := lifecycle.ExpectedExpiryTime(time.Now().UTC(), rreq.Days) + metadata := cloneMSS(objInfo.UserDefined) + + // update self with restore metadata + if rreq.Type != SelectRestoreRequest { + objInfo.metadataOnly = true // Perform only metadata updates. + metadata[xhttp.AmzRestoreExpiryDays] = strconv.Itoa(rreq.Days) + metadata[xhttp.AmzRestoreRequestDate] = time.Now().UTC().Format(http.TimeFormat) + if alreadyRestored { + metadata[xhttp.AmzRestore] = completedRestoreObj(restoreExpiry).String() + } else { + metadata[xhttp.AmzRestore] = ongoingRestoreObj().String() + } + objInfo.UserDefined = metadata + if _, err := objectAPI.CopyObject(GlobalContext, bucket, object, bucket, object, objInfo, ObjectOptions{ + VersionID: objInfo.VersionID, + }, ObjectOptions{ + VersionID: objInfo.VersionID, + MTime: objInfo.ModTime, + }); err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidObjectState), r.URL) + return + } + // for previously restored object, just update the restore expiry + if alreadyRestored { + return + } + } + + restoreObject := mustGetUUID() + if rreq.OutputLocation.S3.BucketName != "" { + w.Header()[xhttp.AmzRestoreOutputPath] = []string{pathJoin(rreq.OutputLocation.S3.BucketName, rreq.OutputLocation.S3.Prefix, restoreObject)} + } + w.WriteHeader(statusCode) + // Notify object restore started via a POST request. + sendEvent(eventArgs{ + EventName: event.ObjectRestorePost, + BucketName: bucket, + Object: objInfo, + ReqParams: extractReqParams(r), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) + // now process the restore in background + go func() { + rctx := GlobalContext + if !rreq.SelectParameters.IsEmpty() { + actualSize, err := objInfo.GetActualSize() + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + objectRSC := s3select.NewObjectReadSeekCloser( + func(offset int64) (io.ReadCloser, error) { + rs := &HTTPRangeSpec{ + IsSuffixLength: false, + Start: offset, + End: -1, + } + return getTransitionedObjectReader(rctx, bucket, object, rs, r.Header, + objInfo, ObjectOptions{VersionID: objInfo.VersionID}) + }, + actualSize, + ) + defer objectRSC.Close() + if err = rreq.SelectParameters.Open(objectRSC); err != nil { + if serr, ok := err.(s3select.SelectError); ok { + encodedErrorResponse := encodeResponse(APIErrorResponse{ + Code: serr.ErrorCode(), + Message: serr.ErrorMessage(), + BucketName: bucket, + Key: object, + Resource: r.URL.Path, + RequestID: w.Header().Get(xhttp.AmzRequestID), + HostID: globalDeploymentID(), + }) + writeResponse(w, serr.HTTPStatusCode(), encodedErrorResponse, mimeXML) + } else { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + } + return + } + nr := httptest.NewRecorder() + rw := xhttp.NewResponseRecorder(nr) + rw.LogErrBody = true + rw.LogAllBody = true + rreq.SelectParameters.Evaluate(rw) + rreq.SelectParameters.Close() + return + } + opts := ObjectOptions{ + Transition: TransitionOptions{ + RestoreRequest: rreq, + RestoreExpiry: restoreExpiry, + }, + VersionID: objInfo.VersionID, + } + if err := objectAPI.RestoreTransitionedObject(rctx, bucket, object, opts); err != nil { + s3LogIf(ctx, fmt.Errorf("Unable to restore transitioned bucket/object %s/%s: %w", bucket, object, err)) + return + } + + // Notify object restore completed via a POST request. + sendEvent(eventArgs{ + EventName: event.ObjectRestoreCompleted, + BucketName: bucket, + Object: objInfo, + ReqParams: extractReqParams(r), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + }) + }() +} diff --git a/cmd/object-handlers_test.go b/cmd/object-handlers_test.go new file mode 100644 index 00000000..ec71c692 --- /dev/null +++ b/cmd/object-handlers_test.go @@ -0,0 +1,4249 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/md5" + "crypto/sha1" + "encoding/base64" + "encoding/hex" + "encoding/xml" + "fmt" + "hash" + "hash/crc32" + "io" + "maps" + "net/http" + "net/http/httptest" + "net/url" + "path" + "runtime" + "strconv" + "strings" + "sync" + "testing" + + "github.com/dustin/go-humanize" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/hash/sha256" + xhttp "github.com/minio/minio/internal/http" + ioutilx "github.com/minio/minio/internal/ioutil" +) + +// Type to capture different modifications to API request to simulate failure cases. +type Fault int + +const ( + None Fault = iota + MissingContentLength + TooBigObject + TooBigDecodedLength + BadSignature + BadMD5 + MissingUploadID +) + +// Wrapper for calling HeadObject API handler tests for both Erasure multiple disks and FS single drive setup. +func TestAPIHeadObjectHandler(t *testing.T) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPIHeadObjectHandler, endpoints: []string{"HeadObject"}}) +} + +func testAPIHeadObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + objectName := "test-object" + // set of byte data for PutObject. + // object has to be created before running tests for HeadObject. + // this is required even to assert the HeadObject data, + // since dataInserted === dataFetched back is a primary criteria for any object storage this assertion is critical. + bytesData := []struct { + byteData []byte + }{ + {generateBytesData(6 * humanize.MiByte)}, + } + // set of inputs for uploading the objects before tests for downloading is done. + putObjectInputs := []struct { + bucketName string + objectName string + contentLength int64 + textData []byte + metaData map[string]string + }{ + {bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)}, + } + // iterate through the above set of inputs and upload the object. + for i, input := range putObjectInputs { + // uploading the object. + _, err := obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewReader(input.textData), input.contentLength, input.metaData[""], ""), ObjectOptions{UserDefined: input.metaData}) + // if object upload fails stop the test. + if err != nil { + t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) + } + } + + // test cases with inputs and expected result for HeadObject. + testCases := []struct { + bucketName string + objectName string + accessKey string + secretKey string + // expected output. + expectedRespStatus int // expected response status body. + }{ + // Test case - 1. + // Fetching stat info of object and validating it. + { + bucketName: bucketName, + objectName: objectName, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusOK, + }, + // Test case - 2. + // Case with non-existent object name. + { + bucketName: bucketName, + objectName: "abcd", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusNotFound, + }, + // Test case - 3. + // Test case to induce a signature mismatch. + // Using invalid accessID. + { + bucketName: bucketName, + objectName: objectName, + accessKey: "Invalid-AccessID", + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusForbidden, + }, + } + + // Iterating over the cases, fetching the object validating the response. + for i, testCase := range testCases { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + // construct HTTP request for Get Object end point. + req, err := newTestSignedRequestV4(http.MethodHead, getHeadObjectURL("", testCase.bucketName, testCase.objectName), + 0, nil, testCase.accessKey, testCase.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for Head Object: %v", i+1, instanceType, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler,`func (api objectAPIHandlers) GetObjectHandler` handles the request. + apiRouter.ServeHTTP(rec, req) + + // Assert the response code with the expected status. + if rec.Code != testCase.expectedRespStatus { + t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, rec.Code) + } + + // Verify response of the V2 signed HTTP request. + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + recV2 := httptest.NewRecorder() + // construct HTTP request for Head Object endpoint. + reqV2, err := newTestSignedRequestV2(http.MethodHead, getHeadObjectURL("", testCase.bucketName, testCase.objectName), + 0, nil, testCase.accessKey, testCase.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for Head Object: %v", i+1, instanceType, err) + } + + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(recV2, reqV2) + if recV2.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code) + } + } + + // Test for Anonymous/unsigned http request. + anonReq, err := newTestRequest(http.MethodHead, getHeadObjectURL("", bucketName, objectName), 0, nil) + if err != nil { + t.Fatalf("MinIO %s: Failed to create an anonymous request for %s/%s: %v", + instanceType, bucketName, objectName, err) + } + + // ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse, + // sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the + // unsigned request goes through and its validated again. + ExecObjectLayerAPIAnonTest(t, obj, "TestAPIHeadObjectHandler", bucketName, objectName, instanceType, apiRouter, anonReq, getAnonReadOnlyObjectPolicy(bucketName, objectName)) + + // HTTP request for testing when `objectLayer` is set to `nil`. + // There is no need to use an existing bucket and valid input for creating the request + // since the `objectLayer==nil` check is performed before any other checks inside the handlers. + // The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called. + + nilBucket := "dummy-bucket" + nilObject := "dummy-object" + nilReq, err := newTestSignedRequestV4(http.MethodHead, getGetObjectURL("", nilBucket, nilObject), + 0, nil, "", "", nil) + if err != nil { + t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType) + } + // execute the object layer set to `nil` test. + // `ExecObjectLayerAPINilTest` manages the operation. + ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq) +} + +func TestAPIHeadObjectHandlerWithEncryption(t *testing.T) { + globalPolicySys = NewPolicySys() + defer func() { globalPolicySys = nil }() + + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPIHeadObjectHandlerWithEncryption, endpoints: []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject", "HeadObject"}}) +} + +func testAPIHeadObjectHandlerWithEncryption(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + // Set SSL to on to do encryption tests + globalIsTLS = true + defer func() { globalIsTLS = false }() + + var ( + oneMiB int64 = 1024 * 1024 + key32Bytes = generateBytesData(32 * humanize.Byte) + key32BytesMd5 = md5.Sum(key32Bytes) + metaWithSSEC = map[string]string{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES, + xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key32Bytes), + xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(key32BytesMd5[:]), + } + mapCopy = func(m map[string]string) map[string]string { + r := make(map[string]string, len(m)) + maps.Copy(r, m) + return r + } + ) + + type ObjectInput struct { + objectName string + partLengths []int64 + + metaData map[string]string + } + + objectLength := func(oi ObjectInput) (sum int64) { + for _, l := range oi.partLengths { + sum += l + } + return sum + } + + // set of inputs for uploading the objects before tests for + // downloading is done. Data bytes are from DummyDataGen. + objectInputs := []ObjectInput{ + // Unencrypted objects + {"nothing", []int64{0}, nil}, + {"small-1", []int64{509}, nil}, + + {"mp-1", []int64{5 * oneMiB, 1}, nil}, + {"mp-2", []int64{5487701, 5487799, 3}, nil}, + + // Encrypted object + {"enc-nothing", []int64{0}, mapCopy(metaWithSSEC)}, + {"enc-small-1", []int64{509}, mapCopy(metaWithSSEC)}, + + {"enc-mp-1", []int64{5 * oneMiB, 1}, mapCopy(metaWithSSEC)}, + {"enc-mp-2", []int64{5487701, 5487799, 3}, mapCopy(metaWithSSEC)}, + } + + // iterate through the above set of inputs and upload the object. + for _, input := range objectInputs { + uploadTestObject(t, apiRouter, credentials, bucketName, input.objectName, input.partLengths, input.metaData, false) + } + + for i, input := range objectInputs { + // initialize HTTP NewRecorder, this records any + // mutations to response writer inside the handler. + rec := httptest.NewRecorder() + // construct HTTP request for HEAD object. + req, err := newTestSignedRequestV4(http.MethodHead, getHeadObjectURL("", bucketName, input.objectName), + 0, nil, credentials.AccessKey, credentials.SecretKey, nil) + if err != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for Head Object: %v", i+1, instanceType, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a + // ServeHTTP to execute the logic of the handler. + apiRouter.ServeHTTP(rec, req) + + isEnc := false + expected := 200 + if strings.HasPrefix(input.objectName, "enc-") { + isEnc = true + expected = 400 + } + if rec.Code != expected { + t.Errorf("Test %d: expected code %d but got %d for object %s", i+1, expected, rec.Code, input.objectName) + } + + contentLength := rec.Header().Get("Content-Length") + if isEnc { + // initialize HTTP NewRecorder, this records any + // mutations to response writer inside the handler. + rec := httptest.NewRecorder() + // construct HTTP request for HEAD object. + req, err := newTestSignedRequestV4(http.MethodHead, getHeadObjectURL("", bucketName, input.objectName), + 0, nil, credentials.AccessKey, credentials.SecretKey, input.metaData) + if err != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for Head Object: %v", i+1, instanceType, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a + // ServeHTTP to execute the logic of the handler. + apiRouter.ServeHTTP(rec, req) + + if rec.Code != 200 { + t.Errorf("Test %d: Did not receive a 200 response: %d", i+1, rec.Code) + } + contentLength = rec.Header().Get("Content-Length") + } + + if contentLength != fmt.Sprintf("%d", objectLength(input)) { + t.Errorf("Test %d: Content length is mismatching: got %s (expected: %d)", i+1, contentLength, objectLength(input)) + } + } +} + +// Wrapper for calling GetObject API handler tests for both Erasure multiple disks and FS single drive setup. +func TestAPIGetObjectHandler(t *testing.T) { + globalPolicySys = NewPolicySys() + defer func() { globalPolicySys = nil }() + + defer DetectTestLeak(t)() + ExecExtendedObjectLayerAPITest(t, testAPIGetObjectHandler, []string{"GetObject"}) +} + +func testAPIGetObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + objectName := "test-object" + // set of byte data for PutObject. + // object has to be created before running tests for GetObject. + // this is required even to assert the GetObject data, + // since dataInserted === dataFetched back is a primary criteria for any object storage this assertion is critical. + bytesData := []struct { + byteData []byte + }{ + {generateBytesData(6 * humanize.MiByte)}, + } + // set of inputs for uploading the objects before tests for downloading is done. + putObjectInputs := []struct { + bucketName string + objectName string + contentLength int64 + textData []byte + metaData map[string]string + }{ + // case - 1. + {bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)}, + } + // iterate through the above set of inputs and upload the object. + for i, input := range putObjectInputs { + // uploading the object. + _, err := obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewReader(input.textData), input.contentLength, input.metaData[""], ""), ObjectOptions{UserDefined: input.metaData}) + // if object upload fails stop the test. + if err != nil { + t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) + } + } + + ctx := context.Background() + + // test cases with inputs and expected result for GetObject. + testCases := []struct { + bucketName string + objectName string + byteRange string // range of bytes to be fetched from GetObject. + accessKey string + secretKey string + // expected output. + expectedContent []byte // expected response body. + expectedRespStatus int // expected response status body. + }{ + // Test case - 1. + // Fetching the entire object and validating its contents. + { + bucketName: bucketName, + objectName: objectName, + byteRange: "", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedContent: bytesData[0].byteData, + expectedRespStatus: http.StatusOK, + }, + // Test case - 2. + // Case with non-existent object name. + { + bucketName: bucketName, + objectName: "abcd", + byteRange: "", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedContent: encodeResponse(getAPIErrorResponse(ctx, + getAPIError(ErrNoSuchKey), + getGetObjectURL("", bucketName, "abcd"), "", "")), + expectedRespStatus: http.StatusNotFound, + }, + // Test case - 3. + // Requesting from range 10-100. + { + bucketName: bucketName, + objectName: objectName, + byteRange: "bytes=10-100", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedContent: bytesData[0].byteData[10:101], + expectedRespStatus: http.StatusPartialContent, + }, + // Test case - 4. + // Test case with invalid range. + { + bucketName: bucketName, + objectName: objectName, + byteRange: "bytes=-0", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedContent: encodeResponse(getAPIErrorResponse(ctx, + getAPIError(ErrInvalidRange), + getGetObjectURL("", bucketName, objectName), "", "")), + expectedRespStatus: http.StatusRequestedRangeNotSatisfiable, + }, + // Test case - 5. + // Test case with byte range exceeding the object size. + // Expected to read till end of the object. + { + bucketName: bucketName, + objectName: objectName, + byteRange: "bytes=10-1000000000000000", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedContent: bytesData[0].byteData[10:], + expectedRespStatus: http.StatusPartialContent, + }, + // Test case - 6. + // Test case to induce a signature mismatch. + // Using invalid accessID. + { + bucketName: bucketName, + objectName: objectName, + byteRange: "", + accessKey: "Invalid-AccessID", + secretKey: credentials.SecretKey, + + expectedContent: encodeResponse(getAPIErrorResponse(ctx, + getAPIError(ErrInvalidAccessKeyID), + getGetObjectURL("", bucketName, objectName), "", "")), + expectedRespStatus: http.StatusForbidden, + }, + // Test case - 7. + // Case with bad components in object name. + { + bucketName: bucketName, + objectName: "../../etc", + byteRange: "", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedContent: encodeResponse(getAPIErrorResponse(ctx, + getAPIError(ErrInvalidObjectName), + getGetObjectURL("", bucketName, "../../etc"), "", "")), + expectedRespStatus: http.StatusBadRequest, + }, + // Test case - 8. + // Case with strange components but returning error as not found. + { + bucketName: bucketName, + objectName: ". ./. ./etc", + byteRange: "", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedContent: encodeResponse(getAPIErrorResponse(ctx, + getAPIError(ErrNoSuchKey), + SlashSeparator+bucketName+SlashSeparator+". ./. ./etc", "", "")), + expectedRespStatus: http.StatusNotFound, + }, + // Test case - 9. + // Case with bad components in object name. + { + bucketName: bucketName, + objectName: ". ./../etc", + byteRange: "", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedContent: encodeResponse(getAPIErrorResponse(ctx, + getAPIError(ErrInvalidObjectName), + SlashSeparator+bucketName+SlashSeparator+". ./../etc", "", "")), + expectedRespStatus: http.StatusBadRequest, + }, + // Test case - 10. + // Case with proper components + { + bucketName: bucketName, + objectName: "etc/path/proper/.../etc", + byteRange: "", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedContent: encodeResponse(getAPIErrorResponse(ctx, + getAPIError(ErrNoSuchKey), + getGetObjectURL("", bucketName, "etc/path/proper/.../etc"), + "", "")), + expectedRespStatus: http.StatusNotFound, + }, + } + + // Iterating over the cases, fetching the object validating the response. + for i, testCase := range testCases { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + // construct HTTP request for Get Object end point. + req, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", testCase.bucketName, testCase.objectName), + 0, nil, testCase.accessKey, testCase.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: Failed to create HTTP request for Get Object: %v", i+1, err) + } + if testCase.byteRange != "" { + req.Header.Set("Range", testCase.byteRange) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler,`func (api objectAPIHandlers) GetObjectHandler` handles the request. + apiRouter.ServeHTTP(rec, req) + // Assert the response code with the expected status. + if rec.Code != testCase.expectedRespStatus { + t.Fatalf("Case %d: Expected the response status to be `%d`, but instead found `%d`", i+1, testCase.expectedRespStatus, rec.Code) + } + // read the response body. + actualContent, err := io.ReadAll(rec.Body) + if err != nil { + t.Fatalf("Test %d: %s: Failed reading response body: %v", i+1, instanceType, err) + } + + if rec.Code == http.StatusOK || rec.Code == http.StatusPartialContent { + if !bytes.Equal(testCase.expectedContent, actualContent) { + t.Errorf("Test %d: %s: Object content differs from expected value %s, got %s", i+1, instanceType, testCase.expectedContent, string(actualContent)) + } + continue + } + + // Verify whether the bucket obtained object is same as the one created. + actualError := &APIErrorResponse{} + if err = xml.Unmarshal(actualContent, actualError); err != nil { + t.Fatalf("Test %d: %s: Failed parsing response body: %v", i+1, instanceType, err) + } + + if path.Clean(actualError.Resource) != pathJoin(SlashSeparator, testCase.bucketName, testCase.objectName) { + t.Fatalf("Test %d: %s: Unexpected resource, expected %s, got %s", i+1, instanceType, pathJoin(SlashSeparator, testCase.bucketName, testCase.objectName), actualError.Resource) + } + + // Verify response of the V2 signed HTTP request. + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + recV2 := httptest.NewRecorder() + // construct HTTP request for GET Object endpoint. + reqV2, err := newTestSignedRequestV2(http.MethodGet, getGetObjectURL("", testCase.bucketName, testCase.objectName), + 0, nil, testCase.accessKey, testCase.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for GetObject: %v", i+1, instanceType, err) + } + + if testCase.byteRange != "" { + reqV2.Header.Set("Range", testCase.byteRange) + } + + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(recV2, reqV2) + if recV2.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code) + } + + // read the response body. + actualContent, err = io.ReadAll(recV2.Body) + if err != nil { + t.Fatalf("Test %d: %s: Failed to read response body: %v", i+1, instanceType, err) + } + + if rec.Code == http.StatusOK || rec.Code == http.StatusPartialContent { + // Verify whether the bucket obtained object is same as the one created. + if !bytes.Equal(testCase.expectedContent, actualContent) { + t.Errorf("Test %d: %s: Object content differs from expected value.", i+1, instanceType) + } + continue + } + + actualError = &APIErrorResponse{} + if err = xml.Unmarshal(actualContent, actualError); err != nil { + t.Fatalf("Test %d: %s: Failed parsing response body: %v", i+1, instanceType, err) + } + + if path.Clean(actualError.Resource) != pathJoin(SlashSeparator, testCase.bucketName, testCase.objectName) { + t.Fatalf("Test %d: %s: Unexpected resource, expected %s, got %s", i+1, instanceType, pathJoin(SlashSeparator, testCase.bucketName, testCase.objectName), actualError.Resource) + } + } + + // Test for Anonymous/unsigned http request. + anonReq, err := newTestRequest(http.MethodGet, getGetObjectURL("", bucketName, objectName), 0, nil) + if err != nil { + t.Fatalf("MinIO %s: Failed to create an anonymous request for %s/%s: %v", + instanceType, bucketName, objectName, err) + } + + // ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse, + // sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the + // unsigned request goes through and its validated again. + ExecObjectLayerAPIAnonTest(t, obj, "TestAPIGetObjectHandler", bucketName, objectName, instanceType, apiRouter, anonReq, getAnonReadOnlyObjectPolicy(bucketName, objectName)) + + // HTTP request for testing when `objectLayer` is set to `nil`. + // There is no need to use an existing bucket and valid input for creating the request + // since the `objectLayer==nil` check is performed before any other checks inside the handlers. + // The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called. + + nilBucket := "dummy-bucket" + nilObject := "dummy-object" + nilReq, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", nilBucket, nilObject), + 0, nil, "", "", nil) + if err != nil { + t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType) + } + // execute the object layer set to `nil` test. + // `ExecObjectLayerAPINilTest` manages the operation. + ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq) +} + +// Wrapper for calling GetObject API handler tests for both Erasure multiple disks and FS single drive setup. +func TestAPIGetObjectWithMPHandler(t *testing.T) { + globalPolicySys = NewPolicySys() + defer func() { globalPolicySys = nil }() + + defer DetectTestLeak(t)() + ExecExtendedObjectLayerAPITest(t, testAPIGetObjectWithMPHandler, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject"}) +} + +func testAPIGetObjectWithMPHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + // Set SSL to on to do encryption tests + globalIsTLS = true + defer func() { globalIsTLS = false }() + + var ( + oneMiB int64 = 1024 * 1024 + key32Bytes = generateBytesData(32 * humanize.Byte) + key32BytesMd5 = md5.Sum(key32Bytes) + metaWithSSEC = map[string]string{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES, + xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key32Bytes), + xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(key32BytesMd5[:]), + } + mapCopy = func(m map[string]string) map[string]string { + r := make(map[string]string, len(m)) + maps.Copy(r, m) + return r + } + ) + + type ObjectInput struct { + objectName string + partLengths []int64 + + metaData map[string]string + } + + objectLength := func(oi ObjectInput) (sum int64) { + for _, l := range oi.partLengths { + sum += l + } + return sum + } + + // set of inputs for uploading the objects before tests for + // downloading is done. Data bytes are from DummyDataGen. + objectInputs := []ObjectInput{ + // // cases 0-3: small single part objects + {"nothing", []int64{0}, make(map[string]string)}, + {"small-0", []int64{11}, make(map[string]string)}, + {"small-1", []int64{509}, make(map[string]string)}, + {"small-2", []int64{5 * oneMiB}, make(map[string]string)}, + // // // cases 4-7: multipart part objects + {"mp-0", []int64{5 * oneMiB, 10}, make(map[string]string)}, + {"mp-1", []int64{5*oneMiB + 1, 10}, make(map[string]string)}, + {"mp-2", []int64{5487701, 5487799, 3}, make(map[string]string)}, + {"mp-3", []int64{10499807, 10499963, 7}, make(map[string]string)}, + // cases 8-11: small single part objects with encryption + {"enc-nothing", []int64{0}, mapCopy(metaWithSSEC)}, + {"enc-small-0", []int64{11}, mapCopy(metaWithSSEC)}, + {"enc-small-1", []int64{509}, mapCopy(metaWithSSEC)}, + {"enc-small-2", []int64{5 * oneMiB}, mapCopy(metaWithSSEC)}, + // cases 12-15: multipart part objects with encryption + {"enc-mp-0", []int64{5 * oneMiB, 10}, mapCopy(metaWithSSEC)}, + {"enc-mp-1", []int64{5*oneMiB + 1, 10}, mapCopy(metaWithSSEC)}, + {"enc-mp-2", []int64{5487701, 5487799, 3}, mapCopy(metaWithSSEC)}, + {"enc-mp-3", []int64{10499807, 10499963, 7}, mapCopy(metaWithSSEC)}, + } + if testing.Short() { + objectInputs = append(objectInputs[0:5], objectInputs[8:11]...) + } + // iterate through the above set of inputs and upload the object. + for _, input := range objectInputs { + uploadTestObject(t, apiRouter, credentials, bucketName, input.objectName, input.partLengths, input.metaData, false) + } + + // function type for creating signed requests - used to repeat + // requests with V2 and V4 signing. + type testSignedReqFn func(method, urlStr string, contentLength int64, + body io.ReadSeeker, accessKey, secretKey string, metamap map[string]string) (*http.Request, + error) + + mkGetReq := func(oi ObjectInput, byteRange string, i int, mkSignedReq testSignedReqFn) { + object := oi.objectName + rec := httptest.NewRecorder() + req, err := mkSignedReq(http.MethodGet, getGetObjectURL("", bucketName, object), + 0, nil, credentials.AccessKey, credentials.SecretKey, oi.metaData) + if err != nil { + t.Fatalf("Object: %s Case %d ByteRange: %s: Failed to create HTTP request for Get Object: %v", + object, i+1, byteRange, err) + } + + if byteRange != "" { + req.Header.Set("Range", byteRange) + } + + apiRouter.ServeHTTP(rec, req) + + // Check response code (we make only valid requests in + // this test) + if rec.Code != http.StatusPartialContent && rec.Code != http.StatusOK { + bd, err1 := io.ReadAll(rec.Body) + t.Fatalf("%s Object: %s Case %d ByteRange: %s: Got response status `%d` and body: %s,%v", + instanceType, object, i+1, byteRange, rec.Code, string(bd), err1) + } + + var off, length int64 + var rs *HTTPRangeSpec + if byteRange != "" { + rs, err = parseRequestRangeSpec(byteRange) + if err != nil { + t.Fatalf("Object: %s Case %d ByteRange: %s: Unexpected err: %v", object, i+1, byteRange, err) + } + } + off, length, err = rs.GetOffsetLength(objectLength(oi)) + if err != nil { + t.Fatalf("Object: %s Case %d ByteRange: %s: Unexpected err: %v", object, i+1, byteRange, err) + } + + readers := []io.Reader{} + cumulativeSum := int64(0) + for _, p := range oi.partLengths { + readers = append(readers, NewDummyDataGen(p, cumulativeSum)) + cumulativeSum += p + } + + refReader := io.LimitReader(ioutilx.NewSkipReader(io.MultiReader(readers...), off), length) + if ok, msg := cmpReaders(refReader, rec.Body); !ok { + t.Fatalf("(%s) Object: %s Case %d ByteRange: %s --> data mismatch! (msg: %s)", instanceType, oi.objectName, i+1, byteRange, msg) + } + } + + // Iterate over each uploaded object and do a bunch of get + // requests on them. + caseNumber := 0 + signFns := []testSignedReqFn{newTestSignedRequestV2, newTestSignedRequestV4} + for _, oi := range objectInputs { + objLen := objectLength(oi) + for _, sf := range signFns { + // Read whole object + mkGetReq(oi, "", caseNumber, sf) + caseNumber++ + + // No range requests are possible if the + // object length is 0 + if objLen == 0 { + continue + } + + // Various ranges to query - all are valid! + rangeHdrs := []string{ + // Read first byte of object + fmt.Sprintf("bytes=%d-%d", 0, 0), + // Read second byte of object + fmt.Sprintf("bytes=%d-%d", 1, 1), + // Read last byte of object + fmt.Sprintf("bytes=-%d", 1), + // Read all but first byte of object + "bytes=1-", + // Read first half of object + fmt.Sprintf("bytes=%d-%d", 0, objLen/2), + // Read last half of object + fmt.Sprintf("bytes=-%d", objLen/2), + // Read middle half of object + fmt.Sprintf("bytes=%d-%d", objLen/4, objLen*3/4), + // Read 100MiB of the object from the beginning + fmt.Sprintf("bytes=%d-%d", 0, 100*humanize.MiByte), + // Read 100MiB of the object from the end + fmt.Sprintf("bytes=-%d", 100*humanize.MiByte), + } + for _, rangeHdr := range rangeHdrs { + mkGetReq(oi, rangeHdr, caseNumber, sf) + caseNumber++ + } + } + } + + // HTTP request for testing when `objectLayer` is set to `nil`. + // There is no need to use an existing bucket and valid input for creating the request + // since the `objectLayer==nil` check is performed before any other checks inside the handlers. + // The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called. + + nilBucket := "dummy-bucket" + nilObject := "dummy-object" + nilReq, err := newTestSignedRequestV4(http.MethodGet, getGetObjectURL("", nilBucket, nilObject), + 0, nil, "", "", nil) + if err != nil { + t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType) + } + // execute the object layer set to `nil` test. + // `ExecObjectLayerAPINilTest` manages the operation. + ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq) +} + +// Wrapper for calling GetObject API handler tests for both Erasure multiple disks and FS single drive setup. +func TestAPIGetObjectWithPartNumberHandler(t *testing.T) { + globalPolicySys = NewPolicySys() + defer func() { globalPolicySys = nil }() + + defer DetectTestLeak(t)() + ExecExtendedObjectLayerAPITest(t, testAPIGetObjectWithPartNumberHandler, []string{"NewMultipart", "PutObjectPart", "CompleteMultipart", "GetObject", "PutObject"}) +} + +func testAPIGetObjectWithPartNumberHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + // Set SSL to on to do encryption tests + globalIsTLS = true + defer func() { globalIsTLS = false }() + + var ( + oneMiB int64 = 1024 * 1024 + key32Bytes = generateBytesData(32 * humanize.Byte) + key32BytesMd5 = md5.Sum(key32Bytes) + metaWithSSEC = map[string]string{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: xhttp.AmzEncryptionAES, + xhttp.AmzServerSideEncryptionCustomerKey: base64.StdEncoding.EncodeToString(key32Bytes), + xhttp.AmzServerSideEncryptionCustomerKeyMD5: base64.StdEncoding.EncodeToString(key32BytesMd5[:]), + } + mapCopy = func(m map[string]string) map[string]string { + r := make(map[string]string, len(m)) + maps.Copy(r, m) + return r + } + ) + + type ObjectInput struct { + objectName string + partLengths []int64 + + metaData map[string]string + } + + // set of inputs for uploading the objects before tests for + // downloading is done. Data bytes are from DummyDataGen. + objectInputs := []ObjectInput{ + // // cases 0-4: small single part objects + {"nothing", []int64{0}, make(map[string]string)}, + {"1byte", []int64{1}, make(map[string]string)}, + {"small-0", []int64{11}, make(map[string]string)}, + {"small-1", []int64{509}, make(map[string]string)}, + {"small-2", []int64{5 * oneMiB}, make(map[string]string)}, + // // // cases 5-8: multipart part objects + {"mp-0", []int64{5 * oneMiB, 1}, make(map[string]string)}, + {"mp-1", []int64{5*oneMiB + 1, 1}, make(map[string]string)}, + {"mp-2", []int64{5487701, 5487799, 3}, make(map[string]string)}, + {"mp-3", []int64{10499807, 10499963, 7}, make(map[string]string)}, + // cases 9-12: small single part objects with encryption + {"enc-nothing", []int64{0}, mapCopy(metaWithSSEC)}, + {"enc-small-0", []int64{11}, mapCopy(metaWithSSEC)}, + {"enc-small-1", []int64{509}, mapCopy(metaWithSSEC)}, + {"enc-small-2", []int64{5 * oneMiB}, mapCopy(metaWithSSEC)}, + // cases 13-16: multipart part objects with encryption + {"enc-mp-0", []int64{5 * oneMiB, 1}, mapCopy(metaWithSSEC)}, + {"enc-mp-1", []int64{5*oneMiB + 1, 1}, mapCopy(metaWithSSEC)}, + {"enc-mp-2", []int64{5487701, 5487799, 3}, mapCopy(metaWithSSEC)}, + {"enc-mp-3", []int64{10499807, 10499963, 7}, mapCopy(metaWithSSEC)}, + } + + // SSEC can't be used with compression + globalCompressConfigMu.Lock() + compressEnabled := globalCompressConfig.Enabled + globalCompressConfigMu.Unlock() + if compressEnabled { + objectInputs = objectInputs[0:9] + } + + // iterate through the above set of inputs and upload the object. + for _, input := range objectInputs { + uploadTestObject(t, apiRouter, credentials, bucketName, input.objectName, input.partLengths, input.metaData, false) + } + + mkGetReqWithPartNumber := func(oindex int, oi ObjectInput, partNumber int) { + object := oi.objectName + + queries := url.Values{} + queries.Add("partNumber", strconv.Itoa(partNumber)) + targetURL := makeTestTargetURL("", bucketName, object, queries) + req, err := newTestSignedRequestV4(http.MethodGet, targetURL, + 0, nil, credentials.AccessKey, credentials.SecretKey, oi.metaData) + if err != nil { + t.Fatalf("Object: %s Object Index %d PartNumber: %d: Failed to create HTTP request for Get Object: %v", + object, oindex, partNumber, err) + } + + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + + // Check response code (we make only valid requests in this test) + if rec.Code != http.StatusPartialContent && rec.Code != http.StatusOK { + bd, err1 := io.ReadAll(rec.Body) + t.Fatalf("%s Object: %s ObjectIndex %d PartNumber: %d: Got response status `%d` and body: %s,%v", + instanceType, object, oindex, partNumber, rec.Code, string(bd), err1) + } + + oinfo, err := obj.GetObjectInfo(context.Background(), bucketName, object, ObjectOptions{}) + if err != nil { + t.Fatalf("Object: %s Object Index %d: Unexpected err: %v", object, oindex, err) + } + + rs := partNumberToRangeSpec(oinfo, partNumber) + size, err := oinfo.GetActualSize() + if err != nil { + t.Fatalf("Object: %s Object Index %d: Unexpected err: %v", object, oindex, err) + } + + off, length, err := rs.GetOffsetLength(size) + if err != nil { + t.Fatalf("Object: %s Object Index %d: Unexpected err: %v", object, oindex, err) + } + + readers := []io.Reader{} + cumulativeSum := int64(0) + for _, p := range oi.partLengths { + readers = append(readers, NewDummyDataGen(p, cumulativeSum)) + cumulativeSum += p + } + + refReader := io.LimitReader(ioutilx.NewSkipReader(io.MultiReader(readers...), off), length) + if ok, msg := cmpReaders(refReader, rec.Body); !ok { + t.Fatalf("(%s) Object: %s ObjectIndex %d PartNumber: %d --> data mismatch! (msg: %s)", instanceType, oi.objectName, oindex, partNumber, msg) + } + } + + for idx, oi := range objectInputs { + for partNum := 1; partNum <= len(oi.partLengths); partNum++ { + mkGetReqWithPartNumber(idx, oi, partNum) + } + } +} + +// Wrapper for calling PutObject API handler tests using streaming signature v4 for both Erasure multiple disks and FS single drive setup. +func TestAPIPutObjectStreamSigV4Handler(t *testing.T) { + defer DetectTestLeak(t)() + ExecExtendedObjectLayerAPITest(t, testAPIPutObjectStreamSigV4Handler, []string{"PutObject"}) +} + +func testAPIPutObjectStreamSigV4Handler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + objectName := "test-object" + bytesDataLen := 65 * humanize.KiByte + bytesData := bytes.Repeat([]byte{'a'}, bytesDataLen) + oneKData := bytes.Repeat([]byte("a"), 1*humanize.KiByte) + + var err error + + type streamFault int + const ( + None streamFault = iota + malformedEncoding + unexpectedEOF + signatureMismatch + chunkDateMismatch + tooBigDecodedLength + ) + + // byte data for PutObject. + // test cases with inputs and expected result for GetObject. + testCases := []struct { + bucketName string + objectName string + data []byte + dataLen int + chunkSize int64 + // expected output. + expectedContent []byte // expected response body. + expectedRespStatus int // expected response status body. + // Access keys + accessKey string + secretKey string + shouldPass bool + removeAuthHeader bool + fault streamFault + // Custom content encoding. + contentEncoding string + }{ + // Test case - 1. + // Fetching the entire object and validating its contents. + { + bucketName: bucketName, + objectName: objectName, + data: bytesData, + dataLen: len(bytesData), + chunkSize: 64 * humanize.KiByte, + expectedContent: []byte{}, + expectedRespStatus: http.StatusOK, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + shouldPass: true, + fault: None, + }, + // Test case - 2 + // Small chunk size. + { + bucketName: bucketName, + objectName: objectName, + data: bytesData, + dataLen: len(bytesData), + chunkSize: 1 * humanize.KiByte, + expectedContent: []byte{}, + expectedRespStatus: http.StatusOK, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + shouldPass: true, + fault: None, + }, + // Test case - 3 + // Empty data + { + bucketName: bucketName, + objectName: objectName, + data: []byte{}, + dataLen: 0, + chunkSize: 64 * humanize.KiByte, + expectedContent: []byte{}, + expectedRespStatus: http.StatusOK, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + shouldPass: true, + }, + // Test case - 4 + // Invalid access key id. + { + bucketName: bucketName, + objectName: objectName, + data: bytesData, + dataLen: len(bytesData), + chunkSize: 64 * humanize.KiByte, + expectedContent: []byte{}, + expectedRespStatus: http.StatusForbidden, + accessKey: "", + secretKey: "", + shouldPass: false, + fault: None, + }, + // Test case - 5 + // Wrong auth header returns as bad request. + { + bucketName: bucketName, + objectName: objectName, + data: bytesData, + dataLen: len(bytesData), + chunkSize: 64 * humanize.KiByte, + expectedContent: []byte{}, + expectedRespStatus: http.StatusBadRequest, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + shouldPass: false, + removeAuthHeader: true, + fault: None, + }, + // Test case - 6 + // Large chunk size.. also passes. + { + bucketName: bucketName, + objectName: objectName, + data: bytesData, + dataLen: len(bytesData), + chunkSize: 100 * humanize.KiByte, + expectedContent: []byte{}, + expectedRespStatus: http.StatusOK, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + shouldPass: true, + fault: None, + }, + // Test case - 7 + // Chunk with malformed encoding. + // Causes signature mismatch. + { + bucketName: bucketName, + objectName: objectName, + data: oneKData, + dataLen: 1024, + chunkSize: 1024, + expectedContent: []byte{}, + expectedRespStatus: http.StatusForbidden, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + shouldPass: false, + fault: malformedEncoding, + }, + // Test case - 8 + // Chunk with shorter than advertised chunk data. + { + bucketName: bucketName, + objectName: objectName, + data: oneKData, + dataLen: 1024, + chunkSize: 1024, + expectedContent: []byte{}, + expectedRespStatus: http.StatusBadRequest, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + shouldPass: false, + fault: unexpectedEOF, + }, + // Test case - 9 + // Chunk with first chunk data byte tampered. + { + bucketName: bucketName, + objectName: objectName, + data: oneKData, + dataLen: 1024, + chunkSize: 1024, + expectedContent: []byte{}, + expectedRespStatus: http.StatusForbidden, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + shouldPass: false, + fault: signatureMismatch, + }, + // Test case - 10 + // Different date (timestamps) used in seed signature calculation + // and chunks signature calculation. + { + bucketName: bucketName, + objectName: objectName, + data: oneKData, + dataLen: 1024, + chunkSize: 1024, + expectedContent: []byte{}, + expectedRespStatus: http.StatusForbidden, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + shouldPass: false, + fault: chunkDateMismatch, + }, + // Test case - 11 + // Set x-amz-decoded-content-length to a value too big to hold in int64. + { + bucketName: bucketName, + objectName: objectName, + data: oneKData, + dataLen: 1024, + chunkSize: 1024, + expectedContent: []byte{}, + expectedRespStatus: http.StatusBadRequest, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + shouldPass: false, + fault: tooBigDecodedLength, + }, + // Test case - 12 + // Set custom content encoding should succeed and save the encoding properly. + { + bucketName: bucketName, + objectName: objectName, + data: bytesData, + dataLen: len(bytesData), + chunkSize: 100 * humanize.KiByte, + expectedContent: []byte{}, + expectedRespStatus: http.StatusOK, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + shouldPass: true, + contentEncoding: "aws-chunked,gzip", + fault: None, + }, + } + // Iterating over the cases, fetching the object validating the response. + for i, testCase := range testCases { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + // construct HTTP request for Put Object end point. + var req *http.Request + switch { + case testCase.fault == chunkDateMismatch: + req, err = newTestStreamingSignedBadChunkDateRequest(http.MethodPut, + getPutObjectURL("", testCase.bucketName, testCase.objectName), + int64(testCase.dataLen), testCase.chunkSize, bytes.NewReader(testCase.data), + testCase.accessKey, testCase.secretKey) + case testCase.contentEncoding == "": + req, err = newTestStreamingSignedRequest(http.MethodPut, + getPutObjectURL("", testCase.bucketName, testCase.objectName), + int64(testCase.dataLen), testCase.chunkSize, bytes.NewReader(testCase.data), + testCase.accessKey, testCase.secretKey) + case testCase.contentEncoding != "": + req, err = newTestStreamingSignedCustomEncodingRequest(http.MethodPut, + getPutObjectURL("", testCase.bucketName, testCase.objectName), + int64(testCase.dataLen), testCase.chunkSize, bytes.NewReader(testCase.data), + testCase.accessKey, testCase.secretKey, testCase.contentEncoding) + } + if err != nil { + t.Fatalf("Test %d: Failed to create HTTP request for Put Object: %v", i+1, err) + } + // Removes auth header if test case requires it. + if testCase.removeAuthHeader { + req.Header.Del("Authorization") + } + switch testCase.fault { + case malformedEncoding: + req, err = malformChunkSizeSigV4(req, testCase.chunkSize-1) + case signatureMismatch: + req, err = malformDataSigV4(req, 'z') + case unexpectedEOF: + req, err = truncateChunkByHalfSigv4(req) + case tooBigDecodedLength: + // Set decoded length to a large value out of int64 range to simulate parse failure. + req.Header.Set("x-amz-decoded-content-length", "9999999999999999999999") + } + + if err != nil { + t.Fatalf("Error injecting faults into the request: %v.", err) + } + + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler,`func (api objectAPIHandlers) GetObjectHandler` handles the request. + apiRouter.ServeHTTP(rec, req) + // Assert the response code with the expected status. + if rec.Code != testCase.expectedRespStatus { + t.Errorf("Test %d %s: Expected the response status to be `%d`, but instead found `%d`: fault case %d", + i+1, instanceType, testCase.expectedRespStatus, rec.Code, testCase.fault) + } + // read the response body. + actualContent, err := io.ReadAll(rec.Body) + if err != nil { + t.Fatalf("Test %d: %s: Failed parsing response body: %v", i+1, instanceType, err) + } + opts := ObjectOptions{} + if testCase.shouldPass { + // Verify whether the bucket obtained object is same as the one created. + if !bytes.Equal(testCase.expectedContent, actualContent) { + t.Errorf("Test %d: %s: Object content differs from expected value.: %s", i+1, instanceType, string(actualContent)) + continue + } + objInfo, err := obj.GetObjectInfo(context.Background(), testCase.bucketName, testCase.objectName, opts) + if err != nil { + t.Fatalf("Test %d: %s: Failed to fetch the copied object: %s", i+1, instanceType, err) + } + if objInfo.ContentEncoding == streamingContentEncoding { + t.Fatalf("Test %d: %s: ContentEncoding is set to \"aws-chunked\" which is unexpected", i+1, instanceType) + } + expectedContentEncoding := trimAwsChunkedContentEncoding(testCase.contentEncoding) + if expectedContentEncoding != objInfo.ContentEncoding { + t.Fatalf("Test %d: %s: ContentEncoding is set to \"%s\" which is unexpected, expected \"%s\"", i+1, instanceType, objInfo.ContentEncoding, expectedContentEncoding) + } + buffer := new(bytes.Buffer) + r, err := obj.GetObjectNInfo(context.Background(), testCase.bucketName, testCase.objectName, nil, nil, opts) + if err != nil { + t.Fatalf("Test %d: %s: Failed to fetch the copied object: %s", i+1, instanceType, err) + } + if _, err = io.Copy(buffer, r); err != nil { + r.Close() + t.Fatalf("Test %d: %s: Failed to fetch the copied object: %s", i+1, instanceType, err) + } + r.Close() + if !bytes.Equal(testCase.data, buffer.Bytes()) { + t.Errorf("Test %d: %s: Data Mismatch: Data fetched back from the uploaded object doesn't match the original one.", i+1, instanceType) + } + } + } +} + +// Wrapper for calling PutObject API handler tests for both Erasure multiple disks and FS single drive setup. +func TestAPIPutObjectHandler(t *testing.T) { + defer DetectTestLeak(t)() + ExecExtendedObjectLayerAPITest(t, testAPIPutObjectHandler, []string{"PutObject"}) +} + +func testAPIPutObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + var err error + objectName := "test-object" + opts := ObjectOptions{} + // byte data for PutObject. + bytesData := generateBytesData(6 * humanize.KiByte) + + copySourceHeader := map[string]string{"X-Amz-Copy-Source": "somewhere"} + invalidMD5Header := map[string]string{"Content-Md5": "42"} + invalidStorageClassHeader := map[string]string{xhttp.AmzStorageClass: "INVALID"} + + addCustomHeaders := func(req *http.Request, customHeaders map[string]string) { + for k, value := range customHeaders { + req.Header.Set(k, value) + } + } + + checksumData := func(b []byte, h hash.Hash) string { + h.Reset() + _, err := h.Write(b) + if err != nil { + t.Fatal(err) + } + return base64.StdEncoding.EncodeToString(h.Sum(nil)) + } + // test cases with inputs and expected result for GetObject. + testCases := []struct { + bucketName string + objectName string + headers map[string]string + data []byte + dataLen int + accessKey string + secretKey string + fault Fault + // expected output. + expectedRespStatus int // expected response status body. + wantAPICode string + wantHeaders map[string]string + }{ + // Fetching the entire object and validating its contents. + 0: { + bucketName: bucketName, + objectName: objectName, + data: bytesData, + dataLen: len(bytesData), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusOK, + }, + // Test Case with invalid accessID. + 1: { + bucketName: bucketName, + objectName: objectName, + data: bytesData, + dataLen: len(bytesData), + accessKey: "Wrong-AccessID", + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusForbidden, + wantAPICode: "InvalidAccessKeyId", + }, + // Test Case with invalid header key X-Amz-Copy-Source. + 2: { + bucketName: bucketName, + objectName: objectName, + headers: copySourceHeader, + data: bytesData, + dataLen: len(bytesData), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusBadRequest, + wantAPICode: "InvalidArgument", + }, + // Test Case with invalid Content-Md5 value + 3: { + bucketName: bucketName, + objectName: objectName, + headers: invalidMD5Header, + data: bytesData, + dataLen: len(bytesData), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusBadRequest, + wantAPICode: "InvalidDigest", + }, + // Test Case with object greater than maximum allowed size. + 4: { + bucketName: bucketName, + objectName: objectName, + data: bytesData, + dataLen: len(bytesData), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + fault: TooBigObject, + expectedRespStatus: http.StatusBadRequest, + wantAPICode: "EntityTooLarge", + }, + // Test Case with missing content length + 5: { + bucketName: bucketName, + objectName: objectName, + data: bytesData, + dataLen: len(bytesData), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + fault: MissingContentLength, + expectedRespStatus: http.StatusLengthRequired, + wantAPICode: "MissingContentLength", + }, + // Test Case with invalid header key X-Amz-Storage-Class + 6: { + bucketName: bucketName, + objectName: objectName, + headers: invalidStorageClassHeader, + data: bytesData, + dataLen: len(bytesData), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusBadRequest, + wantAPICode: "InvalidStorageClass", + }, + + // Invalid crc32 + 7: { + bucketName: bucketName, + objectName: objectName, + headers: map[string]string{"x-amz-checksum-crc32": "123"}, + data: bytesData, + dataLen: len(bytesData), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusBadRequest, + wantAPICode: "InvalidArgument", + }, + // Wrong crc32 + 8: { + bucketName: bucketName, + objectName: objectName, + headers: map[string]string{"x-amz-checksum-crc32": "MTIzNA=="}, + data: bytesData, + dataLen: len(bytesData), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusBadRequest, + wantAPICode: "XAmzContentChecksumMismatch", + }, + // Correct crc32 + 9: { + bucketName: bucketName, + objectName: objectName, + headers: map[string]string{"x-amz-checksum-crc32": checksumData(bytesData, crc32.New(crc32.IEEETable))}, + data: bytesData, + dataLen: len(bytesData), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusOK, + wantHeaders: map[string]string{"x-amz-checksum-crc32": checksumData(bytesData, crc32.New(crc32.IEEETable))}, + }, + // Correct crc32c + 10: { + bucketName: bucketName, + objectName: objectName, + headers: map[string]string{"x-amz-checksum-crc32c": checksumData(bytesData, crc32.New(crc32.MakeTable(crc32.Castagnoli)))}, + data: bytesData, + dataLen: len(bytesData), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusOK, + wantHeaders: map[string]string{"x-amz-checksum-crc32c": checksumData(bytesData, crc32.New(crc32.MakeTable(crc32.Castagnoli)))}, + }, + // CRC32 as CRC32C + 11: { + bucketName: bucketName, + objectName: objectName, + headers: map[string]string{"x-amz-checksum-crc32c": checksumData(bytesData, crc32.New(crc32.IEEETable))}, + data: bytesData, + dataLen: len(bytesData), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusBadRequest, + wantAPICode: "XAmzContentChecksumMismatch", + }, + // SHA1 + 12: { + bucketName: bucketName, + objectName: objectName, + headers: map[string]string{"x-amz-checksum-sha1": checksumData(bytesData, sha1.New())}, + data: bytesData, + dataLen: len(bytesData), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusOK, + wantHeaders: map[string]string{"x-amz-checksum-sha1": checksumData(bytesData, sha1.New())}, + }, + // SHA256 + 13: { + bucketName: bucketName, + objectName: objectName, + headers: map[string]string{"x-amz-checksum-sha256": checksumData(bytesData, sha256.New())}, + data: bytesData, + dataLen: len(bytesData), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusOK, + wantHeaders: map[string]string{"x-amz-checksum-sha256": checksumData(bytesData, sha256.New())}, + }, + } + // Iterating over the cases, fetching the object validating the response. + for i, testCase := range testCases { + var req, reqV2 *http.Request + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + // construct HTTP request for Get Object end point. + req, err = newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", testCase.bucketName, testCase.objectName), + int64(testCase.dataLen), bytes.NewReader(testCase.data), testCase.accessKey, testCase.secretKey, testCase.headers) + if err != nil { + t.Fatalf("Test %d: Failed to create HTTP request for Put Object: %v", i, err) + } + // Add test case specific headers to the request. + addCustomHeaders(req, testCase.headers) + + // Inject faults if specified in testCase.fault + switch testCase.fault { + case MissingContentLength: + req.ContentLength = -1 + req.TransferEncoding = []string{} + case TooBigObject: + req.ContentLength = globalMaxObjectSize + 1 + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler,`func (api objectAPIHandlers) GetObjectHandler` handles the request. + apiRouter.ServeHTTP(rec, req) + // Assert the response code with the expected status. + if rec.Code != testCase.expectedRespStatus { + b, _ := io.ReadAll(rec.Body) + t.Fatalf("Test %d: Expected the response status to be `%d`, but instead found `%d`: %s", i, testCase.expectedRespStatus, rec.Code, string(b)) + } + if testCase.expectedRespStatus != http.StatusOK { + b, err := io.ReadAll(rec.Body) + if err != nil { + t.Fatal(err) + } + var apiErr APIErrorResponse + err = xml.Unmarshal(b, &apiErr) + if err != nil { + t.Fatal(err) + } + gotErr := apiErr.Code + wantErr := testCase.wantAPICode + if gotErr != wantErr { + t.Errorf("test %d: want api error %q, got %q", i, wantErr, gotErr) + } + if testCase.wantHeaders != nil { + for k, v := range testCase.wantHeaders { + got := rec.Header().Get(k) + if got != v { + t.Errorf("Want header %s = %s, got %#v", k, v, rec.Header()) + } + } + } + } + + if testCase.expectedRespStatus == http.StatusOK { + buffer := new(bytes.Buffer) + // Fetch the object to check whether the content is same as the one uploaded via PutObject. + gr, err := obj.GetObjectNInfo(context.Background(), testCase.bucketName, testCase.objectName, nil, nil, opts) + if err != nil { + t.Fatalf("Test %d: %s: Failed to fetch the copied object: %s", i, instanceType, err) + } + if _, err = io.Copy(buffer, gr); err != nil { + gr.Close() + t.Fatalf("Test %d: %s: Failed to fetch the copied object: %s", i, instanceType, err) + } + gr.Close() + if !bytes.Equal(bytesData, buffer.Bytes()) { + t.Errorf("Test %d: %s: Data Mismatch: Data fetched back from the uploaded object doesn't match the original one.", i, instanceType) + } + buffer.Reset() + } + + // Verify response of the V2 signed HTTP request. + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + recV2 := httptest.NewRecorder() + // construct HTTP request for PUT Object endpoint. + reqV2, err = newTestSignedRequestV2(http.MethodPut, getPutObjectURL("", testCase.bucketName, testCase.objectName), + int64(testCase.dataLen), bytes.NewReader(testCase.data), testCase.accessKey, testCase.secretKey, testCase.headers) + if err != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for PutObject: %v", i, instanceType, err) + } + + // Add test case specific headers to the request. + addCustomHeaders(reqV2, testCase.headers) + + // Inject faults if specified in testCase.fault + switch testCase.fault { + case MissingContentLength: + reqV2.ContentLength = -1 + reqV2.TransferEncoding = []string{} + case TooBigObject: + reqV2.ContentLength = globalMaxObjectSize + 1 + } + + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(recV2, reqV2) + if recV2.Code != testCase.expectedRespStatus { + b, _ := io.ReadAll(rec.Body) + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`: %s", i, instanceType, testCase.expectedRespStatus, recV2.Code, string(b)) + } + + if testCase.expectedRespStatus == http.StatusOK { + buffer := new(bytes.Buffer) + // Fetch the object to check whether the content is same as the one uploaded via PutObject. + gr, err := obj.GetObjectNInfo(context.Background(), testCase.bucketName, testCase.objectName, nil, nil, opts) + if err != nil { + t.Fatalf("Test %d: %s: Failed to fetch the copied object: %s", i, instanceType, err) + } + if _, err = io.Copy(buffer, gr); err != nil { + gr.Close() + t.Fatalf("Test %d: %s: Failed to fetch the copied object: %s", i, instanceType, err) + } + gr.Close() + if !bytes.Equal(bytesData, buffer.Bytes()) { + t.Errorf("Test %d: %s: Data Mismatch: Data fetched back from the uploaded object doesn't match the original one.", i, instanceType) + } + buffer.Reset() + + if testCase.wantHeaders != nil { + for k, v := range testCase.wantHeaders { + got := recV2.Header().Get(k) + if got != v { + t.Errorf("Want header %s = %s, got %#v", k, v, recV2.Header()) + } + } + } + } + } + + // Test for Anonymous/unsigned http request. + anonReq, err := newTestRequest(http.MethodPut, getPutObjectURL("", bucketName, objectName), + int64(len("hello")), bytes.NewReader([]byte("hello"))) + if err != nil { + t.Fatalf("MinIO %s: Failed to create an anonymous request for %s/%s: %v", + instanceType, bucketName, objectName, err) + } + + // ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse, + // sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the + // unsigned request goes through and its validated again. + ExecObjectLayerAPIAnonTest(t, obj, "TestAPIPutObjectHandler", bucketName, objectName, instanceType, apiRouter, anonReq, getAnonWriteOnlyObjectPolicy(bucketName, objectName)) + + // HTTP request to test the case of `objectLayer` being set to `nil`. + // There is no need to use an existing bucket or valid input for creating the request, + // since the `objectLayer==nil` check is performed before any other checks inside the handlers. + // The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called. + nilBucket := "dummy-bucket" + nilObject := "dummy-object" + + nilReq, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", nilBucket, nilObject), + 0, nil, "", "", nil) + if err != nil { + t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType) + } + // execute the object layer set to `nil` test. + // `ExecObjectLayerAPINilTest` manages the operation. + ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq) +} + +// Tests sanity of attempting to copying each parts at offsets from an existing +// file and create a new object. Also validates if the written is same as what we +// expected. +func TestAPICopyObjectPartHandlerSanity(t *testing.T) { + defer DetectTestLeak(t)() + ExecExtendedObjectLayerAPITest(t, testAPICopyObjectPartHandlerSanity, []string{"NewMultipart", "CompleteMultipart", "CopyObjectPart"}) +} + +func testAPICopyObjectPartHandlerSanity(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + objectName := "test-object" + var err error + // set of byte data for PutObject. + // object has to be created before running tests for Copy Object. + // this is required even to assert the copied object, + bytesData := []struct { + byteData []byte + }{ + {generateBytesData(6 * humanize.MiByte)}, + } + + // set of inputs for uploading the objects before tests for downloading is done. + putObjectInputs := []struct { + bucketName string + objectName string + contentLength int64 + textData []byte + metaData map[string]string + }{ + // case - 1. + {bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)}, + } + // iterate through the above set of inputs and upload the object. + for i, input := range putObjectInputs { + // uploading the object. + _, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, + mustGetPutObjReader(t, bytes.NewReader(input.textData), input.contentLength, input.metaData[""], ""), ObjectOptions{UserDefined: input.metaData}) + // if object upload fails stop the test. + if err != nil { + t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) + } + } + + testObject := "testobject" + + // Initiate Multipart upload for testing CopyObjectPartHandler. + rec := httptest.NewRecorder() + req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, testObject), + 0, nil, credentials.AccessKey, credentials.SecretKey, nil) + if err != nil { + t.Fatalf("Failed to create HTTP request for NewMultipart Request: %v", err) + } + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("%s: Expected the response status to be `%d`, but instead found `%d`", instanceType, http.StatusOK, rec.Code) + } + decoder := xml.NewDecoder(rec.Body) + multipartResponse := &InitiateMultipartUploadResponse{} + err = decoder.Decode(multipartResponse) + if err != nil { + t.Fatalf("Error decoding the recorded response Body") + } + uploadID := multipartResponse.UploadID + + a := 0 + b := globalMinPartSize + var parts []CompletePart + for partNumber := 1; partNumber <= 2; partNumber++ { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + cpPartURL := getCopyObjectPartURL("", bucketName, testObject, uploadID, fmt.Sprintf("%d", partNumber)) + + // construct HTTP request for copy object. + var req *http.Request + req, err = newTestSignedRequestV4(http.MethodPut, cpPartURL, 0, nil, credentials.AccessKey, credentials.SecretKey, nil) + if err != nil { + t.Fatalf("Test failed to create HTTP request for copy object part: %v", err) + } + + // "X-Amz-Copy-Source" header contains the information about the source bucket and the object to copied. + req.Header.Set("X-Amz-Copy-Source", url.QueryEscape(pathJoin(bucketName, objectName))) + req.Header.Set("X-Amz-Copy-Source-Range", fmt.Sprintf("bytes=%d-%d", a, b)) + + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler, `func (api objectAPIHandlers) CopyObjectHandler` handles the request. + a = globalMinPartSize + 1 + b = len(bytesData[0].byteData) - 1 + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Fatalf("Test failed to create HTTP request for copy %d", rec.Code) + } + + resp := &CopyObjectPartResponse{} + if err = xmlDecoder(rec.Body, resp, rec.Result().ContentLength); err != nil { + t.Fatalf("Test failed to decode XML response: %v", err) + } + + parts = append(parts, CompletePart{ + PartNumber: partNumber, + ETag: canonicalizeETag(resp.ETag), + }) + } + + var completeBytes []byte + // Complete multipart upload parts. + completeUploads := &CompleteMultipartUpload{ + Parts: parts, + } + completeBytes, err = xml.Marshal(completeUploads) + if err != nil { + t.Fatalf("Error XML encoding of parts: %s.", err) + } + // Indicating that all parts are uploaded and initiating CompleteMultipartUpload. + req, err = newTestSignedRequestV4(http.MethodPost, getCompleteMultipartUploadURL("", bucketName, testObject, uploadID), + int64(len(completeBytes)), bytes.NewReader(completeBytes), credentials.AccessKey, credentials.SecretKey, nil) + if err != nil { + t.Fatalf("Failed to create HTTP request for CompleteMultipartUpload: %v", err) + } + + rec = httptest.NewRecorder() + + apiRouter.ServeHTTP(rec, req) + // Assert the response code with the expected status. + if rec.Code != http.StatusOK { + t.Errorf("Test %s: Expected the response status to be `%d`, but instead found `%d`", instanceType, http.StatusOK, rec.Code) + } + + var buf bytes.Buffer + r, err := obj.GetObjectNInfo(context.Background(), bucketName, testObject, nil, nil, ObjectOptions{}) + if err != nil { + t.Fatalf("Test %s: reading completed file failed: %v", instanceType, err) + } + if _, err = io.Copy(&buf, r); err != nil { + r.Close() + t.Fatalf("Test %s: Failed to fetch the copied object: %s", instanceType, err) + } + r.Close() + if !bytes.Equal(buf.Bytes(), bytesData[0].byteData) { + t.Fatalf("Test %s: returned data is not expected corruption detected:", instanceType) + } + + globalCompressConfigMu.Lock() + compressEnabled := globalCompressConfig.Enabled + globalCompressConfigMu.Unlock() + if compressEnabled && !r.ObjInfo.IsCompressed() { + t.Errorf("Test %s: object found to be uncompressed though compression was enabled", instanceType) + } +} + +// Wrapper for calling Copy Object Part API handler tests for both Erasure multiple disks and single node setup. +func TestAPICopyObjectPartHandler(t *testing.T) { + defer DetectTestLeak(t)() + ExecExtendedObjectLayerAPITest(t, testAPICopyObjectPartHandler, []string{"CopyObjectPart"}) +} + +func testAPICopyObjectPartHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + objectName := "test-object" + var err error + opts := ObjectOptions{} + // set of byte data for PutObject. + // object has to be created before running tests for Copy Object. + // this is required even to assert the copied object, + bytesData := []struct { + byteData []byte + }{ + {generateBytesData(6 * humanize.KiByte)}, + } + + // set of inputs for uploading the objects before tests for downloading is done. + putObjectInputs := []struct { + bucketName string + objectName string + contentLength int64 + textData []byte + metaData map[string]string + }{ + // case - 1. + {bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)}, + } + // iterate through the above set of inputs and upload the object. + for i, input := range putObjectInputs { + // uploading the object. + _, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewReader(input.textData), input.contentLength, input.metaData[""], ""), ObjectOptions{UserDefined: input.metaData}) + // if object upload fails stop the test. + if err != nil { + t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) + } + } + + // Initiate Multipart upload for testing PutObjectPartHandler. + testObject := "testobject" + + // PutObjectPart API HTTP Handler has to be tested in isolation, + // that is without any other handler being registered, + // That's why NewMultipartUpload is initiated using ObjectLayer. + res, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, opts) + if err != nil { + // Failed to create NewMultipartUpload, abort. + t.Fatalf("MinIO %s : %s", instanceType, err) + } + uploadID := res.UploadID + + // test cases with inputs and expected result for Copy Object. + testCases := []struct { + bucketName string + copySourceHeader string // data for "X-Amz-Copy-Source" header. Contains the object to be copied in the URL. + copySourceRange string // data for "X-Amz-Copy-Source-Range" header, contains the byte range offsets of data to be copied. + uploadID string // uploadID of the transaction. + invalidPartNumber bool // Sets an invalid multipart. + maximumPartNumber bool // Sets a maximum parts. + accessKey string + secretKey string + // expected output. + expectedRespStatus int + }{ + // Test case - 1, copy part 1 from newObject1, ignore request headers. + { + bucketName: bucketName, + uploadID: uploadID, + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusOK, + }, + + // Test case - 2. + // Test case with invalid source object. + { + bucketName: bucketName, + uploadID: uploadID, + copySourceHeader: url.QueryEscape(SlashSeparator), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusBadRequest, + }, + + // Test case - 3. + // Test case with new object name is same as object to be copied. + // Fail with file not found. + { + bucketName: bucketName, + uploadID: uploadID, + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + testObject), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusNotFound, + }, + + // Test case - 4. + // Test case with valid byte range. + { + bucketName: bucketName, + uploadID: uploadID, + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + copySourceRange: "bytes=500-4096", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusOK, + }, + + // Test case - 5. + // Test case with invalid byte range. + { + bucketName: bucketName, + uploadID: uploadID, + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + copySourceRange: "bytes=6145-", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusBadRequest, + }, + + // Test case - 6. + // Test case with invalid byte range for exceeding source size boundaries. + { + bucketName: bucketName, + uploadID: uploadID, + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + copySourceRange: "bytes=0-6144", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusBadRequest, + }, + + // Test case - 7. + // Test case with object name missing from source. + // fail with BadRequest. + { + bucketName: bucketName, + uploadID: uploadID, + copySourceHeader: url.QueryEscape("//123"), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusBadRequest, + }, + + // Test case - 8. + // Test case with non-existent source file. + // Case for the purpose of failing `api.ObjectAPI.GetObjectInfo`. + // Expecting the response status code to http.StatusNotFound (404). + { + bucketName: bucketName, + uploadID: uploadID, + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + "non-existent-object"), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusNotFound, + }, + + // Test case - 9. + // Test case with non-existent source file. + // Case for the purpose of failing `api.ObjectAPI.PutObjectPart`. + // Expecting the response status code to http.StatusNotFound (404). + { + bucketName: "non-existent-destination-bucket", + uploadID: uploadID, + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusNotFound, + }, + + // Test case - 10. + // Case with invalid AccessKey. + { + bucketName: bucketName, + uploadID: uploadID, + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + accessKey: "Invalid-AccessID", + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusForbidden, + }, + + // Test case - 11. + // Case with non-existent upload id. + { + bucketName: bucketName, + uploadID: "-1", + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusNotFound, + }, + // Test case - 12. + // invalid part number. + { + bucketName: bucketName, + uploadID: uploadID, + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + invalidPartNumber: true, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusOK, + }, + // Test case - 13. + // maximum part number. + { + bucketName: bucketName, + uploadID: uploadID, + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + maximumPartNumber: true, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusOK, + }, + // Test case - 14, copy part 1 from newObject1 with null versionId + { + bucketName: bucketName, + uploadID: uploadID, + copySourceHeader: url.QueryEscape(SlashSeparator+bucketName+SlashSeparator+objectName) + "?versionId=null", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusOK, + }, + // Test case - 15, copy part 1 from newObject1 with non null versionId + { + bucketName: bucketName, + uploadID: uploadID, + copySourceHeader: url.QueryEscape(SlashSeparator+bucketName+SlashSeparator+objectName) + "?versionId=17", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusNotFound, + }, + // Test case - 16, Test case with invalid byte range empty value. + { + bucketName: bucketName, + uploadID: uploadID, + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + copySourceRange: "empty", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusBadRequest, + }, + } + + for i, testCase := range testCases { + var req *http.Request + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + switch { + case !testCase.invalidPartNumber || !testCase.maximumPartNumber: + // construct HTTP request for copy object. + req, err = newTestSignedRequestV4(http.MethodPut, getCopyObjectPartURL("", testCase.bucketName, testObject, testCase.uploadID, "1"), 0, nil, testCase.accessKey, testCase.secretKey, nil) + case testCase.invalidPartNumber: + req, err = newTestSignedRequestV4(http.MethodPut, getCopyObjectPartURL("", testCase.bucketName, testObject, testCase.uploadID, "abc"), 0, nil, testCase.accessKey, testCase.secretKey, nil) + case testCase.maximumPartNumber: + req, err = newTestSignedRequestV4(http.MethodPut, getCopyObjectPartURL("", testCase.bucketName, testObject, testCase.uploadID, "99999"), 0, nil, testCase.accessKey, testCase.secretKey, nil) + } + if err != nil { + t.Fatalf("Test %d: Failed to create HTTP request for copy Object: %v", i+1, err) + } + + // "X-Amz-Copy-Source" header contains the information about the source bucket and the object to copied. + if testCase.copySourceHeader != "" { + req.Header.Set("X-Amz-Copy-Source", testCase.copySourceHeader) + } + if testCase.copySourceRange != "" { + if testCase.copySourceRange == "empty" { + req.Header.Set("X-Amz-Copy-Source-Range", "") // specifically test for S3 errors in this scenario. + } else { + req.Header.Set("X-Amz-Copy-Source-Range", testCase.copySourceRange) + } + } + + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler, `func (api objectAPIHandlers) CopyObjectHandler` handles the request. + apiRouter.ServeHTTP(rec, req) + // Assert the response code with the expected status. + if rec.Code != testCase.expectedRespStatus { + t.Fatalf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) + } + if rec.Code == http.StatusOK { + // See if the new part has been uploaded. + // testing whether the copy was successful. + var results ListPartsInfo + results, err = obj.ListObjectParts(context.Background(), testCase.bucketName, testObject, testCase.uploadID, 0, 1, ObjectOptions{}) + if err != nil { + t.Fatalf("Test %d: %s: Failed to look for copied object part: %s", i+1, instanceType, err) + } + if len(results.Parts) != 1 { + t.Fatalf("Test %d: %s: Expected only one entry returned %d entries", i+1, instanceType, len(results.Parts)) + } + } + } + + // HTTP request for testing when `ObjectLayer` is set to `nil`. + // There is no need to use an existing bucket and valid input for creating the request + // since the `objectLayer==nil` check is performed before any other checks inside the handlers. + // The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called. + nilBucket := "dummy-bucket" + nilObject := "dummy-object" + + nilReq, err := newTestSignedRequestV4(http.MethodPut, getCopyObjectPartURL("", nilBucket, nilObject, "0", "0"), + 0, bytes.NewReader([]byte("testNilObjLayer")), "", "", nil) + if err != nil { + t.Errorf("MinIO %s: Failed to create http request for testing the response when object Layer is set to `nil`.", instanceType) + } + + // Below is how CopyObjectPartHandler is registered. + // bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(api.CopyObjectPartHandler).Queries("partNumber", "{partNumber:[0-9]+}", "uploadId", "{uploadId:.*}") + // Its necessary to set the "X-Amz-Copy-Source" header for the request to be accepted by the handler. + nilReq.Header.Set("X-Amz-Copy-Source", url.QueryEscape(SlashSeparator+nilBucket+SlashSeparator+nilObject)) + + // execute the object layer set to `nil` test. + // `ExecObjectLayerAPINilTest` manages the operation. + ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq) +} + +// Wrapper for calling Copy Object API handler tests for both Erasure multiple disks and single node setup. +func TestAPICopyObjectHandler(t *testing.T) { + defer DetectTestLeak(t)() + ExecExtendedObjectLayerAPITest(t, testAPICopyObjectHandler, []string{"CopyObject", "PutObject"}) +} + +func testAPICopyObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + objectName := "test?object" // use file with ? to test URL parsing... + if runtime.GOOS == "windows" { + objectName = "test-object" // ...except on Windows + } + // object used for anonymous HTTP request test. + anonObject := "anon-object" + var err error + opts := ObjectOptions{} + // set of byte data for PutObject. + // object has to be created before running tests for Copy Object. + // this is required even to assert the copied object, + bytesData := []struct { + byteData []byte + md5sum string + }{ + {byteData: generateBytesData(6 * humanize.KiByte)}, + } + h := md5.New() + h.Write(bytesData[0].byteData) + bytesData[0].md5sum = hex.EncodeToString(h.Sum(nil)) + + buffers := []*bytes.Buffer{ + new(bytes.Buffer), + new(bytes.Buffer), + } + bucketInfo, err := obj.GetBucketInfo(context.Background(), bucketName, BucketOptions{}) + if err != nil { + t.Fatalf("Test -1: %s: Failed to get bucket info: %s", instanceType, err) + } + // set of inputs for uploading the objects before tests for downloading is done. + putObjectInputs := []struct { + bucketName string + objectName string + contentLength int64 + textData []byte + md5sum string + metaData map[string]string + }{ + // case - 1. + {bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, bytesData[0].md5sum, make(map[string]string)}, + + // case - 2. + // used for anonymous HTTP request test. + {bucketName, anonObject, int64(len(bytesData[0].byteData)), bytesData[0].byteData, bytesData[0].md5sum, make(map[string]string)}, + } + // iterate through the above set of inputs and upload the object. + for i, input := range putObjectInputs { + rec := httptest.NewRecorder() + req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", input.bucketName, input.objectName), + input.contentLength, bytes.NewReader(input.textData), credentials.AccessKey, credentials.SecretKey, nil) + if err != nil { + t.Fatalf("Test %d: Failed to create HTTP request for Put Object: %v", i, err) + } + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + b, err := io.ReadAll(rec.Body) + if err != nil { + t.Fatal(err) + } + var apiErr APIErrorResponse + err = xml.Unmarshal(b, &apiErr) + if err != nil { + t.Fatal(err) + } + gotErr := apiErr.Code + t.Errorf("test %d: want api got %q", i, gotErr) + } + } + // test cases with inputs and expected result for Copy Object. + testCases := []struct { + bucketName string + newObjectName string // name of the newly copied object. + copySourceHeader string // data for "X-Amz-Copy-Source" header. Contains the object to be copied in the URL. + copyModifiedHeader string // data for "X-Amz-Copy-Source-If-Modified-Since" header + copyUnmodifiedHeader string // data for "X-Amz-Copy-Source-If-Unmodified-Since" header + copySourceSame bool + metadataGarbage bool + metadataReplace bool + metadataCopy bool + metadata map[string]string + accessKey string + secretKey string + // expected output. + expectedRespStatus int + }{ + 0: { + expectedRespStatus: http.StatusMethodNotAllowed, + }, + // Test case - 1, copy metadata from newObject1, ignore request headers. + 1: { + bucketName: bucketName, + newObjectName: "newObject1", + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + metadata: map[string]string{ + "Content-Type": "application/json", + }, + expectedRespStatus: http.StatusOK, + }, + + // Test case - 2. + // Test case with invalid source object. + 2: { + bucketName: bucketName, + newObjectName: "newObject1", + copySourceHeader: url.QueryEscape(SlashSeparator), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusBadRequest, + }, + // Test case - 3. + // Test case with invalid source object. + 3: { + bucketName: bucketName, + newObjectName: "dir//newObject1", + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusBadRequest, + }, + // Test case - 4. + // Test case with new object name is same as object to be copied. + 4: { + bucketName: bucketName, + newObjectName: objectName, + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + accessKey: credentials.AccessKey, + copySourceSame: true, + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusBadRequest, + }, + + // Test case - 5. + // Test case with new object name is same as object to be copied. + // But source copy is without leading slash + 5: { + bucketName: bucketName, + newObjectName: objectName, + copySourceSame: true, + copySourceHeader: url.QueryEscape(bucketName + SlashSeparator + objectName), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusBadRequest, + }, + + // Test case - 6. + // Test case with new object name is same as object to be copied + // but metadata is updated. + 6: { + bucketName: bucketName, + newObjectName: objectName, + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + metadata: map[string]string{ + "Content-Type": "application/json", + }, + metadataReplace: true, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusOK, + }, + + // Test case - 7. + // Test case with invalid metadata-directive. + 7: { + bucketName: bucketName, + newObjectName: "newObject1", + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + metadata: map[string]string{ + "Content-Type": "application/json", + }, + metadataGarbage: true, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusBadRequest, + }, + + // Test case - 8. + // Test case with new object name is same as object to be copied + // fail with BadRequest. + 8: { + bucketName: bucketName, + newObjectName: objectName, + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + metadata: map[string]string{ + "Content-Type": "application/json", + }, + copySourceSame: true, + metadataCopy: true, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusBadRequest, + }, + + // Test case - 9. + // Test case with non-existent source file. + // Case for the purpose of failing `api.ObjectAPI.GetObjectInfo`. + // Expecting the response status code to http.StatusNotFound (404). + 9: { + bucketName: bucketName, + newObjectName: objectName, + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + "non-existent-object"), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusNotFound, + }, + + // Test case - 10. + // Test case with non-existent source file. + // Case for the purpose of failing `api.ObjectAPI.PutObject`. + // Expecting the response status code to http.StatusNotFound (404). + 10: { + bucketName: "non-existent-destination-bucket", + newObjectName: objectName, + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusNotFound, + }, + + // Test case - 11. + // Case with invalid AccessKey. + 11: { + bucketName: bucketName, + newObjectName: objectName, + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + accessKey: "Invalid-AccessID", + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusForbidden, + }, + // Test case - 12, copy metadata from newObject1 with satisfying modified header. + 12: { + bucketName: bucketName, + newObjectName: "newObject1", + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + copyModifiedHeader: "Mon, 02 Jan 2006 15:04:05 GMT", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusOK, + }, + // Test case - 13, copy metadata from newObject1 with unsatisfying modified header. + 13: { + bucketName: bucketName, + newObjectName: "newObject1", + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + copyModifiedHeader: "Mon, 02 Jan 2217 15:04:05 GMT", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusPreconditionFailed, + }, + // Test case - 14, copy metadata from newObject1 with wrong modified header format + 14: { + bucketName: bucketName, + newObjectName: "newObject1", + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + copyModifiedHeader: "Mon, 02 Jan 2217 15:04:05 +00:00", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusOK, + }, + // Test case - 15, copy metadata from newObject1 with satisfying unmodified header. + 15: { + bucketName: bucketName, + newObjectName: "newObject1", + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + copyUnmodifiedHeader: "Mon, 02 Jan 2217 15:04:05 GMT", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusOK, + }, + // Test case - 16, copy metadata from newObject1 with unsatisfying unmodified header. + 16: { + bucketName: bucketName, + newObjectName: "newObject1", + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + copyUnmodifiedHeader: "Mon, 02 Jan 2007 15:04:05 GMT", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusPreconditionFailed, + }, + // Test case - 17, copy metadata from newObject1 with incorrect unmodified header format. + 17: { + bucketName: bucketName, + newObjectName: "newObject1", + copySourceHeader: url.QueryEscape(SlashSeparator + bucketName + SlashSeparator + objectName), + copyUnmodifiedHeader: "Mon, 02 Jan 2007 15:04:05 +00:00", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusOK, + }, + // Test case - 18, copy metadata from newObject1 with null versionId + 18: { + bucketName: bucketName, + newObjectName: "newObject1", + copySourceHeader: url.QueryEscape(SlashSeparator+bucketName+SlashSeparator+objectName) + "?versionId=null", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusOK, + }, + // Test case - 19, copy metadata from newObject1 with non null versionId + 19: { + bucketName: bucketName, + newObjectName: "newObject1", + copySourceHeader: url.QueryEscape(SlashSeparator+bucketName+SlashSeparator+objectName) + "?versionId=17", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusNotFound, + }, + } + + for i, testCase := range testCases { + if bucketInfo.Versioning { + if strings.Contains(testCase.copySourceHeader, "versionId=null") { + testCase.expectedRespStatus = http.StatusNotFound + } + } + values := url.Values{} + if testCase.expectedRespStatus == http.StatusOK { + r, err := obj.GetObjectNInfo(context.Background(), testCase.bucketName, objectName, nil, nil, opts) + if err != nil { + t.Fatalf("Test %d: %s reading completed file failed: %v", i, instanceType, err) + } + r.Close() + if r.ObjInfo.VersionID != "" { + values.Set(xhttp.VersionID, r.ObjInfo.VersionID) + } + } + var req *http.Request + var reqV2 *http.Request + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + // construct HTTP request for copy object. + req, err = newTestSignedRequestV4(http.MethodPut, getCopyObjectURL("", testCase.bucketName, testCase.newObjectName), + 0, nil, testCase.accessKey, testCase.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: Failed to create HTTP request for copy Object: %v", i, err) + } + // "X-Amz-Copy-Source" header contains the information about the source bucket and the object to copied. + if testCase.copySourceHeader != "" { + if values.Encode() != "" && !strings.Contains(testCase.copySourceHeader, "?") { + req.Header.Set("X-Amz-Copy-Source", testCase.copySourceHeader+"?"+values.Encode()) + } else { + req.Header.Set("X-Amz-Copy-Source", testCase.copySourceHeader) + } + } + if testCase.copyModifiedHeader != "" { + req.Header.Set("X-Amz-Copy-Source-If-Modified-Since", testCase.copyModifiedHeader) + } + if testCase.copyUnmodifiedHeader != "" { + req.Header.Set("X-Amz-Copy-Source-If-Unmodified-Since", testCase.copyUnmodifiedHeader) + } + // Add custom metadata. + for k, v := range testCase.metadata { + req.Header.Set(k, v) + } + if testCase.metadataReplace { + req.Header.Set("X-Amz-Metadata-Directive", "REPLACE") + } + if testCase.metadataCopy { + req.Header.Set("X-Amz-Metadata-Directive", "COPY") + } + if testCase.metadataGarbage { + req.Header.Set("X-Amz-Metadata-Directive", "Unknown") + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler, `func (api objectAPIHandlers) CopyObjectHandler` handles the request. + apiRouter.ServeHTTP(rec, req) + // Assert the response code with the expected status. + if rec.Code != testCase.expectedRespStatus { + if testCase.copySourceSame { + // encryption will rotate creds, so fail only for non-encryption scenario. + if GlobalKMS == nil { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i, instanceType, testCase.expectedRespStatus, rec.Code) + continue + } + } else { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i, instanceType, testCase.expectedRespStatus, rec.Code) + continue + } + } + if rec.Code == http.StatusOK { + var cpObjResp CopyObjectResponse + if err = xml.Unmarshal(rec.Body.Bytes(), &cpObjResp); err != nil { + t.Fatalf("Test %d: %s: Failed to parse the CopyObjectResult response: %s", i, instanceType, err) + } + + // See if the new object is formed. + // testing whether the copy was successful. + // Note that this goes directly to the file system, + // so encryption/compression may interfere at some point. + buffers[0].Reset() + r, err := obj.GetObjectNInfo(context.Background(), testCase.bucketName, testCase.newObjectName, nil, nil, opts) + if err != nil { + t.Fatalf("Test %d: %s reading completed file failed: %v", i, instanceType, err) + } + if _, err = io.Copy(buffers[0], r); err != nil { + r.Close() + t.Fatalf("Test %d %s: Failed to fetch the copied object: %s", i, instanceType, err) + } + r.Close() + if !bytes.Equal(bytesData[0].byteData, buffers[0].Bytes()) { + t.Errorf("Test %d: %s: Data Mismatch: Data fetched back from the copied object doesn't match the original one.", i, instanceType) + } + + globalCompressConfigMu.Lock() + compressEnabled := globalCompressConfig.Enabled + globalCompressConfigMu.Unlock() + if compressEnabled && !r.ObjInfo.IsCompressed() { + t.Errorf("Test %d %s: object found to be uncompressed though compression was enabled", i, instanceType) + } + } + + // Verify response of the V2 signed HTTP request. + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + recV2 := httptest.NewRecorder() + + reqV2, err = newTestRequest(http.MethodPut, getCopyObjectURL("", testCase.bucketName, testCase.newObjectName), 0, nil) + if err != nil { + t.Fatalf("Test %d: Failed to create HTTP request for copy Object: %v", i, err) + } + // "X-Amz-Copy-Source" header contains the information about the source bucket and the object to copied. + if testCase.copySourceHeader != "" { + reqV2.Header.Set("X-Amz-Copy-Source", testCase.copySourceHeader) + } + if testCase.copyModifiedHeader != "" { + reqV2.Header.Set("X-Amz-Copy-Source-If-Modified-Since", testCase.copyModifiedHeader) + } + if testCase.copyUnmodifiedHeader != "" { + reqV2.Header.Set("X-Amz-Copy-Source-If-Unmodified-Since", testCase.copyUnmodifiedHeader) + } + + // Add custom metadata. + for k, v := range testCase.metadata { + reqV2.Header.Set(k, v+"+x") + } + if testCase.metadataReplace { + reqV2.Header.Set("X-Amz-Metadata-Directive", "REPLACE") + } + if testCase.metadataCopy { + reqV2.Header.Set("X-Amz-Metadata-Directive", "COPY") + } + if testCase.metadataGarbage { + reqV2.Header.Set("X-Amz-Metadata-Directive", "Unknown") + } + + err = signRequestV2(reqV2, testCase.accessKey, testCase.secretKey) + if err != nil { + t.Fatalf("Failed to V2 Sign the HTTP request: %v.", err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(recV2, reqV2) + if recV2.Code != testCase.expectedRespStatus { + if testCase.copySourceSame { + // encryption will rotate creds, so fail only for non-encryption scenario. + if GlobalKMS == nil { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i, instanceType, testCase.expectedRespStatus, recV2.Code) + } + } else { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, recV2.Code) + } + } + } + + // HTTP request to test the case of `objectLayer` being set to `nil`. + // There is no need to use an existing bucket or valid input for creating the request, + // since the `objectLayer==nil` check is performed before any other checks inside the handlers. + // The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called. + nilBucket := "dummy-bucket" + nilObject := "dummy-object" + + nilReq, err := newTestSignedRequestV4(http.MethodPut, getCopyObjectURL("", nilBucket, nilObject), + 0, nil, "", "", nil) + + // Below is how CopyObjectHandler is registered. + // bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?") + // Its necessary to set the "X-Amz-Copy-Source" header for the request to be accepted by the handler. + nilReq.Header.Set("X-Amz-Copy-Source", url.QueryEscape(SlashSeparator+nilBucket+SlashSeparator+nilObject)) + if err != nil { + t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType) + } + + // execute the object layer set to `nil` test. + // `ExecObjectLayerAPINilTest` manages the operation. + ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq) +} + +// Wrapper for calling NewMultipartUpload tests for both Erasure multiple disks and single node setup. +// First register the HTTP handler for NewMultipartUpload, then a HTTP request for NewMultipart upload is made. +// The UploadID from the response body is parsed and its existence is asserted with an attempt to ListParts using it. +func TestAPINewMultipartHandler(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPINewMultipartHandler, endpoints: []string{"NewMultipart"}}) +} + +func testAPINewMultipartHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + objectName := "test-object-new-multipart" + rec := httptest.NewRecorder() + // construct HTTP request for NewMultipart upload. + req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, objectName), + 0, nil, credentials.AccessKey, credentials.SecretKey, nil) + if err != nil { + t.Fatalf("Failed to create HTTP request for NewMultipart Request: %v", err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to executes the registered handler. + apiRouter.ServeHTTP(rec, req) + // Assert the response code with the expected status. + if rec.Code != http.StatusOK { + t.Fatalf("%s: Expected the response status to be `%d`, but instead found `%d`", instanceType, http.StatusOK, rec.Code) + } + + // decode the response body. + decoder := xml.NewDecoder(rec.Body) + multipartResponse := &InitiateMultipartUploadResponse{} + + err = decoder.Decode(multipartResponse) + if err != nil { + t.Fatalf("Error decoding the recorded response Body") + } + // verify the uploadID my making an attempt to list parts. + _, err = obj.ListObjectParts(context.Background(), bucketName, objectName, multipartResponse.UploadID, 0, 1, ObjectOptions{}) + if err != nil { + t.Fatalf("Invalid UploadID: %s", err) + } + + // Testing the response for Invalid AccessID. + // Forcing the signature check to fail. + rec = httptest.NewRecorder() + // construct HTTP request for NewMultipart upload. + // Setting an invalid accessID. + req, err = newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, objectName), + 0, nil, "Invalid-AccessID", credentials.SecretKey, nil) + if err != nil { + t.Fatalf("Failed to create HTTP request for NewMultipart Request: %v", err) + } + + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP method to execute the logic of the handler. + // Call the ServeHTTP to executes the registered handler. + apiRouter.ServeHTTP(rec, req) + // Assert the response code with the expected status. + if rec.Code != http.StatusForbidden { + t.Fatalf("%s: Expected the response status to be `%d`, but instead found `%d`", instanceType, http.StatusForbidden, rec.Code) + } + + // Verify response of the V2 signed HTTP request. + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + recV2 := httptest.NewRecorder() + // construct HTTP request for NewMultipartUpload endpoint. + reqV2, err := newTestSignedRequestV2(http.MethodPost, getNewMultipartURL("", bucketName, objectName), + 0, nil, credentials.AccessKey, credentials.SecretKey, nil) + if err != nil { + t.Fatalf("Failed to create HTTP request for NewMultipart Request: %v", err) + } + + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(recV2, reqV2) + // Assert the response code with the expected status. + if recV2.Code != http.StatusOK { + t.Fatalf("%s: Expected the response status to be `%d`, but instead found `%d`", instanceType, http.StatusOK, recV2.Code) + } + // decode the response body. + decoder = xml.NewDecoder(recV2.Body) + multipartResponse = &InitiateMultipartUploadResponse{} + + err = decoder.Decode(multipartResponse) + if err != nil { + t.Fatalf("Error decoding the recorded response Body") + } + // verify the uploadID my making an attempt to list parts. + _, err = obj.ListObjectParts(context.Background(), bucketName, objectName, multipartResponse.UploadID, 0, 1, ObjectOptions{}) + if err != nil { + t.Fatalf("Invalid UploadID: %s", err) + } + + // Testing the response for invalid AccessID. + // Forcing the V2 signature check to fail. + recV2 = httptest.NewRecorder() + // construct HTTP request for NewMultipartUpload endpoint. + // Setting invalid AccessID. + reqV2, err = newTestSignedRequestV2(http.MethodPost, getNewMultipartURL("", bucketName, objectName), + 0, nil, "Invalid-AccessID", credentials.SecretKey, nil) + if err != nil { + t.Fatalf("Failed to create HTTP request for NewMultipart Request: %v", err) + } + + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(recV2, reqV2) + // Assert the response code with the expected status. + if recV2.Code != http.StatusForbidden { + t.Fatalf("%s: Expected the response status to be `%d`, but instead found `%d`", instanceType, http.StatusForbidden, recV2.Code) + } + + // Test for Anonymous/unsigned http request. + anonReq, err := newTestRequest(http.MethodPost, getNewMultipartURL("", bucketName, objectName), 0, nil) + if err != nil { + t.Fatalf("MinIO %s: Failed to create an anonymous request for %s/%s: %v", + instanceType, bucketName, objectName, err) + } + + // ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse, + // sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the + // unsigned request goes through and its validated again. + ExecObjectLayerAPIAnonTest(t, obj, "TestAPINewMultipartHandler", bucketName, objectName, instanceType, apiRouter, anonReq, getAnonWriteOnlyObjectPolicy(bucketName, objectName)) + + // HTTP request to test the case of `objectLayer` being set to `nil`. + // There is no need to use an existing bucket or valid input for creating the request, + // since the `objectLayer==nil` check is performed before any other checks inside the handlers. + // The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called. + nilBucket := "dummy-bucket" + nilObject := "dummy-object" + + nilReq, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", nilBucket, nilObject), + 0, nil, "", "", nil) + if err != nil { + t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType) + } + // execute the object layer set to `nil` test. + // `ExecObjectLayerAPINilTest` manages the operation. + ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq) +} + +// Wrapper for calling NewMultipartUploadParallel tests for both Erasure multiple disks and single node setup. +// The objective of the test is to initialte multipart upload on the same object 10 times concurrently, +// The UploadID from the response body is parsed and its existence is asserted with an attempt to ListParts using it. +func TestAPINewMultipartHandlerParallel(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPINewMultipartHandlerParallel, endpoints: []string{"NewMultipart"}}) +} + +func testAPINewMultipartHandlerParallel(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + // used for storing the uploadID's parsed on concurrent HTTP requests for NewMultipart upload on the same object. + testUploads := struct { + sync.Mutex + uploads []string + }{} + + objectName := "test-object-new-multipart-parallel" + var wg sync.WaitGroup + for range 10 { + wg.Add(1) + // Initiate NewMultipart upload on the same object 10 times concurrrently. + go func() { + defer wg.Done() + rec := httptest.NewRecorder() + // construct HTTP request NewMultipartUpload. + req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, objectName), 0, nil, credentials.AccessKey, credentials.SecretKey, nil) + if err != nil { + t.Errorf("Failed to create HTTP request for NewMultipart request: %v", err) + return + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to executes the registered handler. + apiRouter.ServeHTTP(rec, req) + // Assert the response code with the expected status. + if rec.Code != http.StatusOK { + t.Errorf("MinIO %s: Expected the response status to be `%d`, but instead found `%d`", instanceType, http.StatusOK, rec.Code) + return + } + // decode the response body. + decoder := xml.NewDecoder(rec.Body) + multipartResponse := &InitiateMultipartUploadResponse{} + + err = decoder.Decode(multipartResponse) + if err != nil { + t.Errorf("MinIO %s: Error decoding the recorded response Body", instanceType) + return + } + // push the obtained upload ID from the response into the array. + testUploads.Lock() + testUploads.uploads = append(testUploads.uploads, multipartResponse.UploadID) + testUploads.Unlock() + }() + } + // Wait till all go routines finishes execution. + wg.Wait() + // Validate the upload ID by an attempt to list parts using it. + for _, uploadID := range testUploads.uploads { + _, err := obj.ListObjectParts(context.Background(), bucketName, objectName, uploadID, 0, 1, ObjectOptions{}) + if err != nil { + t.Fatalf("Invalid UploadID: %s", err) + } + } +} + +// The UploadID from the response body is parsed and its existence is asserted with an attempt to ListParts using it. +func TestAPICompleteMultipartHandler(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPICompleteMultipartHandler, endpoints: []string{"CompleteMultipart"}}) +} + +func testAPICompleteMultipartHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + var err error + + var opts ObjectOptions + // object used for the test. + objectName := "test-object-new-multipart" + + // upload IDs collected. + var uploadIDs []string + + for range 2 { + // initiate new multipart uploadID. + res, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, opts) + if err != nil { + // Failed to create NewMultipartUpload, abort. + t.Fatalf("MinIO %s : %s", instanceType, err) + } + + uploadIDs = append(uploadIDs, res.UploadID) + } + + // Parts with size greater than 5 MiB. + // Generating a 6 MiB byte array. + validPart := bytes.Repeat([]byte("abcdef"), 1*humanize.MiByte) + validPartMD5 := getMD5Hash(validPart) + // Create multipart parts. + // Need parts to be uploaded before CompleteMultiPartUpload can be called tested. + parts := []struct { + bucketName string + objName string + uploadID string + PartID int + inputReaderData string + inputMd5 string + inputDataSize int64 + }{ + // Case 1-4. + // Creating sequence of parts for same uploadID. + {bucketName, objectName, uploadIDs[0], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd"))}, + {bucketName, objectName, uploadIDs[0], 2, "efgh", "1f7690ebdd9b4caf8fab49ca1757bf27", int64(len("efgh"))}, + {bucketName, objectName, uploadIDs[0], 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("abcd"))}, + {bucketName, objectName, uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd"))}, + // Part with size larger than 5 MiB. + {bucketName, objectName, uploadIDs[0], 5, string(validPart), validPartMD5, int64(len(validPart))}, + {bucketName, objectName, uploadIDs[0], 6, string(validPart), validPartMD5, int64(len(validPart))}, + + // Part with size larger than 5 MiB. + // Parts uploaded for anonymous/unsigned API handler test. + {bucketName, objectName, uploadIDs[1], 1, string(validPart), validPartMD5, int64(len(validPart))}, + {bucketName, objectName, uploadIDs[1], 2, string(validPart), validPartMD5, int64(len(validPart))}, + } + // Iterating over creatPartCases to generate multipart chunks. + for _, part := range parts { + _, err = obj.PutObjectPart(context.Background(), part.bucketName, part.objName, part.uploadID, part.PartID, + mustGetPutObjReader(t, strings.NewReader(part.inputReaderData), part.inputDataSize, part.inputMd5, ""), opts) + if err != nil { + t.Fatalf("%s : %s", instanceType, err) + } + } + // Parts to be sent as input for CompleteMultipartUpload. + inputParts := []struct { + parts []CompletePart + }{ + // inputParts - 0. + // Case for replicating ETag mismatch. + { + []CompletePart{ + {ETag: "abcd", PartNumber: 1}, + }, + }, + // inputParts - 1. + // should error out with part too small. + { + []CompletePart{ + {ETag: "e2fc714c4727ee9395f324cd2e7f331f", PartNumber: 1}, + {ETag: "1f7690ebdd9b4caf8fab49ca1757bf27", PartNumber: 2}, + }, + }, + // inputParts - 2. + // Case with invalid Part number. + { + []CompletePart{ + {ETag: "e2fc714c4727ee9395f324cd2e7f331f", PartNumber: 10}, + }, + }, + // inputParts - 3. + // Case with valid parts,but parts are unsorted. + // Part size greater than 5 MiB. + { + []CompletePart{ + {ETag: validPartMD5, PartNumber: 6}, + {ETag: validPartMD5, PartNumber: 5}, + }, + }, + // inputParts - 4. + // Case with valid part. + // Part size greater than 5 MiB. + { + []CompletePart{ + {ETag: validPartMD5, PartNumber: 5}, + {ETag: validPartMD5, PartNumber: 6}, + }, + }, + + // inputParts - 5. + // Used for the case of testing for anonymous API request. + // Part size greater than 5 MiB. + { + []CompletePart{ + {ETag: validPartMD5, PartNumber: 1}, + {ETag: validPartMD5, PartNumber: 2}, + }, + }, + } + + // on successful complete multipart operation the s3MD5 for the parts uploaded will be returned. + s3MD5 := getCompleteMultipartMD5(inputParts[3].parts) + + // generating the response body content for the success case. + successResponse := generateCompleteMultipartUploadResponse(bucketName, objectName, getGetObjectURL("", bucketName, objectName), ObjectInfo{ETag: s3MD5}, nil) + encodedSuccessResponse := encodeResponse(successResponse) + + ctx := context.Background() + + testCases := []struct { + bucket string + object string + uploadID string + parts []CompletePart + accessKey string + secretKey string + // Expected output of CompleteMultipartUpload. + expectedContent []byte + // Expected HTTP Response status. + expectedRespStatus int + }{ + // Test case - 1. + // Upload and PartNumber exists, But a deliberate ETag mismatch is introduced. + { + bucket: bucketName, + object: objectName, + uploadID: uploadIDs[0], + parts: inputParts[0].parts, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedContent: encodeResponse(getAPIErrorResponse(ctx, + toAPIError(ctx, InvalidPart{}), + getGetObjectURL("", bucketName, objectName), "", "")), + expectedRespStatus: http.StatusBadRequest, + }, + // Test case - 2. + // No parts specified in CompletePart{}. + // Should return ErrMalformedXML in the response body. + { + bucket: bucketName, + object: objectName, + uploadID: uploadIDs[0], + parts: []CompletePart{}, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedContent: encodeResponse(getAPIErrorResponse(ctx, + getAPIError(ErrMalformedXML), + getGetObjectURL("", bucketName, objectName), "", "")), + expectedRespStatus: http.StatusBadRequest, + }, + // Test case - 3. + // Non-Existent uploadID. + // 404 Not Found response status expected. + { + bucket: bucketName, + object: objectName, + uploadID: "abc", + parts: inputParts[0].parts, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedContent: encodeResponse(getAPIErrorResponse(ctx, + toAPIError(ctx, InvalidUploadID{UploadID: "abc"}), + getGetObjectURL("", bucketName, objectName), "", "")), + expectedRespStatus: http.StatusNotFound, + }, + // Test case - 4. + // Case with part size being less than minimum allowed size. + { + bucket: bucketName, + object: objectName, + uploadID: uploadIDs[0], + parts: inputParts[1].parts, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedContent: encodeResponse(getAPIErrorResponse(ctx, + toAPIError(ctx, PartTooSmall{PartNumber: 1}), + getGetObjectURL("", bucketName, objectName), "", "")), + expectedRespStatus: http.StatusBadRequest, + }, + // Test case - 5. + // TestCase with invalid Part Number. + { + bucket: bucketName, + object: objectName, + uploadID: uploadIDs[0], + parts: inputParts[2].parts, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedContent: encodeResponse(getAPIErrorResponse(ctx, + toAPIError(ctx, InvalidPart{}), + getGetObjectURL("", bucketName, objectName), "", "")), + expectedRespStatus: http.StatusBadRequest, + }, + // Test case - 6. + // Parts are not sorted according to the part number. + // This should return ErrInvalidPartOrder in the response body. + { + bucket: bucketName, + object: objectName, + uploadID: uploadIDs[0], + parts: inputParts[3].parts, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedContent: encodeResponse(getAPIErrorResponse(ctx, + getAPIError(ErrInvalidPartOrder), + getGetObjectURL("", bucketName, objectName), "", "")), + expectedRespStatus: http.StatusBadRequest, + }, + // Test case - 7. + // Test case with proper parts. + // Should succeeded and the content in the response body is asserted. + { + bucket: bucketName, + object: objectName, + uploadID: uploadIDs[0], + parts: inputParts[4].parts, + accessKey: "Invalid-AccessID", + secretKey: credentials.SecretKey, + + expectedContent: encodeResponse(getAPIErrorResponse(ctx, + getAPIError(ErrInvalidAccessKeyID), + getGetObjectURL("", bucketName, objectName), "", "")), + expectedRespStatus: http.StatusForbidden, + }, + // Test case - 8. + // Test case with proper parts. + // Should succeeded and the content in the response body is asserted. + { + bucket: bucketName, + object: objectName, + uploadID: uploadIDs[0], + parts: inputParts[4].parts, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedContent: encodedSuccessResponse, + expectedRespStatus: http.StatusOK, + }, + } + + for i, testCase := range testCases { + var req *http.Request + var completeBytes, actualContent []byte + // Complete multipart upload parts. + completeUploads := &CompleteMultipartUpload{ + Parts: testCase.parts, + } + completeBytes, err = xml.Marshal(completeUploads) + if err != nil { + t.Fatalf("Error XML encoding of parts: %s.", err) + } + // Indicating that all parts are uploaded and initiating CompleteMultipartUpload. + req, err = newTestSignedRequestV4(http.MethodPost, getCompleteMultipartUploadURL("", bucketName, objectName, testCase.uploadID), + int64(len(completeBytes)), bytes.NewReader(completeBytes), testCase.accessKey, testCase.secretKey, nil) + if err != nil { + t.Fatalf("Failed to create HTTP request for CompleteMultipartUpload: %v", err) + } + + rec := httptest.NewRecorder() + + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to executes the registered handler. + apiRouter.ServeHTTP(rec, req) + // Assert the response code with the expected status. + if rec.Code != testCase.expectedRespStatus { + t.Errorf("Case %d: MinIO %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) + } + + // read the response body. + actualContent, err = io.ReadAll(rec.Body) + if err != nil { + t.Fatalf("Test %d : MinIO %s: Failed parsing response body: %v", i+1, instanceType, err) + } + + if rec.Code == http.StatusOK { + // Verify whether the bucket obtained object is same as the one created. + if !bytes.Equal(testCase.expectedContent, actualContent) { + t.Errorf("Test %d : MinIO %s: CompleteMultipart response content differs from expected value. got %s, expected %s", i+1, instanceType, + string(actualContent), string(testCase.expectedContent)) + } + continue + } + + actualError := &APIErrorResponse{} + if err = xml.Unmarshal(actualContent, actualError); err != nil { + t.Errorf("MinIO %s: error response failed to parse error XML", instanceType) + } + + if actualError.BucketName != bucketName { + t.Errorf("MinIO %s: error response bucket name differs from expected value", instanceType) + } + + if actualError.Key != objectName { + t.Errorf("MinIO %s: error response object name (%s) differs from expected value (%s)", instanceType, actualError.Key, objectName) + } + } + + // Testing for anonymous API request. + var completeBytes []byte + // Complete multipart upload parts. + completeUploads := &CompleteMultipartUpload{ + Parts: inputParts[5].parts, + } + completeBytes, err = xml.Marshal(completeUploads) + if err != nil { + t.Fatalf("Error XML encoding of parts: %s.", err) + } + + // create unsigned HTTP request for CompleteMultipart upload. + anonReq, err := newTestRequest(http.MethodPost, getCompleteMultipartUploadURL("", bucketName, objectName, uploadIDs[1]), + int64(len(completeBytes)), bytes.NewReader(completeBytes)) + if err != nil { + t.Fatalf("MinIO %s: Failed to create an anonymous request for %s/%s: %v", + instanceType, bucketName, objectName, err) + } + + // ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse, + // sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the + // unsigned request goes through and its validated again. + ExecObjectLayerAPIAnonTest(t, obj, "TestAPICompleteMultipartHandler", bucketName, objectName, instanceType, + apiRouter, anonReq, getAnonWriteOnlyObjectPolicy(bucketName, objectName)) + + // HTTP request to test the case of `objectLayer` being set to `nil`. + // There is no need to use an existing bucket or valid input for creating the request, + // since the `objectLayer==nil` check is performed before any other checks inside the handlers. + // The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called. + // Indicating that all parts are uploaded and initiating CompleteMultipartUpload. + nilBucket := "dummy-bucket" + nilObject := "dummy-object" + + nilReq, err := newTestSignedRequestV4(http.MethodPost, getCompleteMultipartUploadURL("", nilBucket, nilObject, "dummy-uploadID"), + 0, nil, "", "", nil) + if err != nil { + t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType) + } + // execute the object layer set to `nil` test. + // `ExecObjectLayerAPINilTest` manages the operation. + ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq) +} + +// The UploadID from the response body is parsed and its existence is asserted with an attempt to ListParts using it. +func TestAPIAbortMultipartHandler(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPIAbortMultipartHandler, endpoints: []string{"AbortMultipart"}}) +} + +func testAPIAbortMultipartHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + var err error + opts := ObjectOptions{} + // object used for the test. + objectName := "test-object-new-multipart" + + // upload IDs collected. + var uploadIDs []string + + for range 2 { + // initiate new multipart uploadID. + res, err := obj.NewMultipartUpload(context.Background(), bucketName, objectName, opts) + if err != nil { + // Failed to create NewMultipartUpload, abort. + t.Fatalf("MinIO %s : %s", instanceType, err) + } + + uploadIDs = append(uploadIDs, res.UploadID) + } + + // Parts with size greater than 5 MiB. + // Generating a 6 MiB byte array. + validPart := bytes.Repeat([]byte("abcdef"), 1*humanize.MiByte) + validPartMD5 := getMD5Hash(validPart) + // Create multipart parts. + // Need parts to be uploaded before AbortMultiPartUpload can be called tested. + parts := []struct { + bucketName string + objName string + uploadID string + PartID int + inputReaderData string + inputMd5 string + inputDataSize int64 + }{ + // Case 1-4. + // Creating sequence of parts for same uploadID. + {bucketName, objectName, uploadIDs[0], 1, "abcd", "e2fc714c4727ee9395f324cd2e7f331f", int64(len("abcd"))}, + {bucketName, objectName, uploadIDs[0], 2, "efgh", "1f7690ebdd9b4caf8fab49ca1757bf27", int64(len("efgh"))}, + {bucketName, objectName, uploadIDs[0], 3, "ijkl", "09a0877d04abf8759f99adec02baf579", int64(len("abcd"))}, + {bucketName, objectName, uploadIDs[0], 4, "mnop", "e132e96a5ddad6da8b07bba6f6131fef", int64(len("abcd"))}, + // Part with size larger than 5 MiB. + {bucketName, objectName, uploadIDs[0], 5, string(validPart), validPartMD5, int64(len(validPart))}, + {bucketName, objectName, uploadIDs[0], 6, string(validPart), validPartMD5, int64(len(validPart))}, + + // Part with size larger than 5 MiB. + // Parts uploaded for anonymous/unsigned API handler test. + {bucketName, objectName, uploadIDs[1], 1, string(validPart), validPartMD5, int64(len(validPart))}, + {bucketName, objectName, uploadIDs[1], 2, string(validPart), validPartMD5, int64(len(validPart))}, + } + // Iterating over createPartCases to generate multipart chunks. + for _, part := range parts { + _, err = obj.PutObjectPart(context.Background(), part.bucketName, part.objName, part.uploadID, part.PartID, + mustGetPutObjReader(t, strings.NewReader(part.inputReaderData), part.inputDataSize, part.inputMd5, ""), opts) + if err != nil { + t.Fatalf("%s : %s", instanceType, err) + } + } + + testCases := []struct { + bucket string + object string + uploadID string + accessKey string + secretKey string + // Expected HTTP Response status. + expectedRespStatus int + }{ + // Test case - 1. + // Abort existing upload ID. + { + bucket: bucketName, + object: objectName, + uploadID: uploadIDs[0], + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusNoContent, + }, + // Test case - 2. + // Abort non-existing upload ID. + { + bucket: bucketName, + object: objectName, + uploadID: "nonexistent-upload-id", + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusNotFound, + }, + // Test case - 3. + // Abort with unknown Access key. + { + bucket: bucketName, + object: objectName, + uploadID: uploadIDs[0], + accessKey: "Invalid-AccessID", + secretKey: credentials.SecretKey, + expectedRespStatus: http.StatusForbidden, + }, + } + + for i, testCase := range testCases { + var req *http.Request + // Indicating that all parts are uploaded and initiating abortMultipartUpload. + req, err = newTestSignedRequestV4(http.MethodDelete, getAbortMultipartUploadURL("", testCase.bucket, testCase.object, testCase.uploadID), + 0, nil, testCase.accessKey, testCase.secretKey, nil) + if err != nil { + t.Fatalf("Failed to create HTTP request for AbortMultipartUpload: %v", err) + } + + rec := httptest.NewRecorder() + + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to executes the registered handler. + apiRouter.ServeHTTP(rec, req) + // Assert the response code with the expected status. + if rec.Code != testCase.expectedRespStatus { + t.Errorf("Case %d: MinIO %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) + } + } + + // create unsigned HTTP request for Abort multipart upload. + anonReq, err := newTestRequest(http.MethodDelete, getAbortMultipartUploadURL("", bucketName, objectName, uploadIDs[1]), + 0, nil) + if err != nil { + t.Fatalf("MinIO %s: Failed to create an anonymous request for %s/%s: %v", + instanceType, bucketName, objectName, err) + } + + // ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse, + // sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the + // unsigned request goes through and its validated again. + ExecObjectLayerAPIAnonTest(t, obj, "TestAPIAbortMultipartHandler", bucketName, objectName, instanceType, + apiRouter, anonReq, getAnonWriteOnlyObjectPolicy(bucketName, objectName)) + + // HTTP request to test the case of `objectLayer` being set to `nil`. + // There is no need to use an existing bucket or valid input for creating the request, + // since the `objectLayer==nil` check is performed before any other checks inside the handlers. + // The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called. + // Indicating that all parts are uploaded and initiating abortMultipartUpload. + nilBucket := "dummy-bucket" + nilObject := "dummy-object" + + nilReq, err := newTestSignedRequestV4(http.MethodDelete, getAbortMultipartUploadURL("", nilBucket, nilObject, "dummy-uploadID"), + 0, nil, "", "", nil) + if err != nil { + t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType) + } + // execute the object layer set to `nil` test. + // `ExecObjectLayerAPINilTest` manages the operation. + ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq) +} + +// Wrapper for calling Delete Object API handler tests for both Erasure multiple disks and FS single drive setup. +func TestAPIDeleteObjectHandler(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPIDeleteObjectHandler, endpoints: []string{"DeleteObject"}}) +} + +func testAPIDeleteObjectHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + var err error + objectName := "test-object" + // Object used for anonymous API request test. + anonObjectName := "test-anon-obj" + // set of byte data for PutObject. + // object has to be created before running tests for Deleting the object. + bytesData := []struct { + byteData []byte + }{ + {generateBytesData(6 * humanize.MiByte)}, + } + + // set of inputs for uploading the objects before tests for deleting them is done. + putObjectInputs := []struct { + bucketName string + objectName string + contentLength int64 + textData []byte + metaData map[string]string + }{ + // case - 1. + {bucketName, objectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)}, + // case - 2. + {bucketName, anonObjectName, int64(len(bytesData[0].byteData)), bytesData[0].byteData, make(map[string]string)}, + } + // iterate through the above set of inputs and upload the object. + for i, input := range putObjectInputs { + // uploading the object. + _, err = obj.PutObject(context.Background(), input.bucketName, input.objectName, mustGetPutObjReader(t, bytes.NewReader(input.textData), input.contentLength, input.metaData[""], ""), ObjectOptions{UserDefined: input.metaData}) + // if object upload fails stop the test. + if err != nil { + t.Fatalf("Put Object case %d: Error uploading object: %v", i+1, err) + } + } + + // test cases with inputs and expected result for DeleteObject. + testCases := []struct { + bucketName string + objectName string + accessKey string + secretKey string + + expectedRespStatus int // expected response status body. + }{ + // Test case - 1. + // Deleting an existing object. + // Expected to return HTTP response status code 204. + { + bucketName: bucketName, + objectName: objectName, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusNoContent, + }, + // Test case - 2. + // Attempt to delete an object which is already deleted. + // Still should return http response status 204. + { + bucketName: bucketName, + objectName: objectName, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusNoContent, + }, + // Test case - 3. + // Setting Invalid AccessKey to force signature check inside the handler to fail. + // Should return HTTP response status 403 forbidden. + { + bucketName: bucketName, + objectName: objectName, + accessKey: "Invalid-AccessKey", + secretKey: credentials.SecretKey, + + expectedRespStatus: http.StatusForbidden, + }, + } + + // Iterating over the cases, call DeleteObjectHandler and validate the HTTP response. + for i, testCase := range testCases { + var req, reqV2 *http.Request + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + // construct HTTP request for Delete Object end point. + req, err = newTestSignedRequestV4(http.MethodDelete, getDeleteObjectURL("", testCase.bucketName, testCase.objectName), + 0, nil, testCase.accessKey, testCase.secretKey, nil) + if err != nil { + t.Fatalf("Test %d: Failed to create HTTP request for Delete Object: %v", i+1, err) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler,`func (api objectAPIHandlers) DeleteObjectHandler` handles the request. + apiRouter.ServeHTTP(rec, req) + // Assert the response code with the expected status. + if rec.Code != testCase.expectedRespStatus { + t.Fatalf("MinIO %s: Case %d: Expected the response status to be `%d`, but instead found `%d`", instanceType, i+1, testCase.expectedRespStatus, rec.Code) + } + + // Verify response of the V2 signed HTTP request. + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + recV2 := httptest.NewRecorder() + // construct HTTP request for Delete Object endpoint. + reqV2, err = newTestSignedRequestV2(http.MethodDelete, getDeleteObjectURL("", testCase.bucketName, testCase.objectName), + 0, nil, testCase.accessKey, testCase.secretKey, nil) + if err != nil { + t.Fatalf("Failed to create HTTP request for NewMultipart Request: %v", err) + } + + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(recV2, reqV2) + // Assert the response code with the expected status. + if recV2.Code != testCase.expectedRespStatus { + t.Errorf("Case %d: MinIO %s: Expected the response status to be `%d`, but instead found `%d`", i+1, + instanceType, testCase.expectedRespStatus, recV2.Code) + } + } + + // Test for Anonymous/unsigned http request. + anonReq, err := newTestRequest(http.MethodDelete, getDeleteObjectURL("", bucketName, anonObjectName), 0, nil) + if err != nil { + t.Fatalf("MinIO %s: Failed to create an anonymous request for %s/%s: %v", + instanceType, bucketName, anonObjectName, err) + } + + // ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse, + // sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the + // unsigned request goes through and its validated again. + ExecObjectLayerAPIAnonTest(t, obj, "TestAPIDeleteObjectHandler", bucketName, anonObjectName, instanceType, apiRouter, anonReq, getAnonWriteOnlyObjectPolicy(bucketName, anonObjectName)) + + // HTTP request to test the case of `objectLayer` being set to `nil`. + // There is no need to use an existing bucket or valid input for creating the request, + // since the `objectLayer==nil` check is performed before any other checks inside the handlers. + // The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called. + nilBucket := "dummy-bucket" + nilObject := "dummy-object" + + nilReq, err := newTestSignedRequestV4(http.MethodDelete, getDeleteObjectURL("", nilBucket, nilObject), + 0, nil, "", "", nil) + if err != nil { + t.Errorf("MinIO %s: Failed to create HTTP request for testing the response when object Layer is set to `nil`.", instanceType) + } + // execute the object layer set to `nil` test. + // `ExecObjectLayerAPINilTest` manages the operation. + ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq) +} + +// TestAPIPutObjectPartHandlerStreaming - Tests validate the response of PutObjectPart HTTP handler +// when the request signature type is `streaming signature`. +func TestAPIPutObjectPartHandlerStreaming(t *testing.T) { + defer DetectTestLeak(t)() + ExecExtendedObjectLayerAPITest(t, testAPIPutObjectPartHandlerStreaming, []string{"NewMultipart", "PutObjectPart"}) +} + +func testAPIPutObjectPartHandlerStreaming(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + testObject := "testobject" + rec := httptest.NewRecorder() + req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, "testobject"), + 0, nil, credentials.AccessKey, credentials.SecretKey, nil) + if err != nil { + t.Fatalf("[%s] - Failed to create a signed request to initiate multipart upload for %s/%s: %v", + instanceType, bucketName, testObject, err) + } + apiRouter.ServeHTTP(rec, req) + + // Get uploadID of the multipart upload initiated. + var mpartResp InitiateMultipartUploadResponse + mpartRespBytes, err := io.ReadAll(rec.Result().Body) + if err != nil { + t.Fatalf("[%s] Failed to read NewMultipartUpload response %v", instanceType, err) + } + err = xml.Unmarshal(mpartRespBytes, &mpartResp) + if err != nil { + t.Fatalf("[%s] Failed to unmarshal NewMultipartUpload response %v", instanceType, err) + } + + noAPIErr := APIError{} + missingDateHeaderErr := getAPIError(ErrMissingDateHeader) + internalErr := getAPIError(ErrBadRequest) + testCases := []struct { + fault Fault + expectedErr APIError + }{ + {BadSignature, missingDateHeaderErr}, + {None, noAPIErr}, + {TooBigDecodedLength, internalErr}, + } + + for i, test := range testCases { + rec = httptest.NewRecorder() + req, err = newTestStreamingSignedRequest(http.MethodPut, + getPutObjectPartURL("", bucketName, testObject, mpartResp.UploadID, "1"), + 5, 1, bytes.NewReader([]byte("hello")), credentials.AccessKey, credentials.SecretKey) + if err != nil { + t.Fatalf("Failed to create new streaming signed HTTP request: %v.", err) + } + switch test.fault { + case BadSignature: + // Reset date field in header to make streaming signature fail. + req.Header.Set("x-amz-date", "") + case TooBigDecodedLength: + // Set decoded length to a large value out of int64 range to simulate parse failure. + req.Header.Set("x-amz-decoded-content-length", "9999999999999999999999") + } + apiRouter.ServeHTTP(rec, req) + + if test.expectedErr != noAPIErr { + errBytes, err := io.ReadAll(rec.Result().Body) + if err != nil { + t.Fatalf("Test %d %s Failed to read error response from upload part request %s/%s: %v", + i+1, instanceType, bucketName, testObject, err) + } + var errXML APIErrorResponse + err = xml.Unmarshal(errBytes, &errXML) + if err != nil { + t.Fatalf("Test %d %s Failed to unmarshal error response from upload part request %s/%s: %v", + i+1, instanceType, bucketName, testObject, err) + } + if test.expectedErr.Code != errXML.Code { + t.Errorf("Test %d %s expected to fail with error %s, but received %s", i+1, instanceType, + test.expectedErr.Code, errXML.Code) + } + } else if rec.Code != http.StatusOK { + t.Errorf("Test %d %s expected to succeed, but failed with HTTP status code %d", + i+1, instanceType, rec.Code) + } + } +} + +// TestAPIPutObjectPartHandler - Tests validate the response of PutObjectPart HTTP handler +// +// for variety of inputs. +func TestAPIPutObjectPartHandler(t *testing.T) { + defer DetectTestLeak(t)() + ExecExtendedObjectLayerAPITest(t, testAPIPutObjectPartHandler, []string{"PutObjectPart"}) +} + +func testAPIPutObjectPartHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + // Initiate Multipart upload for testing PutObjectPartHandler. + testObject := "testobject" + var opts ObjectOptions + // PutObjectPart API HTTP Handler has to be tested in isolation, + // that is without any other handler being registered, + // That's why NewMultipartUpload is initiated using ObjectLayer. + res, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, opts) + if err != nil { + // Failed to create NewMultipartUpload, abort. + t.Fatalf("MinIO %s : %s", instanceType, err) + } + uploadID := res.UploadID + + uploadIDCopy := uploadID + + // SignatureMismatch for various signing types + testCases := []struct { + objectName string + content string + partNumber string + fault Fault + accessKey string + secretKey string + + expectedAPIError APIErrorCode + }{ + // Success case. + 0: { + objectName: testObject, + content: "hello", + partNumber: "1", + fault: None, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedAPIError: -1, + }, + // Case where part number is invalid. + 1: { + objectName: testObject, + content: "hello", + partNumber: "9999999999999999999", + fault: None, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedAPIError: ErrInvalidPart, + }, + // Case where the part number has exceeded the max allowed parts in an upload. + 2: { + objectName: testObject, + content: "hello", + partNumber: strconv.Itoa(globalMaxPartID + 1), + fault: None, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedAPIError: ErrInvalidMaxParts, + }, + // Case where the content length is not set in the HTTP request. + 3: { + objectName: testObject, + content: "hello", + partNumber: "1", + fault: MissingContentLength, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedAPIError: ErrMissingContentLength, + }, + // case where the object size is set to a value greater than the max allowed size. + 4: { + objectName: testObject, + content: "hello", + partNumber: "1", + fault: TooBigObject, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedAPIError: ErrEntityTooLarge, + }, + // case where a signature mismatch is introduced and the response is validated. + 5: { + objectName: testObject, + content: "hello", + partNumber: "1", + fault: BadSignature, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedAPIError: ErrSignatureDoesNotMatch, + }, + // Case where incorrect checksum is set and the error response + // is asserted with the expected error response. + 6: { + objectName: testObject, + content: "hello", + partNumber: "1", + fault: BadMD5, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedAPIError: ErrInvalidDigest, + }, + // case where the a non-existent uploadID is set. + 7: { + objectName: testObject, + content: "hello", + partNumber: "1", + fault: MissingUploadID, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedAPIError: ErrNoSuchUpload, + }, + // case with invalid AccessID. + // Forcing the signature check inside the handler to fail. + 8: { + objectName: testObject, + content: "hello", + partNumber: "1", + fault: None, + accessKey: "Invalid-AccessID", + secretKey: credentials.SecretKey, + + expectedAPIError: ErrInvalidAccessKeyID, + }, + // Case where part number is invalid. + 9: { + objectName: testObject, + content: "hello", + partNumber: "0", + fault: None, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedAPIError: ErrInvalidPart, + }, + 10: { + objectName: testObject, + content: "hello", + partNumber: "-10", + fault: None, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + + expectedAPIError: ErrInvalidPart, + }, + } + + reqV2Str := "V2 Signed HTTP request" + reqV4Str := "V4 Signed HTTP request" + + type inputReqRec struct { + req *http.Request + rec *httptest.ResponseRecorder + reqType string + } + + for i, test := range testCases { + // Using sub-tests introduced in Go 1.7. + t.Run(fmt.Sprintf("MinIO-%s-Test-%d.", instanceType, i), func(t *testing.T) { + // collection of input HTTP request, ResponseRecorder and request type. + // Used to make a collection of V4 and V4 HTTP request. + var reqV4, reqV2 *http.Request + var recV4, recV2 *httptest.ResponseRecorder + + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + recV4 = httptest.NewRecorder() + recV2 = httptest.NewRecorder() + // setting a non-existent uploadID. + // deliberately introducing the invalid value to be able to assert the response with the expected error response. + if test.fault == MissingUploadID { + uploadID = "upload1" + } + // constructing a v4 signed HTTP request. + reqV4, err = newTestSignedRequestV4(http.MethodPut, + getPutObjectPartURL("", bucketName, test.objectName, uploadID, test.partNumber), + int64(len(test.content)), bytes.NewReader([]byte(test.content)), test.accessKey, test.secretKey, nil) + if err != nil { + t.Fatalf("Failed to create a signed V4 request to upload part for %s/%s: %v", + bucketName, test.objectName, err) + } + // Verify response of the V2 signed HTTP request. + // construct HTTP request for PutObject Part Object endpoint. + reqV2, err = newTestSignedRequestV2(http.MethodPut, + getPutObjectPartURL("", bucketName, test.objectName, uploadID, test.partNumber), + int64(len(test.content)), bytes.NewReader([]byte(test.content)), test.accessKey, test.secretKey, nil) + if err != nil { + t.Fatalf("Test %d %s Failed to create a V2 signed request to upload part for %s/%s: %v", i, instanceType, + bucketName, test.objectName, err) + } + + // collection of input HTTP request, ResponseRecorder and request type. + reqRecs := []inputReqRec{ + { + req: reqV4, + rec: recV4, + reqType: reqV4Str, + }, + { + req: reqV2, + rec: recV2, + reqType: reqV2Str, + }, + } + + for _, reqRec := range reqRecs { + // Response recorder to record the response of the handler. + rec := reqRec.rec + // HTTP request used to call the handler. + req := reqRec.req + // HTTP request type string for V4/V2 requests. + reqType := reqRec.reqType + + // Clone so we don't retain values we do not want. + req.Header = req.Header.Clone() + + // introduce faults in the request. + // deliberately introducing the invalid value to be able to assert the response with the expected error response. + switch test.fault { + case MissingContentLength: + req.ContentLength = -1 + // Setting the content length to a value greater than the max allowed size of a part. + // Used in test case 4. + case TooBigObject: + req.ContentLength = globalMaxObjectSize + 1 + // Malformed signature. + // Used in test case 6. + case BadSignature: + req.Header.Set("authorization", req.Header.Get("authorization")+"a") + // Setting an invalid Content-MD5 to force a Md5 Mismatch error. + // Used in tesr case 7. + case BadMD5: + req.Header.Set("Content-MD5", "badmd5") + } + + // invoke the PutObjectPart HTTP handler. + apiRouter.ServeHTTP(rec, req) + + // validate the error response. + want := getAPIError(test.expectedAPIError) + if test.expectedAPIError == -1 { + want.HTTPStatusCode = 200 + want.Code = "" + want.Description = "" + } + if rec.Code != http.StatusOK { + var errBytes []byte + // read the response body. + errBytes, err = io.ReadAll(rec.Result().Body) + if err != nil { + t.Fatalf("%s, Failed to read error response from upload part request \"%s\"/\"%s\": %v.", + reqType, bucketName, test.objectName, err) + } + // parse the XML error response. + var errXML APIErrorResponse + err = xml.Unmarshal(errBytes, &errXML) + if err != nil { + t.Fatalf("%s, Failed to unmarshal error response from upload part request \"%s\"/\"%s\": %v.", + reqType, bucketName, test.objectName, err) + } + // Validate whether the error has occurred for the expected reason. + if want.Code != errXML.Code { + t.Errorf("%s, Expected to fail with error \"%s\", but received \"%s\": %q.", + reqType, want.Code, errXML.Code, errXML.Message) + } + // Validate the HTTP response status code with the expected one. + if want.HTTPStatusCode != rec.Code { + t.Errorf("%s, Expected the HTTP response status code to be %d, got %d.", reqType, want.HTTPStatusCode, rec.Code) + } + } else if want.HTTPStatusCode != http.StatusOK { + t.Errorf("got 200 ok, want %d", rec.Code) + } + } + }) + } + + // Test for Anonymous/unsigned http request. + anonReq, err := newTestRequest(http.MethodPut, getPutObjectPartURL("", bucketName, testObject, uploadIDCopy, "1"), + int64(len("hello")), bytes.NewReader([]byte("hello"))) + if err != nil { + t.Fatalf("MinIO %s: Failed to create an anonymous request for %s/%s: %v", + instanceType, bucketName, testObject, err) + } + + // ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse, + // sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the + // unsigned request goes through and its validated again. + ExecObjectLayerAPIAnonTest(t, obj, "TestAPIPutObjectPartHandler", bucketName, testObject, instanceType, apiRouter, anonReq, getAnonWriteOnlyObjectPolicy(bucketName, testObject)) + + // HTTP request for testing when `ObjectLayer` is set to `nil`. + // There is no need to use an existing bucket and valid input for creating the request + // since the `objectLayer==nil` check is performed before any other checks inside the handlers. + // The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called. + nilBucket := "dummy-bucket" + nilObject := "dummy-object" + + nilReq, err := newTestSignedRequestV4(http.MethodPut, getPutObjectPartURL("", nilBucket, nilObject, "0", "0"), + 0, bytes.NewReader([]byte("testNilObjLayer")), "", "", nil) + if err != nil { + t.Errorf("MinIO %s: Failed to create http request for testing the response when object Layer is set to `nil`.", instanceType) + } + // execute the object layer set to `nil` test. + // `ExecObjectLayerAPINilTest` manages the operation. + ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq) +} + +// TestAPIListObjectPartsHandlerPreSign - Tests validate the response of ListObjectParts HTTP handler +// +// when signature type of the HTTP request is `Presigned`. +func TestAPIListObjectPartsHandlerPreSign(t *testing.T) { + defer DetectTestLeak(t)() + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: testAPIListObjectPartsHandlerPreSign, endpoints: []string{"PutObjectPart", "NewMultipart", "ListObjectParts"}}) +} + +func testAPIListObjectPartsHandlerPreSign(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + testObject := "testobject" + rec := httptest.NewRecorder() + req, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, testObject), + 0, nil, credentials.AccessKey, credentials.SecretKey, nil) + if err != nil { + t.Fatalf("[%s] - Failed to create a signed request to initiate multipart upload for %s/%s: %v", + instanceType, bucketName, testObject, err) + } + apiRouter.ServeHTTP(rec, req) + + // Get uploadID of the multipart upload initiated. + var mpartResp InitiateMultipartUploadResponse + mpartRespBytes, err := io.ReadAll(rec.Result().Body) + if err != nil { + t.Fatalf("[%s] Failed to read NewMultipartUpload response %v", instanceType, err) + } + err = xml.Unmarshal(mpartRespBytes, &mpartResp) + if err != nil { + t.Fatalf("[%s] Failed to unmarshal NewMultipartUpload response %v", instanceType, err) + } + + // Upload a part for listing purposes. + rec = httptest.NewRecorder() + req, err = newTestSignedRequestV4(http.MethodPut, + getPutObjectPartURL("", bucketName, testObject, mpartResp.UploadID, "1"), + int64(len("hello")), bytes.NewReader([]byte("hello")), credentials.AccessKey, credentials.SecretKey, nil) + if err != nil { + t.Fatalf("[%s] - Failed to create a signed request to initiate multipart upload for %s/%s: %v", + instanceType, bucketName, testObject, err) + } + apiRouter.ServeHTTP(rec, req) + + if rec.Code != http.StatusOK { + t.Fatalf("[%s] - Failed to PutObjectPart bucket: %s object: %s HTTP status code: %d", + instanceType, bucketName, testObject, rec.Code) + } + rec = httptest.NewRecorder() + req, err = newTestRequest(http.MethodGet, + getListMultipartURLWithParams("", bucketName, testObject, mpartResp.UploadID, "", "", ""), + 0, nil) + if err != nil { + t.Fatalf("[%s] - Failed to create an unsigned request to list object parts for bucket %s, uploadId %s", + instanceType, bucketName, mpartResp.UploadID) + } + + req.Header = http.Header{} + err = preSignV2(req, credentials.AccessKey, credentials.SecretKey, int64(10*60*60)) + if err != nil { + t.Fatalf("[%s] - Failed to presignV2 an unsigned request to list object parts for bucket %s, uploadId %s", + instanceType, bucketName, mpartResp.UploadID) + } + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Errorf("Test %d %s expected to succeed but failed with HTTP status code %d", + 1, instanceType, rec.Code) + } + + rec = httptest.NewRecorder() + req, err = newTestRequest(http.MethodGet, + getListMultipartURLWithParams("", bucketName, testObject, mpartResp.UploadID, "", "", ""), + 0, nil) + if err != nil { + t.Fatalf("[%s] - Failed to create an unsigned request to list object parts for bucket %s, uploadId %s", + instanceType, bucketName, mpartResp.UploadID) + } + + err = preSignV4(req, credentials.AccessKey, credentials.SecretKey, int64(10*60*60)) + if err != nil { + t.Fatalf("[%s] - Failed to presignV2 an unsigned request to list object parts for bucket %s, uploadId %s", + instanceType, bucketName, mpartResp.UploadID) + } + apiRouter.ServeHTTP(rec, req) + if rec.Code != http.StatusOK { + t.Errorf("Test %d %s expected to succeed but failed with HTTP status code %d", + 1, instanceType, rec.Code) + } +} + +// TestAPIListObjectPartsHandler - Tests validate the response of ListObjectParts HTTP handler +// +// for variety of success/failure cases. +func TestAPIListObjectPartsHandler(t *testing.T) { + defer DetectTestLeak(t)() + ExecExtendedObjectLayerAPITest(t, testAPIListObjectPartsHandler, []string{"ListObjectParts"}) +} + +func testAPIListObjectPartsHandler(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, + credentials auth.Credentials, t *testing.T, +) { + testObject := "testobject" + var opts ObjectOptions + // PutObjectPart API HTTP Handler has to be tested in isolation, + // that is without any other handler being registered, + // That's why NewMultipartUpload is initiated using ObjectLayer. + res, err := obj.NewMultipartUpload(context.Background(), bucketName, testObject, opts) + if err != nil { + // Failed to create NewMultipartUpload, abort. + t.Fatalf("MinIO %s : %s", instanceType, err) + } + uploadID := res.UploadID + uploadIDCopy := uploadID + + // create an object Part, will be used to test list object parts. + _, err = obj.PutObjectPart(context.Background(), bucketName, testObject, uploadID, 1, mustGetPutObjReader(t, bytes.NewReader([]byte("hello")), int64(len("hello")), "5d41402abc4b2a76b9719d911017c592", ""), opts) + if err != nil { + t.Fatalf("MinIO %s : %s.", instanceType, err) + } + + // expected error types for invalid inputs to ListObjectParts handler. + noAPIErr := APIError{} + // expected error when the signature check fails. + signatureMismatchErr := getAPIError(ErrSignatureDoesNotMatch) + // expected error the when the uploadID is invalid. + noSuchUploadErr := getAPIError(ErrNoSuchUpload) + // expected error the part number marker use in the ListObjectParts request is invalid. + invalidPartMarkerErr := getAPIError(ErrInvalidPartNumberMarker) + // expected error when the maximum number of parts requested to listed in invalid. + invalidMaxPartsErr := getAPIError(ErrInvalidMaxParts) + + testCases := []struct { + fault Fault + partNumberMarker string + maxParts string + expectedErr APIError + }{ + // Test case - 1. + // case where a signature mismatch is introduced and the response is validated. + { + fault: BadSignature, + partNumberMarker: "", + maxParts: "", + expectedErr: signatureMismatchErr, + }, + + // Test case - 2. + // Marker is set to invalid value of -1, error response is asserted. + { + fault: None, + partNumberMarker: "-1", + maxParts: "", + expectedErr: invalidPartMarkerErr, + }, + // Test case - 3. + // Max Parts is set a negative value, error response is validated. + { + fault: None, + partNumberMarker: "", + maxParts: "-1", + expectedErr: invalidMaxPartsErr, + }, + // Test case - 4. + // Invalid UploadID is set and the error response is validated. + { + fault: MissingUploadID, + partNumberMarker: "", + maxParts: "", + expectedErr: noSuchUploadErr, + }, + } + + // string to represent V2 signed HTTP request. + reqV2Str := "V2 Signed HTTP request" + // string to represent V4 signed HTTP request. + reqV4Str := "V4 Signed HTTP request" + // Collection of HTTP request and ResponseRecorder and request type string. + type inputReqRec struct { + req *http.Request + rec *httptest.ResponseRecorder + reqType string + } + + for i, test := range testCases { + var reqV4, reqV2 *http.Request + // Using sub-tests introduced in Go 1.7. + t.Run(fmt.Sprintf("MinIO %s: Test case %d failed.", instanceType, i+1), func(t *testing.T) { + recV2 := httptest.NewRecorder() + recV4 := httptest.NewRecorder() + + // setting a non-existent uploadID. + // deliberately introducing the invalid value to be able to assert the response with the expected error response. + if test.fault == MissingUploadID { + uploadID = "upload1" + } + + // constructing a v4 signed HTTP request for ListMultipartUploads. + reqV4, err = newTestSignedRequestV4(http.MethodGet, + getListMultipartURLWithParams("", bucketName, testObject, uploadID, test.maxParts, test.partNumberMarker, ""), + 0, nil, credentials.AccessKey, credentials.SecretKey, nil) + if err != nil { + t.Fatalf("Failed to create a V4 signed request to list object parts for %s/%s: %v.", + bucketName, testObject, err) + } + // Verify response of the V2 signed HTTP request. + // construct HTTP request for PutObject Part Object endpoint. + reqV2, err = newTestSignedRequestV2(http.MethodGet, + getListMultipartURLWithParams("", bucketName, testObject, uploadID, test.maxParts, test.partNumberMarker, ""), + 0, nil, credentials.AccessKey, credentials.SecretKey, nil) + if err != nil { + t.Fatalf("Failed to create a V2 signed request to list object parts for %s/%s: %v.", + bucketName, testObject, err) + } + + // collection of input HTTP request, ResponseRecorder and request type. + reqRecs := []inputReqRec{ + { + req: reqV4, + rec: recV4, + reqType: reqV4Str, + }, + { + req: reqV2, + rec: recV2, + reqType: reqV2Str, + }, + } + for _, reqRec := range reqRecs { + // Response recorder to record the response of the handler. + rec := reqRec.rec + // HTTP request used to call the handler. + req := reqRec.req + // HTTP request type string for V4/V2 requests. + reqType := reqRec.reqType + // Malformed signature. + if test.fault == BadSignature { + req.Header.Set("authorization", req.Header.Get("authorization")+"a") + } + + // invoke the PutObjectPart HTTP handler with the given HTTP request. + apiRouter.ServeHTTP(rec, req) + + // validate the error response. + if test.expectedErr != noAPIErr { + var errBytes []byte + // read the response body. + errBytes, err = io.ReadAll(rec.Result().Body) + if err != nil { + t.Fatalf("%s,Failed to read error response list object parts request %s/%s: %v", reqType, bucketName, testObject, err) + } + // parse the error response. + var errXML APIErrorResponse + err = xml.Unmarshal(errBytes, &errXML) + if err != nil { + t.Fatalf("%s, Failed to unmarshal error response from list object partsest %s/%s: %v", + reqType, bucketName, testObject, err) + } + // Validate whether the error has occurred for the expected reason. + if test.expectedErr.Code != errXML.Code { + t.Errorf("%s, Expected to fail with %s but received %s", + reqType, test.expectedErr.Code, errXML.Code) + } + // in case error is not expected response status should be 200OK. + } else if rec.Code != http.StatusOK { + t.Errorf("%s, Expected to succeed with response HTTP status 200OK, but failed with HTTP status code %d.", reqType, rec.Code) + } + } + }) + } + + // Test for Anonymous/unsigned http request. + anonReq, err := newTestRequest(http.MethodGet, + getListMultipartURLWithParams("", bucketName, testObject, uploadIDCopy, "", "", ""), 0, nil) + if err != nil { + t.Fatalf("MinIO %s: Failed to create an anonymous request for %s/%s: %v", + instanceType, bucketName, testObject, err) + } + + // ExecObjectLayerAPIAnonTest - Calls the HTTP API handler using the anonymous request, validates the ErrAccessDeniedResponse, + // sets the bucket policy using the policy statement generated from `getWriteOnlyObjectStatement` so that the + // unsigned request goes through and its validated again. + ExecObjectLayerAPIAnonTest(t, obj, "TestAPIListObjectPartsHandler", bucketName, testObject, instanceType, apiRouter, anonReq, getAnonWriteOnlyObjectPolicy(bucketName, testObject)) + + // HTTP request for testing when `objectLayer` is set to `nil`. + // There is no need to use an existing bucket and valid input for creating the request + // since the `objectLayer==nil` check is performed before any other checks inside the handlers. + // The only aim is to generate an HTTP request in a way that the relevant/registered end point is evoked/called. + nilBucket := "dummy-bucket" + nilObject := "dummy-object" + + nilReq, err := newTestSignedRequestV4(http.MethodGet, + getListMultipartURLWithParams("", nilBucket, nilObject, "dummy-uploadID", "0", "0", ""), + 0, nil, "", "", nil) + if err != nil { + t.Errorf("MinIO %s:Failed to create http request for testing the response when object Layer is set to `nil`.", instanceType) + } + // execute the object layer set to `nil` test. + // `ExecObjectLayerAPINilTest` sets the Object Layer to `nil` and calls the handler. + ExecObjectLayerAPINilTest(t, nilBucket, nilObject, instanceType, apiRouter, nilReq) +} diff --git a/cmd/object-lambda-handlers.go b/cmd/object-lambda-handlers.go new file mode 100644 index 00000000..1ced5165 --- /dev/null +++ b/cmd/object-lambda-handlers.go @@ -0,0 +1,227 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "crypto/subtle" + "encoding/hex" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/klauspost/compress/gzhttp" + "github.com/lithammer/shortuuid/v4" + miniogo "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/mux" + "github.com/minio/pkg/v3/policy" + + "github.com/minio/minio/internal/auth" + levent "github.com/minio/minio/internal/config/lambda/event" + "github.com/minio/minio/internal/hash/sha256" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" +) + +var getLambdaEventData = func(bucket, object string, cred auth.Credentials, r *http.Request) (levent.Event, error) { + host := globalLocalNodeName + secure := globalIsTLS + if globalMinioEndpointURL != nil { + host = globalMinioEndpointURL.Host + secure = globalMinioEndpointURL.Scheme == "https" + } + + duration := time.Until(cred.Expiration) + if duration > time.Hour || duration < time.Hour { + // Always limit to 1 hour. + duration = time.Hour + } + + clnt, err := miniogo.New(host, &miniogo.Options{ + Creds: credentials.NewStaticV4(cred.AccessKey, cred.SecretKey, cred.SessionToken), + Secure: secure, + Transport: globalRemoteTargetTransport, + Region: globalSite.Region(), + }) + if err != nil { + return levent.Event{}, err + } + + reqParams := url.Values{} + if partNumberStr := r.Form.Get("partNumber"); partNumberStr != "" { + reqParams.Set("partNumber", partNumberStr) + } + for k := range supportedHeadGetReqParams { + if v := r.Form.Get(k); v != "" { + reqParams.Set(k, v) + } + } + + extraHeaders := http.Header{} + u, err := clnt.PresignHeader(r.Context(), http.MethodGet, bucket, object, duration, reqParams, extraHeaders) + if err != nil { + return levent.Event{}, err + } + + ckSum := sha256.Sum256([]byte(cred.AccessKey + u.RawQuery)) + + eventData := levent.Event{ + GetObjectContext: &levent.GetObjectContext{ + InputS3URL: u.String(), + OutputRoute: shortuuid.New(), + OutputToken: hex.EncodeToString(ckSum[:]), + }, + UserRequest: levent.UserRequest{ + URL: r.URL.String(), + Headers: r.Header.Clone(), + }, + UserIdentity: levent.Identity{ + Type: "IAMUser", + PrincipalID: cred.ParentUser, + AccessKeyID: cred.AccessKey, + }, + } + return eventData, nil +} + +func fwdHeadersToS3(h http.Header, w http.ResponseWriter) { + const trim = "x-amz-fwd-header-" + for k, v := range h { + if stringsHasPrefixFold(k, trim) { + w.Header()[k[len(trim):]] = v + } + } +} + +func fwdStatusToAPIError(statusCode int, resp *http.Response) *APIError { + if statusCode < http.StatusBadRequest { + return nil + } + desc := resp.Header.Get(xhttp.AmzFwdErrorMessage) + if strings.TrimSpace(desc) == "" { + apiErr := errorCodes.ToAPIErr(ErrInvalidRequest) + return &apiErr + } + code := resp.Header.Get(xhttp.AmzFwdErrorCode) + if strings.TrimSpace(code) == "" { + apiErr := errorCodes.ToAPIErr(ErrInvalidRequest) + apiErr.Description = desc + return &apiErr + } + return &APIError{ + HTTPStatusCode: statusCode, + Description: desc, + Code: code, + } +} + +// GetObjectLambdaHandler - GET Object with transformed data via lambda functions +// ---------- +// This implementation of the GET operation applies lambda functions and returns the +// response generated via the lambda functions. To use this API, you must have READ access +// to the object. +func (api objectAPIHandlers) GetObjectLambdaHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "GetObjectLambda") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Check for auth type to return S3 compatible error. + cred, _, s3Error := checkRequestAuthTypeCredential(ctx, r, policy.GetObjectAction) + if s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + target, err := globalLambdaTargetList.Lookup(r.Form.Get("lambdaArn")) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + eventData, err := getLambdaEventData(bucket, object, cred, r) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + resp, err := target.Send(eventData) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + defer resp.Body.Close() + + if eventData.GetObjectContext.OutputRoute != resp.Header.Get(xhttp.AmzRequestRoute) { + tokenErr := errorCodes.ToAPIErr(ErrInvalidRequest) + tokenErr.Description = "The request route included in the request is invalid" + writeErrorResponse(ctx, w, tokenErr, r.URL) + return + } + + if subtle.ConstantTimeCompare([]byte(resp.Header.Get(xhttp.AmzRequestToken)), []byte(eventData.GetObjectContext.OutputToken)) != 1 { + tokenErr := errorCodes.ToAPIErr(ErrInvalidToken) + tokenErr.Description = "The request token included in the request is invalid" + writeErrorResponse(ctx, w, tokenErr, r.URL) + return + } + + statusCode := resp.StatusCode + if status := resp.Header.Get(xhttp.AmzFwdStatus); status != "" { + statusCode, err = strconv.Atoi(status) + if err != nil { + writeErrorResponse(ctx, w, APIError{ + Code: "LambdaFunctionStatusError", + HTTPStatusCode: http.StatusBadRequest, + Description: err.Error(), + }, r.URL) + return + } + } + + // Set all the relevant lambda forward headers if found. + fwdHeadersToS3(resp.Header, w) + + if apiErr := fwdStatusToAPIError(statusCode, resp); apiErr != nil { + writeErrorResponse(ctx, w, *apiErr, r.URL) + return + } + + if !globalAPIConfig.shouldGzipObjects() { + w.Header().Set(gzhttp.HeaderNoCompression, "true") + } + + w.WriteHeader(statusCode) + io.Copy(w, resp.Body) +} diff --git a/cmd/object-lambda-handlers_test.go b/cmd/object-lambda-handlers_test.go new file mode 100644 index 00000000..458034fe --- /dev/null +++ b/cmd/object-lambda-handlers_test.go @@ -0,0 +1,174 @@ +// Copyright (c) 2015-2025 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "testing" + "time" + + "github.com/minio/minio-go/v7/pkg/signer" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/config/lambda" + levent "github.com/minio/minio/internal/config/lambda/event" + xhttp "github.com/minio/minio/internal/http" +) + +func TestGetObjectLambdaHandler(t *testing.T) { + testCases := []struct { + name string + statusCode int + body string + contentType string + expectStatus int + }{ + { + name: "Success 206 Partial Content", + statusCode: 206, + body: "partial-object-data", + contentType: "text/plain", + expectStatus: 206, + }, + { + name: "Success 200 OK", + statusCode: 200, + body: "full-object-data", + contentType: "application/json", + expectStatus: 200, + }, + { + name: "Client Error 400", + statusCode: 400, + body: "bad-request", + contentType: "application/xml", + expectStatus: 400, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + runObjectLambdaTest(t, tc.statusCode, tc.body, tc.contentType, tc.expectStatus) + }) + } +} + +func runObjectLambdaTest(t *testing.T, lambdaStatus int, lambdaBody, contentType string, expectStatus int) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{ + t: t, + objAPITest: func(obj ObjectLayer, instanceType, bucketName string, apiRouter http.Handler, credentials auth.Credentials, t *testing.T) { + objectName := "dummy-object" + functionID := "lambda1" + functionToken := "token123" + + // Lambda mock server + lambdaServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set(xhttp.AmzRequestRoute, functionID) + w.Header().Set(xhttp.AmzRequestToken, functionToken) + w.Header().Set(xhttp.AmzFwdHeaderContentType, contentType) + w.Header().Set(xhttp.AmzFwdStatus, strconv.Itoa(lambdaStatus)) + w.WriteHeader(lambdaStatus) + w.Write([]byte(lambdaBody)) + })) + defer lambdaServer.Close() + + lambdaARN := "arn:minio:s3-object-lambda::lambda1:webhook" + + cfg := config.New() + cfg[config.LambdaWebhookSubSys] = map[string]config.KVS{ + functionID: { + {Key: "endpoint", Value: lambdaServer.URL}, + {Key: "enable", Value: config.EnableOn}, + }, + } + cfg[config.APISubSys] = map[string]config.KVS{ + "api": { + {Key: "gzip", Value: config.EnableOff}, + }, + } + + var err error + globalLambdaTargetList, err = lambda.FetchEnabledTargets(context.Background(), cfg, http.DefaultTransport.(*http.Transport)) + if err != nil { + t.Fatalf("failed to load lambda targets: %v", err) + } + + getLambdaEventData = func(_, _ string, _ auth.Credentials, _ *http.Request) (levent.Event, error) { + return levent.Event{ + GetObjectContext: &levent.GetObjectContext{ + OutputRoute: functionID, + OutputToken: functionToken, + InputS3URL: "http://localhost/dummy", + }, + UserRequest: levent.UserRequest{ + Headers: map[string][]string{}, + }, + UserIdentity: levent.Identity{ + PrincipalID: "test-user", + }, + }, nil + } + + body := []byte{} + req := httptest.NewRequest("GET", "/objectlambda/"+bucketName+"/"+objectName+"?lambdaArn="+url.QueryEscape(lambdaARN), bytes.NewReader(body)) + req.Form = url.Values{"lambdaArn": []string{lambdaARN}} + req.Header.Set("Host", "localhost") + req.Header.Set("X-Amz-Date", time.Now().UTC().Format("20060102T150405Z")) + sum := sha256.Sum256(body) + req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum[:])) + req = signer.SignV4(*req, credentials.AccessKey, credentials.SecretKey, "", "us-east-1") + + rec := httptest.NewRecorder() + api := objectAPIHandlers{ + ObjectAPI: func() ObjectLayer { + return obj + }, + } + api.GetObjectLambdaHandler(rec, req) + + res := rec.Result() + defer res.Body.Close() + respBody, _ := io.ReadAll(res.Body) + + if res.StatusCode != expectStatus { + t.Errorf("Expected status %d, got %d", expectStatus, res.StatusCode) + } + + if contentType != "" { + if ct := res.Header.Get("Content-Type"); ct != contentType { + t.Errorf("Expected Content-Type %q, got %q", contentType, ct) + } + } + + if res.StatusCode < 400 { + if string(respBody) != lambdaBody { + t.Errorf("Expected body %q, got %q", lambdaBody, string(respBody)) + } + } + }, + endpoints: []string{"GetObject"}, + }) +} diff --git a/cmd/object-multipart-handlers.go b/cmd/object-multipart-handlers.go new file mode 100644 index 00000000..12a99b31 --- /dev/null +++ b/cmd/object-multipart-handlers.go @@ -0,0 +1,1227 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bufio" + "context" + "fmt" + "io" + "maps" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "time" + + "github.com/google/uuid" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/encrypt" + "github.com/minio/minio-go/v7/pkg/tags" + "github.com/minio/minio/internal/amztime" + sse "github.com/minio/minio/internal/bucket/encryption" + objectlock "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/config/dns" + "github.com/minio/minio/internal/config/storageclass" + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/etag" + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/handlers" + "github.com/minio/minio/internal/hash" + "github.com/minio/minio/internal/hash/sha256" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" + "github.com/minio/mux" + "github.com/minio/pkg/v3/policy" + "github.com/minio/sio" +) + +// Multipart objectAPIHandlers + +// NewMultipartUploadHandler - New multipart upload. +// Notice: The S3 client can send secret keys in headers for encryption related jobs, +// the handler should ensure to remove these keys before sending them to the object layer. +// Currently these keys are: +// - X-Amz-Server-Side-Encryption-Customer-Key +// - X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key +func (api objectAPIHandlers) NewMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "NewMultipartUpload") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectAction, bucket, object); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Check if bucket encryption is enabled + sseConfig, _ := globalBucketSSEConfigSys.Get(bucket) + sseConfig.Apply(r.Header, sse.ApplyOptions{ + AutoEncrypt: globalAutoEncryption, + }) + + // Validate storage class metadata if present + if sc := r.Header.Get(xhttp.AmzStorageClass); sc != "" { + if !storageclass.IsValid(sc) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidStorageClass), r.URL) + return + } + } + + encMetadata := map[string]string{} + + if crypto.Requested(r.Header) { + if crypto.SSECopy.IsRequested(r.Header) { + writeErrorResponse(ctx, w, toAPIError(ctx, errInvalidEncryptionParameters), r.URL) + return + } + + if crypto.SSEC.IsRequested(r.Header) && crypto.S3.IsRequested(r.Header) { + writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionMethod), r.URL) + return + } + + if crypto.SSEC.IsRequested(r.Header) && crypto.S3KMS.IsRequested(r.Header) { + writeErrorResponse(ctx, w, toAPIError(ctx, crypto.ErrIncompatibleEncryptionMethod), r.URL) + return + } + + _, sourceReplReq := r.Header[xhttp.MinIOSourceReplicationRequest] + ssecRepHeaders := []string{ + "X-Minio-Replication-Server-Side-Encryption-Seal-Algorithm", + "X-Minio-Replication-Server-Side-Encryption-Sealed-Key", + "X-Minio-Replication-Server-Side-Encryption-Iv", + } + ssecRep := false + for _, header := range ssecRepHeaders { + if val := r.Header.Get(header); val != "" { + ssecRep = true + break + } + } + if !ssecRep || !sourceReplReq { + if err = setEncryptionMetadata(r, bucket, object, encMetadata); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + // Set this for multipart only operations, we need to differentiate during + // decryption if the file was actually multipart or not. + encMetadata[ReservedMetadataPrefix+"Encrypted-Multipart"] = "" + } + + // Extract metadata that needs to be saved. + metadata, err := extractMetadataFromReq(ctx, r) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if objTags := r.Header.Get(xhttp.AmzObjectTagging); objTags != "" { + if _, err := tags.ParseObjectTags(objTags); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + metadata[xhttp.AmzObjectTagging] = objTags + } + if r.Header.Get(xhttp.AmzBucketReplicationStatus) == replication.Replica.String() { + metadata[ReservedMetadataPrefixLower+ReplicaStatus] = replication.Replica.String() + metadata[ReservedMetadataPrefixLower+ReplicaTimestamp] = UTCNow().Format(time.RFC3339Nano) + } + retPerms := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.PutObjectRetentionAction) + holdPerms := isPutActionAllowed(ctx, getRequestAuthType(r), bucket, object, r, policy.PutObjectLegalHoldAction) + + getObjectInfo := objectAPI.GetObjectInfo + + retentionMode, retentionDate, legalHold, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, getObjectInfo, retPerms, holdPerms) + if s3Err == ErrNone && retentionMode.Valid() { + metadata[strings.ToLower(xhttp.AmzObjectLockMode)] = string(retentionMode) + metadata[strings.ToLower(xhttp.AmzObjectLockRetainUntilDate)] = amztime.ISO8601Format(retentionDate.UTC()) + } + if s3Err == ErrNone && legalHold.Status.Valid() { + metadata[strings.ToLower(xhttp.AmzObjectLockLegalHold)] = string(legalHold.Status) + } + if s3Err != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + if dsc := mustReplicate(ctx, bucket, object, getMustReplicateOptions(metadata, "", "", replication.ObjectReplicationType, ObjectOptions{})); dsc.ReplicateAny() { + metadata[ReservedMetadataPrefixLower+ReplicationTimestamp] = UTCNow().Format(time.RFC3339Nano) + metadata[ReservedMetadataPrefixLower+ReplicationStatus] = dsc.PendingStatus() + } + + // We need to preserve the encryption headers set in EncryptRequest, + // so we do not want to override them, copy them instead. + maps.Copy(metadata, encMetadata) + + // Ensure that metadata does not contain sensitive information + crypto.RemoveSensitiveEntries(metadata) + + if isCompressible(r.Header, object) { + // Storing the compression metadata. + metadata[ReservedMetadataPrefix+"compression"] = compressionAlgorithmV2 + } + + opts, err := putOptsFromReq(ctx, r, bucket, object, metadata) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if r.Header.Get(xhttp.IfMatch) != "" { + opts.HasIfMatch = true + } + if opts.PreserveETag != "" || + r.Header.Get(xhttp.IfMatch) != "" || + r.Header.Get(xhttp.IfNoneMatch) != "" { + opts.CheckPrecondFn = func(oi ObjectInfo) bool { + if _, err := DecryptObjectInfo(&oi, r); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return true + } + return checkPreconditionsPUT(ctx, w, r, oi, opts) + } + } + + checksumType := hash.NewChecksumHeader(r.Header) + if checksumType.Is(hash.ChecksumInvalid) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL) + return + } else if checksumType.IsSet() && !checksumType.Is(hash.ChecksumTrailing) { + opts.WantChecksum = &hash.Checksum{Type: checksumType} + } + + if opts.WantChecksum != nil { + opts.WantChecksum.Type |= hash.ChecksumMultipart | hash.ChecksumIncludesMultipart + } + + newMultipartUpload := objectAPI.NewMultipartUpload + + res, err := newMultipartUpload(ctx, bucket, object, opts) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + response := generateInitiateMultipartUploadResponse(bucket, object, res.UploadID) + if res.ChecksumAlgo != "" { + w.Header().Set(xhttp.AmzChecksumAlgo, res.ChecksumAlgo) + if res.ChecksumType != "" { + w.Header().Set(xhttp.AmzChecksumType, res.ChecksumType) + } + } + encodedSuccessResponse := encodeResponse(response) + + // Write success response. + writeSuccessResponseXML(w, encodedSuccessResponse) +} + +// CopyObjectPartHandler - uploads a part by copying data from an existing object as data source. +func (api objectAPIHandlers) CopyObjectPartHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "CopyObjectPart") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if crypto.S3KMS.IsRequested(r.Header) { // SSE-KMS is not supported + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNotImplemented), r.URL) + return + } + + vars := mux.Vars(r) + dstBucket := vars["bucket"] + dstObject, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectAction, dstBucket, dstObject); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Read escaped copy source path to check for parameters. + cpSrcPath := r.Header.Get(xhttp.AmzCopySource) + var vid string + if u, err := url.Parse(cpSrcPath); err == nil { + vid = strings.TrimSpace(u.Query().Get(xhttp.VersionID)) + // Note that url.Parse does the unescaping + cpSrcPath = u.Path + } + + srcBucket, srcObject := path2BucketObject(cpSrcPath) + // If source object is empty or bucket is empty, reply back invalid copy source. + if srcObject == "" || srcBucket == "" { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopySource), r.URL) + return + } + + if vid != "" && vid != nullVersionID { + _, err := uuid.Parse(vid) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, VersionNotFound{ + Bucket: srcBucket, + Object: srcObject, + VersionID: vid, + }), r.URL) + return + } + } + + if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, srcBucket, srcObject); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + uploadID := r.Form.Get(xhttp.UploadID) + partIDString := r.Form.Get(xhttp.PartNumber) + + partID, err := strconv.Atoi(partIDString) + if err != nil || partID <= 0 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidPart), r.URL) + return + } + + // check partID with maximum part ID for multipart objects + if isMaxPartID(partID) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidMaxParts), r.URL) + return + } + + var srcOpts, dstOpts ObjectOptions + srcOpts, err = copySrcOpts(ctx, r, srcBucket, srcObject) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + srcOpts.VersionID = vid + + // convert copy src and dst encryption options for GET/PUT calls + getOpts := ObjectOptions{VersionID: srcOpts.VersionID} + if srcOpts.ServerSideEncryption != nil { + getOpts.ServerSideEncryption = encrypt.SSE(srcOpts.ServerSideEncryption) + } + + dstOpts, err = copyDstOpts(ctx, r, dstBucket, dstObject, nil) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + getObjectNInfo := objectAPI.GetObjectNInfo + + // Get request range. + var rs *HTTPRangeSpec + var parseRangeErr error + if rangeHeader := r.Header.Get(xhttp.AmzCopySourceRange); rangeHeader != "" { + rs, parseRangeErr = parseCopyPartRangeSpec(rangeHeader) + } else { + // This check is to see if client specified a header but the value + // is empty for 'x-amz-copy-source-range' + _, ok := r.Header[xhttp.AmzCopySourceRange] + if ok { + parseRangeErr = errInvalidRange + } + } + + checkCopyPartPrecondFn := func(o ObjectInfo) bool { + if _, err := DecryptObjectInfo(&o, r); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return true + } + if checkCopyObjectPartPreconditions(ctx, w, r, o) { + return true + } + if parseRangeErr != nil { + writeCopyPartErr(ctx, w, parseRangeErr, r.URL) + // Range header mismatch is pre-condition like failure + // so return true to indicate Range precondition failed. + return true + } + return false + } + getOpts.CheckPrecondFn = checkCopyPartPrecondFn + gr, err := getObjectNInfo(ctx, srcBucket, srcObject, rs, r.Header, getOpts) + if err != nil { + if isErrPreconditionFailed(err) { + return + } + if globalBucketVersioningSys.PrefixEnabled(srcBucket, srcObject) && gr != nil { + // Versioning enabled quite possibly object is deleted might be delete-marker + // if present set the headers, no idea why AWS S3 sets these headers. + if gr.ObjInfo.VersionID != "" && gr.ObjInfo.DeleteMarker { + w.Header()[xhttp.AmzVersionID] = []string{gr.ObjInfo.VersionID} + w.Header()[xhttp.AmzDeleteMarker] = []string{strconv.FormatBool(gr.ObjInfo.DeleteMarker)} + } + } + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + defer gr.Close() + srcInfo := gr.ObjInfo + + actualPartSize, err := srcInfo.GetActualSize() + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if err := enforceBucketQuotaHard(ctx, dstBucket, actualPartSize); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Special care for CopyObjectPart + if partRangeErr := checkCopyPartRangeWithSize(rs, actualPartSize); partRangeErr != nil { + writeCopyPartErr(ctx, w, partRangeErr, r.URL) + return + } + + // Get the object offset & length + startOffset, length, err := rs.GetOffsetLength(actualPartSize) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // maximum copy size for multipart objects in a single operation + if isMaxObjectSize(length) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL) + return + } + + if isRemoteCopyRequired(ctx, srcBucket, dstBucket, objectAPI) { + var dstRecords []dns.SrvRecord + dstRecords, err = globalDNSConfig.Get(dstBucket) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Send PutObject request to appropriate instance (in federated deployment) + core, rerr := getRemoteInstanceClient(r, getHostFromSrv(dstRecords)) + if rerr != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, rerr), r.URL) + return + } + + popts := minio.PutObjectPartOptions{ + SSE: dstOpts.ServerSideEncryption, + } + + partInfo, err := core.PutObjectPart(ctx, dstBucket, dstObject, uploadID, partID, gr, length, popts) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + response := generateCopyObjectPartResponse(partInfo.ETag, partInfo.LastModified) + encodedSuccessResponse := encodeResponse(response) + + // Write success response. + writeSuccessResponseXML(w, encodedSuccessResponse) + return + } + + actualPartSize = length + var reader io.Reader = etag.NewReader(ctx, gr, nil, nil) + + mi, err := objectAPI.GetMultipartInfo(ctx, dstBucket, dstObject, uploadID, dstOpts) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + _, isEncrypted := crypto.IsEncrypted(mi.UserDefined) + + // Read compression metadata preserved in the init multipart for the decision. + _, isCompressed := mi.UserDefined[ReservedMetadataPrefix+"compression"] + // Compress only if the compression is enabled during initial multipart. + var idxCb func() []byte + if isCompressed { + wantEncryption := crypto.Requested(r.Header) || isEncrypted + s2c, cb := newS2CompressReader(reader, actualPartSize, wantEncryption) + idxCb = cb + defer s2c.Close() + reader = etag.Wrap(s2c, reader) + length = -1 + } + + srcInfo.Reader, err = hash.NewReader(ctx, reader, length, "", "", actualPartSize) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + dstOpts, err = copyDstOpts(ctx, r, dstBucket, dstObject, mi.UserDefined) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + dstOpts.IndexCB = idxCb + + rawReader := srcInfo.Reader + pReader := NewPutObjReader(rawReader) + + var objectEncryptionKey crypto.ObjectKey + if isEncrypted { + if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(mi.UserDefined) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrSSEMultipartEncrypted), r.URL) + return + } + if crypto.S3.IsEncrypted(mi.UserDefined) && crypto.SSEC.IsRequested(r.Header) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrSSEMultipartEncrypted), r.URL) + return + } + var key []byte + if crypto.SSEC.IsRequested(r.Header) { + key, err = ParseSSECustomerRequest(r) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + key, err = decryptObjectMeta(key, dstBucket, dstObject, mi.UserDefined) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + copy(objectEncryptionKey[:], key) + + var nonce [12]byte + tmp := sha256.Sum256(fmt.Append(nil, uploadID, partID)) + copy(nonce[:], tmp[:12]) + + partEncryptionKey := objectEncryptionKey.DerivePartKey(uint32(partID)) + encReader, err := sio.EncryptReader(reader, sio.Config{ + Key: partEncryptionKey[:], + Nonce: &nonce, + }) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + reader = etag.Wrap(encReader, reader) + + wantSize := int64(-1) + if length >= 0 { + info := ObjectInfo{Size: length} + wantSize = info.EncryptedSize() + } + + srcInfo.Reader, err = hash.NewReader(ctx, reader, wantSize, "", "", actualPartSize) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + pReader, err = pReader.WithEncryption(srcInfo.Reader, &objectEncryptionKey) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if dstOpts.IndexCB != nil { + dstOpts.IndexCB = compressionIndexEncrypter(objectEncryptionKey, dstOpts.IndexCB) + } + } + + srcInfo.PutObjReader = pReader + copyObjectPart := objectAPI.CopyObjectPart + + // Copy source object to destination, if source and destination + // object is same then only metadata is updated. + partInfo, err := copyObjectPart(ctx, srcBucket, srcObject, dstBucket, dstObject, uploadID, partID, + startOffset, length, srcInfo, srcOpts, dstOpts) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if isEncrypted { + sseS3 := crypto.S3.IsRequested(r.Header) || crypto.S3.IsEncrypted(mi.UserDefined) + partInfo.ETag = tryDecryptETag(objectEncryptionKey[:], partInfo.ETag, sseS3) + } + + response := generateCopyObjectPartResponse(partInfo.ETag, partInfo.LastModified) + encodedSuccessResponse := encodeResponse(response) + + // Write success response. + writeSuccessResponseXML(w, encodedSuccessResponse) +} + +// PutObjectPartHandler - uploads an incoming part for an ongoing multipart operation. +func (api objectAPIHandlers) PutObjectPartHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "PutObjectPart") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // X-Amz-Copy-Source shouldn't be set for this call. + if _, ok := r.Header[xhttp.AmzCopySource]; ok { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidCopySource), r.URL) + return + } + + clientETag, err := etag.FromContentMD5(r.Header) + if err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidDigest), r.URL) + return + } + + // if Content-Length is unknown/missing, throw away + size := r.ContentLength + + rAuthType := getRequestAuthType(r) + // For auth type streaming signature, we need to gather a different content length. + switch rAuthType { + // Check signature types that must have content length + case authTypeStreamingSigned, authTypeStreamingSignedTrailer, authTypeStreamingUnsignedTrailer: + if sizeStr, ok := r.Header[xhttp.AmzDecodedContentLength]; ok { + if sizeStr[0] == "" { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL) + return + } + size, err = strconv.ParseInt(sizeStr[0], 10, 64) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + } + + if size == -1 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingContentLength), r.URL) + return + } + + uploadID := r.Form.Get(xhttp.UploadID) + partIDString := r.Form.Get(xhttp.PartNumber) + + partID, err := strconv.Atoi(partIDString) + if err != nil || partID <= 0 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidPart), r.URL) + return + } + + // maximum size for multipart objects in a single operation + if isMaxObjectSize(size) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrEntityTooLarge), r.URL) + return + } + + // check partID with maximum part ID for multipart objects + if isMaxPartID(partID) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidMaxParts), r.URL) + return + } + + var ( + md5hex = clientETag.String() + sha256hex = "" + reader io.Reader = r.Body + s3Error APIErrorCode + ) + if s3Error = isPutActionAllowed(ctx, rAuthType, bucket, object, r, policy.PutObjectAction); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + switch rAuthType { + case authTypeStreamingSigned, authTypeStreamingSignedTrailer: + // Initialize stream signature verifier. + reader, s3Error = newSignV4ChunkedReader(r, rAuthType == authTypeStreamingSignedTrailer) + if s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + case authTypeStreamingUnsignedTrailer: + // Initialize stream signature verifier. + reader, s3Error = newUnsignedV4ChunkedReader(r, true, r.Header.Get(xhttp.Authorization) != "") + if s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + case authTypeSignedV2, authTypePresignedV2: + if s3Error = isReqAuthenticatedV2(r); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + case authTypePresigned, authTypeSigned: + if s3Error = reqSignatureV4Verify(r, globalSite.Region(), serviceS3); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + if !skipContentSha256Cksum(r) { + sha256hex = getContentSha256Cksum(r, serviceS3) + } + } + + if err := enforceBucketQuotaHard(ctx, bucket, size); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + actualSize := size + + // get encryption options + var opts ObjectOptions + if crypto.SSEC.IsRequested(r.Header) { + opts, err = getOpts(ctx, r, bucket, object) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + + mi, err := objectAPI.GetMultipartInfo(ctx, bucket, object, uploadID, opts) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // Read compression metadata preserved in the init multipart for the decision. + _, isCompressed := mi.UserDefined[ReservedMetadataPrefix+"compression"] + var idxCb func() []byte + if isCompressed { + actualReader, err := hash.NewReader(ctx, reader, size, md5hex, sha256hex, actualSize) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if err = actualReader.AddChecksum(r, false); err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL) + return + } + + // Set compression metrics. + wantEncryption := crypto.Requested(r.Header) + s2c, cb := newS2CompressReader(actualReader, actualSize, wantEncryption) + idxCb = cb + defer s2c.Close() + reader = etag.Wrap(s2c, actualReader) + size = -1 // Since compressed size is un-predictable. + md5hex = "" // Do not try to verify the content. + sha256hex = "" + } + + var forceMD5 []byte + // Optimization: If SSE-KMS and SSE-C did not request Content-Md5. Use uuid as etag. Optionally enable this also + // for server that is started with `--no-compat`. + if !etag.ContentMD5Requested(r.Header) && (crypto.S3KMS.IsEncrypted(mi.UserDefined) || crypto.SSEC.IsRequested(r.Header) || !globalServerCtxt.StrictS3Compat) { + forceMD5 = mustGetUUIDBytes() + } + + hashReader, err := hash.NewReaderWithOpts(ctx, reader, hash.Options{ + Size: size, + MD5Hex: md5hex, + SHA256Hex: sha256hex, + ActualSize: actualSize, + DisableMD5: false, + ForceMD5: forceMD5, + }) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if err := hashReader.AddChecksum(r, size < 0); err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL) + return + } + + pReader := NewPutObjReader(hashReader) + + _, isEncrypted := crypto.IsEncrypted(mi.UserDefined) + _, replicationStatus := mi.UserDefined[xhttp.AmzBucketReplicationStatus] + _, sourceReplReq := r.Header[xhttp.MinIOSourceReplicationRequest] + var objectEncryptionKey crypto.ObjectKey + if isEncrypted { + if !crypto.SSEC.IsRequested(r.Header) && crypto.SSEC.IsEncrypted(mi.UserDefined) && !replicationStatus { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrSSEMultipartEncrypted), r.URL) + return + } + + opts, err = putOptsFromReq(ctx, r, bucket, object, mi.UserDefined) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + var key []byte + if crypto.SSEC.IsRequested(r.Header) { + key, err = ParseSSECustomerRequest(r) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + + if !sourceReplReq || !crypto.SSEC.IsEncrypted(mi.UserDefined) { + // Calculating object encryption key + key, err = decryptObjectMeta(key, bucket, object, mi.UserDefined) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + copy(objectEncryptionKey[:], key) + + partEncryptionKey := objectEncryptionKey.DerivePartKey(uint32(partID)) + in := io.Reader(hashReader) + if size > encryptBufferThreshold { + // The encryption reads in blocks of 64KB. + // We add a buffer on bigger files to reduce the number of syscalls upstream. + in = bufio.NewReaderSize(hashReader, encryptBufferSize) + } + + var nonce [12]byte + tmp := sha256.Sum256(fmt.Append(nil, uploadID, partID)) + copy(nonce[:], tmp[:12]) + + reader, err = sio.EncryptReader(in, sio.Config{ + Key: partEncryptionKey[:], + Nonce: &nonce, + }) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + wantSize := int64(-1) + if size >= 0 { + info := ObjectInfo{Size: size} + wantSize = info.EncryptedSize() + } + // do not try to verify encrypted content + hashReader, err = hash.NewReader(ctx, etag.Wrap(reader, hashReader), wantSize, "", "", actualSize) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if err := hashReader.AddChecksum(r, true); err != nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidChecksum), r.URL) + return + } + + pReader, err = pReader.WithEncryption(hashReader, &objectEncryptionKey) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + if idxCb != nil { + idxCb = compressionIndexEncrypter(objectEncryptionKey, idxCb) + } + opts.EncryptFn = metadataEncrypter(objectEncryptionKey) + } + } + opts.IndexCB = idxCb + + opts.ReplicationRequest = sourceReplReq + putObjectPart := objectAPI.PutObjectPart + + partInfo, err := putObjectPart(ctx, bucket, object, uploadID, partID, pReader, opts) + if err != nil { + // Verify if the underlying error is signature mismatch. + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + etag := partInfo.ETag + if kind, encrypted := crypto.IsEncrypted(mi.UserDefined); encrypted { + switch kind { + case crypto.S3KMS: + w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS) + w.Header().Set(xhttp.AmzServerSideEncryptionKmsID, mi.KMSKeyID()) + if kmsCtx, ok := mi.UserDefined[crypto.MetaContext]; ok { + w.Header().Set(xhttp.AmzServerSideEncryptionKmsContext, kmsCtx) + } + if len(etag) >= 32 && strings.Count(etag, "-") != 1 { + etag = etag[len(etag)-32:] + } + case crypto.S3: + w.Header().Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES) + etag, _ = DecryptETag(objectEncryptionKey, ObjectInfo{ETag: etag}) + case crypto.SSEC: + w.Header().Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerAlgorithm)) + w.Header().Set(xhttp.AmzServerSideEncryptionCustomerKeyMD5, r.Header.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5)) + + if len(etag) >= 32 && strings.Count(etag, "-") != 1 { + etag = etag[len(etag)-32:] + } + } + } + + // We must not use the http.Header().Set method here because some (broken) + // clients expect the ETag header key to be literally "ETag" - not "Etag" (case-sensitive). + // Therefore, we have to set the ETag directly as map entry. + w.Header()[xhttp.ETag] = []string{"\"" + etag + "\""} + hash.TransferChecksumHeader(w, r) + + writeSuccessResponseHeadersOnly(w) +} + +// CompleteMultipartUploadHandler - Complete multipart upload. +func (api objectAPIHandlers) CompleteMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "CompleteMultipartUpload") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.PutObjectAction, bucket, object); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Get upload id. + uploadID, _, _, _, s3Error := getObjectResources(r.Form) + if s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // Content-Length is required and should be non-zero + if r.ContentLength <= 0 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingPart), r.URL) + return + } + + complMultipartUpload := &CompleteMultipartUpload{} + if err = xmlDecoder(r.Body, complMultipartUpload, r.ContentLength); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if len(complMultipartUpload.Parts) == 0 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrMissingPart), r.URL) + return + } + + if !sort.SliceIsSorted(complMultipartUpload.Parts, func(i, j int) bool { + return complMultipartUpload.Parts[i].PartNumber < complMultipartUpload.Parts[j].PartNumber + }) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidPartOrder), r.URL) + return + } + + // Reject retention or governance headers if set, CompleteMultipartUpload spec + // does not use these headers, and should not be passed down to checkPutObjectLockAllowed + if objectlock.IsObjectLockRequested(r.Header) || objectlock.IsObjectLockGovernanceBypassSet(r.Header) { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRequest), r.URL) + return + } + + if _, _, _, s3Err := checkPutObjectLockAllowed(ctx, r, bucket, object, objectAPI.GetObjectInfo, ErrNone, ErrNone); s3Err != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Err), r.URL) + return + } + + completeMultiPartUpload := objectAPI.CompleteMultipartUpload + + versioned := globalBucketVersioningSys.PrefixEnabled(bucket, object) + suspended := globalBucketVersioningSys.PrefixSuspended(bucket, object) + os := newObjSweeper(bucket, object).WithVersioning(versioned, suspended) + if !globalTierConfigMgr.Empty() { + // Get appropriate object info to identify the remote object to delete + goiOpts := os.GetOpts() + if goi, gerr := objectAPI.GetObjectInfo(ctx, bucket, object, goiOpts); gerr == nil { + os.SetTransitionState(goi.TransitionedObject) + } + } + + opts, err := completeMultipartOpts(ctx, r, bucket, object) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + opts.Versioned = versioned + opts.VersionSuspended = suspended + + // First, we compute the ETag of the multipart object. + // The ETag of a multi-part object is always: + // ETag := MD5(ETag_p1, ETag_p2, ...)+"-N" (N being the number of parts) + // + // This is independent of encryption. An encrypted multipart + // object also has an ETag that is the MD5 of its part ETags. + // The fact the in case of encryption the ETag of a part is + // not the MD5 of the part content does not change that. + var completeETags []etag.ETag + for _, part := range complMultipartUpload.Parts { + ETag, err := etag.Parse(part.ETag) + if err != nil { + continue + } + completeETags = append(completeETags, ETag) + } + multipartETag := etag.Multipart(completeETags...) + opts.UserDefined["etag"] = multipartETag.String() + + if r.Header.Get(xhttp.IfMatch) != "" { + opts.HasIfMatch = true + } + if opts.PreserveETag != "" || + r.Header.Get(xhttp.IfMatch) != "" || + r.Header.Get(xhttp.IfNoneMatch) != "" { + opts.CheckPrecondFn = func(oi ObjectInfo) bool { + if _, err := DecryptObjectInfo(&oi, r); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return true + } + return checkPreconditionsPUT(ctx, w, r, oi, opts) + } + } + + objInfo, err := completeMultiPartUpload(ctx, bucket, object, uploadID, complMultipartUpload.Parts, opts) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + opts.EncryptFn, err = objInfo.metadataEncryptFn(r.Header) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + if r.Header.Get(xMinIOExtract) == "true" && HasSuffix(object, archiveExt) { + opts := ObjectOptions{VersionID: objInfo.VersionID, MTime: objInfo.ModTime} + if _, err := updateObjectMetadataWithZipInfo(ctx, objectAPI, bucket, object, opts); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + + setPutObjHeaders(w, objInfo, false, r.Header) + if dsc := mustReplicate(ctx, bucket, object, objInfo.getMustReplicateOptions(replication.ObjectReplicationType, opts)); dsc.ReplicateAny() { + scheduleReplication(ctx, objInfo, objectAPI, dsc, replication.ObjectReplicationType) + } + if _, ok := r.Header[xhttp.MinIOSourceReplicationRequest]; ok { + actualSize, _ := objInfo.GetActualSize() + defer globalReplicationStats.Load().UpdateReplicaStat(bucket, actualSize) + } + + // Get object location. + location := getObjectLocation(r, globalDomainNames, bucket, object) + // Generate complete multipart response. + response := generateCompleteMultipartUploadResponse(bucket, object, location, objInfo, r.Header) + encodedSuccessResponse := encodeResponse(response) + + // Write success response. + writeSuccessResponseXML(w, encodedSuccessResponse) + + // Notify object created event. + evt := eventArgs{ + EventName: event.ObjectCreatedCompleteMultipartUpload, + BucketName: bucket, + Object: objInfo, + ReqParams: extractReqParams(r), + RespElements: extractRespElements(w), + UserAgent: r.UserAgent(), + Host: handlers.GetSourceIP(r), + } + sendEvent(evt) + + if objInfo.NumVersions > int(scannerExcessObjectVersions.Load()) { + evt.EventName = event.ObjectManyVersions + sendEvent(evt) + + auditLogInternal(context.Background(), AuditLogOptions{ + Event: "scanner:manyversions", + APIName: "CompleteMultipartUpload", + Bucket: objInfo.Bucket, + Object: objInfo.Name, + VersionID: objInfo.VersionID, + Status: http.StatusText(http.StatusOK), + }) + } + + // Remove the transitioned object whose object version is being overwritten. + if !globalTierConfigMgr.Empty() { + // Schedule object for immediate transition if eligible. + enqueueTransitionImmediate(objInfo, lcEventSrc_s3CompleteMultipartUpload) + os.Sweep() + } +} + +// AbortMultipartUploadHandler - Abort multipart upload +func (api objectAPIHandlers) AbortMultipartUploadHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "AbortMultipartUpload") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + abortMultipartUpload := objectAPI.AbortMultipartUpload + + if s3Error := checkRequestAuthType(ctx, r, policy.AbortMultipartUploadAction, bucket, object); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + uploadID, _, _, _, s3Error := getObjectResources(r.Form) + if s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + opts := ObjectOptions{} + if err := abortMultipartUpload(ctx, bucket, object, uploadID, opts); err != nil { + switch err.(type) { + case InvalidUploadID: + // Do not have return an error for non-existent upload-id + default: + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + + writeSuccessNoContent(w) +} + +// ListObjectPartsHandler - List object parts +func (api objectAPIHandlers) ListObjectPartsHandler(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "ListObjectParts") + + defer logger.AuditLog(ctx, w, r, mustGetClaimsFromToken(r)) + + vars := mux.Vars(r) + bucket := vars["bucket"] + object, err := unescapePath(vars["object"]) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + objectAPI := api.ObjectAPI() + if objectAPI == nil { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrServerNotInitialized), r.URL) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.ListMultipartUploadPartsAction, bucket, object); s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + uploadID, partNumberMarker, maxParts, encodingType, s3Error := getObjectResources(r.Form) + if s3Error != ErrNone { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + if partNumberMarker < 0 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidPartNumberMarker), r.URL) + return + } + if maxParts < 0 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidMaxParts), r.URL) + return + } + + opts := ObjectOptions{} + listPartsInfo, err := objectAPI.ListObjectParts(ctx, bucket, object, uploadID, partNumberMarker, maxParts, opts) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + // We have to adjust the size of encrypted parts since encrypted parts + // are slightly larger due to encryption overhead. + // Further, we have to adjust the ETags of parts when using SSE-S3. + // Due to AWS S3, SSE-S3 encrypted parts return the plaintext ETag + // being the content MD5 of that particular part. This is not the + // case for SSE-C and SSE-KMS objects. + if kind, ok := crypto.IsEncrypted(listPartsInfo.UserDefined); ok { + var objectEncryptionKey []byte + if kind == crypto.S3 { + objectEncryptionKey, err = decryptObjectMeta(nil, bucket, object, listPartsInfo.UserDefined) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } + for i, p := range listPartsInfo.Parts { + listPartsInfo.Parts[i].ETag = tryDecryptETag(objectEncryptionKey, p.ETag, kind == crypto.S3) + listPartsInfo.Parts[i].Size = p.ActualSize + } + } else if _, ok := listPartsInfo.UserDefined[ReservedMetadataPrefix+"compression"]; ok { + for i, p := range listPartsInfo.Parts { + listPartsInfo.Parts[i].Size = p.ActualSize + } + } + + response := generateListPartsResponse(listPartsInfo, encodingType) + encodedSuccessResponse := encodeResponse(response) + + // Write success response. + writeSuccessResponseXML(w, encodedSuccessResponse) +} diff --git a/cmd/object_api_suite_test.go b/cmd/object_api_suite_test.go new file mode 100644 index 00000000..3377ad20 --- /dev/null +++ b/cmd/object_api_suite_test.go @@ -0,0 +1,947 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "fmt" + "io" + "math/rand" + "strconv" + "strings" + "testing" + "time" + + "github.com/dustin/go-humanize" + "github.com/minio/minio/internal/kms" +) + +// Return pointer to testOneByteReadEOF{} +func newTestReaderEOF(data []byte) io.Reader { + return &testOneByteReadEOF{false, data} +} + +// OneByteReadEOF - implements io.Reader which returns 1 byte along with io.EOF error. +type testOneByteReadEOF struct { + eof bool + data []byte +} + +func (r *testOneByteReadEOF) Read(p []byte) (n int, err error) { + if r.eof { + return 0, io.EOF + } + n = copy(p, r.data) + r.eof = true + return n, io.EOF +} + +// Return pointer to testOneByteReadNoEOF{} +func newTestReaderNoEOF(data []byte) io.Reader { + return &testOneByteReadNoEOF{false, data} +} + +// testOneByteReadNoEOF - implements io.Reader which returns 1 byte and nil error, but +// returns io.EOF on the next Read(). +type testOneByteReadNoEOF struct { + eof bool + data []byte +} + +func (r *testOneByteReadNoEOF) Read(p []byte) (n int, err error) { + if r.eof { + return 0, io.EOF + } + n = copy(p, r.data) + r.eof = true + return n, nil +} + +// Wrapper for calling testMakeBucket for both Erasure and FS. +func TestMakeBucket(t *testing.T) { + ExecObjectLayerTest(t, testMakeBucket) +} + +// Tests validate bucket creation. +func testMakeBucket(obj ObjectLayer, instanceType string, t TestErrHandler) { + err := obj.MakeBucket(context.Background(), "bucket-unknown", MakeBucketOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } +} + +// Wrapper for calling testMultipartObjectCreation for both Erasure and FS. +func TestMultipartObjectCreation(t *testing.T) { + ExecExtendedObjectLayerTest(t, testMultipartObjectCreation) +} + +// Tests validate creation of part files during Multipart operation. +func testMultipartObjectCreation(obj ObjectLayer, instanceType string, t TestErrHandler) { + var opts ObjectOptions + err := obj.MakeBucket(context.Background(), "bucket", MakeBucketOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + res, err := obj.NewMultipartUpload(context.Background(), "bucket", "key", opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + uploadID := res.UploadID + + // Create a byte array of 5MiB. + data := bytes.Repeat([]byte("0123456789abcdef"), 5*humanize.MiByte/16) + completedParts := CompleteMultipartUpload{} + for i := 1; i <= 10; i++ { + expectedETaghex := getMD5Hash(data) + + var calcPartInfo PartInfo + calcPartInfo, err = obj.PutObjectPart(context.Background(), "bucket", "key", uploadID, i, mustGetPutObjReader(t, bytes.NewBuffer(data), int64(len(data)), expectedETaghex, ""), opts) + if err != nil { + t.Errorf("%s: %s", instanceType, err) + } + if calcPartInfo.ETag != expectedETaghex { + t.Errorf("MD5 Mismatch") + } + completedParts.Parts = append(completedParts.Parts, CompletePart{ + PartNumber: i, + ETag: calcPartInfo.ETag, + }) + } + objInfo, err := obj.CompleteMultipartUpload(context.Background(), "bucket", "key", uploadID, completedParts.Parts, ObjectOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if objInfo.ETag != "7d364cb728ce42a74a96d22949beefb2-10" { + t.Errorf("Md5 mismtch") + } +} + +// Wrapper for calling testMultipartObjectAbort for both Erasure and FS. +func TestMultipartObjectAbort(t *testing.T) { + ExecObjectLayerTest(t, testMultipartObjectAbort) +} + +// Tests validate abortion of Multipart operation. +func testMultipartObjectAbort(obj ObjectLayer, instanceType string, t TestErrHandler) { + var opts ObjectOptions + err := obj.MakeBucket(context.Background(), "bucket", MakeBucketOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + res, err := obj.NewMultipartUpload(context.Background(), "bucket", "key", opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + uploadID := res.UploadID + + parts := make(map[int]string) + metadata := make(map[string]string) + for i := 1; i <= 10; i++ { + randomPerm := rand.Perm(10) + randomString := "" + for _, num := range randomPerm { + randomString += strconv.Itoa(num) + } + + expectedETaghex := getMD5Hash([]byte(randomString)) + + metadata["md5"] = expectedETaghex + var calcPartInfo PartInfo + calcPartInfo, err = obj.PutObjectPart(context.Background(), "bucket", "key", uploadID, i, mustGetPutObjReader(t, bytes.NewBufferString(randomString), int64(len(randomString)), expectedETaghex, ""), opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if calcPartInfo.ETag != expectedETaghex { + t.Errorf("Md5 Mismatch") + } + parts[i] = expectedETaghex + } + err = obj.AbortMultipartUpload(context.Background(), "bucket", "key", uploadID, ObjectOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } +} + +// Wrapper for calling testMultipleObjectCreation for both Erasure and FS. +func TestMultipleObjectCreation(t *testing.T) { + ExecExtendedObjectLayerTest(t, testMultipleObjectCreation) +} + +// Tests validate object creation. +func testMultipleObjectCreation(obj ObjectLayer, instanceType string, t TestErrHandler) { + objects := make(map[string][]byte) + var opts ObjectOptions + err := obj.MakeBucket(context.Background(), "bucket", MakeBucketOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + for i := range 10 { + randomPerm := rand.Perm(100) + randomString := "" + for _, num := range randomPerm { + randomString += strconv.Itoa(num) + } + + expectedETaghex := getMD5Hash([]byte(randomString)) + + key := "obj" + strconv.Itoa(i) + objects[key] = []byte(randomString) + metadata := make(map[string]string) + metadata["etag"] = expectedETaghex + var objInfo ObjectInfo + objInfo, err = obj.PutObject(context.Background(), "bucket", key, mustGetPutObjReader(t, bytes.NewBufferString(randomString), int64(len(randomString)), metadata["etag"], ""), ObjectOptions{UserDefined: metadata}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if objInfo.ETag != expectedETaghex { + t.Errorf("Md5 Mismatch") + } + } + + for key, value := range objects { + var byteBuffer bytes.Buffer + err = GetObject(context.Background(), obj, "bucket", key, 0, int64(len(value)), &byteBuffer, "", opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if !bytes.Equal(byteBuffer.Bytes(), value) { + t.Errorf("%s: Mismatch of GetObject data with the expected one.", instanceType) + } + + objInfo, err := obj.GetObjectInfo(context.Background(), "bucket", key, opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if objInfo.Size != int64(len(value)) { + t.Errorf("%s: Size mismatch of the GetObject data.", instanceType) + } + } +} + +// Wrapper for calling TestPaging for both Erasure and FS. +func TestPaging(t *testing.T) { + ExecObjectLayerTest(t, testPaging) +} + +// Tests validate creation of objects and the order of listing using various filters for ListObjects operation. +func testPaging(obj ObjectLayer, instanceType string, t TestErrHandler) { + obj.MakeBucket(context.Background(), "bucket", MakeBucketOptions{}) + result, err := obj.ListObjects(context.Background(), "bucket", "", "", "", 0) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if len(result.Objects) != 0 { + t.Errorf("%s: Number of objects in the result different from expected value.", instanceType) + } + if result.IsTruncated { + t.Errorf("%s: Expected IsTruncated to be `false`, but instead found it to be `%v`", instanceType, result.IsTruncated) + } + + uploadContent := "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed." + var opts ObjectOptions + // check before paging occurs. + for i := range 5 { + key := "obj" + strconv.Itoa(i) + _, err = obj.PutObject(context.Background(), "bucket", key, mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + + result, err = obj.ListObjects(context.Background(), "bucket", "", "", "", 5) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if len(result.Objects) != i+1 { + t.Errorf("%s: Expected length of objects to be %d, instead found to be %d", instanceType, len(result.Objects), i+1) + } + if result.IsTruncated { + t.Errorf("%s: Expected IsTruncated to be `false`, but instead found it to be `%v`", instanceType, result.IsTruncated) + } + } + + // check after paging occurs pages work. + for i := 6; i <= 10; i++ { + key := "obj" + strconv.Itoa(i) + _, err = obj.PutObject(context.Background(), "bucket", key, mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + result, err = obj.ListObjects(context.Background(), "bucket", "obj", "", "", 5) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if len(result.Objects) != 5 { + t.Errorf("%s: Expected length of objects to be %d, instead found to be %d", instanceType, 5, len(result.Objects)) + } + if !result.IsTruncated { + t.Errorf("%s: Expected IsTruncated to be `true`, but instead found it to be `%v`", instanceType, result.IsTruncated) + } + } + // check paging with prefix at end returns less objects. + { + _, err = obj.PutObject(context.Background(), "bucket", "newPrefix", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + _, err = obj.PutObject(context.Background(), "bucket", "newPrefix2", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + result, err = obj.ListObjects(context.Background(), "bucket", "new", "", "", 5) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if len(result.Objects) != 2 { + t.Errorf("%s: Expected length of objects to be %d, instead found to be %d", instanceType, 2, len(result.Objects)) + } + } + + // check ordering of pages. + { + result, err = obj.ListObjects(context.Background(), "bucket", "", "", "", 1000) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if result.Objects[0].Name != "newPrefix" { + t.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix", result.Objects[0].Name) + } + if result.Objects[1].Name != "newPrefix2" { + t.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix", result.Objects[1].Name) + } + if result.Objects[2].Name != "obj0" { + t.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix", result.Objects[2].Name) + } + if result.Objects[3].Name != "obj1" { + t.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix", result.Objects[3].Name) + } + if result.Objects[4].Name != "obj10" { + t.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix", result.Objects[4].Name) + } + } + + // check delimited results with delimiter and prefix. + { + _, err = obj.PutObject(context.Background(), "bucket", "this/is/delimited", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + _, err = obj.PutObject(context.Background(), "bucket", "this/is/also/a/delimited/file", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + result, err = obj.ListObjects(context.Background(), "bucket", "this/is/", "", SlashSeparator, 10) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if len(result.Objects) != 1 { + t.Errorf("%s: Expected the number of objects in the result to be %d, but instead found %d", instanceType, 1, len(result.Objects)) + } + if result.Prefixes[0] != "this/is/also/" { + t.Errorf("%s: Expected prefix to be `%s`, but instead found `%s`", instanceType, "this/is/also/", result.Prefixes[0]) + } + } + + // check delimited results with delimiter without prefix. + { + result, err = obj.ListObjects(context.Background(), "bucket", "", "", SlashSeparator, 1000) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + + if result.Objects[0].Name != "newPrefix" { + t.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix", result.Objects[0].Name) + } + if result.Objects[1].Name != "newPrefix2" { + t.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix", result.Objects[1].Name) + } + if result.Objects[2].Name != "obj0" { + t.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix", result.Objects[2].Name) + } + if result.Objects[3].Name != "obj1" { + t.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix", result.Objects[3].Name) + } + if result.Objects[4].Name != "obj10" { + t.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix", result.Objects[4].Name) + } + if result.Prefixes[0] != "this/" { + t.Errorf("%s: Expected the prefix to be `%s`, but instead found `%s`", instanceType, "this/", result.Prefixes[0]) + } + } + + // check results with Marker. + { + result, err = obj.ListObjects(context.Background(), "bucket", "", "newPrefix", "", 3) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if result.Objects[0].Name != "newPrefix2" { + t.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix2", result.Objects[0].Name) + } + if result.Objects[1].Name != "obj0" { + t.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "obj0", result.Objects[1].Name) + } + if result.Objects[2].Name != "obj1" { + t.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "obj1", result.Objects[2].Name) + } + } + // check ordering of results with prefix. + { + result, err = obj.ListObjects(context.Background(), "bucket", "obj", "", "", 1000) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if result.Objects[0].Name != "obj0" { + t.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "obj0", result.Objects[0].Name) + } + if result.Objects[1].Name != "obj1" { + t.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "obj1", result.Objects[1].Name) + } + if result.Objects[2].Name != "obj10" { + t.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "obj10", result.Objects[2].Name) + } + if result.Objects[3].Name != "obj2" { + t.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "obj2", result.Objects[3].Name) + } + if result.Objects[4].Name != "obj3" { + t.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "obj3", result.Objects[4].Name) + } + } + // check ordering of results with prefix and no paging. + { + result, err = obj.ListObjects(context.Background(), "bucket", "new", "", "", 5) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if result.Objects[0].Name != "newPrefix" { + t.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix", result.Objects[0].Name) + } + if result.Objects[1].Name != "newPrefix2" { + t.Errorf("%s: Expected the object name to be `%s`, but instead found `%s`", instanceType, "newPrefix2", result.Objects[0].Name) + } + } + + // check paging works. + ag := []string{"a", "b", "c", "d", "e", "f", "g"} + checkObjCount := make(map[string]int) + for i := range 7 { + dirName := strings.Repeat(ag[i], 3) + key := fmt.Sprintf("testPrefix/%s/obj%s", dirName, dirName) + checkObjCount[key]++ + _, err = obj.PutObject(context.Background(), "bucket", key, mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + } + { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + token := "" + for ctx.Err() == nil { + result, err := obj.ListObjectsV2(ctx, "bucket", "testPrefix", token, "", 2, false, "") + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + token = result.NextContinuationToken + if len(result.Objects) == 0 { + break + } + for _, obj := range result.Objects { + checkObjCount[obj.Name]-- + } + if token == "" { + break + } + } + for key, value := range checkObjCount { + if value != 0 { + t.Errorf("%s: Expected value of objects to be %d, instead found to be %d", instanceType, 0, value) + } + delete(checkObjCount, key) + } + } +} + +// Wrapper for calling testObjectOverwriteWorks for both Erasure and FS. +func TestObjectOverwriteWorks(t *testing.T) { + ExecObjectLayerTest(t, testObjectOverwriteWorks) +} + +// Tests validate overwriting of an existing object. +func testObjectOverwriteWorks(obj ObjectLayer, instanceType string, t TestErrHandler) { + err := obj.MakeBucket(context.Background(), "bucket", MakeBucketOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + + var opts ObjectOptions + uploadContent := "The list of parts was not in ascending order. The parts list must be specified in order by part number." + length := int64(len(uploadContent)) + _, err = obj.PutObject(context.Background(), "bucket", "object", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), length, "", ""), opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + + uploadContent = "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed." + length = int64(len(uploadContent)) + _, err = obj.PutObject(context.Background(), "bucket", "object", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), length, "", ""), opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + + var bytesBuffer bytes.Buffer + err = GetObject(context.Background(), obj, "bucket", "object", 0, length, &bytesBuffer, "", opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if bytesBuffer.String() != "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed." { + t.Errorf("%s: Invalid upload ID error mismatch.", instanceType) + } +} + +// Wrapper for calling testNonExistentBucketOperations for both Erasure and FS. +func TestNonExistentBucketOperations(t *testing.T) { + ExecObjectLayerTest(t, testNonExistentBucketOperations) +} + +// Tests validate that bucket operation on non-existent bucket fails. +func testNonExistentBucketOperations(obj ObjectLayer, instanceType string, t TestErrHandler) { + var opts ObjectOptions + _, err := obj.PutObject(context.Background(), "bucket1", "object", mustGetPutObjReader(t, bytes.NewBufferString("one"), int64(len("one")), "", ""), opts) + if err == nil { + t.Fatal("Expected error but found nil") + } + if err.Error() != "Bucket not found: bucket1" { + t.Errorf("%s: Expected the error msg to be `%s`, but instead found `%s`", instanceType, "Bucket not found: bucket1", err.Error()) + } +} + +// Wrapper for calling testBucketRecreateFails for both Erasure and FS. +func TestBucketRecreateFails(t *testing.T) { + ExecObjectLayerTest(t, testBucketRecreateFails) +} + +// Tests validate that recreation of the bucket fails. +func testBucketRecreateFails(obj ObjectLayer, instanceType string, t TestErrHandler) { + err := obj.MakeBucket(context.Background(), "string", MakeBucketOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + err = obj.MakeBucket(context.Background(), "string", MakeBucketOptions{}) + if err == nil { + t.Fatalf("%s: Expected error but found nil.", instanceType) + } + + if err.Error() != "Bucket exists: string" { + t.Errorf("%s: Expected the error message to be `%s`, but instead found `%s`", instanceType, "Bucket exists: string", err.Error()) + } +} + +func enableCompression(t *testing.T, encrypt bool, mimeTypes []string, extensions []string) { + // Enable compression and exec... + globalCompressConfigMu.Lock() + globalCompressConfig.Enabled = true + globalCompressConfig.MimeTypes = mimeTypes + globalCompressConfig.Extensions = extensions + globalCompressConfig.AllowEncrypted = encrypt + globalCompressConfigMu.Unlock() + if encrypt { + globalAutoEncryption = encrypt + KMS, err := kms.ParseSecretKey("my-minio-key:5lF+0pJM0OWwlQrvK2S/I7W9mO4a6rJJI7wzj7v09cw=") + if err != nil { + t.Fatal(err) + } + GlobalKMS = KMS + } +} + +func enableEncryption(t *testing.T) { + // Exec with default settings... + globalCompressConfigMu.Lock() + globalCompressConfig.Enabled = false + globalCompressConfigMu.Unlock() + + globalAutoEncryption = true + KMS, err := kms.ParseSecretKey("my-minio-key:5lF+0pJM0OWwlQrvK2S/I7W9mO4a6rJJI7wzj7v09cw=") + if err != nil { + t.Fatal(err) + } + GlobalKMS = KMS +} + +func resetCompressEncryption() { + // Reset... + globalCompressConfigMu.Lock() + globalCompressConfig.Enabled = false + globalCompressConfig.AllowEncrypted = false + globalCompressConfigMu.Unlock() + globalAutoEncryption = false + GlobalKMS = nil +} + +func execExtended(t *testing.T, fn func(t *testing.T, init func(), bucketOptions MakeBucketOptions)) { + // Exec with default settings... + resetCompressEncryption() + t.Run("default", func(t *testing.T) { + fn(t, nil, MakeBucketOptions{}) + }) + t.Run("default+versioned", func(t *testing.T) { + fn(t, nil, MakeBucketOptions{VersioningEnabled: true}) + }) + + t.Run("compressed", func(t *testing.T) { + fn(t, func() { + resetCompressEncryption() + enableCompression(t, false, []string{"*"}, []string{"*"}) + }, MakeBucketOptions{}) + }) + t.Run("compressed+versioned", func(t *testing.T) { + fn(t, func() { + resetCompressEncryption() + enableCompression(t, false, []string{"*"}, []string{"*"}) + }, MakeBucketOptions{ + VersioningEnabled: true, + }) + }) + + t.Run("encrypted", func(t *testing.T) { + fn(t, func() { + resetCompressEncryption() + enableEncryption(t) + }, MakeBucketOptions{}) + }) + t.Run("encrypted+versioned", func(t *testing.T) { + fn(t, func() { + resetCompressEncryption() + enableEncryption(t) + }, MakeBucketOptions{ + VersioningEnabled: true, + }) + }) + + t.Run("compressed+encrypted", func(t *testing.T) { + fn(t, func() { + resetCompressEncryption() + enableCompression(t, true, []string{"*"}, []string{"*"}) + }, MakeBucketOptions{}) + }) + t.Run("compressed+encrypted+versioned", func(t *testing.T) { + fn(t, func() { + resetCompressEncryption() + enableCompression(t, true, []string{"*"}, []string{"*"}) + }, MakeBucketOptions{ + VersioningEnabled: true, + }) + }) +} + +// ExecExtendedObjectLayerTest will execute the tests with combinations of encrypted & compressed. +// This can be used to test functionality when reading and writing data. +func ExecExtendedObjectLayerTest(t *testing.T, objTest objTestType) { + execExtended(t, func(t *testing.T, init func(), bucketOptions MakeBucketOptions) { + ExecObjectLayerTest(t, objTest) + }) +} + +// Wrapper for calling testPutObject for both Erasure and FS. +func TestPutObject(t *testing.T) { + ExecExtendedObjectLayerTest(t, testPutObject) +} + +// Tests validate PutObject without prefix. +func testPutObject(obj ObjectLayer, instanceType string, t TestErrHandler) { + content := []byte("testcontent") + length := int64(len(content)) + readerEOF := newTestReaderEOF(content) + readerNoEOF := newTestReaderNoEOF(content) + err := obj.MakeBucket(context.Background(), "bucket", MakeBucketOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + + var bytesBuffer1 bytes.Buffer + var opts ObjectOptions + _, err = obj.PutObject(context.Background(), "bucket", "object", mustGetPutObjReader(t, readerEOF, length, "", ""), opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + err = GetObject(context.Background(), obj, "bucket", "object", 0, length, &bytesBuffer1, "", opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if len(bytesBuffer1.Bytes()) != len(content) { + t.Errorf("%s: Expected content length to be `%d`, but instead found `%d`", instanceType, len(content), len(bytesBuffer1.Bytes())) + } + + var bytesBuffer2 bytes.Buffer + _, err = obj.PutObject(context.Background(), "bucket", "object", mustGetPutObjReader(t, readerNoEOF, length, "", ""), opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + err = GetObject(context.Background(), obj, "bucket", "object", 0, length, &bytesBuffer2, "", opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if len(bytesBuffer2.Bytes()) != len(content) { + t.Errorf("%s: Expected content length to be `%d`, but instead found `%d`", instanceType, len(content), len(bytesBuffer2.Bytes())) + } +} + +// Wrapper for calling testPutObjectInSubdir for both Erasure and FS. +func TestPutObjectInSubdir(t *testing.T) { + ExecExtendedObjectLayerTest(t, testPutObjectInSubdir) +} + +// Tests validate PutObject with subdirectory prefix. +func testPutObjectInSubdir(obj ObjectLayer, instanceType string, t TestErrHandler) { + err := obj.MakeBucket(context.Background(), "bucket", MakeBucketOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + + var opts ObjectOptions + uploadContent := `The specified multipart upload does not exist. The upload ID might be invalid, or the multipart + upload might have been aborted or completed.` + length := int64(len(uploadContent)) + _, err = obj.PutObject(context.Background(), "bucket", "dir1/dir2/object", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), length, "", ""), opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + + var bytesBuffer bytes.Buffer + err = GetObject(context.Background(), obj, "bucket", "dir1/dir2/object", 0, length, &bytesBuffer, "", opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if len(bytesBuffer.Bytes()) != len(uploadContent) { + t.Errorf("%s: Expected length of downloaded data to be `%d`, but instead found `%d`", + instanceType, len(uploadContent), len(bytesBuffer.Bytes())) + } +} + +// Wrapper for calling testListBuckets for both Erasure and FS. +func TestListBuckets(t *testing.T) { + ExecObjectLayerTest(t, testListBuckets) +} + +// Tests validate ListBuckets. +func testListBuckets(obj ObjectLayer, instanceType string, t TestErrHandler) { + // test empty list. + buckets, err := obj.ListBuckets(context.Background(), BucketOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if len(buckets) != 0 { + t.Errorf("%s: Expected number of bucket to be `%d`, but instead found `%d`", instanceType, 0, len(buckets)) + } + + // add one and test exists. + err = obj.MakeBucket(context.Background(), "bucket1", MakeBucketOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + + buckets, err = obj.ListBuckets(context.Background(), BucketOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if len(buckets) != 1 { + t.Errorf("%s: Expected number of bucket to be `%d`, but instead found `%d`", instanceType, 1, len(buckets)) + } + + // add two and test exists. + err = obj.MakeBucket(context.Background(), "bucket2", MakeBucketOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + + buckets, err = obj.ListBuckets(context.Background(), BucketOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if len(buckets) != 2 { + t.Errorf("%s: Expected number of bucket to be `%d`, but instead found `%d`", instanceType, 2, len(buckets)) + } + + // add three and test exists + prefix. + err = obj.MakeBucket(context.Background(), "bucket22", MakeBucketOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + + buckets, err = obj.ListBuckets(context.Background(), BucketOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if len(buckets) != 3 { + t.Errorf("%s: Expected number of bucket to be `%d`, but instead found `%d`", instanceType, 3, len(buckets)) + } +} + +// Wrapper for calling testListBucketsOrder for both Erasure and FS. +func TestListBucketsOrder(t *testing.T) { + ExecObjectLayerTest(t, testListBucketsOrder) +} + +// Tests validate the order of result of ListBuckets. +func testListBucketsOrder(obj ObjectLayer, instanceType string, t TestErrHandler) { + // if implementation contains a map, order of map keys will vary. + // this ensures they return in the same order each time. + // add one and test exists. + err := obj.MakeBucket(context.Background(), "bucket1", MakeBucketOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + err = obj.MakeBucket(context.Background(), "bucket2", MakeBucketOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + buckets, err := obj.ListBuckets(context.Background(), BucketOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + if len(buckets) != 2 { + t.Errorf("%s: Expected number of bucket to be `%d`, but instead found `%d`", instanceType, 2, len(buckets)) + } + + if buckets[0].Name != "bucket1" { + t.Errorf("%s: Expected bucket name to be `%s`, but instead found `%s`", instanceType, "bucket1", buckets[0].Name) + } + if buckets[1].Name != "bucket2" { + t.Errorf("%s: Expected bucket name to be `%s`, but instead found `%s`", instanceType, "bucket2", buckets[1].Name) + } +} + +// Wrapper for calling testListObjectsTestsForNonExistentBucket for both Erasure and FS. +func TestListObjectsTestsForNonExistentBucket(t *testing.T) { + ExecObjectLayerTest(t, testListObjectsTestsForNonExistentBucket) +} + +// Tests validate that ListObjects operation on a non-existent bucket fails as expected. +func testListObjectsTestsForNonExistentBucket(obj ObjectLayer, instanceType string, t TestErrHandler) { + result, err := obj.ListObjects(context.Background(), "bucket", "", "", "", 1000) + if err == nil { + t.Fatalf("%s: Expected error but found nil.", instanceType) + } + if len(result.Objects) != 0 { + t.Fatalf("%s: Expected number of objects in the result to be `%d`, but instead found `%d`", instanceType, 0, len(result.Objects)) + } + if result.IsTruncated { + t.Fatalf("%s: Expected IsTruncated to be `false`, but instead found it to be `%v`", instanceType, result.IsTruncated) + } + if err.Error() != "Bucket not found: bucket" { + t.Errorf("%s: Expected the error msg to be `%s`, but instead found `%s`", instanceType, "Bucket not found: bucket", err.Error()) + } +} + +// Wrapper for calling testNonExistentObjectInBucket for both Erasure and FS. +func TestNonExistentObjectInBucket(t *testing.T) { + ExecObjectLayerTest(t, testNonExistentObjectInBucket) +} + +// Tests validate that GetObject fails on a non-existent bucket as expected. +func testNonExistentObjectInBucket(obj ObjectLayer, instanceType string, t TestErrHandler) { + err := obj.MakeBucket(context.Background(), "bucket", MakeBucketOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + + _, err = obj.GetObjectInfo(context.Background(), "bucket", "dir1", ObjectOptions{}) + if err == nil { + t.Fatalf("%s: Expected error but found nil", instanceType) + } + if isErrObjectNotFound(err) { + if err.Error() != "Object not found: bucket/dir1" { + t.Errorf("%s: Expected the Error message to be `%s`, but instead found `%s`", instanceType, "Object not found: bucket/dir1", err.Error()) + } + } else { + if err.Error() != "fails" { + t.Errorf("%s: Expected the Error message to be `%s`, but instead found it to be `%s`", instanceType, "fails", err.Error()) + } + } +} + +// Wrapper for calling testGetDirectoryReturnsObjectNotFound for both Erasure and FS. +func TestGetDirectoryReturnsObjectNotFound(t *testing.T) { + ExecObjectLayerTest(t, testGetDirectoryReturnsObjectNotFound) +} + +// Tests validate that GetObject on an existing directory fails as expected. +func testGetDirectoryReturnsObjectNotFound(obj ObjectLayer, instanceType string, t TestErrHandler) { + bucketName := "bucket" + err := obj.MakeBucket(context.Background(), bucketName, MakeBucketOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + content := "One or more of the specified parts could not be found. The part might not have been uploaded, or the specified entity tag might not have matched the part's entity tag." + length := int64(len(content)) + var opts ObjectOptions + _, err = obj.PutObject(context.Background(), bucketName, "dir1/dir3/object", mustGetPutObjReader(t, bytes.NewBufferString(content), length, "", ""), opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + + testCases := []struct { + dir string + err error + }{ + { + dir: "dir1/", + err: ObjectNotFound{Bucket: bucketName, Object: "dir1/"}, + }, + { + dir: "dir1/dir3/", + err: ObjectNotFound{Bucket: bucketName, Object: "dir1/dir3/"}, + }, + } + + for i, testCase := range testCases { + _, expectedErr := obj.GetObjectInfo(context.Background(), bucketName, testCase.dir, opts) + if expectedErr != nil && expectedErr.Error() != testCase.err.Error() { + t.Errorf("Test %d, %s: Expected error %s, got %s", i+1, instanceType, testCase.err, expectedErr) + } + } +} + +// Wrapper for calling testContentType for both Erasure and FS. +func TestContentType(t *testing.T) { + ExecObjectLayerTest(t, testContentType) +} + +// Test content-type. +func testContentType(obj ObjectLayer, instanceType string, t TestErrHandler) { + err := obj.MakeBucket(context.Background(), "bucket", MakeBucketOptions{}) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + var opts ObjectOptions + uploadContent := "The specified multipart upload does not exist. The upload ID might be invalid, or the multipart upload might have been aborted or completed." + // Test empty. + _, err = obj.PutObject(context.Background(), "bucket", "minio.png", mustGetPutObjReader(t, bytes.NewBufferString(uploadContent), int64(len(uploadContent)), "", ""), opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + objInfo, err := obj.GetObjectInfo(context.Background(), "bucket", "minio.png", opts) + if err != nil { + t.Fatalf("%s: %s", instanceType, err) + } + + if objInfo.ContentType != "image/png" { + t.Errorf("%s: Expected Content type to be `%s`, but instead found `%s`", instanceType, "image/png", objInfo.ContentType) + } +} diff --git a/cmd/os-dirent_fileino.go b/cmd/os-dirent_fileino.go new file mode 100644 index 00000000..a3522e90 --- /dev/null +++ b/cmd/os-dirent_fileino.go @@ -0,0 +1,27 @@ +//go:build freebsd || openbsd || netbsd +// +build freebsd openbsd netbsd + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import "syscall" + +func direntInode(dirent *syscall.Dirent) uint64 { + return uint64(dirent.Fileno) +} diff --git a/cmd/os-dirent_ino.go b/cmd/os-dirent_ino.go new file mode 100644 index 00000000..09f2debe --- /dev/null +++ b/cmd/os-dirent_ino.go @@ -0,0 +1,28 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +//go:build (linux || darwin) && !appengine +// +build linux darwin +// +build !appengine + +package cmd + +import "syscall" + +func direntInode(dirent *syscall.Dirent) uint64 { + return dirent.Ino +} diff --git a/cmd/os-dirent_namelen_bsd.go b/cmd/os-dirent_namelen_bsd.go new file mode 100644 index 00000000..7ac6ecad --- /dev/null +++ b/cmd/os-dirent_namelen_bsd.go @@ -0,0 +1,27 @@ +//go:build darwin || freebsd || openbsd || netbsd +// +build darwin freebsd openbsd netbsd + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import "syscall" + +func direntNamlen(dirent *syscall.Dirent) (uint64, error) { + return uint64(dirent.Namlen), nil +} diff --git a/cmd/os-dirent_namelen_linux.go b/cmd/os-dirent_namelen_linux.go new file mode 100644 index 00000000..14c44239 --- /dev/null +++ b/cmd/os-dirent_namelen_linux.go @@ -0,0 +1,45 @@ +//go:build linux && !appengine +// +build linux,!appengine + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "fmt" + "syscall" + "unsafe" +) + +func direntNamlen(dirent *syscall.Dirent) (uint64, error) { + const fixedHdr = uint16(unsafe.Offsetof(syscall.Dirent{}.Name)) + nameBuf := (*[unsafe.Sizeof(dirent.Name)]byte)(unsafe.Pointer(&dirent.Name[0])) + const nameBufLen = uint16(len(nameBuf)) + limit := dirent.Reclen - fixedHdr + if limit > nameBufLen { + limit = nameBufLen + } + // Avoid bugs in long file names + // https://github.com/golang/tools/commit/5f9a5413737ba4b4f692214aebee582b47c8be74 + nameLen := bytes.IndexByte(nameBuf[:limit], 0) + if nameLen < 0 { + return 0, fmt.Errorf("failed to find terminating 0 byte in dirent") + } + return uint64(nameLen), nil +} diff --git a/cmd/os-instrumented.go b/cmd/os-instrumented.go new file mode 100644 index 00000000..9a369f86 --- /dev/null +++ b/cmd/os-instrumented.go @@ -0,0 +1,237 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "os" + "strings" + "sync/atomic" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/disk" + ioutilx "github.com/minio/minio/internal/ioutil" +) + +//go:generate stringer -type=osMetric -trimprefix=osMetric $GOFILE + +type osMetric uint8 + +const ( + osMetricRemoveAll osMetric = iota + osMetricMkdirAll + osMetricMkdir + osMetricRename + osMetricOpenFileW + osMetricOpenFileR + osMetricOpenFileWFd + osMetricOpenFileRFd + osMetricOpen + osMetricOpenFileDirectIO + osMetricLstat + osMetricRemove + osMetricStat + osMetricAccess + osMetricCreate + osMetricReadDirent + osMetricFdatasync + osMetricSync + + // .... add more + + osMetricLast +) + +var globalOSMetrics osMetrics + +func init() { + // Inject metrics. + ioutilx.OsOpenFile = OpenFile + ioutilx.OpenFileDirectIO = OpenFileDirectIO + ioutilx.OsOpen = Open +} + +type osMetrics struct { + // All fields must be accessed atomically and aligned. + operations [osMetricLast]uint64 + latency [osMetricLast]lockedLastMinuteLatency +} + +// time an os action. +func (o *osMetrics) time(s osMetric) func() { + startTime := time.Now() + return func() { + duration := time.Since(startTime) + + atomic.AddUint64(&o.operations[s], 1) + o.latency[s].add(duration) + } +} + +// incTime will increment time on metric s with a specific duration. +func (o *osMetrics) incTime(s osMetric, d time.Duration) { + atomic.AddUint64(&o.operations[s], 1) + o.latency[s].add(d) +} + +func osTrace(s osMetric, startTime time.Time, duration time.Duration, path string, err error) madmin.TraceInfo { + var errStr string + if err != nil { + errStr = err.Error() + } + return madmin.TraceInfo{ + TraceType: madmin.TraceOS, + Time: startTime, + NodeName: globalLocalNodeName, + FuncName: "os." + s.String(), + Duration: duration, + Path: path, + Error: errStr, + } +} + +func updateOSMetrics(s osMetric, paths ...string) func(err error) { + if globalTrace.NumSubscribers(madmin.TraceOS) == 0 { + osAction := globalOSMetrics.time(s) + return func(err error) { osAction() } + } + + startTime := time.Now() + return func(err error) { + duration := time.Since(startTime) + globalOSMetrics.incTime(s, duration) + globalTrace.Publish(osTrace(s, startTime, duration, strings.Join(paths, " -> "), err)) + } +} + +// RemoveAll captures time taken to call the underlying os.RemoveAll +func RemoveAll(dirPath string) (err error) { + defer updateOSMetrics(osMetricRemoveAll, dirPath)(err) + return os.RemoveAll(dirPath) +} + +// Mkdir captures time taken to call os.Mkdir +func Mkdir(dirPath string, mode os.FileMode) (err error) { + defer updateOSMetrics(osMetricMkdir, dirPath)(err) + return os.Mkdir(dirPath, mode) +} + +// MkdirAll captures time taken to call os.MkdirAll +func MkdirAll(dirPath string, mode os.FileMode, baseDir string) (err error) { + defer updateOSMetrics(osMetricMkdirAll, dirPath)(err) + return osMkdirAll(dirPath, mode, baseDir) +} + +// Rename captures time taken to call os.Rename +func Rename(src, dst string) (err error) { + defer updateOSMetrics(osMetricRename, src, dst)(err) + return RenameSys(src, dst) +} + +// OpenFile captures time taken to call os.OpenFile +func OpenFile(name string, flag int, perm os.FileMode) (f *os.File, err error) { + switch flag & writeMode { + case writeMode: + defer updateOSMetrics(osMetricOpenFileW, name)(err) + default: + defer updateOSMetrics(osMetricOpenFileR, name)(err) + } + return os.OpenFile(name, flag, perm) +} + +// Access captures time taken to call syscall.Access() +// on windows, plan9 and solaris syscall.Access uses +// os.Lstat() +func Access(name string) (err error) { + defer updateOSMetrics(osMetricAccess, name)(err) + return access(name) +} + +// Open captures time taken to call os.Open +func Open(name string) (f *os.File, err error) { + defer updateOSMetrics(osMetricOpen, name)(err) + return os.Open(name) +} + +// OpenFileDirectIO captures time taken to call disk.OpenFileDirectIO +func OpenFileDirectIO(name string, flag int, perm os.FileMode) (f *os.File, err error) { + defer updateOSMetrics(osMetricOpenFileDirectIO, name)(err) + return disk.OpenFileDirectIO(name, flag, perm) +} + +// Lstat captures time taken to call os.Lstat +func Lstat(name string) (info os.FileInfo, err error) { + defer updateOSMetrics(osMetricLstat, name)(err) + return os.Lstat(name) +} + +// Remove captures time taken to call os.Remove +func Remove(deletePath string) (err error) { + defer updateOSMetrics(osMetricRemove, deletePath)(err) + return os.Remove(deletePath) +} + +// Stat captures time taken to call os.Stat +func Stat(name string) (info os.FileInfo, err error) { + defer updateOSMetrics(osMetricStat, name)(err) + return os.Stat(name) +} + +// Create captures time taken to call os.Create +func Create(name string) (f *os.File, err error) { + defer updateOSMetrics(osMetricCreate, name)(err) + return os.Create(name) +} + +// Fdatasync captures time taken to call Fdatasync +func Fdatasync(f *os.File) (err error) { + fn := "" + if f != nil { + fn = f.Name() + } + defer updateOSMetrics(osMetricFdatasync, fn)(err) + return disk.Fdatasync(f) +} + +// report returns all os metrics. +func (o *osMetrics) report() madmin.OSMetrics { + var m madmin.OSMetrics + m.CollectedAt = time.Now() + m.LifeTimeOps = make(map[string]uint64, osMetricLast) + for i := range osMetricLast { + if n := atomic.LoadUint64(&o.operations[i]); n > 0 { + m.LifeTimeOps[i.String()] = n + } + } + if len(m.LifeTimeOps) == 0 { + m.LifeTimeOps = nil + } + + m.LastMinute.Operations = make(map[string]madmin.TimedAction, osMetricLast) + for i := range osMetricLast { + lm := o.latency[i].total() + if lm.N > 0 { + m.LastMinute.Operations[i.String()] = lm.asTimedAction() + } + } + if len(m.LastMinute.Operations) == 0 { + m.LastMinute.Operations = nil + } + + return m +} diff --git a/cmd/os-readdir-common.go b/cmd/os-readdir-common.go new file mode 100644 index 00000000..0caca905 --- /dev/null +++ b/cmd/os-readdir-common.go @@ -0,0 +1,36 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +// Options for readDir function call +type readDirOpts struct { + // The maximum number of entries to return + count int + // Follow directory symlink + followDirSymlink bool +} + +// Return all the entries at the directory dirPath. +func readDir(dirPath string) (entries []string, err error) { + return readDirWithOpts(dirPath, readDirOpts{count: -1}) +} + +// Return up to count entries at the directory dirPath. +func readDirN(dirPath string, count int) (entries []string, err error) { + return readDirWithOpts(dirPath, readDirOpts{count: count}) +} diff --git a/cmd/os-readdir_test.go b/cmd/os-readdir_test.go new file mode 100644 index 00000000..5649b539 --- /dev/null +++ b/cmd/os-readdir_test.go @@ -0,0 +1,255 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "os" + "path" + "path/filepath" + "runtime" + "sort" + "testing" +) + +// Test to check for different input arguments. +func TestReadDirFail(t *testing.T) { + // Check non existent directory. + if _, err := readDir("/tmp/non-existent-directory"); err != errFileNotFound { + t.Fatalf("expected = %s, got: %s", errFileNotFound, err) + } + + file := path.Join(os.TempDir(), "issue") + if err := os.WriteFile(file, []byte(""), 0o644); err != nil { + t.Fatal(err) + } + defer os.RemoveAll(file) + + // Check if file is given. + if _, err := readDir(path.Join(file, "mydir")); err != errFileNotFound { + t.Fatalf("expected = %s, got: %s", errFileNotFound, err) + } + + // Only valid for linux. + if runtime.GOOS == "linux" { + permDir := path.Join(os.TempDir(), "perm-dir") + if err := os.MkdirAll(permDir, os.FileMode(0o200)); err != nil { + t.Fatal(err) + } + defer os.RemoveAll(permDir) + + // Check if permission denied. + if _, err := readDir(permDir); err == nil { + t.Fatalf("expected = an error, got: nil") + } + } +} + +// Represents data type for all the test results. +type result struct { + dir string + entries []string +} + +// Test to read empty directory. +func setupTestReadDirEmpty(t *testing.T) (testResults []result) { + // Add empty entry slice for this test directory. + testResults = append(testResults, result{t.TempDir(), []string{}}) + return testResults +} + +// Test to read non-empty directory with only files. +func setupTestReadDirFiles(t *testing.T) (testResults []result) { + dir := t.TempDir() + entries := []string{} + for i := range 10 { + name := fmt.Sprintf("file-%d", i) + if err := os.WriteFile(filepath.Join(dir, name), []byte{}, os.ModePerm); err != nil { + // For cleanup, its required to add these entries into test results. + testResults = append(testResults, result{dir, entries}) + t.Fatalf("Unable to create file, %s", err) + } + entries = append(entries, name) + } + + // Keep entries sorted for easier comparison. + sort.Strings(entries) + + // Add entries slice for this test directory. + testResults = append(testResults, result{dir, entries}) + return testResults +} + +// Test to read non-empty directory with directories and files. +func setupTestReadDirGeneric(t *testing.T) (testResults []result) { + dir := t.TempDir() + if err := os.MkdirAll(filepath.Join(dir, "mydir"), 0o777); err != nil { + t.Fatalf("Unable to create prefix directory \"mydir\", %s", err) + } + entries := []string{"mydir/"} + for i := range 10 { + name := fmt.Sprintf("file-%d", i) + if err := os.WriteFile(filepath.Join(dir, "mydir", name), []byte{}, os.ModePerm); err != nil { + // For cleanup, its required to add these entries into test results. + testResults = append(testResults, result{dir, entries}) + t.Fatalf("Unable to write file, %s", err) + } + } + // Keep entries sorted for easier comparison. + sort.Strings(entries) + + // Add entries slice for this test directory. + testResults = append(testResults, result{dir, entries}) + return testResults +} + +// Test to read non-empty directory with symlinks. +func setupTestReadDirSymlink(t *testing.T) (testResults []result) { + if runtime.GOOS == globalWindowsOSName { + t.Skip("symlinks not available on windows") + return nil + } + dir := t.TempDir() + entries := []string{} + for i := range 10 { + name1 := fmt.Sprintf("file-%d", i) + name2 := fmt.Sprintf("file-%d", i+10) + if err := os.WriteFile(filepath.Join(dir, name1), []byte{}, os.ModePerm); err != nil { + // For cleanup, its required to add these entries into test results. + testResults = append(testResults, result{dir, entries}) + t.Fatalf("Unable to create a file, %s", err) + } + // Symlink will not be added to entries. + if err := os.Symlink(filepath.Join(dir, name1), filepath.Join(dir, name2)); err != nil { + t.Fatalf("Unable to create a symlink, %s", err) + } + // Add to entries. + entries = append(entries, name1) + // Symlinks are preserved for regular files + entries = append(entries, name2) + } + if err := os.MkdirAll(filepath.Join(dir, "mydir"), 0o777); err != nil { + t.Fatalf("Unable to create \"mydir\", %s", err) + } + entries = append(entries, "mydir/") + + // Keep entries sorted for easier comparison. + sort.Strings(entries) + + // Add entries slice for this test directory. + testResults = append(testResults, result{dir, entries}) + return testResults +} + +// checkResult - checks whether entries are got are same as expected entries. +func checkResult(expected []string, got []string) bool { + // If length of expected and got slice are different, the test actually failed. + if len(expected) != len(got) { + return false + } + + for i := range expected { + // If entry in expected is not same as entry it got, the test is failed. + if expected[i] != got[i] { + return false + } + } + + // expected and got have same entries. + return true +} + +// teardown - cleans up test directories. +func teardown(testResults []result) { + for _, r := range testResults { + os.RemoveAll(r.dir) + } +} + +// TestReadDir - test function to run various readDir() tests. +func TestReadDir(t *testing.T) { + var testResults []result + + // Setup and capture test results for empty directory. + testResults = append(testResults, setupTestReadDirEmpty(t)...) + // Setup and capture test results for directory with only files. + testResults = append(testResults, setupTestReadDirFiles(t)...) + // Setup and capture test results for directory with files and directories. + testResults = append(testResults, setupTestReadDirGeneric(t)...) + // Setup and capture test results for directory with files and symlink. + testResults = append(testResults, setupTestReadDirSymlink(t)...) + + // Remove all dirs once tests are over. + defer teardown(testResults) + + // Validate all the results. + for _, r := range testResults { + if entries, err := readDir(r.dir); err != nil { + t.Fatal("failed to run test.", err) + } else { + // Keep entries sorted for easier comparison. + sort.Strings(entries) + if !checkResult(r.entries, entries) { + t.Fatalf("expected = %s, got: %s", r.entries, entries) + } + } + } +} + +func TestReadDirN(t *testing.T) { + testCases := []struct { + numFiles int + n int + expectedNum int + }{ + {0, 0, 0}, + {0, 1, 0}, + {1, 0, 0}, + {0, -1, 0}, + {1, -1, 1}, + {10, -1, 10}, + {1, 1, 1}, + {2, 1, 1}, + {10, 9, 9}, + {10, 10, 10}, + {10, 11, 10}, + } + + for i, testCase := range testCases { + dir := t.TempDir() + + for c := 1; c <= testCase.numFiles; c++ { + err := os.WriteFile(filepath.Join(dir, fmt.Sprintf("%d", c)), []byte{}, os.ModePerm) + if err != nil { + os.RemoveAll(dir) + t.Fatalf("Unable to create a file, %s", err) + } + } + entries, err := readDirN(dir, testCase.n) + if err != nil { + os.RemoveAll(dir) + t.Fatalf("Unable to read entries, %s", err) + } + if len(entries) != testCase.expectedNum { + os.RemoveAll(dir) + t.Fatalf("Test %d: unexpected number of entries, waiting for %d, but found %d", + i+1, testCase.expectedNum, len(entries)) + } + os.RemoveAll(dir) + } +} diff --git a/cmd/os-reliable.go b/cmd/os-reliable.go new file mode 100644 index 00000000..3561cbd3 --- /dev/null +++ b/cmd/os-reliable.go @@ -0,0 +1,193 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "os" + "path" +) + +// Wrapper functions to os.RemoveAll, which calls reliableRemoveAll +// this is to ensure that if there is a racy parent directory +// create in between we can simply retry the operation. +func removeAll(dirPath string) (err error) { + if dirPath == "" { + return errInvalidArgument + } + + if err = checkPathLength(dirPath); err != nil { + return err + } + + if err = reliableRemoveAll(dirPath); err != nil { + switch { + case isSysErrNotDir(err): + // File path cannot be verified since one of + // the parents is a file. + return errFileAccessDenied + case isSysErrPathNotFound(err): + // This is a special case should be handled only for + // windows, because windows API does not return "not a + // directory" error message. Handle this specifically + // here. + return errFileAccessDenied + } + } + return err +} + +// Reliably retries os.RemoveAll if for some reason os.RemoveAll returns +// syscall.ENOTEMPTY (children has files). +func reliableRemoveAll(dirPath string) (err error) { + i := 0 + for { + // Removes all the directories and files. + if err = RemoveAll(dirPath); err != nil { + // Retry only for the first retryable error. + if isSysErrNotEmpty(err) && i == 0 { + i++ + continue + } + } + break + } + return err +} + +// Wrapper functions to os.MkdirAll, which calls reliableMkdirAll +// this is to ensure that if there is a racy parent directory +// delete in between we can simply retry the operation. +func mkdirAll(dirPath string, mode os.FileMode, baseDir string) (err error) { + if dirPath == "" { + return errInvalidArgument + } + + if err = checkPathLength(dirPath); err != nil { + return err + } + + if err = reliableMkdirAll(dirPath, mode, baseDir); err != nil { + // File path cannot be verified since one of the parents is a file. + if isSysErrNotDir(err) { + return errFileAccessDenied + } else if isSysErrPathNotFound(err) { + // This is a special case should be handled only for + // windows, because windows API does not return "not a + // directory" error message. Handle this specifically here. + return errFileAccessDenied + } + return osErrToFileErr(err) + } + + return nil +} + +// Reliably retries os.MkdirAll if for some reason os.MkdirAll returns +// syscall.ENOENT (parent does not exist). +func reliableMkdirAll(dirPath string, mode os.FileMode, baseDir string) (err error) { + i := 0 + for { + // Creates all the parent directories, with mode 0777 mkdir honors system umask. + if err = osMkdirAll(dirPath, mode, baseDir); err != nil { + // Retry only for the first retryable error. + if osIsNotExist(err) && i == 0 { + i++ + // Determine if os.NotExist error is because of + // baseDir's parent being present, retry it once such + // that the MkdirAll is retried once for the parent + // of dirPath. + // Because it is worth a retry to skip a different + // baseDir which is slightly higher up the depth. + nbaseDir := path.Dir(baseDir) + if baseDir != "" && nbaseDir != "" && nbaseDir != SlashSeparator { + baseDir = nbaseDir + } + continue + } + } + break + } + return err +} + +// Wrapper function to os.Rename, which calls reliableMkdirAll +// and reliableRenameAll. This is to ensure that if there is a +// racy parent directory delete in between we can simply retry +// the operation. +func renameAll(srcFilePath, dstFilePath, baseDir string) (err error) { + if srcFilePath == "" || dstFilePath == "" { + return errInvalidArgument + } + + if err = checkPathLength(srcFilePath); err != nil { + return err + } + if err = checkPathLength(dstFilePath); err != nil { + return err + } + + if err = reliableRename(srcFilePath, dstFilePath, baseDir); err != nil { + switch { + case isSysErrNotDir(err) && !osIsNotExist(err): + // Windows can have both isSysErrNotDir(err) and osIsNotExist(err) returning + // true if the source file path contains an non-existent directory. In that case, + // we want to return errFileNotFound instead, which will honored in subsequent + // switch cases + return errFileAccessDenied + case isSysErrPathNotFound(err): + // This is a special case should be handled only for + // windows, because windows API does not return "not a + // directory" error message. Handle this specifically here. + return errFileAccessDenied + case isSysErrCrossDevice(err): + return fmt.Errorf("%w (%s)->(%s)", errCrossDeviceLink, srcFilePath, dstFilePath) + case osIsNotExist(err): + return errFileNotFound + case osIsExist(err): + // This is returned only when destination is a directory and we + // are attempting a rename from file to directory. + return errIsNotRegular + default: + return err + } + } + return nil +} + +// Reliably retries os.RenameAll if for some reason os.RenameAll returns +// syscall.ENOENT (parent does not exist). +func reliableRename(srcFilePath, dstFilePath, baseDir string) (err error) { + if err = reliableMkdirAll(path.Dir(dstFilePath), 0o777, baseDir); err != nil { + return err + } + + i := 0 + for { + // After a successful parent directory create attempt a renameAll. + if err = Rename(srcFilePath, dstFilePath); err != nil { + // Retry only for the first retryable error. + if osIsNotExist(err) && i == 0 { + i++ + continue + } + } + break + } + return err +} diff --git a/cmd/os-reliable_test.go b/cmd/os-reliable_test.go new file mode 100644 index 00000000..7fd29588 --- /dev/null +++ b/cmd/os-reliable_test.go @@ -0,0 +1,74 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "testing" +) + +// Tests - mkdirAll() +func TestOSMkdirAll(t *testing.T) { + // create xlStorage test setup + _, path, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to create xlStorage test setup, %s", err) + } + + if err = mkdirAll("", 0o777, ""); err != errInvalidArgument { + t.Fatal("Unexpected error", err) + } + + if err = mkdirAll(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), 0o777, ""); err != errFileNameTooLong { + t.Fatal("Unexpected error", err) + } + + if err = mkdirAll(pathJoin(path, "success-vol", "success-object"), 0o777, ""); err != nil { + t.Fatal("Unexpected error", err) + } +} + +// Tests - renameAll() +func TestOSRenameAll(t *testing.T) { + // create xlStorage test setup + _, path, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to create xlStorage test setup, %s", err) + } + + if err = mkdirAll(pathJoin(path, "testvolume1"), 0o777, ""); err != nil { + t.Fatal(err) + } + if err = renameAll("", "foo", ""); err != errInvalidArgument { + t.Fatal(err) + } + if err = renameAll("foo", "", ""); err != errInvalidArgument { + t.Fatal(err) + } + if err = renameAll(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2"), ""); err != nil { + t.Fatal(err) + } + if err = renameAll(pathJoin(path, "testvolume1"), pathJoin(path, "testvolume2"), ""); err != errFileNotFound { + t.Fatal(err) + } + if err = renameAll(pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), pathJoin(path, "testvolume2"), ""); err != errFileNameTooLong { + t.Fatal("Unexpected error", err) + } + if err = renameAll(pathJoin(path, "testvolume1"), pathJoin(path, "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001"), ""); err != errFileNameTooLong { + t.Fatal("Unexpected error", err) + } +} diff --git a/cmd/os-rename_linux.go b/cmd/os-rename_linux.go new file mode 100644 index 00000000..b5805f7a --- /dev/null +++ b/cmd/os-rename_linux.go @@ -0,0 +1,30 @@ +//go:build linux +// +build linux + +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "syscall" +) + +// RenameSys is low level call in case of Linux this uses syscall.Rename() directly. +func RenameSys(src, dst string) (err error) { + return syscall.Rename(src, dst) +} diff --git a/cmd/os-rename_nolinux.go b/cmd/os-rename_nolinux.go new file mode 100644 index 00000000..5dd023d6 --- /dev/null +++ b/cmd/os-rename_nolinux.go @@ -0,0 +1,30 @@ +//go:build !linux +// +build !linux + +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "os" +) + +// RenameSys is low level call in case of non-Linux this just uses os.Rename() +func RenameSys(src, dst string) (err error) { + return os.Rename(src, dst) +} diff --git a/cmd/os_other.go b/cmd/os_other.go new file mode 100644 index 00000000..340ca1b9 --- /dev/null +++ b/cmd/os_other.go @@ -0,0 +1,163 @@ +//go:build plan9 || solaris +// +build plan9 solaris + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "io" + "os" + "syscall" +) + +func access(name string) error { + _, err := os.Lstat(name) + return err +} + +func osMkdirAll(dirPath string, perm os.FileMode, _ string) error { + // baseDir is not honored in plan9 and solaris platforms. + return os.MkdirAll(dirPath, perm) +} + +// readDirFn applies the fn() function on each entries at dirPath, doesn't recurse into +// the directory itself, if the dirPath doesn't exist this function doesn't return +// an error. +func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error) error { + d, err := Open(dirPath) + if err != nil { + if osErrToFileErr(err) == errFileNotFound { + return nil + } + return osErrToFileErr(err) + } + defer d.Close() + + maxEntries := 1000 + for { + // Read up to max number of entries. + fis, err := d.Readdir(maxEntries) + if err != nil { + if err == io.EOF { + break + } + err = osErrToFileErr(err) + if err == errFileNotFound { + return nil + } + return err + } + for _, fi := range fis { + if fi.Mode()&os.ModeSymlink == os.ModeSymlink { + fi, err = Stat(pathJoin(dirPath, fi.Name())) + if err != nil { + // It got deleted in the meantime, not found + // or returns too many symlinks ignore this + // file/directory. + if osIsNotExist(err) || isSysErrPathNotFound(err) || + isSysErrTooManySymlinks(err) { + continue + } + return err + } + + // Ignore symlinked directories. + if fi.IsDir() { + continue + } + } + if err = filter(fi.Name(), fi.Mode()); err == errDoneForNow { + // filtering requested to return by caller. + return nil + } + } + } + return nil +} + +// Return entries at the directory dirPath. +func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err error) { + d, err := Open(dirPath) + if err != nil { + return nil, osErrToFileErr(err) + } + defer d.Close() + + maxEntries := 1000 + if opts.count > 0 && opts.count < maxEntries { + maxEntries = opts.count + } + + done := false + remaining := opts.count + + for !done { + // Read up to max number of entries. + fis, err := d.Readdir(maxEntries) + if err != nil { + if err == io.EOF { + break + } + return nil, osErrToFileErr(err) + } + if opts.count > -1 { + if remaining <= len(fis) { + fis = fis[:remaining] + done = true + } + } + for _, fi := range fis { + if fi.Mode()&os.ModeSymlink == os.ModeSymlink { + fi, err = Stat(pathJoin(dirPath, fi.Name())) + if err != nil { + // It got deleted in the meantime, not found + // or returns too many symlinks ignore this + // file/directory. + if osIsNotExist(err) || isSysErrPathNotFound(err) || + isSysErrTooManySymlinks(err) { + continue + } + return nil, err + } + + // Ignore symlinked directories. + if !opts.followDirSymlink && fi.IsDir() { + continue + } + } + + if fi.IsDir() { + // Append SlashSeparator instead of "\" so that sorting is achieved as expected. + entries = append(entries, fi.Name()+SlashSeparator) + } else if fi.Mode().IsRegular() { + entries = append(entries, fi.Name()) + } + if opts.count > 0 { + remaining-- + } + } + } + return entries, nil +} + +func globalSync() { + // no-op not sure about plan9/solaris support for syscall support + defer globalOSMetrics.time(osMetricSync)() + syscall.Sync() +} diff --git a/cmd/os_unix.go b/cmd/os_unix.go new file mode 100644 index 00000000..9728493f --- /dev/null +++ b/cmd/os_unix.go @@ -0,0 +1,358 @@ +//go:build (linux && !appengine) || darwin || freebsd || netbsd || openbsd +// +build linux,!appengine darwin freebsd netbsd openbsd + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "fmt" + "os" + "strings" + "syscall" + "unsafe" + + "github.com/minio/minio/internal/bpool" + "golang.org/x/sys/unix" +) + +func access(name string) error { + if err := unix.Access(name, unix.F_OK); err != nil { + return &os.PathError{Op: "lstat", Path: name, Err: err} + } + return nil +} + +// openFileWithFD return 'fd' based file descriptor +func openFileWithFD(name string, flag int, perm os.FileMode) (fd int, err error) { + switch flag & writeMode { + case writeMode: + defer updateOSMetrics(osMetricOpenFileWFd, name)(err) + default: + defer updateOSMetrics(osMetricOpenFileRFd, name)(err) + } + var e error + fd, e = syscall.Open(name, flag|syscall.O_CLOEXEC, uint32(perm)) + if e != nil { + return -1, &os.PathError{Op: "open", Path: name, Err: e} + } + return fd, nil +} + +// Forked from Golang but chooses to avoid performing lookup +// +// osMkdirAll creates a directory named path, +// along with any necessary parents, and returns nil, +// or else returns an error. +// The permission bits perm (before umask) are used for all +// directories that MkdirAll creates. +// If path is already a directory, MkdirAll does nothing +// and returns nil. +func osMkdirAll(dirPath string, perm os.FileMode, baseDir string) error { + if baseDir != "" { + if strings.HasPrefix(baseDir, dirPath) { + return nil + } + } + + // Slow path: make sure parent exists and then call Mkdir for path. + i := len(dirPath) + for i > 0 && os.IsPathSeparator(dirPath[i-1]) { // Skip trailing path separator. + i-- + } + + j := i + for j > 0 && !os.IsPathSeparator(dirPath[j-1]) { // Scan backward over element. + j-- + } + + if j > 1 { + // Create parent. + if err := osMkdirAll(dirPath[:j-1], perm, baseDir); err != nil { + return err + } + } + + // Parent now exists; invoke Mkdir and use its result. + if err := Mkdir(dirPath, perm); err != nil { + if osIsExist(err) { + return nil + } + return err + } + + return nil +} + +// The buffer must be at least a block long. +// refer https://github.com/golang/go/issues/24015 +const blockSize = 8 << 10 // 8192 + +// By default at least 128 entries in single getdents call (1MiB buffer) +var ( + direntPool = bpool.Pool[*[]byte]{ + New: func() *[]byte { + buf := make([]byte, blockSize*128) + return &buf + }, + } + + direntNamePool = bpool.Pool[*[]byte]{ + New: func() *[]byte { + buf := make([]byte, blockSize) + return &buf + }, + } +) + +// unexpectedFileMode is a sentinel (and bogus) os.FileMode +// value used to represent a syscall.DT_UNKNOWN Dirent.Type. +const unexpectedFileMode os.FileMode = os.ModeNamedPipe | os.ModeSocket | os.ModeDevice + +func parseDirEnt(buf []byte) (consumed int, name []byte, typ os.FileMode, err error) { + // golang.org/issue/15653 + dirent := (*syscall.Dirent)(unsafe.Pointer(&buf[0])) + if v := unsafe.Offsetof(dirent.Reclen) + unsafe.Sizeof(dirent.Reclen); uintptr(len(buf)) < v { + return consumed, nil, typ, fmt.Errorf("buf size of %d smaller than dirent header size %d", len(buf), v) + } + if len(buf) < int(dirent.Reclen) { + return consumed, nil, typ, fmt.Errorf("buf size %d < record length %d", len(buf), dirent.Reclen) + } + consumed = int(dirent.Reclen) + if direntInode(dirent) == 0 { // File absent in directory. + return consumed, name, typ, err + } + switch dirent.Type { + case syscall.DT_REG: + typ = 0 + case syscall.DT_DIR: + typ = os.ModeDir + case syscall.DT_LNK: + typ = os.ModeSymlink + default: + // Skip all other file types. Revisit if/when this code needs + // to handle such files, MinIO is only interested in + // files and directories. + typ = unexpectedFileMode + } + + nameBuf := (*[unsafe.Sizeof(dirent.Name)]byte)(unsafe.Pointer(&dirent.Name[0])) + nameLen, err := direntNamlen(dirent) + if err != nil { + return consumed, nil, typ, err + } + + return consumed, nameBuf[:nameLen], typ, nil +} + +// readDirFn applies the fn() function on each entries at dirPath, doesn't recurse into +// the directory itself, if the dirPath doesn't exist this function doesn't return +// an error. +func readDirFn(dirPath string, fn func(name string, typ os.FileMode) error) error { + fd, err := openFileWithFD(dirPath, readMode, 0o666) + if err != nil { + if osErrToFileErr(err) == errFileNotFound { + return nil + } + if !osIsPermission(err) { + return osErrToFileErr(err) + } + // There may be permission error when dirPath + // is at the root of the disk mount that may + // not have the permissions to avoid 'noatime' + fd, err = openFileWithFD(dirPath, os.O_RDONLY, 0o666) + if err != nil { + if osErrToFileErr(err) == errFileNotFound { + return nil + } + return osErrToFileErr(err) + } + } + defer syscall.Close(fd) + + bufp := direntPool.Get() + defer direntPool.Put(bufp) + buf := *bufp + + boff := 0 // starting read position in buf + nbuf := 0 // end valid data in buf + + for { + if boff >= nbuf { + boff = 0 + stop := globalOSMetrics.time(osMetricReadDirent) + nbuf, err = syscall.ReadDirent(fd, buf) + stop() + if err != nil { + if isSysErrNotDir(err) { + return nil + } + err = osErrToFileErr(err) + if err == errFileNotFound { + return nil + } + return err + } + if nbuf <= 0 { + break // EOF + } + } + consumed, name, typ, err := parseDirEnt(buf[boff:nbuf]) + if err != nil { + return err + } + boff += consumed + if len(name) == 0 || bytes.Equal(name, []byte{'.'}) || bytes.Equal(name, []byte{'.', '.'}) { + continue + } + + // Fallback for filesystems (like old XFS) that don't + // support Dirent.Type and have DT_UNKNOWN (0) there + // instead. + if typ == unexpectedFileMode || typ&os.ModeSymlink == os.ModeSymlink { + fi, err := Stat(pathJoin(dirPath, string(name))) + if err != nil { + // It got deleted in the meantime, not found + // or returns too many symlinks ignore this + // file/directory. + if osIsNotExist(err) || isSysErrPathNotFound(err) || + isSysErrTooManySymlinks(err) { + continue + } + return err + } + + // Ignore symlinked directories. + if typ&os.ModeSymlink == os.ModeSymlink && fi.IsDir() { + continue + } + + typ = fi.Mode() & os.ModeType + } + if err = fn(string(name), typ); err == errDoneForNow { + // fn() requested to return by caller. + return nil + } + } + + return err +} + +// Return count entries at the directory dirPath and all entries +// if count is set to -1 +func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err error) { + fd, err := openFileWithFD(dirPath, readMode, 0o666) + if err != nil { + if !osIsPermission(err) { + return nil, osErrToFileErr(err) + } + // There may be permission error when dirPath + // is at the root of the disk mount that may + // not have the permissions to avoid 'noatime' + fd, err = openFileWithFD(dirPath, os.O_RDONLY, 0o666) + if err != nil { + return nil, osErrToFileErr(err) + } + } + defer syscall.Close(fd) + + bufp := direntPool.Get() + defer direntPool.Put(bufp) + buf := *bufp + + nameTmp := direntNamePool.Get() + defer direntNamePool.Put(nameTmp) + tmp := *nameTmp + + boff := 0 // starting read position in buf + nbuf := 0 // end valid data in buf + + count := opts.count + + for count != 0 { + if boff >= nbuf { + boff = 0 + stop := globalOSMetrics.time(osMetricReadDirent) + nbuf, err = syscall.ReadDirent(fd, buf) + stop() + if err != nil { + if isSysErrNotDir(err) { + return nil, errFileNotFound + } + return nil, osErrToFileErr(err) + } + if nbuf <= 0 { + break + } + } + consumed, name, typ, err := parseDirEnt(buf[boff:nbuf]) + if err != nil { + return nil, err + } + boff += consumed + if len(name) == 0 || bytes.Equal(name, []byte{'.'}) || bytes.Equal(name, []byte{'.', '.'}) { + continue + } + + // Fallback for filesystems (like old XFS) that don't + // support Dirent.Type and have DT_UNKNOWN (0) there + // instead. + if typ == unexpectedFileMode || typ&os.ModeSymlink == os.ModeSymlink { + fi, err := Stat(pathJoin(dirPath, string(name))) + if err != nil { + // It got deleted in the meantime, not found + // or returns too many symlinks ignore this + // file/directory. + if osIsNotExist(err) || isSysErrPathNotFound(err) || + isSysErrTooManySymlinks(err) { + continue + } + return nil, err + } + + // Ignore symlinked directories. + if !opts.followDirSymlink && typ&os.ModeSymlink == os.ModeSymlink && fi.IsDir() { + continue + } + + typ = fi.Mode() & os.ModeType + } + + var nameStr string + if typ.IsRegular() { + nameStr = string(name) + } else if typ.IsDir() { + // Use temp buffer to append a slash to avoid string concat. + tmp = tmp[:len(name)+1] + copy(tmp, name) + tmp[len(tmp)-1] = '/' // SlashSeparator + nameStr = string(tmp) + } + + count-- + entries = append(entries, nameStr) + } + + return entries, err +} + +func globalSync() { + defer globalOSMetrics.time(osMetricSync)() + syscall.Sync() +} diff --git a/cmd/os_windows.go b/cmd/os_windows.go new file mode 100644 index 00000000..bf03f2b9 --- /dev/null +++ b/cmd/os_windows.go @@ -0,0 +1,201 @@ +//go:build windows +// +build windows + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "os" + "path/filepath" + "syscall" +) + +func access(name string) error { + _, err := os.Lstat(name) + return err +} + +func osMkdirAll(dirPath string, perm os.FileMode, _ string) error { + // baseDir is not honored in windows platform + return os.MkdirAll(dirPath, perm) +} + +// readDirFn applies the fn() function on each entries at dirPath, doesn't recurse into +// the directory itself, if the dirPath doesn't exist this function doesn't return +// an error. +func readDirFn(dirPath string, filter func(name string, typ os.FileMode) error) error { + // Ensure we don't pick up files as directories. + globAll := filepath.Clean(dirPath) + `\*` + globAllP, err := syscall.UTF16PtrFromString(globAll) + if err != nil { + return errInvalidArgument + } + data := &syscall.Win32finddata{} + handle, err := syscall.FindFirstFile(globAllP, data) + if err != nil { + if err = syscallErrToFileErr(dirPath, err); err == errFileNotFound { + return nil + } + return err + } + defer syscall.FindClose(handle) + + for ; ; err = syscall.FindNextFile(handle, data) { + if err != nil { + if err == syscall.ERROR_NO_MORE_FILES { + break + } + if isSysErrPathNotFound(err) { + return nil + } + err = osErrToFileErr(&os.PathError{ + Op: "FindNextFile", + Path: dirPath, + Err: err, + }) + if err == errFileNotFound { + return nil + } + return err + } + name := syscall.UTF16ToString(data.FileName[0:]) + if name == "" || name == "." || name == ".." { // Useless names + continue + } + + var typ os.FileMode // regular file + switch { + case data.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT != 0: + // Reparse point is a symlink + fi, err := os.Stat(pathJoin(dirPath, name)) + if err != nil { + // It got deleted in the meantime, not found + // or returns too many symlinks ignore this + // file/directory. + if osIsNotExist(err) || isSysErrPathNotFound(err) || + isSysErrTooManySymlinks(err) { + continue + } + return err + } + + if fi.IsDir() { + // Ignore symlinked directories. + continue + } + + typ = fi.Mode() + case data.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY != 0: + typ = os.ModeDir + } + + if err = filter(name, typ); err == errDoneForNow { + // filtering requested to return by caller. + return nil + } + } + + return nil +} + +// Return N entries at the directory dirPath. +func readDirWithOpts(dirPath string, opts readDirOpts) (entries []string, err error) { + // Ensure we don't pick up files as directories. + globAll := filepath.Clean(dirPath) + `\*` + globAllP, err := syscall.UTF16PtrFromString(globAll) + if err != nil { + return nil, errInvalidArgument + } + data := &syscall.Win32finddata{} + handle, err := syscall.FindFirstFile(globAllP, data) + if err != nil { + return nil, syscallErrToFileErr(dirPath, err) + } + + defer syscall.FindClose(handle) + + count := opts.count + for ; count != 0; err = syscall.FindNextFile(handle, data) { + if err != nil { + if err == syscall.ERROR_NO_MORE_FILES { + break + } + return nil, osErrToFileErr(&os.PathError{ + Op: "FindNextFile", + Path: dirPath, + Err: err, + }) + } + + name := syscall.UTF16ToString(data.FileName[0:]) + if name == "" || name == "." || name == ".." { // Useless names + continue + } + + switch { + case data.FileAttributes&syscall.FILE_ATTRIBUTE_REPARSE_POINT != 0: + // Reparse point is a symlink + fi, err := os.Stat(pathJoin(dirPath, name)) + if err != nil { + // It got deleted in the meantime, not found + // or returns too many symlinks ignore this + // file/directory. + if osIsNotExist(err) || isSysErrPathNotFound(err) || + isSysErrTooManySymlinks(err) { + continue + } + return nil, err + } + + if !opts.followDirSymlink && fi.IsDir() { + // directory symlinks are ignored. + continue + } + case data.FileAttributes&syscall.FILE_ATTRIBUTE_DIRECTORY != 0: + name += SlashSeparator + } + + count-- + entries = append(entries, name) + } + + return entries, nil +} + +func globalSync() { + // no-op on windows +} + +func syscallErrToFileErr(dirPath string, err error) error { + switch err { + case nil: + return nil + case syscall.ERROR_FILE_NOT_FOUND: + return errFileNotFound + case syscall.ERROR_ACCESS_DENIED: + return errFileAccessDenied + default: + // Fails on file not found and when not a directory. + return osErrToFileErr(&os.PathError{ + Op: "FindNextFile", + Path: dirPath, + Err: err, + }) + } +} diff --git a/cmd/osmetric_string.go b/cmd/osmetric_string.go new file mode 100644 index 00000000..2f5f49fc --- /dev/null +++ b/cmd/osmetric_string.go @@ -0,0 +1,41 @@ +// Code generated by "stringer -type=osMetric -trimprefix=osMetric os-instrumented.go"; DO NOT EDIT. + +package cmd + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[osMetricRemoveAll-0] + _ = x[osMetricMkdirAll-1] + _ = x[osMetricMkdir-2] + _ = x[osMetricRename-3] + _ = x[osMetricOpenFileW-4] + _ = x[osMetricOpenFileR-5] + _ = x[osMetricOpenFileWFd-6] + _ = x[osMetricOpenFileRFd-7] + _ = x[osMetricOpen-8] + _ = x[osMetricOpenFileDirectIO-9] + _ = x[osMetricLstat-10] + _ = x[osMetricRemove-11] + _ = x[osMetricStat-12] + _ = x[osMetricAccess-13] + _ = x[osMetricCreate-14] + _ = x[osMetricReadDirent-15] + _ = x[osMetricFdatasync-16] + _ = x[osMetricSync-17] + _ = x[osMetricLast-18] +} + +const _osMetric_name = "RemoveAllMkdirAllMkdirRenameOpenFileWOpenFileROpenFileWFdOpenFileRFdOpenOpenFileDirectIOLstatRemoveStatAccessCreateReadDirentFdatasyncSyncLast" + +var _osMetric_index = [...]uint8{0, 9, 17, 22, 28, 37, 46, 57, 68, 72, 88, 93, 99, 103, 109, 115, 125, 134, 138, 142} + +func (i osMetric) String() string { + if i >= osMetric(len(_osMetric_index)-1) { + return "osMetric(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _osMetric_name[_osMetric_index[i]:_osMetric_index[i+1]] +} diff --git a/cmd/peer-rest-client.go b/cmd/peer-rest-client.go new file mode 100644 index 00000000..eb65a40a --- /dev/null +++ b/cmd/peer-rest-client.go @@ -0,0 +1,847 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/gob" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/url" + "strconv" + "sync/atomic" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/bucket/bandwidth" + "github.com/minio/minio/internal/grid" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/rest" + xnet "github.com/minio/pkg/v3/net" +) + +// client to talk to peer Nodes. +type peerRESTClient struct { + host *xnet.Host + restClient *rest.Client + gridHost string + // Function that returns the grid connection for this peer when initialized. + // Will return nil if the grid connection is not initialized yet. + gridConn func() *grid.Connection +} + +// Returns a peer rest client. +func newPeerRESTClient(peer *xnet.Host, gridHost string) *peerRESTClient { + scheme := "http" + if globalIsTLS { + scheme = "https" + } + + serverURL := &url.URL{ + Scheme: scheme, + Host: peer.String(), + Path: peerRESTPath, + } + + restClient := rest.NewClient(serverURL, globalInternodeTransport, newCachedAuthToken()) + // Use a separate client to avoid recursive calls. + healthClient := rest.NewClient(serverURL, globalInternodeTransport, newCachedAuthToken()) + healthClient.NoMetrics = true + + // Construct a new health function. + restClient.HealthCheckFn = func() bool { + ctx, cancel := context.WithTimeout(context.Background(), restClient.HealthCheckTimeout) + defer cancel() + respBody, err := healthClient.Call(ctx, peerRESTMethodHealth, nil, nil, -1) + xhttp.DrainBody(respBody) + return !isNetworkError(err) + } + var gridConn atomic.Pointer[grid.Connection] + + return &peerRESTClient{ + host: peer, restClient: restClient, gridHost: gridHost, + gridConn: func() *grid.Connection { + // Lazy initialization of grid connection. + // When we create this peer client, the grid connection is likely not yet initialized. + if gridHost == "" { + bugLogIf(context.Background(), fmt.Errorf("gridHost is empty for peer %s", peer.String()), peer.String()+":gridHost") + return nil + } + gc := gridConn.Load() + if gc != nil { + return gc + } + gm := globalGrid.Load() + if gm == nil { + return nil + } + gc = gm.Connection(gridHost) + if gc == nil { + bugLogIf(context.Background(), fmt.Errorf("gridHost %q not found for peer %s", gridHost, peer.String()), peer.String()+":gridHost") + return nil + } + gridConn.Store(gc) + return gc + }, + } +} + +// Wrapper to restClient.Call to handle network errors, in case of network error the connection is marked disconnected +// permanently. The only way to restore the connection is at the xl-sets layer by xlsets.monitorAndConnectEndpoints() +// after verifying format.json +func (client *peerRESTClient) callWithContext(ctx context.Context, method string, values url.Values, body io.Reader, length int64) (respBody io.ReadCloser, err error) { + if client == nil { + return nil, errPeerNotReachable + } + + if values == nil { + values = make(url.Values) + } + + respBody, err = client.restClient.Call(ctx, method, values, body, length) + if err == nil { + return respBody, nil + } + + if xnet.IsNetworkOrHostDown(err, true) { + return nil, errPeerNotReachable + } + + return nil, err +} + +// Stringer provides a canonicalized representation of node. +func (client *peerRESTClient) String() string { + return client.host.String() +} + +// IsOnline returns true if the peer client is online. +func (client *peerRESTClient) IsOnline() bool { + conn := client.gridConn() + if conn == nil { + return false + } + return client.restClient.IsOnline() || conn.State() == grid.StateConnected +} + +// Close - marks the client as closed. +func (client *peerRESTClient) Close() error { + client.restClient.Close() + return nil +} + +// GetLocks - fetch older locks for a remote node. +func (client *peerRESTClient) GetLocks(ctx context.Context) (lockMap map[string][]lockRequesterInfo, err error) { + resp, err := getLocksRPC.Call(ctx, client.gridConn(), grid.NewMSS()) + if err != nil || resp == nil { + return nil, err + } + return *resp, nil +} + +// LocalStorageInfo - fetch server information for a remote node. +func (client *peerRESTClient) LocalStorageInfo(ctx context.Context, metrics bool) (info StorageInfo, err error) { + resp, err := localStorageInfoRPC.Call(ctx, client.gridConn(), grid.NewMSSWith(map[string]string{ + peerRESTMetrics: strconv.FormatBool(metrics), + })) + return resp.ValueOrZero(), err +} + +// ServerInfo - fetch server information for a remote node. +func (client *peerRESTClient) ServerInfo(ctx context.Context, metrics bool) (info madmin.ServerProperties, err error) { + resp, err := serverInfoRPC.Call(ctx, client.gridConn(), grid.NewMSSWith(map[string]string{peerRESTMetrics: strconv.FormatBool(metrics)})) + return resp.ValueOrZero(), err +} + +// GetCPUs - fetch CPU information for a remote node. +func (client *peerRESTClient) GetCPUs(ctx context.Context) (info madmin.CPUs, err error) { + resp, err := getCPUsHandler.Call(ctx, client.gridConn(), grid.NewMSS()) + return resp.ValueOrZero(), err +} + +// GetNetInfo - fetch network information for a remote node. +func (client *peerRESTClient) GetNetInfo(ctx context.Context) (info madmin.NetInfo, err error) { + resp, err := getNetInfoRPC.Call(ctx, client.gridConn(), grid.NewMSS()) + return resp.ValueOrZero(), err +} + +// GetPartitions - fetch disk partition information for a remote node. +func (client *peerRESTClient) GetPartitions(ctx context.Context) (info madmin.Partitions, err error) { + resp, err := getPartitionsRPC.Call(ctx, client.gridConn(), grid.NewMSS()) + return resp.ValueOrZero(), err +} + +// GetOSInfo - fetch OS information for a remote node. +func (client *peerRESTClient) GetOSInfo(ctx context.Context) (info madmin.OSInfo, err error) { + resp, err := getOSInfoRPC.Call(ctx, client.gridConn(), grid.NewMSS()) + return resp.ValueOrZero(), err +} + +// GetSELinuxInfo - fetch SELinux information for a remote node. +func (client *peerRESTClient) GetSELinuxInfo(ctx context.Context) (info madmin.SysServices, err error) { + resp, err := getSysServicesRPC.Call(ctx, client.gridConn(), grid.NewMSS()) + return resp.ValueOrZero(), err +} + +// GetSysConfig - fetch sys config for a remote node. +func (client *peerRESTClient) GetSysConfig(ctx context.Context) (info madmin.SysConfig, err error) { + sent := time.Now() + resp, err := getSysConfigRPC.Call(ctx, client.gridConn(), grid.NewMSS()) + info = resp.ValueOrZero() + if ti, ok := info.Config["time-info"].(madmin.TimeInfo); ok { + rt := int32(time.Since(sent).Milliseconds()) + ti.RoundtripDuration = rt + info.Config["time-info"] = ti + } + return info, err +} + +// GetSysErrors - fetch sys errors for a remote node. +func (client *peerRESTClient) GetSysErrors(ctx context.Context) (info madmin.SysErrors, err error) { + resp, err := getSysErrorsRPC.Call(ctx, client.gridConn(), grid.NewMSS()) + return resp.ValueOrZero(), err +} + +// GetMemInfo - fetch memory information for a remote node. +func (client *peerRESTClient) GetMemInfo(ctx context.Context) (info madmin.MemInfo, err error) { + resp, err := getMemInfoRPC.Call(ctx, client.gridConn(), grid.NewMSS()) + return resp.ValueOrZero(), err +} + +// GetMetrics - fetch metrics from a remote node. +func (client *peerRESTClient) GetMetrics(ctx context.Context, t madmin.MetricType, opts collectMetricsOpts) (info madmin.RealtimeMetrics, err error) { + values := make(url.Values) + values.Set(peerRESTMetricsTypes, strconv.FormatUint(uint64(t), 10)) + for disk := range opts.disks { + values.Add(peerRESTDisk, disk) + } + for host := range opts.hosts { + values.Add(peerRESTHost, host) + } + values.Set(peerRESTJobID, opts.jobID) + values.Set(peerRESTDepID, opts.depID) + v, err := getMetricsRPC.Call(ctx, client.gridConn(), grid.NewURLValuesWith(values)) + return v.ValueOrZero(), err +} + +// GetProcInfo - fetch MinIO process information for a remote node. +func (client *peerRESTClient) GetProcInfo(ctx context.Context) (info madmin.ProcInfo, err error) { + resp, err := getProcInfoRPC.Call(ctx, client.gridConn(), grid.NewMSS()) + return resp.ValueOrZero(), err +} + +// StartProfiling - Issues profiling command on the peer node. +func (client *peerRESTClient) StartProfiling(ctx context.Context, profiler string) error { + values := make(url.Values) + values.Set(peerRESTProfiler, profiler) + respBody, err := client.callWithContext(ctx, peerRESTMethodStartProfiling, values, nil, -1) + if err != nil { + return err + } + defer xhttp.DrainBody(respBody) + return nil +} + +// DownloadProfileData - download profiled data from a remote node. +func (client *peerRESTClient) DownloadProfileData(ctx context.Context) (data map[string][]byte, err error) { + respBody, err := client.callWithContext(ctx, peerRESTMethodDownloadProfilingData, nil, nil, -1) + if err != nil { + return data, err + } + defer xhttp.DrainBody(respBody) + err = gob.NewDecoder(respBody).Decode(&data) + return data, err +} + +// GetBucketStats - load bucket statistics +func (client *peerRESTClient) GetBucketStats(ctx context.Context, bucket string) (BucketStats, error) { + resp, err := getBucketStatsRPC.Call(ctx, client.gridConn(), grid.NewMSSWith(map[string]string{ + peerRESTBucket: bucket, + })) + if err != nil || resp == nil { + return BucketStats{}, err + } + return *resp, nil +} + +// GetSRMetrics loads site replication metrics, optionally for a specific bucket +func (client *peerRESTClient) GetSRMetrics(ctx context.Context) (SRMetricsSummary, error) { + resp, err := getSRMetricsRPC.Call(ctx, client.gridConn(), grid.NewMSS()) + if err != nil || resp == nil { + return SRMetricsSummary{}, err + } + return *resp, nil +} + +// GetAllBucketStats - load replication stats for all buckets +func (client *peerRESTClient) GetAllBucketStats(ctx context.Context) (BucketStatsMap, error) { + resp, err := getAllBucketStatsRPC.Call(ctx, client.gridConn(), grid.NewMSS()) + if err != nil || resp == nil { + return BucketStatsMap{}, err + } + return *resp, nil +} + +// LoadBucketMetadata - load bucket metadata +func (client *peerRESTClient) LoadBucketMetadata(ctx context.Context, bucket string) error { + _, err := loadBucketMetadataRPC.Call(ctx, client.gridConn(), grid.NewMSSWith(map[string]string{ + peerRESTBucket: bucket, + })) + return err +} + +// DeleteBucketMetadata - Delete bucket metadata +func (client *peerRESTClient) DeleteBucketMetadata(ctx context.Context, bucket string) error { + _, err := deleteBucketMetadataRPC.Call(ctx, client.gridConn(), grid.NewMSSWith(map[string]string{ + peerRESTBucket: bucket, + })) + return err +} + +// DeletePolicy - delete a specific canned policy. +func (client *peerRESTClient) DeletePolicy(ctx context.Context, policyName string) (err error) { + _, err = deletePolicyRPC.Call(ctx, client.gridConn(), grid.NewMSSWith(map[string]string{ + peerRESTPolicy: policyName, + })) + return err +} + +// LoadPolicy - reload a specific canned policy. +func (client *peerRESTClient) LoadPolicy(ctx context.Context, policyName string) (err error) { + _, err = loadPolicyRPC.Call(ctx, client.gridConn(), grid.NewMSSWith(map[string]string{ + peerRESTPolicy: policyName, + })) + return err +} + +// LoadPolicyMapping - reload a specific policy mapping +func (client *peerRESTClient) LoadPolicyMapping(ctx context.Context, userOrGroup string, userType IAMUserType, isGroup bool) error { + _, err := loadPolicyMappingRPC.Call(ctx, client.gridConn(), grid.NewMSSWith(map[string]string{ + peerRESTUserOrGroup: userOrGroup, + peerRESTUserType: strconv.Itoa(int(userType)), + peerRESTIsGroup: strconv.FormatBool(isGroup), + })) + return err +} + +// DeleteUser - delete a specific user. +func (client *peerRESTClient) DeleteUser(ctx context.Context, accessKey string) (err error) { + _, err = deleteUserRPC.Call(ctx, client.gridConn(), grid.NewMSSWith(map[string]string{ + peerRESTUser: accessKey, + })) + return err +} + +// DeleteServiceAccount - delete a specific service account. +func (client *peerRESTClient) DeleteServiceAccount(ctx context.Context, accessKey string) (err error) { + _, err = deleteSvcActRPC.Call(ctx, client.gridConn(), grid.NewMSSWith(map[string]string{ + peerRESTUser: accessKey, + })) + return err +} + +// LoadUser - reload a specific user. +func (client *peerRESTClient) LoadUser(ctx context.Context, accessKey string, temp bool) (err error) { + _, err = loadUserRPC.Call(ctx, client.gridConn(), grid.NewMSSWith(map[string]string{ + peerRESTUser: accessKey, + peerRESTUserTemp: strconv.FormatBool(temp), + })) + return err +} + +// LoadServiceAccount - reload a specific service account. +func (client *peerRESTClient) LoadServiceAccount(ctx context.Context, accessKey string) (err error) { + _, err = loadSvcActRPC.Call(ctx, client.gridConn(), grid.NewMSSWith(map[string]string{ + peerRESTUser: accessKey, + })) + return err +} + +// LoadGroup - send load group command to peers. +func (client *peerRESTClient) LoadGroup(ctx context.Context, group string) error { + _, err := loadGroupRPC.Call(ctx, client.gridConn(), grid.NewMSSWith(map[string]string{ + peerRESTGroup: group, + })) + return err +} + +func (client *peerRESTClient) ReloadSiteReplicationConfig(ctx context.Context) error { + conn := client.gridConn() + if conn == nil { + return nil + } + + _, err := reloadSiteReplicationConfigRPC.Call(ctx, conn, grid.NewMSS()) + return err +} + +// VerifyBinary - sends verify binary message to remote peers. +func (client *peerRESTClient) VerifyBinary(ctx context.Context, u *url.URL, sha256Sum []byte, releaseInfo string, reader io.Reader) error { + values := make(url.Values) + values.Set(peerRESTURL, u.String()) + values.Set(peerRESTSha256Sum, hex.EncodeToString(sha256Sum)) + values.Set(peerRESTReleaseInfo, releaseInfo) + + respBody, err := client.callWithContext(ctx, peerRESTMethodVerifyBinary, values, reader, -1) + if err != nil { + return err + } + defer xhttp.DrainBody(respBody) + return nil +} + +// CommitBinary - sends commit binary message to remote peers. +func (client *peerRESTClient) CommitBinary(ctx context.Context) error { + respBody, err := client.callWithContext(ctx, peerRESTMethodCommitBinary, nil, nil, -1) + if err != nil { + return err + } + defer xhttp.DrainBody(respBody) + return nil +} + +// SignalService - sends signal to peer nodes. +func (client *peerRESTClient) SignalService(sig serviceSignal, subSys string, dryRun bool, execAt *time.Time) error { + values := grid.NewMSS() + values.Set(peerRESTSignal, strconv.Itoa(int(sig))) + values.Set(peerRESTDryRun, strconv.FormatBool(dryRun)) + values.Set(peerRESTSubSys, subSys) + if execAt != nil { + values.Set(peerRESTExecAt, execAt.Format(time.RFC3339Nano)) + } + _, err := signalServiceRPC.Call(context.Background(), client.gridConn(), values) + return err +} + +func (client *peerRESTClient) BackgroundHealStatus(ctx context.Context) (madmin.BgHealState, error) { + resp, err := getBackgroundHealStatusRPC.Call(ctx, client.gridConn(), grid.NewMSS()) + return resp.ValueOrZero(), err +} + +// GetMetacacheListing - get a new or existing metacache. +func (client *peerRESTClient) GetMetacacheListing(ctx context.Context, o listPathOptions) (*metacache, error) { + if client == nil { + resp := localMetacacheMgr.getBucket(ctx, o.Bucket).findCache(o) + return &resp, nil + } + return getMetacacheListingRPC.Call(ctx, client.gridConn(), &o) +} + +// UpdateMetacacheListing - update an existing metacache it will unconditionally be updated to the new state. +func (client *peerRESTClient) UpdateMetacacheListing(ctx context.Context, m metacache) (metacache, error) { + if client == nil { + return localMetacacheMgr.updateCacheEntry(m) + } + resp, err := updateMetacacheListingRPC.Call(ctx, client.gridConn(), &m) + if err != nil || resp == nil { + return metacache{}, err + } + return *resp, nil +} + +func (client *peerRESTClient) ReloadPoolMeta(ctx context.Context) error { + conn := client.gridConn() + if conn == nil { + return nil + } + _, err := reloadPoolMetaRPC.Call(ctx, conn, grid.NewMSSWith(map[string]string{})) + return err +} + +func (client *peerRESTClient) DeleteUploadID(ctx context.Context, uploadID string) error { + conn := client.gridConn() + if conn == nil { + return nil + } + _, err := cleanupUploadIDCacheMetaRPC.Call(ctx, conn, grid.NewMSSWith(map[string]string{ + peerRESTUploadID: uploadID, + })) + return err +} + +func (client *peerRESTClient) StopRebalance(ctx context.Context) error { + conn := client.gridConn() + if conn == nil { + return nil + } + _, err := stopRebalanceRPC.Call(ctx, conn, grid.NewMSSWith(map[string]string{})) + return err +} + +func (client *peerRESTClient) LoadRebalanceMeta(ctx context.Context, startRebalance bool) error { + conn := client.gridConn() + if conn == nil { + return nil + } + _, err := loadRebalanceMetaRPC.Call(ctx, conn, grid.NewMSSWith(map[string]string{ + peerRESTStartRebalance: strconv.FormatBool(startRebalance), + })) + return err +} + +func (client *peerRESTClient) LoadTransitionTierConfig(ctx context.Context) error { + conn := client.gridConn() + if conn == nil { + return nil + } + _, err := loadTransitionTierConfigRPC.Call(ctx, conn, grid.NewMSSWith(map[string]string{})) + return err +} + +func (client *peerRESTClient) doTrace(ctx context.Context, traceCh chan<- []byte, traceOpts madmin.ServiceTraceOpts) { + gridConn := client.gridConn() + if gridConn == nil { + return + } + + payload, err := json.Marshal(traceOpts) + if err != nil { + bugLogIf(ctx, err) + return + } + + st, err := gridConn.NewStream(ctx, grid.HandlerTrace, payload) + if err != nil { + return + } + st.Results(func(b []byte) error { + select { + case traceCh <- b: + default: + // Do not block on slow receivers. + // Just recycle the buffer. + grid.PutByteBuffer(b) + } + return nil + }) +} + +func (client *peerRESTClient) doListen(ctx context.Context, listenCh chan<- []byte, v url.Values) { + conn := client.gridConn() + if conn == nil { + return + } + st, err := listenRPC.Call(ctx, conn, grid.NewURLValuesWith(v)) + if err != nil { + return + } + st.Results(func(b *grid.Bytes) error { + select { + case listenCh <- *b: + default: + // Do not block on slow receivers. + b.Recycle() + } + return nil + }) +} + +// Listen - listen on peers. +func (client *peerRESTClient) Listen(ctx context.Context, listenCh chan<- []byte, v url.Values) { + go func() { + for { + client.doListen(ctx, listenCh, v) + select { + case <-ctx.Done(): + return + default: + // There was error in the REST request, retry after sometime as probably the peer is down. + time.Sleep(5 * time.Second) + } + } + }() +} + +// Trace - send http trace request to peer nodes +func (client *peerRESTClient) Trace(ctx context.Context, traceCh chan<- []byte, traceOpts madmin.ServiceTraceOpts) { + go func() { + for { + // Blocks until context is canceled or an error occurs. + client.doTrace(ctx, traceCh, traceOpts) + select { + case <-ctx.Done(): + return + default: + // There was error in the REST request, retry after sometime as probably the peer is down. + time.Sleep(5 * time.Second) + } + } + }() +} + +func (client *peerRESTClient) doConsoleLog(ctx context.Context, kind madmin.LogMask, logCh chan<- []byte) { + st, err := consoleLogRPC.Call(ctx, client.gridConn(), grid.NewMSSWith(map[string]string{ + peerRESTLogMask: strconv.Itoa(int(kind)), + })) + if err != nil { + return + } + st.Results(func(b *grid.Bytes) error { + select { + case logCh <- *b: + default: + consoleLogRPC.PutResponse(b) + // Do not block on slow receivers. + } + return nil + }) +} + +// ConsoleLog - sends request to peer nodes to get console logs +func (client *peerRESTClient) ConsoleLog(ctx context.Context, kind madmin.LogMask, logCh chan<- []byte) { + go func() { + for { + client.doConsoleLog(ctx, kind, logCh) + select { + case <-ctx.Done(): + return + default: + // There was error in the REST request, retry after sometime as probably the peer is down. + time.Sleep(5 * time.Second) + } + } + }() +} + +// newPeerRestClients creates new peer clients. +// The two slices will point to the same clients, +// but 'all' will contain nil entry for local client. +// The 'all' slice will be in the same order across the cluster. +func newPeerRestClients(endpoints EndpointServerPools) (remote, all []*peerRESTClient) { + if !globalIsDistErasure { + // Only useful in distributed setups + return nil, nil + } + + hosts := endpoints.hostsSorted() + remote = make([]*peerRESTClient, 0, len(hosts)) + all = make([]*peerRESTClient, len(hosts)) + for i, host := range hosts { + if host == nil { + continue + } + all[i] = newPeerRESTClient(host, endpoints.FindGridHostsFromPeer(host)) + remote = append(remote, all[i]) + } + if len(all) != len(remote)+1 { + peersLogIf(context.Background(), fmt.Errorf("Expected number of all hosts (%v) to be remote +1 (%v)", len(all), len(remote)), logger.WarningKind) + } + return remote, all +} + +// MonitorBandwidth - send http trace request to peer nodes +func (client *peerRESTClient) MonitorBandwidth(ctx context.Context, buckets []string) (*bandwidth.BucketBandwidthReport, error) { + values := grid.NewURLValuesWith(map[string][]string{ + peerRESTBuckets: buckets, + }) + return getBandwidthRPC.Call(ctx, client.gridConn(), values) +} + +func (client *peerRESTClient) GetResourceMetrics(ctx context.Context) (<-chan MetricV2, error) { + resp, err := getResourceMetricsRPC.Call(ctx, client.gridConn(), grid.NewMSS()) + if err != nil { + return nil, err + } + ch := make(chan MetricV2) + go func(ch chan<- MetricV2) { + defer close(ch) + for _, m := range resp.Value() { + if m == nil { + continue + } + select { + case <-ctx.Done(): + return + case ch <- *m: + } + } + }(ch) + return ch, nil +} + +func (client *peerRESTClient) GetPeerMetrics(ctx context.Context) (<-chan MetricV2, error) { + resp, err := getPeerMetricsRPC.Call(ctx, client.gridConn(), grid.NewMSS()) + if err != nil { + return nil, err + } + ch := make(chan MetricV2) + go func() { + defer close(ch) + for _, m := range resp.Value() { + if m == nil { + continue + } + select { + case <-ctx.Done(): + return + case ch <- *m: + } + } + }() + return ch, nil +} + +func (client *peerRESTClient) GetPeerBucketMetrics(ctx context.Context) (<-chan MetricV2, error) { + resp, err := getPeerBucketMetricsRPC.Call(ctx, client.gridConn(), grid.NewMSS()) + if err != nil { + return nil, err + } + ch := make(chan MetricV2) + go func() { + defer close(ch) + for _, m := range resp.Value() { + if m == nil { + continue + } + select { + case <-ctx.Done(): + return + case ch <- *m: + } + } + }() + return ch, nil +} + +func (client *peerRESTClient) SpeedTest(ctx context.Context, opts speedTestOpts) (SpeedTestResult, error) { + values := make(url.Values) + values.Set(peerRESTSize, strconv.Itoa(opts.objectSize)) + values.Set(peerRESTConcurrent, strconv.Itoa(opts.concurrency)) + values.Set(peerRESTDuration, opts.duration.String()) + values.Set(peerRESTStorageClass, opts.storageClass) + values.Set(peerRESTBucket, opts.bucketName) + values.Set(peerRESTEnableSha256, strconv.FormatBool(opts.enableSha256)) + values.Set(peerRESTEnableMultipart, strconv.FormatBool(opts.enableMultipart)) + values.Set(peerRESTAccessKey, opts.creds.AccessKey) + + respBody, err := client.callWithContext(context.Background(), peerRESTMethodSpeedTest, values, nil, -1) + if err != nil { + return SpeedTestResult{}, err + } + defer xhttp.DrainBody(respBody) + waitReader, err := waitForHTTPResponse(respBody) + if err != nil { + return SpeedTestResult{}, err + } + + var result SpeedTestResult + err = gob.NewDecoder(waitReader).Decode(&result) + if err != nil { + return result, err + } + if result.Error != "" { + return result, errors.New(result.Error) + } + return result, nil +} + +func (client *peerRESTClient) DriveSpeedTest(ctx context.Context, opts madmin.DriveSpeedTestOpts) (madmin.DriveSpeedTestResult, error) { + queryVals := make(url.Values) + if opts.Serial { + queryVals.Set("serial", "true") + } + queryVals.Set("blocksize", strconv.FormatUint(opts.BlockSize, 10)) + queryVals.Set("filesize", strconv.FormatUint(opts.FileSize, 10)) + + respBody, err := client.callWithContext(ctx, peerRESTMethodDriveSpeedTest, queryVals, nil, -1) + if err != nil { + return madmin.DriveSpeedTestResult{}, err + } + defer xhttp.DrainBody(respBody) + waitReader, err := waitForHTTPResponse(respBody) + if err != nil { + return madmin.DriveSpeedTestResult{}, err + } + + var result madmin.DriveSpeedTestResult + err = gob.NewDecoder(waitReader).Decode(&result) + if err != nil { + return result, err + } + if result.Error != "" { + return result, errors.New(result.Error) + } + return result, nil +} + +func (client *peerRESTClient) GetLastDayTierStats(ctx context.Context) (DailyAllTierStats, error) { + resp, err := getLastDayTierStatsRPC.Call(ctx, client.gridConn(), grid.NewMSS()) + if err != nil || resp == nil { + return DailyAllTierStats{}, err + } + return *resp, nil +} + +// DevNull - Used by netperf to pump data to peer +func (client *peerRESTClient) DevNull(ctx context.Context, r io.Reader) error { + respBody, err := client.callWithContext(ctx, peerRESTMethodDevNull, nil, r, -1) + if err != nil { + return err + } + defer xhttp.DrainBody(respBody) + return err +} + +// Netperf - To initiate netperf on peer +func (client *peerRESTClient) Netperf(ctx context.Context, duration time.Duration) (madmin.NetperfNodeResult, error) { + var result madmin.NetperfNodeResult + values := make(url.Values) + values.Set(peerRESTDuration, duration.String()) + respBody, err := client.callWithContext(context.Background(), peerRESTMethodNetperf, values, nil, -1) + if err != nil { + return result, err + } + defer xhttp.DrainBody(respBody) + err = gob.NewDecoder(respBody).Decode(&result) + return result, err +} + +// GetReplicationMRF - get replication MRF for bucket +func (client *peerRESTClient) GetReplicationMRF(ctx context.Context, bucket string) (chan madmin.ReplicationMRF, error) { + values := make(url.Values) + values.Set(peerRESTBucket, bucket) + + respBody, err := client.callWithContext(ctx, peerRESTMethodGetReplicationMRF, values, nil, -1) + if err != nil { + return nil, err + } + dec := gob.NewDecoder(respBody) + ch := make(chan madmin.ReplicationMRF) + go func(ch chan madmin.ReplicationMRF) { + defer func() { + xhttp.DrainBody(respBody) + close(ch) + }() + for { + var entry madmin.ReplicationMRF + if err := dec.Decode(&entry); err != nil { + return + } + select { + case <-ctx.Done(): + return + case ch <- entry: + } + } + }(ch) + return ch, nil +} diff --git a/cmd/peer-rest-common.go b/cmd/peer-rest-common.go new file mode 100644 index 00000000..b643d68a --- /dev/null +++ b/cmd/peer-rest-common.go @@ -0,0 +1,84 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import "time" + +const ( + peerRESTVersion = "v39" // add more flags to speedtest API + peerRESTVersionPrefix = SlashSeparator + peerRESTVersion + peerRESTPrefix = minioReservedBucketPath + "/peer" + peerRESTPath = peerRESTPrefix + peerRESTVersionPrefix +) + +const ( + peerRESTMethodHealth = "/health" + peerRESTMethodVerifyBinary = "/verifybinary" + peerRESTMethodCommitBinary = "/commitbinary" + peerRESTMethodStartProfiling = "/startprofiling" + peerRESTMethodDownloadProfilingData = "/downloadprofilingdata" + peerRESTMethodSpeedTest = "/speedtest" + peerRESTMethodDriveSpeedTest = "/drivespeedtest" + peerRESTMethodDevNull = "/devnull" + peerRESTMethodNetperf = "/netperf" + peerRESTMethodGetReplicationMRF = "/getreplicationmrf" +) + +const ( + peerRESTBucket = "bucket" + peerRESTBuckets = "buckets" + peerRESTUser = "user" + peerRESTGroup = "group" + peerRESTUserTemp = "user-temp" + peerRESTPolicy = "policy" + peerRESTUserOrGroup = "user-or-group" + peerRESTUserType = "user-type" + peerRESTIsGroup = "is-group" + peerRESTSignal = "signal" + peerRESTSubSys = "sub-sys" + peerRESTProfiler = "profiler" + peerRESTSize = "size" + peerRESTConcurrent = "concurrent" + peerRESTDuration = "duration" + peerRESTStorageClass = "storage-class" + peerRESTEnableSha256 = "enableSha256" + peerRESTEnableMultipart = "enableMultipart" + peerRESTAccessKey = "access-key" + peerRESTMetricsTypes = "types" + peerRESTDisk = "disk" + peerRESTHost = "host" + peerRESTJobID = "job-id" + peerRESTDepID = "depID" + peerRESTStartRebalance = "start-rebalance" + peerRESTMetrics = "metrics" + peerRESTDryRun = "dry-run" + peerRESTUploadID = "up-id" + + peerRESTURL = "url" + peerRESTSha256Sum = "sha256sum" + peerRESTReleaseInfo = "releaseinfo" + peerRESTExecAt = "exec-at" + + peerRESTListenBucket = "bucket" + peerRESTListenPrefix = "prefix" + peerRESTListenSuffix = "suffix" + peerRESTListenEvents = "events" + peerRESTLogMask = "log-mask" +) + +const restartUpdateDelay = 250 * time.Millisecond diff --git a/cmd/peer-rest-server.go b/cmd/peer-rest-server.go new file mode 100644 index 00000000..9335b761 --- /dev/null +++ b/cmd/peer-rest-server.go @@ -0,0 +1,1424 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/gob" + "encoding/hex" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/dustin/go-humanize" + "github.com/klauspost/compress/zstd" + "github.com/minio/madmin-go/v3" + "github.com/minio/madmin-go/v3/logger/log" + "github.com/minio/minio/internal/bucket/bandwidth" + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/grid" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/pubsub" + "github.com/minio/mux" +) + +// To abstract a node over network. +type peerRESTServer struct{} + +var ( + // Types & Wrappers + aoBucketInfo = grid.NewArrayOf[*BucketInfo](func() *BucketInfo { return &BucketInfo{} }) + aoMetricsGroup = grid.NewArrayOf[*MetricV2](func() *MetricV2 { return &MetricV2{} }) + madminBgHealState = grid.NewJSONPool[madmin.BgHealState]() + madminHealResultItem = grid.NewJSONPool[madmin.HealResultItem]() + madminCPUs = grid.NewJSONPool[madmin.CPUs]() + madminMemInfo = grid.NewJSONPool[madmin.MemInfo]() + madminNetInfo = grid.NewJSONPool[madmin.NetInfo]() + madminOSInfo = grid.NewJSONPool[madmin.OSInfo]() + madminPartitions = grid.NewJSONPool[madmin.Partitions]() + madminProcInfo = grid.NewJSONPool[madmin.ProcInfo]() + madminRealtimeMetrics = grid.NewJSONPool[madmin.RealtimeMetrics]() + madminServerProperties = grid.NewJSONPool[madmin.ServerProperties]() + madminStorageInfo = grid.NewJSONPool[madmin.StorageInfo]() + madminSysConfig = grid.NewJSONPool[madmin.SysConfig]() + madminSysErrors = grid.NewJSONPool[madmin.SysErrors]() + madminSysServices = grid.NewJSONPool[madmin.SysServices]() + + // Request -> Response RPC calls + deleteBucketMetadataRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerDeleteBucketMetadata, grid.NewMSS, grid.NewNoPayload).IgnoreNilConn() + deleteBucketRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerDeleteBucket, grid.NewMSS, grid.NewNoPayload) + deletePolicyRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerDeletePolicy, grid.NewMSS, grid.NewNoPayload).IgnoreNilConn() + deleteSvcActRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerDeleteServiceAccount, grid.NewMSS, grid.NewNoPayload).IgnoreNilConn() + deleteUserRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerDeleteUser, grid.NewMSS, grid.NewNoPayload).IgnoreNilConn() + getAllBucketStatsRPC = grid.NewSingleHandler[*grid.MSS, *BucketStatsMap](grid.HandlerGetAllBucketStats, grid.NewMSS, func() *BucketStatsMap { return &BucketStatsMap{} }) + getBackgroundHealStatusRPC = grid.NewSingleHandler[*grid.MSS, *grid.JSON[madmin.BgHealState]](grid.HandlerBackgroundHealStatus, grid.NewMSS, madminBgHealState.NewJSON) + getBandwidthRPC = grid.NewSingleHandler[*grid.URLValues, *bandwidth.BucketBandwidthReport](grid.HandlerGetBandwidth, grid.NewURLValues, func() *bandwidth.BucketBandwidthReport { return &bandwidth.BucketBandwidthReport{} }) + getBucketStatsRPC = grid.NewSingleHandler[*grid.MSS, *BucketStats](grid.HandlerGetBucketStats, grid.NewMSS, func() *BucketStats { return &BucketStats{} }) + getCPUsHandler = grid.NewSingleHandler[*grid.MSS, *grid.JSON[madmin.CPUs]](grid.HandlerGetCPUs, grid.NewMSS, madminCPUs.NewJSON) + getLastDayTierStatsRPC = grid.NewSingleHandler[*grid.MSS, *DailyAllTierStats](grid.HandlerGetLastDayTierStats, grid.NewMSS, func() *DailyAllTierStats { return &DailyAllTierStats{} }) + getLocksRPC = grid.NewSingleHandler[*grid.MSS, *localLockMap](grid.HandlerGetLocks, grid.NewMSS, func() *localLockMap { return &localLockMap{} }) + getMemInfoRPC = grid.NewSingleHandler[*grid.MSS, *grid.JSON[madmin.MemInfo]](grid.HandlerGetMemInfo, grid.NewMSS, madminMemInfo.NewJSON) + getMetacacheListingRPC = grid.NewSingleHandler[*listPathOptions, *metacache](grid.HandlerGetMetacacheListing, func() *listPathOptions { return &listPathOptions{} }, func() *metacache { return &metacache{} }) + getMetricsRPC = grid.NewSingleHandler[*grid.URLValues, *grid.JSON[madmin.RealtimeMetrics]](grid.HandlerGetMetrics, grid.NewURLValues, madminRealtimeMetrics.NewJSON) + getNetInfoRPC = grid.NewSingleHandler[*grid.MSS, *grid.JSON[madmin.NetInfo]](grid.HandlerGetNetInfo, grid.NewMSS, madminNetInfo.NewJSON) + getOSInfoRPC = grid.NewSingleHandler[*grid.MSS, *grid.JSON[madmin.OSInfo]](grid.HandlerGetOSInfo, grid.NewMSS, madminOSInfo.NewJSON) + getPartitionsRPC = grid.NewSingleHandler[*grid.MSS, *grid.JSON[madmin.Partitions]](grid.HandlerGetPartitions, grid.NewMSS, madminPartitions.NewJSON) + getPeerBucketMetricsRPC = grid.NewSingleHandler[*grid.MSS, *grid.Array[*MetricV2]](grid.HandlerGetPeerBucketMetrics, grid.NewMSS, aoMetricsGroup.New) + getPeerMetricsRPC = grid.NewSingleHandler[*grid.MSS, *grid.Array[*MetricV2]](grid.HandlerGetPeerMetrics, grid.NewMSS, aoMetricsGroup.New) + getResourceMetricsRPC = grid.NewSingleHandler[*grid.MSS, *grid.Array[*MetricV2]](grid.HandlerGetResourceMetrics, grid.NewMSS, aoMetricsGroup.New) + getProcInfoRPC = grid.NewSingleHandler[*grid.MSS, *grid.JSON[madmin.ProcInfo]](grid.HandlerGetProcInfo, grid.NewMSS, madminProcInfo.NewJSON) + getSRMetricsRPC = grid.NewSingleHandler[*grid.MSS, *SRMetricsSummary](grid.HandlerGetSRMetrics, grid.NewMSS, func() *SRMetricsSummary { return &SRMetricsSummary{} }) + getSysConfigRPC = grid.NewSingleHandler[*grid.MSS, *grid.JSON[madmin.SysConfig]](grid.HandlerGetSysConfig, grid.NewMSS, madminSysConfig.NewJSON) + getSysErrorsRPC = grid.NewSingleHandler[*grid.MSS, *grid.JSON[madmin.SysErrors]](grid.HandlerGetSysErrors, grid.NewMSS, madminSysErrors.NewJSON) + getSysServicesRPC = grid.NewSingleHandler[*grid.MSS, *grid.JSON[madmin.SysServices]](grid.HandlerGetSysServices, grid.NewMSS, madminSysServices.NewJSON) + headBucketRPC = grid.NewSingleHandler[*grid.MSS, *VolInfo](grid.HandlerHeadBucket, grid.NewMSS, func() *VolInfo { return &VolInfo{} }) + healBucketRPC = grid.NewSingleHandler[*grid.MSS, *grid.JSON[madmin.HealResultItem]](grid.HandlerHealBucket, grid.NewMSS, madminHealResultItem.NewJSON) + listBucketsRPC = grid.NewSingleHandler[*BucketOptions, *grid.Array[*BucketInfo]](grid.HandlerListBuckets, func() *BucketOptions { return &BucketOptions{} }, aoBucketInfo.New) + loadBucketMetadataRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerLoadBucketMetadata, grid.NewMSS, grid.NewNoPayload).IgnoreNilConn() + loadGroupRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerLoadGroup, grid.NewMSS, grid.NewNoPayload) + loadPolicyMappingRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerLoadPolicyMapping, grid.NewMSS, grid.NewNoPayload).IgnoreNilConn() + loadPolicyRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerLoadPolicy, grid.NewMSS, grid.NewNoPayload).IgnoreNilConn() + loadRebalanceMetaRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerLoadRebalanceMeta, grid.NewMSS, grid.NewNoPayload) + loadSvcActRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerLoadServiceAccount, grid.NewMSS, grid.NewNoPayload).IgnoreNilConn() + loadTransitionTierConfigRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerLoadTransitionTierConfig, grid.NewMSS, grid.NewNoPayload) + loadUserRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerLoadUser, grid.NewMSS, grid.NewNoPayload).IgnoreNilConn() + localStorageInfoRPC = grid.NewSingleHandler[*grid.MSS, *grid.JSON[madmin.StorageInfo]](grid.HandlerStorageInfo, grid.NewMSS, madminStorageInfo.NewJSON) + makeBucketRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerMakeBucket, grid.NewMSS, grid.NewNoPayload) + reloadPoolMetaRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerReloadPoolMeta, grid.NewMSS, grid.NewNoPayload) + reloadSiteReplicationConfigRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerReloadSiteReplicationConfig, grid.NewMSS, grid.NewNoPayload) + serverInfoRPC = grid.NewSingleHandler[*grid.MSS, *grid.JSON[madmin.ServerProperties]](grid.HandlerServerInfo, grid.NewMSS, madminServerProperties.NewJSON) + signalServiceRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerSignalService, grid.NewMSS, grid.NewNoPayload) + stopRebalanceRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerStopRebalance, grid.NewMSS, grid.NewNoPayload) + updateMetacacheListingRPC = grid.NewSingleHandler[*metacache, *metacache](grid.HandlerUpdateMetacacheListing, func() *metacache { return &metacache{} }, func() *metacache { return &metacache{} }) + cleanupUploadIDCacheMetaRPC = grid.NewSingleHandler[*grid.MSS, grid.NoPayload](grid.HandlerClearUploadID, grid.NewMSS, grid.NewNoPayload) + + // STREAMS + // Set an output capacity of 100 for consoleLog and listenRPC + // There is another buffer that will buffer events. + consoleLogRPC = grid.NewStream[*grid.MSS, grid.NoPayload, *grid.Bytes](grid.HandlerConsoleLog, grid.NewMSS, nil, grid.NewBytes).WithOutCapacity(100) + listenRPC = grid.NewStream[*grid.URLValues, grid.NoPayload, *grid.Bytes](grid.HandlerListen, grid.NewURLValues, nil, grid.NewBytes).WithOutCapacity(100) +) + +// GetLocksHandler - returns list of lock from the server. +func (s *peerRESTServer) GetLocksHandler(_ *grid.MSS) (*localLockMap, *grid.RemoteErr) { + res := globalLockServer.DupLockMap() + return &res, nil +} + +// DeletePolicyHandler - deletes a policy on the server. +func (s *peerRESTServer) DeletePolicyHandler(mss *grid.MSS) (np grid.NoPayload, nerr *grid.RemoteErr) { + objAPI := newObjectLayerFn() + if objAPI == nil { + return np, grid.NewRemoteErr(errServerNotInitialized) + } + + policyName := mss.Get(peerRESTPolicy) + if policyName == "" { + return np, grid.NewRemoteErr(errors.New("policyName is missing")) + } + + if err := globalIAMSys.DeletePolicy(context.Background(), policyName, false); err != nil { + return np, grid.NewRemoteErr(err) + } + + return np, nerr +} + +// LoadPolicyHandler - reloads a policy on the server. +func (s *peerRESTServer) LoadPolicyHandler(mss *grid.MSS) (np grid.NoPayload, nerr *grid.RemoteErr) { + objAPI := newObjectLayerFn() + if objAPI == nil { + return np, grid.NewRemoteErr(errServerNotInitialized) + } + + policyName := mss.Get(peerRESTPolicy) + if policyName == "" { + return np, grid.NewRemoteErr(errors.New("policyName is missing")) + } + + if err := globalIAMSys.LoadPolicy(context.Background(), objAPI, policyName); err != nil { + return np, grid.NewRemoteErr(err) + } + + return np, nerr +} + +// LoadPolicyMappingHandler - reloads a policy mapping on the server. +func (s *peerRESTServer) LoadPolicyMappingHandler(mss *grid.MSS) (np grid.NoPayload, nerr *grid.RemoteErr) { + objAPI := newObjectLayerFn() + if objAPI == nil { + return np, grid.NewRemoteErr(errServerNotInitialized) + } + userOrGroup := mss.Get(peerRESTUserOrGroup) + if userOrGroup == "" { + return np, grid.NewRemoteErr(errors.New("user-or-group is missing")) + } + + userType, err := strconv.Atoi(mss.Get(peerRESTUserType)) + if err != nil { + return np, grid.NewRemoteErr(fmt.Errorf("user-type `%s` is invalid: %w", mss.Get(peerRESTUserType), err)) + } + + isGroup := mss.Get(peerRESTIsGroup) == "true" + if err := globalIAMSys.LoadPolicyMapping(context.Background(), objAPI, userOrGroup, IAMUserType(userType), isGroup); err != nil { + return np, grid.NewRemoteErr(err) + } + + return np, nerr +} + +// DeleteServiceAccountHandler - deletes a service account on the server. +func (s *peerRESTServer) DeleteServiceAccountHandler(mss *grid.MSS) (np grid.NoPayload, nerr *grid.RemoteErr) { + objAPI := newObjectLayerFn() + if objAPI == nil { + return np, grid.NewRemoteErr(errServerNotInitialized) + } + + accessKey := mss.Get(peerRESTUser) + if accessKey == "" { + return np, grid.NewRemoteErr(errors.New("service account name is missing")) + } + + if err := globalIAMSys.DeleteServiceAccount(context.Background(), accessKey, false); err != nil { + return np, grid.NewRemoteErr(err) + } + + return np, nerr +} + +// LoadServiceAccountHandler - reloads a service account on the server. +func (s *peerRESTServer) LoadServiceAccountHandler(mss *grid.MSS) (np grid.NoPayload, nerr *grid.RemoteErr) { + objAPI := newObjectLayerFn() + if objAPI == nil { + return np, grid.NewRemoteErr(errServerNotInitialized) + } + + accessKey := mss.Get(peerRESTUser) + if accessKey == "" { + return np, grid.NewRemoteErr(errors.New("service account name is missing")) + } + + if err := globalIAMSys.LoadServiceAccount(context.Background(), accessKey); err != nil { + return np, grid.NewRemoteErr(err) + } + + return np, nerr +} + +// DeleteUserHandler - deletes a user on the server. +func (s *peerRESTServer) DeleteUserHandler(mss *grid.MSS) (np grid.NoPayload, nerr *grid.RemoteErr) { + objAPI := newObjectLayerFn() + if objAPI == nil { + return np, grid.NewRemoteErr(errServerNotInitialized) + } + + accessKey := mss.Get(peerRESTUser) + if accessKey == "" { + return np, grid.NewRemoteErr(errors.New("username is missing")) + } + + if err := globalIAMSys.DeleteUser(context.Background(), accessKey, false); err != nil { + return np, grid.NewRemoteErr(err) + } + + return np, nerr +} + +// LoadUserHandler - reloads a user on the server. +func (s *peerRESTServer) LoadUserHandler(mss *grid.MSS) (np grid.NoPayload, nerr *grid.RemoteErr) { + objAPI := newObjectLayerFn() + if objAPI == nil { + return np, grid.NewRemoteErr(errServerNotInitialized) + } + + accessKey := mss.Get(peerRESTUser) + if accessKey == "" { + return np, grid.NewRemoteErr(errors.New("username is missing")) + } + + temp, err := strconv.ParseBool(mss.Get(peerRESTUserTemp)) + if err != nil { + return np, grid.NewRemoteErr(err) + } + + userType := regUser + if temp { + userType = stsUser + } + + if err = globalIAMSys.LoadUser(context.Background(), objAPI, accessKey, userType); err != nil { + return np, grid.NewRemoteErr(err) + } + + return np, nerr +} + +// LoadGroupHandler - reloads group along with members list. +func (s *peerRESTServer) LoadGroupHandler(mss *grid.MSS) (np grid.NoPayload, nerr *grid.RemoteErr) { + objAPI := newObjectLayerFn() + if objAPI == nil { + return np, grid.NewRemoteErr(errServerNotInitialized) + } + + group := mss.Get(peerRESTGroup) + if group == "" { + return np, grid.NewRemoteErr(errors.New("group is missing")) + } + + err := globalIAMSys.LoadGroup(context.Background(), objAPI, group) + if err != nil { + return np, grid.NewRemoteErr(err) + } + + return np, nerr +} + +// StartProfilingHandler - Issues the start profiling command. +func (s *peerRESTServer) StartProfilingHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + s.writeErrorResponse(w, errors.New("Invalid request")) + return + } + + vars := mux.Vars(r) + profiles := strings.Split(vars[peerRESTProfiler], ",") + if len(profiles) == 0 { + s.writeErrorResponse(w, errors.New("profiler name is missing")) + return + } + globalProfilerMu.Lock() + defer globalProfilerMu.Unlock() + if globalProfiler == nil { + globalProfiler = make(map[string]minioProfiler, 10) + } + + // Stop profiler of all types if already running + for k, v := range globalProfiler { + for _, p := range profiles { + if p == k { + v.Stop() + delete(globalProfiler, k) + } + } + } + + for _, profiler := range profiles { + prof, err := startProfiler(profiler) + if err != nil { + s.writeErrorResponse(w, err) + return + } + globalProfiler[profiler] = prof + } +} + +// DownloadProfilingDataHandler - returns profiled data. +func (s *peerRESTServer) DownloadProfilingDataHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + s.writeErrorResponse(w, errors.New("Invalid request")) + return + } + + ctx := newContext(r, w, "DownloadProfiling") + profileData, err := getProfileData() + if err != nil { + s.writeErrorResponse(w, err) + return + } + peersLogIf(ctx, gob.NewEncoder(w).Encode(profileData)) +} + +func (s *peerRESTServer) LocalStorageInfoHandler(mss *grid.MSS) (*grid.JSON[madmin.StorageInfo], *grid.RemoteErr) { + objLayer := newObjectLayerFn() + if objLayer == nil { + return nil, grid.NewRemoteErr(errServerNotInitialized) + } + + metrics, err := strconv.ParseBool(mss.Get(peerRESTMetrics)) + if err != nil { + return nil, grid.NewRemoteErr(err) + } + info := objLayer.LocalStorageInfo(context.Background(), metrics) + return madminStorageInfo.NewJSONWith(&info), nil +} + +// ServerInfoHandler - returns Server Info +func (s *peerRESTServer) ServerInfoHandler(params *grid.MSS) (*grid.JSON[madmin.ServerProperties], *grid.RemoteErr) { + r := http.Request{Host: globalLocalNodeName} + metrics, err := strconv.ParseBool(params.Get(peerRESTMetrics)) + if err != nil { + return nil, grid.NewRemoteErr(err) + } + info := getLocalServerProperty(globalEndpoints, &r, metrics) + return madminServerProperties.NewJSONWith(&info), nil +} + +// GetCPUsHandler - returns CPU info. +func (s *peerRESTServer) GetCPUsHandler(_ *grid.MSS) (*grid.JSON[madmin.CPUs], *grid.RemoteErr) { + info := madmin.GetCPUs(context.Background(), globalLocalNodeName) + return madminCPUs.NewJSONWith(&info), nil +} + +// GetNetInfoHandler - returns network information. +func (s *peerRESTServer) GetNetInfoHandler(_ *grid.MSS) (*grid.JSON[madmin.NetInfo], *grid.RemoteErr) { + info := madmin.GetNetInfo(globalLocalNodeName, globalInternodeInterface) + return madminNetInfo.NewJSONWith(&info), nil +} + +// GetPartitionsHandler - returns disk partition information. +func (s *peerRESTServer) GetPartitionsHandler(_ *grid.MSS) (*grid.JSON[madmin.Partitions], *grid.RemoteErr) { + info := madmin.GetPartitions(context.Background(), globalLocalNodeName) + return madminPartitions.NewJSONWith(&info), nil +} + +// GetOSInfoHandler - returns operating system's information. +func (s *peerRESTServer) GetOSInfoHandler(_ *grid.MSS) (*grid.JSON[madmin.OSInfo], *grid.RemoteErr) { + info := madmin.GetOSInfo(context.Background(), globalLocalNodeName) + return madminOSInfo.NewJSONWith(&info), nil +} + +// GetProcInfoHandler - returns this MinIO process information. +func (s *peerRESTServer) GetProcInfoHandler(_ *grid.MSS) (*grid.JSON[madmin.ProcInfo], *grid.RemoteErr) { + info := madmin.GetProcInfo(context.Background(), globalLocalNodeName) + return madminProcInfo.NewJSONWith(&info), nil +} + +// GetMemInfoHandler - returns memory information. +func (s *peerRESTServer) GetMemInfoHandler(_ *grid.MSS) (*grid.JSON[madmin.MemInfo], *grid.RemoteErr) { + info := madmin.GetMemInfo(context.Background(), globalLocalNodeName) + return madminMemInfo.NewJSONWith(&info), nil +} + +// GetMetricsHandler - returns server metrics. +func (s *peerRESTServer) GetMetricsHandler(v *grid.URLValues) (*grid.JSON[madmin.RealtimeMetrics], *grid.RemoteErr) { + values := v.Values() + var types madmin.MetricType + if t, _ := strconv.ParseUint(values.Get(peerRESTMetricsTypes), 10, 64); t != 0 { + types = madmin.MetricType(t) + } else { + types = madmin.MetricsAll + } + + diskMap := make(map[string]struct{}) + for _, disk := range values[peerRESTDisk] { + diskMap[disk] = struct{}{} + } + + hostMap := make(map[string]struct{}) + for _, host := range values[peerRESTHost] { + hostMap[host] = struct{}{} + } + + info := collectLocalMetrics(types, collectMetricsOpts{ + disks: diskMap, + hosts: hostMap, + jobID: values.Get(peerRESTJobID), + depID: values.Get(peerRESTDepID), + }) + return madminRealtimeMetrics.NewJSONWith(&info), nil +} + +// GetSysConfigHandler - returns system config information. +// (only the config that are of concern to minio) +func (s *peerRESTServer) GetSysConfigHandler(_ *grid.MSS) (*grid.JSON[madmin.SysConfig], *grid.RemoteErr) { + info := madmin.GetSysConfig(context.Background(), globalLocalNodeName) + return madminSysConfig.NewJSONWith(&info), nil +} + +// GetSysServicesHandler - returns system services information. +// (only the services that are of concern to minio) +func (s *peerRESTServer) GetSysServicesHandler(_ *grid.MSS) (*grid.JSON[madmin.SysServices], *grid.RemoteErr) { + info := madmin.GetSysServices(context.Background(), globalLocalNodeName) + return madminSysServices.NewJSONWith(&info), nil +} + +// GetSysErrorsHandler - returns system level errors +func (s *peerRESTServer) GetSysErrorsHandler(_ *grid.MSS) (*grid.JSON[madmin.SysErrors], *grid.RemoteErr) { + info := madmin.GetSysErrors(context.Background(), globalLocalNodeName) + return madminSysErrors.NewJSONWith(&info), nil +} + +// DeleteBucketMetadataHandler - Delete in memory bucket metadata +func (s *peerRESTServer) DeleteBucketMetadataHandler(mss *grid.MSS) (np grid.NoPayload, nerr *grid.RemoteErr) { + bucketName := mss.Get(peerRESTBucket) + if bucketName == "" { + return np, grid.NewRemoteErr(errors.New("Bucket name is missing")) + } + + globalReplicationStats.Load().Delete(bucketName) + globalBucketMetadataSys.Remove(bucketName) + globalBucketTargetSys.Delete(bucketName) + globalEventNotifier.RemoveNotification(bucketName) + globalBucketConnStats.delete(bucketName) + globalBucketHTTPStats.delete(bucketName) + if localMetacacheMgr != nil { + localMetacacheMgr.deleteBucketCache(bucketName) + } + return np, nerr +} + +// GetAllBucketStatsHandler - fetches bucket replication stats for all buckets from this peer. +func (s *peerRESTServer) GetAllBucketStatsHandler(mss *grid.MSS) (*BucketStatsMap, *grid.RemoteErr) { + replicationStats := globalReplicationStats.Load().GetAll() + bucketStatsMap := make(map[string]BucketStats, len(replicationStats)) + for k, v := range replicationStats { + bucketStatsMap[k] = BucketStats{ + ReplicationStats: v, + ProxyStats: globalReplicationStats.Load().getProxyStats(k), + } + } + return &BucketStatsMap{Stats: bucketStatsMap, Timestamp: time.Now()}, nil +} + +// GetBucketStatsHandler - fetches current in-memory bucket stats, currently only +// returns BucketStats, that currently includes ReplicationStats. +func (s *peerRESTServer) GetBucketStatsHandler(vars *grid.MSS) (*BucketStats, *grid.RemoteErr) { + bucketName := vars.Get(peerRESTBucket) + if bucketName == "" { + return nil, grid.NewRemoteErrString("Bucket name is missing") + } + st := globalReplicationStats.Load() + if st == nil { + return &BucketStats{}, nil + } + bs := BucketStats{ + ReplicationStats: st.Get(bucketName), + QueueStats: ReplicationQueueStats{Nodes: []ReplQNodeStats{st.getNodeQueueStats(bucketName)}}, + ProxyStats: st.getProxyStats(bucketName), + } + return &bs, nil +} + +// GetSRMetricsHandler - fetches current in-memory replication stats at site level from this peer +func (s *peerRESTServer) GetSRMetricsHandler(mss *grid.MSS) (*SRMetricsSummary, *grid.RemoteErr) { + objAPI := newObjectLayerFn() + if objAPI == nil { + return nil, grid.NewRemoteErr(errServerNotInitialized) + } + if st := globalReplicationStats.Load(); st != nil { + sm := st.getSRMetricsForNode() + return &sm, nil + } + return &SRMetricsSummary{}, nil +} + +// LoadBucketMetadataHandler - reloads in memory bucket metadata +func (s *peerRESTServer) LoadBucketMetadataHandler(mss *grid.MSS) (np grid.NoPayload, nerr *grid.RemoteErr) { + bucketName := mss.Get(peerRESTBucket) + if bucketName == "" { + return np, grid.NewRemoteErr(errors.New("Bucket name is missing")) + } + + objAPI := newObjectLayerFn() + if objAPI == nil { + return np, grid.NewRemoteErr(errServerNotInitialized) + } + + meta, err := loadBucketMetadata(context.Background(), objAPI, bucketName) + if err != nil { + return np, grid.NewRemoteErr(err) + } + + globalBucketMetadataSys.Set(bucketName, meta) + + if meta.notificationConfig != nil { + globalEventNotifier.AddRulesMap(bucketName, meta.notificationConfig.ToRulesMap()) + } + + if meta.bucketTargetConfig != nil { + globalBucketTargetSys.UpdateAllTargets(bucketName, meta.bucketTargetConfig) + } + + return np, nerr +} + +func (s *peerRESTServer) GetMetacacheListingHandler(opts *listPathOptions) (*metacache, *grid.RemoteErr) { + resp := localMetacacheMgr.getBucket(context.Background(), opts.Bucket).findCache(*opts) + return &resp, nil +} + +func (s *peerRESTServer) UpdateMetacacheListingHandler(req *metacache) (*metacache, *grid.RemoteErr) { + cache, err := localMetacacheMgr.updateCacheEntry(*req) + if err != nil { + return nil, grid.NewRemoteErr(err) + } + return &cache, nil +} + +// PutBucketNotificationHandler - Set bucket policy. +func (s *peerRESTServer) PutBucketNotificationHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + s.writeErrorResponse(w, errors.New("Invalid request")) + return + } + + vars := mux.Vars(r) + bucketName := vars[peerRESTBucket] + if bucketName == "" { + s.writeErrorResponse(w, errors.New("Bucket name is missing")) + return + } + + var rulesMap event.RulesMap + if r.ContentLength < 0 { + s.writeErrorResponse(w, errInvalidArgument) + return + } + + err := gob.NewDecoder(r.Body).Decode(&rulesMap) + if err != nil { + s.writeErrorResponse(w, err) + return + } + + globalEventNotifier.AddRulesMap(bucketName, rulesMap) +} + +// HealthHandler - returns true of health +func (s *peerRESTServer) HealthHandler(w http.ResponseWriter, r *http.Request) { + s.IsValid(w, r) +} + +// VerifyBinary - verifies the downloaded binary is in-tact +func (s *peerRESTServer) VerifyBinaryHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + s.writeErrorResponse(w, errors.New("Invalid request")) + return + } + + if r.ContentLength < 0 { + s.writeErrorResponse(w, errInvalidArgument) + return + } + + u, err := url.Parse(r.Form.Get(peerRESTURL)) + if err != nil { + s.writeErrorResponse(w, err) + return + } + + sha256Sum, err := hex.DecodeString(r.Form.Get(peerRESTSha256Sum)) + if err != nil { + s.writeErrorResponse(w, err) + return + } + releaseInfo := r.Form.Get(peerRESTReleaseInfo) + + lrTime, err := releaseInfoToReleaseTime(releaseInfo) + if err != nil { + s.writeErrorResponse(w, err) + return + } + + if lrTime.Sub(currentReleaseTime) <= 0 { + s.writeErrorResponse(w, fmt.Errorf("server is running the latest version: %s", Version)) + return + } + + zr, err := zstd.NewReader(r.Body) + if err != nil { + s.writeErrorResponse(w, err) + return + } + defer zr.Close() + + if err = verifyBinary(u, sha256Sum, releaseInfo, getMinioMode(), zr); err != nil { + s.writeErrorResponse(w, err) + return + } +} + +// CommitBinary - overwrites the current binary with the new one. +func (s *peerRESTServer) CommitBinaryHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + s.writeErrorResponse(w, errors.New("Invalid request")) + return + } + + if err := commitBinary(); err != nil { + s.writeErrorResponse(w, err) + return + } +} + +var errUnsupportedSignal = fmt.Errorf("unsupported signal") + +func waitingDrivesNode() map[string]madmin.DiskMetrics { + globalLocalDrivesMu.RLock() + localDrives := cloneDrives(globalLocalDrivesMap) + globalLocalDrivesMu.RUnlock() + + errs := make([]error, len(localDrives)) + infos := make([]DiskInfo, len(localDrives)) + for i, drive := range localDrives { + infos[i], errs[i] = drive.DiskInfo(GlobalContext, DiskInfoOptions{}) + } + infoMaps := make(map[string]madmin.DiskMetrics) + for i := range infos { + if infos[i].Metrics.TotalWaiting >= 1 && errors.Is(errs[i], errFaultyDisk) { + infoMaps[infos[i].Endpoint] = madmin.DiskMetrics{ + TotalWaiting: infos[i].Metrics.TotalWaiting, + } + } + } + return infoMaps +} + +// SignalServiceHandler - signal service handler. +func (s *peerRESTServer) SignalServiceHandler(vars *grid.MSS) (np grid.NoPayload, nerr *grid.RemoteErr) { + signalString := vars.Get(peerRESTSignal) + if signalString == "" { + return np, grid.NewRemoteErrString("signal name is missing") + } + si, err := strconv.Atoi(signalString) + if err != nil { + return np, grid.NewRemoteErr(err) + } + + // Wait until the specified time before executing the signal. + if t := vars.Get(peerRESTExecAt); t != "" { + execAt, err := time.Parse(time.RFC3339Nano, vars.Get(peerRESTExecAt)) + if err != nil { + logger.LogIf(GlobalContext, "signalservice", err) + execAt = time.Now().Add(restartUpdateDelay) + } + if d := time.Until(execAt); d > 0 { + time.Sleep(d) + } + } + signal := serviceSignal(si) + switch signal { + case serviceRestart, serviceStop: + dryRun := vars.Get("dry-run") == "true" // This is only supported for `restart/stop` + + waitingDisks := waitingDrivesNode() + if len(waitingDisks) > 0 { + buf, err := json.Marshal(waitingDisks) + if err != nil { + return np, grid.NewRemoteErr(err) + } + return np, grid.NewRemoteErrString(string(buf)) + } + if !dryRun { + globalServiceSignalCh <- signal + } + case serviceFreeze: + freezeServices() + case serviceUnFreeze: + unfreezeServices() + case serviceReloadDynamic: + objAPI := newObjectLayerFn() + if objAPI == nil { + return np, grid.NewRemoteErr(errServerNotInitialized) + } + srvCfg, err := getValidConfig(objAPI) + if err != nil { + return np, grid.NewRemoteErr(err) + } + subSys := vars.Get(peerRESTSubSys) + // Apply dynamic values. + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + if subSys == "" { + err = applyDynamicConfig(ctx, objAPI, srvCfg) + } else { + err = applyDynamicConfigForSubSys(ctx, objAPI, srvCfg, subSys) + } + if err != nil { + return np, grid.NewRemoteErr(err) + } + default: + return np, grid.NewRemoteErr(errUnsupportedSignal) + } + return np, nil +} + +// ListenHandler sends http trace messages back to peer rest client +func (s *peerRESTServer) ListenHandler(ctx context.Context, v *grid.URLValues, out chan<- *grid.Bytes) *grid.RemoteErr { + values := v.Values() + defer v.Recycle() + var prefix string + if len(values[peerRESTListenPrefix]) > 1 { + return grid.NewRemoteErrString("invalid request (peerRESTListenPrefix)") + } + globalAPIConfig.getRequestsPoolCapacity() + if len(values[peerRESTListenPrefix]) == 1 { + if err := event.ValidateFilterRuleValue(values[peerRESTListenPrefix][0]); err != nil { + return grid.NewRemoteErr(err) + } + + prefix = values[peerRESTListenPrefix][0] + } + + var suffix string + if len(values[peerRESTListenSuffix]) > 1 { + return grid.NewRemoteErrString("invalid request (peerRESTListenSuffix)") + } + + if len(values[peerRESTListenSuffix]) == 1 { + if err := event.ValidateFilterRuleValue(values[peerRESTListenSuffix][0]); err != nil { + return grid.NewRemoteErr(err) + } + + suffix = values[peerRESTListenSuffix][0] + } + + pattern := event.NewPattern(prefix, suffix) + + var eventNames []event.Name + var mask pubsub.Mask + for _, ev := range values[peerRESTListenEvents] { + eventName, err := event.ParseName(ev) + if err != nil { + return grid.NewRemoteErr(err) + } + mask.MergeMaskable(eventName) + eventNames = append(eventNames, eventName) + } + + rulesMap := event.NewRulesMap(eventNames, pattern, event.TargetID{ID: mustGetUUID()}) + + // Listen Publisher uses nonblocking publish and hence does not wait for slow subscribers. + // Use buffered channel to take care of burst sends or slow w.Write() + ch := make(chan event.Event, globalAPIConfig.getRequestsPoolCapacity()) + err := globalHTTPListen.Subscribe(mask, ch, ctx.Done(), func(ev event.Event) bool { + if ev.S3.Bucket.Name != "" && values.Get(peerRESTListenBucket) != "" { + if ev.S3.Bucket.Name != values.Get(peerRESTListenBucket) { + return false + } + } + return rulesMap.MatchSimple(ev.EventName, ev.S3.Object.Key) + }) + if err != nil { + return grid.NewRemoteErr(err) + } + + // Process until remote disconnects. + // Blocks on upstream (out) congestion. + // We have however a dynamic downstream buffer (ch). + buf := bytes.NewBuffer(grid.GetByteBuffer()) + enc := json.NewEncoder(buf) + tmpEvt := struct{ Records []event.Event }{[]event.Event{{}}} + for { + select { + case <-ctx.Done(): + grid.PutByteBuffer(buf.Bytes()) + return nil + case ev := <-ch: + buf.Reset() + tmpEvt.Records[0] = ev + if err := enc.Encode(tmpEvt); err != nil { + peersLogOnceIf(ctx, err, "event: Encode failed") + continue + } + out <- grid.NewBytesWithCopyOf(buf.Bytes()) + } + } +} + +// TraceHandler sends http trace messages back to peer rest client +func (s *peerRESTServer) TraceHandler(ctx context.Context, payload []byte, _ <-chan []byte, out chan<- []byte) *grid.RemoteErr { + var traceOpts madmin.ServiceTraceOpts + err := json.Unmarshal(payload, &traceOpts) + if err != nil { + return grid.NewRemoteErr(err) + } + var wg sync.WaitGroup + + // Trace Publisher uses nonblocking publish and hence does not wait for slow subscribers. + // Use buffered channel to take care of burst sends or slow w.Write() + err = globalTrace.SubscribeJSON(traceOpts.TraceTypes(), out, ctx.Done(), func(entry madmin.TraceInfo) bool { + return shouldTrace(entry, traceOpts) + }, &wg) + if err != nil { + return grid.NewRemoteErr(err) + } + + // Publish bootstrap events that have already occurred before client could subscribe. + if traceOpts.TraceTypes().Contains(madmin.TraceBootstrap) { + go globalBootstrapTracer.Publish(ctx, globalTrace) + } + + // Wait for remote to cancel and SubscribeJSON to exit. + wg.Wait() + return nil +} + +func (s *peerRESTServer) BackgroundHealStatusHandler(_ *grid.MSS) (*grid.JSON[madmin.BgHealState], *grid.RemoteErr) { + state, ok := getLocalBackgroundHealStatus(context.Background(), newObjectLayerFn()) + if !ok { + return nil, grid.NewRemoteErr(errServerNotInitialized) + } + return madminBgHealState.NewJSONWith(&state), nil +} + +// ReloadSiteReplicationConfigHandler - reloads site replication configuration from the disks +func (s *peerRESTServer) ReloadSiteReplicationConfigHandler(mss *grid.MSS) (np grid.NoPayload, nerr *grid.RemoteErr) { + objAPI := newObjectLayerFn() + if objAPI == nil { + return np, grid.NewRemoteErr(errServerNotInitialized) + } + + peersLogIf(context.Background(), globalSiteReplicationSys.Init(context.Background(), objAPI)) + return np, nerr +} + +func (s *peerRESTServer) ReloadPoolMetaHandler(mss *grid.MSS) (np grid.NoPayload, nerr *grid.RemoteErr) { + objAPI := newObjectLayerFn() + if objAPI == nil { + return np, grid.NewRemoteErr(errServerNotInitialized) + } + + pools, ok := objAPI.(*erasureServerPools) + if !ok { + return np, nerr + } + + if err := pools.ReloadPoolMeta(context.Background()); err != nil { + return np, grid.NewRemoteErr(err) + } + + return np, nerr +} + +func (s *peerRESTServer) HandlerClearUploadID(mss *grid.MSS) (np grid.NoPayload, nerr *grid.RemoteErr) { + objAPI := newObjectLayerFn() + if objAPI == nil { + return np, grid.NewRemoteErr(errServerNotInitialized) + } + + pools, ok := objAPI.(*erasureServerPools) + if !ok { + return np, nerr + } + + // No need to return errors, this is not a highly strict operation. + uploadID := mss.Get(peerRESTUploadID) + if uploadID != "" { + pools.ClearUploadID(uploadID) + } + + return np, nerr +} + +func (s *peerRESTServer) StopRebalanceHandler(mss *grid.MSS) (np grid.NoPayload, nerr *grid.RemoteErr) { + objAPI := newObjectLayerFn() + if objAPI == nil { + return np, grid.NewRemoteErr(errServerNotInitialized) + } + + pools, ok := objAPI.(*erasureServerPools) + if !ok { + return np, grid.NewRemoteErr(errors.New("not a pooled setup")) + } + + pools.StopRebalance() + return np, nerr +} + +func (s *peerRESTServer) LoadRebalanceMetaHandler(mss *grid.MSS) (np grid.NoPayload, nerr *grid.RemoteErr) { + objAPI := newObjectLayerFn() + if objAPI == nil { + return np, grid.NewRemoteErr(errServerNotInitialized) + } + + pools, ok := objAPI.(*erasureServerPools) + if !ok { + return np, grid.NewRemoteErr(errors.New("not a pooled setup")) + } + + startRebalance, err := strconv.ParseBool(mss.Get(peerRESTStartRebalance)) + if err != nil { + return np, grid.NewRemoteErr(err) + } + + if err := pools.loadRebalanceMeta(context.Background()); err != nil { + return np, grid.NewRemoteErr(err) + } + + if startRebalance { + go pools.StartRebalance() + } + + return np, nerr +} + +func (s *peerRESTServer) LoadTransitionTierConfigHandler(mss *grid.MSS) (np grid.NoPayload, nerr *grid.RemoteErr) { + objAPI := newObjectLayerFn() + if objAPI == nil { + return np, grid.NewRemoteErr(errServerNotInitialized) + } + + go func() { + err := globalTierConfigMgr.Reload(context.Background(), newObjectLayerFn()) + if err != nil { + peersLogIf(context.Background(), fmt.Errorf("Failed to reload remote tier config %s", err)) + } + }() + + return np, nerr +} + +// ConsoleLogHandler sends console logs of this node back to peer rest client +func (s *peerRESTServer) ConsoleLogHandler(ctx context.Context, params *grid.MSS, out chan<- *grid.Bytes) *grid.RemoteErr { + mask, err := strconv.Atoi(params.Get(peerRESTLogMask)) + if err != nil { + mask = int(madmin.LogMaskAll) + } + ch := make(chan log.Info, 1000) + err = globalConsoleSys.Subscribe(ch, ctx.Done(), "", 0, madmin.LogMask(mask), nil) + if err != nil { + return grid.NewRemoteErr(err) + } + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + for { + select { + case entry, ok := <-ch: + if !ok { + return grid.NewRemoteErrString("console log channel closed") + } + if !entry.SendLog("", madmin.LogMask(mask)) { + continue + } + buf.Reset() + if err := enc.Encode(entry); err != nil { + return grid.NewRemoteErr(err) + } + out <- grid.NewBytesWithCopyOf(buf.Bytes()) + case <-ctx.Done(): + return grid.NewRemoteErr(ctx.Err()) + } + } +} + +func (s *peerRESTServer) writeErrorResponse(w http.ResponseWriter, err error) { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte(err.Error())) +} + +// IsValid - To authenticate and verify the time difference. +func (s *peerRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool { + if err := storageServerRequestValidate(r); err != nil { + s.writeErrorResponse(w, err) + return false + } + return true +} + +// GetBandwidth gets the bandwidth for the buckets requested. +func (s *peerRESTServer) GetBandwidth(params *grid.URLValues) (*bandwidth.BucketBandwidthReport, *grid.RemoteErr) { + buckets := params.Values().Get("buckets") + selectBuckets := bandwidth.SelectBuckets(buckets) + return globalBucketMonitor.GetReport(selectBuckets), nil +} + +func (s *peerRESTServer) GetResourceMetrics(_ *grid.MSS) (*grid.Array[*MetricV2], *grid.RemoteErr) { + res := make([]*MetricV2, 0, len(resourceMetricsGroups)) + populateAndPublish(resourceMetricsGroups, func(m MetricV2) bool { + if m.VariableLabels == nil { + m.VariableLabels = make(map[string]string, 1) + } + m.VariableLabels[serverName] = globalLocalNodeName + res = append(res, &m) + return true + }) + return aoMetricsGroup.NewWith(res), nil +} + +// GetPeerMetrics gets the metrics to be federated across peers. +func (s *peerRESTServer) GetPeerMetrics(_ *grid.MSS) (*grid.Array[*MetricV2], *grid.RemoteErr) { + res := make([]*MetricV2, 0, len(peerMetricsGroups)) + populateAndPublish(peerMetricsGroups, func(m MetricV2) bool { + if m.VariableLabels == nil { + m.VariableLabels = make(map[string]string, 1) + } + m.VariableLabels[serverName] = globalLocalNodeName + res = append(res, &m) + return true + }) + return aoMetricsGroup.NewWith(res), nil +} + +// GetPeerBucketMetrics gets the metrics to be federated across peers. +func (s *peerRESTServer) GetPeerBucketMetrics(_ *grid.MSS) (*grid.Array[*MetricV2], *grid.RemoteErr) { + res := make([]*MetricV2, 0, len(bucketPeerMetricsGroups)) + populateAndPublish(bucketPeerMetricsGroups, func(m MetricV2) bool { + if m.VariableLabels == nil { + m.VariableLabels = make(map[string]string, 1) + } + m.VariableLabels[serverName] = globalLocalNodeName + res = append(res, &m) + return true + }) + return aoMetricsGroup.NewWith(res), nil +} + +func (s *peerRESTServer) SpeedTestHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + s.writeErrorResponse(w, errors.New("invalid request")) + return + } + + objAPI := newObjectLayerFn() + if objAPI == nil { + s.writeErrorResponse(w, errServerNotInitialized) + return + } + + sizeStr := r.Form.Get(peerRESTSize) + durationStr := r.Form.Get(peerRESTDuration) + concurrentStr := r.Form.Get(peerRESTConcurrent) + storageClass := r.Form.Get(peerRESTStorageClass) + bucketName := r.Form.Get(peerRESTBucket) + enableSha256 := r.Form.Get(peerRESTEnableSha256) == "true" + enableMultipart := r.Form.Get(peerRESTEnableMultipart) == "true" + + u, ok := globalIAMSys.GetUser(r.Context(), r.Form.Get(peerRESTAccessKey)) + if !ok { + s.writeErrorResponse(w, errAuthentication) + return + } + + size, err := strconv.Atoi(sizeStr) + if err != nil { + size = 64 * humanize.MiByte + } + + concurrent, err := strconv.Atoi(concurrentStr) + if err != nil { + concurrent = 32 + } + + duration, err := time.ParseDuration(durationStr) + if err != nil { + duration = time.Second * 10 + } + + done := keepHTTPResponseAlive(w) + + result, err := selfSpeedTest(r.Context(), speedTestOpts{ + objectSize: size, + concurrency: concurrent, + duration: duration, + storageClass: storageClass, + bucketName: bucketName, + enableSha256: enableSha256, + enableMultipart: enableMultipart, + creds: u.Credentials, + }) + if err != nil { + result.Error = err.Error() + } + + done(nil) + peersLogIf(r.Context(), gob.NewEncoder(w).Encode(result)) +} + +// GetLastDayTierStatsHandler - returns per-tier stats in the last 24hrs for this server +func (s *peerRESTServer) GetLastDayTierStatsHandler(_ *grid.MSS) (*DailyAllTierStats, *grid.RemoteErr) { + if objAPI := newObjectLayerFn(); objAPI == nil || globalTransitionState == nil { + return nil, grid.NewRemoteErr(errServerNotInitialized) + } + + result := globalTransitionState.getDailyAllTierStats() + return &result, nil +} + +func (s *peerRESTServer) DriveSpeedTestHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + s.writeErrorResponse(w, errors.New("invalid request")) + return + } + + objAPI := newObjectLayerFn() + if objAPI == nil { + s.writeErrorResponse(w, errServerNotInitialized) + return + } + + serial := r.Form.Get("serial") == "true" + blockSizeStr := r.Form.Get("blocksize") + fileSizeStr := r.Form.Get("filesize") + + blockSize, err := strconv.ParseUint(blockSizeStr, 10, 64) + if err != nil { + blockSize = 4 * humanize.MiByte // default value + } + + fileSize, err := strconv.ParseUint(fileSizeStr, 10, 64) + if err != nil { + fileSize = 1 * humanize.GiByte // default value + } + + opts := madmin.DriveSpeedTestOpts{ + Serial: serial, + BlockSize: blockSize, + FileSize: fileSize, + } + + done := keepHTTPResponseAlive(w) + result := driveSpeedTest(r.Context(), opts) + done(nil) + + peersLogIf(r.Context(), gob.NewEncoder(w).Encode(result)) +} + +// GetReplicationMRFHandler - returns replication MRF for bucket +func (s *peerRESTServer) GetReplicationMRFHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + s.writeErrorResponse(w, errors.New("invalid request")) + return + } + + vars := mux.Vars(r) + bucketName := vars[peerRESTBucket] + ctx := newContext(r, w, "GetReplicationMRF") + re, err := globalReplicationPool.Get().getMRF(ctx, bucketName) + if err != nil { + s.writeErrorResponse(w, err) + return + } + enc := gob.NewEncoder(w) + + for m := range re { + if err := enc.Encode(m); err != nil { + s.writeErrorResponse(w, errors.New("Encoding mrf failed: "+err.Error())) + return + } + } +} + +// DevNull - everything goes to io.Discard +func (s *peerRESTServer) DevNull(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + s.writeErrorResponse(w, errors.New("invalid request")) + return + } + + globalNetPerfRX.Connect() + defer globalNetPerfRX.Disconnect() + + connectTime := time.Now() + ctx := newContext(r, w, "DevNull") + for { + n, err := io.CopyN(xioutil.Discard, r.Body, 128*humanize.KiByte) + atomic.AddUint64(&globalNetPerfRX.RX, uint64(n)) + if err != nil && err != io.EOF { + // If there is a disconnection before globalNetPerfMinDuration (we give a margin of error of 1 sec) + // would mean the network is not stable. Logging here will help in debugging network issues. + if time.Since(connectTime) < (globalNetPerfMinDuration - time.Second) { + peersLogIf(ctx, err) + } + } + if err != nil { + break + } + } +} + +// NetSpeedTestHandlers - perform network speedtest +func (s *peerRESTServer) NetSpeedTestHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + s.writeErrorResponse(w, errors.New("invalid request")) + return + } + + durationStr := r.Form.Get(peerRESTDuration) + duration, err := time.ParseDuration(durationStr) + if err != nil || duration.Seconds() == 0 { + duration = time.Second * 10 + } + result := netperf(r.Context(), duration.Round(time.Second)) + peersLogIf(r.Context(), gob.NewEncoder(w).Encode(result)) +} + +func (s *peerRESTServer) HealBucketHandler(mss *grid.MSS) (np *grid.JSON[madmin.HealResultItem], nerr *grid.RemoteErr) { + bucket := mss.Get(peerS3Bucket) + if isMinioMetaBucket(bucket) { + return np, grid.NewRemoteErr(errInvalidArgument) + } + + bucketDeleted := mss.Get(peerS3BucketDeleted) == "true" + res, err := healBucketLocal(context.Background(), bucket, madmin.HealOpts{ + Remove: bucketDeleted, + }) + if err != nil { + return np, grid.NewRemoteErr(err) + } + + return madminHealResultItem.NewJSONWith(&res), nil +} + +func (s *peerRESTServer) ListBucketsHandler(opts *BucketOptions) (*grid.Array[*BucketInfo], *grid.RemoteErr) { + buckets, err := listBucketsLocal(context.Background(), *opts) + if err != nil { + return nil, grid.NewRemoteErr(err) + } + res := aoBucketInfo.New() + for i := range buckets { + bucket := buckets[i] + res.Append(&bucket) + } + return res, nil +} + +// HeadBucketHandler implements peer BucketInfo call, returns bucket create date. +func (s *peerRESTServer) HeadBucketHandler(mss *grid.MSS) (info *VolInfo, nerr *grid.RemoteErr) { + bucket := mss.Get(peerS3Bucket) + if isMinioMetaBucket(bucket) { + return info, grid.NewRemoteErr(errInvalidArgument) + } + + bucketDeleted := mss.Get(peerS3BucketDeleted) == "true" + + bucketInfo, err := getBucketInfoLocal(context.Background(), bucket, BucketOptions{ + Deleted: bucketDeleted, + }) + if err != nil { + return info, grid.NewRemoteErr(err) + } + + return &VolInfo{ + Name: bucketInfo.Name, + Created: bucketInfo.Created, + Deleted: bucketInfo.Deleted, // needed for site replication + }, nil +} + +// DeleteBucketHandler implements peer delete bucket call. +func (s *peerRESTServer) DeleteBucketHandler(mss *grid.MSS) (np grid.NoPayload, nerr *grid.RemoteErr) { + bucket := mss.Get(peerS3Bucket) + if isMinioMetaBucket(bucket) { + return np, grid.NewRemoteErr(errInvalidArgument) + } + + forceDelete := mss.Get(peerS3BucketForceDelete) == "true" + err := deleteBucketLocal(context.Background(), bucket, DeleteBucketOptions{ + Force: forceDelete, + }) + if err != nil { + return np, grid.NewRemoteErr(err) + } + + return np, nil +} + +// MakeBucketHandler implements peer create bucket call. +func (s *peerRESTServer) MakeBucketHandler(mss *grid.MSS) (np grid.NoPayload, nerr *grid.RemoteErr) { + bucket := mss.Get(peerS3Bucket) + if isMinioMetaBucket(bucket) { + return np, grid.NewRemoteErr(errInvalidArgument) + } + + forceCreate := mss.Get(peerS3BucketForceCreate) == "true" + err := makeBucketLocal(context.Background(), bucket, MakeBucketOptions{ + ForceCreate: forceCreate, + }) + if err != nil { + return np, grid.NewRemoteErr(err) + } + + return np, nil +} + +// registerPeerRESTHandlers - register peer rest router. +func registerPeerRESTHandlers(router *mux.Router, gm *grid.Manager) { + h := func(f http.HandlerFunc) http.HandlerFunc { + return collectInternodeStats(httpTraceHdrs(f)) + } + + server := &peerRESTServer{} + subrouter := router.PathPrefix(peerRESTPrefix).Subrouter() + subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodHealth).HandlerFunc(h(server.HealthHandler)) + subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodVerifyBinary).HandlerFunc(h(server.VerifyBinaryHandler)).Queries(restQueries(peerRESTURL, peerRESTSha256Sum, peerRESTReleaseInfo)...) + subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodCommitBinary).HandlerFunc(h(server.CommitBinaryHandler)) + subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodGetReplicationMRF).HandlerFunc(httpTraceHdrs(server.GetReplicationMRFHandler)).Queries(restQueries(peerRESTBucket)...) + subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodStartProfiling).HandlerFunc(h(server.StartProfilingHandler)).Queries(restQueries(peerRESTProfiler)...) + subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDownloadProfilingData).HandlerFunc(h(server.DownloadProfilingDataHandler)) + subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodSpeedTest).HandlerFunc(h(server.SpeedTestHandler)) + subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDriveSpeedTest).HandlerFunc(h(server.DriveSpeedTestHandler)) + subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodNetperf).HandlerFunc(h(server.NetSpeedTestHandler)) + subrouter.Methods(http.MethodPost).Path(peerRESTVersionPrefix + peerRESTMethodDevNull).HandlerFunc(h(server.DevNull)) + + logger.FatalIf(consoleLogRPC.RegisterNoInput(gm, server.ConsoleLogHandler), "unable to register handler") + logger.FatalIf(deleteBucketMetadataRPC.Register(gm, server.DeleteBucketMetadataHandler), "unable to register handler") + logger.FatalIf(deleteBucketRPC.Register(gm, server.DeleteBucketHandler), "unable to register handler") + logger.FatalIf(deletePolicyRPC.Register(gm, server.DeletePolicyHandler), "unable to register handler") + logger.FatalIf(deleteSvcActRPC.Register(gm, server.DeleteServiceAccountHandler), "unable to register handler") + logger.FatalIf(deleteUserRPC.Register(gm, server.DeleteUserHandler), "unable to register handler") + logger.FatalIf(getAllBucketStatsRPC.Register(gm, server.GetAllBucketStatsHandler), "unable to register handler") + logger.FatalIf(getBackgroundHealStatusRPC.Register(gm, server.BackgroundHealStatusHandler), "unable to register handler") + logger.FatalIf(getBandwidthRPC.Register(gm, server.GetBandwidth), "unable to register handler") + logger.FatalIf(getBucketStatsRPC.Register(gm, server.GetBucketStatsHandler), "unable to register handler") + logger.FatalIf(getCPUsHandler.Register(gm, server.GetCPUsHandler), "unable to register handler") + logger.FatalIf(getLastDayTierStatsRPC.Register(gm, server.GetLastDayTierStatsHandler), "unable to register handler") + logger.FatalIf(getLocksRPC.Register(gm, server.GetLocksHandler), "unable to register handler") + logger.FatalIf(getMemInfoRPC.Register(gm, server.GetMemInfoHandler), "unable to register handler") + logger.FatalIf(getMetacacheListingRPC.Register(gm, server.GetMetacacheListingHandler), "unable to register handler") + logger.FatalIf(getMetricsRPC.Register(gm, server.GetMetricsHandler), "unable to register handler") + logger.FatalIf(getNetInfoRPC.Register(gm, server.GetNetInfoHandler), "unable to register handler") + logger.FatalIf(getOSInfoRPC.Register(gm, server.GetOSInfoHandler), "unable to register handler") + logger.FatalIf(getPartitionsRPC.Register(gm, server.GetPartitionsHandler), "unable to register handler") + logger.FatalIf(getPeerBucketMetricsRPC.Register(gm, server.GetPeerBucketMetrics), "unable to register handler") + logger.FatalIf(getPeerMetricsRPC.Register(gm, server.GetPeerMetrics), "unable to register handler") + logger.FatalIf(getProcInfoRPC.Register(gm, server.GetProcInfoHandler), "unable to register handler") + logger.FatalIf(getResourceMetricsRPC.Register(gm, server.GetResourceMetrics), "unable to register handler") + logger.FatalIf(getSRMetricsRPC.Register(gm, server.GetSRMetricsHandler), "unable to register handler") + logger.FatalIf(getSysConfigRPC.Register(gm, server.GetSysConfigHandler), "unable to register handler") + logger.FatalIf(getSysErrorsRPC.Register(gm, server.GetSysErrorsHandler), "unable to register handler") + logger.FatalIf(getSysServicesRPC.Register(gm, server.GetSysServicesHandler), "unable to register handler") + logger.FatalIf(headBucketRPC.Register(gm, server.HeadBucketHandler), "unable to register handler") + logger.FatalIf(healBucketRPC.Register(gm, server.HealBucketHandler), "unable to register handler") + logger.FatalIf(listBucketsRPC.Register(gm, server.ListBucketsHandler), "unable to register handler") + logger.FatalIf(listenRPC.RegisterNoInput(gm, server.ListenHandler), "unable to register handler") + logger.FatalIf(loadBucketMetadataRPC.Register(gm, server.LoadBucketMetadataHandler), "unable to register handler") + logger.FatalIf(loadGroupRPC.Register(gm, server.LoadGroupHandler), "unable to register handler") + logger.FatalIf(loadPolicyMappingRPC.Register(gm, server.LoadPolicyMappingHandler), "unable to register handler") + logger.FatalIf(loadPolicyRPC.Register(gm, server.LoadPolicyHandler), "unable to register handler") + logger.FatalIf(loadRebalanceMetaRPC.Register(gm, server.LoadRebalanceMetaHandler), "unable to register handler") + logger.FatalIf(loadSvcActRPC.Register(gm, server.LoadServiceAccountHandler), "unable to register handler") + logger.FatalIf(loadTransitionTierConfigRPC.Register(gm, server.LoadTransitionTierConfigHandler), "unable to register handler") + logger.FatalIf(loadUserRPC.Register(gm, server.LoadUserHandler), "unable to register handler") + logger.FatalIf(localStorageInfoRPC.Register(gm, server.LocalStorageInfoHandler), "unable to register handler") + logger.FatalIf(makeBucketRPC.Register(gm, server.MakeBucketHandler), "unable to register handler") + logger.FatalIf(reloadPoolMetaRPC.Register(gm, server.ReloadPoolMetaHandler), "unable to register handler") + logger.FatalIf(reloadSiteReplicationConfigRPC.Register(gm, server.ReloadSiteReplicationConfigHandler), "unable to register handler") + logger.FatalIf(serverInfoRPC.Register(gm, server.ServerInfoHandler), "unable to register handler") + logger.FatalIf(signalServiceRPC.Register(gm, server.SignalServiceHandler), "unable to register handler") + logger.FatalIf(stopRebalanceRPC.Register(gm, server.StopRebalanceHandler), "unable to register handler") + logger.FatalIf(updateMetacacheListingRPC.Register(gm, server.UpdateMetacacheListingHandler), "unable to register handler") + logger.FatalIf(cleanupUploadIDCacheMetaRPC.Register(gm, server.HandlerClearUploadID), "unable to register handler") + + logger.FatalIf(gm.RegisterStreamingHandler(grid.HandlerTrace, grid.StreamHandler{ + Handle: server.TraceHandler, + Subroute: "", + OutCapacity: 100000, + InCapacity: 0, + }), "unable to register handler") +} diff --git a/cmd/peer-s3-client.go b/cmd/peer-s3-client.go new file mode 100644 index 00000000..438b4d33 --- /dev/null +++ b/cmd/peer-s3-client.go @@ -0,0 +1,552 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "slices" + "sort" + "strconv" + "sync/atomic" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/grid" + "github.com/minio/pkg/v3/sync/errgroup" +) + +var errPeerOffline = errors.New("peer is offline") + +type peerS3Client interface { + ListBuckets(ctx context.Context, opts BucketOptions) ([]BucketInfo, error) + HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, error) + GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error) + MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error + DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error + + GetHost() string + SetPools([]int) + GetPools() []int +} + +type localPeerS3Client struct { + node Node + pools []int +} + +func (l *localPeerS3Client) GetHost() string { + return l.node.Host +} + +func (l *localPeerS3Client) SetPools(p []int) { + l.pools = make([]int, len(p)) + copy(l.pools, p) +} + +func (l localPeerS3Client) GetPools() []int { + return l.pools +} + +func (l localPeerS3Client) ListBuckets(ctx context.Context, opts BucketOptions) ([]BucketInfo, error) { + return listBucketsLocal(ctx, opts) +} + +func (l localPeerS3Client) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, error) { + return healBucketLocal(ctx, bucket, opts) +} + +func (l localPeerS3Client) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error) { + return getBucketInfoLocal(ctx, bucket, opts) +} + +func (l localPeerS3Client) MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error { + return makeBucketLocal(ctx, bucket, opts) +} + +func (l localPeerS3Client) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error { + return deleteBucketLocal(ctx, bucket, opts) +} + +// client to talk to peer Nodes. +type remotePeerS3Client struct { + node Node + pools []int + + // Function that returns the grid connection for this peer when initialized. + // Will return nil if the grid connection is not initialized yet. + gridConn func() *grid.Connection +} + +// S3PeerSys - S3 peer call system. +type S3PeerSys struct { + peerClients []peerS3Client // Excludes self + poolsCount int +} + +// NewS3PeerSys - creates new S3 peer calls. +func NewS3PeerSys(endpoints EndpointServerPools) *S3PeerSys { + return &S3PeerSys{ + peerClients: newPeerS3Clients(endpoints), + poolsCount: len(endpoints), + } +} + +// HealBucket - heals buckets at node level +func (sys *S3PeerSys) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, error) { + g := errgroup.WithNErrs(len(sys.peerClients)) + + for idx, client := range sys.peerClients { + client := client + g.Go(func() error { + if client == nil { + return errPeerOffline + } + _, err := client.GetBucketInfo(ctx, bucket, BucketOptions{}) + return err + }, idx) + } + + errs := g.Wait() + + var poolErrs []error + for poolIdx := 0; poolIdx < sys.poolsCount; poolIdx++ { + perPoolErrs := make([]error, 0, len(sys.peerClients)) + for i, client := range sys.peerClients { + if slices.Contains(client.GetPools(), poolIdx) { + perPoolErrs = append(perPoolErrs, errs[i]) + } + } + quorum := len(perPoolErrs) / 2 + poolErrs = append(poolErrs, reduceWriteQuorumErrs(ctx, perPoolErrs, bucketOpIgnoredErrs, quorum)) + } + + if !opts.Recreate { + // when there is no force recreate look for pool + // errors to recreate the bucket on all pools. + opts.Remove = isAllBucketsNotFound(poolErrs) + opts.Recreate = !opts.Remove + } + + g = errgroup.WithNErrs(len(sys.peerClients)) + healBucketResults := make([]madmin.HealResultItem, len(sys.peerClients)) + for idx, client := range sys.peerClients { + client := client + g.Go(func() error { + if client == nil { + return errPeerOffline + } + res, err := client.HealBucket(ctx, bucket, opts) + if err != nil { + return err + } + healBucketResults[idx] = res + return nil + }, idx) + } + + errs = g.Wait() + + for poolIdx := 0; poolIdx < sys.poolsCount; poolIdx++ { + perPoolErrs := make([]error, 0, len(sys.peerClients)) + for i, client := range sys.peerClients { + if slices.Contains(client.GetPools(), poolIdx) { + perPoolErrs = append(perPoolErrs, errs[i]) + } + } + quorum := len(perPoolErrs) / 2 + if poolErr := reduceWriteQuorumErrs(ctx, perPoolErrs, bucketOpIgnoredErrs, quorum); poolErr != nil { + return madmin.HealResultItem{}, poolErr + } + } + + if healBucketErr := reduceWriteQuorumErrs(ctx, errs, bucketOpIgnoredErrs, len(errs)/2+1); healBucketErr != nil { + return madmin.HealResultItem{}, toObjectErr(healBucketErr, bucket) + } + + res := madmin.HealResultItem{ + Type: madmin.HealItemBucket, + Bucket: bucket, + SetCount: -1, // explicitly set an invalid value -1, for bucket heal scenario + } + + for i, err := range errs { + if err == nil { + res.Before.Drives = append(res.Before.Drives, healBucketResults[i].Before.Drives...) + res.After.Drives = append(res.After.Drives, healBucketResults[i].After.Drives...) + } + } + + return res, nil +} + +// ListBuckets lists buckets across all nodes and returns a consistent view: +// - Return an error when a pool cannot return N/2+1 valid bucket information +// - For each pool, check if the bucket exists in N/2+1 nodes before including it in the final result +func (sys *S3PeerSys) ListBuckets(ctx context.Context, opts BucketOptions) ([]BucketInfo, error) { + g := errgroup.WithNErrs(len(sys.peerClients)) + + nodeBuckets := make([][]BucketInfo, len(sys.peerClients)) + + for idx, client := range sys.peerClients { + client := client + g.Go(func() error { + if client == nil { + return errPeerOffline + } + localBuckets, err := client.ListBuckets(ctx, opts) + if err != nil { + return err + } + nodeBuckets[idx] = localBuckets + return nil + }, idx) + } + + errs := g.Wait() + + // The list of buckets in a map to avoid duplication + resultMap := make(map[string]BucketInfo) + + for poolIdx := 0; poolIdx < sys.poolsCount; poolIdx++ { + perPoolErrs := make([]error, 0, len(sys.peerClients)) + for i, client := range sys.peerClients { + if slices.Contains(client.GetPools(), poolIdx) { + perPoolErrs = append(perPoolErrs, errs[i]) + } + } + quorum := len(perPoolErrs) / 2 + if poolErr := reduceWriteQuorumErrs(ctx, perPoolErrs, bucketOpIgnoredErrs, quorum); poolErr != nil { + return nil, poolErr + } + + bucketsMap := make(map[string]int) + for idx, buckets := range nodeBuckets { + if buckets == nil { + continue + } + if !slices.Contains(sys.peerClients[idx].GetPools(), poolIdx) { + continue + } + for _, bi := range buckets { + _, ok := resultMap[bi.Name] + if ok { + // Skip it, this bucket is found in another pool + continue + } + bucketsMap[bi.Name]++ + if bucketsMap[bi.Name] >= quorum { + resultMap[bi.Name] = bi + } + } + } + // loop through buckets and see if some with lost quorum + // these could be stale buckets lying around, queue a heal + // of such a bucket. This is needed here as we identify such + // buckets here while listing buckets. As part of regular + // globalBucketMetadataSys.Init() call would get a valid + // buckets only and not the quourum lost ones like this, so + // explicit call + for bktName, count := range bucketsMap { + if count < quorum { + // Queue a bucket heal task + globalMRFState.addPartialOp(PartialOperation{ + Bucket: bktName, + Queued: time.Now(), + }) + } + } + } + + result := make([]BucketInfo, 0, len(resultMap)) + for _, bi := range resultMap { + result = append(result, bi) + } + + sort.Slice(result, func(i, j int) bool { + return result[i].Name < result[j].Name + }) + + return result, nil +} + +// GetBucketInfo returns bucket stat info about bucket on disk across all peers +func (sys *S3PeerSys) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (binfo BucketInfo, err error) { + g := errgroup.WithNErrs(len(sys.peerClients)) + + bucketInfos := make([]BucketInfo, len(sys.peerClients)) + for idx, client := range sys.peerClients { + client := client + g.Go(func() error { + if client == nil { + return errPeerOffline + } + bucketInfo, err := client.GetBucketInfo(ctx, bucket, opts) + if err != nil { + return err + } + bucketInfos[idx] = bucketInfo + return nil + }, idx) + } + + errs := g.Wait() + + for poolIdx := 0; poolIdx < sys.poolsCount; poolIdx++ { + perPoolErrs := make([]error, 0, len(sys.peerClients)) + for i, client := range sys.peerClients { + if slices.Contains(client.GetPools(), poolIdx) { + perPoolErrs = append(perPoolErrs, errs[i]) + } + } + quorum := len(perPoolErrs) / 2 + if poolErr := reduceWriteQuorumErrs(ctx, perPoolErrs, bucketOpIgnoredErrs, quorum); poolErr != nil { + return BucketInfo{}, poolErr + } + } + + for i, err := range errs { + if err == nil { + return bucketInfos[i], nil + } + } + + return BucketInfo{}, toObjectErr(errVolumeNotFound, bucket) +} + +func (client *remotePeerS3Client) ListBuckets(ctx context.Context, opts BucketOptions) ([]BucketInfo, error) { + ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout()) + defer cancel() + + bi, err := listBucketsRPC.Call(ctx, client.gridConn(), &opts) + if err != nil { + return nil, toStorageErr(err) + } + buckets := make([]BucketInfo, 0, len(bi.Value())) + for _, b := range bi.Value() { + if b != nil { + buckets = append(buckets, *b) + } + } + bi.Recycle() // BucketInfo has no internal pointers, so it's safe to recycle. + return buckets, nil +} + +func (client *remotePeerS3Client) HealBucket(ctx context.Context, bucket string, opts madmin.HealOpts) (madmin.HealResultItem, error) { + conn := client.gridConn() + if conn == nil { + return madmin.HealResultItem{}, nil + } + + mss := grid.NewMSSWith(map[string]string{ + peerS3Bucket: bucket, + peerS3BucketDeleted: strconv.FormatBool(opts.Remove), + }) + + ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout()) + defer cancel() + + resp, err := healBucketRPC.Call(ctx, conn, mss) + return resp.ValueOrZero(), toStorageErr(err) +} + +// GetBucketInfo returns bucket stat info from a peer +func (client *remotePeerS3Client) GetBucketInfo(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error) { + conn := client.gridConn() + if conn == nil { + return BucketInfo{}, nil + } + + mss := grid.NewMSSWith(map[string]string{ + peerS3Bucket: bucket, + peerS3BucketDeleted: strconv.FormatBool(opts.Deleted), + }) + + ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout()) + defer cancel() + + volInfo, err := headBucketRPC.Call(ctx, conn, mss) + if err != nil { + return BucketInfo{}, toStorageErr(err) + } + + return BucketInfo{ + Name: volInfo.Name, + Created: volInfo.Created, + Deleted: volInfo.Deleted, + }, nil +} + +// MakeBucket creates bucket across all peers +func (sys *S3PeerSys) MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error { + g := errgroup.WithNErrs(len(sys.peerClients)) + for idx, client := range sys.peerClients { + g.Go(func() error { + if client == nil { + return errPeerOffline + } + return client.MakeBucket(ctx, bucket, opts) + }, idx) + } + errs := g.Wait() + + for poolIdx := 0; poolIdx < sys.poolsCount; poolIdx++ { + perPoolErrs := make([]error, 0, len(sys.peerClients)) + for i, client := range sys.peerClients { + if slices.Contains(client.GetPools(), poolIdx) { + perPoolErrs = append(perPoolErrs, errs[i]) + } + } + if poolErr := reduceWriteQuorumErrs(ctx, perPoolErrs, bucketOpIgnoredErrs, len(perPoolErrs)/2+1); poolErr != nil { + return toObjectErr(poolErr, bucket) + } + } + return nil +} + +// MakeBucket creates a bucket on a peer +func (client *remotePeerS3Client) MakeBucket(ctx context.Context, bucket string, opts MakeBucketOptions) error { + conn := client.gridConn() + if conn == nil { + return nil + } + + mss := grid.NewMSSWith(map[string]string{ + peerS3Bucket: bucket, + peerS3BucketForceCreate: strconv.FormatBool(opts.ForceCreate), + }) + + ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout()) + defer cancel() + + _, err := makeBucketRPC.Call(ctx, conn, mss) + return toStorageErr(err) +} + +// DeleteBucket deletes bucket across all peers +func (sys *S3PeerSys) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error { + g := errgroup.WithNErrs(len(sys.peerClients)) + for idx, client := range sys.peerClients { + g.Go(func() error { + if client == nil { + return errPeerOffline + } + return client.DeleteBucket(ctx, bucket, opts) + }, idx) + } + errs := g.Wait() + + for poolIdx := 0; poolIdx < sys.poolsCount; poolIdx++ { + perPoolErrs := make([]error, 0, len(sys.peerClients)) + for i, client := range sys.peerClients { + if slices.Contains(client.GetPools(), poolIdx) { + perPoolErrs = append(perPoolErrs, errs[i]) + } + } + poolErr := reduceWriteQuorumErrs(ctx, perPoolErrs, bucketOpIgnoredErrs, len(perPoolErrs)/2+1) + if poolErr != nil && !errors.Is(poolErr, errVolumeNotFound) { + if !opts.NoRecreate { + // re-create successful deletes, since we are return an error. + sys.MakeBucket(ctx, bucket, MakeBucketOptions{}) + } + return toObjectErr(poolErr, bucket) + } + } + return nil +} + +// DeleteBucket deletes bucket on a peer +func (client *remotePeerS3Client) DeleteBucket(ctx context.Context, bucket string, opts DeleteBucketOptions) error { + conn := client.gridConn() + if conn == nil { + return nil + } + + mss := grid.NewMSSWith(map[string]string{ + peerS3Bucket: bucket, + peerS3BucketForceDelete: strconv.FormatBool(opts.Force), + }) + + ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout()) + defer cancel() + + _, err := deleteBucketRPC.Call(ctx, conn, mss) + return toStorageErr(err) +} + +func (client remotePeerS3Client) GetHost() string { + return client.node.Host +} + +func (client remotePeerS3Client) GetPools() []int { + return client.pools +} + +func (client *remotePeerS3Client) SetPools(p []int) { + client.pools = make([]int, len(p)) + copy(client.pools, p) +} + +// newPeerS3Clients creates new peer clients. +func newPeerS3Clients(endpoints EndpointServerPools) (peers []peerS3Client) { + nodes := endpoints.GetNodes() + peers = make([]peerS3Client, len(nodes)) + for i, node := range nodes { + if node.IsLocal { + peers[i] = &localPeerS3Client{node: node} + } else { + peers[i] = newPeerS3Client(node) + } + peers[i].SetPools(node.Pools) + } + + return peers +} + +// Returns a peer S3 client. +func newPeerS3Client(node Node) peerS3Client { + var gridConn atomic.Pointer[grid.Connection] + + return &remotePeerS3Client{ + node: node, + gridConn: func() *grid.Connection { + // Lazy initialization of grid connection. + // When we create this peer client, the grid connection is likely not yet initialized. + if node.GridHost == "" { + bugLogIf(context.Background(), fmt.Errorf("gridHost is empty for peer %s", node.Host), node.Host+":gridHost") + return nil + } + gc := gridConn.Load() + if gc != nil { + return gc + } + gm := globalGrid.Load() + if gm == nil { + return nil + } + gc = gm.Connection(node.GridHost) + if gc == nil { + bugLogIf(context.Background(), fmt.Errorf("gridHost %s not found for peer %s", node.GridHost, node.Host), node.Host+":gridHost") + return nil + } + gridConn.Store(gc) + return gc + }, + } +} diff --git a/cmd/peer-s3-server.go b/cmd/peer-s3-server.go new file mode 100644 index 00000000..227aebe8 --- /dev/null +++ b/cmd/peer-s3-server.go @@ -0,0 +1,308 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + + "github.com/minio/madmin-go/v3" + "github.com/minio/pkg/v3/sync/errgroup" + "github.com/puzpuzpuz/xsync/v3" +) + +const ( + peerS3Bucket = "bucket" + peerS3BucketDeleted = "bucket-deleted" + peerS3BucketForceCreate = "force-create" + peerS3BucketForceDelete = "force-delete" +) + +func healBucketLocal(ctx context.Context, bucket string, opts madmin.HealOpts) (res madmin.HealResultItem, err error) { + globalLocalDrivesMu.RLock() + localDrives := cloneDrives(globalLocalDrivesMap) + globalLocalDrivesMu.RUnlock() + + // Initialize sync waitgroup. + g := errgroup.WithNErrs(len(localDrives)) + + // Disk states slices + beforeState := make([]string, len(localDrives)) + afterState := make([]string, len(localDrives)) + + // Make a volume entry on all underlying storage disks. + for index := range localDrives { + g.Go(func() (serr error) { + if localDrives[index] == nil { + beforeState[index] = madmin.DriveStateOffline + afterState[index] = madmin.DriveStateOffline + return errDiskNotFound + } + + beforeState[index] = madmin.DriveStateOk + afterState[index] = madmin.DriveStateOk + + if bucket == minioReservedBucket { + return nil + } + + _, serr = localDrives[index].StatVol(ctx, bucket) + if serr != nil { + if serr == errDiskNotFound { + beforeState[index] = madmin.DriveStateOffline + afterState[index] = madmin.DriveStateOffline + return serr + } + if serr != errVolumeNotFound { + beforeState[index] = madmin.DriveStateCorrupt + afterState[index] = madmin.DriveStateCorrupt + return serr + } + + beforeState[index] = madmin.DriveStateMissing + afterState[index] = madmin.DriveStateMissing + + return serr + } + return nil + }, index) + } + + errs := g.Wait() + + // Initialize heal result info + res = madmin.HealResultItem{ + Type: madmin.HealItemBucket, + Bucket: bucket, + DiskCount: len(localDrives), + SetCount: -1, // explicitly set an invalid value -1, for bucket heal scenario + } + + // mutate only if not a dry-run + if opts.DryRun { + return res, nil + } + + for i := range beforeState { + res.Before.Drives = append(res.Before.Drives, madmin.HealDriveInfo{ + UUID: "", + Endpoint: localDrives[i].Endpoint().String(), + State: beforeState[i], + }) + } + + // check dangling and delete bucket only if its not a meta bucket + if !isMinioMetaBucketName(bucket) && !isAllBucketsNotFound(errs) && opts.Remove { + g := errgroup.WithNErrs(len(localDrives)) + for index := range localDrives { + g.Go(func() error { + if localDrives[index] == nil { + return errDiskNotFound + } + localDrives[index].DeleteVol(ctx, bucket, false) + return nil + }, index) + } + + g.Wait() + } + + // Create the lost volume only if its not marked for delete + if !opts.Remove { + // Initialize sync waitgroup. + g = errgroup.WithNErrs(len(localDrives)) + + // Make a volume entry on all underlying storage disks. + for index := range localDrives { + g.Go(func() error { + if beforeState[index] == madmin.DriveStateMissing { + err := localDrives[index].MakeVol(ctx, bucket) + if err == nil { + afterState[index] = madmin.DriveStateOk + } + return err + } + return errs[index] + }, index) + } + + errs = g.Wait() + } + + for i := range afterState { + res.After.Drives = append(res.After.Drives, madmin.HealDriveInfo{ + UUID: "", + Endpoint: localDrives[i].Endpoint().String(), + State: afterState[i], + }) + } + return res, nil +} + +func listBucketsLocal(ctx context.Context, opts BucketOptions) (buckets []BucketInfo, err error) { + globalLocalDrivesMu.RLock() + localDrives := cloneDrives(globalLocalDrivesMap) + globalLocalDrivesMu.RUnlock() + + quorum := (len(localDrives) / 2) + + buckets = make([]BucketInfo, 0, 32) + healBuckets := xsync.NewMapOf[string, VolInfo]() + + // lists all unique buckets across drives. + if err := listAllBuckets(ctx, localDrives, healBuckets, quorum); err != nil { + return nil, err + } + + // include deleted buckets in listBuckets output + deletedBuckets := xsync.NewMapOf[string, VolInfo]() + + if opts.Deleted { + // lists all deleted buckets across drives. + if err := listDeletedBuckets(ctx, localDrives, deletedBuckets, quorum); err != nil { + return nil, err + } + } + + healBuckets.Range(func(_ string, volInfo VolInfo) bool { + bi := BucketInfo{ + Name: volInfo.Name, + Created: volInfo.Created, + } + if vi, ok := deletedBuckets.Load(volInfo.Name); ok { + bi.Deleted = vi.Created + } + buckets = append(buckets, bi) + return true + }) + + deletedBuckets.Range(func(_ string, v VolInfo) bool { + if _, ok := healBuckets.Load(v.Name); !ok { + buckets = append(buckets, BucketInfo{ + Name: v.Name, + Deleted: v.Created, + }) + } + return true + }) + + return buckets, nil +} + +func cloneDrives(drives map[string]StorageAPI) []StorageAPI { + copyDrives := make([]StorageAPI, 0, len(drives)) + for _, drive := range drives { + copyDrives = append(copyDrives, drive) + } + return copyDrives +} + +func getBucketInfoLocal(ctx context.Context, bucket string, opts BucketOptions) (BucketInfo, error) { + globalLocalDrivesMu.RLock() + localDrives := cloneDrives(globalLocalDrivesMap) + globalLocalDrivesMu.RUnlock() + + g := errgroup.WithNErrs(len(localDrives)).WithConcurrency(32) + bucketsInfo := make([]BucketInfo, len(localDrives)) + + // Make a volume entry on all underlying storage disks. + for index := range localDrives { + g.Go(func() error { + if localDrives[index] == nil { + return errDiskNotFound + } + volInfo, err := localDrives[index].StatVol(ctx, bucket) + if err != nil { + if opts.Deleted { + dvi, derr := localDrives[index].StatVol(ctx, pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, bucket)) + if derr != nil { + return err + } + bucketsInfo[index] = BucketInfo{Name: bucket, Deleted: dvi.Created} + return nil + } + return err + } + + bucketsInfo[index] = BucketInfo{Name: bucket, Created: volInfo.Created} + return nil + }, index) + } + + errs := g.Wait() + if err := reduceReadQuorumErrs(ctx, errs, bucketOpIgnoredErrs, (len(localDrives) / 2)); err != nil { + return BucketInfo{}, err + } + + var bucketInfo BucketInfo + for i, err := range errs { + if err == nil { + bucketInfo = bucketsInfo[i] + break + } + } + + return bucketInfo, nil +} + +func deleteBucketLocal(ctx context.Context, bucket string, opts DeleteBucketOptions) error { + globalLocalDrivesMu.RLock() + localDrives := cloneDrives(globalLocalDrivesMap) + globalLocalDrivesMu.RUnlock() + + g := errgroup.WithNErrs(len(localDrives)).WithConcurrency(32) + + // Make a volume entry on all underlying storage disks. + for index := range localDrives { + g.Go(func() error { + if localDrives[index] == nil { + return errDiskNotFound + } + return localDrives[index].DeleteVol(ctx, bucket, opts.Force) + }, index) + } + + return reduceWriteQuorumErrs(ctx, g.Wait(), bucketOpIgnoredErrs, (len(localDrives)/2)+1) +} + +func makeBucketLocal(ctx context.Context, bucket string, opts MakeBucketOptions) error { + globalLocalDrivesMu.RLock() + localDrives := cloneDrives(globalLocalDrivesMap) + globalLocalDrivesMu.RUnlock() + + g := errgroup.WithNErrs(len(localDrives)).WithConcurrency(32) + + // Make a volume entry on all underlying storage disks. + for index := range localDrives { + g.Go(func() error { + if localDrives[index] == nil { + return errDiskNotFound + } + err := localDrives[index].MakeVol(ctx, bucket) + if opts.ForceCreate && errors.Is(err, errVolumeExists) { + // No need to return error when force create was + // requested. + return nil + } + return err + }, index) + } + + errs := g.Wait() + return reduceWriteQuorumErrs(ctx, errs, bucketOpIgnoredErrs, (len(localDrives)/2)+1) +} diff --git a/cmd/perf-tests.go b/cmd/perf-tests.go new file mode 100644 index 00000000..f9b4663f --- /dev/null +++ b/cmd/perf-tests.go @@ -0,0 +1,455 @@ +// Copyright (c) 2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/gob" + "errors" + "fmt" + "io" + "math/rand" + "net/http" + "net/url" + "sync" + "sync/atomic" + "time" + + "github.com/dustin/go-humanize" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + xhttp "github.com/minio/minio/internal/http" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/pkg/v3/randreader" +) + +// SpeedTestResult return value of the speedtest function +type SpeedTestResult struct { + Endpoint string + Uploads uint64 + Downloads uint64 + UploadTimes madmin.TimeDurations + DownloadTimes madmin.TimeDurations + DownloadTTFB madmin.TimeDurations + Error string +} + +func newRandomReader(size int) io.Reader { + return io.LimitReader(randreader.New(), int64(size)) +} + +type firstByteRecorder struct { + t *time.Time + r io.Reader +} + +func (f *firstByteRecorder) Read(p []byte) (n int, err error) { + if f.t != nil || len(p) == 0 { + return f.r.Read(p) + } + // Read a single byte. + n, err = f.r.Read(p[:1]) + if n > 0 { + t := time.Now() + f.t = &t + } + return n, err +} + +// Runs the speedtest on local MinIO process. +func selfSpeedTest(ctx context.Context, opts speedTestOpts) (res SpeedTestResult, err error) { + objAPI := newObjectLayerFn() + if objAPI == nil { + return SpeedTestResult{}, errServerNotInitialized + } + + var wg sync.WaitGroup + var errOnce sync.Once + var retError string + var totalBytesWritten uint64 + var totalBytesRead uint64 + + objCountPerThread := make([]uint64, opts.concurrency) + + uploadsCtx, uploadsCancel := context.WithTimeout(ctx, opts.duration) + defer uploadsCancel() + + objNamePrefix := pathJoin(speedTest, mustGetUUID()) + + userMetadata := make(map[string]string) + userMetadata[globalObjectPerfUserMetadata] = "true" // Bypass S3 API freeze + popts := minio.PutObjectOptions{ + UserMetadata: userMetadata, + DisableContentSha256: !opts.enableSha256, + DisableMultipart: !opts.enableMultipart, + } + + clnt := globalMinioClient + if !globalAPIConfig.permitRootAccess() { + region := globalSite.Region() + if region == "" { + region = "us-east-1" + } + clnt, err = minio.New(globalLocalNodeName, &minio.Options{ + Creds: credentials.NewStaticV4(opts.creds.AccessKey, opts.creds.SecretKey, opts.creds.SessionToken), + Secure: globalIsTLS, + Transport: globalRemoteTargetTransport, + Region: region, + }) + if err != nil { + return res, err + } + } + + var mu sync.Mutex + var uploadTimes madmin.TimeDurations + wg.Add(opts.concurrency) + for i := 0; i < opts.concurrency; i++ { + go func(i int) { + defer wg.Done() + for { + t := time.Now() + reader := newRandomReader(opts.objectSize) + tmpObjName := pathJoin(objNamePrefix, fmt.Sprintf("%d/%d", i, objCountPerThread[i])) + info, err := clnt.PutObject(uploadsCtx, opts.bucketName, tmpObjName, reader, int64(opts.objectSize), popts) + if err != nil { + if !contextCanceled(uploadsCtx) && !errors.Is(err, context.Canceled) { + errOnce.Do(func() { + retError = err.Error() + }) + } + uploadsCancel() + return + } + response := time.Since(t) + atomic.AddUint64(&totalBytesWritten, uint64(info.Size)) + objCountPerThread[i]++ + mu.Lock() + uploadTimes = append(uploadTimes, response) + mu.Unlock() + } + }(i) + } + wg.Wait() + + // We already saw write failures, no need to proceed into read's + if retError != "" { + return SpeedTestResult{ + Uploads: totalBytesWritten, + Downloads: totalBytesRead, + UploadTimes: uploadTimes, + Error: retError, + }, nil + } + + downloadsCtx, downloadsCancel := context.WithTimeout(ctx, opts.duration) + defer downloadsCancel() + + gopts := minio.GetObjectOptions{} + gopts.Set(globalObjectPerfUserMetadata, "true") // Bypass S3 API freeze + + var downloadTimes madmin.TimeDurations + var downloadTTFB madmin.TimeDurations + wg.Add(opts.concurrency) + + c := minio.Core{Client: clnt} + for i := 0; i < opts.concurrency; i++ { + go func(i int) { + defer wg.Done() + var j uint64 + if objCountPerThread[i] == 0 { + return + } + for { + if objCountPerThread[i] == j { + j = 0 + } + tmpObjName := pathJoin(objNamePrefix, fmt.Sprintf("%d/%d", i, j)) + t := time.Now() + + r, _, _, err := c.GetObject(downloadsCtx, opts.bucketName, tmpObjName, gopts) + if err != nil { + errResp, ok := err.(minio.ErrorResponse) + if ok && errResp.StatusCode == http.StatusNotFound { + continue + } + if !contextCanceled(downloadsCtx) && !errors.Is(err, context.Canceled) { + errOnce.Do(func() { + retError = err.Error() + }) + } + downloadsCancel() + return + } + fbr := firstByteRecorder{ + r: r, + } + n, err := xioutil.Copy(xioutil.Discard, &fbr) + r.Close() + if err == nil { + response := time.Since(t) + ttfb := time.Since(*fbr.t) + // Only capture success criteria - do not + // have to capture failed reads, truncated + // reads etc. + atomic.AddUint64(&totalBytesRead, uint64(n)) + mu.Lock() + downloadTimes = append(downloadTimes, response) + downloadTTFB = append(downloadTTFB, ttfb) + mu.Unlock() + } + if err != nil { + if !contextCanceled(downloadsCtx) && !errors.Is(err, context.Canceled) { + errOnce.Do(func() { + retError = err.Error() + }) + } + downloadsCancel() + return + } + j++ + } + }(i) + } + wg.Wait() + + return SpeedTestResult{ + Uploads: totalBytesWritten, + Downloads: totalBytesRead, + UploadTimes: uploadTimes, + DownloadTimes: downloadTimes, + DownloadTTFB: downloadTTFB, + Error: retError, + }, nil +} + +// To collect RX stats during "mc support perf net" +// RXSample holds the RX bytes for the duration between +// the last peer to connect and the first peer to disconnect. +// This is to improve the RX throughput accuracy. +type netPerfRX struct { + RX uint64 // RX bytes + lastToConnect time.Time // time at which last peer to connect to us + firstToDisconnect time.Time // time at which the first peer disconnects from us + RXSample uint64 // RX bytes between lastToConnect and firstToDisconnect + activeConnections uint64 + sync.RWMutex +} + +func (n *netPerfRX) Connect() { + n.Lock() + defer n.Unlock() + n.activeConnections++ + atomic.StoreUint64(&n.RX, 0) + n.lastToConnect = time.Now() +} + +func (n *netPerfRX) Disconnect() { + n.Lock() + defer n.Unlock() + n.activeConnections-- + if n.firstToDisconnect.IsZero() { + n.RXSample = atomic.LoadUint64(&n.RX) + n.firstToDisconnect = time.Now() + } +} + +func (n *netPerfRX) ActiveConnections() uint64 { + n.RLock() + defer n.RUnlock() + return n.activeConnections +} + +func (n *netPerfRX) Reset() { + n.Lock() + defer n.Unlock() + n.RX = 0 + n.RXSample = 0 + n.lastToConnect = time.Time{} + n.firstToDisconnect = time.Time{} +} + +// Reader to read random data. +type netperfReader struct { + n uint64 + eof chan struct{} + buf []byte +} + +func (m *netperfReader) Read(b []byte) (int, error) { + select { + case <-m.eof: + return 0, io.EOF + default: + } + n := copy(b, m.buf) + atomic.AddUint64(&m.n, uint64(n)) + return n, nil +} + +func netperf(ctx context.Context, duration time.Duration) madmin.NetperfNodeResult { + r := &netperfReader{eof: make(chan struct{})} + r.buf = make([]byte, 128*humanize.KiByte) + rand.Read(r.buf) + + connectionsPerPeer := 16 + + if len(globalNotificationSys.peerClients) > 16 { + // For a large cluster it's enough to have 1 connection per peer to saturate the network. + connectionsPerPeer = 1 + } + + errStr := "" + var wg sync.WaitGroup + for index := range globalNotificationSys.peerClients { + if globalNotificationSys.peerClients[index] == nil { + continue + } + go func(index int) { + for i := 0; i < connectionsPerPeer; i++ { + wg.Add(1) + go func() { + defer wg.Done() + err := globalNotificationSys.peerClients[index].DevNull(ctx, r) + if err != nil { + errStr = fmt.Sprintf("error with %s: %s", globalNotificationSys.peerClients[index].String(), err.Error()) + } + }() + } + }(index) + } + + time.Sleep(duration) + xioutil.SafeClose(r.eof) + wg.Wait() + for globalNetPerfRX.ActiveConnections() != 0 { + time.Sleep(time.Second) + } + rx := float64(globalNetPerfRX.RXSample) + delta := globalNetPerfRX.firstToDisconnect.Sub(globalNetPerfRX.lastToConnect) + if delta < 0 { + rx = 0 + errStr = "network disconnection issues detected" + } + + globalNetPerfRX.Reset() + return madmin.NetperfNodeResult{Endpoint: "", TX: r.n / uint64(duration.Seconds()), RX: uint64(rx / delta.Seconds()), Error: errStr} +} + +func siteNetperf(ctx context.Context, duration time.Duration) madmin.SiteNetPerfNodeResult { + r := &netperfReader{eof: make(chan struct{})} + r.buf = make([]byte, 128*humanize.KiByte) + rand.Read(r.buf) + + clusterInfos, err := globalSiteReplicationSys.GetClusterInfo(ctx) + if err != nil { + return madmin.SiteNetPerfNodeResult{Error: err.Error()} + } + + // Scale the number of connections from 32 -> 4 from small to large clusters. + connectionsPerPeer := 3 + (29+len(clusterInfos.Sites)-1)/len(clusterInfos.Sites) + + errStr := "" + var wg sync.WaitGroup + + for _, info := range clusterInfos.Sites { + // skip self + if globalDeploymentID() == info.DeploymentID { + continue + } + info := info + wg.Add(connectionsPerPeer) + for range connectionsPerPeer { + go func() { + defer wg.Done() + ctx, cancel := context.WithTimeout(ctx, duration+10*time.Second) + defer cancel() + perfNetRequest( + ctx, + info.DeploymentID, + adminPathPrefix+adminAPIVersionPrefix+adminAPISiteReplicationDevNull, + r, + ) + }() + } + } + + time.Sleep(duration) + xioutil.SafeClose(r.eof) + wg.Wait() + for globalSiteNetPerfRX.ActiveConnections() != 0 && !contextCanceled(ctx) { + time.Sleep(time.Second) + } + rx := float64(globalSiteNetPerfRX.RXSample) + delta := globalSiteNetPerfRX.firstToDisconnect.Sub(globalSiteNetPerfRX.lastToConnect) + // If the first disconnected before the last connected, we likely had a network issue. + if delta <= 0 { + rx = 0 + errStr = "detected network disconnections, possibly an unstable network" + } + + globalSiteNetPerfRX.Reset() + return madmin.SiteNetPerfNodeResult{ + Endpoint: "", + TX: r.n, + TXTotalDuration: duration, + RX: uint64(rx), + RXTotalDuration: delta, + Error: errStr, + TotalConn: uint64(connectionsPerPeer), + } +} + +// perfNetRequest - reader for http.request.body +func perfNetRequest(ctx context.Context, deploymentID, reqPath string, reader io.Reader) (result madmin.SiteNetPerfNodeResult) { + result = madmin.SiteNetPerfNodeResult{} + cli, err := globalSiteReplicationSys.getAdminClient(ctx, deploymentID) + if err != nil { + result.Error = err.Error() + return result + } + rp := cli.GetEndpointURL() + reqURL := &url.URL{ + Scheme: rp.Scheme, + Host: rp.Host, + Path: reqPath, + } + result.Endpoint = rp.String() + req, err := http.NewRequestWithContext(ctx, http.MethodPost, reqURL.String(), reader) + if err != nil { + result.Error = err.Error() + return result + } + client := &http.Client{ + Transport: globalRemoteTargetTransport, + } + resp, err := client.Do(req) + if err != nil { + result.Error = err.Error() + return result + } + defer xhttp.DrainBody(resp.Body) + err = gob.NewDecoder(resp.Body).Decode(&result) + // endpoint have been overwritten + result.Endpoint = rp.String() + if err != nil { + result.Error = err.Error() + } + return result +} diff --git a/cmd/policy_test.go b/cmd/policy_test.go new file mode 100644 index 00000000..bd9c9add --- /dev/null +++ b/cmd/policy_test.go @@ -0,0 +1,303 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "reflect" + "testing" + + miniogopolicy "github.com/minio/minio-go/v7/pkg/policy" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/pkg/v3/policy" + "github.com/minio/pkg/v3/policy/condition" +) + +func TestPolicySysIsAllowed(t *testing.T) { + p := &policy.BucketPolicy{ + Version: policy.DefaultVersion, + Statements: []policy.BPStatement{ + policy.NewBPStatement("", + policy.Allow, + policy.NewPrincipal("*"), + policy.NewActionSet(policy.GetBucketLocationAction), + policy.NewResourceSet(policy.NewResource("mybucket")), + condition.NewFunctions(), + ), + policy.NewBPStatement("", + policy.Allow, + policy.NewPrincipal("*"), + policy.NewActionSet(policy.PutObjectAction), + policy.NewResourceSet(policy.NewResource("mybucket/myobject*")), + condition.NewFunctions(), + ), + }, + } + + anonGetBucketLocationArgs := policy.BucketPolicyArgs{ + AccountName: "Q3AM3UQ867SPQQA43P2F", + Action: policy.GetBucketLocationAction, + BucketName: "mybucket", + ConditionValues: map[string][]string{}, + } + + anonPutObjectActionArgs := policy.BucketPolicyArgs{ + AccountName: "Q3AM3UQ867SPQQA43P2F", + Action: policy.PutObjectAction, + BucketName: "mybucket", + ConditionValues: map[string][]string{ + "x-amz-copy-source": {"mybucket/myobject"}, + "SourceIp": {"192.168.1.10"}, + }, + ObjectName: "myobject", + } + + anonGetObjectActionArgs := policy.BucketPolicyArgs{ + AccountName: "Q3AM3UQ867SPQQA43P2F", + Action: policy.GetObjectAction, + BucketName: "mybucket", + ConditionValues: map[string][]string{}, + ObjectName: "myobject", + } + + getBucketLocationArgs := policy.BucketPolicyArgs{ + AccountName: "Q3AM3UQ867SPQQA43P2F", + Action: policy.GetBucketLocationAction, + BucketName: "mybucket", + ConditionValues: map[string][]string{}, + IsOwner: true, + } + + putObjectActionArgs := policy.BucketPolicyArgs{ + AccountName: "Q3AM3UQ867SPQQA43P2F", + Action: policy.PutObjectAction, + BucketName: "mybucket", + ConditionValues: map[string][]string{ + "x-amz-copy-source": {"mybucket/myobject"}, + "SourceIp": {"192.168.1.10"}, + }, + IsOwner: true, + ObjectName: "myobject", + } + + getObjectActionArgs := policy.BucketPolicyArgs{ + AccountName: "Q3AM3UQ867SPQQA43P2F", + Action: policy.GetObjectAction, + BucketName: "mybucket", + ConditionValues: map[string][]string{}, + IsOwner: true, + ObjectName: "myobject", + } + + yourbucketAnonGetObjectActionArgs := policy.BucketPolicyArgs{ + AccountName: "Q3AM3UQ867SPQQA43P2F", + Action: policy.GetObjectAction, + BucketName: "yourbucket", + ConditionValues: map[string][]string{}, + ObjectName: "yourobject", + } + + yourbucketGetObjectActionArgs := policy.BucketPolicyArgs{ + AccountName: "Q3AM3UQ867SPQQA43P2F", + Action: policy.GetObjectAction, + BucketName: "yourbucket", + ConditionValues: map[string][]string{}, + IsOwner: true, + ObjectName: "yourobject", + } + + testCases := []struct { + args policy.BucketPolicyArgs + expectedResult bool + }{ + {anonGetBucketLocationArgs, true}, + {anonPutObjectActionArgs, true}, + {anonGetObjectActionArgs, false}, + {getBucketLocationArgs, true}, + {putObjectActionArgs, true}, + {getObjectActionArgs, true}, + {yourbucketAnonGetObjectActionArgs, false}, + {yourbucketGetObjectActionArgs, true}, + } + + for i, testCase := range testCases { + result := p.IsAllowed(testCase.args) + + if result != testCase.expectedResult { + t.Fatalf("case %v: expected: %v, got: %v\n", i+1, testCase.expectedResult, result) + } + } +} + +func getReadOnlyStatement(bucketName, prefix string) []miniogopolicy.Statement { + return []miniogopolicy.Statement{ + { + Effect: string(policy.Allow), + Principal: miniogopolicy.User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet(policy.NewResource(bucketName).String()), + Actions: set.CreateStringSet("s3:GetBucketLocation", "s3:ListBucket"), + }, + { + Effect: string(policy.Allow), + Principal: miniogopolicy.User{AWS: set.CreateStringSet("*")}, + Resources: set.CreateStringSet(policy.NewResource(bucketName + "/" + prefix).String()), + Actions: set.CreateStringSet("s3:GetObject"), + }, + } +} + +func TestPolicyToBucketAccessPolicy(t *testing.T) { + case1Policy := &policy.BucketPolicy{ + Version: policy.DefaultVersion, + Statements: []policy.BPStatement{ + policy.NewBPStatement("", + policy.Allow, + policy.NewPrincipal("*"), + policy.NewActionSet(policy.GetBucketLocationAction, policy.ListBucketAction), + policy.NewResourceSet(policy.NewResource("mybucket")), + condition.NewFunctions(), + ), + policy.NewBPStatement("", + policy.Allow, + policy.NewPrincipal("*"), + policy.NewActionSet(policy.GetObjectAction), + policy.NewResourceSet(policy.NewResource("mybucket/myobject*")), + condition.NewFunctions(), + ), + }, + } + + case1Result := &miniogopolicy.BucketAccessPolicy{ + Version: policy.DefaultVersion, + Statements: getReadOnlyStatement("mybucket", "myobject*"), + } + + case2Policy := &policy.BucketPolicy{ + Version: policy.DefaultVersion, + Statements: []policy.BPStatement{}, + } + + case2Result := &miniogopolicy.BucketAccessPolicy{ + Version: policy.DefaultVersion, + Statements: []miniogopolicy.Statement{}, + } + + case3Policy := &policy.BucketPolicy{ + Version: "12-10-2012", + Statements: []policy.BPStatement{ + policy.NewBPStatement("", + policy.Allow, + policy.NewPrincipal("*"), + policy.NewActionSet(policy.PutObjectAction), + policy.NewResourceSet(policy.NewResource("mybucket/myobject*")), + condition.NewFunctions(), + ), + }, + } + + testCases := []struct { + bucketPolicy *policy.BucketPolicy + expectedResult *miniogopolicy.BucketAccessPolicy + expectErr bool + }{ + {case1Policy, case1Result, false}, + {case2Policy, case2Result, false}, + {case3Policy, nil, true}, + } + + for i, testCase := range testCases { + result, err := PolicyToBucketAccessPolicy(testCase.bucketPolicy) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("case %v: error: expected: %v, got: %v\n", i+1, testCase.expectErr, expectErr) + } + + if !testCase.expectErr { + if !reflect.DeepEqual(result, testCase.expectedResult) { + t.Fatalf("case %v: result: expected: %+v, got: %+v\n", i+1, testCase.expectedResult, result) + } + } + } +} + +func TestBucketAccessPolicyToPolicy(t *testing.T) { + case1PolicyInfo := &miniogopolicy.BucketAccessPolicy{ + Version: policy.DefaultVersion, + Statements: getReadOnlyStatement("mybucket", "myobject*"), + } + + case1Result := &policy.BucketPolicy{ + Version: policy.DefaultVersion, + Statements: []policy.BPStatement{ + policy.NewBPStatement("", + policy.Allow, + policy.NewPrincipal("*"), + policy.NewActionSet(policy.GetBucketLocationAction, policy.ListBucketAction), + policy.NewResourceSet(policy.NewResource("mybucket")), + condition.NewFunctions(), + ), + policy.NewBPStatement("", + policy.Allow, + policy.NewPrincipal("*"), + policy.NewActionSet(policy.GetObjectAction), + policy.NewResourceSet(policy.NewResource("mybucket/myobject*")), + condition.NewFunctions(), + ), + }, + } + + case2PolicyInfo := &miniogopolicy.BucketAccessPolicy{ + Version: policy.DefaultVersion, + Statements: []miniogopolicy.Statement{}, + } + + case2Result := &policy.BucketPolicy{ + Version: policy.DefaultVersion, + Statements: []policy.BPStatement{}, + } + + case3PolicyInfo := &miniogopolicy.BucketAccessPolicy{ + Version: "12-10-2012", + Statements: getReadOnlyStatement("mybucket", "/myobject*"), + } + + testCases := []struct { + policyInfo *miniogopolicy.BucketAccessPolicy + expectedResult *policy.BucketPolicy + expectErr bool + }{ + {case1PolicyInfo, case1Result, false}, + {case2PolicyInfo, case2Result, false}, + {case3PolicyInfo, nil, true}, + } + + for i, testCase := range testCases { + result, err := BucketAccessPolicyToPolicy(testCase.policyInfo) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("case %v: error: expected: %v, got: %v\n", i+1, testCase.expectErr, expectErr) + } + + if !testCase.expectErr { + if !reflect.DeepEqual(result, testCase.expectedResult) { + t.Fatalf("case %v: result: expected: %+v, got: %+v\n", i+1, testCase.expectedResult, result) + } + } + } +} diff --git a/cmd/post-policy-fan-out.go b/cmd/post-policy-fan-out.go new file mode 100644 index 00000000..23a11898 --- /dev/null +++ b/cmd/post-policy-fan-out.go @@ -0,0 +1,128 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "maps" + "sync" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/tags" + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/kms" +) + +type fanOutOptions struct { + Kind crypto.Type + KeyID string + Key []byte + KmsCtx kms.Context + Checksum *hash.Checksum + MD5Hex string +} + +// fanOutPutObject takes an input source reader and fans out multiple PUT operations +// based on the incoming fan-out request, a context cancellation by the caller +// would ensure all fan-out operations are canceled. +func fanOutPutObject(ctx context.Context, bucket string, objectAPI ObjectLayer, fanOutEntries []minio.PutObjectFanOutEntry, fanOutBuf []byte, opts fanOutOptions) ([]ObjectInfo, []error) { + errs := make([]error, len(fanOutEntries)) + objInfos := make([]ObjectInfo, len(fanOutEntries)) + + var wg sync.WaitGroup + for i, req := range fanOutEntries { + wg.Add(1) + go func(idx int, req minio.PutObjectFanOutEntry) { + defer wg.Done() + + objInfos[idx] = ObjectInfo{Name: req.Key} + + hopts := hash.Options{ + Size: int64(len(fanOutBuf)), + MD5Hex: opts.MD5Hex, + SHA256Hex: "", + ActualSize: -1, + DisableMD5: true, + } + hr, err := hash.NewReaderWithOpts(ctx, bytes.NewReader(fanOutBuf), hopts) + if err != nil { + errs[idx] = err + return + } + + reader := NewPutObjReader(hr) + defer func() { + if err := reader.Close(); err != nil { + errs[idx] = err + } + if err := hr.Close(); err != nil { + errs[idx] = err + } + }() + + userDefined := make(map[string]string, len(req.UserMetadata)) + maps.Copy(userDefined, req.UserMetadata) + + tgs, err := tags.NewTags(req.UserTags, true) + if err != nil { + errs[idx] = err + return + } + + userDefined[xhttp.AmzObjectTagging] = tgs.String() + + if opts.Kind != nil { + encrd, objectEncryptionKey, err := newEncryptReader(ctx, hr, opts.Kind, opts.KeyID, opts.Key, bucket, req.Key, userDefined, opts.KmsCtx) + if err != nil { + errs[idx] = err + return + } + + // do not try to verify encrypted content/ + hr, err = hash.NewReader(ctx, encrd, -1, "", "", -1) + if err != nil { + errs[idx] = err + return + } + + reader, err = reader.WithEncryption(hr, &objectEncryptionKey) + if err != nil { + errs[idx] = err + return + } + } + + objInfo, err := objectAPI.PutObject(ctx, bucket, req.Key, reader, ObjectOptions{ + Versioned: globalBucketVersioningSys.PrefixEnabled(bucket, req.Key), + VersionSuspended: globalBucketVersioningSys.PrefixSuspended(bucket, req.Key), + UserDefined: userDefined, + }) + if err != nil { + errs[idx] = err + return + } + objInfos[idx] = objInfo + }(i, req) + } + wg.Wait() + + return objInfos, errs +} diff --git a/cmd/post-policy_test.go b/cmd/post-policy_test.go new file mode 100644 index 00000000..9a02ca84 --- /dev/null +++ b/cmd/post-policy_test.go @@ -0,0 +1,760 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/base64" + "fmt" + "io" + "maps" + "mime/multipart" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "testing" + "time" + + "github.com/dustin/go-humanize" +) + +const ( + iso8601DateFormat = "20060102T150405Z" +) + +func newPostPolicyBytesV4WithContentRange(credential, bucketName, objectKey string, expiration time.Time) []byte { + t := UTCNow() + // Add the expiration date. + expirationStr := fmt.Sprintf(`"expiration": "%s"`, expiration.Format(iso8601TimeFormat)) + // Add the bucket condition, only accept buckets equal to the one passed. + bucketConditionStr := fmt.Sprintf(`["eq", "$bucket", "%s"]`, bucketName) + // Add the key condition, only accept keys equal to the one passed. + keyConditionStr := fmt.Sprintf(`["eq", "$key", "%s/upload.txt"]`, objectKey) + // Add content length condition, only accept content sizes of a given length. + contentLengthCondStr := `["content-length-range", 1024, 1048576]` + // Add the algorithm condition, only accept AWS SignV4 Sha256. + algorithmConditionStr := `["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"]` + // Add the date condition, only accept the current date. + dateConditionStr := fmt.Sprintf(`["eq", "$x-amz-date", "%s"]`, t.Format(iso8601DateFormat)) + // Add the credential string, only accept the credential passed. + credentialConditionStr := fmt.Sprintf(`["eq", "$x-amz-credential", "%s"]`, credential) + // Add the meta-uuid string, set to 1234 + uuidConditionStr := fmt.Sprintf(`["eq", "$x-amz-meta-uuid", "%s"]`, "1234") + // Add the content-encoding string, set to gzip. + contentEncodingConditionStr := fmt.Sprintf(`["eq", "$content-encoding", "%s"]`, "gzip") + + // Combine all conditions into one string. + conditionStr := fmt.Sprintf(`"conditions":[%s, %s, %s, %s, %s, %s, %s, %s]`, bucketConditionStr, + keyConditionStr, contentLengthCondStr, algorithmConditionStr, dateConditionStr, credentialConditionStr, uuidConditionStr, contentEncodingConditionStr) + retStr := "{" + retStr = retStr + expirationStr + "," + retStr += conditionStr + retStr += "}" + + return []byte(retStr) +} + +// newPostPolicyBytesV4 - creates a bare bones postpolicy string with key and bucket matches. +func newPostPolicyBytesV4(credential, bucketName, objectKey string, expiration time.Time) []byte { + t := UTCNow() + // Add the expiration date. + expirationStr := fmt.Sprintf(`"expiration": "%s"`, expiration.Format(iso8601TimeFormat)) + // Add the bucket condition, only accept buckets equal to the one passed. + bucketConditionStr := fmt.Sprintf(`["eq", "$bucket", "%s"]`, bucketName) + // Add the key condition, only accept keys equal to the one passed. + keyConditionStr := fmt.Sprintf(`["eq", "$key", "%s/upload.txt"]`, objectKey) + // Add the algorithm condition, only accept AWS SignV4 Sha256. + algorithmConditionStr := `["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"]` + // Add the date condition, only accept the current date. + dateConditionStr := fmt.Sprintf(`["eq", "$x-amz-date", "%s"]`, t.Format(iso8601DateFormat)) + // Add the credential string, only accept the credential passed. + credentialConditionStr := fmt.Sprintf(`["eq", "$x-amz-credential", "%s"]`, credential) + // Add the meta-uuid string, set to 1234 + uuidConditionStr := fmt.Sprintf(`["eq", "$x-amz-meta-uuid", "%s"]`, "1234") + // Add the content-encoding string, set to gzip + contentEncodingConditionStr := fmt.Sprintf(`["eq", "$content-encoding", "%s"]`, "gzip") + + // Combine all conditions into one string. + conditionStr := fmt.Sprintf(`"conditions":[%s, %s, %s, %s, %s, %s, %s]`, bucketConditionStr, keyConditionStr, algorithmConditionStr, dateConditionStr, credentialConditionStr, uuidConditionStr, contentEncodingConditionStr) + retStr := "{" + retStr = retStr + expirationStr + "," + retStr += conditionStr + retStr += "}" + + return []byte(retStr) +} + +// newPostPolicyBytesV2 - creates a bare bones postpolicy string with key and bucket matches. +func newPostPolicyBytesV2(bucketName, objectKey string, expiration time.Time) []byte { + // Add the expiration date. + expirationStr := fmt.Sprintf(`"expiration": "%s"`, expiration.Format(iso8601TimeFormat)) + // Add the bucket condition, only accept buckets equal to the one passed. + bucketConditionStr := fmt.Sprintf(`["eq", "$bucket", "%s"]`, bucketName) + // Add the key condition, only accept keys equal to the one passed. + keyConditionStr := fmt.Sprintf(`["starts-with", "$key", "%s/upload.txt"]`, objectKey) + + // Combine all conditions into one string. + conditionStr := fmt.Sprintf(`"conditions":[%s, %s]`, bucketConditionStr, keyConditionStr) + retStr := "{" + retStr = retStr + expirationStr + "," + retStr += conditionStr + retStr += "}" + + return []byte(retStr) +} + +// Wrapper +func TestPostPolicyReservedBucketExploit(t *testing.T) { + ExecObjectLayerTestWithDirs(t, testPostPolicyReservedBucketExploit) +} + +// testPostPolicyReservedBucketExploit is a test for the exploit fixed in PR +// #16849 +func testPostPolicyReservedBucketExploit(obj ObjectLayer, instanceType string, dirs []string, t TestErrHandler) { + if err := newTestConfig(globalMinioDefaultRegion, obj); err != nil { + t.Fatalf("Initializing config.json failed") + } + + // Register the API end points with Erasure/FS object layer. + apiRouter := initTestAPIEndPoints(obj, []string{"PostPolicy"}) + + credentials := globalActiveCred + bucketName := minioMetaBucket + objectName := "config/x" + + // This exploit needs browser to be enabled. + if !globalBrowserEnabled { + globalBrowserEnabled = true + defer func() { globalBrowserEnabled = false }() + } + + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + req, perr := newPostRequestV4("", bucketName, objectName, []byte("pwned"), credentials.AccessKey, credentials.SecretKey) + if perr != nil { + t.Fatalf("Test %s: Failed to create HTTP request for PostPolicyHandler: %v", instanceType, perr) + } + + contentTypeHdr := req.Header.Get("Content-Type") + contentTypeHdr = strings.Replace(contentTypeHdr, "multipart/form-data", "multipart/form-datA", 1) + req.Header.Set("Content-Type", contentTypeHdr) + req.Header.Set("User-Agent", "Mozilla") + + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + + ctx, cancel := context.WithCancel(GlobalContext) + defer cancel() + + // Now check if we actually wrote to backend (regardless of the response + // returned by the server). + z := obj.(*erasureServerPools) + xl := z.serverPools[0].sets[0] + erasureDisks := xl.getDisks() + parts, errs := readAllFileInfo(ctx, erasureDisks, "", bucketName, objectName+"/upload.txt", "", false, false) + for i := range parts { + if errs[i] == nil { + if parts[i].Name == objectName+"/upload.txt" { + t.Errorf("Test %s: Failed to stop post policy handler from writing to minioMetaBucket", instanceType) + } + } + } +} + +// Wrapper for calling TestPostPolicyBucketHandler tests for both Erasure multiple disks and single node setup. +func TestPostPolicyBucketHandler(t *testing.T) { + ExecObjectLayerTest(t, testPostPolicyBucketHandler) +} + +// testPostPolicyBucketHandler - Tests validate post policy handler uploading objects. +func testPostPolicyBucketHandler(obj ObjectLayer, instanceType string, t TestErrHandler) { + if err := newTestConfig(globalMinioDefaultRegion, obj); err != nil { + t.Fatalf("Initializing config.json failed") + } + + // get random bucket name. + bucketName := getRandomBucketName() + + var opts ObjectOptions + // Register the API end points with Erasure/FS object layer. + apiRouter := initTestAPIEndPoints(obj, []string{"PostPolicy"}) + + credentials := globalActiveCred + + curTime := UTCNow() + curTimePlus5Min := curTime.Add(time.Minute * 5) + + // bucketnames[0]. + // objectNames[0]. + // uploadIds [0]. + // Create bucket before initiating NewMultipartUpload. + err := obj.MakeBucket(context.Background(), bucketName, MakeBucketOptions{}) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + // Test cases for signature-V2. + testCasesV2 := []struct { + expectedStatus int + secretKey string + formData map[string]string + }{ + {http.StatusForbidden, credentials.SecretKey, map[string]string{"AWSAccessKeyId": "invalidaccesskey"}}, + {http.StatusForbidden, "invalidsecretkey", map[string]string{"AWSAccessKeyId": credentials.AccessKey}}, + {http.StatusNoContent, credentials.SecretKey, map[string]string{"AWSAccessKeyId": credentials.AccessKey}}, + {http.StatusForbidden, credentials.SecretKey, map[string]string{"Awsaccesskeyid": "invalidaccesskey"}}, + {http.StatusForbidden, "invalidsecretkey", map[string]string{"Awsaccesskeyid": credentials.AccessKey}}, + {http.StatusNoContent, credentials.SecretKey, map[string]string{"Awsaccesskeyid": credentials.AccessKey}}, + // Forbidden with key not in policy.conditions for signed requests V2. + {http.StatusForbidden, credentials.SecretKey, map[string]string{"Awsaccesskeyid": credentials.AccessKey, "AnotherKey": "AnotherContent"}}, + } + + for i, test := range testCasesV2 { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + req, perr := newPostRequestV2("", bucketName, "testobject", test.secretKey, test.formData) + if perr != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for PostPolicyHandler: %v", i+1, instanceType, perr) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + if rec.Code != test.expectedStatus { + t.Fatalf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`, Resp: %s", i+1, instanceType, test.expectedStatus, rec.Code, rec.Body) + } + } + + // Test cases for signature-V4. + testCasesV4 := []struct { + objectName string + data []byte + expectedHeaders map[string]string + expectedRespStatus int + accessKey string + secretKey string + malformedBody bool + }{ + // Success case. + { + objectName: "test", + data: []byte("Hello, World"), + expectedRespStatus: http.StatusNoContent, + expectedHeaders: map[string]string{"X-Amz-Meta-Uuid": "1234"}, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + malformedBody: false, + }, + // Bad case invalid request. + { + objectName: "test", + data: []byte("Hello, World"), + expectedRespStatus: http.StatusForbidden, + accessKey: "", + secretKey: "", + malformedBody: false, + }, + // Bad case malformed input. + { + objectName: "test", + data: []byte("Hello, World"), + expectedRespStatus: http.StatusBadRequest, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + malformedBody: true, + }, + } + + for i, testCase := range testCasesV4 { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + req, perr := newPostRequestV4("", bucketName, testCase.objectName, testCase.data, testCase.accessKey, testCase.secretKey) + if perr != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for PostPolicyHandler: %v", i+1, instanceType, perr) + } + if testCase.malformedBody { + // Change the request body. + req.Body = io.NopCloser(bytes.NewReader([]byte("Hello,"))) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + if rec.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) + } + // When the operation is successful, check if sending metadata is successful too + if rec.Code == http.StatusNoContent { + objInfo, err := obj.GetObjectInfo(context.Background(), bucketName, testCase.objectName+"/upload.txt", opts) + if err != nil { + t.Error("Unexpected error: ", err) + } + for k, v := range testCase.expectedHeaders { + if objInfo.UserDefined[k] != v { + t.Errorf("Expected to have header %s with value %s, but found value `%s` instead", k, v, objInfo.UserDefined[k]) + } + } + } + } + + region := "us-east-1" + // Test cases for signature-V4. + testCasesV4BadData := []struct { + objectName string + data []byte + expectedRespStatus int + accessKey string + secretKey string + dates []any + policy string + noFilename bool + corruptedBase64 bool + corruptedMultipart bool + }{ + // Success case. + { + objectName: "test", + data: []byte("Hello, World"), + expectedRespStatus: http.StatusNoContent, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + dates: []any{curTimePlus5Min.Format(iso8601TimeFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)}, + policy: `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"],["eq", "$x-amz-meta-uuid", "1234"],["eq", "$content-encoding", "gzip"]]}`, + }, + // Success case, no multipart filename. + { + objectName: "test", + data: []byte("Hello, World"), + expectedRespStatus: http.StatusNoContent, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + dates: []any{curTimePlus5Min.Format(iso8601TimeFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)}, + policy: `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"],["eq", "$x-amz-meta-uuid", "1234"],["eq", "$content-encoding", "gzip"]]}`, + noFilename: true, + }, + // Success case, big body. + { + objectName: "test", + data: bytes.Repeat([]byte("a"), 10<<20), + expectedRespStatus: http.StatusNoContent, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + dates: []any{curTimePlus5Min.Format(iso8601TimeFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)}, + policy: `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"],["eq", "$x-amz-meta-uuid", "1234"],["eq", "$content-encoding", "gzip"]]}`, + }, + // Corrupted Base 64 result + { + objectName: "test", + data: []byte("Hello, World"), + expectedRespStatus: http.StatusBadRequest, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + dates: []any{curTimePlus5Min.Format(iso8601TimeFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)}, + policy: `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"]]}`, + corruptedBase64: true, + }, + // Corrupted Multipart body + { + objectName: "test", + data: []byte("Hello, World"), + expectedRespStatus: http.StatusBadRequest, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + dates: []any{curTimePlus5Min.Format(iso8601TimeFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)}, + policy: `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"]]}`, + corruptedMultipart: true, + }, + + // Bad case invalid request. + { + objectName: "test", + data: []byte("Hello, World"), + expectedRespStatus: http.StatusForbidden, + accessKey: "", + secretKey: "", + dates: []any{}, + policy: ``, + }, + // Expired document + { + objectName: "test", + data: []byte("Hello, World"), + expectedRespStatus: http.StatusForbidden, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + dates: []any{curTime.Add(-1 * time.Minute * 5).Format(iso8601TimeFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)}, + policy: `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], ["starts-with", "$key", "test/"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"]]}`, + }, + // Corrupted policy document + { + objectName: "test", + data: []byte("Hello, World"), + expectedRespStatus: http.StatusForbidden, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + dates: []any{curTimePlus5Min.Format(iso8601TimeFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)}, + policy: `{"3/aws4_request"]]}`, + }, + } + + for i, testCase := range testCasesV4BadData { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + + testCase.policy = fmt.Sprintf(testCase.policy, testCase.dates...) + + req, perr := newPostRequestV4Generic("", bucketName, testCase.objectName, testCase.data, testCase.accessKey, + testCase.secretKey, region, curTime, []byte(testCase.policy), nil, testCase.noFilename, testCase.corruptedBase64, testCase.corruptedMultipart) + if perr != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for PostPolicyHandler: %v", i+1, instanceType, perr) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + if rec.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) + } + } + + testCases2 := []struct { + objectName string + data []byte + expectedRespStatus int + accessKey string + secretKey string + malformedBody bool + ignoreContentLength bool + }{ + // Success case. + { + objectName: "test", + data: bytes.Repeat([]byte("a"), 1025), + expectedRespStatus: http.StatusNoContent, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + malformedBody: false, + ignoreContentLength: false, + }, + // Success with Content-Length not specified. + { + objectName: "test", + data: bytes.Repeat([]byte("a"), 1025), + expectedRespStatus: http.StatusNoContent, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + malformedBody: false, + ignoreContentLength: true, + }, + // Failed with entity too small. + { + objectName: "test", + data: bytes.Repeat([]byte("a"), 1023), + expectedRespStatus: http.StatusBadRequest, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + malformedBody: false, + ignoreContentLength: false, + }, + // Failed with entity too large. + { + objectName: "test", + data: bytes.Repeat([]byte("a"), (1*humanize.MiByte)+1), + expectedRespStatus: http.StatusBadRequest, + accessKey: credentials.AccessKey, + secretKey: credentials.SecretKey, + malformedBody: false, + ignoreContentLength: false, + }, + } + + for i, testCase := range testCases2 { + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + var req *http.Request + var perr error + if testCase.ignoreContentLength { + req, perr = newPostRequestV4("", bucketName, testCase.objectName, testCase.data, testCase.accessKey, testCase.secretKey) + } else { + req, perr = newPostRequestV4WithContentLength("", bucketName, testCase.objectName, testCase.data, testCase.accessKey, testCase.secretKey) + } + if perr != nil { + t.Fatalf("Test %d: %s: Failed to create HTTP request for PostPolicyHandler: %v", i+1, instanceType, perr) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + if rec.Code != testCase.expectedRespStatus { + t.Errorf("Test %d: %s: Expected the response status to be `%d`, but instead found `%d`", i+1, instanceType, testCase.expectedRespStatus, rec.Code) + } + } +} + +// Wrapper for calling TestPostPolicyBucketHandlerRedirect tests for both Erasure multiple disks and single node setup. +func TestPostPolicyBucketHandlerRedirect(t *testing.T) { + ExecObjectLayerTest(t, testPostPolicyBucketHandlerRedirect) +} + +// testPostPolicyBucketHandlerRedirect tests POST Object when success_action_redirect is specified +func testPostPolicyBucketHandlerRedirect(obj ObjectLayer, instanceType string, t TestErrHandler) { + if err := newTestConfig(globalMinioDefaultRegion, obj); err != nil { + t.Fatalf("Initializing config.json failed") + } + + // get random bucket name. + bucketName := getRandomBucketName() + + // Key specified in Form data + keyName := "test/object" + + var opts ObjectOptions + + // The final name of the upload object + targetObj := keyName + "/upload.txt" + + // The url of success_action_redirect field + redirectURL, err := url.Parse("http://www.google.com?query=value") + if err != nil { + t.Fatal(err) + } + + // Register the API end points with Erasure/FS object layer. + apiRouter := initTestAPIEndPoints(obj, []string{"PostPolicy"}) + + credentials := globalActiveCred + + curTime := UTCNow() + curTimePlus5Min := curTime.Add(time.Minute * 5) + + err = obj.MakeBucket(context.Background(), bucketName, MakeBucketOptions{}) + if err != nil { + // Failed to create newbucket, abort. + t.Fatalf("%s : %s", instanceType, err.Error()) + } + + // initialize HTTP NewRecorder, this records any mutations to response writer inside the handler. + rec := httptest.NewRecorder() + + dates := []any{curTimePlus5Min.Format(iso8601TimeFormat), curTime.Format(iso8601DateFormat), curTime.Format(yyyymmdd)} + policy := `{"expiration": "%s","conditions":[["eq", "$bucket", "` + bucketName + `"], {"success_action_redirect":"` + redirectURL.String() + `"},["starts-with", "$key", "test/"], ["eq", "$x-amz-meta-uuid", "1234"], ["eq", "$x-amz-algorithm", "AWS4-HMAC-SHA256"], ["eq", "$x-amz-date", "%s"], ["eq", "$x-amz-credential", "` + credentials.AccessKey + `/%s/us-east-1/s3/aws4_request"],["eq", "$content-encoding", "gzip"]]}` + + // Generate the final policy document + policy = fmt.Sprintf(policy, dates...) + + region := "us-east-1" + // Create a new POST request with success_action_redirect field specified + req, perr := newPostRequestV4Generic("", bucketName, keyName, []byte("objData"), + credentials.AccessKey, credentials.SecretKey, region, curTime, + []byte(policy), map[string]string{"success_action_redirect": redirectURL.String()}, false, false, false) + + if perr != nil { + t.Fatalf("%s: Failed to create HTTP request for PostPolicyHandler: %v", instanceType, perr) + } + // Since `apiRouter` satisfies `http.Handler` it has a ServeHTTP to execute the logic of the handler. + // Call the ServeHTTP to execute the handler. + apiRouter.ServeHTTP(rec, req) + + // Check the status code, which must be 303 because success_action_redirect is specified + if rec.Code != http.StatusSeeOther { + t.Errorf("%s: Expected the response status to be `%d`, but instead found `%d`", instanceType, http.StatusSeeOther, rec.Code) + } + + // Get the uploaded object info + info, err := obj.GetObjectInfo(context.Background(), bucketName, targetObj, opts) + if err != nil { + t.Error("Unexpected error: ", err) + } + + v := redirectURL.Query() + v.Add("bucket", info.Bucket) + v.Add("key", info.Name) + v.Add("etag", "\""+info.ETag+"\"") + redirectURL.RawQuery = v.Encode() + expectedLocation := redirectURL.String() + + // Check the new location url + if rec.Header().Get("Location") != expectedLocation { + t.Errorf("Unexpected location, expected = %s, found = `%s`", rec.Header().Get("Location"), expectedLocation) + } +} + +// postPresignSignatureV4 - presigned signature for PostPolicy requests. +func postPresignSignatureV4(policyBase64 string, t time.Time, secretAccessKey, location string) string { + // Get signining key. + signingkey := getSigningKey(secretAccessKey, t, location, "s3") + // Calculate signature. + signature := getSignature(signingkey, policyBase64) + return signature +} + +func newPostRequestV2(endPoint, bucketName, objectName string, secretKey string, formInputData map[string]string) (*http.Request, error) { + // Expire the request five minutes from now. + expirationTime := UTCNow().Add(time.Minute * 5) + // Create a new post policy. + policy := newPostPolicyBytesV2(bucketName, objectName, expirationTime) + // Only need the encoding. + encodedPolicy := base64.StdEncoding.EncodeToString(policy) + + // Presign with V4 signature based on the policy. + signature := calculateSignatureV2(encodedPolicy, secretKey) + + formData := map[string]string{ + "bucket": bucketName, + "key": objectName + "/${filename}", + "policy": encodedPolicy, + "signature": signature, + } + + maps.Copy(formData, formInputData) + + // Create the multipart form. + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + + // Set the normal formData + for k, v := range formData { + w.WriteField(k, v) + } + // Set the File formData + writer, err := w.CreateFormFile("file", "upload.txt") + if err != nil { + // return nil, err + return nil, err + } + writer.Write([]byte("hello world")) + // Close before creating the new request. + w.Close() + + // Set the body equal to the created policy. + reader := bytes.NewReader(buf.Bytes()) + + req, err := http.NewRequest(http.MethodPost, makeTestTargetURL(endPoint, bucketName, "", nil), reader) + if err != nil { + return nil, err + } + + // Set form content-type. + req.Header.Set("Content-Type", w.FormDataContentType()) + return req, nil +} + +func buildGenericPolicy(t time.Time, accessKey, region, bucketName, objectName string, contentLengthRange bool) []byte { + // Expire the request five minutes from now. + expirationTime := t.Add(time.Minute * 5) + + credStr := getCredentialString(accessKey, region, t) + // Create a new post policy. + policy := newPostPolicyBytesV4(credStr, bucketName, objectName, expirationTime) + if contentLengthRange { + policy = newPostPolicyBytesV4WithContentRange(credStr, bucketName, objectName, expirationTime) + } + return policy +} + +func newPostRequestV4Generic(endPoint, bucketName, objectName string, objData []byte, accessKey, secretKey string, region string, + t time.Time, policy []byte, addFormData map[string]string, noFilename bool, corruptedB64 bool, corruptedMultipart bool, +) (*http.Request, error) { + // Get the user credential. + credStr := getCredentialString(accessKey, region, t) + + // Only need the encoding. + encodedPolicy := base64.StdEncoding.EncodeToString(policy) + + if corruptedB64 { + encodedPolicy = "%!~&" + encodedPolicy + } + + // Presign with V4 signature based on the policy. + signature := postPresignSignatureV4(encodedPolicy, t, secretKey, region) + + // If there is no filename on multipart, get the filename from the key. + key := objectName + if noFilename { + key += "/upload.txt" + } else { + key += "/${filename}" + } + + formData := map[string]string{ + "bucket": bucketName, + "key": key, + "x-amz-credential": credStr, + "policy": encodedPolicy, + "x-amz-signature": signature, + "x-amz-date": t.Format(iso8601DateFormat), + "x-amz-algorithm": "AWS4-HMAC-SHA256", + "x-amz-meta-uuid": "1234", + "Content-Encoding": "gzip", + } + + // Add form data + maps.Copy(formData, addFormData) + + // Create the multipart form. + var buf bytes.Buffer + w := multipart.NewWriter(&buf) + + // Set the normal formData + for k, v := range formData { + w.WriteField(k, v) + } + // Set the File formData but don't if we want send an incomplete multipart request + if !corruptedMultipart { + var writer io.Writer + var err error + if noFilename { + writer, err = w.CreateFormField("file") + } else { + writer, err = w.CreateFormFile("file", "upload.txt") + } + if err != nil { + // return nil, err + return nil, err + } + writer.Write(objData) + // Close before creating the new request. + w.Close() + } + + // Set the body equal to the created policy. + reader := bytes.NewReader(buf.Bytes()) + + req, err := http.NewRequest(http.MethodPost, makeTestTargetURL(endPoint, bucketName, "", nil), reader) + if err != nil { + return nil, err + } + + // Set form content-type. + req.Header.Set("Content-Type", w.FormDataContentType()) + return req, nil +} + +func newPostRequestV4WithContentLength(endPoint, bucketName, objectName string, objData []byte, accessKey, secretKey string) (*http.Request, error) { + t := UTCNow() + region := "us-east-1" + policy := buildGenericPolicy(t, accessKey, region, bucketName, objectName, true) + return newPostRequestV4Generic(endPoint, bucketName, objectName, objData, accessKey, secretKey, region, t, policy, nil, false, false, false) +} + +func newPostRequestV4(endPoint, bucketName, objectName string, objData []byte, accessKey, secretKey string) (*http.Request, error) { + t := UTCNow() + region := "us-east-1" + policy := buildGenericPolicy(t, accessKey, region, bucketName, objectName, false) + return newPostRequestV4Generic(endPoint, bucketName, objectName, objData, accessKey, secretKey, region, t, policy, nil, false, false, false) +} diff --git a/cmd/postpolicyform.go b/cmd/postpolicyform.go new file mode 100644 index 00000000..e9b616b7 --- /dev/null +++ b/cmd/postpolicyform.go @@ -0,0 +1,357 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "reflect" + "strconv" + "strings" + "time" + + "github.com/minio/minio-go/v7/pkg/encrypt" + "github.com/minio/minio-go/v7/pkg/set" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/s3select/jstream" +) + +// startWithConds - map which indicates if a given condition supports starts-with policy operator +var startsWithConds = map[string]bool{ + "$acl": true, + "$bucket": false, + "$cache-control": true, + "$content-type": true, + "$content-disposition": true, + "$content-encoding": true, + "$expires": true, + "$key": true, + "$success_action_redirect": true, + "$redirect": true, + "$success_action_status": true, + "$x-amz-algorithm": false, + "$x-amz-credential": false, + "$x-amz-date": false, +} + +// Add policy conditionals. +const ( + policyCondEqual = "eq" + policyCondStartsWith = "starts-with" + policyCondContentLength = "content-length-range" +) + +// toString - Safely convert interface to string without causing panic. +func toString(val any) string { + switch v := val.(type) { + case string: + return v + default: + return "" + } +} + +// toLowerString - safely convert interface to lower string +func toLowerString(val any) string { + return strings.ToLower(toString(val)) +} + +// toInteger _ Safely convert interface to integer without causing panic. +func toInteger(val any) (int64, error) { + switch v := val.(type) { + case float64: + return int64(v), nil + case int64: + return v, nil + case int: + return int64(v), nil + case string: + i, err := strconv.Atoi(v) + return int64(i), err + default: + return 0, errors.New("Invalid number format") + } +} + +// isString - Safely check if val is of type string without causing panic. +func isString(val any) bool { + _, ok := val.(string) + return ok +} + +// ContentLengthRange - policy content-length-range field. +type contentLengthRange struct { + Min int64 + Max int64 + Valid bool // If content-length-range was part of policy +} + +// PostPolicyForm provides strict static type conversion and validation for Amazon S3's POST policy JSON string. +type PostPolicyForm struct { + Expiration time.Time // Expiration date and time of the POST policy. + Conditions struct { // Conditional policy structure. + Policies []struct { + Operator string + Key string + Value string + } + ContentLengthRange contentLengthRange + } +} + +// implemented to ensure that duplicate keys in JSON +// are merged together into a single JSON key, also +// to remove any extraneous JSON bodies. +// +// Go stdlib doesn't support parsing JSON with duplicate +// keys, so we need to use this technique to merge the +// keys. +func sanitizePolicy(r io.Reader) (io.Reader, error) { + var buf bytes.Buffer + e := json.NewEncoder(&buf) + d := jstream.NewDecoder(r, 0).ObjectAsKVS().MaxDepth(10) + sset := set.NewStringSet() + for mv := range d.Stream() { + if mv.ValueType == jstream.Object { + // This is a JSON object type (that preserves key order) + kvs, ok := mv.Value.(jstream.KVS) + if ok { + for _, kv := range kvs { + if sset.Contains(kv.Key) { + // Reject duplicate conditions or expiration. + return nil, fmt.Errorf("input policy has multiple %s, please fix your client code", kv.Key) + } + sset.Add(kv.Key) + } + } + e.Encode(kvs) + } + } + return &buf, d.Err() +} + +// parsePostPolicyForm - Parse JSON policy string into typed PostPolicyForm structure. +func parsePostPolicyForm(r io.Reader) (PostPolicyForm, error) { + reader, err := sanitizePolicy(r) + if err != nil { + return PostPolicyForm{}, err + } + + d := json.NewDecoder(reader) + + // Convert po into interfaces and + // perform strict type conversion using reflection. + var rawPolicy struct { + Expiration string `json:"expiration"` + Conditions []any `json:"conditions"` + } + + d.DisallowUnknownFields() + if err := d.Decode(&rawPolicy); err != nil { + return PostPolicyForm{}, err + } + + parsedPolicy := PostPolicyForm{} + + // Parse expiry time. + parsedPolicy.Expiration, err = time.Parse(time.RFC3339Nano, rawPolicy.Expiration) + if err != nil { + return PostPolicyForm{}, err + } + + // Parse conditions. + for _, val := range rawPolicy.Conditions { + switch condt := val.(type) { + case map[string]any: // Handle key:value map types. + for k, v := range condt { + if !isString(v) { // Pre-check value type. + // All values must be of type string. + return parsedPolicy, fmt.Errorf("Unknown type %s of conditional field value %s found in POST policy form", reflect.TypeOf(condt).String(), condt) + } + // {"acl": "public-read" } is an alternate way to indicate - [ "eq", "$acl", "public-read" ] + // In this case we will just collapse this into "eq" for all use cases. + parsedPolicy.Conditions.Policies = append(parsedPolicy.Conditions.Policies, struct { + Operator string + Key string + Value string + }{ + policyCondEqual, "$" + strings.ToLower(k), toString(v), + }) + } + case []any: // Handle array types. + if len(condt) != 3 { // Return error if we have insufficient elements. + return parsedPolicy, fmt.Errorf("Malformed conditional fields %s of type %s found in POST policy form", condt, reflect.TypeOf(condt).String()) + } + switch toLowerString(condt[0]) { + case policyCondEqual, policyCondStartsWith: + for _, v := range condt { // Pre-check all values for type. + if !isString(v) { + // All values must be of type string. + return parsedPolicy, fmt.Errorf("Unknown type %s of conditional field value %s found in POST policy form", reflect.TypeOf(condt).String(), condt) + } + } + operator, matchType, value := toLowerString(condt[0]), toLowerString(condt[1]), toString(condt[2]) + if !strings.HasPrefix(matchType, "$") { + return parsedPolicy, fmt.Errorf("Invalid according to Policy: Policy Condition failed: [%s, %s, %s]", operator, matchType, value) + } + parsedPolicy.Conditions.Policies = append(parsedPolicy.Conditions.Policies, struct { + Operator string + Key string + Value string + }{ + operator, matchType, value, + }) + case policyCondContentLength: + minLen, err := toInteger(condt[1]) + if err != nil { + return parsedPolicy, err + } + + maxLen, err := toInteger(condt[2]) + if err != nil { + return parsedPolicy, err + } + + parsedPolicy.Conditions.ContentLengthRange = contentLengthRange{ + Min: minLen, + Max: maxLen, + Valid: true, + } + default: + // Condition should be valid. + return parsedPolicy, fmt.Errorf("Unknown type %s of conditional field value %s found in POST policy form", + reflect.TypeOf(condt).String(), condt) + } + default: + return parsedPolicy, fmt.Errorf("Unknown field %s of type %s found in POST policy form", + condt, reflect.TypeOf(condt).String()) + } + } + return parsedPolicy, nil +} + +// checkPolicyCond returns a boolean to indicate if a condition is satisfied according +// to the passed operator +func checkPolicyCond(op string, input1, input2 string) bool { + switch op { + case policyCondEqual: + return input1 == input2 + case policyCondStartsWith: + return strings.HasPrefix(input1, input2) + } + return false +} + +// S3 docs: "Each form field that you specify in a form (except x-amz-signature, file, policy, and field names +// that have an x-ignore- prefix) must appear in the list of conditions." +// https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html +// keyInPolicyExceptions - list of keys that, when present in the form, can be missing in the conditions of the policy. +var keyInPolicyExceptions = map[string]bool{ + xhttp.AmzSignature: true, + "File": true, + "Policy": true, + + // MinIO specific exceptions to the general S3 rule above. + encrypt.SseKmsKeyID: true, + encrypt.SseEncryptionContext: true, + encrypt.SseCustomerAlgorithm: true, + encrypt.SseCustomerKey: true, + encrypt.SseCustomerKeyMD5: true, +} + +// checkPostPolicy - apply policy conditions and validate input values. +// Note that content-length-range is checked in the API handler function PostPolicyBucketHandler. +// formValues is the already-canonicalized form values from the POST request. +func checkPostPolicy(formValues http.Header, postPolicyForm PostPolicyForm) error { + // Check if policy document expiry date is still not reached + if !postPolicyForm.Expiration.After(UTCNow()) { + return fmt.Errorf("Invalid according to Policy: Policy expired") + } + + // mustFindInPolicy is a map to list all the keys that we must find in the policy as + // we process it below. At the end of checkPostPolicy function, if any key is left in + // this map, that's an error. + mustFindInPolicy := make(map[string][]string, len(formValues)) + for key, values := range formValues { + if keyInPolicyExceptions[key] || strings.HasPrefix(key, "X-Ignore-") { + continue + } + mustFindInPolicy[key] = values + } + + // Iterate over policy conditions and check them against received form fields + for _, policy := range postPolicyForm.Conditions.Policies { + // Form fields names are in canonical format, convert conditions names + // to canonical for simplification purpose, so `$key` will become `Key` + formCanonicalName := http.CanonicalHeaderKey(strings.TrimPrefix(policy.Key, "$")) + + // Operator for the current policy condition + op := policy.Operator + + // Multiple values are not allowed for a single form field + if len(mustFindInPolicy[formCanonicalName]) >= 2 { + return fmt.Errorf("Invalid according to Policy: Policy Condition failed: [%s, %s, %s]. FormValues have multiple values: [%s]", op, policy.Key, policy.Value, strings.Join(mustFindInPolicy[formCanonicalName], ", ")) + } + + // If the current policy condition is known + if startsWithSupported, condFound := startsWithConds[policy.Key]; condFound { + // Check if the current condition supports starts-with operator + if op == policyCondStartsWith && !startsWithSupported { + return fmt.Errorf("Invalid according to Policy: Policy Condition failed") + } + // Check if current policy condition is satisfied + if !checkPolicyCond(op, formValues.Get(formCanonicalName), policy.Value) { + return fmt.Errorf("Invalid according to Policy: Policy Condition failed") + } + } else if strings.HasPrefix(policy.Key, "$x-amz-meta-") || strings.HasPrefix(policy.Key, "$x-amz-") { + // This covers all conditions X-Amz-Meta-* and X-Amz-* + // Check if policy condition is satisfied + if !checkPolicyCond(op, formValues.Get(formCanonicalName), policy.Value) { + return fmt.Errorf("Invalid according to Policy: Policy Condition failed: [%s, %s, %s]", op, policy.Key, policy.Value) + } + } + delete(mustFindInPolicy, formCanonicalName) + } + + // For SignV2 - Signature/AWSAccessKeyId fields do not need to be in the policy + if _, ok := formValues[xhttp.AmzSignatureV2]; ok { + delete(mustFindInPolicy, xhttp.AmzSignatureV2) + for k := range mustFindInPolicy { + // case-insensitivity for AWSAccessKeyId + if strings.EqualFold(k, xhttp.AmzAccessKeyID) { + delete(mustFindInPolicy, k) + break + } + } + } + + // Check mustFindInPolicy to see if any key is left, if so, it was not found in policy and we return an error. + if len(mustFindInPolicy) != 0 { + logKeys := make([]string, 0, len(mustFindInPolicy)) + for key := range mustFindInPolicy { + logKeys = append(logKeys, key) + } + return fmt.Errorf("Each form field that you specify in a form must appear in the list of policy conditions. %q not specified in the policy.", strings.Join(logKeys, ", ")) + } + + return nil +} diff --git a/cmd/postpolicyform_test.go b/cmd/postpolicyform_test.go new file mode 100644 index 00000000..721718bc --- /dev/null +++ b/cmd/postpolicyform_test.go @@ -0,0 +1,298 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "encoding/base64" + "net/http" + "strings" + "testing" + + minio "github.com/minio/minio-go/v7" + xhttp "github.com/minio/minio/internal/http" +) + +func TestParsePostPolicyForm(t *testing.T) { + testCases := []struct { + policy string + success bool + }{ + // missing expiration, will fail. + { + policy: `{"conditions":[["eq","$bucket","asdf"],["eq","$key","hello.txt"]],"conditions":[["eq","$success_action_status","201"],["eq","$Content-Type","plain/text"],["eq","$success_action_status","201"],["eq","$x-amz-algorithm","AWS4-HMAC-SHA256"],["eq","$x-amz-credential","Q3AM3UQ867SPQQA43P2F/20210315/us-east-1/s3/aws4_request"],["eq","$x-amz-date","20210315T091621Z"]]}`, + success: false, + }, + // invalid json. + { + policy: `{"conditions":[["eq","$bucket","asdf"],["eq","$key","hello.txt"]],"conditions":[["eq","$success_action_status","201"],["eq","$Content-Type","plain/text"],["eq","$success_action_status","201"],["eq","$x-amz-algorithm","AWS4-HMAC-SHA256"],["eq","$x-amz-credential","Q3AM3UQ867SPQQA43P2F/20210315/us-east-1/s3/aws4_request"],["eq","$x-amz-date","20210315T091621Z"]]`, + success: false, + }, + // duplicate 'expiration' reject + { + policy: `{"expiration":"2021-03-22T09:16:21.310Z","expiration":"2021-03-22T09:16:21.310Z","conditions":[["eq","$bucket","evil"],["eq","$key","hello.txt"],["eq","$success_action_status","201"],["eq","$Content-Type","plain/text"],["eq","$success_action_status","201"],["eq","$x-amz-algorithm","AWS4-HMAC-SHA256"],["eq","$x-amz-credential","Q3AM3UQ867SPQQA43P2F/20210315/us-east-1/s3/aws4_request"],["eq","$x-amz-date","20210315T091621Z"]]}`, + }, + // duplicate '$bucket' reject + { + policy: `{"expiration":"2021-03-22T09:16:21.310Z","conditions":[["eq","$bucket","good"],["eq","$key","hello.txt"]],"conditions":[["eq","$bucket","evil"],["eq","$key","hello.txt"],["eq","$success_action_status","201"],["eq","$Content-Type","plain/text"],["eq","$success_action_status","201"],["eq","$x-amz-algorithm","AWS4-HMAC-SHA256"],["eq","$x-amz-credential","Q3AM3UQ867SPQQA43P2F/20210315/us-east-1/s3/aws4_request"],["eq","$x-amz-date","20210315T091621Z"]]}`, + success: false, + }, + // duplicate conditions, reject + { + policy: `{"expiration":"2021-03-22T09:16:21.310Z","conditions":[["eq","$bucket","asdf"],["eq","$key","hello.txt"]],"conditions":[["eq","$success_action_status","201"],["eq","$Content-Type","plain/text"],["eq","$success_action_status","201"],["eq","$x-amz-algorithm","AWS4-HMAC-SHA256"],["eq","$x-amz-credential","Q3AM3UQ867SPQQA43P2F/20210315/us-east-1/s3/aws4_request"],["eq","$x-amz-date","20210315T091621Z"]]}`, + success: false, + }, + // no duplicates, shall be parsed properly. + { + policy: `{"expiration":"2021-03-27T20:35:28.458Z","conditions":[["eq","$bucket","testbucket"],["eq","$key","wtf.txt"],["eq","$x-amz-date","20210320T203528Z"],["eq","$x-amz-algorithm","AWS4-HMAC-SHA256"],["eq","$x-amz-credential","Q3AM3UQ867SPQQA43P2F/20210320/us-east-1/s3/aws4_request"]]}`, + success: true, + }, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + _, err := parsePostPolicyForm(strings.NewReader(testCase.policy)) + if testCase.success && err != nil { + t.Errorf("Expected success but failed with %s", err) + } + if !testCase.success && err == nil { + t.Errorf("Expected failed but succeeded") + } + }) + } +} + +type formValues struct { + http.Header +} + +func newFormValues() formValues { + return formValues{make(http.Header)} +} + +func (f formValues) Set(key, value string) formValues { + f.Header.Set(key, value) + return f +} + +func (f formValues) Add(key, value string) formValues { + f.Header.Add(key, value) + return f +} + +func (f formValues) Clone() formValues { + return formValues{f.Header.Clone()} +} + +// Test Post Policy parsing and checking conditions +func TestPostPolicyForm(t *testing.T) { + pp := minio.NewPostPolicy() + pp.SetBucket("testbucket") + pp.SetContentType("image/jpeg") + pp.SetUserMetadata("uuid", "14365123651274") + pp.SetKeyStartsWith("user/user1/filename") + pp.SetContentLengthRange(100, 999999) // not testable from this layer, condition is checked in the API handler. + pp.SetSuccessStatusAction("201") + pp.SetCondition("eq", "X-Amz-Credential", "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request") + pp.SetCondition("eq", "X-Amz-Algorithm", "AWS4-HMAC-SHA256") + pp.SetCondition("eq", xhttp.AmzDate, "20160727T000000Z") + + defaultFormVals := newFormValues() + defaultFormVals.Set("Bucket", "testbucket") + defaultFormVals.Set("Content-Type", "image/jpeg") + defaultFormVals.Set(xhttp.AmzMetaUUID, "14365123651274") + defaultFormVals.Set("Key", "user/user1/filename/${filename}/myfile.txt") + defaultFormVals.Set("X-Amz-Credential", "KVGKMDUQ23TCZXTLTHLP/20160727/us-east-1/s3/aws4_request") + defaultFormVals.Set("X-Amz-Algorithm", "AWS4-HMAC-SHA256") + defaultFormVals.Set(xhttp.AmzDate, "20160727T000000Z") + defaultFormVals.Set("Success_action_status", "201") + + policyCondFailedErr := "Invalid according to Policy: Policy Condition failed" + + type testCase struct { + name string + fv formValues + expired bool + wantErr string + } + + // Test case just contains fields we override from defaultFormVals. + testCases := []testCase{ + { + name: "happy path no errors", + fv: defaultFormVals.Clone(), + wantErr: "", + }, + { + name: "expired policy document", + fv: defaultFormVals.Clone(), + expired: true, + wantErr: "Invalid according to Policy: Policy expired", + }, + { + name: "different AMZ date", + fv: defaultFormVals.Clone().Set(xhttp.AmzDate, "2017T000000Z"), + wantErr: policyCondFailedErr, + }, + { + name: "incorrect date", + fv: defaultFormVals.Clone().Set(xhttp.AmzDate, "incorrect"), + wantErr: policyCondFailedErr, + }, + { + name: "key which doesn't start with user/user1/filename", + fv: defaultFormVals.Clone().Set("Key", "myfile.txt"), + wantErr: policyCondFailedErr, + }, + { + name: "incorrect key name", + fv: defaultFormVals.Clone().Set("Key", "incorrect"), + wantErr: policyCondFailedErr, + }, + { + name: "incorrect bucket name", + fv: defaultFormVals.Clone().Set("Bucket", "incorrect"), + wantErr: policyCondFailedErr, + }, + { + name: "incorrect ContentType", + fv: defaultFormVals.Clone().Set(xhttp.ContentType, "incorrect"), + wantErr: policyCondFailedErr, + }, + { + name: "incorrect X-Amz-Algorithm", + fv: defaultFormVals.Clone().Set(xhttp.AmzAlgorithm, "incorrect"), + wantErr: policyCondFailedErr, + }, + { + name: "incorrect X-Amz-Credential", + fv: defaultFormVals.Clone().Set(xhttp.AmzCredential, "incorrect"), + wantErr: policyCondFailedErr, + }, + { + name: "incorrect metadata uuid", + fv: defaultFormVals.Clone().Set(xhttp.AmzMetaUUID, "151274"), + wantErr: "Invalid according to Policy: Policy Condition failed: [eq, $x-amz-meta-uuid, 14365123651274]", + }, + { + name: "unknown key XAmzMetaName is error as it does not appear in policy", + fv: defaultFormVals.Clone().Set(xhttp.AmzMetaName, "my-name"), + wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "X-Amz-Meta-Name" not specified in the policy.`, + }, + { + name: "unknown key XAmzChecksumAlgo is error as it does not appear in policy", + fv: defaultFormVals.Clone().Set(http.CanonicalHeaderKey(xhttp.AmzChecksumAlgo), "algo-val"), + wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "X-Amz-Checksum-Algorithm" not specified in the policy.`, + }, + { + name: "unknown key XAmzChecksumCRC32 is error as it does not appear in policy", + fv: defaultFormVals.Clone().Set(http.CanonicalHeaderKey(xhttp.AmzChecksumCRC32), "crc32-val"), + wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "X-Amz-Checksum-Crc32" not specified in the policy.`, + }, + { + name: "unknown key XAmzChecksumCRC32C is error as it does not appear in policy", + fv: defaultFormVals.Clone().Set(http.CanonicalHeaderKey(xhttp.AmzChecksumCRC32C), "crc32c-val"), + wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "X-Amz-Checksum-Crc32c" not specified in the policy.`, + }, + { + name: "unknown key XAmzChecksumSHA1 is error as it does not appear in policy", + fv: defaultFormVals.Clone().Set(http.CanonicalHeaderKey(xhttp.AmzChecksumSHA1), "sha1-val"), + wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "X-Amz-Checksum-Sha1" not specified in the policy.`, + }, + { + name: "unknown key XAmzChecksumSHA256 is error as it does not appear in policy", + fv: defaultFormVals.Clone().Set(http.CanonicalHeaderKey(xhttp.AmzChecksumSHA256), "sha256-val"), + wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "X-Amz-Checksum-Sha256" not specified in the policy.`, + }, + { + name: "unknown key XAmzChecksumMode is error as it does not appear in policy", + fv: defaultFormVals.Clone().Set(http.CanonicalHeaderKey(xhttp.AmzChecksumMode), "mode-val"), + wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "X-Amz-Checksum-Mode" not specified in the policy.`, + }, + { + name: "unknown key Content-Encoding is error as it does not appear in policy", + fv: defaultFormVals.Clone().Set(http.CanonicalHeaderKey(xhttp.ContentEncoding), "encoding-val"), + wantErr: `Each form field that you specify in a form must appear in the list of policy conditions. "Content-Encoding" not specified in the policy.`, + }, + { + name: "many bucket values", + fv: defaultFormVals.Clone().Add("Bucket", "anotherbucket"), + wantErr: "Invalid according to Policy: Policy Condition failed: [eq, $bucket, testbucket]. FormValues have multiple values: [testbucket, anotherbucket]", + }, + { + name: "XAmzSignature does not have to appear in policy", + fv: defaultFormVals.Clone().Set(xhttp.AmzSignature, "my-signature"), + }, + { + name: "XIgnoreFoo does not have to appear in policy", + fv: defaultFormVals.Clone().Set("X-Ignore-Foo", "my-foo-value"), + }, + { + name: "File does not have to appear in policy", + fv: defaultFormVals.Clone().Set("File", "file-value"), + }, + { + name: "Signature does not have to appear in policy", + fv: defaultFormVals.Clone().Set(xhttp.AmzSignatureV2, "signature-value"), + }, + { + name: "AWSAccessKeyID does not have to appear in policy", + fv: defaultFormVals.Clone().Set(xhttp.AmzAccessKeyID, "access").Set(xhttp.AmzSignatureV2, "signature-value"), + }, + { + name: "any form value starting with X-Amz-Server-Side-Encryption- does not have to appear in policy", + fv: defaultFormVals.Clone(). + Set(xhttp.AmzServerSideEncryptionKmsContext, "context-val"). + Set(xhttp.AmzServerSideEncryptionCustomerAlgorithm, "algo-val"), + }, + } + + // Run tests + for _, tt := range testCases { + t.Run(tt.name, func(t *testing.T) { + if tt.expired { + // Expired already. + pp.SetExpires(UTCNow().AddDate(0, 0, -10)) + } else { + // Expires in 10 days. + pp.SetExpires(UTCNow().AddDate(0, 0, 10)) + } + + tt.fv.Set("Policy", base64.StdEncoding.EncodeToString([]byte(pp.String()))) + + policyBytes, err := base64.StdEncoding.DecodeString(base64.StdEncoding.EncodeToString([]byte(pp.String()))) + if err != nil { + t.Fatal(err) + } + + postPolicyForm, err := parsePostPolicyForm(bytes.NewReader(policyBytes)) + if err != nil { + t.Fatal(err) + } + + errStr := "" + err = checkPostPolicy(tt.fv.Header, postPolicyForm) + if err != nil { + errStr = err.Error() + } + if errStr != tt.wantErr { + t.Errorf("test: '%s', want error: '%s', got error: '%s'", tt.name, tt.wantErr, errStr) + } + }) + } +} diff --git a/cmd/prepare-storage.go b/cmd/prepare-storage.go new file mode 100644 index 00000000..578a5fad --- /dev/null +++ b/cmd/prepare-storage.go @@ -0,0 +1,327 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "net/http" + "net/url" + "path/filepath" + "sync" + "time" + + "github.com/dustin/go-humanize" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" +) + +var printEndpointError = func() func(Endpoint, error, bool) { + var mutex sync.Mutex + printOnce := make(map[Endpoint]map[string]int) + + return func(endpoint Endpoint, err error, once bool) { + reqInfo := (&logger.ReqInfo{}).AppendTags("endpoint", endpoint.String()) + ctx := logger.SetReqInfo(GlobalContext, reqInfo) + mutex.Lock() + defer mutex.Unlock() + + m, ok := printOnce[endpoint] + if !ok { + m = make(map[string]int) + printOnce[endpoint] = m + if once { + m[err.Error()]++ + peersLogAlwaysIf(ctx, err) + return + } + } + // Once is set and we are here means error was already + // printed once. + if once { + return + } + // once not set, check if same error occurred 3 times in + // a row, then make sure we print it to call attention. + if m[err.Error()] > 2 { + peersLogAlwaysIf(ctx, fmt.Errorf("Following error has been printed %d times.. %w", m[err.Error()], err)) + // Reduce the count to introduce further delay in printing + // but let it again print after the 2th attempt + m[err.Error()]-- + m[err.Error()]-- + } + m[err.Error()]++ + } +}() + +// Cleans up tmp directory of the local disk. +func bgFormatErasureCleanupTmp(diskPath string) { + // Need to move temporary objects left behind from previous run of minio + // server to a unique directory under `minioMetaTmpBucket-old` to clean + // up `minioMetaTmpBucket` for the current run. + // + // /disk1/.minio.sys/tmp-old/ + // |__ 33a58b40-aecc-4c9f-a22f-ff17bfa33b62 + // |__ e870a2c1-d09c-450c-a69c-6eaa54a89b3e + // + // In this example, `33a58b40-aecc-4c9f-a22f-ff17bfa33b62` directory contains + // temporary objects from one of the previous runs of minio server. + tmpID := mustGetUUID() + tmpOld := pathJoin(diskPath, minioMetaTmpBucket+"-old", tmpID) + if err := renameAll(pathJoin(diskPath, minioMetaTmpBucket), + tmpOld, diskPath); err != nil && !errors.Is(err, errFileNotFound) { + storageLogIf(GlobalContext, fmt.Errorf("unable to rename (%s -> %s) %w, drive may be faulty, please investigate", + pathJoin(diskPath, minioMetaTmpBucket), + tmpOld, + osErrToFileErr(err))) + } + + if err := mkdirAll(pathJoin(diskPath, minioMetaTmpDeletedBucket), 0o777, diskPath); err != nil { + storageLogIf(GlobalContext, fmt.Errorf("unable to create (%s) %w, drive may be faulty, please investigate", + pathJoin(diskPath, minioMetaTmpBucket), + err)) + } + + // Delete all temporary files created for DirectIO write check + files, _ := filepath.Glob(filepath.Join(diskPath, ".writable-check-*.tmp")) + for _, file := range files { + go removeAll(file) + } + + // Remove the entire folder in case there are leftovers that didn't get cleaned up before restart. + go removeAll(pathJoin(diskPath, minioMetaTmpBucket+"-old")) + + // Renames and schedules for purging all bucket metacache. + go renameAllBucketMetacache(diskPath) +} + +// Following error message is added to fix a regression in release +// RELEASE.2018-03-16T22-52-12Z after migrating v1 to v2 to v3. This +// migration failed to capture '.This' field properly which indicates +// the disk UUID association. Below error message is returned when +// we see this situation in format.json, for more info refer +// https://github.com/minio/minio/issues/5667 +var errErasureV3ThisEmpty = fmt.Errorf("Erasure format version 3 has This field empty") + +// isServerResolvable - checks if the endpoint is resolvable +// by sending a naked HTTP request with liveness checks. +func isServerResolvable(endpoint Endpoint, timeout time.Duration) error { + serverURL := &url.URL{ + Scheme: endpoint.Scheme, + Host: endpoint.Host, + Path: pathJoin(healthCheckPathPrefix, healthCheckLivenessPath), + } + + httpClient := &http.Client{ + Transport: globalInternodeTransport, + } + + ctx, cancel := context.WithTimeout(GlobalContext, timeout) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodGet, serverURL.String(), nil) + if err != nil { + return err + } + // Indicate that the liveness check for a peer call + req.Header.Set(xhttp.MinIOPeerCall, "true") + + resp, err := httpClient.Do(req) + if err != nil { + return err + } + xhttp.DrainBody(resp.Body) + + return nil +} + +// connect to list of endpoints and load all Erasure disk formats, validate the formats are correct +// and are in quorum, if no formats are found attempt to initialize all of them for the first +// time. additionally make sure to close all the disks used in this attempt. +func connectLoadInitFormats(verboseLogging bool, firstDisk bool, storageDisks []StorageAPI, endpoints Endpoints, poolCount, setCount, setDriveCount int, deploymentID string) (format *formatErasureV3, err error) { + // Attempt to load all `format.json` from all disks. + formatConfigs, sErrs := loadFormatErasureAll(storageDisks, false) + + if err := checkDiskFatalErrs(sErrs); err != nil { + return nil, err + } + + for i, err := range sErrs { + if err != nil && !errors.Is(err, errXLBackend) && !errors.Is(err, errUnformattedDisk) { + if errors.Is(err, errDiskNotFound) && verboseLogging { + if globalEndpoints.NEndpoints() > 1 { + logger.Info("Unable to connect to %s: %v, will be retried", endpoints[i], isServerResolvable(endpoints[i], time.Second)) + } else { + logger.Fatal(err, "Unable to connect to %s: %v", endpoints[i], isServerResolvable(endpoints[i], time.Second)) + } + } else { + if globalEndpoints.NEndpoints() > 1 { + logger.Info("Unable to use the drive %s: %v, will be retried", endpoints[i], err) + } else { + logger.Fatal(errInvalidArgument, "Unable to use the drive %s: %v", endpoints[i], err) + } + } + } + } + + // Pre-emptively check if one of the formatted disks + // is invalid. This function returns success for the + // most part unless one of the formats is not consistent + // with expected Erasure format. For example if a user is + // trying to pool FS backend into an Erasure set. + if err = checkFormatErasureValues(formatConfigs, storageDisks, setDriveCount); err != nil { + return nil, err + } + + // All disks report unformatted we should initialized everyone. + if shouldInitErasureDisks(sErrs) && firstDisk { + logger.Info("Formatting %s pool, %v set(s), %v drives per set.", + humanize.Ordinal(poolCount), setCount, setDriveCount) + + // Initialize erasure code format on disks + format, err = initFormatErasure(GlobalContext, storageDisks, setCount, setDriveCount, deploymentID, sErrs) + if err != nil { + return nil, err + } + + return format, nil + } + + // Return error when quorum unformatted disks - indicating we are + // waiting for first server to be online. + unformattedDisks := quorumUnformattedDisks(sErrs) + if unformattedDisks && !firstDisk { + return nil, errNotFirstDisk + } + + // Return error when quorum unformatted disks but waiting for rest + // of the servers to be online. + if unformattedDisks && firstDisk { + return nil, errFirstDiskWait + } + + format, err = getFormatErasureInQuorum(formatConfigs) + if err != nil { + var drivesNotFound int + for _, format := range formatConfigs { + if format != nil { + continue + } + drivesNotFound++ + } + return nil, fmt.Errorf("%w (offline-drives=%d/%d)", err, drivesNotFound, len(formatConfigs)) + } + + if format.ID == "" { + return nil, errors.New("deployment ID missing from disk format, unable to start the server") + } + + return format, nil +} + +// Format disks before initialization of object layer. +func waitForFormatErasure(firstDisk bool, endpoints Endpoints, poolCount, setCount, setDriveCount int, deploymentID string) (storageDisks []StorageAPI, format *formatErasureV3, err error) { + if len(endpoints) == 0 || setCount == 0 || setDriveCount == 0 { + return nil, nil, errInvalidArgument + } + + // prepare getElapsedTime() to calculate elapsed time since we started trying formatting disks. + // All times are rounded to avoid showing milli, micro and nano seconds + formatStartTime := time.Now().Round(time.Second) + getElapsedTime := func() string { + return time.Now().Round(time.Second).Sub(formatStartTime).String() + } + + var ( + tries int + verbose bool + ) + + // Initialize all storage disks + storageDisks, errs := initStorageDisksWithErrors(endpoints, storageOpts{cleanUp: true, healthCheck: true}) + + if err := checkDiskFatalErrs(errs); err != nil { + return nil, nil, err + } + + defer func() { + if err == nil && format != nil { + // Assign globalDeploymentID() on first run for the + // minio server managing the first disk + globalDeploymentIDPtr.Store(&format.ID) + + // Set the deployment ID here to avoid races. + xhttp.SetDeploymentID(format.ID) + xhttp.SetMinIOVersion(Version) + } + }() + + format, err = connectLoadInitFormats(verbose, firstDisk, storageDisks, endpoints, poolCount, setCount, setDriveCount, deploymentID) + if err == nil { + return storageDisks, format, nil + } + + tries++ // tried already once + + // Wait on each try for an update. + ticker := time.NewTicker(1 * time.Second) + defer ticker.Stop() + + for { + // Only log once every 10 iterations, then reset the tries count. + verbose = tries >= 10 + if verbose { + tries = 1 + } + + format, err = connectLoadInitFormats(verbose, firstDisk, storageDisks, endpoints, poolCount, setCount, setDriveCount, deploymentID) + if err == nil { + return storageDisks, format, nil + } + + tries++ + switch { + case errors.Is(err, errNotFirstDisk): + // Fresh setup, wait for first server to be up. + logger.Info("Waiting for the first server to format the drives (elapsed %s)\n", getElapsedTime()) + case errors.Is(err, errFirstDiskWait): + // Fresh setup, wait for other servers to come up. + logger.Info("Waiting for all other servers to be online to format the drives (elapses %s)\n", getElapsedTime()) + case errors.Is(err, errErasureReadQuorum): + // no quorum available continue to wait for minimum number of servers. + logger.Info("Waiting for a minimum of %d drives to come online (elapsed %s)\n", + len(endpoints)/2, getElapsedTime()) + case errors.Is(err, errErasureWriteQuorum): + // no quorum available continue to wait for minimum number of servers. + logger.Info("Waiting for a minimum of %d drives to come online (elapsed %s)\n", + (len(endpoints)/2)+1, getElapsedTime()) + case errors.Is(err, errErasureV3ThisEmpty): + // need to wait for this error to be healed, so continue. + default: + // For all other unhandled errors we exit and fail. + return nil, nil, err + } + + select { + case <-ticker.C: + case <-globalOSSignalCh: + return nil, nil, fmt.Errorf("Initializing data volumes gracefully stopped") + } + } +} diff --git a/cmd/rebalance-admin.go b/cmd/rebalance-admin.go new file mode 100644 index 00000000..4fa372ce --- /dev/null +++ b/cmd/rebalance-admin.go @@ -0,0 +1,113 @@ +// Copyright (c) 2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "time" +) + +type rebalPoolProgress struct { + NumObjects uint64 `json:"objects"` + NumVersions uint64 `json:"versions"` + Bytes uint64 `json:"bytes"` + Bucket string `json:"bucket"` + Object string `json:"object"` + Elapsed time.Duration `json:"elapsed"` + ETA time.Duration `json:"eta"` +} + +type rebalancePoolStatus struct { + ID int `json:"id"` // Pool index (zero-based) + Status string `json:"status"` // Active if rebalance is running, empty otherwise + Used float64 `json:"used"` // Percentage used space + Progress rebalPoolProgress `json:"progress"` // is empty when rebalance is not running +} + +// rebalanceAdminStatus holds rebalance status related information exported to mc, console, etc. +type rebalanceAdminStatus struct { + ID string // identifies the ongoing rebalance operation by a uuid + Pools []rebalancePoolStatus `json:"pools"` // contains all pools, including inactive + StoppedAt time.Time `json:"stoppedAt"` +} + +func rebalanceStatus(ctx context.Context, z *erasureServerPools) (r rebalanceAdminStatus, err error) { + // Load latest rebalance status + meta := &rebalanceMeta{} + err = meta.load(ctx, z.serverPools[0]) + if err != nil { + return r, err + } + + // Compute disk usage percentage + si := z.StorageInfo(ctx, true) + diskStats := make([]struct { + AvailableSpace uint64 + TotalSpace uint64 + }, len(z.serverPools)) + for _, disk := range si.Disks { + // Ignore invalid. + if disk.PoolIndex < 0 || len(diskStats) <= disk.PoolIndex { + // https://github.com/minio/minio/issues/16500 + continue + } + diskStats[disk.PoolIndex].AvailableSpace += disk.AvailableSpace + diskStats[disk.PoolIndex].TotalSpace += disk.TotalSpace + } + + stopTime := meta.StoppedAt + r = rebalanceAdminStatus{ + ID: meta.ID, + StoppedAt: meta.StoppedAt, + Pools: make([]rebalancePoolStatus, len(meta.PoolStats)), + } + for i, ps := range meta.PoolStats { + r.Pools[i] = rebalancePoolStatus{ + ID: i, + Status: ps.Info.Status.String(), + Used: float64(diskStats[i].TotalSpace-diskStats[i].AvailableSpace) / float64(diskStats[i].TotalSpace), + } + if !ps.Participating { + continue + } + // for participating pools, total bytes to be rebalanced by this pool is given by, + // pf_c = (f_i + x)/c_i, + // pf_c - percentage free space across pools, f_i - ith pool's free space, c_i - ith pool's capacity + // i.e. x = c_i*pfc -f_i + totalBytesToRebal := float64(ps.InitCapacity)*meta.PercentFreeGoal - float64(ps.InitFreeSpace) + elapsed := time.Since(ps.Info.StartTime) + eta := time.Duration(totalBytesToRebal * float64(elapsed) / float64(ps.Bytes)) + if !ps.Info.EndTime.IsZero() { + stopTime = ps.Info.EndTime + } + + if !stopTime.IsZero() { // rebalance is stopped or completed + elapsed = stopTime.Sub(ps.Info.StartTime) + eta = 0 + } + + r.Pools[i].Progress = rebalPoolProgress{ + NumObjects: ps.NumObjects, + NumVersions: ps.NumVersions, + Bytes: ps.Bytes, + Elapsed: elapsed, + ETA: eta, + } + } + return r, nil +} diff --git a/cmd/rebalancemetric_string.go b/cmd/rebalancemetric_string.go new file mode 100644 index 00000000..930e0434 --- /dev/null +++ b/cmd/rebalancemetric_string.go @@ -0,0 +1,27 @@ +// Code generated by "stringer -type=rebalanceMetric -trimprefix=rebalanceMetric erasure-server-pool-rebalance.go"; DO NOT EDIT. + +package cmd + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[rebalanceMetricRebalanceBuckets-0] + _ = x[rebalanceMetricRebalanceBucket-1] + _ = x[rebalanceMetricRebalanceObject-2] + _ = x[rebalanceMetricRebalanceRemoveObject-3] + _ = x[rebalanceMetricSaveMetadata-4] +} + +const _rebalanceMetric_name = "RebalanceBucketsRebalanceBucketRebalanceObjectRebalanceRemoveObjectSaveMetadata" + +var _rebalanceMetric_index = [...]uint8{0, 16, 31, 46, 67, 79} + +func (i rebalanceMetric) String() string { + if i >= rebalanceMetric(len(_rebalanceMetric_index)-1) { + return "rebalanceMetric(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _rebalanceMetric_name[_rebalanceMetric_index[i]:_rebalanceMetric_index[i+1]] +} diff --git a/cmd/rebalstatus_string.go b/cmd/rebalstatus_string.go new file mode 100644 index 00000000..0dc74b21 --- /dev/null +++ b/cmd/rebalstatus_string.go @@ -0,0 +1,27 @@ +// Code generated by "stringer -type=rebalStatus -trimprefix=rebal erasure-server-pool-rebalance.go"; DO NOT EDIT. + +package cmd + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[rebalNone-0] + _ = x[rebalStarted-1] + _ = x[rebalCompleted-2] + _ = x[rebalStopped-3] + _ = x[rebalFailed-4] +} + +const _rebalStatus_name = "NoneStartedCompletedStoppedFailed" + +var _rebalStatus_index = [...]uint8{0, 4, 11, 20, 27, 33} + +func (i rebalStatus) String() string { + if i >= rebalStatus(len(_rebalStatus_index)-1) { + return "rebalStatus(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _rebalStatus_name[_rebalStatus_index[i]:_rebalStatus_index[i+1]] +} diff --git a/cmd/routers.go b/cmd/routers.go new file mode 100644 index 00000000..c0bfc91d --- /dev/null +++ b/cmd/routers.go @@ -0,0 +1,115 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "net/http" + + "github.com/minio/minio/internal/grid" + "github.com/minio/mux" +) + +// Composed function registering routers for only distributed Erasure setup. +func registerDistErasureRouters(router *mux.Router, endpointServerPools EndpointServerPools) { + var ( + lockGrid = globalLockGrid.Load() + commonGrid = globalGrid.Load() + ) + + // Register storage REST router only if its a distributed setup. + registerStorageRESTHandlers(router, endpointServerPools, commonGrid) + + // Register peer REST router only if its a distributed setup. + registerPeerRESTHandlers(router, commonGrid) + + // Register bootstrap REST router for distributed setups. + registerBootstrapRESTHandlers(commonGrid) + + // Register distributed namespace lock routers. + registerLockRESTHandlers(lockGrid) + + // Add lock grid to router + router.Handle(grid.RouteLockPath, adminMiddleware(lockGrid.Handler(storageServerRequestValidate), noGZFlag, noObjLayerFlag)) + + // Add grid to router + router.Handle(grid.RoutePath, adminMiddleware(commonGrid.Handler(storageServerRequestValidate), noGZFlag, noObjLayerFlag)) +} + +// List of some generic middlewares which are applied for all incoming requests. +var globalMiddlewares = []mux.MiddlewareFunc{ + // set x-amz-request-id header and others + addCustomHeadersMiddleware, + // The generic tracer needs to be the first middleware to catch all requests + // returned early by any other middleware (but after the middleware that + // sets the amz request id). + httpTracerMiddleware, + // Auth middleware verifies incoming authorization headers and routes them + // accordingly. Client receives a HTTP error for invalid/unsupported + // signatures. + // + // Validates all incoming requests to have a valid date header. + setAuthMiddleware, + // Redirect some pre-defined browser request paths to a static location + // prefix. + setBrowserRedirectMiddleware, + // Adds 'crossdomain.xml' policy middleware to serve legacy flash clients. + setCrossDomainPolicyMiddleware, + // Limits all body and header sizes to a maximum fixed limit + setRequestLimitMiddleware, + // Validate all the incoming requests. + setRequestValidityMiddleware, + // Add upload forwarding middleware for site replication + setUploadForwardingMiddleware, + // Add bucket forwarding middleware + setBucketForwardingMiddleware, + // Add new middlewares here. +} + +// configureServer handler returns final handler for the http server. +func configureServerHandler(endpointServerPools EndpointServerPools) (http.Handler, error) { + // Initialize router. `SkipClean(true)` stops minio/mux from + // normalizing URL path minio/minio#3256 + router := mux.NewRouter().SkipClean(true).UseEncodedPath() + + // Initialize distributed NS lock. + if globalIsDistErasure { + registerDistErasureRouters(router, endpointServerPools) + } + + // Add Admin router, all APIs are enabled in server mode. + registerAdminRouter(router, true) + + // Add healthCheck router + registerHealthCheckRouter(router) + + // Add server metrics router + registerMetricsRouter(router) + + // Add STS router always. + registerSTSRouter(router) + + // Add KMS router + registerKMSRouter(router) + + // Add API router + registerAPIRouter(router) + + router.Use(globalMiddlewares...) + + return router, nil +} diff --git a/cmd/s3-zip-handlers.go b/cmd/s3-zip-handlers.go new file mode 100644 index 00000000..e2c91226 --- /dev/null +++ b/cmd/s3-zip-handlers.go @@ -0,0 +1,520 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "errors" + "io" + "mime" + "net/http" + "path/filepath" + "sort" + "strings" + + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/crypto" + xhttp "github.com/minio/minio/internal/http" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/pkg/v3/policy" + "github.com/minio/zipindex" +) + +const ( + archiveType = "zip" + archiveTypeEnc = "zip-enc" + archiveExt = "." + archiveType // ".zip" + archiveSeparator = "/" + archivePattern = archiveExt + archiveSeparator // ".zip/" + archiveTypeMetadataKey = ReservedMetadataPrefixLower + "archive-type" // "x-minio-internal-archive-type" + archiveInfoMetadataKey = ReservedMetadataPrefixLower + "archive-info" // "x-minio-internal-archive-info" + + // Peek into a zip archive + xMinIOExtract = "x-minio-extract" +) + +// splitZipExtensionPath splits the S3 path to the zip file and the path inside the zip: +// +// e.g /path/to/archive.zip/backup-2021/myimage.png => /path/to/archive.zip, backup/myimage.png +func splitZipExtensionPath(input string) (zipPath, object string, err error) { + idx := strings.Index(input, archivePattern) + if idx < 0 { + // Should never happen + return "", "", errors.New("unable to parse zip path") + } + return input[:idx+len(archivePattern)-1], input[idx+len(archivePattern):], nil +} + +// getObjectInArchiveFileHandler - GET Object in the archive file +func (api objectAPIHandlers) getObjectInArchiveFileHandler(ctx context.Context, objectAPI ObjectLayer, bucket, object string, w http.ResponseWriter, r *http.Request) { + if crypto.S3.IsRequested(r.Header) || crypto.S3KMS.IsRequested(r.Header) { // If SSE-S3 or SSE-KMS present -> AWS fails with undefined error + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrBadRequest), r.URL) + return + } + + zipPath, object, err := splitZipExtensionPath(object) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + opts, err := getOpts(ctx, r, bucket, zipPath) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + getObjectInfo := objectAPI.GetObjectInfo + + // Check for auth type to return S3 compatible error. + // type to return the correct error (NoSuchKey vs AccessDenied) + if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, bucket, zipPath); s3Error != ErrNone { + if getRequestAuthType(r) == authTypeAnonymous { + // As per "Permission" section in + // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html + // If the object you request does not exist, + // the error Amazon S3 returns depends on + // whether you also have the s3:ListBucket + // permission. + // * If you have the s3:ListBucket permission + // on the bucket, Amazon S3 will return an + // HTTP status code 404 ("no such key") + // error. + // * if you don’t have the s3:ListBucket + // permission, Amazon S3 will return an HTTP + // status code 403 ("access denied") error.` + if globalPolicySys.IsAllowed(policy.BucketPolicyArgs{ + Action: policy.ListBucketAction, + BucketName: bucket, + ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials), + IsOwner: false, + }) { + _, err = getObjectInfo(ctx, bucket, zipPath, opts) + if toAPIError(ctx, err).Code == "NoSuchKey" { + s3Error = ErrNoSuchKey + } + } + } + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(s3Error), r.URL) + return + } + + // We do not allow offsetting into extracted files. + if opts.PartNumber != 0 { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidPartNumber), r.URL) + return + } + + if r.Header.Get(xhttp.Range) != "" { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrInvalidRange), r.URL) + return + } + + // Validate pre-conditions if any. + opts.CheckPrecondFn = func(oi ObjectInfo) bool { + if _, err := DecryptObjectInfo(&oi, r); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return true + } + + return checkPreconditions(ctx, w, r, oi, opts) + } + + zipObjInfo, err := getObjectInfo(ctx, bucket, zipPath, opts) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + zipInfo := zipObjInfo.ArchiveInfo(r.Header) + if len(zipInfo) == 0 { + opts.EncryptFn, err = zipObjInfo.metadataEncryptFn(r.Header) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + + zipInfo, err = updateObjectMetadataWithZipInfo(ctx, objectAPI, bucket, zipPath, opts) + } + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + file, err := zipindex.FindSerialized(zipInfo, object) + if err != nil { + if err == io.EOF { + writeErrorResponse(ctx, w, errorCodes.ToAPIErr(ErrNoSuchKey), r.URL) + } else { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + } + return + } + + // New object info + fileObjInfo := ObjectInfo{ + Bucket: bucket, + Name: object, + Size: int64(file.UncompressedSize64), + ModTime: zipObjInfo.ModTime, + ContentType: mime.TypeByExtension(filepath.Ext(object)), + } + + var rc io.ReadCloser + + if file.UncompressedSize64 > 0 { + // There may be number of header bytes before the content. + // Reading 64K extra. This should more than cover name and any "extra" details. + end := min(file.Offset+int64(file.CompressedSize64)+64<<10, zipObjInfo.Size) + rs := &HTTPRangeSpec{Start: file.Offset, End: end} + gr, err := objectAPI.GetObjectNInfo(ctx, bucket, zipPath, rs, nil, opts) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + defer gr.Close() + rc, err = file.Open(gr) + if err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + } else { + rc = io.NopCloser(bytes.NewReader([]byte{})) + } + + defer rc.Close() + + if err = setObjectHeaders(ctx, w, fileObjInfo, nil, opts); err != nil { + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + // s3zip does not allow ranges + w.Header().Del(xhttp.AcceptRanges) + + setHeadGetRespHeaders(w, r.Form) + + httpWriter := xioutil.WriteOnClose(w) + + // Write object content to response body + if _, err = xioutil.Copy(httpWriter, rc); err != nil { + if !httpWriter.HasWritten() { + // write error response only if no data or headers has been written to client yet + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + return + } + + if err = httpWriter.Close(); err != nil { + if !httpWriter.HasWritten() { // write error response only if no data or headers has been written to client yet + writeErrorResponse(ctx, w, toAPIError(ctx, err), r.URL) + return + } + return + } +} + +// listObjectsV2InArchive generates S3 listing result ListObjectsV2Info from zip file, all parameters are already validated by the caller. +func listObjectsV2InArchive(ctx context.Context, objectAPI ObjectLayer, bucket, prefix, token, delimiter string, maxKeys int, startAfter string, h http.Header) (ListObjectsV2Info, error) { + zipPath, _, err := splitZipExtensionPath(prefix) + if err != nil { + // Return empty listing + return ListObjectsV2Info{}, nil + } + + zipObjInfo, err := objectAPI.GetObjectInfo(ctx, bucket, zipPath, ObjectOptions{}) + if err != nil { + // Return empty listing + return ListObjectsV2Info{}, nil + } + + zipInfo := zipObjInfo.ArchiveInfo(h) + if len(zipInfo) == 0 { + // Always update the latest version + zipInfo, err = updateObjectMetadataWithZipInfo(ctx, objectAPI, bucket, zipPath, ObjectOptions{}) + } + if err != nil { + return ListObjectsV2Info{}, err + } + + files, err := zipindex.DeserializeFiles(zipInfo) + if err != nil { + return ListObjectsV2Info{}, err + } + + sort.Slice(files, func(i, j int) bool { + return files[i].Name < files[j].Name + }) + + var ( + count int + isTruncated bool + nextToken string + listObjectsInfo ListObjectsV2Info + ) + + // Always set this + listObjectsInfo.ContinuationToken = token + + // Open and iterate through the files in the archive. + for _, file := range files { + objName := zipObjInfo.Name + archiveSeparator + file.Name + if objName <= startAfter || objName <= token { + continue + } + if strings.HasPrefix(objName, prefix) { + if count == maxKeys { + isTruncated = true + break + } + if delimiter != "" { + i := strings.Index(objName[len(prefix):], delimiter) + if i >= 0 { + commonPrefix := objName[:len(prefix)+i+1] + if len(listObjectsInfo.Prefixes) == 0 || commonPrefix != listObjectsInfo.Prefixes[len(listObjectsInfo.Prefixes)-1] { + listObjectsInfo.Prefixes = append(listObjectsInfo.Prefixes, commonPrefix) + count++ + } + goto next + } + } + listObjectsInfo.Objects = append(listObjectsInfo.Objects, ObjectInfo{ + Bucket: bucket, + Name: objName, + Size: int64(file.UncompressedSize64), + ModTime: zipObjInfo.ModTime, + }) + count++ + } + next: + nextToken = objName + } + + if isTruncated { + listObjectsInfo.IsTruncated = true + listObjectsInfo.NextContinuationToken = nextToken + } + + return listObjectsInfo, nil +} + +// getFilesFromZIPObject reads a partial stream of a zip file to build the zipindex.Files index +func getFilesListFromZIPObject(ctx context.Context, objectAPI ObjectLayer, bucket, object string, opts ObjectOptions) (zipindex.Files, ObjectInfo, error) { + size := 1 << 20 + var objSize int64 + for { + rs := &HTTPRangeSpec{IsSuffixLength: true, Start: int64(-size)} + gr, err := objectAPI.GetObjectNInfo(ctx, bucket, object, rs, nil, opts) + if err != nil { + return nil, ObjectInfo{}, err + } + b, err := io.ReadAll(gr) + gr.Close() + if err != nil { + return nil, ObjectInfo{}, err + } + if size > len(b) { + size = len(b) + } + + // Calculate the object real size if encrypted + if _, ok := crypto.IsEncrypted(gr.ObjInfo.UserDefined); ok { + objSize, err = gr.ObjInfo.DecryptedSize() + if err != nil { + return nil, ObjectInfo{}, err + } + } else { + objSize = gr.ObjInfo.Size + } + + files, err := zipindex.ReadDir(b[len(b)-size:], objSize, nil) + if err == nil { + return files, gr.ObjInfo, nil + } + var terr zipindex.ErrNeedMoreData + if errors.As(err, &terr) { + size = int(terr.FromEnd) + if size <= 0 || size > 100<<20 { + return nil, ObjectInfo{}, errors.New("zip directory too large") + } + } else { + return nil, ObjectInfo{}, err + } + } +} + +// headObjectInArchiveFileHandler - HEAD Object in an archive file +func (api objectAPIHandlers) headObjectInArchiveFileHandler(ctx context.Context, objectAPI ObjectLayer, bucket, object string, w http.ResponseWriter, r *http.Request) { + if crypto.S3.IsRequested(r.Header) || crypto.S3KMS.IsRequested(r.Header) { // If SSE-S3 or SSE-KMS present -> AWS fails with undefined error + writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrBadRequest)) + return + } + + zipPath, object, err := splitZipExtensionPath(object) + if err != nil { + writeErrorResponseHeadersOnly(w, toAPIError(ctx, err)) + return + } + + getObjectInfo := objectAPI.GetObjectInfo + + opts, err := getOpts(ctx, r, bucket, zipPath) + if err != nil { + writeErrorResponseHeadersOnly(w, toAPIError(ctx, err)) + return + } + + if s3Error := checkRequestAuthType(ctx, r, policy.GetObjectAction, bucket, zipPath); s3Error != ErrNone { + if getRequestAuthType(r) == authTypeAnonymous { + // As per "Permission" section in + // https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectHEAD.html + // If the object you request does not exist, + // the error Amazon S3 returns depends on + // whether you also have the s3:ListBucket + // permission. + // * If you have the s3:ListBucket permission + // on the bucket, Amazon S3 will return an + // HTTP status code 404 ("no such key") + // error. + // * if you don’t have the s3:ListBucket + // permission, Amazon S3 will return an HTTP + // status code 403 ("access denied") error.` + if globalPolicySys.IsAllowed(policy.BucketPolicyArgs{ + Action: policy.ListBucketAction, + BucketName: bucket, + ConditionValues: getConditionValues(r, "", auth.AnonymousCredentials), + IsOwner: false, + }) { + _, err = getObjectInfo(ctx, bucket, zipPath, opts) + if toAPIError(ctx, err).Code == "NoSuchKey" { + s3Error = ErrNoSuchKey + } + } + } + errCode := errorCodes.ToAPIErr(s3Error) + w.Header().Set(xMinIOErrCodeHeader, errCode.Code) + w.Header().Set(xMinIOErrDescHeader, "\""+errCode.Description+"\"") + writeErrorResponseHeadersOnly(w, errCode) + return + } + + // Validate pre-conditions if any. + opts.CheckPrecondFn = func(oi ObjectInfo) bool { + return checkPreconditions(ctx, w, r, oi, opts) + } + + // We do not allow offsetting into extracted files. + if opts.PartNumber != 0 { + writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrInvalidPartNumber)) + return + } + + if r.Header.Get(xhttp.Range) != "" { + writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrInvalidRange)) + return + } + + zipObjInfo, err := getObjectInfo(ctx, bucket, zipPath, opts) + if err != nil { + writeErrorResponseHeadersOnly(w, toAPIError(ctx, err)) + return + } + + zipInfo := zipObjInfo.ArchiveInfo(r.Header) + if len(zipInfo) == 0 { + opts.EncryptFn, err = zipObjInfo.metadataEncryptFn(r.Header) + if err != nil { + writeErrorResponseHeadersOnly(w, toAPIError(ctx, err)) + return + } + zipInfo, err = updateObjectMetadataWithZipInfo(ctx, objectAPI, bucket, zipPath, opts) + } + if err != nil { + writeErrorResponseHeadersOnly(w, toAPIError(ctx, err)) + return + } + + file, err := zipindex.FindSerialized(zipInfo, object) + if err != nil { + if err == io.EOF { + writeErrorResponseHeadersOnly(w, errorCodes.ToAPIErr(ErrNoSuchKey)) + } else { + writeErrorResponseHeadersOnly(w, toAPIError(ctx, err)) + } + return + } + + objInfo := ObjectInfo{ + Bucket: bucket, + Name: file.Name, + Size: int64(file.UncompressedSize64), + ModTime: zipObjInfo.ModTime, + } + + // Set standard object headers. + if err = setObjectHeaders(ctx, w, objInfo, nil, opts); err != nil { + writeErrorResponseHeadersOnly(w, toAPIError(ctx, err)) + return + } + + // s3zip does not allow ranges. + w.Header().Del(xhttp.AcceptRanges) + + // Set any additional requested response headers. + setHeadGetRespHeaders(w, r.Form) + + // Successful response. + w.WriteHeader(http.StatusOK) +} + +// Update the passed zip object metadata with the zip contents info, file name, modtime, size, etc. +// The returned zip index will de decrypted. +func updateObjectMetadataWithZipInfo(ctx context.Context, objectAPI ObjectLayer, bucket, object string, opts ObjectOptions) ([]byte, error) { + files, srcInfo, err := getFilesListFromZIPObject(ctx, objectAPI, bucket, object, opts) + if err != nil { + return nil, err + } + files.OptimizeSize() + zipInfo, err := files.Serialize() + if err != nil { + return nil, err + } + at := archiveType + zipInfoStr := string(zipInfo) + if opts.EncryptFn != nil { + at = archiveTypeEnc + zipInfoStr = string(opts.EncryptFn(archiveTypeEnc, zipInfo)) + } + srcInfo.UserDefined[archiveTypeMetadataKey] = at + popts := ObjectOptions{ + MTime: srcInfo.ModTime, + VersionID: srcInfo.VersionID, + EvalMetadataFn: func(oi *ObjectInfo, gerr error) (dsc ReplicateDecision, err error) { + oi.UserDefined[archiveTypeMetadataKey] = at + oi.UserDefined[archiveInfoMetadataKey] = zipInfoStr + return dsc, nil + }, + } + + // For all other modes use in-place update to update metadata on a specific version. + if _, err = objectAPI.PutObjectMetadata(ctx, bucket, object, popts); err != nil { + return nil, err + } + + return zipInfo, nil +} diff --git a/cmd/scannermetric_string.go b/cmd/scannermetric_string.go new file mode 100644 index 00000000..32c7a4f8 --- /dev/null +++ b/cmd/scannermetric_string.go @@ -0,0 +1,44 @@ +// Code generated by "stringer -type=scannerMetric -trimprefix=scannerMetric data-scanner-metric.go"; DO NOT EDIT. + +package cmd + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[scannerMetricReadMetadata-0] + _ = x[scannerMetricCheckMissing-1] + _ = x[scannerMetricSaveUsage-2] + _ = x[scannerMetricApplyAll-3] + _ = x[scannerMetricApplyVersion-4] + _ = x[scannerMetricTierObjSweep-5] + _ = x[scannerMetricHealCheck-6] + _ = x[scannerMetricILM-7] + _ = x[scannerMetricCheckReplication-8] + _ = x[scannerMetricYield-9] + _ = x[scannerMetricCleanAbandoned-10] + _ = x[scannerMetricApplyNonCurrent-11] + _ = x[scannerMetricHealAbandonedVersion-12] + _ = x[scannerMetricStartTrace-13] + _ = x[scannerMetricScanObject-14] + _ = x[scannerMetricHealAbandonedObject-15] + _ = x[scannerMetricLastRealtime-16] + _ = x[scannerMetricScanFolder-17] + _ = x[scannerMetricScanCycle-18] + _ = x[scannerMetricScanBucketDrive-19] + _ = x[scannerMetricCompactFolder-20] + _ = x[scannerMetricLast-21] +} + +const _scannerMetric_name = "ReadMetadataCheckMissingSaveUsageApplyAllApplyVersionTierObjSweepHealCheckILMCheckReplicationYieldCleanAbandonedApplyNonCurrentHealAbandonedVersionStartTraceScanObjectHealAbandonedObjectLastRealtimeScanFolderScanCycleScanBucketDriveCompactFolderLast" + +var _scannerMetric_index = [...]uint8{0, 12, 24, 33, 41, 53, 65, 74, 77, 93, 98, 112, 127, 147, 157, 167, 186, 198, 208, 217, 232, 245, 249} + +func (i scannerMetric) String() string { + if i >= scannerMetric(len(_scannerMetric_index)-1) { + return "scannerMetric(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _scannerMetric_name[_scannerMetric_index[i]:_scannerMetric_index[i+1]] +} diff --git a/cmd/server-main.go b/cmd/server-main.go new file mode 100644 index 00000000..ade86cca --- /dev/null +++ b/cmd/server-main.go @@ -0,0 +1,1201 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/hex" + "errors" + "fmt" + "io" + "log" + "math/rand" + "net" + "os" + "os/signal" + "path/filepath" + "runtime" + "slices" + "strings" + "syscall" + "time" + + "github.com/coreos/go-systemd/v22/daemon" + "github.com/dustin/go-humanize" + "github.com/minio/cli" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/bucket/bandwidth" + "github.com/minio/minio/internal/color" + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/config/api" + "github.com/minio/minio/internal/handlers" + "github.com/minio/minio/internal/hash/sha256" + xhttp "github.com/minio/minio/internal/http" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/certs" + "github.com/minio/pkg/v3/env" + "gopkg.in/yaml.v2" +) + +// ServerFlags - server command specific flags +var ServerFlags = []cli.Flag{ + cli.StringFlag{ + Name: "config", + Usage: "specify server configuration via YAML configuration", + EnvVar: "MINIO_CONFIG", + }, + cli.StringFlag{ + Name: "address", + Value: ":" + GlobalMinioDefaultPort, + Usage: "bind to a specific ADDRESS:PORT, ADDRESS can be an IP or hostname", + EnvVar: "MINIO_ADDRESS", + }, + cli.IntFlag{ + Name: "listeners", // Deprecated Oct 2022 + Value: 1, + Usage: "bind N number of listeners per ADDRESS:PORT", + EnvVar: "MINIO_LISTENERS", + Hidden: true, + }, + cli.StringFlag{ + Name: "console-address", + Usage: "bind to a specific ADDRESS:PORT for embedded Console UI, ADDRESS can be an IP or hostname", + EnvVar: "MINIO_CONSOLE_ADDRESS", + }, + cli.DurationFlag{ + Name: "shutdown-timeout", + Value: time.Second * 30, + Usage: "shutdown timeout to gracefully shutdown server (DEPRECATED)", + EnvVar: "MINIO_SHUTDOWN_TIMEOUT", + Hidden: true, + }, + + cli.DurationFlag{ + Name: "idle-timeout", + Value: xhttp.DefaultIdleTimeout, + Usage: "idle timeout is the maximum amount of time to wait for the next request when keep-alive are enabled", + EnvVar: "MINIO_IDLE_TIMEOUT", + Hidden: true, + }, + cli.DurationFlag{ + Name: "read-header-timeout", + Value: xhttp.DefaultReadHeaderTimeout, + Usage: "read header timeout is the amount of time allowed to read request headers", + EnvVar: "MINIO_READ_HEADER_TIMEOUT", + Hidden: true, + }, + cli.DurationFlag{ + Name: "conn-user-timeout", + Usage: "custom TCP_USER_TIMEOUT for socket buffers", + Hidden: true, + Value: 10 * time.Minute, + EnvVar: "MINIO_CONN_USER_TIMEOUT", + }, + cli.StringFlag{ + Name: "interface", + Usage: "bind to right VRF device for MinIO services", + Hidden: true, + EnvVar: "MINIO_INTERFACE", + }, + cli.DurationFlag{ + Name: "dns-cache-ttl", + Usage: "custom DNS cache TTL", + Hidden: true, + Value: func() time.Duration { + if orchestrated { + return 30 * time.Second + } + return 10 * time.Minute + }(), + EnvVar: "MINIO_DNS_CACHE_TTL", + }, + cli.IntFlag{ + Name: "max-idle-conns-per-host", + Usage: "set a custom max idle connections per host value", + Hidden: true, + Value: 2048, + EnvVar: "MINIO_MAX_IDLE_CONNS_PER_HOST", + }, + cli.StringSliceFlag{ + Name: "ftp", + Usage: "enable and configure an FTP(Secure) server", + }, + cli.StringSliceFlag{ + Name: "sftp", + Usage: "enable and configure an SFTP server", + }, + cli.StringFlag{ + Name: "crossdomain-xml", + Usage: "provide a custom crossdomain-xml configuration to report at http://endpoint/crossdomain.xml", + Hidden: true, + EnvVar: "MINIO_CROSSDOMAIN_XML", + }, + cli.StringFlag{ + Name: "memlimit", + Usage: "set global memory limit per server via GOMEMLIMIT", + Hidden: true, + EnvVar: "MINIO_MEMLIMIT", + }, + cli.IntFlag{ + Name: "send-buf-size", + Value: 4 * humanize.MiByte, + EnvVar: "MINIO_SEND_BUF_SIZE", + Hidden: true, + }, + cli.IntFlag{ + Name: "recv-buf-size", + Value: 4 * humanize.MiByte, + EnvVar: "MINIO_RECV_BUF_SIZE", + Hidden: true, + }, + cli.StringFlag{ + Name: "log-dir", + Usage: "specify the directory to save the server log", + EnvVar: "MINIO_LOG_DIR", + Hidden: true, + }, + cli.IntFlag{ + Name: "log-size", + Usage: "specify the maximum server log file size in bytes before its rotated", + Value: 10 * humanize.MiByte, + EnvVar: "MINIO_LOG_SIZE", + Hidden: true, + }, + cli.BoolFlag{ + Name: "log-compress", + Usage: "specify if we want the rotated logs to be gzip compressed or not", + EnvVar: "MINIO_LOG_COMPRESS", + Hidden: true, + }, + cli.StringFlag{ + Name: "log-prefix", + Usage: "specify the log prefix name for the server log", + EnvVar: "MINIO_LOG_PREFIX", + Hidden: true, + }, +} + +var serverCmd = cli.Command{ + Name: "server", + Usage: "start object storage server", + Flags: append(ServerFlags, GlobalFlags...), + Action: serverMain, + CustomHelpTemplate: `NAME: + {{.HelpName}} - {{.Usage}} + +USAGE: + {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}DIR1 [DIR2..] + {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}DIR{1...64} + {{.HelpName}} {{if .VisibleFlags}}[FLAGS] {{end}}DIR{1...64} DIR{65...128} + +DIR: + DIR points to a directory on a filesystem. When you want to combine + multiple drives into a single large system, pass one directory per + filesystem separated by space. You may also use a '...' convention + to abbreviate the directory arguments. Remote directories in a + distributed setup are encoded as HTTP(s) URIs. +{{if .VisibleFlags}} +FLAGS: + {{range .VisibleFlags}}{{.}} + {{end}}{{end}} +EXAMPLES: + 1. Start MinIO server on "/home/shared" directory. + {{.Prompt}} {{.HelpName}} /home/shared + + 2. Start single node server with 64 local drives "/mnt/data1" to "/mnt/data64". + {{.Prompt}} {{.HelpName}} /mnt/data{1...64} + + 3. Start distributed MinIO server on an 32 node setup with 32 drives each, run following command on all the nodes + {{.Prompt}} {{.HelpName}} http://node{1...32}.example.com/mnt/export{1...32} + + 4. Start distributed MinIO server in an expanded setup, run the following command on all the nodes + {{.Prompt}} {{.HelpName}} http://node{1...16}.example.com/mnt/export{1...32} \ + http://node{17...64}.example.com/mnt/export{1...64} + + 5. Start distributed MinIO server, with FTP and SFTP servers on all interfaces via port 8021, 8022 respectively + {{.Prompt}} {{.HelpName}} http://node{1...4}.example.com/mnt/export{1...4} \ + --ftp="address=:8021" --ftp="passive-port-range=30000-40000" \ + --sftp="address=:8022" --sftp="ssh-private-key=${HOME}/.ssh/id_rsa" +`, +} + +func serverCmdArgs(ctx *cli.Context) []string { + v, _, _, err := env.LookupEnv(config.EnvArgs) + if err != nil { + logger.FatalIf(err, "Unable to validate passed arguments in %s:%s", + config.EnvArgs, os.Getenv(config.EnvArgs)) + } + if v == "" { + v, _, _, err = env.LookupEnv(config.EnvVolumes) + if err != nil { + logger.FatalIf(err, "Unable to validate passed arguments in %s:%s", + config.EnvVolumes, os.Getenv(config.EnvVolumes)) + } + } + if v == "" { + // Fall back to older environment value MINIO_ENDPOINTS + v, _, _, err = env.LookupEnv(config.EnvEndpoints) + if err != nil { + logger.FatalIf(err, "Unable to validate passed arguments in %s:%s", + config.EnvEndpoints, os.Getenv(config.EnvEndpoints)) + } + } + if v == "" { + if !ctx.Args().Present() || ctx.Args().First() == "help" { + cli.ShowCommandHelpAndExit(ctx, ctx.Command.Name, 1) + } + return ctx.Args() + } + return strings.Fields(v) +} + +func configCommonToSrvCtx(cf config.ServerConfigCommon, ctxt *serverCtxt) { + ctxt.RootUser = cf.RootUser + ctxt.RootPwd = cf.RootPwd + + if cf.Addr != "" { + ctxt.Addr = cf.Addr + } + if cf.ConsoleAddr != "" { + ctxt.ConsoleAddr = cf.ConsoleAddr + } + if cf.CertsDir != "" { + ctxt.CertsDir = cf.CertsDir + ctxt.certsDirSet = true + } + + if cf.Options.FTP.Address != "" { + ctxt.FTP = append(ctxt.FTP, fmt.Sprintf("address=%s", cf.Options.FTP.Address)) + } + if cf.Options.FTP.PassivePortRange != "" { + ctxt.FTP = append(ctxt.FTP, fmt.Sprintf("passive-port-range=%s", cf.Options.FTP.PassivePortRange)) + } + + if cf.Options.SFTP.Address != "" { + ctxt.SFTP = append(ctxt.SFTP, fmt.Sprintf("address=%s", cf.Options.SFTP.Address)) + } + if cf.Options.SFTP.SSHPrivateKey != "" { + ctxt.SFTP = append(ctxt.SFTP, fmt.Sprintf("ssh-private-key=%s", cf.Options.SFTP.SSHPrivateKey)) + } +} + +func mergeServerCtxtFromConfigFile(configFile string, ctxt *serverCtxt) error { + rd, err := xioutil.ReadFile(configFile) + if err != nil { + return err + } + + cfReader := bytes.NewReader(rd) + + cv := config.ServerConfigVersion{} + if err = yaml.Unmarshal(rd, &cv); err != nil { + return err + } + + switch cv.Version { + case "v1", "v2": + default: + return fmt.Errorf("unexpected version: %s", cv.Version) + } + + cfCommon := config.ServerConfigCommon{} + if err = yaml.Unmarshal(rd, &cfCommon); err != nil { + return err + } + + configCommonToSrvCtx(cfCommon, ctxt) + + v, err := env.GetInt(EnvErasureSetDriveCount, 0) + if err != nil { + return err + } + setDriveCount := uint64(v) + + var pools []poolArgs + switch cv.Version { + case "v1": + cfV1 := config.ServerConfigV1{} + if err = yaml.Unmarshal(rd, &cfV1); err != nil { + return err + } + + pools = make([]poolArgs, 0, len(cfV1.Pools)) + for _, list := range cfV1.Pools { + pools = append(pools, poolArgs{ + args: list, + setDriveCount: setDriveCount, + }) + } + case "v2": + cf := config.ServerConfig{} + cfReader.Seek(0, io.SeekStart) + if err = yaml.Unmarshal(rd, &cf); err != nil { + return err + } + + pools = make([]poolArgs, 0, len(cf.Pools)) + for _, list := range cf.Pools { + driveCount := list.SetDriveCount + if setDriveCount > 0 { + driveCount = setDriveCount + } + pools = append(pools, poolArgs{ + args: list.Args, + setDriveCount: driveCount, + }) + } + } + ctxt.Layout, err = buildDisksLayoutFromConfFile(pools) + return err +} + +func serverHandleCmdArgs(ctxt serverCtxt) { + handleCommonArgs(ctxt) + + logger.FatalIf(CheckLocalServerAddr(globalMinioAddr), "Unable to validate passed arguments") + + var err error + var setupType SetupType + + // Check and load TLS certificates. + globalPublicCerts, globalTLSCerts, globalIsTLS, err = getTLSConfig() + logger.FatalIf(err, "Unable to load the TLS configuration") + + // Check and load Root CAs. + globalRootCAs, err = certs.GetRootCAs(globalCertsCADir.Get()) + logger.FatalIf(err, "Failed to read root CAs (%v)", err) + + // Add the global public crts as part of global root CAs + for _, publicCrt := range globalPublicCerts { + globalRootCAs.AddCert(publicCrt) + } + + // Register root CAs for remote ENVs + env.RegisterGlobalCAs(globalRootCAs) + + globalEndpoints, setupType, err = createServerEndpoints(globalMinioAddr, ctxt.Layout.pools, ctxt.Layout.legacy) + logger.FatalIf(err, "Invalid command line arguments") + globalNodes = globalEndpoints.GetNodes() + + globalIsErasure = (setupType == ErasureSetupType) + globalIsDistErasure = (setupType == DistErasureSetupType) + if globalIsDistErasure { + globalIsErasure = true + } + globalIsErasureSD = (setupType == ErasureSDSetupType) + if globalDynamicAPIPort && globalIsDistErasure { + logger.FatalIf(errInvalidArgument, "Invalid --address=\"%s\", port '0' is not allowed in a distributed erasure coded setup", ctxt.Addr) + } + + globalLocalNodeName = GetLocalPeer(globalEndpoints, globalMinioHost, globalMinioPort) + nodeNameSum := sha256.Sum256([]byte(globalLocalNodeName)) + globalLocalNodeNameHex = hex.EncodeToString(nodeNameSum[:]) + + // Initialize, see which NIC the service is running on, and save it as global value + setGlobalInternodeInterface(ctxt.Interface) + + globalTCPOptions = xhttp.TCPOptions{ + UserTimeout: int(ctxt.UserTimeout.Milliseconds()), + // FIXME: Bring this back when we have valid way to handle deadlines + // DriveOPTimeout: globalDriveConfig.GetOPTimeout, + Interface: ctxt.Interface, + SendBufSize: ctxt.SendBufSize, + RecvBufSize: ctxt.RecvBufSize, + IdleTimeout: ctxt.IdleTimeout, + } + + // allow transport to be HTTP/1.1 for proxying. + globalInternodeTransport = NewInternodeHTTPTransport(ctxt.MaxIdleConnsPerHost)() + globalRemoteTargetTransport = NewRemoteTargetHTTPTransport(false)() + globalProxyEndpoints = GetProxyEndpoints(globalEndpoints, globalRemoteTargetTransport) + + globalForwarder = handlers.NewForwarder(&handlers.Forwarder{ + PassHost: true, + RoundTripper: globalRemoteTargetTransport, + Logger: func(err error) { + if err != nil && !errors.Is(err, context.Canceled) { + proxyLogIf(GlobalContext, err) + } + }, + }) + + // On macOS, if a process already listens on LOCALIPADDR:PORT, net.Listen() falls back + // to IPv6 address ie minio will start listening on IPv6 address whereas another + // (non-)minio process is listening on IPv4 of given port. + // To avoid this error situation we check for port availability. + logger.FatalIf(xhttp.CheckPortAvailability(globalMinioHost, globalMinioPort, globalTCPOptions), "Unable to start the server") +} + +func initAllSubsystems(ctx context.Context) { + // Initialize notification peer targets + globalNotificationSys = NewNotificationSys(globalEndpoints) + + // Create new notification system + if globalEventNotifier == nil { + globalEventNotifier = NewEventNotifier(GlobalContext) + } + + // Create new bucket metadata system. + if globalBucketMetadataSys == nil { + globalBucketMetadataSys = NewBucketMetadataSys() + } else { + // Reinitialize safely when testing. + globalBucketMetadataSys.Reset() + } + + // Create the bucket bandwidth monitor + globalBucketMonitor = bandwidth.NewMonitor(ctx, uint64(totalNodeCount())) + + // Create a new config system. + globalConfigSys = NewConfigSys() + + // Create new IAM system. + globalIAMSys = NewIAMSys() + + // Create new policy system. + globalPolicySys = NewPolicySys() + + // Create new lifecycle system. + globalLifecycleSys = NewLifecycleSys() + + // Create new bucket encryption subsystem + globalBucketSSEConfigSys = NewBucketSSEConfigSys() + + // Create new bucket object lock subsystem + globalBucketObjectLockSys = NewBucketObjectLockSys() + + // Create new bucket quota subsystem + globalBucketQuotaSys = NewBucketQuotaSys() + + // Create new bucket versioning subsystem + if globalBucketVersioningSys == nil { + globalBucketVersioningSys = NewBucketVersioningSys() + } + + // Create new bucket replication subsystem + globalBucketTargetSys = NewBucketTargetSys(GlobalContext) + + // Create new ILM tier configuration subsystem + globalTierConfigMgr = NewTierConfigMgr() + + globalTransitionState = newTransitionState(GlobalContext) + globalSiteResyncMetrics = newSiteResyncMetrics(GlobalContext) +} + +func configRetriableErrors(err error) bool { + if err == nil { + return false + } + + notInitialized := strings.Contains(err.Error(), "Server not initialized, please try again") || + errors.Is(err, errServerNotInitialized) + + // Initializing sub-systems needs a retry mechanism for + // the following reasons: + // - Read quorum is lost just after the initialization + // of the object layer. + // - Write quorum not met when upgrading configuration + // version is needed, migration is needed etc. + rquorum := InsufficientReadQuorum{} + wquorum := InsufficientWriteQuorum{} + + // One of these retriable errors shall be retried. + return errors.Is(err, errDiskNotFound) || + errors.Is(err, errConfigNotFound) || + errors.Is(err, context.DeadlineExceeded) || + errors.Is(err, errErasureWriteQuorum) || + errors.Is(err, errErasureReadQuorum) || + errors.Is(err, io.ErrUnexpectedEOF) || + errors.As(err, &rquorum) || + errors.As(err, &wquorum) || + isErrObjectNotFound(err) || + isErrBucketNotFound(err) || + errors.Is(err, os.ErrDeadlineExceeded) || + notInitialized +} + +func bootstrapTraceMsg(msg string) { + info := madmin.TraceInfo{ + TraceType: madmin.TraceBootstrap, + Time: UTCNow(), + NodeName: globalLocalNodeName, + FuncName: "BOOTSTRAP", + Message: fmt.Sprintf("%s %s", getSource(2), msg), + } + globalBootstrapTracer.Record(info) + + if serverDebugLog { + fmt.Println(time.Now().Round(time.Millisecond).Format(time.RFC3339), " bootstrap: ", msg) + } + + noSubs := globalTrace.NumSubscribers(madmin.TraceBootstrap) == 0 + if noSubs { + return + } + + globalTrace.Publish(info) +} + +func bootstrapTrace(msg string, worker func()) { + if serverDebugLog { + fmt.Println(time.Now().Round(time.Millisecond).Format(time.RFC3339), " bootstrap: ", msg) + } + + now := time.Now() + worker() + dur := time.Since(now) + + info := madmin.TraceInfo{ + TraceType: madmin.TraceBootstrap, + Time: UTCNow(), + NodeName: globalLocalNodeName, + FuncName: "BOOTSTRAP", + Message: fmt.Sprintf("%s %s (duration: %s)", getSource(2), msg, dur), + } + globalBootstrapTracer.Record(info) + + if globalTrace.NumSubscribers(madmin.TraceBootstrap) == 0 { + return + } + + globalTrace.Publish(info) +} + +func initServerConfig(ctx context.Context, newObject ObjectLayer) error { + t1 := time.Now() + + r := rand.New(rand.NewSource(time.Now().UnixNano())) + + for { + select { + case <-ctx.Done(): + // Retry was canceled successfully. + return fmt.Errorf("Initializing sub-systems stopped gracefully %w", ctx.Err()) + default: + } + + // These messages only meant primarily for distributed setup, so only log during distributed setup. + if globalIsDistErasure { + logger.Info("Waiting for all MinIO sub-systems to be initialize...") + } + + // Upon success migrating the config, initialize all sub-systems + // if all sub-systems initialized successfully return right away + err := initConfigSubsystem(ctx, newObject) + if err == nil { + // All successful return. + if globalIsDistErasure { + // These messages only meant primarily for distributed setup, so only log during distributed setup. + logger.Info("All MinIO sub-systems initialized successfully in %s", time.Since(t1)) + } + return nil + } + + if configRetriableErrors(err) { + logger.Info("Waiting for all MinIO sub-systems to be initialized.. possible cause (%v)", err) + time.Sleep(time.Duration(r.Float64() * float64(5*time.Second))) + continue + } + + // Any other unhandled return right here. + return fmt.Errorf("Unable to initialize sub-systems: %w", err) + } +} + +func initConfigSubsystem(ctx context.Context, newObject ObjectLayer) error { + // %w is used by all error returns here to make sure + // we wrap the underlying error, make sure when you + // are modifying this code that you do so, if and when + // you want to add extra context to your error. This + // ensures top level retry works accordingly. + + // Initialize config system. + if err := globalConfigSys.Init(newObject); err != nil { + if configRetriableErrors(err) { + return fmt.Errorf("Unable to initialize config system: %w", err) + } + + // Any other config errors we simply print a message and proceed forward. + configLogIf(ctx, fmt.Errorf("Unable to initialize config, some features may be missing: %w", err)) + } + + return nil +} + +func setGlobalInternodeInterface(interfaceName string) { + globalInternodeInterfaceOnce.Do(func() { + if interfaceName != "" { + globalInternodeInterface = interfaceName + return + } + ip := "127.0.0.1" + host, _ := mustSplitHostPort(globalLocalNodeName) + if host != "" { + if net.ParseIP(host) != nil { + ip = host + } else { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + haddrs, err := globalDNSCache.LookupHost(ctx, host) + if err == nil { + ip = haddrs[0] + } + } + } + ifs, _ := net.Interfaces() + for _, interf := range ifs { + addrs, err := interf.Addrs() + if err == nil { + for _, addr := range addrs { + if strings.SplitN(addr.String(), "/", 2)[0] == ip { + globalInternodeInterface = interf.Name + } + } + } + } + }) +} + +// Return the list of address that MinIO server needs to listen on: +// - Returning 127.0.0.1 is necessary so Console will be able to send +// requests to the local S3 API. +// - The returned List needs to be deduplicated as well. +func getServerListenAddrs() []string { + // Use a string set to avoid duplication + addrs := set.NewStringSet() + // Listen on local interface to receive requests from Console + for _, ip := range localLoopbacks.ToSlice() { + addrs.Add(net.JoinHostPort(ip, globalMinioPort)) + } + host, _ := mustSplitHostPort(globalMinioAddr) + if host != "" { + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + + haddrs, err := globalDNSCache.LookupHost(ctx, host) + if err == nil { + for _, addr := range haddrs { + addrs.Add(net.JoinHostPort(addr, globalMinioPort)) + } + } else { + // Unable to lookup host in 2-secs, let it fail later anyways. + addrs.Add(globalMinioAddr) + } + } else { + addrs.Add(globalMinioAddr) + } + return addrs.ToSlice() +} + +var globalLoggerOutput io.WriteCloser + +func initializeLogRotate(ctx *cli.Context) (io.WriteCloser, error) { + lgDir := ctx.String("log-dir") + if lgDir == "" { + return os.Stderr, nil + } + lgDirAbs, err := filepath.Abs(lgDir) + if err != nil { + return nil, err + } + lgSize := ctx.Int("log-size") + + var fileNameFunc func() string + if ctx.IsSet("log-prefix") { + fileNameFunc = func() string { + return fmt.Sprintf("%s-%s.log", ctx.String("log-prefix"), fmt.Sprintf("%X", time.Now().UTC().UnixNano())) + } + } + + output, err := logger.NewDir(logger.Options{ + Directory: lgDirAbs, + MaximumFileSize: int64(lgSize), + Compress: ctx.Bool("log-compress"), + FileNameFunc: fileNameFunc, + }) + if err != nil { + return nil, err + } + logger.EnableJSON() + return output, nil +} + +// serverMain handler called for 'minio server' command. +func serverMain(ctx *cli.Context) { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + + var warnings []string + + signal.Notify(globalOSSignalCh, os.Interrupt, syscall.SIGTERM, syscall.SIGQUIT) + + go handleSignals() + + setDefaultProfilerRates() + + // Initialize globalConsoleSys system + bootstrapTrace("newConsoleLogger", func() { + output, err := initializeLogRotate(ctx) + if err == nil { + logger.Output = output + globalConsoleSys = NewConsoleLogger(GlobalContext, output) + globalLoggerOutput = output + } else { + logger.Output = os.Stderr + globalConsoleSys = NewConsoleLogger(GlobalContext, os.Stderr) + } + logger.AddSystemTarget(GlobalContext, globalConsoleSys) + + // Set node name, only set for distributed setup. + globalConsoleSys.SetNodeName(globalLocalNodeName) + if err != nil { + // We can only log here since we need globalConsoleSys initialized + logger.Fatal(err, "invalid --logrorate-dir option") + } + }) + + // Always load ENV variables from files first. + loadEnvVarsFromFiles() + + // Handle early server environment vars + serverHandleEarlyEnvVars() + + // Handle all server command args and build the disks layout + bootstrapTrace("serverHandleCmdArgs", func() { + err := buildServerCtxt(ctx, &globalServerCtxt) + logger.FatalIf(err, "Unable to prepare the list of endpoints") + + serverHandleCmdArgs(globalServerCtxt) + }) + + // DNS cache subsystem to reduce outgoing DNS requests + runDNSCache(ctx) + + // Handle all server environment vars. + serverHandleEnvVars() + + // Perform any self-tests + bootstrapTrace("selftests", func() { + bitrotSelfTest() + erasureSelfTest() + compressSelfTest() + }) + + // Initialize KMS configuration + bootstrapTrace("handleKMSConfig", handleKMSConfig) + + // Load the root credentials from the shell environment or from + // the config file if not defined, set the default one. + bootstrapTrace("rootCredentials", func() { + cred := loadRootCredentials() + if !cred.IsValid() && (env.Get(api.EnvAPIRootAccess, config.EnableOn) == config.EnableOff) { + // Generate KMS based credentials if root access is disabled + // and no ENV is set. + cred = autoGenerateRootCredentials() + } + + if !cred.IsValid() { + cred = auth.DefaultCredentials + } + + var err error + globalNodeAuthToken, err = authenticateNode(cred.AccessKey, cred.SecretKey) + if err != nil { + logger.Fatal(err, "Unable to generate internode credentials") + } + + globalActiveCred = cred + }) + + // Initialize all help + bootstrapTrace("initHelp", initHelp) + + // Initialize all sub-systems + bootstrapTrace("initAllSubsystems", func() { + initAllSubsystems(GlobalContext) + }) + + // Is distributed setup, error out if no certificates are found for HTTPS endpoints. + if globalIsDistErasure { + if globalEndpoints.HTTPS() && !globalIsTLS { + logger.Fatal(config.ErrNoCertsAndHTTPSEndpoints(nil), "Unable to start the server") + } + if !globalEndpoints.HTTPS() && globalIsTLS { + logger.Fatal(config.ErrCertsAndHTTPEndpoints(nil), "Unable to start the server") + } + } + + var getCert certs.GetCertificateFunc + if globalTLSCerts != nil { + getCert = globalTLSCerts.GetCertificate + } + + // Check for updates in non-blocking manner. + go func() { + if !globalServerCtxt.Quiet && !globalInplaceUpdateDisabled { + // Check for new updates from dl.min.io. + bootstrapTrace("checkUpdate", func() { + checkUpdate(getMinioMode()) + }) + } + }() + + // Set system resources to maximum. + bootstrapTrace("setMaxResources", func() { + _ = setMaxResources(globalServerCtxt) + }) + + // Verify kernel release and version. + if oldLinux() { + warnings = append(warnings, color.YellowBold("Detected Linux kernel version older than 4.0 release, there are some known potential performance problems with this kernel version. MinIO recommends a minimum of 4.x linux kernel version for best performance")) + } + + maxProcs := runtime.GOMAXPROCS(0) + cpuProcs := runtime.NumCPU() + if maxProcs < cpuProcs { + warnings = append(warnings, color.YellowBold("Detected GOMAXPROCS(%d) < NumCPU(%d), please make sure to provide all PROCS to MinIO for optimal performance", + maxProcs, cpuProcs)) + } + + // Initialize grid + bootstrapTrace("initGrid", func() { + logger.FatalIf(initGlobalGrid(GlobalContext, globalEndpoints), "Unable to configure server grid RPC services") + }) + + // Initialize lock grid + bootstrapTrace("initLockGrid", func() { + logger.FatalIf(initGlobalLockGrid(GlobalContext, globalEndpoints), "Unable to configure server lock grid RPC services") + }) + + // Configure server. + bootstrapTrace("configureServer", func() { + handler, err := configureServerHandler(globalEndpoints) + if err != nil { + logger.Fatal(config.ErrUnexpectedError(err), "Unable to configure one of server's RPC services") + } + // Allow grid to start after registering all services. + close(globalGridStart) + close(globalLockGridStart) + + httpServer := xhttp.NewServer(getServerListenAddrs()). + UseHandler(setCriticalErrorHandler(corsHandler(handler))). + UseTLSConfig(newTLSConfig(getCert)). + UseIdleTimeout(globalServerCtxt.IdleTimeout). + UseReadTimeout(globalServerCtxt.IdleTimeout). + UseWriteTimeout(globalServerCtxt.IdleTimeout). + UseReadHeaderTimeout(globalServerCtxt.ReadHeaderTimeout). + UseBaseContext(GlobalContext). + UseCustomLogger(log.New(io.Discard, "", 0)). // Turn-off random logging by Go stdlib + UseTCPOptions(globalTCPOptions) + + httpServer.TCPOptions.Trace = bootstrapTraceMsg + go func() { + serveFn, err := httpServer.Init(GlobalContext, func(listenAddr string, err error) { + bootLogIf(GlobalContext, fmt.Errorf("Unable to listen on `%s`: %v", listenAddr, err)) + }) + if err != nil { + globalHTTPServerErrorCh <- err + return + } + globalHTTPServerErrorCh <- serveFn() + }() + + setHTTPServer(httpServer) + }) + + if globalIsDistErasure { + bootstrapTrace("verifying system configuration", func() { + // Additionally in distributed setup, validate the setup and configuration. + if err := verifyServerSystemConfig(GlobalContext, globalEndpoints, globalGrid.Load()); err != nil { + logger.Fatal(err, "Unable to start the server") + } + }) + } + + if globalEnableSyncBoot { + // Freeze the services until the bucket notification subsystem gets initialized. + bootstrapTrace("freezeServices", freezeServices) + } + + var newObject ObjectLayer + bootstrapTrace("newObjectLayer", func() { + var err error + newObject, err = newObjectLayer(GlobalContext, globalEndpoints) + if err != nil { + logFatalErrs(err, Endpoint{}, true) + } + }) + + for _, n := range globalNodes { + nodeName := n.Host + if n.IsLocal { + nodeName = globalLocalNodeName + } + nodeNameSum := sha256.Sum256([]byte(nodeName + globalDeploymentID())) + globalNodeNamesHex[hex.EncodeToString(nodeNameSum[:])] = struct{}{} + } + + bootstrapTrace("waitForQuorum", func() { + result := newObject.Health(context.Background(), HealthOptions{NoLogging: true}) + for !result.HealthyRead { + if debugNoExit { + logger.Info("Not waiting for quorum since we are debugging.. possible cause unhealthy sets") + logger.Info(result.String()) + break + } + d := time.Duration(r.Float64() * float64(time.Second)) + logger.Info("Waiting for quorum READ healthcheck to succeed retrying in %s.. possible cause unhealthy sets", d) + logger.Info(result.String()) + time.Sleep(d) + result = newObject.Health(context.Background(), HealthOptions{NoLogging: true}) + } + }) + + var err error + bootstrapTrace("initServerConfig", func() { + if err = initServerConfig(GlobalContext, newObject); err != nil { + var cerr config.Err + // For any config error, we don't need to drop into safe-mode + // instead its a user error and should be fixed by user. + if errors.As(err, &cerr) { + logger.FatalIf(err, "Unable to initialize the server") + } + + // If context was canceled + if errors.Is(err, context.Canceled) { + logger.FatalIf(err, "Server startup canceled upon user request") + } + + bootLogIf(GlobalContext, err) + } + + if !globalServerCtxt.StrictS3Compat { + warnings = append(warnings, color.YellowBold("Strict AWS S3 compatible incoming PUT, POST content payload validation is turned off, caution is advised do not use in production")) + } + }) + if globalActiveCred.Equal(auth.DefaultCredentials) { + msg := fmt.Sprintf("Detected default credentials '%s', we recommend that you change these values with 'MINIO_ROOT_USER' and 'MINIO_ROOT_PASSWORD' environment variables", + globalActiveCred) + warnings = append(warnings, color.YellowBold(msg)) + } + + // Initialize users credentials and policies in background right after config has initialized. + go func() { + bootstrapTrace("globalIAMSys.Init", func() { + globalIAMSys.Init(GlobalContext, newObject, globalEtcdClient, globalRefreshIAMInterval) + }) + + // Initialize Console UI + if globalBrowserEnabled { + bootstrapTrace("initConsoleServer", func() { + srv, err := initConsoleServer() + if err != nil { + logger.FatalIf(err, "Unable to initialize console service") + } + + setConsoleSrv(srv) + + go func() { + logger.FatalIf(newConsoleServerFn().Serve(), "Unable to initialize console server") + }() + }) + } + + // if we see FTP args, start FTP if possible + if len(globalServerCtxt.FTP) > 0 { + bootstrapTrace("go startFTPServer", func() { + go startFTPServer(globalServerCtxt.FTP) + }) + } + + // If we see SFTP args, start SFTP if possible + if len(globalServerCtxt.SFTP) > 0 { + bootstrapTrace("go startSFTPServer", func() { + go startSFTPServer(globalServerCtxt.SFTP) + }) + } + }() + + go func() { + if globalEnableSyncBoot { + defer bootstrapTrace("unfreezeServices", unfreezeServices) + t := time.AfterFunc(5*time.Minute, func() { + warnings = append(warnings, + color.YellowBold("- Initializing the config subsystem is taking longer than 5 minutes. Please remove 'MINIO_SYNC_BOOT=on' to not freeze the APIs")) + }) + defer t.Stop() + } + + // Initialize data scanner. + bootstrapTrace("initDataScanner", func() { + if v := env.Get("_MINIO_SCANNER", config.EnableOn); v == config.EnableOn { + initDataScanner(GlobalContext, newObject) + } + }) + + // Initialize background replication + bootstrapTrace("initBackgroundReplication", func() { + initBackgroundReplication(GlobalContext, newObject) + }) + + // Initialize background ILM worker poool + bootstrapTrace("initBackgroundExpiry", func() { + initBackgroundExpiry(GlobalContext, newObject) + }) + + bootstrapTrace("globalTransitionState.Init", func() { + globalTransitionState.Init(newObject) + }) + + go func() { + // Initialize transition tier configuration manager + bootstrapTrace("globalTierConfigMgr.Init", func() { + if err := globalTierConfigMgr.Init(GlobalContext, newObject); err != nil { + bootLogIf(GlobalContext, err) + } + }) + }() + + // Initialize bucket notification system. + bootstrapTrace("initBucketTargets", func() { + bootLogIf(GlobalContext, globalEventNotifier.InitBucketTargets(GlobalContext, newObject)) + }) + + var buckets []string + // List buckets to initialize bucket metadata sub-sys. + bootstrapTrace("listBuckets", func() { + for { + bucketsList, err := newObject.ListBuckets(GlobalContext, BucketOptions{NoMetadata: true}) + if err != nil { + if configRetriableErrors(err) { + logger.Info("Waiting for list buckets to succeed to initialize buckets.. possible cause (%v)", err) + time.Sleep(time.Duration(r.Float64() * float64(time.Second))) + continue + } + bootLogIf(GlobalContext, fmt.Errorf("Unable to list buckets to initialize bucket metadata sub-system: %w", err)) + } + + buckets = make([]string, len(bucketsList)) + for i := range bucketsList { + buckets[i] = bucketsList[i].Name + } + break + } + }) + + // Initialize bucket metadata sub-system. + bootstrapTrace("globalBucketMetadataSys.Init", func() { + globalBucketMetadataSys.Init(GlobalContext, buckets, newObject) + }) + + // initialize replication resync state. + bootstrapTrace("initResync", func() { + globalReplicationPool.Get().initResync(GlobalContext, buckets, newObject) + }) + + // Initialize site replication manager after bucket metadata + bootstrapTrace("globalSiteReplicationSys.Init", func() { + globalSiteReplicationSys.Init(GlobalContext, newObject) + }) + + // Populate existing buckets to the etcd backend + if globalDNSConfig != nil { + // Background this operation. + bootstrapTrace("go initFederatorBackend", func() { + go initFederatorBackend(buckets, newObject) + }) + } + + // Initialize batch job pool. + bootstrapTrace("newBatchJobPool", func() { + globalBatchJobPool = newBatchJobPool(GlobalContext, newObject, 100) + globalBatchJobsMetrics = batchJobMetrics{ + metrics: make(map[string]*batchJobInfo), + } + go globalBatchJobsMetrics.init(GlobalContext, newObject) + go globalBatchJobsMetrics.purgeJobMetrics() + }) + + // Prints the formatted startup message, if err is not nil then it prints additional information as well. + printStartupMessage(getAPIEndpoints(), err) + + // Print a warning at the end of the startup banner so it is more noticeable + if newObject.BackendInfo().StandardSCParity == 0 && !globalIsErasureSD { + warnings = append(warnings, color.YellowBold("The standard parity is set to 0. This can lead to data loss.")) + } + + for _, warn := range warnings { + logger.Warning(warn) + } + }() + + region := globalSite.Region() + if region == "" { + region = "us-east-1" + } + bootstrapTrace("globalMinioClient", func() { + globalMinioClient, err = minio.New(globalLocalNodeName, &minio.Options{ + Creds: credentials.NewStaticV4(globalActiveCred.AccessKey, globalActiveCred.SecretKey, ""), + Secure: globalIsTLS, + Transport: globalRemoteTargetTransport, + Region: region, + }) + logger.FatalIf(err, "Unable to initialize MinIO client") + }) + + go bootstrapTrace("startResourceMetricsCollection", func() { + startResourceMetricsCollection() + }) + + // Add User-Agent to differentiate the requests. + globalMinioClient.SetAppInfo("minio-perf-test", ReleaseTag) + + if serverDebugLog { + fmt.Println("== DEBUG Mode enabled ==") + fmt.Println("Currently set environment settings:") + ks := []string{ + config.EnvAccessKey, + config.EnvSecretKey, + config.EnvRootUser, + config.EnvRootPassword, + } + for _, v := range os.Environ() { + // Do not print sensitive creds in debug. + if slices.Contains(ks, strings.Split(v, "=")[0]) { + continue + } + fmt.Println(v) + } + fmt.Println("======") + } + + daemon.SdNotify(false, daemon.SdNotifyReady) + + <-globalOSSignalCh +} + +// Initialize object layer with the supplied disks, objectLayer is nil upon any error. +func newObjectLayer(ctx context.Context, endpointServerPools EndpointServerPools) (newObject ObjectLayer, err error) { + return newErasureServerPools(ctx, endpointServerPools) +} diff --git a/cmd/server-main_test.go b/cmd/server-main_test.go new file mode 100644 index 00000000..fd397b9f --- /dev/null +++ b/cmd/server-main_test.go @@ -0,0 +1,117 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "reflect" + "testing" +) + +func TestServerConfigFile(t *testing.T) { + for _, testcase := range []struct { + config string + expectedErr bool + hash string + }{ + { + config: "testdata/config/1.yaml", + expectedErr: false, + hash: "hash:02bf70285dc71f76", + }, + { + config: "testdata/config/2.yaml", + expectedErr: false, + hash: "hash:676d2da00f71f205", + }, + { + config: "testdata/config/invalid.yaml", + expectedErr: true, + }, + { + config: "testdata/config/invalid-types.yaml", + expectedErr: true, + }, + { + config: "testdata/config/invalid-disks.yaml", + expectedErr: true, + }, + } { + t.Run(testcase.config, func(t *testing.T) { + sctx := &serverCtxt{} + err := mergeServerCtxtFromConfigFile(testcase.config, sctx) + if testcase.expectedErr && err == nil { + t.Error("expected failure, got success") + } + if !testcase.expectedErr && err != nil { + t.Error("expected success, got failure", err) + } + if err == nil { + if len(sctx.Layout.pools) != 2 { + t.Error("expected parsed pools to be 2, not", len(sctx.Layout.pools)) + } + if sctx.Layout.pools[0].cmdline != testcase.hash { + t.Error("expected hash", testcase.hash, "got", sctx.Layout.pools[0].cmdline) + } + } + }) + } +} + +// Tests initializing new object layer. +func TestNewObjectLayer(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + // Tests for ErasureSD object layer. + nDisks := 1 + disks, err := getRandomDisks(nDisks) + if err != nil { + t.Fatal("Failed to create drives for the backend") + } + defer removeRoots(disks) + + obj, err := newObjectLayer(ctx, mustGetPoolEndpoints(0, disks...)) + if err != nil { + t.Fatal("Unexpected object layer initialization error", err) + } + + _, ok := obj.(*erasureServerPools) + if !ok { + t.Fatal("Unexpected object layer detected", reflect.TypeOf(obj)) + } + + // Tests for Erasure object layer initialization. + + // Create temporary backend for the test server. + nDisks = 16 + disks, err = getRandomDisks(nDisks) + if err != nil { + t.Fatal("Failed to create drives for the backend") + } + defer removeRoots(disks) + + obj, err = newObjectLayer(ctx, mustGetPoolEndpoints(0, disks...)) + if err != nil { + t.Fatal("Unexpected object layer initialization error", err) + } + + _, ok = obj.(*erasureServerPools) + if !ok { + t.Fatal("Unexpected object layer detected", reflect.TypeOf(obj)) + } +} diff --git a/cmd/server-rlimit.go b/cmd/server-rlimit.go new file mode 100644 index 00000000..ecb779e1 --- /dev/null +++ b/cmd/server-rlimit.go @@ -0,0 +1,94 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "runtime" + "runtime/debug" + + "github.com/dustin/go-humanize" + "github.com/minio/madmin-go/v3/kernel" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/sys" +) + +func oldLinux() bool { + currentKernel, err := kernel.CurrentVersion() + if err != nil { + // Could not probe the kernel version + return false + } + + if currentKernel == 0 { + // We could not get any valid value return false + return false + } + + // legacy linux indicator for printing warnings + // about older Linux kernels and Go runtime. + return currentKernel < kernel.Version(4, 0, 0) +} + +func setMaxResources(ctx serverCtxt) (err error) { + // Set the Go runtime max threads threshold to 90% of kernel setting. + sysMaxThreads, err := sys.GetMaxThreads() + if err == nil { + minioMaxThreads := (sysMaxThreads * 90) / 100 + // Only set max threads if it is greater than the default one + if minioMaxThreads > 10000 { + debug.SetMaxThreads(minioMaxThreads) + } + } + + var maxLimit uint64 + + // Set open files limit to maximum. + if _, maxLimit, err = sys.GetMaxOpenFileLimit(); err != nil { + return err + } + + if maxLimit < 4096 && runtime.GOOS != globalWindowsOSName { + logger.Info("WARNING: maximum file descriptor limit %d is too low for production servers. At least 4096 is recommended. Fix with \"ulimit -n 4096\"", + maxLimit) + } + + if err = sys.SetMaxOpenFileLimit(maxLimit, maxLimit); err != nil { + return err + } + + _, vssLimit, err := sys.GetMaxMemoryLimit() + if err != nil { + return err + } + + if vssLimit > 0 && vssLimit < humanize.GiByte { + logger.Info("WARNING: maximum virtual memory limit (%s) is too small for 'go runtime', please consider setting `ulimit -v` to unlimited", + humanize.IBytes(vssLimit)) + } + + if ctx.MemLimit > 0 { + debug.SetMemoryLimit(int64(ctx.MemLimit)) + } + + // Do not use RLIMIT_AS as that is not useful and at times on systems < 4Gi + // this can crash the Go runtime if the value is smaller refer + // - https://github.com/golang/go/issues/38010 + // - https://github.com/golang/go/issues/43699 + // So do not add `sys.SetMaxMemoryLimit()` this is not useful for any practical purposes. + return nil +} diff --git a/cmd/server-startup-msg.go b/cmd/server-startup-msg.go new file mode 100644 index 00000000..ce98d008 --- /dev/null +++ b/cmd/server-startup-msg.go @@ -0,0 +1,196 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "net" + "net/url" + "strings" + + xnet "github.com/minio/pkg/v3/net" + + "github.com/minio/minio/internal/color" + "github.com/minio/minio/internal/logger" +) + +// generates format string depending on the string length and padding. +func getFormatStr(strLen int, padding int) string { + formatStr := fmt.Sprintf("%ds", strLen+padding) + return "%" + formatStr +} + +// Prints the formatted startup message. +func printStartupMessage(apiEndpoints []string, err error) { + banner := strings.Repeat("-", len(MinioBannerName)) + if globalIsDistErasure { + logger.Startup(color.Bold(banner)) + } + logger.Startup(color.Bold(MinioBannerName)) + if err != nil { + if globalConsoleSys != nil { + globalConsoleSys.Send(GlobalContext, fmt.Sprintf("Server startup failed with '%v', some features may be missing", err)) + } + } + + if !globalSubnetConfig.Registered() { + var builder strings.Builder + startupBanner(&builder) + logger.Startup(builder.String()) + } + + strippedAPIEndpoints := stripStandardPorts(apiEndpoints, globalMinioHost) + + // Prints credential, region and browser access. + printServerCommonMsg(strippedAPIEndpoints) + + // Prints `mc` cli configuration message chooses + // first endpoint as default. + printCLIAccessMsg(strippedAPIEndpoints[0], "myminio") + + // Prints documentation message. + printObjectAPIMsg() + if globalIsDistErasure { + logger.Startup(color.Bold(banner)) + } +} + +// Returns true if input is IPv6 +func isIPv6(host string) bool { + h, _, err := net.SplitHostPort(host) + if err != nil { + h = host + } + ip := net.ParseIP(h) + return ip.To16() != nil && ip.To4() == nil +} + +// strip api endpoints list with standard ports such as +// port "80" and "443" before displaying on the startup +// banner. Returns a new list of API endpoints. +func stripStandardPorts(apiEndpoints []string, host string) (newAPIEndpoints []string) { + if len(apiEndpoints) == 1 { + return apiEndpoints + } + newAPIEndpoints = make([]string, len(apiEndpoints)) + // Check all API endpoints for standard ports and strip them. + for i, apiEndpoint := range apiEndpoints { + _, err := xnet.ParseHTTPURL(apiEndpoint) + if err != nil { + continue + } + u, err := url.Parse(apiEndpoint) + if err != nil { + continue + } + if host == "" && isIPv6(u.Hostname()) { + // Skip all IPv6 endpoints + continue + } + if u.Port() == "80" && u.Scheme == "http" || u.Port() == "443" && u.Scheme == "https" { + u.Host = u.Hostname() + } + newAPIEndpoints[i] = u.String() + } + return newAPIEndpoints +} + +// Prints common server startup message. Prints credential, region and browser access. +func printServerCommonMsg(apiEndpoints []string) { + // Get saved credentials. + cred := globalActiveCred + + // Get saved region. + region := globalSite.Region() + + apiEndpointStr := strings.TrimSpace(strings.Join(apiEndpoints, " ")) + // Colorize the message and print. + logger.Startup(color.Blue("API: ") + color.Bold(fmt.Sprintf("%s ", apiEndpointStr))) + if color.IsTerminal() && (!globalServerCtxt.Anonymous && !globalServerCtxt.JSON && globalAPIConfig.permitRootAccess()) { + logger.Startup(color.Blue(" RootUser: ") + color.Bold("%s ", cred.AccessKey)) + logger.Startup(color.Blue(" RootPass: ") + color.Bold("%s \n", cred.SecretKey)) + if region != "" { + logger.Startup(color.Blue(" Region: ") + color.Bold("%s", fmt.Sprintf(getFormatStr(len(region), 2), region))) + } + } + + if globalBrowserEnabled { + consoleEndpointStr := strings.Join(stripStandardPorts(getConsoleEndpoints(), globalMinioConsoleHost), " ") + logger.Startup(color.Blue("WebUI: ") + color.Bold(fmt.Sprintf("%s ", consoleEndpointStr))) + if color.IsTerminal() && (!globalServerCtxt.Anonymous && !globalServerCtxt.JSON && globalAPIConfig.permitRootAccess()) { + logger.Startup(color.Blue(" RootUser: ") + color.Bold("%s ", cred.AccessKey)) + logger.Startup(color.Blue(" RootPass: ") + color.Bold("%s ", cred.SecretKey)) + } + } + + printEventNotifiers() + printLambdaTargets() +} + +// Prints startup message for Object API access, prints link to our SDK documentation. +func printObjectAPIMsg() { + logger.Startup(color.Blue("\nDocs: ") + "https://docs.min.io") +} + +func printLambdaTargets() { + if globalLambdaTargetList == nil || globalLambdaTargetList.Empty() { + return + } + + arnMsg := color.Blue("Object Lambda ARNs: ") + for _, arn := range globalLambdaTargetList.List(globalSite.Region()) { + arnMsg += color.Bold(fmt.Sprintf("%s ", arn)) + } + logger.Startup(arnMsg + "\n") +} + +// Prints bucket notification configurations. +func printEventNotifiers() { + if globalNotificationSys == nil { + return + } + + arns := globalEventNotifier.GetARNList() + if len(arns) == 0 { + return + } + + arnMsg := color.Blue("SQS ARNs: ") + for _, arn := range arns { + arnMsg += color.Bold(fmt.Sprintf("%s ", arn)) + } + + logger.Startup(arnMsg + "\n") +} + +// Prints startup message for command line access. Prints link to our documentation +// and custom platform specific message. +func printCLIAccessMsg(endPoint string, alias string) { + // Get saved credentials. + cred := globalActiveCred + + const mcQuickStartGuide = "https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart" + + // Configure 'mc', following block prints platform specific information for minio client. + if color.IsTerminal() && (!globalServerCtxt.Anonymous && globalAPIConfig.permitRootAccess()) { + logger.Startup(color.Blue("\nCLI: ") + mcQuickStartGuide) + mcMessage := fmt.Sprintf("$ mc alias set '%s' '%s' '%s' '%s'", alias, + endPoint, cred.AccessKey, cred.SecretKey) + logger.Startup(fmt.Sprintf(getFormatStr(len(mcMessage), 3), mcMessage)) + } +} diff --git a/cmd/server-startup-msg_test.go b/cmd/server-startup-msg_test.go new file mode 100644 index 00000000..08b45182 --- /dev/null +++ b/cmd/server-startup-msg_test.go @@ -0,0 +1,102 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "os" + "reflect" + "testing" +) + +// Tests stripping standard ports from apiEndpoints. +func TestStripStandardPorts(t *testing.T) { + apiEndpoints := []string{"http://127.0.0.1:9000", "http://127.0.0.2:80", "https://127.0.0.3:443"} + expectedAPIEndpoints := []string{"http://127.0.0.1:9000", "http://127.0.0.2", "https://127.0.0.3"} + newAPIEndpoints := stripStandardPorts(apiEndpoints, "") + + if !reflect.DeepEqual(expectedAPIEndpoints, newAPIEndpoints) { + t.Fatalf("Expected %#v, got %#v", expectedAPIEndpoints, newAPIEndpoints) + } + + apiEndpoints = []string{"http://%%%%%:9000"} + newAPIEndpoints = stripStandardPorts(apiEndpoints, "") + if !reflect.DeepEqual(apiEndpoints, newAPIEndpoints) { + t.Fatalf("Expected %#v, got %#v", apiEndpoints, newAPIEndpoints) + } + + apiEndpoints = []string{"http://127.0.0.1:443", "https://127.0.0.1:80"} + newAPIEndpoints = stripStandardPorts(apiEndpoints, "") + if !reflect.DeepEqual(apiEndpoints, newAPIEndpoints) { + t.Fatalf("Expected %#v, got %#v", apiEndpoints, newAPIEndpoints) + } +} + +// Test printing server common message. +func TestPrintServerCommonMessage(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + obj, fsDir, err := prepareFS(ctx) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(fsDir) + if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil { + t.Fatal(err) + } + + apiEndpoints := []string{"http://127.0.0.1:9000"} + printServerCommonMsg(apiEndpoints) +} + +// Tests print cli access message. +func TestPrintCLIAccessMsg(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + obj, fsDir, err := prepareFS(ctx) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(fsDir) + if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil { + t.Fatal(err) + } + + apiEndpoints := []string{"http://127.0.0.1:9000"} + printCLIAccessMsg(apiEndpoints[0], "myminio") +} + +// Test print startup message. +func TestPrintStartupMessage(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + obj, fsDir, err := prepareFS(ctx) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(fsDir) + if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil { + t.Fatal(err) + } + + apiEndpoints := []string{"http://127.0.0.1:9000"} + printStartupMessage(apiEndpoints, nil) +} diff --git a/cmd/server_test.go b/cmd/server_test.go new file mode 100644 index 00000000..e6911735 --- /dev/null +++ b/cmd/server_test.go @@ -0,0 +1,3005 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/xml" + "fmt" + "io" + "math/rand" + "net/http" + "net/url" + "reflect" + "regexp" + "runtime" + "strings" + "sync" + "testing" + "time" + + "github.com/dustin/go-humanize" + jwtgo "github.com/golang-jwt/jwt/v4" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio-go/v7/pkg/signer" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/pkg/v3/policy" +) + +// API suite container common to both ErasureSD and Erasure. +type TestSuiteCommon struct { + serverType string + testServer TestServer + endPoint string + accessKey string + secretKey string + signer signerType + secure bool + client *http.Client +} + +type check struct { + *testing.T + testType string +} + +// Assert - checks if gotValue is same as expectedValue, if not fails the test. +func (c *check) Assert(gotValue any, expectedValue any) { + c.Helper() + if !reflect.DeepEqual(gotValue, expectedValue) { + c.Fatalf("Test %s expected %v, got %v", c.testType, expectedValue, gotValue) + } +} + +func verifyError(c *check, response *http.Response, code, description string, statusCode int) { + c.Helper() + data, err := io.ReadAll(response.Body) + c.Assert(err, nil) + errorResponse := APIErrorResponse{} + err = xml.Unmarshal(data, &errorResponse) + c.Assert(err, nil) + c.Assert(errorResponse.Code, code) + c.Assert(errorResponse.Message, description) + c.Assert(response.StatusCode, statusCode) +} + +func runAllTests(suite *TestSuiteCommon, c *check) { + suite.SetUpSuite(c) + suite.TestCors(c) + suite.TestObjectDir(c) + suite.TestBucketPolicy(c) + suite.TestDeleteBucket(c) + suite.TestDeleteBucketNotEmpty(c) + suite.TestDeleteMultipleObjects(c) + suite.TestDeleteObject(c) + suite.TestNonExistentBucket(c) + suite.TestEmptyObject(c) + suite.TestBucket(c) + suite.TestObjectGetAnonymous(c) + suite.TestMultipleObjects(c) + suite.TestHeader(c) + suite.TestPutBucket(c) + suite.TestCopyObject(c) + suite.TestPutObject(c) + suite.TestListBuckets(c) + suite.TestValidateSignature(c) + suite.TestSHA256Mismatch(c) + suite.TestPutObjectLongName(c) + suite.TestNotBeAbleToCreateObjectInNonexistentBucket(c) + suite.TestHeadOnObjectLastModified(c) + suite.TestHeadOnBucket(c) + suite.TestContentTypePersists(c) + suite.TestPartialContent(c) + suite.TestListObjectsHandler(c) + suite.TestListObjectVersionsOutputOrderHandler(c) + suite.TestListObjectsHandlerErrors(c) + suite.TestListObjectsV2HadoopUAHandler(c) + suite.TestPutBucketErrors(c) + suite.TestGetObjectLarge10MiB(c) + suite.TestGetObjectLarge11MiB(c) + suite.TestGetPartialObjectMisAligned(c) + suite.TestGetPartialObjectLarge11MiB(c) + suite.TestGetPartialObjectLarge10MiB(c) + suite.TestGetObjectErrors(c) + suite.TestGetObjectRangeErrors(c) + suite.TestObjectMultipartAbort(c) + suite.TestBucketMultipartList(c) + suite.TestValidateObjectMultipartUploadID(c) + suite.TestObjectMultipartListError(c) + suite.TestObjectValidMD5(c) + suite.TestObjectMultipart(c) + suite.TestMetricsV3Handler(c) + suite.TestBucketSQSNotificationWebHook(c) + suite.TestBucketSQSNotificationAMQP(c) + suite.TestUnsignedCVE(c) + suite.TearDownSuite(c) +} + +func TestServerSuite(t *testing.T) { + testCases := []*TestSuiteCommon{ + // Init and run test on ErasureSD backend with signature v4. + {serverType: "ErasureSD", signer: signerV4}, + // Init and run test on ErasureSD backend with signature v2. + {serverType: "ErasureSD", signer: signerV2}, + // Init and run test on ErasureSD backend, with tls enabled. + {serverType: "ErasureSD", signer: signerV4, secure: true}, + // Init and run test on Erasure backend. + {serverType: "Erasure", signer: signerV4}, + // Init and run test on ErasureSet backend. + {serverType: "ErasureSet", signer: signerV4}, + } + globalServerCtxt.StrictS3Compat = true + for i, testCase := range testCases { + t.Run(fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.serverType), func(t *testing.T) { + runAllTests(testCase, &check{t, testCase.serverType}) + }) + } +} + +// Setting up the test suite. +// Starting the Test server with temporary backend. +func (s *TestSuiteCommon) SetUpSuite(c *check) { + if s.secure { + cert, key, err := generateTLSCertKey("127.0.0.1") + c.Assert(err, nil) + + s.testServer = StartTestTLSServer(c, s.serverType, cert, key) + } else { + s.testServer = StartTestServer(c, s.serverType) + } + + s.client = s.testServer.Server.Client() + s.endPoint = s.testServer.Server.URL + s.accessKey = s.testServer.AccessKey + s.secretKey = s.testServer.SecretKey +} + +func (s *TestSuiteCommon) RestartTestServer(c *check) { + // Shutdown. + s.testServer.cancel() + s.testServer.Server.Close() + s.testServer.Obj.Shutdown(context.Background()) + + // Restart. + ctx, cancel := context.WithCancel(context.Background()) + + s.testServer.cancel = cancel + s.testServer = initTestServerWithBackend(ctx, c, s.testServer, s.testServer.Obj, s.testServer.rawDiskPaths) + if s.secure { + s.testServer.Server.StartTLS() + } else { + s.testServer.Server.Start() + } + + s.client = s.testServer.Server.Client() + s.endPoint = s.testServer.Server.URL +} + +func (s *TestSuiteCommon) TearDownSuite(c *check) { + s.testServer.Stop() +} + +const ( + defaultPrometheusJWTExpiry = 100 * 365 * 24 * time.Hour +) + +func (s *TestSuiteCommon) TestMetricsV3Handler(c *check) { + jwt := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, jwtgo.StandardClaims{ + ExpiresAt: time.Now().UTC().Add(defaultPrometheusJWTExpiry).Unix(), + Subject: s.accessKey, + Issuer: "prometheus", + }) + + token, err := jwt.SignedString([]byte(s.secretKey)) + c.Assert(err, nil) + + for _, cpath := range globalMetricsV3CollectorPaths { + request, err := newTestSignedRequest(http.MethodGet, s.endPoint+minioReservedBucketPath+metricsV3Path+string(cpath), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + request.Header.Set("Authorization", "Bearer "+token) + + // execute the request. + response, err := s.client.Do(request) + c.Assert(err, nil) + + // assert the http response status code. + c.Assert(response.StatusCode, http.StatusOK) + } +} + +func (s *TestSuiteCommon) TestBucketSQSNotificationWebHook(c *check) { + // Sample bucket notification. + bucketNotificationBuf := `s3:ObjectCreated:Putprefiximages/1arn:minio:sqs:us-east-1:444455556666:webhook` + // generate a random bucket Name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the request. + response, err := s.client.Do(request) + c.Assert(err, nil) + + // assert the http response status code. + c.Assert(response.StatusCode, http.StatusOK) + + request, err = newTestSignedRequest(http.MethodPut, getPutNotificationURL(s.endPoint, bucketName), + int64(len(bucketNotificationBuf)), bytes.NewReader([]byte(bucketNotificationBuf)), s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request. + response, err = s.client.Do(request) + + c.Assert(err, nil) + verifyError(c, response, "InvalidArgument", "A specified destination ARN does not exist or is not well-formed. Verify the destination ARN.", http.StatusBadRequest) +} + +func (s *TestSuiteCommon) TestCors(c *check) { + expectedMap := http.Header{} + expectedMap.Set("Access-Control-Allow-Credentials", "true") + expectedMap.Set("Access-Control-Allow-Origin", "http://foobar.com") + expectedMap["Access-Control-Expose-Headers"] = []string{ + "Date", + "Etag", + "Server", + "Connection", + "Accept-Ranges", + "Content-Range", + "Content-Encoding", + "Content-Length", + "Content-Type", + "Content-Disposition", + "Last-Modified", + "Content-Language", + "Cache-Control", + "Retry-After", + "X-Amz-Bucket-Region", + "Expires", + "X-Amz*", + "X-Amz*", + "*", + } + expectedMap.Set("Vary", "Origin") + + req, _ := http.NewRequest(http.MethodOptions, s.endPoint, nil) + req.Header.Set("Origin", "http://foobar.com") + res, err := s.client.Do(req) + if err != nil { + c.Fatal(err) + } + + for k := range expectedMap { + if v, ok := res.Header[k]; !ok { + c.Errorf("Expected key %s missing from %v", k, res.Header) + } else { + expectedSet := set.CreateStringSet(expectedMap[k]...) + gotSet := set.CreateStringSet(strings.Split(v[0], ", ")...) + if !expectedSet.Equals(gotSet) { + c.Errorf("Expected value %v, got %v", strings.Join(expectedMap[k], ", "), v) + } + } + } +} + +func (s *TestSuiteCommon) TestObjectDir(c *check) { + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the request. + response, err := s.client.Do(request) + c.Assert(err, nil) + + // assert the http response status code. + c.Assert(response.StatusCode, http.StatusOK) + + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, "my-object-directory/"), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request. + response, err = s.client.Do(request) + + c.Assert(err, nil) + // assert the http response status code. + c.Assert(response.StatusCode, http.StatusOK) + + request, err = newTestSignedRequest(http.MethodHead, getHeadObjectURL(s.endPoint, bucketName, "my-object-directory/"), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request. + response, err = s.client.Do(request) + + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + request, err = newTestSignedRequest(http.MethodGet, getGetObjectURL(s.endPoint, bucketName, "my-object-directory/"), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request. + response, err = s.client.Do(request) + + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + request, err = newTestSignedRequest(http.MethodDelete, getDeleteObjectURL(s.endPoint, bucketName, "my-object-directory/"), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request. + response, err = s.client.Do(request) + + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusNoContent) +} + +func (s *TestSuiteCommon) TestUnsignedCVE(c *check) { + c.Helper() + + // generate a random bucket Name. + bucketName := getRandomBucketName() + + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the request. + response, err := s.client.Do(request) + c.Assert(err, nil) + + // assert the http response status code. + c.Assert(response.StatusCode, http.StatusOK) + + req, err := http.NewRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, "test-cve-object.txt"), nil) + c.Assert(err, nil) + + req.Body = io.NopCloser(bytes.NewReader([]byte("foobar!\n"))) + req.Trailer = http.Header{} + req.Trailer.Set("x-amz-checksum-crc32", "rK0DXg==") + + now := UTCNow() + + req = signer.StreamingUnsignedV4(req, "", 8, now) + + maliciousHeaders := http.Header{ + "Authorization": []string{fmt.Sprintf("AWS4-HMAC-SHA256 Credential=%s/%s/us-east-1/s3/aws4_request, SignedHeaders=invalidheader, Signature=deadbeefdeadbeefdeadbeeddeadbeeddeadbeefdeadbeefdeadbeefdeadbeef", s.accessKey, now.Format(yyyymmdd))}, + "User-Agent": []string{"A malicious request"}, + "X-Amz-Decoded-Content-Length": []string{"8"}, + "Content-Encoding": []string{"aws-chunked"}, + "X-Amz-Trailer": []string{"x-amz-checksum-crc32"}, + "x-amz-content-sha256": []string{unsignedPayloadTrailer}, + } + + for k, v := range maliciousHeaders { + req.Header.Set(k, v[0]) + } + + // execute the request. + response, err = s.client.Do(req) + c.Assert(err, nil) + + // out, err = httputil.DumpResponse(response, true) + // fmt.Println("RESPONSE ===\n", string(out), err) + + // assert the http response status code. + c.Assert(response.StatusCode, http.StatusBadRequest) +} + +func (s *TestSuiteCommon) TestBucketSQSNotificationAMQP(c *check) { + // Sample bucket notification. + bucketNotificationBuf := `s3:ObjectCreated:Putprefiximages/1arn:minio:sqs:us-east-1:444455556666:amqp` + // generate a random bucket Name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the request. + response, err := s.client.Do(request) + c.Assert(err, nil) + + // assert the http response status code. + c.Assert(response.StatusCode, http.StatusOK) + + request, err = newTestSignedRequest(http.MethodPut, getPutNotificationURL(s.endPoint, bucketName), + int64(len(bucketNotificationBuf)), bytes.NewReader([]byte(bucketNotificationBuf)), s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request. + response, err = s.client.Do(request) + + c.Assert(err, nil) + verifyError(c, response, "InvalidArgument", "A specified destination ARN does not exist or is not well-formed. Verify the destination ARN.", http.StatusBadRequest) +} + +// TestBucketPolicy - Inserts the bucket policy and verifies it by fetching the policy back. +// Deletes the policy and verifies the deletion by fetching it back. +func (s *TestSuiteCommon) TestBucketPolicy(c *check) { + // Sample bucket policy. + bucketPolicyBuf := `{"Version":"2012-10-17","Statement":[{"Action":["s3:GetBucketLocation","s3:ListBucket"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s"]},{"Action":["s3:GetObject"],"Effect":"Allow","Principal":{"AWS":["*"]},"Resource":["arn:aws:s3:::%s/this*"]}]}` + + // generate a random bucket Name. + bucketName := getRandomBucketName() + // create the policy statement string with the randomly generated bucket name. + bucketPolicyStr := fmt.Sprintf(bucketPolicyBuf, bucketName, bucketName) + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the request. + response, err := s.client.Do(request) + c.Assert(err, nil) + // assert the http response status code. + c.Assert(response.StatusCode, http.StatusOK) + + // Put a new bucket policy. + request, err = newTestSignedRequest(http.MethodPut, getPutPolicyURL(s.endPoint, bucketName), + int64(len(bucketPolicyStr)), bytes.NewReader([]byte(bucketPolicyStr)), s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create bucket. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusNoContent) + + // Fetch the uploaded policy. + request, err = newTestSignedRequest(http.MethodGet, getGetPolicyURL(s.endPoint, bucketName), 0, nil, + s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + bucketPolicyReadBuf, err := io.ReadAll(response.Body) + c.Assert(err, nil) + // Verify if downloaded policy matches with previously uploaded. + expectedPolicy, err := policy.ParseBucketPolicyConfig(strings.NewReader(bucketPolicyStr), bucketName) + c.Assert(err, nil) + gotPolicy, err := policy.ParseBucketPolicyConfig(bytes.NewReader(bucketPolicyReadBuf), bucketName) + c.Assert(err, nil) + c.Assert(reflect.DeepEqual(expectedPolicy, gotPolicy), true) + + // Delete policy. + request, err = newTestSignedRequest(http.MethodDelete, getDeletePolicyURL(s.endPoint, bucketName), 0, nil, + s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusNoContent) + + // Verify if the policy was indeed deleted. + request, err = newTestSignedRequest(http.MethodGet, getGetPolicyURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusNotFound) +} + +// TestDeleteBucket - validates DELETE bucket operation. +func (s *TestSuiteCommon) TestDeleteBucket(c *check) { + bucketName := getRandomBucketName() + + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + response, err := s.client.Do(request) + c.Assert(err, nil) + // assert the response status code. + c.Assert(response.StatusCode, http.StatusOK) + + // construct request to delete the bucket. + request, err = newTestSignedRequest(http.MethodDelete, getDeleteBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + response, err = s.client.Do(request) + c.Assert(err, nil) + // Assert the response status code. + c.Assert(response.StatusCode, http.StatusNoContent) +} + +// TestDeleteBucketNotEmpty - Validates the operation during an attempt to delete a non-empty bucket. +func (s *TestSuiteCommon) TestDeleteBucketNotEmpty(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the request. + response, err := s.client.Do(request) + c.Assert(err, nil) + // assert the response status code. + c.Assert(response.StatusCode, http.StatusOK) + + // generate http request for an object upload. + // "test-object" is the object name. + objectName := "test-object" + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the request to complete object upload. + response, err = s.client.Do(request) + c.Assert(err, nil) + // assert the status code of the response. + c.Assert(response.StatusCode, http.StatusOK) + + // constructing http request to delete the bucket. + // making an attempt to delete an non-empty bucket. + // expected to fail. + request, err = newTestSignedRequest(http.MethodDelete, getDeleteBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusConflict) +} + +func (s *TestSuiteCommon) TestListenNotificationHandler(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + req, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the request. + response, err := s.client.Do(req) + c.Assert(err, nil) + // assert the http response status code. + c.Assert(response.StatusCode, http.StatusOK) + + invalidBucket := "Invalid\\Bucket" + tooByte := bytes.Repeat([]byte("a"), 1025) + tooBigPrefix := string(tooByte) + validEvents := []string{"s3:ObjectCreated:*", "s3:ObjectRemoved:*"} + invalidEvents := []string{"invalidEvent"} + + req, err = newTestSignedRequest(http.MethodGet, + getListenNotificationURL(s.endPoint, invalidBucket, []string{}, []string{}, []string{}), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the request. + response, err = s.client.Do(req) + c.Assert(err, nil) + verifyError(c, response, "InvalidBucketName", "The specified bucket is not valid.", http.StatusBadRequest) + + req, err = newTestSignedRequest(http.MethodGet, + getListenNotificationURL(s.endPoint, bucketName, []string{}, []string{}, invalidEvents), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the request. + response, err = s.client.Do(req) + c.Assert(err, nil) + verifyError(c, response, "InvalidArgument", "A specified event is not supported for notifications.", http.StatusBadRequest) + + req, err = newTestSignedRequest(http.MethodGet, + getListenNotificationURL(s.endPoint, bucketName, []string{tooBigPrefix}, []string{}, validEvents), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the request. + response, err = s.client.Do(req) + c.Assert(err, nil) + verifyError(c, response, "InvalidArgument", "Size of filter rule value cannot exceed 1024 bytes in UTF-8 representation", http.StatusBadRequest) + + req, err = newTestSignedBadSHARequest(http.MethodGet, + getListenNotificationURL(s.endPoint, bucketName, []string{}, []string{}, validEvents), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the request. + response, err = s.client.Do(req) + c.Assert(err, nil) + if s.signer == signerV4 { + verifyError(c, response, "XAmzContentSHA256Mismatch", "The provided 'x-amz-content-sha256' header does not match what was computed.", http.StatusBadRequest) + } +} + +// Test deletes multiple objects and verifies server response. +func (s *TestSuiteCommon) TestDeleteMultipleObjects(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the request. + response, err := s.client.Do(request) + c.Assert(err, nil) + // assert the http response status code. + c.Assert(response.StatusCode, http.StatusOK) + + objectName := "prefix/myobject" + delObjReq := DeleteObjectsRequest{ + Quiet: false, + } + for i := range 10 { + // Obtain http request to upload object. + // object Name contains a prefix. + objName := fmt.Sprintf("%d/%s", i, objectName) + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the http request. + response, err = s.client.Do(request) + c.Assert(err, nil) + // assert the status of http response. + c.Assert(response.StatusCode, http.StatusOK) + // Append all objects. + delObjReq.Objects = append(delObjReq.Objects, ObjectToDelete{ + ObjectV: ObjectV{ + ObjectName: objName, + }, + }) + } + // Marshal delete request. + deleteReqBytes, err := xml.Marshal(delObjReq) + c.Assert(err, nil) + + // Delete list of objects. + request, err = newTestSignedRequest(http.MethodPost, getMultiDeleteObjectURL(s.endPoint, bucketName), + int64(len(deleteReqBytes)), bytes.NewReader(deleteReqBytes), s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + deleteResp := DeleteObjectsResponse{} + delRespBytes, err := io.ReadAll(response.Body) + c.Assert(err, nil) + err = xml.Unmarshal(delRespBytes, &deleteResp) + c.Assert(err, nil) + for i := range 10 { + // All the objects should be under deleted list (including non-existent object) + c.Assert(deleteResp.DeletedObjects[i], DeletedObject{ + ObjectName: delObjReq.Objects[i].ObjectName, + VersionID: delObjReq.Objects[i].VersionID, + }) + } + c.Assert(len(deleteResp.Errors), 0) + + // Attempt second time results should be same, NoSuchKey for objects not found + // shouldn't be set. + request, err = newTestSignedRequest(http.MethodPost, getMultiDeleteObjectURL(s.endPoint, bucketName), + int64(len(deleteReqBytes)), bytes.NewReader(deleteReqBytes), s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + deleteResp = DeleteObjectsResponse{} + delRespBytes, err = io.ReadAll(response.Body) + c.Assert(err, nil) + err = xml.Unmarshal(delRespBytes, &deleteResp) + c.Assert(err, nil) + c.Assert(len(deleteResp.DeletedObjects), len(delObjReq.Objects)) + for i := range 10 { + c.Assert(deleteResp.DeletedObjects[i], DeletedObject{ + ObjectName: delObjReq.Objects[i].ObjectName, + VersionID: delObjReq.Objects[i].VersionID, + }) + } + c.Assert(len(deleteResp.Errors), 0) +} + +// Tests delete object responses and success. +func (s *TestSuiteCommon) TestDeleteObject(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the request. + response, err := s.client.Do(request) + c.Assert(err, nil) + // assert the http response status code. + c.Assert(response.StatusCode, http.StatusOK) + + objectName := "prefix/myobject" + // obtain http request to upload object. + // object Name contains a prefix. + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the http request. + response, err = s.client.Do(request) + c.Assert(err, nil) + // assert the status of http response. + c.Assert(response.StatusCode, http.StatusOK) + + // object name was "prefix/myobject", an attempt to delete "prefix" + // Should not delete "prefix/myobject" + request, err = newTestSignedRequest(http.MethodDelete, getDeleteObjectURL(s.endPoint, bucketName, "prefix"), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusNoContent) + + // create http request to HEAD on the object. + // this helps to validate the existence of the bucket. + request, err = newTestSignedRequest(http.MethodHead, getHeadObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + response, err = s.client.Do(request) + c.Assert(err, nil) + // Assert the HTTP response status code. + c.Assert(response.StatusCode, http.StatusOK) + + // create HTTP request to delete the object. + request, err = newTestSignedRequest(http.MethodDelete, getDeleteObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the http request. + response, err = s.client.Do(request) + c.Assert(err, nil) + // assert the http response status code. + c.Assert(response.StatusCode, http.StatusNoContent) + + // Delete of non-existent data should return success. + request, err = newTestSignedRequest(http.MethodDelete, getDeleteObjectURL(s.endPoint, bucketName, "prefix/myobject1"), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the http request. + response, err = s.client.Do(request) + c.Assert(err, nil) + // assert the http response status. + c.Assert(response.StatusCode, http.StatusNoContent) +} + +// TestNonExistentBucket - Asserts response for HEAD on non-existent bucket. +func (s *TestSuiteCommon) TestNonExistentBucket(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // create request to HEAD on the bucket. + // HEAD on an bucket helps validate the existence of the bucket. + request, err := newTestSignedRequest(http.MethodHead, getHEADBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the http request. + response, err := s.client.Do(request) + c.Assert(err, nil) + // Assert the response. + c.Assert(response.StatusCode, http.StatusNotFound) +} + +// TestEmptyObject - Asserts the response for operation on a 0 byte object. +func (s *TestSuiteCommon) TestEmptyObject(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the http request. + response, err := s.client.Do(request) + c.Assert(err, nil) + // assert the http response status code. + c.Assert(response.StatusCode, http.StatusOK) + + objectName := "test-object" + // construct http request for uploading the object. + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the upload request. + response, err = s.client.Do(request) + c.Assert(err, nil) + // assert the http response. + c.Assert(response.StatusCode, http.StatusOK) + + // make HTTP request to fetch the object. + request, err = newTestSignedRequest(http.MethodGet, getGetObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the http request to fetch object. + response, err = s.client.Do(request) + c.Assert(err, nil) + // assert the http response status code. + c.Assert(response.StatusCode, http.StatusOK) + + var buffer bytes.Buffer + // extract the body of the response. + responseBody, err := io.ReadAll(response.Body) + c.Assert(err, nil) + // assert the http response body content. + c.Assert(true, bytes.Equal(responseBody, buffer.Bytes())) +} + +func (s *TestSuiteCommon) TestBucket(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + request, err = newTestSignedRequest(http.MethodHead, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) +} + +// Tests get anonymous object. +func (s *TestSuiteCommon) TestObjectGetAnonymous(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + buffer := bytes.NewReader([]byte("hello world")) + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the make bucket http request. + response, err := s.client.Do(request) + c.Assert(err, nil) + // assert the response http status code. + c.Assert(response.StatusCode, http.StatusOK) + + objectName := "testObject" + // create HTTP request to upload the object. + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buffer.Len()), buffer, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to upload the object. + response, err = s.client.Do(request) + c.Assert(err, nil) + // assert the HTTP response status code. + c.Assert(response.StatusCode, http.StatusOK) + + // initiate anonymous HTTP request to fetch the object which does not exist. We need to return AccessDenied. + response, err = s.client.Get(getGetObjectURL(s.endPoint, bucketName, objectName+".1")) + c.Assert(err, nil) + // assert the http response status code. + verifyError(c, response, "AccessDenied", "Access Denied.", http.StatusForbidden) + + // initiate anonymous HTTP request to fetch the object which does exist. We need to return AccessDenied. + response, err = s.client.Get(getGetObjectURL(s.endPoint, bucketName, objectName)) + c.Assert(err, nil) + // assert the http response status code. + verifyError(c, response, "AccessDenied", "Access Denied.", http.StatusForbidden) +} + +// TestMultipleObjects - Validates upload and fetching of multiple object into the bucket. +func (s *TestSuiteCommon) TestMultipleObjects(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create the bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // constructing HTTP request to fetch a non-existent object. + // expected to fail, error response asserted for expected error values later. + objectName := "testObject" + request, err = newTestSignedRequest(http.MethodGet, getGetObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + // Asserting the error response with the expected values. + verifyError(c, response, "NoSuchKey", "The specified key does not exist.", http.StatusNotFound) + + objectName = "testObject1" + // content for the object to be uploaded. + buffer1 := bytes.NewReader([]byte("hello one")) + // create HTTP request for the object upload. + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request for object upload. + response, err = s.client.Do(request) + c.Assert(err, nil) + // assert the returned values. + c.Assert(response.StatusCode, http.StatusOK) + + // create HTTP request to fetch the object which was uploaded above. + request, err = newTestSignedRequest(http.MethodGet, getGetObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + // assert whether 200 OK response status is obtained. + c.Assert(response.StatusCode, http.StatusOK) + + // extract the response body. + responseBody, err := io.ReadAll(response.Body) + c.Assert(err, nil) + // assert the content body for the expected object data. + c.Assert(true, bytes.Equal(responseBody, []byte("hello one"))) + + // data for new object to be uploaded. + buffer2 := bytes.NewReader([]byte("hello two")) + objectName = "testObject2" + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buffer2.Len()), buffer2, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request for object upload. + response, err = s.client.Do(request) + c.Assert(err, nil) + // assert the response status code for expected value 200 OK. + c.Assert(response.StatusCode, http.StatusOK) + // fetch the object which was uploaded above. + request, err = newTestSignedRequest(http.MethodGet, getGetObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to fetch the object. + response, err = s.client.Do(request) + c.Assert(err, nil) + // assert the response status code for expected value 200 OK. + c.Assert(response.StatusCode, http.StatusOK) + + // verify response data + responseBody, err = io.ReadAll(response.Body) + c.Assert(err, nil) + c.Assert(true, bytes.Equal(responseBody, []byte("hello two"))) + + // data for new object to be uploaded. + buffer3 := bytes.NewReader([]byte("hello three")) + objectName = "testObject3" + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buffer3.Len()), buffer3, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + // verify the response code with the expected value of 200 OK. + c.Assert(response.StatusCode, http.StatusOK) + + // fetch the object which was uploaded above. + request, err = newTestSignedRequest(http.MethodGet, getPutObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // verify object. + responseBody, err = io.ReadAll(response.Body) + c.Assert(err, nil) + c.Assert(true, bytes.Equal(responseBody, []byte("hello three"))) +} + +// TestHeader - Validates the error response for an attempt to fetch non-existent object. +func (s *TestSuiteCommon) TestHeader(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // obtain HTTP request to fetch an object from non-existent bucket/object. + request, err := newTestSignedRequest(http.MethodGet, getGetObjectURL(s.endPoint, bucketName, "testObject"), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + response, err := s.client.Do(request) + c.Assert(err, nil) + // asserting for the expected error response. + verifyError(c, response, "NoSuchBucket", "The specified bucket does not exist", http.StatusNotFound) +} + +func (s *TestSuiteCommon) TestPutBucket(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // Block 1: Testing for racy access + // The assertion is removed from this block since the purpose of this block is to find races + // The purpose this block is not to check for correctness of functionality + // Run the test with -race flag to utilize this + var wg sync.WaitGroup + for range testConcurrencyLevel { + wg.Add(1) + go func() { + defer wg.Done() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + response, err := s.client.Do(request) + if err != nil { + c.Errorf("Put bucket Failed: %s", err) + return + } + defer response.Body.Close() + }() + } + wg.Wait() + + bucketName = getRandomBucketName() + // Block 2: testing for correctness of the functionality + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + response.Body.Close() +} + +// TestCopyObject - Validates copy object. +// The following is the test flow. +// 1. Create bucket. +// 2. Insert Object. +// 3. Use "X-Amz-Copy-Source" header to copy the previously created object. +// 4. Validate the content of copied object. +func (s *TestSuiteCommon) TestCopyObject(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // content for the object to be created. + buffer1 := bytes.NewReader([]byte("hello world")) + objectName := "testObject" + // create HTTP request for object upload. + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey, s.signer) + request.Header.Set("Content-Type", "application/json") + if s.signer == signerV2 { + c.Assert(err, nil) + err = signRequestV2(request, s.accessKey, s.secretKey) + } + c.Assert(err, nil) + // execute the HTTP request for object upload. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + objectName2 := "testObject2" + // Unlike the actual PUT object request, the request to Copy Object doesn't contain request body, + // empty body with the "X-Amz-Copy-Source" header pointing to the object to copies it in the backend. + request, err = newTestRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName2), 0, nil) + c.Assert(err, nil) + // setting the "X-Amz-Copy-Source" to allow copying the content of previously uploaded object. + request.Header.Set("X-Amz-Copy-Source", url.QueryEscape(SlashSeparator+bucketName+SlashSeparator+objectName)) + if s.signer == signerV4 { + err = signRequestV4(request, s.accessKey, s.secretKey) + } else { + err = signRequestV2(request, s.accessKey, s.secretKey) + } + c.Assert(err, nil) + // execute the HTTP request. + // the content is expected to have the content of previous disk. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // creating HTTP request to fetch the previously uploaded object. + request, err = newTestSignedRequest(http.MethodGet, getGetObjectURL(s.endPoint, bucketName, objectName2), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // executing the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + // validating the response status code. + c.Assert(response.StatusCode, http.StatusOK) + // reading the response body. + // response body is expected to have the copied content of the first uploaded object. + object, err := io.ReadAll(response.Body) + c.Assert(err, nil) + c.Assert(string(object), "hello world") +} + +// TestPutObject - Tests successful put object request. +func (s *TestSuiteCommon) TestPutObject(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // content for new object upload. + buffer1 := bytes.NewReader([]byte("hello world")) + objectName := "testObject" + // creating HTTP request for object upload. + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request for object upload. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // fetch the object back and verify its contents. + request, err = newTestSignedRequest(http.MethodGet, getGetObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request to fetch the object. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + c.Assert(response.ContentLength, int64(len([]byte("hello world")))) + var buffer2 bytes.Buffer + // retrieve the contents of response body. + n, err := io.Copy(&buffer2, response.Body) + c.Assert(err, nil) + c.Assert(n, int64(len([]byte("hello world")))) + // asserted the contents of the fetched object with the expected result. + c.Assert(true, bytes.Equal(buffer2.Bytes(), []byte("hello world"))) + + // Test the response when object name ends with a slash. + // This is a special case with size as '0' and object ends with + // a slash separator, we treat it like a valid operation and + // return success. + // The response Etag headers should contain Md5Sum of empty string. + objectName = "objectwith/" + // create HTTP request for object upload. + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + if s.signer == signerV2 { + c.Assert(err, nil) + err = signRequestV2(request, s.accessKey, s.secretKey) + } + c.Assert(err, nil) + // execute the HTTP request for object upload. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + // The response Etag header should contain Md5sum of an empty string. + c.Assert(response.Header.Get(xhttp.ETag), "\""+emptyETag+"\"") +} + +// TestListBuckets - Make request for listing of all buckets. +// XML response is parsed. +// Its success verifies the format of the response. +func (s *TestSuiteCommon) TestListBuckets(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request to list buckets. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // create HTTP request for listing buckets. + request, err = newTestSignedRequest(http.MethodGet, getListBucketURL(s.endPoint), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to list buckets. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + var results ListBucketsResponse + // parse the list bucket response. + decoder := xml.NewDecoder(response.Body) + err = decoder.Decode(&results) + // validating that the xml-decoding/parsing was successful. + c.Assert(err, nil) + + // Fetch the bucket created above + var createdBucket Bucket + for _, b := range results.Buckets.Buckets { + if b.Name == bucketName { + createdBucket = b + } + } + c.Assert(createdBucket.Name != "", true) + + // Parse the bucket modtime + creationTime, err := time.Parse(iso8601TimeFormat, createdBucket.CreationDate) + c.Assert(err, nil) + + // Check if bucket modtime is consistent (not less than current time and not late more than 5 minutes) + timeNow := time.Now().UTC() + c.Assert(creationTime.Before(timeNow), true) + c.Assert(timeNow.Sub(creationTime) < time.Minute*5, true) +} + +// This tests validate if PUT handler can successfully detect signature mismatch. +func (s *TestSuiteCommon) TestValidateSignature(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // Execute the HTTP request to create bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + objName := "test-object" + + // Body is on purpose set to nil so that we get payload generated for empty bytes. + + // Create new HTTP request with incorrect secretKey to generate an incorrect signature. + secretKey := s.secretKey + "a" + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objName), 0, nil, s.accessKey, secretKey, s.signer) + c.Assert(err, nil) + response, err = s.client.Do(request) + c.Assert(err, nil) + verifyError(c, response, "SignatureDoesNotMatch", "The request signature we calculated does not match the signature you provided. Check your key and signing method.", http.StatusForbidden) +} + +// This tests validate if PUT handler can successfully detect SHA256 mismatch. +func (s *TestSuiteCommon) TestSHA256Mismatch(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // Execute the HTTP request to create bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + objName := "test-object" + + // Body is on purpose set to nil so that we get payload generated for empty bytes. + + // Create new HTTP request with incorrect secretKey to generate an incorrect signature. + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objName), 0, nil, s.accessKey, s.secretKey, s.signer) + if s.signer == signerV4 { + c.Assert(request.Header.Get("x-amz-content-sha256"), "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") + } + // Set the body to generate signature mismatch. + helloReader := bytes.NewReader([]byte("Hello, World")) + request.ContentLength = helloReader.Size() + request.Body = io.NopCloser(helloReader) + c.Assert(err, nil) + + // execute the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + if s.signer == signerV4 { + verifyError(c, response, "XAmzContentSHA256Mismatch", "The provided 'x-amz-content-sha256' header does not match what was computed.", http.StatusBadRequest) + } +} + +// TestPutObjectLongName - Validates the error response +// on an attempt to upload an object with long name. +func (s *TestSuiteCommon) TestPutObjectLongName(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // Execute the HTTP request to create bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + // Content for the object to be uploaded. + buffer := bytes.NewReader([]byte("hello world")) + // make long object name. + longObjName := fmt.Sprintf("%0255d/%0255d/%0255d", 1, 1, 1) + if IsDocker() || IsKubernetes() { + longObjName = fmt.Sprintf("%0242d/%0242d/%0242d", 1, 1, 1) + } + // create new HTTP request to insert the object. + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, longObjName), + int64(buffer.Len()), buffer, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // make long object name. + longObjName = fmt.Sprintf("%0255d/%0255d/%0255d/%0255d/%0255d", 1, 1, 1, 1, 1) + if IsDocker() || IsKubernetes() { + longObjName = fmt.Sprintf("%0242d/%0242d/%0242d/%0242d/%0242d", 1, 1, 1, 1, 1) + } + // create new HTTP request to insert the object. + buffer = bytes.NewReader([]byte("hello world")) + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, longObjName), + int64(buffer.Len()), buffer, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusBadRequest) + verifyError(c, response, "KeyTooLongError", "Your key is too long", http.StatusBadRequest) + + // make object name as unsupported + longObjName = fmt.Sprintf("%0256d", 1) + buffer = bytes.NewReader([]byte("hello world")) + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, longObjName), + int64(buffer.Len()), buffer, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + response, err = s.client.Do(request) + c.Assert(err, nil) + verifyError(c, response, "XMinioInvalidObjectName", "Object name contains unsupported characters.", http.StatusBadRequest) +} + +// TestNotBeAbleToCreateObjectInNonexistentBucket - Validates the error response +// on an attempt to upload an object into a non-existent bucket. +func (s *TestSuiteCommon) TestNotBeAbleToCreateObjectInNonexistentBucket(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // content of the object to be uploaded. + buffer1 := bytes.NewReader([]byte("hello world")) + + // preparing for upload by generating the upload URL. + objectName := "test-object" + request, err := newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // Execute the HTTP request. + response, err := s.client.Do(request) + c.Assert(err, nil) + // Assert the response error message. + verifyError(c, response, "NoSuchBucket", "The specified bucket does not exist", http.StatusNotFound) +} + +// TestHeadOnObjectLastModified - Asserts response for HEAD on an object. +// HEAD requests on an object validates the existence of the object. +// The responses for fetching the object when If-Modified-Since +// and If-Unmodified-Since headers set are validated. +// If-Modified-Since - Return the object only if it has been modified since the specified time, else return a 304 (not modified). +// If-Unmodified-Since - Return the object only if it has not been modified since the specified time, else return a 412 (precondition failed). +func (s *TestSuiteCommon) TestHeadOnObjectLastModified(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // preparing for object upload. + objectName := "test-object" + // content for the object to be uploaded. + buffer1 := bytes.NewReader([]byte("hello world")) + // obtaining URL for uploading the object. + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // executing the HTTP request to download the object. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + // make HTTP request to obtain object info. + request, err = newTestSignedRequest(http.MethodHead, getHeadObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + // verify the status of the HTTP response. + c.Assert(response.StatusCode, http.StatusOK) + + // retrieve the info of last modification time of the object from the response header. + lastModified := response.Header.Get("Last-Modified") + // Parse it into time.Time structure. + t, err := time.Parse(http.TimeFormat, lastModified) + c.Assert(err, nil) + + // make HTTP request to obtain object info. + // But this time set the "If-Modified-Since" header to be 10 minute more than the actual + // last modified time of the object. + request, err = newTestSignedRequest(http.MethodHead, getHeadObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + request.Header.Set("If-Modified-Since", t.Add(10*time.Minute).UTC().Format(http.TimeFormat)) + response, err = s.client.Do(request) + c.Assert(err, nil) + // Since the "If-Modified-Since" header was ahead in time compared to the actual + // modified time of the object expecting the response status to be http.StatusNotModified. + c.Assert(response.StatusCode, http.StatusNotModified) + + // Again, obtain the object info. + // This time setting "If-Unmodified-Since" to a time after the object is modified. + // As documented above, expecting http.StatusPreconditionFailed. + request, err = newTestSignedRequest(http.MethodHead, getHeadObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + request.Header.Set("If-Unmodified-Since", t.Add(-10*time.Minute).UTC().Format(http.TimeFormat)) + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusPreconditionFailed) + + // make HTTP request to obtain object info. + // But this time set a date with unrecognized format to the "If-Modified-Since" header + request, err = newTestSignedRequest(http.MethodHead, getHeadObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + request.Header.Set("If-Unmodified-Since", "Mon, 02 Jan 2006 15:04:05 +00:00") + response, err = s.client.Do(request) + c.Assert(err, nil) + // Since the "If-Modified-Since" header was ahead in time compared to the actual + // modified time of the object expecting the response status to be http.StatusNotModified. + c.Assert(response.StatusCode, http.StatusOK) +} + +// TestHeadOnBucket - Validates response for HEAD on the bucket. +// HEAD request on the bucket validates the existence of the bucket. +func (s *TestSuiteCommon) TestHeadOnBucket(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getHEADBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + // make HEAD request on the bucket. + request, err = newTestSignedRequest(http.MethodHead, getHEADBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + // Asserting the response status for expected value of http.StatusOK. + c.Assert(response.StatusCode, http.StatusOK) +} + +// TestContentTypePersists - Object upload with different Content-type is first done. +// And then a HEAD and GET request on these objects are done to validate if the same Content-Type set during upload persists. +func (s *TestSuiteCommon) TestContentTypePersists(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // Uploading a new object with Content-Type "image/png". + // content for the object to be uploaded. + buffer1 := bytes.NewReader([]byte("hello world")) + objectName := "test-object.png" + // constructing HTTP request for object upload. + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + request.Header.Set("Content-Type", "image/png") + if s.signer == signerV2 { + err = signRequestV2(request, s.accessKey, s.secretKey) + c.Assert(err, nil) + } + + // execute the HTTP request for object upload. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // Fetching the object info using HEAD request for the object which was uploaded above. + request, err = newTestSignedRequest(http.MethodHead, getHeadObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // Execute the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + // Verify if the Content-Type header is set during the object persists. + c.Assert(response.Header.Get("Content-Type"), "image/png") + + // Fetching the object itself and then verify the Content-Type header. + request, err = newTestSignedRequest(http.MethodGet, getGetObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // Execute the HTTP to fetch the object. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + // Verify if the Content-Type header is set during the object persists. + c.Assert(response.Header.Get("Content-Type"), "image/png") + + // Uploading a new object with Content-Type "application/json". + objectName = "test-object.json" + buffer2 := bytes.NewReader([]byte("hello world")) + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buffer2.Len()), buffer2, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // setting the request header to be application/json. + request.Header.Set("Content-Type", "application/json") + if s.signer == signerV2 { + err = signRequestV2(request, s.accessKey, s.secretKey) + c.Assert(err, nil) + } + + // Execute the HTTP request to upload the object. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // Obtain the info of the object which was uploaded above using HEAD request. + request, err = newTestSignedRequest(http.MethodHead, getHeadObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // Execute the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + // Assert if the content-type header set during the object upload persists. + c.Assert(response.Header.Get("Content-Type"), "application/json") + + // Fetch the object and assert whether the Content-Type header persists. + request, err = newTestSignedRequest(http.MethodGet, getGetObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // Execute the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + // Assert if the content-type header set during the object upload persists. + c.Assert(response.Header.Get("Content-Type"), "application/json") +} + +// TestPartialContent - Validating for GetObject with partial content request. +// By setting the Range header, A request to send specific bytes range of data from an +// already uploaded object can be done. +func (s *TestSuiteCommon) TestPartialContent(c *check) { + bucketName := getRandomBucketName() + + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + buffer1 := bytes.NewReader([]byte("Hello World")) + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, "bar"), + int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // Prepare request + request, err = newTestSignedRequest(http.MethodGet, getGetObjectURL(s.endPoint, bucketName, "bar"), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + request.Header.Set("Range", "bytes=6-7") + + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusPartialContent) + partialObject, err := io.ReadAll(response.Body) + c.Assert(err, nil) + + c.Assert(string(partialObject), "Wo") +} + +// TestListObjectsHandler - Setting valid parameters to List Objects +// and then asserting the response with the expected one. +func (s *TestSuiteCommon) TestListObjectsHandler(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + for _, objectName := range []string{"foo bar 1", "foo bar 2", "obj2", "obj2/"} { + buffer := bytes.NewReader([]byte("Hello World")) + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buffer.Len()), buffer, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + } + + testCases := []struct { + getURL string + expectedStrings []string + }{ + {getListObjectsV1URL(s.endPoint, bucketName, "", "1000", ""), []string{"foo bar 1", "foo bar 2"}}, + {getListObjectsV1URL(s.endPoint, bucketName, "", "1000", "url"), []string{"foo+bar+1", "foo+bar+2"}}, + { + getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "", "", ""), + []string{ + "foo bar 1", + "foo bar 2", + }, + }, + { + getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "true", "", ""), + []string{ + "foo bar 1", + "foo bar 2", + fmt.Sprintf("%sminio", globalMinioDefaultOwnerID), + }, + }, + {getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "", "url", ""), []string{"foo+bar+1", "foo+bar+2"}}, + { + getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "", "", ""), + []string{ + "obj2", + "obj2/", + }, + }, + } + + for _, testCase := range testCases { + // create listObjectsV1 request with valid parameters + request, err = newTestSignedRequest(http.MethodGet, testCase.getURL, 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + getContent, err := io.ReadAll(response.Body) + c.Assert(err, nil) + + for _, expectedStr := range testCase.expectedStrings { + c.Assert(strings.Contains(string(getContent), expectedStr), true) + } + } +} + +// TestListObjectsV2HadoopUAHandler - Test ListObjectsV2 call with max-keys=2 and Hadoop User-Agent +func (s *TestSuiteCommon) TestListObjectsV2HadoopUAHandler(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // enable versioning on the bucket. + enableVersioningBody := []byte("Enabled") + enableVersioningBucketRequest, err := newTestSignedRequest(http.MethodPut, getBucketVersioningConfigURL(s.endPoint, bucketName), + int64(len(enableVersioningBody)), bytes.NewReader(enableVersioningBody), s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request to create bucket. + response, err = s.client.Do(enableVersioningBucketRequest) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + for _, objectName := range []string{"pfx/a/1.txt", "pfx/b/2.txt", "pfx/", "pfx2/c/3.txt", "pfx2/d/3.txt", "pfx1/1.txt", "pfx2/", "pfx3/", "pfx4/"} { + buffer := bytes.NewReader([]byte("")) + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buffer.Len()), buffer, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + } + for _, objectName := range []string{"pfx2/c/3.txt", "pfx2/d/3.txt", "pfx2/", "pfx3/"} { + delRequest, err := newTestSignedRequest(http.MethodDelete, getDeleteObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + response, err = s.client.Do(delRequest) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusNoContent) + } + testCases := []struct { + getURL string + expectedStrings []string + userAgent string + }{ + { + getListObjectsV2URL(s.endPoint, bucketName, "pfx/a/", "2", "", "", "/"), + []string{ + "pfx/a/1.txt", + }, + "Hadoop 3.3.2, aws-sdk-java/1.12.262 Linux/5.14.0-362.24.1.el9_3.x86_64 OpenJDK_64-Bit_Server_VM/11.0.22+7 java/11.0.22 scala/2.12.15 vendor/Eclipse_Adoptium cfg/retry-mode/legacy", + }, + { + getListObjectsV2URL(s.endPoint, bucketName, "pfx/a/", "2", "", "", "/"), + []string{ + "pfx/a/", + "pfx/a/1.txt", + }, + "", + }, + { + getListObjectsV2URL(s.endPoint, bucketName, "pfx2/c", "2", "true", "", "/"), + []string{ + "pfx2/c12/falsepfx2/c/", + }, + "", + }, + + { + getListObjectsV2URL(s.endPoint, bucketName, "pfx2/c", "2", "true", "", "/"), + []string{ + "pfx2/c12/false", + "pfx2/c/", + }, + "Hadoop 3.3.2, aws-sdk-java/1.12.262 Linux/5.14.0-362.24.1.el9_3.x86_64 OpenJDK_64-Bit_Server_VM/11.0.22+7 java/11.0.22 scala/2.12.15 vendor/Eclipse_Adoptium cfg/retry-mode/legacy", + }, + { + getListObjectsV2URL(s.endPoint, bucketName, "pfx2/c", "2", "true", "", "/"), + []string{ + "pfx2/c12/false", + "pfx2/c/", + }, + "Hadoop 3.3.2, aws-sdk-java/1.12.262 Linux/5.14.0-362.24.1.el9_3.x86_64 OpenJDK_64-Bit_Server_VM/11.0.22+7 java/11.0.22 scala/2.12.15 vendor/Eclipse_Adoptium cfg/retry-mode/legacy", + }, + { + getListObjectsV2URL(s.endPoint, bucketName, "pfx2/", "2", "false", "", "/"), + []string{ + "pfx2/22/false", + "pfx2/c/pfx2/d/", + }, + "Hadoop 3.3.2, aws-sdk-java/1.12.262 Linux/5.14.0-362.24.1.el9_3.x86_64 OpenJDK_64-Bit_Server_VM/11.0.22+7 java/11.0.22 scala/2.12.15 vendor/Eclipse_Adoptium cfg/retry-mode/legacy", + }, + { + getListObjectsV2URL(s.endPoint, bucketName, "pfx2/", "2", "false", "", "/"), + []string{ + "pfx2/c/", + }, + "", + }, + { + getListObjectsV2URL(s.endPoint, bucketName, "pfx2/", "2", "false", "", "/"), + []string{ + "pfx2/22/false", + "pfx2/c/pfx2/d/", + }, + "Hadoop 3.3.2, aws-sdk-java/1.12.262 Linux/5.14.0-362.24.1.el9_3.x86_64 OpenJDK_64-Bit_Server_VM/11.0.22+7 java/11.0.22 scala/2.12.15 vendor/Eclipse_Adoptium cfg/retry-mode/legacy", + }, + { + getListObjectsV2URL(s.endPoint, bucketName, "pfx3/", "2", "false", "", "/"), + []string{ + "pfx3/02/false", + }, + "Hadoop 3.3.2, aws-sdk-java/1.12.262 Linux/5.14.0-362.24.1.el9_3.x86_64 OpenJDK_64-Bit_Server_VM/11.0.22+7 java/11.0.22 scala/2.12.15 vendor/Eclipse_Adoptium cfg/retry-mode/legacy", + }, + { + getListObjectsV2URL(s.endPoint, bucketName, "pfx4/", "2", "false", "", "/"), + []string{ + "pfx4/12/falsepfx4/", + }, + "Hadoop 3.3.2, aws-sdk-java/1.12.262 Linux/5.14.0-362.24.1.el9_3.x86_64 OpenJDK_64-Bit_Server_VM/11.0.22+7 java/11.0.22 scala/2.12.15 vendor/Eclipse_Adoptium cfg/retry-mode/legacy", + }, + { + getListObjectsV2URL(s.endPoint, bucketName, "pfx3/", "2", "false", "", "/"), + []string{ + "pfx3/02/false", + }, + "", + }, + { + getListObjectsV2URL(s.endPoint, bucketName, "pfx4/", "2", "false", "", "/"), + []string{ + "pfx4/12/falsepfx4/", + }, + "", + }, + } + for _, testCase := range testCases { + // create listObjectsV1 request with valid parameters + request, err = newTestSignedRequest(http.MethodGet, testCase.getURL, 0, nil, s.accessKey, s.secretKey, s.signer) + request.Header.Set("User-Agent", testCase.userAgent) + c.Assert(err, nil) + // execute the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + getContent, err := io.ReadAll(response.Body) + c.Assert(err, nil) + + for _, expectedStr := range testCase.expectedStrings { + c.Assert(strings.Contains(string(getContent), expectedStr), true) + } + } +} + +// TestListObjectVersionsHandler - checks the order of +// and XML tags in a version listing +func (s *TestSuiteCommon) TestListObjectVersionsOutputOrderHandler(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + makeBucketRequest, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request to create bucket. + response, err := s.client.Do(makeBucketRequest) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // HTTP request to create the bucket. + enableVersioningBody := []byte("Enabled") + enableVersioningBucketRequest, err := newTestSignedRequest(http.MethodPut, getBucketVersioningConfigURL(s.endPoint, bucketName), + int64(len(enableVersioningBody)), bytes.NewReader(enableVersioningBody), s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request to create bucket. + response, err = s.client.Do(enableVersioningBucketRequest) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + for _, objectName := range []string{"file.1", "file.2"} { + buffer := bytes.NewReader([]byte("testcontent")) + putRequest, err := newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buffer.Len()), buffer, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + response, err = s.client.Do(putRequest) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + delRequest, err := newTestSignedRequest(http.MethodDelete, getDeleteObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + response, err = s.client.Do(delRequest) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusNoContent) + } + + // create listObjectsV1 request with valid parameters + request, err := newTestSignedRequest(http.MethodGet, getListObjectVersionsURL(s.endPoint, bucketName, "", "1000", ""), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + getContent, err := io.ReadAll(response.Body) + c.Assert(err, nil) + + r := regexp.MustCompile( + `.*` + + `file.1.*true.*` + + `file.1.*false.*` + + `file.2.*true.*` + + `file.2.*false.*` + + ``) + + c.Assert(r.MatchString(string(getContent)), true) +} + +// TestListObjectsSpecialCharactersHandler - Setting valid parameters to List Objects +// and then asserting the response with the expected one. +func (s *TestSuiteCommon) TestListObjectsSpecialCharactersHandler(c *check) { + if runtime.GOOS == globalWindowsOSName { + c.Skip("skip special character test for windows") + } + + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + for _, objectName := range []string{"foo bar 1", "foo bar 2", "foo \x01 bar"} { + buffer := bytes.NewReader([]byte("Hello World")) + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buffer.Len()), buffer, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + } + + testCases := []struct { + getURL string + expectedStrings []string + }{ + {getListObjectsV1URL(s.endPoint, bucketName, "", "1000", ""), []string{"foo bar 1", "foo bar 2", "foo  bar"}}, + {getListObjectsV1URL(s.endPoint, bucketName, "", "1000", "url"), []string{"foo+bar+1", "foo+bar+2", "foo+%01+bar"}}, + { + getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "", "", ""), + []string{ + "foo bar 1", + "foo bar 2", + "foo  bar", + fmt.Sprintf("%sminio", globalMinioDefaultOwnerID), + }, + }, + { + getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "true", "", ""), + []string{ + "foo bar 1", + "foo bar 2", + "foo  bar", + fmt.Sprintf("%sminio", globalMinioDefaultOwnerID), + }, + }, + {getListObjectsV2URL(s.endPoint, bucketName, "", "1000", "", "url", ""), []string{"foo+bar+1", "foo+bar+2", "foo+%01+bar"}}, + } + + for _, testCase := range testCases { + // create listObjectsV1 request with valid parameters + request, err = newTestSignedRequest(http.MethodGet, testCase.getURL, 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + getContent, err := io.ReadAll(response.Body) + c.Assert(err, nil) + + for _, expectedStr := range testCase.expectedStrings { + c.Assert(strings.Contains(string(getContent), expectedStr), true) + } + } +} + +// TestListObjectsHandlerErrors - Setting invalid parameters to List Objects +// and then asserting the error response with the expected one. +func (s *TestSuiteCommon) TestListObjectsHandlerErrors(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // create listObjectsV1 request with invalid value of max-keys parameter. max-keys is set to -2. + request, err = newTestSignedRequest(http.MethodGet, getListObjectsV1URL(s.endPoint, bucketName, "", "-2", ""), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + // validating the error response. + verifyError(c, response, "InvalidArgument", "Argument maxKeys must be an integer between 0 and 2147483647", http.StatusBadRequest) + + // create listObjectsV2 request with invalid value of max-keys parameter. max-keys is set to -2. + request, err = newTestSignedRequest(http.MethodGet, getListObjectsV2URL(s.endPoint, bucketName, "", "-2", "", "", ""), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + // validating the error response. + verifyError(c, response, "InvalidArgument", "Argument maxKeys must be an integer between 0 and 2147483647", http.StatusBadRequest) +} + +// TestPutBucketErrors - request for non valid bucket operation +// and validate it with expected error result. +func (s *TestSuiteCommon) TestPutBucketErrors(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // generating a HTTP request to create bucket. + // using invalid bucket name. + request, err := newTestSignedRequest(http.MethodPut, s.endPoint+"/putbucket-.", + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + response, err := s.client.Do(request) + c.Assert(err, nil) + // expected to fail with error message "InvalidBucketName". + verifyError(c, response, "InvalidBucketName", "The specified bucket is not valid.", http.StatusBadRequest) + // HTTP request to create the bucket. + request, err = newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create bucket. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + // make HTTP request to create the same bucket again. + // expected to fail with error message "BucketAlreadyOwnedByYou". + request, err = newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + response, err = s.client.Do(request) + c.Assert(err, nil) + verifyError(c, response, "BucketAlreadyOwnedByYou", "Your previous request to create the named bucket succeeded and you already own it.", + http.StatusConflict) +} + +func (s *TestSuiteCommon) TestGetObjectLarge10MiB(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // form HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create the bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + var buffer bytes.Buffer + line := `1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,123"` + // Create 10MiB content where each line contains 1024 characters. + for i := range 10 * 1024 { + buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line)) + } + putContent := buffer.String() + + buf := bytes.NewReader([]byte(putContent)) + + objectName := "test-big-object" + // create HTTP request for object upload. + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buf.Len()), buf, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + // Assert the status code to verify successful upload. + c.Assert(response.StatusCode, http.StatusOK) + + // prepare HTTP requests to download the object. + request, err = newTestSignedRequest(http.MethodGet, getPutObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to download the object. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + // extract the content from response body. + getContent, err := io.ReadAll(response.Body) + c.Assert(err, nil) + + // Compare putContent and getContent. + c.Assert(string(getContent), putContent) +} + +// TestGetObjectLarge11MiB - Tests validate fetching of an object of size 11MB. +func (s *TestSuiteCommon) TestGetObjectLarge11MiB(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + var buffer bytes.Buffer + line := `1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,123` + // Create 11MiB content where each line contains 1024 characters. + for i := range 11 * 1024 { + buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line)) + } + putMD5 := getMD5Hash(buffer.Bytes()) + + objectName := "test-11Mb-object" + // Put object + buf := bytes.NewReader(buffer.Bytes()) + // create HTTP request foe object upload. + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buf.Len()), buf, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request for object upload. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // create HTTP request to download the object. + request, err = newTestSignedRequest(http.MethodGet, getGetObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + // fetch the content from response body. + getContent, err := io.ReadAll(response.Body) + c.Assert(err, nil) + + // Get etag of the response content. + getMD5 := getMD5Hash(getContent) + + // Compare putContent and getContent. + c.Assert(putMD5, getMD5) +} + +// TestGetPartialObjectMisAligned - tests get object partially miss-aligned. +// create a large buffer of miss-aligned data and upload it. +// then make partial range requests to while fetching it back and assert the response content. +func (s *TestSuiteCommon) TestGetPartialObjectMisAligned(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create the bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + var buffer bytes.Buffer + // data to be written into buffer. + data := "1234567890" + // seed the random number generator once. + rand.Seed(3) + // generate a random number between 13 and 200. + randInt := getRandomRange(13, 200, -1) + // write into buffer till length of the buffer is greater than the generated random number. + for i := 0; i <= randInt; i += 10 { + buffer.WriteString(data) + } + // String content which is used for put object range test. + putBytes := buffer.Bytes() + putBytes = putBytes[:randInt] + // randomize the order of bytes in the byte array and create a reader. + putBytes = randomizeBytes(putBytes, -1) + buf := bytes.NewReader(putBytes) + putContent := string(putBytes) + objectName := "test-big-file" + // HTTP request to upload the object. + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buf.Len()), buf, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to upload the object. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // test Cases containing data to make partial range requests. + // also has expected response data. + testCases := []struct { + byteRange string + expectedString string + }{ + // request for byte range 10-11. + // expecting the result to contain only putContent[10:12] bytes. + {"10-11", putContent[10:12]}, + // request for object data after the first byte. + {"1-", putContent[1:]}, + // request for object data after the first byte. + {"6-", putContent[6:]}, + // request for last 2 bytes of the object. + {"-2", putContent[len(putContent)-2:]}, + // request for last 7 bytes of the object. + {"-7", putContent[len(putContent)-7:]}, + } + + for _, t := range testCases { + // HTTP request to download the object. + request, err = newTestSignedRequest(http.MethodGet, getGetObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // Get partial content based on the byte range set. + request.Header.Set("Range", "bytes="+t.byteRange) + + // execute the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + // Since only part of the object is requested, expecting response status to be http.StatusPartialContent . + c.Assert(response.StatusCode, http.StatusPartialContent) + // parse the HTTP response body. + getContent, err := io.ReadAll(response.Body) + c.Assert(err, nil) + + // Compare putContent and getContent. + c.Assert(string(getContent), t.expectedString) + } +} + +// TestGetPartialObjectLarge11MiB - Test validates partial content request for a 11MiB object. +func (s *TestSuiteCommon) TestGetPartialObjectLarge11MiB(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create the bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + var buffer bytes.Buffer + line := `234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,123` + // Create 11MiB content where each line contains 1024 + // characters. + for i := range 11 * 1024 { + buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line)) + } + putContent := buffer.String() + + objectName := "test-large-11Mb-object" + + buf := bytes.NewReader([]byte(putContent)) + // HTTP request to upload the object. + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buf.Len()), buf, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to upload the object. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // HTTP request to download the object. + request, err = newTestSignedRequest(http.MethodGet, getGetObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // This range spans into first two blocks. + request.Header.Set("Range", "bytes=10485750-10485769") + + // execute the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + // Since only part of the object is requested, expecting response status to be http.StatusPartialContent . + c.Assert(response.StatusCode, http.StatusPartialContent) + // read the downloaded content from the response body. + getContent, err := io.ReadAll(response.Body) + c.Assert(err, nil) + + // Compare putContent and getContent. + c.Assert(string(getContent), putContent[10485750:10485770]) +} + +// TestGetPartialObjectLarge11MiB - Test validates partial content request for a 10MiB object. +func (s *TestSuiteCommon) TestGetPartialObjectLarge10MiB(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create bucket. + response, err := s.client.Do(request) + // expecting the error to be nil. + c.Assert(err, nil) + // expecting the HTTP response status code to 200 OK. + c.Assert(response.StatusCode, http.StatusOK) + + var buffer bytes.Buffer + line := `1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890,1234567890, + 1234567890,1234567890,1234567890,123` + // Create 10MiB content where each line contains 1024 characters. + for i := range 10 * 1024 { + buffer.WriteString(fmt.Sprintf("[%05d] %s\n", i, line)) + } + + putContent := buffer.String() + buf := bytes.NewReader([]byte(putContent)) + + objectName := "test-big-10Mb-file" + // HTTP request to upload the object. + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buf.Len()), buf, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to upload the object. + response, err = s.client.Do(request) + c.Assert(err, nil) + // verify whether upload was successful. + c.Assert(response.StatusCode, http.StatusOK) + + // HTTP request to download the object. + request, err = newTestSignedRequest(http.MethodGet, getGetObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // Get partial content based on the byte range set. + request.Header.Set("Range", "bytes=2048-2058") + + // execute the HTTP request to download the partial content. + response, err = s.client.Do(request) + c.Assert(err, nil) + // Since only part of the object is requested, expecting response status to be http.StatusPartialContent . + c.Assert(response.StatusCode, http.StatusPartialContent) + // read the downloaded content from the response body. + getContent, err := io.ReadAll(response.Body) + c.Assert(err, nil) + + // Compare putContent and getContent. + c.Assert(string(getContent), putContent[2048:2059]) +} + +// TestGetObjectErrors - Tests validate error response for invalid object operations. +func (s *TestSuiteCommon) TestGetObjectErrors(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + objectName := "test-non-exitent-object" + // HTTP request to download the object. + // Since the specified object doesn't exist in the given bucket, + // expected to fail with error message "NoSuchKey" + request, err = newTestSignedRequest(http.MethodGet, getGetObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + response, err = s.client.Do(request) + c.Assert(err, nil) + verifyError(c, response, "NoSuchKey", "The specified key does not exist.", http.StatusNotFound) + + // request to download an object, but an invalid bucket name is set. + request, err = newTestSignedRequest(http.MethodGet, getGetObjectURL(s.endPoint, "getobjecterrors-.", objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + // expected to fail with "InvalidBucketName". + verifyError(c, response, "InvalidBucketName", "The specified bucket is not valid.", http.StatusBadRequest) +} + +// TestGetObjectRangeErrors - Validate error response when object is fetched with incorrect byte range value. +func (s *TestSuiteCommon) TestGetObjectRangeErrors(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // content for the object to be uploaded. + buffer1 := bytes.NewReader([]byte("Hello World")) + + objectName := "test-object" + // HTTP request to upload the object. + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to upload the object. + response, err = s.client.Do(request) + c.Assert(err, nil) + // verify whether upload was successful. + c.Assert(response.StatusCode, http.StatusOK) + + // HTTP request to download the object. + request, err = newTestSignedRequest(http.MethodGet, getGetObjectURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + // Invalid byte range set. + request.Header.Set("Range", "bytes=-0") + c.Assert(err, nil) + + // execute the HTTP request. + response, err = s.client.Do(request) + c.Assert(err, nil) + // expected to fail with "InvalidRange" error message. + verifyError(c, response, "InvalidRange", "The requested range is not satisfiable", http.StatusRequestedRangeNotSatisfiable) +} + +// TestObjectMultipartAbort - Test validates abortion of a multipart upload after uploading 2 parts. +func (s *TestSuiteCommon) TestObjectMultipartAbort(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + objectName := "test-multipart-object" + + // 1. Initiate 2 uploads for the same object + // 2. Upload 2 parts for the second upload + // 3. Abort the second upload. + // 4. Abort the first upload. + // This will test abort upload when there are more than one upload IDs + // and the case where there is only one upload ID. + + // construct HTTP request to initiate a NewMultipart upload. + request, err = newTestSignedRequest(http.MethodPost, getNewMultipartURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request initiating the new multipart upload. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // parse the response body and obtain the new upload ID. + decoder := xml.NewDecoder(response.Body) + newResponse := &InitiateMultipartUploadResponse{} + + err = decoder.Decode(newResponse) + c.Assert(err, nil) + c.Assert(len(newResponse.UploadID) > 0, true) + + // construct HTTP request to initiate a NewMultipart upload. + request, err = newTestSignedRequest(http.MethodPost, getNewMultipartURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request initiating the new multipart upload. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // parse the response body and obtain the new upload ID. + decoder = xml.NewDecoder(response.Body) + newResponse = &InitiateMultipartUploadResponse{} + + err = decoder.Decode(newResponse) + c.Assert(err, nil) + c.Assert(len(newResponse.UploadID) > 0, true) + // uploadID to be used for rest of the multipart operations on the object. + uploadID := newResponse.UploadID + + // content for the part to be uploaded. + buffer1 := bytes.NewReader([]byte("hello world")) + // HTTP request for the part to be uploaded. + request, err = newTestSignedRequest(http.MethodPut, getPartUploadURL(s.endPoint, bucketName, objectName, uploadID, "1"), + int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request to upload the first part. + response1, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response1.StatusCode, http.StatusOK) + + // content for the second part to be uploaded. + buffer2 := bytes.NewReader([]byte("hello world")) + // HTTP request for the second part to be uploaded. + request, err = newTestSignedRequest(http.MethodPut, getPartUploadURL(s.endPoint, bucketName, objectName, uploadID, "2"), + int64(buffer2.Len()), buffer2, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request to upload the second part. + response2, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response2.StatusCode, http.StatusOK) + // HTTP request for aborting the multipart upload. + request, err = newTestSignedRequest(http.MethodDelete, getAbortMultipartUploadURL(s.endPoint, bucketName, objectName, uploadID), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request to abort the multipart upload. + response3, err := s.client.Do(request) + c.Assert(err, nil) + // expecting the response status code to be http.StatusNoContent. + // The assertion validates the success of Abort Multipart operation. + c.Assert(response3.StatusCode, http.StatusNoContent) +} + +// TestBucketMultipartList - Initiates a NewMultipart upload, uploads parts and validates listing of the parts. +func (s *TestSuiteCommon) TestBucketMultipartList(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), 0, + nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, 200) + + objectName := "test-multipart-object" + // construct HTTP request to initiate a NewMultipart upload. + request, err = newTestSignedRequest(http.MethodPost, getNewMultipartURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request initiating the new multipart upload. + response, err = s.client.Do(request) + c.Assert(err, nil) + // expecting the response status code to be http.StatusOK(200 OK) . + c.Assert(response.StatusCode, http.StatusOK) + + // parse the response body and obtain the new upload ID. + decoder := xml.NewDecoder(response.Body) + newResponse := &InitiateMultipartUploadResponse{} + + err = decoder.Decode(newResponse) + c.Assert(err, nil) + c.Assert(len(newResponse.UploadID) > 0, true) + // uploadID to be used for rest of the multipart operations on the object. + uploadID := newResponse.UploadID + + // content for the part to be uploaded. + buffer1 := bytes.NewReader([]byte("hello world")) + // HTTP request for the part to be uploaded. + request, err = newTestSignedRequest(http.MethodPut, getPartUploadURL(s.endPoint, bucketName, objectName, uploadID, "1"), + int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request to upload the first part. + response1, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response1.StatusCode, http.StatusOK) + + // content for the second part to be uploaded. + buffer2 := bytes.NewReader([]byte("hello world")) + // HTTP request for the second part to be uploaded. + request, err = newTestSignedRequest(http.MethodPut, getPartUploadURL(s.endPoint, bucketName, objectName, uploadID, "2"), + int64(buffer2.Len()), buffer2, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request to upload the second part. + response2, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response2.StatusCode, http.StatusOK) + + // HTTP request to ListMultipart Uploads. + request, err = newTestSignedRequest(http.MethodGet, getListMultipartURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request. + response3, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response3.StatusCode, http.StatusOK) + + // The reason to duplicate this structure here is to verify if the + // unmarshalling works from a client perspective, specifically + // while unmarshalling time.Time type for 'Initiated' field. + // time.Time does not honor xml marshaller, it means that we need + // to encode/format it before giving it to xml marshaling. + + // This below check adds client side verification to see if its + // truly parsable. + + // listMultipartUploadsResponse - format for list multipart uploads response. + type listMultipartUploadsResponse struct { + XMLName xml.Name `xml:"http://s3.amazonaws.com/doc/2006-03-01/ ListMultipartUploadsResult" json:"-"` + + Bucket string + KeyMarker string + UploadIDMarker string `xml:"UploadIdMarker"` + NextKeyMarker string + NextUploadIDMarker string `xml:"NextUploadIdMarker"` + EncodingType string + MaxUploads int + IsTruncated bool + // All the in progress multipart uploads. + Uploads []struct { + Key string + UploadID string `xml:"UploadId"` + Initiator Initiator + Owner Owner + StorageClass string + Initiated time.Time // Keep this native to be able to parse properly. + } + Prefix string + Delimiter string + CommonPrefixes []CommonPrefix + } + + // parse the response body. + decoder = xml.NewDecoder(response3.Body) + newResponse3 := &listMultipartUploadsResponse{} + err = decoder.Decode(newResponse3) + c.Assert(err, nil) + // Assert the bucket name in the response with the expected bucketName. + c.Assert(newResponse3.Bucket, bucketName) + // Assert the bucket name in the response with the expected bucketName. + c.Assert(newResponse3.IsTruncated, false) +} + +// TestValidateObjectMultipartUploadID - Test Initiates a new multipart upload and validates the uploadID. +func (s *TestSuiteCommon) TestValidateObjectMultipartUploadID(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, 200) + + objectName := "directory1/directory2/object" + // construct HTTP request to initiate a NewMultipart upload. + request, err = newTestSignedRequest(http.MethodPost, getNewMultipartURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request initiating the new multipart upload. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + + // parse the response body and obtain the new upload ID. + decoder := xml.NewDecoder(response.Body) + newResponse := &InitiateMultipartUploadResponse{} + err = decoder.Decode(newResponse) + // expecting the decoding error to be nil. + c.Assert(err, nil) + // Verifying for Upload ID value to be greater than 0. + c.Assert(len(newResponse.UploadID) > 0, true) +} + +// TestObjectMultipartListError - Initiates a NewMultipart upload, uploads parts and validates +// error response for an incorrect max-parts parameter . +func (s *TestSuiteCommon) TestObjectMultipartListError(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, 200) + + objectName := "test-multipart-object" + // construct HTTP request to initiate a NewMultipart upload. + request, err = newTestSignedRequest(http.MethodPost, getNewMultipartURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request initiating the new multipart upload. + response, err = s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, http.StatusOK) + // parse the response body and obtain the new upload ID. + decoder := xml.NewDecoder(response.Body) + newResponse := &InitiateMultipartUploadResponse{} + + err = decoder.Decode(newResponse) + c.Assert(err, nil) + c.Assert(len(newResponse.UploadID) > 0, true) + // uploadID to be used for rest of the multipart operations on the object. + uploadID := newResponse.UploadID + + // content for the part to be uploaded. + buffer1 := bytes.NewReader([]byte("hello world")) + // HTTP request for the part to be uploaded. + request, err = newTestSignedRequest(http.MethodPut, getPartUploadURL(s.endPoint, bucketName, objectName, uploadID, "1"), + int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request to upload the first part. + response1, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response1.StatusCode, http.StatusOK) + + // content for the second part to be uploaded. + buffer2 := bytes.NewReader([]byte("hello world")) + // HTTP request for the second part to be uploaded. + request, err = newTestSignedRequest(http.MethodPut, getPartUploadURL(s.endPoint, bucketName, objectName, uploadID, "2"), + int64(buffer2.Len()), buffer2, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to upload the second part. + response2, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response2.StatusCode, http.StatusOK) + + // HTTP request to ListMultipart Uploads. + // max-keys is set to valid value of 1 + request, err = newTestSignedRequest(http.MethodGet, getListMultipartURLWithParams(s.endPoint, bucketName, objectName, uploadID, "1", "", ""), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request. + response3, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response3.StatusCode, http.StatusOK) + + // HTTP request to ListMultipart Uploads. + // max-keys is set to invalid value of -2. + request, err = newTestSignedRequest(http.MethodGet, getListMultipartURLWithParams(s.endPoint, bucketName, objectName, uploadID, "-2", "", ""), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // execute the HTTP request. + response4, err := s.client.Do(request) + c.Assert(err, nil) + // Since max-keys parameter in the ListMultipart request set to invalid value of -2, + // its expected to fail with error message "InvalidArgument". + verifyError(c, response4, "InvalidArgument", "Part number must be an integer between 1 and 10000, inclusive", http.StatusBadRequest) +} + +// TestObjectValidMD5 - First uploads an object with a valid Content-Md5 header and verifies the status, +// then upload an object in a wrong Content-Md5 and validate the error response. +func (s *TestSuiteCommon) TestObjectValidMD5(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, 200) + + // Create a byte array of 5MB. + // content for the object to be uploaded. + data := bytes.Repeat([]byte("0123456789abcdef"), 5*humanize.MiByte/16) + // calculate etag of the data. + etagBase64 := getMD5HashBase64(data) + + buffer1 := bytes.NewReader(data) + objectName := "test-1-object" + // HTTP request for the object to be uploaded. + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // set the Content-Md5 to be the hash to content. + request.Header.Set("Content-Md5", etagBase64) + response, err = s.client.Do(request) + c.Assert(err, nil) + // expecting a successful upload. + c.Assert(response.StatusCode, http.StatusOK) + objectName = "test-2-object" + buffer1 = bytes.NewReader(data) + // HTTP request for the object to be uploaded. + request, err = newTestSignedRequest(http.MethodPut, getPutObjectURL(s.endPoint, bucketName, objectName), + int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // set Content-Md5 to invalid value. + request.Header.Set("Content-Md5", "kvLTlMrX9NpYDQlEIFlnDA==") + // expecting a failure during upload. + response, err = s.client.Do(request) + c.Assert(err, nil) + // Since Content-Md5 header was wrong, expecting to fail with "SignatureDoesNotMatch" error. + verifyError(c, response, "SignatureDoesNotMatch", "The request signature we calculated does not match the signature you provided. Check your key and signing method.", http.StatusForbidden) +} + +// TestObjectMultipart - Initiates a NewMultipart upload, uploads 2 parts, +// completes the multipart upload and validates the status of the operation. +func (s *TestSuiteCommon) TestObjectMultipart(c *check) { + // generate a random bucket name. + bucketName := getRandomBucketName() + // HTTP request to create the bucket. + request, err := newTestSignedRequest(http.MethodPut, getMakeBucketURL(s.endPoint, bucketName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request to create bucket. + response, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response.StatusCode, 200) + + objectName := "test-multipart-object" + // construct HTTP request to initiate a NewMultipart upload. + request, err = newTestSignedRequest(http.MethodPost, getNewMultipartURL(s.endPoint, bucketName, objectName), + 0, nil, s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + + // execute the HTTP request initiating the new multipart upload. + response, err = s.client.Do(request) + c.Assert(err, nil) + // expecting the response status code to be http.StatusOK(200 OK). + c.Assert(response.StatusCode, http.StatusOK) + // parse the response body and obtain the new upload ID. + decoder := xml.NewDecoder(response.Body) + newResponse := &InitiateMultipartUploadResponse{} + + err = decoder.Decode(newResponse) + c.Assert(err, nil) + c.Assert(len(newResponse.UploadID) > 0, true) + // uploadID to be used for rest of the multipart operations on the object. + uploadID := newResponse.UploadID + + // content for the part to be uploaded. + // Create a byte array of 5MB. + data := bytes.Repeat([]byte("0123456789abcdef"), 5*humanize.MiByte/16) + // calculate etag of the data. + md5SumBase64 := getMD5HashBase64(data) + + buffer1 := bytes.NewReader(data) + // HTTP request for the part to be uploaded. + request, err = newTestSignedRequest(http.MethodPut, getPartUploadURL(s.endPoint, bucketName, objectName, uploadID, "1"), + int64(buffer1.Len()), buffer1, s.accessKey, s.secretKey, s.signer) + // set the Content-Md5 header to the base64 encoding the etag of the content. + request.Header.Set("Content-Md5", md5SumBase64) + c.Assert(err, nil) + + // execute the HTTP request to upload the first part. + response1, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response1.StatusCode, http.StatusOK) + + // content for the second part to be uploaded. + // Create a byte array of 1 byte. + data = []byte("0") + + // calculate etag of the data. + md5SumBase64 = getMD5HashBase64(data) + + buffer2 := bytes.NewReader(data) + // HTTP request for the second part to be uploaded. + request, err = newTestSignedRequest(http.MethodPut, getPartUploadURL(s.endPoint, bucketName, objectName, uploadID, "2"), + int64(buffer2.Len()), buffer2, s.accessKey, s.secretKey, s.signer) + // set the Content-Md5 header to the base64 encoding the etag of the content. + request.Header.Set("Content-Md5", md5SumBase64) + c.Assert(err, nil) + + // execute the HTTP request to upload the second part. + response2, err := s.client.Do(request) + c.Assert(err, nil) + c.Assert(response2.StatusCode, http.StatusOK) + + // Complete multipart upload + completeUploads := &CompleteMultipartUpload{ + Parts: []CompletePart{ + { + PartNumber: 1, + ETag: response1.Header.Get("ETag"), + }, + { + PartNumber: 2, + ETag: response2.Header.Get("ETag"), + }, + }, + } + + completeBytes, err := xml.Marshal(completeUploads) + c.Assert(err, nil) + // Indicating that all parts are uploaded and initiating CompleteMultipartUpload. + request, err = newTestSignedRequest(http.MethodPost, getCompleteMultipartUploadURL(s.endPoint, bucketName, objectName, uploadID), + int64(len(completeBytes)), bytes.NewReader(completeBytes), s.accessKey, s.secretKey, s.signer) + c.Assert(err, nil) + // Execute the complete multipart request. + response, err = s.client.Do(request) + c.Assert(err, nil) + // verify whether complete multipart was successful. + c.Assert(response.StatusCode, http.StatusOK) + var parts []CompletePart + for _, part := range completeUploads.Parts { + part.ETag = canonicalizeETag(part.ETag) + parts = append(parts, part) + } + etag := getCompleteMultipartMD5(parts) + c.Assert(canonicalizeETag(response.Header.Get(xhttp.ETag)), etag) +} diff --git a/cmd/service.go b/cmd/service.go new file mode 100644 index 00000000..5561cfc9 --- /dev/null +++ b/cmd/service.go @@ -0,0 +1,113 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "os" + "os/exec" + "runtime" + "syscall" + + xioutil "github.com/minio/minio/internal/ioutil" +) + +// Type of service signals currently supported. +type serviceSignal int + +const ( + serviceRestart serviceSignal = iota // Restarts the server. + serviceStop // Stops the server. + serviceReloadDynamic // Reload dynamic config values. + serviceFreeze // Freeze all S3 API calls. + serviceUnFreeze // Un-Freeze previously frozen S3 API calls. + // Add new service requests here. +) + +// Global service signal channel. +var globalServiceSignalCh = make(chan serviceSignal) + +// GlobalContext context that is canceled when server is requested to shut down. +// cancelGlobalContext can be used to indicate server shutdown. +var GlobalContext, cancelGlobalContext = context.WithCancel(context.Background()) + +// restartProcess starts a new process passing it the active fd's. It +// doesn't fork, but starts a new process using the same environment and +// arguments as when it was originally started. This allows for a newly +// deployed binary to be started. It returns the pid of the newly started +// process when successful. +func restartProcess() error { + if runtime.GOOS == globalWindowsOSName { + cmd := exec.Command(os.Args[0], os.Args[1:]...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + cmd.Stdin = os.Stdin + cmd.Env = os.Environ() + err := cmd.Run() + if err == nil { + os.Exit(0) + } + return err + } + + // Use the original binary location. This works with symlinks such that if + // the file it points to has been changed we will use the updated symlink. + argv0, err := exec.LookPath(os.Args[0]) + if err != nil { + return err + } + + // Invokes the execve system call. + // Re-uses the same pid. This preserves the pid over multiple server-respawns. + return syscall.Exec(argv0, os.Args, os.Environ()) +} + +// freezeServices will freeze all incoming S3 API calls. +// For each call, unfreezeServices must be called once. +func freezeServices() { + // Use atomics for globalServiceFreeze, so we can read without locking. + // We need a lock since we are need the 2 atomic values to remain in sync. + globalServiceFreezeMu.Lock() + // If multiple calls, first one creates channel. + globalServiceFreezeCnt++ + if globalServiceFreezeCnt == 1 { + globalServiceFreeze.Store(make(chan struct{})) + } + globalServiceFreezeMu.Unlock() +} + +// unfreezeServices will unfreeze all incoming S3 API calls. +// For each call, unfreezeServices must be called once. +func unfreezeServices() { + // We need a lock since we need the 2 atomic values to remain in sync. + globalServiceFreezeMu.Lock() + // Close when we reach 0 + globalServiceFreezeCnt-- + if globalServiceFreezeCnt <= 0 { + // Set to a nil channel. + var _ch chan struct{} + if val := globalServiceFreeze.Swap(_ch); val != nil { + if ch, ok := val.(chan struct{}); ok && ch != nil { + // Close previous non-nil channel. + xioutil.SafeClose(ch) + } + } + globalServiceFreezeCnt = 0 // Don't risk going negative. + } + globalServiceFreezeMu.Unlock() +} diff --git a/cmd/setup-type.go b/cmd/setup-type.go new file mode 100644 index 00000000..1da83da7 --- /dev/null +++ b/cmd/setup-type.go @@ -0,0 +1,53 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +// SetupType - enum for setup type. +type SetupType int + +const ( + // UnknownSetupType - starts with unknown setup type. + UnknownSetupType SetupType = iota + + // FSSetupType - FS setup type enum. + FSSetupType + + // ErasureSDSetupType - Erasure single drive setup enum. + ErasureSDSetupType + + // ErasureSetupType - Erasure setup type enum. + ErasureSetupType + + // DistErasureSetupType - Distributed Erasure setup type enum. + DistErasureSetupType +) + +func (setupType SetupType) String() string { + switch setupType { + case FSSetupType: + return globalMinioModeFS + case ErasureSDSetupType: + return globalMinioModeErasureSD + case ErasureSetupType: + return globalMinioModeErasure + case DistErasureSetupType: + return globalMinioModeDistErasure + } + + return "unknown" +} diff --git a/cmd/sftp-server-driver.go b/cmd/sftp-server-driver.go new file mode 100644 index 00000000..3ce7c0b4 --- /dev/null +++ b/cmd/sftp-server-driver.go @@ -0,0 +1,503 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/http" + "os" + "path" + "strings" + "sync" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/pkg/v3/mimedb" + "github.com/pkg/sftp" + "golang.org/x/crypto/ssh" +) + +// Maximum write offset for incoming SFTP blocks. +// Set to 100MiB to prevent hostile DOS attacks. +const ftpMaxWriteOffset = 100 << 20 + +type sftpDriver struct { + permissions *ssh.Permissions + endpoint string + remoteIP string +} + +//msgp:ignore sftpMetrics +type sftpMetrics struct{} + +var globalSftpMetrics sftpMetrics + +func sftpTrace(s *sftp.Request, startTime time.Time, source string, user string, err error, sz int64) madmin.TraceInfo { + var errStr string + if err != nil { + errStr = err.Error() + } + return madmin.TraceInfo{ + TraceType: madmin.TraceFTP, + Time: startTime, + NodeName: globalLocalNodeName, + FuncName: s.Method, + Duration: time.Since(startTime), + Path: s.Filepath, + Error: errStr, + Bytes: sz, + Custom: map[string]string{ + "user": user, + "cmd": s.Method, + "param": s.Filepath, + "source": source, + }, + } +} + +func (m *sftpMetrics) log(s *sftp.Request, user string) func(sz int64, err error) { + startTime := time.Now() + source := getSource(2) + return func(sz int64, err error) { + globalTrace.Publish(sftpTrace(s, startTime, source, user, err, sz)) + } +} + +// NewSFTPDriver initializes sftp.Handlers implementation of following interfaces +// +// - sftp.Fileread +// - sftp.Filewrite +// - sftp.Filelist +// - sftp.Filecmd +func NewSFTPDriver(perms *ssh.Permissions, remoteIP string) sftp.Handlers { + handler := &sftpDriver{ + endpoint: fmt.Sprintf("127.0.0.1:%s", globalMinioPort), + permissions: perms, + remoteIP: remoteIP, + } + return sftp.Handlers{ + FileGet: handler, + FilePut: handler, + FileCmd: handler, + FileList: handler, + } +} + +type forwardForTransport struct { + tr http.RoundTripper + fwd string +} + +func (f forwardForTransport) RoundTrip(r *http.Request) (*http.Response, error) { + r.Header.Set("X-Forwarded-For", f.fwd) + return f.tr.RoundTrip(r) +} + +func (f *sftpDriver) getMinIOClient() (*minio.Client, error) { + mcreds := credentials.NewStaticV4( + f.permissions.CriticalOptions["AccessKey"], + f.permissions.CriticalOptions["SecretKey"], + f.permissions.CriticalOptions["SessionToken"], + ) + // Set X-Forwarded-For on all requests. + tr := http.RoundTripper(globalRemoteFTPClientTransport) + if f.remoteIP != "" { + tr = forwardForTransport{tr: tr, fwd: f.remoteIP} + } + return minio.New(f.endpoint, &minio.Options{ + TrailingHeaders: true, + Creds: mcreds, + Secure: globalIsTLS, + Transport: tr, + }) +} + +func (f *sftpDriver) AccessKey() string { + return f.permissions.CriticalOptions["AccessKey"] +} + +func (f *sftpDriver) Fileread(r *sftp.Request) (ra io.ReaderAt, err error) { + // This is not timing the actual read operation, but the time it takes to prepare the reader. + stopFn := globalSftpMetrics.log(r, f.AccessKey()) + defer stopFn(0, err) + + flags := r.Pflags() + if !flags.Read { + // sanity check + return nil, os.ErrInvalid + } + + bucket, object := path2BucketObject(r.Filepath) + if bucket == "" { + return nil, errors.New("bucket name cannot be empty") + } + + clnt, err := f.getMinIOClient() + if err != nil { + return nil, err + } + + obj, err := clnt.GetObject(context.Background(), bucket, object, minio.GetObjectOptions{}) + if err != nil { + return nil, err + } + + _, err = obj.Stat() + if err != nil { + return nil, err + } + + return obj, nil +} + +// TransferError will catch network errors during transfer. +// When TransferError() is called Close() will also +// be called, so we do not need to Wait() here. +func (w *writerAt) TransferError(err error) { + _ = w.w.CloseWithError(err) + _ = w.r.CloseWithError(err) + w.err = err +} + +func (w *writerAt) Close() (err error) { + switch { + case len(w.buffer) > 0: + err = errors.New("some file segments were not flushed from the queue") + _ = w.w.CloseWithError(err) + case w.err != nil: + // No need to close here since both pipes were + // closing inside TransferError() + err = w.err + default: + err = w.w.Close() + } + for i := range w.buffer { + delete(w.buffer, i) + } + w.wg.Wait() + return err +} + +type writerAt struct { + w *io.PipeWriter + r *io.PipeReader + wg *sync.WaitGroup + buffer map[int64][]byte + err error + + nextOffset int64 + m sync.Mutex +} + +func (w *writerAt) WriteAt(b []byte, offset int64) (n int, err error) { + w.m.Lock() + defer w.m.Unlock() + + if w.nextOffset == offset { + n, err = w.w.Write(b) + w.nextOffset += int64(n) + } else { + if offset > w.nextOffset+ftpMaxWriteOffset { + return 0, fmt.Errorf("write offset %d is too far ahead of next offset %d", offset, w.nextOffset) + } + w.buffer[offset] = make([]byte, len(b)) + copy(w.buffer[offset], b) + n = len(b) + } + +again: + nextOut, ok := w.buffer[w.nextOffset] + if ok { + n, err = w.w.Write(nextOut) + delete(w.buffer, w.nextOffset) + w.nextOffset += int64(n) + if n != len(nextOut) { + return 0, fmt.Errorf("expected write size %d but wrote %d bytes", len(nextOut), n) + } + if err != nil { + return 0, err + } + goto again + } + + return len(b), nil +} + +func (f *sftpDriver) Filewrite(r *sftp.Request) (w io.WriterAt, err error) { + stopFn := globalSftpMetrics.log(r, f.AccessKey()) + defer func() { + if err != nil { + // If there is an error, we never started the goroutine. + stopFn(0, err) + } + }() + + flags := r.Pflags() + if !flags.Write { + // sanity check + return nil, os.ErrInvalid + } + + bucket, object := path2BucketObject(r.Filepath) + if bucket == "" { + return nil, errors.New("bucket name cannot be empty") + } + + clnt, err := f.getMinIOClient() + if err != nil { + return nil, err + } + ok, err := clnt.BucketExists(r.Context(), bucket) + if err != nil { + return nil, err + } + if !ok { + return nil, os.ErrNotExist + } + + pr, pw := io.Pipe() + + wa := &writerAt{ + buffer: make(map[int64][]byte), + w: pw, + r: pr, + wg: &sync.WaitGroup{}, + } + wa.wg.Add(1) + go func() { + oi, err := clnt.PutObject(r.Context(), bucket, object, pr, -1, minio.PutObjectOptions{ + ContentType: mimedb.TypeByExtension(path.Ext(object)), + DisableContentSha256: true, + Checksum: minio.ChecksumFullObjectCRC32C, + }) + stopFn(oi.Size, err) + pr.CloseWithError(err) + wa.wg.Done() + }() + return wa, nil +} + +func (f *sftpDriver) Filecmd(r *sftp.Request) (err error) { + stopFn := globalSftpMetrics.log(r, f.AccessKey()) + defer stopFn(0, err) + + clnt, err := f.getMinIOClient() + if err != nil { + return err + } + + switch r.Method { + case "Setstat", "Rename", "Link", "Symlink": + return sftp.ErrSSHFxOpUnsupported + + case "Rmdir": + bucket, prefix := path2BucketObject(r.Filepath) + if bucket == "" { + return errors.New("deleting all buckets not allowed") + } + + cctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + if prefix == "" { + // if all objects are not deleted yet this call may fail. + return clnt.RemoveBucket(cctx, bucket) + } + + objectsCh := make(chan minio.ObjectInfo) + + // Send object names that are needed to be removed to objectsCh + go func() { + defer xioutil.SafeClose(objectsCh) + opts := minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: true, + } + for object := range clnt.ListObjects(cctx, bucket, opts) { + if object.Err != nil { + return + } + objectsCh <- object + } + }() + + // Call RemoveObjects API + for err := range clnt.RemoveObjects(context.Background(), bucket, objectsCh, minio.RemoveObjectsOptions{}) { + if err.Err != nil { + return err.Err + } + } + return err + + case "Remove": + bucket, object := path2BucketObject(r.Filepath) + if bucket == "" { + return errors.New("bucket name cannot be empty") + } + + return clnt.RemoveObject(context.Background(), bucket, object, minio.RemoveObjectOptions{}) + + case "Mkdir": + bucket, prefix := path2BucketObject(r.Filepath) + if bucket == "" { + return errors.New("bucket name cannot be empty") + } + + if prefix == "" { + return clnt.MakeBucket(context.Background(), bucket, minio.MakeBucketOptions{Region: globalSite.Region()}) + } + + dirPath := buildMinioDir(prefix) + + _, err = clnt.PutObject(context.Background(), bucket, dirPath, bytes.NewReader([]byte("")), 0, + minio.PutObjectOptions{DisableContentSha256: true}, + ) + return err + } + + return NotImplemented{} +} + +type listerAt []os.FileInfo + +// Modeled after strings.Reader's ReadAt() implementation +func (f listerAt) ListAt(ls []os.FileInfo, offset int64) (int, error) { + var n int + if offset >= int64(len(f)) { + return 0, io.EOF + } + n = copy(ls, f[offset:]) + if n < len(ls) { + return n, io.EOF + } + return n, nil +} + +func (f *sftpDriver) Filelist(r *sftp.Request) (la sftp.ListerAt, err error) { + stopFn := globalSftpMetrics.log(r, f.AccessKey()) + defer stopFn(0, err) + + clnt, err := f.getMinIOClient() + if err != nil { + return nil, err + } + + switch r.Method { + case "List": + var files []os.FileInfo + + bucket, prefix := path2BucketObject(r.Filepath) + if bucket == "" { + buckets, err := clnt.ListBuckets(r.Context()) + if err != nil { + return nil, err + } + + for _, bucket := range buckets { + files = append(files, &minioFileInfo{ + p: bucket.Name, + info: minio.ObjectInfo{Key: bucket.Name, LastModified: bucket.CreationDate}, + isDir: true, + }) + } + + return listerAt(files), nil + } + + prefix = retainSlash(prefix) + + for object := range clnt.ListObjects(r.Context(), bucket, minio.ListObjectsOptions{ + Prefix: prefix, + Recursive: false, + }) { + if object.Err != nil { + return nil, object.Err + } + + if object.Key == prefix { + continue + } + + isDir := strings.HasSuffix(object.Key, SlashSeparator) + files = append(files, &minioFileInfo{ + p: pathClean(strings.TrimPrefix(object.Key, prefix)), + info: object, + isDir: isDir, + }) + } + + return listerAt(files), nil + + case "Stat": + if r.Filepath == SlashSeparator { + return listerAt{&minioFileInfo{ + p: r.Filepath, + isDir: true, + }}, nil + } + + bucket, object := path2BucketObject(r.Filepath) + if bucket == "" { + return nil, errors.New("bucket name cannot be empty") + } + + if object == "" { + ok, err := clnt.BucketExists(context.Background(), bucket) + if err != nil { + return nil, err + } + if !ok { + return nil, os.ErrNotExist + } + return listerAt{&minioFileInfo{ + p: pathClean(bucket), + info: minio.ObjectInfo{Key: bucket}, + isDir: true, + }}, nil + } + + objInfo, err := clnt.StatObject(context.Background(), bucket, object, minio.StatObjectOptions{}) + if err != nil { + if minio.ToErrorResponse(err).Code == "NoSuchKey" { + // dummy return to satisfy LIST (stat -> list) behavior. + return listerAt{&minioFileInfo{ + p: pathClean(object), + info: minio.ObjectInfo{Key: object}, + isDir: true, + }}, nil + } + return nil, err + } + + isDir := strings.HasSuffix(objInfo.Key, SlashSeparator) + return listerAt{&minioFileInfo{ + p: pathClean(object), + info: objInfo, + isDir: isDir, + }}, nil + } + + return nil, NotImplemented{} +} diff --git a/cmd/sftp-server.go b/cmd/sftp-server.go new file mode 100644 index 00000000..0a0164a4 --- /dev/null +++ b/cmd/sftp-server.go @@ -0,0 +1,523 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "crypto/subtle" + "errors" + "fmt" + "net" + "os" + "strconv" + "strings" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/logger" + xldap "github.com/minio/pkg/v3/ldap" + xsftp "github.com/minio/pkg/v3/sftp" + "github.com/pkg/sftp" + "golang.org/x/crypto/ssh" +) + +const ( + kexAlgoDH1SHA1 = "diffie-hellman-group1-sha1" + kexAlgoDH14SHA1 = "diffie-hellman-group14-sha1" + kexAlgoDH14SHA256 = "diffie-hellman-group14-sha256" + kexAlgoDH16SHA512 = "diffie-hellman-group16-sha512" + kexAlgoECDH256 = "ecdh-sha2-nistp256" + kexAlgoECDH384 = "ecdh-sha2-nistp384" + kexAlgoECDH521 = "ecdh-sha2-nistp521" + kexAlgoCurve25519SHA256LibSSH = "curve25519-sha256@libssh.org" + kexAlgoCurve25519SHA256 = "curve25519-sha256" + + chacha20Poly1305ID = "chacha20-poly1305@openssh.com" + gcm256CipherID = "aes256-gcm@openssh.com" + aes128cbcID = "aes128-cbc" + tripledescbcID = "3des-cbc" +) + +var ( + errSFTPPublicKeyBadFormat = errors.New("the public key provided could not be parsed") + errSFTPUserHasNoPolicies = errors.New("no policies present on this account") + errSFTPLDAPNotEnabled = errors.New("ldap authentication is not enabled") +) + +// if the sftp parameter --trusted-user-ca-key is set, then +// the final form of the key file will be set as this variable. +var globalSFTPTrustedCAPubkey ssh.PublicKey + +// https://cs.opensource.google/go/x/crypto/+/refs/tags/v0.22.0:ssh/common.go;l=46 +// preferredKexAlgos specifies the default preference for key-exchange +// algorithms in preference order. The diffie-hellman-group16-sha512 algorithm +// is disabled by default because it is a bit slower than the others. +var preferredKexAlgos = []string{ + kexAlgoCurve25519SHA256, kexAlgoCurve25519SHA256LibSSH, + kexAlgoECDH256, kexAlgoECDH384, kexAlgoECDH521, + kexAlgoDH14SHA256, kexAlgoDH14SHA1, +} + +// supportedKexAlgos specifies the supported key-exchange algorithms in +// preference order. +// https://cs.opensource.google/go/x/crypto/+/refs/tags/v0.22.0:ssh/common.go;l=44 +var supportedKexAlgos = []string{ + kexAlgoCurve25519SHA256, kexAlgoCurve25519SHA256LibSSH, + // P384 and P521 are not constant-time yet, but since we don't + // reuse ephemeral keys, using them for ECDH should be OK. + kexAlgoECDH256, kexAlgoECDH384, kexAlgoECDH521, + kexAlgoDH14SHA256, kexAlgoDH16SHA512, kexAlgoDH14SHA1, + kexAlgoDH1SHA1, +} + +// supportedPubKeyAuthAlgos specifies the supported client public key +// authentication algorithms. Note that this doesn't include certificate types +// since those use the underlying algorithm. This list is sent to the client if +// it supports the server-sig-algs extension. Order is irrelevant. +// https://cs.opensource.google/go/x/crypto/+/refs/tags/v0.22.0:ssh/common.go;l=142 +var supportedPubKeyAuthAlgos = []string{ + ssh.KeyAlgoED25519, + ssh.KeyAlgoSKED25519, ssh.KeyAlgoSKECDSA256, + ssh.KeyAlgoECDSA256, ssh.KeyAlgoECDSA384, ssh.KeyAlgoECDSA521, + ssh.KeyAlgoRSASHA256, ssh.KeyAlgoRSASHA512, ssh.KeyAlgoRSA, + ssh.KeyAlgoDSA, +} + +// supportedCiphers lists ciphers we support but might not recommend. +// https://cs.opensource.google/go/x/crypto/+/refs/tags/v0.22.0:ssh/common.go;l=28 +var supportedCiphers = []string{ + "aes128-ctr", "aes192-ctr", "aes256-ctr", + "aes128-gcm@openssh.com", gcm256CipherID, + chacha20Poly1305ID, + "arcfour256", "arcfour128", "arcfour", + aes128cbcID, + tripledescbcID, +} + +// preferredCiphers specifies the default preference for ciphers. +// https://cs.opensource.google/go/x/crypto/+/refs/tags/v0.22.0:ssh/common.go;l=37 +var preferredCiphers = []string{ + "aes128-gcm@openssh.com", gcm256CipherID, + chacha20Poly1305ID, + "aes128-ctr", "aes192-ctr", "aes256-ctr", +} + +// supportedMACs specifies a default set of MAC algorithms in preference order. +// This is based on RFC 4253, section 6.4, but with hmac-md5 variants removed +// because they have reached the end of their useful life. +// https://cs.opensource.google/go/x/crypto/+/refs/tags/v0.22.0:ssh/common.go;l=85 +var supportedMACs = []string{ + "hmac-sha2-256-etm@openssh.com", "hmac-sha2-512-etm@openssh.com", "hmac-sha2-256", "hmac-sha2-512", "hmac-sha1", "hmac-sha1-96", +} + +func sshPubKeyAuth(c ssh.ConnMetadata, key ssh.PublicKey) (*ssh.Permissions, error) { + return authenticateSSHConnection(c, key, nil) +} + +func sshPasswordAuth(c ssh.ConnMetadata, pass []byte) (*ssh.Permissions, error) { + return authenticateSSHConnection(c, nil, pass) +} + +func authenticateSSHConnection(c ssh.ConnMetadata, key ssh.PublicKey, pass []byte) (*ssh.Permissions, error) { + user, found := strings.CutSuffix(c.User(), "=ldap") + if found { + if !globalIAMSys.LDAPConfig.Enabled() { + return nil, errSFTPLDAPNotEnabled + } + return processLDAPAuthentication(key, pass, user) + } + + user, found = strings.CutSuffix(c.User(), "=svc") + if found { + goto internalAuth + } + + if globalIAMSys.LDAPConfig.Enabled() { + perms, _ := processLDAPAuthentication(key, pass, user) + if perms != nil { + return perms, nil + } + } + +internalAuth: + ui, ok := globalIAMSys.GetUser(context.Background(), user) + if !ok { + return nil, errNoSuchUser + } + + if globalSFTPTrustedCAPubkey != nil && pass == nil { + err := validateClientKeyIsTrusted(c, key) + if err != nil { + return nil, errAuthentication + } + } else { + // Temporary credentials are not allowed. + if ui.Credentials.IsTemp() { + return nil, errAuthentication + } + if subtle.ConstantTimeCompare([]byte(ui.Credentials.SecretKey), pass) != 1 { + return nil, errAuthentication + } + } + + copts := map[string]string{ + "AccessKey": ui.Credentials.AccessKey, + "SecretKey": ui.Credentials.SecretKey, + } + if ui.Credentials.IsTemp() { + copts["SessionToken"] = ui.Credentials.SessionToken + } + + return &ssh.Permissions{ + CriticalOptions: copts, + Extensions: make(map[string]string), + }, nil +} + +func processLDAPAuthentication(key ssh.PublicKey, pass []byte, user string) (perms *ssh.Permissions, err error) { + var lookupResult *xldap.DNSearchResult + var targetGroups []string + + if pass == nil && key == nil { + return nil, errAuthentication + } + + if pass != nil { + sa, _, err := globalIAMSys.getServiceAccount(context.Background(), user) + if err == nil { + if subtle.ConstantTimeCompare([]byte(sa.Credentials.SecretKey), pass) != 1 { + return nil, errAuthentication + } + + return &ssh.Permissions{ + CriticalOptions: map[string]string{ + "AccessKey": sa.Credentials.AccessKey, + "SecretKey": sa.Credentials.SecretKey, + }, + Extensions: make(map[string]string), + }, nil + } + + if !errors.Is(err, errNoSuchServiceAccount) { + return nil, err + } + + lookupResult, targetGroups, err = globalIAMSys.LDAPConfig.Bind(user, string(pass)) + if err != nil { + return nil, err + } + } else if key != nil { + lookupResult, targetGroups, err = globalIAMSys.LDAPConfig.LookupUserDN(user) + if err != nil { + return nil, err + } + } + + if lookupResult == nil { + return nil, errNoSuchUser + } + + ldapPolicies, _ := globalIAMSys.PolicyDBGet(lookupResult.NormDN, targetGroups...) + if len(ldapPolicies) == 0 { + return nil, errSFTPUserHasNoPolicies + } + + claims := make(map[string]any) + for attribKey, attribValue := range lookupResult.Attributes { + // we skip multi-value attributes here, as they cannot + // be stored in the critical options. + if len(attribValue) != 1 { + continue + } + + if attribKey == "sshPublicKey" && key != nil { + key2, _, _, _, err := ssh.ParseAuthorizedKey([]byte(attribValue[0])) + if err != nil { + return nil, errSFTPPublicKeyBadFormat + } + + if subtle.ConstantTimeCompare(key2.Marshal(), key.Marshal()) != 1 { + return nil, errAuthentication + } + } + // Save each attribute to claims. + claims[ldapAttribPrefix+attribKey] = attribValue[0] + } + + if key != nil { + // If a key was provided, we expect the user to have an sshPublicKey + // attribute. + if _, ok := claims[ldapAttribPrefix+"sshPublicKey"]; !ok { + return nil, errAuthentication + } + } + + expiryDur, err := globalIAMSys.LDAPConfig.GetExpiryDuration("") + if err != nil { + return nil, err + } + + claims[expClaim] = UTCNow().Add(expiryDur).Unix() + claims[ldapUserN] = user + claims[ldapUser] = lookupResult.NormDN + + cred, err := auth.GetNewCredentialsWithMetadata(claims, globalActiveCred.SecretKey) + if err != nil { + return nil, err + } + + // Set the parent of the temporary access key, this is useful + // in obtaining service accounts by this cred. + cred.ParentUser = lookupResult.NormDN + + // Set this value to LDAP groups, LDAP user can be part + // of large number of groups + cred.Groups = targetGroups + + // Set the newly generated credentials, policyName is empty on purpose + // LDAP policies are applied automatically using their ldapUser, ldapGroups + // mapping. + updatedAt, err := globalIAMSys.SetTempUser(context.Background(), cred.AccessKey, cred, "") + if err != nil { + return nil, err + } + + replLogIf(context.Background(), globalSiteReplicationSys.IAMChangeHook(context.Background(), madmin.SRIAMItem{ + Type: madmin.SRIAMItemSTSAcc, + STSCredential: &madmin.SRSTSCredential{ + AccessKey: cred.AccessKey, + SecretKey: cred.SecretKey, + SessionToken: cred.SessionToken, + ParentUser: cred.ParentUser, + }, + UpdatedAt: updatedAt, + })) + + return &ssh.Permissions{ + CriticalOptions: map[string]string{ + "AccessKey": cred.AccessKey, + "SecretKey": cred.SecretKey, + "SessionToken": cred.SessionToken, + }, + Extensions: make(map[string]string), + }, nil +} + +func validateClientKeyIsTrusted(c ssh.ConnMetadata, clientKey ssh.PublicKey) (err error) { + if globalSFTPTrustedCAPubkey == nil { + return errors.New("public key authority validation requested but no ca public key specified.") + } + + cert, ok := clientKey.(*ssh.Certificate) + if !ok { + return errSftpPublicKeyWithoutCert + } + + // ssh.CheckCert called by ssh.Authenticate accepts certificates + // with empty principles list so we block those in here. + if len(cert.ValidPrincipals) == 0 { + return errSftpCertWithoutPrincipals + } + + // Verify that certificate provided by user is issued by trusted CA, + // username in authentication request matches to identities in certificate + // and that certificate type is correct. + checker := ssh.CertChecker{} + checker.IsUserAuthority = func(k ssh.PublicKey) bool { + return subtle.ConstantTimeCompare(globalSFTPTrustedCAPubkey.Marshal(), k.Marshal()) == 1 + } + + _, err = checker.Authenticate(c, clientKey) + return err +} + +type sftpLogger struct{} + +func (s *sftpLogger) Info(tag xsftp.LogType, msg string) { + logger.Info(msg) +} + +func (s *sftpLogger) Error(tag xsftp.LogType, err error) { + switch tag { + case xsftp.AcceptNetworkError: + sftpLogOnceIf(context.Background(), err, "accept-limit-sftp") + case xsftp.AcceptChannelError: + sftpLogOnceIf(context.Background(), err, "accept-channel-sftp") + case xsftp.SSHKeyExchangeError: + sftpLogOnceIf(context.Background(), err, "key-exchange-sftp") + default: + sftpLogOnceIf(context.Background(), err, "unknown-error-sftp") + } +} + +func filterAlgos(arg string, want []string, allowed []string) []string { + var filteredAlgos []string + found := false + for _, algo := range want { + if len(algo) == 0 { + continue + } + for _, allowedAlgo := range allowed { + algo := strings.ToLower(strings.TrimSpace(algo)) + if algo == allowedAlgo { + filteredAlgos = append(filteredAlgos, algo) + found = true + break + } + } + if !found { + logger.Fatal(fmt.Errorf("unknown algorithm %q passed to --sftp=%s\nValid algorithms: %v", algo, arg, strings.Join(allowed, ", ")), "unable to start SFTP server") + } + } + if len(filteredAlgos) == 0 { + logger.Fatal(fmt.Errorf("no valid algorithms passed to --sftp=%s\nValid algorithms: %v", arg, strings.Join(allowed, ", ")), "unable to start SFTP server") + } + return filteredAlgos +} + +func startSFTPServer(args []string) { + var ( + port int + publicIP string + sshPrivateKey string + userCaKeyFile string + disablePassAuth bool + ) + + allowPubKeys := supportedPubKeyAuthAlgos + allowKexAlgos := preferredKexAlgos + allowCiphers := preferredCiphers + allowMACs := supportedMACs + var err error + + for _, arg := range args { + tokens := strings.SplitN(arg, "=", 2) + if len(tokens) != 2 { + logger.Fatal(fmt.Errorf("invalid arguments passed to --sftp=%s", arg), "unable to start SFTP server") + } + switch tokens[0] { + case "address": + host, portStr, err := net.SplitHostPort(tokens[1]) + if err != nil { + logger.Fatal(fmt.Errorf("invalid arguments passed to --sftp=%s (%v)", arg, err), "unable to start SFTP server") + } + port, err = strconv.Atoi(portStr) + if err != nil { + logger.Fatal(fmt.Errorf("invalid arguments passed to --sftp=%s (%v)", arg, err), "unable to start SFTP server") + } + if port < 1 || port > 65535 { + logger.Fatal(fmt.Errorf("invalid arguments passed to --sftp=%s, (port number must be between 1 to 65535)", arg), "unable to start SFTP server") + } + publicIP = host + case "ssh-private-key": + sshPrivateKey = tokens[1] + case "pub-key-algos": + allowPubKeys = filterAlgos(arg, strings.Split(tokens[1], ","), supportedPubKeyAuthAlgos) + case "kex-algos": + allowKexAlgos = filterAlgos(arg, strings.Split(tokens[1], ","), supportedKexAlgos) + case "cipher-algos": + allowCiphers = filterAlgos(arg, strings.Split(tokens[1], ","), supportedCiphers) + case "mac-algos": + allowMACs = filterAlgos(arg, strings.Split(tokens[1], ","), supportedMACs) + case "trusted-user-ca-key": + userCaKeyFile = tokens[1] + case "disable-password-auth": + disablePassAuth, _ = strconv.ParseBool(tokens[1]) + } + } + + if port == 0 { + port = 8022 // Default SFTP port, since no port was given. + } + + if sshPrivateKey == "" { + logger.Fatal(fmt.Errorf("invalid arguments passed, private key file is mandatory for --sftp='ssh-private-key=path/to/id_ecdsa'"), "unable to start SFTP server") + } + + privateBytes, err := os.ReadFile(sshPrivateKey) + if err != nil { + logger.Fatal(fmt.Errorf("invalid arguments passed, private key file is not accessible: %v", err), "unable to start SFTP server") + } + + private, err := ssh.ParsePrivateKey(privateBytes) + if err != nil { + logger.Fatal(fmt.Errorf("invalid arguments passed, private key file is not parseable: %v", err), "unable to start SFTP server") + } + + if userCaKeyFile != "" { + keyBytes, err := os.ReadFile(userCaKeyFile) + if err != nil { + logger.Fatal(fmt.Errorf("invalid arguments passed, trusted user certificate authority public key file is not accessible: %v", err), "unable to start SFTP server") + } + + globalSFTPTrustedCAPubkey, _, _, _, err = ssh.ParseAuthorizedKey(keyBytes) + if err != nil { + logger.Fatal(fmt.Errorf("invalid arguments passed, trusted user certificate authority public key file is not parseable: %v", err), "unable to start SFTP server") + } + } + + // An SSH server is represented by a ServerConfig, which holds + // certificate details and handles authentication of ServerConns. + sshConfig := &ssh.ServerConfig{ + Config: ssh.Config{ + KeyExchanges: allowKexAlgos, + Ciphers: allowCiphers, + MACs: allowMACs, + }, + PublicKeyAuthAlgorithms: allowPubKeys, + PublicKeyCallback: sshPubKeyAuth, + } + + if !disablePassAuth { + sshConfig.PasswordCallback = sshPasswordAuth + } else { + sshConfig.PasswordCallback = nil + } + + sshConfig.AddHostKey(private) + + handleSFTPSession := func(channel ssh.Channel, sconn *ssh.ServerConn) { + var remoteIP string + + if host, _, err := net.SplitHostPort(sconn.RemoteAddr().String()); err == nil { + remoteIP = host + } + server := sftp.NewRequestServer(channel, NewSFTPDriver(sconn.Permissions, remoteIP), sftp.WithRSAllocator()) + defer server.Close() + server.Serve() + } + + sftpServer, err := xsftp.NewServer(&xsftp.Options{ + PublicIP: publicIP, + Port: port, + // OpensSSH default handshake timeout is 2 minutes. + SSHHandshakeDeadline: 2 * time.Minute, + Logger: new(sftpLogger), + SSHConfig: sshConfig, + HandleSFTPSession: handleSFTPSession, + }) + if err != nil { + logger.Fatal(err, "Unable to start SFTP Server") + } + + err = sftpServer.Listen() + if err != nil { + logger.Fatal(err, "SFTP Server had an unrecoverable error while accepting connections") + } +} diff --git a/cmd/sftp-server_test.go b/cmd/sftp-server_test.go new file mode 100644 index 00000000..00642308 --- /dev/null +++ b/cmd/sftp-server_test.go @@ -0,0 +1,349 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "net" + "os" + "testing" + + "github.com/minio/madmin-go/v3" + "golang.org/x/crypto/ssh" +) + +type MockConnMeta struct { + username string +} + +func (m *MockConnMeta) User() string { + return m.username +} + +func (m *MockConnMeta) SessionID() []byte { + return []byte{} +} + +func (m *MockConnMeta) ClientVersion() []byte { + return []byte{} +} + +func (m *MockConnMeta) ServerVersion() []byte { + return []byte{} +} + +func (m *MockConnMeta) RemoteAddr() net.Addr { + return nil +} + +func (m *MockConnMeta) LocalAddr() net.Addr { + return nil +} + +func newSSHConnMock(username string) ssh.ConnMetadata { + return &MockConnMeta{username: username} +} + +func TestSFTPAuthentication(t *testing.T) { + for i, testCase := range iamTestSuites { + t.Run( + fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription), + func(t *testing.T) { + c := &check{t, testCase.serverType} + suite := testCase + + suite.SetUpSuite(c) + + suite.SFTPServiceAccountLogin(c) + suite.SFTPInvalidServiceAccountPassword(c) + + // LDAP tests + ldapServer := os.Getenv(EnvTestLDAPServer) + if ldapServer == "" { + c.Skipf("Skipping LDAP test as no LDAP server is provided via %s", EnvTestLDAPServer) + } + + suite.SetUpLDAP(c, ldapServer) + + suite.SFTPFailedAuthDueToMissingPolicy(c) + suite.SFTPFailedAuthDueToInvalidUser(c) + suite.SFTPFailedForcedServiceAccountAuthOnLDAPUser(c) + suite.SFTPFailedAuthDueToInvalidPassword(c) + + suite.SFTPValidLDAPLoginWithPassword(c) + + suite.SFTPPublicKeyAuthentication(c) + suite.SFTPFailedPublicKeyAuthenticationInvalidKey(c) + suite.SFTPPublicKeyAuthNoPubKey(c) + + suite.TearDownSuite(c) + }, + ) + } +} + +func (s *TestSuiteIAM) SFTPFailedPublicKeyAuthenticationInvalidKey(c *check) { + keyBytes, err := os.ReadFile("./testdata/invalid_test_key.pub") + if err != nil { + c.Fatalf("could not read test key file: %s", err) + } + + testKey, _, _, _, err := ssh.ParseAuthorizedKey(keyBytes) + if err != nil { + c.Fatalf("could not parse test key file: %s", err) + } + + newSSHCon := newSSHConnMock("dillon=ldap") + _, err = sshPubKeyAuth(newSSHCon, testKey) + if err == nil || !errors.Is(err, errAuthentication) { + c.Fatalf("expected err(%s) but got (%s)", errAuthentication, err) + } + + newSSHCon = newSSHConnMock("dillon") + _, err = sshPubKeyAuth(newSSHCon, testKey) + if err == nil || !errors.Is(err, errNoSuchUser) { + c.Fatalf("expected err(%s) but got (%s)", errNoSuchUser, err) + } +} + +func (s *TestSuiteIAM) SFTPPublicKeyAuthentication(c *check) { + keyBytes, err := os.ReadFile("./testdata/dillon_test_key.pub") + if err != nil { + c.Fatalf("could not read test key file: %s", err) + } + + testKey, _, _, _, err := ssh.ParseAuthorizedKey(keyBytes) + if err != nil { + c.Fatalf("could not parse test key file: %s", err) + } + + newSSHCon := newSSHConnMock("dillon=ldap") + _, err = sshPubKeyAuth(newSSHCon, testKey) + if err != nil { + c.Fatalf("expected no error but got(%s)", err) + } + + newSSHCon = newSSHConnMock("dillon") + _, err = sshPubKeyAuth(newSSHCon, testKey) + if err != nil { + c.Fatalf("expected no error but got(%s)", err) + } +} + +// A user without an sshpubkey attribute in LDAP (here: fahim) should not be +// able to authenticate. +func (s *TestSuiteIAM) SFTPPublicKeyAuthNoPubKey(c *check) { + keyBytes, err := os.ReadFile("./testdata/dillon_test_key.pub") + if err != nil { + c.Fatalf("could not read test key file: %s", err) + } + + testKey, _, _, _, err := ssh.ParseAuthorizedKey(keyBytes) + if err != nil { + c.Fatalf("could not parse test key file: %s", err) + } + + newSSHCon := newSSHConnMock("fahim=ldap") + _, err = sshPubKeyAuth(newSSHCon, testKey) + if err == nil { + c.Fatalf("expected error but got none") + } + + newSSHCon = newSSHConnMock("fahim") + _, err = sshPubKeyAuth(newSSHCon, testKey) + if err == nil { + c.Fatalf("expected error but got none") + } +} + +func (s *TestSuiteIAM) SFTPFailedAuthDueToMissingPolicy(c *check) { + newSSHCon := newSSHConnMock("dillon=ldap") + _, err := sshPasswordAuth(newSSHCon, []byte("dillon")) + if err == nil || !errors.Is(err, errSFTPUserHasNoPolicies) { + c.Fatalf("expected err(%s) but got (%s)", errSFTPUserHasNoPolicies, err) + } + + newSSHCon = newSSHConnMock("dillon") + _, err = sshPasswordAuth(newSSHCon, []byte("dillon")) + if err == nil || !errors.Is(err, errNoSuchUser) { + c.Fatalf("expected err(%s) but got (%s)", errNoSuchUser, err) + } +} + +func (s *TestSuiteIAM) SFTPFailedAuthDueToInvalidUser(c *check) { + newSSHCon := newSSHConnMock("dillon_error") + _, err := sshPasswordAuth(newSSHCon, []byte("dillon_error")) + if err == nil || !errors.Is(err, errNoSuchUser) { + c.Fatalf("expected err(%s) but got (%s)", errNoSuchUser, err) + } +} + +func (s *TestSuiteIAM) SFTPFailedForcedServiceAccountAuthOnLDAPUser(c *check) { + newSSHCon := newSSHConnMock("dillon=svc") + _, err := sshPasswordAuth(newSSHCon, []byte("dillon")) + if err == nil || !errors.Is(err, errNoSuchUser) { + c.Fatalf("expected err(%s) but got (%s)", errNoSuchUser, err) + } +} + +func (s *TestSuiteIAM) SFTPFailedAuthDueToInvalidPassword(c *check) { + newSSHCon := newSSHConnMock("dillon") + _, err := sshPasswordAuth(newSSHCon, []byte("dillon_error")) + if err == nil || !errors.Is(err, errNoSuchUser) { + c.Fatalf("expected err(%s) but got (%s)", errNoSuchUser, err) + } +} + +func (s *TestSuiteIAM) SFTPInvalidServiceAccountPassword(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + accessKey, secretKey := mustGenerateCredentials(c) + err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled) + if err != nil { + c.Fatalf("Unable to set user: %v", err) + } + + userReq := madmin.PolicyAssociationReq{ + Policies: []string{"readwrite"}, + User: accessKey, + } + if _, err := s.adm.AttachPolicy(ctx, userReq); err != nil { + c.Fatalf("Unable to attach policy: %v", err) + } + + newSSHCon := newSSHConnMock(accessKey + "=svc") + _, err = sshPasswordAuth(newSSHCon, []byte("invalid")) + if err == nil || !errors.Is(err, errAuthentication) { + c.Fatalf("expected err(%s) but got (%s)", errAuthentication, err) + } + + newSSHCon = newSSHConnMock(accessKey) + _, err = sshPasswordAuth(newSSHCon, []byte("invalid")) + if err == nil || !errors.Is(err, errAuthentication) { + c.Fatalf("expected err(%s) but got (%s)", errAuthentication, err) + } +} + +func (s *TestSuiteIAM) SFTPServiceAccountLogin(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + accessKey, secretKey := mustGenerateCredentials(c) + err := s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled) + if err != nil { + c.Fatalf("Unable to set user: %v", err) + } + + userReq := madmin.PolicyAssociationReq{ + Policies: []string{"readwrite"}, + User: accessKey, + } + if _, err := s.adm.AttachPolicy(ctx, userReq); err != nil { + c.Fatalf("Unable to attach policy: %v", err) + } + + newSSHCon := newSSHConnMock(accessKey + "=svc") + _, err = sshPasswordAuth(newSSHCon, []byte(secretKey)) + if err != nil { + c.Fatalf("expected no error but got (%s)", err) + } + + newSSHCon = newSSHConnMock(accessKey) + _, err = sshPasswordAuth(newSSHCon, []byte(secretKey)) + if err != nil { + c.Fatalf("expected no error but got (%s)", err) + } +} + +func (s *TestSuiteIAM) SFTPValidLDAPLoginWithPassword(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + // we need to do this so that the user has a policy before authentication. + // ldap user accounts without policies are denied access in sftp. + policy := "mypolicy" + policyBytes := []byte(`{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::BUCKET/*" + ] + } + ] +}`) + + err := s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + { + userDN := "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" + userReq := madmin.PolicyAssociationReq{ + Policies: []string{policy}, + User: userDN, + } + if _, err := s.adm.AttachPolicyLDAP(ctx, userReq); err != nil { + c.Fatalf("Unable to attach policy: %v", err) + } + + newSSHCon := newSSHConnMock("dillon=ldap") + _, err = sshPasswordAuth(newSSHCon, []byte("dillon")) + if err != nil { + c.Fatal("Password authentication failed for user (dillon):", err) + } + + newSSHCon = newSSHConnMock("dillon") + _, err = sshPasswordAuth(newSSHCon, []byte("dillon")) + if err != nil { + c.Fatal("Password authentication failed for user (dillon):", err) + } + } + { + userDN := "uid=fahim,ou=people,ou=swengg,dc=min,dc=io" + userReq := madmin.PolicyAssociationReq{ + Policies: []string{policy}, + User: userDN, + } + if _, err := s.adm.AttachPolicyLDAP(ctx, userReq); err != nil { + c.Fatalf("Unable to attach policy: %v", err) + } + + newSSHCon := newSSHConnMock("fahim=ldap") + _, err = sshPasswordAuth(newSSHCon, []byte("fahim")) + if err != nil { + c.Fatal("Password authentication failed for user (fahim):", err) + } + + newSSHCon = newSSHConnMock("fahim") + _, err = sshPasswordAuth(newSSHCon, []byte("fahim")) + if err != nil { + c.Fatal("Password authentication failed for user (fahim):", err) + } + } +} diff --git a/cmd/shared-lock.go b/cmd/shared-lock.go new file mode 100644 index 00000000..664b68ee --- /dev/null +++ b/cmd/shared-lock.go @@ -0,0 +1,87 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "time" +) + +var sharedLockTimeout = newDynamicTimeoutWithOpts(dynamicTimeoutOpts{ + timeout: 30 * time.Second, + minimum: 10 * time.Second, + retryInterval: time.Minute, +}) + +type sharedLock struct { + lockContext chan LockContext +} + +func (ld sharedLock) backgroundRoutine(ctx context.Context, objAPI ObjectLayer, lockName string) { + for { + locker := objAPI.NewNSLock(minioMetaBucket, lockName) + lkctx, err := locker.GetLock(ctx, sharedLockTimeout) + if err != nil { + continue + } + + keepLock: + for { + select { + case <-ctx.Done(): + return + case <-lkctx.Context().Done(): + // The context of the lock is canceled, this can happen + // if one lock lost quorum due to cluster instability + // in that case, try to lock again. + break keepLock + case ld.lockContext <- lkctx: + // Send the lock context to anyone asking for it + } + } + } +} + +func mergeContext(ctx1, ctx2 context.Context) (context.Context, context.CancelFunc) { + ctx, cancel := context.WithCancel(context.Background()) + go func() { + select { + case <-ctx1.Done(): + case <-ctx2.Done(): + // The lock acquirer decides to cancel, exit this goroutine + case <-ctx.Done(): + } + + cancel() + }() + return ctx, cancel +} + +func (ld sharedLock) GetLock(ctx context.Context) (context.Context, context.CancelFunc) { + l := <-ld.lockContext + return mergeContext(l.Context(), ctx) +} + +func newSharedLock(ctx context.Context, objAPI ObjectLayer, lockName string) *sharedLock { + l := &sharedLock{ + lockContext: make(chan LockContext), + } + go l.backgroundRoutine(ctx, objAPI, lockName) + + return l +} diff --git a/cmd/signals.go b/cmd/signals.go new file mode 100644 index 00000000..1b2f3ffa --- /dev/null +++ b/cmd/signals.go @@ -0,0 +1,125 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "net/http" + "os" + "strings" + "time" + + "github.com/coreos/go-systemd/v22/daemon" + "github.com/minio/minio/internal/logger" +) + +func shutdownHealMRFWithTimeout() { + const shutdownTimeout = time.Minute + + finished := make(chan struct{}) + go func() { + globalMRFState.shutdown() + close(finished) + }() + select { + case <-time.After(shutdownTimeout): + case <-finished: + } +} + +func handleSignals() { + // Custom exit function + exit := func(success bool) { + if globalLoggerOutput != nil { + globalLoggerOutput.Close() + } + + // If global profiler is set stop before we exit. + globalProfilerMu.Lock() + defer globalProfilerMu.Unlock() + for _, p := range globalProfiler { + p.Stop() + } + + if success { + os.Exit(0) + } + + os.Exit(1) + } + + stopProcess := func() bool { + shutdownHealMRFWithTimeout() // this can take time sometimes, it needs to be executed + // before stopping s3 operations + + // send signal to various go-routines that they need to quit. + cancelGlobalContext() + + if httpServer := newHTTPServerFn(); httpServer != nil { + if err := httpServer.Shutdown(); err != nil && !errors.Is(err, http.ErrServerClosed) { + shutdownLogIf(context.Background(), err) + } + } + + if objAPI := newObjectLayerFn(); objAPI != nil { + shutdownLogIf(context.Background(), objAPI.Shutdown(context.Background())) + } + + if globalBrowserEnabled { + if srv := newConsoleServerFn(); srv != nil { + shutdownLogIf(context.Background(), srv.Shutdown()) + } + } + + if globalEventNotifier != nil { + globalEventNotifier.RemoveAllBucketTargets() + } + + return true + } + + for { + select { + case err := <-globalHTTPServerErrorCh: + shutdownLogIf(context.Background(), err) + exit(stopProcess()) + case osSignal := <-globalOSSignalCh: + logger.Info("Exiting on signal: %s", strings.ToUpper(osSignal.String())) + daemon.SdNotify(false, daemon.SdNotifyStopping) + exit(stopProcess()) + case signal := <-globalServiceSignalCh: + switch signal { + case serviceRestart: + logger.Info("Restarting on service signal") + daemon.SdNotify(false, daemon.SdNotifyReloading) + stop := stopProcess() + rerr := restartProcess() + if rerr == nil { + daemon.SdNotify(false, daemon.SdNotifyReady) + } + shutdownLogIf(context.Background(), rerr) + exit(stop && rerr == nil) + case serviceStop: + logger.Info("Stopping on service signal") + daemon.SdNotify(false, daemon.SdNotifyStopping) + exit(stopProcess()) + } + } + } +} diff --git a/cmd/signature-v2.go b/cmd/signature-v2.go new file mode 100644 index 00000000..1fd42ba7 --- /dev/null +++ b/cmd/signature-v2.go @@ -0,0 +1,414 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "crypto/hmac" + "crypto/sha1" + "crypto/subtle" + "encoding/base64" + "fmt" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + + xhttp "github.com/minio/minio/internal/http" + + "github.com/minio/minio/internal/auth" +) + +// Whitelist resource list that will be used in query string for signature-V2 calculation. +// +// This list should be kept alphabetically sorted, do not hastily edit. +var resourceList = []string{ + "acl", + "cors", + "delete", + "encryption", + "legal-hold", + "lifecycle", + "location", + "logging", + "notification", + "partNumber", + "policy", + "requestPayment", + "response-cache-control", + "response-content-disposition", + "response-content-encoding", + "response-content-language", + "response-content-type", + "response-expires", + "retention", + "select", + "select-type", + "tagging", + "torrent", + "uploadId", + "uploads", + "versionId", + "versioning", + "versions", + "website", +} + +// Signature and API related constants. +const ( + signV2Algorithm = "AWS" +) + +// AWS S3 Signature V2 calculation rule is give here: +// http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationStringToSign +func doesPolicySignatureV2Match(formValues http.Header) (auth.Credentials, APIErrorCode) { + accessKey := formValues.Get(xhttp.AmzAccessKeyID) + + r := &http.Request{Header: formValues} + cred, _, s3Err := checkKeyValid(r, accessKey) + if s3Err != ErrNone { + return cred, s3Err + } + policy := formValues.Get("Policy") + signature := formValues.Get(xhttp.AmzSignatureV2) + if !compareSignatureV2(signature, calculateSignatureV2(policy, cred.SecretKey)) { + return cred, ErrSignatureDoesNotMatch + } + return cred, ErrNone +} + +// Escape encodedQuery string into unescaped list of query params, returns error +// if any while unescaping the values. +func unescapeQueries(encodedQuery string) (unescapedQueries []string, err error) { + for query := range strings.SplitSeq(encodedQuery, "&") { + var unescapedQuery string + unescapedQuery, err = url.QueryUnescape(query) + if err != nil { + return nil, err + } + unescapedQueries = append(unescapedQueries, unescapedQuery) + } + return unescapedQueries, nil +} + +// doesPresignV2SignatureMatch - Verify query headers with presigned signature +// - http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAuthentication.html#RESTAuthenticationQueryStringAuth +// +// returns ErrNone if matches. S3 errors otherwise. +func doesPresignV2SignatureMatch(r *http.Request) APIErrorCode { + // r.RequestURI will have raw encoded URI as sent by the client. + tokens := strings.SplitN(r.RequestURI, "?", 2) + encodedResource := tokens[0] + encodedQuery := "" + if len(tokens) == 2 { + encodedQuery = tokens[1] + } + + var ( + filteredQueries []string + gotSignature string + expires string + accessKey string + err error + ) + + var unescapedQueries []string + unescapedQueries, err = unescapeQueries(encodedQuery) + if err != nil { + return ErrInvalidQueryParams + } + + // Extract the necessary values from presigned query, construct a list of new filtered queries. + for _, query := range unescapedQueries { + keyval := strings.SplitN(query, "=", 2) + if len(keyval) != 2 { + return ErrInvalidQueryParams + } + switch keyval[0] { + case xhttp.AmzAccessKeyID: + accessKey = keyval[1] + case xhttp.AmzSignatureV2: + gotSignature = keyval[1] + case xhttp.Expires: + expires = keyval[1] + default: + filteredQueries = append(filteredQueries, query) + } + } + + // Invalid values returns error. + if accessKey == "" || gotSignature == "" || expires == "" { + return ErrInvalidQueryParams + } + + cred, _, s3Err := checkKeyValid(r, accessKey) + if s3Err != ErrNone { + return s3Err + } + + // Make sure the request has not expired. + expiresInt, err := strconv.ParseInt(expires, 10, 64) + if err != nil { + return ErrMalformedExpires + } + + // Check if the presigned URL has expired. + if expiresInt < UTCNow().Unix() { + return ErrExpiredPresignRequest + } + + encodedResource, err = getResource(encodedResource, r.Host, globalDomainNames) + if err != nil { + return ErrInvalidRequest + } + + expectedSignature := preSignatureV2(cred, r.Method, encodedResource, strings.Join(filteredQueries, "&"), r.Header, expires) + if !compareSignatureV2(gotSignature, expectedSignature) { + return ErrSignatureDoesNotMatch + } + + r.Form.Del(xhttp.Expires) + + return ErrNone +} + +func getReqAccessKeyV2(r *http.Request) (auth.Credentials, bool, APIErrorCode) { + if accessKey := r.Form.Get(xhttp.AmzAccessKeyID); accessKey != "" { + return checkKeyValid(r, accessKey) + } + + // below is V2 Signed Auth header format, splitting on `space` (after the `AWS` string). + // Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature + authFields := strings.Split(r.Header.Get(xhttp.Authorization), " ") + if len(authFields) != 2 { + return auth.Credentials{}, false, ErrMissingFields + } + + // Then will be splitting on ":", this will separate `AWSAccessKeyId` and `Signature` string. + keySignFields := strings.Split(strings.TrimSpace(authFields[1]), ":") + if len(keySignFields) != 2 { + return auth.Credentials{}, false, ErrMissingFields + } + + return checkKeyValid(r, keySignFields[0]) +} + +// Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature; +// Signature = Base64( HMAC-SHA1( YourSecretKey, UTF-8-Encoding-Of( StringToSign ) ) ); +// +// StringToSign = HTTP-Verb + "\n" + +// Content-Md5 + "\n" + +// Content-Type + "\n" + +// Date + "\n" + +// CanonicalizedProtocolHeaders + +// CanonicalizedResource; +// +// CanonicalizedResource = [ SlashSeparator + Bucket ] + +// + +// [ subresource, if present. For example "?acl", "?location", "?logging", or "?torrent"]; +// +// CanonicalizedProtocolHeaders = + +// doesSignV2Match - Verify authorization header with calculated header in accordance with +// - http://docs.aws.amazon.com/AmazonS3/latest/dev/auth-request-sig-v2.html +// returns true if matches, false otherwise. if error is not nil then it is always false + +func validateV2AuthHeader(r *http.Request) (auth.Credentials, APIErrorCode) { + var cred auth.Credentials + v2Auth := r.Header.Get(xhttp.Authorization) + if v2Auth == "" { + return cred, ErrAuthHeaderEmpty + } + + // Verify if the header algorithm is supported or not. + if !strings.HasPrefix(v2Auth, signV2Algorithm) { + return cred, ErrSignatureVersionNotSupported + } + + cred, _, apiErr := getReqAccessKeyV2(r) + if apiErr != ErrNone { + return cred, apiErr + } + + return cred, ErrNone +} + +func doesSignV2Match(r *http.Request) APIErrorCode { + v2Auth := r.Header.Get(xhttp.Authorization) + cred, apiError := validateV2AuthHeader(r) + if apiError != ErrNone { + return apiError + } + + // r.RequestURI will have raw encoded URI as sent by the client. + tokens := strings.SplitN(r.RequestURI, "?", 2) + encodedResource := tokens[0] + encodedQuery := "" + if len(tokens) == 2 { + encodedQuery = tokens[1] + } + + unescapedQueries, err := unescapeQueries(encodedQuery) + if err != nil { + return ErrInvalidQueryParams + } + + encodedResource, err = getResource(encodedResource, r.Host, globalDomainNames) + if err != nil { + return ErrInvalidRequest + } + + prefix := fmt.Sprintf("%s %s:", signV2Algorithm, cred.AccessKey) + if !strings.HasPrefix(v2Auth, prefix) { + return ErrSignatureDoesNotMatch + } + v2Auth = v2Auth[len(prefix):] + expectedAuth := signatureV2(cred, r.Method, encodedResource, strings.Join(unescapedQueries, "&"), r.Header) + if !compareSignatureV2(v2Auth, expectedAuth) { + return ErrSignatureDoesNotMatch + } + return ErrNone +} + +func calculateSignatureV2(stringToSign string, secret string) string { + hm := hmac.New(sha1.New, []byte(secret)) + hm.Write([]byte(stringToSign)) + return base64.StdEncoding.EncodeToString(hm.Sum(nil)) +} + +// Return signature-v2 for the presigned request. +func preSignatureV2(cred auth.Credentials, method string, encodedResource string, encodedQuery string, headers http.Header, expires string) string { + stringToSign := getStringToSignV2(method, encodedResource, encodedQuery, headers, expires) + return calculateSignatureV2(stringToSign, cred.SecretKey) +} + +// Return the signature v2 of a given request. +func signatureV2(cred auth.Credentials, method string, encodedResource string, encodedQuery string, headers http.Header) string { + stringToSign := getStringToSignV2(method, encodedResource, encodedQuery, headers, "") + signature := calculateSignatureV2(stringToSign, cred.SecretKey) + return signature +} + +// compareSignatureV2 returns true if and only if both signatures +// are equal. The signatures are expected to be base64 encoded strings +// according to the AWS S3 signature V2 spec. +func compareSignatureV2(sig1, sig2 string) bool { + // Decode signature string to binary byte-sequence representation is required + // as Base64 encoding of a value is not unique: + // For example "aGVsbG8=" and "aGVsbG8=\r" will result in the same byte slice. + signature1, err := base64.StdEncoding.DecodeString(sig1) + if err != nil { + return false + } + signature2, err := base64.StdEncoding.DecodeString(sig2) + if err != nil { + return false + } + return subtle.ConstantTimeCompare(signature1, signature2) == 1 +} + +// Return canonical headers. +func canonicalizedAmzHeadersV2(headers http.Header) string { + var keys []string + keyval := make(map[string]string, len(headers)) + for key := range headers { + lkey := strings.ToLower(key) + if !strings.HasPrefix(lkey, "x-amz-") { + continue + } + keys = append(keys, lkey) + keyval[lkey] = strings.Join(headers[key], ",") + } + sort.Strings(keys) + var canonicalHeaders []string + for _, key := range keys { + canonicalHeaders = append(canonicalHeaders, key+":"+keyval[key]) + } + return strings.Join(canonicalHeaders, "\n") +} + +// Return canonical resource string. +func canonicalizedResourceV2(encodedResource, encodedQuery string) string { + queries := strings.Split(encodedQuery, "&") + keyval := make(map[string]string) + for _, query := range queries { + key := query + val := "" + index := strings.Index(query, "=") + if index != -1 { + key = query[:index] + val = query[index+1:] + } + keyval[key] = val + } + + var canonicalQueries []string + for _, key := range resourceList { + val, ok := keyval[key] + if !ok { + continue + } + if val == "" { + canonicalQueries = append(canonicalQueries, key) + continue + } + canonicalQueries = append(canonicalQueries, key+"="+val) + } + + // The queries will be already sorted as resourceList is sorted, if canonicalQueries + // is empty strings.Join returns empty. + canonicalQuery := strings.Join(canonicalQueries, "&") + if canonicalQuery != "" { + return encodedResource + "?" + canonicalQuery + } + return encodedResource +} + +// Return string to sign under two different conditions. +// - if expires string is set then string to sign includes date instead of the Date header. +// - if expires string is empty then string to sign includes date header instead. +func getStringToSignV2(method string, encodedResource, encodedQuery string, headers http.Header, expires string) string { + canonicalHeaders := canonicalizedAmzHeadersV2(headers) + if len(canonicalHeaders) > 0 { + canonicalHeaders += "\n" + } + + date := expires // Date is set to expires date for presign operations. + if date == "" { + // If expires date is empty then request header Date is used. + date = headers.Get(xhttp.Date) + } + + // From the Amazon docs: + // + // StringToSign = HTTP-Verb + "\n" + + // Content-Md5 + "\n" + + // Content-Type + "\n" + + // Date/Expires + "\n" + + // CanonicalizedProtocolHeaders + + // CanonicalizedResource; + stringToSign := strings.Join([]string{ + method, + headers.Get(xhttp.ContentMD5), + headers.Get(xhttp.ContentType), + date, + canonicalHeaders, + }, "\n") + + return stringToSign + canonicalizedResourceV2(encodedResource, encodedQuery) +} diff --git a/cmd/signature-v2_test.go b/cmd/signature-v2_test.go new file mode 100644 index 00000000..b2c2b012 --- /dev/null +++ b/cmd/signature-v2_test.go @@ -0,0 +1,275 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "fmt" + "net/http" + "net/url" + "os" + "sort" + "testing" +) + +// Tests for 'func TestResourceListSorting(t *testing.T)'. +func TestResourceListSorting(t *testing.T) { + sortedResourceList := make([]string, len(resourceList)) + copy(sortedResourceList, resourceList) + sort.Strings(sortedResourceList) + for i := range resourceList { + if resourceList[i] != sortedResourceList[i] { + t.Errorf("Expected resourceList[%d] = \"%s\", resourceList is not correctly sorted.", i, sortedResourceList[i]) + break + } + } +} + +// Tests presigned v2 signature. +func TestDoesPresignedV2SignatureMatch(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + obj, fsDir, err := prepareFS(ctx) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(fsDir) + if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil { + t.Fatal(err) + } + + now := UTCNow() + + var ( + accessKey = globalActiveCred.AccessKey + secretKey = globalActiveCred.SecretKey + ) + testCases := []struct { + queryParams map[string]string + expected APIErrorCode + }{ + // (0) Should error without a set URL query. + { + expected: ErrInvalidQueryParams, + }, + // (1) Should error on an invalid access key. + { + queryParams: map[string]string{ + "Expires": "60", + "Signature": "badsignature", + "AWSAccessKeyId": "Z7IXGOO6BZ0REAN1Q26I", + }, + expected: ErrInvalidAccessKeyID, + }, + // (2) Should error with malformed expires. + { + queryParams: map[string]string{ + "Expires": "60s", + "Signature": "badsignature", + "AWSAccessKeyId": accessKey, + }, + expected: ErrMalformedExpires, + }, + // (3) Should give an expired request if it has expired. + { + queryParams: map[string]string{ + "Expires": "60", + "Signature": "badsignature", + "AWSAccessKeyId": accessKey, + }, + expected: ErrExpiredPresignRequest, + }, + // (4) Should error when the signature does not match. + { + queryParams: map[string]string{ + "Expires": fmt.Sprintf("%d", now.Unix()+60), + "Signature": "badsignature", + "AWSAccessKeyId": accessKey, + }, + expected: ErrSignatureDoesNotMatch, + }, + // (5) Should error when the signature does not match. + { + queryParams: map[string]string{ + "Expires": fmt.Sprintf("%d", now.Unix()+60), + "Signature": "zOM2YrY/yAQe15VWmT78OlBrK6g=", + "AWSAccessKeyId": accessKey, + }, + expected: ErrSignatureDoesNotMatch, + }, + // (6) Should not error signature matches with extra query params. + { + queryParams: map[string]string{ + "response-content-disposition": "attachment; filename=\"4K%2d4M.txt\"", + }, + expected: ErrNone, + }, + // (7) Should not error signature matches with no special query params. + { + queryParams: map[string]string{}, + expected: ErrNone, + }, + } + + // Run each test case individually. + for i, testCase := range testCases { + // Turn the map[string]string into map[string][]string, because Go. + query := url.Values{} + for key, value := range testCase.queryParams { + query.Set(key, value) + } + // Create a request to use. + req, err := http.NewRequest(http.MethodGet, "http://host/a/b?"+query.Encode(), nil) + if err != nil { + t.Errorf("(%d) failed to create http.Request, got %v", i, err) + } + if testCase.expected != ErrNone { + // Should be set since we are simulating a http server. + req.RequestURI = req.URL.RequestURI() + // Check if it matches! + errCode := doesPresignV2SignatureMatch(req) + if errCode != testCase.expected { + t.Errorf("(%d) expected to get %s, instead got %s", i, niceError(testCase.expected), niceError(errCode)) + } + } else { + err = preSignV2(req, accessKey, secretKey, now.Unix()+60) + if err != nil { + t.Fatalf("(%d) failed to preSignV2 http request, got %v", i, err) + } + // Should be set since we are simulating a http server. + req.RequestURI = req.URL.RequestURI() + errCode := doesPresignV2SignatureMatch(req) + if errCode != testCase.expected { + t.Errorf("(%d) expected to get success, instead got %s", i, niceError(errCode)) + } + } + } +} + +// TestValidateV2AuthHeader - Tests validate the logic of V2 Authorization header validator. +func TestValidateV2AuthHeader(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + obj, fsDir, err := prepareFS(ctx) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(fsDir) + if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil { + t.Fatal(err) + } + + accessID := globalActiveCred.AccessKey + testCases := []struct { + authString string + expectedError APIErrorCode + }{ + // Test case - 1. + // Case with empty V2AuthString. + { + authString: "", + expectedError: ErrAuthHeaderEmpty, + }, + // Test case - 2. + // Test case with `signV2Algorithm` ("AWS") not being the prefix. + { + authString: "NoV2Prefix", + expectedError: ErrSignatureVersionNotSupported, + }, + // Test case - 3. + // Test case with missing parts in the Auth string. + // below is the correct format of V2 Authorization header. + // Authorization = "AWS" + " " + AWSAccessKeyId + ":" + Signature + { + authString: signV2Algorithm, + expectedError: ErrMissingFields, + }, + // Test case - 4. + // Test case with signature part missing. + { + authString: fmt.Sprintf("%s %s", signV2Algorithm, accessID), + expectedError: ErrMissingFields, + }, + // Test case - 5. + // Test case with wrong accessID. + { + authString: fmt.Sprintf("%s %s:%s", signV2Algorithm, "InvalidAccessID", "signature"), + expectedError: ErrInvalidAccessKeyID, + }, + // Test case - 6. + // Case with right accessID and format. + { + authString: fmt.Sprintf("%s %s:%s", signV2Algorithm, accessID, "signature"), + expectedError: ErrNone, + }, + } + + for i, testCase := range testCases { + t.Run(fmt.Sprintf("Case %d AuthStr \"%s\".", i+1, testCase.authString), func(t *testing.T) { + req := &http.Request{ + Header: make(http.Header), + URL: &url.URL{}, + } + req.Header.Set("Authorization", testCase.authString) + _, actualErrCode := validateV2AuthHeader(req) + + if testCase.expectedError != actualErrCode { + t.Errorf("Expected the error code to be %v, got %v.", testCase.expectedError, actualErrCode) + } + }) + } +} + +func TestDoesPolicySignatureV2Match(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + obj, fsDir, err := prepareFS(ctx) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(fsDir) + if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil { + t.Fatal(err) + } + + creds := globalActiveCred + policy := "policy" + testCases := []struct { + accessKey string + policy string + signature string + errCode APIErrorCode + }{ + {"invalidAccessKey", policy, calculateSignatureV2(policy, creds.SecretKey), ErrInvalidAccessKeyID}, + {creds.AccessKey, policy, calculateSignatureV2("random", creds.SecretKey), ErrSignatureDoesNotMatch}, + {creds.AccessKey, policy, calculateSignatureV2(policy, creds.SecretKey), ErrNone}, + } + for i, test := range testCases { + formValues := make(http.Header) + formValues.Set("Awsaccesskeyid", test.accessKey) + formValues.Set("Signature", test.signature) + formValues.Set("Policy", test.policy) + _, errCode := doesPolicySignatureV2Match(formValues) + if errCode != test.errCode { + t.Fatalf("(%d) expected to get %s, instead got %s", i+1, niceError(test.errCode), niceError(errCode)) + } + } +} diff --git a/cmd/signature-v4-parser.go b/cmd/signature-v4-parser.go new file mode 100644 index 00000000..f6866cb8 --- /dev/null +++ b/cmd/signature-v4-parser.go @@ -0,0 +1,306 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "net/http" + "net/url" + "strings" + "time" + + "github.com/minio/minio/internal/auth" + xhttp "github.com/minio/minio/internal/http" +) + +// credentialHeader data type represents structured form of Credential +// string from authorization header. +type credentialHeader struct { + accessKey string + scope struct { + date time.Time + region string + service string + request string + } +} + +// Return scope string. +func (c credentialHeader) getScope() string { + return strings.Join([]string{ + c.scope.date.Format(yyyymmdd), + c.scope.region, + c.scope.service, + c.scope.request, + }, SlashSeparator) +} + +func getReqAccessKeyV4(r *http.Request, region string, stype serviceType) (auth.Credentials, bool, APIErrorCode) { + ch, s3Err := parseCredentialHeader("Credential="+r.Form.Get(xhttp.AmzCredential), region, stype) + if s3Err != ErrNone { + // Strip off the Algorithm prefix. + v4Auth := strings.TrimPrefix(r.Header.Get("Authorization"), signV4Algorithm) + authFields := strings.Split(strings.TrimSpace(v4Auth), ",") + if len(authFields) != 3 { + return auth.Credentials{}, false, ErrMissingFields + } + ch, s3Err = parseCredentialHeader(authFields[0], region, stype) + if s3Err != ErrNone { + return auth.Credentials{}, false, s3Err + } + } + return checkKeyValid(r, ch.accessKey) +} + +// parse credentialHeader string into its structured form. +func parseCredentialHeader(credElement string, region string, stype serviceType) (ch credentialHeader, aec APIErrorCode) { + creds := strings.SplitN(strings.TrimSpace(credElement), "=", 2) + if len(creds) != 2 { + return ch, ErrMissingFields + } + if creds[0] != "Credential" { + return ch, ErrMissingCredTag + } + credElements := strings.Split(strings.TrimRight(strings.TrimSpace(creds[1]), SlashSeparator), SlashSeparator) + if len(credElements) < 5 { + return ch, ErrCredMalformed + } + accessKey := strings.Join(credElements[:len(credElements)-4], SlashSeparator) // The access key may contain one or more `/` + if !auth.IsAccessKeyValid(accessKey) { + return ch, ErrInvalidAccessKeyID + } + // Save access key id. + cred := credentialHeader{ + accessKey: accessKey, + } + credElements = credElements[len(credElements)-4:] + var e error + cred.scope.date, e = time.Parse(yyyymmdd, credElements[0]) + if e != nil { + return ch, ErrMalformedCredentialDate + } + + cred.scope.region = credElements[1] + // Verify if region is valid. + sRegion := cred.scope.region + // Region is set to be empty, we use whatever was sent by the + // request and proceed further. This is a work-around to address + // an important problem for ListBuckets() getting signed with + // different regions. + if region == "" { + region = sRegion + } + // Should validate region, only if region is set. + if !isValidRegion(sRegion, region) { + return ch, ErrAuthorizationHeaderMalformed + } + if credElements[2] != string(stype) { + if stype == serviceSTS { + return ch, ErrInvalidServiceSTS + } + return ch, ErrInvalidServiceS3 + } + cred.scope.service = credElements[2] + if credElements[3] != "aws4_request" { + return ch, ErrInvalidRequestVersion + } + cred.scope.request = credElements[3] + return cred, ErrNone +} + +// Parse signature from signature tag. +func parseSignature(signElement string) (string, APIErrorCode) { + signFields := strings.Split(strings.TrimSpace(signElement), "=") + if len(signFields) != 2 { + return "", ErrMissingFields + } + if signFields[0] != "Signature" { + return "", ErrMissingSignTag + } + if signFields[1] == "" { + return "", ErrMissingFields + } + signature := signFields[1] + return signature, ErrNone +} + +// Parse slice of signed headers from signed headers tag. +func parseSignedHeader(signedHdrElement string) ([]string, APIErrorCode) { + signedHdrFields := strings.Split(strings.TrimSpace(signedHdrElement), "=") + if len(signedHdrFields) != 2 { + return nil, ErrMissingFields + } + if signedHdrFields[0] != "SignedHeaders" { + return nil, ErrMissingSignHeadersTag + } + if signedHdrFields[1] == "" { + return nil, ErrMissingFields + } + signedHeaders := strings.Split(signedHdrFields[1], ";") + return signedHeaders, ErrNone +} + +// signValues data type represents structured form of AWS Signature V4 header. +type signValues struct { + Credential credentialHeader + SignedHeaders []string + Signature string +} + +// preSignValues data type represents structured form of AWS Signature V4 query string. +type preSignValues struct { + signValues + Date time.Time + Expires time.Duration +} + +// Parses signature version '4' query string of the following form. +// +// querystring = X-Amz-Algorithm=algorithm +// querystring += &X-Amz-Credential= urlencode(accessKey + '/' + credential_scope) +// querystring += &X-Amz-Date=date +// querystring += &X-Amz-Expires=timeout interval +// querystring += &X-Amz-SignedHeaders=signed_headers +// querystring += &X-Amz-Signature=signature +// +// verifies if any of the necessary query params are missing in the presigned request. +func doesV4PresignParamsExist(query url.Values) APIErrorCode { + v4PresignQueryParams := []string{xhttp.AmzAlgorithm, xhttp.AmzCredential, xhttp.AmzSignature, xhttp.AmzDate, xhttp.AmzSignedHeaders, xhttp.AmzExpires} + for _, v4PresignQueryParam := range v4PresignQueryParams { + if _, ok := query[v4PresignQueryParam]; !ok { + return ErrInvalidQueryParams + } + } + return ErrNone +} + +// Parses all the presigned signature values into separate elements. +func parsePreSignV4(query url.Values, region string, stype serviceType) (psv preSignValues, aec APIErrorCode) { + // verify whether the required query params exist. + aec = doesV4PresignParamsExist(query) + if aec != ErrNone { + return psv, aec + } + + // Verify if the query algorithm is supported or not. + if query.Get(xhttp.AmzAlgorithm) != signV4Algorithm { + return psv, ErrInvalidQuerySignatureAlgo + } + + // Initialize signature version '4' structured header. + preSignV4Values := preSignValues{} + + // Save credential. + preSignV4Values.Credential, aec = parseCredentialHeader("Credential="+query.Get(xhttp.AmzCredential), region, stype) + if aec != ErrNone { + return psv, aec + } + + var e error + // Save date in native time.Time. + preSignV4Values.Date, e = time.Parse(iso8601Format, query.Get(xhttp.AmzDate)) + if e != nil { + return psv, ErrMalformedPresignedDate + } + + // Save expires in native time.Duration. + preSignV4Values.Expires, e = time.ParseDuration(query.Get(xhttp.AmzExpires) + "s") + if e != nil { + return psv, ErrMalformedExpires + } + + if preSignV4Values.Expires < 0 { + return psv, ErrNegativeExpires + } + + // Check if Expiry time is less than 7 days (value in seconds). + if preSignV4Values.Expires.Seconds() > 604800 { + return psv, ErrMaximumExpires + } + + if preSignV4Values.Date.IsZero() || preSignV4Values.Date.Equal(timeSentinel) { + return psv, ErrMalformedPresignedDate + } + + // Save signed headers. + preSignV4Values.SignedHeaders, aec = parseSignedHeader("SignedHeaders=" + query.Get(xhttp.AmzSignedHeaders)) + if aec != ErrNone { + return psv, aec + } + + // Save signature. + preSignV4Values.Signature, aec = parseSignature("Signature=" + query.Get(xhttp.AmzSignature)) + if aec != ErrNone { + return psv, aec + } + + // Return structured form of signature query string. + return preSignV4Values, ErrNone +} + +// Parses signature version '4' header of the following form. +// +// Authorization: algorithm Credential=accessKeyID/credScope, \ +// SignedHeaders=signedHeaders, Signature=signature +func parseSignV4(v4Auth string, region string, stype serviceType) (sv signValues, aec APIErrorCode) { + // credElement is fetched first to skip replacing the space in access key. + credElement := strings.TrimPrefix(strings.Split(strings.TrimSpace(v4Auth), ",")[0], signV4Algorithm) + // Replace all spaced strings, some clients can send spaced + // parameters and some won't. So we pro-actively remove any spaces + // to make parsing easier. + v4Auth = strings.ReplaceAll(v4Auth, " ", "") + if v4Auth == "" { + return sv, ErrAuthHeaderEmpty + } + + // Verify if the header algorithm is supported or not. + if !strings.HasPrefix(v4Auth, signV4Algorithm) { + return sv, ErrSignatureVersionNotSupported + } + + // Strip off the Algorithm prefix. + v4Auth = strings.TrimPrefix(v4Auth, signV4Algorithm) + authFields := strings.Split(strings.TrimSpace(v4Auth), ",") + if len(authFields) != 3 { + return sv, ErrMissingFields + } + + // Initialize signature version '4' structured header. + signV4Values := signValues{} + + var s3Err APIErrorCode + // Save credential values. + signV4Values.Credential, s3Err = parseCredentialHeader(strings.TrimSpace(credElement), region, stype) + if s3Err != ErrNone { + return sv, s3Err + } + + // Save signed headers. + signV4Values.SignedHeaders, s3Err = parseSignedHeader(authFields[1]) + if s3Err != ErrNone { + return sv, s3Err + } + + // Save signature. + signV4Values.Signature, s3Err = parseSignature(authFields[2]) + if s3Err != ErrNone { + return sv, s3Err + } + + // Return the structure here. + return signV4Values, ErrNone +} diff --git a/cmd/signature-v4-parser_test.go b/cmd/signature-v4-parser_test.go new file mode 100644 index 00000000..3d900133 --- /dev/null +++ b/cmd/signature-v4-parser_test.go @@ -0,0 +1,900 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "net/url" + "strconv" + "strings" + "testing" + "time" +) + +// generates credential string from its fields. +func generateCredentialStr(accessKey, date, region, service, requestVersion string) string { + return "Credential=" + joinWithSlash(accessKey, date, region, service, requestVersion) +} + +// joins the argument strings with a '/' and returns it. +func joinWithSlash(accessKey, date, region, service, requestVersion string) string { + return strings.Join([]string{ + accessKey, + date, + region, + service, + requestVersion, + }, SlashSeparator) +} + +// generate CredentialHeader from its fields. +func generateCredentials(t *testing.T, accessKey string, date string, region, service, requestVersion string) credentialHeader { + cred := credentialHeader{ + accessKey: accessKey, + } + parsedDate, err := time.Parse(yyyymmdd, date) + if err != nil { + t.Fatalf("Failed to parse date") + } + cred.scope.date = parsedDate + cred.scope.region = region + cred.scope.service = service + cred.scope.request = requestVersion + + return cred +} + +// validates the credential fields against the expected credential. +func validateCredentialfields(t *testing.T, testNum int, expectedCredentials credentialHeader, actualCredential credentialHeader) { + if expectedCredentials.accessKey != actualCredential.accessKey { + t.Errorf("Test %d: AccessKey mismatch: Expected \"%s\", got \"%s\"", testNum, expectedCredentials.accessKey, actualCredential.accessKey) + } + if !expectedCredentials.scope.date.Equal(actualCredential.scope.date) { + t.Errorf("Test %d: Date mismatch:Expected \"%s\", got \"%s\"", testNum, expectedCredentials.scope.date, actualCredential.scope.date) + } + if expectedCredentials.scope.region != actualCredential.scope.region { + t.Errorf("Test %d: region mismatch:Expected \"%s\", got \"%s\"", testNum, expectedCredentials.scope.region, actualCredential.scope.region) + } + if expectedCredentials.scope.service != actualCredential.scope.service { + t.Errorf("Test %d: service mismatch:Expected \"%s\", got \"%s\"", testNum, expectedCredentials.scope.service, actualCredential.scope.service) + } + + if expectedCredentials.scope.request != actualCredential.scope.request { + t.Errorf("Test %d: scope request mismatch:Expected \"%s\", got \"%s\"", testNum, expectedCredentials.scope.request, actualCredential.scope.request) + } +} + +// TestParseCredentialHeader - validates the format validator and extractor for the Credential header in an aws v4 request. +// A valid format of credential should be of the following format. +// Credential = accessKey + SlashSeparator+ scope +// where scope = string.Join([]string{ currTime.Format(yyyymmdd), +// +// globalMinioDefaultRegion, +// "s3", +// "aws4_request", +// },SlashSeparator) +func TestParseCredentialHeader(t *testing.T) { + sampleTimeStr := UTCNow().Format(yyyymmdd) + + testCases := []struct { + inputCredentialStr string + expectedCredentials credentialHeader + expectedErrCode APIErrorCode + }{ + // Test Case - 1. + // Test case with no '=' in te inputCredentialStr. + { + inputCredentialStr: "Credential", + expectedCredentials: credentialHeader{}, + expectedErrCode: ErrMissingFields, + }, + // Test Case - 2. + // Test case with no "Credential" string in te inputCredentialStr. + { + inputCredentialStr: "Cred=", + expectedCredentials: credentialHeader{}, + expectedErrCode: ErrMissingCredTag, + }, + // Test Case - 3. + // Test case with malformed credentials. + { + inputCredentialStr: "Credential=abc", + expectedCredentials: credentialHeader{}, + expectedErrCode: ErrCredMalformed, + }, + // Test Case - 4. + // Test case with AccessKey of length 2. + { + inputCredentialStr: generateCredentialStr( + "^#", + UTCNow().Format(yyyymmdd), + "ABCD", + "ABCD", + "ABCD"), + expectedCredentials: credentialHeader{}, + expectedErrCode: ErrInvalidAccessKeyID, + }, + // Test Case - 5. + // Test case with invalid date format date. + // a valid date format for credentials is "yyyymmdd". + { + inputCredentialStr: generateCredentialStr( + "Z7IXGOO6BZ0REAN1Q26I", + UTCNow().String(), + "ABCD", + "ABCD", + "ABCD"), + expectedCredentials: credentialHeader{}, + expectedErrCode: ErrMalformedCredentialDate, + }, + // Test Case - 6. + // Test case with invalid service. + // "s3" is the valid service string. + { + inputCredentialStr: generateCredentialStr( + "Z7IXGOO6BZ0REAN1Q26I", + UTCNow().Format(yyyymmdd), + "us-west-1", + "ABCD", + "ABCD"), + expectedCredentials: credentialHeader{}, + expectedErrCode: ErrInvalidServiceS3, + }, + // Test Case - 7. + // Test case with invalid region. + { + inputCredentialStr: generateCredentialStr( + "Z7IXGOO6BZ0REAN1Q26I", + UTCNow().Format(yyyymmdd), + "us-west-2", + "s3", + "aws4_request"), + expectedCredentials: credentialHeader{}, + expectedErrCode: ErrAuthorizationHeaderMalformed, + }, + // Test Case - 8. + // Test case with invalid request version. + // "aws4_request" is the valid request version. + { + inputCredentialStr: generateCredentialStr( + "Z7IXGOO6BZ0REAN1Q26I", + UTCNow().Format(yyyymmdd), + "us-west-1", + "s3", + "ABCD"), + expectedCredentials: credentialHeader{}, + expectedErrCode: ErrInvalidRequestVersion, + }, + // Test Case - 9. + // Test case with right inputs. Expected to return a valid CredentialHeader. + // "aws4_request" is the valid request version. + { + inputCredentialStr: generateCredentialStr( + "Z7IXGOO6BZ0REAN1Q26I", + sampleTimeStr, + "us-west-1", + "s3", + "aws4_request"), + expectedCredentials: generateCredentials( + t, + "Z7IXGOO6BZ0REAN1Q26I", + sampleTimeStr, + "us-west-1", + "s3", + "aws4_request"), + expectedErrCode: ErrNone, + }, + // Test Case - 10. + // Test case with right inputs -> AccessKey contains `/`. See minio/#6443 + // "aws4_request" is the valid request version. + { + inputCredentialStr: generateCredentialStr( + "LOCALKEY/DEV/1", + sampleTimeStr, + "us-west-1", + "s3", + "aws4_request"), + expectedCredentials: generateCredentials( + t, + "LOCALKEY/DEV/1", + sampleTimeStr, + "us-west-1", + "s3", + "aws4_request"), + expectedErrCode: ErrNone, + }, + // Test Case - 11. + // Test case with right inputs -> AccessKey contains `=`. See minio/#7376 + // "aws4_request" is the valid request version. + { + inputCredentialStr: generateCredentialStr( + "LOCALKEY/DEV/1=", + sampleTimeStr, + "us-west-1", + "s3", + "aws4_request"), + expectedCredentials: generateCredentials( + t, + "LOCALKEY/DEV/1=", + sampleTimeStr, + "us-west-1", + "s3", + "aws4_request"), + expectedErrCode: ErrNone, + }, + // Test Case - 12. + // Test case with right inputs but trailing `/`. Expected to return a valid CredentialHeader. + // "aws4_request" is the valid request version. + { + inputCredentialStr: generateCredentialStr( + "Z7IXGOO6BZ0REAN1Q26I", + sampleTimeStr, + "us-west-1", + "s3", + "aws4_request/"), + expectedCredentials: generateCredentials( + t, + "Z7IXGOO6BZ0REAN1Q26I", + sampleTimeStr, + "us-west-1", + "s3", + "aws4_request"), + expectedErrCode: ErrNone, + }, + } + + for i, testCase := range testCases { + actualCredential, actualErrCode := parseCredentialHeader(testCase.inputCredentialStr, "us-west-1", "s3") + // validating the credential fields. + if testCase.expectedErrCode != actualErrCode { + t.Fatalf("Test %d: Expected the APIErrCode to be %s, got %s", i+1, errorCodes[testCase.expectedErrCode].Code, errorCodes[actualErrCode].Code) + } + if actualErrCode == ErrNone { + validateCredentialfields(t, i+1, testCase.expectedCredentials, actualCredential) + } + } +} + +// TestParseSignature - validates the logic for extracting the signature string. +func TestParseSignature(t *testing.T) { + testCases := []struct { + inputSignElement string + expectedSignStr string + expectedErrCode APIErrorCode + }{ + // Test case - 1. + // SignElement doesn't have 2 parts on an attempt to split at '='. + // ErrMissingFields expected. + { + inputSignElement: "Signature", + expectedSignStr: "", + expectedErrCode: ErrMissingFields, + }, + // Test case - 2. + // SignElement does have 2 parts but doesn't have valid signature value. + // ErrMissingFields expected. + { + inputSignElement: "Signature=", + expectedSignStr: "", + expectedErrCode: ErrMissingFields, + }, + // Test case - 3. + // SignElement with missing "SignatureTag",ErrMissingSignTag expected. + { + inputSignElement: "Sign=", + expectedSignStr: "", + expectedErrCode: ErrMissingSignTag, + }, + // Test case - 4. + // Test case with valid inputs. + { + inputSignElement: "Signature=abcd", + expectedSignStr: "abcd", + expectedErrCode: ErrNone, + }, + } + for i, testCase := range testCases { + actualSignStr, actualErrCode := parseSignature(testCase.inputSignElement) + if testCase.expectedErrCode != actualErrCode { + t.Fatalf("Test %d: Expected the APIErrCode to be %d, got %d", i+1, testCase.expectedErrCode, actualErrCode) + } + if actualErrCode == ErrNone { + if testCase.expectedSignStr != actualSignStr { + t.Errorf("Test %d: Expected the result to be \"%s\", but got \"%s\". ", i+1, testCase.expectedSignStr, actualSignStr) + } + } + } +} + +// TestParseSignedHeaders - validates the logic for extracting the signature string. +func TestParseSignedHeaders(t *testing.T) { + testCases := []struct { + inputSignElement string + expectedSignedHeaders []string + expectedErrCode APIErrorCode + }{ + // Test case - 1. + // SignElement doesn't have 2 parts on an attempt to split at '='. + // ErrMissingFields expected. + { + inputSignElement: "SignedHeaders", + expectedSignedHeaders: nil, + expectedErrCode: ErrMissingFields, + }, + // Test case - 2. + // SignElement with missing "SigHeaderTag",ErrMissingSignHeadersTag expected. + { + inputSignElement: "Sign=", + expectedSignedHeaders: nil, + expectedErrCode: ErrMissingSignHeadersTag, + }, + // Test case - 3. + // Test case with valid inputs. + { + inputSignElement: "SignedHeaders=host;x-amz-content-sha256;x-amz-date", + expectedSignedHeaders: []string{"host", "x-amz-content-sha256", "x-amz-date"}, + expectedErrCode: ErrNone, + }, + } + + for i, testCase := range testCases { + actualSignedHeaders, actualErrCode := parseSignedHeader(testCase.inputSignElement) + if testCase.expectedErrCode != actualErrCode { + t.Errorf("Test %d: Expected the APIErrCode to be %d, got %d", i+1, testCase.expectedErrCode, actualErrCode) + } + if actualErrCode == ErrNone { + if strings.Join(testCase.expectedSignedHeaders, ",") != strings.Join(actualSignedHeaders, ",") { + t.Errorf("Test %d: Expected the result to be \"%v\", but got \"%v\". ", i+1, testCase.expectedSignedHeaders, actualSignedHeaders) + } + } + } +} + +// TestParseSignV4 - Tests Parsing of v4 signature form the authorization string. +func TestParseSignV4(t *testing.T) { + sampleTimeStr := UTCNow().Format(yyyymmdd) + testCases := []struct { + inputV4AuthStr string + expectedAuthField signValues + expectedErrCode APIErrorCode + }{ + // Test case - 1. + // Test case with empty auth string. + { + inputV4AuthStr: "", + expectedAuthField: signValues{}, + expectedErrCode: ErrAuthHeaderEmpty, + }, + // Test case - 2. + // Test case with no sign v4 Algorithm prefix. + // A valid authorization string should begin(prefix) + { + inputV4AuthStr: "no-singv4AlgorithmPrefix", + expectedAuthField: signValues{}, + expectedErrCode: ErrSignatureVersionNotSupported, + }, + // Test case - 3. + // Test case with missing fields. + // A valid authorization string should have 3 fields. + { + inputV4AuthStr: signV4Algorithm, + expectedAuthField: signValues{}, + expectedErrCode: ErrMissingFields, + }, + // Test case - 4. + // Test case with invalid credential field. + { + inputV4AuthStr: signV4Algorithm + " Cred=,a,b", + expectedAuthField: signValues{}, + expectedErrCode: ErrMissingCredTag, + }, + // Test case - 5. + // Auth field with missing "SigHeaderTag",ErrMissingSignHeadersTag expected. + // A valid credential is generated. + // Test case with invalid credential field. + { + inputV4AuthStr: signV4Algorithm + + strings.Join([]string{ + // generating a valid credential field. + generateCredentialStr( + "Z7IXGOO6BZ0REAN1Q26I", + sampleTimeStr, + "us-west-1", + "s3", + "aws4_request"), + // Incorrect SignedHeader field. + "SignIncorrectHeader=", + "b", + }, ","), + + expectedAuthField: signValues{}, + expectedErrCode: ErrMissingSignHeadersTag, + }, + // Test case - 6. + // Auth string with missing "SignatureTag",ErrMissingSignTag expected. + // A valid credential is generated. + // Test case with invalid credential field. + { + inputV4AuthStr: signV4Algorithm + + strings.Join([]string{ + // generating a valid credential. + generateCredentialStr( + "Z7IXGOO6BZ0REAN1Q26I", + sampleTimeStr, + "us-west-1", + "s3", + "aws4_request"), + // valid SignedHeader. + "SignedHeaders=host;x-amz-content-sha256;x-amz-date", + // invalid Signature field. + // a valid signature is of form "Signature=" + "Sign=", + }, ","), + + expectedAuthField: signValues{}, + expectedErrCode: ErrMissingSignTag, + }, + // Test case - 7. + { + inputV4AuthStr: signV4Algorithm + + strings.Join([]string{ + // generating a valid credential. + generateCredentialStr( + "Z7IXGOO6BZ0REAN1Q26I", + sampleTimeStr, + "us-west-1", + "s3", + "aws4_request"), + // valid SignedHeader. + "SignedHeaders=host;x-amz-content-sha256;x-amz-date", + // valid Signature field. + // a valid signature is of form "Signature=" + "Signature=abcd", + }, ","), + expectedAuthField: signValues{ + Credential: generateCredentials( + t, + "Z7IXGOO6BZ0REAN1Q26I", + sampleTimeStr, + "us-west-1", + "s3", + "aws4_request"), + SignedHeaders: []string{"host", "x-amz-content-sha256", "x-amz-date"}, + Signature: "abcd", + }, + expectedErrCode: ErrNone, + }, + // Test case - 8. + { + inputV4AuthStr: signV4Algorithm + + strings.Join([]string{ + // generating a valid credential. + generateCredentialStr( + "access key", + sampleTimeStr, + "us-west-1", + "s3", + "aws4_request"), + // valid SignedHeader. + "SignedHeaders=host;x-amz-content-sha256;x-amz-date", + // valid Signature field. + // a valid signature is of form "Signature=" + "Signature=abcd", + }, ","), + expectedAuthField: signValues{ + Credential: generateCredentials( + t, + "access key", + sampleTimeStr, + "us-west-1", + "s3", + "aws4_request"), + SignedHeaders: []string{"host", "x-amz-content-sha256", "x-amz-date"}, + Signature: "abcd", + }, + expectedErrCode: ErrNone, + }, + } + + for i, testCase := range testCases { + parsedAuthField, actualErrCode := parseSignV4(testCase.inputV4AuthStr, "", "s3") + + if testCase.expectedErrCode != actualErrCode { + t.Fatalf("Test %d: Expected the APIErrCode to be %d, got %d", i+1, testCase.expectedErrCode, actualErrCode) + } + + if actualErrCode == ErrNone { + // validating the extracted/parsed credential fields. + validateCredentialfields(t, i+1, testCase.expectedAuthField.Credential, parsedAuthField.Credential) + + // validating the extraction/parsing of signature field. + if !compareSignatureV4(testCase.expectedAuthField.Signature, parsedAuthField.Signature) { + t.Errorf("Test %d: Parsed Signature field mismatch: Expected \"%s\", got \"%s\"", i+1, testCase.expectedAuthField.Signature, parsedAuthField.Signature) + } + + // validating the extracted signed headers. + if strings.Join(testCase.expectedAuthField.SignedHeaders, ",") != strings.Join(parsedAuthField.SignedHeaders, ",") { + t.Errorf("Test %d: Expected the result to be \"%v\", but got \"%v\". ", i+1, testCase.expectedAuthField, parsedAuthField.SignedHeaders) + } + } + } +} + +// TestDoesV4PresignParamsExist - tests validate the logic to +func TestDoesV4PresignParamsExist(t *testing.T) { + testCases := []struct { + inputQueryKeyVals []string + expectedErrCode APIErrorCode + }{ + // Test case - 1. + // contains all query param keys which are necessary for v4 presign request. + { + inputQueryKeyVals: []string{ + "X-Amz-Algorithm", "", + "X-Amz-Credential", "", + "X-Amz-Signature", "", + "X-Amz-Date", "", + "X-Amz-SignedHeaders", "", + "X-Amz-Expires", "", + }, + expectedErrCode: ErrNone, + }, + // Test case - 2. + // missing "X-Amz-Algorithm" in tdhe query param. + // contains all query param keys which are necessary for v4 presign request. + { + inputQueryKeyVals: []string{ + "X-Amz-Credential", "", + "X-Amz-Signature", "", + "X-Amz-Date", "", + "X-Amz-SignedHeaders", "", + "X-Amz-Expires", "", + }, + expectedErrCode: ErrInvalidQueryParams, + }, + // Test case - 3. + // missing "X-Amz-Credential" in the query param. + { + inputQueryKeyVals: []string{ + "X-Amz-Algorithm", "", + "X-Amz-Signature", "", + "X-Amz-Date", "", + "X-Amz-SignedHeaders", "", + "X-Amz-Expires", "", + }, + expectedErrCode: ErrInvalidQueryParams, + }, + // Test case - 4. + // missing "X-Amz-Signature" in the query param. + { + inputQueryKeyVals: []string{ + "X-Amz-Algorithm", "", + "X-Amz-Credential", "", + "X-Amz-Date", "", + "X-Amz-SignedHeaders", "", + "X-Amz-Expires", "", + }, + expectedErrCode: ErrInvalidQueryParams, + }, + // Test case - 5. + // missing "X-Amz-Date" in the query param. + { + inputQueryKeyVals: []string{ + "X-Amz-Algorithm", "", + "X-Amz-Credential", "", + "X-Amz-Signature", "", + "X-Amz-SignedHeaders", "", + "X-Amz-Expires", "", + }, + expectedErrCode: ErrInvalidQueryParams, + }, + // Test case - 6. + // missing "X-Amz-SignedHeaders" in the query param. + { + inputQueryKeyVals: []string{ + "X-Amz-Algorithm", "", + "X-Amz-Credential", "", + "X-Amz-Signature", "", + "X-Amz-Date", "", + "X-Amz-Expires", "", + }, + expectedErrCode: ErrInvalidQueryParams, + }, + // Test case - 7. + // missing "X-Amz-Expires" in the query param. + { + inputQueryKeyVals: []string{ + "X-Amz-Algorithm", "", + "X-Amz-Credential", "", + "X-Amz-Signature", "", + "X-Amz-Date", "", + "X-Amz-SignedHeaders", "", + }, + expectedErrCode: ErrInvalidQueryParams, + }, + } + + for i, testCase := range testCases { + inputQuery := url.Values{} + // iterating through input query key value and setting the inputQuery of type url.Values. + for j := 0; j < len(testCase.inputQueryKeyVals)-1; j += 2 { + inputQuery.Set(testCase.inputQueryKeyVals[j], testCase.inputQueryKeyVals[j+1]) + } + + actualErrCode := doesV4PresignParamsExist(inputQuery) + + if testCase.expectedErrCode != actualErrCode { + t.Fatalf("Test %d: Expected the APIErrCode to be %d, got %d", i+1, testCase.expectedErrCode, actualErrCode) + } + } +} + +// TestParsePreSignV4 - Validates the parsing logic of Presignied v4 request from its url query values. +func TestParsePreSignV4(t *testing.T) { + // converts the duration in seconds into string format. + getDurationStr := strconv.Itoa + + // used in expected preSignValues, preSignValues.Date is of type time.Time . + queryTime := UTCNow() + + sampleTimeStr := UTCNow().Format(yyyymmdd) + + testCases := []struct { + inputQueryKeyVals []string + expectedPreSignValues preSignValues + expectedErrCode APIErrorCode + }{ + // Test case - 1. + // A Valid v4 presign URL requires the following params to be in the query. + // "X-Amz-Algorithm", "X-Amz-Credential", "X-Amz-Signature", " X-Amz-Date", "X-Amz-SignedHeaders", "X-Amz-Expires". + // If these params are missing its expected to get ErrInvalidQueryParams . + // In the following test case 2 out of 6 query params are missing. + { + inputQueryKeyVals: []string{ + "X-Amz-Algorithm", "", + "X-Amz-Credential", "", + "X-Amz-Signature", "", + "X-Amz-Expires", "", + }, + expectedPreSignValues: preSignValues{}, + expectedErrCode: ErrInvalidQueryParams, + }, + // Test case - 2. + // Test case with invalid "X-Amz-Algorithm" query value. + // The other query params should exist, other wise ErrInvalidQueryParams will be returned because of missing fields. + { + inputQueryKeyVals: []string{ + "X-Amz-Algorithm", "InvalidValue", + "X-Amz-Credential", "", + "X-Amz-Signature", "", + "X-Amz-Date", "", + "X-Amz-SignedHeaders", "", + "X-Amz-Expires", "", + }, + expectedPreSignValues: preSignValues{}, + expectedErrCode: ErrInvalidQuerySignatureAlgo, + }, + // Test case - 3. + // Test case with valid "X-Amz-Algorithm" query value, but invalid "X-Amz-Credential" header. + // Malformed crenential. + { + inputQueryKeyVals: []string{ + // valid "X-Amz-Algorithm" header. + "X-Amz-Algorithm", signV4Algorithm, + // valid "X-Amz-Credential" header. + "X-Amz-Credential", "invalid-credential", + "X-Amz-Signature", "", + "X-Amz-Date", "", + "X-Amz-SignedHeaders", "", + "X-Amz-Expires", "", + }, + expectedPreSignValues: preSignValues{}, + expectedErrCode: ErrCredMalformed, + }, + + // Test case - 4. + // Test case with valid "X-Amz-Algorithm" query value. + // Malformed date. + { + inputQueryKeyVals: []string{ + // valid "X-Amz-Algorithm" header. + "X-Amz-Algorithm", signV4Algorithm, + // valid "X-Amz-Credential" header. + "X-Amz-Credential", joinWithSlash( + "Z7IXGOO6BZ0REAN1Q26I", + sampleTimeStr, + "us-west-1", + "s3", + "aws4_request"), + // invalid "X-Amz-Date" query. + "X-Amz-Date", "invalid-time", + "X-Amz-SignedHeaders", "", + "X-Amz-Expires", "", + "X-Amz-Signature", "", + }, + expectedPreSignValues: preSignValues{}, + expectedErrCode: ErrMalformedPresignedDate, + }, + // Test case - 5. + // Test case with valid "X-Amz-Algorithm", "X-Amz-Credential", "X-Amz-Date" query value. + // Malformed Expiry, a valid expiry should be of format "s". + { + inputQueryKeyVals: []string{ + // valid "X-Amz-Algorithm" header. + "X-Amz-Algorithm", signV4Algorithm, + // valid "X-Amz-Credential" header. + "X-Amz-Credential", joinWithSlash( + "Z7IXGOO6BZ0REAN1Q26I", + sampleTimeStr, + "us-west-1", + "s3", + "aws4_request"), + // valid "X-Amz-Date" query. + "X-Amz-Date", UTCNow().Format(iso8601Format), + "X-Amz-Expires", "MalformedExpiry", + "X-Amz-SignedHeaders", "", + "X-Amz-Signature", "", + }, + expectedPreSignValues: preSignValues{}, + expectedErrCode: ErrMalformedExpires, + }, + // Test case - 6. + // Test case with negative X-Amz-Expires header. + { + inputQueryKeyVals: []string{ + // valid "X-Amz-Algorithm" header. + "X-Amz-Algorithm", signV4Algorithm, + // valid "X-Amz-Credential" header. + "X-Amz-Credential", joinWithSlash( + "Z7IXGOO6BZ0REAN1Q26I", + sampleTimeStr, + "us-west-1", + "s3", + "aws4_request"), + // valid "X-Amz-Date" query. + "X-Amz-Date", queryTime.UTC().Format(iso8601Format), + "X-Amz-Expires", getDurationStr(-1), + "X-Amz-Signature", "abcd", + "X-Amz-SignedHeaders", "host;x-amz-content-sha256;x-amz-date", + }, + expectedPreSignValues: preSignValues{}, + expectedErrCode: ErrNegativeExpires, + }, + // Test case - 7. + // Test case with empty X-Amz-SignedHeaders. + { + inputQueryKeyVals: []string{ + // valid "X-Amz-Algorithm" header. + "X-Amz-Algorithm", signV4Algorithm, + // valid "X-Amz-Credential" header. + "X-Amz-Credential", joinWithSlash( + "Z7IXGOO6BZ0REAN1Q26I", + sampleTimeStr, + "us-west-1", + "s3", + "aws4_request"), + // valid "X-Amz-Date" query. + "X-Amz-Date", queryTime.UTC().Format(iso8601Format), + "X-Amz-Expires", getDurationStr(100), + "X-Amz-Signature", "abcd", + "X-Amz-SignedHeaders", "", + }, + expectedPreSignValues: preSignValues{}, + expectedErrCode: ErrMissingFields, + }, + // Test case - 8. + // Test case with valid "X-Amz-Algorithm", "X-Amz-Credential", "X-Amz-Date" query value. + // Malformed Expiry, a valid expiry should be of format "s". + { + inputQueryKeyVals: []string{ + // valid "X-Amz-Algorithm" header. + "X-Amz-Algorithm", signV4Algorithm, + // valid "X-Amz-Credential" header. + "X-Amz-Credential", joinWithSlash( + "Z7IXGOO6BZ0REAN1Q26I", + sampleTimeStr, + "us-west-1", + "s3", + "aws4_request"), + // valid "X-Amz-Date" query. + "X-Amz-Date", queryTime.UTC().Format(iso8601Format), + "X-Amz-Expires", getDurationStr(100), + "X-Amz-Signature", "abcd", + "X-Amz-SignedHeaders", "host;x-amz-content-sha256;x-amz-date", + }, + expectedPreSignValues: preSignValues{ + signValues{ + // Credentials. + generateCredentials( + t, + "Z7IXGOO6BZ0REAN1Q26I", + sampleTimeStr, + "us-west-1", + "s3", + "aws4_request", + ), + // SignedHeaders. + []string{"host", "x-amz-content-sha256", "x-amz-date"}, + // Signature. + "abcd", + }, + // Date + queryTime, + // Expires. + 100 * time.Second, + }, + expectedErrCode: ErrNone, + }, + + // Test case - 9. + // Test case with value greater than 604800 in X-Amz-Expires header. + { + inputQueryKeyVals: []string{ + // valid "X-Amz-Algorithm" header. + "X-Amz-Algorithm", signV4Algorithm, + // valid "X-Amz-Credential" header. + "X-Amz-Credential", joinWithSlash( + "Z7IXGOO6BZ0REAN1Q26I", + sampleTimeStr, + "us-west-1", + "s3", + "aws4_request"), + // valid "X-Amz-Date" query. + "X-Amz-Date", queryTime.UTC().Format(iso8601Format), + // Invalid Expiry time greater than 7 days (604800 in seconds). + "X-Amz-Expires", getDurationStr(605000), + "X-Amz-Signature", "abcd", + "X-Amz-SignedHeaders", "host;x-amz-content-sha256;x-amz-date", + }, + expectedPreSignValues: preSignValues{}, + expectedErrCode: ErrMaximumExpires, + }, + } + + for i, testCase := range testCases { + inputQuery := url.Values{} + // iterating through input query key value and setting the inputQuery of type url.Values. + for j := 0; j < len(testCase.inputQueryKeyVals)-1; j += 2 { + inputQuery.Set(testCase.inputQueryKeyVals[j], testCase.inputQueryKeyVals[j+1]) + } + // call the function under test. + parsedPreSign, actualErrCode := parsePreSignV4(inputQuery, "", serviceS3) + if testCase.expectedErrCode != actualErrCode { + t.Fatalf("Test %d: Expected the APIErrCode to be %d, got %d", i+1, testCase.expectedErrCode, actualErrCode) + } + if actualErrCode == ErrNone { + // validating credentials. + validateCredentialfields(t, i+1, testCase.expectedPreSignValues.Credential, parsedPreSign.Credential) + // validating signed headers. + if strings.Join(testCase.expectedPreSignValues.SignedHeaders, ",") != strings.Join(parsedPreSign.SignedHeaders, ",") { + t.Errorf("Test %d: Expected the result to be \"%v\", but got \"%v\". ", i+1, testCase.expectedPreSignValues.SignedHeaders, parsedPreSign.SignedHeaders) + } + // validating signature field. + if !compareSignatureV4(testCase.expectedPreSignValues.Signature, parsedPreSign.Signature) { + t.Errorf("Test %d: Signature field mismatch: Expected \"%s\", got \"%s\"", i+1, testCase.expectedPreSignValues.Signature, parsedPreSign.Signature) + } + // validating expiry duration. + if testCase.expectedPreSignValues.Expires != parsedPreSign.Expires { + t.Errorf("Test %d: Expected expiry time to be %v, but got %v", i+1, testCase.expectedPreSignValues.Expires, parsedPreSign.Expires) + } + // validating presign date field. + if testCase.expectedPreSignValues.Date.UTC().Format(iso8601Format) != parsedPreSign.Date.UTC().Format(iso8601Format) { + t.Errorf("Test %d: Expected date to be %v, but got %v", i+1, testCase.expectedPreSignValues.Date.UTC().Format(iso8601Format), parsedPreSign.Date.UTC().Format(iso8601Format)) + } + } + } +} diff --git a/cmd/signature-v4-utils.go b/cmd/signature-v4-utils.go new file mode 100644 index 00000000..1569dec1 --- /dev/null +++ b/cmd/signature-v4-utils.go @@ -0,0 +1,280 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "crypto/hmac" + "encoding/hex" + "io" + "net/http" + "slices" + "strconv" + "strings" + + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/hash/sha256" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/policy" +) + +// http Header "x-amz-content-sha256" == "UNSIGNED-PAYLOAD" indicates that the +// client did not calculate sha256 of the payload. +const unsignedPayload = "UNSIGNED-PAYLOAD" + +// http Header "x-amz-content-sha256" == "STREAMING-UNSIGNED-PAYLOAD-TRAILER" indicates that the +// client did not calculate sha256 of the payload and there is a trailer. +const unsignedPayloadTrailer = "STREAMING-UNSIGNED-PAYLOAD-TRAILER" + +// skipContentSha256Cksum returns true if caller needs to skip +// payload checksum, false if not. +func skipContentSha256Cksum(r *http.Request) bool { + var ( + v []string + ok bool + ) + + if isRequestPresignedSignatureV4(r) { + v, ok = r.Form[xhttp.AmzContentSha256] + if !ok { + v, ok = r.Header[xhttp.AmzContentSha256] + } + } else { + v, ok = r.Header[xhttp.AmzContentSha256] + } + + // Skip if no header was set. + if !ok { + return true + } + + // If x-amz-content-sha256 is set and the value is not + // 'UNSIGNED-PAYLOAD' we should validate the content sha256. + switch v[0] { + case unsignedPayload, unsignedPayloadTrailer: + return true + case emptySHA256: + // some broken clients set empty-sha256 + // with > 0 content-length in the body, + // we should skip such clients and allow + // blindly such insecure clients only if + // S3 strict compatibility is disabled. + + // We return true only in situations when + // deployment has asked MinIO to allow for + // such broken clients and content-length > 0. + return r.ContentLength > 0 && !globalServerCtxt.StrictS3Compat + } + return false +} + +// Returns SHA256 for calculating canonical-request. +func getContentSha256Cksum(r *http.Request, stype serviceType) string { + if stype == serviceSTS { + payload, err := io.ReadAll(io.LimitReader(r.Body, stsRequestBodyLimit)) + if err != nil { + logger.CriticalIf(GlobalContext, err) + } + sum256 := sha256.Sum256(payload) + r.Body = io.NopCloser(bytes.NewReader(payload)) + return hex.EncodeToString(sum256[:]) + } + + var ( + defaultSha256Cksum string + v []string + ok bool + ) + + // For a presigned request we look at the query param for sha256. + if isRequestPresignedSignatureV4(r) { + // X-Amz-Content-Sha256, if not set in presigned requests, checksum + // will default to 'UNSIGNED-PAYLOAD'. + defaultSha256Cksum = unsignedPayload + v, ok = r.Form[xhttp.AmzContentSha256] + if !ok { + v, ok = r.Header[xhttp.AmzContentSha256] + } + } else { + // X-Amz-Content-Sha256, if not set in signed requests, checksum + // will default to sha256([]byte("")). + defaultSha256Cksum = emptySHA256 + v, ok = r.Header[xhttp.AmzContentSha256] + } + + // We found 'X-Amz-Content-Sha256' return the captured value. + if ok { + return v[0] + } + + // We couldn't find 'X-Amz-Content-Sha256'. + return defaultSha256Cksum +} + +// isValidRegion - verify if incoming region value is valid with configured Region. +func isValidRegion(reqRegion string, confRegion string) bool { + if confRegion == "" { + return true + } + if confRegion == "US" { + confRegion = globalMinioDefaultRegion + } + // Some older s3 clients set region as "US" instead of + // globalMinioDefaultRegion, handle it. + if reqRegion == "US" { + reqRegion = globalMinioDefaultRegion + } + return reqRegion == confRegion +} + +// check if the access key is valid and recognized, additionally +// also returns if the access key is owner/admin. +func checkKeyValid(r *http.Request, accessKey string) (auth.Credentials, bool, APIErrorCode) { + cred := globalActiveCred + if cred.AccessKey != accessKey { + if !globalIAMSys.Initialized() { + // Check if server has initialized, then only proceed + // to check for IAM users otherwise its okay for clients + // to retry with 503 errors when server is coming up. + return auth.Credentials{}, false, ErrIAMNotInitialized + } + + // Check if the access key is part of users credentials. + u, ok, err := globalIAMSys.CheckKey(r.Context(), accessKey) + if err != nil { + return auth.Credentials{}, false, ErrIAMNotInitialized + } + if !ok { + // Credentials could be valid but disabled - return a different + // error in such a scenario. + if u.Credentials.Status == auth.AccountOff { + return cred, false, ErrAccessKeyDisabled + } + return cred, false, ErrInvalidAccessKeyID + } + cred = u.Credentials + } + + claims, s3Err := checkClaimsFromToken(r, cred) + if s3Err != ErrNone { + return cred, false, s3Err + } + cred.Claims = claims + + owner := cred.AccessKey == globalActiveCred.AccessKey || (cred.ParentUser == globalActiveCred.AccessKey && cred.AccessKey != siteReplicatorSvcAcc) + if owner && !globalAPIConfig.permitRootAccess() { + // We disable root access and its service accounts if asked for. + return cred, owner, ErrAccessKeyDisabled + } + + if _, ok := claims[policy.SessionPolicyName]; ok { + owner = false + } + + return cred, owner, ErrNone +} + +// sumHMAC calculate hmac between two input byte array. +func sumHMAC(key []byte, data []byte) []byte { + hash := hmac.New(sha256.New, key) + hash.Write(data) + return hash.Sum(nil) +} + +// extractSignedHeaders extract signed headers from Authorization header +func extractSignedHeaders(signedHeaders []string, r *http.Request) (http.Header, APIErrorCode) { + reqHeaders := r.Header + reqQueries := r.Form + // find whether "host" is part of list of signed headers. + // if not return ErrUnsignedHeaders. "host" is mandatory. + if !slices.Contains(signedHeaders, "host") { + return nil, ErrUnsignedHeaders + } + extractedSignedHeaders := make(http.Header) + for _, header := range signedHeaders { + // `host` will not be found in the headers, can be found in r.Host. + // but its always necessary that the list of signed headers containing host in it. + val, ok := reqHeaders[http.CanonicalHeaderKey(header)] + if !ok { + // try to set headers from Query String + val, ok = reqQueries[header] + } + if ok { + extractedSignedHeaders[http.CanonicalHeaderKey(header)] = val + continue + } + switch header { + case "expect": + // Golang http server strips off 'Expect' header, if the + // client sent this as part of signed headers we need to + // handle otherwise we would see a signature mismatch. + // `aws-cli` sets this as part of signed headers. + // + // According to + // http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.20 + // Expect header is always of form: + // + // Expect = "Expect" ":" 1#expectation + // expectation = "100-continue" | expectation-extension + // + // So it safe to assume that '100-continue' is what would + // be sent, for the time being keep this work around. + // Adding a *TODO* to remove this later when Golang server + // doesn't filter out the 'Expect' header. + extractedSignedHeaders.Set(header, "100-continue") + case "host": + // Go http server removes "host" from Request.Header + extractedSignedHeaders.Set(header, r.Host) + case "transfer-encoding": + // Go http server removes "host" from Request.Header + extractedSignedHeaders[http.CanonicalHeaderKey(header)] = r.TransferEncoding + case "content-length": + // Signature-V4 spec excludes Content-Length from signed headers list for signature calculation. + // But some clients deviate from this rule. Hence we consider Content-Length for signature + // calculation to be compatible with such clients. + extractedSignedHeaders.Set(header, strconv.FormatInt(r.ContentLength, 10)) + default: + return nil, ErrUnsignedHeaders + } + } + return extractedSignedHeaders, ErrNone +} + +// Trim leading and trailing spaces and replace sequential spaces with one space, following Trimall() +// in http://docs.aws.amazon.com/general/latest/gr/sigv4-create-canonical-request.html +func signV4TrimAll(input string) string { + // Compress adjacent spaces (a space is determined by + // unicode.IsSpace() internally here) to one space and return + return strings.Join(strings.Fields(input), " ") +} + +// checkMetaHeaders will check if the metadata from header/url is the same with the one from signed headers +func checkMetaHeaders(signedHeadersMap http.Header, r *http.Request) APIErrorCode { + // check values from http header + for k, val := range r.Header { + if stringsHasPrefixFold(k, "X-Amz-Meta-") { + if signedHeadersMap.Get(k) == val[0] { + continue + } + return ErrUnsignedHeaders + } + } + + return ErrNone +} diff --git a/cmd/signature-v4-utils_test.go b/cmd/signature-v4-utils_test.go new file mode 100644 index 00000000..74830fc9 --- /dev/null +++ b/cmd/signature-v4-utils_test.go @@ -0,0 +1,420 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "net/http" + "os" + "testing" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/auth" + xhttp "github.com/minio/minio/internal/http" +) + +func TestCheckValid(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + objLayer, fsDir, err := prepareFS(ctx) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(fsDir) + if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil { + t.Fatalf("unable initialize config file, %s", err) + } + + initAllSubsystems(ctx) + initConfigSubsystem(ctx, objLayer) + + globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second) + + req, err := newTestRequest(http.MethodGet, "http://example.com:9000/bucket/object", 0, nil) + if err != nil { + t.Fatal(err) + } + + if err = signRequestV4(req, globalActiveCred.AccessKey, globalActiveCred.SecretKey); err != nil { + t.Fatal(err) + } + + _, owner, s3Err := checkKeyValid(req, globalActiveCred.AccessKey) + if s3Err != ErrNone { + t.Fatalf("Unexpected failure with %v", errorCodes.ToAPIErr(s3Err)) + } + + if !owner { + t.Fatalf("Expected owner to be 'true', found %t", owner) + } + + _, _, s3Err = checkKeyValid(req, "does-not-exist") + if s3Err != ErrInvalidAccessKeyID { + t.Fatalf("Expected error 'ErrInvalidAccessKeyID', found %v", s3Err) + } + + ucreds, err := auth.CreateCredentials("myuser1", "mypassword1") + if err != nil { + t.Fatalf("unable create credential, %s", err) + } + + _, err = globalIAMSys.CreateUser(ctx, ucreds.AccessKey, madmin.AddOrUpdateUserReq{ + SecretKey: ucreds.SecretKey, + Status: madmin.AccountEnabled, + }) + if err != nil { + t.Fatalf("unable create credential, %s", err) + } + + _, owner, s3Err = checkKeyValid(req, ucreds.AccessKey) + if s3Err != ErrNone { + t.Fatalf("Unexpected failure with %v", errorCodes.ToAPIErr(s3Err)) + } + + if owner { + t.Fatalf("Expected owner to be 'false', found %t", owner) + } + + _, err = globalIAMSys.PolicyDBSet(ctx, ucreds.AccessKey, "consoleAdmin", regUser, false) + if err != nil { + t.Fatalf("unable to attach policy to credential, %s", err) + } + + time.Sleep(4 * time.Second) + + policies, err := globalIAMSys.PolicyDBGet(ucreds.AccessKey) + if err != nil { + t.Fatalf("unable to get policy to credential, %s", err) + } + + if len(policies) == 0 { + t.Fatal("no policies found") + } + + if policies[0] != "consoleAdmin" { + t.Fatalf("expected 'consoleAdmin', %s", policies[0]) + } +} + +// TestSkipContentSha256Cksum - Test validate the logic which decides whether +// to skip checksum validation based on the request header. +func TestSkipContentSha256Cksum(t *testing.T) { + testCases := []struct { + inputHeaderKey string + inputHeaderValue string + + inputQueryKey string + inputQueryValue string + + expectedResult bool + }{ + // Test case - 1. + // Test case with "X-Amz-Content-Sha256" header set, but to empty value but we can't skip. + {"X-Amz-Content-Sha256", "", "", "", false}, + + // Test case - 2. + // Test case with "X-Amz-Content-Sha256" not set so we can skip. + {"", "", "", "", true}, + + // Test case - 3. + // Test case with "X-Amz-Content-Sha256" header set to "UNSIGNED-PAYLOAD" + // When "X-Amz-Content-Sha256" header is set to "UNSIGNED-PAYLOAD", validation of content sha256 has to be skipped. + {"X-Amz-Content-Sha256", unsignedPayload, "X-Amz-Credential", "", true}, + + // Test case - 4. + // Enabling PreSigned Signature v4, but X-Amz-Content-Sha256 not set has to be skipped. + {"", "", "X-Amz-Credential", "", true}, + + // Test case - 5. + // Enabling PreSigned Signature v4, but X-Amz-Content-Sha256 set and its not UNSIGNED-PAYLOAD, we shouldn't skip. + {"X-Amz-Content-Sha256", "somevalue", "X-Amz-Credential", "", false}, + + // Test case - 6. + // Test case with "X-Amz-Content-Sha256" header set to "UNSIGNED-PAYLOAD" and its not presigned, we should skip. + {"X-Amz-Content-Sha256", unsignedPayload, "", "", true}, + + // Test case - 7. + // "X-Amz-Content-Sha256" not set and PreSigned Signature v4 not enabled, sha256 checksum calculation is not skipped. + {"", "", "X-Amz-Credential", "", true}, + + // Test case - 8. + // "X-Amz-Content-Sha256" has a proper value cannot skip. + {"X-Amz-Content-Sha256", "somevalue", "", "", false}, + } + + for i, testCase := range testCases { + // creating an input HTTP request. + // Only the headers are relevant for this particular test. + inputReq, err := http.NewRequest(http.MethodGet, "http://example.com", nil) + if err != nil { + t.Fatalf("Error initializing input HTTP request: %v", err) + } + if testCase.inputQueryKey != "" { + q := inputReq.URL.Query() + q.Add(testCase.inputQueryKey, testCase.inputQueryValue) + if testCase.inputHeaderKey != "" { + q.Add(testCase.inputHeaderKey, testCase.inputHeaderValue) + } + inputReq.URL.RawQuery = q.Encode() + } else if testCase.inputHeaderKey != "" { + inputReq.Header.Set(testCase.inputHeaderKey, testCase.inputHeaderValue) + } + inputReq.ParseForm() + + actualResult := skipContentSha256Cksum(inputReq) + if testCase.expectedResult != actualResult { + t.Errorf("Test %d: Expected the result to `%v`, but instead got `%v`", i+1, testCase.expectedResult, actualResult) + } + } +} + +// TestIsValidRegion - Tests validate the comparison logic for asserting whether the region from http request is valid. +func TestIsValidRegion(t *testing.T) { + testCases := []struct { + inputReqRegion string + inputConfRegion string + + expectedResult bool + }{ + {"", "", true}, + {globalMinioDefaultRegion, "", true}, + {globalMinioDefaultRegion, "US", true}, + {"us-west-1", "US", false}, + {"us-west-1", "us-west-1", true}, + // "US" was old naming convention for 'us-east-1'. + {"US", "US", true}, + } + + for i, testCase := range testCases { + actualResult := isValidRegion(testCase.inputReqRegion, testCase.inputConfRegion) + if testCase.expectedResult != actualResult { + t.Errorf("Test %d: Expected the result to `%v`, but instead got `%v`", i+1, testCase.expectedResult, actualResult) + } + } +} + +// TestExtractSignedHeaders - Tests validate extraction of signed headers using list of signed header keys. +func TestExtractSignedHeaders(t *testing.T) { + signedHeaders := []string{"host", "x-amz-content-sha256", "x-amz-date", "transfer-encoding"} + + // If the `expect` key exists in the signed headers then golang server would have stripped out the value, expecting the `expect` header set to `100-continue` in the result. + signedHeaders = append(signedHeaders, "expect") + // expected header values. + expectedHost := "play.min.io:9000" + expectedContentSha256 := "1234abcd" + expectedTime := UTCNow().Format(iso8601Format) + expectedTransferEncoding := "gzip" + expectedExpect := "100-continue" + + r, err := http.NewRequest(http.MethodGet, "http://play.min.io:9000", nil) + if err != nil { + t.Fatal("Unable to create http.Request :", err) + } + r.TransferEncoding = []string{expectedTransferEncoding} + + // Creating input http header. + inputHeader := r.Header + inputHeader.Set("x-amz-content-sha256", expectedContentSha256) + inputHeader.Set("x-amz-date", expectedTime) + // calling the function being tested. + extractedSignedHeaders, errCode := extractSignedHeaders(signedHeaders, r) + if errCode != ErrNone { + t.Fatalf("Expected the APIErrorCode to be %d, but got %d", ErrNone, errCode) + } + + inputQuery := r.URL.Query() + // case where some headers need to get from request query + signedHeaders = append(signedHeaders, "x-amz-server-side-encryption") + // expect to fail with `ErrUnsignedHeaders` because couldn't find some header + _, errCode = extractSignedHeaders(signedHeaders, r) + if errCode != ErrUnsignedHeaders { + t.Fatalf("Expected the APIErrorCode to %d, but got %d", ErrUnsignedHeaders, errCode) + } + // set headers value through Get parameter + inputQuery.Add("x-amz-server-side-encryption", xhttp.AmzEncryptionAES) + r.URL.RawQuery = inputQuery.Encode() + r.ParseForm() + _, errCode = extractSignedHeaders(signedHeaders, r) + if errCode != ErrNone { + t.Fatalf("Expected the APIErrorCode to be %d, but got %d", ErrNone, errCode) + } + + // "x-amz-content-sha256" header value from the extracted result. + extractedContentSha256 := extractedSignedHeaders.Get("x-amz-content-sha256") + // "host" header value from the extracted result. + extractedHost := extractedSignedHeaders.Get("host") + // "x-amz-date" header from the extracted result. + extractedDate := extractedSignedHeaders.Get("x-amz-date") + // extracted `expect` header. + extractedExpect := extractedSignedHeaders.Get("expect") + + extractedTransferEncoding := extractedSignedHeaders.Get("transfer-encoding") + + if expectedHost != extractedHost { + t.Errorf("host header mismatch: expected `%s`, got `%s`", expectedHost, extractedHost) + } + // assert the result with the expected value. + if expectedContentSha256 != extractedContentSha256 { + t.Errorf("x-amz-content-sha256 header mismatch: expected `%s`, got `%s`", expectedContentSha256, extractedContentSha256) + } + if expectedTime != extractedDate { + t.Errorf("x-amz-date header mismatch: expected `%s`, got `%s`", expectedTime, extractedDate) + } + if extractedTransferEncoding != expectedTransferEncoding { + t.Errorf("transfer-encoding mismatch: expected %s, got %s", expectedTransferEncoding, extractedTransferEncoding) + } + + // Since the list of signed headers value contained `expect`, the default value of `100-continue` will be added to extracted signed headers. + if extractedExpect != expectedExpect { + t.Errorf("expect header incorrect value: expected `%s`, got `%s`", expectedExpect, extractedExpect) + } + + // case where the headers don't contain the one of the signed header in the signed headers list. + signedHeaders = append(signedHeaders, "X-Amz-Credential") + // expected to fail with `ErrUnsignedHeaders`. + _, errCode = extractSignedHeaders(signedHeaders, r) + if errCode != ErrUnsignedHeaders { + t.Fatalf("Expected the APIErrorCode to %d, but got %d", ErrUnsignedHeaders, errCode) + } + + // case where the list of signed headers doesn't contain the host field. + signedHeaders = signedHeaders[2:5] + // expected to fail with `ErrUnsignedHeaders`. + _, errCode = extractSignedHeaders(signedHeaders, r) + if errCode != ErrUnsignedHeaders { + t.Fatalf("Expected the APIErrorCode to %d, but got %d", ErrUnsignedHeaders, errCode) + } +} + +// TestSignV4TrimAll - tests the logic of TrimAll() function +func TestSignV4TrimAll(t *testing.T) { + testCases := []struct { + // Input. + inputStr string + // Expected result. + result string + }{ + {"本語", "本語"}, + {" abc ", "abc"}, + {" a b ", "a b"}, + {"a b ", "a b"}, + {"a b", "a b"}, + {"a b", "a b"}, + {" a b c ", "a b c"}, + {"a \t b c ", "a b c"}, + {"\"a \t b c ", "\"a b c"}, + {" \t\n\u000b\r\fa \t\n\u000b\r\f b \t\n\u000b\r\f c \t\n\u000b\r\f", "a b c"}, + } + + // Tests generated values from url encoded name. + for i, testCase := range testCases { + result := signV4TrimAll(testCase.inputStr) + if testCase.result != result { + t.Errorf("Test %d: Expected signV4TrimAll result to be \"%s\", but found it to be \"%s\" instead", i+1, testCase.result, result) + } + } +} + +// Test getContentSha256Cksum +func TestGetContentSha256Cksum(t *testing.T) { + testCases := []struct { + h string // header SHA256 + q string // query SHA256 + expected string // expected SHA256 + }{ + {"shastring", "", "shastring"}, + {emptySHA256, "", emptySHA256}, + {"", "", emptySHA256}, + {"", "X-Amz-Credential=random", unsignedPayload}, + {"", "X-Amz-Credential=random&X-Amz-Content-Sha256=" + unsignedPayload, unsignedPayload}, + {"", "X-Amz-Credential=random&X-Amz-Content-Sha256=shastring", "shastring"}, + } + + for i, testCase := range testCases { + r, err := http.NewRequest(http.MethodGet, "http://localhost/?"+testCase.q, nil) + if err != nil { + t.Fatal(err) + } + if testCase.h != "" { + r.Header.Set("x-amz-content-sha256", testCase.h) + } + r.ParseForm() + got := getContentSha256Cksum(r, serviceS3) + if got != testCase.expected { + t.Errorf("Test %d: got:%s expected:%s", i+1, got, testCase.expected) + } + } +} + +// Test TestCheckMetaHeaders tests the logic of checkMetaHeaders() function +func TestCheckMetaHeaders(t *testing.T) { + signedHeadersMap := map[string][]string{ + "X-Amz-Meta-Test": {"test"}, + "X-Amz-Meta-Extension": {"png"}, + "X-Amz-Meta-Name": {"imagepng"}, + } + expectedMetaTest := "test" + expectedMetaExtension := "png" + expectedMetaName := "imagepng" + r, err := http.NewRequest(http.MethodPut, "http://play.min.io:9000", nil) + if err != nil { + t.Fatal("Unable to create http.Request :", err) + } + + // Creating input http header. + inputHeader := r.Header + inputHeader.Set("X-Amz-Meta-Test", expectedMetaTest) + inputHeader.Set("X-Amz-Meta-Extension", expectedMetaExtension) + inputHeader.Set("X-Amz-Meta-Name", expectedMetaName) + // calling the function being tested. + errCode := checkMetaHeaders(signedHeadersMap, r) + if errCode != ErrNone { + t.Fatalf("Expected the APIErrorCode to be %d, but got %d", ErrNone, errCode) + } + + // Add new metadata in inputHeader + inputHeader.Set("X-Amz-Meta-Clone", "fail") + // calling the function being tested. + errCode = checkMetaHeaders(signedHeadersMap, r) + if errCode != ErrUnsignedHeaders { + t.Fatalf("Expected the APIErrorCode to be %d, but got %d", ErrUnsignedHeaders, errCode) + } + + // Delete extra metadata from header to don't affect other test + inputHeader.Del("X-Amz-Meta-Clone") + // calling the function being tested. + errCode = checkMetaHeaders(signedHeadersMap, r) + if errCode != ErrNone { + t.Fatalf("Expected the APIErrorCode to be %d, but got %d", ErrNone, errCode) + } + + // Creating input url values + r, err = http.NewRequest(http.MethodPut, "http://play.min.io:9000?x-amz-meta-test=test&x-amz-meta-extension=png&x-amz-meta-name=imagepng", nil) + if err != nil { + t.Fatal("Unable to create http.Request :", err) + } + + r.ParseForm() + // calling the function being tested. + errCode = checkMetaHeaders(signedHeadersMap, r) + if errCode != ErrNone { + t.Fatalf("Expected the APIErrorCode to be %d, but got %d", ErrNone, errCode) + } +} diff --git a/cmd/signature-v4.go b/cmd/signature-v4.go new file mode 100644 index 00000000..ceb8b4b2 --- /dev/null +++ b/cmd/signature-v4.go @@ -0,0 +1,408 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Package cmd This file implements helper functions to validate AWS +// Signature Version '4' authorization header. +// +// This package provides comprehensive helpers for following signature +// types. +// - Based on Authorization header. +// - Based on Query parameters. +// - Based on Form POST policy. +package cmd + +import ( + "bytes" + "crypto/subtle" + "encoding/hex" + "net/http" + "net/url" + "sort" + "strconv" + "strings" + "time" + + "github.com/minio/minio-go/v7/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/hash/sha256" + xhttp "github.com/minio/minio/internal/http" +) + +// AWS Signature Version '4' constants. +const ( + signV4Algorithm = "AWS4-HMAC-SHA256" + iso8601Format = "20060102T150405Z" + yyyymmdd = "20060102" +) + +type serviceType string + +const ( + serviceS3 serviceType = "s3" + serviceSTS serviceType = "sts" +) + +// getCanonicalHeaders generate a list of request headers with their values +func getCanonicalHeaders(signedHeaders http.Header) string { + var headers []string + vals := make(http.Header, len(signedHeaders)) + for k, vv := range signedHeaders { + k = strings.ToLower(k) + headers = append(headers, k) + vals[k] = vv + } + sort.Strings(headers) + + var buf bytes.Buffer + for _, k := range headers { + buf.WriteString(k) + buf.WriteByte(':') + for idx, v := range vals[k] { + if idx > 0 { + buf.WriteByte(',') + } + buf.WriteString(signV4TrimAll(v)) + } + buf.WriteByte('\n') + } + return buf.String() +} + +// getSignedHeaders generate a string i.e alphabetically sorted, semicolon-separated list of lowercase request header names +func getSignedHeaders(signedHeaders http.Header) string { + var headers []string + for k := range signedHeaders { + headers = append(headers, strings.ToLower(k)) + } + sort.Strings(headers) + return strings.Join(headers, ";") +} + +// getCanonicalRequest generate a canonical request of style +// +// canonicalRequest = +// +// \n +// \n +// \n +// \n +// \n +// +func getCanonicalRequest(extractedSignedHeaders http.Header, payload, queryStr, urlPath, method string) string { + rawQuery := strings.ReplaceAll(queryStr, "+", "%20") + encodedPath := s3utils.EncodePath(urlPath) + canonicalRequest := strings.Join([]string{ + method, + encodedPath, + rawQuery, + getCanonicalHeaders(extractedSignedHeaders), + getSignedHeaders(extractedSignedHeaders), + payload, + }, "\n") + return canonicalRequest +} + +// getScope generate a string of a specific date, an AWS region, and a service. +func getScope(t time.Time, region string) string { + scope := strings.Join([]string{ + t.Format(yyyymmdd), + region, + string(serviceS3), + "aws4_request", + }, SlashSeparator) + return scope +} + +// getStringToSign a string based on selected query values. +func getStringToSign(canonicalRequest string, t time.Time, scope string) string { + stringToSign := signV4Algorithm + "\n" + t.Format(iso8601Format) + "\n" + stringToSign += scope + "\n" + canonicalRequestBytes := sha256.Sum256([]byte(canonicalRequest)) + stringToSign += hex.EncodeToString(canonicalRequestBytes[:]) + return stringToSign +} + +// getSigningKey hmac seed to calculate final signature. +func getSigningKey(secretKey string, t time.Time, region string, stype serviceType) []byte { + date := sumHMAC([]byte("AWS4"+secretKey), []byte(t.Format(yyyymmdd))) + regionBytes := sumHMAC(date, []byte(region)) + service := sumHMAC(regionBytes, []byte(stype)) + signingKey := sumHMAC(service, []byte("aws4_request")) + return signingKey +} + +// getSignature final signature in hexadecimal form. +func getSignature(signingKey []byte, stringToSign string) string { + return hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign))) +} + +// Check to see if Policy is signed correctly. +func doesPolicySignatureMatch(formValues http.Header) (auth.Credentials, APIErrorCode) { + // For SignV2 - Signature field will be valid + if _, ok := formValues[xhttp.AmzSignatureV2]; ok { + return doesPolicySignatureV2Match(formValues) + } + return doesPolicySignatureV4Match(formValues) +} + +// compareSignatureV4 returns true if and only if both signatures +// are equal. The signatures are expected to be HEX encoded strings +// according to the AWS S3 signature V4 spec. +func compareSignatureV4(sig1, sig2 string) bool { + // The CTC using []byte(str) works because the hex encoding + // is unique for a sequence of bytes. See also compareSignatureV2. + return subtle.ConstantTimeCompare([]byte(sig1), []byte(sig2)) == 1 +} + +// doesPolicySignatureMatch - Verify query headers with post policy +// - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-HTTPPOSTConstructPolicy.html +// +// returns ErrNone if the signature matches. +func doesPolicySignatureV4Match(formValues http.Header) (auth.Credentials, APIErrorCode) { + // Server region. + region := globalSite.Region() + + // Parse credential tag. + credHeader, s3Err := parseCredentialHeader("Credential="+formValues.Get(xhttp.AmzCredential), region, serviceS3) + if s3Err != ErrNone { + return auth.Credentials{}, s3Err + } + + r := &http.Request{Header: formValues} + cred, _, s3Err := checkKeyValid(r, credHeader.accessKey) + if s3Err != ErrNone { + return cred, s3Err + } + + // Get signing key. + signingKey := getSigningKey(cred.SecretKey, credHeader.scope.date, credHeader.scope.region, serviceS3) + + // Get signature. + newSignature := getSignature(signingKey, formValues.Get("Policy")) + + // Verify signature. + if !compareSignatureV4(newSignature, formValues.Get(xhttp.AmzSignature)) { + return cred, ErrSignatureDoesNotMatch + } + + // Success. + return cred, ErrNone +} + +// doesPresignedSignatureMatch - Verify query headers with presigned signature +// - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html +// +// returns ErrNone if the signature matches. +func doesPresignedSignatureMatch(hashedPayload string, r *http.Request, region string, stype serviceType) APIErrorCode { + // Copy request + req := *r + + // Parse request query string. + pSignValues, err := parsePreSignV4(req.Form, region, stype) + if err != ErrNone { + return err + } + + cred, _, s3Err := checkKeyValid(r, pSignValues.Credential.accessKey) + if s3Err != ErrNone { + return s3Err + } + + // Extract all the signed headers along with its values. + extractedSignedHeaders, errCode := extractSignedHeaders(pSignValues.SignedHeaders, r) + if errCode != ErrNone { + return errCode + } + + // Check if the metadata headers are equal with signedheaders + errMetaCode := checkMetaHeaders(extractedSignedHeaders, r) + if errMetaCode != ErrNone { + return errMetaCode + } + + // If the host which signed the request is slightly ahead in time (by less than globalMaxSkewTime) the + // request should still be allowed. + if pSignValues.Date.After(UTCNow().Add(globalMaxSkewTime)) { + return ErrRequestNotReadyYet + } + + if UTCNow().Sub(pSignValues.Date) > pSignValues.Expires { + return ErrExpiredPresignRequest + } + + // Save the date and expires. + t := pSignValues.Date + expireSeconds := int(pSignValues.Expires / time.Second) + + // Construct new query. + query := make(url.Values) + clntHashedPayload := req.Form.Get(xhttp.AmzContentSha256) + if clntHashedPayload != "" { + query.Set(xhttp.AmzContentSha256, hashedPayload) + } + + token := req.Form.Get(xhttp.AmzSecurityToken) + if token != "" { + query.Set(xhttp.AmzSecurityToken, cred.SessionToken) + } + + query.Set(xhttp.AmzAlgorithm, signV4Algorithm) + + // Construct the query. + query.Set(xhttp.AmzDate, t.Format(iso8601Format)) + query.Set(xhttp.AmzExpires, strconv.Itoa(expireSeconds)) + query.Set(xhttp.AmzSignedHeaders, strings.Join(pSignValues.SignedHeaders, ";")) + query.Set(xhttp.AmzCredential, cred.AccessKey+SlashSeparator+pSignValues.Credential.getScope()) + + defaultSigParams := set.CreateStringSet( + xhttp.AmzContentSha256, + xhttp.AmzSecurityToken, + xhttp.AmzAlgorithm, + xhttp.AmzDate, + xhttp.AmzExpires, + xhttp.AmzSignedHeaders, + xhttp.AmzCredential, + xhttp.AmzSignature, + ) + + // Add missing query parameters if any provided in the request URL + for k, v := range req.Form { + if !defaultSigParams.Contains(k) { + query[k] = v + } + } + + // Get the encoded query. + encodedQuery := query.Encode() + + // Verify if date query is same. + if req.Form.Get(xhttp.AmzDate) != query.Get(xhttp.AmzDate) { + return ErrSignatureDoesNotMatch + } + // Verify if expires query is same. + if req.Form.Get(xhttp.AmzExpires) != query.Get(xhttp.AmzExpires) { + return ErrSignatureDoesNotMatch + } + // Verify if signed headers query is same. + if req.Form.Get(xhttp.AmzSignedHeaders) != query.Get(xhttp.AmzSignedHeaders) { + return ErrSignatureDoesNotMatch + } + // Verify if credential query is same. + if req.Form.Get(xhttp.AmzCredential) != query.Get(xhttp.AmzCredential) { + return ErrSignatureDoesNotMatch + } + // Verify if sha256 payload query is same. + if clntHashedPayload != "" && clntHashedPayload != query.Get(xhttp.AmzContentSha256) { + return ErrContentSHA256Mismatch + } + // Verify if security token is correct. + if token != "" && subtle.ConstantTimeCompare([]byte(token), []byte(cred.SessionToken)) != 1 { + return ErrInvalidToken + } + + // Verify finally if signature is same. + + // Get canonical request. + presignedCanonicalReq := getCanonicalRequest(extractedSignedHeaders, hashedPayload, encodedQuery, req.URL.Path, req.Method) + + // Get string to sign from canonical request. + presignedStringToSign := getStringToSign(presignedCanonicalReq, t, pSignValues.Credential.getScope()) + + // Get hmac presigned signing key. + presignedSigningKey := getSigningKey(cred.SecretKey, pSignValues.Credential.scope.date, + pSignValues.Credential.scope.region, stype) + + // Get new signature. + newSignature := getSignature(presignedSigningKey, presignedStringToSign) + + // Verify signature. + if !compareSignatureV4(req.Form.Get(xhttp.AmzSignature), newSignature) { + return ErrSignatureDoesNotMatch + } + + r.Header.Set("x-amz-signature-age", strconv.FormatInt(UTCNow().Sub(pSignValues.Date).Milliseconds(), 10)) + + return ErrNone +} + +// doesSignatureMatch - Verify authorization header with calculated header in accordance with +// - http://docs.aws.amazon.com/AmazonS3/latest/API/sig-v4-authenticating-requests.html +// +// returns ErrNone if signature matches. +func doesSignatureMatch(hashedPayload string, r *http.Request, region string, stype serviceType) APIErrorCode { + // Copy request. + req := *r + + // Save authorization header. + v4Auth := req.Header.Get(xhttp.Authorization) + + // Parse signature version '4' header. + signV4Values, err := parseSignV4(v4Auth, region, stype) + if err != ErrNone { + return err + } + + // Extract all the signed headers along with its values. + extractedSignedHeaders, errCode := extractSignedHeaders(signV4Values.SignedHeaders, r) + if errCode != ErrNone { + return errCode + } + + cred, _, s3Err := checkKeyValid(r, signV4Values.Credential.accessKey) + if s3Err != ErrNone { + return s3Err + } + + // Extract date, if not present throw error. + var date string + if date = req.Header.Get(xhttp.AmzDate); date == "" { + if date = r.Header.Get(xhttp.Date); date == "" { + return ErrMissingDateHeader + } + } + + // Parse date header. + t, e := time.Parse(iso8601Format, date) + if e != nil { + return ErrMalformedDate + } + + // Query string. + queryStr := req.Form.Encode() + + // Get canonical request. + canonicalRequest := getCanonicalRequest(extractedSignedHeaders, hashedPayload, queryStr, req.URL.Path, req.Method) + + // Get string to sign from canonical request. + stringToSign := getStringToSign(canonicalRequest, t, signV4Values.Credential.getScope()) + + // Get hmac signing key. + signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date, + signV4Values.Credential.scope.region, stype) + + // Calculate signature. + newSignature := getSignature(signingKey, stringToSign) + + // Verify if signature match. + if !compareSignatureV4(newSignature, signV4Values.Signature) { + return ErrSignatureDoesNotMatch + } + + // Return error none. + return ErrNone +} diff --git a/cmd/signature-v4_test.go b/cmd/signature-v4_test.go new file mode 100644 index 00000000..a0f5d815 --- /dev/null +++ b/cmd/signature-v4_test.go @@ -0,0 +1,315 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "fmt" + "net/http" + "net/url" + "os" + "testing" + "time" +) + +func niceError(code APIErrorCode) string { + // Special-handle ErrNone + if code == ErrNone { + return "ErrNone" + } + + return fmt.Sprintf("%s (%s)", errorCodes[code].Code, errorCodes[code].Description) +} + +func TestDoesPolicySignatureMatch(t *testing.T) { + _, fsDir, err := prepareFS(t.Context()) + if err != nil { + t.Fatal(err) + } + defer removeRoots([]string{fsDir}) + + credentialTemplate := "%s/%s/%s/s3/aws4_request" + now := UTCNow() + accessKey := globalActiveCred.AccessKey + + testCases := []struct { + form http.Header + expected APIErrorCode + }{ + // (0) It should fail if 'X-Amz-Credential' is missing. + { + form: http.Header{}, + expected: ErrCredMalformed, + }, + // (1) It should fail if the access key is incorrect. + { + form: http.Header{ + "X-Amz-Credential": []string{fmt.Sprintf(credentialTemplate, "EXAMPLEINVALIDEXAMPL", now.Format(yyyymmdd), globalMinioDefaultRegion)}, + }, + expected: ErrInvalidAccessKeyID, + }, + // (2) It should fail with a bad signature. + { + form: http.Header{ + "X-Amz-Credential": []string{fmt.Sprintf(credentialTemplate, accessKey, now.Format(yyyymmdd), globalMinioDefaultRegion)}, + "X-Amz-Date": []string{now.Format(iso8601Format)}, + "X-Amz-Signature": []string{"invalidsignature"}, + "Policy": []string{"policy"}, + }, + expected: ErrSignatureDoesNotMatch, + }, + // (3) It should succeed if everything is correct. + { + form: http.Header{ + "X-Amz-Credential": []string{ + fmt.Sprintf(credentialTemplate, accessKey, now.Format(yyyymmdd), globalMinioDefaultRegion), + }, + "X-Amz-Date": []string{now.Format(iso8601Format)}, + "X-Amz-Signature": []string{ + getSignature(getSigningKey(globalActiveCred.SecretKey, now, + globalMinioDefaultRegion, serviceS3), "policy"), + }, + "Policy": []string{"policy"}, + }, + expected: ErrNone, + }, + } + + // Run each test case individually. + for i, testCase := range testCases { + _, code := doesPolicySignatureMatch(testCase.form) + if code != testCase.expected { + t.Errorf("(%d) expected to get %s, instead got %s", i, niceError(testCase.expected), niceError(code)) + } + } +} + +func TestDoesPresignedSignatureMatch(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + obj, fsDir, err := prepareFS(ctx) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(fsDir) + if err = newTestConfig(globalMinioDefaultRegion, obj); err != nil { + t.Fatal(err) + } + + // sha256 hash of "payload" + payloadSHA256 := "239f59ed55e737c77147cf55ad0c1b030b6d7ee748a7426952f9b852d5a935e5" + now := UTCNow() + credentialTemplate := "%s/%s/%s/s3/aws4_request" + + region := globalSite.Region() + accessKeyID := globalActiveCred.AccessKey + testCases := []struct { + queryParams map[string]string + headers map[string]string + region string + expected APIErrorCode + }{ + // (0) Should error without a set URL query. + { + region: globalMinioDefaultRegion, + expected: ErrInvalidQueryParams, + }, + // (1) Should error on an invalid access key. + { + queryParams: map[string]string{ + "X-Amz-Algorithm": signV4Algorithm, + "X-Amz-Date": now.Format(iso8601Format), + "X-Amz-Expires": "60", + "X-Amz-Signature": "badsignature", + "X-Amz-SignedHeaders": "host;x-amz-content-sha256;x-amz-date", + "X-Amz-Credential": fmt.Sprintf(credentialTemplate, "Z7IXGOO6BZ0REAN1Q26I", now.Format(yyyymmdd), "us-west-1"), + }, + region: "us-west-1", + expected: ErrInvalidAccessKeyID, + }, + // (2) Should NOT fail with an invalid region if it doesn't verify it. + { + queryParams: map[string]string{ + "X-Amz-Algorithm": signV4Algorithm, + "X-Amz-Date": now.Format(iso8601Format), + "X-Amz-Expires": "60", + "X-Amz-Signature": "badsignature", + "X-Amz-SignedHeaders": "host;x-amz-content-sha256;x-amz-date", + "X-Amz-Credential": fmt.Sprintf(credentialTemplate, accessKeyID, now.Format(yyyymmdd), "us-west-1"), + "X-Amz-Content-Sha256": payloadSHA256, + }, + region: "us-west-1", + expected: ErrUnsignedHeaders, + }, + // (3) Should fail to extract headers if the host header is not signed. + { + queryParams: map[string]string{ + "X-Amz-Algorithm": signV4Algorithm, + "X-Amz-Date": now.Format(iso8601Format), + "X-Amz-Expires": "60", + "X-Amz-Signature": "badsignature", + "X-Amz-SignedHeaders": "x-amz-content-sha256;x-amz-date", + "X-Amz-Credential": fmt.Sprintf(credentialTemplate, accessKeyID, now.Format(yyyymmdd), region), + "X-Amz-Content-Sha256": payloadSHA256, + }, + region: region, + expected: ErrUnsignedHeaders, + }, + // (4) Should give an expired request if it has expired. + { + queryParams: map[string]string{ + "X-Amz-Algorithm": signV4Algorithm, + "X-Amz-Date": now.AddDate(0, 0, -2).Format(iso8601Format), + "X-Amz-Expires": "60", + "X-Amz-Signature": "badsignature", + "X-Amz-SignedHeaders": "host;x-amz-content-sha256;x-amz-date", + "X-Amz-Credential": fmt.Sprintf(credentialTemplate, accessKeyID, now.Format(yyyymmdd), region), + "X-Amz-Content-Sha256": payloadSHA256, + }, + headers: map[string]string{ + "X-Amz-Date": now.AddDate(0, 0, -2).Format(iso8601Format), + "X-Amz-Content-Sha256": payloadSHA256, + }, + region: region, + expected: ErrExpiredPresignRequest, + }, + // (5) Should error if the signature is incorrect. + { + queryParams: map[string]string{ + "X-Amz-Algorithm": signV4Algorithm, + "X-Amz-Date": now.Format(iso8601Format), + "X-Amz-Expires": "60", + "X-Amz-Signature": "badsignature", + "X-Amz-SignedHeaders": "host;x-amz-content-sha256;x-amz-date", + "X-Amz-Credential": fmt.Sprintf(credentialTemplate, accessKeyID, now.Format(yyyymmdd), region), + "X-Amz-Content-Sha256": payloadSHA256, + }, + headers: map[string]string{ + "X-Amz-Date": now.Format(iso8601Format), + "X-Amz-Content-Sha256": payloadSHA256, + }, + region: region, + expected: ErrSignatureDoesNotMatch, + }, + // (6) Should error if the request is not ready yet, ie X-Amz-Date is in the future. + { + queryParams: map[string]string{ + "X-Amz-Algorithm": signV4Algorithm, + "X-Amz-Date": now.Add(1 * time.Hour).Format(iso8601Format), + "X-Amz-Expires": "60", + "X-Amz-Signature": "badsignature", + "X-Amz-SignedHeaders": "host;x-amz-content-sha256;x-amz-date", + "X-Amz-Credential": fmt.Sprintf(credentialTemplate, accessKeyID, now.Format(yyyymmdd), region), + "X-Amz-Content-Sha256": payloadSHA256, + }, + headers: map[string]string{ + "X-Amz-Date": now.Format(iso8601Format), + "X-Amz-Content-Sha256": payloadSHA256, + }, + region: region, + expected: ErrRequestNotReadyYet, + }, + // (7) Should not error with invalid region instead, call should proceed + // with signature does not match. + { + queryParams: map[string]string{ + "X-Amz-Algorithm": signV4Algorithm, + "X-Amz-Date": now.Format(iso8601Format), + "X-Amz-Expires": "60", + "X-Amz-Signature": "badsignature", + "X-Amz-SignedHeaders": "host;x-amz-content-sha256;x-amz-date", + "X-Amz-Credential": fmt.Sprintf(credentialTemplate, accessKeyID, now.Format(yyyymmdd), region), + "X-Amz-Content-Sha256": payloadSHA256, + }, + headers: map[string]string{ + "X-Amz-Date": now.Format(iso8601Format), + "X-Amz-Content-Sha256": payloadSHA256, + }, + region: "", + expected: ErrSignatureDoesNotMatch, + }, + // (8) Should error with signature does not match. But handles + // query params which do not precede with "x-amz-" header. + { + queryParams: map[string]string{ + "X-Amz-Algorithm": signV4Algorithm, + "X-Amz-Date": now.Format(iso8601Format), + "X-Amz-Expires": "60", + "X-Amz-Signature": "badsignature", + "X-Amz-SignedHeaders": "host;x-amz-content-sha256;x-amz-date", + "X-Amz-Credential": fmt.Sprintf(credentialTemplate, accessKeyID, now.Format(yyyymmdd), region), + "X-Amz-Content-Sha256": payloadSHA256, + "response-content-type": "application/json", + }, + headers: map[string]string{ + "X-Amz-Date": now.Format(iso8601Format), + "X-Amz-Content-Sha256": payloadSHA256, + }, + region: "", + expected: ErrSignatureDoesNotMatch, + }, + // (9) Should error with unsigned headers. + { + queryParams: map[string]string{ + "X-Amz-Algorithm": signV4Algorithm, + "X-Amz-Date": now.Format(iso8601Format), + "X-Amz-Expires": "60", + "X-Amz-Signature": "badsignature", + "X-Amz-SignedHeaders": "host;x-amz-content-sha256;x-amz-date", + "X-Amz-Credential": fmt.Sprintf(credentialTemplate, accessKeyID, now.Format(yyyymmdd), region), + "X-Amz-Content-Sha256": payloadSHA256, + "response-content-type": "application/json", + }, + headers: map[string]string{ + "X-Amz-Date": now.Format(iso8601Format), + }, + region: "", + expected: ErrUnsignedHeaders, + }, + } + + // Run each test case individually. + for i, testCase := range testCases { + // Turn the map[string]string into map[string][]string, because Go. + query := url.Values{} + for key, value := range testCase.queryParams { + query.Set(key, value) + } + + // Create a request to use. + req, e := http.NewRequest(http.MethodGet, "http://host/a/b?"+query.Encode(), nil) + if e != nil { + t.Errorf("(%d) failed to create http.Request, got %v", i, e) + } + + // Do the same for the headers. + for key, value := range testCase.headers { + req.Header.Set(key, value) + } + + // parse form. + req.ParseForm() + + // Check if it matches! + err := doesPresignedSignatureMatch(payloadSHA256, req, testCase.region, serviceS3) + if err != testCase.expected { + t.Errorf("(%d) expected to get %s, instead got %s", i, niceError(testCase.expected), niceError(err)) + } + } +} diff --git a/cmd/site-replication-metrics.go b/cmd/site-replication-metrics.go new file mode 100644 index 00000000..bb371531 --- /dev/null +++ b/cmd/site-replication-metrics.go @@ -0,0 +1,288 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "maps" + "sync" + "sync/atomic" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7" +) + +//go:generate msgp -file $GOFILE + +// RStat has replication error stats +type RStat struct { + Count int64 `json:"count"` + Bytes int64 `json:"bytes"` +} + +// RTimedMetrics has replication error stats for various time windows +type RTimedMetrics struct { + LastHour ReplicationLastHour `json:"lastHour"` + SinceUptime RStat `json:"sinceUptime"` + LastMinute ReplicationLastMinute + // Error counts + ErrCounts map[string]int `json:"errCounts"` // Count of credential errors +} + +func (rt *RTimedMetrics) String() string { + s := rt.toMetric() + return fmt.Sprintf("Errors in LastMinute: %v, LastHour: %v, SinceUptime: %v", s.LastMinute.Count, s.LastHour.Count, s.Totals.Count) +} + +func (rt *RTimedMetrics) toMetric() madmin.TimedErrStats { + if rt == nil { + return madmin.TimedErrStats{} + } + errCounts := make(map[string]int) + maps.Copy(errCounts, rt.ErrCounts) + minuteTotals := rt.LastMinute.getTotal() + hourTotals := rt.LastHour.getTotal() + return madmin.TimedErrStats{ + LastMinute: madmin.RStat{ + Count: float64(minuteTotals.N), + Bytes: minuteTotals.Size, + }, + LastHour: madmin.RStat{ + Count: float64(hourTotals.N), + Bytes: hourTotals.Size, + }, + Totals: madmin.RStat{ + Count: float64(rt.SinceUptime.Count), + Bytes: rt.SinceUptime.Bytes, + }, + ErrCounts: errCounts, + } +} + +func (rt *RTimedMetrics) addsize(size int64, err error) { + // failures seen since uptime + atomic.AddInt64(&rt.SinceUptime.Bytes, size) + atomic.AddInt64(&rt.SinceUptime.Count, 1) + rt.LastMinute.addsize(size) + rt.LastHour.addsize(size) + if err != nil && minio.ToErrorResponse(err).Code == "AccessDenied" { + if rt.ErrCounts == nil { + rt.ErrCounts = make(map[string]int) + } + rt.ErrCounts["AccessDenied"]++ + } +} + +func (rt *RTimedMetrics) merge(o RTimedMetrics) (n RTimedMetrics) { + n.SinceUptime.Bytes = atomic.LoadInt64(&rt.SinceUptime.Bytes) + atomic.LoadInt64(&o.SinceUptime.Bytes) + n.SinceUptime.Count = atomic.LoadInt64(&rt.SinceUptime.Count) + atomic.LoadInt64(&o.SinceUptime.Count) + + n.LastMinute = n.LastMinute.merge(rt.LastMinute) + n.LastMinute = n.LastMinute.merge(o.LastMinute) + n.LastHour = n.LastHour.merge(rt.LastHour) + n.LastHour = n.LastHour.merge(o.LastHour) + n.ErrCounts = make(map[string]int) + maps.Copy(n.ErrCounts, rt.ErrCounts) + for k, v := range o.ErrCounts { + n.ErrCounts[k] += v + } + return n +} + +// SRStats has replication stats at site level +type SRStats struct { + // Total Replica size in bytes + ReplicaSize int64 `json:"replicaSize"` + // Total Replica received + ReplicaCount int64 `json:"replicaCount"` + M map[string]*SRStatus `json:"srStatusMap"` + + movingAvgTicker *time.Ticker // Ticker for calculating moving averages + lock sync.RWMutex // mutex for srStats +} + +// SRStatus has replication stats at deployment level +type SRStatus struct { + ReplicatedSize int64 `json:"completedReplicationSize"` + // Total number of failed operations including metadata updates in the last minute + Failed RTimedMetrics `json:"failedReplication"` + // Total number of completed operations + ReplicatedCount int64 `json:"replicationCount"` + // Replication latency information + Latency ReplicationLatency `json:"replicationLatency"` + // transfer rate for large uploads + XferRateLrg *XferStats `json:"largeTransferRate" msg:"lt"` + // transfer rate for small uploads + XferRateSml *XferStats `json:"smallTransferRate" msg:"st"` + // Endpoint is the replication target endpoint + Endpoint string `json:"-"` + // Secure is true if the replication target endpoint is secure + Secure bool `json:"-"` +} + +func (sr *SRStats) update(st replStat, dID string) { + sr.lock.Lock() + defer sr.lock.Unlock() + srs, ok := sr.M[dID] + if !ok { + srs = &SRStatus{ + XferRateLrg: newXferStats(), + XferRateSml: newXferStats(), + } + } + srs.Endpoint = st.Endpoint + srs.Secure = st.Secure + switch { + case st.Completed: + srs.ReplicatedSize += st.TransferSize + srs.ReplicatedCount++ + if st.TransferDuration > 0 { + srs.Latency.update(st.TransferSize, st.TransferDuration) + srs.updateXferRate(st.TransferSize, st.TransferDuration) + } + case st.Failed: + srs.Failed.addsize(st.TransferSize, st.Err) + case st.Pending: + } + sr.M[dID] = srs +} + +func (sr *SRStats) get() map[string]SRMetric { + epMap := globalBucketTargetSys.healthStats() + + sr.lock.RLock() + defer sr.lock.RUnlock() + m := make(map[string]SRMetric, len(sr.M)) + for dID, v := range sr.M { + t := newXferStats() + mx := make(map[RMetricName]XferStats) + + if v.XferRateLrg != nil { + mx[Large] = *v.XferRateLrg.Clone() + m := t.merge(*v.XferRateLrg) + t = &m + } + if v.XferRateSml != nil { + mx[Small] = *v.XferRateSml.Clone() + m := t.merge(*v.XferRateSml) + t = &m + } + + mx[Total] = *t + metric := SRMetric{ + ReplicatedSize: v.ReplicatedSize, + ReplicatedCount: v.ReplicatedCount, + DeploymentID: dID, + Failed: v.Failed.toMetric(), + XferStats: mx, + } + epHealth, ok := epMap[v.Endpoint] + if ok { + metric.Endpoint = epHealth.Endpoint + metric.TotalDowntime = epHealth.offlineDuration + metric.LastOnline = epHealth.lastOnline + metric.Online = epHealth.Online + metric.Latency = madmin.LatencyStat{ + Curr: epHealth.latency.curr, + Avg: epHealth.latency.avg, + Max: epHealth.latency.peak, + } + } + m[dID] = metric + } + return m +} + +func (srs *SRStatus) updateXferRate(sz int64, duration time.Duration) { + if sz > minLargeObjSize { + srs.XferRateLrg.addSize(sz, duration) + } else { + srs.XferRateSml.addSize(sz, duration) + } +} + +func newSRStats() *SRStats { + s := SRStats{ + M: make(map[string]*SRStatus), + movingAvgTicker: time.NewTicker(time.Second * 2), + } + go s.trackEWMA() + return &s +} + +func (sr *SRStats) trackEWMA() { + for { + select { + case <-sr.movingAvgTicker.C: + sr.updateMovingAvg() + case <-GlobalContext.Done(): + return + } + } +} + +func (sr *SRStats) updateMovingAvg() { + sr.lock.Lock() + defer sr.lock.Unlock() + for _, s := range sr.M { + s.XferRateLrg.measure.updateExponentialMovingAverage(time.Now()) + s.XferRateSml.measure.updateExponentialMovingAverage(time.Now()) + } +} + +// SRMetric captures replication metrics for a deployment +type SRMetric struct { + DeploymentID string `json:"deploymentID"` + Endpoint string `json:"endpoint"` + TotalDowntime time.Duration `json:"totalDowntime"` + LastOnline time.Time `json:"lastOnline"` + Online bool `json:"isOnline"` + Latency madmin.LatencyStat `json:"latency"` + + // replication metrics across buckets roll up + ReplicatedSize int64 `json:"replicatedSize"` + // Total number of completed operations + ReplicatedCount int64 `json:"replicatedCount"` + // Failed captures replication errors in various time windows + + Failed madmin.TimedErrStats `json:"failed"` + + XferStats map[RMetricName]XferStats `json:"transferSummary"` +} + +// SRMetricsSummary captures summary of replication counts across buckets on site +// along with op metrics rollup. +type SRMetricsSummary struct { + // op metrics roll up + ActiveWorkers ActiveWorkerStat `json:"activeWorkers"` + + // Total Replica size in bytes + ReplicaSize int64 `json:"replicaSize"` + + // Total number of replica received + ReplicaCount int64 `json:"replicaCount"` + // Queued operations + Queued InQueueMetric `json:"queued"` + // Proxy stats + Proxied ProxyMetric `json:"proxied"` + // replication metrics summary for each site replication peer + Metrics map[string]SRMetric `json:"replMetrics"` + // uptime of node being queried for site replication metrics + Uptime int64 `json:"uptime"` +} diff --git a/cmd/site-replication-metrics_gen.go b/cmd/site-replication-metrics_gen.go new file mode 100644 index 00000000..2b6272e3 --- /dev/null +++ b/cmd/site-replication-metrics_gen.go @@ -0,0 +1,1750 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *RStat) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Count": + z.Count, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Count") + return + } + case "Bytes": + z.Bytes, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Bytes") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z RStat) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "Count" + err = en.Append(0x82, 0xa5, 0x43, 0x6f, 0x75, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteInt64(z.Count) + if err != nil { + err = msgp.WrapError(err, "Count") + return + } + // write "Bytes" + err = en.Append(0xa5, 0x42, 0x79, 0x74, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteInt64(z.Bytes) + if err != nil { + err = msgp.WrapError(err, "Bytes") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z RStat) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "Count" + o = append(o, 0x82, 0xa5, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendInt64(o, z.Count) + // string "Bytes" + o = append(o, 0xa5, 0x42, 0x79, 0x74, 0x65, 0x73) + o = msgp.AppendInt64(o, z.Bytes) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *RStat) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Count": + z.Count, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Count") + return + } + case "Bytes": + z.Bytes, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bytes") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z RStat) Msgsize() (s int) { + s = 1 + 6 + msgp.Int64Size + 6 + msgp.Int64Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *RTimedMetrics) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "LastHour": + err = z.LastHour.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "LastHour") + return + } + case "SinceUptime": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "SinceUptime") + return + } + for zb0002 > 0 { + zb0002-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "SinceUptime") + return + } + switch msgp.UnsafeString(field) { + case "Count": + z.SinceUptime.Count, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "SinceUptime", "Count") + return + } + case "Bytes": + z.SinceUptime.Bytes, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "SinceUptime", "Bytes") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "SinceUptime") + return + } + } + } + case "LastMinute": + err = z.LastMinute.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "LastMinute") + return + } + case "ErrCounts": + var zb0003 uint32 + zb0003, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "ErrCounts") + return + } + if z.ErrCounts == nil { + z.ErrCounts = make(map[string]int, zb0003) + } else if len(z.ErrCounts) > 0 { + clear(z.ErrCounts) + } + for zb0003 > 0 { + zb0003-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "ErrCounts") + return + } + var za0002 int + za0002, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "ErrCounts", za0001) + return + } + z.ErrCounts[za0001] = za0002 + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *RTimedMetrics) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 4 + // write "LastHour" + err = en.Append(0x84, 0xa8, 0x4c, 0x61, 0x73, 0x74, 0x48, 0x6f, 0x75, 0x72) + if err != nil { + return + } + err = z.LastHour.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "LastHour") + return + } + // write "SinceUptime" + err = en.Append(0xab, 0x53, 0x69, 0x6e, 0x63, 0x65, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65) + if err != nil { + return + } + // map header, size 2 + // write "Count" + err = en.Append(0x82, 0xa5, 0x43, 0x6f, 0x75, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteInt64(z.SinceUptime.Count) + if err != nil { + err = msgp.WrapError(err, "SinceUptime", "Count") + return + } + // write "Bytes" + err = en.Append(0xa5, 0x42, 0x79, 0x74, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteInt64(z.SinceUptime.Bytes) + if err != nil { + err = msgp.WrapError(err, "SinceUptime", "Bytes") + return + } + // write "LastMinute" + err = en.Append(0xaa, 0x4c, 0x61, 0x73, 0x74, 0x4d, 0x69, 0x6e, 0x75, 0x74, 0x65) + if err != nil { + return + } + err = z.LastMinute.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "LastMinute") + return + } + // write "ErrCounts" + err = en.Append(0xa9, 0x45, 0x72, 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.ErrCounts))) + if err != nil { + err = msgp.WrapError(err, "ErrCounts") + return + } + for za0001, za0002 := range z.ErrCounts { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "ErrCounts") + return + } + err = en.WriteInt(za0002) + if err != nil { + err = msgp.WrapError(err, "ErrCounts", za0001) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *RTimedMetrics) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 4 + // string "LastHour" + o = append(o, 0x84, 0xa8, 0x4c, 0x61, 0x73, 0x74, 0x48, 0x6f, 0x75, 0x72) + o, err = z.LastHour.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "LastHour") + return + } + // string "SinceUptime" + o = append(o, 0xab, 0x53, 0x69, 0x6e, 0x63, 0x65, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65) + // map header, size 2 + // string "Count" + o = append(o, 0x82, 0xa5, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendInt64(o, z.SinceUptime.Count) + // string "Bytes" + o = append(o, 0xa5, 0x42, 0x79, 0x74, 0x65, 0x73) + o = msgp.AppendInt64(o, z.SinceUptime.Bytes) + // string "LastMinute" + o = append(o, 0xaa, 0x4c, 0x61, 0x73, 0x74, 0x4d, 0x69, 0x6e, 0x75, 0x74, 0x65) + o, err = z.LastMinute.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "LastMinute") + return + } + // string "ErrCounts" + o = append(o, 0xa9, 0x45, 0x72, 0x72, 0x43, 0x6f, 0x75, 0x6e, 0x74, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.ErrCounts))) + for za0001, za0002 := range z.ErrCounts { + o = msgp.AppendString(o, za0001) + o = msgp.AppendInt(o, za0002) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *RTimedMetrics) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "LastHour": + bts, err = z.LastHour.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "LastHour") + return + } + case "SinceUptime": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SinceUptime") + return + } + for zb0002 > 0 { + zb0002-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "SinceUptime") + return + } + switch msgp.UnsafeString(field) { + case "Count": + z.SinceUptime.Count, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "SinceUptime", "Count") + return + } + case "Bytes": + z.SinceUptime.Bytes, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "SinceUptime", "Bytes") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "SinceUptime") + return + } + } + } + case "LastMinute": + bts, err = z.LastMinute.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "LastMinute") + return + } + case "ErrCounts": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ErrCounts") + return + } + if z.ErrCounts == nil { + z.ErrCounts = make(map[string]int, zb0003) + } else if len(z.ErrCounts) > 0 { + clear(z.ErrCounts) + } + for zb0003 > 0 { + var za0002 int + zb0003-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ErrCounts") + return + } + za0002, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ErrCounts", za0001) + return + } + z.ErrCounts[za0001] = za0002 + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *RTimedMetrics) Msgsize() (s int) { + s = 1 + 9 + z.LastHour.Msgsize() + 12 + 1 + 6 + msgp.Int64Size + 6 + msgp.Int64Size + 11 + z.LastMinute.Msgsize() + 10 + msgp.MapHeaderSize + if z.ErrCounts != nil { + for za0001, za0002 := range z.ErrCounts { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + msgp.IntSize + } + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *SRMetric) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "DeploymentID": + z.DeploymentID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DeploymentID") + return + } + case "Endpoint": + z.Endpoint, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + case "TotalDowntime": + z.TotalDowntime, err = dc.ReadDuration() + if err != nil { + err = msgp.WrapError(err, "TotalDowntime") + return + } + case "LastOnline": + z.LastOnline, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "LastOnline") + return + } + case "Online": + z.Online, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Online") + return + } + case "Latency": + err = z.Latency.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Latency") + return + } + case "ReplicatedSize": + z.ReplicatedSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ReplicatedSize") + return + } + case "ReplicatedCount": + z.ReplicatedCount, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ReplicatedCount") + return + } + case "Failed": + err = z.Failed.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Failed") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *SRMetric) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 9 + // write "DeploymentID" + err = en.Append(0x89, 0xac, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x44) + if err != nil { + return + } + err = en.WriteString(z.DeploymentID) + if err != nil { + err = msgp.WrapError(err, "DeploymentID") + return + } + // write "Endpoint" + err = en.Append(0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Endpoint) + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + // write "TotalDowntime" + err = en.Append(0xad, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x44, 0x6f, 0x77, 0x6e, 0x74, 0x69, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteDuration(z.TotalDowntime) + if err != nil { + err = msgp.WrapError(err, "TotalDowntime") + return + } + // write "LastOnline" + err = en.Append(0xaa, 0x4c, 0x61, 0x73, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65) + if err != nil { + return + } + err = en.WriteTime(z.LastOnline) + if err != nil { + err = msgp.WrapError(err, "LastOnline") + return + } + // write "Online" + err = en.Append(0xa6, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65) + if err != nil { + return + } + err = en.WriteBool(z.Online) + if err != nil { + err = msgp.WrapError(err, "Online") + return + } + // write "Latency" + err = en.Append(0xa7, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79) + if err != nil { + return + } + err = z.Latency.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Latency") + return + } + // write "ReplicatedSize" + err = en.Append(0xae, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.ReplicatedSize) + if err != nil { + err = msgp.WrapError(err, "ReplicatedSize") + return + } + // write "ReplicatedCount" + err = en.Append(0xaf, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteInt64(z.ReplicatedCount) + if err != nil { + err = msgp.WrapError(err, "ReplicatedCount") + return + } + // write "Failed" + err = en.Append(0xa6, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64) + if err != nil { + return + } + err = z.Failed.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Failed") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *SRMetric) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 9 + // string "DeploymentID" + o = append(o, 0x89, 0xac, 0x44, 0x65, 0x70, 0x6c, 0x6f, 0x79, 0x6d, 0x65, 0x6e, 0x74, 0x49, 0x44) + o = msgp.AppendString(o, z.DeploymentID) + // string "Endpoint" + o = append(o, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74) + o = msgp.AppendString(o, z.Endpoint) + // string "TotalDowntime" + o = append(o, 0xad, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x44, 0x6f, 0x77, 0x6e, 0x74, 0x69, 0x6d, 0x65) + o = msgp.AppendDuration(o, z.TotalDowntime) + // string "LastOnline" + o = append(o, 0xaa, 0x4c, 0x61, 0x73, 0x74, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65) + o = msgp.AppendTime(o, z.LastOnline) + // string "Online" + o = append(o, 0xa6, 0x4f, 0x6e, 0x6c, 0x69, 0x6e, 0x65) + o = msgp.AppendBool(o, z.Online) + // string "Latency" + o = append(o, 0xa7, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79) + o, err = z.Latency.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Latency") + return + } + // string "ReplicatedSize" + o = append(o, 0xae, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.ReplicatedSize) + // string "ReplicatedCount" + o = append(o, 0xaf, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendInt64(o, z.ReplicatedCount) + // string "Failed" + o = append(o, 0xa6, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64) + o, err = z.Failed.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Failed") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *SRMetric) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "DeploymentID": + z.DeploymentID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DeploymentID") + return + } + case "Endpoint": + z.Endpoint, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + case "TotalDowntime": + z.TotalDowntime, bts, err = msgp.ReadDurationBytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalDowntime") + return + } + case "LastOnline": + z.LastOnline, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "LastOnline") + return + } + case "Online": + z.Online, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Online") + return + } + case "Latency": + bts, err = z.Latency.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Latency") + return + } + case "ReplicatedSize": + z.ReplicatedSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicatedSize") + return + } + case "ReplicatedCount": + z.ReplicatedCount, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicatedCount") + return + } + case "Failed": + bts, err = z.Failed.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Failed") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *SRMetric) Msgsize() (s int) { + s = 1 + 13 + msgp.StringPrefixSize + len(z.DeploymentID) + 9 + msgp.StringPrefixSize + len(z.Endpoint) + 14 + msgp.DurationSize + 11 + msgp.TimeSize + 7 + msgp.BoolSize + 8 + z.Latency.Msgsize() + 15 + msgp.Int64Size + 16 + msgp.Int64Size + 7 + z.Failed.Msgsize() + return +} + +// DecodeMsg implements msgp.Decodable +func (z *SRMetricsSummary) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ActiveWorkers": + err = z.ActiveWorkers.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "ActiveWorkers") + return + } + case "ReplicaSize": + z.ReplicaSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ReplicaSize") + return + } + case "ReplicaCount": + z.ReplicaCount, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ReplicaCount") + return + } + case "Queued": + err = z.Queued.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Queued") + return + } + case "Proxied": + err = z.Proxied.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Proxied") + return + } + case "Metrics": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Metrics") + return + } + if z.Metrics == nil { + z.Metrics = make(map[string]SRMetric, zb0002) + } else if len(z.Metrics) > 0 { + clear(z.Metrics) + } + for zb0002 > 0 { + zb0002-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Metrics") + return + } + var za0002 SRMetric + err = za0002.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Metrics", za0001) + return + } + z.Metrics[za0001] = za0002 + } + case "Uptime": + z.Uptime, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Uptime") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *SRMetricsSummary) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 7 + // write "ActiveWorkers" + err = en.Append(0x87, 0xad, 0x41, 0x63, 0x74, 0x69, 0x76, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73) + if err != nil { + return + } + err = z.ActiveWorkers.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "ActiveWorkers") + return + } + // write "ReplicaSize" + err = en.Append(0xab, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.ReplicaSize) + if err != nil { + err = msgp.WrapError(err, "ReplicaSize") + return + } + // write "ReplicaCount" + err = en.Append(0xac, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x43, 0x6f, 0x75, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteInt64(z.ReplicaCount) + if err != nil { + err = msgp.WrapError(err, "ReplicaCount") + return + } + // write "Queued" + err = en.Append(0xa6, 0x51, 0x75, 0x65, 0x75, 0x65, 0x64) + if err != nil { + return + } + err = z.Queued.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Queued") + return + } + // write "Proxied" + err = en.Append(0xa7, 0x50, 0x72, 0x6f, 0x78, 0x69, 0x65, 0x64) + if err != nil { + return + } + err = z.Proxied.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Proxied") + return + } + // write "Metrics" + err = en.Append(0xa7, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.Metrics))) + if err != nil { + err = msgp.WrapError(err, "Metrics") + return + } + for za0001, za0002 := range z.Metrics { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "Metrics") + return + } + err = za0002.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Metrics", za0001) + return + } + } + // write "Uptime" + err = en.Append(0xa6, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.Uptime) + if err != nil { + err = msgp.WrapError(err, "Uptime") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *SRMetricsSummary) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 7 + // string "ActiveWorkers" + o = append(o, 0x87, 0xad, 0x41, 0x63, 0x74, 0x69, 0x76, 0x65, 0x57, 0x6f, 0x72, 0x6b, 0x65, 0x72, 0x73) + o, err = z.ActiveWorkers.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "ActiveWorkers") + return + } + // string "ReplicaSize" + o = append(o, 0xab, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.ReplicaSize) + // string "ReplicaCount" + o = append(o, 0xac, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendInt64(o, z.ReplicaCount) + // string "Queued" + o = append(o, 0xa6, 0x51, 0x75, 0x65, 0x75, 0x65, 0x64) + o, err = z.Queued.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Queued") + return + } + // string "Proxied" + o = append(o, 0xa7, 0x50, 0x72, 0x6f, 0x78, 0x69, 0x65, 0x64) + o, err = z.Proxied.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Proxied") + return + } + // string "Metrics" + o = append(o, 0xa7, 0x4d, 0x65, 0x74, 0x72, 0x69, 0x63, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.Metrics))) + for za0001, za0002 := range z.Metrics { + o = msgp.AppendString(o, za0001) + o, err = za0002.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Metrics", za0001) + return + } + } + // string "Uptime" + o = append(o, 0xa6, 0x55, 0x70, 0x74, 0x69, 0x6d, 0x65) + o = msgp.AppendInt64(o, z.Uptime) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *SRMetricsSummary) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ActiveWorkers": + bts, err = z.ActiveWorkers.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ActiveWorkers") + return + } + case "ReplicaSize": + z.ReplicaSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicaSize") + return + } + case "ReplicaCount": + z.ReplicaCount, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicaCount") + return + } + case "Queued": + bts, err = z.Queued.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Queued") + return + } + case "Proxied": + bts, err = z.Proxied.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Proxied") + return + } + case "Metrics": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Metrics") + return + } + if z.Metrics == nil { + z.Metrics = make(map[string]SRMetric, zb0002) + } else if len(z.Metrics) > 0 { + clear(z.Metrics) + } + for zb0002 > 0 { + var za0002 SRMetric + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Metrics") + return + } + bts, err = za0002.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Metrics", za0001) + return + } + z.Metrics[za0001] = za0002 + } + case "Uptime": + z.Uptime, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Uptime") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *SRMetricsSummary) Msgsize() (s int) { + s = 1 + 14 + z.ActiveWorkers.Msgsize() + 12 + msgp.Int64Size + 13 + msgp.Int64Size + 7 + z.Queued.Msgsize() + 8 + z.Proxied.Msgsize() + 8 + msgp.MapHeaderSize + if z.Metrics != nil { + for za0001, za0002 := range z.Metrics { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize() + } + } + s += 7 + msgp.Int64Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *SRStats) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ReplicaSize": + z.ReplicaSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ReplicaSize") + return + } + case "ReplicaCount": + z.ReplicaCount, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ReplicaCount") + return + } + case "M": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "M") + return + } + if z.M == nil { + z.M = make(map[string]*SRStatus, zb0002) + } else if len(z.M) > 0 { + clear(z.M) + } + for zb0002 > 0 { + zb0002-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "M") + return + } + var za0002 *SRStatus + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "M", za0001) + return + } + za0002 = nil + } else { + if za0002 == nil { + za0002 = new(SRStatus) + } + err = za0002.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "M", za0001) + return + } + } + z.M[za0001] = za0002 + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *SRStats) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "ReplicaSize" + err = en.Append(0x83, 0xab, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.ReplicaSize) + if err != nil { + err = msgp.WrapError(err, "ReplicaSize") + return + } + // write "ReplicaCount" + err = en.Append(0xac, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x43, 0x6f, 0x75, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteInt64(z.ReplicaCount) + if err != nil { + err = msgp.WrapError(err, "ReplicaCount") + return + } + // write "M" + err = en.Append(0xa1, 0x4d) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.M))) + if err != nil { + err = msgp.WrapError(err, "M") + return + } + for za0001, za0002 := range z.M { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "M") + return + } + if za0002 == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = za0002.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "M", za0001) + return + } + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *SRStats) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "ReplicaSize" + o = append(o, 0x83, 0xab, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.ReplicaSize) + // string "ReplicaCount" + o = append(o, 0xac, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendInt64(o, z.ReplicaCount) + // string "M" + o = append(o, 0xa1, 0x4d) + o = msgp.AppendMapHeader(o, uint32(len(z.M))) + for za0001, za0002 := range z.M { + o = msgp.AppendString(o, za0001) + if za0002 == nil { + o = msgp.AppendNil(o) + } else { + o, err = za0002.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "M", za0001) + return + } + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *SRStats) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ReplicaSize": + z.ReplicaSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicaSize") + return + } + case "ReplicaCount": + z.ReplicaCount, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicaCount") + return + } + case "M": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "M") + return + } + if z.M == nil { + z.M = make(map[string]*SRStatus, zb0002) + } else if len(z.M) > 0 { + clear(z.M) + } + for zb0002 > 0 { + var za0002 *SRStatus + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "M") + return + } + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + za0002 = nil + } else { + if za0002 == nil { + za0002 = new(SRStatus) + } + bts, err = za0002.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "M", za0001) + return + } + } + z.M[za0001] = za0002 + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *SRStats) Msgsize() (s int) { + s = 1 + 12 + msgp.Int64Size + 13 + msgp.Int64Size + 2 + msgp.MapHeaderSize + if z.M != nil { + for za0001, za0002 := range z.M { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + if za0002 == nil { + s += msgp.NilSize + } else { + s += za0002.Msgsize() + } + } + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *SRStatus) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ReplicatedSize": + z.ReplicatedSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ReplicatedSize") + return + } + case "Failed": + err = z.Failed.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Failed") + return + } + case "ReplicatedCount": + z.ReplicatedCount, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ReplicatedCount") + return + } + case "Latency": + err = z.Latency.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Latency") + return + } + case "lt": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "XferRateLrg") + return + } + z.XferRateLrg = nil + } else { + if z.XferRateLrg == nil { + z.XferRateLrg = new(XferStats) + } + err = z.XferRateLrg.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "XferRateLrg") + return + } + } + case "st": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "XferRateSml") + return + } + z.XferRateSml = nil + } else { + if z.XferRateSml == nil { + z.XferRateSml = new(XferStats) + } + err = z.XferRateSml.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "XferRateSml") + return + } + } + case "Endpoint": + z.Endpoint, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + case "Secure": + z.Secure, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Secure") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *SRStatus) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 8 + // write "ReplicatedSize" + err = en.Append(0x88, 0xae, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.ReplicatedSize) + if err != nil { + err = msgp.WrapError(err, "ReplicatedSize") + return + } + // write "Failed" + err = en.Append(0xa6, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64) + if err != nil { + return + } + err = z.Failed.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Failed") + return + } + // write "ReplicatedCount" + err = en.Append(0xaf, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteInt64(z.ReplicatedCount) + if err != nil { + err = msgp.WrapError(err, "ReplicatedCount") + return + } + // write "Latency" + err = en.Append(0xa7, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79) + if err != nil { + return + } + err = z.Latency.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Latency") + return + } + // write "lt" + err = en.Append(0xa2, 0x6c, 0x74) + if err != nil { + return + } + if z.XferRateLrg == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.XferRateLrg.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "XferRateLrg") + return + } + } + // write "st" + err = en.Append(0xa2, 0x73, 0x74) + if err != nil { + return + } + if z.XferRateSml == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.XferRateSml.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "XferRateSml") + return + } + } + // write "Endpoint" + err = en.Append(0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Endpoint) + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + // write "Secure" + err = en.Append(0xa6, 0x53, 0x65, 0x63, 0x75, 0x72, 0x65) + if err != nil { + return + } + err = en.WriteBool(z.Secure) + if err != nil { + err = msgp.WrapError(err, "Secure") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *SRStatus) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 8 + // string "ReplicatedSize" + o = append(o, 0x88, 0xae, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.ReplicatedSize) + // string "Failed" + o = append(o, 0xa6, 0x46, 0x61, 0x69, 0x6c, 0x65, 0x64) + o, err = z.Failed.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Failed") + return + } + // string "ReplicatedCount" + o = append(o, 0xaf, 0x52, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x65, 0x64, 0x43, 0x6f, 0x75, 0x6e, 0x74) + o = msgp.AppendInt64(o, z.ReplicatedCount) + // string "Latency" + o = append(o, 0xa7, 0x4c, 0x61, 0x74, 0x65, 0x6e, 0x63, 0x79) + o, err = z.Latency.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Latency") + return + } + // string "lt" + o = append(o, 0xa2, 0x6c, 0x74) + if z.XferRateLrg == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.XferRateLrg.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "XferRateLrg") + return + } + } + // string "st" + o = append(o, 0xa2, 0x73, 0x74) + if z.XferRateSml == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.XferRateSml.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "XferRateSml") + return + } + } + // string "Endpoint" + o = append(o, 0xa8, 0x45, 0x6e, 0x64, 0x70, 0x6f, 0x69, 0x6e, 0x74) + o = msgp.AppendString(o, z.Endpoint) + // string "Secure" + o = append(o, 0xa6, 0x53, 0x65, 0x63, 0x75, 0x72, 0x65) + o = msgp.AppendBool(o, z.Secure) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *SRStatus) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ReplicatedSize": + z.ReplicatedSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicatedSize") + return + } + case "Failed": + bts, err = z.Failed.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Failed") + return + } + case "ReplicatedCount": + z.ReplicatedCount, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicatedCount") + return + } + case "Latency": + bts, err = z.Latency.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Latency") + return + } + case "lt": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.XferRateLrg = nil + } else { + if z.XferRateLrg == nil { + z.XferRateLrg = new(XferStats) + } + bts, err = z.XferRateLrg.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "XferRateLrg") + return + } + } + case "st": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.XferRateSml = nil + } else { + if z.XferRateSml == nil { + z.XferRateSml = new(XferStats) + } + bts, err = z.XferRateSml.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "XferRateSml") + return + } + } + case "Endpoint": + z.Endpoint, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + case "Secure": + z.Secure, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Secure") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *SRStatus) Msgsize() (s int) { + s = 1 + 15 + msgp.Int64Size + 7 + z.Failed.Msgsize() + 16 + msgp.Int64Size + 8 + z.Latency.Msgsize() + 3 + if z.XferRateLrg == nil { + s += msgp.NilSize + } else { + s += z.XferRateLrg.Msgsize() + } + s += 3 + if z.XferRateSml == nil { + s += msgp.NilSize + } else { + s += z.XferRateSml.Msgsize() + } + s += 9 + msgp.StringPrefixSize + len(z.Endpoint) + 7 + msgp.BoolSize + return +} diff --git a/cmd/site-replication-metrics_gen_test.go b/cmd/site-replication-metrics_gen_test.go new file mode 100644 index 00000000..9e47381c --- /dev/null +++ b/cmd/site-replication-metrics_gen_test.go @@ -0,0 +1,688 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalRStat(t *testing.T) { + v := RStat{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgRStat(b *testing.B) { + v := RStat{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgRStat(b *testing.B) { + v := RStat{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalRStat(b *testing.B) { + v := RStat{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeRStat(t *testing.T) { + v := RStat{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeRStat Msgsize() is inaccurate") + } + + vn := RStat{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeRStat(b *testing.B) { + v := RStat{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeRStat(b *testing.B) { + v := RStat{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalRTimedMetrics(t *testing.T) { + v := RTimedMetrics{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgRTimedMetrics(b *testing.B) { + v := RTimedMetrics{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgRTimedMetrics(b *testing.B) { + v := RTimedMetrics{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalRTimedMetrics(b *testing.B) { + v := RTimedMetrics{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeRTimedMetrics(t *testing.T) { + v := RTimedMetrics{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeRTimedMetrics Msgsize() is inaccurate") + } + + vn := RTimedMetrics{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeRTimedMetrics(b *testing.B) { + v := RTimedMetrics{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeRTimedMetrics(b *testing.B) { + v := RTimedMetrics{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalSRMetric(t *testing.T) { + v := SRMetric{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgSRMetric(b *testing.B) { + v := SRMetric{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgSRMetric(b *testing.B) { + v := SRMetric{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalSRMetric(b *testing.B) { + v := SRMetric{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeSRMetric(t *testing.T) { + v := SRMetric{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeSRMetric Msgsize() is inaccurate") + } + + vn := SRMetric{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeSRMetric(b *testing.B) { + v := SRMetric{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeSRMetric(b *testing.B) { + v := SRMetric{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalSRMetricsSummary(t *testing.T) { + v := SRMetricsSummary{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgSRMetricsSummary(b *testing.B) { + v := SRMetricsSummary{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgSRMetricsSummary(b *testing.B) { + v := SRMetricsSummary{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalSRMetricsSummary(b *testing.B) { + v := SRMetricsSummary{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeSRMetricsSummary(t *testing.T) { + v := SRMetricsSummary{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeSRMetricsSummary Msgsize() is inaccurate") + } + + vn := SRMetricsSummary{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeSRMetricsSummary(b *testing.B) { + v := SRMetricsSummary{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeSRMetricsSummary(b *testing.B) { + v := SRMetricsSummary{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalSRStats(t *testing.T) { + v := SRStats{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgSRStats(b *testing.B) { + v := SRStats{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgSRStats(b *testing.B) { + v := SRStats{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalSRStats(b *testing.B) { + v := SRStats{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeSRStats(t *testing.T) { + v := SRStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeSRStats Msgsize() is inaccurate") + } + + vn := SRStats{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeSRStats(b *testing.B) { + v := SRStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeSRStats(b *testing.B) { + v := SRStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalSRStatus(t *testing.T) { + v := SRStatus{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgSRStatus(b *testing.B) { + v := SRStatus{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgSRStatus(b *testing.B) { + v := SRStatus{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalSRStatus(b *testing.B) { + v := SRStatus{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeSRStatus(t *testing.T) { + v := SRStatus{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeSRStatus Msgsize() is inaccurate") + } + + vn := SRStatus{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeSRStatus(b *testing.B) { + v := SRStatus{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeSRStatus(b *testing.B) { + v := SRStatus{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/site-replication-utils.go b/cmd/site-replication-utils.go new file mode 100644 index 00000000..19227584 --- /dev/null +++ b/cmd/site-replication-utils.go @@ -0,0 +1,343 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "maps" + "math/rand" + "sync" + "time" + + "github.com/minio/madmin-go/v3" +) + +//go:generate msgp -file=$GOFILE + +// SiteResyncStatus captures current replication resync status for a target site +type SiteResyncStatus struct { + Version int `json:"version" msg:"v"` + // Overall site status + Status ResyncStatusType `json:"st" msg:"ss"` + DeplID string `json:"dId" msg:"did"` + BucketStatuses map[string]ResyncStatusType `json:"buckets" msg:"bkts"` + TotBuckets int `json:"totbuckets" msg:"tb"` + TargetReplicationResyncStatus `json:"currSt" msg:"cst"` +} + +func (s *SiteResyncStatus) clone() SiteResyncStatus { + if s == nil { + return SiteResyncStatus{} + } + o := *s + o.BucketStatuses = make(map[string]ResyncStatusType, len(s.BucketStatuses)) + maps.Copy(o.BucketStatuses, s.BucketStatuses) + return o +} + +const ( + siteResyncPrefix = bucketMetaPrefix + "/site-replication/resync" +) + +type resyncState struct { + resyncID string + LastSaved time.Time +} + +//msgp:ignore siteResyncMetrics +type siteResyncMetrics struct { + sync.RWMutex + // resyncStatus maps resync ID to resync status for peer + resyncStatus map[string]SiteResyncStatus + // map peer deployment ID to resync ID + peerResyncMap map[string]resyncState +} + +func newSiteResyncMetrics(ctx context.Context) *siteResyncMetrics { + s := siteResyncMetrics{ + resyncStatus: make(map[string]SiteResyncStatus), + peerResyncMap: make(map[string]resyncState), + } + go s.save(ctx) + go s.init(ctx) + return &s +} + +// init site resync metrics +func (sm *siteResyncMetrics) init(ctx context.Context) { + r := rand.New(rand.NewSource(time.Now().UnixNano())) + // Run the site resync metrics load in a loop + for { + if err := sm.load(ctx, newObjectLayerFn()); err == nil { + <-ctx.Done() + return + } + duration := max(time.Duration(r.Float64()*float64(time.Second*10)), + // Make sure to sleep at least a second to avoid high CPU ticks. + time.Second) + time.Sleep(duration) + } +} + +// load resync metrics saved on disk into memory +func (sm *siteResyncMetrics) load(ctx context.Context, objAPI ObjectLayer) error { + if objAPI == nil { + return errServerNotInitialized + } + info, err := globalSiteReplicationSys.GetClusterInfo(ctx) + if err != nil { + return err + } + if !info.Enabled { + return nil + } + for _, peer := range info.Sites { + if peer.DeploymentID == globalDeploymentID() { + continue + } + rs, err := loadSiteResyncMetadata(ctx, objAPI, peer.DeploymentID) + if err != nil { + return err + } + sm.Lock() + if _, ok := sm.peerResyncMap[peer.DeploymentID]; !ok { + sm.peerResyncMap[peer.DeploymentID] = resyncState{resyncID: rs.ResyncID, LastSaved: time.Time{}} + sm.resyncStatus[rs.ResyncID] = rs + } + sm.Unlock() + } + return nil +} + +func (sm *siteResyncMetrics) report(dID string) *madmin.SiteResyncMetrics { + sm.RLock() + defer sm.RUnlock() + rst, ok := sm.peerResyncMap[dID] + if !ok { + return nil + } + rs, ok := sm.resyncStatus[rst.resyncID] + if !ok { + return nil + } + m := madmin.SiteResyncMetrics{ + CollectedAt: rs.LastUpdate, + StartTime: rs.StartTime, + LastUpdate: rs.LastUpdate, + ResyncStatus: rs.Status.String(), + ResyncID: rst.resyncID, + DeplID: rs.DeplID, + ReplicatedSize: rs.ReplicatedSize, + ReplicatedCount: rs.ReplicatedCount, + FailedSize: rs.FailedSize, + FailedCount: rs.FailedCount, + Bucket: rs.Bucket, + Object: rs.Object, + NumBuckets: int64(rs.TotBuckets), + } + for b, st := range rs.BucketStatuses { + if st == ResyncFailed { + m.FailedBuckets = append(m.FailedBuckets, b) + } + } + return &m +} + +// save in-memory stats to disk +func (sm *siteResyncMetrics) save(ctx context.Context) { + sTimer := time.NewTimer(siteResyncSaveInterval) + defer sTimer.Stop() + for { + select { + case <-sTimer.C: + if globalSiteReplicationSys.isEnabled() { + sm.Lock() + wg := sync.WaitGroup{} + for dID, rs := range sm.peerResyncMap { + st, ok := sm.resyncStatus[rs.resyncID] + if ok { + updt := st.Status.isValid() && st.LastUpdate.After(rs.LastSaved) + if !updt { + continue + } + rs.LastSaved = UTCNow() + sm.peerResyncMap[dID] = rs + wg.Add(1) + go func() { + defer wg.Done() + saveSiteResyncMetadata(ctx, st, newObjectLayerFn()) + }() + } + } + wg.Wait() + sm.Unlock() + } + sTimer.Reset(siteResyncSaveInterval) + case <-ctx.Done(): + return + } + } +} + +// update overall site resync state +func (sm *siteResyncMetrics) updateState(s SiteResyncStatus) error { + if !globalSiteReplicationSys.isEnabled() { + return nil + } + sm.Lock() + defer sm.Unlock() + switch s.Status { + case ResyncStarted: + sm.peerResyncMap[s.DeplID] = resyncState{resyncID: s.ResyncID, LastSaved: time.Time{}} + sm.resyncStatus[s.ResyncID] = s + case ResyncCompleted, ResyncCanceled, ResyncFailed: + st, ok := sm.resyncStatus[s.ResyncID] + if ok { + st.LastUpdate = s.LastUpdate + st.Status = s.Status + return nil + } + sm.resyncStatus[s.ResyncID] = st + return saveSiteResyncMetadata(GlobalContext, st, newObjectLayerFn()) + } + return nil +} + +// increment SyncedBuckets count +func (sm *siteResyncMetrics) incBucket(o resyncOpts, bktStatus ResyncStatusType) { + if !globalSiteReplicationSys.isEnabled() { + return + } + sm.Lock() + defer sm.Unlock() + st, ok := sm.resyncStatus[o.resyncID] + if ok { + if st.BucketStatuses == nil { + st.BucketStatuses = map[string]ResyncStatusType{} + } + switch bktStatus { + case ResyncCompleted: + st.BucketStatuses[o.bucket] = ResyncCompleted + st.Status = siteResyncStatus(st.Status, st.BucketStatuses) + st.LastUpdate = UTCNow() + sm.resyncStatus[o.resyncID] = st + case ResyncFailed: + st.BucketStatuses[o.bucket] = ResyncFailed + st.Status = siteResyncStatus(st.Status, st.BucketStatuses) + st.LastUpdate = UTCNow() + sm.resyncStatus[o.resyncID] = st + } + } +} + +// remove deleted bucket from active resync tracking +func (sm *siteResyncMetrics) deleteBucket(b string) { + if !globalSiteReplicationSys.isEnabled() { + return + } + sm.Lock() + defer sm.Unlock() + for _, rs := range sm.peerResyncMap { + st, ok := sm.resyncStatus[rs.resyncID] + if !ok { + return + } + switch st.Status { + case ResyncCompleted, ResyncFailed: + return + default: + delete(st.BucketStatuses, b) + } + } +} + +// returns overall resync status from individual bucket resync status map +func siteResyncStatus(currSt ResyncStatusType, m map[string]ResyncStatusType) ResyncStatusType { + // avoid overwriting canceled resync status + if currSt != ResyncStarted { + return currSt + } + totBuckets := len(m) + var cmpCount, failCount int + for _, st := range m { + switch st { + case ResyncCompleted: + cmpCount++ + case ResyncFailed: + failCount++ + } + } + if cmpCount == totBuckets { + return ResyncCompleted + } + if cmpCount+failCount == totBuckets { + return ResyncFailed + } + return ResyncStarted +} + +// update resync metrics per object +func (sm *siteResyncMetrics) updateMetric(r TargetReplicationResyncStatus, resyncID string) { + if !globalSiteReplicationSys.isEnabled() { + return + } + sm.Lock() + defer sm.Unlock() + s := sm.resyncStatus[resyncID] + if r.ReplicatedCount > 0 { + s.ReplicatedCount++ + s.ReplicatedSize += r.ReplicatedSize + } else { + s.FailedCount++ + s.FailedSize += r.FailedSize + } + s.Bucket = r.Bucket + s.Object = r.Object + s.LastUpdate = UTCNow() + sm.resyncStatus[resyncID] = s +} + +// Status returns current in-memory resync status for this deployment +func (sm *siteResyncMetrics) status(dID string) (rs SiteResyncStatus, err error) { + sm.RLock() + defer sm.RUnlock() + if rst, ok1 := sm.peerResyncMap[dID]; ok1 { + if st, ok2 := sm.resyncStatus[rst.resyncID]; ok2 { + return st.clone(), nil + } + } + return rs, errSRNoResync +} + +// Status returns latest resync status for this deployment +func (sm *siteResyncMetrics) siteStatus(ctx context.Context, objAPI ObjectLayer, dID string) (rs SiteResyncStatus, err error) { + if !globalSiteReplicationSys.isEnabled() { + return rs, errSRNotEnabled + } + // check in-memory status + rs, err = sm.status(dID) + if err == nil { + return rs, nil + } + // check disk resync status + rs, err = loadSiteResyncMetadata(ctx, objAPI, dID) + if err != nil && err == errConfigNotFound { + return rs, nil + } + return rs, err +} diff --git a/cmd/site-replication-utils_gen.go b/cmd/site-replication-utils_gen.go new file mode 100644 index 00000000..f02eb5c7 --- /dev/null +++ b/cmd/site-replication-utils_gen.go @@ -0,0 +1,314 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *SiteResyncStatus) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "v": + z.Version, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Version") + return + } + case "ss": + err = z.Status.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Status") + return + } + case "did": + z.DeplID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DeplID") + return + } + case "bkts": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "BucketStatuses") + return + } + if z.BucketStatuses == nil { + z.BucketStatuses = make(map[string]ResyncStatusType, zb0002) + } else if len(z.BucketStatuses) > 0 { + clear(z.BucketStatuses) + } + for zb0002 > 0 { + zb0002-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "BucketStatuses") + return + } + var za0002 ResyncStatusType + err = za0002.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "BucketStatuses", za0001) + return + } + z.BucketStatuses[za0001] = za0002 + } + case "tb": + z.TotBuckets, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "TotBuckets") + return + } + case "cst": + err = z.TargetReplicationResyncStatus.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "TargetReplicationResyncStatus") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *SiteResyncStatus) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 6 + // write "v" + err = en.Append(0x86, 0xa1, 0x76) + if err != nil { + return + } + err = en.WriteInt(z.Version) + if err != nil { + err = msgp.WrapError(err, "Version") + return + } + // write "ss" + err = en.Append(0xa2, 0x73, 0x73) + if err != nil { + return + } + err = z.Status.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Status") + return + } + // write "did" + err = en.Append(0xa3, 0x64, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteString(z.DeplID) + if err != nil { + err = msgp.WrapError(err, "DeplID") + return + } + // write "bkts" + err = en.Append(0xa4, 0x62, 0x6b, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.BucketStatuses))) + if err != nil { + err = msgp.WrapError(err, "BucketStatuses") + return + } + for za0001, za0002 := range z.BucketStatuses { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "BucketStatuses") + return + } + err = za0002.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "BucketStatuses", za0001) + return + } + } + // write "tb" + err = en.Append(0xa2, 0x74, 0x62) + if err != nil { + return + } + err = en.WriteInt(z.TotBuckets) + if err != nil { + err = msgp.WrapError(err, "TotBuckets") + return + } + // write "cst" + err = en.Append(0xa3, 0x63, 0x73, 0x74) + if err != nil { + return + } + err = z.TargetReplicationResyncStatus.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "TargetReplicationResyncStatus") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *SiteResyncStatus) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 6 + // string "v" + o = append(o, 0x86, 0xa1, 0x76) + o = msgp.AppendInt(o, z.Version) + // string "ss" + o = append(o, 0xa2, 0x73, 0x73) + o, err = z.Status.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Status") + return + } + // string "did" + o = append(o, 0xa3, 0x64, 0x69, 0x64) + o = msgp.AppendString(o, z.DeplID) + // string "bkts" + o = append(o, 0xa4, 0x62, 0x6b, 0x74, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.BucketStatuses))) + for za0001, za0002 := range z.BucketStatuses { + o = msgp.AppendString(o, za0001) + o, err = za0002.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "BucketStatuses", za0001) + return + } + } + // string "tb" + o = append(o, 0xa2, 0x74, 0x62) + o = msgp.AppendInt(o, z.TotBuckets) + // string "cst" + o = append(o, 0xa3, 0x63, 0x73, 0x74) + o, err = z.TargetReplicationResyncStatus.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "TargetReplicationResyncStatus") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *SiteResyncStatus) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "v": + z.Version, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Version") + return + } + case "ss": + bts, err = z.Status.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Status") + return + } + case "did": + z.DeplID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DeplID") + return + } + case "bkts": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "BucketStatuses") + return + } + if z.BucketStatuses == nil { + z.BucketStatuses = make(map[string]ResyncStatusType, zb0002) + } else if len(z.BucketStatuses) > 0 { + clear(z.BucketStatuses) + } + for zb0002 > 0 { + var za0002 ResyncStatusType + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "BucketStatuses") + return + } + bts, err = za0002.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "BucketStatuses", za0001) + return + } + z.BucketStatuses[za0001] = za0002 + } + case "tb": + z.TotBuckets, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotBuckets") + return + } + case "cst": + bts, err = z.TargetReplicationResyncStatus.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "TargetReplicationResyncStatus") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *SiteResyncStatus) Msgsize() (s int) { + s = 1 + 2 + msgp.IntSize + 3 + z.Status.Msgsize() + 4 + msgp.StringPrefixSize + len(z.DeplID) + 5 + msgp.MapHeaderSize + if z.BucketStatuses != nil { + for za0001, za0002 := range z.BucketStatuses { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize() + } + } + s += 3 + msgp.IntSize + 4 + z.TargetReplicationResyncStatus.Msgsize() + return +} diff --git a/cmd/site-replication-utils_gen_test.go b/cmd/site-replication-utils_gen_test.go new file mode 100644 index 00000000..a982c6c2 --- /dev/null +++ b/cmd/site-replication-utils_gen_test.go @@ -0,0 +1,123 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalSiteResyncStatus(t *testing.T) { + v := SiteResyncStatus{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgSiteResyncStatus(b *testing.B) { + v := SiteResyncStatus{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgSiteResyncStatus(b *testing.B) { + v := SiteResyncStatus{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalSiteResyncStatus(b *testing.B) { + v := SiteResyncStatus{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeSiteResyncStatus(t *testing.T) { + v := SiteResyncStatus{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeSiteResyncStatus Msgsize() is inaccurate") + } + + vn := SiteResyncStatus{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeSiteResyncStatus(b *testing.B) { + v := SiteResyncStatus{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeSiteResyncStatus(b *testing.B) { + v := SiteResyncStatus{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/site-replication.go b/cmd/site-replication.go new file mode 100644 index 00000000..0b2f26f7 --- /dev/null +++ b/cmd/site-replication.go @@ -0,0 +1,6284 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/binary" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "maps" + "math/rand" + "net/url" + "reflect" + "runtime" + "slices" + "sort" + "strings" + "sync" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio-go/v7/pkg/replication" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/bucket/lifecycle" + sreplication "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/logger" + xldap "github.com/minio/pkg/v3/ldap" + "github.com/minio/pkg/v3/policy" + "github.com/puzpuzpuz/xsync/v3" +) + +const ( + srStatePrefix = minioConfigPrefix + "/site-replication" + srStateFile = "state.json" +) + +const ( + srStateFormatVersion1 = 1 +) + +var ( + errSRCannotJoin = SRError{ + Cause: errors.New("this site is already configured for site-replication"), + Code: ErrSiteReplicationInvalidRequest, + } + errSRDuplicateSites = SRError{ + Cause: errors.New("duplicate sites provided for site-replication"), + Code: ErrSiteReplicationInvalidRequest, + } + errSRSelfNotFound = SRError{ + Cause: errors.New("none of the given sites correspond to the current one"), + Code: ErrSiteReplicationInvalidRequest, + } + errSRPeerNotFound = SRError{ + Cause: errors.New("peer not found"), + Code: ErrSiteReplicationInvalidRequest, + } + errSRRequestorNotFound = SRError{ + Cause: errors.New("requesting site not found in site replication config"), + Code: ErrSiteReplicationInvalidRequest, + } + errSRNotEnabled = SRError{ + Cause: errors.New("site replication is not enabled"), + Code: ErrSiteReplicationInvalidRequest, + } + errSRResyncStarted = SRError{ + Cause: errors.New("site replication resync is already in progress"), + Code: ErrSiteReplicationInvalidRequest, + } + errSRResyncCanceled = SRError{ + Cause: errors.New("site replication resync is already canceled"), + Code: ErrSiteReplicationInvalidRequest, + } + errSRNoResync = SRError{ + Cause: errors.New("no resync in progress"), + Code: ErrSiteReplicationInvalidRequest, + } + errSRResyncToSelf = SRError{ + Cause: errors.New("invalid peer specified - cannot resync to self"), + Code: ErrSiteReplicationInvalidRequest, + } +) + +func errSRInvalidRequest(err error) SRError { + return SRError{ + Cause: err, + Code: ErrSiteReplicationInvalidRequest, + } +} + +func errSRPeerResp(err error) SRError { + return SRError{ + Cause: err, + Code: ErrSiteReplicationPeerResp, + } +} + +func errSRBackendIssue(err error) SRError { + return SRError{ + Cause: err, + Code: ErrSiteReplicationBackendIssue, + } +} + +func errSRServiceAccount(err error) SRError { + return SRError{ + Cause: err, + Code: ErrSiteReplicationServiceAccountError, + } +} + +func errSRBucketConfigError(err error) SRError { + return SRError{ + Cause: err, + Code: ErrSiteReplicationBucketConfigError, + } +} + +func errSRBucketMetaError(err error) SRError { + return SRError{ + Cause: err, + Code: ErrSiteReplicationBucketMetaError, + } +} + +func errSRIAMError(err error) SRError { + return SRError{ + Cause: err, + Code: ErrSiteReplicationIAMError, + } +} + +func errSRConfigMissingError(err error) SRError { + return SRError{ + Cause: err, + Code: ErrSiteReplicationConfigMissing, + } +} + +func errSRIAMConfigMismatch(peer1, peer2 string, s1, s2 madmin.IDPSettings) SRError { + return SRError{ + Cause: fmt.Errorf("IAM/IDP settings mismatch between %s and %s: %#v vs %#v", peer1, peer2, s1, s2), + Code: ErrSiteReplicationIAMConfigMismatch, + } +} + +var errSRObjectLayerNotReady = SRError{ + Cause: fmt.Errorf("object layer not ready"), + Code: ErrServerNotInitialized, +} + +func getSRStateFilePath() string { + return srStatePrefix + SlashSeparator + srStateFile +} + +// SRError - wrapped error for site replication. +type SRError struct { + Cause error + Code APIErrorCode +} + +func (c SRError) Error() string { + if c.Cause != nil { + return c.Cause.Error() + } + return "" +} + +func (c SRError) Unwrap() error { + return c.Cause +} + +func wrapSRErr(err error) SRError { + return SRError{Cause: err, Code: ErrInternalError} +} + +// SiteReplicationSys - manages cluster-level replication. +type SiteReplicationSys struct { + sync.RWMutex + + enabled bool + + // In-memory and persisted multi-site replication state. + state srState + + iamMetaCache srIAMCache +} + +type srState srStateV1 + +// srStateV1 represents version 1 of the site replication state persistence +// format. +type srStateV1 struct { + Name string `json:"name"` + + // Peers maps peers by their deploymentID + Peers map[string]madmin.PeerInfo `json:"peers"` + ServiceAccountAccessKey string `json:"serviceAccountAccessKey"` + UpdatedAt time.Time `json:"updatedAt"` +} + +// srStateData represents the format of the current `srStateFile`. +type srStateData struct { + Version int `json:"version"` + + SRState srStateV1 `json:"srState"` +} + +// Init - initialize the site replication manager. +func (c *SiteReplicationSys) Init(ctx context.Context, objAPI ObjectLayer) error { + go c.startHealRoutine(ctx, objAPI) + r := rand.New(rand.NewSource(time.Now().UnixNano())) + for { + err := c.loadFromDisk(ctx, objAPI) + if err == errConfigNotFound { + return nil + } + if err == nil { + break + } + replLogOnceIf(context.Background(), fmt.Errorf("unable to initialize site replication subsystem: (%w)", err), "site-relication-init") + + duration := max(time.Duration(r.Float64()*float64(time.Minute)), + // Make sure to sleep at least a second to avoid high CPU ticks. + time.Second) + time.Sleep(duration) + } + c.RLock() + defer c.RUnlock() + if c.enabled { + logger.Info("Cluster replication initialized") + } + return nil +} + +func (c *SiteReplicationSys) loadFromDisk(ctx context.Context, objAPI ObjectLayer) error { + buf, err := readConfig(ctx, objAPI, getSRStateFilePath()) + if err != nil { + if errors.Is(err, errConfigNotFound) { + c.Lock() + defer c.Unlock() + c.state = srState{} + c.enabled = false + } + return err + } + + // attempt to read just the version key in the state file to ensure we + // are reading a compatible version. + var ver struct { + Version int `json:"version"` + } + err = json.Unmarshal(buf, &ver) + if err != nil { + return err + } + if ver.Version != srStateFormatVersion1 { + return fmt.Errorf("Unexpected ClusterRepl state version: %d", ver.Version) + } + + var sdata srStateData + err = json.Unmarshal(buf, &sdata) + if err != nil { + return err + } + + c.Lock() + defer c.Unlock() + c.state = srState(sdata.SRState) + c.enabled = len(c.state.Peers) != 0 + return nil +} + +func (c *SiteReplicationSys) saveToDisk(ctx context.Context, state srState) error { + sdata := srStateData{ + Version: srStateFormatVersion1, + SRState: srStateV1(state), + } + buf, err := json.Marshal(sdata) + if err != nil { + return err + } + + objAPI := newObjectLayerFn() + if objAPI == nil { + return errServerNotInitialized + } + + if err = saveConfig(ctx, objAPI, getSRStateFilePath(), buf); err != nil { + return err + } + + for _, err := range globalNotificationSys.ReloadSiteReplicationConfig(ctx) { + replLogIf(ctx, err) + } + + c.Lock() + defer c.Unlock() + c.state = state + c.enabled = len(c.state.Peers) != 0 + return nil +} + +func (c *SiteReplicationSys) removeFromDisk(ctx context.Context) error { + objAPI := newObjectLayerFn() + if objAPI == nil { + return errServerNotInitialized + } + + if err := deleteConfig(ctx, objAPI, getSRStateFilePath()); err != nil { + return err + } + + for _, err := range globalNotificationSys.ReloadSiteReplicationConfig(ctx) { + replLogIf(ctx, err) + } + + c.Lock() + defer c.Unlock() + c.state = srState{} + c.enabled = false + return nil +} + +const ( + // Access key of service account used for perform cluster-replication + // operations. + siteReplicatorSvcAcc = "site-replicator-0" +) + +// PeerSiteInfo is a wrapper struct around madmin.PeerSite with extra info on site status +type PeerSiteInfo struct { + madmin.PeerSite + self bool + DeploymentID string + Replicated bool // true if already participating in site replication + Empty bool // true if cluster has no buckets +} + +// getSiteStatuses gathers more info on the sites being added +func (c *SiteReplicationSys) getSiteStatuses(ctx context.Context, sites ...madmin.PeerSite) (psi []PeerSiteInfo, err error) { + psi = make([]PeerSiteInfo, 0, len(sites)) + for _, v := range sites { + admClient, err := getAdminClient(v.Endpoint, v.AccessKey, v.SecretKey) + if err != nil { + return psi, errSRPeerResp(fmt.Errorf("unable to create admin client for %s: %w", v.Name, err)) + } + + info, err := admClient.ServerInfo(ctx) + if err != nil { + return psi, errSRPeerResp(fmt.Errorf("unable to fetch server info for %s: %w", v.Name, err)) + } + + s3Client, err := getS3Client(v) + if err != nil { + return psi, errSRPeerResp(fmt.Errorf("unable to create s3 client for %s: %w", v.Name, err)) + } + + buckets, err := s3Client.ListBuckets(ctx) + if err != nil { + return psi, errSRPeerResp(fmt.Errorf("unable to list buckets for %s: %v", v.Name, err)) + } + + psi = append(psi, PeerSiteInfo{ + PeerSite: v, + DeploymentID: info.DeploymentID, + Empty: len(buckets) == 0, + self: info.DeploymentID == globalDeploymentID(), + }) + } + return psi, err +} + +// AddPeerClusters - add cluster sites for replication configuration. +func (c *SiteReplicationSys) AddPeerClusters(ctx context.Context, psites []madmin.PeerSite, opts madmin.SRAddOptions) (madmin.ReplicateAddStatus, error) { + sites, serr := c.getSiteStatuses(ctx, psites...) + if serr != nil { + return madmin.ReplicateAddStatus{}, serr + } + var ( + currSites madmin.SiteReplicationInfo + currDeploymentIDsSet = set.NewStringSet() + err error + ) + currSites, err = c.GetClusterInfo(ctx) + if err != nil { + return madmin.ReplicateAddStatus{}, errSRBackendIssue(err) + } + for _, v := range currSites.Sites { + currDeploymentIDsSet.Add(v.DeploymentID) + } + deploymentIDsSet := set.NewStringSet() + localHasBuckets := false + nonLocalPeerWithBuckets := "" + selfIdx := -1 + for i, v := range sites { + // deploymentIDs must be unique + if deploymentIDsSet.Contains(v.DeploymentID) { + return madmin.ReplicateAddStatus{}, errSRDuplicateSites + } + deploymentIDsSet.Add(v.DeploymentID) + + if v.self { + selfIdx = i + localHasBuckets = !v.Empty + continue + } + if !v.Empty && !currDeploymentIDsSet.Contains(v.DeploymentID) { + nonLocalPeerWithBuckets = v.Name + } + } + if selfIdx == -1 { + return madmin.ReplicateAddStatus{}, errSRBackendIssue(fmt.Errorf("global deployment ID %s mismatch, expected one of %s", globalDeploymentID(), deploymentIDsSet)) + } + if !currDeploymentIDsSet.IsEmpty() { + // If current cluster is already SR enabled and no new site being added ,fail. + if currDeploymentIDsSet.Equals(deploymentIDsSet) { + return madmin.ReplicateAddStatus{}, errSRCannotJoin + } + if len(currDeploymentIDsSet.Intersection(deploymentIDsSet)) != len(currDeploymentIDsSet) { + diffSlc := getMissingSiteNames(currDeploymentIDsSet, deploymentIDsSet, currSites.Sites) + return madmin.ReplicateAddStatus{}, errSRInvalidRequest(fmt.Errorf("all existing replicated sites must be specified - missing %s", strings.Join(diffSlc, " "))) + } + } + + // validate that all clusters are using the same IDP settings. + err = c.validateIDPSettings(ctx, sites) + if err != nil { + return madmin.ReplicateAddStatus{}, err + } + + // For this `add` API, either all clusters must be empty or the local + // cluster must be the only one having some buckets. + if localHasBuckets && nonLocalPeerWithBuckets != "" { + return madmin.ReplicateAddStatus{}, errSRInvalidRequest(errors.New("only one cluster may have data when configuring site replication")) + } + + if !localHasBuckets && nonLocalPeerWithBuckets != "" { + return madmin.ReplicateAddStatus{}, errSRInvalidRequest(fmt.Errorf("please send your request to the cluster containing data/buckets: %s", nonLocalPeerWithBuckets)) + } + + // FIXME: Ideally, we also need to check if there are any global IAM + // policies and any (LDAP user created) service accounts on the other + // peer clusters, and if so, reject the cluster replicate add request. + // This is not yet implemented. + + // VALIDATIONS COMPLETE. + + // Create a common service account for all clusters, with root + // permissions. + + // Create a local service account. + + // Generate a secret key for the service account if not created already. + var secretKey string + var svcCred auth.Credentials + sa, _, err := globalIAMSys.getServiceAccount(ctx, siteReplicatorSvcAcc) + switch err { + case errNoSuchServiceAccount: + _, secretKey, err = auth.GenerateCredentials() + if err != nil { + return madmin.ReplicateAddStatus{}, errSRServiceAccount(fmt.Errorf("unable to create local service account: %w", err)) + } + svcCred, _, err = globalIAMSys.NewServiceAccount(ctx, sites[selfIdx].AccessKey, nil, newServiceAccountOpts{ + accessKey: siteReplicatorSvcAcc, + secretKey: secretKey, + allowSiteReplicatorAccount: true, + }) + if err != nil { + return madmin.ReplicateAddStatus{}, errSRServiceAccount(fmt.Errorf("unable to create local service account: %w", err)) + } + case nil: + svcCred = sa.Credentials + secretKey = svcCred.SecretKey + default: + return madmin.ReplicateAddStatus{}, errSRBackendIssue(err) + } + + currTime := time.Now() + joinReq := madmin.SRPeerJoinReq{ + SvcAcctAccessKey: svcCred.AccessKey, + SvcAcctSecretKey: secretKey, + Peers: make(map[string]madmin.PeerInfo), + UpdatedAt: currTime, + } + // check if few peers exist already and ILM expiry replcation is set to true + replicateILMExpirySet := false + if c.state.Peers != nil { + for _, pi := range c.state.Peers { + if pi.ReplicateILMExpiry { + replicateILMExpirySet = true + break + } + } + } + for _, v := range sites { + var peerReplicateILMExpiry bool + // if peers already exist and for one of them ReplicateILMExpiry + // set true, that means earlier replication of ILM expiry was set + // for the site replication. All new sites added to the setup should + // get this enabled as well + if replicateILMExpirySet { + peerReplicateILMExpiry = replicateILMExpirySet + } else { + peerReplicateILMExpiry = opts.ReplicateILMExpiry + } + joinReq.Peers[v.DeploymentID] = madmin.PeerInfo{ + Endpoint: v.Endpoint, + Name: v.Name, + DeploymentID: v.DeploymentID, + ReplicateILMExpiry: peerReplicateILMExpiry, + } + } + + addedCount := 0 + var ( + peerAddErr error + admClient *madmin.AdminClient + ) + + for _, v := range sites { + if v.self { + continue + } + switch { + case currDeploymentIDsSet.Contains(v.DeploymentID): + admClient, err = c.getAdminClient(ctx, v.DeploymentID) + default: + admClient, err = getAdminClient(v.Endpoint, v.AccessKey, v.SecretKey) + } + if err != nil { + peerAddErr = errSRPeerResp(fmt.Errorf("unable to create admin client for %s: %w", v.Name, err)) + break + } + joinReq.SvcAcctParent = v.AccessKey + err = admClient.SRPeerJoin(ctx, joinReq) + if err != nil { + peerAddErr = errSRPeerResp(fmt.Errorf("unable to link with peer %s: %w", v.Name, err)) + break + } + addedCount++ + } + + if peerAddErr != nil { + if addedCount == 0 { + return madmin.ReplicateAddStatus{}, peerAddErr + } + // In this case, it means at least one cluster was added + // successfully, we need to send a response to the client with + // some details - FIXME: the disks on this cluster would need to + // be cleaned to recover. + partial := madmin.ReplicateAddStatus{ + Status: madmin.ReplicateAddStatusPartial, + ErrDetail: peerAddErr.Error(), + } + + return partial, nil + } + + // Other than handling existing buckets, we can now save the cluster + // replication configuration state. + state := srState{ + Name: sites[selfIdx].Name, + Peers: joinReq.Peers, + ServiceAccountAccessKey: svcCred.AccessKey, + UpdatedAt: currTime, + } + + if err = c.saveToDisk(ctx, state); err != nil { + return madmin.ReplicateAddStatus{ + Status: madmin.ReplicateAddStatusPartial, + ErrDetail: fmt.Sprintf("unable to save cluster-replication state on local: %v", err), + }, nil + } + + if !globalSiteReplicatorCred.IsValid() { + globalSiteReplicatorCred.Set(svcCred.SecretKey) + } + result := madmin.ReplicateAddStatus{ + Success: true, + Status: madmin.ReplicateAddStatusSuccess, + } + + if err := c.syncToAllPeers(ctx, opts); err != nil { + result.InitialSyncErrorMessage = err.Error() + } + + return result, nil +} + +// PeerJoinReq - internal API handler to respond to a peer cluster's request to join. +func (c *SiteReplicationSys) PeerJoinReq(ctx context.Context, arg madmin.SRPeerJoinReq) error { + var ourName string + for d, p := range arg.Peers { + if d == globalDeploymentID() { + ourName = p.Name + break + } + } + if ourName == "" { + return errSRSelfNotFound + } + + sa, _, err := globalIAMSys.GetServiceAccount(ctx, arg.SvcAcctAccessKey) + if err == errNoSuchServiceAccount { + sa, _, err = globalIAMSys.NewServiceAccount(ctx, arg.SvcAcctParent, nil, newServiceAccountOpts{ + accessKey: arg.SvcAcctAccessKey, + secretKey: arg.SvcAcctSecretKey, + allowSiteReplicatorAccount: arg.SvcAcctAccessKey == siteReplicatorSvcAcc, + }) + } + if err != nil { + return errSRServiceAccount(fmt.Errorf("unable to create service account on %s: %v", ourName, err)) + } + + peers := make(map[string]madmin.PeerInfo, len(arg.Peers)) + for dID, pi := range arg.Peers { + if c.state.Peers != nil { + if existingPeer, ok := c.state.Peers[dID]; ok { + // retain existing ReplicateILMExpiry of peer if its already set + // and incoming arg has it false. it could be default false + if !pi.ReplicateILMExpiry && existingPeer.ReplicateILMExpiry { + pi.ReplicateILMExpiry = existingPeer.ReplicateILMExpiry + } + } + } + peers[dID] = pi + } + state := srState{ + Name: ourName, + Peers: peers, + ServiceAccountAccessKey: arg.SvcAcctAccessKey, + UpdatedAt: arg.UpdatedAt, + } + if err = c.saveToDisk(ctx, state); err != nil { + return errSRBackendIssue(fmt.Errorf("unable to save cluster-replication state to drive on %s: %v", ourName, err)) + } + if !globalSiteReplicatorCred.IsValid() { + globalSiteReplicatorCred.Set(sa.SecretKey) + } + + return nil +} + +// GetIDPSettings returns info about the configured identity provider. It is +// used to validate that all peers have the same IDP. +func (c *SiteReplicationSys) GetIDPSettings(ctx context.Context) madmin.IDPSettings { + s := madmin.IDPSettings{} + s.LDAP = madmin.LDAPSettings{ + IsLDAPEnabled: globalIAMSys.LDAPConfig.Enabled(), + LDAPUserDNSearchBase: globalIAMSys.LDAPConfig.LDAP.UserDNSearchBaseDistName, + LDAPUserDNSearchFilter: globalIAMSys.LDAPConfig.LDAP.UserDNSearchFilter, + LDAPGroupSearchBase: globalIAMSys.LDAPConfig.LDAP.GroupSearchBaseDistName, + LDAPGroupSearchFilter: globalIAMSys.LDAPConfig.LDAP.GroupSearchFilter, + } + s.OpenID = globalIAMSys.OpenIDConfig.GetSettings() + if s.OpenID.Enabled { + s.OpenID.Region = globalSite.Region() + } + return s +} + +func (c *SiteReplicationSys) validateIDPSettings(ctx context.Context, peers []PeerSiteInfo) error { + s := make([]madmin.IDPSettings, 0, len(peers)) + for _, v := range peers { + if v.self { + s = append(s, c.GetIDPSettings(ctx)) + continue + } + + admClient, err := getAdminClient(v.Endpoint, v.AccessKey, v.SecretKey) + if err != nil { + return errSRPeerResp(fmt.Errorf("unable to create admin client for %s: %w", v.Name, err)) + } + + is, err := admClient.SRPeerGetIDPSettings(ctx) + if err != nil { + return errSRPeerResp(fmt.Errorf("unable to fetch IDP settings from %s: %v", v.Name, err)) + } + s = append(s, is) + } + + for i := 1; i < len(s); i++ { + if !reflect.DeepEqual(s[i], s[0]) { + return errSRIAMConfigMismatch(peers[0].Name, peers[i].Name, s[0], s[i]) + } + } + + return nil +} + +// Netperf for site-replication net perf +func (c *SiteReplicationSys) Netperf(ctx context.Context, duration time.Duration) (results madmin.SiteNetPerfResult, err error) { + infos, err := globalSiteReplicationSys.GetClusterInfo(ctx) + if err != nil { + return results, err + } + var wg sync.WaitGroup + var resultsMu sync.RWMutex + for _, info := range infos.Sites { + // will call siteNetperf, means call others's adminAPISiteReplicationDevNull + if globalDeploymentID() == info.DeploymentID { + wg.Add(1) + go func() { + defer wg.Done() + result := madmin.SiteNetPerfNodeResult{} + cli, err := globalSiteReplicationSys.getAdminClient(ctx, info.DeploymentID) + if err != nil { + result.Error = err.Error() + } else { + result = siteNetperf(ctx, duration) + result.Endpoint = cli.GetEndpointURL().String() + } + resultsMu.Lock() + results.NodeResults = append(results.NodeResults, result) + resultsMu.Unlock() + }() + continue + } + wg.Add(1) + go func() { + defer wg.Done() + ctx, cancel := context.WithTimeout(ctx, duration+10*time.Second) + defer cancel() + result := perfNetRequest( + ctx, + info.DeploymentID, + adminPathPrefix+adminAPIVersionPrefix+adminAPISiteReplicationNetPerf, + nil, + ) + resultsMu.Lock() + results.NodeResults = append(results.NodeResults, result) + resultsMu.Unlock() + }() + } + wg.Wait() + return results, err +} + +// GetClusterInfo - returns site replication information. +func (c *SiteReplicationSys) GetClusterInfo(ctx context.Context) (info madmin.SiteReplicationInfo, err error) { + c.RLock() + defer c.RUnlock() + if !c.enabled { + return info, nil + } + + info.Enabled = true + info.Name = c.state.Name + info.Sites = make([]madmin.PeerInfo, 0, len(c.state.Peers)) + for _, peer := range c.state.Peers { + info.Sites = append(info.Sites, peer) + } + sort.Slice(info.Sites, func(i, j int) bool { + return info.Sites[i].Name < info.Sites[j].Name + }) + + info.ServiceAccountAccessKey = c.state.ServiceAccountAccessKey + return info, nil +} + +const ( + makeBucketWithVersion = "MakeBucketWithVersioning" + configureReplication = "ConfigureReplication" + deleteBucket = "DeleteBucket" + replicateIAMItem = "SRPeerReplicateIAMItem" + replicateBucketMetadata = "SRPeerReplicateBucketMeta" + siteReplicationEdit = "SiteReplicationEdit" +) + +// MakeBucketHook - called during a regular make bucket call when cluster +// replication is enabled. It is responsible for the creation of the same bucket +// on remote clusters, and creating replication rules on local and peer +// clusters. +func (c *SiteReplicationSys) MakeBucketHook(ctx context.Context, bucket string, opts MakeBucketOptions) error { + // At this point, the local bucket is created. + + c.RLock() + defer c.RUnlock() + if !c.enabled { + return nil + } + + optsMap := make(map[string]string) + if opts.LockEnabled { + optsMap["lockEnabled"] = "true" + optsMap["versioningEnabled"] = "true" + } + if opts.VersioningEnabled { + optsMap["versioningEnabled"] = "true" + } + if opts.ForceCreate { + optsMap["forceCreate"] = "true" + } + createdAt, _ := globalBucketMetadataSys.CreatedAt(bucket) + optsMap["createdAt"] = createdAt.UTC().Format(time.RFC3339Nano) + opts.CreatedAt = createdAt + + // Create bucket and enable versioning on all peers. + makeBucketConcErr := c.concDo( + func() error { + return c.annotateErr(makeBucketWithVersion, c.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts)) + }, + func(deploymentID string, p madmin.PeerInfo) error { + admClient, err := c.getAdminClient(ctx, deploymentID) + if err != nil { + return err + } + + return c.annotatePeerErr(p.Name, makeBucketWithVersion, admClient.SRPeerBucketOps(ctx, bucket, madmin.MakeWithVersioningBktOp, optsMap)) + }, + makeBucketWithVersion, + ) + + // Create bucket remotes and add replication rules for the bucket on self and peers. + makeRemotesConcErr := c.concDo( + func() error { + return c.annotateErr(configureReplication, c.PeerBucketConfigureReplHandler(ctx, bucket)) + }, + func(deploymentID string, p madmin.PeerInfo) error { + admClient, err := c.getAdminClient(ctx, deploymentID) + if err != nil { + return err + } + + return c.annotatePeerErr(p.Name, configureReplication, admClient.SRPeerBucketOps(ctx, bucket, madmin.ConfigureReplBktOp, nil)) + }, + configureReplication, + ) + + if err := errors.Unwrap(makeBucketConcErr); err != nil { + return err + } + return errors.Unwrap(makeRemotesConcErr) +} + +// DeleteBucketHook - called during a regular delete bucket call when cluster +// replication is enabled. It is responsible for the deletion of the same bucket +// on remote clusters. +func (c *SiteReplicationSys) DeleteBucketHook(ctx context.Context, bucket string, forceDelete bool) error { + // At this point, the local bucket is deleted. + + c.RLock() + defer c.RUnlock() + if !c.enabled { + return nil + } + + op := madmin.DeleteBucketBktOp + if forceDelete { + op = madmin.ForceDeleteBucketBktOp + } + + // Send bucket delete to other clusters. + cerr := c.concDo(nil, func(deploymentID string, p madmin.PeerInfo) error { + admClient, err := c.getAdminClient(ctx, deploymentID) + if err != nil { + return wrapSRErr(err) + } + + return c.annotatePeerErr(p.Name, deleteBucket, admClient.SRPeerBucketOps(ctx, bucket, op, nil)) + }, + deleteBucket, + ) + return errors.Unwrap(cerr) +} + +// PeerBucketMakeWithVersioningHandler - creates bucket and enables versioning. +func (c *SiteReplicationSys) PeerBucketMakeWithVersioningHandler(ctx context.Context, bucket string, opts MakeBucketOptions) error { + objAPI := newObjectLayerFn() + if objAPI == nil { + return errServerNotInitialized + } + + err := objAPI.MakeBucket(ctx, bucket, opts) + if err != nil { + // Check if this is a bucket exists error. + _, ok1 := err.(BucketExists) + _, ok2 := err.(BucketAlreadyExists) + if !ok1 && !ok2 { + return wrapSRErr(c.annotateErr(makeBucketWithVersion, err)) + } + } else { + // Load updated bucket metadata into memory as new + // bucket was created. + globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket) + } + + meta, err := globalBucketMetadataSys.Get(bucket) + if err != nil { + return wrapSRErr(c.annotateErr(makeBucketWithVersion, err)) + } + + meta.SetCreatedAt(opts.CreatedAt) + + meta.VersioningConfigXML = enabledBucketVersioningConfig + if opts.LockEnabled { + meta.ObjectLockConfigXML = enabledBucketObjectLockConfig + } + + if err := meta.Save(context.Background(), objAPI); err != nil { + return wrapSRErr(err) + } + + globalBucketMetadataSys.Set(bucket, meta) + + // Load updated bucket metadata into memory as new metadata updated. + globalNotificationSys.LoadBucketMetadata(GlobalContext, bucket) + return nil +} + +// PeerBucketConfigureReplHandler - configures replication remote and +// replication rules to all other peers for the local bucket. +func (c *SiteReplicationSys) PeerBucketConfigureReplHandler(ctx context.Context, bucket string) error { + creds, err := c.getPeerCreds() + if err != nil { + return wrapSRErr(err) + } + + // The following function, creates a bucket remote and sets up a bucket + // replication rule for the given peer. + configurePeerFn := func(d string, peer madmin.PeerInfo) error { + // Create bucket replication rule to this peer. + + // To add the bucket replication rule, we fetch the current + // server configuration, and convert it to minio-go's + // replication configuration type (by converting to xml and + // parsing it back), use minio-go's add rule function, and + // finally convert it back to the server type (again via xml). + // This is needed as there is no add-rule function in the server + // yet. + + // Though we do not check if the rule already exists, this is + // not a problem as we are always using the same replication + // rule ID - if the rule already exists, it is just replaced. + replicationConfigS, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket) + if err != nil { + _, ok := err.(BucketReplicationConfigNotFound) + if !ok { + return err + } + } + var replicationConfig replication.Config + if replicationConfigS != nil { + replCfgSBytes, err := xml.Marshal(replicationConfigS) + if err != nil { + return err + } + err = xml.Unmarshal(replCfgSBytes, &replicationConfig) + if err != nil { + return err + } + } + var ( + ruleID = fmt.Sprintf("site-repl-%s", d) + hasRule bool + ) + var ruleARN string + for _, r := range replicationConfig.Rules { + if r.ID == ruleID { + hasRule = true + ruleARN = r.Destination.Bucket + } + } + + ep, _ := url.Parse(peer.Endpoint) + var targets []madmin.BucketTarget + if targetsPtr, _ := globalBucketTargetSys.ListBucketTargets(ctx, bucket); targetsPtr != nil { + targets = targetsPtr.Targets + } + targetARN := "" + var updateTgt, updateBW bool + var targetToUpdate madmin.BucketTarget + for _, target := range targets { + if target.Arn == ruleARN { + targetARN = ruleARN + updateBW = peer.DefaultBandwidth.Limit != 0 && target.BandwidthLimit == 0 + if (target.URL().String() != peer.Endpoint) || updateBW { + updateTgt = true + targetToUpdate = target + } + break + } + } + // replication config had a stale target ARN - update the endpoint + if updateTgt { + targetToUpdate.Endpoint = ep.Host + targetToUpdate.Secure = ep.Scheme == "https" + targetToUpdate.Credentials = &madmin.Credentials{ + AccessKey: creds.AccessKey, + SecretKey: creds.SecretKey, + } + if !peer.SyncState.Empty() { + targetToUpdate.ReplicationSync = (peer.SyncState == madmin.SyncEnabled) + } + if updateBW { + targetToUpdate.BandwidthLimit = int64(peer.DefaultBandwidth.Limit) + } + err := globalBucketTargetSys.SetTarget(ctx, bucket, &targetToUpdate, true) + if err != nil { + return c.annotatePeerErr(peer.Name, "Bucket target update error", err) + } + targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket) + if err != nil { + return wrapSRErr(err) + } + tgtBytes, err := json.Marshal(&targets) + if err != nil { + return wrapSRErr(err) + } + if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil { + return wrapSRErr(err) + } + } + // no replication rule for this peer or target ARN missing in bucket targets + if targetARN == "" { + bucketTarget := madmin.BucketTarget{ + SourceBucket: bucket, + Endpoint: ep.Host, + Credentials: &madmin.Credentials{ + AccessKey: creds.AccessKey, + SecretKey: creds.SecretKey, + }, + TargetBucket: bucket, + Secure: ep.Scheme == "https", + API: "s3v4", + Type: madmin.ReplicationService, + Region: "", + ReplicationSync: peer.SyncState == madmin.SyncEnabled, + DeploymentID: d, + BandwidthLimit: int64(peer.DefaultBandwidth.Limit), + } + var exists bool // true if ARN already exists + bucketTarget.Arn, exists = globalBucketTargetSys.getRemoteARN(bucket, &bucketTarget, peer.DeploymentID) + if !exists { // persist newly generated ARN to targets and metadata on disk + err := globalBucketTargetSys.SetTarget(ctx, bucket, &bucketTarget, false) + if err != nil { + return c.annotatePeerErr(peer.Name, "Bucket target creation error", err) + } + targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket) + if err != nil { + return err + } + tgtBytes, err := json.Marshal(&targets) + if err != nil { + return err + } + if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil { + return err + } + } + targetARN = bucketTarget.Arn + } + opts := replication.Options{ + // Set the ID so we can identify the rule as being + // created for site-replication and include the + // destination cluster's deployment ID. + ID: ruleID, + + // Use a helper to generate unique priority numbers. + Priority: fmt.Sprintf("%d", getPriorityHelper(replicationConfig)), + + Op: replication.AddOption, + RuleStatus: "enable", + DestBucket: targetARN, + // Replicate everything! + ReplicateDeletes: "enable", + ReplicateDeleteMarkers: "enable", + ReplicaSync: "enable", + ExistingObjectReplicate: "enable", + } + + switch { + case hasRule: + if ruleARN != opts.DestBucket { + // remove stale replication rule and replace rule with correct target ARN + if len(replicationConfig.Rules) > 1 { + err = replicationConfig.RemoveRule(opts) + } else { + replicationConfig = replication.Config{} + } + if err == nil { + err = replicationConfig.AddRule(opts) + } + } else { + err = replicationConfig.EditRule(opts) + } + default: + err = replicationConfig.AddRule(opts) + } + if err != nil { + return c.annotatePeerErr(peer.Name, "Error adding bucket replication rule", err) + } + + // Now convert the configuration back to server's type so we can + // do some validation. + newReplCfgBytes, err := xml.Marshal(replicationConfig) + if err != nil { + return err + } + newReplicationConfig, err := sreplication.ParseConfig(bytes.NewReader(newReplCfgBytes)) + if err != nil { + return err + } + sameTarget, apiErr := validateReplicationDestination(ctx, bucket, newReplicationConfig, &validateReplicationDestinationOptions{CheckRemoteBucket: true}) + if apiErr != noError { + return fmt.Errorf("bucket replication config validation error: %#v", apiErr) + } + err = newReplicationConfig.Validate(bucket, sameTarget) + if err != nil { + return err + } + // Config looks good, so we save it. + replCfgData, err := xml.Marshal(newReplicationConfig) + if err != nil { + return err + } + + _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketReplicationConfig, replCfgData) + return c.annotatePeerErr(peer.Name, "Error updating replication configuration", err) + } + + c.RLock() + defer c.RUnlock() + errMap := make(map[string]error, len(c.state.Peers)) + for d, peer := range c.state.Peers { + if d == globalDeploymentID() { + continue + } + errMap[d] = configurePeerFn(d, peer) + } + return c.toErrorFromErrMap(errMap, configureReplication) +} + +// PeerBucketDeleteHandler - deletes bucket on local in response to a delete +// bucket request from a peer. +func (c *SiteReplicationSys) PeerBucketDeleteHandler(ctx context.Context, bucket string, opts DeleteBucketOptions) error { + c.RLock() + defer c.RUnlock() + if !c.enabled { + return errSRNotEnabled + } + + objAPI := newObjectLayerFn() + if objAPI == nil { + return errServerNotInitialized + } + + if globalDNSConfig != nil { + if err := globalDNSConfig.Delete(bucket); err != nil { + return err + } + } + err := objAPI.DeleteBucket(ctx, bucket, opts) + if err != nil { + if globalDNSConfig != nil { + if err2 := globalDNSConfig.Put(bucket); err2 != nil { + replLogIf(ctx, fmt.Errorf("Unable to restore bucket DNS entry %w, please fix it manually", err2)) + } + } + return err + } + + globalNotificationSys.DeleteBucketMetadata(ctx, bucket) + + return nil +} + +// IAMChangeHook - called when IAM items need to be replicated to peer clusters. +// This includes named policy creation, policy mapping changes and service +// account changes. +// +// All policies are replicated. +// +// Policy mappings are only replicated when they are for LDAP users or groups +// (as an external IDP is always assumed when SR is used). In the case of +// OpenID, such mappings are provided from the IDP directly and so are not +// applicable here. +// +// Service accounts are replicated as long as they are not meant for the root +// user. +// +// STS accounts are replicated, but only if the session token is verifiable +// using the local cluster's root credential. +func (c *SiteReplicationSys) IAMChangeHook(ctx context.Context, item madmin.SRIAMItem) error { + // The IAM item has already been applied to the local cluster at this + // point, and only needs to be updated on all remote peer clusters. + + c.RLock() + defer c.RUnlock() + if !c.enabled { + return nil + } + + cerr := c.concDo(nil, func(d string, p madmin.PeerInfo) error { + admClient, err := c.getAdminClient(ctx, d) + if err != nil { + return wrapSRErr(err) + } + + return c.annotatePeerErr(p.Name, replicateIAMItem, admClient.SRPeerReplicateIAMItem(ctx, item)) + }, + replicateIAMItem, + ) + return errors.Unwrap(cerr) +} + +// PeerAddPolicyHandler - copies IAM policy to local. A nil policy argument, +// causes the named policy to be deleted. +func (c *SiteReplicationSys) PeerAddPolicyHandler(ctx context.Context, policyName string, p *policy.Policy, updatedAt time.Time) error { + var err error + // skip overwrite of local update if peer sent stale info + if !updatedAt.IsZero() { + if p, err := globalIAMSys.store.GetPolicyDoc(policyName); err == nil && p.UpdateDate.After(updatedAt) { + return nil + } + } + if p == nil { + err = globalIAMSys.DeletePolicy(ctx, policyName, true) + } else { + _, err = globalIAMSys.SetPolicy(ctx, policyName, *p) + } + if err != nil { + return wrapSRErr(err) + } + return nil +} + +// PeerIAMUserChangeHandler - copies IAM user to local. +func (c *SiteReplicationSys) PeerIAMUserChangeHandler(ctx context.Context, change *madmin.SRIAMUser, updatedAt time.Time) error { + if change == nil { + return errSRInvalidRequest(errInvalidArgument) + } + // skip overwrite of local update if peer sent stale info + if !updatedAt.IsZero() { + if ui, err := globalIAMSys.GetUserInfo(ctx, change.AccessKey); err == nil && ui.UpdatedAt.After(updatedAt) { + return nil + } + } + + var err error + if change.IsDeleteReq { + err = globalIAMSys.DeleteUser(ctx, change.AccessKey, true) + } else { + if change.UserReq == nil { + return errSRInvalidRequest(errInvalidArgument) + } + userReq := *change.UserReq + if userReq.Status != "" && userReq.SecretKey == "" { + // Status is set without secretKey updates means we are + // only changing the account status. + _, err = globalIAMSys.SetUserStatus(ctx, change.AccessKey, userReq.Status) + } else { + // We don't allow internal user creation with LDAP enabled for now + // (both sites must have LDAP disabled). + if globalIAMSys.LDAPConfig.Enabled() { + err = errIAMActionNotAllowed + } else { + _, err = globalIAMSys.CreateUser(ctx, change.AccessKey, userReq) + } + } + } + if err != nil { + return wrapSRErr(err) + } + return nil +} + +// PeerGroupInfoChangeHandler - copies group changes to local. +func (c *SiteReplicationSys) PeerGroupInfoChangeHandler(ctx context.Context, change *madmin.SRGroupInfo, updatedAt time.Time) error { + if change == nil { + return errSRInvalidRequest(errInvalidArgument) + } + updReq := change.UpdateReq + var err error + + // skip overwrite of local update if peer sent stale info + if !updatedAt.IsZero() { + if gd, err := globalIAMSys.GetGroupDescription(updReq.Group); err == nil && gd.UpdatedAt.After(updatedAt) { + return nil + } + } + + if updReq.IsRemove { + _, err = globalIAMSys.RemoveUsersFromGroup(ctx, updReq.Group, updReq.Members) + } else { + if updReq.Status != "" && len(updReq.Members) == 0 { + _, err = globalIAMSys.SetGroupStatus(ctx, updReq.Group, updReq.Status == madmin.GroupEnabled) + } else { + if globalIAMSys.LDAPConfig.Enabled() { + // We don't allow internal group manipulation in this API when + // LDAP is enabled for now (both sites must have LDAP disabled). + err = errIAMActionNotAllowed + } else { + _, err = globalIAMSys.AddUsersToGroup(ctx, updReq.Group, updReq.Members) + } + if err == nil && updReq.Status != "" { + _, err = globalIAMSys.SetGroupStatus(ctx, updReq.Group, updReq.Status == madmin.GroupEnabled) + } + } + } + if err != nil && !errors.Is(err, errNoSuchGroup) { + return wrapSRErr(err) + } + return nil +} + +// PeerSvcAccChangeHandler - copies service-account change to local. +func (c *SiteReplicationSys) PeerSvcAccChangeHandler(ctx context.Context, change *madmin.SRSvcAccChange, updatedAt time.Time) error { + if change == nil { + return errSRInvalidRequest(errInvalidArgument) + } + switch { + case change.Create != nil: + var sp *policy.Policy + var err error + if len(change.Create.SessionPolicy) > 0 { + sp, err = policy.ParseConfig(bytes.NewReader(change.Create.SessionPolicy)) + if err != nil { + return wrapSRErr(err) + } + } + // skip overwrite of local update if peer sent stale info + if !updatedAt.IsZero() && change.Create.AccessKey != "" { + if sa, _, err := globalIAMSys.getServiceAccount(ctx, change.Create.AccessKey); err == nil && sa.UpdatedAt.After(updatedAt) { + return nil + } + } + opts := newServiceAccountOpts{ + accessKey: change.Create.AccessKey, + secretKey: change.Create.SecretKey, + sessionPolicy: sp, + claims: change.Create.Claims, + name: change.Create.Name, + description: change.Create.Description, + expiration: change.Create.Expiration, + } + _, _, err = globalIAMSys.NewServiceAccount(ctx, change.Create.Parent, change.Create.Groups, opts) + if err != nil { + return wrapSRErr(err) + } + + case change.Update != nil: + var sp *policy.Policy + var err error + if len(change.Update.SessionPolicy) > 0 { + sp, err = policy.ParseConfig(bytes.NewReader(change.Update.SessionPolicy)) + if err != nil { + return wrapSRErr(err) + } + } + // skip overwrite of local update if peer sent stale info + if !updatedAt.IsZero() { + if sa, _, err := globalIAMSys.getServiceAccount(ctx, change.Update.AccessKey); err == nil && sa.UpdatedAt.After(updatedAt) { + return nil + } + } + opts := updateServiceAccountOpts{ + secretKey: change.Update.SecretKey, + status: change.Update.Status, + name: change.Update.Name, + description: change.Update.Description, + sessionPolicy: sp, + expiration: change.Update.Expiration, + } + + _, err = globalIAMSys.UpdateServiceAccount(ctx, change.Update.AccessKey, opts) + if err != nil { + return wrapSRErr(err) + } + + case change.Delete != nil: + // skip overwrite of local update if peer sent stale info + if !updatedAt.IsZero() { + if sa, _, err := globalIAMSys.getServiceAccount(ctx, change.Delete.AccessKey); err == nil && sa.UpdatedAt.After(updatedAt) { + return nil + } + } + if err := globalIAMSys.DeleteServiceAccount(ctx, change.Delete.AccessKey, true); err != nil { + return wrapSRErr(err) + } + } + + return nil +} + +// PeerPolicyMappingHandler - copies policy mapping to local. +func (c *SiteReplicationSys) PeerPolicyMappingHandler(ctx context.Context, mapping *madmin.SRPolicyMapping, updatedAt time.Time) error { + if mapping == nil { + return errSRInvalidRequest(errInvalidArgument) + } + // skip overwrite of local update if peer sent stale info + if !updatedAt.IsZero() { + mp, ok := globalIAMSys.store.GetMappedPolicy(mapping.Policy, mapping.IsGroup) + if ok && mp.UpdatedAt.After(updatedAt) { + return nil + } + } + + // When LDAP is enabled, we verify that the user or group exists in LDAP and + // use the normalized form of the entityName (which will be an LDAP DN). + userType := IAMUserType(mapping.UserType) + isGroup := mapping.IsGroup + entityName := mapping.UserOrGroup + + if globalIAMSys.GetUsersSysType() == LDAPUsersSysType && userType == stsUser { + // Validate that the user or group exists in LDAP and use the normalized + // form of the entityName (which will be an LDAP DN). + var err error + if isGroup { + var foundGroupDN *xldap.DNSearchResult + var underBaseDN bool + if foundGroupDN, underBaseDN, err = globalIAMSys.LDAPConfig.GetValidatedGroupDN(nil, entityName); err != nil { + iamLogIf(ctx, err) + } else if foundGroupDN == nil || !underBaseDN { + return wrapSRErr(errNoSuchGroup) + } + entityName = foundGroupDN.NormDN + } else { + var foundUserDN *xldap.DNSearchResult + if foundUserDN, err = globalIAMSys.LDAPConfig.GetValidatedDNForUsername(entityName); err != nil { + iamLogIf(ctx, err) + } else if foundUserDN == nil { + return wrapSRErr(errNoSuchUser) + } + entityName = foundUserDN.NormDN + } + if err != nil { + return wrapSRErr(err) + } + } + + _, err := globalIAMSys.PolicyDBSet(ctx, entityName, mapping.Policy, userType, isGroup) + if err != nil { + return wrapSRErr(err) + } + return nil +} + +// PeerSTSAccHandler - replicates STS credential locally. +func (c *SiteReplicationSys) PeerSTSAccHandler(ctx context.Context, stsCred *madmin.SRSTSCredential, updatedAt time.Time) error { + if stsCred == nil { + return errSRInvalidRequest(errInvalidArgument) + } + // skip overwrite of local update if peer sent stale info + if !updatedAt.IsZero() { + if u, _, err := globalIAMSys.getTempAccount(ctx, stsCred.AccessKey); err == nil { + if u.UpdatedAt.After(updatedAt) { + return nil + } + } + } + secretKey, err := getTokenSigningKey() + if err != nil { + return errSRInvalidRequest(err) + } + + // Verify the session token of the stsCred + claims, err := auth.ExtractClaims(stsCred.SessionToken, secretKey) + if err != nil { + return fmt.Errorf("STS credential could not be verified: %w", err) + } + + mapClaims := claims.Map() + expiry, err := auth.ExpToInt64(mapClaims["exp"]) + if err != nil { + return fmt.Errorf("Expiry claim was not found: %v: %w", mapClaims, err) + } + + cred := auth.Credentials{ + AccessKey: stsCred.AccessKey, + SecretKey: stsCred.SecretKey, + Expiration: time.Unix(expiry, 0).UTC(), + SessionToken: stsCred.SessionToken, + ParentUser: stsCred.ParentUser, + Status: auth.AccountOn, + } + + // Extract the username and lookup DN and groups in LDAP. + ldapUser, isLDAPSTS := claims.Lookup(ldapUserN) + if isLDAPSTS { + // Need to lookup the groups from LDAP. + _, ldapGroups, err := globalIAMSys.LDAPConfig.LookupUserDN(ldapUser) + if err != nil { + return fmt.Errorf("unable to query LDAP server for %s: %w", ldapUser, err) + } + + cred.Groups = ldapGroups + } + + // Set these credentials to IAM. + if _, err := globalIAMSys.SetTempUser(ctx, cred.AccessKey, cred, stsCred.ParentPolicyMapping); err != nil { + return fmt.Errorf("unable to save STS credential and/or parent policy mapping: %w", err) + } + + return nil +} + +// BucketMetaHook - called when bucket meta changes happen and need to be +// replicated to peer clusters. +func (c *SiteReplicationSys) BucketMetaHook(ctx context.Context, item madmin.SRBucketMeta) error { + // The change has already been applied to the local cluster at this + // point, and only needs to be updated on all remote peer clusters. + + c.RLock() + defer c.RUnlock() + if !c.enabled { + return nil + } + + cerr := c.concDo(nil, func(d string, p madmin.PeerInfo) error { + admClient, err := c.getAdminClient(ctx, d) + if err != nil { + return wrapSRErr(err) + } + + return c.annotatePeerErr(p.Name, replicateBucketMetadata, admClient.SRPeerReplicateBucketMeta(ctx, item)) + }, + replicateBucketMetadata, + ) + return errors.Unwrap(cerr) +} + +// PeerBucketVersioningHandler - updates versioning config to local cluster. +func (c *SiteReplicationSys) PeerBucketVersioningHandler(ctx context.Context, bucket string, versioning *string, updatedAt time.Time) error { + if versioning != nil { + // skip overwrite if local update is newer than peer update. + if !updatedAt.IsZero() { + if _, updateTm, err := globalBucketMetadataSys.GetVersioningConfig(bucket); err == nil && updateTm.After(updatedAt) { + return nil + } + } + configData, err := base64.StdEncoding.DecodeString(*versioning) + if err != nil { + return wrapSRErr(err) + } + _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, configData) + if err != nil { + return wrapSRErr(err) + } + return nil + } + + return nil +} + +// PeerBucketMetadataUpdateHandler - merges the bucket metadata, save and ping other nodes +func (c *SiteReplicationSys) PeerBucketMetadataUpdateHandler(ctx context.Context, item madmin.SRBucketMeta) error { + objectAPI := newObjectLayerFn() + if objectAPI == nil { + return errSRObjectLayerNotReady + } + + if item.Bucket == "" || item.UpdatedAt.IsZero() { + return wrapSRErr(errInvalidArgument) + } + + meta, err := readBucketMetadata(ctx, objectAPI, item.Bucket) + if err != nil { + return wrapSRErr(err) + } + + if meta.Created.After(item.UpdatedAt) { + return nil + } + + if item.Policy != nil { + meta.PolicyConfigJSON = item.Policy + meta.PolicyConfigUpdatedAt = item.UpdatedAt + } + + if item.Versioning != nil { + configData, err := base64.StdEncoding.DecodeString(*item.Versioning) + if err != nil { + return wrapSRErr(err) + } + meta.VersioningConfigXML = configData + meta.VersioningConfigUpdatedAt = item.UpdatedAt + } + + if item.Tags != nil { + configData, err := base64.StdEncoding.DecodeString(*item.Tags) + if err != nil { + return wrapSRErr(err) + } + meta.TaggingConfigXML = configData + meta.TaggingConfigUpdatedAt = item.UpdatedAt + } + + if item.ObjectLockConfig != nil { + configData, err := base64.StdEncoding.DecodeString(*item.ObjectLockConfig) + if err != nil { + return wrapSRErr(err) + } + meta.ObjectLockConfigXML = configData + meta.ObjectLockConfigUpdatedAt = item.UpdatedAt + } + + if item.SSEConfig != nil { + configData, err := base64.StdEncoding.DecodeString(*item.SSEConfig) + if err != nil { + return wrapSRErr(err) + } + meta.EncryptionConfigXML = configData + meta.EncryptionConfigUpdatedAt = item.UpdatedAt + } + + if item.Quota != nil { + meta.QuotaConfigJSON = item.Quota + meta.QuotaConfigUpdatedAt = item.UpdatedAt + } + + return globalBucketMetadataSys.save(ctx, meta) +} + +// PeerBucketPolicyHandler - copies/deletes policy to local cluster. +func (c *SiteReplicationSys) PeerBucketPolicyHandler(ctx context.Context, bucket string, policy *policy.BucketPolicy, updatedAt time.Time) error { + // skip overwrite if local update is newer than peer update. + if !updatedAt.IsZero() { + if _, updateTm, err := globalBucketMetadataSys.GetPolicyConfig(bucket); err == nil && updateTm.After(updatedAt) { + return nil + } + } + + if policy != nil { + configData, err := json.Marshal(policy) + if err != nil { + return wrapSRErr(err) + } + + _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, configData) + if err != nil { + return wrapSRErr(err) + } + return nil + } + + // Delete the bucket policy + _, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketPolicyConfig) + if err != nil { + return wrapSRErr(err) + } + + return nil +} + +// PeerBucketTaggingHandler - copies/deletes tags to local cluster. +func (c *SiteReplicationSys) PeerBucketTaggingHandler(ctx context.Context, bucket string, tags *string, updatedAt time.Time) error { + // skip overwrite if local update is newer than peer update. + if !updatedAt.IsZero() { + if _, updateTm, err := globalBucketMetadataSys.GetTaggingConfig(bucket); err == nil && updateTm.After(updatedAt) { + return nil + } + } + + if tags != nil { + configData, err := base64.StdEncoding.DecodeString(*tags) + if err != nil { + return wrapSRErr(err) + } + _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, configData) + if err != nil { + return wrapSRErr(err) + } + return nil + } + + // Delete the tags + _, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketTaggingConfig) + if err != nil { + return wrapSRErr(err) + } + + return nil +} + +// PeerBucketObjectLockConfigHandler - sets object lock on local bucket. +func (c *SiteReplicationSys) PeerBucketObjectLockConfigHandler(ctx context.Context, bucket string, objectLockData *string, updatedAt time.Time) error { + if objectLockData != nil { + // skip overwrite if local update is newer than peer update. + if !updatedAt.IsZero() { + if _, updateTm, err := globalBucketMetadataSys.GetObjectLockConfig(bucket); err == nil && updateTm.After(updatedAt) { + return nil + } + } + + configData, err := base64.StdEncoding.DecodeString(*objectLockData) + if err != nil { + return wrapSRErr(err) + } + _, err = globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, configData) + if err != nil { + return wrapSRErr(err) + } + return nil + } + + return nil +} + +// PeerBucketSSEConfigHandler - copies/deletes SSE config to local cluster. +func (c *SiteReplicationSys) PeerBucketSSEConfigHandler(ctx context.Context, bucket string, sseConfig *string, updatedAt time.Time) error { + // skip overwrite if local update is newer than peer update. + if !updatedAt.IsZero() { + if _, updateTm, err := globalBucketMetadataSys.GetSSEConfig(bucket); err == nil && updateTm.After(updatedAt) { + return nil + } + } + + if sseConfig != nil { + configData, err := base64.StdEncoding.DecodeString(*sseConfig) + if err != nil { + return wrapSRErr(err) + } + _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, configData) + if err != nil { + return wrapSRErr(err) + } + return nil + } + + // Delete sse config + _, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketSSEConfig) + if err != nil { + return wrapSRErr(err) + } + return nil +} + +// PeerBucketQuotaConfigHandler - copies/deletes policy to local cluster. +func (c *SiteReplicationSys) PeerBucketQuotaConfigHandler(ctx context.Context, bucket string, quota *madmin.BucketQuota, updatedAt time.Time) error { + // skip overwrite if local update is newer than peer update. + if !updatedAt.IsZero() { + if _, updateTm, err := globalBucketMetadataSys.GetQuotaConfig(ctx, bucket); err == nil && updateTm.After(updatedAt) { + return nil + } + } + + if quota != nil { + quotaData, err := json.Marshal(quota) + if err != nil { + return wrapSRErr(err) + } + + if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, quotaData); err != nil { + return wrapSRErr(err) + } + + return nil + } + + // Delete the bucket policy + _, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketQuotaConfigFile) + if err != nil { + return wrapSRErr(err) + } + + return nil +} + +// PeerBucketLCConfigHandler - copies/deletes lifecycle config to local cluster +func (c *SiteReplicationSys) PeerBucketLCConfigHandler(ctx context.Context, bucket string, expLCConfig *string, updatedAt time.Time) error { + // skip overwrite if local update is newer than peer update. + if !updatedAt.IsZero() { + if cfg, _, err := globalBucketMetadataSys.GetLifecycleConfig(bucket); err == nil && (cfg.ExpiryUpdatedAt != nil && cfg.ExpiryUpdatedAt.After(updatedAt)) { + return nil + } + } + + if expLCConfig != nil { + configData, err := mergeWithCurrentLCConfig(ctx, bucket, expLCConfig, updatedAt) + if err != nil { + return wrapSRErr(err) + } + _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, configData) + if err != nil { + return wrapSRErr(err) + } + return nil + } + + // Delete ILM config + _, err := globalBucketMetadataSys.Delete(ctx, bucket, bucketLifecycleConfig) + if err != nil { + return wrapSRErr(err) + } + return nil +} + +// getAdminClient - NOTE: ensure to take at least a read lock on SiteReplicationSys +// before calling this. +func (c *SiteReplicationSys) getAdminClient(ctx context.Context, deploymentID string) (*madmin.AdminClient, error) { + creds, err := c.getPeerCreds() + if err != nil { + return nil, err + } + + peer, ok := c.state.Peers[deploymentID] + if !ok { + return nil, errSRPeerNotFound + } + + return getAdminClient(peer.Endpoint, creds.AccessKey, creds.SecretKey) +} + +// getAdminClientWithEndpoint - NOTE: ensure to take at least a read lock on SiteReplicationSys +// before calling this. +func (c *SiteReplicationSys) getAdminClientWithEndpoint(ctx context.Context, deploymentID, endpoint string) (*madmin.AdminClient, error) { + creds, err := c.getPeerCreds() + if err != nil { + return nil, err + } + + if _, ok := c.state.Peers[deploymentID]; !ok { + return nil, errSRPeerNotFound + } + return getAdminClient(endpoint, creds.AccessKey, creds.SecretKey) +} + +func (c *SiteReplicationSys) getPeerCreds() (*auth.Credentials, error) { + u, ok := globalIAMSys.store.GetUser(c.state.ServiceAccountAccessKey) + if !ok { + return nil, errors.New("site replication service account not found") + } + return &u.Credentials, nil +} + +// listBuckets returns a consistent common view of latest unique buckets across +// sites, this is used for replication. +func (c *SiteReplicationSys) listBuckets(ctx context.Context) ([]BucketInfo, error) { + // If local has buckets, enable versioning on them, create them on peers + // and setup replication rules. + objAPI := newObjectLayerFn() + if objAPI == nil { + return nil, errSRObjectLayerNotReady + } + return objAPI.ListBuckets(ctx, BucketOptions{Deleted: true}) +} + +// syncToAllPeers is used for syncing local data to all remote peers, it is +// called once during initial "AddPeerClusters" request. +func (c *SiteReplicationSys) syncToAllPeers(ctx context.Context, addOpts madmin.SRAddOptions) error { + objAPI := newObjectLayerFn() + if objAPI == nil { + return errSRObjectLayerNotReady + } + + buckets, err := objAPI.ListBuckets(ctx, BucketOptions{}) + if err != nil { + return err + } + + for _, bucketInfo := range buckets { + bucket := bucketInfo.Name + + meta, err := globalBucketMetadataSys.GetConfigFromDisk(ctx, bucket) + if err != nil && !errors.Is(err, errConfigNotFound) { + return errSRBackendIssue(err) + } + + opts := MakeBucketOptions{ + LockEnabled: meta.ObjectLocking(), + CreatedAt: bucketInfo.Created.UTC(), + } + + // Now call the MakeBucketHook on existing bucket - this will + // create buckets and replication rules on peer clusters. + if err = c.MakeBucketHook(ctx, bucket, opts); err != nil { + return errSRBucketConfigError(err) + } + + // Replicate bucket policy if present. + policyJSON, tm := meta.PolicyConfigJSON, meta.PolicyConfigUpdatedAt + if len(policyJSON) > 0 { + err = c.BucketMetaHook(ctx, madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaTypePolicy, + Bucket: bucket, + Policy: policyJSON, + UpdatedAt: tm, + }) + if err != nil { + return errSRBucketMetaError(err) + } + } + + // Replicate bucket tags if present. + tagCfg, tm := meta.TaggingConfigXML, meta.TaggingConfigUpdatedAt + if len(tagCfg) > 0 { + tagCfgStr := base64.StdEncoding.EncodeToString(tagCfg) + err = c.BucketMetaHook(ctx, madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaTypeTags, + Bucket: bucket, + Tags: &tagCfgStr, + UpdatedAt: tm, + }) + if err != nil { + return errSRBucketMetaError(err) + } + } + + // Replicate object-lock config if present. + objLockCfgData, tm := meta.ObjectLockConfigXML, meta.ObjectLockConfigUpdatedAt + if len(objLockCfgData) > 0 { + objLockStr := base64.StdEncoding.EncodeToString(objLockCfgData) + err = c.BucketMetaHook(ctx, madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaTypeObjectLockConfig, + Bucket: bucket, + Tags: &objLockStr, + UpdatedAt: tm, + }) + if err != nil { + return errSRBucketMetaError(err) + } + } + + // Replicate existing bucket bucket encryption settings + sseConfigData, tm := meta.EncryptionConfigXML, meta.EncryptionConfigUpdatedAt + if len(sseConfigData) > 0 { + sseConfigStr := base64.StdEncoding.EncodeToString(sseConfigData) + err = c.BucketMetaHook(ctx, madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaTypeSSEConfig, + Bucket: bucket, + SSEConfig: &sseConfigStr, + UpdatedAt: tm, + }) + if err != nil { + return errSRBucketMetaError(err) + } + } + + // Replicate existing bucket quotas settings + quotaConfigJSON, tm := meta.QuotaConfigJSON, meta.QuotaConfigUpdatedAt + if len(quotaConfigJSON) > 0 { + err = c.BucketMetaHook(ctx, madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaTypeQuotaConfig, + Bucket: bucket, + Quota: quotaConfigJSON, + UpdatedAt: tm, + }) + if err != nil { + return errSRBucketMetaError(err) + } + } + + // Replicate ILM expiry rules if needed + if addOpts.ReplicateILMExpiry && (meta.lifecycleConfig != nil && meta.lifecycleConfig.HasExpiry()) { + var expLclCfg lifecycle.Lifecycle + expLclCfg.XMLName = meta.lifecycleConfig.XMLName + for _, rule := range meta.lifecycleConfig.Rules { + if !rule.Expiration.IsNull() || !rule.NoncurrentVersionExpiration.IsNull() { + // copy the non transition details of the rule + expLclCfg.Rules = append(expLclCfg.Rules, rule.CloneNonTransition()) + } + } + currtime := time.Now() + expLclCfg.ExpiryUpdatedAt = &currtime + ilmConfigData, err := xml.Marshal(expLclCfg) + if err != nil { + return errSRBucketMetaError(err) + } + if len(ilmConfigData) > 0 { + configStr := base64.StdEncoding.EncodeToString(ilmConfigData) + err = c.BucketMetaHook(ctx, madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaLCConfig, + Bucket: bucket, + ExpiryLCConfig: &configStr, + UpdatedAt: time.Now(), + }) + if err != nil { + return errSRBucketMetaError(err) + } + } + } + } + + // Order matters from now on how the information is + // synced to remote sites. + + // Policies should be synced first. + { + // Replicate IAM policies on local to all peers. + allPolicyDocs, err := globalIAMSys.ListPolicyDocs(ctx, "") + if err != nil { + return errSRBackendIssue(err) + } + + for pname, pdoc := range allPolicyDocs { + policyJSON, err := json.Marshal(pdoc.Policy) + if err != nil { + return wrapSRErr(err) + } + err = c.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemPolicy, + Name: pname, + Policy: policyJSON, + UpdatedAt: pdoc.UpdateDate, + }) + if err != nil { + return errSRIAMError(err) + } + } + } + + // Next should be userAccounts those are local users, OIDC and LDAP will not + // may not have any local users. + { + userAccounts := make(map[string]UserIdentity) + err := globalIAMSys.store.loadUsers(ctx, regUser, userAccounts) + if err != nil { + return errSRBackendIssue(err) + } + + for _, acc := range userAccounts { + if err := c.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemIAMUser, + IAMUser: &madmin.SRIAMUser{ + AccessKey: acc.Credentials.AccessKey, + IsDeleteReq: false, + UserReq: &madmin.AddOrUpdateUserReq{ + SecretKey: acc.Credentials.SecretKey, + Status: madmin.AccountStatus(acc.Credentials.Status), + }, + }, + UpdatedAt: acc.UpdatedAt, + }); err != nil { + return errSRIAMError(err) + } + } + } + + // Next should be Groups for some of these users, LDAP might have some Group + // DNs here + { + groups := make(map[string]GroupInfo) + err := globalIAMSys.store.loadGroups(ctx, groups) + if err != nil { + return errSRBackendIssue(err) + } + + for gname, group := range groups { + if err := c.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemGroupInfo, + GroupInfo: &madmin.SRGroupInfo{ + UpdateReq: madmin.GroupAddRemove{ + Group: gname, + Members: group.Members, + Status: madmin.GroupStatus(group.Status), + IsRemove: false, + }, + }, + UpdatedAt: group.UpdatedAt, + }); err != nil { + return errSRIAMError(err) + } + } + } + + // Followed by group policy mapping + { + // Replicate policy mappings on local to all peers. + groupPolicyMap := xsync.NewMapOf[string, MappedPolicy]() + errG := globalIAMSys.store.loadMappedPolicies(ctx, unknownIAMUserType, true, groupPolicyMap) + if errG != nil { + return errSRBackendIssue(errG) + } + + var err error + groupPolicyMap.Range(func(k string, mp MappedPolicy) bool { + err = c.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemPolicyMapping, + PolicyMapping: &madmin.SRPolicyMapping{ + UserOrGroup: k, + UserType: int(unknownIAMUserType), + IsGroup: true, + Policy: mp.Policies, + }, + UpdatedAt: mp.UpdatedAt, + }) + return err == nil + }) + if err != nil { + return errSRIAMError(err) + } + } + + // Service accounts are the static accounts that should be synced with + // valid claims. + { + serviceAccounts := make(map[string]UserIdentity) + err := globalIAMSys.store.loadUsers(ctx, svcUser, serviceAccounts) + if err != nil { + return errSRBackendIssue(err) + } + + for user, acc := range serviceAccounts { + if user == siteReplicatorSvcAcc { + // skip the site replicate svc account as it is + // already replicated. + continue + } + + claims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, acc.Credentials.AccessKey) + if err != nil { + return errSRBackendIssue(err) + } + + _, policy, err := globalIAMSys.GetServiceAccount(ctx, acc.Credentials.AccessKey) + if err != nil { + return errSRBackendIssue(err) + } + + var policyJSON []byte + if policy != nil { + policyJSON, err = json.Marshal(policy) + if err != nil { + return wrapSRErr(err) + } + } + + err = c.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemSvcAcc, + SvcAccChange: &madmin.SRSvcAccChange{ + Create: &madmin.SRSvcAccCreate{ + Parent: acc.Credentials.ParentUser, + AccessKey: user, + SecretKey: acc.Credentials.SecretKey, + Groups: acc.Credentials.Groups, + Claims: claims, + SessionPolicy: policyJSON, + Status: acc.Credentials.Status, + Name: acc.Credentials.Name, + Description: acc.Credentials.Description, + Expiration: &acc.Credentials.Expiration, + }, + }, + UpdatedAt: acc.UpdatedAt, + }) + if err != nil { + return errSRIAMError(err) + } + } + } + + // Followed by policy mapping for the userAccounts we previously synced. + { + // Replicate policy mappings on local to all peers. + userPolicyMap := xsync.NewMapOf[string, MappedPolicy]() + errU := globalIAMSys.store.loadMappedPolicies(ctx, regUser, false, userPolicyMap) + if errU != nil { + return errSRBackendIssue(errU) + } + var err error + userPolicyMap.Range(func(user string, mp MappedPolicy) bool { + err = c.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemPolicyMapping, + PolicyMapping: &madmin.SRPolicyMapping{ + UserOrGroup: user, + UserType: int(regUser), + IsGroup: false, + Policy: mp.Policies, + }, + UpdatedAt: mp.UpdatedAt, + }) + return err == nil + }) + if err != nil { + return errSRIAMError(err) + } + } + + // and finally followed by policy mappings for for STS users. + { + // Replicate policy mappings on local to all peers. + stsPolicyMap := xsync.NewMapOf[string, MappedPolicy]() + errU := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, false, stsPolicyMap) + if errU != nil { + return errSRBackendIssue(errU) + } + + var err error + stsPolicyMap.Range(func(user string, mp MappedPolicy) bool { + err = c.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemPolicyMapping, + PolicyMapping: &madmin.SRPolicyMapping{ + UserOrGroup: user, + UserType: int(stsUser), + IsGroup: false, + Policy: mp.Policies, + }, + UpdatedAt: mp.UpdatedAt, + }) + return err == nil + }) + if err != nil { + return errSRIAMError(err) + } + } + + return nil +} + +// Concurrency helpers + +type concErr struct { + errMap map[string]error + summaryErr error +} + +func (c concErr) Error() string { + if c.summaryErr != nil { + return c.summaryErr.Error() + } + return "" +} + +func (c concErr) Unwrap() error { + return c.summaryErr +} + +func (c *SiteReplicationSys) toErrorFromErrMap(errMap map[string]error, actionName string) error { + if len(errMap) == 0 { + return nil + } + + // Get ordered list of keys of errMap + keys := []string{} + for d := range errMap { + keys = append(keys, d) + } + sort.Strings(keys) + + var success int + msgs := []string{} + for _, d := range keys { + name := c.state.Peers[d].Name + err := errMap[d] + if err == nil { + msgs = append(msgs, fmt.Sprintf("'%s' on site %s (%s): succeeded", actionName, name, d)) + success++ + } else { + msgs = append(msgs, fmt.Sprintf("'%s' on site %s (%s): failed(%v)", actionName, name, d, err)) + } + } + if success == len(keys) { + return nil + } + return fmt.Errorf("Site replication error(s): \n%s", strings.Join(msgs, "\n")) +} + +func (c *SiteReplicationSys) newConcErr(errMap map[string]error, actionName string) error { + return concErr{ + errMap: errMap, + summaryErr: c.toErrorFromErrMap(errMap, actionName), + } +} + +// concDo calls actions concurrently. selfActionFn is run for the current +// cluster and peerActionFn is run for each peer replication cluster. +func (c *SiteReplicationSys) concDo(selfActionFn func() error, peerActionFn func(deploymentID string, p madmin.PeerInfo) error, actionName string) error { + depIDs := make([]string, 0, len(c.state.Peers)) + for d := range c.state.Peers { + depIDs = append(depIDs, d) + } + errs := make([]error, len(c.state.Peers)) + var wg sync.WaitGroup + wg.Add(len(depIDs)) + for i := range depIDs { + go func(i int) { + defer wg.Done() + if depIDs[i] == globalDeploymentID() { + if selfActionFn != nil { + errs[i] = selfActionFn() + } + } else { + errs[i] = peerActionFn(depIDs[i], c.state.Peers[depIDs[i]]) + } + }(i) + } + wg.Wait() + errMap := make(map[string]error, len(c.state.Peers)) + for i, depID := range depIDs { + errMap[depID] = errs[i] + if errs[i] != nil && minio.IsNetworkOrHostDown(errs[i], true) { + ep := c.state.Peers[depID].Endpoint + epURL, _ := url.Parse(ep) + if !globalBucketTargetSys.isOffline(epURL) { + globalBucketTargetSys.markOffline(epURL) + } + } + } + return c.newConcErr(errMap, actionName) +} + +func (c *SiteReplicationSys) annotateErr(annotation string, err error) error { + if err == nil { + return nil + } + return fmt.Errorf("%s: %s: %w", c.state.Name, annotation, err) +} + +func (c *SiteReplicationSys) annotatePeerErr(dstPeer string, annotation string, err error) error { + if err == nil { + return nil + } + return fmt.Errorf("%s->%s: %s: %w", c.state.Name, dstPeer, annotation, err) +} + +// isEnabled returns true if site replication is enabled +func (c *SiteReplicationSys) isEnabled() bool { + c.RLock() + defer c.RUnlock() + return c.enabled +} + +var errMissingSRConfig = fmt.Errorf("unable to find site replication configuration") + +// RemovePeerCluster - removes one or more clusters from site replication configuration. +func (c *SiteReplicationSys) RemovePeerCluster(ctx context.Context, objectAPI ObjectLayer, rreq madmin.SRRemoveReq) (st madmin.ReplicateRemoveStatus, err error) { + if !c.isEnabled() { + return st, errSRNotEnabled + } + info, err := c.GetClusterInfo(ctx) + if err != nil { + return st, errSRBackendIssue(err) + } + peerMap := make(map[string]madmin.PeerInfo) + var rmvEndpoints []string + siteNames := rreq.SiteNames + updatedPeers := make(map[string]madmin.PeerInfo) + + for _, pi := range info.Sites { + updatedPeers[pi.DeploymentID] = pi + peerMap[pi.Name] = pi + if rreq.RemoveAll { + siteNames = append(siteNames, pi.Name) + } + } + for _, s := range siteNames { + pinfo, ok := peerMap[s] + if !ok { + return st, errSRConfigMissingError(errMissingSRConfig) + } + rmvEndpoints = append(rmvEndpoints, pinfo.Endpoint) + delete(updatedPeers, pinfo.DeploymentID) + } + var wg sync.WaitGroup + errs := make(map[string]error, len(c.state.Peers)) + + for _, v := range info.Sites { + wg.Add(1) + if v.DeploymentID == globalDeploymentID() { + go func() { + defer wg.Done() + err := c.RemoveRemoteTargetsForEndpoint(ctx, objectAPI, rmvEndpoints, false) + errs[globalDeploymentID()] = err + }() + continue + } + go func(pi madmin.PeerInfo) { + defer wg.Done() + admClient, err := c.getAdminClient(ctx, pi.DeploymentID) + if err != nil { + errs[pi.DeploymentID] = errSRPeerResp(fmt.Errorf("unable to create admin client for %s: %w", pi.Name, err)) + return + } + // set the requesting site's deploymentID for verification of peer request + rreq.RequestingDepID = globalDeploymentID() + if _, err = admClient.SRPeerRemove(ctx, rreq); err != nil { + if errors.Is(err, errMissingSRConfig) { + // ignore if peer is already removed. + return + } + errs[pi.DeploymentID] = errSRPeerResp(fmt.Errorf("unable to update peer %s: %w", pi.Name, err)) + return + } + }(v) + } + wg.Wait() + + errdID := "" + selfTgtsDeleted := errs[globalDeploymentID()] == nil // true if all remote targets and replication config cleared successfully on local cluster + + for dID, err := range errs { + if err != nil { + if !rreq.RemoveAll && !selfTgtsDeleted { + return madmin.ReplicateRemoveStatus{ + ErrDetail: err.Error(), + Status: madmin.ReplicateRemoveStatusPartial, + }, errSRPeerResp(fmt.Errorf("unable to update peer %s: %w", c.state.Peers[dID].Name, err)) + } + errdID = dID + } + } + + // force local config to be cleared even if peers failed since the remote targets are deleted + // by now from the replication config and user intended to forcibly clear all site replication + if rreq.RemoveAll { + if err = c.removeFromDisk(ctx); err != nil { + return madmin.ReplicateRemoveStatus{ + Status: madmin.ReplicateRemoveStatusPartial, + ErrDetail: fmt.Sprintf("unable to remove cluster-replication state on local: %v", err), + }, nil + } + if errdID != "" { + err := errs[errdID] + return madmin.ReplicateRemoveStatus{ + Status: madmin.ReplicateRemoveStatusPartial, + ErrDetail: err.Error(), + }, nil + } + return madmin.ReplicateRemoveStatus{ + Status: madmin.ReplicateRemoveStatusSuccess, + }, nil + } + + // Update cluster state + var state srState + if len(updatedPeers) > 1 { + state = srState{ + Name: info.Name, + Peers: updatedPeers, + ServiceAccountAccessKey: info.ServiceAccountAccessKey, + } + } + if err = c.saveToDisk(ctx, state); err != nil { + return madmin.ReplicateRemoveStatus{ + Status: madmin.ReplicateRemoveStatusPartial, + ErrDetail: fmt.Sprintf("unable to save cluster-replication state on local: %v", err), + }, err + } + + st = madmin.ReplicateRemoveStatus{ + Status: madmin.ReplicateRemoveStatusSuccess, + } + if errs[errdID] != nil { + st.Status = madmin.ReplicateRemoveStatusPartial + st.ErrDetail = errs[errdID].Error() + } + return st, nil +} + +// InternalRemoveReq - sends an unlink request to peer cluster to remove one or more sites +// from the site replication configuration. +func (c *SiteReplicationSys) InternalRemoveReq(ctx context.Context, objectAPI ObjectLayer, rreq madmin.SRRemoveReq) error { + if !c.isEnabled() { + return errSRNotEnabled + } + if rreq.RequestingDepID != "" { + // validate if requesting site is still part of site replication + var foundRequestor bool + for _, p := range c.state.Peers { + if p.DeploymentID == rreq.RequestingDepID { + foundRequestor = true + break + } + } + if !foundRequestor { + return errSRRequestorNotFound + } + } + + ourName := "" + peerMap := make(map[string]madmin.PeerInfo) + updatedPeers := make(map[string]madmin.PeerInfo) + siteNames := rreq.SiteNames + + for _, p := range c.state.Peers { + peerMap[p.Name] = p + if p.DeploymentID == globalDeploymentID() { + ourName = p.Name + } + updatedPeers[p.DeploymentID] = p + if rreq.RemoveAll { + siteNames = append(siteNames, p.Name) + } + } + var rmvEndpoints []string + var unlinkSelf bool + + for _, s := range siteNames { + info, ok := peerMap[s] + if !ok { + return errMissingSRConfig + } + if info.DeploymentID == globalDeploymentID() { + unlinkSelf = true + continue + } + delete(updatedPeers, info.DeploymentID) + rmvEndpoints = append(rmvEndpoints, info.Endpoint) + } + if err := c.RemoveRemoteTargetsForEndpoint(ctx, objectAPI, rmvEndpoints, unlinkSelf); err != nil { + return err + } + var state srState + if !unlinkSelf { + state = srState{ + Name: c.state.Name, + Peers: updatedPeers, + ServiceAccountAccessKey: c.state.ServiceAccountAccessKey, + } + } + + if err := c.saveToDisk(ctx, state); err != nil { + return errSRBackendIssue(fmt.Errorf("unable to save cluster-replication state to drive on %s: %v", ourName, err)) + } + return nil +} + +// RemoveRemoteTargetsForEndpoint removes replication targets corresponding to endpoint +func (c *SiteReplicationSys) RemoveRemoteTargetsForEndpoint(ctx context.Context, objectAPI ObjectLayer, endpoints []string, unlinkSelf bool) (err error) { + targets := globalBucketTargetSys.ListTargets(ctx, "", string(madmin.ReplicationService)) + m := make(map[string]madmin.BucketTarget) + for _, t := range targets { + for _, endpoint := range endpoints { + ep, _ := url.Parse(endpoint) + if t.Endpoint == ep.Host && + t.Secure == (ep.Scheme == "https") && + t.Type == madmin.ReplicationService { + m[t.Arn] = t + } + } + // all remote targets from self are to be delinked + if unlinkSelf { + m[t.Arn] = t + } + } + buckets, err := objectAPI.ListBuckets(ctx, BucketOptions{}) + if err != nil { + return errSRBackendIssue(err) + } + + for _, b := range buckets { + config, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, b.Name) + if err != nil { + if errors.Is(err, BucketReplicationConfigNotFound{Bucket: b.Name}) { + continue + } + return err + } + var nRules []sreplication.Rule + for _, r := range config.Rules { + if _, ok := m[r.Destination.Bucket]; !ok { + nRules = append(nRules, r) + } + } + if len(nRules) > 0 { + config.Rules = nRules + configData, err := xml.Marshal(config) + if err != nil { + return err + } + if _, err = globalBucketMetadataSys.Update(ctx, b.Name, bucketReplicationConfig, configData); err != nil { + return err + } + } else { + if _, err := globalBucketMetadataSys.Delete(ctx, b.Name, bucketReplicationConfig); err != nil { + return err + } + } + } + for arn, t := range m { + if err := globalBucketTargetSys.RemoveTarget(ctx, t.SourceBucket, arn); err != nil { + if errors.Is(err, BucketRemoteTargetNotFound{Bucket: t.SourceBucket}) { + continue + } + return err + } + targets, terr := globalBucketTargetSys.ListBucketTargets(ctx, t.SourceBucket) + if terr != nil { + return terr + } + tgtBytes, terr := json.Marshal(&targets) + if terr != nil { + return terr + } + if _, err = globalBucketMetadataSys.Update(ctx, t.SourceBucket, bucketTargetsFile, tgtBytes); err != nil { + return err + } + } + return err +} + +// Other helpers + +func getAdminClient(endpoint, accessKey, secretKey string) (*madmin.AdminClient, error) { + epURL, err := url.Parse(endpoint) + if err != nil { + return nil, err + } + if globalBucketTargetSys.isOffline(epURL) { + return nil, RemoteTargetConnectionErr{Endpoint: epURL.String(), Err: fmt.Errorf("remote target is offline for endpoint %s", epURL.String())} + } + return madmin.NewWithOptions(epURL.Host, &madmin.Options{ + Creds: credentials.NewStaticV4(accessKey, secretKey, ""), + Secure: epURL.Scheme == "https", + Transport: globalRemoteTargetTransport, + }) +} + +func getS3Client(pc madmin.PeerSite) (*minio.Client, error) { + ep, err := url.Parse(pc.Endpoint) + if err != nil { + return nil, err + } + if globalBucketTargetSys.isOffline(ep) { + return nil, RemoteTargetConnectionErr{Endpoint: ep.String(), Err: fmt.Errorf("remote target is offline for endpoint %s", ep.String())} + } + + return minio.New(ep.Host, &minio.Options{ + Creds: credentials.NewStaticV4(pc.AccessKey, pc.SecretKey, ""), + Secure: ep.Scheme == "https", + Transport: globalRemoteTargetTransport, + }) +} + +func getPriorityHelper(replicationConfig replication.Config) int { + maxPrio := 0 + for _, rule := range replicationConfig.Rules { + if rule.Priority > maxPrio { + maxPrio = rule.Priority + } + } + + // leave some gaps in priority numbers for flexibility + return maxPrio + 10 +} + +// returns a slice with site names participating in site replciation but unspecified while adding +// a new site. +func getMissingSiteNames(oldDeps, newDeps set.StringSet, currSites []madmin.PeerInfo) []string { + diff := oldDeps.Difference(newDeps) + var diffSlc []string + for _, v := range currSites { + if diff.Contains(v.DeploymentID) { + diffSlc = append(diffSlc, v.Name) + } + } + return diffSlc +} + +type srBucketMetaInfo struct { + madmin.SRBucketInfo + DeploymentID string +} + +type srPolicy struct { + madmin.SRIAMPolicy + DeploymentID string +} + +type srPolicyMapping struct { + madmin.SRPolicyMapping + DeploymentID string +} + +type srUserInfo struct { + madmin.UserInfo + DeploymentID string +} + +type srGroupDesc struct { + madmin.GroupDesc + DeploymentID string +} + +type srILMExpiryRule struct { + madmin.ILMExpiryRule + DeploymentID string +} + +// SiteReplicationStatus returns the site replication status across clusters participating in site replication. +func (c *SiteReplicationSys) SiteReplicationStatus(ctx context.Context, objAPI ObjectLayer, opts madmin.SRStatusOptions) (info madmin.SRStatusInfo, err error) { + sinfo, err := c.siteReplicationStatus(ctx, objAPI, opts) + if err != nil { + return info, err + } + info = madmin.SRStatusInfo{ + Enabled: sinfo.Enabled, + MaxBuckets: sinfo.MaxBuckets, + MaxUsers: sinfo.MaxUsers, + MaxGroups: sinfo.MaxGroups, + MaxPolicies: sinfo.MaxPolicies, + MaxILMExpiryRules: sinfo.MaxILMExpiryRules, + Sites: sinfo.Sites, + StatsSummary: sinfo.StatsSummary, + Metrics: sinfo.Metrics, + } + info.BucketStats = make(map[string]map[string]madmin.SRBucketStatsSummary, len(sinfo.Sites)) + info.PolicyStats = make(map[string]map[string]madmin.SRPolicyStatsSummary) + info.UserStats = make(map[string]map[string]madmin.SRUserStatsSummary) + info.GroupStats = make(map[string]map[string]madmin.SRGroupStatsSummary) + info.ILMExpiryStats = make(map[string]map[string]madmin.SRILMExpiryStatsSummary) + numSites := len(info.Sites) + for b, stat := range sinfo.BucketStats { + for dID, st := range stat { + if st.TagMismatch || + st.VersioningConfigMismatch || + st.OLockConfigMismatch || + st.SSEConfigMismatch || + st.PolicyMismatch || + st.ReplicationCfgMismatch || + st.QuotaCfgMismatch || + opts.Entity == madmin.SRBucketEntity { + if _, ok := info.BucketStats[b]; !ok { + info.BucketStats[b] = make(map[string]madmin.SRBucketStatsSummary, numSites) + } + info.BucketStats[b][dID] = st.SRBucketStatsSummary + } + } + } + for u, stat := range sinfo.UserStats { + for dID, st := range stat { + if st.PolicyMismatch || st.UserInfoMismatch || opts.Entity == madmin.SRUserEntity { + if _, ok := info.UserStats[u]; !ok { + info.UserStats[u] = make(map[string]madmin.SRUserStatsSummary, numSites) + } + info.UserStats[u][dID] = st.SRUserStatsSummary + } + } + } + for g, stat := range sinfo.GroupStats { + for dID, st := range stat { + if st.PolicyMismatch || st.GroupDescMismatch || opts.Entity == madmin.SRGroupEntity { + if _, ok := info.GroupStats[g]; !ok { + info.GroupStats[g] = make(map[string]madmin.SRGroupStatsSummary, numSites) + } + info.GroupStats[g][dID] = st.SRGroupStatsSummary + } + } + } + for p, stat := range sinfo.PolicyStats { + for dID, st := range stat { + if st.PolicyMismatch || opts.Entity == madmin.SRPolicyEntity { + if _, ok := info.PolicyStats[p]; !ok { + info.PolicyStats[p] = make(map[string]madmin.SRPolicyStatsSummary, numSites) + } + info.PolicyStats[p][dID] = st.SRPolicyStatsSummary + } + } + } + for p, stat := range sinfo.ILMExpiryRulesStats { + for dID, st := range stat { + if st.ILMExpiryRuleMismatch || opts.Entity == madmin.SRILMExpiryRuleEntity { + if _, ok := info.ILMExpiryStats[p]; !ok { + info.ILMExpiryStats[p] = make(map[string]madmin.SRILMExpiryStatsSummary, numSites) + } + info.ILMExpiryStats[p][dID] = st.SRILMExpiryStatsSummary + } + } + } + + return info, err +} + +const ( + replicationStatus = "ReplicationStatus" +) + +// siteReplicationStatus returns the site replication status across clusters participating in site replication. +func (c *SiteReplicationSys) siteReplicationStatus(ctx context.Context, objAPI ObjectLayer, opts madmin.SRStatusOptions) (info srStatusInfo, err error) { + c.RLock() + defer c.RUnlock() + if !c.enabled { + return info, err + } + + sris := make([]madmin.SRInfo, len(c.state.Peers)) + depIdx := make(map[string]int, len(c.state.Peers)) + i := 0 + for d := range c.state.Peers { + depIdx[d] = i + i++ + } + + metaInfoConcErr := c.concDo( + func() error { + srInfo, err := c.SiteReplicationMetaInfo(ctx, objAPI, opts) + if err != nil { + return err + } + sris[depIdx[globalDeploymentID()]] = srInfo + return nil + }, + func(deploymentID string, p madmin.PeerInfo) error { + admClient, err := c.getAdminClient(ctx, deploymentID) + if err != nil { + switch err.(type) { + case RemoteTargetConnectionErr: + sris[depIdx[deploymentID]] = madmin.SRInfo{} + return nil + default: + return err + } + } + srInfo, err := admClient.SRMetaInfo(ctx, opts) + if err != nil { + return err + } + sris[depIdx[deploymentID]] = srInfo + return nil + }, + replicationStatus, + ) + if err := errors.Unwrap(metaInfoConcErr); err != nil { + return info, errSRBackendIssue(err) + } + + info.Enabled = true + info.Sites = make(map[string]madmin.PeerInfo, len(c.state.Peers)) + maps.Copy(info.Sites, c.state.Peers) + info.UpdatedAt = c.state.UpdatedAt + + var maxBuckets int + for _, sri := range sris { + if len(sri.Buckets) > maxBuckets { + maxBuckets = len(sri.Buckets) + } + } + // mapping b/w entity and entity config across sites + bucketStats := make(map[string][]srBucketMetaInfo) + policyStats := make(map[string][]srPolicy) + userPolicyStats := make(map[string][]srPolicyMapping) + groupPolicyStats := make(map[string][]srPolicyMapping) + userInfoStats := make(map[string][]srUserInfo) + groupDescStats := make(map[string][]srGroupDesc) + ilmExpiryRuleStats := make(map[string][]srILMExpiryRule) + + numSites := len(sris) + allBuckets := set.NewStringSet() // across sites + allUsers := set.NewStringSet() + allUserWPolicies := set.NewStringSet() + allGroups := set.NewStringSet() + allGroupWPolicies := set.NewStringSet() + allILMExpiryRules := set.NewStringSet() + + allPolicies := set.NewStringSet() + for _, sri := range sris { + for b := range sri.Buckets { + allBuckets.Add(b) + } + for u := range sri.UserInfoMap { + allUsers.Add(u) + } + for g := range sri.GroupDescMap { + allGroups.Add(g) + } + for p := range sri.Policies { + allPolicies.Add(p) + } + for u := range sri.UserPolicies { + allUserWPolicies.Add(u) + } + for g := range sri.GroupPolicies { + allGroupWPolicies.Add(g) + } + for r := range sri.ILMExpiryRules { + allILMExpiryRules.Add(r) + } + } + + for i, sri := range sris { + for b := range allBuckets { + if _, ok := bucketStats[b]; !ok { + bucketStats[b] = make([]srBucketMetaInfo, numSites) + } + si, ok := sri.Buckets[b] + if !ok { + si = madmin.SRBucketInfo{Bucket: b} + } + bucketStats[b][i] = srBucketMetaInfo{SRBucketInfo: si, DeploymentID: sri.DeploymentID} + } + + for pname := range allPolicies { + if _, ok := policyStats[pname]; !ok { + policyStats[pname] = make([]srPolicy, numSites) + } + + // if pname is not present in the map, the zero value + // will be returned. + pi := sri.Policies[pname] + policyStats[pname][i] = srPolicy{SRIAMPolicy: pi, DeploymentID: sri.DeploymentID} + } + for user := range allUserWPolicies { + if _, ok := userPolicyStats[user]; !ok { + userPolicyStats[user] = make([]srPolicyMapping, numSites) + } + up := sri.UserPolicies[user] + userPolicyStats[user][i] = srPolicyMapping{SRPolicyMapping: up, DeploymentID: sri.DeploymentID} + } + for group := range allGroupWPolicies { + if _, ok := groupPolicyStats[group]; !ok { + groupPolicyStats[group] = make([]srPolicyMapping, numSites) + } + up := sri.GroupPolicies[group] + groupPolicyStats[group][i] = srPolicyMapping{SRPolicyMapping: up, DeploymentID: sri.DeploymentID} + } + for u := range allUsers { + if _, ok := userInfoStats[u]; !ok { + userInfoStats[u] = make([]srUserInfo, numSites) + } + ui := sri.UserInfoMap[u] + userInfoStats[u][i] = srUserInfo{UserInfo: ui, DeploymentID: sri.DeploymentID} + } + for g := range allGroups { + if _, ok := groupDescStats[g]; !ok { + groupDescStats[g] = make([]srGroupDesc, numSites) + } + gd := sri.GroupDescMap[g] + groupDescStats[g][i] = srGroupDesc{GroupDesc: gd, DeploymentID: sri.DeploymentID} + } + for r := range allILMExpiryRules { + if _, ok := ilmExpiryRuleStats[r]; !ok { + ilmExpiryRuleStats[r] = make([]srILMExpiryRule, numSites) + } + rl := sri.ILMExpiryRules[r] + ilmExpiryRuleStats[r][i] = srILMExpiryRule{ILMExpiryRule: rl, DeploymentID: sri.DeploymentID} + } + } + + info.StatsSummary = make(map[string]madmin.SRSiteSummary, len(c.state.Peers)) + info.BucketStats = make(map[string]map[string]srBucketStatsSummary) + info.PolicyStats = make(map[string]map[string]srPolicyStatsSummary) + info.UserStats = make(map[string]map[string]srUserStatsSummary) + info.GroupStats = make(map[string]map[string]srGroupStatsSummary) + info.ILMExpiryRulesStats = make(map[string]map[string]srILMExpiryRuleStatsSummary) + // collect user policy mapping replication status across sites + if opts.Users || opts.Entity == madmin.SRUserEntity { + for u, pslc := range userPolicyStats { + if len(info.UserStats[u]) == 0 { + info.UserStats[u] = make(map[string]srUserStatsSummary) + } + var policyMappings []madmin.SRPolicyMapping + uPolicyCount := 0 + for _, ps := range pslc { + policyMappings = append(policyMappings, ps.SRPolicyMapping) + uPolicyCount++ + sum := info.StatsSummary[ps.DeploymentID] + sum.TotalUserPolicyMappingCount++ + info.StatsSummary[ps.DeploymentID] = sum + } + userPolicyMismatch := !isPolicyMappingReplicated(uPolicyCount, numSites, policyMappings) + for _, ps := range pslc { + dID := depIdx[ps.DeploymentID] + _, hasUser := sris[dID].UserPolicies[u] + info.UserStats[u][ps.DeploymentID] = srUserStatsSummary{ + SRUserStatsSummary: madmin.SRUserStatsSummary{ + PolicyMismatch: userPolicyMismatch, + HasUser: hasUser, + HasPolicyMapping: ps.Policy != "", + }, + userPolicy: ps, + } + if !userPolicyMismatch || opts.Entity != madmin.SRUserEntity { + sum := info.StatsSummary[ps.DeploymentID] + if !ps.IsGroup { + sum.ReplicatedUserPolicyMappings++ + } + info.StatsSummary[ps.DeploymentID] = sum + } + } + } + + // collect user info replication status across sites + for u, pslc := range userInfoStats { + var uiSlc []madmin.UserInfo + userCount := 0 + for _, ps := range pslc { + uiSlc = append(uiSlc, ps.UserInfo) + userCount++ + sum := info.StatsSummary[ps.DeploymentID] + sum.TotalUsersCount++ + info.StatsSummary[ps.DeploymentID] = sum + } + userInfoMismatch := !isUserInfoReplicated(userCount, numSites, uiSlc) + for _, ps := range pslc { + dID := depIdx[ps.DeploymentID] + _, hasUser := sris[dID].UserInfoMap[u] + if len(info.UserStats[u]) == 0 { + info.UserStats[u] = make(map[string]srUserStatsSummary) + } + umis, ok := info.UserStats[u][ps.DeploymentID] + if !ok { + umis = srUserStatsSummary{ + SRUserStatsSummary: madmin.SRUserStatsSummary{ + HasUser: hasUser, + }, + } + } + umis.UserInfoMismatch = userInfoMismatch + umis.userInfo = ps + info.UserStats[u][ps.DeploymentID] = umis + if !userInfoMismatch || opts.Entity != madmin.SRUserEntity { + sum := info.StatsSummary[ps.DeploymentID] + sum.ReplicatedUsers++ + info.StatsSummary[ps.DeploymentID] = sum + } + } + } + } + if opts.Groups || opts.Entity == madmin.SRGroupEntity { + // collect group policy mapping replication status across sites + for g, pslc := range groupPolicyStats { + var policyMappings []madmin.SRPolicyMapping + gPolicyCount := 0 + for _, ps := range pslc { + policyMappings = append(policyMappings, ps.SRPolicyMapping) + gPolicyCount++ + sum := info.StatsSummary[ps.DeploymentID] + sum.TotalGroupPolicyMappingCount++ + info.StatsSummary[ps.DeploymentID] = sum + } + groupPolicyMismatch := !isPolicyMappingReplicated(gPolicyCount, numSites, policyMappings) + if len(info.GroupStats[g]) == 0 { + info.GroupStats[g] = make(map[string]srGroupStatsSummary) + } + for _, ps := range pslc { + dID := depIdx[ps.DeploymentID] + _, hasGroup := sris[dID].GroupPolicies[g] + info.GroupStats[g][ps.DeploymentID] = srGroupStatsSummary{ + SRGroupStatsSummary: madmin.SRGroupStatsSummary{ + PolicyMismatch: groupPolicyMismatch, + HasGroup: hasGroup, + HasPolicyMapping: ps.Policy != "", + DeploymentID: ps.DeploymentID, + }, + groupPolicy: ps, + } + if !groupPolicyMismatch && opts.Entity != madmin.SRGroupEntity { + sum := info.StatsSummary[ps.DeploymentID] + sum.ReplicatedGroupPolicyMappings++ + info.StatsSummary[ps.DeploymentID] = sum + } + } + } + + // collect group desc replication status across sites + for g, pslc := range groupDescStats { + var gds []madmin.GroupDesc + groupCount := 0 + for _, ps := range pslc { + groupCount++ + sum := info.StatsSummary[ps.DeploymentID] + sum.TotalGroupsCount++ + info.StatsSummary[ps.DeploymentID] = sum + gds = append(gds, ps.GroupDesc) + } + gdMismatch := !isGroupDescReplicated(groupCount, numSites, gds) + for _, ps := range pslc { + dID := depIdx[ps.DeploymentID] + _, hasGroup := sris[dID].GroupDescMap[g] + if len(info.GroupStats[g]) == 0 { + info.GroupStats[g] = make(map[string]srGroupStatsSummary) + } + gmis, ok := info.GroupStats[g][ps.DeploymentID] + if !ok { + gmis = srGroupStatsSummary{ + SRGroupStatsSummary: madmin.SRGroupStatsSummary{ + HasGroup: hasGroup, + }, + } + } + gmis.GroupDescMismatch = gdMismatch + gmis.groupDesc = ps + info.GroupStats[g][ps.DeploymentID] = gmis + if !gdMismatch && opts.Entity != madmin.SRGroupEntity { + sum := info.StatsSummary[ps.DeploymentID] + sum.ReplicatedGroups++ + info.StatsSummary[ps.DeploymentID] = sum + } + } + } + } + if opts.Policies || opts.Entity == madmin.SRPolicyEntity { + // collect IAM policy replication status across sites + for p, pslc := range policyStats { + var policies []*policy.Policy + uPolicyCount := 0 + for _, ps := range pslc { + plcy, err := policy.ParseConfig(bytes.NewReader([]byte(ps.Policy))) + if err != nil { + continue + } + policies = append(policies, plcy) + uPolicyCount++ + sum := info.StatsSummary[ps.DeploymentID] + sum.TotalIAMPoliciesCount++ + info.StatsSummary[ps.DeploymentID] = sum + } + if len(info.PolicyStats[p]) == 0 { + info.PolicyStats[p] = make(map[string]srPolicyStatsSummary) + } + policyMismatch := !isIAMPolicyReplicated(uPolicyCount, numSites, policies) + for _, ps := range pslc { + dID := depIdx[ps.DeploymentID] + _, hasPolicy := sris[dID].Policies[p] + info.PolicyStats[p][ps.DeploymentID] = srPolicyStatsSummary{ + SRPolicyStatsSummary: madmin.SRPolicyStatsSummary{ + PolicyMismatch: policyMismatch, + HasPolicy: hasPolicy, + }, + policy: ps, + } + switch { + case policyMismatch, opts.Entity == madmin.SRPolicyEntity: + default: + sum := info.StatsSummary[ps.DeploymentID] + if !policyMismatch { + sum.ReplicatedIAMPolicies++ + } + info.StatsSummary[ps.DeploymentID] = sum + } + } + } + } + if opts.Buckets || opts.Entity == madmin.SRBucketEntity { + // collect bucket metadata replication stats across sites + for b, slc := range bucketStats { + tagSet := set.NewStringSet() + olockConfigSet := set.NewStringSet() + policies := make([]*policy.BucketPolicy, numSites) + replCfgs := make([]*sreplication.Config, numSites) + quotaCfgs := make([]*madmin.BucketQuota, numSites) + sseCfgSet := set.NewStringSet() + versionCfgSet := set.NewStringSet() + var tagCount, olockCfgCount, sseCfgCount, versionCfgCount int + for i, s := range slc { + if s.ReplicationConfig != nil { + cfgBytes, err := base64.StdEncoding.DecodeString(*s.ReplicationConfig) + if err != nil { + continue + } + cfg, err := sreplication.ParseConfig(bytes.NewReader(cfgBytes)) + if err != nil { + continue + } + replCfgs[i] = cfg + } + if s.Versioning != nil { + configData, err := base64.StdEncoding.DecodeString(*s.Versioning) + if err != nil { + continue + } + versionCfgCount++ + if !versionCfgSet.Contains(string(configData)) { + versionCfgSet.Add(string(configData)) + } + } + if s.QuotaConfig != nil { + cfgBytes, err := base64.StdEncoding.DecodeString(*s.QuotaConfig) + if err != nil { + continue + } + cfg, err := parseBucketQuota(b, cfgBytes) + if err != nil { + continue + } + quotaCfgs[i] = cfg + } + if s.Tags != nil { + tagBytes, err := base64.StdEncoding.DecodeString(*s.Tags) + if err != nil { + continue + } + tagCount++ + if !tagSet.Contains(string(tagBytes)) { + tagSet.Add(string(tagBytes)) + } + } + if len(s.Policy) > 0 { + plcy, err := policy.ParseBucketPolicyConfig(bytes.NewReader(s.Policy), b) + if err != nil { + continue + } + policies[i] = plcy + } + if s.ObjectLockConfig != nil { + configData, err := base64.StdEncoding.DecodeString(*s.ObjectLockConfig) + if err != nil { + continue + } + olockCfgCount++ + if !olockConfigSet.Contains(string(configData)) { + olockConfigSet.Add(string(configData)) + } + } + if s.SSEConfig != nil { + configData, err := base64.StdEncoding.DecodeString(*s.SSEConfig) + if err != nil { + continue + } + sseCfgCount++ + if !sseCfgSet.Contains(string(configData)) { + sseCfgSet.Add(string(configData)) + } + } + ss, ok := info.StatsSummary[s.DeploymentID] + if !ok { + ss = madmin.SRSiteSummary{} + } + // increment total number of replicated buckets + if len(slc) == numSites { + ss.ReplicatedBuckets++ + } + ss.TotalBucketsCount++ + if tagCount > 0 { + ss.TotalTagsCount++ + } + if olockCfgCount > 0 { + ss.TotalLockConfigCount++ + } + if sseCfgCount > 0 { + ss.TotalSSEConfigCount++ + } + if versionCfgCount > 0 { + ss.TotalVersioningConfigCount++ + } + if len(policies) > 0 { + ss.TotalBucketPoliciesCount++ + } + info.StatsSummary[s.DeploymentID] = ss + } + tagMismatch := !isReplicated(tagCount, numSites, tagSet) + olockCfgMismatch := !isReplicated(olockCfgCount, numSites, olockConfigSet) + sseCfgMismatch := !isReplicated(sseCfgCount, numSites, sseCfgSet) + versionCfgMismatch := !isReplicated(versionCfgCount, numSites, versionCfgSet) + policyMismatch := !isBktPolicyReplicated(numSites, policies) + replCfgMismatch := !isBktReplCfgReplicated(numSites, replCfgs) + quotaCfgMismatch := !isBktQuotaCfgReplicated(numSites, quotaCfgs) + info.BucketStats[b] = make(map[string]srBucketStatsSummary, numSites) + for i, s := range slc { + dIdx := depIdx[s.DeploymentID] + var hasBucket, isBucketMarkedDeleted bool + + bi, ok := sris[dIdx].Buckets[s.Bucket] + if ok { + isBucketMarkedDeleted = !bi.DeletedAt.IsZero() && (bi.CreatedAt.IsZero() || bi.DeletedAt.After(bi.CreatedAt)) + hasBucket = !bi.CreatedAt.IsZero() + } + quotaCfgSet := hasBucket && quotaCfgs[i] != nil && *quotaCfgs[i] != madmin.BucketQuota{} + ss := madmin.SRBucketStatsSummary{ + DeploymentID: s.DeploymentID, + HasBucket: hasBucket, + BucketMarkedDeleted: isBucketMarkedDeleted, + TagMismatch: tagMismatch, + OLockConfigMismatch: olockCfgMismatch, + SSEConfigMismatch: sseCfgMismatch, + VersioningConfigMismatch: versionCfgMismatch, + PolicyMismatch: policyMismatch, + ReplicationCfgMismatch: replCfgMismatch, + QuotaCfgMismatch: quotaCfgMismatch, + HasReplicationCfg: s.ReplicationConfig != nil, + HasTagsSet: s.Tags != nil, + HasOLockConfigSet: s.ObjectLockConfig != nil, + HasPolicySet: s.Policy != nil, + HasQuotaCfgSet: quotaCfgSet, + HasSSECfgSet: s.SSEConfig != nil, + } + var m srBucketMetaInfo + if len(bucketStats[s.Bucket]) > dIdx { + m = bucketStats[s.Bucket][dIdx] + } + info.BucketStats[b][s.DeploymentID] = srBucketStatsSummary{ + SRBucketStatsSummary: ss, + meta: m, + } + } + // no mismatch + for _, s := range slc { + sum := info.StatsSummary[s.DeploymentID] + if !olockCfgMismatch && olockCfgCount == numSites { + sum.ReplicatedLockConfig++ + } + if !versionCfgMismatch && versionCfgCount == numSites { + sum.ReplicatedVersioningConfig++ + } + if !sseCfgMismatch && sseCfgCount == numSites { + sum.ReplicatedSSEConfig++ + } + if !policyMismatch && len(policies) == numSites { + sum.ReplicatedBucketPolicies++ + } + if !tagMismatch && tagCount == numSites { + sum.ReplicatedTags++ + } + info.StatsSummary[s.DeploymentID] = sum + } + } + } + if opts.ILMExpiryRules || opts.Entity == madmin.SRILMExpiryRuleEntity { + // collect ILM expiry rules replication status across sites + for id, ilmExpRules := range ilmExpiryRuleStats { + var rules []*lifecycle.Rule + uRuleCount := 0 + for _, rl := range ilmExpRules { + var rule lifecycle.Rule + if err := xml.Unmarshal([]byte(rl.ILMRule), &rule); err != nil { + continue + } + rules = append(rules, &rule) + uRuleCount++ + sum := info.StatsSummary[rl.DeploymentID] + sum.TotalILMExpiryRulesCount++ + info.StatsSummary[rl.DeploymentID] = sum + } + if len(info.ILMExpiryRulesStats[id]) == 0 { + info.ILMExpiryRulesStats[id] = make(map[string]srILMExpiryRuleStatsSummary) + } + ilmExpRuleMismatch := !isILMExpRuleReplicated(uRuleCount, numSites, rules) + for _, rl := range ilmExpRules { + dID := depIdx[rl.DeploymentID] + _, hasILMExpRule := sris[dID].ILMExpiryRules[id] + info.ILMExpiryRulesStats[id][rl.DeploymentID] = srILMExpiryRuleStatsSummary{ + SRILMExpiryStatsSummary: madmin.SRILMExpiryStatsSummary{ + ILMExpiryRuleMismatch: ilmExpRuleMismatch, + HasILMExpiryRules: hasILMExpRule, + }, + ilmExpiryRule: rl, + } + switch { + case ilmExpRuleMismatch, opts.Entity == madmin.SRILMExpiryRuleEntity: + default: + sum := info.StatsSummary[rl.DeploymentID] + if !ilmExpRuleMismatch { + sum.ReplicatedILMExpiryRules++ + } + info.StatsSummary[rl.DeploymentID] = sum + } + } + } + } + if opts.PeerState { + info.PeerStates = make(map[string]madmin.SRStateInfo, numSites) + for _, sri := range sris { + info.PeerStates[sri.DeploymentID] = sri.State + } + } + + if opts.Metrics { + m, err := globalSiteReplicationSys.getSiteMetrics(ctx) + if err != nil { + return info, err + } + info.Metrics = m + } + + // maximum buckets users etc seen across sites + info.MaxBuckets = len(bucketStats) + info.MaxUsers = len(userInfoStats) + info.MaxGroups = len(groupDescStats) + info.MaxPolicies = len(policyStats) + info.MaxILMExpiryRules = len(ilmExpiryRuleStats) + return info, err +} + +// isReplicated returns true if count of replicated matches the number of +// sites and there is atmost one unique entry in the set. +func isReplicated(cntReplicated, total int, valSet set.StringSet) bool { + if cntReplicated > 0 && cntReplicated < total { + return false + } + if len(valSet) > 1 { + // mismatch - one or more sites has differing tags/policy + return false + } + return true +} + +// isIAMPolicyReplicated returns true if count of replicated IAM policies matches total +// number of sites and IAM policies are identical. +func isIAMPolicyReplicated(cntReplicated, total int, policies []*policy.Policy) bool { + if cntReplicated > 0 && cntReplicated != total { + return false + } + // check if policies match between sites + var prev *policy.Policy + for i, p := range policies { + if i == 0 { + prev = p + continue + } + if !prev.Equals(*p) { + return false + } + } + return true +} + +// isPolicyMappingReplicated returns true if count of replicated IAM policy mappings matches total +// number of sites and IAM policy mappings are identical. +func isPolicyMappingReplicated(cntReplicated, total int, policies []madmin.SRPolicyMapping) bool { + if cntReplicated > 0 && cntReplicated != total { + return false + } + // check if policies match between sites + var prev madmin.SRPolicyMapping + for i, p := range policies { + if i == 0 { + prev = p + continue + } + if prev.IsGroup != p.IsGroup || + prev.Policy != p.Policy || + prev.UserOrGroup != p.UserOrGroup { + return false + } + } + return true +} + +func isUserInfoReplicated(cntReplicated, total int, uis []madmin.UserInfo) bool { + if cntReplicated > 0 && cntReplicated != total { + return false + } + // check if policies match between sites + var prev madmin.UserInfo + for i, ui := range uis { + if i == 0 { + prev = ui + continue + } + if !isUserInfoEqual(prev, ui) { + return false + } + } + return true +} + +func isGroupDescReplicated(cntReplicated, total int, gds []madmin.GroupDesc) bool { + if cntReplicated > 0 && cntReplicated != total { + return false + } + // check if policies match between sites + var prev madmin.GroupDesc + for i, gd := range gds { + if i == 0 { + prev = gd + continue + } + if !isGroupDescEqual(prev, gd) { + return false + } + } + return true +} + +func isBktQuotaCfgReplicated(total int, quotaCfgs []*madmin.BucketQuota) bool { + numquotaCfgs := 0 + for _, q := range quotaCfgs { + if q == nil { + continue + } + numquotaCfgs++ + } + if numquotaCfgs == 0 { + return true + } + if numquotaCfgs > 0 && numquotaCfgs != total { + return false + } + var prev *madmin.BucketQuota + for i, q := range quotaCfgs { + if q == nil { + return false + } + if i == 0 { + prev = q + continue + } + if prev.Quota != q.Quota || prev.Type != q.Type { + return false + } + } + return true +} + +// isBktPolicyReplicated returns true if count of replicated bucket policies matches total +// number of sites and bucket policies are identical. +func isBktPolicyReplicated(total int, policies []*policy.BucketPolicy) bool { + numPolicies := 0 + for _, p := range policies { + if p == nil { + continue + } + numPolicies++ + } + if numPolicies > 0 && numPolicies != total { + return false + } + // check if policies match between sites + var prev *policy.BucketPolicy + for i, p := range policies { + if p == nil { + continue + } + if i == 0 { + prev = p + continue + } + if !prev.Equals(*p) { + return false + } + } + return true +} + +// isBktReplCfgReplicated returns true if all the sites have same number +// of replication rules with all replication features enabled. +func isBktReplCfgReplicated(total int, cfgs []*sreplication.Config) bool { + cntReplicated := 0 + for _, c := range cfgs { + if c == nil { + continue + } + cntReplicated++ + } + + if cntReplicated > 0 && cntReplicated != total { + return false + } + // check if policies match between sites + var prev *sreplication.Config + for i, c := range cfgs { + if c == nil { + continue + } + if i == 0 { + prev = c + continue + } + if len(prev.Rules) != len(c.Rules) { + return false + } + if len(c.Rules) != total-1 { + return false + } + for _, r := range c.Rules { + if !strings.HasPrefix(r.ID, "site-repl-") { + return false + } + if r.DeleteMarkerReplication.Status == sreplication.Disabled || + r.DeleteReplication.Status == sreplication.Disabled || + r.ExistingObjectReplication.Status == sreplication.Disabled || + r.SourceSelectionCriteria.ReplicaModifications.Status == sreplication.Disabled { + return false + } + } + } + return true +} + +// isILMExpRuleReplicated returns true if count of replicated ILM Expiry rules matches total +// number of sites and ILM expiry rules are identical. +func isILMExpRuleReplicated(cntReplicated, total int, rules []*lifecycle.Rule) bool { + if cntReplicated > 0 && cntReplicated != total { + return false + } + // check if policies match between sites + var prev *lifecycle.Rule + for i, r := range rules { + if i == 0 { + prev = r + continue + } + // Check equality of rules + prevRData, err := xml.Marshal(prev) + if err != nil { + return false + } + rData, err := xml.Marshal(*r) + if err != nil { + return false + } + if string(prevRData) != string(rData) { + return false + } + } + return true +} + +// cache of IAM info fetched in last SiteReplicationMetaInfo call +type srIAMCache struct { + sync.RWMutex + lastUpdate time.Time + srIAMInfo madmin.SRInfo // caches IAM info +} + +func (c *SiteReplicationSys) getSRCachedIAMInfo() (info madmin.SRInfo, ok bool) { + c.iamMetaCache.RLock() + defer c.iamMetaCache.RUnlock() + if c.iamMetaCache.lastUpdate.IsZero() { + return info, false + } + if time.Since(c.iamMetaCache.lastUpdate) < siteHealTimeInterval { + return c.iamMetaCache.srIAMInfo, true + } + return info, false +} + +func (c *SiteReplicationSys) srCacheIAMInfo(info madmin.SRInfo) { + c.iamMetaCache.Lock() + defer c.iamMetaCache.Unlock() + c.iamMetaCache.srIAMInfo = info + c.iamMetaCache.lastUpdate = time.Now() +} + +// SiteReplicationMetaInfo returns the metadata info on buckets, policies etc for the replicated site +func (c *SiteReplicationSys) SiteReplicationMetaInfo(ctx context.Context, objAPI ObjectLayer, opts madmin.SRStatusOptions) (info madmin.SRInfo, err error) { + if objAPI == nil { + return info, errSRObjectLayerNotReady + } + c.RLock() + defer c.RUnlock() + if !c.enabled { + return info, nil + } + info.DeploymentID = globalDeploymentID() + if opts.Buckets || opts.Entity == madmin.SRBucketEntity { + var ( + buckets []BucketInfo + err error + ) + if opts.Entity == madmin.SRBucketEntity { + bi, err := objAPI.GetBucketInfo(ctx, opts.EntityValue, BucketOptions{Deleted: opts.ShowDeleted}) + if err != nil { + if isErrBucketNotFound(err) { + return info, nil + } + return info, errSRBackendIssue(err) + } + buckets = append(buckets, bi) + } else { + buckets, err = objAPI.ListBuckets(ctx, BucketOptions{Deleted: opts.ShowDeleted}) + if err != nil { + return info, errSRBackendIssue(err) + } + } + info.Buckets = make(map[string]madmin.SRBucketInfo, len(buckets)) + for _, bucketInfo := range buckets { + bucket := bucketInfo.Name + bucketExists := bucketInfo.Deleted.IsZero() || (!bucketInfo.Created.IsZero() && bucketInfo.Created.After(bucketInfo.Deleted)) + bms := madmin.SRBucketInfo{ + Bucket: bucket, + CreatedAt: bucketInfo.Created.UTC(), + DeletedAt: bucketInfo.Deleted.UTC(), + } + if !bucketExists { + info.Buckets[bucket] = bms + continue + } + + meta, err := globalBucketMetadataSys.GetConfigFromDisk(ctx, bucket) + if err != nil && !errors.Is(err, errConfigNotFound) { + return info, errSRBackendIssue(err) + } + + bms.Policy = meta.PolicyConfigJSON + bms.PolicyUpdatedAt = meta.PolicyConfigUpdatedAt + + if len(meta.TaggingConfigXML) > 0 { + tagCfgStr := base64.StdEncoding.EncodeToString(meta.TaggingConfigXML) + bms.Tags = &tagCfgStr + bms.TagConfigUpdatedAt = meta.TaggingConfigUpdatedAt + } + + if len(meta.VersioningConfigXML) > 0 { + versioningCfgStr := base64.StdEncoding.EncodeToString(meta.VersioningConfigXML) + bms.Versioning = &versioningCfgStr + bms.VersioningConfigUpdatedAt = meta.VersioningConfigUpdatedAt + } + + if len(meta.ObjectLockConfigXML) > 0 { + objLockStr := base64.StdEncoding.EncodeToString(meta.ObjectLockConfigXML) + bms.ObjectLockConfig = &objLockStr + bms.ObjectLockConfigUpdatedAt = meta.ObjectLockConfigUpdatedAt + } + + if len(meta.QuotaConfigJSON) > 0 { + quotaConfigStr := base64.StdEncoding.EncodeToString(meta.QuotaConfigJSON) + bms.QuotaConfig = "aConfigStr + bms.QuotaConfigUpdatedAt = meta.QuotaConfigUpdatedAt + } + + if len(meta.EncryptionConfigXML) > 0 { + sseConfigStr := base64.StdEncoding.EncodeToString(meta.EncryptionConfigXML) + bms.SSEConfig = &sseConfigStr + bms.SSEConfigUpdatedAt = meta.EncryptionConfigUpdatedAt + } + + if len(meta.ReplicationConfigXML) > 0 { + rcfgXMLStr := base64.StdEncoding.EncodeToString(meta.ReplicationConfigXML) + bms.ReplicationConfig = &rcfgXMLStr + bms.ReplicationConfigUpdatedAt = meta.ReplicationConfigUpdatedAt + } + + if meta.lifecycleConfig != nil { + var expLclCfg lifecycle.Lifecycle + expLclCfg.XMLName = meta.lifecycleConfig.XMLName + for _, rule := range meta.lifecycleConfig.Rules { + if !rule.Expiration.IsNull() || !rule.NoncurrentVersionExpiration.IsNull() { + // copy the non transition details of the rule + expLclCfg.Rules = append(expLclCfg.Rules, rule.CloneNonTransition()) + } + } + expLclCfg.ExpiryUpdatedAt = meta.lifecycleConfig.ExpiryUpdatedAt + ilmConfigData, err := xml.Marshal(expLclCfg) + if err != nil { + return info, errSRBackendIssue(err) + } + + expLclCfgStr := base64.StdEncoding.EncodeToString(ilmConfigData) + bms.ExpiryLCConfig = &expLclCfgStr + // if all non expiry rules only, ExpiryUpdatedAt would be nil + if meta.lifecycleConfig.ExpiryUpdatedAt != nil { + bms.ExpiryLCConfigUpdatedAt = *(meta.lifecycleConfig.ExpiryUpdatedAt) + } + } + + info.Buckets[bucket] = bms + } + } + + if opts.Users && opts.Groups && opts.Policies && !opts.Buckets { + // serialize SiteReplicationMetaInfo calls - if data in cache is within + // healing interval, avoid fetching IAM data again from disk. + if metaInfo, ok := c.getSRCachedIAMInfo(); ok { + return metaInfo, nil + } + } + if opts.Policies || opts.Entity == madmin.SRPolicyEntity { + var allPolicies map[string]PolicyDoc + if opts.Entity == madmin.SRPolicyEntity { + if p, err := globalIAMSys.store.GetPolicyDoc(opts.EntityValue); err == nil { + allPolicies = map[string]PolicyDoc{opts.EntityValue: p} + } + } else { + // Replicate IAM policies on local to all peers. + allPolicies, err = globalIAMSys.store.listPolicyDocs(ctx, "") + if err != nil { + return info, errSRBackendIssue(err) + } + } + info.Policies = make(map[string]madmin.SRIAMPolicy, len(allPolicies)) + for pname, policyDoc := range allPolicies { + policyJSON, err := json.Marshal(policyDoc.Policy) + if err != nil { + return info, wrapSRErr(err) + } + info.Policies[pname] = madmin.SRIAMPolicy{Policy: json.RawMessage(policyJSON), UpdatedAt: policyDoc.UpdateDate} + } + } + if opts.ILMExpiryRules || opts.Entity == madmin.SRILMExpiryRuleEntity { + info.ILMExpiryRules = make(map[string]madmin.ILMExpiryRule) + buckets, err := objAPI.ListBuckets(ctx, BucketOptions{Deleted: opts.ShowDeleted}) + if err != nil { + return info, errSRBackendIssue(err) + } + + allRules := make(map[string]madmin.ILMExpiryRule) + for _, bucketInfo := range buckets { + bucket := bucketInfo.Name + bucketExists := bucketInfo.Deleted.IsZero() || (!bucketInfo.Created.IsZero() && bucketInfo.Created.After(bucketInfo.Deleted)) + if !bucketExists { + continue + } + + meta, err := globalBucketMetadataSys.GetConfigFromDisk(ctx, bucket) + if err != nil && !errors.Is(err, errConfigNotFound) { + return info, errSRBackendIssue(err) + } + + if meta.lifecycleConfig != nil && meta.lifecycleConfig.HasExpiry() { + var updatedAt time.Time + if meta.lifecycleConfig.ExpiryUpdatedAt != nil { + updatedAt = *meta.lifecycleConfig.ExpiryUpdatedAt + } + for _, rule := range meta.lifecycleConfig.Rules { + if !rule.Expiration.IsNull() || !rule.NoncurrentVersionExpiration.IsNull() { + // copy the non transition details of the rule + ruleData, err := xml.Marshal(rule.CloneNonTransition()) + if err != nil { + return info, errSRBackendIssue(err) + } + allRules[rule.ID] = madmin.ILMExpiryRule{ILMRule: string(ruleData), Bucket: bucket, UpdatedAt: updatedAt} + } + } + } + } + if opts.Entity == madmin.SRILMExpiryRuleEntity { + if rule, ok := allRules[opts.EntityValue]; ok { + info.ILMExpiryRules[opts.EntityValue] = rule + } + } else { + maps.Copy(info.ILMExpiryRules, allRules) + } + } + if opts.PeerState { + info.State = madmin.SRStateInfo{ + Name: c.state.Name, + Peers: c.state.Peers, + UpdatedAt: c.state.UpdatedAt, + } + } + + if opts.Users || opts.Entity == madmin.SRUserEntity { + // Replicate policy mappings on local to all peers. + userPolicyMap := xsync.NewMapOf[string, MappedPolicy]() + stsPolicyMap := xsync.NewMapOf[string, MappedPolicy]() + svcPolicyMap := xsync.NewMapOf[string, MappedPolicy]() + if opts.Entity == madmin.SRUserEntity { + if mp, ok := globalIAMSys.store.GetMappedPolicy(opts.EntityValue, false); ok { + userPolicyMap.Store(opts.EntityValue, mp) + } + } else { + stsErr := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, false, stsPolicyMap) + if stsErr != nil { + return info, errSRBackendIssue(stsErr) + } + usrErr := globalIAMSys.store.loadMappedPolicies(ctx, regUser, false, userPolicyMap) + if usrErr != nil { + return info, errSRBackendIssue(usrErr) + } + svcErr := globalIAMSys.store.loadMappedPolicies(ctx, svcUser, false, svcPolicyMap) + if svcErr != nil { + return info, errSRBackendIssue(svcErr) + } + } + info.UserPolicies = make(map[string]madmin.SRPolicyMapping, userPolicyMap.Size()) + addPolicy := func(t IAMUserType, mp *xsync.MapOf[string, MappedPolicy]) { + mp.Range(func(k string, mp MappedPolicy) bool { + info.UserPolicies[k] = madmin.SRPolicyMapping{ + IsGroup: false, + UserOrGroup: k, + UserType: int(t), + Policy: mp.Policies, + UpdatedAt: mp.UpdatedAt, + } + return true + }) + } + addPolicy(regUser, userPolicyMap) + addPolicy(stsUser, stsPolicyMap) + addPolicy(svcUser, svcPolicyMap) + + info.UserInfoMap = make(map[string]madmin.UserInfo) + if opts.Entity == madmin.SRUserEntity { + if ui, err := globalIAMSys.GetUserInfo(ctx, opts.EntityValue); err == nil { + info.UserInfoMap[opts.EntityValue] = ui + } + } else { + userAccounts := make(map[string]UserIdentity) + uerr := globalIAMSys.store.loadUsers(ctx, regUser, userAccounts) + if uerr != nil { + return info, errSRBackendIssue(uerr) + } + + serr := globalIAMSys.store.loadUsers(ctx, svcUser, userAccounts) + if serr != nil { + return info, errSRBackendIssue(serr) + } + + terr := globalIAMSys.store.loadUsers(ctx, stsUser, userAccounts) + if terr != nil { + return info, errSRBackendIssue(terr) + } + + for k, v := range userAccounts { + if k == siteReplicatorSvcAcc { + // skip the site replicate svc account as it is + // already replicated. + continue + } + + if v.Credentials.ParentUser != "" && v.Credentials.ParentUser == globalActiveCred.AccessKey { + // skip all root user service accounts. + continue + } + + info.UserInfoMap[k] = madmin.UserInfo{ + Status: madmin.AccountStatus(v.Credentials.Status), + } + } + } + } + + if opts.Groups || opts.Entity == madmin.SRGroupEntity { + // Replicate policy mappings on local to all peers. + groupPolicyMap := xsync.NewMapOf[string, MappedPolicy]() + if opts.Entity == madmin.SRGroupEntity { + if mp, ok := globalIAMSys.store.GetMappedPolicy(opts.EntityValue, true); ok { + groupPolicyMap.Store(opts.EntityValue, mp) + } + } else { + stsErr := globalIAMSys.store.loadMappedPolicies(ctx, stsUser, true, groupPolicyMap) + if stsErr != nil { + return info, errSRBackendIssue(stsErr) + } + userErr := globalIAMSys.store.loadMappedPolicies(ctx, regUser, true, groupPolicyMap) + if userErr != nil { + return info, errSRBackendIssue(userErr) + } + } + + info.GroupPolicies = make(map[string]madmin.SRPolicyMapping, groupPolicyMap.Size()) + groupPolicyMap.Range(func(group string, mp MappedPolicy) bool { + info.GroupPolicies[group] = madmin.SRPolicyMapping{ + IsGroup: true, + UserOrGroup: group, + Policy: mp.Policies, + UpdatedAt: mp.UpdatedAt, + } + return true + }) + info.GroupDescMap = make(map[string]madmin.GroupDesc) + if opts.Entity == madmin.SRGroupEntity { + if gd, err := globalIAMSys.GetGroupDescription(opts.EntityValue); err == nil { + info.GroupDescMap[opts.EntityValue] = gd + } + } else { + // get users/group info on local. + groups, errG := globalIAMSys.store.listGroups(ctx) + if errG != nil { + return info, errSRBackendIssue(errG) + } + groupDescMap := make(map[string]madmin.GroupDesc, len(groups)) + for _, g := range groups { + groupDescMap[g], errG = globalIAMSys.GetGroupDescription(g) + if errG != nil { + return info, errSRBackendIssue(errG) + } + } + maps.Copy(info.GroupDescMap, groupDescMap) + } + } + // cache SR metadata info for IAM + if opts.Users && opts.Groups && opts.Policies && !opts.Buckets { + c.srCacheIAMInfo(info) + } + + return info, nil +} + +// EditPeerCluster - edits replication configuration and updates peer endpoint. +func (c *SiteReplicationSys) EditPeerCluster(ctx context.Context, peer madmin.PeerInfo, opts madmin.SREditOptions) (madmin.ReplicateEditStatus, error) { + sites, err := c.GetClusterInfo(ctx) + if err != nil { + return madmin.ReplicateEditStatus{}, errSRBackendIssue(err) + } + if !sites.Enabled { + return madmin.ReplicateEditStatus{}, errSRNotEnabled + } + + var ( + found bool + admClient *madmin.AdminClient + ) + + if globalDeploymentID() == peer.DeploymentID && !peer.SyncState.Empty() && !peer.DefaultBandwidth.IsSet { + return madmin.ReplicateEditStatus{}, errSRInvalidRequest(fmt.Errorf("a peer cluster, rather than the local cluster (endpoint=%s, deployment-id=%s) needs to be specified while setting a 'sync' replication mode", peer.Endpoint, peer.DeploymentID)) + } + + for _, v := range sites.Sites { + if peer.DeploymentID == v.DeploymentID { + found = true + if (!peer.SyncState.Empty() || peer.DefaultBandwidth.IsSet) && peer.Endpoint == "" { // peer.Endpoint may be "" if only sync state/bandwidth is being updated + break + } + if peer.Endpoint == v.Endpoint && peer.SyncState.Empty() && !peer.DefaultBandwidth.IsSet { + return madmin.ReplicateEditStatus{}, errSRInvalidRequest(fmt.Errorf("Endpoint %s entered for deployment id %s already configured in site replication", v.Endpoint, v.DeploymentID)) + } + admClient, err = c.getAdminClientWithEndpoint(ctx, v.DeploymentID, peer.Endpoint) + if err != nil { + return madmin.ReplicateEditStatus{}, errSRPeerResp(fmt.Errorf("unable to create admin client for %s: %w", v.Name, err)) + } + // check if endpoint is reachable + info, err := admClient.ServerInfo(ctx) + if err != nil { + return madmin.ReplicateEditStatus{}, errSRInvalidRequest(fmt.Errorf("Endpoint %s not reachable: %w", peer.Endpoint, err)) + } + if info.DeploymentID != v.DeploymentID { + return madmin.ReplicateEditStatus{}, errSRInvalidRequest(fmt.Errorf("Endpoint %s does not belong to deployment expected: %s (found %s) ", peer.Endpoint, v.DeploymentID, info.DeploymentID)) + } + } + } + + // if disable/enable ILM expiry replication, deployment id not needed. + // check for below error only if other options being updated (e.g. endpoint, sync, bandwidth) + if !opts.DisableILMExpiryReplication && !opts.EnableILMExpiryReplication && !found { + return madmin.ReplicateEditStatus{}, errSRInvalidRequest(fmt.Errorf("%s not found in existing replicated sites", peer.DeploymentID)) + } + successMsg := "Cluster replication configuration updated successfully with:" + var state srState + c.RLock() + state = c.state + c.RUnlock() + + // in case of --disable-ilm-expiry-replication and --enable-ilm-expiry-replication + // --deployment-id is not passed + var ( + prevPeerInfo, pi madmin.PeerInfo + ) + if peer.DeploymentID != "" { + pi = c.state.Peers[peer.DeploymentID] + prevPeerInfo = pi + if !peer.SyncState.Empty() { // update replication to peer to be sync/async + pi.SyncState = peer.SyncState + successMsg = fmt.Sprintf("%s\n- sync state %s for peer %s", successMsg, peer.SyncState, peer.Name) + } + if peer.Endpoint != "" { // `admin replicate update` requested an endpoint change + pi.Endpoint = peer.Endpoint + successMsg = fmt.Sprintf("%s\n- endpoint %s for peer %s", successMsg, peer.Endpoint, peer.Name) + } + + if peer.DefaultBandwidth.IsSet { + if peer.DeploymentID == globalDeploymentID() { + return madmin.ReplicateEditStatus{}, errSRInvalidRequest(fmt.Errorf("invalid deployment id specified: expecting a peer deployment-id to be specified for restricting bandwidth from %s, found self %s", peer.Name, globalDeploymentID())) + } + pi.DefaultBandwidth = peer.DefaultBandwidth + pi.DefaultBandwidth.UpdatedAt = UTCNow() + successMsg = fmt.Sprintf("%s\n- default bandwidth %v for peer %s", successMsg, peer.DefaultBandwidth.Limit, peer.Name) + } + state.Peers[peer.DeploymentID] = pi + } + + // If ILM expiry replications enabled/disabled, set accordingly + if opts.DisableILMExpiryReplication { + for dID, pi := range state.Peers { + if !pi.ReplicateILMExpiry { + return madmin.ReplicateEditStatus{ + Status: madmin.ReplicateAddStatusPartial, + ErrDetail: "ILM expiry already set to false", + }, nil + } + pi.ReplicateILMExpiry = false + state.Peers[dID] = pi + } + successMsg = fmt.Sprintf("%s\n- replicate-ilm-expiry: false", successMsg) + } + if opts.EnableILMExpiryReplication { + for dID, pi := range state.Peers { + if pi.ReplicateILMExpiry { + return madmin.ReplicateEditStatus{ + Status: madmin.ReplicateAddStatusPartial, + ErrDetail: "ILM expiry already set to true", + }, nil + } + pi.ReplicateILMExpiry = true + state.Peers[dID] = pi + } + successMsg = fmt.Sprintf("%s\n- replicate-ilm-expiry: true", successMsg) + } + state.UpdatedAt = time.Now() + + errs := make(map[string]error, len(state.Peers)) + var wg sync.WaitGroup + + for dID, v := range state.Peers { + if v.DeploymentID == globalDeploymentID() { + continue + } + // if individual deployment change like mode, endpoint, default bandwidth + // send it to all sites. Else send the current node details to all sites + // for ILM expiry flag update + var p madmin.PeerInfo + if peer.DeploymentID != "" { + p = pi + } else { + p = v + } + wg.Add(1) + go func(pi madmin.PeerInfo, dID string) { + defer wg.Done() + admClient, err := c.getAdminClient(ctx, dID) + if dID == peer.DeploymentID { + admClient, err = c.getAdminClientWithEndpoint(ctx, dID, pi.Endpoint) + } + if err != nil { + errs[dID] = errSRPeerResp(fmt.Errorf("unable to create admin client for %s: %w", pi.Name, err)) + return + } + if err = admClient.SRPeerEdit(ctx, pi); err != nil { + errs[dID] = errSRPeerResp(fmt.Errorf("unable to update peer %s: %w", pi.Name, err)) + return + } + }(p, dID) + } + + wg.Wait() + for dID, err := range errs { + replLogOnceIf(ctx, fmt.Errorf("unable to update peer %s: %w", state.Peers[dID].Name, err), "site-relication-edit") + } + + // we can now save the cluster replication configuration state. + if err = c.saveToDisk(ctx, state); err != nil { + return madmin.ReplicateEditStatus{ + Status: madmin.ReplicateAddStatusPartial, + ErrDetail: fmt.Sprintf("unable to save cluster-replication state on local: %v", err), + }, nil + } + + if peer.DeploymentID != "" { + if err = c.updateTargetEndpoints(ctx, prevPeerInfo, pi); err != nil { + return madmin.ReplicateEditStatus{ + Status: madmin.ReplicateAddStatusPartial, + ErrDetail: fmt.Sprintf("unable to update peer targets on local: %v", err), + }, nil + } + } + + // set partial error message if remote site updates failed for few cases + if len(errs) > 0 { + successMsg = fmt.Sprintf("%s\n- partially failed for few remote sites as they could be down/unreachable at the moment", successMsg) + } + result := madmin.ReplicateEditStatus{ + Success: true, + Status: successMsg, + } + return result, nil +} + +func (c *SiteReplicationSys) updateTargetEndpoints(ctx context.Context, prevInfo, peer madmin.PeerInfo) error { + objAPI := newObjectLayerFn() + if objAPI == nil { + return errSRObjectLayerNotReady + } + + buckets, err := objAPI.ListBuckets(ctx, BucketOptions{}) + if err != nil { + return err + } + + for _, bucketInfo := range buckets { + bucket := bucketInfo.Name + ep, _ := url.Parse(peer.Endpoint) + prevEp, _ := url.Parse(prevInfo.Endpoint) + targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket) + if err != nil { + continue // site healing will take care of configuring new targets + } + for _, target := range targets.Targets { + if target.SourceBucket == bucket && + target.TargetBucket == bucket && + target.Endpoint == prevEp.Host && + target.Secure == (prevEp.Scheme == "https") && + target.Type == madmin.ReplicationService { + bucketTarget := target + bucketTarget.Secure = ep.Scheme == "https" + bucketTarget.Endpoint = ep.Host + if peer.DefaultBandwidth.IsSet && target.BandwidthLimit == 0 { + bucketTarget.BandwidthLimit = int64(peer.DefaultBandwidth.Limit) + } + if !peer.SyncState.Empty() { + bucketTarget.ReplicationSync = (peer.SyncState == madmin.SyncEnabled) + } + err := globalBucketTargetSys.SetTarget(ctx, bucket, &bucketTarget, true) + if err != nil { + replLogIf(ctx, c.annotatePeerErr(peer.Name, "Bucket target creation error", err)) + continue + } + targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket) + if err != nil { + replLogIf(ctx, err) + continue + } + tgtBytes, err := json.Marshal(&targets) + if err != nil { + bugLogIf(ctx, err) + continue + } + if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil { + replLogIf(ctx, err) + continue + } + } + } + } + return nil +} + +// PeerEditReq - internal API handler to respond to a peer cluster's request +// to edit endpoint. +func (c *SiteReplicationSys) PeerEditReq(ctx context.Context, arg madmin.PeerInfo) error { + ourName := "" + + // Set ReplicateILMExpiry for all peers + currTime := time.Now() + for i := range c.state.Peers { + p := c.state.Peers[i] + if p.ReplicateILMExpiry == arg.ReplicateILMExpiry { + // its already set due to previous edit req + break + } + p.ReplicateILMExpiry = arg.ReplicateILMExpiry + c.state.UpdatedAt = currTime + c.state.Peers[i] = p + } + + for i := range c.state.Peers { + p := c.state.Peers[i] + if p.DeploymentID == arg.DeploymentID { + p.Endpoint = arg.Endpoint + c.state.Peers[arg.DeploymentID] = p + } + if p.DeploymentID == globalDeploymentID() { + ourName = p.Name + } + } + if err := c.saveToDisk(ctx, c.state); err != nil { + return errSRBackendIssue(fmt.Errorf("unable to save cluster-replication state to drive on %s: %v", ourName, err)) + } + return nil +} + +// PeerStateEditReq - internal API handler to respond to a peer cluster's request +// to edit state. +func (c *SiteReplicationSys) PeerStateEditReq(ctx context.Context, arg madmin.SRStateEditReq) error { + if arg.UpdatedAt.After(c.state.UpdatedAt) { + state := c.state + // update only the ReplicateILMExpiry flag for the peers from incoming request + for _, peer := range arg.Peers { + currPeer := c.state.Peers[peer.DeploymentID] + currPeer.ReplicateILMExpiry = peer.ReplicateILMExpiry + state.Peers[peer.DeploymentID] = currPeer + } + state.UpdatedAt = arg.UpdatedAt + if err := c.saveToDisk(ctx, state); err != nil { + return errSRBackendIssue(fmt.Errorf("unable to save cluster-replication state to drive on %s: %v", state.Name, err)) + } + } + return nil +} + +const siteHealTimeInterval = 30 * time.Second + +func (c *SiteReplicationSys) startHealRoutine(ctx context.Context, objAPI ObjectLayer) { + ctx, cancel := globalLeaderLock.GetLock(ctx) + defer cancel() + + healTimer := time.NewTimer(siteHealTimeInterval) + defer healTimer.Stop() + + var maxRefreshDurationSecondsForLog float64 = 10 // 10 seconds.. + + for { + select { + case <-healTimer.C: + c.RLock() + enabled := c.enabled + c.RUnlock() + if enabled { + refreshStart := time.Now() + c.healIAMSystem(ctx, objAPI) // heal IAM system first + c.healBuckets(ctx, objAPI) // heal buckets subsequently + + took := time.Since(refreshStart).Seconds() + if took > maxRefreshDurationSecondsForLog { + // Log if we took a lot of time. + logger.Info("Site replication healing refresh took %.2fs", took) + } + + // wait for 200 millisecond, if we are experience lot of I/O + waitForLowIO(runtime.GOMAXPROCS(0), 200*time.Millisecond, currentHTTPIO) + } + healTimer.Reset(siteHealTimeInterval) + + case <-ctx.Done(): + return + } + } +} + +type srBucketStatsSummary struct { + madmin.SRBucketStatsSummary + meta srBucketMetaInfo +} + +type srPolicyStatsSummary struct { + madmin.SRPolicyStatsSummary + policy srPolicy +} + +type srUserStatsSummary struct { + madmin.SRUserStatsSummary + userInfo srUserInfo + userPolicy srPolicyMapping +} + +type srGroupStatsSummary struct { + madmin.SRGroupStatsSummary + groupDesc srGroupDesc + groupPolicy srPolicyMapping +} + +type srILMExpiryRuleStatsSummary struct { + madmin.SRILMExpiryStatsSummary + ilmExpiryRule srILMExpiryRule +} + +type srStatusInfo struct { + // SRStatusInfo returns detailed status on site replication status + Enabled bool + MaxBuckets int // maximum buckets seen across sites + MaxUsers int // maximum users seen across sites + MaxGroups int // maximum groups seen across sites + MaxPolicies int // maximum policies across sites + MaxILMExpiryRules int // maximum ILM expiry rules across sites + Sites map[string]madmin.PeerInfo // deployment->sitename + StatsSummary map[string]madmin.SRSiteSummary // map of deployment id -> site stat + // BucketStats map of bucket to slice of deployment IDs with stats. This is populated only if there are + // mismatches or if a specific bucket's stats are requested + BucketStats map[string]map[string]srBucketStatsSummary + // PolicyStats map of policy to slice of deployment IDs with stats. This is populated only if there are + // mismatches or if a specific bucket's stats are requested + PolicyStats map[string]map[string]srPolicyStatsSummary + // UserStats map of user to slice of deployment IDs with stats. This is populated only if there are + // mismatches or if a specific bucket's stats are requested + UserStats map[string]map[string]srUserStatsSummary + // GroupStats map of group to slice of deployment IDs with stats. This is populated only if there are + // mismatches or if a specific bucket's stats are requested + GroupStats map[string]map[string]srGroupStatsSummary + // ILMExpiryRulesStats map of ILM expiry rules to slice of deployment IDs with stats. This is populated only if there are + // mismatches or if a specific ILM expiry rule's stats are requested + ILMExpiryRulesStats map[string]map[string]srILMExpiryRuleStatsSummary + // PeerStates map of site replication sites to their site replication states + PeerStates map[string]madmin.SRStateInfo + Metrics madmin.SRMetricsSummary + UpdatedAt time.Time +} + +// SRBucketDeleteOp - type of delete op +type SRBucketDeleteOp string + +const ( + // MarkDelete creates .minio.sys/buckets/.deleted/ vol entry to hold onto deleted bucket's state + // until peers are synced in site replication setup. + MarkDelete SRBucketDeleteOp = "MarkDelete" + + // Purge deletes the .minio.sys/buckets/.deleted/ vol entry + Purge SRBucketDeleteOp = "Purge" + // NoOp no action needed + NoOp SRBucketDeleteOp = "NoOp" +) + +// Empty returns true if this Op is not set +func (s SRBucketDeleteOp) Empty() bool { + return string(s) == "" || string(s) == string(NoOp) +} + +func getSRBucketDeleteOp(isSiteReplicated bool) SRBucketDeleteOp { + if !isSiteReplicated { + return NoOp + } + return MarkDelete +} + +func (c *SiteReplicationSys) healILMExpiryConfig(ctx context.Context, objAPI ObjectLayer, info srStatusInfo) error { + c.RLock() + defer c.RUnlock() + if !c.enabled { + return nil + } + var ( + latestID, latestPeerName string + lastUpdate time.Time + latestPeers map[string]madmin.PeerInfo + ) + + for dID, ps := range info.PeerStates { + if lastUpdate.IsZero() { + lastUpdate = ps.UpdatedAt + latestID = dID + latestPeers = ps.Peers + } + if ps.UpdatedAt.After(lastUpdate) { + lastUpdate = ps.UpdatedAt + latestID = dID + latestPeers = ps.Peers + } + } + latestPeerName = info.Sites[latestID].Name + + for dID, ps := range info.PeerStates { + // If latest peers ILM expiry flags are equal to current peer, no need to heal + flagEqual := true + for id, peer := range latestPeers { + if ps.Peers[id].ReplicateILMExpiry != peer.ReplicateILMExpiry { + flagEqual = false + break + } + } + if flagEqual { + continue + } + + // Dont apply the self state to self + if dID == globalDeploymentID() { + continue + } + + // Send details to other sites for healing + admClient, err := c.getAdminClient(ctx, dID) + if err != nil { + return wrapSRErr(err) + } + if err = admClient.SRStateEdit(ctx, madmin.SRStateEditReq{Peers: latestPeers, UpdatedAt: lastUpdate}); err != nil { + replLogIf(ctx, c.annotatePeerErr(ps.Name, siteReplicationEdit, + fmt.Errorf("Unable to heal site replication state for peer %s from peer %s : %w", + ps.Name, latestPeerName, err))) + } + } + return nil +} + +func (c *SiteReplicationSys) healBuckets(ctx context.Context, objAPI ObjectLayer) error { + buckets, err := c.listBuckets(ctx) + if err != nil { + return err + } + ilmExpiryCfgHealed := false + opts := validateReplicationDestinationOptions{CheckReady: true} + for _, bi := range buckets { + bucket := bi.Name + info, err := c.siteReplicationStatus(ctx, objAPI, madmin.SRStatusOptions{ + Entity: madmin.SRBucketEntity, + EntityValue: bucket, + ShowDeleted: true, + ILMExpiryRules: true, + PeerState: true, + }) + if err != nil { + return err + } + + c.healBucket(ctx, objAPI, bucket, info) + + if bi.Deleted.IsZero() || (!bi.Created.IsZero() && bi.Deleted.Before(bi.Created)) { + c.healVersioningMetadata(ctx, objAPI, bucket, info) + c.healOLockConfigMetadata(ctx, objAPI, bucket, info) + c.healSSEMetadata(ctx, objAPI, bucket, info) + c.healBucketReplicationConfig(ctx, objAPI, bucket, info, &opts) + c.healBucketPolicies(ctx, objAPI, bucket, info) + c.healTagMetadata(ctx, objAPI, bucket, info) + c.healBucketQuotaConfig(ctx, objAPI, bucket, info) + if !ilmExpiryCfgHealed { + c.healILMExpiryConfig(ctx, objAPI, info) + ilmExpiryCfgHealed = true + } + if ilmExpiryReplicationEnabled(c.state.Peers) { + c.healBucketILMExpiry(ctx, objAPI, bucket, info) + } + } + // Notification and ILM are site specific settings. + } + return nil +} + +func (c *SiteReplicationSys) healBucketILMExpiry(ctx context.Context, objAPI ObjectLayer, bucket string, info srStatusInfo) error { + bs := info.BucketStats[bucket] + + c.RLock() + defer c.RUnlock() + if !c.enabled { + return nil + } + + var ( + latestID, latestPeerName string + lastUpdate time.Time + latestExpLCConfig *string + ) + + for dID, ss := range bs { + if lastUpdate.IsZero() { + lastUpdate = ss.meta.ExpiryLCConfigUpdatedAt + latestID = dID + latestExpLCConfig = ss.meta.ExpiryLCConfig + } + // avoid considering just created buckets as latest. Perhaps this site + // just joined cluster replication and yet to be sync'd + if ss.meta.CreatedAt.Equal(ss.meta.ExpiryLCConfigUpdatedAt) { + continue + } + if ss.meta.ExpiryLCConfigUpdatedAt.After(lastUpdate) { + lastUpdate = ss.meta.ExpiryLCConfigUpdatedAt + latestID = dID + latestExpLCConfig = ss.meta.ExpiryLCConfig + } + } + latestPeerName = info.Sites[latestID].Name + var err error + if latestExpLCConfig != nil { + _, err = base64.StdEncoding.DecodeString(*latestExpLCConfig) + if err != nil { + return err + } + } + + for dID, bStatus := range bs { + if latestExpLCConfig != nil && bStatus.meta.ExpiryLCConfig != nil && strings.EqualFold(*latestExpLCConfig, *bStatus.meta.ExpiryLCConfig) { + continue + } + + finalConfigData, err := mergeWithCurrentLCConfig(ctx, bucket, latestExpLCConfig, lastUpdate) + if err != nil { + return wrapSRErr(err) + } + + if dID == globalDeploymentID() { + if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketLifecycleConfig, finalConfigData); err != nil { + replLogIf(ctx, fmt.Errorf("Unable to heal bucket ILM expiry data from peer site %s : %w", latestPeerName, err)) + } + continue + } + + admClient, err := c.getAdminClient(ctx, dID) + if err != nil { + return wrapSRErr(err) + } + peerName := info.Sites[dID].Name + if err = admClient.SRPeerReplicateBucketMeta(ctx, madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaLCConfig, + Bucket: bucket, + ExpiryLCConfig: latestExpLCConfig, + UpdatedAt: lastUpdate, + }); err != nil { + replLogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata, + fmt.Errorf("Unable to heal bucket ILM expiry data for peer %s from peer %s : %w", + peerName, latestPeerName, err))) + } + } + return nil +} + +func (c *SiteReplicationSys) healTagMetadata(ctx context.Context, objAPI ObjectLayer, bucket string, info srStatusInfo) error { + bs := info.BucketStats[bucket] + + c.RLock() + defer c.RUnlock() + if !c.enabled { + return nil + } + var ( + latestID, latestPeerName string + lastUpdate time.Time + latestTaggingConfig *string + ) + + for dID, ss := range bs { + if lastUpdate.IsZero() { + lastUpdate = ss.meta.TagConfigUpdatedAt + latestID = dID + latestTaggingConfig = ss.meta.Tags + } + // avoid considering just created buckets as latest. Perhaps this site + // just joined cluster replication and yet to be sync'd + if ss.meta.CreatedAt.Equal(ss.meta.TagConfigUpdatedAt) { + continue + } + if ss.meta.TagConfigUpdatedAt.After(lastUpdate) { + lastUpdate = ss.meta.TagConfigUpdatedAt + latestID = dID + latestTaggingConfig = ss.meta.Tags + } + } + latestPeerName = info.Sites[latestID].Name + var latestTaggingConfigBytes []byte + var err error + if latestTaggingConfig != nil { + latestTaggingConfigBytes, err = base64.StdEncoding.DecodeString(*latestTaggingConfig) + if err != nil { + return err + } + } + for dID, bStatus := range bs { + if !bStatus.TagMismatch { + continue + } + if isBucketMetadataEqual(latestTaggingConfig, bStatus.meta.Tags) { + continue + } + if dID == globalDeploymentID() { + if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketTaggingConfig, latestTaggingConfigBytes); err != nil { + replLogIf(ctx, fmt.Errorf("Unable to heal tagging metadata from peer site %s : %w", latestPeerName, err)) + } + continue + } + + admClient, err := c.getAdminClient(ctx, dID) + if err != nil { + return wrapSRErr(err) + } + peerName := info.Sites[dID].Name + err = admClient.SRPeerReplicateBucketMeta(ctx, madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaTypeTags, + Bucket: bucket, + Tags: latestTaggingConfig, + }) + if err != nil { + replLogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata, + fmt.Errorf("Unable to heal tagging metadata for peer %s from peer %s : %w", peerName, latestPeerName, err))) + } + } + return nil +} + +func (c *SiteReplicationSys) healBucketPolicies(ctx context.Context, objAPI ObjectLayer, bucket string, info srStatusInfo) error { + bs := info.BucketStats[bucket] + + c.RLock() + defer c.RUnlock() + if !c.enabled { + return nil + } + var ( + latestID, latestPeerName string + lastUpdate time.Time + latestIAMPolicy json.RawMessage + ) + + for dID, ss := range bs { + if lastUpdate.IsZero() { + lastUpdate = ss.meta.PolicyUpdatedAt + latestID = dID + latestIAMPolicy = ss.meta.Policy + } + // avoid considering just created buckets as latest. Perhaps this site + // just joined cluster replication and yet to be sync'd + if ss.meta.CreatedAt.Equal(ss.meta.PolicyUpdatedAt) { + continue + } + if ss.meta.PolicyUpdatedAt.After(lastUpdate) { + lastUpdate = ss.meta.PolicyUpdatedAt + latestID = dID + latestIAMPolicy = ss.meta.Policy + } + } + latestPeerName = info.Sites[latestID].Name + for dID, bStatus := range bs { + if !bStatus.PolicyMismatch { + continue + } + if strings.EqualFold(string(latestIAMPolicy), string(bStatus.meta.Policy)) { + continue + } + if dID == globalDeploymentID() { + if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketPolicyConfig, latestIAMPolicy); err != nil { + replLogIf(ctx, fmt.Errorf("Unable to heal bucket policy metadata from peer site %s : %w", latestPeerName, err)) + } + continue + } + + admClient, err := c.getAdminClient(ctx, dID) + if err != nil { + return wrapSRErr(err) + } + peerName := info.Sites[dID].Name + if err = admClient.SRPeerReplicateBucketMeta(ctx, madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaTypePolicy, + Bucket: bucket, + Policy: latestIAMPolicy, + UpdatedAt: lastUpdate, + }); err != nil { + replLogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata, + fmt.Errorf("Unable to heal bucket policy metadata for peer %s from peer %s : %w", + peerName, latestPeerName, err))) + } + } + return nil +} + +func (c *SiteReplicationSys) healBucketQuotaConfig(ctx context.Context, objAPI ObjectLayer, bucket string, info srStatusInfo) error { + bs := info.BucketStats[bucket] + + c.RLock() + defer c.RUnlock() + if !c.enabled { + return nil + } + var ( + latestID, latestPeerName string + lastUpdate time.Time + latestQuotaConfig *string + latestQuotaConfigBytes []byte + ) + + for dID, ss := range bs { + if lastUpdate.IsZero() { + lastUpdate = ss.meta.QuotaConfigUpdatedAt + latestID = dID + latestQuotaConfig = ss.meta.QuotaConfig + } + // avoid considering just created buckets as latest. Perhaps this site + // just joined cluster replication and yet to be sync'd + if ss.meta.CreatedAt.Equal(ss.meta.QuotaConfigUpdatedAt) { + continue + } + if ss.meta.QuotaConfigUpdatedAt.After(lastUpdate) { + lastUpdate = ss.meta.QuotaConfigUpdatedAt + latestID = dID + latestQuotaConfig = ss.meta.QuotaConfig + } + } + + var err error + if latestQuotaConfig != nil { + latestQuotaConfigBytes, err = base64.StdEncoding.DecodeString(*latestQuotaConfig) + if err != nil { + return err + } + } + + latestPeerName = info.Sites[latestID].Name + for dID, bStatus := range bs { + if !bStatus.QuotaCfgMismatch { + continue + } + if isBucketMetadataEqual(latestQuotaConfig, bStatus.meta.QuotaConfig) { + continue + } + if dID == globalDeploymentID() { + if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketQuotaConfigFile, latestQuotaConfigBytes); err != nil { + replLogIf(ctx, fmt.Errorf("Unable to heal quota metadata from peer site %s : %w", latestPeerName, err)) + } + continue + } + + admClient, err := c.getAdminClient(ctx, dID) + if err != nil { + return wrapSRErr(err) + } + peerName := info.Sites[dID].Name + + if err = admClient.SRPeerReplicateBucketMeta(ctx, madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaTypeQuotaConfig, + Bucket: bucket, + Quota: latestQuotaConfigBytes, + UpdatedAt: lastUpdate, + }); err != nil { + replLogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata, + fmt.Errorf("Unable to heal quota config metadata for peer %s from peer %s : %w", + peerName, latestPeerName, err))) + } + } + return nil +} + +func (c *SiteReplicationSys) healVersioningMetadata(ctx context.Context, objAPI ObjectLayer, bucket string, info srStatusInfo) error { + c.RLock() + defer c.RUnlock() + if !c.enabled { + return nil + } + var ( + latestID, latestPeerName string + lastUpdate time.Time + latestVersioningConfig *string + ) + + bs := info.BucketStats[bucket] + for dID, ss := range bs { + if lastUpdate.IsZero() { + lastUpdate = ss.meta.VersioningConfigUpdatedAt + latestID = dID + latestVersioningConfig = ss.meta.Versioning + } + // avoid considering just created buckets as latest. Perhaps this site + // just joined cluster replication and yet to be sync'd + if ss.meta.CreatedAt.Equal(ss.meta.VersioningConfigUpdatedAt) { + continue + } + if ss.meta.VersioningConfigUpdatedAt.After(lastUpdate) { + lastUpdate = ss.meta.VersioningConfigUpdatedAt + latestID = dID + latestVersioningConfig = ss.meta.Versioning + } + } + + latestPeerName = info.Sites[latestID].Name + var latestVersioningConfigBytes []byte + var err error + if latestVersioningConfig != nil { + latestVersioningConfigBytes, err = base64.StdEncoding.DecodeString(*latestVersioningConfig) + if err != nil { + return err + } + } + + for dID, bStatus := range bs { + if !bStatus.VersioningConfigMismatch { + continue + } + if isBucketMetadataEqual(latestVersioningConfig, bStatus.meta.Versioning) { + continue + } + if dID == globalDeploymentID() { + if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketVersioningConfig, latestVersioningConfigBytes); err != nil { + replLogIf(ctx, fmt.Errorf("Unable to heal versioning metadata from peer site %s : %w", latestPeerName, err)) + } + continue + } + + admClient, err := c.getAdminClient(ctx, dID) + if err != nil { + return wrapSRErr(err) + } + peerName := info.Sites[dID].Name + err = admClient.SRPeerReplicateBucketMeta(ctx, madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaTypeVersionConfig, + Bucket: bucket, + Versioning: latestVersioningConfig, + UpdatedAt: lastUpdate, + }) + if err != nil { + replLogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata, + fmt.Errorf("Unable to heal versioning config metadata for peer %s from peer %s : %w", + peerName, latestPeerName, err))) + } + } + return nil +} + +func (c *SiteReplicationSys) healSSEMetadata(ctx context.Context, objAPI ObjectLayer, bucket string, info srStatusInfo) error { + c.RLock() + defer c.RUnlock() + if !c.enabled { + return nil + } + var ( + latestID, latestPeerName string + lastUpdate time.Time + latestSSEConfig *string + ) + + bs := info.BucketStats[bucket] + for dID, ss := range bs { + if lastUpdate.IsZero() { + lastUpdate = ss.meta.SSEConfigUpdatedAt + latestID = dID + latestSSEConfig = ss.meta.SSEConfig + } + // avoid considering just created buckets as latest. Perhaps this site + // just joined cluster replication and yet to be sync'd + if ss.meta.CreatedAt.Equal(ss.meta.SSEConfigUpdatedAt) { + continue + } + if ss.meta.SSEConfigUpdatedAt.After(lastUpdate) { + lastUpdate = ss.meta.SSEConfigUpdatedAt + latestID = dID + latestSSEConfig = ss.meta.SSEConfig + } + } + + latestPeerName = info.Sites[latestID].Name + var latestSSEConfigBytes []byte + var err error + if latestSSEConfig != nil { + latestSSEConfigBytes, err = base64.StdEncoding.DecodeString(*latestSSEConfig) + if err != nil { + return err + } + } + + for dID, bStatus := range bs { + if !bStatus.SSEConfigMismatch { + continue + } + if isBucketMetadataEqual(latestSSEConfig, bStatus.meta.SSEConfig) { + continue + } + if dID == globalDeploymentID() { + if _, err := globalBucketMetadataSys.Update(ctx, bucket, bucketSSEConfig, latestSSEConfigBytes); err != nil { + replLogIf(ctx, fmt.Errorf("Unable to heal sse metadata from peer site %s : %w", latestPeerName, err)) + } + continue + } + + admClient, err := c.getAdminClient(ctx, dID) + if err != nil { + return wrapSRErr(err) + } + peerName := info.Sites[dID].Name + err = admClient.SRPeerReplicateBucketMeta(ctx, madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaTypeSSEConfig, + Bucket: bucket, + SSEConfig: latestSSEConfig, + UpdatedAt: lastUpdate, + }) + if err != nil { + replLogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata, + fmt.Errorf("Unable to heal SSE config metadata for peer %s from peer %s : %w", + peerName, latestPeerName, err))) + } + } + return nil +} + +func (c *SiteReplicationSys) healOLockConfigMetadata(ctx context.Context, objAPI ObjectLayer, bucket string, info srStatusInfo) error { + bs := info.BucketStats[bucket] + + c.RLock() + defer c.RUnlock() + if !c.enabled { + return nil + } + var ( + latestID, latestPeerName string + lastUpdate time.Time + latestObjLockConfig *string + ) + + for dID, ss := range bs { + if lastUpdate.IsZero() { + lastUpdate = ss.meta.ObjectLockConfigUpdatedAt + latestID = dID + latestObjLockConfig = ss.meta.ObjectLockConfig + } + // avoid considering just created buckets as latest. Perhaps this site + // just joined cluster replication and yet to be sync'd + if ss.meta.CreatedAt.Equal(ss.meta.ObjectLockConfigUpdatedAt) { + continue + } + if ss.meta.ObjectLockConfig != nil && ss.meta.ObjectLockConfigUpdatedAt.After(lastUpdate) { + lastUpdate = ss.meta.ObjectLockConfigUpdatedAt + latestID = dID + latestObjLockConfig = ss.meta.ObjectLockConfig + } + } + latestPeerName = info.Sites[latestID].Name + var latestObjLockConfigBytes []byte + var err error + if latestObjLockConfig != nil { + latestObjLockConfigBytes, err = base64.StdEncoding.DecodeString(*latestObjLockConfig) + if err != nil { + return err + } + } + + for dID, bStatus := range bs { + if !bStatus.OLockConfigMismatch { + continue + } + if isBucketMetadataEqual(latestObjLockConfig, bStatus.meta.ObjectLockConfig) { + continue + } + if dID == globalDeploymentID() { + if _, err := globalBucketMetadataSys.Update(ctx, bucket, objectLockConfig, latestObjLockConfigBytes); err != nil { + replLogIf(ctx, fmt.Errorf("Unable to heal objectlock config metadata from peer site %s : %w", latestPeerName, err)) + } + continue + } + + admClient, err := c.getAdminClient(ctx, dID) + if err != nil { + return wrapSRErr(err) + } + peerName := info.Sites[dID].Name + err = admClient.SRPeerReplicateBucketMeta(ctx, madmin.SRBucketMeta{ + Type: madmin.SRBucketMetaTypeObjectLockConfig, + Bucket: bucket, + Tags: latestObjLockConfig, + UpdatedAt: lastUpdate, + }) + if err != nil { + replLogIf(ctx, c.annotatePeerErr(peerName, replicateBucketMetadata, + fmt.Errorf("Unable to heal object lock config metadata for peer %s from peer %s : %w", + peerName, latestPeerName, err))) + } + } + return nil +} + +func (c *SiteReplicationSys) purgeDeletedBucket(ctx context.Context, objAPI ObjectLayer, bucket string) { + z, ok := objAPI.(*erasureServerPools) + if !ok { + return + } + z.s3Peer.DeleteBucket(context.Background(), pathJoin(minioMetaBucket, bucketMetaPrefix, deletedBucketsPrefix, bucket), DeleteBucketOptions{}) +} + +// healBucket creates/deletes the bucket according to latest state across clusters participating in site replication. +func (c *SiteReplicationSys) healBucket(ctx context.Context, objAPI ObjectLayer, bucket string, info srStatusInfo) error { + bs := info.BucketStats[bucket] + c.RLock() + defer c.RUnlock() + if !c.enabled { + return nil + } + numSites := len(c.state.Peers) + mostRecent := func(d1, d2 time.Time) time.Time { + if d1.IsZero() { + return d2 + } + if d2.IsZero() { + return d1 + } + if d1.After(d2) { + return d1 + } + return d2 + } + + var ( + latestID string + lastUpdate time.Time + withB []string + missingB []string + deletedCnt int + ) + for dID, ss := range bs { + if lastUpdate.IsZero() { + lastUpdate = mostRecent(ss.meta.CreatedAt, ss.meta.DeletedAt) + latestID = dID + } + recentUpdt := mostRecent(ss.meta.CreatedAt, ss.meta.DeletedAt) + if recentUpdt.After(lastUpdate) { + lastUpdate = recentUpdt + latestID = dID + } + if ss.BucketMarkedDeleted { + deletedCnt++ + } + if ss.HasBucket { + withB = append(withB, dID) + } else { + missingB = append(missingB, dID) + } + } + + latestPeerName := info.Sites[latestID].Name + bStatus := info.BucketStats[bucket][latestID].meta + isMakeBucket := len(missingB) > 0 + deleteOp := NoOp + if latestID != globalDeploymentID() { + return nil + } + if lastUpdate.Equal(bStatus.DeletedAt) { + isMakeBucket = false + switch { + case len(withB) == numSites && deletedCnt == numSites: + deleteOp = NoOp + case len(withB) == 0 && len(missingB) == numSites: + deleteOp = Purge + default: + deleteOp = MarkDelete + } + } + if isMakeBucket { + var opts MakeBucketOptions + optsMap := make(map[string]string) + optsMap["versioningEnabled"] = "true" + opts.VersioningEnabled = true + opts.CreatedAt = bStatus.CreatedAt + optsMap["createdAt"] = bStatus.CreatedAt.UTC().Format(time.RFC3339Nano) + + if bStatus.ObjectLockConfig != nil { + config, err := base64.StdEncoding.DecodeString(*bStatus.ObjectLockConfig) + if err != nil { + return err + } + if bytes.Equal([]byte(string(config)), enabledBucketObjectLockConfig) { + optsMap["lockEnabled"] = "true" + opts.LockEnabled = true + } + } + for _, dID := range missingB { + peerName := info.Sites[dID].Name + if dID == globalDeploymentID() { + err := c.PeerBucketMakeWithVersioningHandler(ctx, bucket, opts) + if err != nil { + return c.annotateErr(makeBucketWithVersion, fmt.Errorf("error healing bucket for site replication %w from %s -> %s", + err, latestPeerName, peerName)) + } + } else { + admClient, err := c.getAdminClient(ctx, dID) + if err != nil { + return c.annotateErr(configureReplication, fmt.Errorf("unable to use admin client for %s: %w", dID, err)) + } + if err = admClient.SRPeerBucketOps(ctx, bucket, madmin.MakeWithVersioningBktOp, optsMap); err != nil { + return c.annotatePeerErr(peerName, makeBucketWithVersion, err) + } + if err = admClient.SRPeerBucketOps(ctx, bucket, madmin.ConfigureReplBktOp, nil); err != nil { + return c.annotatePeerErr(peerName, configureReplication, err) + } + } + } + if len(missingB) > 0 { + // configure replication from current cluster to other clusters + err := c.PeerBucketConfigureReplHandler(ctx, bucket) + if err != nil { + return c.annotateErr(configureReplication, err) + } + } + return nil + } + // all buckets are marked deleted across sites at this point. It should be safe to purge the .minio.sys/buckets/.deleted/ entry + // from disk + if deleteOp == Purge { + for _, dID := range missingB { + peerName := info.Sites[dID].Name + if dID == globalDeploymentID() { + c.purgeDeletedBucket(ctx, objAPI, bucket) + } else { + admClient, err := c.getAdminClient(ctx, dID) + if err != nil { + return c.annotateErr(configureReplication, fmt.Errorf("unable to use admin client for %s: %w", dID, err)) + } + if err = admClient.SRPeerBucketOps(ctx, bucket, madmin.PurgeDeletedBucketOp, nil); err != nil { + return c.annotatePeerErr(peerName, deleteBucket, err) + } + } + } + } + // Mark buckets deleted on remaining peers + if deleteOp == MarkDelete { + for _, dID := range withB { + peerName := info.Sites[dID].Name + if dID == globalDeploymentID() { + err := c.PeerBucketDeleteHandler(ctx, bucket, DeleteBucketOptions{ + Force: true, + }) + if err != nil { + return c.annotateErr(deleteBucket, fmt.Errorf("error healing bucket for site replication %w from %s -> %s", + err, latestPeerName, peerName)) + } + } else { + admClient, err := c.getAdminClient(ctx, dID) + if err != nil { + return c.annotateErr(configureReplication, fmt.Errorf("unable to use admin client for %s: %w", dID, err)) + } + if err = admClient.SRPeerBucketOps(ctx, bucket, madmin.ForceDeleteBucketBktOp, nil); err != nil { + return c.annotatePeerErr(peerName, deleteBucket, err) + } + } + } + } + + return nil +} + +func (c *SiteReplicationSys) healBucketReplicationConfig(ctx context.Context, objAPI ObjectLayer, bucket string, info srStatusInfo, opts *validateReplicationDestinationOptions) error { + bs := info.BucketStats[bucket] + + c.RLock() + defer c.RUnlock() + if !c.enabled { + return nil + } + + var replMismatch bool + for _, ss := range bs { + if ss.ReplicationCfgMismatch { + replMismatch = true + break + } + } + rcfg, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket) + if err != nil { + _, ok := err.(BucketReplicationConfigNotFound) + if !ok { + return err + } + replMismatch = true + } + + var ( + epDeplIDMap = make(map[string]string) + arnTgtMap = make(map[string]madmin.BucketTarget) + ) + if targetsPtr, _ := globalBucketTargetSys.ListBucketTargets(ctx, bucket); targetsPtr != nil { + for _, t := range targetsPtr.Targets { + arnTgtMap[t.Arn] = t + } + } + for _, p := range c.state.Peers { + epDeplIDMap[p.Endpoint] = p.DeploymentID + } + // fix stale ARN's in replication config and endpoint mismatch between site config and + // targets associated to this config. + if rcfg != nil { + for _, rule := range rcfg.Rules { + if rule.Status != sreplication.Status(replication.Disabled) { + tgt, isValidARN := arnTgtMap[rule.Destination.ARN] // detect stale ARN in replication config + _, epFound := epDeplIDMap[tgt.URL().String()] // detect end point change at site level + if !isValidARN || !epFound { + replMismatch = true + break + } + } + } + } + + if rcfg != nil && !replMismatch { + // validate remote targets on current cluster for this bucket + _, apiErr := validateReplicationDestination(ctx, bucket, rcfg, opts) + if apiErr != noError { + replMismatch = true + } + } + + if replMismatch { + replLogOnceIf(ctx, c.annotateErr(configureReplication, c.PeerBucketConfigureReplHandler(ctx, bucket)), "heal-bucket-relication-config") + } + return nil +} + +func isBucketMetadataEqual(one, two *string) bool { + switch { + case one == nil && two == nil: + return true + case one == nil || two == nil: + return false + default: + return strings.EqualFold(*one, *two) + } +} + +func (c *SiteReplicationSys) healIAMSystem(ctx context.Context, objAPI ObjectLayer) error { + info, err := c.siteReplicationStatus(ctx, objAPI, madmin.SRStatusOptions{ + Users: true, + Policies: true, + Groups: true, + }) + if err != nil { + return err + } + for policy := range info.PolicyStats { + c.healPolicies(ctx, objAPI, policy, info) + } + for user := range info.UserStats { + c.healUsers(ctx, objAPI, user, info) + } + for group := range info.GroupStats { + c.healGroups(ctx, objAPI, group, info) + } + for user := range info.UserStats { + c.healUserPolicies(ctx, objAPI, user, info) + } + for group := range info.GroupStats { + c.healGroupPolicies(ctx, objAPI, group, info) + } + + return nil +} + +// heal iam policies present on this site to peers, provided current cluster has the most recent update. +func (c *SiteReplicationSys) healPolicies(ctx context.Context, objAPI ObjectLayer, policy string, info srStatusInfo) error { + // create IAM policy on peer cluster if missing + ps := info.PolicyStats[policy] + + c.RLock() + defer c.RUnlock() + if !c.enabled { + return nil + } + + var ( + latestID, latestPeerName string + lastUpdate time.Time + latestPolicyStat srPolicyStatsSummary + ) + for dID, ss := range ps { + if lastUpdate.IsZero() { + lastUpdate = ss.policy.UpdatedAt + latestID = dID + latestPolicyStat = ss + } + if !ss.policy.UpdatedAt.IsZero() && ss.policy.UpdatedAt.After(lastUpdate) { + lastUpdate = ss.policy.UpdatedAt + latestID = dID + latestPolicyStat = ss + } + } + if latestID != globalDeploymentID() { + // heal only from the site with latest info. + return nil + } + latestPeerName = info.Sites[latestID].Name + // heal policy of peers if peer does not have it. + for dID, pStatus := range ps { + if dID == globalDeploymentID() { + continue + } + if !pStatus.PolicyMismatch && pStatus.HasPolicy { + continue + } + peerName := info.Sites[dID].Name + err := c.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemPolicy, + Name: policy, + Policy: latestPolicyStat.policy.Policy, + UpdatedAt: lastUpdate, + }) + if err != nil { + replLogOnceIf( + ctx, + fmt.Errorf("Unable to heal IAM policy %s from peer site %s -> site %s : %w", policy, latestPeerName, peerName, err), + fmt.Sprintf("heal-policy-%s", policy)) + } + } + return nil +} + +// heal user policy mappings present on this site to peers, provided current cluster has the most recent update. +func (c *SiteReplicationSys) healUserPolicies(ctx context.Context, objAPI ObjectLayer, user string, info srStatusInfo) error { + // create user policy mapping on peer cluster if missing + us := info.UserStats[user] + + c.RLock() + defer c.RUnlock() + if !c.enabled { + return nil + } + var ( + latestID, latestPeerName string + lastUpdate time.Time + latestUserStat srUserStatsSummary + ) + for dID, ss := range us { + if lastUpdate.IsZero() { + lastUpdate = ss.userPolicy.UpdatedAt + latestID = dID + latestUserStat = ss + } + if !ss.userPolicy.UpdatedAt.IsZero() && ss.userPolicy.UpdatedAt.After(lastUpdate) { + lastUpdate = ss.userPolicy.UpdatedAt + latestID = dID + latestUserStat = ss + } + } + if latestID != globalDeploymentID() { + // heal only from the site with latest info. + return nil + } + latestPeerName = info.Sites[latestID].Name + // heal policy of peers if peer does not have it. + for dID, pStatus := range us { + if dID == globalDeploymentID() { + continue + } + if !pStatus.PolicyMismatch && pStatus.HasPolicyMapping { + continue + } + if isPolicyMappingEqual(pStatus.userPolicy, latestUserStat.userPolicy) { + continue + } + peerName := info.Sites[dID].Name + err := c.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemPolicyMapping, + PolicyMapping: &madmin.SRPolicyMapping{ + UserOrGroup: user, + IsGroup: false, + UserType: latestUserStat.userPolicy.UserType, + Policy: latestUserStat.userPolicy.Policy, + }, + UpdatedAt: lastUpdate, + }) + if err != nil { + replLogOnceIf(ctx, fmt.Errorf("Unable to heal IAM user policy mapping from peer site %s -> site %s : %w", latestPeerName, peerName, err), + fmt.Sprintf("heal-user-policy-%s", user)) + } + } + return nil +} + +// heal group policy mappings present on this site to peers, provided current cluster has the most recent update. +func (c *SiteReplicationSys) healGroupPolicies(ctx context.Context, objAPI ObjectLayer, group string, info srStatusInfo) error { + // create group policy mapping on peer cluster if missing + gs := info.GroupStats[group] + + c.RLock() + defer c.RUnlock() + if !c.enabled { + return nil + } + + var ( + latestID, latestPeerName string + lastUpdate time.Time + latestGroupStat srGroupStatsSummary + ) + for dID, ss := range gs { + if lastUpdate.IsZero() { + lastUpdate = ss.groupPolicy.UpdatedAt + latestID = dID + latestGroupStat = ss + } + if !ss.groupPolicy.UpdatedAt.IsZero() && ss.groupPolicy.UpdatedAt.After(lastUpdate) { + lastUpdate = ss.groupPolicy.UpdatedAt + latestID = dID + latestGroupStat = ss + } + } + if latestID != globalDeploymentID() { + // heal only from the site with latest info. + return nil + } + latestPeerName = info.Sites[latestID].Name + // heal policy of peers if peer does not have it. + for dID, pStatus := range gs { + if dID == globalDeploymentID() { + continue + } + if !pStatus.PolicyMismatch && pStatus.HasPolicyMapping { + continue + } + if isPolicyMappingEqual(pStatus.groupPolicy, latestGroupStat.groupPolicy) { + continue + } + peerName := info.Sites[dID].Name + + err := c.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemPolicyMapping, + PolicyMapping: &madmin.SRPolicyMapping{ + UserOrGroup: group, + IsGroup: true, + UserType: int(unknownIAMUserType), + Policy: latestGroupStat.groupPolicy.Policy, + }, + UpdatedAt: lastUpdate, + }) + if err != nil { + replLogOnceIf(ctx, + fmt.Errorf("Unable to heal IAM group policy mapping for from peer site %s -> site %s : %w", latestPeerName, peerName, err), + fmt.Sprintf("heal-group-policy-%s", group)) + } + } + return nil +} + +// heal all users and their service accounts that are present on this site, +// provided current cluster has the most recent update. +func (c *SiteReplicationSys) healUsers(ctx context.Context, objAPI ObjectLayer, user string, info srStatusInfo) error { + // create user if missing; fix user policy mapping if missing + us := info.UserStats[user] + + c.RLock() + defer c.RUnlock() + if !c.enabled { + return nil + } + var ( + latestID, latestPeerName string + lastUpdate time.Time + latestUserStat srUserStatsSummary + ) + for dID, ss := range us { + if lastUpdate.IsZero() { + lastUpdate = ss.userInfo.UpdatedAt + latestID = dID + latestUserStat = ss + } + if !ss.userInfo.UpdatedAt.IsZero() && ss.userInfo.UpdatedAt.After(lastUpdate) { + lastUpdate = ss.userInfo.UpdatedAt + latestID = dID + latestUserStat = ss + } + } + if latestID != globalDeploymentID() { + // heal only from the site with latest info. + return nil + } + latestPeerName = info.Sites[latestID].Name + for dID, uStatus := range us { + if dID == globalDeploymentID() { + continue + } + if !uStatus.UserInfoMismatch { + continue + } + + if isUserInfoEqual(latestUserStat.userInfo.UserInfo, uStatus.userInfo.UserInfo) { + continue + } + + peerName := info.Sites[dID].Name + + u, ok := globalIAMSys.GetUser(ctx, user) + if !ok { + continue + } + creds := u.Credentials + if creds.IsServiceAccount() { + claims, err := globalIAMSys.GetClaimsForSvcAcc(ctx, creds.AccessKey) + if err != nil { + replLogOnceIf(ctx, + fmt.Errorf("Unable to heal service account from peer site %s -> %s : %w", latestPeerName, peerName, err), + fmt.Sprintf("heal-user-%s", user)) + continue + } + + _, policy, err := globalIAMSys.GetServiceAccount(ctx, creds.AccessKey) + if err != nil { + replLogOnceIf(ctx, + fmt.Errorf("Unable to heal service account from peer site %s -> %s : %w", latestPeerName, peerName, err), + fmt.Sprintf("heal-user-%s", user)) + continue + } + + var policyJSON []byte + if policy != nil { + policyJSON, err = json.Marshal(policy) + if err != nil { + replLogOnceIf(ctx, + fmt.Errorf("Unable to heal service account from peer site %s -> %s : %w", latestPeerName, peerName, err), + fmt.Sprintf("heal-user-%s", user)) + continue + } + } + + if err := c.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemSvcAcc, + SvcAccChange: &madmin.SRSvcAccChange{ + Create: &madmin.SRSvcAccCreate{ + Parent: creds.ParentUser, + AccessKey: creds.AccessKey, + SecretKey: creds.SecretKey, + Groups: creds.Groups, + Claims: claims, + SessionPolicy: policyJSON, + Status: creds.Status, + Name: creds.Name, + Description: creds.Description, + Expiration: &creds.Expiration, + }, + }, + UpdatedAt: lastUpdate, + }); err != nil { + replLogOnceIf(ctx, + fmt.Errorf("Unable to heal service account from peer site %s -> %s : %w", latestPeerName, peerName, err), + fmt.Sprintf("heal-user-%s", user)) + } + continue + } + if creds.IsTemp() && !creds.IsExpired() { + var parentPolicy string + u, err := globalIAMSys.GetUserInfo(ctx, creds.ParentUser) + if err != nil { + // Parent may be "virtual" (for ldap, oidc, client tls auth, + // custom auth plugin), so in such cases we apply no parent + // policy. The session token will contain info about policy to + // be applied. + if !errors.Is(err, errNoSuchUser) { + replLogOnceIf(ctx, + fmt.Errorf("Unable to heal temporary credentials from peer site %s -> %s : %w", latestPeerName, peerName, err), + fmt.Sprintf("heal-user-%s", user)) + continue + } + } else { + parentPolicy = u.PolicyName + } + // Call hook for site replication. + if err := c.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemSTSAcc, + STSCredential: &madmin.SRSTSCredential{ + AccessKey: creds.AccessKey, + SecretKey: creds.SecretKey, + SessionToken: creds.SessionToken, + ParentUser: creds.ParentUser, + ParentPolicyMapping: parentPolicy, + }, + UpdatedAt: lastUpdate, + }); err != nil { + replLogOnceIf(ctx, + fmt.Errorf("Unable to heal temporary credentials from peer site %s -> %s : %w", latestPeerName, peerName, err), + fmt.Sprintf("heal-user-%s", user)) + } + continue + } + if err := c.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemIAMUser, + IAMUser: &madmin.SRIAMUser{ + AccessKey: user, + IsDeleteReq: false, + UserReq: &madmin.AddOrUpdateUserReq{ + SecretKey: creds.SecretKey, + Status: latestUserStat.userInfo.Status, + }, + }, + UpdatedAt: lastUpdate, + }); err != nil { + replLogOnceIf(ctx, + fmt.Errorf("Unable to heal user from peer site %s -> %s : %w", latestPeerName, peerName, err), + fmt.Sprintf("heal-user-%s", user)) + } + } + return nil +} + +func (c *SiteReplicationSys) healGroups(ctx context.Context, objAPI ObjectLayer, group string, info srStatusInfo) error { + c.RLock() + defer c.RUnlock() + if !c.enabled { + return nil + } + + var ( + latestID, latestPeerName string + lastUpdate time.Time + latestGroupStat srGroupStatsSummary + ) + // create group if missing; fix group policy mapping if missing + gs, ok := info.GroupStats[group] + if !ok { + return nil + } + for dID, ss := range gs { + if lastUpdate.IsZero() { + lastUpdate = ss.groupDesc.UpdatedAt + latestID = dID + latestGroupStat = ss + } + if !ss.groupDesc.UpdatedAt.IsZero() && ss.groupDesc.UpdatedAt.After(lastUpdate) { + lastUpdate = ss.groupDesc.UpdatedAt + latestID = dID + latestGroupStat = ss + } + } + if latestID != globalDeploymentID() { + // heal only from the site with latest info. + return nil + } + latestPeerName = info.Sites[latestID].Name + for dID, gStatus := range gs { + if dID == globalDeploymentID() { + continue + } + if !gStatus.GroupDescMismatch { + continue + } + + if isGroupDescEqual(latestGroupStat.groupDesc.GroupDesc, gStatus.groupDesc.GroupDesc) { + continue + } + peerName := info.Sites[dID].Name + if err := c.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemGroupInfo, + GroupInfo: &madmin.SRGroupInfo{ + UpdateReq: madmin.GroupAddRemove{ + Group: group, + Status: madmin.GroupStatus(latestGroupStat.groupDesc.Status), + Members: latestGroupStat.groupDesc.Members, + IsRemove: false, + }, + }, + UpdatedAt: lastUpdate, + }); err != nil { + replLogOnceIf(ctx, + fmt.Errorf("Unable to heal group from peer site %s -> site %s : %w", latestPeerName, peerName, err), + fmt.Sprintf("heal-group-%s", group)) + } + } + return nil +} + +func isGroupDescEqual(g1, g2 madmin.GroupDesc) bool { + if g1.Name != g2.Name || + g1.Status != g2.Status || + g1.Policy != g2.Policy { + return false + } + if len(g1.Members) != len(g2.Members) { + return false + } + for _, v1 := range g1.Members { + var found bool + if slices.Contains(g2.Members, v1) { + found = true + } + if !found { + return false + } + } + return true +} + +func isUserInfoEqual(u1, u2 madmin.UserInfo) bool { + if u1.PolicyName != u2.PolicyName || + u1.Status != u2.Status || + u1.SecretKey != u2.SecretKey { + return false + } + for len(u1.MemberOf) != len(u2.MemberOf) { + return false + } + for _, v1 := range u1.MemberOf { + var found bool + if slices.Contains(u2.MemberOf, v1) { + found = true + } + if !found { + return false + } + } + return true +} + +func isPolicyMappingEqual(p1, p2 srPolicyMapping) bool { + return p1.Policy == p2.Policy && p1.IsGroup == p2.IsGroup && p1.UserOrGroup == p2.UserOrGroup +} + +type srPeerInfo struct { + madmin.PeerInfo + EndpointURL *url.URL +} + +// getPeerForUpload returns the site replication peer handling this upload. Defaults to local cluster otherwise +func (c *SiteReplicationSys) getPeerForUpload(deplID string) (pi srPeerInfo, local bool) { + ci, _ := c.GetClusterInfo(GlobalContext) + if !ci.Enabled { + return pi, true + } + for _, site := range ci.Sites { + if deplID == site.DeploymentID { + ep, _ := url.Parse(site.Endpoint) + pi = srPeerInfo{ + PeerInfo: site, + EndpointURL: ep, + } + return pi, site.DeploymentID == globalDeploymentID() + } + } + return pi, true +} + +// startResync initiates resync of data to peerSite specified. The overall site resync status +// is maintained in .minio.sys/buckets/site-replication/resync/, while collecting +// individual bucket resync status in .minio.sys/buckets//replication/resync.bin +func (c *SiteReplicationSys) startResync(ctx context.Context, objAPI ObjectLayer, peer madmin.PeerInfo) (res madmin.SRResyncOpStatus, err error) { + if !c.isEnabled() { + return res, errSRNotEnabled + } + if objAPI == nil { + return res, errSRObjectLayerNotReady + } + + if peer.DeploymentID == globalDeploymentID() { + return res, errSRResyncToSelf + } + if _, ok := c.state.Peers[peer.DeploymentID]; !ok { + return res, errSRPeerNotFound + } + rs, err := globalSiteResyncMetrics.siteStatus(ctx, objAPI, peer.DeploymentID) + if err != nil { + return res, err + } + if rs.Status == ResyncStarted { + return res, errSRResyncStarted + } + var buckets []BucketInfo + buckets, err = objAPI.ListBuckets(ctx, BucketOptions{}) + if err != nil { + return res, err + } + rs = newSiteResyncStatus(peer.DeploymentID, buckets) + defer func() { + if err != nil { + rs.Status = ResyncFailed + saveSiteResyncMetadata(ctx, rs, objAPI) + globalSiteResyncMetrics.updateState(rs) + } + }() + + if err := globalSiteResyncMetrics.updateState(rs); err != nil { + return res, err + } + + for _, bi := range buckets { + bucket := bi.Name + if _, _, err := globalBucketMetadataSys.GetReplicationConfig(ctx, bucket); err != nil { + res.Buckets = append(res.Buckets, madmin.ResyncBucketStatus{ + ErrDetail: err.Error(), + Bucket: bucket, + Status: ResyncFailed.String(), + }) + continue + } + // mark remote target for this deployment with the new reset id + tgtArn := globalBucketTargetSys.getRemoteARNForPeer(bucket, peer) + if tgtArn == "" { + res.Buckets = append(res.Buckets, madmin.ResyncBucketStatus{ + ErrDetail: fmt.Sprintf("no valid remote target found for this peer %s (%s)", peer.Name, peer.DeploymentID), + Bucket: bucket, + }) + continue + } + target := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, bucket, tgtArn) + target.ResetBeforeDate = UTCNow() + target.ResetID = rs.ResyncID + if err = globalBucketTargetSys.SetTarget(ctx, bucket, &target, true); err != nil { + res.Buckets = append(res.Buckets, madmin.ResyncBucketStatus{ + ErrDetail: err.Error(), + Bucket: bucket, + }) + continue + } + targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket) + if err != nil { + res.Buckets = append(res.Buckets, madmin.ResyncBucketStatus{ + ErrDetail: err.Error(), + Bucket: bucket, + }) + continue + } + tgtBytes, err := json.Marshal(&targets) + if err != nil { + res.Buckets = append(res.Buckets, madmin.ResyncBucketStatus{ + ErrDetail: err.Error(), + Bucket: bucket, + }) + continue + } + if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil { + res.Buckets = append(res.Buckets, madmin.ResyncBucketStatus{ + ErrDetail: err.Error(), + Bucket: bucket, + }) + continue + } + if err := globalReplicationPool.Get().resyncer.start(ctx, objAPI, resyncOpts{ + bucket: bucket, + arn: tgtArn, + resyncID: rs.ResyncID, + }); err != nil { + res.Buckets = append(res.Buckets, madmin.ResyncBucketStatus{ + ErrDetail: err.Error(), + Bucket: bucket, + }) + continue + } + } + res = madmin.SRResyncOpStatus{ + Status: ResyncStarted.String(), + OpType: "start", + ResyncID: rs.ResyncID, + Buckets: res.Buckets, + } + if len(res.Buckets) > 0 { + res.ErrDetail = "partial failure in starting site resync" + } + if len(buckets) != 0 && len(res.Buckets) == len(buckets) { + return res, fmt.Errorf("all buckets resync failed") + } + return res, nil +} + +// cancelResync stops an ongoing site level resync for the peer specified. +func (c *SiteReplicationSys) cancelResync(ctx context.Context, objAPI ObjectLayer, peer madmin.PeerInfo) (res madmin.SRResyncOpStatus, err error) { + if !c.isEnabled() { + return res, errSRNotEnabled + } + if objAPI == nil { + return res, errSRObjectLayerNotReady + } + if peer.DeploymentID == globalDeploymentID() { + return res, errSRResyncToSelf + } + if _, ok := c.state.Peers[peer.DeploymentID]; !ok { + return res, errSRPeerNotFound + } + rs, err := globalSiteResyncMetrics.siteStatus(ctx, objAPI, peer.DeploymentID) + if err != nil { + return res, err + } + res = madmin.SRResyncOpStatus{ + Status: rs.Status.String(), + OpType: "cancel", + ResyncID: rs.ResyncID, + } + switch rs.Status { + case ResyncCanceled: + return res, errSRResyncCanceled + case ResyncCompleted, NoResync: + return res, errSRNoResync + } + targets := globalBucketTargetSys.ListTargets(ctx, "", string(madmin.ReplicationService)) + // clear the remote target resetID set while initiating resync to stop replication + for _, t := range targets { + if t.ResetID == rs.ResyncID { + // get tgt with credentials + tgt := globalBucketTargetSys.GetRemoteBucketTargetByArn(ctx, t.SourceBucket, t.Arn) + tgt.ResetID = "" + bucket := t.SourceBucket + if err = globalBucketTargetSys.SetTarget(ctx, bucket, &tgt, true); err != nil { + res.Buckets = append(res.Buckets, madmin.ResyncBucketStatus{ + ErrDetail: err.Error(), + Bucket: bucket, + }) + continue + } + targets, err := globalBucketTargetSys.ListBucketTargets(ctx, bucket) + if err != nil { + res.Buckets = append(res.Buckets, madmin.ResyncBucketStatus{ + ErrDetail: err.Error(), + Bucket: bucket, + }) + continue + } + tgtBytes, err := json.Marshal(&targets) + if err != nil { + res.Buckets = append(res.Buckets, madmin.ResyncBucketStatus{ + ErrDetail: err.Error(), + Bucket: bucket, + }) + continue + } + if _, err = globalBucketMetadataSys.Update(ctx, bucket, bucketTargetsFile, tgtBytes); err != nil { + res.Buckets = append(res.Buckets, madmin.ResyncBucketStatus{ + ErrDetail: err.Error(), + Bucket: bucket, + }) + continue + } + // update resync state for the bucket + globalReplicationPool.Get().resyncer.Lock() + m, ok := globalReplicationPool.Get().resyncer.statusMap[bucket] + if !ok { + m = newBucketResyncStatus(bucket) + } + if st, ok := m.TargetsMap[t.Arn]; ok { + st.LastUpdate = UTCNow() + st.ResyncStatus = ResyncCanceled + m.TargetsMap[t.Arn] = st + m.LastUpdate = UTCNow() + } + globalReplicationPool.Get().resyncer.statusMap[bucket] = m + globalReplicationPool.Get().resyncer.Unlock() + } + } + + rs.Status = ResyncCanceled + rs.LastUpdate = UTCNow() + if err := saveSiteResyncMetadata(ctx, rs, objAPI); err != nil { + return res, err + } + select { + case globalReplicationPool.Get().resyncer.resyncCancelCh <- struct{}{}: + case <-ctx.Done(): + } + + globalSiteResyncMetrics.updateState(rs) + + res.Status = rs.Status.String() + return res, nil +} + +const ( + siteResyncMetaFormat = 1 + siteResyncMetaVersionV1 = 1 + siteResyncMetaVersion = siteResyncMetaVersionV1 + siteResyncSaveInterval = 10 * time.Second +) + +func newSiteResyncStatus(dID string, buckets []BucketInfo) SiteResyncStatus { + now := UTCNow() + s := SiteResyncStatus{ + Version: siteResyncMetaVersion, + Status: ResyncStarted, + DeplID: dID, + TotBuckets: len(buckets), + BucketStatuses: make(map[string]ResyncStatusType), + } + for _, bi := range buckets { + s.BucketStatuses[bi.Name] = ResyncPending + } + s.ResyncID = mustGetUUID() + s.StartTime = now + s.LastUpdate = now + return s +} + +// load site resync metadata from disk +func loadSiteResyncMetadata(ctx context.Context, objAPI ObjectLayer, dID string) (rs SiteResyncStatus, e error) { + data, err := readConfig(GlobalContext, objAPI, getSRResyncFilePath(dID)) + if err != nil { + return rs, err + } + if len(data) == 0 { + // Seems to be empty. + return rs, nil + } + if len(data) <= 4 { + return rs, fmt.Errorf("site resync: no data") + } + // Read resync meta header + switch binary.LittleEndian.Uint16(data[0:2]) { + case siteResyncMetaFormat: + default: + return rs, fmt.Errorf("resyncMeta: unknown format: %d", binary.LittleEndian.Uint16(data[0:2])) + } + switch binary.LittleEndian.Uint16(data[2:4]) { + case siteResyncMetaVersion: + default: + return rs, fmt.Errorf("resyncMeta: unknown version: %d", binary.LittleEndian.Uint16(data[2:4])) + } + // OK, parse data. + if _, err = rs.UnmarshalMsg(data[4:]); err != nil { + return rs, err + } + + switch rs.Version { + case siteResyncMetaVersionV1: + default: + return rs, fmt.Errorf("unexpected resync meta version: %d", rs.Version) + } + return rs, nil +} + +// save resync status of peer to resync/depl-id.meta +func saveSiteResyncMetadata(ctx context.Context, ss SiteResyncStatus, objectAPI ObjectLayer) error { + if objectAPI == nil { + return errSRObjectLayerNotReady + } + data := make([]byte, 4, ss.Msgsize()+4) + + // Initialize the resync meta header. + binary.LittleEndian.PutUint16(data[0:2], siteResyncMetaFormat) + binary.LittleEndian.PutUint16(data[2:4], siteResyncMetaVersion) + + buf, err := ss.MarshalMsg(data) + if err != nil { + return err + } + return saveConfig(ctx, objectAPI, getSRResyncFilePath(ss.DeplID), buf) +} + +func getSRResyncFilePath(dID string) string { + return pathJoin(siteResyncPrefix, dID+".meta") +} + +func (c *SiteReplicationSys) getDeplIDForEndpoint(ep string) (dID string, err error) { + if ep == "" { + return dID, fmt.Errorf("no deployment id found for endpoint %s", ep) + } + c.RLock() + defer c.RUnlock() + if !c.enabled { + return dID, errSRNotEnabled + } + for _, peer := range c.state.Peers { + if ep == peer.Endpoint { + return peer.DeploymentID, nil + } + } + return dID, fmt.Errorf("no deployment id found for endpoint %s", ep) +} + +func (c *SiteReplicationSys) getSiteMetrics(ctx context.Context) (madmin.SRMetricsSummary, error) { + if !c.isEnabled() { + return madmin.SRMetricsSummary{}, errSRNotEnabled + } + peerSMetricsList := globalNotificationSys.GetClusterSiteMetrics(ctx) + var sm madmin.SRMetricsSummary + sm.Metrics = make(map[string]madmin.SRMetric) + + for _, peer := range peerSMetricsList { + sm.ActiveWorkers.Avg += peer.ActiveWorkers.Avg + sm.ActiveWorkers.Curr += peer.ActiveWorkers.Curr + if peer.ActiveWorkers.Max > sm.ActiveWorkers.Max { + sm.ActiveWorkers.Max += peer.ActiveWorkers.Max + } + sm.Queued.Avg.Bytes += peer.Queued.Avg.Bytes + sm.Queued.Avg.Count += peer.Queued.Avg.Count + sm.Queued.Curr.Bytes += peer.Queued.Curr.Bytes + sm.Queued.Curr.Count += peer.Queued.Curr.Count + if peer.Queued.Max.Count > sm.Queued.Max.Count { + sm.Queued.Max.Bytes = peer.Queued.Max.Bytes + sm.Queued.Max.Count = peer.Queued.Max.Count + } + sm.ReplicaCount += peer.ReplicaCount + sm.ReplicaSize += peer.ReplicaSize + sm.Proxied.Add(madmin.ReplProxyMetric(peer.Proxied)) + for dID, v := range peer.Metrics { + v2, ok := sm.Metrics[dID] + if !ok { + v2 = madmin.SRMetric{} + v2.Failed.ErrCounts = make(map[string]int) + } + + // use target endpoint metrics from node which has been up the longest + if v2.LastOnline.After(v.LastOnline) || v2.LastOnline.IsZero() { + v2.Endpoint = v.Endpoint + v2.LastOnline = v.LastOnline + v2.Latency = v.Latency + v2.Online = v.Online + v2.TotalDowntime = v.TotalDowntime + v2.DeploymentID = v.DeploymentID + } + v2.ReplicatedCount += v.ReplicatedCount + v2.ReplicatedSize += v.ReplicatedSize + v2.Failed = v2.Failed.Add(v.Failed) + for k, v := range v.Failed.ErrCounts { + v2.Failed.ErrCounts[k] += v + } + if v2.XferStats == nil { + v2.XferStats = make(map[replication.MetricName]replication.XferStats) + } + for rm, x := range v.XferStats { + x2, ok := v2.XferStats[replication.MetricName(rm)] + if !ok { + x2 = replication.XferStats{} + } + x2.AvgRate += x.Avg + x2.CurrRate += x.Curr + if x.Peak > x2.PeakRate { + x2.PeakRate = x.Peak + } + v2.XferStats[replication.MetricName(rm)] = x2 + } + sm.Metrics[dID] = v2 + } + } + sm.Uptime = UTCNow().Unix() - globalBootTime.Unix() + return sm, nil +} + +// mergeWithCurrentLCConfig - merges the given ilm expiry configuration with existing for the current site and returns +func mergeWithCurrentLCConfig(ctx context.Context, bucket string, expLCCfg *string, updatedAt time.Time) ([]byte, error) { + // Get bucket config from current site + meta, e := globalBucketMetadataSys.GetConfigFromDisk(ctx, bucket) + if e != nil && !errors.Is(e, errConfigNotFound) { + return []byte{}, e + } + rMap := make(map[string]lifecycle.Rule) + var xmlName xml.Name + if len(meta.LifecycleConfigXML) > 0 { + var lcCfg lifecycle.Lifecycle + if err := xml.Unmarshal(meta.LifecycleConfigXML, &lcCfg); err != nil { + return []byte{}, err + } + for _, rl := range lcCfg.Rules { + rMap[rl.ID] = rl + } + xmlName = meta.lifecycleConfig.XMLName + } + + // get latest expiry rules + newRMap := make(map[string]lifecycle.Rule) + if expLCCfg != nil { + var cfg lifecycle.Lifecycle + expLcCfgData, err := base64.StdEncoding.DecodeString(*expLCCfg) + if err != nil { + return []byte{}, err + } + if err := xml.Unmarshal(expLcCfgData, &cfg); err != nil { + return []byte{}, err + } + for _, rl := range cfg.Rules { + newRMap[rl.ID] = rl + } + xmlName = cfg.XMLName + } + + // check if current expiry rules are there in new one. if not remove the expiration + // part of rule as they may have been removed from latest updated one + for id, rl := range rMap { + if !rl.Expiration.IsNull() || !rl.NoncurrentVersionExpiration.IsNull() { + if _, ok := newRMap[id]; !ok { + // if rule getting removed was pure expiry rule (may be got to this site + // as part of replication of expiry rules), remove it. Otherwise remove + // only the expiry part of it + if rl.Transition.IsNull() && rl.NoncurrentVersionTransition.IsNull() { + delete(rMap, id) + } else { + rl.Expiration = lifecycle.Expiration{} + rl.NoncurrentVersionExpiration = lifecycle.NoncurrentVersionExpiration{} + rMap[id] = rl + } + } + } + } + + // append now + for id, rl := range newRMap { + // if rule is already in original list update non tranisition details with latest + // else simply add to the map. This may happen if ILM expiry replication + // was disabled for sometime and rules were updated independently in different + // sites. Latest changes would get applied but merge only the non transition details + if existingRl, ok := rMap[id]; ok { + clonedRl := rl.CloneNonTransition() + clonedRl.Transition = existingRl.Transition + clonedRl.NoncurrentVersionTransition = existingRl.NoncurrentVersionTransition + rMap[id] = clonedRl + } else { + rMap[id] = rl + } + } + + var rules []lifecycle.Rule + for _, rule := range rMap { + rules = append(rules, rule) + } + + // no rules, return + if len(rules) == 0 { + return []byte{}, nil + } + + // get final list for write + finalLcCfg := lifecycle.Lifecycle{ + XMLName: xmlName, + Rules: rules, + ExpiryUpdatedAt: &updatedAt, + } + + rcfg, err := globalBucketObjectLockSys.Get(bucket) + if err != nil { + return nil, err + } + + if err := finalLcCfg.Validate(rcfg); err != nil { + return []byte{}, err + } + finalConfigData, err := xml.Marshal(finalLcCfg) + if err != nil { + return []byte{}, err + } + + return finalConfigData, nil +} + +func ilmExpiryReplicationEnabled(sites map[string]madmin.PeerInfo) bool { + flag := true + for _, pi := range sites { + flag = flag && pi.ReplicateILMExpiry + } + return flag +} + +type siteReplicatorCred struct { + secretKey string + sync.RWMutex +} + +// Get or attempt to load site replicator credentials from disk. +func (s *siteReplicatorCred) Get(ctx context.Context) (string, error) { + s.RLock() + secretKey := s.secretKey + s.RUnlock() + + if secretKey != "" { + return secretKey, nil + } + + secretKey, err := globalIAMSys.store.loadSecretKey(ctx, siteReplicatorSvcAcc, svcUser) + if err != nil { + return "", err + } + s.Set(secretKey) + return secretKey, nil +} + +func (s *siteReplicatorCred) Set(secretKey string) { + s.Lock() + defer s.Unlock() + s.secretKey = secretKey +} + +func (s *siteReplicatorCred) IsValid() bool { + s.RLock() + defer s.RUnlock() + return s.secretKey != "" +} diff --git a/cmd/site-replication_test.go b/cmd/site-replication_test.go new file mode 100644 index 00000000..397bb9f9 --- /dev/null +++ b/cmd/site-replication_test.go @@ -0,0 +1,68 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "testing" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7/pkg/set" +) + +// TestGetMissingSiteNames +func TestGetMissingSiteNames(t *testing.T) { + testCases := []struct { + currSites []madmin.PeerInfo + oldDepIDs set.StringSet + newDepIDs set.StringSet + expNames []string + }{ + // Test1: missing some sites in replicated setup + { + []madmin.PeerInfo{ + {Endpoint: "minio1:9000", Name: "minio1", DeploymentID: "dep1"}, + {Endpoint: "minio2:9000", Name: "minio2", DeploymentID: "dep2"}, + {Endpoint: "minio3:9000", Name: "minio3", DeploymentID: "dep3"}, + }, + set.CreateStringSet("dep1", "dep2", "dep3"), + set.CreateStringSet("dep1"), + []string{"minio2", "minio3"}, + }, + // Test2: new site added that is not in replicated setup + { + []madmin.PeerInfo{{Endpoint: "minio1:9000", Name: "minio1", DeploymentID: "dep1"}, {Endpoint: "minio2:9000", Name: "minio2", DeploymentID: "dep2"}, {Endpoint: "minio3:9000", Name: "minio3", DeploymentID: "dep3"}}, + set.CreateStringSet("dep1", "dep2", "dep3"), + set.CreateStringSet("dep1", "dep2", "dep3", "dep4"), + []string{}, + }, + // Test3: not currently under site replication. + { + []madmin.PeerInfo{}, + set.CreateStringSet(), + set.CreateStringSet("dep1", "dep2", "dep3", "dep4"), + []string{}, + }, + } + + for i, tc := range testCases { + names := getMissingSiteNames(tc.oldDepIDs, tc.newDepIDs, tc.currSites) + if len(names) != len(tc.expNames) { + t.Errorf("Test %d: Expected `%v`, got `%v`", i+1, tc.expNames, names) + } + } +} diff --git a/cmd/speedtest.go b/cmd/speedtest.go new file mode 100644 index 00000000..f91db787 --- /dev/null +++ b/cmd/speedtest.go @@ -0,0 +1,308 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "fmt" + "net/url" + "runtime" + "sort" + "time" + + "github.com/minio/dperf/pkg/dperf" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/auth" + xioutil "github.com/minio/minio/internal/ioutil" +) + +const speedTest = "speedtest" + +type speedTestOpts struct { + objectSize int + concurrencyStart int + concurrency int + duration time.Duration + autotune bool + storageClass string + bucketName string + enableSha256 bool + enableMultipart bool + creds auth.Credentials +} + +// Get the max throughput and iops numbers. +func objectSpeedTest(ctx context.Context, opts speedTestOpts) chan madmin.SpeedTestResult { + ch := make(chan madmin.SpeedTestResult, 1) + go func() { + defer xioutil.SafeClose(ch) + + concurrency := opts.concurrencyStart + + if opts.autotune { + // if we have less drives than concurrency then choose + // only the concurrency to be number of drives to start + // with - since default '32' might be big and may not + // complete in total time of 10s. + if globalEndpoints.NEndpoints() < concurrency { + concurrency = globalEndpoints.NEndpoints() + } + + // Check if we have local disks per pool less than + // the concurrency make sure we choose only the "start" + // concurrency to be equal to the lowest number of + // local disks per server. + for _, localDiskCount := range globalEndpoints.NLocalDisksPathsPerPool() { + if localDiskCount < concurrency { + concurrency = localDiskCount + } + } + + // Any concurrency less than '4' just stick to '4' concurrent + // operations for now to begin with. + if concurrency < 4 { + concurrency = 4 + } + + // if GOMAXPROCS is set to a lower value then choose to use + // concurrency == GOMAXPROCS instead. + if runtime.GOMAXPROCS(0) < concurrency { + concurrency = runtime.GOMAXPROCS(0) + } + } + + throughputHighestGet := uint64(0) + throughputHighestPut := uint64(0) + var throughputHighestResults []SpeedTestResult + + sendResult := func() { + var result madmin.SpeedTestResult + + durationSecs := opts.duration.Seconds() + + result.GETStats.ThroughputPerSec = throughputHighestGet / uint64(durationSecs) + result.GETStats.ObjectsPerSec = throughputHighestGet / uint64(opts.objectSize) / uint64(durationSecs) + result.PUTStats.ThroughputPerSec = throughputHighestPut / uint64(durationSecs) + result.PUTStats.ObjectsPerSec = throughputHighestPut / uint64(opts.objectSize) / uint64(durationSecs) + var totalUploadTimes madmin.TimeDurations + var totalDownloadTimes madmin.TimeDurations + var totalDownloadTTFB madmin.TimeDurations + for i := range len(throughputHighestResults) { + errStr := "" + if throughputHighestResults[i].Error != "" { + errStr = throughputHighestResults[i].Error + } + + // if the default concurrency yields zero results, throw an error. + if throughputHighestResults[i].Downloads == 0 && opts.concurrencyStart == concurrency { + errStr = fmt.Sprintf("no results for downloads upon first attempt, concurrency %d and duration %s", + opts.concurrencyStart, opts.duration) + } + + // if the default concurrency yields zero results, throw an error. + if throughputHighestResults[i].Uploads == 0 && opts.concurrencyStart == concurrency { + errStr = fmt.Sprintf("no results for uploads upon first attempt, concurrency %d and duration %s", + opts.concurrencyStart, opts.duration) + } + + result.PUTStats.Servers = append(result.PUTStats.Servers, madmin.SpeedTestStatServer{ + Endpoint: throughputHighestResults[i].Endpoint, + ThroughputPerSec: throughputHighestResults[i].Uploads / uint64(durationSecs), + ObjectsPerSec: throughputHighestResults[i].Uploads / uint64(opts.objectSize) / uint64(durationSecs), + Err: errStr, + }) + + result.GETStats.Servers = append(result.GETStats.Servers, madmin.SpeedTestStatServer{ + Endpoint: throughputHighestResults[i].Endpoint, + ThroughputPerSec: throughputHighestResults[i].Downloads / uint64(durationSecs), + ObjectsPerSec: throughputHighestResults[i].Downloads / uint64(opts.objectSize) / uint64(durationSecs), + Err: errStr, + }) + + totalUploadTimes = append(totalUploadTimes, throughputHighestResults[i].UploadTimes...) + totalDownloadTimes = append(totalDownloadTimes, throughputHighestResults[i].DownloadTimes...) + totalDownloadTTFB = append(totalDownloadTTFB, throughputHighestResults[i].DownloadTTFB...) + } + + result.PUTStats.Response = totalUploadTimes.Measure() + result.GETStats.Response = totalDownloadTimes.Measure() + result.GETStats.TTFB = totalDownloadTTFB.Measure() + + result.Size = opts.objectSize + result.Disks = globalEndpoints.NEndpoints() + result.Servers = len(globalNotificationSys.peerClients) + 1 + result.Version = Version + result.Concurrent = concurrency + + select { + case ch <- result: + case <-ctx.Done(): + return + } + } + + for { + select { + case <-ctx.Done(): + // If the client got disconnected stop the speedtest. + return + default: + } + + sopts := speedTestOpts{ + objectSize: opts.objectSize, + concurrency: concurrency, + duration: opts.duration, + storageClass: opts.storageClass, + bucketName: opts.bucketName, + enableSha256: opts.enableSha256, + enableMultipart: opts.enableMultipart, + creds: opts.creds, + } + + results := globalNotificationSys.SpeedTest(ctx, sopts) + sort.Slice(results, func(i, j int) bool { + return results[i].Endpoint < results[j].Endpoint + }) + + totalPut := uint64(0) + totalGet := uint64(0) + for _, result := range results { + totalPut += result.Uploads + totalGet += result.Downloads + } + + if totalGet < throughputHighestGet { + // Following check is for situations + // when Writes() scale higher than Reads() + // - practically speaking this never happens + // and should never happen - however it has + // been seen recently due to hardware issues + // causes Reads() to go slower than Writes(). + // + // Send such results anyways as this shall + // expose a problem underneath. + if totalPut > throughputHighestPut { + throughputHighestResults = results + throughputHighestPut = totalPut + // let the client see lower value as well + throughputHighestGet = totalGet + } + sendResult() + break + } + + // We break if we did not see 2.5% growth rate in total GET + // requests, we have reached our peak at this point. + doBreak := float64(totalGet-throughputHighestGet)/float64(totalGet) < 0.025 + + throughputHighestGet = totalGet + throughputHighestResults = results + throughputHighestPut = totalPut + + if doBreak { + sendResult() + break + } + + for _, result := range results { + if result.Error != "" { + // Break out on errors. + sendResult() + return + } + } + + sendResult() + if !opts.autotune { + break + } + + // Try with a higher concurrency to see if we get better throughput + concurrency += (concurrency + 1) / 2 + } + }() + return ch +} + +func driveSpeedTest(ctx context.Context, opts madmin.DriveSpeedTestOpts) madmin.DriveSpeedTestResult { + perf := &dperf.DrivePerf{ + Serial: opts.Serial, + BlockSize: opts.BlockSize, + FileSize: opts.FileSize, + } + + localPaths := globalEndpoints.LocalDisksPaths() + var ignoredPaths []string + paths := func() (tmpPaths []string) { + for _, lp := range localPaths { + if _, err := Lstat(pathJoin(lp, minioMetaBucket, formatConfigFile)); err == nil { + tmpPaths = append(tmpPaths, pathJoin(lp, minioMetaTmpBucket)) + } else { + // Use dperf on only formatted drives. + ignoredPaths = append(ignoredPaths, lp) + } + } + return tmpPaths + }() + + scheme := "http" + if globalIsTLS { + scheme = "https" + } + + u := &url.URL{ + Scheme: scheme, + Host: globalLocalNodeName, + } + + perfs, err := perf.Run(ctx, paths...) + return madmin.DriveSpeedTestResult{ + Endpoint: u.String(), + Version: Version, + DrivePerf: func() (results []madmin.DrivePerf) { + for idx, r := range perfs { + result := madmin.DrivePerf{ + Path: localPaths[idx], + ReadThroughput: r.ReadThroughput, + WriteThroughput: r.WriteThroughput, + Error: func() string { + if r.Error != nil { + return r.Error.Error() + } + return "" + }(), + } + results = append(results, result) + } + for _, inp := range ignoredPaths { + results = append(results, madmin.DrivePerf{ + Path: inp, + Error: errFaultyDisk.Error(), + }) + } + return results + }(), + Error: func() string { + if err != nil { + return err.Error() + } + return "" + }(), + } +} diff --git a/cmd/storage-datatypes.go b/cmd/storage-datatypes.go new file mode 100644 index 00000000..be36fa35 --- /dev/null +++ b/cmd/storage-datatypes.go @@ -0,0 +1,586 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "time" + + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/grid" + xioutil "github.com/minio/minio/internal/ioutil" +) + +//msgp:clearomitted + +//go:generate msgp -file=$GOFILE + +// DeleteOptions represents the disk level delete options available for the APIs +type DeleteOptions struct { + BaseOptions + Recursive bool `msg:"r"` + Immediate bool `msg:"i"` + UndoWrite bool `msg:"u"` + // OldDataDir of the previous object + OldDataDir string `msg:"o,omitempty"` // old data dir used only when to revert a rename() +} + +// BaseOptions represents common options for all Storage API calls +type BaseOptions struct{} + +// RenameOptions represents rename API options, currently its same as BaseOptions +type RenameOptions struct { + BaseOptions +} + +// DiskInfoOptions options for requesting custom results. +type DiskInfoOptions struct { + DiskID string `msg:"id"` + Metrics bool `msg:"m"` + NoOp bool `msg:"np"` +} + +// DiskInfo is an extended type which returns current +// disk usage per path. +// The above means that any added/deleted fields are incompatible. +// +// The above means that any added/deleted fields are incompatible. +// +//msgp:tuple DiskInfo +type DiskInfo struct { + Total uint64 + Free uint64 + Used uint64 + UsedInodes uint64 + FreeInodes uint64 + Major uint32 + Minor uint32 + NRRequests uint64 + FSType string + RootDisk bool + Healing bool + Scanning bool + Endpoint string + MountPath string + ID string + Rotational bool + Metrics DiskMetrics + Error string // carries the error over the network +} + +// DiskMetrics has the information about XL Storage APIs +// the number of calls of each API and the moving average of +// the duration of each API. +type DiskMetrics struct { + LastMinute map[string]AccElem `json:"apiLatencies,omitempty"` + APICalls map[string]uint64 `json:"apiCalls,omitempty"` + TotalWaiting uint32 `json:"totalWaiting,omitempty"` + TotalErrorsAvailability uint64 `json:"totalErrsAvailability"` + TotalErrorsTimeout uint64 `json:"totalErrsTimeout"` + TotalWrites uint64 `json:"totalWrites"` + TotalDeletes uint64 `json:"totalDeletes"` +} + +// VolsInfo is a collection of volume(bucket) information +type VolsInfo []VolInfo + +// VolInfo - represents volume stat information. +// The above means that any added/deleted fields are incompatible. +// +//msgp:tuple VolInfo +type VolInfo struct { + // Name of the volume. + Name string + + // Date and time when the volume was created. + Created time.Time + + // total VolInfo counts + count int + + // Date and time when the volume was deleted, if Deleted + Deleted time.Time +} + +// FilesInfo represent a list of files, additionally +// indicates if the list is last. +// +//msgp:tuple FileInfo +type FilesInfo struct { + Files []FileInfo + IsTruncated bool +} + +// Size returns size of all versions for the object 'Name' +func (f FileInfoVersions) Size() (size int64) { + for _, v := range f.Versions { + size += v.Size + } + return size +} + +// FileInfoVersions represent a list of versions for a given file. +// The above means that any added/deleted fields are incompatible. +// +// The above means that any added/deleted fields are incompatible. +// +//msgp:tuple FileInfoVersions +type FileInfoVersions struct { + // Name of the volume. + Volume string `msg:"v,omitempty"` + + // Name of the file. + Name string `msg:"n,omitempty"` + + // Represents the latest mod time of the + // latest version. + LatestModTime time.Time `msg:"lm"` + + Versions []FileInfo `msg:"vs"` + FreeVersions []FileInfo `msg:"fvs"` +} + +// findVersionIndex will return the version index where the version +// was found. Returns -1 if not found. +func (f *FileInfoVersions) findVersionIndex(v string) int { + if f == nil || v == "" { + return -1 + } + if v == nullVersionID { + for i, ver := range f.Versions { + if ver.VersionID == "" { + return i + } + } + return -1 + } + + for i, ver := range f.Versions { + if ver.VersionID == v { + return i + } + } + return -1 +} + +// RawFileInfo - represents raw file stat information as byte array. +// The above means that any added/deleted fields are incompatible. +// Make sure to bump the internode version at storage-rest-common.go +type RawFileInfo struct { + // Content of entire xl.meta (may contain data depending on what was requested by the caller. + Buf []byte `msg:"b,allownil"` +} + +// FileInfo - represents file stat information. +// The above means that any added/deleted fields are incompatible. +// Make sure to bump the internode version at storage-rest-common.go +type FileInfo struct { + // Name of the volume. + Volume string `msg:"v,omitempty"` + + // Name of the file. + Name string `msg:"n,omitempty"` + + // Version of the file. + VersionID string `msg:"vid,omitempty"` + + // Indicates if the version is the latest + IsLatest bool `msg:"is"` + + // Deleted is set when this FileInfo represents + // a deleted marker for a versioned bucket. + Deleted bool `msg:"del"` + + // TransitionStatus is set to Pending/Complete for transitioned + // entries based on state of transition + TransitionStatus string `msg:"ts"` + // TransitionedObjName is the object name on the remote tier corresponding + // to object (version) on the source tier. + TransitionedObjName string `msg:"to"` + // TransitionTier is the storage class label assigned to remote tier. + TransitionTier string `msg:"tt"` + // TransitionVersionID stores a version ID of the object associate + // with the remote tier. + TransitionVersionID string `msg:"tv"` + // ExpireRestored indicates that the restored object is to be expired. + ExpireRestored bool `msg:"exp"` + + // DataDir of the file + DataDir string `msg:"dd"` + + // Indicates if this object is still in V1 format. + XLV1 bool `msg:"v1"` + + // Date and time when the file was last modified, if Deleted + // is 'true' this value represents when while was deleted. + ModTime time.Time `msg:"mt"` + + // Total file size. + Size int64 `msg:"sz"` + + // File mode bits. + Mode uint32 `msg:"m"` + + // WrittenByVersion is the unix time stamp of the MinIO + // version that created this version of the object. + WrittenByVersion uint64 `msg:"wv"` + + // File metadata + Metadata map[string]string `msg:"meta"` + + // All the parts per object. + Parts []ObjectPartInfo `msg:"parts"` + + // Erasure info for all objects. + Erasure ErasureInfo `msg:"ei"` + + MarkDeleted bool `msg:"md"` // mark this version as deleted + ReplicationState ReplicationState `msg:"rs"` // Internal replication state to be passed back in ObjectInfo + + Data []byte `msg:"d,allownil"` // optionally carries object data + + NumVersions int `msg:"nv"` + SuccessorModTime time.Time `msg:"smt"` + + Fresh bool `msg:"fr"` // indicates this is a first time call to write FileInfo. + + // Position of this version or object in a multi-object delete call, + // no other caller must set this value other than multi-object delete call. + // usage in other calls in undefined please avoid. + Idx int `msg:"i"` + + // Combined checksum when object was uploaded. + Checksum []byte `msg:"cs,allownil"` + + // Versioned - indicates if this file is versioned or not. + Versioned bool `msg:"vs"` +} + +func (fi FileInfo) shardSize() int64 { + return ceilFrac(fi.Erasure.BlockSize, int64(fi.Erasure.DataBlocks)) +} + +// ShardFileSize - returns final erasure size from original size. +func (fi FileInfo) ShardFileSize(totalLength int64) int64 { + if totalLength == 0 { + return 0 + } + if totalLength == -1 { + return -1 + } + numShards := totalLength / fi.Erasure.BlockSize + lastBlockSize := totalLength % fi.Erasure.BlockSize + lastShardSize := ceilFrac(lastBlockSize, int64(fi.Erasure.DataBlocks)) + return numShards*fi.shardSize() + lastShardSize +} + +// ShallowCopy - copies minimal information for READ MRF checks. +func (fi FileInfo) ShallowCopy() (n FileInfo) { + n.Volume = fi.Volume + n.Name = fi.Name + n.VersionID = fi.VersionID + n.Deleted = fi.Deleted + n.Erasure = fi.Erasure + return n +} + +// WriteQuorum returns expected write quorum for this FileInfo +func (fi FileInfo) WriteQuorum(dquorum int) int { + if fi.Deleted { + return dquorum + } + quorum := fi.Erasure.DataBlocks + if fi.Erasure.DataBlocks == fi.Erasure.ParityBlocks { + quorum++ + } + return quorum +} + +// ReadQuorum returns expected read quorum for this FileInfo +func (fi FileInfo) ReadQuorum(dquorum int) int { + if fi.Deleted { + return dquorum + } + return fi.Erasure.DataBlocks +} + +// Equals checks if fi(FileInfo) matches ofi(FileInfo) +func (fi FileInfo) Equals(ofi FileInfo) (ok bool) { + typ1, ok1 := crypto.IsEncrypted(fi.Metadata) + typ2, ok2 := crypto.IsEncrypted(ofi.Metadata) + if ok1 != ok2 { + return false + } + if typ1 != typ2 { + return false + } + if fi.IsCompressed() != ofi.IsCompressed() { + return false + } + if !fi.TransitionInfoEquals(ofi) { + return false + } + if !fi.ModTime.Equal(ofi.ModTime) { + return false + } + return fi.Erasure.Equal(ofi.Erasure) +} + +// GetDataDir returns an expected dataDir given FileInfo +// - deleteMarker returns "delete-marker" +// - returns "legacy" if FileInfo is XLV1 and DataDir is +// empty, returns DataDir otherwise +// - returns "dataDir" +func (fi FileInfo) GetDataDir() string { + if fi.Deleted { + return "delete-marker" + } + if fi.XLV1 && fi.DataDir == "" { + return "legacy" + } + return fi.DataDir +} + +// IsCompressed returns true if the object is marked as compressed. +func (fi FileInfo) IsCompressed() bool { + _, ok := fi.Metadata[ReservedMetadataPrefix+"compression"] + return ok +} + +// InlineData returns true if object contents are inlined alongside its metadata. +func (fi FileInfo) InlineData() bool { + _, ok := fi.Metadata[ReservedMetadataPrefixLower+"inline-data"] + // Earlier MinIO versions didn't reset "x-minio-internal-inline-data" + // from fi.Metadata when the object was tiered. So, tiered objects + // would return true for InlineData() in these versions even though the + // object isn't inlined in xl.meta + return ok && !fi.IsRemote() +} + +// SetInlineData marks object (version) as inline. +func (fi *FileInfo) SetInlineData() { + if fi.Metadata == nil { + fi.Metadata = make(map[string]string, 1) + } + fi.Metadata[ReservedMetadataPrefixLower+"inline-data"] = "true" +} + +// VersionPurgeStatusKey denotes purge status in metadata +const ( + VersionPurgeStatusKey = ReservedMetadataPrefixLower + "purgestatus" +) + +// newFileInfo - initializes new FileInfo, allocates a fresh erasure info. +func newFileInfo(object string, dataBlocks, parityBlocks int) (fi FileInfo) { + fi.Erasure = ErasureInfo{ + Algorithm: erasureAlgorithm, + DataBlocks: dataBlocks, + ParityBlocks: parityBlocks, + BlockSize: blockSizeV2, + Distribution: hashOrder(object, dataBlocks+parityBlocks), + } + return fi +} + +// ReadMultipleReq contains information of multiple files to read from disk. +type ReadMultipleReq struct { + Bucket string `msg:"bk"` // Bucket. Can be empty if multiple buckets. + Prefix string `msg:"pr,omitempty"` // Shared prefix of all files. Can be empty. Will be joined to filename without modification. + Files []string `msg:"fl"` // Individual files to read. + MaxSize int64 `msg:"ms"` // Return error if size is exceed. + MetadataOnly bool `msg:"mo"` // Read as XL meta and truncate data. + AbortOn404 bool `msg:"ab"` // Stop reading after first file not found. + MaxResults int `msg:"mr"` // Stop after this many successful results. <= 0 means all. +} + +// ReadMultipleResp contains a single response from a ReadMultipleReq. +type ReadMultipleResp struct { + Bucket string `msg:"bk"` // Bucket as given by request. + Prefix string `msg:"pr,omitempty"` // Prefix as given by request. + File string `msg:"fl"` // File name as given in request. + Exists bool `msg:"ex"` // Returns whether the file existed on disk. + Error string `msg:"er,omitempty"` // Returns any error when reading. + Data []byte `msg:"d"` // Contains all data of file. + Modtime time.Time `msg:"m"` // Modtime of file on disk. +} + +// DeleteVersionHandlerParams are parameters for DeleteVersionHandler +type DeleteVersionHandlerParams struct { + DiskID string `msg:"id"` + Volume string `msg:"v"` + FilePath string `msg:"fp"` + ForceDelMarker bool `msg:"fdm"` + Opts DeleteOptions `msg:"do"` + FI FileInfo `msg:"fi"` +} + +// MetadataHandlerParams is request info for UpdateMetadataHandle and WriteMetadataHandler. +type MetadataHandlerParams struct { + DiskID string `msg:"id"` + Volume string `msg:"v"` + OrigVolume string `msg:"ov"` + FilePath string `msg:"fp"` + UpdateOpts UpdateMetadataOpts `msg:"uo"` + FI FileInfo `msg:"fi"` +} + +// UpdateMetadataOpts provides an optional input to indicate if xl.meta updates need to be fully synced to disk. +type UpdateMetadataOpts struct { + NoPersistence bool `msg:"np"` +} + +// CheckPartsHandlerParams are parameters for CheckPartsHandler +type CheckPartsHandlerParams struct { + DiskID string `msg:"id"` + Volume string `msg:"v"` + FilePath string `msg:"fp"` + FI FileInfo `msg:"fi"` +} + +// DeleteFileHandlerParams are parameters for DeleteFileHandler +type DeleteFileHandlerParams struct { + DiskID string `msg:"id"` + Volume string `msg:"v"` + FilePath string `msg:"fp"` + Opts DeleteOptions `msg:"do"` +} + +// RenameDataHandlerParams are parameters for RenameDataHandler. +type RenameDataHandlerParams struct { + DiskID string `msg:"id"` + SrcVolume string `msg:"sv"` + SrcPath string `msg:"sp"` + DstVolume string `msg:"dv"` + DstPath string `msg:"dp"` + FI FileInfo `msg:"fi"` + Opts RenameOptions `msg:"ro"` +} + +// RenameDataInlineHandlerParams are parameters for RenameDataHandler with a buffer for inline data. +type RenameDataInlineHandlerParams struct { + RenameDataHandlerParams `msg:"p"` +} + +func newRenameDataInlineHandlerParams() *RenameDataInlineHandlerParams { + buf := grid.GetByteBufferCap(32 + 16<<10) + return &RenameDataInlineHandlerParams{RenameDataHandlerParams{FI: FileInfo{Data: buf[:0]}}} +} + +// Recycle will reuse the memory allocated for the FileInfo data. +func (r *RenameDataInlineHandlerParams) Recycle() { + if r == nil { + return + } + if cap(r.FI.Data) >= xioutil.SmallBlock { + grid.PutByteBuffer(r.FI.Data) + r.FI.Data = nil + } +} + +// RenameFileHandlerParams are parameters for RenameFileHandler. +type RenameFileHandlerParams struct { + DiskID string `msg:"id"` + SrcVolume string `msg:"sv"` + SrcFilePath string `msg:"sp"` + DstVolume string `msg:"dv"` + DstFilePath string `msg:"dp"` +} + +// RenamePartHandlerParams are parameters for RenamePartHandler. +type RenamePartHandlerParams struct { + DiskID string `msg:"id"` + SrcVolume string `msg:"sv"` + SrcFilePath string `msg:"sp"` + DstVolume string `msg:"dv"` + DstFilePath string `msg:"dp"` + Meta []byte `msg:"m"` + SkipParent string `msg:"kp"` +} + +// ReadAllHandlerParams are parameters for ReadAllHandler. +type ReadAllHandlerParams struct { + DiskID string `msg:"id"` + Volume string `msg:"v"` + FilePath string `msg:"fp"` +} + +// WriteAllHandlerParams are parameters for WriteAllHandler. +type WriteAllHandlerParams struct { + DiskID string `msg:"id"` + Volume string `msg:"v"` + FilePath string `msg:"fp"` + Buf []byte `msg:"b"` +} + +// RenameDataResp - RenameData()'s response. +// Provides information about the final state of Rename() +// - on xl.meta (array of versions) on disk to check for version disparity +// - on rewrite dataDir on disk that must be additionally purged +// only after as a 2-phase call, allowing the older dataDir to +// hang-around in-case we need some form of recovery. +type RenameDataResp struct { + Sign []byte `msg:"s"` + OldDataDir string `msg:"od"` // contains '', it is designed to be passed as value to Delete(bucket, pathJoin(object, dataDir)) +} + +const ( + checkPartUnknown int = iota + + // Changing the order can cause a data loss + // when running two nodes with incompatible versions + checkPartSuccess + checkPartDiskNotFound + checkPartVolumeNotFound + checkPartFileNotFound + checkPartFileCorrupt +) + +// CheckPartsResp is a response of the storage CheckParts and VerifyFile APIs +type CheckPartsResp struct { + Results []int `msg:"r"` +} + +// LocalDiskIDs - GetLocalIDs response. +type LocalDiskIDs struct { + IDs []string `msg:"i"` +} + +// ListDirResult - ListDir()'s response. +type ListDirResult struct { + Entries []string `msg:"e"` +} + +// ReadPartsReq - send multiple part paths to read from +type ReadPartsReq struct { + Paths []string `msg:"p"` +} + +// ReadPartsResp - is the response for ReadPartsReq +type ReadPartsResp struct { + Infos []*ObjectPartInfo `msg:"is"` +} + +// DeleteBulkReq - send multiple paths in same delete request. +type DeleteBulkReq struct { + Paths []string `msg:"p"` +} + +// DeleteVersionsErrsResp - collection of delete errors +// for bulk version deletes +type DeleteVersionsErrsResp struct { + Errs []string `msg:"e"` +} diff --git a/cmd/storage-datatypes_gen.go b/cmd/storage-datatypes_gen.go new file mode 100644 index 00000000..3671bb2e --- /dev/null +++ b/cmd/storage-datatypes_gen.go @@ -0,0 +1,6933 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *BaseOptions) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z BaseOptions) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 0 + _ = z + err = en.Append(0x80) + if err != nil { + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z BaseOptions) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 0 + _ = z + o = append(o, 0x80) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BaseOptions) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z BaseOptions) Msgsize() (s int) { + s = 1 + return +} + +// DecodeMsg implements msgp.Decodable +func (z *CheckPartsHandlerParams) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.DiskID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + case "v": + z.Volume, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + case "fp": + z.FilePath, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "FilePath") + return + } + case "fi": + err = z.FI.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "FI") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *CheckPartsHandlerParams) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 4 + // write "id" + err = en.Append(0x84, 0xa2, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteString(z.DiskID) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + // write "v" + err = en.Append(0xa1, 0x76) + if err != nil { + return + } + err = en.WriteString(z.Volume) + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + // write "fp" + err = en.Append(0xa2, 0x66, 0x70) + if err != nil { + return + } + err = en.WriteString(z.FilePath) + if err != nil { + err = msgp.WrapError(err, "FilePath") + return + } + // write "fi" + err = en.Append(0xa2, 0x66, 0x69) + if err != nil { + return + } + err = z.FI.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "FI") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *CheckPartsHandlerParams) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 4 + // string "id" + o = append(o, 0x84, 0xa2, 0x69, 0x64) + o = msgp.AppendString(o, z.DiskID) + // string "v" + o = append(o, 0xa1, 0x76) + o = msgp.AppendString(o, z.Volume) + // string "fp" + o = append(o, 0xa2, 0x66, 0x70) + o = msgp.AppendString(o, z.FilePath) + // string "fi" + o = append(o, 0xa2, 0x66, 0x69) + o, err = z.FI.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "FI") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *CheckPartsHandlerParams) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.DiskID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + case "v": + z.Volume, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + case "fp": + z.FilePath, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "FilePath") + return + } + case "fi": + bts, err = z.FI.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "FI") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *CheckPartsHandlerParams) Msgsize() (s int) { + s = 1 + 3 + msgp.StringPrefixSize + len(z.DiskID) + 2 + msgp.StringPrefixSize + len(z.Volume) + 3 + msgp.StringPrefixSize + len(z.FilePath) + 3 + z.FI.Msgsize() + return +} + +// DecodeMsg implements msgp.Decodable +func (z *CheckPartsResp) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "r": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Results") + return + } + if cap(z.Results) >= int(zb0002) { + z.Results = (z.Results)[:zb0002] + } else { + z.Results = make([]int, zb0002) + } + for za0001 := range z.Results { + z.Results[za0001], err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Results", za0001) + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *CheckPartsResp) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "r" + err = en.Append(0x81, 0xa1, 0x72) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Results))) + if err != nil { + err = msgp.WrapError(err, "Results") + return + } + for za0001 := range z.Results { + err = en.WriteInt(z.Results[za0001]) + if err != nil { + err = msgp.WrapError(err, "Results", za0001) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *CheckPartsResp) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "r" + o = append(o, 0x81, 0xa1, 0x72) + o = msgp.AppendArrayHeader(o, uint32(len(z.Results))) + for za0001 := range z.Results { + o = msgp.AppendInt(o, z.Results[za0001]) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *CheckPartsResp) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "r": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Results") + return + } + if cap(z.Results) >= int(zb0002) { + z.Results = (z.Results)[:zb0002] + } else { + z.Results = make([]int, zb0002) + } + for za0001 := range z.Results { + z.Results[za0001], bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Results", za0001) + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *CheckPartsResp) Msgsize() (s int) { + s = 1 + 2 + msgp.ArrayHeaderSize + (len(z.Results) * (msgp.IntSize)) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *DeleteBulkReq) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "p": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Paths") + return + } + if cap(z.Paths) >= int(zb0002) { + z.Paths = (z.Paths)[:zb0002] + } else { + z.Paths = make([]string, zb0002) + } + for za0001 := range z.Paths { + z.Paths[za0001], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Paths", za0001) + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *DeleteBulkReq) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "p" + err = en.Append(0x81, 0xa1, 0x70) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Paths))) + if err != nil { + err = msgp.WrapError(err, "Paths") + return + } + for za0001 := range z.Paths { + err = en.WriteString(z.Paths[za0001]) + if err != nil { + err = msgp.WrapError(err, "Paths", za0001) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *DeleteBulkReq) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "p" + o = append(o, 0x81, 0xa1, 0x70) + o = msgp.AppendArrayHeader(o, uint32(len(z.Paths))) + for za0001 := range z.Paths { + o = msgp.AppendString(o, z.Paths[za0001]) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *DeleteBulkReq) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "p": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Paths") + return + } + if cap(z.Paths) >= int(zb0002) { + z.Paths = (z.Paths)[:zb0002] + } else { + z.Paths = make([]string, zb0002) + } + for za0001 := range z.Paths { + z.Paths[za0001], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Paths", za0001) + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *DeleteBulkReq) Msgsize() (s int) { + s = 1 + 2 + msgp.ArrayHeaderSize + for za0001 := range z.Paths { + s += msgp.StringPrefixSize + len(z.Paths[za0001]) + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *DeleteFileHandlerParams) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.DiskID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + case "v": + z.Volume, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + case "fp": + z.FilePath, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "FilePath") + return + } + case "do": + err = z.Opts.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Opts") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *DeleteFileHandlerParams) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 4 + // write "id" + err = en.Append(0x84, 0xa2, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteString(z.DiskID) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + // write "v" + err = en.Append(0xa1, 0x76) + if err != nil { + return + } + err = en.WriteString(z.Volume) + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + // write "fp" + err = en.Append(0xa2, 0x66, 0x70) + if err != nil { + return + } + err = en.WriteString(z.FilePath) + if err != nil { + err = msgp.WrapError(err, "FilePath") + return + } + // write "do" + err = en.Append(0xa2, 0x64, 0x6f) + if err != nil { + return + } + err = z.Opts.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Opts") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *DeleteFileHandlerParams) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 4 + // string "id" + o = append(o, 0x84, 0xa2, 0x69, 0x64) + o = msgp.AppendString(o, z.DiskID) + // string "v" + o = append(o, 0xa1, 0x76) + o = msgp.AppendString(o, z.Volume) + // string "fp" + o = append(o, 0xa2, 0x66, 0x70) + o = msgp.AppendString(o, z.FilePath) + // string "do" + o = append(o, 0xa2, 0x64, 0x6f) + o, err = z.Opts.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Opts") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *DeleteFileHandlerParams) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.DiskID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + case "v": + z.Volume, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + case "fp": + z.FilePath, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "FilePath") + return + } + case "do": + bts, err = z.Opts.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Opts") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *DeleteFileHandlerParams) Msgsize() (s int) { + s = 1 + 3 + msgp.StringPrefixSize + len(z.DiskID) + 2 + msgp.StringPrefixSize + len(z.Volume) + 3 + msgp.StringPrefixSize + len(z.FilePath) + 3 + z.Opts.Msgsize() + return +} + +// DecodeMsg implements msgp.Decodable +func (z *DeleteOptions) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 1 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "BaseOptions": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "BaseOptions") + return + } + for zb0002 > 0 { + zb0002-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "BaseOptions") + return + } + switch msgp.UnsafeString(field) { + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "BaseOptions") + return + } + } + } + case "r": + z.Recursive, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Recursive") + return + } + case "i": + z.Immediate, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Immediate") + return + } + case "u": + z.UndoWrite, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "UndoWrite") + return + } + case "o": + z.OldDataDir, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "OldDataDir") + return + } + zb0001Mask |= 0x1 + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if (zb0001Mask & 0x1) == 0 { + z.OldDataDir = "" + } + + return +} + +// EncodeMsg implements msgp.Encodable +func (z *DeleteOptions) EncodeMsg(en *msgp.Writer) (err error) { + // check for omitted fields + zb0001Len := uint32(5) + var zb0001Mask uint8 /* 5 bits */ + _ = zb0001Mask + if z.OldDataDir == "" { + zb0001Len-- + zb0001Mask |= 0x10 + } + // variable map header, size zb0001Len + err = en.Append(0x80 | uint8(zb0001Len)) + if err != nil { + return + } + + // skip if no fields are to be emitted + if zb0001Len != 0 { + // write "BaseOptions" + err = en.Append(0xab, 0x42, 0x61, 0x73, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73) + if err != nil { + return + } + // map header, size 0 + _ = z.BaseOptions + err = en.Append(0x80) + if err != nil { + return + } + // write "r" + err = en.Append(0xa1, 0x72) + if err != nil { + return + } + err = en.WriteBool(z.Recursive) + if err != nil { + err = msgp.WrapError(err, "Recursive") + return + } + // write "i" + err = en.Append(0xa1, 0x69) + if err != nil { + return + } + err = en.WriteBool(z.Immediate) + if err != nil { + err = msgp.WrapError(err, "Immediate") + return + } + // write "u" + err = en.Append(0xa1, 0x75) + if err != nil { + return + } + err = en.WriteBool(z.UndoWrite) + if err != nil { + err = msgp.WrapError(err, "UndoWrite") + return + } + if (zb0001Mask & 0x10) == 0 { // if not omitted + // write "o" + err = en.Append(0xa1, 0x6f) + if err != nil { + return + } + err = en.WriteString(z.OldDataDir) + if err != nil { + err = msgp.WrapError(err, "OldDataDir") + return + } + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *DeleteOptions) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // check for omitted fields + zb0001Len := uint32(5) + var zb0001Mask uint8 /* 5 bits */ + _ = zb0001Mask + if z.OldDataDir == "" { + zb0001Len-- + zb0001Mask |= 0x10 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + + // skip if no fields are to be emitted + if zb0001Len != 0 { + // string "BaseOptions" + o = append(o, 0xab, 0x42, 0x61, 0x73, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73) + // map header, size 0 + _ = z.BaseOptions + o = append(o, 0x80) + // string "r" + o = append(o, 0xa1, 0x72) + o = msgp.AppendBool(o, z.Recursive) + // string "i" + o = append(o, 0xa1, 0x69) + o = msgp.AppendBool(o, z.Immediate) + // string "u" + o = append(o, 0xa1, 0x75) + o = msgp.AppendBool(o, z.UndoWrite) + if (zb0001Mask & 0x10) == 0 { // if not omitted + // string "o" + o = append(o, 0xa1, 0x6f) + o = msgp.AppendString(o, z.OldDataDir) + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *DeleteOptions) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 1 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "BaseOptions": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "BaseOptions") + return + } + for zb0002 > 0 { + zb0002-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "BaseOptions") + return + } + switch msgp.UnsafeString(field) { + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "BaseOptions") + return + } + } + } + case "r": + z.Recursive, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Recursive") + return + } + case "i": + z.Immediate, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Immediate") + return + } + case "u": + z.UndoWrite, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UndoWrite") + return + } + case "o": + z.OldDataDir, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "OldDataDir") + return + } + zb0001Mask |= 0x1 + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if (zb0001Mask & 0x1) == 0 { + z.OldDataDir = "" + } + + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *DeleteOptions) Msgsize() (s int) { + s = 1 + 12 + 1 + 2 + msgp.BoolSize + 2 + msgp.BoolSize + 2 + msgp.BoolSize + 2 + msgp.StringPrefixSize + len(z.OldDataDir) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *DeleteVersionHandlerParams) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.DiskID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + case "v": + z.Volume, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + case "fp": + z.FilePath, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "FilePath") + return + } + case "fdm": + z.ForceDelMarker, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "ForceDelMarker") + return + } + case "do": + err = z.Opts.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Opts") + return + } + case "fi": + err = z.FI.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "FI") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *DeleteVersionHandlerParams) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 6 + // write "id" + err = en.Append(0x86, 0xa2, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteString(z.DiskID) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + // write "v" + err = en.Append(0xa1, 0x76) + if err != nil { + return + } + err = en.WriteString(z.Volume) + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + // write "fp" + err = en.Append(0xa2, 0x66, 0x70) + if err != nil { + return + } + err = en.WriteString(z.FilePath) + if err != nil { + err = msgp.WrapError(err, "FilePath") + return + } + // write "fdm" + err = en.Append(0xa3, 0x66, 0x64, 0x6d) + if err != nil { + return + } + err = en.WriteBool(z.ForceDelMarker) + if err != nil { + err = msgp.WrapError(err, "ForceDelMarker") + return + } + // write "do" + err = en.Append(0xa2, 0x64, 0x6f) + if err != nil { + return + } + err = z.Opts.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Opts") + return + } + // write "fi" + err = en.Append(0xa2, 0x66, 0x69) + if err != nil { + return + } + err = z.FI.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "FI") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *DeleteVersionHandlerParams) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 6 + // string "id" + o = append(o, 0x86, 0xa2, 0x69, 0x64) + o = msgp.AppendString(o, z.DiskID) + // string "v" + o = append(o, 0xa1, 0x76) + o = msgp.AppendString(o, z.Volume) + // string "fp" + o = append(o, 0xa2, 0x66, 0x70) + o = msgp.AppendString(o, z.FilePath) + // string "fdm" + o = append(o, 0xa3, 0x66, 0x64, 0x6d) + o = msgp.AppendBool(o, z.ForceDelMarker) + // string "do" + o = append(o, 0xa2, 0x64, 0x6f) + o, err = z.Opts.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Opts") + return + } + // string "fi" + o = append(o, 0xa2, 0x66, 0x69) + o, err = z.FI.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "FI") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *DeleteVersionHandlerParams) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.DiskID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + case "v": + z.Volume, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + case "fp": + z.FilePath, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "FilePath") + return + } + case "fdm": + z.ForceDelMarker, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ForceDelMarker") + return + } + case "do": + bts, err = z.Opts.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Opts") + return + } + case "fi": + bts, err = z.FI.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "FI") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *DeleteVersionHandlerParams) Msgsize() (s int) { + s = 1 + 3 + msgp.StringPrefixSize + len(z.DiskID) + 2 + msgp.StringPrefixSize + len(z.Volume) + 3 + msgp.StringPrefixSize + len(z.FilePath) + 4 + msgp.BoolSize + 3 + z.Opts.Msgsize() + 3 + z.FI.Msgsize() + return +} + +// DecodeMsg implements msgp.Decodable +func (z *DeleteVersionsErrsResp) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "e": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Errs") + return + } + if cap(z.Errs) >= int(zb0002) { + z.Errs = (z.Errs)[:zb0002] + } else { + z.Errs = make([]string, zb0002) + } + for za0001 := range z.Errs { + z.Errs[za0001], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Errs", za0001) + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *DeleteVersionsErrsResp) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "e" + err = en.Append(0x81, 0xa1, 0x65) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Errs))) + if err != nil { + err = msgp.WrapError(err, "Errs") + return + } + for za0001 := range z.Errs { + err = en.WriteString(z.Errs[za0001]) + if err != nil { + err = msgp.WrapError(err, "Errs", za0001) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *DeleteVersionsErrsResp) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "e" + o = append(o, 0x81, 0xa1, 0x65) + o = msgp.AppendArrayHeader(o, uint32(len(z.Errs))) + for za0001 := range z.Errs { + o = msgp.AppendString(o, z.Errs[za0001]) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *DeleteVersionsErrsResp) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "e": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Errs") + return + } + if cap(z.Errs) >= int(zb0002) { + z.Errs = (z.Errs)[:zb0002] + } else { + z.Errs = make([]string, zb0002) + } + for za0001 := range z.Errs { + z.Errs[za0001], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Errs", za0001) + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *DeleteVersionsErrsResp) Msgsize() (s int) { + s = 1 + 2 + msgp.ArrayHeaderSize + for za0001 := range z.Errs { + s += msgp.StringPrefixSize + len(z.Errs[za0001]) + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *DiskInfo) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0001 uint32 + zb0001, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 18 { + err = msgp.ArrayError{Wanted: 18, Got: zb0001} + return + } + z.Total, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Total") + return + } + z.Free, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Free") + return + } + z.Used, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "Used") + return + } + z.UsedInodes, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "UsedInodes") + return + } + z.FreeInodes, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "FreeInodes") + return + } + z.Major, err = dc.ReadUint32() + if err != nil { + err = msgp.WrapError(err, "Major") + return + } + z.Minor, err = dc.ReadUint32() + if err != nil { + err = msgp.WrapError(err, "Minor") + return + } + z.NRRequests, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "NRRequests") + return + } + z.FSType, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "FSType") + return + } + z.RootDisk, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "RootDisk") + return + } + z.Healing, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Healing") + return + } + z.Scanning, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Scanning") + return + } + z.Endpoint, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + z.MountPath, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "MountPath") + return + } + z.ID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + z.Rotational, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Rotational") + return + } + err = z.Metrics.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Metrics") + return + } + z.Error, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Error") + return + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *DiskInfo) EncodeMsg(en *msgp.Writer) (err error) { + // array header, size 18 + err = en.Append(0xdc, 0x0, 0x12) + if err != nil { + return + } + err = en.WriteUint64(z.Total) + if err != nil { + err = msgp.WrapError(err, "Total") + return + } + err = en.WriteUint64(z.Free) + if err != nil { + err = msgp.WrapError(err, "Free") + return + } + err = en.WriteUint64(z.Used) + if err != nil { + err = msgp.WrapError(err, "Used") + return + } + err = en.WriteUint64(z.UsedInodes) + if err != nil { + err = msgp.WrapError(err, "UsedInodes") + return + } + err = en.WriteUint64(z.FreeInodes) + if err != nil { + err = msgp.WrapError(err, "FreeInodes") + return + } + err = en.WriteUint32(z.Major) + if err != nil { + err = msgp.WrapError(err, "Major") + return + } + err = en.WriteUint32(z.Minor) + if err != nil { + err = msgp.WrapError(err, "Minor") + return + } + err = en.WriteUint64(z.NRRequests) + if err != nil { + err = msgp.WrapError(err, "NRRequests") + return + } + err = en.WriteString(z.FSType) + if err != nil { + err = msgp.WrapError(err, "FSType") + return + } + err = en.WriteBool(z.RootDisk) + if err != nil { + err = msgp.WrapError(err, "RootDisk") + return + } + err = en.WriteBool(z.Healing) + if err != nil { + err = msgp.WrapError(err, "Healing") + return + } + err = en.WriteBool(z.Scanning) + if err != nil { + err = msgp.WrapError(err, "Scanning") + return + } + err = en.WriteString(z.Endpoint) + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + err = en.WriteString(z.MountPath) + if err != nil { + err = msgp.WrapError(err, "MountPath") + return + } + err = en.WriteString(z.ID) + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + err = en.WriteBool(z.Rotational) + if err != nil { + err = msgp.WrapError(err, "Rotational") + return + } + err = z.Metrics.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Metrics") + return + } + err = en.WriteString(z.Error) + if err != nil { + err = msgp.WrapError(err, "Error") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *DiskInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // array header, size 18 + o = append(o, 0xdc, 0x0, 0x12) + o = msgp.AppendUint64(o, z.Total) + o = msgp.AppendUint64(o, z.Free) + o = msgp.AppendUint64(o, z.Used) + o = msgp.AppendUint64(o, z.UsedInodes) + o = msgp.AppendUint64(o, z.FreeInodes) + o = msgp.AppendUint32(o, z.Major) + o = msgp.AppendUint32(o, z.Minor) + o = msgp.AppendUint64(o, z.NRRequests) + o = msgp.AppendString(o, z.FSType) + o = msgp.AppendBool(o, z.RootDisk) + o = msgp.AppendBool(o, z.Healing) + o = msgp.AppendBool(o, z.Scanning) + o = msgp.AppendString(o, z.Endpoint) + o = msgp.AppendString(o, z.MountPath) + o = msgp.AppendString(o, z.ID) + o = msgp.AppendBool(o, z.Rotational) + o, err = z.Metrics.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Metrics") + return + } + o = msgp.AppendString(o, z.Error) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *DiskInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0001 uint32 + zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 18 { + err = msgp.ArrayError{Wanted: 18, Got: zb0001} + return + } + z.Total, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Total") + return + } + z.Free, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Free") + return + } + z.Used, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Used") + return + } + z.UsedInodes, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "UsedInodes") + return + } + z.FreeInodes, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "FreeInodes") + return + } + z.Major, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Major") + return + } + z.Minor, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Minor") + return + } + z.NRRequests, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "NRRequests") + return + } + z.FSType, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "FSType") + return + } + z.RootDisk, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "RootDisk") + return + } + z.Healing, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Healing") + return + } + z.Scanning, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Scanning") + return + } + z.Endpoint, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Endpoint") + return + } + z.MountPath, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "MountPath") + return + } + z.ID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + z.Rotational, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Rotational") + return + } + bts, err = z.Metrics.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Metrics") + return + } + z.Error, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Error") + return + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *DiskInfo) Msgsize() (s int) { + s = 3 + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint64Size + msgp.Uint32Size + msgp.Uint32Size + msgp.Uint64Size + msgp.StringPrefixSize + len(z.FSType) + msgp.BoolSize + msgp.BoolSize + msgp.BoolSize + msgp.StringPrefixSize + len(z.Endpoint) + msgp.StringPrefixSize + len(z.MountPath) + msgp.StringPrefixSize + len(z.ID) + msgp.BoolSize + z.Metrics.Msgsize() + msgp.StringPrefixSize + len(z.Error) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *DiskInfoOptions) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.DiskID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + case "m": + z.Metrics, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Metrics") + return + } + case "np": + z.NoOp, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "NoOp") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z DiskInfoOptions) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "id" + err = en.Append(0x83, 0xa2, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteString(z.DiskID) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + // write "m" + err = en.Append(0xa1, 0x6d) + if err != nil { + return + } + err = en.WriteBool(z.Metrics) + if err != nil { + err = msgp.WrapError(err, "Metrics") + return + } + // write "np" + err = en.Append(0xa2, 0x6e, 0x70) + if err != nil { + return + } + err = en.WriteBool(z.NoOp) + if err != nil { + err = msgp.WrapError(err, "NoOp") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z DiskInfoOptions) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "id" + o = append(o, 0x83, 0xa2, 0x69, 0x64) + o = msgp.AppendString(o, z.DiskID) + // string "m" + o = append(o, 0xa1, 0x6d) + o = msgp.AppendBool(o, z.Metrics) + // string "np" + o = append(o, 0xa2, 0x6e, 0x70) + o = msgp.AppendBool(o, z.NoOp) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *DiskInfoOptions) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.DiskID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + case "m": + z.Metrics, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Metrics") + return + } + case "np": + z.NoOp, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "NoOp") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z DiskInfoOptions) Msgsize() (s int) { + s = 1 + 3 + msgp.StringPrefixSize + len(z.DiskID) + 2 + msgp.BoolSize + 3 + msgp.BoolSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *DiskMetrics) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "LastMinute": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "LastMinute") + return + } + if z.LastMinute == nil { + z.LastMinute = make(map[string]AccElem, zb0002) + } else if len(z.LastMinute) > 0 { + clear(z.LastMinute) + } + for zb0002 > 0 { + zb0002-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "LastMinute") + return + } + var za0002 AccElem + err = za0002.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "LastMinute", za0001) + return + } + z.LastMinute[za0001] = za0002 + } + case "APICalls": + var zb0003 uint32 + zb0003, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "APICalls") + return + } + if z.APICalls == nil { + z.APICalls = make(map[string]uint64, zb0003) + } else if len(z.APICalls) > 0 { + clear(z.APICalls) + } + for zb0003 > 0 { + zb0003-- + var za0003 string + za0003, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "APICalls") + return + } + var za0004 uint64 + za0004, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "APICalls", za0003) + return + } + z.APICalls[za0003] = za0004 + } + case "TotalWaiting": + z.TotalWaiting, err = dc.ReadUint32() + if err != nil { + err = msgp.WrapError(err, "TotalWaiting") + return + } + case "TotalErrorsAvailability": + z.TotalErrorsAvailability, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "TotalErrorsAvailability") + return + } + case "TotalErrorsTimeout": + z.TotalErrorsTimeout, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "TotalErrorsTimeout") + return + } + case "TotalWrites": + z.TotalWrites, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "TotalWrites") + return + } + case "TotalDeletes": + z.TotalDeletes, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "TotalDeletes") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *DiskMetrics) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 7 + // write "LastMinute" + err = en.Append(0x87, 0xaa, 0x4c, 0x61, 0x73, 0x74, 0x4d, 0x69, 0x6e, 0x75, 0x74, 0x65) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.LastMinute))) + if err != nil { + err = msgp.WrapError(err, "LastMinute") + return + } + for za0001, za0002 := range z.LastMinute { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "LastMinute") + return + } + err = za0002.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "LastMinute", za0001) + return + } + } + // write "APICalls" + err = en.Append(0xa8, 0x41, 0x50, 0x49, 0x43, 0x61, 0x6c, 0x6c, 0x73) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.APICalls))) + if err != nil { + err = msgp.WrapError(err, "APICalls") + return + } + for za0003, za0004 := range z.APICalls { + err = en.WriteString(za0003) + if err != nil { + err = msgp.WrapError(err, "APICalls") + return + } + err = en.WriteUint64(za0004) + if err != nil { + err = msgp.WrapError(err, "APICalls", za0003) + return + } + } + // write "TotalWaiting" + err = en.Append(0xac, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x57, 0x61, 0x69, 0x74, 0x69, 0x6e, 0x67) + if err != nil { + return + } + err = en.WriteUint32(z.TotalWaiting) + if err != nil { + err = msgp.WrapError(err, "TotalWaiting") + return + } + // write "TotalErrorsAvailability" + err = en.Append(0xb7, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79) + if err != nil { + return + } + err = en.WriteUint64(z.TotalErrorsAvailability) + if err != nil { + err = msgp.WrapError(err, "TotalErrorsAvailability") + return + } + // write "TotalErrorsTimeout" + err = en.Append(0xb2, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74) + if err != nil { + return + } + err = en.WriteUint64(z.TotalErrorsTimeout) + if err != nil { + err = msgp.WrapError(err, "TotalErrorsTimeout") + return + } + // write "TotalWrites" + err = en.Append(0xab, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteUint64(z.TotalWrites) + if err != nil { + err = msgp.WrapError(err, "TotalWrites") + return + } + // write "TotalDeletes" + err = en.Append(0xac, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteUint64(z.TotalDeletes) + if err != nil { + err = msgp.WrapError(err, "TotalDeletes") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *DiskMetrics) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 7 + // string "LastMinute" + o = append(o, 0x87, 0xaa, 0x4c, 0x61, 0x73, 0x74, 0x4d, 0x69, 0x6e, 0x75, 0x74, 0x65) + o = msgp.AppendMapHeader(o, uint32(len(z.LastMinute))) + for za0001, za0002 := range z.LastMinute { + o = msgp.AppendString(o, za0001) + o, err = za0002.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "LastMinute", za0001) + return + } + } + // string "APICalls" + o = append(o, 0xa8, 0x41, 0x50, 0x49, 0x43, 0x61, 0x6c, 0x6c, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.APICalls))) + for za0003, za0004 := range z.APICalls { + o = msgp.AppendString(o, za0003) + o = msgp.AppendUint64(o, za0004) + } + // string "TotalWaiting" + o = append(o, 0xac, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x57, 0x61, 0x69, 0x74, 0x69, 0x6e, 0x67) + o = msgp.AppendUint32(o, z.TotalWaiting) + // string "TotalErrorsAvailability" + o = append(o, 0xb7, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x41, 0x76, 0x61, 0x69, 0x6c, 0x61, 0x62, 0x69, 0x6c, 0x69, 0x74, 0x79) + o = msgp.AppendUint64(o, z.TotalErrorsAvailability) + // string "TotalErrorsTimeout" + o = append(o, 0xb2, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x45, 0x72, 0x72, 0x6f, 0x72, 0x73, 0x54, 0x69, 0x6d, 0x65, 0x6f, 0x75, 0x74) + o = msgp.AppendUint64(o, z.TotalErrorsTimeout) + // string "TotalWrites" + o = append(o, 0xab, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x57, 0x72, 0x69, 0x74, 0x65, 0x73) + o = msgp.AppendUint64(o, z.TotalWrites) + // string "TotalDeletes" + o = append(o, 0xac, 0x54, 0x6f, 0x74, 0x61, 0x6c, 0x44, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x73) + o = msgp.AppendUint64(o, z.TotalDeletes) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *DiskMetrics) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "LastMinute": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "LastMinute") + return + } + if z.LastMinute == nil { + z.LastMinute = make(map[string]AccElem, zb0002) + } else if len(z.LastMinute) > 0 { + clear(z.LastMinute) + } + for zb0002 > 0 { + var za0002 AccElem + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "LastMinute") + return + } + bts, err = za0002.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "LastMinute", za0001) + return + } + z.LastMinute[za0001] = za0002 + } + case "APICalls": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "APICalls") + return + } + if z.APICalls == nil { + z.APICalls = make(map[string]uint64, zb0003) + } else if len(z.APICalls) > 0 { + clear(z.APICalls) + } + for zb0003 > 0 { + var za0004 uint64 + zb0003-- + var za0003 string + za0003, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "APICalls") + return + } + za0004, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "APICalls", za0003) + return + } + z.APICalls[za0003] = za0004 + } + case "TotalWaiting": + z.TotalWaiting, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalWaiting") + return + } + case "TotalErrorsAvailability": + z.TotalErrorsAvailability, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalErrorsAvailability") + return + } + case "TotalErrorsTimeout": + z.TotalErrorsTimeout, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalErrorsTimeout") + return + } + case "TotalWrites": + z.TotalWrites, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalWrites") + return + } + case "TotalDeletes": + z.TotalDeletes, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "TotalDeletes") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *DiskMetrics) Msgsize() (s int) { + s = 1 + 11 + msgp.MapHeaderSize + if z.LastMinute != nil { + for za0001, za0002 := range z.LastMinute { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize() + } + } + s += 9 + msgp.MapHeaderSize + if z.APICalls != nil { + for za0003, za0004 := range z.APICalls { + _ = za0004 + s += msgp.StringPrefixSize + len(za0003) + msgp.Uint64Size + } + } + s += 13 + msgp.Uint32Size + 24 + msgp.Uint64Size + 19 + msgp.Uint64Size + 12 + msgp.Uint64Size + 13 + msgp.Uint64Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *FileInfo) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0001 uint32 + zb0001, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 28 { + err = msgp.ArrayError{Wanted: 28, Got: zb0001} + return + } + z.Volume, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + z.Name, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + z.VersionID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + z.IsLatest, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "IsLatest") + return + } + z.Deleted, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Deleted") + return + } + z.TransitionStatus, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "TransitionStatus") + return + } + z.TransitionedObjName, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "TransitionedObjName") + return + } + z.TransitionTier, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "TransitionTier") + return + } + z.TransitionVersionID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "TransitionVersionID") + return + } + z.ExpireRestored, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "ExpireRestored") + return + } + z.DataDir, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DataDir") + return + } + z.XLV1, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "XLV1") + return + } + z.ModTime, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + z.Size, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + z.Mode, err = dc.ReadUint32() + if err != nil { + err = msgp.WrapError(err, "Mode") + return + } + z.WrittenByVersion, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "WrittenByVersion") + return + } + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Metadata") + return + } + if z.Metadata == nil { + z.Metadata = make(map[string]string, zb0002) + } else if len(z.Metadata) > 0 { + clear(z.Metadata) + } + var field []byte + _ = field + for zb0002 > 0 { + zb0002-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Metadata") + return + } + var za0002 string + za0002, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Metadata", za0001) + return + } + z.Metadata[za0001] = za0002 + } + var zb0003 uint32 + zb0003, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Parts") + return + } + if cap(z.Parts) >= int(zb0003) { + z.Parts = (z.Parts)[:zb0003] + } else { + z.Parts = make([]ObjectPartInfo, zb0003) + } + for za0003 := range z.Parts { + err = z.Parts[za0003].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Parts", za0003) + return + } + } + err = z.Erasure.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Erasure") + return + } + z.MarkDeleted, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "MarkDeleted") + return + } + err = z.ReplicationState.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "ReplicationState") + return + } + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err) + return + } + z.Data = nil + } else { + z.Data, err = dc.ReadBytes(z.Data) + if err != nil { + err = msgp.WrapError(err, "Data") + return + } + if z.Data == nil { + z.Data = make([]byte, 0) + } + } + z.NumVersions, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "NumVersions") + return + } + z.SuccessorModTime, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "SuccessorModTime") + return + } + z.Fresh, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Fresh") + return + } + z.Idx, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Idx") + return + } + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err) + return + } + z.Checksum = nil + } else { + z.Checksum, err = dc.ReadBytes(z.Checksum) + if err != nil { + err = msgp.WrapError(err, "Checksum") + return + } + if z.Checksum == nil { + z.Checksum = make([]byte, 0) + } + } + z.Versioned, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Versioned") + return + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *FileInfo) EncodeMsg(en *msgp.Writer) (err error) { + // array header, size 28 + err = en.Append(0xdc, 0x0, 0x1c) + if err != nil { + return + } + err = en.WriteString(z.Volume) + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + err = en.WriteString(z.Name) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + err = en.WriteString(z.VersionID) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + err = en.WriteBool(z.IsLatest) + if err != nil { + err = msgp.WrapError(err, "IsLatest") + return + } + err = en.WriteBool(z.Deleted) + if err != nil { + err = msgp.WrapError(err, "Deleted") + return + } + err = en.WriteString(z.TransitionStatus) + if err != nil { + err = msgp.WrapError(err, "TransitionStatus") + return + } + err = en.WriteString(z.TransitionedObjName) + if err != nil { + err = msgp.WrapError(err, "TransitionedObjName") + return + } + err = en.WriteString(z.TransitionTier) + if err != nil { + err = msgp.WrapError(err, "TransitionTier") + return + } + err = en.WriteString(z.TransitionVersionID) + if err != nil { + err = msgp.WrapError(err, "TransitionVersionID") + return + } + err = en.WriteBool(z.ExpireRestored) + if err != nil { + err = msgp.WrapError(err, "ExpireRestored") + return + } + err = en.WriteString(z.DataDir) + if err != nil { + err = msgp.WrapError(err, "DataDir") + return + } + err = en.WriteBool(z.XLV1) + if err != nil { + err = msgp.WrapError(err, "XLV1") + return + } + err = en.WriteTime(z.ModTime) + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + err = en.WriteInt64(z.Size) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + err = en.WriteUint32(z.Mode) + if err != nil { + err = msgp.WrapError(err, "Mode") + return + } + err = en.WriteUint64(z.WrittenByVersion) + if err != nil { + err = msgp.WrapError(err, "WrittenByVersion") + return + } + err = en.WriteMapHeader(uint32(len(z.Metadata))) + if err != nil { + err = msgp.WrapError(err, "Metadata") + return + } + for za0001, za0002 := range z.Metadata { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "Metadata") + return + } + err = en.WriteString(za0002) + if err != nil { + err = msgp.WrapError(err, "Metadata", za0001) + return + } + } + err = en.WriteArrayHeader(uint32(len(z.Parts))) + if err != nil { + err = msgp.WrapError(err, "Parts") + return + } + for za0003 := range z.Parts { + err = z.Parts[za0003].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Parts", za0003) + return + } + } + err = z.Erasure.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Erasure") + return + } + err = en.WriteBool(z.MarkDeleted) + if err != nil { + err = msgp.WrapError(err, "MarkDeleted") + return + } + err = z.ReplicationState.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "ReplicationState") + return + } + if z.Data == nil { // allownil: if nil + err = en.WriteNil() + if err != nil { + return + } + } else { + err = en.WriteBytes(z.Data) + if err != nil { + err = msgp.WrapError(err, "Data") + return + } + } + err = en.WriteInt(z.NumVersions) + if err != nil { + err = msgp.WrapError(err, "NumVersions") + return + } + err = en.WriteTime(z.SuccessorModTime) + if err != nil { + err = msgp.WrapError(err, "SuccessorModTime") + return + } + err = en.WriteBool(z.Fresh) + if err != nil { + err = msgp.WrapError(err, "Fresh") + return + } + err = en.WriteInt(z.Idx) + if err != nil { + err = msgp.WrapError(err, "Idx") + return + } + if z.Checksum == nil { // allownil: if nil + err = en.WriteNil() + if err != nil { + return + } + } else { + err = en.WriteBytes(z.Checksum) + if err != nil { + err = msgp.WrapError(err, "Checksum") + return + } + } + err = en.WriteBool(z.Versioned) + if err != nil { + err = msgp.WrapError(err, "Versioned") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *FileInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // array header, size 28 + o = append(o, 0xdc, 0x0, 0x1c) + o = msgp.AppendString(o, z.Volume) + o = msgp.AppendString(o, z.Name) + o = msgp.AppendString(o, z.VersionID) + o = msgp.AppendBool(o, z.IsLatest) + o = msgp.AppendBool(o, z.Deleted) + o = msgp.AppendString(o, z.TransitionStatus) + o = msgp.AppendString(o, z.TransitionedObjName) + o = msgp.AppendString(o, z.TransitionTier) + o = msgp.AppendString(o, z.TransitionVersionID) + o = msgp.AppendBool(o, z.ExpireRestored) + o = msgp.AppendString(o, z.DataDir) + o = msgp.AppendBool(o, z.XLV1) + o = msgp.AppendTime(o, z.ModTime) + o = msgp.AppendInt64(o, z.Size) + o = msgp.AppendUint32(o, z.Mode) + o = msgp.AppendUint64(o, z.WrittenByVersion) + o = msgp.AppendMapHeader(o, uint32(len(z.Metadata))) + for za0001, za0002 := range z.Metadata { + o = msgp.AppendString(o, za0001) + o = msgp.AppendString(o, za0002) + } + o = msgp.AppendArrayHeader(o, uint32(len(z.Parts))) + for za0003 := range z.Parts { + o, err = z.Parts[za0003].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Parts", za0003) + return + } + } + o, err = z.Erasure.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Erasure") + return + } + o = msgp.AppendBool(o, z.MarkDeleted) + o, err = z.ReplicationState.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "ReplicationState") + return + } + if z.Data == nil { // allownil: if nil + o = msgp.AppendNil(o) + } else { + o = msgp.AppendBytes(o, z.Data) + } + o = msgp.AppendInt(o, z.NumVersions) + o = msgp.AppendTime(o, z.SuccessorModTime) + o = msgp.AppendBool(o, z.Fresh) + o = msgp.AppendInt(o, z.Idx) + if z.Checksum == nil { // allownil: if nil + o = msgp.AppendNil(o) + } else { + o = msgp.AppendBytes(o, z.Checksum) + } + o = msgp.AppendBool(o, z.Versioned) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *FileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0001 uint32 + zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 28 { + err = msgp.ArrayError{Wanted: 28, Got: zb0001} + return + } + z.Volume, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + z.Name, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + z.VersionID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + z.IsLatest, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "IsLatest") + return + } + z.Deleted, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Deleted") + return + } + z.TransitionStatus, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "TransitionStatus") + return + } + z.TransitionedObjName, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "TransitionedObjName") + return + } + z.TransitionTier, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "TransitionTier") + return + } + z.TransitionVersionID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "TransitionVersionID") + return + } + z.ExpireRestored, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ExpireRestored") + return + } + z.DataDir, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DataDir") + return + } + z.XLV1, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "XLV1") + return + } + z.ModTime, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + z.Size, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + z.Mode, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Mode") + return + } + z.WrittenByVersion, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "WrittenByVersion") + return + } + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Metadata") + return + } + if z.Metadata == nil { + z.Metadata = make(map[string]string, zb0002) + } else if len(z.Metadata) > 0 { + clear(z.Metadata) + } + var field []byte + _ = field + for zb0002 > 0 { + var za0002 string + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Metadata") + return + } + za0002, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Metadata", za0001) + return + } + z.Metadata[za0001] = za0002 + } + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Parts") + return + } + if cap(z.Parts) >= int(zb0003) { + z.Parts = (z.Parts)[:zb0003] + } else { + z.Parts = make([]ObjectPartInfo, zb0003) + } + for za0003 := range z.Parts { + bts, err = z.Parts[za0003].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Parts", za0003) + return + } + } + bts, err = z.Erasure.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Erasure") + return + } + z.MarkDeleted, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "MarkDeleted") + return + } + bts, err = z.ReplicationState.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ReplicationState") + return + } + if msgp.IsNil(bts) { + bts = bts[1:] + z.Data = nil + } else { + z.Data, bts, err = msgp.ReadBytesBytes(bts, z.Data) + if err != nil { + err = msgp.WrapError(err, "Data") + return + } + if z.Data == nil { + z.Data = make([]byte, 0) + } + } + z.NumVersions, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "NumVersions") + return + } + z.SuccessorModTime, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SuccessorModTime") + return + } + z.Fresh, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Fresh") + return + } + z.Idx, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Idx") + return + } + if msgp.IsNil(bts) { + bts = bts[1:] + z.Checksum = nil + } else { + z.Checksum, bts, err = msgp.ReadBytesBytes(bts, z.Checksum) + if err != nil { + err = msgp.WrapError(err, "Checksum") + return + } + if z.Checksum == nil { + z.Checksum = make([]byte, 0) + } + } + z.Versioned, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Versioned") + return + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *FileInfo) Msgsize() (s int) { + s = 3 + msgp.StringPrefixSize + len(z.Volume) + msgp.StringPrefixSize + len(z.Name) + msgp.StringPrefixSize + len(z.VersionID) + msgp.BoolSize + msgp.BoolSize + msgp.StringPrefixSize + len(z.TransitionStatus) + msgp.StringPrefixSize + len(z.TransitionedObjName) + msgp.StringPrefixSize + len(z.TransitionTier) + msgp.StringPrefixSize + len(z.TransitionVersionID) + msgp.BoolSize + msgp.StringPrefixSize + len(z.DataDir) + msgp.BoolSize + msgp.TimeSize + msgp.Int64Size + msgp.Uint32Size + msgp.Uint64Size + msgp.MapHeaderSize + if z.Metadata != nil { + for za0001, za0002 := range z.Metadata { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + msgp.StringPrefixSize + len(za0002) + } + } + s += msgp.ArrayHeaderSize + for za0003 := range z.Parts { + s += z.Parts[za0003].Msgsize() + } + s += z.Erasure.Msgsize() + msgp.BoolSize + z.ReplicationState.Msgsize() + msgp.BytesPrefixSize + len(z.Data) + msgp.IntSize + msgp.TimeSize + msgp.BoolSize + msgp.IntSize + msgp.BytesPrefixSize + len(z.Checksum) + msgp.BoolSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *FileInfoVersions) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0001 uint32 + zb0001, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 5 { + err = msgp.ArrayError{Wanted: 5, Got: zb0001} + return + } + z.Volume, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + z.Name, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + z.LatestModTime, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "LatestModTime") + return + } + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + if cap(z.Versions) >= int(zb0002) { + z.Versions = (z.Versions)[:zb0002] + } else { + z.Versions = make([]FileInfo, zb0002) + } + for za0001 := range z.Versions { + err = z.Versions[za0001].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Versions", za0001) + return + } + } + var zb0003 uint32 + zb0003, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "FreeVersions") + return + } + if cap(z.FreeVersions) >= int(zb0003) { + z.FreeVersions = (z.FreeVersions)[:zb0003] + } else { + z.FreeVersions = make([]FileInfo, zb0003) + } + for za0002 := range z.FreeVersions { + err = z.FreeVersions[za0002].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "FreeVersions", za0002) + return + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *FileInfoVersions) EncodeMsg(en *msgp.Writer) (err error) { + // array header, size 5 + err = en.Append(0x95) + if err != nil { + return + } + err = en.WriteString(z.Volume) + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + err = en.WriteString(z.Name) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + err = en.WriteTime(z.LatestModTime) + if err != nil { + err = msgp.WrapError(err, "LatestModTime") + return + } + err = en.WriteArrayHeader(uint32(len(z.Versions))) + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + for za0001 := range z.Versions { + err = z.Versions[za0001].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Versions", za0001) + return + } + } + err = en.WriteArrayHeader(uint32(len(z.FreeVersions))) + if err != nil { + err = msgp.WrapError(err, "FreeVersions") + return + } + for za0002 := range z.FreeVersions { + err = z.FreeVersions[za0002].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "FreeVersions", za0002) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *FileInfoVersions) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // array header, size 5 + o = append(o, 0x95) + o = msgp.AppendString(o, z.Volume) + o = msgp.AppendString(o, z.Name) + o = msgp.AppendTime(o, z.LatestModTime) + o = msgp.AppendArrayHeader(o, uint32(len(z.Versions))) + for za0001 := range z.Versions { + o, err = z.Versions[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Versions", za0001) + return + } + } + o = msgp.AppendArrayHeader(o, uint32(len(z.FreeVersions))) + for za0002 := range z.FreeVersions { + o, err = z.FreeVersions[za0002].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "FreeVersions", za0002) + return + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *FileInfoVersions) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0001 uint32 + zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 5 { + err = msgp.ArrayError{Wanted: 5, Got: zb0001} + return + } + z.Volume, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + z.Name, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + z.LatestModTime, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "LatestModTime") + return + } + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Versions") + return + } + if cap(z.Versions) >= int(zb0002) { + z.Versions = (z.Versions)[:zb0002] + } else { + z.Versions = make([]FileInfo, zb0002) + } + for za0001 := range z.Versions { + bts, err = z.Versions[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Versions", za0001) + return + } + } + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "FreeVersions") + return + } + if cap(z.FreeVersions) >= int(zb0003) { + z.FreeVersions = (z.FreeVersions)[:zb0003] + } else { + z.FreeVersions = make([]FileInfo, zb0003) + } + for za0002 := range z.FreeVersions { + bts, err = z.FreeVersions[za0002].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "FreeVersions", za0002) + return + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *FileInfoVersions) Msgsize() (s int) { + s = 1 + msgp.StringPrefixSize + len(z.Volume) + msgp.StringPrefixSize + len(z.Name) + msgp.TimeSize + msgp.ArrayHeaderSize + for za0001 := range z.Versions { + s += z.Versions[za0001].Msgsize() + } + s += msgp.ArrayHeaderSize + for za0002 := range z.FreeVersions { + s += z.FreeVersions[za0002].Msgsize() + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *FilesInfo) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Files": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Files") + return + } + if cap(z.Files) >= int(zb0002) { + z.Files = (z.Files)[:zb0002] + } else { + z.Files = make([]FileInfo, zb0002) + } + for za0001 := range z.Files { + err = z.Files[za0001].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Files", za0001) + return + } + } + case "IsTruncated": + z.IsTruncated, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "IsTruncated") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *FilesInfo) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "Files" + err = en.Append(0x82, 0xa5, 0x46, 0x69, 0x6c, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Files))) + if err != nil { + err = msgp.WrapError(err, "Files") + return + } + for za0001 := range z.Files { + err = z.Files[za0001].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Files", za0001) + return + } + } + // write "IsTruncated" + err = en.Append(0xab, 0x49, 0x73, 0x54, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteBool(z.IsTruncated) + if err != nil { + err = msgp.WrapError(err, "IsTruncated") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *FilesInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "Files" + o = append(o, 0x82, 0xa5, 0x46, 0x69, 0x6c, 0x65, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Files))) + for za0001 := range z.Files { + o, err = z.Files[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Files", za0001) + return + } + } + // string "IsTruncated" + o = append(o, 0xab, 0x49, 0x73, 0x54, 0x72, 0x75, 0x6e, 0x63, 0x61, 0x74, 0x65, 0x64) + o = msgp.AppendBool(o, z.IsTruncated) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *FilesInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Files": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Files") + return + } + if cap(z.Files) >= int(zb0002) { + z.Files = (z.Files)[:zb0002] + } else { + z.Files = make([]FileInfo, zb0002) + } + for za0001 := range z.Files { + bts, err = z.Files[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Files", za0001) + return + } + } + case "IsTruncated": + z.IsTruncated, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "IsTruncated") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *FilesInfo) Msgsize() (s int) { + s = 1 + 6 + msgp.ArrayHeaderSize + for za0001 := range z.Files { + s += z.Files[za0001].Msgsize() + } + s += 12 + msgp.BoolSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ListDirResult) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "e": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Entries") + return + } + if cap(z.Entries) >= int(zb0002) { + z.Entries = (z.Entries)[:zb0002] + } else { + z.Entries = make([]string, zb0002) + } + for za0001 := range z.Entries { + z.Entries[za0001], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Entries", za0001) + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *ListDirResult) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "e" + err = en.Append(0x81, 0xa1, 0x65) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Entries))) + if err != nil { + err = msgp.WrapError(err, "Entries") + return + } + for za0001 := range z.Entries { + err = en.WriteString(z.Entries[za0001]) + if err != nil { + err = msgp.WrapError(err, "Entries", za0001) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ListDirResult) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "e" + o = append(o, 0x81, 0xa1, 0x65) + o = msgp.AppendArrayHeader(o, uint32(len(z.Entries))) + for za0001 := range z.Entries { + o = msgp.AppendString(o, z.Entries[za0001]) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ListDirResult) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "e": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Entries") + return + } + if cap(z.Entries) >= int(zb0002) { + z.Entries = (z.Entries)[:zb0002] + } else { + z.Entries = make([]string, zb0002) + } + for za0001 := range z.Entries { + z.Entries[za0001], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Entries", za0001) + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ListDirResult) Msgsize() (s int) { + s = 1 + 2 + msgp.ArrayHeaderSize + for za0001 := range z.Entries { + s += msgp.StringPrefixSize + len(z.Entries[za0001]) + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *LocalDiskIDs) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "i": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "IDs") + return + } + if cap(z.IDs) >= int(zb0002) { + z.IDs = (z.IDs)[:zb0002] + } else { + z.IDs = make([]string, zb0002) + } + for za0001 := range z.IDs { + z.IDs[za0001], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "IDs", za0001) + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *LocalDiskIDs) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "i" + err = en.Append(0x81, 0xa1, 0x69) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.IDs))) + if err != nil { + err = msgp.WrapError(err, "IDs") + return + } + for za0001 := range z.IDs { + err = en.WriteString(z.IDs[za0001]) + if err != nil { + err = msgp.WrapError(err, "IDs", za0001) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *LocalDiskIDs) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "i" + o = append(o, 0x81, 0xa1, 0x69) + o = msgp.AppendArrayHeader(o, uint32(len(z.IDs))) + for za0001 := range z.IDs { + o = msgp.AppendString(o, z.IDs[za0001]) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *LocalDiskIDs) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "i": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "IDs") + return + } + if cap(z.IDs) >= int(zb0002) { + z.IDs = (z.IDs)[:zb0002] + } else { + z.IDs = make([]string, zb0002) + } + for za0001 := range z.IDs { + z.IDs[za0001], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "IDs", za0001) + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *LocalDiskIDs) Msgsize() (s int) { + s = 1 + 2 + msgp.ArrayHeaderSize + for za0001 := range z.IDs { + s += msgp.StringPrefixSize + len(z.IDs[za0001]) + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *MetadataHandlerParams) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.DiskID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + case "v": + z.Volume, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + case "ov": + z.OrigVolume, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "OrigVolume") + return + } + case "fp": + z.FilePath, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "FilePath") + return + } + case "uo": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "UpdateOpts") + return + } + for zb0002 > 0 { + zb0002-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "UpdateOpts") + return + } + switch msgp.UnsafeString(field) { + case "np": + z.UpdateOpts.NoPersistence, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "UpdateOpts", "NoPersistence") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "UpdateOpts") + return + } + } + } + case "fi": + err = z.FI.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "FI") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *MetadataHandlerParams) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 6 + // write "id" + err = en.Append(0x86, 0xa2, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteString(z.DiskID) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + // write "v" + err = en.Append(0xa1, 0x76) + if err != nil { + return + } + err = en.WriteString(z.Volume) + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + // write "ov" + err = en.Append(0xa2, 0x6f, 0x76) + if err != nil { + return + } + err = en.WriteString(z.OrigVolume) + if err != nil { + err = msgp.WrapError(err, "OrigVolume") + return + } + // write "fp" + err = en.Append(0xa2, 0x66, 0x70) + if err != nil { + return + } + err = en.WriteString(z.FilePath) + if err != nil { + err = msgp.WrapError(err, "FilePath") + return + } + // write "uo" + err = en.Append(0xa2, 0x75, 0x6f) + if err != nil { + return + } + // map header, size 1 + // write "np" + err = en.Append(0x81, 0xa2, 0x6e, 0x70) + if err != nil { + return + } + err = en.WriteBool(z.UpdateOpts.NoPersistence) + if err != nil { + err = msgp.WrapError(err, "UpdateOpts", "NoPersistence") + return + } + // write "fi" + err = en.Append(0xa2, 0x66, 0x69) + if err != nil { + return + } + err = z.FI.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "FI") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *MetadataHandlerParams) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 6 + // string "id" + o = append(o, 0x86, 0xa2, 0x69, 0x64) + o = msgp.AppendString(o, z.DiskID) + // string "v" + o = append(o, 0xa1, 0x76) + o = msgp.AppendString(o, z.Volume) + // string "ov" + o = append(o, 0xa2, 0x6f, 0x76) + o = msgp.AppendString(o, z.OrigVolume) + // string "fp" + o = append(o, 0xa2, 0x66, 0x70) + o = msgp.AppendString(o, z.FilePath) + // string "uo" + o = append(o, 0xa2, 0x75, 0x6f) + // map header, size 1 + // string "np" + o = append(o, 0x81, 0xa2, 0x6e, 0x70) + o = msgp.AppendBool(o, z.UpdateOpts.NoPersistence) + // string "fi" + o = append(o, 0xa2, 0x66, 0x69) + o, err = z.FI.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "FI") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *MetadataHandlerParams) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.DiskID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + case "v": + z.Volume, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + case "ov": + z.OrigVolume, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "OrigVolume") + return + } + case "fp": + z.FilePath, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "FilePath") + return + } + case "uo": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UpdateOpts") + return + } + for zb0002 > 0 { + zb0002-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "UpdateOpts") + return + } + switch msgp.UnsafeString(field) { + case "np": + z.UpdateOpts.NoPersistence, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UpdateOpts", "NoPersistence") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "UpdateOpts") + return + } + } + } + case "fi": + bts, err = z.FI.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "FI") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *MetadataHandlerParams) Msgsize() (s int) { + s = 1 + 3 + msgp.StringPrefixSize + len(z.DiskID) + 2 + msgp.StringPrefixSize + len(z.Volume) + 3 + msgp.StringPrefixSize + len(z.OrigVolume) + 3 + msgp.StringPrefixSize + len(z.FilePath) + 3 + 1 + 3 + msgp.BoolSize + 3 + z.FI.Msgsize() + return +} + +// DecodeMsg implements msgp.Decodable +func (z *RawFileInfo) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "b": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "Buf") + return + } + z.Buf = nil + } else { + z.Buf, err = dc.ReadBytes(z.Buf) + if err != nil { + err = msgp.WrapError(err, "Buf") + return + } + if z.Buf == nil { + z.Buf = make([]byte, 0) + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *RawFileInfo) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "b" + err = en.Append(0x81, 0xa1, 0x62) + if err != nil { + return + } + if z.Buf == nil { // allownil: if nil + err = en.WriteNil() + if err != nil { + return + } + } else { + err = en.WriteBytes(z.Buf) + if err != nil { + err = msgp.WrapError(err, "Buf") + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *RawFileInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "b" + o = append(o, 0x81, 0xa1, 0x62) + if z.Buf == nil { // allownil: if nil + o = msgp.AppendNil(o) + } else { + o = msgp.AppendBytes(o, z.Buf) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *RawFileInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "b": + if msgp.IsNil(bts) { + bts = bts[1:] + z.Buf = nil + } else { + z.Buf, bts, err = msgp.ReadBytesBytes(bts, z.Buf) + if err != nil { + err = msgp.WrapError(err, "Buf") + return + } + if z.Buf == nil { + z.Buf = make([]byte, 0) + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *RawFileInfo) Msgsize() (s int) { + s = 1 + 2 + msgp.BytesPrefixSize + len(z.Buf) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ReadAllHandlerParams) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.DiskID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + case "v": + z.Volume, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + case "fp": + z.FilePath, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "FilePath") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z ReadAllHandlerParams) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "id" + err = en.Append(0x83, 0xa2, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteString(z.DiskID) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + // write "v" + err = en.Append(0xa1, 0x76) + if err != nil { + return + } + err = en.WriteString(z.Volume) + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + // write "fp" + err = en.Append(0xa2, 0x66, 0x70) + if err != nil { + return + } + err = en.WriteString(z.FilePath) + if err != nil { + err = msgp.WrapError(err, "FilePath") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z ReadAllHandlerParams) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "id" + o = append(o, 0x83, 0xa2, 0x69, 0x64) + o = msgp.AppendString(o, z.DiskID) + // string "v" + o = append(o, 0xa1, 0x76) + o = msgp.AppendString(o, z.Volume) + // string "fp" + o = append(o, 0xa2, 0x66, 0x70) + o = msgp.AppendString(o, z.FilePath) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ReadAllHandlerParams) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.DiskID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + case "v": + z.Volume, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + case "fp": + z.FilePath, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "FilePath") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z ReadAllHandlerParams) Msgsize() (s int) { + s = 1 + 3 + msgp.StringPrefixSize + len(z.DiskID) + 2 + msgp.StringPrefixSize + len(z.Volume) + 3 + msgp.StringPrefixSize + len(z.FilePath) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ReadMultipleReq) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 1 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "bk": + z.Bucket, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "pr": + z.Prefix, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + zb0001Mask |= 0x1 + case "fl": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Files") + return + } + if cap(z.Files) >= int(zb0002) { + z.Files = (z.Files)[:zb0002] + } else { + z.Files = make([]string, zb0002) + } + for za0001 := range z.Files { + z.Files[za0001], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Files", za0001) + return + } + } + case "ms": + z.MaxSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "MaxSize") + return + } + case "mo": + z.MetadataOnly, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "MetadataOnly") + return + } + case "ab": + z.AbortOn404, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "AbortOn404") + return + } + case "mr": + z.MaxResults, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "MaxResults") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if (zb0001Mask & 0x1) == 0 { + z.Prefix = "" + } + + return +} + +// EncodeMsg implements msgp.Encodable +func (z *ReadMultipleReq) EncodeMsg(en *msgp.Writer) (err error) { + // check for omitted fields + zb0001Len := uint32(7) + var zb0001Mask uint8 /* 7 bits */ + _ = zb0001Mask + if z.Prefix == "" { + zb0001Len-- + zb0001Mask |= 0x2 + } + // variable map header, size zb0001Len + err = en.Append(0x80 | uint8(zb0001Len)) + if err != nil { + return + } + + // skip if no fields are to be emitted + if zb0001Len != 0 { + // write "bk" + err = en.Append(0xa2, 0x62, 0x6b) + if err != nil { + return + } + err = en.WriteString(z.Bucket) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + if (zb0001Mask & 0x2) == 0 { // if not omitted + // write "pr" + err = en.Append(0xa2, 0x70, 0x72) + if err != nil { + return + } + err = en.WriteString(z.Prefix) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + } + // write "fl" + err = en.Append(0xa2, 0x66, 0x6c) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Files))) + if err != nil { + err = msgp.WrapError(err, "Files") + return + } + for za0001 := range z.Files { + err = en.WriteString(z.Files[za0001]) + if err != nil { + err = msgp.WrapError(err, "Files", za0001) + return + } + } + // write "ms" + err = en.Append(0xa2, 0x6d, 0x73) + if err != nil { + return + } + err = en.WriteInt64(z.MaxSize) + if err != nil { + err = msgp.WrapError(err, "MaxSize") + return + } + // write "mo" + err = en.Append(0xa2, 0x6d, 0x6f) + if err != nil { + return + } + err = en.WriteBool(z.MetadataOnly) + if err != nil { + err = msgp.WrapError(err, "MetadataOnly") + return + } + // write "ab" + err = en.Append(0xa2, 0x61, 0x62) + if err != nil { + return + } + err = en.WriteBool(z.AbortOn404) + if err != nil { + err = msgp.WrapError(err, "AbortOn404") + return + } + // write "mr" + err = en.Append(0xa2, 0x6d, 0x72) + if err != nil { + return + } + err = en.WriteInt(z.MaxResults) + if err != nil { + err = msgp.WrapError(err, "MaxResults") + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ReadMultipleReq) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // check for omitted fields + zb0001Len := uint32(7) + var zb0001Mask uint8 /* 7 bits */ + _ = zb0001Mask + if z.Prefix == "" { + zb0001Len-- + zb0001Mask |= 0x2 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + + // skip if no fields are to be emitted + if zb0001Len != 0 { + // string "bk" + o = append(o, 0xa2, 0x62, 0x6b) + o = msgp.AppendString(o, z.Bucket) + if (zb0001Mask & 0x2) == 0 { // if not omitted + // string "pr" + o = append(o, 0xa2, 0x70, 0x72) + o = msgp.AppendString(o, z.Prefix) + } + // string "fl" + o = append(o, 0xa2, 0x66, 0x6c) + o = msgp.AppendArrayHeader(o, uint32(len(z.Files))) + for za0001 := range z.Files { + o = msgp.AppendString(o, z.Files[za0001]) + } + // string "ms" + o = append(o, 0xa2, 0x6d, 0x73) + o = msgp.AppendInt64(o, z.MaxSize) + // string "mo" + o = append(o, 0xa2, 0x6d, 0x6f) + o = msgp.AppendBool(o, z.MetadataOnly) + // string "ab" + o = append(o, 0xa2, 0x61, 0x62) + o = msgp.AppendBool(o, z.AbortOn404) + // string "mr" + o = append(o, 0xa2, 0x6d, 0x72) + o = msgp.AppendInt(o, z.MaxResults) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ReadMultipleReq) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 1 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "bk": + z.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "pr": + z.Prefix, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + zb0001Mask |= 0x1 + case "fl": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Files") + return + } + if cap(z.Files) >= int(zb0002) { + z.Files = (z.Files)[:zb0002] + } else { + z.Files = make([]string, zb0002) + } + for za0001 := range z.Files { + z.Files[za0001], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Files", za0001) + return + } + } + case "ms": + z.MaxSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "MaxSize") + return + } + case "mo": + z.MetadataOnly, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "MetadataOnly") + return + } + case "ab": + z.AbortOn404, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "AbortOn404") + return + } + case "mr": + z.MaxResults, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "MaxResults") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if (zb0001Mask & 0x1) == 0 { + z.Prefix = "" + } + + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ReadMultipleReq) Msgsize() (s int) { + s = 1 + 3 + msgp.StringPrefixSize + len(z.Bucket) + 3 + msgp.StringPrefixSize + len(z.Prefix) + 3 + msgp.ArrayHeaderSize + for za0001 := range z.Files { + s += msgp.StringPrefixSize + len(z.Files[za0001]) + } + s += 3 + msgp.Int64Size + 3 + msgp.BoolSize + 3 + msgp.BoolSize + 3 + msgp.IntSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ReadMultipleResp) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 2 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "bk": + z.Bucket, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "pr": + z.Prefix, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + zb0001Mask |= 0x1 + case "fl": + z.File, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "File") + return + } + case "ex": + z.Exists, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Exists") + return + } + case "er": + z.Error, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Error") + return + } + zb0001Mask |= 0x2 + case "d": + z.Data, err = dc.ReadBytes(z.Data) + if err != nil { + err = msgp.WrapError(err, "Data") + return + } + case "m": + z.Modtime, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "Modtime") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if zb0001Mask != 0x3 { + if (zb0001Mask & 0x1) == 0 { + z.Prefix = "" + } + if (zb0001Mask & 0x2) == 0 { + z.Error = "" + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *ReadMultipleResp) EncodeMsg(en *msgp.Writer) (err error) { + // check for omitted fields + zb0001Len := uint32(7) + var zb0001Mask uint8 /* 7 bits */ + _ = zb0001Mask + if z.Prefix == "" { + zb0001Len-- + zb0001Mask |= 0x2 + } + if z.Error == "" { + zb0001Len-- + zb0001Mask |= 0x10 + } + // variable map header, size zb0001Len + err = en.Append(0x80 | uint8(zb0001Len)) + if err != nil { + return + } + + // skip if no fields are to be emitted + if zb0001Len != 0 { + // write "bk" + err = en.Append(0xa2, 0x62, 0x6b) + if err != nil { + return + } + err = en.WriteString(z.Bucket) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + if (zb0001Mask & 0x2) == 0 { // if not omitted + // write "pr" + err = en.Append(0xa2, 0x70, 0x72) + if err != nil { + return + } + err = en.WriteString(z.Prefix) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + } + // write "fl" + err = en.Append(0xa2, 0x66, 0x6c) + if err != nil { + return + } + err = en.WriteString(z.File) + if err != nil { + err = msgp.WrapError(err, "File") + return + } + // write "ex" + err = en.Append(0xa2, 0x65, 0x78) + if err != nil { + return + } + err = en.WriteBool(z.Exists) + if err != nil { + err = msgp.WrapError(err, "Exists") + return + } + if (zb0001Mask & 0x10) == 0 { // if not omitted + // write "er" + err = en.Append(0xa2, 0x65, 0x72) + if err != nil { + return + } + err = en.WriteString(z.Error) + if err != nil { + err = msgp.WrapError(err, "Error") + return + } + } + // write "d" + err = en.Append(0xa1, 0x64) + if err != nil { + return + } + err = en.WriteBytes(z.Data) + if err != nil { + err = msgp.WrapError(err, "Data") + return + } + // write "m" + err = en.Append(0xa1, 0x6d) + if err != nil { + return + } + err = en.WriteTime(z.Modtime) + if err != nil { + err = msgp.WrapError(err, "Modtime") + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ReadMultipleResp) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // check for omitted fields + zb0001Len := uint32(7) + var zb0001Mask uint8 /* 7 bits */ + _ = zb0001Mask + if z.Prefix == "" { + zb0001Len-- + zb0001Mask |= 0x2 + } + if z.Error == "" { + zb0001Len-- + zb0001Mask |= 0x10 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + + // skip if no fields are to be emitted + if zb0001Len != 0 { + // string "bk" + o = append(o, 0xa2, 0x62, 0x6b) + o = msgp.AppendString(o, z.Bucket) + if (zb0001Mask & 0x2) == 0 { // if not omitted + // string "pr" + o = append(o, 0xa2, 0x70, 0x72) + o = msgp.AppendString(o, z.Prefix) + } + // string "fl" + o = append(o, 0xa2, 0x66, 0x6c) + o = msgp.AppendString(o, z.File) + // string "ex" + o = append(o, 0xa2, 0x65, 0x78) + o = msgp.AppendBool(o, z.Exists) + if (zb0001Mask & 0x10) == 0 { // if not omitted + // string "er" + o = append(o, 0xa2, 0x65, 0x72) + o = msgp.AppendString(o, z.Error) + } + // string "d" + o = append(o, 0xa1, 0x64) + o = msgp.AppendBytes(o, z.Data) + // string "m" + o = append(o, 0xa1, 0x6d) + o = msgp.AppendTime(o, z.Modtime) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ReadMultipleResp) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 2 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "bk": + z.Bucket, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bucket") + return + } + case "pr": + z.Prefix, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Prefix") + return + } + zb0001Mask |= 0x1 + case "fl": + z.File, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "File") + return + } + case "ex": + z.Exists, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Exists") + return + } + case "er": + z.Error, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Error") + return + } + zb0001Mask |= 0x2 + case "d": + z.Data, bts, err = msgp.ReadBytesBytes(bts, z.Data) + if err != nil { + err = msgp.WrapError(err, "Data") + return + } + case "m": + z.Modtime, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Modtime") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if zb0001Mask != 0x3 { + if (zb0001Mask & 0x1) == 0 { + z.Prefix = "" + } + if (zb0001Mask & 0x2) == 0 { + z.Error = "" + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ReadMultipleResp) Msgsize() (s int) { + s = 1 + 3 + msgp.StringPrefixSize + len(z.Bucket) + 3 + msgp.StringPrefixSize + len(z.Prefix) + 3 + msgp.StringPrefixSize + len(z.File) + 3 + msgp.BoolSize + 3 + msgp.StringPrefixSize + len(z.Error) + 2 + msgp.BytesPrefixSize + len(z.Data) + 2 + msgp.TimeSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ReadPartsReq) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "p": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Paths") + return + } + if cap(z.Paths) >= int(zb0002) { + z.Paths = (z.Paths)[:zb0002] + } else { + z.Paths = make([]string, zb0002) + } + for za0001 := range z.Paths { + z.Paths[za0001], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Paths", za0001) + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *ReadPartsReq) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "p" + err = en.Append(0x81, 0xa1, 0x70) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Paths))) + if err != nil { + err = msgp.WrapError(err, "Paths") + return + } + for za0001 := range z.Paths { + err = en.WriteString(z.Paths[za0001]) + if err != nil { + err = msgp.WrapError(err, "Paths", za0001) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ReadPartsReq) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "p" + o = append(o, 0x81, 0xa1, 0x70) + o = msgp.AppendArrayHeader(o, uint32(len(z.Paths))) + for za0001 := range z.Paths { + o = msgp.AppendString(o, z.Paths[za0001]) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ReadPartsReq) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "p": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Paths") + return + } + if cap(z.Paths) >= int(zb0002) { + z.Paths = (z.Paths)[:zb0002] + } else { + z.Paths = make([]string, zb0002) + } + for za0001 := range z.Paths { + z.Paths[za0001], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Paths", za0001) + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ReadPartsReq) Msgsize() (s int) { + s = 1 + 2 + msgp.ArrayHeaderSize + for za0001 := range z.Paths { + s += msgp.StringPrefixSize + len(z.Paths[za0001]) + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ReadPartsResp) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "is": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Infos") + return + } + if cap(z.Infos) >= int(zb0002) { + z.Infos = (z.Infos)[:zb0002] + } else { + z.Infos = make([]*ObjectPartInfo, zb0002) + } + for za0001 := range z.Infos { + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "Infos", za0001) + return + } + z.Infos[za0001] = nil + } else { + if z.Infos[za0001] == nil { + z.Infos[za0001] = new(ObjectPartInfo) + } + err = z.Infos[za0001].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Infos", za0001) + return + } + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *ReadPartsResp) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "is" + err = en.Append(0x81, 0xa2, 0x69, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Infos))) + if err != nil { + err = msgp.WrapError(err, "Infos") + return + } + for za0001 := range z.Infos { + if z.Infos[za0001] == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.Infos[za0001].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Infos", za0001) + return + } + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ReadPartsResp) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "is" + o = append(o, 0x81, 0xa2, 0x69, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Infos))) + for za0001 := range z.Infos { + if z.Infos[za0001] == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.Infos[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Infos", za0001) + return + } + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ReadPartsResp) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "is": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Infos") + return + } + if cap(z.Infos) >= int(zb0002) { + z.Infos = (z.Infos)[:zb0002] + } else { + z.Infos = make([]*ObjectPartInfo, zb0002) + } + for za0001 := range z.Infos { + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.Infos[za0001] = nil + } else { + if z.Infos[za0001] == nil { + z.Infos[za0001] = new(ObjectPartInfo) + } + bts, err = z.Infos[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Infos", za0001) + return + } + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ReadPartsResp) Msgsize() (s int) { + s = 1 + 3 + msgp.ArrayHeaderSize + for za0001 := range z.Infos { + if z.Infos[za0001] == nil { + s += msgp.NilSize + } else { + s += z.Infos[za0001].Msgsize() + } + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *RenameDataHandlerParams) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.DiskID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + case "sv": + z.SrcVolume, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "SrcVolume") + return + } + case "sp": + z.SrcPath, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "SrcPath") + return + } + case "dv": + z.DstVolume, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DstVolume") + return + } + case "dp": + z.DstPath, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DstPath") + return + } + case "fi": + err = z.FI.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "FI") + return + } + case "ro": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Opts") + return + } + for zb0002 > 0 { + zb0002-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "Opts") + return + } + switch msgp.UnsafeString(field) { + case "BaseOptions": + var zb0003 uint32 + zb0003, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Opts", "BaseOptions") + return + } + for zb0003 > 0 { + zb0003-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "Opts", "BaseOptions") + return + } + switch msgp.UnsafeString(field) { + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "Opts", "BaseOptions") + return + } + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "Opts") + return + } + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *RenameDataHandlerParams) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 7 + // write "id" + err = en.Append(0x87, 0xa2, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteString(z.DiskID) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + // write "sv" + err = en.Append(0xa2, 0x73, 0x76) + if err != nil { + return + } + err = en.WriteString(z.SrcVolume) + if err != nil { + err = msgp.WrapError(err, "SrcVolume") + return + } + // write "sp" + err = en.Append(0xa2, 0x73, 0x70) + if err != nil { + return + } + err = en.WriteString(z.SrcPath) + if err != nil { + err = msgp.WrapError(err, "SrcPath") + return + } + // write "dv" + err = en.Append(0xa2, 0x64, 0x76) + if err != nil { + return + } + err = en.WriteString(z.DstVolume) + if err != nil { + err = msgp.WrapError(err, "DstVolume") + return + } + // write "dp" + err = en.Append(0xa2, 0x64, 0x70) + if err != nil { + return + } + err = en.WriteString(z.DstPath) + if err != nil { + err = msgp.WrapError(err, "DstPath") + return + } + // write "fi" + err = en.Append(0xa2, 0x66, 0x69) + if err != nil { + return + } + err = z.FI.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "FI") + return + } + // write "ro" + err = en.Append(0xa2, 0x72, 0x6f) + if err != nil { + return + } + // map header, size 1 + // write "BaseOptions" + err = en.Append(0x81, 0xab, 0x42, 0x61, 0x73, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73) + if err != nil { + return + } + // map header, size 0 + _ = z.Opts.BaseOptions + err = en.Append(0x80) + if err != nil { + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *RenameDataHandlerParams) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 7 + // string "id" + o = append(o, 0x87, 0xa2, 0x69, 0x64) + o = msgp.AppendString(o, z.DiskID) + // string "sv" + o = append(o, 0xa2, 0x73, 0x76) + o = msgp.AppendString(o, z.SrcVolume) + // string "sp" + o = append(o, 0xa2, 0x73, 0x70) + o = msgp.AppendString(o, z.SrcPath) + // string "dv" + o = append(o, 0xa2, 0x64, 0x76) + o = msgp.AppendString(o, z.DstVolume) + // string "dp" + o = append(o, 0xa2, 0x64, 0x70) + o = msgp.AppendString(o, z.DstPath) + // string "fi" + o = append(o, 0xa2, 0x66, 0x69) + o, err = z.FI.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "FI") + return + } + // string "ro" + o = append(o, 0xa2, 0x72, 0x6f) + // map header, size 1 + // string "BaseOptions" + o = append(o, 0x81, 0xab, 0x42, 0x61, 0x73, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73) + // map header, size 0 + _ = z.Opts.BaseOptions + o = append(o, 0x80) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *RenameDataHandlerParams) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.DiskID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + case "sv": + z.SrcVolume, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SrcVolume") + return + } + case "sp": + z.SrcPath, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SrcPath") + return + } + case "dv": + z.DstVolume, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DstVolume") + return + } + case "dp": + z.DstPath, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DstPath") + return + } + case "fi": + bts, err = z.FI.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "FI") + return + } + case "ro": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Opts") + return + } + for zb0002 > 0 { + zb0002-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "Opts") + return + } + switch msgp.UnsafeString(field) { + case "BaseOptions": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Opts", "BaseOptions") + return + } + for zb0003 > 0 { + zb0003-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "Opts", "BaseOptions") + return + } + switch msgp.UnsafeString(field) { + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "Opts", "BaseOptions") + return + } + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "Opts") + return + } + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *RenameDataHandlerParams) Msgsize() (s int) { + s = 1 + 3 + msgp.StringPrefixSize + len(z.DiskID) + 3 + msgp.StringPrefixSize + len(z.SrcVolume) + 3 + msgp.StringPrefixSize + len(z.SrcPath) + 3 + msgp.StringPrefixSize + len(z.DstVolume) + 3 + msgp.StringPrefixSize + len(z.DstPath) + 3 + z.FI.Msgsize() + 3 + 1 + 12 + 1 + return +} + +// DecodeMsg implements msgp.Decodable +func (z *RenameDataInlineHandlerParams) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "p": + err = z.RenameDataHandlerParams.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "RenameDataHandlerParams") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *RenameDataInlineHandlerParams) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "p" + err = en.Append(0x81, 0xa1, 0x70) + if err != nil { + return + } + err = z.RenameDataHandlerParams.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "RenameDataHandlerParams") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *RenameDataInlineHandlerParams) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "p" + o = append(o, 0x81, 0xa1, 0x70) + o, err = z.RenameDataHandlerParams.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "RenameDataHandlerParams") + return + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *RenameDataInlineHandlerParams) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "p": + bts, err = z.RenameDataHandlerParams.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "RenameDataHandlerParams") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *RenameDataInlineHandlerParams) Msgsize() (s int) { + s = 1 + 2 + z.RenameDataHandlerParams.Msgsize() + return +} + +// DecodeMsg implements msgp.Decodable +func (z *RenameDataResp) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "s": + z.Sign, err = dc.ReadBytes(z.Sign) + if err != nil { + err = msgp.WrapError(err, "Sign") + return + } + case "od": + z.OldDataDir, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "OldDataDir") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *RenameDataResp) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "s" + err = en.Append(0x82, 0xa1, 0x73) + if err != nil { + return + } + err = en.WriteBytes(z.Sign) + if err != nil { + err = msgp.WrapError(err, "Sign") + return + } + // write "od" + err = en.Append(0xa2, 0x6f, 0x64) + if err != nil { + return + } + err = en.WriteString(z.OldDataDir) + if err != nil { + err = msgp.WrapError(err, "OldDataDir") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *RenameDataResp) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "s" + o = append(o, 0x82, 0xa1, 0x73) + o = msgp.AppendBytes(o, z.Sign) + // string "od" + o = append(o, 0xa2, 0x6f, 0x64) + o = msgp.AppendString(o, z.OldDataDir) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *RenameDataResp) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "s": + z.Sign, bts, err = msgp.ReadBytesBytes(bts, z.Sign) + if err != nil { + err = msgp.WrapError(err, "Sign") + return + } + case "od": + z.OldDataDir, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "OldDataDir") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *RenameDataResp) Msgsize() (s int) { + s = 1 + 2 + msgp.BytesPrefixSize + len(z.Sign) + 3 + msgp.StringPrefixSize + len(z.OldDataDir) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *RenameFileHandlerParams) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.DiskID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + case "sv": + z.SrcVolume, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "SrcVolume") + return + } + case "sp": + z.SrcFilePath, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "SrcFilePath") + return + } + case "dv": + z.DstVolume, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DstVolume") + return + } + case "dp": + z.DstFilePath, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DstFilePath") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *RenameFileHandlerParams) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 5 + // write "id" + err = en.Append(0x85, 0xa2, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteString(z.DiskID) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + // write "sv" + err = en.Append(0xa2, 0x73, 0x76) + if err != nil { + return + } + err = en.WriteString(z.SrcVolume) + if err != nil { + err = msgp.WrapError(err, "SrcVolume") + return + } + // write "sp" + err = en.Append(0xa2, 0x73, 0x70) + if err != nil { + return + } + err = en.WriteString(z.SrcFilePath) + if err != nil { + err = msgp.WrapError(err, "SrcFilePath") + return + } + // write "dv" + err = en.Append(0xa2, 0x64, 0x76) + if err != nil { + return + } + err = en.WriteString(z.DstVolume) + if err != nil { + err = msgp.WrapError(err, "DstVolume") + return + } + // write "dp" + err = en.Append(0xa2, 0x64, 0x70) + if err != nil { + return + } + err = en.WriteString(z.DstFilePath) + if err != nil { + err = msgp.WrapError(err, "DstFilePath") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *RenameFileHandlerParams) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 5 + // string "id" + o = append(o, 0x85, 0xa2, 0x69, 0x64) + o = msgp.AppendString(o, z.DiskID) + // string "sv" + o = append(o, 0xa2, 0x73, 0x76) + o = msgp.AppendString(o, z.SrcVolume) + // string "sp" + o = append(o, 0xa2, 0x73, 0x70) + o = msgp.AppendString(o, z.SrcFilePath) + // string "dv" + o = append(o, 0xa2, 0x64, 0x76) + o = msgp.AppendString(o, z.DstVolume) + // string "dp" + o = append(o, 0xa2, 0x64, 0x70) + o = msgp.AppendString(o, z.DstFilePath) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *RenameFileHandlerParams) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.DiskID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + case "sv": + z.SrcVolume, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SrcVolume") + return + } + case "sp": + z.SrcFilePath, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SrcFilePath") + return + } + case "dv": + z.DstVolume, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DstVolume") + return + } + case "dp": + z.DstFilePath, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DstFilePath") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *RenameFileHandlerParams) Msgsize() (s int) { + s = 1 + 3 + msgp.StringPrefixSize + len(z.DiskID) + 3 + msgp.StringPrefixSize + len(z.SrcVolume) + 3 + msgp.StringPrefixSize + len(z.SrcFilePath) + 3 + msgp.StringPrefixSize + len(z.DstVolume) + 3 + msgp.StringPrefixSize + len(z.DstFilePath) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *RenameOptions) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "BaseOptions": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "BaseOptions") + return + } + for zb0002 > 0 { + zb0002-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "BaseOptions") + return + } + switch msgp.UnsafeString(field) { + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "BaseOptions") + return + } + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *RenameOptions) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "BaseOptions" + err = en.Append(0x81, 0xab, 0x42, 0x61, 0x73, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73) + if err != nil { + return + } + // map header, size 0 + _ = z.BaseOptions + err = en.Append(0x80) + if err != nil { + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *RenameOptions) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "BaseOptions" + o = append(o, 0x81, 0xab, 0x42, 0x61, 0x73, 0x65, 0x4f, 0x70, 0x74, 0x69, 0x6f, 0x6e, 0x73) + // map header, size 0 + _ = z.BaseOptions + o = append(o, 0x80) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *RenameOptions) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "BaseOptions": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "BaseOptions") + return + } + for zb0002 > 0 { + zb0002-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "BaseOptions") + return + } + switch msgp.UnsafeString(field) { + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "BaseOptions") + return + } + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *RenameOptions) Msgsize() (s int) { + s = 1 + 12 + 1 + return +} + +// DecodeMsg implements msgp.Decodable +func (z *RenamePartHandlerParams) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.DiskID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + case "sv": + z.SrcVolume, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "SrcVolume") + return + } + case "sp": + z.SrcFilePath, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "SrcFilePath") + return + } + case "dv": + z.DstVolume, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DstVolume") + return + } + case "dp": + z.DstFilePath, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DstFilePath") + return + } + case "m": + z.Meta, err = dc.ReadBytes(z.Meta) + if err != nil { + err = msgp.WrapError(err, "Meta") + return + } + case "kp": + z.SkipParent, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "SkipParent") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *RenamePartHandlerParams) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 7 + // write "id" + err = en.Append(0x87, 0xa2, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteString(z.DiskID) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + // write "sv" + err = en.Append(0xa2, 0x73, 0x76) + if err != nil { + return + } + err = en.WriteString(z.SrcVolume) + if err != nil { + err = msgp.WrapError(err, "SrcVolume") + return + } + // write "sp" + err = en.Append(0xa2, 0x73, 0x70) + if err != nil { + return + } + err = en.WriteString(z.SrcFilePath) + if err != nil { + err = msgp.WrapError(err, "SrcFilePath") + return + } + // write "dv" + err = en.Append(0xa2, 0x64, 0x76) + if err != nil { + return + } + err = en.WriteString(z.DstVolume) + if err != nil { + err = msgp.WrapError(err, "DstVolume") + return + } + // write "dp" + err = en.Append(0xa2, 0x64, 0x70) + if err != nil { + return + } + err = en.WriteString(z.DstFilePath) + if err != nil { + err = msgp.WrapError(err, "DstFilePath") + return + } + // write "m" + err = en.Append(0xa1, 0x6d) + if err != nil { + return + } + err = en.WriteBytes(z.Meta) + if err != nil { + err = msgp.WrapError(err, "Meta") + return + } + // write "kp" + err = en.Append(0xa2, 0x6b, 0x70) + if err != nil { + return + } + err = en.WriteString(z.SkipParent) + if err != nil { + err = msgp.WrapError(err, "SkipParent") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *RenamePartHandlerParams) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 7 + // string "id" + o = append(o, 0x87, 0xa2, 0x69, 0x64) + o = msgp.AppendString(o, z.DiskID) + // string "sv" + o = append(o, 0xa2, 0x73, 0x76) + o = msgp.AppendString(o, z.SrcVolume) + // string "sp" + o = append(o, 0xa2, 0x73, 0x70) + o = msgp.AppendString(o, z.SrcFilePath) + // string "dv" + o = append(o, 0xa2, 0x64, 0x76) + o = msgp.AppendString(o, z.DstVolume) + // string "dp" + o = append(o, 0xa2, 0x64, 0x70) + o = msgp.AppendString(o, z.DstFilePath) + // string "m" + o = append(o, 0xa1, 0x6d) + o = msgp.AppendBytes(o, z.Meta) + // string "kp" + o = append(o, 0xa2, 0x6b, 0x70) + o = msgp.AppendString(o, z.SkipParent) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *RenamePartHandlerParams) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.DiskID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + case "sv": + z.SrcVolume, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SrcVolume") + return + } + case "sp": + z.SrcFilePath, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SrcFilePath") + return + } + case "dv": + z.DstVolume, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DstVolume") + return + } + case "dp": + z.DstFilePath, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DstFilePath") + return + } + case "m": + z.Meta, bts, err = msgp.ReadBytesBytes(bts, z.Meta) + if err != nil { + err = msgp.WrapError(err, "Meta") + return + } + case "kp": + z.SkipParent, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "SkipParent") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *RenamePartHandlerParams) Msgsize() (s int) { + s = 1 + 3 + msgp.StringPrefixSize + len(z.DiskID) + 3 + msgp.StringPrefixSize + len(z.SrcVolume) + 3 + msgp.StringPrefixSize + len(z.SrcFilePath) + 3 + msgp.StringPrefixSize + len(z.DstVolume) + 3 + msgp.StringPrefixSize + len(z.DstFilePath) + 2 + msgp.BytesPrefixSize + len(z.Meta) + 3 + msgp.StringPrefixSize + len(z.SkipParent) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *UpdateMetadataOpts) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "np": + z.NoPersistence, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "NoPersistence") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z UpdateMetadataOpts) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "np" + err = en.Append(0x81, 0xa2, 0x6e, 0x70) + if err != nil { + return + } + err = en.WriteBool(z.NoPersistence) + if err != nil { + err = msgp.WrapError(err, "NoPersistence") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z UpdateMetadataOpts) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "np" + o = append(o, 0x81, 0xa2, 0x6e, 0x70) + o = msgp.AppendBool(o, z.NoPersistence) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *UpdateMetadataOpts) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "np": + z.NoPersistence, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "NoPersistence") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z UpdateMetadataOpts) Msgsize() (s int) { + s = 1 + 3 + msgp.BoolSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *VolInfo) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0001 uint32 + zb0001, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 3 { + err = msgp.ArrayError{Wanted: 3, Got: zb0001} + return + } + z.Name, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + z.Created, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "Created") + return + } + z.Deleted, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "Deleted") + return + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z VolInfo) EncodeMsg(en *msgp.Writer) (err error) { + // array header, size 3 + err = en.Append(0x93) + if err != nil { + return + } + err = en.WriteString(z.Name) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + err = en.WriteTime(z.Created) + if err != nil { + err = msgp.WrapError(err, "Created") + return + } + err = en.WriteTime(z.Deleted) + if err != nil { + err = msgp.WrapError(err, "Deleted") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z VolInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // array header, size 3 + o = append(o, 0x93) + o = msgp.AppendString(o, z.Name) + o = msgp.AppendTime(o, z.Created) + o = msgp.AppendTime(o, z.Deleted) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *VolInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0001 uint32 + zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 3 { + err = msgp.ArrayError{Wanted: 3, Got: zb0001} + return + } + z.Name, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + z.Created, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Created") + return + } + z.Deleted, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Deleted") + return + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z VolInfo) Msgsize() (s int) { + s = 1 + msgp.StringPrefixSize + len(z.Name) + msgp.TimeSize + msgp.TimeSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *VolsInfo) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if cap((*z)) >= int(zb0002) { + (*z) = (*z)[:zb0002] + } else { + (*z) = make(VolsInfo, zb0002) + } + for zb0001 := range *z { + var zb0003 uint32 + zb0003, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, zb0001) + return + } + if zb0003 != 3 { + err = msgp.ArrayError{Wanted: 3, Got: zb0003} + return + } + (*z)[zb0001].Name, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, zb0001, "Name") + return + } + (*z)[zb0001].Created, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, zb0001, "Created") + return + } + (*z)[zb0001].Deleted, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, zb0001, "Deleted") + return + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z VolsInfo) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteArrayHeader(uint32(len(z))) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0004 := range z { + // array header, size 3 + err = en.Append(0x93) + if err != nil { + return + } + err = en.WriteString(z[zb0004].Name) + if err != nil { + err = msgp.WrapError(err, zb0004, "Name") + return + } + err = en.WriteTime(z[zb0004].Created) + if err != nil { + err = msgp.WrapError(err, zb0004, "Created") + return + } + err = en.WriteTime(z[zb0004].Deleted) + if err != nil { + err = msgp.WrapError(err, zb0004, "Deleted") + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z VolsInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendArrayHeader(o, uint32(len(z))) + for zb0004 := range z { + // array header, size 3 + o = append(o, 0x93) + o = msgp.AppendString(o, z[zb0004].Name) + o = msgp.AppendTime(o, z[zb0004].Created) + o = msgp.AppendTime(o, z[zb0004].Deleted) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *VolsInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if cap((*z)) >= int(zb0002) { + (*z) = (*z)[:zb0002] + } else { + (*z) = make(VolsInfo, zb0002) + } + for zb0001 := range *z { + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, zb0001) + return + } + if zb0003 != 3 { + err = msgp.ArrayError{Wanted: 3, Got: zb0003} + return + } + (*z)[zb0001].Name, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, zb0001, "Name") + return + } + (*z)[zb0001].Created, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, zb0001, "Created") + return + } + (*z)[zb0001].Deleted, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, zb0001, "Deleted") + return + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z VolsInfo) Msgsize() (s int) { + s = msgp.ArrayHeaderSize + for zb0004 := range z { + s += 1 + msgp.StringPrefixSize + len(z[zb0004].Name) + msgp.TimeSize + msgp.TimeSize + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *WriteAllHandlerParams) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.DiskID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + case "v": + z.Volume, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + case "fp": + z.FilePath, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "FilePath") + return + } + case "b": + z.Buf, err = dc.ReadBytes(z.Buf) + if err != nil { + err = msgp.WrapError(err, "Buf") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *WriteAllHandlerParams) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 4 + // write "id" + err = en.Append(0x84, 0xa2, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteString(z.DiskID) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + // write "v" + err = en.Append(0xa1, 0x76) + if err != nil { + return + } + err = en.WriteString(z.Volume) + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + // write "fp" + err = en.Append(0xa2, 0x66, 0x70) + if err != nil { + return + } + err = en.WriteString(z.FilePath) + if err != nil { + err = msgp.WrapError(err, "FilePath") + return + } + // write "b" + err = en.Append(0xa1, 0x62) + if err != nil { + return + } + err = en.WriteBytes(z.Buf) + if err != nil { + err = msgp.WrapError(err, "Buf") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *WriteAllHandlerParams) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 4 + // string "id" + o = append(o, 0x84, 0xa2, 0x69, 0x64) + o = msgp.AppendString(o, z.DiskID) + // string "v" + o = append(o, 0xa1, 0x76) + o = msgp.AppendString(o, z.Volume) + // string "fp" + o = append(o, 0xa2, 0x66, 0x70) + o = msgp.AppendString(o, z.FilePath) + // string "b" + o = append(o, 0xa1, 0x62) + o = msgp.AppendBytes(o, z.Buf) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *WriteAllHandlerParams) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.DiskID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + case "v": + z.Volume, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Volume") + return + } + case "fp": + z.FilePath, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "FilePath") + return + } + case "b": + z.Buf, bts, err = msgp.ReadBytesBytes(bts, z.Buf) + if err != nil { + err = msgp.WrapError(err, "Buf") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *WriteAllHandlerParams) Msgsize() (s int) { + s = 1 + 3 + msgp.StringPrefixSize + len(z.DiskID) + 2 + msgp.StringPrefixSize + len(z.Volume) + 3 + msgp.StringPrefixSize + len(z.FilePath) + 2 + msgp.BytesPrefixSize + len(z.Buf) + return +} diff --git a/cmd/storage-datatypes_gen_test.go b/cmd/storage-datatypes_gen_test.go new file mode 100644 index 00000000..78f53f36 --- /dev/null +++ b/cmd/storage-datatypes_gen_test.go @@ -0,0 +1,3739 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalBaseOptions(t *testing.T) { + v := BaseOptions{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBaseOptions(b *testing.B) { + v := BaseOptions{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBaseOptions(b *testing.B) { + v := BaseOptions{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBaseOptions(b *testing.B) { + v := BaseOptions{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBaseOptions(t *testing.T) { + v := BaseOptions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBaseOptions Msgsize() is inaccurate") + } + + vn := BaseOptions{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBaseOptions(b *testing.B) { + v := BaseOptions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBaseOptions(b *testing.B) { + v := BaseOptions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalCheckPartsHandlerParams(t *testing.T) { + v := CheckPartsHandlerParams{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgCheckPartsHandlerParams(b *testing.B) { + v := CheckPartsHandlerParams{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgCheckPartsHandlerParams(b *testing.B) { + v := CheckPartsHandlerParams{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalCheckPartsHandlerParams(b *testing.B) { + v := CheckPartsHandlerParams{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeCheckPartsHandlerParams(t *testing.T) { + v := CheckPartsHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeCheckPartsHandlerParams Msgsize() is inaccurate") + } + + vn := CheckPartsHandlerParams{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeCheckPartsHandlerParams(b *testing.B) { + v := CheckPartsHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeCheckPartsHandlerParams(b *testing.B) { + v := CheckPartsHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalCheckPartsResp(t *testing.T) { + v := CheckPartsResp{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgCheckPartsResp(b *testing.B) { + v := CheckPartsResp{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgCheckPartsResp(b *testing.B) { + v := CheckPartsResp{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalCheckPartsResp(b *testing.B) { + v := CheckPartsResp{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeCheckPartsResp(t *testing.T) { + v := CheckPartsResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeCheckPartsResp Msgsize() is inaccurate") + } + + vn := CheckPartsResp{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeCheckPartsResp(b *testing.B) { + v := CheckPartsResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeCheckPartsResp(b *testing.B) { + v := CheckPartsResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalDeleteBulkReq(t *testing.T) { + v := DeleteBulkReq{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgDeleteBulkReq(b *testing.B) { + v := DeleteBulkReq{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgDeleteBulkReq(b *testing.B) { + v := DeleteBulkReq{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalDeleteBulkReq(b *testing.B) { + v := DeleteBulkReq{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeDeleteBulkReq(t *testing.T) { + v := DeleteBulkReq{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeDeleteBulkReq Msgsize() is inaccurate") + } + + vn := DeleteBulkReq{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeDeleteBulkReq(b *testing.B) { + v := DeleteBulkReq{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeDeleteBulkReq(b *testing.B) { + v := DeleteBulkReq{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalDeleteFileHandlerParams(t *testing.T) { + v := DeleteFileHandlerParams{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgDeleteFileHandlerParams(b *testing.B) { + v := DeleteFileHandlerParams{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgDeleteFileHandlerParams(b *testing.B) { + v := DeleteFileHandlerParams{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalDeleteFileHandlerParams(b *testing.B) { + v := DeleteFileHandlerParams{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeDeleteFileHandlerParams(t *testing.T) { + v := DeleteFileHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeDeleteFileHandlerParams Msgsize() is inaccurate") + } + + vn := DeleteFileHandlerParams{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeDeleteFileHandlerParams(b *testing.B) { + v := DeleteFileHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeDeleteFileHandlerParams(b *testing.B) { + v := DeleteFileHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalDeleteOptions(t *testing.T) { + v := DeleteOptions{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgDeleteOptions(b *testing.B) { + v := DeleteOptions{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgDeleteOptions(b *testing.B) { + v := DeleteOptions{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalDeleteOptions(b *testing.B) { + v := DeleteOptions{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeDeleteOptions(t *testing.T) { + v := DeleteOptions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeDeleteOptions Msgsize() is inaccurate") + } + + vn := DeleteOptions{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeDeleteOptions(b *testing.B) { + v := DeleteOptions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeDeleteOptions(b *testing.B) { + v := DeleteOptions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalDeleteVersionHandlerParams(t *testing.T) { + v := DeleteVersionHandlerParams{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgDeleteVersionHandlerParams(b *testing.B) { + v := DeleteVersionHandlerParams{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgDeleteVersionHandlerParams(b *testing.B) { + v := DeleteVersionHandlerParams{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalDeleteVersionHandlerParams(b *testing.B) { + v := DeleteVersionHandlerParams{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeDeleteVersionHandlerParams(t *testing.T) { + v := DeleteVersionHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeDeleteVersionHandlerParams Msgsize() is inaccurate") + } + + vn := DeleteVersionHandlerParams{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeDeleteVersionHandlerParams(b *testing.B) { + v := DeleteVersionHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeDeleteVersionHandlerParams(b *testing.B) { + v := DeleteVersionHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalDeleteVersionsErrsResp(t *testing.T) { + v := DeleteVersionsErrsResp{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgDeleteVersionsErrsResp(b *testing.B) { + v := DeleteVersionsErrsResp{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgDeleteVersionsErrsResp(b *testing.B) { + v := DeleteVersionsErrsResp{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalDeleteVersionsErrsResp(b *testing.B) { + v := DeleteVersionsErrsResp{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeDeleteVersionsErrsResp(t *testing.T) { + v := DeleteVersionsErrsResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeDeleteVersionsErrsResp Msgsize() is inaccurate") + } + + vn := DeleteVersionsErrsResp{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeDeleteVersionsErrsResp(b *testing.B) { + v := DeleteVersionsErrsResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeDeleteVersionsErrsResp(b *testing.B) { + v := DeleteVersionsErrsResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalDiskInfo(t *testing.T) { + v := DiskInfo{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgDiskInfo(b *testing.B) { + v := DiskInfo{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgDiskInfo(b *testing.B) { + v := DiskInfo{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalDiskInfo(b *testing.B) { + v := DiskInfo{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeDiskInfo(t *testing.T) { + v := DiskInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeDiskInfo Msgsize() is inaccurate") + } + + vn := DiskInfo{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeDiskInfo(b *testing.B) { + v := DiskInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeDiskInfo(b *testing.B) { + v := DiskInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalDiskInfoOptions(t *testing.T) { + v := DiskInfoOptions{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgDiskInfoOptions(b *testing.B) { + v := DiskInfoOptions{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgDiskInfoOptions(b *testing.B) { + v := DiskInfoOptions{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalDiskInfoOptions(b *testing.B) { + v := DiskInfoOptions{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeDiskInfoOptions(t *testing.T) { + v := DiskInfoOptions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeDiskInfoOptions Msgsize() is inaccurate") + } + + vn := DiskInfoOptions{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeDiskInfoOptions(b *testing.B) { + v := DiskInfoOptions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeDiskInfoOptions(b *testing.B) { + v := DiskInfoOptions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalDiskMetrics(t *testing.T) { + v := DiskMetrics{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgDiskMetrics(b *testing.B) { + v := DiskMetrics{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgDiskMetrics(b *testing.B) { + v := DiskMetrics{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalDiskMetrics(b *testing.B) { + v := DiskMetrics{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeDiskMetrics(t *testing.T) { + v := DiskMetrics{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeDiskMetrics Msgsize() is inaccurate") + } + + vn := DiskMetrics{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeDiskMetrics(b *testing.B) { + v := DiskMetrics{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeDiskMetrics(b *testing.B) { + v := DiskMetrics{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalFileInfo(t *testing.T) { + v := FileInfo{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgFileInfo(b *testing.B) { + v := FileInfo{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgFileInfo(b *testing.B) { + v := FileInfo{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalFileInfo(b *testing.B) { + v := FileInfo{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeFileInfo(t *testing.T) { + v := FileInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeFileInfo Msgsize() is inaccurate") + } + + vn := FileInfo{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeFileInfo(b *testing.B) { + v := FileInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeFileInfo(b *testing.B) { + v := FileInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalFileInfoVersions(t *testing.T) { + v := FileInfoVersions{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgFileInfoVersions(b *testing.B) { + v := FileInfoVersions{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgFileInfoVersions(b *testing.B) { + v := FileInfoVersions{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalFileInfoVersions(b *testing.B) { + v := FileInfoVersions{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeFileInfoVersions(t *testing.T) { + v := FileInfoVersions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeFileInfoVersions Msgsize() is inaccurate") + } + + vn := FileInfoVersions{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeFileInfoVersions(b *testing.B) { + v := FileInfoVersions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeFileInfoVersions(b *testing.B) { + v := FileInfoVersions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalFilesInfo(t *testing.T) { + v := FilesInfo{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgFilesInfo(b *testing.B) { + v := FilesInfo{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgFilesInfo(b *testing.B) { + v := FilesInfo{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalFilesInfo(b *testing.B) { + v := FilesInfo{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeFilesInfo(t *testing.T) { + v := FilesInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeFilesInfo Msgsize() is inaccurate") + } + + vn := FilesInfo{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeFilesInfo(b *testing.B) { + v := FilesInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeFilesInfo(b *testing.B) { + v := FilesInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalListDirResult(t *testing.T) { + v := ListDirResult{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgListDirResult(b *testing.B) { + v := ListDirResult{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgListDirResult(b *testing.B) { + v := ListDirResult{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalListDirResult(b *testing.B) { + v := ListDirResult{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeListDirResult(t *testing.T) { + v := ListDirResult{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeListDirResult Msgsize() is inaccurate") + } + + vn := ListDirResult{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeListDirResult(b *testing.B) { + v := ListDirResult{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeListDirResult(b *testing.B) { + v := ListDirResult{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalLocalDiskIDs(t *testing.T) { + v := LocalDiskIDs{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgLocalDiskIDs(b *testing.B) { + v := LocalDiskIDs{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgLocalDiskIDs(b *testing.B) { + v := LocalDiskIDs{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalLocalDiskIDs(b *testing.B) { + v := LocalDiskIDs{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeLocalDiskIDs(t *testing.T) { + v := LocalDiskIDs{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeLocalDiskIDs Msgsize() is inaccurate") + } + + vn := LocalDiskIDs{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeLocalDiskIDs(b *testing.B) { + v := LocalDiskIDs{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeLocalDiskIDs(b *testing.B) { + v := LocalDiskIDs{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalMetadataHandlerParams(t *testing.T) { + v := MetadataHandlerParams{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgMetadataHandlerParams(b *testing.B) { + v := MetadataHandlerParams{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgMetadataHandlerParams(b *testing.B) { + v := MetadataHandlerParams{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalMetadataHandlerParams(b *testing.B) { + v := MetadataHandlerParams{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeMetadataHandlerParams(t *testing.T) { + v := MetadataHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeMetadataHandlerParams Msgsize() is inaccurate") + } + + vn := MetadataHandlerParams{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeMetadataHandlerParams(b *testing.B) { + v := MetadataHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeMetadataHandlerParams(b *testing.B) { + v := MetadataHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalRawFileInfo(t *testing.T) { + v := RawFileInfo{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgRawFileInfo(b *testing.B) { + v := RawFileInfo{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgRawFileInfo(b *testing.B) { + v := RawFileInfo{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalRawFileInfo(b *testing.B) { + v := RawFileInfo{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeRawFileInfo(t *testing.T) { + v := RawFileInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeRawFileInfo Msgsize() is inaccurate") + } + + vn := RawFileInfo{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeRawFileInfo(b *testing.B) { + v := RawFileInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeRawFileInfo(b *testing.B) { + v := RawFileInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalReadAllHandlerParams(t *testing.T) { + v := ReadAllHandlerParams{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgReadAllHandlerParams(b *testing.B) { + v := ReadAllHandlerParams{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgReadAllHandlerParams(b *testing.B) { + v := ReadAllHandlerParams{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalReadAllHandlerParams(b *testing.B) { + v := ReadAllHandlerParams{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeReadAllHandlerParams(t *testing.T) { + v := ReadAllHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeReadAllHandlerParams Msgsize() is inaccurate") + } + + vn := ReadAllHandlerParams{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeReadAllHandlerParams(b *testing.B) { + v := ReadAllHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeReadAllHandlerParams(b *testing.B) { + v := ReadAllHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalReadMultipleReq(t *testing.T) { + v := ReadMultipleReq{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgReadMultipleReq(b *testing.B) { + v := ReadMultipleReq{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgReadMultipleReq(b *testing.B) { + v := ReadMultipleReq{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalReadMultipleReq(b *testing.B) { + v := ReadMultipleReq{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeReadMultipleReq(t *testing.T) { + v := ReadMultipleReq{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeReadMultipleReq Msgsize() is inaccurate") + } + + vn := ReadMultipleReq{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeReadMultipleReq(b *testing.B) { + v := ReadMultipleReq{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeReadMultipleReq(b *testing.B) { + v := ReadMultipleReq{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalReadMultipleResp(t *testing.T) { + v := ReadMultipleResp{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgReadMultipleResp(b *testing.B) { + v := ReadMultipleResp{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgReadMultipleResp(b *testing.B) { + v := ReadMultipleResp{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalReadMultipleResp(b *testing.B) { + v := ReadMultipleResp{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeReadMultipleResp(t *testing.T) { + v := ReadMultipleResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeReadMultipleResp Msgsize() is inaccurate") + } + + vn := ReadMultipleResp{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeReadMultipleResp(b *testing.B) { + v := ReadMultipleResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeReadMultipleResp(b *testing.B) { + v := ReadMultipleResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalReadPartsReq(t *testing.T) { + v := ReadPartsReq{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgReadPartsReq(b *testing.B) { + v := ReadPartsReq{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgReadPartsReq(b *testing.B) { + v := ReadPartsReq{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalReadPartsReq(b *testing.B) { + v := ReadPartsReq{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeReadPartsReq(t *testing.T) { + v := ReadPartsReq{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeReadPartsReq Msgsize() is inaccurate") + } + + vn := ReadPartsReq{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeReadPartsReq(b *testing.B) { + v := ReadPartsReq{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeReadPartsReq(b *testing.B) { + v := ReadPartsReq{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalReadPartsResp(t *testing.T) { + v := ReadPartsResp{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgReadPartsResp(b *testing.B) { + v := ReadPartsResp{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgReadPartsResp(b *testing.B) { + v := ReadPartsResp{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalReadPartsResp(b *testing.B) { + v := ReadPartsResp{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeReadPartsResp(t *testing.T) { + v := ReadPartsResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeReadPartsResp Msgsize() is inaccurate") + } + + vn := ReadPartsResp{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeReadPartsResp(b *testing.B) { + v := ReadPartsResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeReadPartsResp(b *testing.B) { + v := ReadPartsResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalRenameDataHandlerParams(t *testing.T) { + v := RenameDataHandlerParams{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgRenameDataHandlerParams(b *testing.B) { + v := RenameDataHandlerParams{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgRenameDataHandlerParams(b *testing.B) { + v := RenameDataHandlerParams{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalRenameDataHandlerParams(b *testing.B) { + v := RenameDataHandlerParams{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeRenameDataHandlerParams(t *testing.T) { + v := RenameDataHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeRenameDataHandlerParams Msgsize() is inaccurate") + } + + vn := RenameDataHandlerParams{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeRenameDataHandlerParams(b *testing.B) { + v := RenameDataHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeRenameDataHandlerParams(b *testing.B) { + v := RenameDataHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalRenameDataInlineHandlerParams(t *testing.T) { + v := RenameDataInlineHandlerParams{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgRenameDataInlineHandlerParams(b *testing.B) { + v := RenameDataInlineHandlerParams{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgRenameDataInlineHandlerParams(b *testing.B) { + v := RenameDataInlineHandlerParams{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalRenameDataInlineHandlerParams(b *testing.B) { + v := RenameDataInlineHandlerParams{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeRenameDataInlineHandlerParams(t *testing.T) { + v := RenameDataInlineHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeRenameDataInlineHandlerParams Msgsize() is inaccurate") + } + + vn := RenameDataInlineHandlerParams{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeRenameDataInlineHandlerParams(b *testing.B) { + v := RenameDataInlineHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeRenameDataInlineHandlerParams(b *testing.B) { + v := RenameDataInlineHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalRenameDataResp(t *testing.T) { + v := RenameDataResp{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgRenameDataResp(b *testing.B) { + v := RenameDataResp{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgRenameDataResp(b *testing.B) { + v := RenameDataResp{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalRenameDataResp(b *testing.B) { + v := RenameDataResp{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeRenameDataResp(t *testing.T) { + v := RenameDataResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeRenameDataResp Msgsize() is inaccurate") + } + + vn := RenameDataResp{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeRenameDataResp(b *testing.B) { + v := RenameDataResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeRenameDataResp(b *testing.B) { + v := RenameDataResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalRenameFileHandlerParams(t *testing.T) { + v := RenameFileHandlerParams{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgRenameFileHandlerParams(b *testing.B) { + v := RenameFileHandlerParams{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgRenameFileHandlerParams(b *testing.B) { + v := RenameFileHandlerParams{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalRenameFileHandlerParams(b *testing.B) { + v := RenameFileHandlerParams{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeRenameFileHandlerParams(t *testing.T) { + v := RenameFileHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeRenameFileHandlerParams Msgsize() is inaccurate") + } + + vn := RenameFileHandlerParams{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeRenameFileHandlerParams(b *testing.B) { + v := RenameFileHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeRenameFileHandlerParams(b *testing.B) { + v := RenameFileHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalRenameOptions(t *testing.T) { + v := RenameOptions{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgRenameOptions(b *testing.B) { + v := RenameOptions{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgRenameOptions(b *testing.B) { + v := RenameOptions{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalRenameOptions(b *testing.B) { + v := RenameOptions{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeRenameOptions(t *testing.T) { + v := RenameOptions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeRenameOptions Msgsize() is inaccurate") + } + + vn := RenameOptions{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeRenameOptions(b *testing.B) { + v := RenameOptions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeRenameOptions(b *testing.B) { + v := RenameOptions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalRenamePartHandlerParams(t *testing.T) { + v := RenamePartHandlerParams{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgRenamePartHandlerParams(b *testing.B) { + v := RenamePartHandlerParams{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgRenamePartHandlerParams(b *testing.B) { + v := RenamePartHandlerParams{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalRenamePartHandlerParams(b *testing.B) { + v := RenamePartHandlerParams{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeRenamePartHandlerParams(t *testing.T) { + v := RenamePartHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeRenamePartHandlerParams Msgsize() is inaccurate") + } + + vn := RenamePartHandlerParams{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeRenamePartHandlerParams(b *testing.B) { + v := RenamePartHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeRenamePartHandlerParams(b *testing.B) { + v := RenamePartHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalUpdateMetadataOpts(t *testing.T) { + v := UpdateMetadataOpts{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgUpdateMetadataOpts(b *testing.B) { + v := UpdateMetadataOpts{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgUpdateMetadataOpts(b *testing.B) { + v := UpdateMetadataOpts{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalUpdateMetadataOpts(b *testing.B) { + v := UpdateMetadataOpts{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeUpdateMetadataOpts(t *testing.T) { + v := UpdateMetadataOpts{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeUpdateMetadataOpts Msgsize() is inaccurate") + } + + vn := UpdateMetadataOpts{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeUpdateMetadataOpts(b *testing.B) { + v := UpdateMetadataOpts{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeUpdateMetadataOpts(b *testing.B) { + v := UpdateMetadataOpts{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalVolInfo(t *testing.T) { + v := VolInfo{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgVolInfo(b *testing.B) { + v := VolInfo{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgVolInfo(b *testing.B) { + v := VolInfo{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalVolInfo(b *testing.B) { + v := VolInfo{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeVolInfo(t *testing.T) { + v := VolInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeVolInfo Msgsize() is inaccurate") + } + + vn := VolInfo{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeVolInfo(b *testing.B) { + v := VolInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeVolInfo(b *testing.B) { + v := VolInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalVolsInfo(t *testing.T) { + v := VolsInfo{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgVolsInfo(b *testing.B) { + v := VolsInfo{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgVolsInfo(b *testing.B) { + v := VolsInfo{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalVolsInfo(b *testing.B) { + v := VolsInfo{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeVolsInfo(t *testing.T) { + v := VolsInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeVolsInfo Msgsize() is inaccurate") + } + + vn := VolsInfo{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeVolsInfo(b *testing.B) { + v := VolsInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeVolsInfo(b *testing.B) { + v := VolsInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalWriteAllHandlerParams(t *testing.T) { + v := WriteAllHandlerParams{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgWriteAllHandlerParams(b *testing.B) { + v := WriteAllHandlerParams{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgWriteAllHandlerParams(b *testing.B) { + v := WriteAllHandlerParams{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalWriteAllHandlerParams(b *testing.B) { + v := WriteAllHandlerParams{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeWriteAllHandlerParams(t *testing.T) { + v := WriteAllHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeWriteAllHandlerParams Msgsize() is inaccurate") + } + + vn := WriteAllHandlerParams{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeWriteAllHandlerParams(b *testing.B) { + v := WriteAllHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeWriteAllHandlerParams(b *testing.B) { + v := WriteAllHandlerParams{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/storage-datatypes_test.go b/cmd/storage-datatypes_test.go new file mode 100644 index 00000000..60470a22 --- /dev/null +++ b/cmd/storage-datatypes_test.go @@ -0,0 +1,222 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "encoding/gob" + "io" + "testing" + "time" + + "github.com/tinylib/msgp/msgp" +) + +func BenchmarkDecodeVolInfoMsgp(b *testing.B) { + v := VolInfo{ + Name: "uuid", + Created: time.Now(), + } + var buf bytes.Buffer + msgp.Encode(&buf, &v) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.Log("Size:", buf.Len(), "bytes") + b.SetBytes(1) + b.ReportAllocs() + + for b.Loop() { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkDecodeDiskInfoMsgp(b *testing.B) { + v := DiskInfo{ + Total: 1000, + Free: 1000, + Used: 1000, + FSType: "xfs", + RootDisk: true, + Healing: true, + Endpoint: "http://localhost:9001/tmp/drive1", + MountPath: "/tmp/drive1", + ID: "uuid", + Error: "", + } + var buf bytes.Buffer + msgp.Encode(&buf, &v) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.Log("Size:", buf.Len(), "bytes") + b.SetBytes(1) + b.ReportAllocs() + + for b.Loop() { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkDecodeDiskInfoGOB(b *testing.B) { + v := DiskInfo{ + Total: 1000, + Free: 1000, + Used: 1000, + FSType: "xfs", + RootDisk: true, + Healing: true, + Endpoint: "http://localhost:9001/tmp/drive1", + MountPath: "/tmp/drive1", + ID: "uuid", + Error: "", + } + + var buf bytes.Buffer + gob.NewEncoder(&buf).Encode(v) + encoded := buf.Bytes() + b.Log("Size:", buf.Len(), "bytes") + b.SetBytes(1) + b.ReportAllocs() + + for b.Loop() { + dec := gob.NewDecoder(bytes.NewBuffer(encoded)) + err := dec.Decode(&v) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkEncodeDiskInfoMsgp(b *testing.B) { + v := DiskInfo{ + Total: 1000, + Free: 1000, + Used: 1000, + FSType: "xfs", + RootDisk: true, + Healing: true, + Endpoint: "http://localhost:9001/tmp/drive1", + MountPath: "/tmp/drive1", + ID: "uuid", + Error: "", + } + + b.SetBytes(1) + b.ReportAllocs() + + for b.Loop() { + err := msgp.Encode(io.Discard, &v) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkEncodeDiskInfoGOB(b *testing.B) { + v := DiskInfo{ + Total: 1000, + Free: 1000, + Used: 1000, + FSType: "xfs", + RootDisk: true, + Healing: true, + Endpoint: "http://localhost:9001/tmp/drive1", + MountPath: "/tmp/drive1", + ID: "uuid", + Error: "", + } + + enc := gob.NewEncoder(io.Discard) + b.SetBytes(1) + b.ReportAllocs() + + for b.Loop() { + err := enc.Encode(&v) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkDecodeFileInfoMsgp(b *testing.B) { + v := FileInfo{Volume: "testbucket", Name: "src/compress/zlib/reader_test.go", VersionID: "", IsLatest: true, Deleted: false, DataDir: "5e0153cc-621a-4267-8cb6-4919140d53b3", XLV1: false, ModTime: UTCNow(), Size: 3430, Mode: 0x0, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption-Iv": "jIJPsrkkVYYMvc7edBrNl+7zcM7+ZwXqMb/YAjBO/ck=", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id": "my-minio-key", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "IAAfAP2p7ZLv3UpLwBnsKkF2mtWba0qoY42tymK0szRgGvAxBNcXyHXYooe9dQpeeEJWgKUa/8R61oCy1mFwIg==", "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key": "IAAfAPFYRDkHVirJBJxBixNj3PLWt78dFuUTyTLIdLG820J7XqLPBO4gpEEEWw/DoTsJIb+apnaem+rKtQ1h3Q==", "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256", "content-type": "application/octet-stream", "etag": "20000f00e2c3709dc94905c6ce31e1cadbd1c064e14acdcd44cf0ac2db777eeedd88d639fcd64de16851ade8b21a9a1a"}, Parts: []ObjectPartInfo{{ETag: "", Number: 1, Size: 3430, ActualSize: 3398}}, Erasure: ErasureInfo{Algorithm: "reedsolomon", DataBlocks: 2, ParityBlocks: 2, BlockSize: 10485760, Index: 3, Distribution: []int{3, 4, 1, 2}, Checksums: []ChecksumInfo{{PartNumber: 1, Algorithm: 0x3, Hash: []uint8{}}}}} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.Log("Size:", buf.Len(), "bytes") + b.SetBytes(1) + b.ReportAllocs() + + for b.Loop() { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkDecodeFileInfoGOB(b *testing.B) { + v := FileInfo{Volume: "testbucket", Name: "src/compress/zlib/reader_test.go", VersionID: "", IsLatest: true, Deleted: false, DataDir: "5e0153cc-621a-4267-8cb6-4919140d53b3", XLV1: false, ModTime: UTCNow(), Size: 3430, Mode: 0x0, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption-Iv": "jIJPsrkkVYYMvc7edBrNl+7zcM7+ZwXqMb/YAjBO/ck=", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id": "my-minio-key", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "IAAfAP2p7ZLv3UpLwBnsKkF2mtWba0qoY42tymK0szRgGvAxBNcXyHXYooe9dQpeeEJWgKUa/8R61oCy1mFwIg==", "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key": "IAAfAPFYRDkHVirJBJxBixNj3PLWt78dFuUTyTLIdLG820J7XqLPBO4gpEEEWw/DoTsJIb+apnaem+rKtQ1h3Q==", "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256", "content-type": "application/octet-stream", "etag": "20000f00e2c3709dc94905c6ce31e1cadbd1c064e14acdcd44cf0ac2db777eeedd88d639fcd64de16851ade8b21a9a1a"}, Parts: []ObjectPartInfo{{ETag: "", Number: 1, Size: 3430, ActualSize: 3398}}, Erasure: ErasureInfo{Algorithm: "reedsolomon", DataBlocks: 2, ParityBlocks: 2, BlockSize: 10485760, Index: 3, Distribution: []int{3, 4, 1, 2}, Checksums: []ChecksumInfo{{PartNumber: 1, Algorithm: 0x3, Hash: []uint8{}}}}} + var buf bytes.Buffer + gob.NewEncoder(&buf).Encode(v) + encoded := buf.Bytes() + b.Log("Size:", buf.Len(), "bytes") + b.SetBytes(1) + b.ReportAllocs() + + for b.Loop() { + dec := gob.NewDecoder(bytes.NewBuffer(encoded)) + err := dec.Decode(&v) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkEncodeFileInfoMsgp(b *testing.B) { + v := FileInfo{Volume: "testbucket", Name: "src/compress/zlib/reader_test.go", VersionID: "", IsLatest: true, Deleted: false, DataDir: "5e0153cc-621a-4267-8cb6-4919140d53b3", XLV1: false, ModTime: UTCNow(), Size: 3430, Mode: 0x0, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption-Iv": "jIJPsrkkVYYMvc7edBrNl+7zcM7+ZwXqMb/YAjBO/ck=", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id": "my-minio-key", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "IAAfAP2p7ZLv3UpLwBnsKkF2mtWba0qoY42tymK0szRgGvAxBNcXyHXYooe9dQpeeEJWgKUa/8R61oCy1mFwIg==", "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key": "IAAfAPFYRDkHVirJBJxBixNj3PLWt78dFuUTyTLIdLG820J7XqLPBO4gpEEEWw/DoTsJIb+apnaem+rKtQ1h3Q==", "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256", "content-type": "application/octet-stream", "etag": "20000f00e2c3709dc94905c6ce31e1cadbd1c064e14acdcd44cf0ac2db777eeedd88d639fcd64de16851ade8b21a9a1a"}, Parts: []ObjectPartInfo{{ETag: "", Number: 1, Size: 3430, ActualSize: 3398}}, Erasure: ErasureInfo{Algorithm: "reedsolomon", DataBlocks: 2, ParityBlocks: 2, BlockSize: 10485760, Index: 3, Distribution: []int{3, 4, 1, 2}, Checksums: []ChecksumInfo{{PartNumber: 1, Algorithm: 0x3, Hash: []uint8{}}}}} + b.SetBytes(1) + b.ReportAllocs() + + for b.Loop() { + err := msgp.Encode(io.Discard, &v) + if err != nil { + b.Fatal(err) + } + } +} + +func BenchmarkEncodeFileInfoGOB(b *testing.B) { + v := FileInfo{Volume: "testbucket", Name: "src/compress/zlib/reader_test.go", VersionID: "", IsLatest: true, Deleted: false, DataDir: "5e0153cc-621a-4267-8cb6-4919140d53b3", XLV1: false, ModTime: UTCNow(), Size: 3430, Mode: 0x0, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption-Iv": "jIJPsrkkVYYMvc7edBrNl+7zcM7+ZwXqMb/YAjBO/ck=", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id": "my-minio-key", "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "IAAfAP2p7ZLv3UpLwBnsKkF2mtWba0qoY42tymK0szRgGvAxBNcXyHXYooe9dQpeeEJWgKUa/8R61oCy1mFwIg==", "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key": "IAAfAPFYRDkHVirJBJxBixNj3PLWt78dFuUTyTLIdLG820J7XqLPBO4gpEEEWw/DoTsJIb+apnaem+rKtQ1h3Q==", "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256", "content-type": "application/octet-stream", "etag": "20000f00e2c3709dc94905c6ce31e1cadbd1c064e14acdcd44cf0ac2db777eeedd88d639fcd64de16851ade8b21a9a1a"}, Parts: []ObjectPartInfo{{ETag: "", Number: 1, Size: 3430, ActualSize: 3398}}, Erasure: ErasureInfo{Algorithm: "reedsolomon", DataBlocks: 2, ParityBlocks: 2, BlockSize: 10485760, Index: 3, Distribution: []int{3, 4, 1, 2}, Checksums: []ChecksumInfo{{PartNumber: 1, Algorithm: 0x3, Hash: []uint8{}}}}} + enc := gob.NewEncoder(io.Discard) + b.SetBytes(1) + b.ReportAllocs() + + for b.Loop() { + err := enc.Encode(&v) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/storage-errors.go b/cmd/storage-errors.go new file mode 100644 index 00000000..b39d7c8a --- /dev/null +++ b/cmd/storage-errors.go @@ -0,0 +1,187 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" +) + +// errMaxVersionsExceeded return error beyond 10000 (default) versions per object +var errMaxVersionsExceeded = StorageErr("maximum versions exceeded, please delete few versions to proceed") + +// errUnexpected - unexpected error, requires manual intervention. +var errUnexpected = StorageErr("unexpected error, please report this issue at https://github.com/minio/minio/issues") + +// errCorruptedFormat - corrupted format. +var errCorruptedFormat = StorageErr("corrupted format") + +// errCorruptedBackend - corrupted backend. +var errCorruptedBackend = StorageErr("corrupted backend") + +// errUnformattedDisk - unformatted disk found. +var errUnformattedDisk = StorageErr("unformatted drive found") + +// errInconsistentDisk - inconsistent disk found. +var errInconsistentDisk = StorageErr("inconsistent drive found") + +// errUnsupporteDisk - when disk does not support O_DIRECT flag. +var errUnsupportedDisk = StorageErr("drive does not support O_DIRECT") + +// errDiskFull - cannot create volume or files when disk is full. +var errDiskFull = StorageErr("drive path full") + +// errDiskNotDir - cannot use storage disk if its not a directory +var errDiskNotDir = StorageErr("drive is not directory or mountpoint") + +// errDiskNotFound - cannot find the underlying configured disk anymore. +var errDiskNotFound = StorageErr("drive not found") + +// errDiskOngoingReq - indicates if the disk has an on-going request in progress. +var errDiskOngoingReq = StorageErr("drive still did not complete the request") + +// errDriveIsRoot - cannot use the disk since its a root disk. +var errDriveIsRoot = StorageErr("drive is part of root drive, will not be used") + +// errFaultyRemoteDisk - remote disk is faulty. +var errFaultyRemoteDisk = StorageErr("remote drive is faulty") + +// errFaultyDisk - disk is faulty. +var errFaultyDisk = StorageErr("drive is faulty") + +// errDiskAccessDenied - we don't have write permissions on disk. +var errDiskAccessDenied = StorageErr("drive access denied") + +// errFileNotFound - cannot find the file. +var errFileNotFound = StorageErr("file not found") + +// errFileNotFound - cannot find requested file version. +var errFileVersionNotFound = StorageErr("file version not found") + +// errTooManyOpenFiles - too many open files. +var errTooManyOpenFiles = StorageErr("too many open files, please increase 'ulimit -n'") + +// errFileNameTooLong - given file name is too long than supported length. +var errFileNameTooLong = StorageErr("file name too long") + +// errVolumeExists - cannot create same volume again. +var errVolumeExists = StorageErr("volume already exists") + +// errIsNotRegular - not of regular file type. +var errIsNotRegular = StorageErr("not of regular file type") + +// errPathNotFound - cannot find the path. +var errPathNotFound = StorageErr("path not found") + +// errVolumeNotFound - cannot find the volume. +var errVolumeNotFound = StorageErr("volume not found") + +// errVolumeNotEmpty - volume not empty. +var errVolumeNotEmpty = StorageErr("volume is not empty") + +// errVolumeAccessDenied - cannot access volume, insufficient permissions. +var errVolumeAccessDenied = StorageErr("volume access denied") + +// errFileAccessDenied - cannot access file, insufficient permissions. +var errFileAccessDenied = StorageErr("file access denied") + +// errFileCorrupt - file has an unexpected size, or is not readable +var errFileCorrupt = StorageErr("file is corrupted") + +// errBitrotHashAlgoInvalid - the algo for bit-rot hash +// verification is empty or invalid. +var errBitrotHashAlgoInvalid = StorageErr("bit-rot hash algorithm is invalid") + +// errCrossDeviceLink - rename across devices not allowed. +var errCrossDeviceLink = StorageErr("Rename across devices not allowed, please fix your backend configuration") + +// errLessData - returned when less data available than what was requested. +var errLessData = StorageErr("less data available than what was requested") + +// errMoreData = returned when more data was sent by the caller than what it was supposed to. +var errMoreData = StorageErr("more data was sent than what was advertised") + +// indicates readDirFn to return without further applying the fn() +var errDoneForNow = errors.New("done for now") + +// errSkipFile returned by the fn() for readDirFn() when it needs +// to proceed to next entry. +var errSkipFile = errors.New("skip this file") + +var errIgnoreFileContrib = errors.New("ignore this file's contribution toward data-usage") + +// errXLBackend XL drive mode requires fresh deployment. +var errXLBackend = errors.New("XL backend requires fresh drive") + +// StorageErr represents error generated by xlStorage call. +type StorageErr string + +func (h StorageErr) Error() string { + return string(h) +} + +// Collection of basic errors. +var baseErrs = []error{ + errDiskNotFound, + errFaultyDisk, + errFaultyRemoteDisk, +} + +var baseIgnoredErrs = baseErrs + +// Is a one place function which converts all os.PathError +// into a more FS object layer friendly form, converts +// known errors into their typed form for top level +// interpretation. +func osErrToFileErr(err error) error { + if err == nil { + return nil + } + if osIsNotExist(err) { + return errFileNotFound + } + if osIsPermission(err) { + return errFileAccessDenied + } + if isSysErrNotDir(err) || isSysErrIsDir(err) { + return errFileNotFound + } + if isSysErrPathNotFound(err) { + return errFileNotFound + } + if isSysErrTooManyFiles(err) { + return errTooManyOpenFiles + } + if isSysErrHandleInvalid(err) { + return errFileNotFound + } + if isSysErrIO(err) { + return errFaultyDisk + } + if isSysErrInvalidArg(err) { + storageLogIf(context.Background(), err) + // For some odd calls with O_DIRECT reads + // filesystems can return EINVAL, handle + // these as FileNotFound instead. + return errFileNotFound + } + if isSysErrNoSpace(err) { + return errDiskFull + } + return err +} diff --git a/cmd/storage-interface.go b/cmd/storage-interface.go new file mode 100644 index 00000000..3c265048 --- /dev/null +++ b/cmd/storage-interface.go @@ -0,0 +1,114 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "io" + "time" + + "github.com/minio/madmin-go/v3" +) + +// StorageAPI interface. +type StorageAPI interface { + // Stringified version of disk. + String() string + + // Storage operations. + + // Returns true if disk is online and its valid i.e valid format.json. + // This has nothing to do with if the drive is hung or not responding. + // For that individual storage API calls will fail properly. The purpose + // of this function is to know if the "drive" has "format.json" or not + // if it has a "format.json" then is it correct "format.json" or not. + IsOnline() bool + + // Returns the last time this disk (re)-connected + LastConn() time.Time + + // Indicates if disk is local or not. + IsLocal() bool + + // Returns hostname if disk is remote. + Hostname() string + + // Returns the entire endpoint. + Endpoint() Endpoint + + // Close the disk, mark it purposefully closed, only implemented for remote disks. + Close() error + + // Returns the unique 'uuid' of this disk. + GetDiskID() (string, error) + + // Set a unique 'uuid' for this disk, only used when + // disk is replaced and formatted. + SetDiskID(id string) + + // Returns healing information for a newly replaced disk, + // returns 'nil' once healing is complete or if the disk + // has never been replaced. + Healing() *healingTracker + DiskInfo(ctx context.Context, opts DiskInfoOptions) (info DiskInfo, err error) + NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry, scanMode madmin.HealScanMode, shouldSleep func() bool) (dataUsageCache, error) + + // Volume operations. + MakeVol(ctx context.Context, volume string) (err error) + MakeVolBulk(ctx context.Context, volumes ...string) (err error) + ListVols(ctx context.Context) (vols []VolInfo, err error) + StatVol(ctx context.Context, volume string) (vol VolInfo, err error) + DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) + + // WalkDir will walk a directory on disk and return a metacache stream on wr. + WalkDir(ctx context.Context, opts WalkDirOptions, wr io.Writer) error + + // Metadata operations + DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool, opts DeleteOptions) error + DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions, opts DeleteOptions) []error + DeleteBulk(ctx context.Context, volume string, paths ...string) error + WriteMetadata(ctx context.Context, origvolume, volume, path string, fi FileInfo) error + UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo, opts UpdateMetadataOpts) error + ReadVersion(ctx context.Context, origvolume, volume, path, versionID string, opts ReadOptions) (FileInfo, error) + ReadXL(ctx context.Context, volume, path string, readData bool) (RawFileInfo, error) + RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (RenameDataResp, error) + + // File operations. + ListDir(ctx context.Context, origvolume, volume, dirPath string, count int) ([]string, error) + ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) + AppendFile(ctx context.Context, volume string, path string, buf []byte) (err error) + CreateFile(ctx context.Context, origvolume, olume, path string, size int64, reader io.Reader) error + ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) + RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) error + RenamePart(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string, meta []byte, skipParent string) error + CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (*CheckPartsResp, error) + Delete(ctx context.Context, volume string, path string, opts DeleteOptions) (err error) + VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error) + StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) + ReadParts(ctx context.Context, bucket string, partMetaPaths ...string) ([]*ObjectPartInfo, error) + ReadMultiple(ctx context.Context, req ReadMultipleReq, resp chan<- ReadMultipleResp) error + CleanAbandonedData(ctx context.Context, volume string, path string) error + + // Write all data, syncs the data to disk. + // Should be used for smaller payloads. + WriteAll(ctx context.Context, volume string, path string, b []byte) (err error) + + // Read all. + ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) + GetDiskLoc() (poolIdx, setIdx, diskIdx int) // Retrieve location indexes. +} diff --git a/cmd/storage-rest-client.go b/cmd/storage-rest-client.go new file mode 100644 index 00000000..d3c17a17 --- /dev/null +++ b/cmd/storage-rest-client.go @@ -0,0 +1,1010 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "path" + "strconv" + "strings" + "sync/atomic" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/bpool" + "github.com/minio/minio/internal/cachevalue" + "github.com/minio/minio/internal/grid" + xhttp "github.com/minio/minio/internal/http" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/rest" + xnet "github.com/minio/pkg/v3/net" + xbufio "github.com/philhofer/fwd" + "github.com/tinylib/msgp/msgp" +) + +func isNetworkError(err error) bool { + if err == nil { + return false + } + + if nerr, ok := err.(*rest.NetworkError); ok { + if down := xnet.IsNetworkOrHostDown(nerr.Err, false); down { + return true + } + if errors.Is(nerr.Err, rest.ErrClientClosed) { + return true + } + } + if errors.Is(err, grid.ErrDisconnected) { + return true + } + // More corner cases suitable for storage REST API + switch { + // A peer node can be in shut down phase and proactively + // return 503 server closed error, consider it as an offline node + case strings.Contains(err.Error(), http.ErrServerClosed.Error()): + return true + // Corner case, the server closed the connection with a keep-alive timeout + // some requests are not retried internally, such as POST request with written body + case strings.Contains(err.Error(), "server closed idle connection"): + return true + } + + return false +} + +// Converts network error to storageErr. This function is +// written so that the storageAPI errors are consistent +// across network disks. +func toStorageErr(err error) error { + if err == nil { + return nil + } + + if isNetworkError(err) { + return errDiskNotFound + } + + switch err.Error() { + case errUploadIDNotFound.Error(): + return errUploadIDNotFound + case errFaultyDisk.Error(): + return errFaultyDisk + case errFaultyRemoteDisk.Error(): + return errFaultyRemoteDisk + case errFileCorrupt.Error(): + return errFileCorrupt + case errUnexpected.Error(): + return errUnexpected + case errDiskFull.Error(): + return errDiskFull + case errVolumeNotFound.Error(): + return errVolumeNotFound + case errVolumeExists.Error(): + return errVolumeExists + case errFileNotFound.Error(): + return errFileNotFound + case errFileVersionNotFound.Error(): + return errFileVersionNotFound + case errFileNameTooLong.Error(): + return errFileNameTooLong + case errFileAccessDenied.Error(): + return errFileAccessDenied + case errPathNotFound.Error(): + return errPathNotFound + case errIsNotRegular.Error(): + return errIsNotRegular + case errVolumeNotEmpty.Error(): + return errVolumeNotEmpty + case errVolumeAccessDenied.Error(): + return errVolumeAccessDenied + case errCorruptedFormat.Error(): + return errCorruptedFormat + case errCorruptedBackend.Error(): + return errCorruptedBackend + case errUnformattedDisk.Error(): + return errUnformattedDisk + case errInvalidAccessKeyID.Error(): + return errInvalidAccessKeyID + case errAuthentication.Error(): + return errAuthentication + case errRPCAPIVersionUnsupported.Error(): + return errRPCAPIVersionUnsupported + case errServerTimeMismatch.Error(): + return errServerTimeMismatch + case io.EOF.Error(): + return io.EOF + case io.ErrUnexpectedEOF.Error(): + return io.ErrUnexpectedEOF + case errDiskStale.Error(): + return errDiskNotFound + case errDiskNotFound.Error(): + return errDiskNotFound + case errMaxVersionsExceeded.Error(): + return errMaxVersionsExceeded + case errInconsistentDisk.Error(): + return errInconsistentDisk + case errDriveIsRoot.Error(): + return errDriveIsRoot + case errDiskOngoingReq.Error(): + return errDiskOngoingReq + case grid.ErrUnknownHandler.Error(): + return errInconsistentDisk + case grid.ErrDisconnected.Error(): + return errDiskNotFound + } + return err +} + +// Abstracts a remote disk. +type storageRESTClient struct { + endpoint Endpoint + restClient *rest.Client + gridConn *grid.Subroute + diskID atomic.Pointer[string] + + diskInfoCache *cachevalue.Cache[DiskInfo] +} + +// Retrieve location indexes. +func (client *storageRESTClient) GetDiskLoc() (poolIdx, setIdx, diskIdx int) { + return client.endpoint.PoolIdx, client.endpoint.SetIdx, client.endpoint.DiskIdx +} + +// Wrapper to restClient.CallWithMethod to handle network errors, in case of network error the connection is disconnected +// and a healthcheck routine gets invoked that would reconnect. +func (client *storageRESTClient) callGet(ctx context.Context, rpcMethod string, values url.Values, body io.Reader, length int64) (io.ReadCloser, error) { + if values == nil { + values = make(url.Values) + } + values.Set(storageRESTDiskID, *client.diskID.Load()) + respBody, err := client.restClient.CallWithHTTPMethod(ctx, http.MethodGet, rpcMethod, values, body, length) + if err != nil { + return nil, toStorageErr(err) + } + return respBody, nil +} + +// Wrapper to restClient.Call to handle network errors, in case of network error the connection is disconnected +// and a healthcheck routine gets invoked that would reconnect. +func (client *storageRESTClient) call(ctx context.Context, rpcMethod string, values url.Values, body io.Reader, length int64) (io.ReadCloser, error) { + if values == nil { + values = make(url.Values) + } + values.Set(storageRESTDiskID, *client.diskID.Load()) + respBody, err := client.restClient.CallWithHTTPMethod(ctx, http.MethodPost, rpcMethod, values, body, length) + if err != nil { + return nil, toStorageErr(err) + } + return respBody, nil +} + +// Stringer provides a canonicalized representation of network device. +func (client *storageRESTClient) String() string { + return client.endpoint.String() +} + +// IsOnline - returns whether client failed to connect or not. +func (client *storageRESTClient) IsOnline() bool { + return client.restClient.IsOnline() || client.IsOnlineWS() +} + +// IsOnlineWS - returns whether websocket client failed to connect or not. +func (client *storageRESTClient) IsOnlineWS() bool { + return client.gridConn.State() == grid.StateConnected +} + +// LastConn - returns when the disk is seen to be connected the last time +func (client *storageRESTClient) LastConn() time.Time { + return client.restClient.LastConn() +} + +func (client *storageRESTClient) IsLocal() bool { + return false +} + +func (client *storageRESTClient) Hostname() string { + return client.endpoint.Host +} + +func (client *storageRESTClient) Endpoint() Endpoint { + return client.endpoint +} + +func (client *storageRESTClient) Healing() *healingTracker { + // This call is not implemented for remote client on purpose. + // healing tracker is always for local disks. + return nil +} + +func (client *storageRESTClient) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry, scanMode madmin.HealScanMode, _ func() bool) (dataUsageCache, error) { + defer xioutil.SafeClose(updates) + + st, err := storageNSScannerRPC.Call(ctx, client.gridConn, &nsScannerOptions{ + DiskID: *client.diskID.Load(), + ScanMode: int(scanMode), + Cache: &cache, + }) + if err != nil { + return cache, toStorageErr(err) + } + var final *dataUsageCache + err = st.Results(func(resp *nsScannerResp) error { + if resp.Update != nil { + select { + case <-ctx.Done(): + case updates <- *resp.Update: + } + } + if resp.Final != nil { + final = resp.Final + } + // We can't reuse the response since it is sent upstream. + return nil + }) + if err != nil { + return cache, toStorageErr(err) + } + if final == nil { + return cache, errors.New("no final cache") + } + return *final, nil +} + +func (client *storageRESTClient) GetDiskID() (string, error) { + if !client.IsOnlineWS() { + // make sure to check if the disk is offline, since the underlying + // value is cached we should attempt to invalidate it if such calls + // were attempted. This can lead to false success under certain conditions + // - this change attempts to avoid stale information if the underlying + // transport is already down. + return "", errDiskNotFound + } + + // This call should never be over the network, this is always + // a cached value - caller should make sure to use this + // function on a fresh disk or make sure to look at the error + // from a different networked call to validate the GetDiskID() + return *client.diskID.Load(), nil +} + +func (client *storageRESTClient) SetDiskID(id string) { + client.diskID.Store(&id) +} + +func (client *storageRESTClient) DiskInfo(ctx context.Context, opts DiskInfoOptions) (info DiskInfo, err error) { + if !client.IsOnlineWS() { + // make sure to check if the disk is offline, since the underlying + // value is cached we should attempt to invalidate it if such calls + // were attempted. This can lead to false success under certain conditions + // - this change attempts to avoid stale information if the underlying + // transport is already down. + return info, errDiskNotFound + } + + // if 'NoOp' we do not cache the value. + if opts.NoOp { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + opts.DiskID = *client.diskID.Load() + + infop, err := storageDiskInfoRPC.Call(ctx, client.gridConn, &opts) + if err != nil { + return info, toStorageErr(err) + } + info = *infop + if info.Error != "" { + return info, toStorageErr(errors.New(info.Error)) + } + return info, nil + } // In all other cases cache the value upto 1sec. + + client.diskInfoCache.InitOnce(time.Second, cachevalue.Opts{}, + func(ctx context.Context) (info DiskInfo, err error) { + ctx, cancel := context.WithTimeout(ctx, 5*time.Second) + defer cancel() + + nopts := DiskInfoOptions{DiskID: *client.diskID.Load(), Metrics: true} + infop, err := storageDiskInfoRPC.Call(ctx, client.gridConn, &nopts) + if err != nil { + return info, toStorageErr(err) + } + info = *infop + if info.Error != "" { + return info, toStorageErr(errors.New(info.Error)) + } + return info, nil + }, + ) + + return client.diskInfoCache.GetWithCtx(ctx) +} + +// MakeVolBulk - create multiple volumes in a bulk operation. +func (client *storageRESTClient) MakeVolBulk(ctx context.Context, volumes ...string) (err error) { + return errInvalidArgument +} + +// MakeVol - create a volume on a remote disk. +func (client *storageRESTClient) MakeVol(ctx context.Context, volume string) (err error) { + return errInvalidArgument +} + +// ListVols - List all volumes on a remote disk. +func (client *storageRESTClient) ListVols(ctx context.Context) (vols []VolInfo, err error) { + return nil, errInvalidArgument +} + +// StatVol - get volume info over the network. +func (client *storageRESTClient) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) { + v, err := storageStatVolRPC.Call(ctx, client.gridConn, grid.NewMSSWith(map[string]string{ + storageRESTDiskID: *client.diskID.Load(), + storageRESTVolume: volume, + })) + if err != nil { + return vol, toStorageErr(err) + } + vol = *v + // Performs shallow copy, so we can reuse. + storageStatVolRPC.PutResponse(v) + return vol, nil +} + +// DeleteVol - Deletes a volume over the network. +func (client *storageRESTClient) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) { + return errInvalidArgument +} + +// AppendFile - append to a file. +func (client *storageRESTClient) AppendFile(ctx context.Context, volume string, path string, buf []byte) error { + values := make(url.Values) + values.Set(storageRESTVolume, volume) + values.Set(storageRESTFilePath, path) + reader := bytes.NewReader(buf) + respBody, err := client.call(ctx, storageRESTMethodAppendFile, values, reader, -1) + defer xhttp.DrainBody(respBody) + return err +} + +func (client *storageRESTClient) CreateFile(ctx context.Context, origvolume, volume, path string, size int64, reader io.Reader) error { + values := make(url.Values) + values.Set(storageRESTVolume, volume) + values.Set(storageRESTFilePath, path) + values.Set(storageRESTLength, strconv.Itoa(int(size))) + values.Set(storageRESTOrigVolume, origvolume) + + respBody, err := client.call(ctx, storageRESTMethodCreateFile, values, io.NopCloser(reader), size) + defer xhttp.DrainBody(respBody) + if err != nil { + return err + } + _, err = waitForHTTPResponse(respBody) + return toStorageErr(err) +} + +func (client *storageRESTClient) WriteMetadata(ctx context.Context, origvolume, volume, path string, fi FileInfo) error { + ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout()) + defer cancel() + + _, err := storageWriteMetadataRPC.Call(ctx, client.gridConn, &MetadataHandlerParams{ + DiskID: *client.diskID.Load(), + OrigVolume: origvolume, + Volume: volume, + FilePath: path, + FI: fi, + }) + return toStorageErr(err) +} + +func (client *storageRESTClient) UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo, opts UpdateMetadataOpts) error { + ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout()) + defer cancel() + + _, err := storageUpdateMetadataRPC.Call(ctx, client.gridConn, &MetadataHandlerParams{ + DiskID: *client.diskID.Load(), + Volume: volume, + FilePath: path, + UpdateOpts: opts, + FI: fi, + }) + return toStorageErr(err) +} + +func (client *storageRESTClient) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool, opts DeleteOptions) (err error) { + ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout()) + defer cancel() + + _, err = storageDeleteVersionRPC.Call(ctx, client.gridConn, &DeleteVersionHandlerParams{ + DiskID: *client.diskID.Load(), + Volume: volume, + FilePath: path, + ForceDelMarker: forceDelMarker, + FI: fi, + Opts: opts, + }) + return toStorageErr(err) +} + +// WriteAll - write all data to a file. +func (client *storageRESTClient) WriteAll(ctx context.Context, volume string, path string, b []byte) error { + ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout()) + defer cancel() + + _, err := storageWriteAllRPC.Call(ctx, client.gridConn, &WriteAllHandlerParams{ + DiskID: *client.diskID.Load(), + Volume: volume, + FilePath: path, + Buf: b, + }) + return toStorageErr(err) +} + +// CheckParts - stat all file parts. +func (client *storageRESTClient) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (*CheckPartsResp, error) { + var resp *CheckPartsResp + st, err := storageCheckPartsRPC.Call(ctx, client.gridConn, &CheckPartsHandlerParams{ + DiskID: *client.diskID.Load(), + Volume: volume, + FilePath: path, + FI: fi, + }) + if err != nil { + return nil, toStorageErr(err) + } + err = st.Results(func(r *CheckPartsResp) error { + resp = r + return nil + }) + return resp, toStorageErr(err) +} + +// RenameData - rename source path to destination path atomically, metadata and data file. +func (client *storageRESTClient) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, + dstVolume, dstPath string, opts RenameOptions, +) (res RenameDataResp, err error) { + params := RenameDataHandlerParams{ + DiskID: *client.diskID.Load(), + SrcVolume: srcVolume, + SrcPath: srcPath, + DstPath: dstPath, + DstVolume: dstVolume, + FI: fi, + Opts: opts, + } + var resp *RenameDataResp + if fi.Data == nil { + resp, err = storageRenameDataRPC.Call(ctx, client.gridConn, ¶ms) + } else { + resp, err = storageRenameDataInlineRPC.Call(ctx, client.gridConn, &RenameDataInlineHandlerParams{params}) + } + if err != nil { + return res, toStorageErr(err) + } + + defer storageRenameDataRPC.PutResponse(resp) + return *resp, nil +} + +// where we keep old *Readers +var readMsgpReaderPool = bpool.Pool[*msgp.Reader]{New: func() *msgp.Reader { return &msgp.Reader{} }} + +// mspNewReader returns a *Reader that reads from the provided reader. +// The reader will be buffered. +// Return with readMsgpReaderPoolPut when done. +func msgpNewReader(r io.Reader) *msgp.Reader { + p := readMsgpReaderPool.Get() + if p.R == nil { + p.R = xbufio.NewReaderSize(r, 32<<10) + } else { + p.R.Reset(r) + } + return p +} + +// readMsgpReaderPoolPut can be used to reuse a *msgp.Reader. +func readMsgpReaderPoolPut(r *msgp.Reader) { + if r != nil { + readMsgpReaderPool.Put(r) + } +} + +func (client *storageRESTClient) ReadVersion(ctx context.Context, origvolume, volume, path, versionID string, opts ReadOptions) (fi FileInfo, err error) { + ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout()) + defer cancel() + + // Use websocket when not reading data. + if !opts.ReadData { + resp, err := storageReadVersionRPC.Call(ctx, client.gridConn, grid.NewMSSWith(map[string]string{ + storageRESTDiskID: *client.diskID.Load(), + storageRESTOrigVolume: origvolume, + storageRESTVolume: volume, + storageRESTFilePath: path, + storageRESTVersionID: versionID, + storageRESTInclFreeVersions: strconv.FormatBool(opts.InclFreeVersions), + storageRESTHealing: strconv.FormatBool(opts.Healing), + })) + if err != nil { + return fi, toStorageErr(err) + } + return *resp, nil + } + + values := make(url.Values) + values.Set(storageRESTOrigVolume, origvolume) + values.Set(storageRESTVolume, volume) + values.Set(storageRESTFilePath, path) + values.Set(storageRESTVersionID, versionID) + values.Set(storageRESTInclFreeVersions, strconv.FormatBool(opts.InclFreeVersions)) + values.Set(storageRESTHealing, strconv.FormatBool(opts.Healing)) + + respBody, err := client.callGet(ctx, storageRESTMethodReadVersion, values, nil, -1) + if err != nil { + return fi, err + } + defer xhttp.DrainBody(respBody) + + dec := msgpNewReader(respBody) + defer readMsgpReaderPoolPut(dec) + + err = fi.DecodeMsg(dec) + return fi, err +} + +// ReadXL - reads all contents of xl.meta of a file. +func (client *storageRESTClient) ReadXL(ctx context.Context, volume string, path string, readData bool) (rf RawFileInfo, err error) { + ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout()) + defer cancel() + + // Use websocket when not reading data. + if !readData { + resp, err := storageReadXLRPC.Call(ctx, client.gridConn, grid.NewMSSWith(map[string]string{ + storageRESTDiskID: *client.diskID.Load(), + storageRESTVolume: volume, + storageRESTFilePath: path, + })) + if err != nil { + return rf, toStorageErr(err) + } + return *resp, nil + } + + values := make(url.Values) + values.Set(storageRESTVolume, volume) + values.Set(storageRESTFilePath, path) + + respBody, err := client.callGet(ctx, storageRESTMethodReadXL, values, nil, -1) + if err != nil { + return rf, toStorageErr(err) + } + defer xhttp.DrainBody(respBody) + + dec := msgpNewReader(respBody) + defer readMsgpReaderPoolPut(dec) + + err = rf.DecodeMsg(dec) + return rf, err +} + +// ReadAll - reads all contents of a file. +func (client *storageRESTClient) ReadAll(ctx context.Context, volume string, path string) ([]byte, error) { + ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout()) + defer cancel() + + gridBytes, err := storageReadAllRPC.Call(ctx, client.gridConn, &ReadAllHandlerParams{ + DiskID: *client.diskID.Load(), + Volume: volume, + FilePath: path, + }) + if err != nil { + return nil, toStorageErr(err) + } + + return *gridBytes, nil +} + +// ReadFileStream - returns a reader for the requested file. +func (client *storageRESTClient) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) { + values := make(url.Values) + values.Set(storageRESTVolume, volume) + values.Set(storageRESTFilePath, path) + values.Set(storageRESTOffset, strconv.Itoa(int(offset))) + values.Set(storageRESTLength, strconv.Itoa(int(length))) + + respBody, err := client.callGet(ctx, storageRESTMethodReadFileStream, values, nil, -1) + if err != nil { + return nil, toStorageErr(err) + } + return respBody, nil +} + +// ReadFile - reads section of a file. +func (client *storageRESTClient) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (int64, error) { + values := make(url.Values) + values.Set(storageRESTVolume, volume) + values.Set(storageRESTFilePath, path) + values.Set(storageRESTOffset, strconv.Itoa(int(offset))) + values.Set(storageRESTLength, strconv.Itoa(len(buf))) + if verifier != nil { + values.Set(storageRESTBitrotAlgo, verifier.algorithm.String()) + values.Set(storageRESTBitrotHash, hex.EncodeToString(verifier.sum)) + } else { + values.Set(storageRESTBitrotAlgo, "") + values.Set(storageRESTBitrotHash, "") + } + respBody, err := client.callGet(ctx, storageRESTMethodReadFile, values, nil, -1) + if err != nil { + return 0, err + } + defer xhttp.DrainBody(respBody) + n, err := io.ReadFull(respBody, buf) + return int64(n), toStorageErr(err) +} + +// ListDir - lists a directory. +func (client *storageRESTClient) ListDir(ctx context.Context, origvolume, volume, dirPath string, count int) (entries []string, err error) { + values := grid.NewMSS() + values.Set(storageRESTVolume, volume) + values.Set(storageRESTDirPath, dirPath) + values.Set(storageRESTCount, strconv.Itoa(count)) + values.Set(storageRESTOrigVolume, origvolume) + values.Set(storageRESTDiskID, *client.diskID.Load()) + + st, err := storageListDirRPC.Call(ctx, client.gridConn, values) + if err != nil { + return nil, toStorageErr(err) + } + err = st.Results(func(resp *ListDirResult) error { + entries = resp.Entries + return nil + }) + return entries, toStorageErr(err) +} + +// DeleteFile - deletes a file. +func (client *storageRESTClient) Delete(ctx context.Context, volume string, path string, deleteOpts DeleteOptions) error { + if !deleteOpts.Immediate { + // add deadlines for all non-immediate purges + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout()) + defer cancel() + } + + _, err := storageDeleteFileRPC.Call(ctx, client.gridConn, &DeleteFileHandlerParams{ + DiskID: *client.diskID.Load(), + Volume: volume, + FilePath: path, + Opts: deleteOpts, + }) + return toStorageErr(err) +} + +// DeleteVersions - deletes list of specified versions if present +func (client *storageRESTClient) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions, opts DeleteOptions) (errs []error) { + if len(versions) == 0 { + return errs + } + + values := make(url.Values) + values.Set(storageRESTVolume, volume) + values.Set(storageRESTTotalVersions, strconv.Itoa(len(versions))) + + var buffer bytes.Buffer + encoder := msgp.NewWriter(&buffer) + for _, version := range versions { + version.EncodeMsg(encoder) + } + storageLogIf(ctx, encoder.Flush()) + + errs = make([]error, len(versions)) + + respBody, err := client.call(ctx, storageRESTMethodDeleteVersions, values, &buffer, -1) + defer xhttp.DrainBody(respBody) + if err != nil { + if contextCanceled(ctx) { + err = ctx.Err() + } + for i := range errs { + errs[i] = err + } + return errs + } + + reader, err := waitForHTTPResponse(respBody) + if err != nil { + for i := range errs { + errs[i] = toStorageErr(err) + } + return errs + } + + dErrResp := &DeleteVersionsErrsResp{} + decoder := msgpNewReader(reader) + defer readMsgpReaderPoolPut(decoder) + if err = dErrResp.DecodeMsg(decoder); err != nil { + for i := range errs { + errs[i] = toStorageErr(err) + } + return errs + } + + for i, dErr := range dErrResp.Errs { + if dErr != "" { + errs[i] = toStorageErr(errors.New(dErr)) + } else { + errs[i] = nil + } + } + + return errs +} + +// RenamePart - renames multipart part file +func (client *storageRESTClient) RenamePart(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string, meta []byte, skipParent string) (err error) { + ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout()) + defer cancel() + + _, err = storageRenamePartRPC.Call(ctx, client.gridConn, &RenamePartHandlerParams{ + DiskID: *client.diskID.Load(), + SrcVolume: srcVolume, + SrcFilePath: srcPath, + DstVolume: dstVolume, + DstFilePath: dstPath, + Meta: meta, + SkipParent: skipParent, + }) + return toStorageErr(err) +} + +// ReadParts - reads various part.N.meta paths from a drive remotely and returns object part info for each of those part.N.meta if found +func (client *storageRESTClient) ReadParts(ctx context.Context, volume string, partMetaPaths ...string) ([]*ObjectPartInfo, error) { + values := make(url.Values) + values.Set(storageRESTVolume, volume) + + rp := &ReadPartsReq{Paths: partMetaPaths} + buf, err := rp.MarshalMsg(nil) + if err != nil { + return nil, err + } + + respBody, err := client.call(ctx, storageRESTMethodReadParts, values, bytes.NewReader(buf), -1) + defer xhttp.DrainBody(respBody) + if err != nil { + return nil, err + } + + respReader, err := waitForHTTPResponse(respBody) + if err != nil { + return nil, toStorageErr(err) + } + + rd := msgpNewReader(respReader) + defer readMsgpReaderPoolPut(rd) + + readPartsResp := &ReadPartsResp{} + if err = readPartsResp.DecodeMsg(rd); err != nil { + return nil, toStorageErr(err) + } + + return readPartsResp.Infos, nil +} + +// RenameFile - renames a file. +func (client *storageRESTClient) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) (err error) { + ctx, cancel := context.WithTimeout(ctx, globalDriveConfig.GetMaxTimeout()) + defer cancel() + + _, err = storageRenameFileRPC.Call(ctx, client.gridConn, &RenameFileHandlerParams{ + DiskID: *client.diskID.Load(), + SrcVolume: srcVolume, + SrcFilePath: srcPath, + DstVolume: dstVolume, + DstFilePath: dstPath, + }) + return toStorageErr(err) +} + +func (client *storageRESTClient) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error) { + values := make(url.Values) + values.Set(storageRESTVolume, volume) + values.Set(storageRESTFilePath, path) + + var reader bytes.Buffer + if err := msgp.Encode(&reader, &fi); err != nil { + return nil, err + } + + respBody, err := client.call(ctx, storageRESTMethodVerifyFile, values, &reader, -1) + defer xhttp.DrainBody(respBody) + if err != nil { + return nil, err + } + + respReader, err := waitForHTTPResponse(respBody) + if err != nil { + return nil, toStorageErr(err) + } + + dec := msgpNewReader(respReader) + defer readMsgpReaderPoolPut(dec) + + verifyResp := CheckPartsResp{} + err = verifyResp.DecodeMsg(dec) + if err != nil { + return nil, toStorageErr(err) + } + + return &verifyResp, nil +} + +func (client *storageRESTClient) DeleteBulk(ctx context.Context, volume string, paths ...string) (err error) { + values := make(url.Values) + values.Set(storageRESTVolume, volume) + + req := &DeleteBulkReq{Paths: paths} + body, err := req.MarshalMsg(nil) + if err != nil { + return err + } + + respBody, err := client.call(ctx, storageRESTMethodDeleteBulk, values, bytes.NewReader(body), int64(len(body))) + if err != nil { + return err + } + defer xhttp.DrainBody(respBody) + + _, err = waitForHTTPResponse(respBody) + return toStorageErr(err) +} + +func (client *storageRESTClient) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) { + values := make(url.Values) + values.Set(storageRESTVolume, volume) + values.Set(storageRESTFilePath, path) + values.Set(storageRESTGlob, strconv.FormatBool(glob)) + respBody, err := client.call(ctx, storageRESTMethodStatInfoFile, values, nil, -1) + if err != nil { + return stat, err + } + defer xhttp.DrainBody(respBody) + respReader, err := waitForHTTPResponse(respBody) + if err != nil { + return stat, toStorageErr(err) + } + rd := msgpNewReader(respReader) + defer readMsgpReaderPoolPut(rd) + + for { + var st StatInfo + err = st.DecodeMsg(rd) + if err != nil { + if errors.Is(err, io.EOF) { + err = nil + } + break + } + + stat = append(stat, st) + } + + return stat, toStorageErr(err) +} + +// ReadMultiple will read multiple files and send each back as response. +// Files are read and returned in the given order. +// The resp channel is closed before the call returns. +// Only a canceled context or network errors returns an error. +func (client *storageRESTClient) ReadMultiple(ctx context.Context, req ReadMultipleReq, resp chan<- ReadMultipleResp) error { + defer xioutil.SafeClose(resp) + body, err := req.MarshalMsg(nil) + if err != nil { + return err + } + respBody, err := client.call(ctx, storageRESTMethodReadMultiple, nil, bytes.NewReader(body), int64(len(body))) + if err != nil { + return err + } + defer xhttp.DrainBody(respBody) + + pr, pw := io.Pipe() + go func() { + pw.CloseWithError(waitForHTTPStream(respBody, xioutil.NewDeadlineWriter(pw, globalDriveConfig.GetMaxTimeout()))) + }() + mr := msgp.NewReader(pr) + defer readMsgpReaderPoolPut(mr) + for { + var file ReadMultipleResp + if err := file.DecodeMsg(mr); err != nil { + if errors.Is(err, io.EOF) { + err = nil + } + pr.CloseWithError(err) + return toStorageErr(err) + } + select { + case <-ctx.Done(): + return ctx.Err() + case resp <- file: + } + } +} + +// CleanAbandonedData will read metadata of the object on disk +// and delete any data directories and inline data that isn't referenced in metadata. +func (client *storageRESTClient) CleanAbandonedData(ctx context.Context, volume string, path string) error { + values := make(url.Values) + values.Set(storageRESTVolume, volume) + values.Set(storageRESTFilePath, path) + respBody, err := client.call(ctx, storageRESTMethodCleanAbandoned, values, nil, -1) + if err != nil { + return err + } + defer xhttp.DrainBody(respBody) + _, err = waitForHTTPResponse(respBody) + return toStorageErr(err) +} + +// Close - marks the client as closed. +func (client *storageRESTClient) Close() error { + client.restClient.Close() + return nil +} + +var emptyDiskID = "" + +// Returns a storage rest client. +func newStorageRESTClient(endpoint Endpoint, healthCheck bool, gm *grid.Manager) (*storageRESTClient, error) { + serverURL := &url.URL{ + Scheme: endpoint.Scheme, + Host: endpoint.Host, + Path: path.Join(storageRESTPrefix, endpoint.Path, storageRESTVersion), + } + + restClient := rest.NewClient(serverURL, globalInternodeTransport, newCachedAuthToken()) + if healthCheck { + // Use a separate client to avoid recursive calls. + healthClient := rest.NewClient(serverURL, globalInternodeTransport, newCachedAuthToken()) + healthClient.NoMetrics = true + restClient.HealthCheckFn = func() bool { + ctx, cancel := context.WithTimeout(context.Background(), restClient.HealthCheckTimeout) + defer cancel() + respBody, err := healthClient.Call(ctx, storageRESTMethodHealth, nil, nil, -1) + xhttp.DrainBody(respBody) + return toStorageErr(err) != errDiskNotFound + } + } + conn := gm.Connection(endpoint.GridHost()).Subroute(endpoint.Path) + if conn == nil { + return nil, fmt.Errorf("unable to find connection for %s in targets: %v", endpoint.GridHost(), gm.Targets()) + } + client := &storageRESTClient{ + endpoint: endpoint, + restClient: restClient, + gridConn: conn, + diskInfoCache: cachevalue.New[DiskInfo](), + } + client.SetDiskID(emptyDiskID) + return client, nil +} diff --git a/cmd/storage-rest-common.go b/cmd/storage-rest-common.go new file mode 100644 index 00000000..361045de --- /dev/null +++ b/cmd/storage-rest-common.go @@ -0,0 +1,86 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +//go:generate msgp -file $GOFILE -unexported + +const ( + storageRESTVersion = "v63" // Introduce RenamePart and ReadParts API + storageRESTVersionPrefix = SlashSeparator + storageRESTVersion + storageRESTPrefix = minioReservedBucketPath + "/storage" +) + +const ( + storageRESTMethodHealth = "/health" + + storageRESTMethodAppendFile = "/afile" + storageRESTMethodCreateFile = "/cfile" + storageRESTMethodWriteAll = "/wall" + storageRESTMethodReadVersion = "/rver" + storageRESTMethodReadXL = "/rxl" + storageRESTMethodReadAll = "/rall" + storageRESTMethodReadFile = "/rfile" + storageRESTMethodReadFileStream = "/rfilest" + storageRESTMethodListDir = "/ls" + storageRESTMethodDeleteVersions = "/dvers" + storageRESTMethodRenameFile = "/rfile" + storageRESTMethodVerifyFile = "/vfile" + storageRESTMethodStatInfoFile = "/sfile" + storageRESTMethodReadMultiple = "/rmpl" + storageRESTMethodCleanAbandoned = "/cln" + storageRESTMethodDeleteBulk = "/dblk" + storageRESTMethodReadParts = "/rps" +) + +const ( + storageRESTVolume = "vol" + storageRESTVolumes = "vols" + storageRESTDirPath = "dpath" + storageRESTFilePath = "fp" + storageRESTVersionID = "vid" + storageRESTHealing = "heal" + storageRESTTotalVersions = "tvers" + storageRESTSrcVolume = "svol" + storageRESTSrcPath = "spath" + storageRESTDstVolume = "dvol" + storageRESTDstPath = "dpath" + storageRESTOffset = "offset" + storageRESTLength = "length" + storageRESTCount = "count" + storageRESTBitrotAlgo = "balg" + storageRESTBitrotHash = "bhash" + storageRESTDiskID = "did" + storageRESTForceDelete = "fdel" + storageRESTGlob = "glob" + storageRESTMetrics = "metrics" + storageRESTDriveQuorum = "dquorum" + storageRESTOrigVolume = "ovol" + storageRESTInclFreeVersions = "incl-fv" + storageRESTRange = "rng" +) + +type nsScannerOptions struct { + DiskID string `msg:"id"` + ScanMode int `msg:"m"` + Cache *dataUsageCache `msg:"c"` +} + +type nsScannerResp struct { + Update *dataUsageEntry `msg:"u"` + Final *dataUsageCache `msg:"f"` +} diff --git a/cmd/storage-rest-common_gen.go b/cmd/storage-rest-common_gen.go new file mode 100644 index 00000000..c29615c7 --- /dev/null +++ b/cmd/storage-rest-common_gen.go @@ -0,0 +1,418 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *nsScannerOptions) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.DiskID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + case "m": + z.ScanMode, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "ScanMode") + return + } + case "c": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + z.Cache = nil + } else { + if z.Cache == nil { + z.Cache = new(dataUsageCache) + } + err = z.Cache.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *nsScannerOptions) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "id" + err = en.Append(0x83, 0xa2, 0x69, 0x64) + if err != nil { + return + } + err = en.WriteString(z.DiskID) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + // write "m" + err = en.Append(0xa1, 0x6d) + if err != nil { + return + } + err = en.WriteInt(z.ScanMode) + if err != nil { + err = msgp.WrapError(err, "ScanMode") + return + } + // write "c" + err = en.Append(0xa1, 0x63) + if err != nil { + return + } + if z.Cache == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.Cache.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *nsScannerOptions) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "id" + o = append(o, 0x83, 0xa2, 0x69, 0x64) + o = msgp.AppendString(o, z.DiskID) + // string "m" + o = append(o, 0xa1, 0x6d) + o = msgp.AppendInt(o, z.ScanMode) + // string "c" + o = append(o, 0xa1, 0x63) + if z.Cache == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.Cache.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *nsScannerOptions) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "id": + z.DiskID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DiskID") + return + } + case "m": + z.ScanMode, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ScanMode") + return + } + case "c": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.Cache = nil + } else { + if z.Cache == nil { + z.Cache = new(dataUsageCache) + } + bts, err = z.Cache.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Cache") + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *nsScannerOptions) Msgsize() (s int) { + s = 1 + 3 + msgp.StringPrefixSize + len(z.DiskID) + 2 + msgp.IntSize + 2 + if z.Cache == nil { + s += msgp.NilSize + } else { + s += z.Cache.Msgsize() + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *nsScannerResp) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "u": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "Update") + return + } + z.Update = nil + } else { + if z.Update == nil { + z.Update = new(dataUsageEntry) + } + err = z.Update.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Update") + return + } + } + case "f": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "Final") + return + } + z.Final = nil + } else { + if z.Final == nil { + z.Final = new(dataUsageCache) + } + err = z.Final.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Final") + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *nsScannerResp) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "u" + err = en.Append(0x82, 0xa1, 0x75) + if err != nil { + return + } + if z.Update == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.Update.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Update") + return + } + } + // write "f" + err = en.Append(0xa1, 0x66) + if err != nil { + return + } + if z.Final == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.Final.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Final") + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *nsScannerResp) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "u" + o = append(o, 0x82, 0xa1, 0x75) + if z.Update == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.Update.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Update") + return + } + } + // string "f" + o = append(o, 0xa1, 0x66) + if z.Final == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.Final.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Final") + return + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *nsScannerResp) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "u": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.Update = nil + } else { + if z.Update == nil { + z.Update = new(dataUsageEntry) + } + bts, err = z.Update.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Update") + return + } + } + case "f": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.Final = nil + } else { + if z.Final == nil { + z.Final = new(dataUsageCache) + } + bts, err = z.Final.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Final") + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *nsScannerResp) Msgsize() (s int) { + s = 1 + 2 + if z.Update == nil { + s += msgp.NilSize + } else { + s += z.Update.Msgsize() + } + s += 2 + if z.Final == nil { + s += msgp.NilSize + } else { + s += z.Final.Msgsize() + } + return +} diff --git a/cmd/storage-rest-common_gen_test.go b/cmd/storage-rest-common_gen_test.go new file mode 100644 index 00000000..4164020a --- /dev/null +++ b/cmd/storage-rest-common_gen_test.go @@ -0,0 +1,236 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalnsScannerOptions(t *testing.T) { + v := nsScannerOptions{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgnsScannerOptions(b *testing.B) { + v := nsScannerOptions{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgnsScannerOptions(b *testing.B) { + v := nsScannerOptions{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalnsScannerOptions(b *testing.B) { + v := nsScannerOptions{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodensScannerOptions(t *testing.T) { + v := nsScannerOptions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodensScannerOptions Msgsize() is inaccurate") + } + + vn := nsScannerOptions{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodensScannerOptions(b *testing.B) { + v := nsScannerOptions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodensScannerOptions(b *testing.B) { + v := nsScannerOptions{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalnsScannerResp(t *testing.T) { + v := nsScannerResp{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgnsScannerResp(b *testing.B) { + v := nsScannerResp{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgnsScannerResp(b *testing.B) { + v := nsScannerResp{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalnsScannerResp(b *testing.B) { + v := nsScannerResp{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodensScannerResp(t *testing.T) { + v := nsScannerResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodensScannerResp Msgsize() is inaccurate") + } + + vn := nsScannerResp{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodensScannerResp(b *testing.B) { + v := nsScannerResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodensScannerResp(b *testing.B) { + v := nsScannerResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/storage-rest-server.go b/cmd/storage-rest-server.go new file mode 100644 index 00000000..927571b3 --- /dev/null +++ b/cmd/storage-rest-server.go @@ -0,0 +1,1438 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bufio" + "context" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "os/user" + "path" + "runtime/debug" + "strconv" + "strings" + "sync" + "time" + + "github.com/minio/minio/internal/bpool" + "github.com/minio/minio/internal/grid" + "github.com/tinylib/msgp/msgp" + + jwtreq "github.com/golang-jwt/jwt/v4/request" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/config" + xhttp "github.com/minio/minio/internal/http" + xioutil "github.com/minio/minio/internal/ioutil" + xjwt "github.com/minio/minio/internal/jwt" + "github.com/minio/minio/internal/logger" + "github.com/minio/mux" + xnet "github.com/minio/pkg/v3/net" +) + +var errDiskStale = errors.New("drive stale") + +// To abstract a disk over network. +type storageRESTServer struct { + endpoint Endpoint +} + +var ( + storageCheckPartsRPC = grid.NewStream[*CheckPartsHandlerParams, grid.NoPayload, *CheckPartsResp](grid.HandlerCheckParts3, func() *CheckPartsHandlerParams { return &CheckPartsHandlerParams{} }, nil, func() *CheckPartsResp { return &CheckPartsResp{} }) + storageDeleteFileRPC = grid.NewSingleHandler[*DeleteFileHandlerParams, grid.NoPayload](grid.HandlerDeleteFile, func() *DeleteFileHandlerParams { return &DeleteFileHandlerParams{} }, grid.NewNoPayload).AllowCallRequestPool(true) + storageDeleteVersionRPC = grid.NewSingleHandler[*DeleteVersionHandlerParams, grid.NoPayload](grid.HandlerDeleteVersion, func() *DeleteVersionHandlerParams { return &DeleteVersionHandlerParams{} }, grid.NewNoPayload) + storageDiskInfoRPC = grid.NewSingleHandler[*DiskInfoOptions, *DiskInfo](grid.HandlerDiskInfo, func() *DiskInfoOptions { return &DiskInfoOptions{} }, func() *DiskInfo { return &DiskInfo{} }).WithSharedResponse().AllowCallRequestPool(true) + storageNSScannerRPC = grid.NewStream[*nsScannerOptions, grid.NoPayload, *nsScannerResp](grid.HandlerNSScanner, func() *nsScannerOptions { return &nsScannerOptions{} }, nil, func() *nsScannerResp { return &nsScannerResp{} }) + storageReadAllRPC = grid.NewSingleHandler[*ReadAllHandlerParams, *grid.Bytes](grid.HandlerReadAll, func() *ReadAllHandlerParams { return &ReadAllHandlerParams{} }, grid.NewBytes).AllowCallRequestPool(true) + storageWriteAllRPC = grid.NewSingleHandler[*WriteAllHandlerParams, grid.NoPayload](grid.HandlerWriteAll, func() *WriteAllHandlerParams { return &WriteAllHandlerParams{} }, grid.NewNoPayload) + storageReadVersionRPC = grid.NewSingleHandler[*grid.MSS, *FileInfo](grid.HandlerReadVersion, grid.NewMSS, func() *FileInfo { return &FileInfo{} }) + storageReadXLRPC = grid.NewSingleHandler[*grid.MSS, *RawFileInfo](grid.HandlerReadXL, grid.NewMSS, func() *RawFileInfo { return &RawFileInfo{} }) + storageRenameDataRPC = grid.NewSingleHandler[*RenameDataHandlerParams, *RenameDataResp](grid.HandlerRenameData2, func() *RenameDataHandlerParams { return &RenameDataHandlerParams{} }, func() *RenameDataResp { return &RenameDataResp{} }) + storageRenameDataInlineRPC = grid.NewSingleHandler[*RenameDataInlineHandlerParams, *RenameDataResp](grid.HandlerRenameDataInline, newRenameDataInlineHandlerParams, func() *RenameDataResp { return &RenameDataResp{} }).AllowCallRequestPool(false) + storageRenameFileRPC = grid.NewSingleHandler[*RenameFileHandlerParams, grid.NoPayload](grid.HandlerRenameFile, func() *RenameFileHandlerParams { return &RenameFileHandlerParams{} }, grid.NewNoPayload).AllowCallRequestPool(true) + storageRenamePartRPC = grid.NewSingleHandler[*RenamePartHandlerParams, grid.NoPayload](grid.HandlerRenamePart, func() *RenamePartHandlerParams { return &RenamePartHandlerParams{} }, grid.NewNoPayload) + storageStatVolRPC = grid.NewSingleHandler[*grid.MSS, *VolInfo](grid.HandlerStatVol, grid.NewMSS, func() *VolInfo { return &VolInfo{} }) + storageUpdateMetadataRPC = grid.NewSingleHandler[*MetadataHandlerParams, grid.NoPayload](grid.HandlerUpdateMetadata, func() *MetadataHandlerParams { return &MetadataHandlerParams{} }, grid.NewNoPayload) + storageWriteMetadataRPC = grid.NewSingleHandler[*MetadataHandlerParams, grid.NoPayload](grid.HandlerWriteMetadata, func() *MetadataHandlerParams { return &MetadataHandlerParams{} }, grid.NewNoPayload) + storageListDirRPC = grid.NewStream[*grid.MSS, grid.NoPayload, *ListDirResult](grid.HandlerListDir, grid.NewMSS, nil, func() *ListDirResult { return &ListDirResult{} }).WithOutCapacity(1) +) + +func getStorageViaEndpoint(endpoint Endpoint) StorageAPI { + globalLocalDrivesMu.RLock() + defer globalLocalDrivesMu.RUnlock() + if len(globalLocalSetDrives) == 0 { + return globalLocalDrivesMap[endpoint.String()] + } + return globalLocalSetDrives[endpoint.PoolIdx][endpoint.SetIdx][endpoint.DiskIdx] +} + +func (s *storageRESTServer) getStorage() StorageAPI { + return getStorageViaEndpoint(s.endpoint) +} + +func (s *storageRESTServer) writeErrorResponse(w http.ResponseWriter, err error) { + err = unwrapAll(err) + switch err { + case errDiskStale: + w.WriteHeader(http.StatusPreconditionFailed) + case errFileNotFound, errFileVersionNotFound: + w.WriteHeader(http.StatusNotFound) + case errInvalidAccessKeyID, errAccessKeyDisabled, errNoAuthToken, errMalformedAuth, errAuthentication, errSkewedAuthTime: + w.WriteHeader(http.StatusUnauthorized) + case context.Canceled, context.DeadlineExceeded: + w.WriteHeader(499) + default: + w.WriteHeader(http.StatusForbidden) + } + w.Write([]byte(err.Error())) +} + +// DefaultSkewTime - skew time is 15 minutes between minio peers. +const DefaultSkewTime = 15 * time.Minute + +// validateStorageRequestToken will validate the token against the provided audience. +func validateStorageRequestToken(token string) error { + claims := xjwt.NewStandardClaims() + if err := xjwt.ParseWithStandardClaims(token, claims, []byte(globalActiveCred.SecretKey)); err != nil { + return errAuthentication + } + + owner := claims.AccessKey == globalActiveCred.AccessKey || claims.Subject == globalActiveCred.AccessKey + if !owner { + return errAuthentication + } + + return nil +} + +// Authenticates storage client's requests and validates for skewed time. +func storageServerRequestValidate(r *http.Request) error { + token, err := jwtreq.AuthorizationHeaderExtractor.ExtractToken(r) + if err != nil { + if err == jwtreq.ErrNoTokenInRequest { + return errNoAuthToken + } + return errMalformedAuth + } + + if err = validateStorageRequestToken(token); err != nil { + return err + } + + nanoTime, err := strconv.ParseInt(r.Header.Get("X-Minio-Time"), 10, 64) + if err != nil { + return errMalformedAuth + } + + localTime := UTCNow() + remoteTime := time.Unix(0, nanoTime) + + delta := remoteTime.Sub(localTime) + if delta < 0 { + delta *= -1 + } + + if delta > DefaultSkewTime { + return errSkewedAuthTime + } + + return nil +} + +// IsAuthValid - To authenticate and verify the time difference. +func (s *storageRESTServer) IsAuthValid(w http.ResponseWriter, r *http.Request) bool { + if s.getStorage() == nil { + s.writeErrorResponse(w, errDiskNotFound) + return false + } + + if err := storageServerRequestValidate(r); err != nil { + s.writeErrorResponse(w, err) + return false + } + + return true +} + +// IsValid - To authenticate and check if the disk-id in the request corresponds to the underlying disk. +func (s *storageRESTServer) IsValid(w http.ResponseWriter, r *http.Request) bool { + if !s.IsAuthValid(w, r) { + return false + } + + if err := r.ParseForm(); err != nil { + s.writeErrorResponse(w, err) + return false + } + + diskID := r.Form.Get(storageRESTDiskID) + if diskID == "" { + // Request sent empty disk-id, we allow the request + // as the peer might be coming up and trying to read format.json + // or create format.json + return true + } + + storedDiskID, err := s.getStorage().GetDiskID() + if err != nil { + s.writeErrorResponse(w, err) + return false + } + + if diskID != storedDiskID { + s.writeErrorResponse(w, errDiskStale) + return false + } + + // If format.json is available and request sent the right disk-id, we allow the request + return true +} + +// checkID - check if the disk-id in the request corresponds to the underlying disk. +func (s *storageRESTServer) checkID(wantID string) bool { + if s.getStorage() == nil { + return false + } + if wantID == "" { + // Request sent empty disk-id, we allow the request + // as the peer might be coming up and trying to read format.json + // or create format.json + return true + } + + storedDiskID, err := s.getStorage().GetDiskID() + if err != nil { + return false + } + + return wantID == storedDiskID +} + +// HealthHandler handler checks if disk is stale +func (s *storageRESTServer) HealthHandler(w http.ResponseWriter, r *http.Request) { + s.IsValid(w, r) +} + +// DiskInfoHandler - returns disk info. +func (s *storageRESTServer) DiskInfoHandler(opts *DiskInfoOptions) (*DiskInfo, *grid.RemoteErr) { + if !s.checkID(opts.DiskID) { + return nil, grid.NewRemoteErr(errDiskNotFound) + } + info, err := s.getStorage().DiskInfo(context.Background(), *opts) + if err != nil { + info.Error = err.Error() + } + return &info, nil +} + +func (s *storageRESTServer) NSScannerHandler(ctx context.Context, params *nsScannerOptions, out chan<- *nsScannerResp) *grid.RemoteErr { + if !s.checkID(params.DiskID) { + return grid.NewRemoteErr(errDiskNotFound) + } + if params.Cache == nil { + return grid.NewRemoteErrString("NSScannerHandler: provided cache is nil") + } + + // Collect updates, stream them before the full cache is sent. + updates := make(chan dataUsageEntry, 1) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for update := range updates { + resp := storageNSScannerRPC.NewResponse() + resp.Update = &update + out <- resp + } + }() + ui, err := s.getStorage().NSScanner(ctx, *params.Cache, updates, madmin.HealScanMode(params.ScanMode), nil) + wg.Wait() + if err != nil { + return grid.NewRemoteErr(err) + } + // Send final response. + resp := storageNSScannerRPC.NewResponse() + resp.Final = &ui + out <- resp + return nil +} + +// MakeVolHandler - make a volume. +func (s *storageRESTServer) MakeVolHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + return + } + volume := r.Form.Get(storageRESTVolume) + err := s.getStorage().MakeVol(r.Context(), volume) + if err != nil { + s.writeErrorResponse(w, err) + } +} + +// MakeVolBulkHandler - create multiple volumes as a bulk operation. +func (s *storageRESTServer) MakeVolBulkHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + return + } + volumes := strings.Split(r.Form.Get(storageRESTVolumes), ",") + err := s.getStorage().MakeVolBulk(r.Context(), volumes...) + if err != nil { + s.writeErrorResponse(w, err) + } +} + +// StatVolHandler - stat a volume. +func (s *storageRESTServer) StatVolHandler(params *grid.MSS) (*VolInfo, *grid.RemoteErr) { + if !s.checkID(params.Get(storageRESTDiskID)) { + return nil, grid.NewRemoteErr(errDiskNotFound) + } + info, err := s.getStorage().StatVol(context.Background(), params.Get(storageRESTVolume)) + if err != nil { + return nil, grid.NewRemoteErr(err) + } + return &info, nil +} + +// AppendFileHandler - append data from the request to the file specified. +func (s *storageRESTServer) AppendFileHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + return + } + volume := r.Form.Get(storageRESTVolume) + filePath := r.Form.Get(storageRESTFilePath) + + buf := make([]byte, r.ContentLength) + _, err := io.ReadFull(r.Body, buf) + if err != nil { + s.writeErrorResponse(w, err) + return + } + err = s.getStorage().AppendFile(r.Context(), volume, filePath, buf) + if err != nil { + s.writeErrorResponse(w, err) + } +} + +// CreateFileHandler - copy the contents from the request. +func (s *storageRESTServer) CreateFileHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + return + } + + volume := r.Form.Get(storageRESTVolume) + filePath := r.Form.Get(storageRESTFilePath) + origvolume := r.Form.Get(storageRESTOrigVolume) + + fileSizeStr := r.Form.Get(storageRESTLength) + fileSize, err := strconv.Atoi(fileSizeStr) + if err != nil { + s.writeErrorResponse(w, err) + return + } + + done, body := keepHTTPReqResponseAlive(w, r) + done(s.getStorage().CreateFile(r.Context(), origvolume, volume, filePath, int64(fileSize), body)) +} + +// DeleteVersionHandler delete updated metadata. +func (s *storageRESTServer) DeleteVersionHandler(p *DeleteVersionHandlerParams) (np grid.NoPayload, gerr *grid.RemoteErr) { + if !s.checkID(p.DiskID) { + return np, grid.NewRemoteErr(errDiskNotFound) + } + volume := p.Volume + filePath := p.FilePath + forceDelMarker := p.ForceDelMarker + + opts := DeleteOptions{} + err := s.getStorage().DeleteVersion(context.Background(), volume, filePath, p.FI, forceDelMarker, opts) + return np, grid.NewRemoteErr(err) +} + +// ReadVersionHandlerWS read metadata of versionID +func (s *storageRESTServer) ReadVersionHandlerWS(params *grid.MSS) (*FileInfo, *grid.RemoteErr) { + if !s.checkID(params.Get(storageRESTDiskID)) { + return nil, grid.NewRemoteErr(errDiskNotFound) + } + origvolume := params.Get(storageRESTOrigVolume) + volume := params.Get(storageRESTVolume) + filePath := params.Get(storageRESTFilePath) + versionID := params.Get(storageRESTVersionID) + + healing, err := strconv.ParseBool(params.Get(storageRESTHealing)) + if err != nil { + return nil, grid.NewRemoteErr(err) + } + + inclFreeVersions, err := strconv.ParseBool(params.Get(storageRESTInclFreeVersions)) + if err != nil { + return nil, grid.NewRemoteErr(err) + } + + fi, err := s.getStorage().ReadVersion(context.Background(), origvolume, volume, filePath, versionID, ReadOptions{ + InclFreeVersions: inclFreeVersions, + ReadData: false, + Healing: healing, + }) + if err != nil { + return nil, grid.NewRemoteErr(err) + } + return &fi, nil +} + +// ReadVersionHandler read metadata of versionID +func (s *storageRESTServer) ReadVersionHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + return + } + origvolume := r.Form.Get(storageRESTOrigVolume) + volume := r.Form.Get(storageRESTVolume) + filePath := r.Form.Get(storageRESTFilePath) + versionID := r.Form.Get(storageRESTVersionID) + healing, err := strconv.ParseBool(r.Form.Get(storageRESTHealing)) + if err != nil { + s.writeErrorResponse(w, err) + return + } + + inclFreeVersions, err := strconv.ParseBool(r.Form.Get(storageRESTInclFreeVersions)) + if err != nil { + s.writeErrorResponse(w, err) + return + } + + fi, err := s.getStorage().ReadVersion(r.Context(), origvolume, volume, filePath, versionID, ReadOptions{ + InclFreeVersions: inclFreeVersions, + ReadData: true, + Healing: healing, + }) + if err != nil { + s.writeErrorResponse(w, err) + return + } + + storageLogIf(r.Context(), msgp.Encode(w, &fi)) +} + +// WriteMetadataHandler rpc handler to write new updated metadata. +func (s *storageRESTServer) WriteMetadataHandler(p *MetadataHandlerParams) (np grid.NoPayload, gerr *grid.RemoteErr) { + if !s.checkID(p.DiskID) { + return grid.NewNPErr(errDiskNotFound) + } + + volume := p.Volume + filePath := p.FilePath + origvolume := p.OrigVolume + + err := s.getStorage().WriteMetadata(context.Background(), origvolume, volume, filePath, p.FI) + return np, grid.NewRemoteErr(err) +} + +// UpdateMetadataHandler update new updated metadata. +func (s *storageRESTServer) UpdateMetadataHandler(p *MetadataHandlerParams) (grid.NoPayload, *grid.RemoteErr) { + if !s.checkID(p.DiskID) { + return grid.NewNPErr(errDiskNotFound) + } + volume := p.Volume + filePath := p.FilePath + + return grid.NewNPErr(s.getStorage().UpdateMetadata(context.Background(), volume, filePath, p.FI, p.UpdateOpts)) +} + +// CheckPartsHandler - check if a file parts exists. +func (s *storageRESTServer) CheckPartsHandler(ctx context.Context, p *CheckPartsHandlerParams, out chan<- *CheckPartsResp) *grid.RemoteErr { + if !s.checkID(p.DiskID) { + return grid.NewRemoteErr(errDiskNotFound) + } + volume := p.Volume + filePath := p.FilePath + + resp, err := s.getStorage().CheckParts(ctx, volume, filePath, p.FI) + if err != nil { + return grid.NewRemoteErr(err) + } + out <- resp + return grid.NewRemoteErr(err) +} + +func (s *storageRESTServer) WriteAllHandler(p *WriteAllHandlerParams) (grid.NoPayload, *grid.RemoteErr) { + if !s.checkID(p.DiskID) { + return grid.NewNPErr(errDiskNotFound) + } + + volume := p.Volume + filePath := p.FilePath + + return grid.NewNPErr(s.getStorage().WriteAll(context.Background(), volume, filePath, p.Buf)) +} + +// ReadAllHandler - read all the contents of a file. +func (s *storageRESTServer) ReadAllHandler(p *ReadAllHandlerParams) (*grid.Bytes, *grid.RemoteErr) { + if !s.checkID(p.DiskID) { + return nil, grid.NewRemoteErr(errDiskNotFound) + } + + volume := p.Volume + filePath := p.FilePath + + buf, err := s.getStorage().ReadAll(context.Background(), volume, filePath) + return grid.NewBytesWith(buf), grid.NewRemoteErr(err) +} + +// ReadXLHandler - read xl.meta for an object at path. +func (s *storageRESTServer) ReadXLHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + return + } + + volume := r.Form.Get(storageRESTVolume) + filePath := r.Form.Get(storageRESTFilePath) + + rf, err := s.getStorage().ReadXL(r.Context(), volume, filePath, true) + if err != nil { + s.writeErrorResponse(w, err) + return + } + + storageLogIf(r.Context(), msgp.Encode(w, &rf)) +} + +// ReadXLHandlerWS - read xl.meta for an object at path. +func (s *storageRESTServer) ReadXLHandlerWS(params *grid.MSS) (*RawFileInfo, *grid.RemoteErr) { + if !s.checkID(params.Get(storageRESTDiskID)) { + return nil, grid.NewRemoteErr(errDiskNotFound) + } + + volume := params.Get(storageRESTVolume) + filePath := params.Get(storageRESTFilePath) + rf, err := s.getStorage().ReadXL(context.Background(), volume, filePath, false) + if err != nil { + return nil, grid.NewRemoteErr(err) + } + + return &rf, nil +} + +// ReadPartsHandler - read section of a file. +func (s *storageRESTServer) ReadPartsHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + return + } + volume := r.Form.Get(storageRESTVolume) + + var preq ReadPartsReq + if err := msgp.Decode(r.Body, &preq); err != nil { + s.writeErrorResponse(w, err) + return + } + + done := keepHTTPResponseAlive(w) + infos, err := s.getStorage().ReadParts(r.Context(), volume, preq.Paths...) + done(nil) + if err != nil { + s.writeErrorResponse(w, err) + return + } + + presp := &ReadPartsResp{Infos: infos} + storageLogIf(r.Context(), msgp.Encode(w, presp)) +} + +// ReadFileHandler - read section of a file. +func (s *storageRESTServer) ReadFileHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + return + } + volume := r.Form.Get(storageRESTVolume) + filePath := r.Form.Get(storageRESTFilePath) + offset, err := strconv.Atoi(r.Form.Get(storageRESTOffset)) + if err != nil { + s.writeErrorResponse(w, err) + return + } + length, err := strconv.Atoi(r.Form.Get(storageRESTLength)) + if err != nil { + s.writeErrorResponse(w, err) + return + } + if offset < 0 || length < 0 { + s.writeErrorResponse(w, errInvalidArgument) + return + } + var verifier *BitrotVerifier + if r.Form.Get(storageRESTBitrotAlgo) != "" { + hashStr := r.Form.Get(storageRESTBitrotHash) + var hash []byte + hash, err = hex.DecodeString(hashStr) + if err != nil { + s.writeErrorResponse(w, err) + return + } + verifier = NewBitrotVerifier(BitrotAlgorithmFromString(r.Form.Get(storageRESTBitrotAlgo)), hash) + } + buf := make([]byte, length) + defer metaDataPoolPut(buf) // Reuse if we can. + _, err = s.getStorage().ReadFile(r.Context(), volume, filePath, int64(offset), buf, verifier) + if err != nil { + s.writeErrorResponse(w, err) + return + } + w.Header().Set(xhttp.ContentLength, strconv.Itoa(len(buf))) + w.Write(buf) +} + +// ReadFileStreamHandler - read section of a file. +func (s *storageRESTServer) ReadFileStreamHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + return + } + volume := r.Form.Get(storageRESTVolume) + filePath := r.Form.Get(storageRESTFilePath) + offset, err := strconv.ParseInt(r.Form.Get(storageRESTOffset), 10, 64) + if err != nil { + s.writeErrorResponse(w, err) + return + } + length, err := strconv.ParseInt(r.Form.Get(storageRESTLength), 10, 64) + if err != nil { + s.writeErrorResponse(w, err) + return + } + + rc, err := s.getStorage().ReadFileStream(r.Context(), volume, filePath, offset, length) + if err != nil { + s.writeErrorResponse(w, err) + return + } + defer rc.Close() + + _, err = xioutil.Copy(w, rc) + if !xnet.IsNetworkOrHostDown(err, true) { // do not need to log disconnected clients + storageLogIf(r.Context(), err) + } +} + +// ListDirHandler - list a directory. +func (s *storageRESTServer) ListDirHandler(ctx context.Context, params *grid.MSS, out chan<- *ListDirResult) *grid.RemoteErr { + if !s.checkID(params.Get(storageRESTDiskID)) { + return grid.NewRemoteErr(errDiskNotFound) + } + volume := params.Get(storageRESTVolume) + dirPath := params.Get(storageRESTDirPath) + origvolume := params.Get(storageRESTOrigVolume) + count, err := strconv.Atoi(params.Get(storageRESTCount)) + if err != nil { + return grid.NewRemoteErr(err) + } + + entries, err := s.getStorage().ListDir(ctx, origvolume, volume, dirPath, count) + if err != nil { + return grid.NewRemoteErr(err) + } + out <- &ListDirResult{Entries: entries} + return nil +} + +// DeleteFileHandler - delete a file. +func (s *storageRESTServer) DeleteFileHandler(p *DeleteFileHandlerParams) (grid.NoPayload, *grid.RemoteErr) { + if !s.checkID(p.DiskID) { + return grid.NewNPErr(errDiskNotFound) + } + return grid.NewNPErr(s.getStorage().Delete(context.Background(), p.Volume, p.FilePath, p.Opts)) +} + +// DeleteVersionsHandler - delete a set of a versions. +func (s *storageRESTServer) DeleteVersionsHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + return + } + + volume := r.Form.Get(storageRESTVolume) + totalVersions, err := strconv.Atoi(r.Form.Get(storageRESTTotalVersions)) + if err != nil { + s.writeErrorResponse(w, err) + return + } + + versions := make([]FileInfoVersions, totalVersions) + decoder := msgpNewReader(r.Body) + defer readMsgpReaderPoolPut(decoder) + for i := range totalVersions { + dst := &versions[i] + if err := dst.DecodeMsg(decoder); err != nil { + s.writeErrorResponse(w, err) + return + } + } + + done := keepHTTPResponseAlive(w) + opts := DeleteOptions{} + errs := s.getStorage().DeleteVersions(r.Context(), volume, versions, opts) + done(nil) + + dErrsResp := &DeleteVersionsErrsResp{Errs: make([]string, totalVersions)} + for idx := range versions { + if errs[idx] != nil { + dErrsResp.Errs[idx] = errs[idx].Error() + } + } + + buf, _ := dErrsResp.MarshalMsg(nil) + w.Write(buf) +} + +// RenameDataHandler - renames a meta object and data dir to destination. +func (s *storageRESTServer) RenameDataHandler(p *RenameDataHandlerParams) (*RenameDataResp, *grid.RemoteErr) { + if !s.checkID(p.DiskID) { + return nil, grid.NewRemoteErr(errDiskNotFound) + } + + resp, err := s.getStorage().RenameData(context.Background(), p.SrcVolume, p.SrcPath, p.FI, p.DstVolume, p.DstPath, p.Opts) + return &resp, grid.NewRemoteErr(err) +} + +// RenameDataInlineHandler - renames a meta object and data dir to destination. +func (s *storageRESTServer) RenameDataInlineHandler(p *RenameDataInlineHandlerParams) (*RenameDataResp, *grid.RemoteErr) { + defer p.Recycle() + return s.RenameDataHandler(&p.RenameDataHandlerParams) +} + +// RenameFileHandler - rename a file from source to destination +func (s *storageRESTServer) RenameFileHandler(p *RenameFileHandlerParams) (grid.NoPayload, *grid.RemoteErr) { + if !s.checkID(p.DiskID) { + return grid.NewNPErr(errDiskNotFound) + } + return grid.NewNPErr(s.getStorage().RenameFile(context.Background(), p.SrcVolume, p.SrcFilePath, p.DstVolume, p.DstFilePath)) +} + +// RenamePartHandler - rename a multipart part from source to destination +func (s *storageRESTServer) RenamePartHandler(p *RenamePartHandlerParams) (grid.NoPayload, *grid.RemoteErr) { + if !s.checkID(p.DiskID) { + return grid.NewNPErr(errDiskNotFound) + } + return grid.NewNPErr(s.getStorage().RenamePart(context.Background(), p.SrcVolume, p.SrcFilePath, p.DstVolume, p.DstFilePath, p.Meta, p.SkipParent)) +} + +// CleanAbandonedDataHandler - Clean unused data directories. +func (s *storageRESTServer) CleanAbandonedDataHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + return + } + volume := r.Form.Get(storageRESTVolume) + filePath := r.Form.Get(storageRESTFilePath) + if volume == "" || filePath == "" { + return // Ignore + } + keepHTTPResponseAlive(w)(s.getStorage().CleanAbandonedData(r.Context(), volume, filePath)) +} + +// closeNotifier is itself a ReadCloser that will notify when either an error occurs or +// the Close() function is called. +type closeNotifier struct { + rc io.ReadCloser + done chan struct{} +} + +func (c *closeNotifier) Read(p []byte) (n int, err error) { + n, err = c.rc.Read(p) + if err != nil { + if c.done != nil { + xioutil.SafeClose(c.done) + c.done = nil + } + } + return n, err +} + +func (c *closeNotifier) Close() error { + if c.done != nil { + xioutil.SafeClose(c.done) + c.done = nil + } + return c.rc.Close() +} + +// keepHTTPReqResponseAlive can be used to avoid timeouts with long storage +// operations, such as bitrot verification or data usage scanning. +// Every 10 seconds a space character is sent. +// keepHTTPReqResponseAlive will wait for the returned body to be read before starting the ticker. +// The returned function should always be called to release resources. +// An optional error can be sent which will be picked as text only error, +// without its original type by the receiver. +// waitForHTTPResponse should be used to the receiving side. +func keepHTTPReqResponseAlive(w http.ResponseWriter, r *http.Request) (resp func(error), body io.ReadCloser) { + bodyDoneCh := make(chan struct{}) + doneCh := make(chan error) + ctx := r.Context() + go func() { + canWrite := true + write := func(b []byte) { + if canWrite { + n, err := w.Write(b) + if err != nil || n != len(b) { + canWrite = false + } + } + } + // Wait for body to be read. + select { + case <-ctx.Done(): + case <-bodyDoneCh: + case err := <-doneCh: + if err != nil { + write([]byte{1}) + write([]byte(err.Error())) + } else { + write([]byte{0}) + } + xioutil.SafeClose(doneCh) + return + } + defer xioutil.SafeClose(doneCh) + // Initiate ticker after body has been read. + ticker := time.NewTicker(time.Second * 10) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + // The done() might have been called + // concurrently, check for it before we + // write the filler byte. + select { + case err := <-doneCh: + if err != nil { + write([]byte{1}) + write([]byte(err.Error())) + } else { + write([]byte{0}) + } + return + default: + } + + // Response not ready, write a filler byte. + write([]byte{32}) + if canWrite { + xhttp.Flush(w) + } + case err := <-doneCh: + if err != nil { + write([]byte{1}) + write([]byte(err.Error())) + } else { + write([]byte{0}) + } + return + } + } + }() + return func(err error) { + if doneCh == nil { + return + } + + // Indicate we are ready to write. + doneCh <- err + + // Wait for channel to be closed so we don't race on writes. + <-doneCh + + // Clear so we can be called multiple times without crashing. + doneCh = nil + }, &closeNotifier{rc: r.Body, done: bodyDoneCh} +} + +// keepHTTPResponseAlive can be used to avoid timeouts with long storage +// operations, such as bitrot verification or data usage scanning. +// keepHTTPResponseAlive may NOT be used until the request body has been read, +// use keepHTTPReqResponseAlive instead. +// Every 10 seconds a space character is sent. +// The returned function should always be called to release resources. +// An optional error can be sent which will be picked as text only error, +// without its original type by the receiver. +// waitForHTTPResponse should be used to the receiving side. +func keepHTTPResponseAlive(w http.ResponseWriter) func(error) { + doneCh := make(chan error) + go func() { + canWrite := true + write := func(b []byte) { + if canWrite { + n, err := w.Write(b) + if err != nil || n != len(b) { + canWrite = false + } + } + } + defer xioutil.SafeClose(doneCh) + ticker := time.NewTicker(time.Second * 10) + defer ticker.Stop() + for { + select { + case <-ticker.C: + // The done() might have been called + // concurrently, check for it before we + // write the filler byte. + select { + case err := <-doneCh: + if err != nil { + write([]byte{1}) + write([]byte(err.Error())) + } else { + write([]byte{0}) + } + return + default: + } + + // Response not ready, write a filler byte. + write([]byte{32}) + if canWrite { + xhttp.Flush(w) + } + case err := <-doneCh: + if err != nil { + write([]byte{1}) + write([]byte(err.Error())) + } else { + write([]byte{0}) + } + return + } + } + }() + return func(err error) { + if doneCh == nil { + return + } + // Indicate we are ready to write. + doneCh <- err + + // Wait for channel to be closed so we don't race on writes. + <-doneCh + + // Clear so we can be called multiple times without crashing. + doneCh = nil + } +} + +// waitForHTTPResponse will wait for responses where keepHTTPResponseAlive +// has been used. +// The returned reader contains the payload. +func waitForHTTPResponse(respBody io.Reader) (io.Reader, error) { + reader := bufio.NewReader(respBody) + for { + b, err := reader.ReadByte() + if err != nil { + return nil, err + } + // Check if we have a response ready or a filler byte. + switch b { + case 0: + return reader, nil + case 1: + errorText, err := io.ReadAll(reader) + if err != nil { + return nil, err + } + return nil, errors.New(string(errorText)) + case 32: + continue + default: + return nil, fmt.Errorf("unexpected filler byte: %d", b) + } + } +} + +// httpStreamResponse allows streaming a response, but still send an error. +type httpStreamResponse struct { + done chan error + block chan []byte + err error +} + +// Write part of the streaming response. +// Note that upstream errors are currently not forwarded, but may be in the future. +func (h *httpStreamResponse) Write(b []byte) (int, error) { + if len(b) == 0 || h.err != nil { + // Ignore 0 length blocks + return 0, h.err + } + tmp := make([]byte, len(b)) + copy(tmp, b) + h.block <- tmp + return len(b), h.err +} + +// CloseWithError will close the stream and return the specified error. +// This can be done several times, but only the first error will be sent. +// After calling this the stream should not be written to. +func (h *httpStreamResponse) CloseWithError(err error) { + if h.done == nil { + return + } + h.done <- err + h.err = err + // Indicates that the response is done. + <-h.done + h.done = nil +} + +// streamHTTPResponse can be used to avoid timeouts with long storage +// operations, such as bitrot verification or data usage scanning. +// Every 10 seconds a space character is sent. +// The returned function should always be called to release resources. +// An optional error can be sent which will be picked as text only error, +// without its original type by the receiver. +// waitForHTTPStream should be used to the receiving side. +func streamHTTPResponse(w http.ResponseWriter) *httpStreamResponse { + doneCh := make(chan error) + blockCh := make(chan []byte) + h := httpStreamResponse{done: doneCh, block: blockCh} + go func() { + canWrite := true + write := func(b []byte) { + if canWrite { + n, err := w.Write(b) + if err != nil || n != len(b) { + canWrite = false + } + } + } + + ticker := time.NewTicker(time.Second * 10) + defer ticker.Stop() + for { + select { + case <-ticker.C: + // Response not ready, write a filler byte. + write([]byte{32}) + if canWrite { + xhttp.Flush(w) + } + case err := <-doneCh: + if err != nil { + write([]byte{1}) + write([]byte(err.Error())) + } else { + write([]byte{0}) + } + xioutil.SafeClose(doneCh) + return + case block := <-blockCh: + var tmp [5]byte + tmp[0] = 2 + binary.LittleEndian.PutUint32(tmp[1:], uint32(len(block))) + write(tmp[:]) + write(block) + if canWrite { + xhttp.Flush(w) + } + } + } + }() + return &h +} + +var poolBuf8k = bpool.Pool[*[]byte]{ + New: func() *[]byte { + b := make([]byte, 8192) + return &b + }, +} + +// waitForHTTPStream will wait for responses where +// streamHTTPResponse has been used. +// The returned reader contains the payload and must be closed if no error is returned. +func waitForHTTPStream(respBody io.ReadCloser, w io.Writer) error { + var tmp [1]byte + // 8K copy buffer, reused for less allocs... + bufp := poolBuf8k.Get() + buf := *bufp + defer poolBuf8k.Put(bufp) + + for { + _, err := io.ReadFull(respBody, tmp[:]) + if err != nil { + return err + } + // Check if we have a response ready or a filler byte. + switch tmp[0] { + case 0: + // 0 is unbuffered, copy the rest. + _, err := io.CopyBuffer(w, respBody, buf) + if err == io.EOF { + return nil + } + return err + case 1: + errorText, err := io.ReadAll(respBody) + if err != nil { + return err + } + return errors.New(string(errorText)) + case 2: + // Block of data + var tmp [4]byte + _, err := io.ReadFull(respBody, tmp[:]) + if err != nil { + return err + } + length := binary.LittleEndian.Uint32(tmp[:]) + n, err := io.CopyBuffer(w, io.LimitReader(respBody, int64(length)), buf) + if err != nil { + return err + } + if n != int64(length) { + return io.ErrUnexpectedEOF + } + continue + case 32: + continue + default: + return fmt.Errorf("unexpected filler byte: %d", tmp[0]) + } + } +} + +// VerifyFileHandler - Verify all part of file for bitrot errors. +func (s *storageRESTServer) VerifyFileHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + return + } + volume := r.Form.Get(storageRESTVolume) + filePath := r.Form.Get(storageRESTFilePath) + + if r.ContentLength < 0 { + s.writeErrorResponse(w, errInvalidArgument) + return + } + + var fi FileInfo + if err := msgp.Decode(r.Body, &fi); err != nil { + s.writeErrorResponse(w, err) + return + } + + done := keepHTTPResponseAlive(w) + resp, err := s.getStorage().VerifyFile(r.Context(), volume, filePath, fi) + done(err) + if err != nil { + return + } + + buf, _ := resp.MarshalMsg(nil) + w.Write(buf) +} + +func checkDiskFatalErrs(errs []error) error { + // This returns a common error if all errors are + // same errors, then there is no point starting + // the server. + if countErrs(errs, errUnsupportedDisk) == len(errs) { + return errUnsupportedDisk + } + + if countErrs(errs, errDiskAccessDenied) == len(errs) { + return errDiskAccessDenied + } + + if countErrs(errs, errFileAccessDenied) == len(errs) { + return errFileAccessDenied + } + + if countErrs(errs, errDiskNotDir) == len(errs) { + return errDiskNotDir + } + + if countErrs(errs, errFaultyDisk) == len(errs) { + return errFaultyDisk + } + + if countErrs(errs, errXLBackend) == len(errs) { + return errXLBackend + } + + return nil +} + +// A single function to write certain errors to be fatal +// or informative based on the `exit` flag, please look +// at each implementation of error for added hints. +// +// FIXME: This is an unusual function but serves its purpose for +// now, need to revisit the overall erroring structure here. +// Do not like it :-( +func logFatalErrs(err error, endpoint Endpoint, exit bool) { + switch { + case errors.Is(err, errXLBackend): + logger.Fatal(config.ErrInvalidXLValue(err), "Unable to initialize backend") + case errors.Is(err, errUnsupportedDisk): + var hint string + if endpoint.URL != nil { + hint = fmt.Sprintf("Drive '%s' does not support O_DIRECT flags, MinIO erasure coding requires filesystems with O_DIRECT support", endpoint.Path) + } else { + hint = "Drives do not support O_DIRECT flags, MinIO erasure coding requires filesystems with O_DIRECT support" + } + logger.Fatal(config.ErrUnsupportedBackend(err).Hint("%s", hint), "Unable to initialize backend") + case errors.Is(err, errDiskNotDir): + var hint string + if endpoint.URL != nil { + hint = fmt.Sprintf("Drive '%s' is not a directory, MinIO erasure coding needs a directory", endpoint.Path) + } else { + hint = "Drives are not directories, MinIO erasure coding needs directories" + } + logger.Fatal(config.ErrUnableToWriteInBackend(err).Hint("%s", hint), "Unable to initialize backend") + case errors.Is(err, errDiskAccessDenied): + // Show a descriptive error with a hint about how to fix it. + var username string + if u, err := user.Current(); err == nil { + username = u.Username + } else { + username = "" + } + var hint string + if endpoint.URL != nil { + hint = fmt.Sprintf("Run the following command to add write permissions: `sudo chown -R %s %s && sudo chmod u+rxw %s`", + username, endpoint.Path, endpoint.Path) + } else { + hint = fmt.Sprintf("Run the following command to add write permissions: `sudo chown -R %s. && sudo chmod u+rxw `", username) + } + if !exit { + storageLogOnceIf(GlobalContext, fmt.Errorf("Drive is not writable %s, %s", endpoint, hint), "log-fatal-errs") + } else { + logger.Fatal(config.ErrUnableToWriteInBackend(err).Hint("%s", hint), "Unable to initialize backend") + } + case errors.Is(err, errFaultyDisk): + if !exit { + storageLogOnceIf(GlobalContext, fmt.Errorf("Drive is faulty at %s, please replace the drive - drive will be offline", endpoint), "log-fatal-errs") + } else { + logger.Fatal(err, "Unable to initialize backend") + } + case errors.Is(err, errDiskFull): + if !exit { + storageLogOnceIf(GlobalContext, fmt.Errorf("Drive is already full at %s, incoming I/O will fail - drive will be offline", endpoint), "log-fatal-errs") + } else { + logger.Fatal(err, "Unable to initialize backend") + } + case errors.Is(err, errInconsistentDisk): + if exit { + logger.Fatal(err, "Unable to initialize backend") + } + default: + if !exit { + storageLogOnceIf(GlobalContext, fmt.Errorf("Drive %s returned an unexpected error: %w, please investigate - drive will be offline", endpoint, err), "log-fatal-errs") + } else { + logger.Fatal(err, "Unable to initialize backend") + } + } +} + +// StatInfoFile returns file stat info. +func (s *storageRESTServer) StatInfoFile(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + return + } + volume := r.Form.Get(storageRESTVolume) + filePath := r.Form.Get(storageRESTFilePath) + glob := r.Form.Get(storageRESTGlob) + done := keepHTTPResponseAlive(w) + stats, err := s.getStorage().StatInfoFile(r.Context(), volume, filePath, glob == "true") + done(err) + if err != nil { + return + } + for _, si := range stats { + msgp.Encode(w, &si) + } +} + +func (s *storageRESTServer) DeleteBulkHandler(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + return + } + + var req DeleteBulkReq + mr := msgpNewReader(r.Body) + defer readMsgpReaderPoolPut(mr) + + if err := req.DecodeMsg(mr); err != nil { + s.writeErrorResponse(w, err) + return + } + + volume := r.Form.Get(storageRESTVolume) + keepHTTPResponseAlive(w)(s.getStorage().DeleteBulk(r.Context(), volume, req.Paths...)) +} + +// ReadMultiple returns multiple files +func (s *storageRESTServer) ReadMultiple(w http.ResponseWriter, r *http.Request) { + if !s.IsValid(w, r) { + return + } + rw := streamHTTPResponse(w) + defer func() { + if r := recover(); r != nil { + debug.PrintStack() + rw.CloseWithError(fmt.Errorf("panic: %v", r)) + } + }() + + var req ReadMultipleReq + mr := msgpNewReader(r.Body) + defer readMsgpReaderPoolPut(mr) + err := req.DecodeMsg(mr) + if err != nil { + rw.CloseWithError(err) + return + } + + mw := msgp.NewWriter(rw) + responses := make(chan ReadMultipleResp, len(req.Files)) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for resp := range responses { + err := resp.EncodeMsg(mw) + if err != nil { + rw.CloseWithError(err) + return + } + mw.Flush() + } + }() + err = s.getStorage().ReadMultiple(r.Context(), req, responses) + wg.Wait() + rw.CloseWithError(err) +} + +// globalLocalSetDrives is used for local drive as well as remote REST +// API caller for other nodes to talk to this node. +// +// Any updates to this must be serialized via globalLocalDrivesMu (locker) +var globalLocalSetDrives [][][]StorageAPI + +// registerStorageRESTHandlers - register storage rpc router. +func registerStorageRESTHandlers(router *mux.Router, endpointServerPools EndpointServerPools, gm *grid.Manager) { + h := func(f http.HandlerFunc) http.HandlerFunc { + return collectInternodeStats(httpTraceHdrs(f)) + } + + globalLocalDrivesMap = make(map[string]StorageAPI) + globalLocalSetDrives = make([][][]StorageAPI, len(endpointServerPools)) + for pool := range globalLocalSetDrives { + globalLocalSetDrives[pool] = make([][]StorageAPI, endpointServerPools[pool].SetCount) + for set := range globalLocalSetDrives[pool] { + globalLocalSetDrives[pool][set] = make([]StorageAPI, endpointServerPools[pool].DrivesPerSet) + } + } + for _, serverPool := range endpointServerPools { + for _, endpoint := range serverPool.Endpoints { + if !endpoint.IsLocal { + continue + } + + server := &storageRESTServer{ + endpoint: endpoint, + } + + subrouter := router.PathPrefix(path.Join(storageRESTPrefix, endpoint.Path)).Subrouter() + + subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodHealth).HandlerFunc(h(server.HealthHandler)) + subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodAppendFile).HandlerFunc(h(server.AppendFileHandler)) + subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCreateFile).HandlerFunc(h(server.CreateFileHandler)) + subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteVersions).HandlerFunc(h(server.DeleteVersionsHandler)) + subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodVerifyFile).HandlerFunc(h(server.VerifyFileHandler)) + subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodStatInfoFile).HandlerFunc(h(server.StatInfoFile)) + subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadMultiple).HandlerFunc(h(server.ReadMultiple)) + subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodCleanAbandoned).HandlerFunc(h(server.CleanAbandonedDataHandler)) + subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodDeleteBulk).HandlerFunc(h(server.DeleteBulkHandler)) + subrouter.Methods(http.MethodPost).Path(storageRESTVersionPrefix + storageRESTMethodReadParts).HandlerFunc(h(server.ReadPartsHandler)) + + subrouter.Methods(http.MethodGet).Path(storageRESTVersionPrefix + storageRESTMethodReadFileStream).HandlerFunc(h(server.ReadFileStreamHandler)) + subrouter.Methods(http.MethodGet).Path(storageRESTVersionPrefix + storageRESTMethodReadVersion).HandlerFunc(h(server.ReadVersionHandler)) + subrouter.Methods(http.MethodGet).Path(storageRESTVersionPrefix + storageRESTMethodReadXL).HandlerFunc(h(server.ReadXLHandler)) + subrouter.Methods(http.MethodGet).Path(storageRESTVersionPrefix + storageRESTMethodReadFile).HandlerFunc(h(server.ReadFileHandler)) + + logger.FatalIf(storageListDirRPC.RegisterNoInput(gm, server.ListDirHandler, endpoint.Path), "unable to register handler") + logger.FatalIf(storageReadAllRPC.Register(gm, server.ReadAllHandler, endpoint.Path), "unable to register handler") + logger.FatalIf(storageWriteAllRPC.Register(gm, server.WriteAllHandler, endpoint.Path), "unable to register handler") + logger.FatalIf(storageRenameFileRPC.Register(gm, server.RenameFileHandler, endpoint.Path), "unable to register handler") + logger.FatalIf(storageRenamePartRPC.Register(gm, server.RenamePartHandler, endpoint.Path), "unable to register handler") + logger.FatalIf(storageRenameDataRPC.Register(gm, server.RenameDataHandler, endpoint.Path), "unable to register handler") + logger.FatalIf(storageRenameDataInlineRPC.Register(gm, server.RenameDataInlineHandler, endpoint.Path), "unable to register handler") + logger.FatalIf(storageDeleteFileRPC.Register(gm, server.DeleteFileHandler, endpoint.Path), "unable to register handler") + logger.FatalIf(storageCheckPartsRPC.RegisterNoInput(gm, server.CheckPartsHandler, endpoint.Path), "unable to register handler") + logger.FatalIf(storageReadVersionRPC.Register(gm, server.ReadVersionHandlerWS, endpoint.Path), "unable to register handler") + logger.FatalIf(storageWriteMetadataRPC.Register(gm, server.WriteMetadataHandler, endpoint.Path), "unable to register handler") + logger.FatalIf(storageUpdateMetadataRPC.Register(gm, server.UpdateMetadataHandler, endpoint.Path), "unable to register handler") + logger.FatalIf(storageDeleteVersionRPC.Register(gm, server.DeleteVersionHandler, endpoint.Path), "unable to register handler") + logger.FatalIf(storageReadXLRPC.Register(gm, server.ReadXLHandlerWS, endpoint.Path), "unable to register handler") + logger.FatalIf(storageNSScannerRPC.RegisterNoInput(gm, server.NSScannerHandler, endpoint.Path), "unable to register handler") + logger.FatalIf(storageDiskInfoRPC.Register(gm, server.DiskInfoHandler, endpoint.Path), "unable to register handler") + logger.FatalIf(storageStatVolRPC.Register(gm, server.StatVolHandler, endpoint.Path), "unable to register handler") + logger.FatalIf(gm.RegisterStreamingHandler(grid.HandlerWalkDir, grid.StreamHandler{ + Subroute: endpoint.Path, + Handle: server.WalkDirHandler, + OutCapacity: 1, + }), "unable to register handler") + + createStorage := func(endpoint Endpoint) bool { + xl, err := newXLStorage(endpoint, false) + if err != nil { + // if supported errors don't fail, we proceed to + // printing message and moving forward. + if errors.Is(err, errDriveIsRoot) { + err = fmt.Errorf("major: %v: minor: %v: %w", xl.major, xl.minor, err) + } + logFatalErrs(err, endpoint, false) + return false + } + storage := newXLStorageDiskIDCheck(xl, true) + storage.SetDiskID(xl.diskID) + // We do not have to do SetFormatData() since 'xl' + // already captures formatData cached. + + globalLocalDrivesMu.Lock() + defer globalLocalDrivesMu.Unlock() + + globalLocalDrivesMap[endpoint.String()] = storage + globalLocalSetDrives[endpoint.PoolIdx][endpoint.SetIdx][endpoint.DiskIdx] = storage + return true + } + + if createStorage(endpoint) { + continue + } + + // Start async goroutine to create storage. + go func(endpoint Endpoint) { + for { + time.Sleep(3 * time.Second) + if createStorage(endpoint) { + return + } + } + }(endpoint) + } + } +} diff --git a/cmd/storage-rest_test.go b/cmd/storage-rest_test.go new file mode 100644 index 00000000..a601d799 --- /dev/null +++ b/cmd/storage-rest_test.go @@ -0,0 +1,413 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "errors" + "math/rand" + "reflect" + "runtime" + "testing" + "time" + + "github.com/minio/minio/internal/grid" + xnet "github.com/minio/pkg/v3/net" +) + +// Storage REST server, storageRESTReceiver and StorageRESTClient are +// inter-dependent, below test functions are sufficient to test all of them. +func testStorageAPIDiskInfo(t *testing.T, storage StorageAPI) { + testCases := []struct { + expectErr bool + }{ + {true}, + } + + for i, testCase := range testCases { + _, err := storage.DiskInfo(t.Context(), DiskInfoOptions{Metrics: true}) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("case %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + if err != errUnformattedDisk { + t.Fatalf("case %v: error: expected: %v, got: %v", i+1, errUnformattedDisk, err) + } + } +} + +func testStorageAPIStatInfoFile(t *testing.T, storage StorageAPI) { + err := storage.AppendFile(t.Context(), "foo", pathJoin("myobject", xlStorageFormatFile), []byte("foo")) + if err != nil { + t.Fatalf("unexpected error %v", err) + } + + testCases := []struct { + volumeName string + objectName string + expectErr bool + }{ + {"foo", "myobject", false}, + // file not found error. + {"foo", "yourobject", true}, + } + + for i, testCase := range testCases { + _, err := storage.StatInfoFile(t.Context(), testCase.volumeName, testCase.objectName+"/"+xlStorageFormatFile, false) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("case %v: error: expected: %v, got: %v, err: %v", i+1, expectErr, testCase.expectErr, err) + } + } +} + +func testStorageAPIListDir(t *testing.T, storage StorageAPI) { + err := storage.AppendFile(t.Context(), "foo", "path/to/myobject", []byte("foo")) + if err != nil { + t.Fatalf("unexpected error %v", err) + } + + testCases := []struct { + volumeName string + prefix string + expectedResult []string + expectErr bool + }{ + {"foo", "path", []string{"to/"}, false}, + // prefix not found error. + {"foo", "nodir", nil, true}, + } + + for i, testCase := range testCases { + result, err := storage.ListDir(t.Context(), "", testCase.volumeName, testCase.prefix, -1) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("case %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + + if !testCase.expectErr { + if !reflect.DeepEqual(result, testCase.expectedResult) { + t.Fatalf("case %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } + } +} + +func testStorageAPIReadAll(t *testing.T, storage StorageAPI) { + err := storage.AppendFile(t.Context(), "foo", "myobject", []byte("foo")) + if err != nil { + t.Fatalf("unexpected error %v", err) + } + + testCases := []struct { + volumeName string + objectName string + expectedResult []byte + expectErr bool + }{ + {"foo", "myobject", []byte("foo"), false}, + // file not found error. + {"foo", "yourobject", nil, true}, + } + + for i, testCase := range testCases { + result, err := storage.ReadAll(t.Context(), testCase.volumeName, testCase.objectName) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("case %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + + if !testCase.expectErr { + if !reflect.DeepEqual(result, testCase.expectedResult) { + t.Fatalf("case %v: result: expected: %v, got: %v", i+1, string(testCase.expectedResult), string(result)) + } + } + } +} + +func testStorageAPIReadFile(t *testing.T, storage StorageAPI) { + err := storage.AppendFile(t.Context(), "foo", "myobject", []byte("foo")) + if err != nil { + t.Fatalf("unexpected error %v", err) + } + + testCases := []struct { + volumeName string + objectName string + offset int64 + expectedResult []byte + expectErr bool + }{ + {"foo", "myobject", 0, []byte("foo"), false}, + {"foo", "myobject", 1, []byte("oo"), false}, + // file not found error. + {"foo", "yourobject", 0, nil, true}, + } + + result := make([]byte, 100) + for i, testCase := range testCases { + result = result[testCase.offset:3] + _, err := storage.ReadFile(t.Context(), testCase.volumeName, testCase.objectName, testCase.offset, result, nil) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("case %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + + if !testCase.expectErr { + if !reflect.DeepEqual(result, testCase.expectedResult) { + t.Fatalf("case %v: result: expected: %v, got: %v", i+1, string(testCase.expectedResult), string(result)) + } + } + } +} + +func testStorageAPIAppendFile(t *testing.T, storage StorageAPI) { + testData := []byte("foo") + testCases := []struct { + volumeName string + objectName string + data []byte + expectErr bool + ignoreIfWindows bool + }{ + {"foo", "myobject", testData, false, false}, + {"foo", "myobject-0byte", []byte{}, false, false}, + // volume not found error. + {"foo-bar", "myobject", testData, true, false}, + // Test some weird characters over the wire. + {"foo", "newline\n", testData, false, true}, + {"foo", "newline\t", testData, false, true}, + {"foo", "newline \n", testData, false, true}, + {"foo", "newline$$$\n", testData, false, true}, + {"foo", "newline%%%\n", testData, false, true}, + {"foo", "newline \t % $ & * ^ # @ \n", testData, false, true}, + {"foo", "\n\tnewline \t % $ & * ^ # @ \n", testData, false, true}, + } + + for i, testCase := range testCases { + if testCase.ignoreIfWindows && runtime.GOOS == "windows" { + continue + } + err := storage.AppendFile(t.Context(), testCase.volumeName, testCase.objectName, testCase.data) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("case %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + + if !testCase.expectErr { + data, err := storage.ReadAll(t.Context(), testCase.volumeName, testCase.objectName) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(data, testCase.data) { + t.Fatalf("case %v: expected %v, got %v", i+1, testCase.data, data) + } + } + } +} + +func testStorageAPIDeleteFile(t *testing.T, storage StorageAPI) { + err := storage.AppendFile(t.Context(), "foo", "myobject", []byte("foo")) + if err != nil { + t.Fatalf("unexpected error %v", err) + } + + testCases := []struct { + volumeName string + objectName string + expectErr bool + }{ + {"foo", "myobject", false}, + // file not found not returned + {"foo", "myobject", false}, + // file not found not returned + {"foo", "yourobject", false}, + } + + for i, testCase := range testCases { + err := storage.Delete(t.Context(), testCase.volumeName, testCase.objectName, DeleteOptions{ + Recursive: false, + Immediate: false, + }) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("case %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + } +} + +func testStorageAPIRenameFile(t *testing.T, storage StorageAPI) { + err := storage.AppendFile(t.Context(), "foo", "myobject", []byte("foo")) + if err != nil { + t.Fatalf("unexpected error %v", err) + } + + err = storage.AppendFile(t.Context(), "foo", "otherobject", []byte("foo")) + if err != nil { + t.Fatalf("unexpected error %v", err) + } + + testCases := []struct { + volumeName string + objectName string + destVolumeName string + destObjectName string + expectErr bool + }{ + {"foo", "myobject", "foo", "yourobject", false}, + {"foo", "yourobject", "bar", "myobject", false}, + // overwrite. + {"foo", "otherobject", "bar", "myobject", false}, + } + + for i, testCase := range testCases { + err := storage.RenameFile(t.Context(), testCase.volumeName, testCase.objectName, testCase.destVolumeName, testCase.destObjectName) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("case %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + } +} + +func newStorageRESTHTTPServerClient(t testing.TB) *storageRESTClient { + // Grid with 2 hosts + tg, err := grid.SetupTestGrid(2) + if err != nil { + t.Fatalf("SetupTestGrid: %v", err) + } + t.Cleanup(tg.Cleanup) + prevHost, prevPort := globalMinioHost, globalMinioPort + defer func() { + globalMinioHost, globalMinioPort = prevHost, prevPort + }() + // tg[0] = local, tg[1] = remote + + // Remote URL + url, err := xnet.ParseHTTPURL(tg.Servers[1].URL) + if err != nil { + t.Fatalf("unexpected error %v", err) + } + url.Path = t.TempDir() + + globalMinioHost, globalMinioPort = mustSplitHostPort(url.Host) + globalNodeAuthToken, _ = authenticateNode(globalActiveCred.AccessKey, globalActiveCred.SecretKey) + + endpoint, err := NewEndpoint(url.String()) + if err != nil { + t.Fatalf("NewEndpoint failed %v", endpoint) + } + + if err = endpoint.UpdateIsLocal(); err != nil { + t.Fatalf("UpdateIsLocal failed %v", err) + } + + endpoint.PoolIdx = 0 + endpoint.SetIdx = 0 + endpoint.DiskIdx = 0 + + poolEps := []PoolEndpoints{{ + Endpoints: Endpoints{endpoint}, + }} + poolEps[0].SetCount = 1 + poolEps[0].DrivesPerSet = 1 + + // Register handlers on newly created servers + registerStorageRESTHandlers(tg.Mux[0], poolEps, tg.Managers[0]) + registerStorageRESTHandlers(tg.Mux[1], poolEps, tg.Managers[1]) + + storage := globalLocalSetDrives[0][0][0] + if err = storage.MakeVol(t.Context(), "foo"); err != nil { + t.Fatalf("unexpected error %v", err) + } + + if err = storage.MakeVol(t.Context(), "bar"); err != nil { + t.Fatalf("unexpected error %v", err) + } + + restClient, err := newStorageRESTClient(endpoint, false, tg.Managers[0]) + if err != nil { + t.Fatal(err) + } + + for { + _, err := restClient.DiskInfo(t.Context(), DiskInfoOptions{}) + if err == nil || errors.Is(err, errUnformattedDisk) { + break + } + time.Sleep(time.Duration(rand.Float64() * float64(100*time.Millisecond))) + } + + return restClient +} + +func TestStorageRESTClientDiskInfo(t *testing.T) { + restClient := newStorageRESTHTTPServerClient(t) + + testStorageAPIDiskInfo(t, restClient) +} + +func TestStorageRESTClientStatInfoFile(t *testing.T) { + restClient := newStorageRESTHTTPServerClient(t) + + testStorageAPIStatInfoFile(t, restClient) +} + +func TestStorageRESTClientListDir(t *testing.T) { + restClient := newStorageRESTHTTPServerClient(t) + + testStorageAPIListDir(t, restClient) +} + +func TestStorageRESTClientReadAll(t *testing.T) { + restClient := newStorageRESTHTTPServerClient(t) + + testStorageAPIReadAll(t, restClient) +} + +func TestStorageRESTClientReadFile(t *testing.T) { + restClient := newStorageRESTHTTPServerClient(t) + + testStorageAPIReadFile(t, restClient) +} + +func TestStorageRESTClientAppendFile(t *testing.T) { + restClient := newStorageRESTHTTPServerClient(t) + + testStorageAPIAppendFile(t, restClient) +} + +func TestStorageRESTClientDeleteFile(t *testing.T) { + restClient := newStorageRESTHTTPServerClient(t) + + testStorageAPIDeleteFile(t, restClient) +} + +func TestStorageRESTClientRenameFile(t *testing.T) { + restClient := newStorageRESTHTTPServerClient(t) + + testStorageAPIRenameFile(t, restClient) +} diff --git a/cmd/storagemetric_string.go b/cmd/storagemetric_string.go new file mode 100644 index 00000000..79478132 --- /dev/null +++ b/cmd/storagemetric_string.go @@ -0,0 +1,54 @@ +// Code generated by "stringer -type=storageMetric -trimprefix=storageMetric xl-storage-disk-id-check.go"; DO NOT EDIT. + +package cmd + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[storageMetricMakeVolBulk-0] + _ = x[storageMetricMakeVol-1] + _ = x[storageMetricListVols-2] + _ = x[storageMetricStatVol-3] + _ = x[storageMetricDeleteVol-4] + _ = x[storageMetricWalkDir-5] + _ = x[storageMetricListDir-6] + _ = x[storageMetricReadFile-7] + _ = x[storageMetricAppendFile-8] + _ = x[storageMetricCreateFile-9] + _ = x[storageMetricReadFileStream-10] + _ = x[storageMetricRenameFile-11] + _ = x[storageMetricRenameData-12] + _ = x[storageMetricCheckParts-13] + _ = x[storageMetricDelete-14] + _ = x[storageMetricDeleteVersions-15] + _ = x[storageMetricVerifyFile-16] + _ = x[storageMetricWriteAll-17] + _ = x[storageMetricDeleteVersion-18] + _ = x[storageMetricWriteMetadata-19] + _ = x[storageMetricUpdateMetadata-20] + _ = x[storageMetricReadVersion-21] + _ = x[storageMetricReadXL-22] + _ = x[storageMetricReadAll-23] + _ = x[storageMetricStatInfoFile-24] + _ = x[storageMetricReadMultiple-25] + _ = x[storageMetricDeleteAbandonedParts-26] + _ = x[storageMetricDiskInfo-27] + _ = x[storageMetricDeleteBulk-28] + _ = x[storageMetricRenamePart-29] + _ = x[storageMetricReadParts-30] + _ = x[storageMetricLast-31] +} + +const _storageMetric_name = "MakeVolBulkMakeVolListVolsStatVolDeleteVolWalkDirListDirReadFileAppendFileCreateFileReadFileStreamRenameFileRenameDataCheckPartsDeleteDeleteVersionsVerifyFileWriteAllDeleteVersionWriteMetadataUpdateMetadataReadVersionReadXLReadAllStatInfoFileReadMultipleDeleteAbandonedPartsDiskInfoDeleteBulkRenamePartReadPartsLast" + +var _storageMetric_index = [...]uint16{0, 11, 18, 26, 33, 42, 49, 56, 64, 74, 84, 98, 108, 118, 128, 134, 148, 158, 166, 179, 192, 206, 217, 223, 230, 242, 254, 274, 282, 292, 302, 311, 315} + +func (i storageMetric) String() string { + if i >= storageMetric(len(_storageMetric_index)-1) { + return "storageMetric(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _storageMetric_name[_storageMetric_index[i]:_storageMetric_index[i+1]] +} diff --git a/cmd/streaming-signature-v4.go b/cmd/streaming-signature-v4.go new file mode 100644 index 00000000..702714be --- /dev/null +++ b/cmd/streaming-signature-v4.go @@ -0,0 +1,667 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Package cmd This file implements helper functions to validate Streaming AWS +// Signature Version '4' authorization header. +package cmd + +import ( + "bufio" + "bytes" + "encoding/hex" + "errors" + "fmt" + "hash" + "io" + "net/http" + "strings" + "time" + + "github.com/dustin/go-humanize" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/hash/sha256" + xhttp "github.com/minio/minio/internal/http" +) + +// Streaming AWS Signature Version '4' constants. +const ( + emptySHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + streamingContentSHA256 = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD" + streamingContentSHA256Trailer = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER" + signV4ChunkedAlgorithm = "AWS4-HMAC-SHA256-PAYLOAD" + signV4ChunkedAlgorithmTrailer = "AWS4-HMAC-SHA256-TRAILER" + streamingContentEncoding = "aws-chunked" + awsTrailerHeader = "X-Amz-Trailer" + trailerKVSeparator = ":" +) + +// getChunkSignature - get chunk signature. +// Does not update anything in cr. +func (cr *s3ChunkedReader) getChunkSignature() string { + hashedChunk := hex.EncodeToString(cr.chunkSHA256Writer.Sum(nil)) + + // Calculate string to sign. + alg := signV4ChunkedAlgorithm + "\n" + stringToSign := alg + + cr.seedDate.Format(iso8601Format) + "\n" + + getScope(cr.seedDate, cr.region) + "\n" + + cr.seedSignature + "\n" + + emptySHA256 + "\n" + + hashedChunk + + // Get hmac signing key. + signingKey := getSigningKey(cr.cred.SecretKey, cr.seedDate, cr.region, serviceS3) + + // Calculate signature. + newSignature := getSignature(signingKey, stringToSign) + + return newSignature +} + +// getTrailerChunkSignature - get trailer chunk signature. +func (cr *s3ChunkedReader) getTrailerChunkSignature() string { + hashedChunk := hex.EncodeToString(cr.chunkSHA256Writer.Sum(nil)) + + // Calculate string to sign. + alg := signV4ChunkedAlgorithmTrailer + "\n" + stringToSign := alg + + cr.seedDate.Format(iso8601Format) + "\n" + + getScope(cr.seedDate, cr.region) + "\n" + + cr.seedSignature + "\n" + + hashedChunk + + // Get hmac signing key. + signingKey := getSigningKey(cr.cred.SecretKey, cr.seedDate, cr.region, serviceS3) + + // Calculate signature. + newSignature := getSignature(signingKey, stringToSign) + + return newSignature +} + +// calculateSeedSignature - Calculate seed signature in accordance with +// - http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html +// +// returns signature, error otherwise if the signature mismatches or any other +// error while parsing and validating. +func calculateSeedSignature(r *http.Request, trailers bool) (cred auth.Credentials, signature string, region string, date time.Time, errCode APIErrorCode) { + // Copy request. + req := *r + + // Save authorization header. + v4Auth := req.Header.Get(xhttp.Authorization) + + // Parse signature version '4' header. + signV4Values, errCode := parseSignV4(v4Auth, globalSite.Region(), serviceS3) + if errCode != ErrNone { + return cred, "", "", time.Time{}, errCode + } + + // Payload streaming. + payload := streamingContentSHA256 + if trailers { + payload = streamingContentSHA256Trailer + } + + // Payload for STREAMING signature should be 'STREAMING-AWS4-HMAC-SHA256-PAYLOAD' + if payload != req.Header.Get(xhttp.AmzContentSha256) { + return cred, "", "", time.Time{}, ErrContentSHA256Mismatch + } + + // Extract all the signed headers along with its values. + extractedSignedHeaders, errCode := extractSignedHeaders(signV4Values.SignedHeaders, r) + if errCode != ErrNone { + return cred, "", "", time.Time{}, errCode + } + + cred, _, errCode = checkKeyValid(r, signV4Values.Credential.accessKey) + if errCode != ErrNone { + return cred, "", "", time.Time{}, errCode + } + + // Verify if region is valid. + region = signV4Values.Credential.scope.region + + // Extract date, if not present throw error. + var dateStr string + if dateStr = req.Header.Get("x-amz-date"); dateStr == "" { + if dateStr = r.Header.Get("Date"); dateStr == "" { + return cred, "", "", time.Time{}, ErrMissingDateHeader + } + } + + // Parse date header. + var err error + date, err = time.Parse(iso8601Format, dateStr) + if err != nil { + return cred, "", "", time.Time{}, ErrMalformedDate + } + + // Query string. + queryStr := req.Form.Encode() + + // Get canonical request. + canonicalRequest := getCanonicalRequest(extractedSignedHeaders, payload, queryStr, req.URL.Path, req.Method) + + // Get string to sign from canonical request. + stringToSign := getStringToSign(canonicalRequest, date, signV4Values.Credential.getScope()) + + // Get hmac signing key. + signingKey := getSigningKey(cred.SecretKey, signV4Values.Credential.scope.date, region, serviceS3) + + // Calculate signature. + newSignature := getSignature(signingKey, stringToSign) + + // Verify if signature match. + if !compareSignatureV4(newSignature, signV4Values.Signature) { + return cred, "", "", time.Time{}, ErrSignatureDoesNotMatch + } + + // Return calculated signature. + return cred, newSignature, region, date, ErrNone +} + +const maxLineLength = 4 * humanize.KiByte // assumed <= bufio.defaultBufSize 4KiB + +// lineTooLong is generated as chunk header is bigger than 4KiB. +var errLineTooLong = errors.New("header line too long") + +// malformed encoding is generated when chunk header is wrongly formed. +var errMalformedEncoding = errors.New("malformed chunked encoding") + +// chunk is considered too big if its bigger than > 16MiB. +var errChunkTooBig = errors.New("chunk too big: choose chunk size <= 16MiB") + +// newSignV4ChunkedReader returns a new s3ChunkedReader that translates the data read from r +// out of HTTP "chunked" format before returning it. +// The s3ChunkedReader returns io.EOF when the final 0-length chunk is read. +// +// NewChunkedReader is not needed by normal applications. The http package +// automatically decodes chunking when reading response bodies. +func newSignV4ChunkedReader(req *http.Request, trailer bool) (io.ReadCloser, APIErrorCode) { + cred, seedSignature, region, seedDate, errCode := calculateSeedSignature(req, trailer) + if errCode != ErrNone { + return nil, errCode + } + + if trailer { + // Discard anything unsigned. + req.Trailer = make(http.Header) + trailers := req.Header.Values(awsTrailerHeader) + for _, key := range trailers { + req.Trailer.Add(key, "") + } + } else { + req.Trailer = nil + } + return &s3ChunkedReader{ + trailers: req.Trailer, + reader: bufio.NewReader(req.Body), + cred: cred, + seedSignature: seedSignature, + seedDate: seedDate, + region: region, + chunkSHA256Writer: sha256.New(), + buffer: make([]byte, 64*1024), + debug: false, + }, ErrNone +} + +// Represents the overall state that is required for decoding a +// AWS Signature V4 chunked reader. +type s3ChunkedReader struct { + reader *bufio.Reader + cred auth.Credentials + seedSignature string + seedDate time.Time + region string + trailers http.Header + + chunkSHA256Writer hash.Hash // Calculates sha256 of chunk data. + buffer []byte + offset int + err error + debug bool // Print details on failure. Add your own if more are needed. +} + +func (cr *s3ChunkedReader) Close() (err error) { + return nil +} + +// Now, we read one chunk from the underlying reader. +// A chunk has the following format: +// +// + ";chunk-signature=" + + "\r\n" + + "\r\n" +// +// First, we read the chunk size but fail if it is larger +// than 16 MiB. We must not accept arbitrary large chunks. +// One 16 MiB is a reasonable max limit. +// +// Then we read the signature and payload data. We compute the SHA256 checksum +// of the payload and verify that it matches the expected signature value. +// +// The last chunk is *always* 0-sized. So, we must only return io.EOF if we have encountered +// a chunk with a chunk size = 0. However, this chunk still has a signature and we must +// verify it. +const maxChunkSize = 16 << 20 // 16 MiB + +// Read - implements `io.Reader`, which transparently decodes +// the incoming AWS Signature V4 streaming signature. +func (cr *s3ChunkedReader) Read(buf []byte) (n int, err error) { + if cr.err != nil { + if cr.debug { + fmt.Printf("s3ChunkedReader: Returning err: %v (%T)\n", cr.err, cr.err) + } + return 0, cr.err + } + defer func() { + if err != nil && err != io.EOF { + if cr.debug { + fmt.Println("Read err:", err) + } + } + }() + // First, if there is any unread data, copy it to the client + // provided buffer. + if cr.offset > 0 { + n = copy(buf, cr.buffer[cr.offset:]) + if n == len(buf) { + cr.offset += n + return n, nil + } + cr.offset = 0 + buf = buf[n:] + } + + var size int + for { + b, err := cr.reader.ReadByte() + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + if err != nil { + cr.err = err + return n, cr.err + } + if b == ';' { // separating character + break + } + + // Manually deserialize the size since AWS specified + // the chunk size to be of variable width. In particular, + // a size of 16 is encoded as `10` while a size of 64 KB + // is `10000`. + switch { + case b >= '0' && b <= '9': + size = size<<4 | int(b-'0') + case b >= 'a' && b <= 'f': + size = size<<4 | int(b-('a'-10)) + case b >= 'A' && b <= 'F': + size = size<<4 | int(b-('A'-10)) + default: + cr.err = errMalformedEncoding + return n, cr.err + } + if size > maxChunkSize { + cr.err = errChunkTooBig + return n, cr.err + } + } + + // Now, we read the signature of the following payload and expect: + // chunk-signature=" + + "\r\n" + // + // The signature is 64 bytes long (hex-encoded SHA256 hash) and + // starts with a 16 byte header: len("chunk-signature=") + 64 == 80. + var signature [80]byte + _, err = io.ReadFull(cr.reader, signature[:]) + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + if err != nil { + cr.err = err + return n, cr.err + } + if !bytes.HasPrefix(signature[:], []byte("chunk-signature=")) { + cr.err = errMalformedEncoding + return n, cr.err + } + b, err := cr.reader.ReadByte() + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + if err != nil { + cr.err = err + return n, cr.err + } + if b != '\r' { + cr.err = errMalformedEncoding + return n, cr.err + } + b, err = cr.reader.ReadByte() + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + if err != nil { + cr.err = err + return n, cr.err + } + if b != '\n' { + cr.err = errMalformedEncoding + return n, cr.err + } + + if cap(cr.buffer) < size { + cr.buffer = make([]byte, size) + } else { + cr.buffer = cr.buffer[:size] + } + + // Now, we read the payload and compute its SHA-256 hash. + _, err = io.ReadFull(cr.reader, cr.buffer) + if err == io.EOF && size != 0 { + err = io.ErrUnexpectedEOF + } + if err != nil && err != io.EOF { + cr.err = err + return n, cr.err + } + + // Once we have read the entire chunk successfully, we verify + // that the received signature matches our computed signature. + cr.chunkSHA256Writer.Write(cr.buffer) + newSignature := cr.getChunkSignature() + if !compareSignatureV4(string(signature[16:]), newSignature) { + cr.err = errSignatureMismatch + return n, cr.err + } + cr.seedSignature = newSignature + cr.chunkSHA256Writer.Reset() + + // If the chunk size is zero we return io.EOF. As specified by AWS, + // only the last chunk is zero-sized. + if len(cr.buffer) == 0 { + if cr.debug { + fmt.Println("EOF. Reading Trailers:", cr.trailers) + } + if cr.trailers != nil { + err = cr.readTrailers() + if cr.debug { + fmt.Println("trailers returned:", err, "now:", cr.trailers) + } + if err != nil { + cr.err = err + return 0, err + } + } + cr.err = io.EOF + return n, cr.err + } + + b, err = cr.reader.ReadByte() + if b != '\r' || err != nil { + if cr.debug { + fmt.Printf("want %q, got %q\n", "\r", string(b)) + } + cr.err = errMalformedEncoding + return n, cr.err + } + b, err = cr.reader.ReadByte() + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + if err != nil { + cr.err = err + return n, cr.err + } + if b != '\n' { + if cr.debug { + fmt.Printf("want %q, got %q\n", "\r", string(b)) + } + cr.err = errMalformedEncoding + return n, cr.err + } + + cr.offset = copy(buf, cr.buffer) + n += cr.offset + return n, err +} + +// readTrailers will read all trailers and populate cr.trailers with actual values. +func (cr *s3ChunkedReader) readTrailers() error { + if cr.debug { + fmt.Printf("pre trailer sig: %s\n", cr.seedSignature) + } + var valueBuffer bytes.Buffer + // Read value + for { + v, err := cr.reader.ReadByte() + if err != nil { + if err == io.EOF { + return io.ErrUnexpectedEOF + } + } + if v != '\r' { + valueBuffer.WriteByte(v) + continue + } + // End of buffer, do not add to value. + v, err = cr.reader.ReadByte() + if err != nil { + if err == io.EOF { + return io.ErrUnexpectedEOF + } + } + if v != '\n' { + return errMalformedEncoding + } + break + } + + // Read signature + var signatureBuffer bytes.Buffer + for { + v, err := cr.reader.ReadByte() + if err != nil { + if err == io.EOF { + return io.ErrUnexpectedEOF + } + } + if v != '\r' { + signatureBuffer.WriteByte(v) + continue + } + var tmp [3]byte + _, err = io.ReadFull(cr.reader, tmp[:]) + if err != nil { + if err == io.EOF { + return io.ErrUnexpectedEOF + } + } + if string(tmp[:]) != "\n\r\n" { + if cr.debug { + fmt.Printf("signature, want %q, got %q", "\n\r\n", string(tmp[:])) + } + return errMalformedEncoding + } + // No need to write final newlines to buffer. + break + } + + // Verify signature. + sig := signatureBuffer.Bytes() + if !bytes.HasPrefix(sig, []byte("x-amz-trailer-signature:")) { + if cr.debug { + fmt.Printf("prefix, want prefix %q, got %q", "x-amz-trailer-signature:", string(sig)) + } + return errMalformedEncoding + } + + // TODO: It seems like we may have to be prepared to rewrite and sort trailing headers: + // https://docs.aws.amazon.com/IAM/latest/UserGuide/create-signed-request.html + + // Any value must end with a newline. + // Not all clients send that. + trailerRaw := valueBuffer.Bytes() + if len(trailerRaw) > 0 && trailerRaw[len(trailerRaw)-1] != '\n' { + valueBuffer.Write([]byte{'\n'}) + } + sig = sig[len("x-amz-trailer-signature:"):] + sig = bytes.TrimSpace(sig) + cr.chunkSHA256Writer.Write(valueBuffer.Bytes()) + wantSig := cr.getTrailerChunkSignature() + if !compareSignatureV4(string(sig), wantSig) { + if cr.debug { + fmt.Printf("signature, want: %q, got %q\nSignature buffer: %q\n", wantSig, string(sig), valueBuffer.String()) + } + return errSignatureMismatch + } + + // Parse trailers. + wantTrailers := make(map[string]struct{}, len(cr.trailers)) + for k := range cr.trailers { + wantTrailers[strings.ToLower(k)] = struct{}{} + } + input := bufio.NewScanner(bytes.NewReader(valueBuffer.Bytes())) + for input.Scan() { + line := strings.TrimSpace(input.Text()) + if line == "" { + continue + } + // Find first separator. + idx := strings.IndexByte(line, trailerKVSeparator[0]) + if idx <= 0 || idx >= len(line) { + if cr.debug { + fmt.Printf("index, ':' not found in %q\n", line) + } + return errMalformedEncoding + } + key := line[:idx] + value := line[idx+1:] + if _, ok := wantTrailers[key]; !ok { + if cr.debug { + fmt.Printf("%q not found in %q\n", key, cr.trailers) + } + return errMalformedEncoding + } + cr.trailers.Set(key, value) + delete(wantTrailers, key) + } + + // Check if we got all we want. + if len(wantTrailers) > 0 { + return io.ErrUnexpectedEOF + } + return nil +} + +// readCRLF - check if reader only has '\r\n' CRLF character. +// returns malformed encoding if it doesn't. +func readCRLF(reader io.Reader) error { + buf := make([]byte, 2) + _, err := io.ReadFull(reader, buf[:2]) + if err != nil { + return err + } + if buf[0] != '\r' || buf[1] != '\n' { + return errMalformedEncoding + } + return nil +} + +// Read a line of bytes (up to \n) from b. +// Give up if the line exceeds maxLineLength. +// The returned bytes are owned by the bufio.Reader +// so they are only valid until the next bufio read. +func readChunkLine(b *bufio.Reader) ([]byte, []byte, error) { + buf, err := b.ReadSlice('\n') + if err != nil { + // We always know when EOF is coming. + // If the caller asked for a line, there should be a line. + switch err { + case io.EOF: + err = io.ErrUnexpectedEOF + case bufio.ErrBufferFull: + err = errLineTooLong + } + return nil, nil, err + } + if len(buf) >= maxLineLength { + return nil, nil, errLineTooLong + } + // Parse s3 specific chunk extension and fetch the values. + hexChunkSize, hexChunkSignature := parseS3ChunkExtension(buf) + return hexChunkSize, hexChunkSignature, nil +} + +// trimTrailingWhitespace - trim trailing white space. +func trimTrailingWhitespace(b []byte) []byte { + for len(b) > 0 && isASCIISpace(b[len(b)-1]) { + b = b[:len(b)-1] + } + return b +} + +// isASCIISpace - is ascii space? +func isASCIISpace(b byte) bool { + return b == ' ' || b == '\t' || b == '\n' || b == '\r' +} + +// Constant s3 chunk encoding signature. +const s3ChunkSignatureStr = ";chunk-signature=" + +// parses3ChunkExtension removes any s3 specific chunk-extension from buf. +// For example, +// +// "10000;chunk-signature=..." => "10000", "chunk-signature=..." +func parseS3ChunkExtension(buf []byte) ([]byte, []byte) { + buf = trimTrailingWhitespace(buf) + semi := bytes.Index(buf, []byte(s3ChunkSignatureStr)) + // Chunk signature not found, return the whole buffer. + if semi == -1 { + return buf, nil + } + return buf[:semi], parseChunkSignature(buf[semi:]) +} + +// parseChunkSignature - parse chunk signature. +func parseChunkSignature(chunk []byte) []byte { + chunkSplits := bytes.SplitN(chunk, []byte(s3ChunkSignatureStr), 2) + return chunkSplits[1] +} + +// parse hex to uint64. +func parseHexUint(v []byte) (n uint64, err error) { + for i, b := range v { + switch { + case '0' <= b && b <= '9': + b -= '0' + case 'a' <= b && b <= 'f': + b = b - 'a' + 10 + case 'A' <= b && b <= 'F': + b = b - 'A' + 10 + default: + return 0, errors.New("invalid byte in chunk length") + } + if i == 16 { + return 0, errors.New("http chunk length too large") + } + n <<= 4 + n |= uint64(b) + } + return n, err +} diff --git a/cmd/streaming-signature-v4_test.go b/cmd/streaming-signature-v4_test.go new file mode 100644 index 00000000..8ea1301c --- /dev/null +++ b/cmd/streaming-signature-v4_test.go @@ -0,0 +1,197 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bufio" + "bytes" + "fmt" + "io" + "strings" + "testing" +) + +// Test read chunk line. +func TestReadChunkLine(t *testing.T) { + type testCase struct { + reader *bufio.Reader + expectedErr error + chunkSize []byte + chunkSignature []byte + } + // List of readers used. + readers := []io.Reader{ + // Test - 1 + bytes.NewReader([]byte("1000;chunk-signature=111123333333333333334444211\r\n")), + // Test - 2 + bytes.NewReader([]byte("1000;")), + // Test - 3 + bytes.NewReader(fmt.Appendf(nil, "%4097d", 1)), + // Test - 4 + bytes.NewReader([]byte("1000;chunk-signature=111123333333333333334444211\r\n")), + } + testCases := []testCase{ + // Test - 1 - small bufio reader. + { + bufio.NewReaderSize(readers[0], 16), + errLineTooLong, + nil, + nil, + }, + // Test - 2 - unexpected end of the reader. + { + bufio.NewReader(readers[1]), + io.ErrUnexpectedEOF, + nil, + nil, + }, + // Test - 3 - line too long bigger than 4k+1 + { + bufio.NewReader(readers[2]), + errLineTooLong, + nil, + nil, + }, + // Test - 4 - parse the chunk reader properly. + { + bufio.NewReader(readers[3]), + nil, + []byte("1000"), + []byte("111123333333333333334444211"), + }, + } + // Valid test cases for each chunk line. + for i, tt := range testCases { + chunkSize, chunkSignature, err := readChunkLine(tt.reader) + if err != tt.expectedErr { + t.Errorf("Test %d: Expected %s, got %s", i+1, tt.expectedErr, err) + } + if !bytes.Equal(chunkSize, tt.chunkSize) { + t.Errorf("Test %d: Expected %s, got %s", i+1, string(tt.chunkSize), string(chunkSize)) + } + if !bytes.Equal(chunkSignature, tt.chunkSignature) { + t.Errorf("Test %d: Expected %s, got %s", i+1, string(tt.chunkSignature), string(chunkSignature)) + } + } +} + +// Test parsing s3 chunk extension. +func TestParseS3ChunkExtension(t *testing.T) { + type testCase struct { + buf []byte + chunkSize []byte + chunkSign []byte + } + + tests := []testCase{ + // Test - 1 valid case. + { + []byte("10000;chunk-signature=ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648"), + []byte("10000"), + []byte("ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648"), + }, + // Test - 2 no chunk extension, return same buffer. + { + []byte("10000;"), + []byte("10000;"), + nil, + }, + // Test - 3 no chunk size, return error. + { + []byte(";chunk-signature="), + nil, + nil, + }, + // Test - 4 removes trailing slash. + { + []byte("10000;chunk-signature=ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648 \t \n"), + []byte("10000"), + []byte("ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648"), + }, + } + // Validate chunk extension removal. + for i, tt := range tests { + // Extract chunk size and chunk signature after parsing a standard chunk-extension format. + hexChunkSize, hexChunkSignature := parseS3ChunkExtension(tt.buf) + if !bytes.Equal(hexChunkSize, tt.chunkSize) { + t.Errorf("Test %d: Expected %s, got %s", i+1, string(tt.chunkSize), string(hexChunkSize)) + } + if !bytes.Equal(hexChunkSignature, tt.chunkSign) { + t.Errorf("Test %d: Expected %s, got %s", i+1, string(tt.chunkSign), string(hexChunkSignature)) + } + } +} + +// Test read CRLF characters on input reader. +func TestReadCRLF(t *testing.T) { + type testCase struct { + reader io.Reader + expectedErr error + } + tests := []testCase{ + // Test - 1 valid buffer with CRLF. + {bytes.NewReader([]byte("\r\n")), nil}, + // Test - 2 invalid buffer with no CRLF. + {bytes.NewReader([]byte("he")), errMalformedEncoding}, + // Test - 3 invalid buffer with more characters. + {bytes.NewReader([]byte("he\r\n")), errMalformedEncoding}, + // Test - 4 smaller buffer than expected. + {bytes.NewReader([]byte("h")), io.ErrUnexpectedEOF}, + } + for i, tt := range tests { + err := readCRLF(tt.reader) + if err != tt.expectedErr { + t.Errorf("Test %d: Expected %s, got %s this", i+1, tt.expectedErr, err) + } + } +} + +// Tests parsing hex number into its uint64 decimal equivalent. +func TestParseHexUint(t *testing.T) { + type testCase struct { + in string + want uint64 + wantErr string + } + tests := []testCase{ + {"x", 0, "invalid byte in chunk length"}, + {"0000000000000000", 0, ""}, + {"0000000000000001", 1, ""}, + {"ffffffffffffffff", 1<<64 - 1, ""}, + {"FFFFFFFFFFFFFFFF", 1<<64 - 1, ""}, + {"000000000000bogus", 0, "invalid byte in chunk length"}, + {"00000000000000000", 0, "http chunk length too large"}, // could accept if we wanted + {"10000000000000000", 0, "http chunk length too large"}, + {"00000000000000001", 0, "http chunk length too large"}, // could accept if we wanted + } + for i := uint64(0); i <= 1234; i++ { + tests = append(tests, testCase{in: fmt.Sprintf("%x", i), want: i}) + } + for _, tt := range tests { + got, err := parseHexUint([]byte(tt.in)) + if tt.wantErr != "" { + if err != nil && !strings.Contains(err.Error(), tt.wantErr) { + t.Errorf("parseHexUint(%q) = %v, %v; want error %q", tt.in, got, err, tt.wantErr) + } + } else { + if err != nil || got != tt.want { + t.Errorf("parseHexUint(%q) = %v, %v; want %v", tt.in, got, err, tt.want) + } + } + } +} diff --git a/cmd/streaming-v4-unsigned.go b/cmd/streaming-v4-unsigned.go new file mode 100644 index 00000000..a3166867 --- /dev/null +++ b/cmd/streaming-v4-unsigned.go @@ -0,0 +1,262 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bufio" + "bytes" + "fmt" + "io" + "net/http" + "strings" +) + +// newUnsignedV4ChunkedReader returns a new s3UnsignedChunkedReader that translates the data read from r +// out of HTTP "chunked" format before returning it. +// The s3ChunkedReader returns io.EOF when the final 0-length chunk is read. +func newUnsignedV4ChunkedReader(req *http.Request, trailer bool, signature bool) (io.ReadCloser, APIErrorCode) { + if signature { + if errCode := doesSignatureMatch(unsignedPayloadTrailer, req, globalSite.Region(), serviceS3); errCode != ErrNone { + return nil, errCode + } + } + if trailer { + // Discard anything unsigned. + req.Trailer = make(http.Header) + trailers := req.Header.Values(awsTrailerHeader) + for _, key := range trailers { + req.Trailer.Add(key, "") + } + } else { + req.Trailer = nil + } + return &s3UnsignedChunkedReader{ + trailers: req.Trailer, + reader: bufio.NewReader(req.Body), + buffer: make([]byte, 64*1024), + }, ErrNone +} + +// Represents the overall state that is required for decoding a +// AWS Signature V4 chunked reader. +type s3UnsignedChunkedReader struct { + reader *bufio.Reader + trailers http.Header + + buffer []byte + offset int + err error + debug bool +} + +func (cr *s3UnsignedChunkedReader) Close() (err error) { + return cr.err +} + +// Read - implements `io.Reader`, which transparently decodes +// the incoming AWS Signature V4 streaming signature. +func (cr *s3UnsignedChunkedReader) Read(buf []byte) (n int, err error) { + // First, if there is any unread data, copy it to the client + // provided buffer. + if cr.offset > 0 { + n = copy(buf, cr.buffer[cr.offset:]) + if n == len(buf) { + cr.offset += n + return n, nil + } + cr.offset = 0 + buf = buf[n:] + } + // mustRead reads from input and compares against provided slice. + mustRead := func(b ...byte) error { + for _, want := range b { + got, err := cr.reader.ReadByte() + if err == io.EOF { + return io.ErrUnexpectedEOF + } + if got != want { + if cr.debug { + fmt.Printf("mustread: want: %q got: %q\n", string(want), string(got)) + } + return errMalformedEncoding + } + if err != nil { + return err + } + } + return nil + } + var size int + for { + b, err := cr.reader.ReadByte() + if err == io.EOF { + err = io.ErrUnexpectedEOF + } + if err != nil { + cr.err = err + return n, cr.err + } + if b == '\r' { // \r\n denotes end of size. + err := mustRead('\n') + if err != nil { + cr.err = err + return n, cr.err + } + break + } + + // Manually deserialize the size since AWS specified + // the chunk size to be of variable width. In particular, + // a size of 16 is encoded as `10` while a size of 64 KB + // is `10000`. + switch { + case b >= '0' && b <= '9': + size = size<<4 | int(b-'0') + case b >= 'a' && b <= 'f': + size = size<<4 | int(b-('a'-10)) + case b >= 'A' && b <= 'F': + size = size<<4 | int(b-('A'-10)) + default: + if cr.debug { + fmt.Printf("err size: %v\n", string(b)) + } + cr.err = errMalformedEncoding + return n, cr.err + } + if size > maxChunkSize { + cr.err = errChunkTooBig + return n, cr.err + } + } + + if cap(cr.buffer) < size { + cr.buffer = make([]byte, size) + } else { + cr.buffer = cr.buffer[:size] + } + + // Now, we read the payload. + _, err = io.ReadFull(cr.reader, cr.buffer) + if err == io.EOF && size != 0 { + err = io.ErrUnexpectedEOF + } + if err != nil && err != io.EOF { + cr.err = err + return n, cr.err + } + + // If the chunk size is zero we return io.EOF. As specified by AWS, + // only the last chunk is zero-sized. + if len(cr.buffer) == 0 { + if cr.debug { + fmt.Println("EOF") + } + if cr.trailers != nil { + err = cr.readTrailers() + if cr.debug { + fmt.Println("trailer returned:", err) + } + if err != nil { + cr.err = err + return 0, err + } + } + cr.err = io.EOF + return n, cr.err + } + // read final terminator. + err = mustRead('\r', '\n') + if err != nil && err != io.EOF { + cr.err = err + return n, cr.err + } + + cr.offset = copy(buf, cr.buffer) + n += cr.offset + return n, err +} + +// readTrailers will read all trailers and populate cr.trailers with actual values. +func (cr *s3UnsignedChunkedReader) readTrailers() error { + var valueBuffer bytes.Buffer + // Read value + for { + v, err := cr.reader.ReadByte() + if err != nil { + if err == io.EOF { + return io.ErrUnexpectedEOF + } + } + if v != '\r' { + valueBuffer.WriteByte(v) + continue + } + // Must end with \r\n\r\n + var tmp [3]byte + _, err = io.ReadFull(cr.reader, tmp[:]) + if err != nil { + if err == io.EOF { + return io.ErrUnexpectedEOF + } + } + if !bytes.Equal(tmp[:], []byte{'\n', '\r', '\n'}) { + if cr.debug { + fmt.Printf("got %q, want %q\n", string(tmp[:]), "\n\r\n") + } + return errMalformedEncoding + } + break + } + + // Parse trailers. + wantTrailers := make(map[string]struct{}, len(cr.trailers)) + for k := range cr.trailers { + wantTrailers[strings.ToLower(k)] = struct{}{} + } + input := bufio.NewScanner(bytes.NewReader(valueBuffer.Bytes())) + for input.Scan() { + line := strings.TrimSpace(input.Text()) + if line == "" { + continue + } + // Find first separator. + idx := strings.IndexByte(line, trailerKVSeparator[0]) + if idx <= 0 || idx >= len(line) { + if cr.debug { + fmt.Printf("Could not find separator, got %q\n", line) + } + return errMalformedEncoding + } + key := strings.ToLower(line[:idx]) + value := line[idx+1:] + if _, ok := wantTrailers[key]; !ok { + if cr.debug { + fmt.Printf("Unknown key %q - expected on of %v\n", key, cr.trailers) + } + return errMalformedEncoding + } + cr.trailers.Set(key, value) + delete(wantTrailers, key) + } + + // Check if we got all we want. + if len(wantTrailers) > 0 { + return io.ErrUnexpectedEOF + } + return nil +} diff --git a/cmd/sts-datatypes.go b/cmd/sts-datatypes.go new file mode 100644 index 00000000..17b4920d --- /dev/null +++ b/cmd/sts-datatypes.go @@ -0,0 +1,218 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "encoding/xml" + + "github.com/minio/minio/internal/auth" +) + +// AssumedRoleUser - The identifiers for the temporary security credentials that +// the operation returns. Please also see https://docs.aws.amazon.com/goto/WebAPI/sts-2011-06-15/AssumedRoleUser +type AssumedRoleUser struct { + // The ARN of the temporary security credentials that are returned from the + // AssumeRole action. For more information about ARNs and how to use them in + // policies, see IAM Identifiers (http://docs.aws.amazon.com/IAM/latest/UserGuide/reference_identifiers.html) + // in Using IAM. + // + // Arn is a required field + Arn string + + // A unique identifier that contains the role ID and the role session name of + // the role that is being assumed. The role ID is generated by AWS when the + // role is created. + // + // AssumedRoleId is a required field + AssumedRoleID string `xml:"AssumeRoleId"` + // contains filtered or unexported fields +} + +// AssumeRoleResponse contains the result of successful AssumeRole request. +type AssumeRoleResponse struct { + XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleResponse" json:"-"` + + Result AssumeRoleResult `xml:"AssumeRoleResult"` + ResponseMetadata struct { + RequestID string `xml:"RequestId,omitempty"` + } `xml:"ResponseMetadata,omitempty"` +} + +// AssumeRoleResult - Contains the response to a successful AssumeRole +// request, including temporary credentials that can be used to make +// MinIO API requests. +type AssumeRoleResult struct { + // The identifiers for the temporary security credentials that the operation + // returns. + AssumedRoleUser AssumedRoleUser `xml:",omitempty"` + + // The temporary security credentials, which include an access key ID, a secret + // access key, and a security (or session) token. + // + // Note: The size of the security token that STS APIs return is not fixed. We + // strongly recommend that you make no assumptions about the maximum size. As + // of this writing, the typical size is less than 4096 bytes, but that can vary. + // Also, future updates to AWS might require larger sizes. + Credentials auth.Credentials `xml:",omitempty"` + + // A percentage value that indicates the size of the policy in packed form. + // The service rejects any policy with a packed size greater than 100 percent, + // which means the policy exceeded the allowed space. + PackedPolicySize int `xml:",omitempty"` +} + +// AssumeRoleWithWebIdentityResponse contains the result of successful AssumeRoleWithWebIdentity request. +type AssumeRoleWithWebIdentityResponse struct { + XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithWebIdentityResponse" json:"-"` + Result WebIdentityResult `xml:"AssumeRoleWithWebIdentityResult"` + ResponseMetadata struct { + RequestID string `xml:"RequestId,omitempty"` + } `xml:"ResponseMetadata,omitempty"` +} + +// WebIdentityResult - Contains the response to a successful AssumeRoleWithWebIdentity +// request, including temporary credentials that can be used to make MinIO API requests. +type WebIdentityResult struct { + // The identifiers for the temporary security credentials that the operation + // returns. + AssumedRoleUser AssumedRoleUser `xml:",omitempty"` + + // The intended audience (also known as client ID) of the web identity token. + // This is traditionally the client identifier issued to the application that + // requested the client grants. + Audience string `xml:",omitempty"` + + // The temporary security credentials, which include an access key ID, a secret + // access key, and a security (or session) token. + // + // Note: The size of the security token that STS APIs return is not fixed. We + // strongly recommend that you make no assumptions about the maximum size. As + // of this writing, the typical size is less than 4096 bytes, but that can vary. + // Also, future updates to AWS might require larger sizes. + Credentials auth.Credentials `xml:",omitempty"` + + // A percentage value that indicates the size of the policy in packed form. + // The service rejects any policy with a packed size greater than 100 percent, + // which means the policy exceeded the allowed space. + PackedPolicySize int `xml:",omitempty"` + + // The issuing authority of the web identity token presented. For OpenID Connect + // ID tokens, this contains the value of the iss field. For OAuth 2.0 id_tokens, + // this contains the value of the ProviderId parameter that was passed in the + // AssumeRoleWithWebIdentity request. + Provider string `xml:",omitempty"` + + // The unique user identifier that is returned by the identity provider. + // This identifier is associated with the Token that was submitted + // with the AssumeRoleWithWebIdentity call. The identifier is typically unique to + // the user and the application that acquired the WebIdentityToken (pairwise identifier). + // For OpenID Connect ID tokens, this field contains the value returned by the identity + // provider as the token's sub (Subject) claim. + SubjectFromWebIdentityToken string `xml:",omitempty"` +} + +// AssumeRoleWithClientGrantsResponse contains the result of successful AssumeRoleWithClientGrants request. +type AssumeRoleWithClientGrantsResponse struct { + XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithClientGrantsResponse" json:"-"` + Result ClientGrantsResult `xml:"AssumeRoleWithClientGrantsResult"` + ResponseMetadata struct { + RequestID string `xml:"RequestId,omitempty"` + } `xml:"ResponseMetadata,omitempty"` +} + +// ClientGrantsResult - Contains the response to a successful AssumeRoleWithClientGrants +// request, including temporary credentials that can be used to make MinIO API requests. +type ClientGrantsResult struct { + // The identifiers for the temporary security credentials that the operation + // returns. + AssumedRoleUser AssumedRoleUser `xml:",omitempty"` + + // The intended audience (also known as client ID) of the web identity token. + // This is traditionally the client identifier issued to the application that + // requested the client grants. + Audience string `xml:",omitempty"` + + // The temporary security credentials, which include an access key ID, a secret + // access key, and a security (or session) token. + // + // Note: The size of the security token that STS APIs return is not fixed. We + // strongly recommend that you make no assumptions about the maximum size. As + // of this writing, the typical size is less than 4096 bytes, but that can vary. + // Also, future updates to AWS might require larger sizes. + Credentials auth.Credentials `xml:",omitempty"` + + // A percentage value that indicates the size of the policy in packed form. + // The service rejects any policy with a packed size greater than 100 percent, + // which means the policy exceeded the allowed space. + PackedPolicySize int `xml:",omitempty"` + + // The issuing authority of the web identity token presented. For OpenID Connect + // ID tokens, this contains the value of the iss field. For OAuth 2.0 id_tokens, + // this contains the value of the ProviderId parameter that was passed in the + // AssumeRoleWithClientGrants request. + Provider string `xml:",omitempty"` + + // The unique user identifier that is returned by the identity provider. + // This identifier is associated with the Token that was submitted + // with the AssumeRoleWithClientGrants call. The identifier is typically unique to + // the user and the application that acquired the ClientGrantsToken (pairwise identifier). + // For OpenID Connect ID tokens, this field contains the value returned by the identity + // provider as the token's sub (Subject) claim. + SubjectFromToken string `xml:",omitempty"` +} + +// AssumeRoleWithLDAPResponse contains the result of successful +// AssumeRoleWithLDAPIdentity request +type AssumeRoleWithLDAPResponse struct { + XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithLDAPIdentityResponse" json:"-"` + Result LDAPIdentityResult `xml:"AssumeRoleWithLDAPIdentityResult"` + ResponseMetadata struct { + RequestID string `xml:"RequestId,omitempty"` + } `xml:"ResponseMetadata,omitempty"` +} + +// LDAPIdentityResult - contains credentials for a successful +// AssumeRoleWithLDAPIdentity request. +type LDAPIdentityResult struct { + Credentials auth.Credentials `xml:",omitempty"` +} + +// AssumeRoleWithCertificateResponse contains the result of +// a successful AssumeRoleWithCertificate request. +type AssumeRoleWithCertificateResponse struct { + XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithCertificateResponse" json:"-"` + Result struct { + Credentials auth.Credentials `xml:"Credentials,omitempty"` + } `xml:"AssumeRoleWithCertificateResult"` + Metadata struct { + RequestID string `xml:"RequestId,omitempty"` + } `xml:"ResponseMetadata,omitempty"` +} + +// AssumeRoleWithCustomTokenResponse contains the result of a successful +// AssumeRoleWithCustomToken request. +type AssumeRoleWithCustomTokenResponse struct { + XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ AssumeRoleWithCustomTokenResponse" json:"-"` + Result struct { + Credentials auth.Credentials `xml:"Credentials,omitempty"` + AssumedUser string `xml:"AssumedUser,omitempty"` + } `xml:"AssumeRoleWithCustomTokenResult"` + Metadata struct { + RequestID string `xml:"RequestId,omitempty"` + } `xml:"ResponseMetadata,omitempty"` +} diff --git a/cmd/sts-errors.go b/cmd/sts-errors.go new file mode 100644 index 00000000..c68b68f1 --- /dev/null +++ b/cmd/sts-errors.go @@ -0,0 +1,174 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "encoding/xml" + "net/http" + + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" +) + +// writeSTSErrorResponse writes error headers +func writeSTSErrorResponse(ctx context.Context, w http.ResponseWriter, errCode STSErrorCode, err error) { + stsErr := stsErrCodes.ToSTSErr(errCode) + + // Generate error response. + stsErrorResponse := STSErrorResponse{} + stsErrorResponse.Error.Code = stsErr.Code + stsErrorResponse.RequestID = w.Header().Get(xhttp.AmzRequestID) + stsErrorResponse.Error.Message = stsErr.Description + if err != nil { + stsErrorResponse.Error.Message = err.Error() + } + switch errCode { + case ErrSTSInternalError, ErrSTSUpstreamError: + stsLogIf(ctx, err, logger.ErrorKind) + } + encodedErrorResponse := encodeResponse(stsErrorResponse) + writeResponse(w, stsErr.HTTPStatusCode, encodedErrorResponse, mimeXML) +} + +// STSError structure +type STSError struct { + Code string + Description string + HTTPStatusCode int +} + +// STSErrorResponse - error response format +type STSErrorResponse struct { + XMLName xml.Name `xml:"https://sts.amazonaws.com/doc/2011-06-15/ ErrorResponse" json:"-"` + Error struct { + Type string `xml:"Type"` + Code string `xml:"Code"` + Message string `xml:"Message"` + } `xml:"Error"` + RequestID string `xml:"RequestId"` +} + +// STSErrorCode type of error status. +type STSErrorCode int + +//go:generate stringer -type=STSErrorCode -trimprefix=Err $GOFILE + +// Error codes, non exhaustive list - http://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithSAML.html +const ( + ErrSTSNone STSErrorCode = iota + ErrSTSAccessDenied + ErrSTSMissingParameter + ErrSTSInvalidParameterValue + ErrSTSWebIdentityExpiredToken + ErrSTSClientGrantsExpiredToken + ErrSTSInvalidClientGrantsToken + ErrSTSMalformedPolicyDocument + ErrSTSInsecureConnection + ErrSTSInvalidClientCertificate + ErrSTSTooManyIntermediateCAs + ErrSTSNotInitialized + ErrSTSIAMNotInitialized + ErrSTSUpstreamError + ErrSTSInternalError +) + +type stsErrorCodeMap map[STSErrorCode]STSError + +func (e stsErrorCodeMap) ToSTSErr(errCode STSErrorCode) STSError { + apiErr, ok := e[errCode] + if !ok { + return e[ErrSTSInternalError] + } + return apiErr +} + +// error code to STSError structure, these fields carry respective +// descriptions for all the error responses. +var stsErrCodes = stsErrorCodeMap{ + ErrSTSAccessDenied: { + Code: "AccessDenied", + Description: "Generating temporary credentials not allowed for this request.", + HTTPStatusCode: http.StatusForbidden, + }, + ErrSTSMissingParameter: { + Code: "MissingParameter", + Description: "A required parameter for the specified action is not supplied.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrSTSInvalidParameterValue: { + Code: "InvalidParameterValue", + Description: "An invalid or out-of-range value was supplied for the input parameter.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrSTSWebIdentityExpiredToken: { + Code: "ExpiredToken", + Description: "The web identity token that was passed is expired or is not valid. Get a new identity token from the identity provider and then retry the request.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrSTSClientGrantsExpiredToken: { + Code: "ExpiredToken", + Description: "The client grants that was passed is expired or is not valid. Get a new client grants token from the identity provider and then retry the request.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrSTSInvalidClientGrantsToken: { + Code: "InvalidClientGrantsToken", + Description: "The client grants token that was passed could not be validated by MinIO.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrSTSMalformedPolicyDocument: { + Code: "MalformedPolicyDocument", + Description: "The request was rejected because the policy document was malformed.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrSTSInsecureConnection: { + Code: "InsecureConnection", + Description: "The request was made over a plain HTTP connection. A TLS connection is required.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrSTSInvalidClientCertificate: { + Code: "InvalidClientCertificate", + Description: "The provided client certificate is invalid. Retry with a different certificate.", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrSTSTooManyIntermediateCAs: { + Code: "TooManyIntermediateCAs", + Description: "The provided client certificate contains too many intermediate CA certificates", + HTTPStatusCode: http.StatusBadRequest, + }, + ErrSTSNotInitialized: { + Code: "STSNotInitialized", + Description: "STS API not initialized, please try again.", + HTTPStatusCode: http.StatusServiceUnavailable, + }, + ErrSTSIAMNotInitialized: { + Code: "STSIAMNotInitialized", + Description: "STS IAM not initialized, please try again.", + HTTPStatusCode: http.StatusServiceUnavailable, + }, + ErrSTSUpstreamError: { + Code: "InternalError", + Description: "An upstream service required for this operation failed - please try again or contact an administrator.", + HTTPStatusCode: http.StatusInternalServerError, + }, + ErrSTSInternalError: { + Code: "InternalError", + Description: "We encountered an internal error generating credentials, please try again.", + HTTPStatusCode: http.StatusInternalServerError, + }, +} diff --git a/cmd/sts-handlers.go b/cmd/sts-handlers.go new file mode 100644 index 00000000..c4092bca --- /dev/null +++ b/cmd/sts-handlers.go @@ -0,0 +1,1120 @@ +/// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/x509" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/config/identity/openid" + "github.com/minio/minio/internal/hash/sha256" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" + "github.com/minio/mux" + "github.com/minio/pkg/v3/policy" + "github.com/minio/pkg/v3/wildcard" +) + +const ( + // STS API version. + stsAPIVersion = "2011-06-15" + stsVersion = "Version" + stsAction = "Action" + stsPolicy = "Policy" + stsToken = "Token" + stsRoleArn = "RoleArn" + stsWebIdentityToken = "WebIdentityToken" + stsWebIdentityAccessToken = "WebIdentityAccessToken" // only valid if UserInfo is enabled. + stsDurationSeconds = "DurationSeconds" + stsLDAPUsername = "LDAPUsername" + stsLDAPPassword = "LDAPPassword" + stsRevokeTokenType = "TokenRevokeType" + + // STS API action constants + clientGrants = "AssumeRoleWithClientGrants" + webIdentity = "AssumeRoleWithWebIdentity" + ldapIdentity = "AssumeRoleWithLDAPIdentity" + clientCertificate = "AssumeRoleWithCertificate" + customTokenIdentity = "AssumeRoleWithCustomToken" + assumeRole = "AssumeRole" + + stsRequestBodyLimit = 10 * (1 << 20) // 10 MiB + + // JWT claim keys + expClaim = "exp" + subClaim = "sub" + audClaim = "aud" + issClaim = "iss" + + // JWT claim to check the parent user + parentClaim = "parent" + + // LDAP claim keys + ldapUser = "ldapUser" // this is a key name for a normalized DN value + ldapActualUser = "ldapActualUser" // this is a key name for the actual DN value + ldapUserN = "ldapUsername" // this is a key name for the short/login username + // Claim key-prefix for LDAP attributes + ldapAttribPrefix = "ldapAttrib_" + + // Role Claim key + roleArnClaim = "roleArn" + + // STS revoke type claim key + tokenRevokeTypeClaim = "tokenRevokeType" + + // maximum supported STS session policy size + maxSTSSessionPolicySize = 2048 +) + +type stsClaims map[string]any + +func (c stsClaims) populateSessionPolicy(form url.Values) error { + if len(form) == 0 { + return nil + } + + sessionPolicyStr := form.Get(stsPolicy) + if len(sessionPolicyStr) == 0 { + return nil + } + + sessionPolicy, err := policy.ParseConfig(bytes.NewReader([]byte(sessionPolicyStr))) + if err != nil { + return err + } + + // Version in policy must not be empty + if sessionPolicy.Version == "" { + return errors.New("Version cannot be empty expecting '2012-10-17'") + } + + policyBuf, err := json.Marshal(sessionPolicy) + if err != nil { + return err + } + + // https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html + // https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html + // The plain text that you use for both inline and managed session + // policies shouldn't exceed maxSTSSessionPolicySize characters. + if len(policyBuf) > maxSTSSessionPolicySize { + return errSessionPolicyTooLarge + } + + c[policy.SessionPolicyName] = base64.StdEncoding.EncodeToString(policyBuf) + return nil +} + +// stsAPIHandlers implements and provides http handlers for AWS STS API. +type stsAPIHandlers struct{} + +// registerSTSRouter - registers AWS STS compatible APIs. +func registerSTSRouter(router *mux.Router) { + // Initialize STS. + sts := &stsAPIHandlers{} + + // STS Router + stsRouter := router.NewRoute().PathPrefix(SlashSeparator).Subrouter() + + // Assume roles with no JWT, handles AssumeRole. + stsRouter.Methods(http.MethodPost).MatcherFunc(func(r *http.Request, rm *mux.RouteMatch) bool { + ctypeOk := wildcard.MatchSimple("application/x-www-form-urlencoded*", r.Header.Get(xhttp.ContentType)) + authOk := wildcard.MatchSimple(signV4Algorithm+"*", r.Header.Get(xhttp.Authorization)) + noQueries := len(r.URL.RawQuery) == 0 + return ctypeOk && authOk && noQueries + }).HandlerFunc(httpTraceAll(sts.AssumeRole)) + + // Assume roles with JWT handler, handles both ClientGrants and WebIdentity. + stsRouter.Methods(http.MethodPost).MatcherFunc(func(r *http.Request, rm *mux.RouteMatch) bool { + ctypeOk := wildcard.MatchSimple("application/x-www-form-urlencoded*", r.Header.Get(xhttp.ContentType)) + noQueries := len(r.URL.RawQuery) == 0 + return ctypeOk && noQueries + }).HandlerFunc(httpTraceAll(sts.AssumeRoleWithSSO)) + + // AssumeRoleWithClientGrants + stsRouter.Methods(http.MethodPost).HandlerFunc(httpTraceAll(sts.AssumeRoleWithClientGrants)). + Queries(stsAction, clientGrants). + Queries(stsVersion, stsAPIVersion). + Queries(stsToken, "{Token:.*}") + + // AssumeRoleWithWebIdentity + stsRouter.Methods(http.MethodPost).HandlerFunc(httpTraceAll(sts.AssumeRoleWithWebIdentity)). + Queries(stsAction, webIdentity). + Queries(stsVersion, stsAPIVersion). + Queries(stsWebIdentityToken, "{Token:.*}") + + // AssumeRoleWithLDAPIdentity + stsRouter.Methods(http.MethodPost).HandlerFunc(httpTraceAll(sts.AssumeRoleWithLDAPIdentity)). + Queries(stsAction, ldapIdentity). + Queries(stsVersion, stsAPIVersion). + Queries(stsLDAPUsername, "{LDAPUsername:.*}"). + Queries(stsLDAPPassword, "{LDAPPassword:.*}") + + // AssumeRoleWithCertificate + stsRouter.Methods(http.MethodPost).HandlerFunc(httpTraceAll(sts.AssumeRoleWithCertificate)). + Queries(stsAction, clientCertificate). + Queries(stsVersion, stsAPIVersion) + + // AssumeRoleWithCustomToken + stsRouter.Methods(http.MethodPost).HandlerFunc(httpTraceAll(sts.AssumeRoleWithCustomToken)). + Queries(stsAction, customTokenIdentity). + Queries(stsVersion, stsAPIVersion) +} + +func apiToSTSError(authErr APIErrorCode) (stsErrCode STSErrorCode) { + switch authErr { + case ErrSignatureDoesNotMatch, ErrInvalidAccessKeyID, ErrAccessKeyDisabled: + return ErrSTSAccessDenied + case ErrServerNotInitialized: + return ErrSTSNotInitialized + case ErrInternalError: + return ErrSTSInternalError + default: + return ErrSTSAccessDenied + } +} + +func checkAssumeRoleAuth(ctx context.Context, r *http.Request) (auth.Credentials, APIErrorCode) { + if !isRequestSignatureV4(r) { + return auth.Credentials{}, ErrAccessDenied + } + + s3Err := isReqAuthenticated(ctx, r, globalSite.Region(), serviceSTS) + if s3Err != ErrNone { + return auth.Credentials{}, s3Err + } + + user, _, s3Err := getReqAccessKeyV4(r, globalSite.Region(), serviceSTS) + if s3Err != ErrNone { + return auth.Credentials{}, s3Err + } + + // Temporary credentials or Service accounts cannot generate further temporary credentials. + if user.IsTemp() || user.IsServiceAccount() { + return auth.Credentials{}, ErrAccessDenied + } + + // Session tokens are not allowed in STS AssumeRole requests. + if getSessionToken(r) != "" { + return auth.Credentials{}, ErrAccessDenied + } + + return user, ErrNone +} + +func parseForm(r *http.Request) error { + if err := r.ParseForm(); err != nil { + return err + } + for k, v := range r.PostForm { + if _, ok := r.Form[k]; !ok { + r.Form[k] = v + } + } + return nil +} + +// getTokenSigningKey returns secret key used to sign JWT session tokens +func getTokenSigningKey() (string, error) { + secret := globalActiveCred.SecretKey + if globalSiteReplicationSys.isEnabled() { + secretKey, err := globalSiteReplicatorCred.Get(GlobalContext) + if err != nil { + return "", err + } + return secretKey, nil + } + return secret, nil +} + +// AssumeRole - implementation of AWS STS API AssumeRole to get temporary +// credentials for regular users on Minio. +// https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html +func (sts *stsAPIHandlers) AssumeRole(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "AssumeRole") + + claims := stsClaims{} + defer logger.AuditLog(ctx, w, r, claims) + + // Check auth here (otherwise r.Form will have unexpected values from + // the call to `parseForm` below), but return failure only after we are + // able to validate that it is a valid STS request, so that we are able + // to send an appropriate audit log. + user, apiErrCode := checkAssumeRoleAuth(ctx, r) + + if err := parseForm(r); err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err) + return + } + + if r.Form.Get(stsVersion) != stsAPIVersion { + writeSTSErrorResponse(ctx, w, ErrSTSMissingParameter, fmt.Errorf("Invalid STS API version %s, expecting %s", r.Form.Get(stsVersion), stsAPIVersion)) + return + } + + action := r.Form.Get(stsAction) + switch action { + case assumeRole: + default: + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Unsupported action %s", action)) + return + } + + ctx = newContext(r, w, action) + + // Validate the authentication result here so that failures will be audit-logged. + if apiErrCode != ErrNone { + stsErr := apiToSTSError(apiErrCode) + // Borrow the description error from the API error code + writeSTSErrorResponse(ctx, w, stsErr, errors.New(errorCodes[apiErrCode].Description)) + return + } + + if err := claims.populateSessionPolicy(r.Form); err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err) + return + } + + duration, err := openid.GetDefaultExpiration(r.Form.Get(stsDurationSeconds)) + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err) + return + } + + claims[expClaim] = UTCNow().Add(duration).Unix() + claims[parentClaim] = user.AccessKey + + tokenRevokeType := r.Form.Get(stsRevokeTokenType) + if tokenRevokeType != "" { + claims[tokenRevokeTypeClaim] = tokenRevokeType + } + + // Validate that user.AccessKey's policies can be retrieved - it may not + // be in case the user is disabled. + if _, err = globalIAMSys.PolicyDBGet(user.AccessKey, user.Groups...); err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err) + return + } + + secret, err := getTokenSigningKey() + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err) + return + } + + cred, err := auth.GetNewCredentialsWithMetadata(claims, secret) + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err) + return + } + + // Set the parent of the temporary access key, so that it's access + // policy is inherited from `user.AccessKey`. + cred.ParentUser = user.AccessKey + + // Set the newly generated credentials. + updatedAt, err := globalIAMSys.SetTempUser(ctx, cred.AccessKey, cred, "") + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err) + return + } + + // Call hook for site replication. + if cred.ParentUser != globalActiveCred.AccessKey { + replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemSTSAcc, + STSCredential: &madmin.SRSTSCredential{ + AccessKey: cred.AccessKey, + SecretKey: cred.SecretKey, + SessionToken: cred.SessionToken, + ParentUser: cred.ParentUser, + }, + UpdatedAt: updatedAt, + })) + } + + assumeRoleResponse := &AssumeRoleResponse{ + Result: AssumeRoleResult{ + Credentials: cred, + }, + } + + assumeRoleResponse.ResponseMetadata.RequestID = w.Header().Get(xhttp.AmzRequestID) + writeSuccessResponseXML(w, encodeResponse(assumeRoleResponse)) +} + +func (sts *stsAPIHandlers) AssumeRoleWithSSO(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "AssumeRoleSSOCommon") + + claims := stsClaims{} + defer logger.AuditLog(ctx, w, r, claims) + + // Parse the incoming form data. + if err := parseForm(r); err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err) + return + } + + if r.Form.Get(stsVersion) != stsAPIVersion { + writeSTSErrorResponse(ctx, w, ErrSTSMissingParameter, fmt.Errorf("Invalid STS API version %s, expecting %s", r.Form.Get("Version"), stsAPIVersion)) + return + } + + action := r.Form.Get(stsAction) + switch action { + case ldapIdentity: + sts.AssumeRoleWithLDAPIdentity(w, r) + return + case clientGrants, webIdentity: + default: + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Unsupported action %s", action)) + return + } + + ctx = newContext(r, w, action) + + token := r.Form.Get(stsToken) + if token == "" { + token = r.Form.Get(stsWebIdentityToken) + } + + accessToken := r.Form.Get(stsWebIdentityAccessToken) + + // RoleARN parameter processing: If a role ARN is given in the request, we + // use that and validate the authentication request. If not, we assume this + // is an STS request for a claim based IDP (if one is present) and set + // roleArn = openid.DummyRoleARN. + // + // Currently, we do not support multiple claim based IDPs, as there is no + // defined parameter to disambiguate the intended IDP in this STS request. + roleArn := openid.DummyRoleARN + roleArnStr := r.Form.Get(stsRoleArn) + isRolePolicyProvider := roleArnStr != "" + if isRolePolicyProvider { + var err error + roleArn, _, err = globalIAMSys.GetRolePolicy(roleArnStr) + if err != nil { + // If there is no claim-based provider configured, then an + // unrecognized roleArn is an error + if strings.TrimSpace(iamPolicyClaimNameOpenID()) == "" { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, + fmt.Errorf("Error processing %s parameter: %v", stsRoleArn, err)) + return + } + // If there *is* a claim-based provider configured, then + // treat an unrecognized roleArn the same as no roleArn + // at all. This is to support clients like the AWS SDKs + // or CLI that will not allow an AssumeRoleWithWebIdentity + // call without a RoleARN parameter - for these cases the + // user can supply a dummy ARN, which Minio will ignore. + roleArn = openid.DummyRoleARN + isRolePolicyProvider = false + } + } + + if !globalIAMSys.Initialized() { + writeSTSErrorResponse(ctx, w, ErrSTSIAMNotInitialized, errIAMNotInitialized) + return + } + + // Validate JWT; check clientID in claims matches the one associated with the roleArn + if err := globalIAMSys.OpenIDConfig.Validate(r.Context(), roleArn, token, accessToken, r.Form.Get(stsDurationSeconds), claims); err != nil { + switch err { + case openid.ErrTokenExpired: + switch action { + case clientGrants: + writeSTSErrorResponse(ctx, w, ErrSTSClientGrantsExpiredToken, err) + case webIdentity: + writeSTSErrorResponse(ctx, w, ErrSTSWebIdentityExpiredToken, err) + } + return + case auth.ErrInvalidDuration: + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err) + return + } + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err) + return + } + + var policyName string + if isRolePolicyProvider { + // If roleArn is used, we set it as a claim, and use the + // associated policy when credentials are used. + claims[roleArnClaim] = roleArn.String() + } else { + // If no role policy is configured, then we use claims from the + // JWT. This is a MinIO STS API specific value, this value + // should be set and configured on your identity provider as + // part of JWT custom claims. + policySet, ok := policy.GetPoliciesFromClaims(claims, iamPolicyClaimNameOpenID()) + policies := strings.Join(policySet.ToSlice(), ",") + if ok { + policyName = globalIAMSys.CurrentPolicies(policies) + } + + if newGlobalAuthZPluginFn() == nil { + if !ok { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, + fmt.Errorf("%s claim missing from the JWT token, credentials will not be generated", iamPolicyClaimNameOpenID())) + return + } else if policyName == "" { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, + fmt.Errorf("None of the given policies (`%s`) are defined, credentials will not be generated", policies)) + return + } + } + claims[iamPolicyClaimNameOpenID()] = policyName + } + + tokenRevokeType := r.Form.Get(stsRevokeTokenType) + if tokenRevokeType != "" { + claims[tokenRevokeTypeClaim] = tokenRevokeType + } + + if err := claims.populateSessionPolicy(r.Form); err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err) + return + } + + secret, err := getTokenSigningKey() + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err) + return + } + cred, err := auth.GetNewCredentialsWithMetadata(claims, secret) + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err) + return + } + + // https://openid.net/specs/openid-connect-core-1_0.html#ClaimStability + // claim is only considered stable when subject and iss are used together + // this is to ensure that ParentUser doesn't change and we get to use + // parentUser as per the requirements for service accounts for OpenID + // based logins. + var subFromToken string + if v, ok := claims[subClaim]; ok { + subFromToken, _ = v.(string) + } + + if subFromToken == "" { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, + errors.New("STS JWT Token has `sub` claim missing, `sub` claim is mandatory")) + return + } + + var issFromToken string + if v, ok := claims[issClaim]; ok { + issFromToken, _ = v.(string) + } + + // Since issFromToken can have `/` characters (it is typically the + // provider URL), we hash and encode it to base64 here. This is needed + // because there will be a policy mapping stored on drives whose + // filename is this parentUser: therefore, it needs to have only valid + // filename characters and needs to have bounded length. + { + h := sha256.New() + h.Write([]byte("openid:" + subFromToken + ":" + issFromToken)) + bs := h.Sum(nil) + cred.ParentUser = base64.RawURLEncoding.EncodeToString(bs) + } + + // Deny this assume role request if the policy that the user intends to bind + // has a sts:DurationSeconds condition, which is not satisfied as well + { + p := policyName + if p == "" { + var err error + _, p, err = globalIAMSys.GetRolePolicy(roleArnStr) + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSAccessDenied, err) + return + } + if newGlobalAuthZPluginFn() == nil { + // if authZ is not set - we expect the policies to be present. + if globalIAMSys.CurrentPolicies(p) == "" { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, + fmt.Errorf("None of the given policies (`%s`) are defined, credentials will not be generated", p)) + return + } + } + } + + if !globalIAMSys.doesPolicyAllow(p, policy.Args{ + DenyOnly: true, + Action: policy.AssumeRoleWithWebIdentityAction, + ConditionValues: getSTSConditionValues(r, "", cred), + Claims: cred.Claims, + }) { + writeSTSErrorResponse(ctx, w, ErrSTSAccessDenied, errors.New("this user does not have enough permission")) + return + } + } + + // Set the newly generated credentials. + updatedAt, err := globalIAMSys.SetTempUser(ctx, cred.AccessKey, cred, policyName) + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err) + return + } + + // Call hook for site replication. + replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemSTSAcc, + STSCredential: &madmin.SRSTSCredential{ + AccessKey: cred.AccessKey, + SecretKey: cred.SecretKey, + SessionToken: cred.SessionToken, + ParentUser: cred.ParentUser, + ParentPolicyMapping: policyName, + }, + UpdatedAt: updatedAt, + })) + + var encodedSuccessResponse []byte + switch action { + case clientGrants: + clientGrantsResponse := &AssumeRoleWithClientGrantsResponse{ + Result: ClientGrantsResult{ + Credentials: cred, + SubjectFromToken: subFromToken, + }, + } + clientGrantsResponse.ResponseMetadata.RequestID = w.Header().Get(xhttp.AmzRequestID) + encodedSuccessResponse = encodeResponse(clientGrantsResponse) + case webIdentity: + webIdentityResponse := &AssumeRoleWithWebIdentityResponse{ + Result: WebIdentityResult{ + Credentials: cred, + SubjectFromWebIdentityToken: subFromToken, + }, + } + webIdentityResponse.ResponseMetadata.RequestID = w.Header().Get(xhttp.AmzRequestID) + encodedSuccessResponse = encodeResponse(webIdentityResponse) + } + + writeSuccessResponseXML(w, encodedSuccessResponse) +} + +// AssumeRoleWithWebIdentity - implementation of AWS STS API supporting OAuth2.0 +// users from web identity provider such as Facebook, Google, or any OpenID +// Connect-compatible identity provider. +// +// Eg:- +// +// $ curl https://minio:9000/?Action=AssumeRoleWithWebIdentity&WebIdentityToken= +func (sts *stsAPIHandlers) AssumeRoleWithWebIdentity(w http.ResponseWriter, r *http.Request) { + sts.AssumeRoleWithSSO(w, r) +} + +// AssumeRoleWithClientGrants - implementation of AWS STS extension API supporting +// OAuth2.0 client credential grants. +// +// Eg:- +// +// $ curl https://minio:9000/?Action=AssumeRoleWithClientGrants&Token= +func (sts *stsAPIHandlers) AssumeRoleWithClientGrants(w http.ResponseWriter, r *http.Request) { + sts.AssumeRoleWithSSO(w, r) +} + +// AssumeRoleWithLDAPIdentity - implements user auth against LDAP server +func (sts *stsAPIHandlers) AssumeRoleWithLDAPIdentity(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "AssumeRoleWithLDAPIdentity") + + claims := stsClaims{} + defer logger.AuditLog(ctx, w, r, claims, stsLDAPPassword) + + // Parse the incoming form data. + if err := parseForm(r); err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err) + return + } + + if r.Form.Get(stsVersion) != stsAPIVersion { + writeSTSErrorResponse(ctx, w, ErrSTSMissingParameter, + fmt.Errorf("Invalid STS API version %s, expecting %s", r.Form.Get("Version"), stsAPIVersion)) + return + } + + ldapUsername := r.Form.Get(stsLDAPUsername) + ldapPassword := r.Form.Get(stsLDAPPassword) + + if ldapUsername == "" || ldapPassword == "" { + writeSTSErrorResponse(ctx, w, ErrSTSMissingParameter, fmt.Errorf("LDAPUsername and LDAPPassword cannot be empty")) + return + } + + action := r.Form.Get(stsAction) + switch action { + case ldapIdentity: + default: + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Unsupported action %s", action)) + return + } + + if err := claims.populateSessionPolicy(r.Form); err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err) + return + } + + if !globalIAMSys.Initialized() { + writeSTSErrorResponse(ctx, w, ErrSTSIAMNotInitialized, errIAMNotInitialized) + return + } + + lookupResult, groupDistNames, err := globalIAMSys.LDAPConfig.Bind(ldapUsername, ldapPassword) + if err != nil { + err = fmt.Errorf("LDAP server error: %w", err) + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err) + return + } + ldapUserDN := lookupResult.NormDN + ldapActualUserDN := lookupResult.ActualDN + + // Check if this user or their groups have a policy applied. + ldapPolicies, err := globalIAMSys.PolicyDBGet(ldapUserDN, groupDistNames...) + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err) + return + } + if len(ldapPolicies) == 0 && newGlobalAuthZPluginFn() == nil { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, + fmt.Errorf("expecting a policy to be set for user `%s` or one of their groups: `%s` - rejecting this request", + ldapActualUserDN, strings.Join(groupDistNames, "`,`"))) + return + } + + expiryDur, err := globalIAMSys.LDAPConfig.GetExpiryDuration(r.Form.Get(stsDurationSeconds)) + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err) + return + } + + claims[expClaim] = UTCNow().Add(expiryDur).Unix() + claims[ldapUser] = ldapUserDN + claims[ldapActualUser] = ldapActualUserDN + claims[ldapUserN] = ldapUsername + // Add lookup up LDAP attributes as claims. + for attrib, value := range lookupResult.Attributes { + claims[ldapAttribPrefix+attrib] = value + } + tokenRevokeType := r.Form.Get(stsRevokeTokenType) + if tokenRevokeType != "" { + claims[tokenRevokeTypeClaim] = tokenRevokeType + } + + secret, err := getTokenSigningKey() + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err) + return + } + + cred, err := auth.GetNewCredentialsWithMetadata(claims, secret) + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err) + return + } + + // Set the parent of the temporary access key, this is useful + // in obtaining service accounts by this cred. + cred.ParentUser = ldapUserDN + + // Set this value to LDAP groups, LDAP user can be part + // of large number of groups + cred.Groups = groupDistNames + + // Set the newly generated credentials, policyName is empty on purpose + // LDAP policies are applied automatically using their ldapUser, ldapGroups + // mapping. + updatedAt, err := globalIAMSys.SetTempUser(ctx, cred.AccessKey, cred, "") + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err) + return + } + + // Call hook for site replication. + replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemSTSAcc, + STSCredential: &madmin.SRSTSCredential{ + AccessKey: cred.AccessKey, + SecretKey: cred.SecretKey, + SessionToken: cred.SessionToken, + ParentUser: cred.ParentUser, + }, + UpdatedAt: updatedAt, + })) + + ldapIdentityResponse := &AssumeRoleWithLDAPResponse{ + Result: LDAPIdentityResult{ + Credentials: cred, + }, + } + ldapIdentityResponse.ResponseMetadata.RequestID = w.Header().Get(xhttp.AmzRequestID) + encodedSuccessResponse := encodeResponse(ldapIdentityResponse) + + writeSuccessResponseXML(w, encodedSuccessResponse) +} + +// AssumeRoleWithCertificate implements user authentication with client certificates. +// It verifies the client-provided X.509 certificate, maps the certificate to an S3 policy +// and returns temp. S3 credentials to the client. +// +// API endpoint: https://minio:9000?Action=AssumeRoleWithCertificate&Version=2011-06-15 +func (sts *stsAPIHandlers) AssumeRoleWithCertificate(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "AssumeRoleWithCertificate") + + claims := make(map[string]any) + defer logger.AuditLog(ctx, w, r, claims) + + if !globalIAMSys.Initialized() { + writeSTSErrorResponse(ctx, w, ErrSTSIAMNotInitialized, errIAMNotInitialized) + return + } + + if !globalIAMSys.STSTLSConfig.Enabled { + writeSTSErrorResponse(ctx, w, ErrSTSNotInitialized, errors.New("STS API 'AssumeRoleWithCertificate' is disabled")) + return + } + + // We have to establish a TLS connection and the + // client must provide exactly one client certificate. + // Otherwise, we don't have a certificate to verify or + // the policy lookup would ambiguous. + if r.TLS == nil { + writeSTSErrorResponse(ctx, w, ErrSTSInsecureConnection, errors.New("No TLS connection attempt")) + return + } + + // A client may send a certificate chain such that we end up + // with multiple peer certificates. However, we can only accept + // a single client certificate. Otherwise, the certificate to + // policy mapping would be ambiguous. + // However, we can filter all CA certificates and only check + // whether they client has sent exactly one (non-CA) leaf certificate. + const MaxIntermediateCAs = 10 + var ( + peerCertificates = make([]*x509.Certificate, 0, len(r.TLS.PeerCertificates)) + intermediates *x509.CertPool + numIntermediates int + ) + for _, cert := range r.TLS.PeerCertificates { + if cert.IsCA { + numIntermediates++ + if numIntermediates > MaxIntermediateCAs { + writeSTSErrorResponse(ctx, w, ErrSTSTooManyIntermediateCAs, fmt.Errorf("client certificate contains more than %d intermediate CAs", MaxIntermediateCAs)) + return + } + if intermediates == nil { + intermediates = x509.NewCertPool() + } + intermediates.AddCert(cert) + } else { + peerCertificates = append(peerCertificates, cert) + } + } + r.TLS.PeerCertificates = peerCertificates + + // Now, we have to check that the client has provided exactly one leaf + // certificate that we can map to a policy. + if len(r.TLS.PeerCertificates) == 0 { + writeSTSErrorResponse(ctx, w, ErrSTSMissingParameter, errors.New("No client certificate provided")) + return + } + if len(r.TLS.PeerCertificates) > 1 { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, errors.New("More than one client certificate provided")) + return + } + + certificate := r.TLS.PeerCertificates[0] + if !globalIAMSys.STSTLSConfig.InsecureSkipVerify { // Verify whether the client certificate has been issued by a trusted CA. + _, err := certificate.Verify(x509.VerifyOptions{ + KeyUsages: []x509.ExtKeyUsage{ + x509.ExtKeyUsageClientAuth, + }, + Intermediates: intermediates, + Roots: globalRootCAs, + }) + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidClientCertificate, err) + return + } + } else { + // Technically, there is no security argument for verifying the key usage + // when we don't verify that the certificate has been issued by a trusted CA. + // Any client can create a certificate with arbitrary key usage settings. + // + // However, this check ensures that a certificate with an invalid key usage + // gets rejected even when we skip certificate verification. This helps + // clients detect malformed certificates during testing instead of e.g. + // a self-signed certificate that works while a comparable certificate + // issued by a trusted CA fails due to the MinIO server being less strict + // w.r.t. key usage verification. + // + // Basically, MinIO is more consistent (from a client perspective) when + // we verify the key usage all the time. + var validKeyUsage bool + for _, usage := range certificate.ExtKeyUsage { + if usage == x509.ExtKeyUsageAny || usage == x509.ExtKeyUsageClientAuth { + validKeyUsage = true + break + } + } + if !validKeyUsage { + writeSTSErrorResponse(ctx, w, ErrSTSMissingParameter, errors.New("certificate is not valid for client authentication")) + return + } + } + + // We map the X.509 subject common name to the policy. So, a client + // with the common name "foo" will be associated with the policy "foo". + // Other mapping functions - e.g. public-key hash based mapping - are + // possible but not implemented. + // + // Group mapping is not possible with standard X.509 certificates. + if certificate.Subject.CommonName == "" { + writeSTSErrorResponse(ctx, w, ErrSTSMissingParameter, errors.New("certificate subject CN cannot be empty")) + return + } + + expiry, err := globalIAMSys.STSTLSConfig.GetExpiryDuration(r.Form.Get(stsDurationSeconds)) + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSMissingParameter, err) + return + } + + // We set the expiry of the temp. credentials to the minimum of the + // configured expiry and the duration until the certificate itself + // expires. + // We must not issue credentials that out-live the certificate. + if validUntil := time.Until(certificate.NotAfter); validUntil < expiry { + expiry = validUntil + } + + // Associate any service accounts to the certificate CN + parentUser := "tls" + getKeySeparator() + certificate.Subject.CommonName + + claims[expClaim] = UTCNow().Add(expiry).Unix() + claims[subClaim] = certificate.Subject.CommonName + claims[audClaim] = certificate.Subject.Organization + claims[issClaim] = certificate.Issuer.CommonName + claims[parentClaim] = parentUser + tokenRevokeType := r.Form.Get(stsRevokeTokenType) + if tokenRevokeType != "" { + claims[tokenRevokeTypeClaim] = tokenRevokeType + } + + secretKey, err := getTokenSigningKey() + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err) + return + } + tmpCredentials, err := auth.GetNewCredentialsWithMetadata(claims, secretKey) + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err) + return + } + + tmpCredentials.ParentUser = parentUser + policyName := certificate.Subject.CommonName + updatedAt, err := globalIAMSys.SetTempUser(ctx, tmpCredentials.AccessKey, tmpCredentials, policyName) + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err) + return + } + + // Call hook for site replication. + replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemSTSAcc, + STSCredential: &madmin.SRSTSCredential{ + AccessKey: tmpCredentials.AccessKey, + SecretKey: tmpCredentials.SecretKey, + SessionToken: tmpCredentials.SessionToken, + ParentUser: tmpCredentials.ParentUser, + ParentPolicyMapping: policyName, + }, + UpdatedAt: updatedAt, + })) + + response := new(AssumeRoleWithCertificateResponse) + response.Result.Credentials = tmpCredentials + response.Metadata.RequestID = w.Header().Get(xhttp.AmzRequestID) + writeSuccessResponseXML(w, encodeResponse(response)) +} + +// AssumeRoleWithCustomToken implements user authentication with custom tokens. +// These tokens are opaque to MinIO and are verified by a configured (external) +// Identity Management Plugin. +// +// API endpoint: https://minio:9000?Action=AssumeRoleWithCustomToken&Token=xxx +func (sts *stsAPIHandlers) AssumeRoleWithCustomToken(w http.ResponseWriter, r *http.Request) { + ctx := newContext(r, w, "AssumeRoleWithCustomToken") + + claims := make(map[string]any) + + auditLogFilterKeys := []string{stsToken} + defer logger.AuditLog(ctx, w, r, claims, auditLogFilterKeys...) + + if !globalIAMSys.Initialized() { + writeSTSErrorResponse(ctx, w, ErrSTSIAMNotInitialized, errIAMNotInitialized) + return + } + + authn := newGlobalAuthNPluginFn() + if authn == nil { + writeSTSErrorResponse(ctx, w, ErrSTSNotInitialized, errors.New("STS API 'AssumeRoleWithCustomToken' is disabled")) + return + } + + action := r.Form.Get(stsAction) + if action != customTokenIdentity { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Unsupported action %s", action)) + return + } + + token := r.Form.Get(stsToken) + if token == "" { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Invalid empty `Token` parameter provided")) + return + } + + durationParam := r.Form.Get(stsDurationSeconds) + var requestedDuration int + if durationParam != "" { + var err error + requestedDuration, err = strconv.Atoi(durationParam) + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, fmt.Errorf("Invalid requested duration: %s", durationParam)) + return + } + } + + roleArnStr := r.Form.Get(stsRoleArn) + roleArn, _, err := globalIAMSys.GetRolePolicy(roleArnStr) + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, + fmt.Errorf("Error processing parameter %s: %v", stsRoleArn, err)) + return + } + + _, policyName, err := globalIAMSys.GetRolePolicy(roleArnStr) + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSAccessDenied, err) + return + } + + if newGlobalAuthZPluginFn() == nil { // if authZ is not set - we expect the policyname to be present. + if globalIAMSys.CurrentPolicies(policyName) == "" { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, + fmt.Errorf("None of the given policies (`%s`) are defined, credentials will not be generated", policyName)) + return + } + } + + res, err := authn.Authenticate(roleArn, token) + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInvalidParameterValue, err) + return + } + + // If authentication failed, return the error message to the user. + if res.Failure != nil { + writeSTSErrorResponse(ctx, w, ErrSTSUpstreamError, errors.New(res.Failure.Reason)) + return + } + + // It is required that parent user be set. + if res.Success.User == "" { + writeSTSErrorResponse(ctx, w, ErrSTSUpstreamError, errors.New("A valid user was not returned by the authenticator.")) + return + } + + // Expiry is set as minimum of requested value and value allowed by auth + // plugin. + expiry := res.Success.MaxValiditySeconds + if durationParam != "" && requestedDuration < expiry { + expiry = requestedDuration + } + + parentUser := "custom" + getKeySeparator() + res.Success.User + + // metadata map + claims[expClaim] = UTCNow().Add(time.Duration(expiry) * time.Second).Unix() + claims[subClaim] = parentUser + claims[roleArnClaim] = roleArn.String() + claims[parentClaim] = parentUser + tokenRevokeType := r.Form.Get(stsRevokeTokenType) + if tokenRevokeType != "" { + claims[tokenRevokeTypeClaim] = tokenRevokeType + } + + // Add all other claims from the plugin **without** replacing any + // existing claims. + for k, v := range res.Success.Claims { + if _, ok := claims[k]; !ok { + claims[k] = v + } + } + secretKey, err := getTokenSigningKey() + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err) + return + } + tmpCredentials, err := auth.GetNewCredentialsWithMetadata(claims, secretKey) + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err) + return + } + + tmpCredentials.ParentUser = parentUser + updatedAt, err := globalIAMSys.SetTempUser(ctx, tmpCredentials.AccessKey, tmpCredentials, "") + if err != nil { + writeSTSErrorResponse(ctx, w, ErrSTSInternalError, err) + return + } + + // Call hook for site replication. + replLogIf(ctx, globalSiteReplicationSys.IAMChangeHook(ctx, madmin.SRIAMItem{ + Type: madmin.SRIAMItemSTSAcc, + STSCredential: &madmin.SRSTSCredential{ + AccessKey: tmpCredentials.AccessKey, + SecretKey: tmpCredentials.SecretKey, + SessionToken: tmpCredentials.SessionToken, + ParentUser: tmpCredentials.ParentUser, + }, + UpdatedAt: updatedAt, + })) + + response := new(AssumeRoleWithCustomTokenResponse) + response.Result.Credentials = tmpCredentials + response.Result.AssumedUser = parentUser + response.Metadata.RequestID = w.Header().Get(xhttp.AmzRequestID) + writeSuccessResponseXML(w, encodeResponse(response)) +} diff --git a/cmd/sts-handlers_test.go b/cmd/sts-handlers_test.go new file mode 100644 index 00000000..a883999e --- /dev/null +++ b/cmd/sts-handlers_test.go @@ -0,0 +1,3597 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "fmt" + "io" + "os" + "reflect" + "slices" + "strings" + "testing" + "time" + + "github.com/klauspost/compress/zip" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7" + cr "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/pkg/v3/ldap" +) + +func runAllIAMSTSTests(suite *TestSuiteIAM, c *check) { + suite.SetUpSuite(c) + // The STS for root test needs to be the first one after setup. + suite.TestSTSForRoot(c) + suite.TestSTS(c) + suite.TestSTSPrivilegeEscalationBug2_2025_10_15(c, true) + suite.TestSTSPrivilegeEscalationBug2_2025_10_15(c, false) + suite.TestSTSWithDenyDeleteVersion(c) + suite.TestSTSWithTags(c) + suite.TestSTSServiceAccountsWithUsername(c) + suite.TestSTSWithGroupPolicy(c) + suite.TestSTSTokenRevoke(c) + suite.TearDownSuite(c) +} + +func TestIAMInternalIDPSTSServerSuite(t *testing.T) { + baseTestCases := []TestSuiteCommon{ + // Init and run test on ErasureSD backend with signature v4. + {serverType: "ErasureSD", signer: signerV4}, + // Init and run test on ErasureSD backend, with tls enabled. + {serverType: "ErasureSD", signer: signerV4, secure: true}, + // Init and run test on Erasure backend. + {serverType: "Erasure", signer: signerV4}, + // Init and run test on ErasureSet backend. + {serverType: "ErasureSet", signer: signerV4}, + } + testCases := []*TestSuiteIAM{} + for _, bt := range baseTestCases { + testCases = append(testCases, + newTestSuiteIAM(bt, false), + newTestSuiteIAM(bt, true), + ) + } + for i, testCase := range testCases { + etcdStr := "" + if testCase.withEtcdBackend { + etcdStr = " (with etcd backend)" + } + t.Run( + fmt.Sprintf("Test: %d, ServerType: %s%s", i+1, testCase.serverType, etcdStr), + func(t *testing.T) { + runAllIAMSTSTests(testCase, &check{t, testCase.serverType}) + }, + ) + } +} + +func (s *TestSuiteIAM) TestSTSServiceAccountsWithUsername(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + bucket := "dillon-bucket" + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket create error: %v", err) + } + + // Create policy + policy := "mypolicy-username" + policyBytes := []byte(`{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:*" + ], + "Resource": [ + "arn:aws:s3:::${aws:username}-*" + ] + } + ] +}`) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + if err = s.adm.AddUser(ctx, "dillon", "dillon-123"); err != nil { + c.Fatalf("policy add error: %v", err) + } + + _, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{ + Policies: []string{policy}, + User: "dillon", + }) + if err != nil { + c.Fatalf("Unable to attach policy: %v", err) + } + + assumeRole := cr.STSAssumeRole{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + Options: cr.STSAssumeRoleOptions{ + AccessKey: "dillon", + SecretKey: "dillon-123", + Location: "", + }, + } + + value, err := assumeRole.Retrieve() + if err != nil { + c.Fatalf("Expected to generate STS creds, got err: %#v", err) + } + + // Check that the LDAP sts cred is actually working. + minioClient, err := minio.New(s.endpoint, &minio.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("Error initializing client: %v", err) + } + + // Validate that the client from sts creds can access the bucket. + c.mustListObjects(ctx, minioClient, bucket) + + // Create an madmin client with user creds + userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + }) + if err != nil { + c.Fatalf("Err creating user admin client: %v", err) + } + userAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport) + + // Create svc acc + cr := c.mustCreateSvcAccount(ctx, value.AccessKeyID, userAdmClient) + + svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "") + + // 1. Check S3 access for service account ListObjects() + c.mustListObjects(ctx, svcClient, bucket) + + // 2. Check S3 access for upload + c.mustUpload(ctx, svcClient, bucket) + + // 3. Check S3 access for download + c.mustDownload(ctx, svcClient, bucket) +} + +func (s *TestSuiteIAM) TestSTSWithDenyDeleteVersion(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{ObjectLocking: true}) + if err != nil { + c.Fatalf("bucket creat error: %v", err) + } + + // Create policy, user and associate policy + policy := "mypolicy" + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ObjectActionsRW", + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:PutObjectTagging", + "s3:AbortMultipartUpload", + "s3:DeleteObject", + "s3:GetObject", + "s3:GetObjectTagging", + "s3:GetObjectVersion", + "s3:ListMultipartUploadParts" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + }, + { + "Sid": "DenyDeleteVersionAction", + "Effect": "Deny", + "Action": [ + "s3:DeleteObjectVersion" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] + } +`, bucket, bucket) + + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + accessKey, secretKey := mustGenerateCredentials(c) + err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled) + if err != nil { + c.Fatalf("Unable to set user: %v", err) + } + + _, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{ + Policies: []string{policy}, + User: accessKey, + }) + if err != nil { + c.Fatalf("Unable to attach policy: %v", err) + } + + // confirm that the user is able to access the bucket + uClient := s.getUserClient(c, accessKey, secretKey, "") + versions := c.mustUploadReturnVersions(ctx, uClient, bucket) + c.mustNotDelete(ctx, uClient, bucket, versions[0]) + + assumeRole := cr.STSAssumeRole{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + Options: cr.STSAssumeRoleOptions{ + AccessKey: accessKey, + SecretKey: secretKey, + Location: "", + }, + } + + value, err := assumeRole.Retrieve() + if err != nil { + c.Fatalf("err calling assumeRole: %v", err) + } + + minioClient, err := minio.New(s.endpoint, &minio.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("Error initializing client: %v", err) + } + + versions = c.mustUploadReturnVersions(ctx, minioClient, bucket) + c.mustNotDelete(ctx, minioClient, bucket, versions[0]) +} + +func (s *TestSuiteIAM) TestSTSPrivilegeEscalationBug2_2025_10_15(c *check, forRoot bool) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + for i := range 3 { + err := s.client.MakeBucket(ctx, fmt.Sprintf("bucket%d", i+1), minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket create error: %v", err) + } + defer func(i int) { + _ = s.client.RemoveBucket(ctx, fmt.Sprintf("bucket%d", i+1)) + }(i) + } + + allow2BucketsPolicyBytes := []byte(`{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "ListBucket1AndBucket2", + "Effect": "Allow", + "Action": ["s3:ListBucket"], + "Resource": ["arn:aws:s3:::bucket1", "arn:aws:s3:::bucket2"] + }, + { + "Sid": "ReadWriteBucket1AndBucket2Objects", + "Effect": "Allow", + "Action": [ + "s3:DeleteObject", + "s3:DeleteObjectVersion", + "s3:GetObject", + "s3:GetObjectVersion", + "s3:PutObject" + ], + "Resource": ["arn:aws:s3:::bucket1/*", "arn:aws:s3:::bucket2/*"] + } + ] +}`) + + var value cr.Value + var err error + if forRoot { + assumeRole := cr.STSAssumeRole{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + Options: cr.STSAssumeRoleOptions{ + AccessKey: globalActiveCred.AccessKey, + SecretKey: globalActiveCred.SecretKey, + Policy: string(allow2BucketsPolicyBytes), + }, + } + value, err = assumeRole.Retrieve() + if err != nil { + c.Fatalf("err calling assumeRole: %v", err) + } + } else { + // Create a regular user and attach consoleAdmin policy + err := s.adm.AddUser(ctx, "foobar", "foobar123") + if err != nil { + c.Fatalf("could not create user") + } + + _, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{ + Policies: []string{"consoleAdmin"}, + User: "foobar", + }) + if err != nil { + c.Fatalf("could not attach policy") + } + + assumeRole := cr.STSAssumeRole{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + Options: cr.STSAssumeRoleOptions{ + AccessKey: "foobar", + SecretKey: "foobar123", + Policy: string(allow2BucketsPolicyBytes), + }, + } + value, err = assumeRole.Retrieve() + if err != nil { + c.Fatalf("err calling assumeRole: %v", err) + } + } + restrictedClient := s.getUserClient(c, value.AccessKeyID, value.SecretAccessKey, value.SessionToken) + + buckets, err := restrictedClient.ListBuckets(ctx) + if err != nil { + c.Fatalf("err fetching buckets %s", err) + } + if len(buckets) != 2 || buckets[0].Name != "bucket1" || buckets[1].Name != "bucket2" { + c.Fatalf("restricted STS account should only have access to bucket1 and bucket2") + } + + // Try to escalate privileges + restrictedAdmClient := s.getAdminClient(c, value.AccessKeyID, value.SecretAccessKey, value.SessionToken) + _, err = restrictedAdmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{ + AccessKey: "newroot", + SecretKey: "newroot123", + }) + if err == nil { + c.Fatalf("restricted STS account was able to create service account bypassing sub-policy!") + } +} + +func (s *TestSuiteIAM) TestSTSWithTags(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + bucket := getRandomBucketName() + object := getRandomObjectName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket creat error: %v", err) + } + + // Create policy, user and associate policy + policy := "mypolicy" + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": "s3:GetObject", + "Resource": "arn:aws:s3:::%s/*", + "Condition": { "StringEquals": {"s3:ExistingObjectTag/security": "public" } } + }, + { + "Effect": "Allow", + "Action": "s3:DeleteObjectTagging", + "Resource": "arn:aws:s3:::%s/*", + "Condition": { "StringEquals": {"s3:ExistingObjectTag/security": "public" } } + }, + { + "Effect": "Allow", + "Action": "s3:DeleteObject", + "Resource": "arn:aws:s3:::%s/*" + }, + { + "Effect": "Allow", + "Action": [ + "s3:PutObject" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ], + "Condition": { + "ForAllValues:StringLike": { + "s3:RequestObjectTagKeys": [ + "security", + "virus" + ] + } + } + } + ] +}`, bucket, bucket, bucket, bucket) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + accessKey, secretKey := mustGenerateCredentials(c) + err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled) + if err != nil { + c.Fatalf("Unable to set user: %v", err) + } + + _, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{ + Policies: []string{policy}, + User: accessKey, + }) + if err != nil { + c.Fatalf("Unable to attach policy: %v", err) + } + + // confirm that the user is able to access the bucket + uClient := s.getUserClient(c, accessKey, secretKey, "") + c.mustPutObjectWithTags(ctx, uClient, bucket, object) + c.mustGetObject(ctx, uClient, bucket, object) + + assumeRole := cr.STSAssumeRole{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + Options: cr.STSAssumeRoleOptions{ + AccessKey: accessKey, + SecretKey: secretKey, + Location: "", + }, + } + + value, err := assumeRole.Retrieve() + if err != nil { + c.Fatalf("err calling assumeRole: %v", err) + } + + minioClient, err := minio.New(s.endpoint, &minio.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("Error initializing client: %v", err) + } + + // Validate sts creds can access the object + c.mustPutObjectWithTags(ctx, minioClient, bucket, object) + c.mustGetObject(ctx, minioClient, bucket, object) + c.mustHeadObject(ctx, minioClient, bucket, object, 2) + + // Validate that the client can remove objects + if err = minioClient.RemoveObjectTagging(ctx, bucket, object, minio.RemoveObjectTaggingOptions{}); err != nil { + c.Fatalf("user is unable to delete the object tags: %v", err) + } + + if err = minioClient.RemoveObject(ctx, bucket, object, minio.RemoveObjectOptions{}); err != nil { + c.Fatalf("user is unable to delete the object: %v", err) + } +} + +func (s *TestSuiteIAM) TestSTS(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket creat error: %v", err) + } + + // Create policy, user and associate policy + policy := "mypolicy" + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + accessKey, secretKey := mustGenerateCredentials(c) + err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled) + if err != nil { + c.Fatalf("Unable to set user: %v", err) + } + + _, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{ + Policies: []string{policy}, + User: accessKey, + }) + if err != nil { + c.Fatalf("Unable to attach policy: %v", err) + } + + // confirm that the user is able to access the bucket + uClient := s.getUserClient(c, accessKey, secretKey, "") + c.mustListObjects(ctx, uClient, bucket) + + assumeRole := cr.STSAssumeRole{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + Options: cr.STSAssumeRoleOptions{ + AccessKey: accessKey, + SecretKey: secretKey, + Location: "", + }, + } + + value, err := assumeRole.Retrieve() + if err != nil { + c.Fatalf("err calling assumeRole: %v", err) + } + + minioClient, err := minio.New(s.endpoint, &minio.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("Error initializing client: %v", err) + } + + // Validate that the client from sts creds can access the bucket. + c.mustListObjects(ctx, minioClient, bucket) + + // Validate that the client cannot remove any objects + err = minioClient.RemoveObject(ctx, bucket, "someobject", minio.RemoveObjectOptions{}) + if err.Error() != "Access Denied." { + c.Fatalf("unexpected non-access-denied err: %v", err) + } +} + +func (s *TestSuiteIAM) TestSTSWithGroupPolicy(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket creat error: %v", err) + } + + // Create policy, user and associate policy + policy := "mypolicy" + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + accessKey, secretKey := mustGenerateCredentials(c) + err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled) + if err != nil { + c.Fatalf("Unable to set user: %v", err) + } + + // confirm that the user is unable to access the bucket - we have not + // yet set any policy + uClient := s.getUserClient(c, accessKey, secretKey, "") + c.mustNotListObjects(ctx, uClient, bucket) + + err = s.adm.UpdateGroupMembers(ctx, madmin.GroupAddRemove{ + Group: "test-group", + Members: []string{accessKey}, + }) + if err != nil { + c.Fatalf("unable to add user to group: %v", err) + } + + _, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{ + Policies: []string{policy}, + Group: "test-group", + }) + if err != nil { + c.Fatalf("Unable to attach policy: %v", err) + } + + // confirm that the user is able to access the bucket - permission comes + // from group. + c.mustListObjects(ctx, uClient, bucket) + + // Create STS user. + assumeRole := cr.STSAssumeRole{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + Options: cr.STSAssumeRoleOptions{ + AccessKey: accessKey, + SecretKey: secretKey, + Location: "", + }, + } + value, err := assumeRole.Retrieve() + if err != nil { + c.Fatalf("err calling assumeRole: %v", err) + } + + // Check that STS user client has access coming from parent user's + // group. + minioClient, err := minio.New(s.endpoint, &minio.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("Error initializing client: %v", err) + } + + // Validate that the client from sts creds can access the bucket. + c.mustListObjects(ctx, minioClient, bucket) + + // Validate that the client cannot remove any objects + err = minioClient.RemoveObject(ctx, bucket, "someobject", minio.RemoveObjectOptions{}) + if err.Error() != "Access Denied." { + c.Fatalf("unexpected non-access-denied err: %v", err) + } +} + +// TestSTSForRoot - needs to be the first test after server setup due to the +// buckets list check. +func (s *TestSuiteIAM) TestSTSForRoot(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket create error: %v", err) + } + + assumeRole := cr.STSAssumeRole{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + Options: cr.STSAssumeRoleOptions{ + AccessKey: globalActiveCred.AccessKey, + SecretKey: globalActiveCred.SecretKey, + Location: "", + }, + } + + value, err := assumeRole.Retrieve() + if err != nil { + c.Fatalf("err calling assumeRole: %v", err) + } + + minioClient, err := minio.New(s.endpoint, &minio.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("Error initializing client: %v", err) + } + + // Validate that the client from sts creds can access the bucket. + c.mustListObjects(ctx, minioClient, bucket) + + // Validate that a bucket can be created + bucket2 := getRandomBucketName() + err = minioClient.MakeBucket(ctx, bucket2, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket creat error: %v", err) + } + + // Validate that admin APIs can be called - create an madmin client with + // user creds + userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + }) + if err != nil { + c.Fatalf("Err creating user admin client: %v", err) + } + userAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport) + + time.Sleep(2 * time.Second) // wait for listbuckets cache to be invalidated + + accInfo, err := userAdmClient.AccountInfo(ctx, madmin.AccountOpts{}) + if err != nil { + c.Fatalf("root user STS should be able to get account info: %v", err) + } + + gotBuckets := set.NewStringSet() + for _, b := range accInfo.Buckets { + gotBuckets.Add(b.Name) + if !b.Access.Read || !b.Access.Write { + c.Fatalf("root user should have read and write access to bucket: %v", b.Name) + } + } + shouldHaveBuckets := set.CreateStringSet(bucket2, bucket) + if !gotBuckets.Equals(shouldHaveBuckets) { + c.Fatalf("root user should have access to all buckets") + } + + // This must fail. + if err := userAdmClient.AddUser(ctx, globalActiveCred.AccessKey, globalActiveCred.SecretKey); err == nil { + c.Fatal("AddUser() for root credential must fail via root STS creds") + } +} + +// TestSTSTokenRevoke - tests the token revoke API +func (s *TestSuiteIAM) TestSTSTokenRevoke(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), 100*testDefaultTimeout) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket create error: %v", err) + } + + // Create policy, user and associate policy + policy := "mypolicy" + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + accessKey, secretKey := mustGenerateCredentials(c) + err = s.adm.SetUser(ctx, accessKey, secretKey, madmin.AccountEnabled) + if err != nil { + c.Fatalf("Unable to set user: %v", err) + } + + _, err = s.adm.AttachPolicy(ctx, madmin.PolicyAssociationReq{ + Policies: []string{policy}, + User: accessKey, + }) + if err != nil { + c.Fatalf("Unable to attach policy: %v", err) + } + + cases := []struct { + tokenType string + fullRevoke bool + selfRevoke bool + }{ + {"", true, false}, // Case 1 + {"", true, true}, // Case 2 + {"type-1", false, false}, // Case 3 + {"type-2", false, true}, // Case 4 + {"type-2", true, true}, // Case 5 - repeat type 2 to ensure previous revoke does not affect it. + } + + for i, tc := range cases { + // Create STS user. + assumeRole := cr.STSAssumeRole{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + Options: cr.STSAssumeRoleOptions{ + AccessKey: accessKey, + SecretKey: secretKey, + TokenRevokeType: tc.tokenType, + }, + } + + value, err := assumeRole.Retrieve() + if err != nil { + c.Fatalf("err calling assumeRole: %v", err) + } + + minioClient, err := minio.New(s.endpoint, &minio.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("Error initializing client: %v", err) + } + + // Validate that the client from sts creds can access the bucket. + c.mustListObjects(ctx, minioClient, bucket) + + // Set up revocation + user := accessKey + tokenType := tc.tokenType + reqAdmClient := s.adm + if tc.fullRevoke { + tokenType = "" + } + if tc.selfRevoke { + user = "" + tokenType = "" + reqAdmClient, err = madmin.NewWithOptions(s.endpoint, &madmin.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + }) + if err != nil { + c.Fatalf("Err creating user admin client: %v", err) + } + reqAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport) + } + + err = reqAdmClient.RevokeTokens(ctx, madmin.RevokeTokensReq{ + User: user, + TokenRevokeType: tokenType, + FullRevoke: tc.fullRevoke, + }) + if err != nil { + c.Fatalf("Case %d: unexpected error: %v", i+1, err) + } + + // Validate that the client cannot access the bucket after revocation. + c.mustNotListObjects(ctx, minioClient, bucket) + } +} + +// SetUpLDAP - expects to setup an LDAP test server using the test LDAP +// container and canned data from https://github.com/minio/minio-ldap-testing +func (s *TestSuiteIAM) SetUpLDAP(c *check, serverAddr string) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + configCmds := []string{ + "identity_ldap", + fmt.Sprintf("server_addr=%s", serverAddr), + "server_insecure=on", + "lookup_bind_dn=cn=admin,dc=min,dc=io", + "lookup_bind_password=admin", + "user_dn_search_base_dn=dc=min,dc=io", + "user_dn_search_filter=(uid=%s)", + "user_dn_attributes=sshPublicKey", + "group_search_base_dn=ou=swengg,dc=min,dc=io", + "group_search_filter=(&(objectclass=groupofnames)(member=%d))", + } + _, err := s.adm.SetConfigKV(ctx, strings.Join(configCmds, " ")) + if err != nil { + c.Fatalf("unable to setup LDAP for tests: %v", err) + } + + s.RestartIAMSuite(c) +} + +// SetUpLDAPWithNonNormalizedBaseDN - expects to setup an LDAP test server using +// the test LDAP container and canned data from +// https://github.com/minio/minio-ldap-testing +// +// Sets up non-normalized base DN configuration for testing. +func (s *TestSuiteIAM) SetUpLDAPWithNonNormalizedBaseDN(c *check, serverAddr string) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + configCmds := []string{ + "identity_ldap", + fmt.Sprintf("server_addr=%s", serverAddr), + "server_insecure=on", + "lookup_bind_dn=cn=admin,dc=min,dc=io", + "lookup_bind_password=admin", + // `DC` is intentionally capitalized here. + "user_dn_search_base_dn=DC=min,DC=io", + "user_dn_search_filter=(uid=%s)", + // `DC` is intentionally capitalized here. + "group_search_base_dn=ou=swengg,DC=min,dc=io", + "group_search_filter=(&(objectclass=groupofnames)(member=%d))", + } + _, err := s.adm.SetConfigKV(ctx, strings.Join(configCmds, " ")) + if err != nil { + c.Fatalf("unable to setup LDAP for tests: %v", err) + } + + s.RestartIAMSuite(c) +} + +const ( + EnvTestLDAPServer = "_MINIO_LDAP_TEST_SERVER" +) + +func TestIAMWithLDAPServerSuite(t *testing.T) { + for i, testCase := range iamTestSuites { + t.Run( + fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription), + func(t *testing.T) { + c := &check{t, testCase.serverType} + suite := testCase + + ldapServer := os.Getenv(EnvTestLDAPServer) + if ldapServer == "" { + c.Skipf("Skipping LDAP test as no LDAP server is provided via %s", EnvTestLDAPServer) + } + + suite.SetUpSuite(c) + suite.SetUpLDAP(c, ldapServer) + suite.TestLDAPSTS(c) + suite.TestLDAPPolicyEntitiesLookup(c) + suite.TestLDAPUnicodeVariations(c) + suite.TestLDAPSTSServiceAccounts(c) + suite.TestLDAPSTSServiceAccountsWithUsername(c) + suite.TestLDAPSTSServiceAccountsWithGroups(c) + suite.TestLDAPAttributesLookup(c) + suite.TestLDAPCyrillicUser(c) + suite.TestLDAPSlashDN(c) + suite.TearDownSuite(c) + }, + ) + } +} + +// This test is for a fix added to handle non-normalized base DN values in the +// LDAP configuration. It runs the existing LDAP sub-tests with a non-normalized +// LDAP configuration. +func TestIAMWithLDAPNonNormalizedBaseDNConfigServerSuite(t *testing.T) { + for i, testCase := range iamTestSuites { + t.Run( + fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription), + func(t *testing.T) { + c := &check{t, testCase.serverType} + suite := testCase + + ldapServer := os.Getenv(EnvTestLDAPServer) + if ldapServer == "" { + c.Skipf("Skipping LDAP test as no LDAP server is provided via %s", EnvTestLDAPServer) + } + + suite.SetUpSuite(c) + suite.SetUpLDAPWithNonNormalizedBaseDN(c, ldapServer) + suite.TestLDAPSTS(c) + suite.TestLDAPPolicyEntitiesLookup(c) + suite.TestLDAPUnicodeVariations(c) + suite.TestLDAPSTSServiceAccounts(c) + suite.TestLDAPSTSServiceAccountsWithUsername(c) + suite.TestLDAPSTSServiceAccountsWithGroups(c) + suite.TestLDAPSlashDN(c) + suite.TearDownSuite(c) + }, + ) + } +} + +func TestIAMExportImportWithLDAP(t *testing.T) { + for i, testCase := range iamTestSuites { + t.Run( + fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription), + func(t *testing.T) { + c := &check{t, testCase.serverType} + suite := testCase + + ldapServer := os.Getenv(EnvTestLDAPServer) + if ldapServer == "" { + c.Skipf("Skipping LDAP test as no LDAP server is provided via %s", EnvTestLDAPServer) + } + + iamTestContentCases := []iamTestContent{ + { + policies: map[string][]byte{ + "mypolicy": []byte(`{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetObject","s3:ListBucket","s3:PutObject"],"Resource":["arn:aws:s3:::mybucket/*"]}]}`), + }, + ldapUserPolicyMappings: map[string][]string{ + "uid=dillon,ou=people,ou=swengg,dc=min,dc=io": {"mypolicy"}, + "uid=liza,ou=people,ou=swengg,dc=min,dc=io": {"consoleAdmin"}, + }, + ldapGroupPolicyMappings: map[string][]string{ + "cn=projectb,ou=groups,ou=swengg,dc=min,dc=io": {"mypolicy"}, + "cn=projecta,ou=groups,ou=swengg,dc=min,dc=io": {"consoleAdmin"}, + }, + }, + } + + for caseNum, content := range iamTestContentCases { + suite.SetUpSuite(c) + suite.SetUpLDAP(c, ldapServer) + exportedContent := suite.TestIAMExport(c, caseNum, content) + suite.TearDownSuite(c) + suite.SetUpSuite(c) + suite.SetUpLDAP(c, ldapServer) + suite.TestIAMImport(c, exportedContent, caseNum, content) + suite.TearDownSuite(c) + } + }, + ) + } +} + +func TestIAMImportAssetWithLDAP(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), testDefaultTimeout) + defer cancel() + + exportContentStrings := map[string]string{ + allPoliciesFile: `{"consoleAdmin":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:*"]},{"Effect":"Allow","Action":["kms:*"]},{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::*"]}]},"diagnostics":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:Prometheus","admin:Profiling","admin:ServerTrace","admin:ConsoleLog","admin:ServerInfo","admin:TopLocksInfo","admin:OBDInfo","admin:BandwidthMonitor"],"Resource":["arn:aws:s3:::*"]}]},"readonly":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:GetBucketLocation","s3:GetObject"],"Resource":["arn:aws:s3:::*"]}]},"readwrite":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::*"]}]},"writeonly":{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["s3:PutObject"],"Resource":["arn:aws:s3:::*"]}]}}`, + + // Built-in user should be imported without errors even if LDAP is + // enabled. + allUsersFile: `{ + "foo": { + "secretKey": "foobar123", + "status": "enabled" + } +} +`, + // Built-in groups should be imported without errors even if LDAP is + // enabled. + allGroupsFile: `{ + "mygroup": { + "version": 1, + "status": "enabled", + "members": [ + "foo" + ], + "updatedAt": "2024-04-23T21:34:43.587429659Z" + } +} +`, + // The `cn=projecty,..` group below is not under a configured DN, but we + // should still import without an error. + allSvcAcctsFile: `{ + "u4ccRswj62HV3Ifwima7": { + "parent": "uid=svc.algorithm,OU=swengg,DC=min,DC=io", + "accessKey": "u4ccRswj62HV3Ifwima7", + "secretKey": "ZoEoZdLlzVbOlT9rbhD7ZN7TLyiYXSAlB79uGEge", + "groups": ["cn=project.c,ou=groups,OU=swengg,DC=min,DC=io", "cn=projecty,ou=groups,ou=hwengg,dc=min,dc=io"], + "claims": { + "accessKey": "u4ccRswj62HV3Ifwima7", + "ldapUser": "uid=svc.algorithm,ou=swengg,dc=min,dc=io", + "ldapUsername": "svc.algorithm", + "parent": "uid=svc.algorithm,ou=swengg,dc=min,dc=io", + "sa-policy": "inherited-policy" + }, + "sessionPolicy": null, + "status": "on", + "name": "", + "description": "" + } +} +`, + // Built-in user-to-policies mapping should be imported without errors + // even if LDAP is enabled. + userPolicyMappingsFile: `{ + "foo": { + "version": 0, + "policy": "readwrite", + "updatedAt": "2024-04-23T21:34:43.815519816Z" + } +} +`, + // Contains: + // + // 1. duplicate mapping with same policy, we should not error out; + // + // 2. non-LDAP group mapping, we should not error out; + groupPolicyMappingsFile: `{ + "cn=project.c,ou=groups,ou=swengg,DC=min,dc=io": { + "version": 0, + "policy": "consoleAdmin", + "updatedAt": "2024-04-17T23:54:28.442998301Z" + }, + "mygroup": { + "version": 0, + "policy": "consoleAdmin", + "updatedAt": "2024-04-23T21:34:43.66922872Z" + }, + "cn=project.c,ou=groups,OU=swengg,DC=min,DC=io": { + "version": 0, + "policy": "consoleAdmin", + "updatedAt": "2024-04-17T20:54:28.442998301Z" + } +} +`, + stsUserPolicyMappingsFile: `{ + "uid=dillon,ou=people,OU=swengg,DC=min,DC=io": { + "version": 0, + "policy": "consoleAdmin", + "updatedAt": "2024-04-17T23:54:10.606645642Z" + } +} +`, + } + exportContent := map[string][]byte{} + for k, v := range exportContentStrings { + exportContent[k] = []byte(v) + } + + var importContent []byte + { + var b bytes.Buffer + zipWriter := zip.NewWriter(&b) + rawDataFn := func(r io.Reader, filename string, sz int) error { + header, zerr := zip.FileInfoHeader(dummyFileInfo{ + name: filename, + size: int64(sz), + mode: 0o600, + modTime: time.Now(), + isDir: false, + sys: nil, + }) + if zerr != nil { + adminLogIf(ctx, zerr) + return nil + } + header.Method = zip.Deflate + zwriter, zerr := zipWriter.CreateHeader(header) + if zerr != nil { + adminLogIf(ctx, zerr) + return nil + } + if _, err := io.Copy(zwriter, r); err != nil { + adminLogIf(ctx, err) + } + return nil + } + for _, f := range iamExportFiles { + iamFile := pathJoin(iamAssetsDir, f) + + fileContent, ok := exportContent[f] + if !ok { + t.Fatalf("missing content for %s", f) + } + + if err := rawDataFn(bytes.NewReader(fileContent), iamFile, len(fileContent)); err != nil { + t.Fatalf("failed to write %s: %v", iamFile, err) + } + } + zipWriter.Close() + importContent = b.Bytes() + } + + for i, testCase := range iamTestSuites { + t.Run( + fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription), + func(t *testing.T) { + c := &check{t, testCase.serverType} + suite := testCase + + ldapServer := os.Getenv(EnvTestLDAPServer) + if ldapServer == "" { + c.Skipf("Skipping LDAP test as no LDAP server is provided via %s", EnvTestLDAPServer) + } + + suite.SetUpSuite(c) + suite.SetUpLDAP(c, ldapServer) + suite.TestIAMImportAssetContent(c, importContent) + suite.TearDownSuite(c) + }, + ) + } +} + +type iamTestContent struct { + policies map[string][]byte + ldapUserPolicyMappings map[string][]string + ldapGroupPolicyMappings map[string][]string +} + +func (s *TestSuiteIAM) TestIAMExport(c *check, caseNum int, content iamTestContent) []byte { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + for policy, policyBytes := range content.policies { + err := s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("export %d: policy add error: %v", caseNum, err) + } + } + + for userDN, policies := range content.ldapUserPolicyMappings { + // No need to detach, we are starting from a clean slate after exporting. + _, err := s.adm.AttachPolicyLDAP(ctx, madmin.PolicyAssociationReq{ + Policies: policies, + User: userDN, + }) + if err != nil { + c.Fatalf("export %d: Unable to attach policy: %v", caseNum, err) + } + } + + for groupDN, policies := range content.ldapGroupPolicyMappings { + _, err := s.adm.AttachPolicyLDAP(ctx, madmin.PolicyAssociationReq{ + Policies: policies, + Group: groupDN, + }) + if err != nil { + c.Fatalf("export %d: Unable to attach group policy: %v", caseNum, err) + } + } + + contentReader, err := s.adm.ExportIAM(ctx) + if err != nil { + c.Fatalf("export %d: Unable to export IAM: %v", caseNum, err) + } + defer contentReader.Close() + + expContent, err := io.ReadAll(contentReader) + if err != nil { + c.Fatalf("export %d: Unable to read exported content: %v", caseNum, err) + } + + return expContent +} + +type dummyCloser struct { + io.Reader +} + +func (d dummyCloser) Close() error { return nil } + +func (s *TestSuiteIAM) TestIAMImportAssetContent(c *check, content []byte) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + dummyCloser := dummyCloser{bytes.NewReader(content)} + err := s.adm.ImportIAM(ctx, dummyCloser) + if err != nil { + c.Fatalf("Unable to import IAM: %v", err) + } + + entRes, err := s.adm.GetLDAPPolicyEntities(ctx, madmin.PolicyEntitiesQuery{}) + if err != nil { + c.Fatalf("Unable to get policy entities: %v", err) + } + + expected := madmin.PolicyEntitiesResult{ + PolicyMappings: []madmin.PolicyEntities{ + { + Policy: "consoleAdmin", + Users: []string{"uid=dillon,ou=people,ou=swengg,dc=min,dc=io"}, + Groups: []string{"cn=project.c,ou=groups,ou=swengg,dc=min,dc=io"}, + }, + }, + } + + entRes.Timestamp = time.Time{} + if !reflect.DeepEqual(expected, entRes) { + c.Fatalf("policy entities mismatch: expected: %v, got: %v", expected, entRes) + } + + dn := "uid=svc.algorithm,ou=swengg,dc=min,dc=io" + res, err := s.adm.ListAccessKeysLDAP(ctx, dn, "") + if err != nil { + c.Fatalf("Unable to list access keys: %v", err) + } + + epochTime := time.Unix(0, 0).UTC() + expectedAccKeys := madmin.ListAccessKeysLDAPResp{ + ServiceAccounts: []madmin.ServiceAccountInfo{ + { + AccessKey: "u4ccRswj62HV3Ifwima7", + Expiration: &epochTime, + }, + }, + } + + if !reflect.DeepEqual(expectedAccKeys, res) { + c.Fatalf("access keys mismatch: expected: %v, got: %v", expectedAccKeys, res) + } + + accKeyInfo, err := s.adm.InfoServiceAccount(ctx, "u4ccRswj62HV3Ifwima7") + if err != nil { + c.Fatalf("Unable to get service account info: %v", err) + } + if accKeyInfo.ParentUser != "uid=svc.algorithm,ou=swengg,dc=min,dc=io" { + c.Fatalf("parent mismatch: expected: %s, got: %s", "uid=svc.algorithm,ou=swengg,dc=min,dc=io", accKeyInfo.ParentUser) + } +} + +func (s *TestSuiteIAM) TestIAMImport(c *check, exportedContent []byte, caseNum int, content iamTestContent) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + dummyCloser := dummyCloser{bytes.NewReader(exportedContent)} + err := s.adm.ImportIAM(ctx, dummyCloser) + if err != nil { + c.Fatalf("import %d: Unable to import IAM: %v", caseNum, err) + } + + gotContent := iamTestContent{ + policies: make(map[string][]byte), + ldapUserPolicyMappings: make(map[string][]string), + ldapGroupPolicyMappings: make(map[string][]string), + } + policyContentMap, err := s.adm.ListCannedPolicies(ctx) + if err != nil { + c.Fatalf("import %d: Unable to list policies: %v", caseNum, err) + } + defaultCannedPolicies := set.CreateStringSet("consoleAdmin", "readwrite", "readonly", + "diagnostics", "writeonly") + for policy, policyBytes := range policyContentMap { + if defaultCannedPolicies.Contains(policy) { + continue + } + gotContent.policies[policy] = policyBytes + } + + policyQueryRes, err := s.adm.GetLDAPPolicyEntities(ctx, madmin.PolicyEntitiesQuery{}) + if err != nil { + c.Fatalf("import %d: Unable to get policy entities: %v", caseNum, err) + } + + for _, entity := range policyQueryRes.PolicyMappings { + m := gotContent.ldapUserPolicyMappings + for _, user := range entity.Users { + m[user] = append(m[user], entity.Policy) + } + m = gotContent.ldapGroupPolicyMappings + for _, group := range entity.Groups { + m[group] = append(m[group], entity.Policy) + } + } + + { + // We don't compare the values of the canned policies because server is + // re-encoding them. (FIXME?) + for k := range content.policies { + content.policies[k] = nil + gotContent.policies[k] = nil + } + if !reflect.DeepEqual(content.policies, gotContent.policies) { + c.Fatalf("import %d: policies mismatch: expected: %v, got: %v", caseNum, content.policies, gotContent.policies) + } + } + + if !reflect.DeepEqual(content.ldapUserPolicyMappings, gotContent.ldapUserPolicyMappings) { + c.Fatalf("import %d: user policy mappings mismatch: expected: %v, got: %v", caseNum, content.ldapUserPolicyMappings, gotContent.ldapUserPolicyMappings) + } + + if !reflect.DeepEqual(content.ldapGroupPolicyMappings, gotContent.ldapGroupPolicyMappings) { + c.Fatalf("import %d: group policy mappings mismatch: expected: %v, got: %v", caseNum, content.ldapGroupPolicyMappings, gotContent.ldapGroupPolicyMappings) + } +} + +func (s *TestSuiteIAM) TestLDAPSTS(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket create error: %v", err) + } + + // Create policy + policy := "mypolicy" + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + ldapID := cr.LDAPIdentity{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + LDAPUsername: "dillon", + LDAPPassword: "dillon", + } + + _, err = ldapID.Retrieve() + if err == nil { + c.Fatalf("Expected to fail to create STS cred with no associated policy!") + } + + // Attempting to set a non-existent policy should fail. + userDN := "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" + _, err = s.adm.AttachPolicyLDAP(ctx, madmin.PolicyAssociationReq{ + Policies: []string{policy + "x"}, + User: userDN, + }) + if err == nil { + c.Fatalf("should not be able to attach non-existent policy") + } + + userReq := madmin.PolicyAssociationReq{ + Policies: []string{policy}, + User: userDN, + } + + if _, err = s.adm.AttachPolicyLDAP(ctx, userReq); err != nil { + c.Fatalf("Unable to attach user policy: %v", err) + } + + value, err := ldapID.Retrieve() + if err != nil { + c.Fatalf("Expected to generate STS creds, got err: %#v", err) + } + + minioClient, err := minio.New(s.endpoint, &minio.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("Error initializing client: %v", err) + } + + // Validate that user listing does not return any entries + usersList, err := s.adm.ListUsers(ctx) + if err != nil { + c.Fatalf("list users should not fail: %v", err) + } + if len(usersList) != 1 { + c.Fatalf("expected user listing output: %v", usersList) + } + uinfo := usersList[userDN] + if uinfo.PolicyName != policy || uinfo.Status != madmin.AccountEnabled { + c.Fatalf("expected user listing content: %v", uinfo) + } + + // Validate that the client from sts creds can access the bucket. + c.mustListObjects(ctx, minioClient, bucket) + + // Validate that the client cannot remove any objects + err = minioClient.RemoveObject(ctx, bucket, "someobject", minio.RemoveObjectOptions{}) + if err.Error() != "Access Denied." { + c.Fatalf("unexpected non-access-denied err: %v", err) + } + + if _, err = s.adm.DetachPolicyLDAP(ctx, userReq); err != nil { + c.Fatalf("Unable to detach user policy: %v", err) + } + + _, err = ldapID.Retrieve() + if err == nil { + c.Fatalf("Expected to fail to create a user with no associated policy!") + } + + // Set policy via group and validate policy assignment. + groupDN := "cn=projectb,ou=groups,ou=swengg,dc=min,dc=io" + groupReq := madmin.PolicyAssociationReq{ + Policies: []string{policy}, + Group: groupDN, + } + + if _, err = s.adm.AttachPolicyLDAP(ctx, groupReq); err != nil { + c.Fatalf("Unable to attach group policy: %v", err) + } + + value, err = ldapID.Retrieve() + if err != nil { + c.Fatalf("Expected to generate STS creds, got err: %#v", err) + } + + minioClient, err = minio.New(s.endpoint, &minio.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("Error initializing client: %v", err) + } + + // Validate that the client from sts creds can access the bucket. + c.mustListObjects(ctx, minioClient, bucket) + + // Validate that the client cannot remove any objects + err = minioClient.RemoveObject(ctx, bucket, "someobject", minio.RemoveObjectOptions{}) + c.Assert(err.Error(), "Access Denied.") + + if _, err = s.adm.DetachPolicyLDAP(ctx, groupReq); err != nil { + c.Fatalf("Unable to detach group policy: %v", err) + } +} + +func (s *TestSuiteIAM) TestLDAPUnicodeVariationsLegacyAPI(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket create error: %v", err) + } + + // Create policy + policy := "mypolicy" + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + ldapID := cr.LDAPIdentity{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + LDAPUsername: "svc.algorithm", + LDAPPassword: "example", + } + + _, err = ldapID.Retrieve() + if err == nil { + c.Fatalf("Expected to fail to create STS cred with no associated policy!") + } + + mustNormalizeDN := func(dn string) string { + normalizedDN, err := ldap.NormalizeDN(dn) + if err != nil { + c.Fatalf("normalize err: %v", err) + } + return normalizedDN + } + + actualUserDN := mustNormalizeDN("uid=svc.algorithm,OU=swengg,DC=min,DC=io") + + // \uFE52 is the unicode dot SMALL FULL STOP used below: + userDNWithUnicodeDot := "uid=svc﹒algorithm,OU=swengg,DC=min,DC=io" + + if err = s.adm.SetPolicy(ctx, policy, userDNWithUnicodeDot, false); err != nil { + c.Fatalf("Unable to set policy: %v", err) + } + + value, err := ldapID.Retrieve() + if err != nil { + c.Fatalf("Expected to generate STS creds, got err: %#v", err) + } + + usersList, err := s.adm.ListUsers(ctx) + if err != nil { + c.Fatalf("list users should not fail: %v", err) + } + if len(usersList) != 1 { + c.Fatalf("expected user listing output: %#v", usersList) + } + uinfo := usersList[actualUserDN] + if uinfo.PolicyName != policy || uinfo.Status != madmin.AccountEnabled { + c.Fatalf("expected user listing content: %v", uinfo) + } + + minioClient, err := minio.New(s.endpoint, &minio.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("Error initializing client: %v", err) + } + + // Validate that the client from sts creds can access the bucket. + c.mustListObjects(ctx, minioClient, bucket) + + // Validate that the client cannot remove any objects + err = minioClient.RemoveObject(ctx, bucket, "someobject", minio.RemoveObjectOptions{}) + if err.Error() != "Access Denied." { + c.Fatalf("unexpected non-access-denied err: %v", err) + } + + // Remove the policy assignment on the user DN: + if err = s.adm.SetPolicy(ctx, "", userDNWithUnicodeDot, false); err != nil { + c.Fatalf("Unable to remove policy setting: %v", err) + } + + _, err = ldapID.Retrieve() + if err == nil { + c.Fatalf("Expected to fail to create a user with no associated policy!") + } + + // Set policy via group and validate policy assignment. + actualGroupDN := mustNormalizeDN("cn=project.c,ou=groups,ou=swengg,dc=min,dc=io") + groupDNWithUnicodeDot := "cn=project﹒c,ou=groups,ou=swengg,dc=min,dc=io" + if err = s.adm.SetPolicy(ctx, policy, groupDNWithUnicodeDot, true); err != nil { + c.Fatalf("Unable to attach group policy: %v", err) + } + + value, err = ldapID.Retrieve() + if err != nil { + c.Fatalf("Expected to generate STS creds, got err: %#v", err) + } + + policyResult, err := s.adm.GetLDAPPolicyEntities(ctx, madmin.PolicyEntitiesQuery{ + Policy: []string{policy}, + }) + if err != nil { + c.Fatalf("GetLDAPPolicyEntities should not fail: %v", err) + } + { + // Check that the mapping we created exists. + idx := slices.IndexFunc(policyResult.PolicyMappings, func(e madmin.PolicyEntities) bool { + return e.Policy == policy && slices.Contains(e.Groups, actualGroupDN) + }) + if idx < 0 { + c.Fatalf("expected groupDN (%s) to be present in mapping list: %#v", actualGroupDN, policyResult) + } + } + + minioClient, err = minio.New(s.endpoint, &minio.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("Error initializing client: %v", err) + } + + // Validate that the client from sts creds can access the bucket. + c.mustListObjects(ctx, minioClient, bucket) + + // Validate that the client cannot remove any objects + err = minioClient.RemoveObject(ctx, bucket, "someobject", minio.RemoveObjectOptions{}) + c.Assert(err.Error(), "Access Denied.") +} + +func (s *TestSuiteIAM) TestLDAPUnicodeVariations(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket create error: %v", err) + } + + // Create policy + policy := "mypolicy" + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + ldapID := cr.LDAPIdentity{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + LDAPUsername: "svc.algorithm", + LDAPPassword: "example", + } + + _, err = ldapID.Retrieve() + if err == nil { + c.Fatalf("Expected to fail to create STS cred with no associated policy!") + } + + mustNormalizeDN := func(dn string) string { + normalizedDN, err := ldap.NormalizeDN(dn) + if err != nil { + c.Fatalf("normalize err: %v", err) + } + return normalizedDN + } + + actualUserDN := mustNormalizeDN("uid=svc.algorithm,OU=swengg,DC=min,DC=io") + + // \uFE52 is the unicode dot SMALL FULL STOP used below: + userDNWithUnicodeDot := "uid=svc﹒algorithm,OU=swengg,DC=min,DC=io" + + userReq := madmin.PolicyAssociationReq{ + Policies: []string{policy}, + User: userDNWithUnicodeDot, + } + + if _, err = s.adm.AttachPolicyLDAP(ctx, userReq); err != nil { + c.Fatalf("Unable to attach user policy: %v", err) + } + + value, err := ldapID.Retrieve() + if err != nil { + c.Fatalf("Expected to generate STS creds, got err: %#v", err) + } + + usersList, err := s.adm.ListUsers(ctx) + if err != nil { + c.Fatalf("list users should not fail: %v", err) + } + if len(usersList) != 1 { + c.Fatalf("expected user listing output: %#v", usersList) + } + uinfo := usersList[actualUserDN] + if uinfo.PolicyName != policy || uinfo.Status != madmin.AccountEnabled { + c.Fatalf("expected user listing content: %v", uinfo) + } + + minioClient, err := minio.New(s.endpoint, &minio.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("Error initializing client: %v", err) + } + + // Validate that the client from sts creds can access the bucket. + c.mustListObjects(ctx, minioClient, bucket) + + // Validate that the client cannot remove any objects + err = minioClient.RemoveObject(ctx, bucket, "someobject", minio.RemoveObjectOptions{}) + if err.Error() != "Access Denied." { + c.Fatalf("unexpected non-access-denied err: %v", err) + } + + // Remove the policy assignment on the user DN: + + if _, err = s.adm.DetachPolicyLDAP(ctx, userReq); err != nil { + c.Fatalf("Unable to detach user policy: %v", err) + } + + _, err = ldapID.Retrieve() + if err == nil { + c.Fatalf("Expected to fail to create a user with no associated policy!") + } + + // Set policy via group and validate policy assignment. + actualGroupDN := mustNormalizeDN("cn=project.c,ou=groups,ou=swengg,dc=min,dc=io") + groupDNWithUnicodeDot := "cn=project﹒c,ou=groups,ou=swengg,dc=min,dc=io" + groupReq := madmin.PolicyAssociationReq{ + Policies: []string{policy}, + Group: groupDNWithUnicodeDot, + } + + if _, err = s.adm.AttachPolicyLDAP(ctx, groupReq); err != nil { + c.Fatalf("Unable to attach group policy: %v", err) + } + + value, err = ldapID.Retrieve() + if err != nil { + c.Fatalf("Expected to generate STS creds, got err: %#v", err) + } + + policyResult, err := s.adm.GetLDAPPolicyEntities(ctx, madmin.PolicyEntitiesQuery{ + Policy: []string{policy}, + }) + if err != nil { + c.Fatalf("GetLDAPPolicyEntities should not fail: %v", err) + } + { + // Check that the mapping we created exists. + idx := slices.IndexFunc(policyResult.PolicyMappings, func(e madmin.PolicyEntities) bool { + return e.Policy == policy && slices.Contains(e.Groups, actualGroupDN) + }) + if idx < 0 { + c.Fatalf("expected groupDN (%s) to be present in mapping list: %#v", actualGroupDN, policyResult) + } + } + + minioClient, err = minio.New(s.endpoint, &minio.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("Error initializing client: %v", err) + } + + // Validate that the client from sts creds can access the bucket. + c.mustListObjects(ctx, minioClient, bucket) + + // Validate that the client cannot remove any objects + err = minioClient.RemoveObject(ctx, bucket, "someobject", minio.RemoveObjectOptions{}) + c.Assert(err.Error(), "Access Denied.") + + if _, err = s.adm.DetachPolicyLDAP(ctx, groupReq); err != nil { + c.Fatalf("Unable to detach group policy: %v", err) + } +} + +func (s *TestSuiteIAM) TestLDAPSTSServiceAccounts(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket create error: %v", err) + } + + // Create policy + policy := "mypolicy" + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + userDN := "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" + userReq := madmin.PolicyAssociationReq{ + Policies: []string{policy}, + User: userDN, + } + + if _, err = s.adm.AttachPolicyLDAP(ctx, userReq); err != nil { + c.Fatalf("Unable to attach user policy: %v", err) + } + + ldapID := cr.LDAPIdentity{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + LDAPUsername: "dillon", + LDAPPassword: "dillon", + } + + value, err := ldapID.Retrieve() + if err != nil { + c.Fatalf("Expected to generate STS creds, got err: %#v", err) + } + + // Check that the LDAP sts cred is actually working. + minioClient, err := minio.New(s.endpoint, &minio.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("Error initializing client: %v", err) + } + + // Validate that the client from sts creds can access the bucket. + c.mustListObjects(ctx, minioClient, bucket) + + // Create an madmin client with user creds + userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + }) + if err != nil { + c.Fatalf("Err creating user admin client: %v", err) + } + userAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport) + + // Create svc acc + cr := c.mustCreateSvcAccount(ctx, value.AccessKeyID, userAdmClient) + + // 1. Check that svc account appears in listing + c.assertSvcAccAppearsInListing(ctx, userAdmClient, value.AccessKeyID, cr.AccessKey) + + // 2. Check that svc account info can be queried + c.assertSvcAccInfoQueryable(ctx, userAdmClient, value.AccessKeyID, cr.AccessKey, true) + + // 3. Check S3 access + c.assertSvcAccS3Access(ctx, s, cr, bucket) + + // 5. Check that service account can be deleted. + c.assertSvcAccDeletion(ctx, s, userAdmClient, value.AccessKeyID, bucket) + + // 6. Check that service account cannot be created for some other user. + c.mustNotCreateSvcAccount(ctx, globalActiveCred.AccessKey, userAdmClient) + + // Detach the policy from the user + if _, err = s.adm.DetachPolicyLDAP(ctx, userReq); err != nil { + c.Fatalf("Unable to detach user policy: %v", err) + } +} + +func (s *TestSuiteIAM) TestLDAPSTSServiceAccountsWithUsername(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + bucket := "dillon" + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket create error: %v", err) + } + + // Create policy + policy := "mypolicy-username" + policyBytes := []byte(`{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::${ldap:username}/*" + ] + } + ] +}`) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + userDN := "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" + + userReq := madmin.PolicyAssociationReq{ + Policies: []string{policy}, + User: userDN, + } + + if _, err = s.adm.AttachPolicyLDAP(ctx, userReq); err != nil { + c.Fatalf("Unable to attach user policy: %v", err) + } + + ldapID := cr.LDAPIdentity{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + LDAPUsername: "dillon", + LDAPPassword: "dillon", + } + + value, err := ldapID.Retrieve() + if err != nil { + c.Fatalf("Expected to generate STS creds, got err: %#v", err) + } + + // Check that the LDAP sts cred is actually working. + minioClient, err := minio.New(s.endpoint, &minio.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("Error initializing client: %v", err) + } + + // Validate that the client from sts creds can access the bucket. + c.mustListObjects(ctx, minioClient, bucket) + + // Create an madmin client with user creds + userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + }) + if err != nil { + c.Fatalf("Err creating user admin client: %v", err) + } + userAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport) + + // Create svc acc + cr := c.mustCreateSvcAccount(ctx, value.AccessKeyID, userAdmClient) + + svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "") + + // 1. Check S3 access for service account ListObjects() + c.mustListObjects(ctx, svcClient, bucket) + + // 2. Check S3 access for upload + c.mustUpload(ctx, svcClient, bucket) + + // 3. Check S3 access for download + c.mustDownload(ctx, svcClient, bucket) + + if _, err = s.adm.DetachPolicyLDAP(ctx, userReq); err != nil { + c.Fatalf("Unable to detach user policy: %v", err) + } +} + +// In this test, the parent users gets their permissions from a group, rather +// than having a policy set directly on them. +func (s *TestSuiteIAM) TestLDAPSTSServiceAccountsWithGroups(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket create error: %v", err) + } + + // Create policy + policy := "mypolicy" + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + groupDN := "cn=projecta,ou=groups,ou=swengg,dc=min,dc=io" + userReq := madmin.PolicyAssociationReq{ + Policies: []string{policy}, + Group: groupDN, + } + + if _, err = s.adm.AttachPolicyLDAP(ctx, userReq); err != nil { + c.Fatalf("Unable to attach user policy: %v", err) + } + + ldapID := cr.LDAPIdentity{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + LDAPUsername: "dillon", + LDAPPassword: "dillon", + } + + value, err := ldapID.Retrieve() + if err != nil { + c.Fatalf("Expected to generate STS creds, got err: %#v", err) + } + + // Check that the LDAP sts cred is actually working. + minioClient, err := minio.New(s.endpoint, &minio.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("Error initializing client: %v", err) + } + + // Validate that the client from sts creds can access the bucket. + c.mustListObjects(ctx, minioClient, bucket) + + // Create an madmin client with user creds + userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + }) + if err != nil { + c.Fatalf("Err creating user admin client: %v", err) + } + userAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport) + + // Create svc acc + cr := c.mustCreateSvcAccount(ctx, value.AccessKeyID, userAdmClient) + + // 1. Check that svc account appears in listing + c.assertSvcAccAppearsInListing(ctx, userAdmClient, value.AccessKeyID, cr.AccessKey) + + // 2. Check that svc account info can be queried + c.assertSvcAccInfoQueryable(ctx, userAdmClient, value.AccessKeyID, cr.AccessKey, true) + + // 3. Check S3 access + c.assertSvcAccS3Access(ctx, s, cr, bucket) + + // 5. Check that service account can be deleted. + c.assertSvcAccDeletion(ctx, s, userAdmClient, value.AccessKeyID, bucket) + + // 6. Check that service account cannot be created for some other user. + c.mustNotCreateSvcAccount(ctx, globalActiveCred.AccessKey, userAdmClient) + + // Detach the user policy + if _, err = s.adm.DetachPolicyLDAP(ctx, userReq); err != nil { + c.Fatalf("Unable to detach user policy: %v", err) + } +} + +func (s *TestSuiteIAM) TestLDAPCyrillicUser(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + userReq := madmin.PolicyAssociationReq{ + Policies: []string{"readwrite"}, + User: "uid=Пользователь,ou=people,ou=swengg,dc=min,dc=io", + } + + if _, err := s.adm.AttachPolicyLDAP(ctx, userReq); err != nil { + c.Fatalf("Unable to attach user policy: %v", err) + } + + cases := []struct { + username string + dn string + }{ + { + username: "Пользователь", + dn: "uid=Пользователь,ou=people,ou=swengg,dc=min,dc=io", + }, + } + + conn, err := globalIAMSys.LDAPConfig.LDAP.Connect() + if err != nil { + c.Fatalf("LDAP connect failed: %v", err) + } + defer conn.Close() + + for i, testCase := range cases { + ldapID := cr.LDAPIdentity{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + LDAPUsername: testCase.username, + LDAPPassword: "example", + } + + value, err := ldapID.Retrieve() + if err != nil { + c.Fatalf("Expected to generate STS creds, got err: %#v", err) + } + + // Retrieve the STS account's credential object. + u, ok := globalIAMSys.GetUser(ctx, value.AccessKeyID) + if !ok { + c.Fatalf("Expected to find user %s", value.AccessKeyID) + } + + if u.Credentials.AccessKey != value.AccessKeyID { + c.Fatalf("Expected access key %s, got %s", value.AccessKeyID, u.Credentials.AccessKey) + } + + // Retrieve the credential's claims. + secret, err := getTokenSigningKey() + if err != nil { + c.Fatalf("Error getting token signing key: %v", err) + } + claims, err := getClaimsFromTokenWithSecret(value.SessionToken, secret) + if err != nil { + c.Fatalf("Error getting claims from token: %v", err) + } + + // Validate claims. + dnClaim := claims.MapClaims[ldapActualUser].(string) + if dnClaim != testCase.dn { + c.Fatalf("Test %d: unexpected dn claim: %s", i+1, dnClaim) + } + } + + if _, err = s.adm.DetachPolicyLDAP(ctx, userReq); err != nil { + c.Fatalf("Unable to detach user policy: %v", err) + } +} + +func (s *TestSuiteIAM) TestLDAPSlashDN(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + policyReq := madmin.PolicyAssociationReq{ + Policies: []string{"readwrite"}, + } + + cases := []struct { + username string + dn string + group string + }{ + { + username: "slashuser", + dn: "uid=slash/user,ou=people,ou=swengg,dc=min,dc=io", + }, + { + username: "dillon", + dn: "uid=dillon,ou=people,ou=swengg,dc=min,dc=io", + group: "cn=project/d,ou=groups,ou=swengg,dc=min,dc=io", + }, + } + + conn, err := globalIAMSys.LDAPConfig.LDAP.Connect() + if err != nil { + c.Fatalf("LDAP connect failed: %v", err) + } + defer conn.Close() + + for i, testCase := range cases { + if testCase.group != "" { + policyReq.Group = testCase.group + policyReq.User = "" + } else { + policyReq.User = testCase.dn + policyReq.Group = "" + } + + if _, err := s.adm.AttachPolicyLDAP(ctx, policyReq); err != nil { + c.Fatalf("Unable to attach policy: %v", err) + } + + ldapID := cr.LDAPIdentity{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + LDAPUsername: testCase.username, + LDAPPassword: testCase.username, + } + + value, err := ldapID.Retrieve() + if err != nil { + c.Fatalf("Expected to generate STS creds, got err: %#v", err) + } + + // Retrieve the STS account's credential object. + u, ok := globalIAMSys.GetUser(ctx, value.AccessKeyID) + if !ok { + c.Fatalf("Expected to find user %s", value.AccessKeyID) + } + + if u.Credentials.AccessKey != value.AccessKeyID { + c.Fatalf("Expected access key %s, got %s", value.AccessKeyID, u.Credentials.AccessKey) + } + + // Retrieve the credential's claims. + secret, err := getTokenSigningKey() + if err != nil { + c.Fatalf("Error getting token signing key: %v", err) + } + claims, err := getClaimsFromTokenWithSecret(value.SessionToken, secret) + if err != nil { + c.Fatalf("Error getting claims from token: %v", err) + } + + // Validate claims. + dnClaim := claims.MapClaims[ldapActualUser].(string) + if dnClaim != testCase.dn { + c.Fatalf("Test %d: unexpected dn claim: %s", i+1, dnClaim) + } + + if _, err = s.adm.DetachPolicyLDAP(ctx, policyReq); err != nil { + c.Fatalf("Unable to detach user policy: %v", err) + } + } +} + +func (s *TestSuiteIAM) TestLDAPAttributesLookup(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + groupDN := "cn=projectb,ou=groups,ou=swengg,dc=min,dc=io" + groupReq := madmin.PolicyAssociationReq{ + Policies: []string{"readwrite"}, + Group: groupDN, + } + + if _, err := s.adm.AttachPolicyLDAP(ctx, groupReq); err != nil { + c.Fatalf("Unable to attach user policy: %v", err) + } + + cases := []struct { + username string + dn string + expectedSSHKeyType string + }{ + { + username: "dillon", + dn: "uid=dillon,ou=people,ou=swengg,dc=min,dc=io", + expectedSSHKeyType: "ssh-ed25519", + }, + { + username: "liza", + dn: "uid=liza,ou=people,ou=swengg,dc=min,dc=io", + expectedSSHKeyType: "ssh-rsa", + }, + } + + conn, err := globalIAMSys.LDAPConfig.LDAP.Connect() + if err != nil { + c.Fatalf("LDAP connect failed: %v", err) + } + defer conn.Close() + + for i, testCase := range cases { + ldapID := cr.LDAPIdentity{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + LDAPUsername: testCase.username, + LDAPPassword: testCase.username, + } + + value, err := ldapID.Retrieve() + if err != nil { + c.Fatalf("Expected to generate STS creds, got err: %#v", err) + } + + // Retrieve the STS account's credential object. + u, ok := globalIAMSys.GetUser(ctx, value.AccessKeyID) + if !ok { + c.Fatalf("Expected to find user %s", value.AccessKeyID) + } + + if u.Credentials.AccessKey != value.AccessKeyID { + c.Fatalf("Expected access key %s, got %s", value.AccessKeyID, u.Credentials.AccessKey) + } + + // Retrieve the credential's claims. + secret, err := getTokenSigningKey() + if err != nil { + c.Fatalf("Error getting token signing key: %v", err) + } + claims, err := getClaimsFromTokenWithSecret(value.SessionToken, secret) + if err != nil { + c.Fatalf("Error getting claims from token: %v", err) + } + + // Validate claims. Check if the sshPublicKey claim is present. + dnClaim := claims.MapClaims[ldapActualUser].(string) + if dnClaim != testCase.dn { + c.Fatalf("Test %d: unexpected dn claim: %s", i+1, dnClaim) + } + sshPublicKeyClaim := claims.MapClaims[ldapAttribPrefix+"sshPublicKey"].([]any)[0].(string) + if sshPublicKeyClaim == "" { + c.Fatalf("Test %d: expected sshPublicKey claim to be present", i+1) + } + parts := strings.Split(sshPublicKeyClaim, " ") + if parts[0] != testCase.expectedSSHKeyType { + c.Fatalf("Test %d: unexpected sshPublicKey type: %s", i+1, parts[0]) + } + } + + if _, err = s.adm.DetachPolicyLDAP(ctx, groupReq); err != nil { + c.Fatalf("Unable to detach group policy: %v", err) + } +} + +func (s *TestSuiteIAM) TestLDAPPolicyEntitiesLookup(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + groupDN := "cn=projectb,ou=groups,ou=swengg,dc=min,dc=io" + groupPolicy := "readwrite" + groupReq := madmin.PolicyAssociationReq{ + Policies: []string{groupPolicy}, + Group: groupDN, + } + _, err := s.adm.AttachPolicyLDAP(ctx, groupReq) + if err != nil { + c.Fatalf("Unable to attach group policy: %v", err) + } + type caseTemplate struct { + inDN string + expectedOutDN string + expectedGroupDN string + expectedGroupPolicy string + } + cases := []caseTemplate{ + { + inDN: "uid=dillon,ou=people,ou=swengg,dc=min,dc=io", + expectedOutDN: "uid=dillon,ou=people,ou=swengg,dc=min,dc=io", + expectedGroupDN: groupDN, + expectedGroupPolicy: groupPolicy, + }, + } + + policy := "readonly" + for _, testCase := range cases { + userReq := madmin.PolicyAssociationReq{ + Policies: []string{policy}, + User: testCase.inDN, + } + _, err := s.adm.AttachPolicyLDAP(ctx, userReq) + if err != nil { + c.Fatalf("Unable to attach policy: %v", err) + } + + entities, err := s.adm.GetLDAPPolicyEntities(ctx, madmin.PolicyEntitiesQuery{ + Users: []string{testCase.inDN}, + Policy: []string{policy}, + }) + if err != nil { + c.Fatalf("Unable to fetch policy entities: %v", err) + } + + // switch statement to check all the conditions + switch { + case len(entities.UserMappings) != 1: + c.Fatalf("Expected to find exactly one user mapping") + case entities.UserMappings[0].User != testCase.expectedOutDN: + c.Fatalf("Expected user DN `%s`, found `%s`", testCase.expectedOutDN, entities.UserMappings[0].User) + case len(entities.UserMappings[0].Policies) != 1: + c.Fatalf("Expected exactly one policy attached to user") + case entities.UserMappings[0].Policies[0] != policy: + c.Fatalf("Expected attached policy `%s`, found `%s`", policy, entities.UserMappings[0].Policies[0]) + case len(entities.UserMappings[0].MemberOfMappings) != 1: + c.Fatalf("Expected exactly one group attached to user") + case entities.UserMappings[0].MemberOfMappings[0].Group != testCase.expectedGroupDN: + c.Fatalf("Expected attached group `%s`, found `%s`", testCase.expectedGroupDN, entities.UserMappings[0].MemberOfMappings[0].Group) + case len(entities.UserMappings[0].MemberOfMappings[0].Policies) != 1: + c.Fatalf("Expected exactly one policy attached to group") + case entities.UserMappings[0].MemberOfMappings[0].Policies[0] != testCase.expectedGroupPolicy: + c.Fatalf("Expected attached policy `%s`, found `%s`", testCase.expectedGroupPolicy, entities.UserMappings[0].MemberOfMappings[0].Policies[0]) + } + + _, err = s.adm.DetachPolicyLDAP(ctx, userReq) + if err != nil { + c.Fatalf("Unable to detach policy: %v", err) + } + } + + _, err = s.adm.DetachPolicyLDAP(ctx, groupReq) + if err != nil { + c.Fatalf("Unable to detach group policy: %v", err) + } +} + +func (s *TestSuiteIAM) TestOpenIDSTS(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket create error: %v", err) + } + + // Generate web identity STS token by interacting with OpenID IDP. + token, err := MockOpenIDTestUserInteraction(ctx, testAppParams, "dillon@example.io", "dillon") + if err != nil { + c.Fatalf("mock user err: %v", err) + } + // fmt.Printf("TOKEN: %s\n", token) + + webID := cr.STSWebIdentity{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + GetWebIDTokenExpiry: func() (*cr.WebIdentityToken, error) { + return &cr.WebIdentityToken{ + Token: token, + }, nil + }, + } + + // Create policy - with name as one of the groups in OpenID the user is + // a member of. + policy := "projecta" + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + value, err := webID.Retrieve() + if err != nil { + c.Fatalf("Expected to generate STS creds, got err: %#v", err) + } + + minioClient, err := minio.New(s.endpoint, &minio.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("Error initializing client: %v", err) + } + + // Validate that the client from sts creds can access the bucket. + c.mustListObjects(ctx, minioClient, bucket) + + // Validate that the client cannot remove any objects + err = minioClient.RemoveObject(ctx, bucket, "someobject", minio.RemoveObjectOptions{}) + if err.Error() != "Access Denied." { + c.Fatalf("unexpected non-access-denied err: %v", err) + } +} + +func (s *TestSuiteIAM) TestOpenIDSTSDurationSeconds(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket create error: %v", err) + } + + // Generate web identity STS token by interacting with OpenID IDP. + token, err := MockOpenIDTestUserInteraction(ctx, testAppParams, "dillon@example.io", "dillon") + if err != nil { + c.Fatalf("mock user err: %v", err) + } + // fmt.Printf("TOKEN: %s\n", token) + + webID := cr.STSWebIdentity{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + GetWebIDTokenExpiry: func() (*cr.WebIdentityToken, error) { + return &cr.WebIdentityToken{ + Token: token, + Expiry: 900, + }, nil + }, + } + + // Create policy - with name as one of the groups in OpenID the user is + // a member of. + policy := "projecta" + policyTmpl := `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Deny", + "Action": ["sts:AssumeRoleWithWebIdentity"], + "Condition": {"NumericGreaterThan": {"sts:DurationSeconds": "%d"}} + }, + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}` + + for i, testCase := range []struct { + durSecs int + expectedErr bool + }{ + {60, true}, + {1800, false}, + } { + policyBytes := fmt.Appendf(nil, policyTmpl, testCase.durSecs, bucket) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("Test %d: policy add error: %v", i+1, err) + } + + value, err := webID.Retrieve() + if err != nil && !testCase.expectedErr { + c.Fatalf("Test %d: Expected to generate STS creds, got err: %#v", i+1, err) + } + if err == nil && testCase.expectedErr { + c.Fatalf("Test %d: An error is unexpected to generate STS creds, got err: %#v", i+1, err) + } + + if err != nil && testCase.expectedErr { + continue + } + + minioClient, err := minio.New(s.endpoint, &minio.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("Test %d: Error initializing client: %v", i+1, err) + } + + c.mustListObjects(ctx, minioClient, bucket) + } +} + +func (s *TestSuiteIAM) TestOpenIDSTSAddUser(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket create error: %v", err) + } + + // Generate web identity STS token by interacting with OpenID IDP. + token, err := MockOpenIDTestUserInteraction(ctx, testAppParams, "dillon@example.io", "dillon") + if err != nil { + c.Fatalf("mock user err: %v", err) + } + + webID := cr.STSWebIdentity{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + GetWebIDTokenExpiry: func() (*cr.WebIdentityToken, error) { + return &cr.WebIdentityToken{ + Token: token, + }, nil + }, + } + + // Create policy - with name as one of the groups in OpenID the user is + // a member of. + policy := "projecta" + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + value, err := webID.Retrieve() + if err != nil { + c.Fatalf("Expected to generate STS creds, got err: %#v", err) + } + + // Create an madmin client with user creds + userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + }) + if err != nil { + c.Fatalf("Err creating user admin client: %v", err) + } + userAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport) + + c.mustNotCreateIAMUser(ctx, userAdmClient) + + // Create admin user policy. + policyBytes = []byte(`{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "admin:*" + ] + } + ] +}`) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + cr := c.mustCreateIAMUser(ctx, userAdmClient) + + userInfo := c.mustGetIAMUserInfo(ctx, userAdmClient, cr.AccessKey) + c.Assert(userInfo.Status, madmin.AccountEnabled) +} + +func (s *TestSuiteIAM) TestOpenIDServiceAcc(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket create error: %v", err) + } + + // Generate web identity STS token by interacting with OpenID IDP. + token, err := MockOpenIDTestUserInteraction(ctx, testAppParams, "dillon@example.io", "dillon") + if err != nil { + c.Fatalf("mock user err: %v", err) + } + + webID := cr.STSWebIdentity{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + GetWebIDTokenExpiry: func() (*cr.WebIdentityToken, error) { + return &cr.WebIdentityToken{ + Token: token, + }, nil + }, + } + + // Create policy - with name as one of the groups in OpenID the user is + // a member of. + policy := "projecta" + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject", + "s3:ListBucket" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket) + err = s.adm.AddCannedPolicy(ctx, policy, policyBytes) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + + value, err := webID.Retrieve() + if err != nil { + c.Fatalf("Expected to generate STS creds, got err: %#v", err) + } + + // Create an madmin client with user creds + userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + }) + if err != nil { + c.Fatalf("Err creating user admin client: %v", err) + } + userAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport) + + // Create svc acc + cr := c.mustCreateSvcAccount(ctx, value.AccessKeyID, userAdmClient) + + // 1. Check that svc account appears in listing + c.assertSvcAccAppearsInListing(ctx, userAdmClient, value.AccessKeyID, cr.AccessKey) + + // 2. Check that svc account info can be queried + c.assertSvcAccInfoQueryable(ctx, userAdmClient, value.AccessKeyID, cr.AccessKey, true) + + // 3. Check S3 access + c.assertSvcAccS3Access(ctx, s, cr, bucket) + + // 5. Check that service account can be deleted. + c.assertSvcAccDeletion(ctx, s, userAdmClient, value.AccessKeyID, bucket) + + // 6. Check that service account cannot be created for some other user. + c.mustNotCreateSvcAccount(ctx, globalActiveCred.AccessKey, userAdmClient) +} + +var testAppParams = OpenIDClientAppParams{ + ClientID: "minio-client-app", + ClientSecret: "minio-client-app-secret", + ProviderURL: "http://127.0.0.1:5556/dex", + RedirectURL: "http://127.0.0.1:10000/oauth_callback", +} + +const ( + EnvTestOpenIDServer = "_MINIO_OPENID_TEST_SERVER" + EnvTestOpenIDServer2 = "_MINIO_OPENID_TEST_SERVER_2" +) + +// SetUpOpenIDs - sets up one or more OpenID test servers using the test OpenID +// container and canned data from https://github.com/minio/minio-ldap-testing +// +// Each set of client app params corresponds to a separate openid server, and +// the i-th server in this will be applied the i-th policy in `rolePolicies`. If +// a rolePolicies entry is an empty string, that server will be configured as +// policy-claim based openid server. NOTE that a valid configuration can have a +// policy claim based provider only if it is the only OpenID provider. +func (s *TestSuiteIAM) SetUpOpenIDs(c *check, testApps []OpenIDClientAppParams, rolePolicies []string) error { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + for i, testApp := range testApps { + configCmds := []string{ + fmt.Sprintf("identity_openid:%d", i), + fmt.Sprintf("config_url=%s/.well-known/openid-configuration", testApp.ProviderURL), + fmt.Sprintf("client_id=%s", testApp.ClientID), + fmt.Sprintf("client_secret=%s", testApp.ClientSecret), + "scopes=openid,groups", + fmt.Sprintf("redirect_uri=%s", testApp.RedirectURL), + } + if rolePolicies[i] != "" { + configCmds = append(configCmds, fmt.Sprintf("role_policy=%s", rolePolicies[i])) + } else { + configCmds = append(configCmds, "claim_name=groups") + } + _, err := s.adm.SetConfigKV(ctx, strings.Join(configCmds, " ")) + if err != nil { + return fmt.Errorf("unable to setup OpenID for tests: %v", err) + } + } + + s.RestartIAMSuite(c) + return nil +} + +// SetUpOpenID - expects to setup an OpenID test server using the test OpenID +// container and canned data from https://github.com/minio/minio-ldap-testing +func (s *TestSuiteIAM) SetUpOpenID(c *check, serverAddr string, rolePolicy string) { + ctx, cancel := context.WithTimeout(context.Background(), testDefaultTimeout) + defer cancel() + + configCmds := []string{ + "identity_openid", + fmt.Sprintf("config_url=%s/.well-known/openid-configuration", serverAddr), + "client_id=minio-client-app", + "client_secret=minio-client-app-secret", + "scopes=openid,groups", + "redirect_uri=http://127.0.0.1:10000/oauth_callback", + } + if rolePolicy != "" { + configCmds = append(configCmds, fmt.Sprintf("role_policy=%s", rolePolicy)) + } else { + configCmds = append(configCmds, "claim_name=groups") + } + _, err := s.adm.SetConfigKV(ctx, strings.Join(configCmds, " ")) + if err != nil { + c.Fatalf("unable to setup OpenID for tests: %v", err) + } + + s.RestartIAMSuite(c) +} + +func TestIAMWithOpenIDServerSuite(t *testing.T) { + for i, testCase := range iamTestSuites { + t.Run( + fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription), + func(t *testing.T) { + c := &check{t, testCase.serverType} + suite := testCase + + openIDServer := os.Getenv(EnvTestOpenIDServer) + if openIDServer == "" { + c.Skip("Skipping OpenID test as no OpenID server is provided.") + } + + suite.SetUpSuite(c) + suite.SetUpOpenID(c, openIDServer, "") + suite.TestOpenIDSTS(c) + suite.TestOpenIDSTSDurationSeconds(c) + suite.TestOpenIDServiceAcc(c) + suite.TestOpenIDSTSAddUser(c) + suite.TearDownSuite(c) + }, + ) + } +} + +func TestIAMWithOpenIDWithRolePolicyServerSuite(t *testing.T) { + for i, testCase := range iamTestSuites { + t.Run( + fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription), + func(t *testing.T) { + c := &check{t, testCase.serverType} + suite := testCase + + openIDServer := os.Getenv(EnvTestOpenIDServer) + if openIDServer == "" { + c.Skip("Skipping OpenID test as no OpenID server is provided.") + } + + suite.SetUpSuite(c) + suite.SetUpOpenID(c, openIDServer, "readwrite") + suite.TestOpenIDSTSWithRolePolicy(c, testRoleARNs[0], testRoleMap[testRoleARNs[0]]) + suite.TestOpenIDServiceAccWithRolePolicy(c) + suite.TearDownSuite(c) + }, + ) + } +} + +func TestIAMWithOpenIDWithRolePolicyWithPolicyVariablesServerSuite(t *testing.T) { + for i, testCase := range iamTestSuites { + t.Run( + fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription), + func(t *testing.T) { + c := &check{t, testCase.serverType} + suite := testCase + + openIDServer := os.Getenv(EnvTestOpenIDServer) + if openIDServer == "" { + c.Skip("Skipping OpenID test as no OpenID server is provided.") + } + + suite.SetUpSuite(c) + suite.SetUpOpenID(c, openIDServer, "projecta,projectb,projectaorb") + suite.TestOpenIDSTSWithRolePolicyWithPolVar(c, testRoleARNs[0], testRoleMap[testRoleARNs[0]]) + suite.TearDownSuite(c) + }, + ) + } +} + +const ( + testRoleARN = "arn:minio:iam:::role/nOybJqMNzNmroqEKq5D0EUsRZw0" + testRoleARN2 = "arn:minio:iam:::role/domXb70kze7Ugc1SaxaeFchhLP4" +) + +var ( + testRoleARNs = []string{testRoleARN, testRoleARN2} + + // Load test client app and test role mapping depending on test + // environment. + testClientApps, testRoleMap = func() ([]OpenIDClientAppParams, map[string]OpenIDClientAppParams) { + var apps []OpenIDClientAppParams + m := map[string]OpenIDClientAppParams{} + + openIDServer := os.Getenv(EnvTestOpenIDServer) + if openIDServer != "" { + apps = append(apps, OpenIDClientAppParams{ + ClientID: "minio-client-app", + ClientSecret: "minio-client-app-secret", + ProviderURL: openIDServer, + RedirectURL: "http://127.0.0.1:10000/oauth_callback", + }) + m[testRoleARNs[len(apps)-1]] = apps[len(apps)-1] + } + + openIDServer2 := os.Getenv(EnvTestOpenIDServer2) + if openIDServer2 != "" { + apps = append(apps, OpenIDClientAppParams{ + ClientID: "minio-client-app-2", + ClientSecret: "minio-client-app-secret-2", + ProviderURL: openIDServer2, + RedirectURL: "http://127.0.0.1:10000/oauth_callback", + }) + m[testRoleARNs[len(apps)-1]] = apps[len(apps)-1] + } + + return apps, m + }() +) + +func (s *TestSuiteIAM) TestOpenIDSTSWithRolePolicy(c *check, roleARN string, clientApp OpenIDClientAppParams) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket create error: %v", err) + } + + // Generate web identity JWT by interacting with OpenID IDP. + token, err := MockOpenIDTestUserInteraction(ctx, clientApp, "dillon@example.io", "dillon") + if err != nil { + c.Fatalf("mock user err: %v", err) + } + + // Generate STS credential. + webID := cr.STSWebIdentity{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + GetWebIDTokenExpiry: func() (*cr.WebIdentityToken, error) { + return &cr.WebIdentityToken{ + Token: token, + }, nil + }, + RoleARN: roleARN, + } + + value, err := webID.Retrieve() + if err != nil { + c.Fatalf("Expected to generate STS creds, got err: %#v", err) + } + // fmt.Printf("value: %#v\n", value) + + minioClient, err := minio.New(s.endpoint, &minio.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("Error initializing client: %v", err) + } + + // Validate that the client from sts creds can access the bucket. + c.mustListObjects(ctx, minioClient, bucket) +} + +func (s *TestSuiteIAM) TestOpenIDServiceAccWithRolePolicy(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket create error: %v", err) + } + + // Generate web identity STS token by interacting with OpenID IDP. + token, err := MockOpenIDTestUserInteraction(ctx, testAppParams, "dillon@example.io", "dillon") + if err != nil { + c.Fatalf("mock user err: %v", err) + } + + webID := cr.STSWebIdentity{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + GetWebIDTokenExpiry: func() (*cr.WebIdentityToken, error) { + return &cr.WebIdentityToken{ + Token: token, + }, nil + }, + RoleARN: testRoleARN, + } + + value, err := webID.Retrieve() + if err != nil { + c.Fatalf("Expected to generate STS creds, got err: %#v", err) + } + + // Create an madmin client with user creds + userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + }) + if err != nil { + c.Fatalf("Err creating user admin client: %v", err) + } + userAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport) + + // Create svc acc + cr := c.mustCreateSvcAccount(ctx, value.AccessKeyID, userAdmClient) + + // 1. Check that svc account appears in listing + c.assertSvcAccAppearsInListing(ctx, userAdmClient, value.AccessKeyID, cr.AccessKey) + + // 2. Check that svc account info can be queried + c.assertSvcAccInfoQueryable(ctx, userAdmClient, value.AccessKeyID, cr.AccessKey, true) + + // 3. Check S3 access + c.assertSvcAccS3Access(ctx, s, cr, bucket) + + // 5. Check that service account can be deleted. + c.assertSvcAccDeletion(ctx, s, userAdmClient, value.AccessKeyID, bucket) +} + +// Constants for Policy Variables test. +var ( + policyProjectA = `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetBucketLocation", + "s3:ListAllMyBuckets" + ], + "Resource": "arn:aws:s3:::*" + }, + { + "Effect": "Allow", + "Action": "s3:*", + "Resource": [ + "arn:aws:s3:::projecta", + "arn:aws:s3:::projecta/*" + ], + "Condition": { + "ForAnyValue:StringEquals": { + "jwt:groups": [ + "projecta" + ] + } + } + } + ] +} +` + policyProjectB = `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetBucketLocation", + "s3:ListAllMyBuckets" + ], + "Resource": "arn:aws:s3:::*" + }, + { + "Effect": "Allow", + "Action": "s3:*", + "Resource": [ + "arn:aws:s3:::projectb", + "arn:aws:s3:::projectb/*" + ], + "Condition": { + "ForAnyValue:StringEquals": { + "jwt:groups": [ + "projectb" + ] + } + } + } + ] +} +` + policyProjectAorB = `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetBucketLocation", + "s3:ListAllMyBuckets" + ], + "Resource": "arn:aws:s3:::*" + }, + { + "Effect": "Allow", + "Action": "s3:*", + "Resource": [ + "arn:aws:s3:::projectaorb", + "arn:aws:s3:::projectaorb/*" + ], + "Condition": { + "ForAnyValue:StringEquals": { + "jwt:groups": [ + "projecta", + "projectb" + ] + } + } + } + ] +}` + + policyProjectsMap = map[string]string{ + // grants access to bucket `projecta` if user is in group `projecta` + "projecta": policyProjectA, + + // grants access to bucket `projectb` if user is in group `projectb` + "projectb": policyProjectB, + + // grants access to bucket `projectaorb` if user is in either group + // `projecta` or `projectb` + "projectaorb": policyProjectAorB, + } +) + +func (s *TestSuiteIAM) TestOpenIDSTSWithRolePolicyWithPolVar(c *check, roleARN string, clientApp OpenIDClientAppParams) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + // Create project buckets + buckets := []string{"projecta", "projectb", "projectaorb", "other"} + for _, bucket := range buckets { + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket create error: %v", err) + } + } + + // Create policies + for polName, polContent := range policyProjectsMap { + err := s.adm.AddCannedPolicy(ctx, polName, []byte(polContent)) + if err != nil { + c.Fatalf("policy add error: %v", err) + } + } + + makeSTSClient := func(user, password string) *minio.Client { + // Generate web identity JWT by interacting with OpenID IDP. + token, err := MockOpenIDTestUserInteraction(ctx, clientApp, user, password) + if err != nil { + c.Fatalf("mock user err: %v", err) + } + + // Generate STS credential. + webID := cr.STSWebIdentity{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + GetWebIDTokenExpiry: func() (*cr.WebIdentityToken, error) { + return &cr.WebIdentityToken{ + Token: token, + }, nil + }, + RoleARN: roleARN, + } + + value, err := webID.Retrieve() + if err != nil { + c.Fatalf("Expected to generate STS creds, got err: %#v", err) + } + // fmt.Printf("value: %#v\n", value) + + minioClient, err := minio.New(s.endpoint, &minio.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("Error initializing client: %v", err) + } + + return minioClient + } + + // user dillon's groups attribute is ["projecta", "projectb"] + dillonClient := makeSTSClient("dillon@example.io", "dillon") + // Validate client's permissions + c.mustListBuckets(ctx, dillonClient) + c.mustListObjects(ctx, dillonClient, "projecta") + c.mustListObjects(ctx, dillonClient, "projectb") + c.mustListObjects(ctx, dillonClient, "projectaorb") + c.mustNotListObjects(ctx, dillonClient, "other") + + // this user's groups attribute is ["projectb"] + lisaClient := makeSTSClient("ejones@example.io", "liza") + // Validate client's permissions + c.mustListBuckets(ctx, lisaClient) + c.mustNotListObjects(ctx, lisaClient, "projecta") + c.mustListObjects(ctx, lisaClient, "projectb") + c.mustListObjects(ctx, lisaClient, "projectaorb") + c.mustNotListObjects(ctx, lisaClient, "other") +} + +func TestIAMWithOpenIDMultipleConfigsValidation1(t *testing.T) { + openIDServer := os.Getenv(EnvTestOpenIDServer) + openIDServer2 := os.Getenv(EnvTestOpenIDServer2) + if openIDServer == "" || openIDServer2 == "" { + t.Skip("Skipping OpenID test as enough OpenID servers are not provided.") + } + testApps := testClientApps + + rolePolicies := []string{ + "", // Treated as claim-based provider as no role policy is given. + "readwrite", + } + + for i, testCase := range iamTestSuites { + t.Run( + fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription), + func(t *testing.T) { + c := &check{t, testCase.serverType} + suite := testCase + + suite.SetUpSuite(c) + defer suite.TearDownSuite(c) + + err := suite.SetUpOpenIDs(c, testApps, rolePolicies) + if err != nil { + c.Fatalf("config with 1 claim based and 1 role based provider should pass but got: %v", err) + } + }, + ) + } +} + +func TestIAMWithOpenIDMultipleConfigsValidation2(t *testing.T) { + openIDServer := os.Getenv(EnvTestOpenIDServer) + openIDServer2 := os.Getenv(EnvTestOpenIDServer2) + if openIDServer == "" || openIDServer2 == "" { + t.Skip("Skipping OpenID test as enough OpenID servers are not provided.") + } + testApps := testClientApps + + rolePolicies := []string{ + "", // Treated as claim-based provider as no role policy is given. + "", // Treated as claim-based provider as no role policy is given. + } + + for i, testCase := range iamTestSuites { + t.Run( + fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription), + func(t *testing.T) { + c := &check{t, testCase.serverType} + suite := testCase + + suite.SetUpSuite(c) + defer suite.TearDownSuite(c) + + err := suite.SetUpOpenIDs(c, testApps, rolePolicies) + if err == nil { + c.Fatalf("config with 2 claim based provider should fail") + } + }, + ) + } +} + +func TestIAMWithOpenIDWithMultipleRolesServerSuite(t *testing.T) { + openIDServer := os.Getenv(EnvTestOpenIDServer) + openIDServer2 := os.Getenv(EnvTestOpenIDServer2) + if openIDServer == "" || openIDServer2 == "" { + t.Skip("Skipping OpenID test as enough OpenID servers are not provided.") + } + testApps := testClientApps + + rolePolicies := []string{ + "consoleAdmin", + "readwrite", + } + + for i, testCase := range iamTestSuites { + t.Run( + fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription), + func(t *testing.T) { + c := &check{t, testCase.serverType} + suite := testCase + + suite.SetUpSuite(c) + err := suite.SetUpOpenIDs(c, testApps, rolePolicies) + if err != nil { + c.Fatalf("Error setting up openid providers for tests: %v", err) + } + suite.TestOpenIDSTSWithRolePolicy(c, testRoleARNs[0], testRoleMap[testRoleARNs[0]]) + suite.TestOpenIDSTSWithRolePolicy(c, testRoleARNs[1], testRoleMap[testRoleARNs[1]]) + suite.TestOpenIDServiceAccWithRolePolicy(c) + suite.TearDownSuite(c) + }, + ) + } +} + +// Access Management Plugin tests +func TestIAM_AMPWithOpenIDWithMultipleRolesServerSuite(t *testing.T) { + openIDServer := os.Getenv(EnvTestOpenIDServer) + openIDServer2 := os.Getenv(EnvTestOpenIDServer2) + if openIDServer == "" || openIDServer2 == "" { + t.Skip("Skipping OpenID test as enough OpenID servers are not provided.") + } + testApps := testClientApps + + rolePolicies := []string{ + "consoleAdmin", + "readwrite", + } + + for i, testCase := range iamTestSuites { + t.Run( + fmt.Sprintf("Test: %d, ServerType: %s", i+1, testCase.ServerTypeDescription), + func(t *testing.T) { + c := &check{t, testCase.serverType} + suite := testCase + + suite.SetUpSuite(c) + defer suite.TearDownSuite(c) + + err := suite.SetUpOpenIDs(c, testApps, rolePolicies) + if err != nil { + c.Fatalf("Error setting up openid providers for tests: %v", err) + } + + suite.SetUpAccMgmtPlugin(c) + + suite.TestOpenIDSTSWithRolePolicyUnderAMP(c, testRoleARNs[0], testRoleMap[testRoleARNs[0]]) + suite.TestOpenIDSTSWithRolePolicyUnderAMP(c, testRoleARNs[1], testRoleMap[testRoleARNs[1]]) + suite.TestOpenIDServiceAccWithRolePolicyUnderAMP(c) + }, + ) + } +} + +func (s *TestSuiteIAM) TestOpenIDSTSWithRolePolicyUnderAMP(c *check, roleARN string, clientApp OpenIDClientAppParams) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket create error: %v", err) + } + + // Generate web identity JWT by interacting with OpenID IDP. + token, err := MockOpenIDTestUserInteraction(ctx, clientApp, "dillon@example.io", "dillon") + if err != nil { + c.Fatalf("mock user err: %v", err) + } + + // Generate STS credential. + webID := cr.STSWebIdentity{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + GetWebIDTokenExpiry: func() (*cr.WebIdentityToken, error) { + return &cr.WebIdentityToken{ + Token: token, + }, nil + }, + RoleARN: roleARN, + } + + value, err := webID.Retrieve() + if err != nil { + c.Fatalf("Expected to generate STS creds, got err: %#v", err) + } + // fmt.Printf("value: %#v\n", value) + + minioClient, err := minio.New(s.endpoint, &minio.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + Transport: s.TestSuiteCommon.client.Transport, + }) + if err != nil { + c.Fatalf("Error initializing client: %v", err) + } + + // Validate that the client from sts creds can access the bucket. + c.mustListObjects(ctx, minioClient, bucket) + + // Validate that the client from STS creds cannot upload any object as + // it is denied by the plugin. + c.mustNotUpload(ctx, minioClient, bucket) +} + +func (s *TestSuiteIAM) TestOpenIDServiceAccWithRolePolicyUnderAMP(c *check) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + + bucket := getRandomBucketName() + err := s.client.MakeBucket(ctx, bucket, minio.MakeBucketOptions{}) + if err != nil { + c.Fatalf("bucket create error: %v", err) + } + + // Generate web identity STS token by interacting with OpenID IDP. + token, err := MockOpenIDTestUserInteraction(ctx, testAppParams, "dillon@example.io", "dillon") + if err != nil { + c.Fatalf("mock user err: %v", err) + } + + webID := cr.STSWebIdentity{ + Client: s.TestSuiteCommon.client, + STSEndpoint: s.endPoint, + GetWebIDTokenExpiry: func() (*cr.WebIdentityToken, error) { + return &cr.WebIdentityToken{ + Token: token, + }, nil + }, + RoleARN: testRoleARN, + } + + value, err := webID.Retrieve() + if err != nil { + c.Fatalf("Expected to generate STS creds, got err: %#v", err) + } + + // Create an madmin client with user creds + userAdmClient, err := madmin.NewWithOptions(s.endpoint, &madmin.Options{ + Creds: cr.NewStaticV4(value.AccessKeyID, value.SecretAccessKey, value.SessionToken), + Secure: s.secure, + }) + if err != nil { + c.Fatalf("Err creating user admin client: %v", err) + } + userAdmClient.SetCustomTransport(s.TestSuiteCommon.client.Transport) + + // Create svc acc + cr := c.mustCreateSvcAccount(ctx, value.AccessKeyID, userAdmClient) + + // 1. Check that svc account appears in listing + c.assertSvcAccAppearsInListing(ctx, userAdmClient, value.AccessKeyID, cr.AccessKey) + + // 2. Check that svc account info can be queried + c.assertSvcAccInfoQueryable(ctx, userAdmClient, value.AccessKeyID, cr.AccessKey, true) + + // 3. Check S3 access + c.assertSvcAccS3Access(ctx, s, cr, bucket) + // 3.1 Validate that the client from STS creds cannot upload any object as + // it is denied by the plugin. + c.mustNotUpload(ctx, s.getUserClient(c, cr.AccessKey, cr.SecretKey, ""), bucket) + + // Check that session policies do not apply - as policy enforcement is + // delegated to plugin. + { + svcAK, svcSK := mustGenerateCredentials(c) + + // This policy does not allow listing objects. + policyBytes := fmt.Appendf(nil, `{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:PutObject", + "s3:GetObject" + ], + "Resource": [ + "arn:aws:s3:::%s/*" + ] + } + ] +}`, bucket) + cr, err := userAdmClient.AddServiceAccount(ctx, madmin.AddServiceAccountReq{ + Policy: policyBytes, + TargetUser: value.AccessKeyID, + AccessKey: svcAK, + SecretKey: svcSK, + }) + if err != nil { + c.Fatalf("Unable to create svc acc: %v", err) + } + svcClient := s.getUserClient(c, cr.AccessKey, cr.SecretKey, "") + // Though the attached policy does not allow listing, it will be + // ignored because the plugin allows it. + c.mustListObjects(ctx, svcClient, bucket) + } + + // 4. Check that service account's secret key and account status can be + // updated. + c.assertSvcAccSecretKeyAndStatusUpdate(ctx, s, userAdmClient, value.AccessKeyID, bucket) + + // 5. Check that service account can be deleted. + c.assertSvcAccDeletion(ctx, s, userAdmClient, value.AccessKeyID, bucket) +} diff --git a/cmd/stserrorcode_string.go b/cmd/stserrorcode_string.go new file mode 100644 index 00000000..b1192835 --- /dev/null +++ b/cmd/stserrorcode_string.go @@ -0,0 +1,37 @@ +// Code generated by "stringer -type=STSErrorCode -trimprefix=Err sts-errors.go"; DO NOT EDIT. + +package cmd + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[ErrSTSNone-0] + _ = x[ErrSTSAccessDenied-1] + _ = x[ErrSTSMissingParameter-2] + _ = x[ErrSTSInvalidParameterValue-3] + _ = x[ErrSTSWebIdentityExpiredToken-4] + _ = x[ErrSTSClientGrantsExpiredToken-5] + _ = x[ErrSTSInvalidClientGrantsToken-6] + _ = x[ErrSTSMalformedPolicyDocument-7] + _ = x[ErrSTSInsecureConnection-8] + _ = x[ErrSTSInvalidClientCertificate-9] + _ = x[ErrSTSTooManyIntermediateCAs-10] + _ = x[ErrSTSNotInitialized-11] + _ = x[ErrSTSIAMNotInitialized-12] + _ = x[ErrSTSUpstreamError-13] + _ = x[ErrSTSInternalError-14] +} + +const _STSErrorCode_name = "STSNoneSTSAccessDeniedSTSMissingParameterSTSInvalidParameterValueSTSWebIdentityExpiredTokenSTSClientGrantsExpiredTokenSTSInvalidClientGrantsTokenSTSMalformedPolicyDocumentSTSInsecureConnectionSTSInvalidClientCertificateSTSTooManyIntermediateCAsSTSNotInitializedSTSIAMNotInitializedSTSUpstreamErrorSTSInternalError" + +var _STSErrorCode_index = [...]uint16{0, 7, 22, 41, 65, 91, 118, 145, 171, 192, 219, 244, 261, 281, 297, 313} + +func (i STSErrorCode) String() string { + if i < 0 || i >= STSErrorCode(len(_STSErrorCode_index)-1) { + return "STSErrorCode(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _STSErrorCode_name[_STSErrorCode_index[i]:_STSErrorCode_index[i+1]] +} diff --git a/cmd/test-utils_test.go b/cmd/test-utils_test.go new file mode 100644 index 00000000..0f903625 --- /dev/null +++ b/cmd/test-utils_test.go @@ -0,0 +1,2417 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "archive/zip" + "bufio" + "bytes" + "context" + "crypto/ecdsa" + "crypto/hmac" + crand "crypto/rand" + "crypto/rsa" + "crypto/sha1" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/hex" + "encoding/pem" + "encoding/xml" + "errors" + "flag" + "fmt" + "io" + "math/big" + "math/rand" + "net" + "net/http" + "net/http/httptest" + "net/url" + "os" + "path" + "path/filepath" + "reflect" + "sort" + "strconv" + "strings" + "sync" + "testing" + "time" + "unsafe" + + "github.com/fatih/color" + + "github.com/minio/minio-go/v7/pkg/s3utils" + "github.com/minio/minio-go/v7/pkg/signer" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/hash" + "github.com/minio/minio/internal/logger" + "github.com/minio/mux" + "github.com/minio/pkg/v3/policy" +) + +// TestMain to set up global env. +func TestMain(m *testing.M) { + flag.Parse() + + // set to 'true' when testing is invoked + globalIsTesting = true + + globalIsCICD = globalIsTesting + + globalActiveCred = auth.Credentials{ + AccessKey: auth.DefaultAccessKey, + SecretKey: auth.DefaultSecretKey, + } + + globalNodeAuthToken, _ = authenticateNode(auth.DefaultAccessKey, auth.DefaultSecretKey) + + // disable ENVs which interfere with tests. + for _, env := range []string{ + crypto.EnvKMSAutoEncryption, + config.EnvAccessKey, + config.EnvSecretKey, + config.EnvRootUser, + config.EnvRootPassword, + } { + os.Unsetenv(env) + } + + // Set as non-distributed. + globalIsDistErasure = false + + // Disable printing console messages during tests. + color.Output = io.Discard + // Disable Error logging in testing. + logger.DisableLog = true + + // Uncomment the following line to see trace logs during unit tests. + // logger.AddTarget(console.New()) + + // Set system resources to maximum. + setMaxResources(serverCtxt{}) + + // Initialize globalConsoleSys system + globalConsoleSys = NewConsoleLogger(context.Background(), io.Discard) + + globalInternodeTransport = NewInternodeHTTPTransport(0)() + + initHelp() + + resetTestGlobals() + + globalIsCICD = true + + os.Exit(m.Run()) +} + +// concurrency level for certain parallel tests. +const testConcurrencyLevel = 10 + +const iso8601TimeFormat = "2006-01-02T15:04:05.000Z" + +// Excerpts from @lsegal - https://github.com/aws/aws-sdk-js/issues/659#issuecomment-120477258 +// +// User-Agent: +// +// This is ignored from signing because signing this causes problems with generating pre-signed URLs +// (that are executed by other agents) or when customers pass requests through proxies, which may +// modify the user-agent. +// +// Authorization: +// +// Is skipped for obvious reasons +var ignoredHeaders = map[string]bool{ + "Authorization": true, + "User-Agent": true, +} + +// Headers to ignore in streaming v4 +var ignoredStreamingHeaders = map[string]bool{ + "Authorization": true, + "Content-Type": true, + "Content-Md5": true, + "User-Agent": true, +} + +// calculateSignedChunkLength - calculates the length of chunk metadata +func calculateSignedChunkLength(chunkDataSize int64) int64 { + return int64(len(fmt.Sprintf("%x", chunkDataSize))) + + 17 + // ";chunk-signature=" + 64 + // e.g. "f2ca1bb6c7e907d06dafe4687e579fce76b37e4e93b7605022da52e6ccc26fd2" + 2 + // CRLF + chunkDataSize + + 2 // CRLF +} + +func mustGetPutObjReader(t TestErrHandler, data io.Reader, size int64, md5hex, sha256hex string) *PutObjReader { + hr, err := hash.NewReader(context.Background(), data, size, md5hex, sha256hex, size) + if err != nil { + t.Fatal(err) + } + return NewPutObjReader(hr) +} + +// calculateSignedChunkLength - calculates the length of the overall stream (data + metadata) +func calculateStreamContentLength(dataLen, chunkSize int64) int64 { + if dataLen <= 0 { + return 0 + } + chunksCount := dataLen / chunkSize + remainingBytes := dataLen % chunkSize + var streamLen int64 + streamLen += chunksCount * calculateSignedChunkLength(chunkSize) + if remainingBytes > 0 { + streamLen += calculateSignedChunkLength(remainingBytes) + } + streamLen += calculateSignedChunkLength(0) + return streamLen +} + +func prepareFS(ctx context.Context) (ObjectLayer, string, error) { + nDisks := 1 + fsDirs, err := getRandomDisks(nDisks) + if err != nil { + return nil, "", err + } + obj, _, err := initObjectLayer(context.Background(), mustGetPoolEndpoints(0, fsDirs...)) + if err != nil { + return nil, "", err + } + + initAllSubsystems(ctx) + + globalIAMSys.Init(ctx, obj, globalEtcdClient, 2*time.Second) + return obj, fsDirs[0], nil +} + +func prepareErasureSets32(ctx context.Context) (ObjectLayer, []string, error) { + return prepareErasure(ctx, 32) +} + +func prepareErasure(ctx context.Context, nDisks int) (ObjectLayer, []string, error) { + fsDirs, err := getRandomDisks(nDisks) + if err != nil { + return nil, nil, err + } + obj, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, fsDirs...)) + if err != nil { + removeRoots(fsDirs) + return nil, nil, err + } + + // Wait up to 10 seconds for disks to come online. + pools := obj.(*erasureServerPools) + t := time.Now() + for _, pool := range pools.serverPools { + for _, sets := range pool.erasureDisks { + for _, s := range sets { + if !s.IsLocal() { + for !s.IsOnline() { + time.Sleep(100 * time.Millisecond) + if time.Since(t) > 10*time.Second { + return nil, nil, errors.New("timeout waiting for disk to come online") + } + } + } + } + } + } + + return obj, fsDirs, nil +} + +func prepareErasure16(ctx context.Context) (ObjectLayer, []string, error) { + return prepareErasure(ctx, 16) +} + +// TestErrHandler - Go testing.T satisfy this interface. +// This makes it easy to run the TestServer from any of the tests. +// Using this interface, functionalities to be used in tests can be +// made generalized, and can be integrated in benchmarks/unit tests/go check suite tests. +type TestErrHandler interface { + testing.TB +} + +const ( + // ErasureSDStr is the string which is used as notation for Single node ObjectLayer in the unit tests. + ErasureSDStr string = "ErasureSD" + + // ErasureTestStr is the string which is used as notation for Erasure ObjectLayer in the unit tests. + ErasureTestStr string = "Erasure" + + // ErasureSetsTestStr is the string which is used as notation for Erasure sets object layer in the unit tests. + ErasureSetsTestStr string = "ErasureSet" +) + +const letterBytes = "abcdefghijklmnopqrstuvwxyz01234569" +const ( + letterIdxBits = 6 // 6 bits to represent a letter index + letterIdxMask = 1< %s", err) + } + + // Run TestServer. + testServer.Server = httptest.NewUnstartedServer(setCriticalErrorHandler(corsHandler(httpHandler))) + + globalObjLayerMutex.Lock() + globalObjectAPI = objLayer + globalObjLayerMutex.Unlock() + + // initialize peer rpc + host, port := mustSplitHostPort(testServer.Server.Listener.Addr().String()) + globalMinioHost = host + globalMinioPort = port + globalMinioAddr = getEndpointsLocalAddr(testServer.Disks) + + initAllSubsystems(ctx) + + globalEtcdClient = nil + + initConfigSubsystem(ctx, objLayer) + + globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second) + + globalEventNotifier.InitBucketTargets(ctx, objLayer) + + return testServer +} + +// testServerCertPEM and testServerKeyPEM are generated by +// https://golang.org/src/crypto/tls/generate_cert.go +// $ go run generate_cert.go -ca --host 127.0.0.1 +// The generated certificate contains IP SAN, that way we don't need +// to enable InsecureSkipVerify in TLS config + +// Starts the test server and returns the TestServer with TLS configured instance. +func StartTestTLSServer(t TestErrHandler, instanceType string, cert, key []byte) TestServer { + // Fetch TLS key and pem files from test-data/ directory. + // dir, _ := os.Getwd() + // testDataDir := filepath.Join(filepath.Dir(dir), "test-data") + // + // pemFile := filepath.Join(testDataDir, "server.pem") + // keyFile := filepath.Join(testDataDir, "server.key") + cer, err := tls.X509KeyPair(cert, key) + if err != nil { + t.Fatalf("Failed to load certificate: %v", err) + } + config := &tls.Config{Certificates: []tls.Certificate{cer}} + + testServer := UnstartedTestServer(t, instanceType) + testServer.Server.TLS = config + testServer.Server.StartTLS() + return testServer +} + +// Starts the test server and returns the TestServer instance. +func StartTestServer(t TestErrHandler, instanceType string) TestServer { + // create an instance of TestServer. + testServer := UnstartedTestServer(t, instanceType) + testServer.Server.Start() + return testServer +} + +// Sets the global config path to empty string. +func resetGlobalConfigPath() { + globalConfigDir = &ConfigDir{path: ""} +} + +// sets globalObjectAPI to `nil`. +func resetGlobalObjectAPI() { + globalObjLayerMutex.Lock() + globalObjectAPI = nil + globalObjLayerMutex.Unlock() +} + +// reset the value of the Global server config. +// set it to `nil`. +func resetGlobalConfig() { + // hold the mutex lock before a new config is assigned. + globalServerConfigMu.Lock() + // Save the loaded config globally. + globalServerConfig = nil + globalServerConfigMu.Unlock() +} + +func resetGlobalEndpoints() { + globalEndpoints = EndpointServerPools{} +} + +func resetGlobalIsErasure() { + globalIsErasure = false +} + +// reset global heal state +func resetGlobalHealState() { + // Init global heal state + if globalAllHealState == nil { + globalAllHealState = newHealState(GlobalContext, false) + } else { + globalAllHealState.Lock() + for _, v := range globalAllHealState.healSeqMap { + if !v.hasEnded() { + v.stop() + } + } + globalAllHealState.Unlock() + } + + // Init background heal state + if globalBackgroundHealState == nil { + globalBackgroundHealState = newHealState(GlobalContext, false) + } else { + globalBackgroundHealState.Lock() + for _, v := range globalBackgroundHealState.healSeqMap { + if !v.hasEnded() { + v.stop() + } + } + globalBackgroundHealState.Unlock() + } +} + +// sets globalIAMSys to `nil`. +func resetGlobalIAMSys() { + globalIAMSys = nil +} + +// Resets all the globals used modified in tests. +// Resetting ensures that the changes made to globals by one test doesn't affect others. +func resetTestGlobals() { + // set globalObjectAPI to `nil`. + resetGlobalObjectAPI() + // Reset config path set. + resetGlobalConfigPath() + // Reset Global server config. + resetGlobalConfig() + // Reset global endpoints. + resetGlobalEndpoints() + // Reset global isErasure flag. + resetGlobalIsErasure() + // Reset global heal state + resetGlobalHealState() + // Reset globalIAMSys to `nil` + resetGlobalIAMSys() +} + +// Configure the server for the test run. +func newTestConfig(bucketLocation string, obj ObjectLayer) (err error) { + // Initialize server config. + if err = newSrvConfig(obj); err != nil { + return err + } + + // Set a default region. + config.SetRegion(globalServerConfig, bucketLocation) + + applyDynamicConfigForSubSys(context.Background(), obj, globalServerConfig, config.StorageClassSubSys) + + // Save config. + return saveServerConfig(context.Background(), obj, globalServerConfig) +} + +// Deleting the temporary backend and stopping the server. +func (testServer TestServer) Stop() { + testServer.cancel() + testServer.Server.Close() + testServer.Obj.Shutdown(context.Background()) + os.RemoveAll(testServer.Root) + for _, ep := range testServer.Disks { + for _, disk := range ep.Endpoints { + os.RemoveAll(disk.Path) + } + } +} + +// Truncate request to simulate unexpected EOF for a request signed using streaming signature v4. +func truncateChunkByHalfSigv4(req *http.Request) (*http.Request, error) { + bufReader := bufio.NewReader(req.Body) + hexChunkSize, chunkSignature, err := readChunkLine(bufReader) + if err != nil { + return nil, err + } + + newChunkHdr := fmt.Appendf(nil, "%s"+s3ChunkSignatureStr+"%s\r\n", + hexChunkSize, chunkSignature) + newChunk, err := io.ReadAll(bufReader) + if err != nil { + return nil, err + } + newReq := req + newReq.Body = io.NopCloser( + bytes.NewReader(bytes.Join([][]byte{newChunkHdr, newChunk[:len(newChunk)/2]}, + []byte(""))), + ) + return newReq, nil +} + +// Malform data given a request signed using streaming signature V4. +func malformDataSigV4(req *http.Request, newByte byte) (*http.Request, error) { + bufReader := bufio.NewReader(req.Body) + hexChunkSize, chunkSignature, err := readChunkLine(bufReader) + if err != nil { + return nil, err + } + + newChunkHdr := fmt.Appendf(nil, "%s"+s3ChunkSignatureStr+"%s\r\n", + hexChunkSize, chunkSignature) + newChunk, err := io.ReadAll(bufReader) + if err != nil { + return nil, err + } + + newChunk[0] = newByte + newReq := req + newReq.Body = io.NopCloser( + bytes.NewReader(bytes.Join([][]byte{newChunkHdr, newChunk}, + []byte(""))), + ) + + return newReq, nil +} + +// Malform chunk size given a request signed using streaming signatureV4. +func malformChunkSizeSigV4(req *http.Request, badSize int64) (*http.Request, error) { + bufReader := bufio.NewReader(req.Body) + _, chunkSignature, err := readChunkLine(bufReader) + if err != nil { + return nil, err + } + + n := badSize + newHexChunkSize := fmt.Appendf(nil, "%x", n) + newChunkHdr := fmt.Appendf(nil, "%s"+s3ChunkSignatureStr+"%s\r\n", + newHexChunkSize, chunkSignature) + newChunk, err := io.ReadAll(bufReader) + if err != nil { + return nil, err + } + + newReq := req + newReq.Body = io.NopCloser( + bytes.NewReader(bytes.Join([][]byte{newChunkHdr, newChunk}, + []byte(""))), + ) + + return newReq, nil +} + +// Sign given request using Signature V4. +func signStreamingRequest(req *http.Request, accessKey, secretKey string, currTime time.Time) (string, error) { + // Get hashed payload. + hashedPayload := req.Header.Get("x-amz-content-sha256") + if hashedPayload == "" { + return "", fmt.Errorf("Invalid hashed payload") + } + + // Set x-amz-date. + req.Header.Set("x-amz-date", currTime.Format(iso8601Format)) + + // Get header map. + headerMap := make(map[string][]string) + for k, vv := range req.Header { + // If request header key is not in ignored headers, then add it. + if _, ok := ignoredStreamingHeaders[http.CanonicalHeaderKey(k)]; !ok { + headerMap[strings.ToLower(k)] = vv + } + } + + // Get header keys. + headers := []string{"host"} + for k := range headerMap { + headers = append(headers, k) + } + sort.Strings(headers) + + // Get canonical headers. + var buf bytes.Buffer + for _, k := range headers { + buf.WriteString(k) + buf.WriteByte(':') + switch k { + case "host": + buf.WriteString(req.URL.Host) + fallthrough + default: + for idx, v := range headerMap[k] { + if idx > 0 { + buf.WriteByte(',') + } + buf.WriteString(v) + } + buf.WriteByte('\n') + } + } + canonicalHeaders := buf.String() + + // Get signed headers. + signedHeaders := strings.Join(headers, ";") + + // Get canonical query string. + req.URL.RawQuery = strings.ReplaceAll(req.URL.Query().Encode(), "+", "%20") + + // Get canonical URI. + canonicalURI := s3utils.EncodePath(req.URL.Path) + + // Get canonical request. + // canonicalRequest = + // \n + // \n + // \n + // \n + // \n + // + // + canonicalRequest := strings.Join([]string{ + req.Method, + canonicalURI, + req.URL.RawQuery, + canonicalHeaders, + signedHeaders, + hashedPayload, + }, "\n") + + // Get scope. + scope := strings.Join([]string{ + currTime.Format(yyyymmdd), + globalMinioDefaultRegion, + string(serviceS3), + "aws4_request", + }, SlashSeparator) + + stringToSign := "AWS4-HMAC-SHA256" + "\n" + currTime.Format(iso8601Format) + "\n" + stringToSign += scope + "\n" + stringToSign += getSHA256Hash([]byte(canonicalRequest)) + + date := sumHMAC([]byte("AWS4"+secretKey), []byte(currTime.Format(yyyymmdd))) + region := sumHMAC(date, []byte(globalMinioDefaultRegion)) + service := sumHMAC(region, []byte(string(serviceS3))) + signingKey := sumHMAC(service, []byte("aws4_request")) + + signature := hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign))) + + // final Authorization header + parts := []string{ + "AWS4-HMAC-SHA256" + " Credential=" + accessKey + SlashSeparator + scope, + "SignedHeaders=" + signedHeaders, + "Signature=" + signature, + } + auth := strings.Join(parts, ", ") + req.Header.Set("Authorization", auth) + + return signature, nil +} + +// Returns new HTTP request object. +func newTestStreamingRequest(method, urlStr string, dataLength, chunkSize int64, body io.ReadSeeker) (*http.Request, error) { + if method == "" { + method = http.MethodPost + } + + req, err := http.NewRequest(method, urlStr, nil) + if err != nil { + return nil, err + } + + if body == nil { + // this is added to avoid panic during io.ReadAll(req.Body). + // th stack trace can be found here https://github.com/minio/minio/pull/2074 . + // This is very similar to https://github.com/golang/go/issues/7527. + req.Body = io.NopCloser(bytes.NewReader([]byte(""))) + } + + contentLength := calculateStreamContentLength(dataLength, chunkSize) + + req.Header.Set("x-amz-content-sha256", "STREAMING-AWS4-HMAC-SHA256-PAYLOAD") + req.Header.Set("content-encoding", "aws-chunked") + req.Header.Set("x-amz-decoded-content-length", strconv.FormatInt(dataLength, 10)) + req.Header.Set("content-length", strconv.FormatInt(contentLength, 10)) + + // Seek back to beginning. + body.Seek(0, 0) + + // Add body + req.Body = io.NopCloser(body) + req.ContentLength = contentLength + + return req, nil +} + +func assembleStreamingChunks(req *http.Request, body io.ReadSeeker, chunkSize int64, + secretKey, signature string, currTime time.Time) (*http.Request, error, +) { + regionStr := globalSite.Region() + var stream []byte + var buffer []byte + body.Seek(0, 0) + for { + buffer = make([]byte, chunkSize) + n, err := body.Read(buffer) + if err != nil && err != io.EOF { + return nil, err + } + + // Get scope. + scope := strings.Join([]string{ + currTime.Format(yyyymmdd), + regionStr, + string(serviceS3), + "aws4_request", + }, SlashSeparator) + + stringToSign := "AWS4-HMAC-SHA256-PAYLOAD" + "\n" + stringToSign = stringToSign + currTime.Format(iso8601Format) + "\n" + stringToSign = stringToSign + scope + "\n" + stringToSign = stringToSign + signature + "\n" + stringToSign = stringToSign + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + "\n" // hex(sum256("")) + stringToSign += getSHA256Hash(buffer[:n]) + + date := sumHMAC([]byte("AWS4"+secretKey), []byte(currTime.Format(yyyymmdd))) + region := sumHMAC(date, []byte(regionStr)) + service := sumHMAC(region, []byte(serviceS3)) + signingKey := sumHMAC(service, []byte("aws4_request")) + + signature = hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign))) + + stream = append(stream, []byte(fmt.Sprintf("%x", n)+";chunk-signature="+signature+"\r\n")...) + stream = append(stream, buffer[:n]...) + stream = append(stream, []byte("\r\n")...) + + if n <= 0 { + break + } + } + req.Body = io.NopCloser(bytes.NewReader(stream)) + return req, nil +} + +func newTestStreamingSignedBadChunkDateRequest(method, urlStr string, contentLength, chunkSize int64, body io.ReadSeeker, accessKey, secretKey string) (*http.Request, error) { + req, err := newTestStreamingRequest(method, urlStr, contentLength, chunkSize, body) + if err != nil { + return nil, err + } + + currTime := UTCNow() + signature, err := signStreamingRequest(req, accessKey, secretKey, currTime) + if err != nil { + return nil, err + } + + // skew the time between the chunk signature calculation and seed signature. + currTime = currTime.Add(1 * time.Second) + req, err = assembleStreamingChunks(req, body, chunkSize, secretKey, signature, currTime) + return req, err +} + +func newTestStreamingSignedCustomEncodingRequest(method, urlStr string, contentLength, chunkSize int64, body io.ReadSeeker, accessKey, secretKey, contentEncoding string) (*http.Request, error) { + req, err := newTestStreamingRequest(method, urlStr, contentLength, chunkSize, body) + if err != nil { + return nil, err + } + + // Set custom encoding. + req.Header.Set("content-encoding", contentEncoding) + + currTime := UTCNow() + signature, err := signStreamingRequest(req, accessKey, secretKey, currTime) + if err != nil { + return nil, err + } + + req, err = assembleStreamingChunks(req, body, chunkSize, secretKey, signature, currTime) + return req, err +} + +// Returns new HTTP request object signed with streaming signature v4. +func newTestStreamingSignedRequest(method, urlStr string, contentLength, chunkSize int64, body io.ReadSeeker, accessKey, secretKey string) (*http.Request, error) { + req, err := newTestStreamingRequest(method, urlStr, contentLength, chunkSize, body) + if err != nil { + return nil, err + } + + currTime := UTCNow() + signature, err := signStreamingRequest(req, accessKey, secretKey, currTime) + if err != nil { + return nil, err + } + + req, err = assembleStreamingChunks(req, body, chunkSize, secretKey, signature, currTime) + return req, err +} + +// preSignV4 presign the request, in accordance with +// http://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-query-string-auth.html. +func preSignV4(req *http.Request, accessKeyID, secretAccessKey string, expires int64) error { + // Presign is not needed for anonymous credentials. + if accessKeyID == "" || secretAccessKey == "" { + return errors.New("Presign cannot be generated without access and secret keys") + } + + region := globalSite.Region() + date := UTCNow() + scope := getScope(date, region) + credential := fmt.Sprintf("%s/%s", accessKeyID, scope) + + // Set URL query. + query := req.URL.Query() + query.Set("X-Amz-Algorithm", signV4Algorithm) + query.Set("X-Amz-Date", date.Format(iso8601Format)) + query.Set("X-Amz-Expires", strconv.FormatInt(expires, 10)) + query.Set("X-Amz-SignedHeaders", "host") + query.Set("X-Amz-Credential", credential) + query.Set("X-Amz-Content-Sha256", unsignedPayload) + + // "host" is the only header required to be signed for Presigned URLs. + extractedSignedHeaders := make(http.Header) + extractedSignedHeaders.Set("host", req.Host) + + queryStr := strings.ReplaceAll(query.Encode(), "+", "%20") + canonicalRequest := getCanonicalRequest(extractedSignedHeaders, unsignedPayload, queryStr, req.URL.Path, req.Method) + stringToSign := getStringToSign(canonicalRequest, date, scope) + signingKey := getSigningKey(secretAccessKey, date, region, serviceS3) + signature := getSignature(signingKey, stringToSign) + + req.URL.RawQuery = query.Encode() + + // Add signature header to RawQuery. + req.URL.RawQuery += "&X-Amz-Signature=" + url.QueryEscape(signature) + + // Construct the final presigned URL. + return nil +} + +// preSignV2 - presign the request in following style. +// https://${S3_BUCKET}.s3.amazonaws.com/${S3_OBJECT}?AWSAccessKeyId=${S3_ACCESS_KEY}&Expires=${TIMESTAMP}&Signature=${SIGNATURE}. +func preSignV2(req *http.Request, accessKeyID, secretAccessKey string, expires int64) error { + // Presign is not needed for anonymous credentials. + if accessKeyID == "" || secretAccessKey == "" { + return errors.New("Presign cannot be generated without access and secret keys") + } + + // FIXME: Remove following portion of code after fixing a bug in minio-go preSignV2. + + d := UTCNow() + // Find epoch expires when the request will expire. + epochExpires := d.Unix() + expires + + // Add expires header if not present. + expiresStr := req.Header.Get("Expires") + if expiresStr == "" { + expiresStr = strconv.FormatInt(epochExpires, 10) + req.Header.Set("Expires", expiresStr) + } + + // url.RawPath will be valid if path has any encoded characters, if not it will + // be empty - in which case we need to consider url.Path (bug in net/http?) + encodedResource := req.URL.RawPath + encodedQuery := req.URL.RawQuery + if encodedResource == "" { + splits := strings.SplitN(req.URL.Path, "?", 2) + encodedResource = splits[0] + if len(splits) == 2 { + encodedQuery = splits[1] + } + } + + unescapedQueries, err := unescapeQueries(encodedQuery) + if err != nil { + return err + } + + // Get presigned string to sign. + stringToSign := getStringToSignV2(req.Method, encodedResource, strings.Join(unescapedQueries, "&"), req.Header, expiresStr) + hm := hmac.New(sha1.New, []byte(secretAccessKey)) + hm.Write([]byte(stringToSign)) + + // Calculate signature. + signature := base64.StdEncoding.EncodeToString(hm.Sum(nil)) + + query := req.URL.Query() + // Handle specially for Google Cloud Storage. + query.Set("AWSAccessKeyId", accessKeyID) + // Fill in Expires for presigned query. + query.Set("Expires", strconv.FormatInt(epochExpires, 10)) + + // Encode query and save. + req.URL.RawQuery = query.Encode() + + // Save signature finally. + req.URL.RawQuery += "&Signature=" + url.QueryEscape(signature) + return nil +} + +// Sign given request using Signature V2. +func signRequestV2(req *http.Request, accessKey, secretKey string) error { + signer.SignV2(*req, accessKey, secretKey, false) + return nil +} + +// Sign given request using Signature V4. +func signRequestV4(req *http.Request, accessKey, secretKey string) error { + // Get hashed payload. + hashedPayload := req.Header.Get("x-amz-content-sha256") + if hashedPayload == "" { + return fmt.Errorf("Invalid hashed payload") + } + + currTime := UTCNow() + + // Set x-amz-date. + req.Header.Set("x-amz-date", currTime.Format(iso8601Format)) + + // Get header map. + headerMap := make(map[string][]string) + for k, vv := range req.Header { + // If request header key is not in ignored headers, then add it. + if _, ok := ignoredHeaders[http.CanonicalHeaderKey(k)]; !ok { + headerMap[strings.ToLower(k)] = vv + } + } + + // Get header keys. + headers := []string{"host"} + for k := range headerMap { + headers = append(headers, k) + } + sort.Strings(headers) + + region := globalSite.Region() + + // Get canonical headers. + var buf bytes.Buffer + for _, k := range headers { + buf.WriteString(k) + buf.WriteByte(':') + switch k { + case "host": + buf.WriteString(req.URL.Host) + fallthrough + default: + for idx, v := range headerMap[k] { + if idx > 0 { + buf.WriteByte(',') + } + buf.WriteString(v) + } + buf.WriteByte('\n') + } + } + canonicalHeaders := buf.String() + + // Get signed headers. + signedHeaders := strings.Join(headers, ";") + + // Get canonical query string. + req.URL.RawQuery = strings.ReplaceAll(req.URL.Query().Encode(), "+", "%20") + + // Get canonical URI. + canonicalURI := s3utils.EncodePath(req.URL.Path) + + // Get canonical request. + // canonicalRequest = + // \n + // \n + // \n + // \n + // \n + // + // + canonicalRequest := strings.Join([]string{ + req.Method, + canonicalURI, + req.URL.RawQuery, + canonicalHeaders, + signedHeaders, + hashedPayload, + }, "\n") + + // Get scope. + scope := strings.Join([]string{ + currTime.Format(yyyymmdd), + region, + string(serviceS3), + "aws4_request", + }, SlashSeparator) + + stringToSign := "AWS4-HMAC-SHA256" + "\n" + currTime.Format(iso8601Format) + "\n" + stringToSign = stringToSign + scope + "\n" + stringToSign += getSHA256Hash([]byte(canonicalRequest)) + + date := sumHMAC([]byte("AWS4"+secretKey), []byte(currTime.Format(yyyymmdd))) + regionHMAC := sumHMAC(date, []byte(region)) + service := sumHMAC(regionHMAC, []byte(serviceS3)) + signingKey := sumHMAC(service, []byte("aws4_request")) + + signature := hex.EncodeToString(sumHMAC(signingKey, []byte(stringToSign))) + + // final Authorization header + parts := []string{ + "AWS4-HMAC-SHA256" + " Credential=" + accessKey + SlashSeparator + scope, + "SignedHeaders=" + signedHeaders, + "Signature=" + signature, + } + auth := strings.Join(parts, ", ") + req.Header.Set("Authorization", auth) + + return nil +} + +// getCredentialString generate a credential string. +func getCredentialString(accessKeyID, location string, t time.Time) string { + return accessKeyID + SlashSeparator + getScope(t, location) +} + +// getMD5HashBase64 returns MD5 hash in base64 encoding of given data. +func getMD5HashBase64(data []byte) string { + return base64.StdEncoding.EncodeToString(getMD5Sum(data)) +} + +// Returns new HTTP request object. +func newTestRequest(method, urlStr string, contentLength int64, body io.ReadSeeker) (*http.Request, error) { + if method == "" { + method = http.MethodPost + } + + // Save for subsequent use + var hashedPayload string + var md5Base64 string + switch body { + case nil: + hashedPayload = getSHA256Hash([]byte{}) + default: + payloadBytes, err := io.ReadAll(body) + if err != nil { + return nil, err + } + hashedPayload = getSHA256Hash(payloadBytes) + md5Base64 = getMD5HashBase64(payloadBytes) + } + // Seek back to beginning. + if body != nil { + body.Seek(0, 0) + } else { + body = bytes.NewReader([]byte("")) + } + req, err := http.NewRequest(method, urlStr, body) + if err != nil { + return nil, err + } + if md5Base64 != "" { + req.Header.Set("Content-Md5", md5Base64) + } + req.Header.Set("x-amz-content-sha256", hashedPayload) + + // Add Content-Length + req.ContentLength = contentLength + + return req, nil +} + +// Various signature types we are supporting, currently +// two main signature types. +type signerType int + +const ( + signerV2 signerType = iota + signerV4 +) + +func newTestSignedRequest(method, urlStr string, contentLength int64, body io.ReadSeeker, accessKey, secretKey string, signer signerType) (*http.Request, error) { + if signer == signerV2 { + return newTestSignedRequestV2(method, urlStr, contentLength, body, accessKey, secretKey, nil) + } + return newTestSignedRequestV4(method, urlStr, contentLength, body, accessKey, secretKey, nil) +} + +// Returns request with correct signature but with incorrect SHA256. +func newTestSignedBadSHARequest(method, urlStr string, contentLength int64, body io.ReadSeeker, accessKey, secretKey string, signer signerType) (*http.Request, error) { + req, err := newTestRequest(method, urlStr, contentLength, body) + if err != nil { + return nil, err + } + + // Anonymous request return early. + if accessKey == "" || secretKey == "" { + return req, nil + } + + if signer == signerV2 { + err = signRequestV2(req, accessKey, secretKey) + req.Header.Del("x-amz-content-sha256") + } else { + req.Header.Set("x-amz-content-sha256", "92b165232fbd011da355eca0b033db22b934ba9af0145a437a832d27310b89f9") + err = signRequestV4(req, accessKey, secretKey) + } + + return req, err +} + +// Returns new HTTP request object signed with signature v2. +func newTestSignedRequestV2(method, urlStr string, contentLength int64, body io.ReadSeeker, accessKey, secretKey string, headers map[string]string) (*http.Request, error) { + req, err := newTestRequest(method, urlStr, contentLength, body) + if err != nil { + return nil, err + } + req.Header.Del("x-amz-content-sha256") + + // Anonymous request return quickly. + if accessKey == "" || secretKey == "" { + return req, nil + } + + for k, v := range headers { + req.Header.Set(k, v) + } + + err = signRequestV2(req, accessKey, secretKey) + if err != nil { + return nil, err + } + + return req, nil +} + +// Returns new HTTP request object signed with signature v4. +func newTestSignedRequestV4(method, urlStr string, contentLength int64, body io.ReadSeeker, accessKey, secretKey string, headers map[string]string) (*http.Request, error) { + req, err := newTestRequest(method, urlStr, contentLength, body) + if err != nil { + return nil, err + } + + // Anonymous request return quickly. + if accessKey == "" || secretKey == "" { + return req, nil + } + + for k, v := range headers { + req.Header.Set(k, v) + } + + err = signRequestV4(req, accessKey, secretKey) + if err != nil { + return nil, err + } + + return req, nil +} + +var src = rand.NewSource(time.Now().UnixNano()) + +func randString(n int) string { + b := make([]byte, n) + // A src.Int63() generates 63 random bits, enough for letterIdxMax characters! + for i, cache, remain := n-1, src.Int63(), letterIdxMax; i >= 0; { + if remain == 0 { + cache, remain = src.Int63(), letterIdxMax + } + if idx := int(cache & letterIdxMask); idx < len(letterBytes) { + b[i] = letterBytes[idx] + i-- + } + cache >>= letterIdxBits + remain-- + } + + return *(*string)(unsafe.Pointer(&b)) +} + +// generate random object name. +func getRandomObjectName() string { + return randString(16) +} + +// generate random bucket name. +func getRandomBucketName() string { + return randString(60) +} + +// construct URL for http requests for bucket operations. +func makeTestTargetURL(endPoint, bucketName, objectName string, queryValues url.Values) string { + urlStr := endPoint + SlashSeparator + if bucketName != "" { + urlStr = urlStr + bucketName + SlashSeparator + } + if objectName != "" { + urlStr += s3utils.EncodePath(objectName) + } + if len(queryValues) > 0 { + urlStr = urlStr + "?" + queryValues.Encode() + } + return urlStr +} + +// return URL for uploading object into the bucket. +func getPutObjectURL(endPoint, bucketName, objectName string) string { + return makeTestTargetURL(endPoint, bucketName, objectName, url.Values{}) +} + +func getPutObjectPartURL(endPoint, bucketName, objectName, uploadID, partNumber string) string { + queryValues := url.Values{} + queryValues.Set("uploadId", uploadID) + queryValues.Set("partNumber", partNumber) + return makeTestTargetURL(endPoint, bucketName, objectName, queryValues) +} + +func getCopyObjectPartURL(endPoint, bucketName, objectName, uploadID, partNumber string) string { + queryValues := url.Values{} + queryValues.Set("uploadId", uploadID) + queryValues.Set("partNumber", partNumber) + return makeTestTargetURL(endPoint, bucketName, objectName, queryValues) +} + +// return URL for fetching object from the bucket. +func getGetObjectURL(endPoint, bucketName, objectName string) string { + return makeTestTargetURL(endPoint, bucketName, objectName, url.Values{}) +} + +// return URL for deleting the object from the bucket. +func getDeleteObjectURL(endPoint, bucketName, objectName string) string { + return makeTestTargetURL(endPoint, bucketName, objectName, url.Values{}) +} + +// return URL for deleting multiple objects from a bucket. +func getMultiDeleteObjectURL(endPoint, bucketName string) string { + queryValue := url.Values{} + queryValue.Set("delete", "") + return makeTestTargetURL(endPoint, bucketName, "", queryValue) +} + +// return URL for HEAD on the object. +func getHeadObjectURL(endPoint, bucketName, objectName string) string { + return makeTestTargetURL(endPoint, bucketName, objectName, url.Values{}) +} + +// return url to be used while copying the object. +func getCopyObjectURL(endPoint, bucketName, objectName string) string { + return makeTestTargetURL(endPoint, bucketName, objectName, url.Values{}) +} + +// return URL for inserting bucket notification. +func getPutNotificationURL(endPoint, bucketName string) string { + queryValue := url.Values{} + queryValue.Set("notification", "") + return makeTestTargetURL(endPoint, bucketName, "", queryValue) +} + +// return URL for inserting bucket policy. +func getPutPolicyURL(endPoint, bucketName string) string { + queryValue := url.Values{} + queryValue.Set("policy", "") + return makeTestTargetURL(endPoint, bucketName, "", queryValue) +} + +// return URL for fetching bucket policy. +func getGetPolicyURL(endPoint, bucketName string) string { + queryValue := url.Values{} + queryValue.Set("policy", "") + return makeTestTargetURL(endPoint, bucketName, "", queryValue) +} + +// return URL for deleting bucket policy. +func getDeletePolicyURL(endPoint, bucketName string) string { + queryValue := url.Values{} + queryValue.Set("policy", "") + return makeTestTargetURL(endPoint, bucketName, "", queryValue) +} + +// return URL for creating the bucket. +func getMakeBucketURL(endPoint, bucketName string) string { + return makeTestTargetURL(endPoint, bucketName, "", url.Values{}) +} + +// return URL for creating the bucket. +func getBucketVersioningConfigURL(endPoint, bucketName string) string { + vals := make(url.Values) + vals.Set("versioning", "") + return makeTestTargetURL(endPoint, bucketName, "", vals) +} + +// return URL for listing buckets. +func getListBucketURL(endPoint string) string { + return makeTestTargetURL(endPoint, "", "", url.Values{}) +} + +// return URL for HEAD on the bucket. +func getHEADBucketURL(endPoint, bucketName string) string { + return makeTestTargetURL(endPoint, bucketName, "", url.Values{}) +} + +// return URL for deleting the bucket. +func getDeleteBucketURL(endPoint, bucketName string) string { + return makeTestTargetURL(endPoint, bucketName, "", url.Values{}) +} + +// return URL for deleting the bucket. +func getDeleteMultipleObjectsURL(endPoint, bucketName string) string { + queryValue := url.Values{} + queryValue.Set("delete", "") + return makeTestTargetURL(endPoint, bucketName, "", queryValue) +} + +// return URL For fetching location of the bucket. +func getBucketLocationURL(endPoint, bucketName string) string { + queryValue := url.Values{} + queryValue.Set("location", "") + return makeTestTargetURL(endPoint, bucketName, "", queryValue) +} + +// return URL For set/get lifecycle of the bucket. +func getBucketLifecycleURL(endPoint, bucketName string) (ret string) { + queryValue := url.Values{} + queryValue.Set("lifecycle", "") + return makeTestTargetURL(endPoint, bucketName, "", queryValue) +} + +// return URL for listing objects in the bucket with V1 legacy API. +func getListObjectsV1URL(endPoint, bucketName, prefix, maxKeys, encodingType string) string { + queryValue := url.Values{} + if maxKeys != "" { + queryValue.Set("max-keys", maxKeys) + } + if encodingType != "" { + queryValue.Set("encoding-type", encodingType) + } + return makeTestTargetURL(endPoint, bucketName, prefix, queryValue) +} + +// return URL for listing objects in the bucket with V1 legacy API. +func getListObjectVersionsURL(endPoint, bucketName, prefix, maxKeys, encodingType string) string { + queryValue := url.Values{} + if maxKeys != "" { + queryValue.Set("max-keys", maxKeys) + } + if encodingType != "" { + queryValue.Set("encoding-type", encodingType) + } + queryValue.Set("versions", "") + return makeTestTargetURL(endPoint, bucketName, prefix, queryValue) +} + +// return URL for listing objects in the bucket with V2 API. +func getListObjectsV2URL(endPoint, bucketName, prefix, maxKeys, fetchOwner, encodingType, delimiter string) string { + queryValue := url.Values{} + queryValue.Set("list-type", "2") // Enables list objects V2 URL. + if maxKeys != "" { + queryValue.Set("max-keys", maxKeys) + } + if fetchOwner != "" { + queryValue.Set("fetch-owner", fetchOwner) + } + if encodingType != "" { + queryValue.Set("encoding-type", encodingType) + } + if prefix != "" { + queryValue.Set("prefix", prefix) + } + if delimiter != "" { + queryValue.Set("delimiter", delimiter) + } + return makeTestTargetURL(endPoint, bucketName, "", queryValue) +} + +// return URL for a new multipart upload. +func getNewMultipartURL(endPoint, bucketName, objectName string) string { + queryValue := url.Values{} + queryValue.Set("uploads", "") + return makeTestTargetURL(endPoint, bucketName, objectName, queryValue) +} + +// return URL for a new multipart upload. +func getPartUploadURL(endPoint, bucketName, objectName, uploadID, partNumber string) string { + queryValues := url.Values{} + queryValues.Set("uploadId", uploadID) + queryValues.Set("partNumber", partNumber) + return makeTestTargetURL(endPoint, bucketName, objectName, queryValues) +} + +// return URL for aborting multipart upload. +func getAbortMultipartUploadURL(endPoint, bucketName, objectName, uploadID string) string { + queryValue := url.Values{} + queryValue.Set("uploadId", uploadID) + return makeTestTargetURL(endPoint, bucketName, objectName, queryValue) +} + +// return URL for a listing pending multipart uploads. +func getListMultipartURL(endPoint, bucketName string) string { + queryValue := url.Values{} + queryValue.Set("uploads", "") + return makeTestTargetURL(endPoint, bucketName, "", queryValue) +} + +// return URL for listing pending multipart uploads with parameters. +func getListMultipartUploadsURLWithParams(endPoint, bucketName, prefix, keyMarker, uploadIDMarker, delimiter, maxUploads string) string { + queryValue := url.Values{} + queryValue.Set("uploads", "") + queryValue.Set("prefix", prefix) + queryValue.Set("delimiter", delimiter) + queryValue.Set("key-marker", keyMarker) + queryValue.Set("upload-id-marker", uploadIDMarker) + queryValue.Set("max-uploads", maxUploads) + return makeTestTargetURL(endPoint, bucketName, "", queryValue) +} + +// return URL for a listing parts on a given upload id. +func getListMultipartURLWithParams(endPoint, bucketName, objectName, uploadID, maxParts, partNumberMarker, encoding string) string { + queryValues := url.Values{} + queryValues.Set("uploadId", uploadID) + queryValues.Set("max-parts", maxParts) + if partNumberMarker != "" { + queryValues.Set("part-number-marker", partNumberMarker) + } + return makeTestTargetURL(endPoint, bucketName, objectName, queryValues) +} + +// return URL for completing multipart upload. +// complete multipart upload request is sent after all parts are uploaded. +func getCompleteMultipartUploadURL(endPoint, bucketName, objectName, uploadID string) string { + queryValue := url.Values{} + queryValue.Set("uploadId", uploadID) + return makeTestTargetURL(endPoint, bucketName, objectName, queryValue) +} + +// return URL for listen bucket notification. +func getListenNotificationURL(endPoint, bucketName string, prefixes, suffixes, events []string) string { + queryValue := url.Values{} + + queryValue["prefix"] = prefixes + queryValue["suffix"] = suffixes + queryValue["events"] = events + return makeTestTargetURL(endPoint, bucketName, "", queryValue) +} + +// getRandomDisks - Creates a slice of N random disks, each of the form - minio-XXX +func getRandomDisks(n int) ([]string, error) { + var erasureDisks []string + for range n { + path, err := os.MkdirTemp(globalTestTmpDir, "minio-") + if err != nil { + // Remove directories created so far. + removeRoots(erasureDisks) + return nil, err + } + erasureDisks = append(erasureDisks, path) + } + return erasureDisks, nil +} + +// Initialize object layer with the supplied disks, objectLayer is nil upon any error. +func newTestObjectLayer(ctx context.Context, endpointServerPools EndpointServerPools) (newObject ObjectLayer, err error) { + initAllSubsystems(ctx) + + return newErasureServerPools(ctx, endpointServerPools) +} + +// initObjectLayer - Instantiates object layer and returns it. +func initObjectLayer(ctx context.Context, endpointServerPools EndpointServerPools) (ObjectLayer, []StorageAPI, error) { + objLayer, err := newTestObjectLayer(ctx, endpointServerPools) + if err != nil { + return nil, nil, err + } + + var formattedDisks []StorageAPI + // Should use the object layer tests for validating cache. + if z, ok := objLayer.(*erasureServerPools); ok { + formattedDisks = z.serverPools[0].GetDisks(0)() + } + + // Success. + return objLayer, formattedDisks, nil +} + +// removeRoots - Cleans up initialized directories during tests. +func removeRoots(roots []string) { + for _, root := range roots { + os.RemoveAll(root) + } +} + +// creates a bucket for the tests and returns the bucket name. +// initializes the specified API endpoints for the tests. +// initializes the root and returns its path. +// return credentials. +func initAPIHandlerTest(ctx context.Context, obj ObjectLayer, endpoints []string, makeBucketOptions MakeBucketOptions) (string, http.Handler, error) { + initAllSubsystems(ctx) + + initConfigSubsystem(ctx, obj) + + globalIAMSys.Init(ctx, obj, globalEtcdClient, 2*time.Second) + + // get random bucket name. + bucketName := getRandomBucketName() + // Create bucket. + err := obj.MakeBucket(context.Background(), bucketName, makeBucketOptions) + if err != nil { + // failed to create newbucket, return err. + return "", nil, err + } + // Register the API end points with Erasure object layer. + // Registering only the GetObject handler. + apiRouter := initTestAPIEndPoints(obj, endpoints) + f := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + r.RequestURI = r.URL.RequestURI() + apiRouter.ServeHTTP(w, r) + }) + return bucketName, f, nil +} + +// prepare test backend. +// create FS/Erasure/ErasureSet backend. +// return object layer, backend disks. +func prepareTestBackend(ctx context.Context, instanceType string) (ObjectLayer, []string, error) { + switch instanceType { + // Total number of disks for Erasure sets backend is set to 32. + case ErasureSetsTestStr: + return prepareErasureSets32(ctx) + // Total number of disks for Erasure backend is set to 16. + case ErasureTestStr: + return prepareErasure16(ctx) + default: + // return FS backend by default. + obj, disk, err := prepareFS(ctx) + if err != nil { + return nil, nil, err + } + return obj, []string{disk}, nil + } +} + +// ExecObjectLayerAPIAnonTest - Helper function to validate object Layer API handler +// response for anonymous/unsigned and unknown signature type HTTP request. + +// Here is the brief description of some of the arguments to the function below. +// +// apiRouter - http.Handler with the relevant API endPoint (API endPoint under test) registered. +// anonReq - unsigned *http.Request to invoke the handler's response for anonymous requests. +// policyFunc - function to return bucketPolicy statement which would permit the anonymous request to be served. +// +// The test works in 2 steps, here is the description of the steps. +// +// STEP 1: Call the handler with the unsigned HTTP request (anonReq), assert for the `ErrAccessDenied` error response. +func ExecObjectLayerAPIAnonTest(t *testing.T, obj ObjectLayer, testName, bucketName, objectName, instanceType string, apiRouter http.Handler, + anonReq *http.Request, bucketPolicy *policy.BucketPolicy, +) { + anonTestStr := "Anonymous HTTP request test" + unknownSignTestStr := "Unknown HTTP signature test" + + // simple function which returns a message which gives the context of the test + // and then followed by the actual error message. + failTestStr := func(testType, failMsg string) string { + return fmt.Sprintf("MinIO %s: %s fail for \"%s\": \n %s", instanceType, testType, testName, failMsg) + } + + // httptest Recorder to capture all the response by the http handler. + rec := httptest.NewRecorder() + // reading the body to preserve it so that it can be used again for second attempt of sending unsigned HTTP request. + // If the body is read in the handler the same request cannot be made use of. + buf, err := io.ReadAll(anonReq.Body) + if err != nil { + t.Fatal(failTestStr(anonTestStr, err.Error())) + } + + // creating 2 read closer (to set as request body) from the body content. + readerOne := io.NopCloser(bytes.NewBuffer(buf)) + readerTwo := io.NopCloser(bytes.NewBuffer(buf)) + + anonReq.Body = readerOne + + // call the HTTP handler. + apiRouter.ServeHTTP(rec, anonReq) + + // expected error response when the unsigned HTTP request is not permitted. + accessDenied := getAPIError(ErrAccessDenied).HTTPStatusCode + if rec.Code != accessDenied { + t.Fatal(failTestStr(anonTestStr, fmt.Sprintf("Object API Nil Test expected to fail with %d, but failed with %d", accessDenied, rec.Code))) + } + + // HEAD HTTP request doesn't contain response body. + if anonReq.Method != http.MethodHead { + // read the response body. + var actualContent []byte + actualContent, err = io.ReadAll(rec.Body) + if err != nil { + t.Fatal(failTestStr(anonTestStr, fmt.Sprintf("Failed parsing response body: %v", err))) + } + + actualError := &APIErrorResponse{} + if err = xml.Unmarshal(actualContent, actualError); err != nil { + t.Fatal(failTestStr(anonTestStr, "error response failed to parse error XML")) + } + + if actualError.BucketName != bucketName { + t.Fatal(failTestStr(anonTestStr, "error response bucket name differs from expected value")) + } + + if actualError.Key != objectName { + t.Fatal(failTestStr(anonTestStr, "error response object name differs from expected value")) + } + } + + // test for unknown auth case. + anonReq.Body = readerTwo + // Setting the `Authorization` header to a random value so that the signature falls into unknown auth case. + anonReq.Header.Set("Authorization", "nothingElse") + // initialize new response recorder. + rec = httptest.NewRecorder() + // call the handler using the HTTP Request. + apiRouter.ServeHTTP(rec, anonReq) + // verify the response body for `ErrAccessDenied` message =. + if anonReq.Method != http.MethodHead { + // read the response body. + actualContent, err := io.ReadAll(rec.Body) + if err != nil { + t.Fatal(failTestStr(unknownSignTestStr, fmt.Sprintf("Failed parsing response body: %v", err))) + } + + actualError := &APIErrorResponse{} + if err = xml.Unmarshal(actualContent, actualError); err != nil { + t.Fatal(failTestStr(unknownSignTestStr, "error response failed to parse error XML")) + } + + if path.Clean(actualError.Resource) != pathJoin(SlashSeparator, bucketName, SlashSeparator, objectName) { + t.Fatal(failTestStr(unknownSignTestStr, "error response resource differs from expected value")) + } + } + + // expected error response when the unsigned HTTP request is not permitted. + unsupportedSignature := getAPIError(ErrSignatureVersionNotSupported).HTTPStatusCode + if rec.Code != unsupportedSignature { + t.Fatal(failTestStr(unknownSignTestStr, fmt.Sprintf("Object API Unknown auth test for \"%s\", expected to fail with %d, but failed with %d", testName, unsupportedSignature, rec.Code))) + } +} + +// ExecObjectLayerAPINilTest - Sets the object layer to `nil`, and calls the registered object layer API endpoint, +// and assert the error response. The purpose is to validate the API handlers response when the object layer is uninitialized. +// Usage hint: Should be used at the end of the API end points tests (ex: check the last few lines of `testAPIListObjectPartsHandler`), +// need a sample HTTP request to be sent as argument so that the relevant handler is called, the handler registration is expected +// to be done since its called from within the API handler tests, the reference to the registered HTTP handler has to be sent +// as an argument. +func ExecObjectLayerAPINilTest(t TestErrHandler, bucketName, objectName, instanceType string, apiRouter http.Handler, req *http.Request) { + // httptest Recorder to capture all the response by the http handler. + rec := httptest.NewRecorder() + + // The API handler gets the reference to the object layer via the global object Layer, + // setting it to `nil` in order test for handlers response for uninitialized object layer. + globalObjLayerMutex.Lock() + globalObjectAPI = nil + globalObjLayerMutex.Unlock() + + // call the HTTP handler. + apiRouter.ServeHTTP(rec, req) + + // expected error response when the API handler is called before the object layer is initialized, + // or when objectLayer is `nil`. + serverNotInitializedErr := getAPIError(ErrServerNotInitialized).HTTPStatusCode + if rec.Code != serverNotInitializedErr { + t.Errorf("Object API Nil Test expected to fail with %d, but failed with %d", serverNotInitializedErr, rec.Code) + } + + // HEAD HTTP Request doesn't contain body in its response, + // for other type of HTTP requests compare the response body content with the expected one. + if req.Method != http.MethodHead { + // read the response body. + actualContent, err := io.ReadAll(rec.Body) + if err != nil { + t.Fatalf("MinIO %s: Failed parsing response body: %v", instanceType, err) + } + + actualError := &APIErrorResponse{} + if err = xml.Unmarshal(actualContent, actualError); err != nil { + t.Errorf("MinIO %s: error response failed to parse error XML", instanceType) + } + + if actualError.BucketName != bucketName { + t.Errorf("MinIO %s: error response bucket name differs from expected value", instanceType) + } + + if actualError.Key != objectName { + t.Errorf("MinIO %s: error response object name differs from expected value", instanceType) + } + } +} + +type ExecObjectLayerAPITestArgs struct { + t *testing.T + objAPITest objAPITestType + endpoints []string + init func() + makeBucketOptions MakeBucketOptions +} + +// ExecObjectLayerAPITest - executes object layer API tests. +// Creates single node and Erasure ObjectLayer instance, registers the specified API end points and runs test for both the layers. +func ExecObjectLayerAPITest(args ExecObjectLayerAPITestArgs) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // reset globals. + // this is to make sure that the tests are not affected by modified value. + resetTestGlobals() + + objLayer, fsDir, err := prepareFS(ctx) + if err != nil { + args.t.Fatalf("Initialization of object layer failed for single node setup: %s", err) + } + + bucketFS, fsAPIRouter, err := initAPIHandlerTest(ctx, objLayer, args.endpoints, args.makeBucketOptions) + if err != nil { + args.t.Fatalf("Initialization of API handler tests failed: %s", err) + } + + if args.init != nil { + args.init() + } + + // initialize the server and obtain the credentials and root. + // credentials are necessary to sign the HTTP request. + if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil { + args.t.Fatalf("Unable to initialize server config. %s", err) + } + + credentials := globalActiveCred + + // Executing the object layer tests for single node setup. + args.objAPITest(objLayer, ErasureSDStr, bucketFS, fsAPIRouter, credentials, args.t) + + // reset globals. + // this is to make sure that the tests are not affected by modified value. + resetTestGlobals() + + objLayer, erasureDisks, err := prepareErasure16(ctx) + if err != nil { + args.t.Fatalf("Initialization of object layer failed for Erasure setup: %s", err) + } + defer objLayer.Shutdown(ctx) + + bucketErasure, erAPIRouter, err := initAPIHandlerTest(ctx, objLayer, args.endpoints, args.makeBucketOptions) + if err != nil { + args.t.Fatalf("Initialization of API handler tests failed: %s", err) + } + + if args.init != nil { + args.init() + } + + // initialize the server and obtain the credentials and root. + // credentials are necessary to sign the HTTP request. + if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil { + args.t.Fatalf("Unable to initialize server config. %s", err) + } + + // Executing the object layer tests for Erasure. + args.objAPITest(objLayer, ErasureTestStr, bucketErasure, erAPIRouter, credentials, args.t) + + // clean up the temporary test backend. + removeRoots(append(erasureDisks, fsDir)) +} + +// ExecExtendedObjectLayerTest will execute the tests with combinations of encrypted & compressed. +// This can be used to test functionality when reading and writing data. +func ExecExtendedObjectLayerAPITest(t *testing.T, objAPITest objAPITestType, endpoints []string) { + execExtended(t, func(t *testing.T, init func(), makeBucketOptions MakeBucketOptions) { + ExecObjectLayerAPITest(ExecObjectLayerAPITestArgs{t: t, objAPITest: objAPITest, endpoints: endpoints, init: init, makeBucketOptions: makeBucketOptions}) + }) +} + +// function to be passed to ExecObjectLayerAPITest, for executing object layr API handler tests. +type objAPITestType func(obj ObjectLayer, instanceType string, bucketName string, + apiRouter http.Handler, credentials auth.Credentials, t *testing.T) + +// Regular object test type. +type objTestType func(obj ObjectLayer, instanceType string, t TestErrHandler) + +// Special test type for test with directories +type objTestTypeWithDirs func(obj ObjectLayer, instanceType string, dirs []string, t TestErrHandler) + +// Special object test type for disk not found situations. +type objTestDiskNotFoundType func(obj ObjectLayer, instanceType string, dirs []string, t *testing.T) + +// ExecObjectLayerTest - executes object layer tests. +// Creates single node and Erasure ObjectLayer instance and runs test for both the layers. +func ExecObjectLayerTest(t TestErrHandler, objTest objTestType) { + { + ctx, cancel := context.WithCancel(context.Background()) + if localMetacacheMgr != nil { + localMetacacheMgr.deleteAll() + } + + objLayer, fsDir, err := prepareFS(ctx) + if err != nil { + t.Fatalf("Initialization of object layer failed for single node setup: %s", err) + } + setObjectLayer(objLayer) + initAllSubsystems(ctx) + + // initialize the server and obtain the credentials and root. + // credentials are necessary to sign the HTTP request. + if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil { + t.Fatal("Unexpected error", err) + } + initConfigSubsystem(ctx, objLayer) + globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second) + + // Executing the object layer tests for single node setup. + objTest(objLayer, ErasureSDStr, t) + + // Call clean up functions + cancel() + setObjectLayer(newObjectLayerFn()) + removeRoots([]string{fsDir}) + } + + { + ctx, cancel := context.WithCancel(context.Background()) + + if localMetacacheMgr != nil { + localMetacacheMgr.deleteAll() + } + + initAllSubsystems(ctx) + objLayer, fsDirs, err := prepareErasureSets32(ctx) + if err != nil { + t.Fatalf("Initialization of object layer failed for Erasure setup: %s", err) + } + setObjectLayer(objLayer) + initConfigSubsystem(ctx, objLayer) + globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second) + + // Executing the object layer tests for Erasure. + objTest(objLayer, ErasureTestStr, t) + + objLayer.Shutdown(context.Background()) + if localMetacacheMgr != nil { + localMetacacheMgr.deleteAll() + } + setObjectLayer(newObjectLayerFn()) + cancel() + removeRoots(fsDirs) + } +} + +// ExecObjectLayerTestWithDirs - executes object layer tests. +// Creates single node and Erasure ObjectLayer instance and runs test for both the layers. +func ExecObjectLayerTestWithDirs(t TestErrHandler, objTest objTestTypeWithDirs) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + if localMetacacheMgr != nil { + localMetacacheMgr.deleteAll() + } + + initAllSubsystems(ctx) + objLayer, fsDirs, err := prepareErasure16(ctx) + if err != nil { + t.Fatalf("Initialization of object layer failed for Erasure setup: %s", err) + } + setObjectLayer(objLayer) + initConfigSubsystem(ctx, objLayer) + globalIAMSys.Init(ctx, objLayer, globalEtcdClient, 2*time.Second) + + // Executing the object layer tests for Erasure. + objTest(objLayer, ErasureTestStr, fsDirs, t) + + objLayer.Shutdown(context.Background()) + if localMetacacheMgr != nil { + localMetacacheMgr.deleteAll() + } + setObjectLayer(newObjectLayerFn()) + cancel() + removeRoots(fsDirs) +} + +// ExecObjectLayerDiskAlteredTest - executes object layer tests while altering +// disks in between tests. Creates Erasure ObjectLayer instance and runs test for Erasure layer. +func ExecObjectLayerDiskAlteredTest(t *testing.T, objTest objTestDiskNotFoundType) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + objLayer, fsDirs, err := prepareErasure16(ctx) + if err != nil { + t.Fatalf("Initialization of object layer failed for Erasure setup: %s", err) + } + defer objLayer.Shutdown(ctx) + + if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil { + t.Fatal("Failed to create config directory", err) + } + + // Executing the object layer tests for Erasure. + objTest(objLayer, ErasureTestStr, fsDirs, t) + defer removeRoots(fsDirs) +} + +// Special object test type for stale files situations. +type objTestStaleFilesType func(obj ObjectLayer, instanceType string, dirs []string, t *testing.T) + +// ExecObjectLayerStaleFilesTest - executes object layer tests those leaves stale +// files/directories under .minio/tmp. Creates Erasure ObjectLayer instance and runs test for Erasure layer. +func ExecObjectLayerStaleFilesTest(t *testing.T, objTest objTestStaleFilesType) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + nDisks := 16 + erasureDisks, err := getRandomDisks(nDisks) + if err != nil { + t.Fatalf("Initialization of drives for Erasure setup: %s", err) + } + objLayer, _, err := initObjectLayer(ctx, mustGetPoolEndpoints(0, erasureDisks...)) + if err != nil { + t.Fatalf("Initialization of object layer failed for Erasure setup: %s", err) + } + if err = newTestConfig(globalMinioDefaultRegion, objLayer); err != nil { + t.Fatal("Failed to create config directory", err) + } + + // Executing the object layer tests for Erasure. + objTest(objLayer, ErasureTestStr, erasureDisks, t) + defer removeRoots(erasureDisks) +} + +func registerBucketLevelFunc(bucket *mux.Router, api objectAPIHandlers, apiFunctions ...string) { + for _, apiFunction := range apiFunctions { + switch apiFunction { + case "PostPolicy": + // Register PostPolicy handler. + bucket.Methods(http.MethodPost).HeadersRegexp("Content-Type", "multipart/form-data*").HandlerFunc(api.PostPolicyBucketHandler) + case "HeadObject": + // Register HeadObject handler. + bucket.Methods("Head").Path("/{object:.+}").HandlerFunc(api.HeadObjectHandler) + case "GetObject": + // Register GetObject handler. + bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(api.GetObjectHandler) + case "PutObject": + // Register PutObject handler. + bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(api.PutObjectHandler) + case "DeleteObject": + // Register Delete Object handler. + bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(api.DeleteObjectHandler) + case "CopyObject": + // Register Copy Object handler. + bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(api.CopyObjectHandler) + case "PutBucketPolicy": + // Register PutBucket Policy handler. + bucket.Methods(http.MethodPut).HandlerFunc(api.PutBucketPolicyHandler).Queries("policy", "") + case "DeleteBucketPolicy": + // Register Delete bucket HTTP policy handler. + bucket.Methods(http.MethodDelete).HandlerFunc(api.DeleteBucketPolicyHandler).Queries("policy", "") + case "GetBucketPolicy": + // Register Get Bucket policy HTTP Handler. + bucket.Methods(http.MethodGet).HandlerFunc(api.GetBucketPolicyHandler).Queries("policy", "") + case "GetBucketLifecycle": + bucket.Methods(http.MethodGet).HandlerFunc(api.GetBucketLifecycleHandler).Queries("lifecycle", "") + case "PutBucketLifecycle": + bucket.Methods(http.MethodPut).HandlerFunc(api.PutBucketLifecycleHandler).Queries("lifecycle", "") + case "DeleteBucketLifecycle": + bucket.Methods(http.MethodDelete).HandlerFunc(api.DeleteBucketLifecycleHandler).Queries("lifecycle", "") + case "GetBucketLocation": + // Register GetBucketLocation handler. + bucket.Methods(http.MethodGet).HandlerFunc(api.GetBucketLocationHandler).Queries("location", "") + case "HeadBucket": + // Register HeadBucket handler. + bucket.Methods(http.MethodHead).HandlerFunc(api.HeadBucketHandler) + case "DeleteMultipleObjects": + // Register DeleteMultipleObjects handler. + bucket.Methods(http.MethodPost).HandlerFunc(api.DeleteMultipleObjectsHandler).Queries("delete", "") + case "NewMultipart": + // Register New Multipart upload handler. + bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(api.NewMultipartUploadHandler).Queries("uploads", "") + case "CopyObjectPart": + // Register CopyObjectPart handler. + bucket.Methods(http.MethodPut).Path("/{object:.+}").HeadersRegexp("X-Amz-Copy-Source", ".*?(\\/|%2F).*?").HandlerFunc(api.CopyObjectPartHandler).Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}") + case "PutObjectPart": + // Register PutObjectPart handler. + bucket.Methods(http.MethodPut).Path("/{object:.+}").HandlerFunc(api.PutObjectPartHandler).Queries("partNumber", "{partNumber:.*}", "uploadId", "{uploadId:.*}") + case "ListObjectParts": + // Register ListObjectParts handler. + bucket.Methods(http.MethodGet).Path("/{object:.+}").HandlerFunc(api.ListObjectPartsHandler).Queries("uploadId", "{uploadId:.*}") + case "ListMultipartUploads": + // Register ListMultipartUploads handler. + bucket.Methods(http.MethodGet).HandlerFunc(api.ListMultipartUploadsHandler).Queries("uploads", "") + case "CompleteMultipart": + // Register Complete Multipart Upload handler. + bucket.Methods(http.MethodPost).Path("/{object:.+}").HandlerFunc(api.CompleteMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}") + case "AbortMultipart": + // Register AbortMultipart Handler. + bucket.Methods(http.MethodDelete).Path("/{object:.+}").HandlerFunc(api.AbortMultipartUploadHandler).Queries("uploadId", "{uploadId:.*}") + case "GetBucketNotification": + // Register GetBucketNotification Handler. + bucket.Methods(http.MethodGet).HandlerFunc(api.GetBucketNotificationHandler).Queries("notification", "") + case "PutBucketNotification": + // Register PutBucketNotification Handler. + bucket.Methods(http.MethodPut).HandlerFunc(api.PutBucketNotificationHandler).Queries("notification", "") + case "ListenNotification": + // Register ListenNotification Handler. + bucket.Methods(http.MethodGet).HandlerFunc(api.ListenNotificationHandler).Queries("events", "{events:.*}") + } + } +} + +// registerAPIFunctions helper function to add API functions identified by name to the routers. +func registerAPIFunctions(muxRouter *mux.Router, objLayer ObjectLayer, apiFunctions ...string) { + if len(apiFunctions) == 0 { + // Register all api endpoints by default. + registerAPIRouter(muxRouter) + return + } + // API Router. + apiRouter := muxRouter.PathPrefix(SlashSeparator).Subrouter() + // Bucket router. + bucketRouter := apiRouter.PathPrefix("/{bucket}").Subrouter() + + // All object storage operations are registered as HTTP handlers on `objectAPIHandlers`. + // When the handlers get a HTTP request they use the underlying ObjectLayer to perform operations. + globalObjLayerMutex.Lock() + globalObjectAPI = objLayer + globalObjLayerMutex.Unlock() + + // When cache is enabled, Put and Get operations are passed + // to underlying cache layer to manage object layer operation and disk caching + // operation + api := objectAPIHandlers{ + ObjectAPI: func() ObjectLayer { + return globalObjectAPI + }, + } + + // Register ListBuckets handler. + apiRouter.Methods(http.MethodGet).HandlerFunc(api.ListBucketsHandler) + // Register all bucket level handlers. + registerBucketLevelFunc(bucketRouter, api, apiFunctions...) +} + +// Takes in Erasure object layer, and the list of API end points to be tested/required, registers the API end points and returns the HTTP handler. +// Need isolated registration of API end points while writing unit tests for end points. +// All the API end points are registered only for the default case. +func initTestAPIEndPoints(objLayer ObjectLayer, apiFunctions []string) http.Handler { + // initialize a new mux router. + // goriilla/mux is the library used to register all the routes and handle them. + muxRouter := mux.NewRouter().SkipClean(true).UseEncodedPath() + if len(apiFunctions) > 0 { + // Iterate the list of API functions requested for and register them in mux HTTP handler. + registerAPIFunctions(muxRouter, objLayer, apiFunctions...) + muxRouter.Use(globalMiddlewares...) + return muxRouter + } + registerAPIRouter(muxRouter) + muxRouter.Use(globalMiddlewares...) + return muxRouter +} + +// generateTLSCertKey creates valid key/cert with registered DNS or IP address +// depending on the passed parameter. That way, we can use tls config without +// passing InsecureSkipVerify flag. This code is a simplified version of +// https://golang.org/src/crypto/tls/generate_cert.go +func generateTLSCertKey(host string) ([]byte, []byte, error) { + validFor := 365 * 24 * time.Hour + rsaBits := 2048 + + if len(host) == 0 { + return nil, nil, fmt.Errorf("Missing host parameter") + } + + publicKey := func(priv any) any { + switch k := priv.(type) { + case *rsa.PrivateKey: + return &k.PublicKey + case *ecdsa.PrivateKey: + return &k.PublicKey + default: + return nil + } + } + + pemBlockForKey := func(priv any) *pem.Block { + switch k := priv.(type) { + case *rsa.PrivateKey: + return &pem.Block{Type: "RSA PRIVATE KEY", Bytes: x509.MarshalPKCS1PrivateKey(k)} + case *ecdsa.PrivateKey: + b, err := x509.MarshalECPrivateKey(k) + if err != nil { + fmt.Fprintf(os.Stderr, "Unable to marshal ECDSA private key: %v", err) + os.Exit(2) + } + return &pem.Block{Type: "EC PRIVATE KEY", Bytes: b} + default: + return nil + } + } + + var priv any + var err error + priv, err = rsa.GenerateKey(crand.Reader, rsaBits) + if err != nil { + return nil, nil, fmt.Errorf("failed to generate private key: %w", err) + } + + notBefore := time.Now() + notAfter := notBefore.Add(validFor) + + serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128) + serialNumber, err := crand.Int(crand.Reader, serialNumberLimit) + if err != nil { + return nil, nil, fmt.Errorf("failed to generate serial number: %w", err) + } + + template := x509.Certificate{ + SerialNumber: serialNumber, + Subject: pkix.Name{ + Organization: []string{"Acme Co"}, + }, + NotBefore: notBefore, + NotAfter: notAfter, + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + } + + hosts := strings.SplitSeq(host, ",") + for h := range hosts { + if ip := net.ParseIP(h); ip != nil { + template.IPAddresses = append(template.IPAddresses, ip) + } else { + template.DNSNames = append(template.DNSNames, h) + } + } + + template.IsCA = true + template.KeyUsage |= x509.KeyUsageCertSign + + derBytes, err := x509.CreateCertificate(crand.Reader, &template, &template, publicKey(priv), priv) + if err != nil { + return nil, nil, fmt.Errorf("Failed to create certificate: %w", err) + } + + certOut := bytes.NewBuffer([]byte{}) + pem.Encode(certOut, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + + keyOut := bytes.NewBuffer([]byte{}) + pem.Encode(keyOut, pemBlockForKey(priv)) + + return certOut.Bytes(), keyOut.Bytes(), nil +} + +func mustGetPoolEndpoints(poolIdx int, args ...string) EndpointServerPools { + drivesPerSet := len(args) + setCount := 1 + if len(args) >= 16 { + drivesPerSet = 16 + setCount = len(args) / 16 + } + endpoints := mustGetNewEndpoints(poolIdx, drivesPerSet, args...) + return []PoolEndpoints{{ + SetCount: setCount, + DrivesPerSet: drivesPerSet, + Endpoints: endpoints, + CmdLine: strings.Join(args, " "), + }} +} + +func mustGetNewEndpoints(poolIdx int, drivesPerSet int, args ...string) (endpoints Endpoints) { + endpoints, err := NewEndpoints(args...) + if err != nil { + panic(err) + } + for i := range endpoints { + endpoints[i].SetPoolIndex(poolIdx) + endpoints[i].SetSetIndex(i / drivesPerSet) + endpoints[i].SetDiskIndex(i % drivesPerSet) + } + return endpoints +} + +func getEndpointsLocalAddr(endpointServerPools EndpointServerPools) string { + for _, endpoints := range endpointServerPools { + for _, endpoint := range endpoints.Endpoints { + if endpoint.IsLocal && endpoint.Type() == URLEndpointType { + return endpoint.Host + } + } + } + + return net.JoinHostPort(globalMinioHost, globalMinioPort) +} + +// fetches a random number between range min-max. +func getRandomRange(minN, maxN int, seed int64) int { + // special value -1 means no explicit seeding. + if seed == -1 { + return rand.New(rand.NewSource(time.Now().UnixNano())).Intn(maxN-minN) + minN + } + return rand.New(rand.NewSource(seed)).Intn(maxN-minN) + minN +} + +// Randomizes the order of bytes in the byte array +// using Knuth Fisher-Yates shuffle algorithm. +func randomizeBytes(s []byte, seed int64) []byte { + // special value -1 means no explicit seeding. + if seed != -1 { + rand.Seed(seed) + } + n := len(s) + var j int + for i := 0; i < n-1; i++ { + j = i + rand.Intn(n-i) + s[i], s[j] = s[j], s[i] + } + return s +} + +func TestToErrIsNil(t *testing.T) { + if toObjectErr(nil) != nil { + t.Errorf("Test expected to return nil, failed instead got a non-nil value %s", toObjectErr(nil)) + } + if toStorageErr(nil) != nil { + t.Errorf("Test expected to return nil, failed instead got a non-nil value %s", toStorageErr(nil)) + } + ctx := t.Context() + if toAPIError(ctx, nil) != noError { + t.Errorf("Test expected error code to be ErrNone, failed instead provided %s", toAPIError(ctx, nil).Code) + } +} + +// Uploads an object using DummyDataGen directly via the http +// handler. Each part in a multipart object is a new DummyDataGen +// instance (so the part sizes are needed to reconstruct the whole +// object). When `len(partSizes) == 1`, asMultipart is used to upload +// the object as multipart with 1 part or as a regular single object. +// +// All upload failures are considered test errors - this function is +// intended as a helper for other tests. +func uploadTestObject(t *testing.T, apiRouter http.Handler, creds auth.Credentials, bucketName, objectName string, + partSizes []int64, metadata map[string]string, asMultipart bool, +) { + if len(partSizes) == 0 { + t.Fatalf("Cannot upload an object without part sizes") + } + if len(partSizes) > 1 { + asMultipart = true + } + + checkRespErr := func(rec *httptest.ResponseRecorder, exp int) { + t.Helper() + if rec.Code != exp { + b, err := io.ReadAll(rec.Body) + t.Fatalf("Expected: %v, Got: %v, Body: %s, err: %v", exp, rec.Code, string(b), err) + } + } + + if !asMultipart { + srcData := NewDummyDataGen(partSizes[0], 0) + req, err := newTestSignedRequestV4(http.MethodPut, getPutObjectURL("", bucketName, objectName), + partSizes[0], srcData, creds.AccessKey, creds.SecretKey, metadata) + if err != nil { + t.Fatalf("Unexpected err: %#v", err) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + checkRespErr(rec, http.StatusOK) + } else { + // Multipart upload - each part is a new DummyDataGen + // (so the part lengths are required to verify the + // object when reading). + + // Initiate mp upload + reqI, err := newTestSignedRequestV4(http.MethodPost, getNewMultipartURL("", bucketName, objectName), + 0, nil, creds.AccessKey, creds.SecretKey, metadata) + if err != nil { + t.Fatalf("Unexpected err: %#v", err) + } + rec := httptest.NewRecorder() + apiRouter.ServeHTTP(rec, reqI) + checkRespErr(rec, http.StatusOK) + decoder := xml.NewDecoder(rec.Body) + multipartResponse := &InitiateMultipartUploadResponse{} + err = decoder.Decode(multipartResponse) + if err != nil { + t.Fatalf("Error decoding the recorded response Body") + } + upID := multipartResponse.UploadID + + // Upload each part + var cp []CompletePart + cumulativeSum := int64(0) + for i, partLen := range partSizes { + partID := i + 1 + partSrc := NewDummyDataGen(partLen, cumulativeSum) + cumulativeSum += partLen + req, errP := newTestSignedRequestV4(http.MethodPut, + getPutObjectPartURL("", bucketName, objectName, upID, fmt.Sprintf("%d", partID)), + partLen, partSrc, creds.AccessKey, creds.SecretKey, metadata) + if errP != nil { + t.Fatalf("Unexpected err: %#v", errP) + } + rec = httptest.NewRecorder() + apiRouter.ServeHTTP(rec, req) + checkRespErr(rec, http.StatusOK) + header := rec.Header() + if v, ok := header["ETag"]; ok { + etag := v[0] + if etag == "" { + t.Fatalf("Unexpected empty etag") + } + cp = append(cp, CompletePart{PartNumber: partID, ETag: etag[1 : len(etag)-1]}) + } else { + t.Fatalf("Missing etag header") + } + } + + // Call CompleteMultipart API + compMpBody, err := xml.Marshal(CompleteMultipartUpload{Parts: cp}) + if err != nil { + t.Fatalf("Unexpected err: %#v", err) + } + reqC, errP := newTestSignedRequestV4(http.MethodPost, + getCompleteMultipartUploadURL("", bucketName, objectName, upID), + int64(len(compMpBody)), bytes.NewReader(compMpBody), + creds.AccessKey, creds.SecretKey, metadata) + if errP != nil { + t.Fatalf("Unexpected err: %#v", errP) + } + rec = httptest.NewRecorder() + apiRouter.ServeHTTP(rec, reqC) + checkRespErr(rec, http.StatusOK) + } +} + +// unzip a file into a specific target dir - used to unzip sample data in cmd/testdata/ +func unzipArchive(zipFilePath, targetDir string) error { + zipReader, err := zip.OpenReader(zipFilePath) + if err != nil { + return err + } + for _, file := range zipReader.File { + zippedFile, err := file.Open() + if err != nil { + return err + } + err = func() (err error) { + defer zippedFile.Close() + extractedFilePath := filepath.Join(targetDir, file.Name) + if file.FileInfo().IsDir() { + return os.MkdirAll(extractedFilePath, file.Mode()) + } + outputFile, err := os.OpenFile(extractedFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, file.Mode()) + if err != nil { + return err + } + defer outputFile.Close() + _, err = io.Copy(outputFile, zippedFile) + return err + }() + if err != nil { + return err + } + } + return nil +} diff --git a/cmd/testdata/config/1.yaml b/cmd/testdata/config/1.yaml new file mode 100644 index 00000000..211f7da1 --- /dev/null +++ b/cmd/testdata/config/1.yaml @@ -0,0 +1,23 @@ +version: v1 +address: ':9000' +console-address: ':9001' +certs-dir: '/home/user/.minio/certs/' +pools: # Specify the nodes and drives with pools + - + - 'https://server-example-pool1:9000/mnt/disk{1...4}/' + - 'https://server{1...2}-pool1:9000/mnt/disk{1...4}/' + - 'https://server3-pool1:9000/mnt/disk{1...4}/' + - 'https://server4-pool1:9000/mnt/disk{1...4}/' + - + - 'https://server-example-pool2:9000/mnt/disk{1...4}/' + - 'https://server{1...2}-pool2:9000/mnt/disk{1...4}/' + - 'https://server3-pool2:9000/mnt/disk{1...4}/' + - 'https://server4-pool2:9000/mnt/disk{1...4}/' + +options: + ftp: # settings for MinIO to act as an ftp server + address: ':8021' + passive-port-range: '30000-40000' + sftp: # settings for MinIO to act as an sftp server + address: ':8022' + ssh-private-key: '/home/user/.ssh/id_rsa' diff --git a/cmd/testdata/config/2.yaml b/cmd/testdata/config/2.yaml new file mode 100644 index 00000000..fadb75cc --- /dev/null +++ b/cmd/testdata/config/2.yaml @@ -0,0 +1,23 @@ +version: v1 +address: ':9000' +console-address: ':9001' +certs-dir: '/home/user/.minio/certs/' +pools: # Specify the nodes and drives with pools + - + - 'https://server-example-pool1:9000/mnt/disk{1...4}/' + - 'https://server1-pool1:9000/mnt/disk{1...4}/' + - 'https://server3-pool1:9000/mnt/disk{1...4}/' + - 'https://server4-pool1:9000/mnt/disk{1...4}/' + - + - 'https://server-example-pool2:9000/mnt/disk{1...4}/' + - 'https://server1-pool2:9000/mnt/disk{1...4}/' + - 'https://server3-pool2:9000/mnt/disk{1...4}/' + - 'https://server4-pool2:9000/mnt/disk{1...4}/' + +options: + ftp: # settings for MinIO to act as an ftp server + address: ':8021' + passive-port-range: '30000-40000' + sftp: # settings for MinIO to act as an sftp server + address: ':8022' + ssh-private-key: '/home/user/.ssh/id_rsa' diff --git a/cmd/testdata/config/invalid-disks.yaml b/cmd/testdata/config/invalid-disks.yaml new file mode 100644 index 00000000..2bd77e0b --- /dev/null +++ b/cmd/testdata/config/invalid-disks.yaml @@ -0,0 +1,23 @@ +version: v1 +address: ':9000' +console-address: ':9001' +certs-dir: '/home/user/.minio/certs/' +pools: # Specify the nodes and drives with pools + - + - 'https://server-example-pool1:9000/mnt/disk1/' + - 'https://server1-pool1:9000/mnt/disk{1...4}/' + - 'https://server3-pool1:9000/mnt/disk{1...4}/' + - 'https://server4-pool1:9000/mnt/disk{1...4}/' + - + - 'https://server-example-pool2:9000/mnt/disk{1...4}/' + - 'https://server1-pool2:9000/mnt/disk{1...4}/' + - 'https://server3-pool2:9000/mnt/disk{1...4}/' + - 'https://server4-pool2:9000/mnt/disk{1...4}/' + +options: + ftp: # settings for MinIO to act as an ftp server + address: ':8021' + passive-port-range: '30000-40000' + sftp: # settings for MinIO to act as an sftp server + address: ':8022' + ssh-private-key: '/home/user/.ssh/id_rsa' diff --git a/cmd/testdata/config/invalid-types.yaml b/cmd/testdata/config/invalid-types.yaml new file mode 100644 index 00000000..10b6ceb2 --- /dev/null +++ b/cmd/testdata/config/invalid-types.yaml @@ -0,0 +1,23 @@ +version: v1 +address: ':9000' +console-address: ':9001' +certs-dir: '/home/user/.minio/certs/' +pools: # Specify the nodes and drives with pools + - + - '/mnt/disk{1...4}/' + - 'https://server1-pool1:9000/mnt/disk{1...4}/' + - 'https://server3-pool1:9000/mnt/disk{1...4}/' + - 'https://server4-pool1:9000/mnt/disk{1...4}/' + - + - 'https://server-example-pool2:9000/mnt/disk{1...4}/' + - 'https://server1-pool2:9000/mnt/disk{1...4}/' + - 'https://server3-pool2:9000/mnt/disk{1...4}/' + - 'https://server4-pool2:9000/mnt/disk{1...4}/' + +options: + ftp: # settings for MinIO to act as an ftp server + address: ':8021' + passive-port-range: '30000-40000' + sftp: # settings for MinIO to act as an sftp server + address: ':8022' + ssh-private-key: '/home/user/.ssh/id_rsa' diff --git a/cmd/testdata/config/invalid.yaml b/cmd/testdata/config/invalid.yaml new file mode 100644 index 00000000..2275d565 --- /dev/null +++ b/cmd/testdata/config/invalid.yaml @@ -0,0 +1,23 @@ +version: +address: ':9000' +console-address: ':9001' +certs-dir: '/home/user/.minio/certs/' +pools: # Specify the nodes and drives with pools + - + - 'https://server-example-pool1:9000/mnt/disk{1...4}/' + - 'https://server1-pool1:9000/mnt/disk{1...4}/' + - 'https://server3-pool1:9000/mnt/disk{1...4}/' + - 'https://server4-pool1:9000/mnt/disk{1...4}/' + - + - 'https://server-example-pool2:9000/mnt/disk{1...4}/' + - 'https://server1-pool2:9000/mnt/disk{1...4}/' + - 'https://server3-pool2:9000/mnt/disk{1...4}/' + - 'https://server4-pool2:9000/mnt/disk{1...4}/' + +options: + ftp: # settings for MinIO to act as an ftp server + address: ':8021' + passive-port-range: '30000-40000' + sftp: # settings for MinIO to act as an sftp server + address: ':8022' + ssh-private-key: '/home/user/.ssh/id_rsa' diff --git a/cmd/testdata/decryptObjectInfo.json.zst b/cmd/testdata/decryptObjectInfo.json.zst new file mode 100644 index 00000000..c27a8279 Binary files /dev/null and b/cmd/testdata/decryptObjectInfo.json.zst differ diff --git a/cmd/testdata/dillon_test_key.pub b/cmd/testdata/dillon_test_key.pub new file mode 100644 index 00000000..dc22abbd --- /dev/null +++ b/cmd/testdata/dillon_test_key.pub @@ -0,0 +1 @@ +ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIDVGk/SRz4fwTPK0+Ra7WYUGf3o08YkpI0yTMPpHwYoq dillon@example.io diff --git a/cmd/testdata/invalid_test_key.pub b/cmd/testdata/invalid_test_key.pub new file mode 100644 index 00000000..182a4766 --- /dev/null +++ b/cmd/testdata/invalid_test_key.pub @@ -0,0 +1 @@ +ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDES4saDDRpoHDVmiYESEQrCYhw8EK7Utj/A/lqxiqZlP6Il3aN2fWu6uJQdWAovZxNeXUf8LIujisW1mJWGZPql0SLKVq6IZ707OAGmKA59IXfF5onRoU9+K4UDL7BJFfix6/3F5OV2WB3ChFrOrXhJ0CZ0sVAfGcV4q72kS19YjZNX3fqCc2HF8UQEaZGKIkw5MtdZI9a1P2bqnPuPGJybRFUzyoQXPge45QT5jnpcsAXOuXcGxbjuqaaHXFNTSKAkCU93TcjAbqUMkTz2mnFz/MnrKJTECN3Fy0GPCCQ5dxmG8p8DyMiNl7JYkX2r3XYgxmioCzkcg8fDs5p0CaQcipu+MA7iK7APKq7v4Zr/wNltXHI3DE9S8J88Hxb2FZAyEhCRfcgGmCVfoZxVNCRHNkGYzfe63BkxtnseUCzpYEhKv02H5u9rjFpdMY37kDfHDVqBbgutdMij+tQAEp1kyqi6TQL+4XHjPHkLaeekW07yB+VI90dK1A9dzTpOvE= liza@example.io diff --git a/cmd/testdata/metacache.s2 b/cmd/testdata/metacache.s2 new file mode 100644 index 00000000..d288db9b Binary files /dev/null and b/cmd/testdata/metacache.s2 differ diff --git a/cmd/testdata/undeleteable-object.tgz b/cmd/testdata/undeleteable-object.tgz new file mode 100644 index 00000000..b6abc0a9 Binary files /dev/null and b/cmd/testdata/undeleteable-object.tgz differ diff --git a/cmd/testdata/xl-many-parts.meta b/cmd/testdata/xl-many-parts.meta new file mode 100644 index 00000000..72afd28c Binary files /dev/null and b/cmd/testdata/xl-many-parts.meta differ diff --git a/cmd/testdata/xl.meta b/cmd/testdata/xl.meta new file mode 100644 index 00000000..694e5a2a Binary files /dev/null and b/cmd/testdata/xl.meta differ diff --git a/cmd/testdata/xl.meta-v1.2.zst b/cmd/testdata/xl.meta-v1.2.zst new file mode 100644 index 00000000..5eb4c5da Binary files /dev/null and b/cmd/testdata/xl.meta-v1.2.zst differ diff --git a/cmd/tier-handlers.go b/cmd/tier-handlers.go new file mode 100644 index 00000000..8c814874 --- /dev/null +++ b/cmd/tier-handlers.go @@ -0,0 +1,264 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "encoding/json" + "io" + "net/http" + "strconv" + + jsoniter "github.com/json-iterator/go" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/config/storageclass" + "github.com/minio/mux" + "github.com/minio/pkg/v3/policy" +) + +var ( + // error returned when remote tier already exists + errTierAlreadyExists = AdminError{ + Code: "XMinioAdminTierAlreadyExists", + Message: "Specified remote tier already exists", + StatusCode: http.StatusConflict, + } + // error returned when remote tier is not found + errTierNotFound = AdminError{ + Code: "XMinioAdminTierNotFound", + Message: "Specified remote tier was not found", + StatusCode: http.StatusNotFound, + } + // error returned when remote tier name is not in uppercase + errTierNameNotUppercase = AdminError{ + Code: "XMinioAdminTierNameNotUpperCase", + Message: "Tier name must be in uppercase", + StatusCode: http.StatusBadRequest, + } + // error returned when remote tier bucket is not found + errTierBucketNotFound = AdminError{ + Code: "XMinioAdminTierBucketNotFound", + Message: "Remote tier bucket not found", + StatusCode: http.StatusBadRequest, + } + // error returned when remote tier credentials are invalid. + errTierInvalidCredentials = AdminError{ + Code: "XMinioAdminTierInvalidCredentials", + Message: "Invalid remote tier credentials", + StatusCode: http.StatusBadRequest, + } + // error returned when reserved internal names are used. + errTierReservedName = AdminError{ + Code: "XMinioAdminTierReserved", + Message: "Cannot use reserved tier name", + StatusCode: http.StatusBadRequest, + } +) + +func (api adminAPIHandlers) AddTierHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objAPI, cred := validateAdminReq(ctx, w, r, policy.SetTierAction) + if objAPI == nil { + return + } + + password := cred.SecretKey + reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength)) + if err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL) + return + } + + var cfg madmin.TierConfig + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err := json.Unmarshal(reqBytes, &cfg); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + var ignoreInUse bool + if forceStr := r.Form.Get("force"); forceStr != "" { + ignoreInUse, _ = strconv.ParseBool(forceStr) + } + + // Disallow remote tiers with internal storage class names + switch cfg.Name { + case storageclass.STANDARD, storageclass.RRS: + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, errTierReservedName), r.URL) + return + } + + // Refresh from the disk in case we had missed notifications about edits from peers. + if err := globalTierConfigMgr.Reload(ctx, objAPI); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + err = globalTierConfigMgr.Add(ctx, cfg, ignoreInUse) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + err = globalTierConfigMgr.Save(ctx, objAPI) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + globalNotificationSys.LoadTransitionTierConfig(ctx) + + writeSuccessNoContent(w) +} + +func (api adminAPIHandlers) ListTierHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objAPI, _ := validateAdminReq(ctx, w, r, policy.ListTierAction) + if objAPI == nil { + return + } + + tiers := globalTierConfigMgr.ListTiers() + data, err := json.Marshal(tiers) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + w.Header().Set(tierCfgRefreshAtHdr, globalTierConfigMgr.refreshedAt().String()) + writeSuccessResponseJSON(w, data) +} + +func (api adminAPIHandlers) EditTierHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objAPI, cred := validateAdminReq(ctx, w, r, policy.SetTierAction) + if objAPI == nil { + return + } + vars := mux.Vars(r) + scName := vars["tier"] + + password := cred.SecretKey + reqBytes, err := madmin.DecryptData(password, io.LimitReader(r.Body, r.ContentLength)) + if err != nil { + writeErrorResponseJSON(ctx, w, errorCodes.ToAPIErrWithErr(ErrAdminConfigBadJSON, err), r.URL) + return + } + + var creds madmin.TierCreds + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err := json.Unmarshal(reqBytes, &creds); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + // Refresh from the disk in case we had missed notifications about edits from peers. + if err := globalTierConfigMgr.Reload(ctx, objAPI); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + if err := globalTierConfigMgr.Edit(ctx, scName, creds); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + if err := globalTierConfigMgr.Save(ctx, objAPI); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + globalNotificationSys.LoadTransitionTierConfig(ctx) + + writeSuccessNoContent(w) +} + +func (api adminAPIHandlers) RemoveTierHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objAPI, _ := validateAdminReq(ctx, w, r, policy.SetTierAction) + if objAPI == nil { + return + } + + vars := mux.Vars(r) + tier := vars["tier"] + force := r.Form.Get("force") == "true" + + if err := globalTierConfigMgr.Reload(ctx, objAPI); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + if err := globalTierConfigMgr.Remove(ctx, tier, force); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + if err := globalTierConfigMgr.Save(ctx, objAPI); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + globalNotificationSys.LoadTransitionTierConfig(ctx) + + writeSuccessNoContent(w) +} + +func (api adminAPIHandlers) VerifyTierHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objAPI, _ := validateAdminReq(ctx, w, r, policy.ListTierAction) + if objAPI == nil { + return + } + + vars := mux.Vars(r) + tier := vars["tier"] + if err := globalTierConfigMgr.Verify(ctx, tier); err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + writeSuccessNoContent(w) +} + +func (api adminAPIHandlers) TierStatsHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + objAPI, _ := validateAdminReq(ctx, w, r, policy.ListTierAction) + if objAPI == nil { + return + } + + dui, err := loadDataUsageFromBackend(ctx, objAPI) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + + tierStats := dui.tierStats() + dailyStats := globalNotificationSys.GetLastDayTierStats(ctx) + tierStats = dailyStats.addToTierInfo(tierStats) + + data, err := json.Marshal(tierStats) + if err != nil { + writeErrorResponseJSON(ctx, w, toAdminAPIErr(ctx, err), r.URL) + return + } + writeSuccessResponseJSON(w, data) +} diff --git a/cmd/tier-last-day-stats.go b/cmd/tier-last-day-stats.go new file mode 100644 index 00000000..c280ecbd --- /dev/null +++ b/cmd/tier-last-day-stats.go @@ -0,0 +1,120 @@ +// Copyright (c) 2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "time" + + "github.com/minio/madmin-go/v3" +) + +//go:generate msgp -file=$GOFILE -unexported + +type lastDayTierStats struct { + Bins [24]tierStats + UpdatedAt time.Time +} + +func (l *lastDayTierStats) addStats(ts tierStats) { + now := time.Now() + l.forwardTo(now) + + nowIdx := now.Hour() + l.Bins[nowIdx] = l.Bins[nowIdx].add(ts) +} + +// forwardTo moves time to t, clearing entries between last update and t. +func (l *lastDayTierStats) forwardTo(t time.Time) { + if t.IsZero() { + t = time.Now() + } + + since := t.Sub(l.UpdatedAt).Hours() + // within the hour since l.UpdatedAt + if since < 1 { + return + } + + idx, lastIdx := t.Hour(), l.UpdatedAt.Hour() + + l.UpdatedAt = t // update to the latest time index + + if since >= 24 { + l.Bins = [24]tierStats{} + return + } + + for lastIdx != idx { + lastIdx = (lastIdx + 1) % 24 + l.Bins[lastIdx] = tierStats{} + } +} + +func (l *lastDayTierStats) clone() lastDayTierStats { + clone := lastDayTierStats{ + UpdatedAt: l.UpdatedAt, + } + copy(clone.Bins[:], l.Bins[:]) + return clone +} + +func (l lastDayTierStats) merge(m lastDayTierStats) (merged lastDayTierStats) { + cl := l.clone() + cm := m.clone() + + if cl.UpdatedAt.After(cm.UpdatedAt) { + cm.forwardTo(cl.UpdatedAt) + merged.UpdatedAt = cl.UpdatedAt + } else { + cl.forwardTo(cm.UpdatedAt) + merged.UpdatedAt = cm.UpdatedAt + } + + for i := range cl.Bins { + merged.Bins[i] = cl.Bins[i].add(cm.Bins[i]) + } + + return merged +} + +// DailyAllTierStats is used to aggregate last day tier stats across MinIO servers +type DailyAllTierStats map[string]lastDayTierStats + +func (l DailyAllTierStats) merge(m DailyAllTierStats) { + for tier, st := range m { + l[tier] = l[tier].merge(st) + } +} + +func (l DailyAllTierStats) addToTierInfo(tierInfos []madmin.TierInfo) []madmin.TierInfo { + for i := range tierInfos { + lst, ok := l[tierInfos[i].Name] + if !ok { + continue + } + for hr, st := range lst.Bins { + tierInfos[i].DailyStats.Bins[hr] = madmin.TierStats{ + TotalSize: st.TotalSize, + NumVersions: st.NumVersions, + NumObjects: st.NumObjects, + } + } + tierInfos[i].DailyStats.UpdatedAt = lst.UpdatedAt + } + return tierInfos +} diff --git a/cmd/tier-last-day-stats_gen.go b/cmd/tier-last-day-stats_gen.go new file mode 100644 index 00000000..24e4dde7 --- /dev/null +++ b/cmd/tier-last-day-stats_gen.go @@ -0,0 +1,413 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *DailyAllTierStats) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0004 uint32 + zb0004, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if (*z) == nil { + (*z) = make(DailyAllTierStats, zb0004) + } else if len((*z)) > 0 { + clear((*z)) + } + var field []byte + _ = field + for zb0004 > 0 { + zb0004-- + var zb0001 string + zb0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0002 lastDayTierStats + var zb0005 uint32 + zb0005, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, zb0001) + return + } + for zb0005 > 0 { + zb0005-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, zb0001) + return + } + switch msgp.UnsafeString(field) { + case "Bins": + var zb0006 uint32 + zb0006, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, zb0001, "Bins") + return + } + if zb0006 != uint32(24) { + err = msgp.ArrayError{Wanted: uint32(24), Got: zb0006} + return + } + for zb0003 := range zb0002.Bins { + err = zb0002.Bins[zb0003].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, zb0001, "Bins", zb0003) + return + } + } + case "UpdatedAt": + zb0002.UpdatedAt, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, zb0001, "UpdatedAt") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, zb0001) + return + } + } + } + (*z)[zb0001] = zb0002 + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z DailyAllTierStats) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteMapHeader(uint32(len(z))) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0007, zb0008 := range z { + err = en.WriteString(zb0007) + if err != nil { + err = msgp.WrapError(err) + return + } + // map header, size 2 + // write "Bins" + err = en.Append(0x82, 0xa4, 0x42, 0x69, 0x6e, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(24)) + if err != nil { + err = msgp.WrapError(err, zb0007, "Bins") + return + } + for zb0009 := range zb0008.Bins { + err = zb0008.Bins[zb0009].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, zb0007, "Bins", zb0009) + return + } + } + // write "UpdatedAt" + err = en.Append(0xa9, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + if err != nil { + return + } + err = en.WriteTime(zb0008.UpdatedAt) + if err != nil { + err = msgp.WrapError(err, zb0007, "UpdatedAt") + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z DailyAllTierStats) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendMapHeader(o, uint32(len(z))) + for zb0007, zb0008 := range z { + o = msgp.AppendString(o, zb0007) + // map header, size 2 + // string "Bins" + o = append(o, 0x82, 0xa4, 0x42, 0x69, 0x6e, 0x73) + o = msgp.AppendArrayHeader(o, uint32(24)) + for zb0009 := range zb0008.Bins { + o, err = zb0008.Bins[zb0009].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, zb0007, "Bins", zb0009) + return + } + } + // string "UpdatedAt" + o = append(o, 0xa9, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + o = msgp.AppendTime(o, zb0008.UpdatedAt) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *DailyAllTierStats) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0004 uint32 + zb0004, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if (*z) == nil { + (*z) = make(DailyAllTierStats, zb0004) + } else if len((*z)) > 0 { + clear((*z)) + } + var field []byte + _ = field + for zb0004 > 0 { + var zb0002 lastDayTierStats + zb0004-- + var zb0001 string + zb0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0005 uint32 + zb0005, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, zb0001) + return + } + for zb0005 > 0 { + zb0005-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, zb0001) + return + } + switch msgp.UnsafeString(field) { + case "Bins": + var zb0006 uint32 + zb0006, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, zb0001, "Bins") + return + } + if zb0006 != uint32(24) { + err = msgp.ArrayError{Wanted: uint32(24), Got: zb0006} + return + } + for zb0003 := range zb0002.Bins { + bts, err = zb0002.Bins[zb0003].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, zb0001, "Bins", zb0003) + return + } + } + case "UpdatedAt": + zb0002.UpdatedAt, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, zb0001, "UpdatedAt") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, zb0001) + return + } + } + } + (*z)[zb0001] = zb0002 + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z DailyAllTierStats) Msgsize() (s int) { + s = msgp.MapHeaderSize + if z != nil { + for zb0007, zb0008 := range z { + _ = zb0008 + s += msgp.StringPrefixSize + len(zb0007) + 1 + 5 + msgp.ArrayHeaderSize + for zb0009 := range zb0008.Bins { + s += zb0008.Bins[zb0009].Msgsize() + } + s += 10 + msgp.TimeSize + } + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *lastDayTierStats) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Bins": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Bins") + return + } + if zb0002 != uint32(24) { + err = msgp.ArrayError{Wanted: uint32(24), Got: zb0002} + return + } + for za0001 := range z.Bins { + err = z.Bins[za0001].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Bins", za0001) + return + } + } + case "UpdatedAt": + z.UpdatedAt, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "UpdatedAt") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *lastDayTierStats) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "Bins" + err = en.Append(0x82, 0xa4, 0x42, 0x69, 0x6e, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(24)) + if err != nil { + err = msgp.WrapError(err, "Bins") + return + } + for za0001 := range z.Bins { + err = z.Bins[za0001].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Bins", za0001) + return + } + } + // write "UpdatedAt" + err = en.Append(0xa9, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + if err != nil { + return + } + err = en.WriteTime(z.UpdatedAt) + if err != nil { + err = msgp.WrapError(err, "UpdatedAt") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *lastDayTierStats) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "Bins" + o = append(o, 0x82, 0xa4, 0x42, 0x69, 0x6e, 0x73) + o = msgp.AppendArrayHeader(o, uint32(24)) + for za0001 := range z.Bins { + o, err = z.Bins[za0001].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Bins", za0001) + return + } + } + // string "UpdatedAt" + o = append(o, 0xa9, 0x55, 0x70, 0x64, 0x61, 0x74, 0x65, 0x64, 0x41, 0x74) + o = msgp.AppendTime(o, z.UpdatedAt) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *lastDayTierStats) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Bins": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Bins") + return + } + if zb0002 != uint32(24) { + err = msgp.ArrayError{Wanted: uint32(24), Got: zb0002} + return + } + for za0001 := range z.Bins { + bts, err = z.Bins[za0001].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Bins", za0001) + return + } + } + case "UpdatedAt": + z.UpdatedAt, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UpdatedAt") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *lastDayTierStats) Msgsize() (s int) { + s = 1 + 5 + msgp.ArrayHeaderSize + for za0001 := range z.Bins { + s += z.Bins[za0001].Msgsize() + } + s += 10 + msgp.TimeSize + return +} diff --git a/cmd/tier-last-day-stats_gen_test.go b/cmd/tier-last-day-stats_gen_test.go new file mode 100644 index 00000000..590c2c4c --- /dev/null +++ b/cmd/tier-last-day-stats_gen_test.go @@ -0,0 +1,236 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalDailyAllTierStats(t *testing.T) { + v := DailyAllTierStats{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgDailyAllTierStats(b *testing.B) { + v := DailyAllTierStats{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgDailyAllTierStats(b *testing.B) { + v := DailyAllTierStats{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalDailyAllTierStats(b *testing.B) { + v := DailyAllTierStats{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeDailyAllTierStats(t *testing.T) { + v := DailyAllTierStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeDailyAllTierStats Msgsize() is inaccurate") + } + + vn := DailyAllTierStats{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeDailyAllTierStats(b *testing.B) { + v := DailyAllTierStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeDailyAllTierStats(b *testing.B) { + v := DailyAllTierStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshallastDayTierStats(t *testing.T) { + v := lastDayTierStats{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsglastDayTierStats(b *testing.B) { + v := lastDayTierStats{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsglastDayTierStats(b *testing.B) { + v := lastDayTierStats{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshallastDayTierStats(b *testing.B) { + v := lastDayTierStats{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodelastDayTierStats(t *testing.T) { + v := lastDayTierStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodelastDayTierStats Msgsize() is inaccurate") + } + + vn := lastDayTierStats{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodelastDayTierStats(b *testing.B) { + v := lastDayTierStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodelastDayTierStats(b *testing.B) { + v := lastDayTierStats{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/tier-sweeper.go b/cmd/tier-sweeper.go new file mode 100644 index 00000000..189c0666 --- /dev/null +++ b/cmd/tier-sweeper.go @@ -0,0 +1,151 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + + "github.com/minio/minio/internal/bucket/lifecycle" +) + +// objSweeper determines if a transitioned object needs to be removed from the remote tier. +// A typical usage would be like, +// os := newObjSweeper(bucket, object) +// // Perform a ObjectLayer.GetObjectInfo to fetch object version information +// goiOpts := os.GetOpts() +// gerr := objAPI.GetObjectInfo(ctx, bucket, object, goiOpts) +// +// if gerr == nil { +// os.SetTransitionState(goi) +// } +// +// // After the overwriting object operation is complete. +// +// if jentry, ok := os.ShouldRemoveRemoteObject(); ok { +// err := globalTierJournal.AddEntry(jentry) +// logger.LogIf(ctx, err) +// } +type objSweeper struct { + Object string + Bucket string + VersionID string // version ID set by application, applies only to DeleteObject and DeleteObjects APIs + Versioned bool + Suspended bool + TransitionStatus string + TransitionTier string + TransitionVersionID string + RemoteObject string +} + +// newObjSweeper returns an objSweeper for a given bucket and object. +// It initializes the versioning information using bucket name. +func newObjSweeper(bucket, object string) *objSweeper { + return &objSweeper{ + Object: object, + Bucket: bucket, + } +} + +// WithVersion sets the version ID from v +func (os *objSweeper) WithVersion(vid string) *objSweeper { + os.VersionID = vid + return os +} + +// WithVersioning sets bucket versioning for sweeper. +func (os *objSweeper) WithVersioning(versioned, suspended bool) *objSweeper { + os.Versioned = versioned + os.Suspended = suspended + return os +} + +// GetOpts returns ObjectOptions to fetch the object version that may be +// overwritten or deleted depending on bucket versioning status. +func (os *objSweeper) GetOpts() ObjectOptions { + opts := ObjectOptions{ + VersionID: os.VersionID, + Versioned: os.Versioned, + VersionSuspended: os.Suspended, + } + if os.Suspended && os.VersionID == "" { + opts.VersionID = nullVersionID + } + return opts +} + +// SetTransitionState sets ILM transition related information from given info. +func (os *objSweeper) SetTransitionState(info TransitionedObject) { + os.TransitionTier = info.Tier + os.TransitionStatus = info.Status + os.RemoteObject = info.Name + os.TransitionVersionID = info.VersionID +} + +// shouldRemoveRemoteObject determines if a transitioned object should be +// removed from remote tier. If remote object is to be deleted, returns the +// corresponding tier deletion journal entry and true. Otherwise returns empty +// jentry value and false. +func (os *objSweeper) shouldRemoveRemoteObject() (jentry, bool) { + if os.TransitionStatus != lifecycle.TransitionComplete { + return jentry{}, false + } + + // 1. If bucket versioning is disabled, remove the remote object. + // 2. If bucket versioning is suspended and + // a. version id is specified, remove its remote object. + // b. version id is not specified, remove null version's remote object if it exists. + // 3. If bucket versioning is enabled and + // a. version id is specified, remove its remote object. + // b. version id is not specified, nothing to be done (a delete marker is added). + delTier := false + switch { + case !os.Versioned, os.Suspended: // 1, 2.a, 2.b + delTier = true + case os.Versioned && os.VersionID != "": // 3.a + delTier = true + } + if delTier { + return jentry{ + ObjName: os.RemoteObject, + VersionID: os.TransitionVersionID, + TierName: os.TransitionTier, + }, true + } + return jentry{}, false +} + +// Sweep removes the transitioned object if it's no longer referred to. +func (os *objSweeper) Sweep() { + if je, ok := os.shouldRemoveRemoteObject(); ok { + globalExpiryState.enqueueTierJournalEntry(je) + } +} + +type jentry struct { + ObjName string + VersionID string + TierName string +} + +func deleteObjectFromRemoteTier(ctx context.Context, objName, rvID, tierName string) error { + w, err := globalTierConfigMgr.getDriver(ctx, tierName) + if err != nil { + return err + } + return w.Remove(ctx, objName, remoteVersionID(rvID)) +} diff --git a/cmd/tier.go b/cmd/tier.go new file mode 100644 index 00000000..b5309526 --- /dev/null +++ b/cmd/tier.go @@ -0,0 +1,594 @@ +// Copyright (c) 2015-2024 MinIO, Inc +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/binary" + "errors" + "fmt" + "maps" + "math/rand" + "net/http" + "path" + "strings" + "sync" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/hash" + "github.com/minio/minio/internal/kms" + "github.com/prometheus/client_golang/prometheus" +) + +//go:generate msgp -file $GOFILE + +var ( + errTierMissingCredentials = AdminError{ + Code: "XMinioAdminTierMissingCredentials", + Message: "Specified remote credentials are empty", + StatusCode: http.StatusForbidden, + } + + errTierBackendInUse = AdminError{ + Code: "XMinioAdminTierBackendInUse", + Message: "Specified remote tier is already in use", + StatusCode: http.StatusConflict, + } + + errTierTypeUnsupported = AdminError{ + Code: "XMinioAdminTierTypeUnsupported", + Message: "Specified tier type is unsupported", + StatusCode: http.StatusBadRequest, + } + + errTierBackendNotEmpty = AdminError{ + Code: "XMinioAdminTierBackendNotEmpty", + Message: "Specified remote backend is not empty", + StatusCode: http.StatusBadRequest, + } + + errTierInvalidConfig = AdminError{ + Code: "XMinioAdminTierInvalidConfig", + Message: "Unable to setup remote tier, check tier configuration", + StatusCode: http.StatusBadRequest, + } +) + +const ( + tierConfigFile = "tier-config.bin" + tierConfigFormat = 1 + tierConfigV1 = 1 + tierConfigVersion = 2 +) + +// tierConfigPath refers to remote tier config object name +var tierConfigPath = path.Join(minioConfigPrefix, tierConfigFile) + +const tierCfgRefreshAtHdr = "X-MinIO-TierCfg-RefreshedAt" + +// TierConfigMgr holds the collection of remote tiers configured in this deployment. +type TierConfigMgr struct { + sync.RWMutex `msg:"-"` + drivercache map[string]WarmBackend `msg:"-"` + + Tiers map[string]madmin.TierConfig `json:"tiers"` + lastRefreshedAt time.Time `msg:"-"` +} + +type tierMetrics struct { + sync.RWMutex // protects requestsCount only + requestsCount map[string]struct { + success int64 + failure int64 + } + histogram *prometheus.HistogramVec +} + +var globalTierMetrics = tierMetrics{ + requestsCount: make(map[string]struct { + success int64 + failure int64 + }), + histogram: prometheus.NewHistogramVec(prometheus.HistogramOpts{ + Name: "tier_ttlb_seconds", + Help: "Time taken by requests served by warm tier", + Buckets: []float64{0.01, 0.1, 1, 2, 5, 10, 60, 5 * 60, 15 * 60, 30 * 60}, + }, []string{"tier"}), +} + +func (t *tierMetrics) Observe(tier string, dur time.Duration) { + t.histogram.With(prometheus.Labels{"tier": tier}).Observe(dur.Seconds()) +} + +func (t *tierMetrics) logSuccess(tier string) { + t.Lock() + defer t.Unlock() + + stat := t.requestsCount[tier] + stat.success++ + t.requestsCount[tier] = stat +} + +func (t *tierMetrics) logFailure(tier string) { + t.Lock() + defer t.Unlock() + + stat := t.requestsCount[tier] + stat.failure++ + t.requestsCount[tier] = stat +} + +var ( + // {minio_node}_{tier}_{ttlb_seconds_distribution} + tierTTLBMD = MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: tierSubsystem, + Name: ttlbDistribution, + Help: "Distribution of time to last byte for objects downloaded from warm tier", + Type: gaugeMetric, + } + + // {minio_node}_{tier}_{requests_success} + tierRequestsSuccessMD = MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: tierSubsystem, + Name: tierRequestsSuccess, + Help: "Number of requests to download object from warm tier that were successful", + Type: counterMetric, + } + // {minio_node}_{tier}_{requests_failure} + tierRequestsFailureMD = MetricDescription{ + Namespace: nodeMetricNamespace, + Subsystem: tierSubsystem, + Name: tierRequestsFailure, + Help: "Number of requests to download object from warm tier that failed", + Type: counterMetric, + } +) + +func (t *tierMetrics) Report() []MetricV2 { + metrics := getHistogramMetrics(t.histogram, tierTTLBMD, true, true) + t.RLock() + defer t.RUnlock() + for tier, stat := range t.requestsCount { + metrics = append(metrics, MetricV2{ + Description: tierRequestsSuccessMD, + Value: float64(stat.success), + VariableLabels: map[string]string{"tier": tier}, + }) + metrics = append(metrics, MetricV2{ + Description: tierRequestsFailureMD, + Value: float64(stat.failure), + VariableLabels: map[string]string{"tier": tier}, + }) + } + return metrics +} + +func (config *TierConfigMgr) refreshedAt() time.Time { + config.RLock() + defer config.RUnlock() + return config.lastRefreshedAt +} + +// IsTierValid returns true if there exists a remote tier by name tierName, +// otherwise returns false. +func (config *TierConfigMgr) IsTierValid(tierName string) bool { + config.RLock() + defer config.RUnlock() + _, valid := config.isTierNameInUse(tierName) + return valid +} + +// isTierNameInUse returns tier type and true if there exists a remote tier by +// name tierName, otherwise returns madmin.Unsupported and false. N B this +// function is meant for internal use, where the caller is expected to take +// appropriate locks. +func (config *TierConfigMgr) isTierNameInUse(tierName string) (madmin.TierType, bool) { + if t, ok := config.Tiers[tierName]; ok { + return t.Type, true + } + return madmin.Unsupported, false +} + +// Add adds tier to config if it passes all validations. +func (config *TierConfigMgr) Add(ctx context.Context, tier madmin.TierConfig, ignoreInUse bool) error { + config.Lock() + defer config.Unlock() + + // check if tier name is in all caps + tierName := tier.Name + if tierName != strings.ToUpper(tierName) { + return errTierNameNotUppercase + } + + // check if tier name already in use + if _, exists := config.isTierNameInUse(tierName); exists { + return errTierAlreadyExists + } + + d, err := newWarmBackend(ctx, tier, true) + if err != nil { + return err + } + + if !ignoreInUse { + // Check if warmbackend is in use by other MinIO tenants + inUse, err := d.InUse(ctx) + if err != nil { + return err + } + if inUse { + return errTierBackendInUse + } + } + + config.Tiers[tierName] = tier + config.drivercache[tierName] = d + + return nil +} + +// Remove removes tier if it is empty. +func (config *TierConfigMgr) Remove(ctx context.Context, tier string, force bool) error { + d, err := config.getDriver(ctx, tier) + if err != nil { + if errors.Is(err, errTierNotFound) { + return nil + } + return err + } + if !force { + if inuse, err := d.InUse(ctx); err != nil { + return err + } else if inuse { + return errTierBackendNotEmpty + } + } + config.Lock() + delete(config.Tiers, tier) + delete(config.drivercache, tier) + config.Unlock() + return nil +} + +// Verify verifies if tier's config is valid by performing all supported +// operations on the corresponding warmbackend. +func (config *TierConfigMgr) Verify(ctx context.Context, tier string) error { + d, err := config.getDriver(ctx, tier) + if err != nil { + return err + } + return checkWarmBackend(ctx, d) +} + +// Empty returns if tier targets are empty +func (config *TierConfigMgr) Empty() bool { + if config == nil { + return true + } + return len(config.ListTiers()) == 0 +} + +// TierType returns the type of tier +func (config *TierConfigMgr) TierType(name string) string { + config.RLock() + defer config.RUnlock() + + cfg, ok := config.Tiers[name] + if !ok { + return "internal" + } + return cfg.Type.String() +} + +// ListTiers lists remote tiers configured in this deployment. +func (config *TierConfigMgr) ListTiers() []madmin.TierConfig { + if config == nil { + return nil + } + + config.RLock() + defer config.RUnlock() + + var tierCfgs []madmin.TierConfig + for _, tier := range config.Tiers { + // This makes a local copy of tier config before + // passing a reference to it. + tier := tier.Clone() + tierCfgs = append(tierCfgs, tier) + } + return tierCfgs +} + +// Edit replaces the credentials of the remote tier specified by tierName with creds. +func (config *TierConfigMgr) Edit(ctx context.Context, tierName string, creds madmin.TierCreds) error { + config.Lock() + defer config.Unlock() + + // check if tier by this name exists + tierType, exists := config.isTierNameInUse(tierName) + if !exists { + return errTierNotFound + } + + cfg := config.Tiers[tierName] + switch tierType { + case madmin.S3: + if creds.AWSRole { + cfg.S3.AWSRole = true + } + if creds.AWSRoleWebIdentityTokenFile != "" && creds.AWSRoleARN != "" { + cfg.S3.AWSRoleARN = creds.AWSRoleARN + cfg.S3.AWSRoleWebIdentityTokenFile = creds.AWSRoleWebIdentityTokenFile + } + if creds.AccessKey != "" && creds.SecretKey != "" { + cfg.S3.AccessKey = creds.AccessKey + cfg.S3.SecretKey = creds.SecretKey + } + case madmin.Azure: + if creds.SecretKey != "" { + cfg.Azure.AccountKey = creds.SecretKey + } + if creds.AzSP.TenantID != "" { + cfg.Azure.SPAuth.TenantID = creds.AzSP.TenantID + } + if creds.AzSP.ClientID != "" { + cfg.Azure.SPAuth.ClientID = creds.AzSP.ClientID + } + if creds.AzSP.ClientSecret != "" { + cfg.Azure.SPAuth.ClientSecret = creds.AzSP.ClientSecret + } + case madmin.GCS: + if creds.CredsJSON == nil { + return errTierMissingCredentials + } + cfg.GCS.Creds = base64.URLEncoding.EncodeToString(creds.CredsJSON) + case madmin.MinIO: + if creds.AccessKey == "" || creds.SecretKey == "" { + return errTierMissingCredentials + } + cfg.MinIO.AccessKey = creds.AccessKey + cfg.MinIO.SecretKey = creds.SecretKey + } + + d, err := newWarmBackend(ctx, cfg, true) + if err != nil { + return err + } + config.Tiers[tierName] = cfg + config.drivercache[tierName] = d + return nil +} + +// Bytes returns msgpack encoded config with format and version headers. +func (config *TierConfigMgr) Bytes() ([]byte, error) { + config.RLock() + defer config.RUnlock() + data := make([]byte, 4, config.Msgsize()+4) + + // Initialize the header. + binary.LittleEndian.PutUint16(data[0:2], tierConfigFormat) + binary.LittleEndian.PutUint16(data[2:4], tierConfigVersion) + + // Marshal the tier config + return config.MarshalMsg(data) +} + +// getDriver returns a warmBackend interface object initialized with remote tier config matching tierName +func (config *TierConfigMgr) getDriver(ctx context.Context, tierName string) (d WarmBackend, err error) { + config.Lock() + defer config.Unlock() + + var ok bool + // Lookup in-memory drivercache + d, ok = config.drivercache[tierName] + if ok { + return d, nil + } + + // Initialize driver from tier config matching tierName + t, ok := config.Tiers[tierName] + if !ok { + return nil, errTierNotFound + } + d, err = newWarmBackend(ctx, t, false) + if err != nil { + return nil, err + } + config.drivercache[tierName] = d + return d, nil +} + +// configReader returns a PutObjReader and ObjectOptions needed to save config +// using a PutObject API. PutObjReader encrypts json encoded tier configurations +// if KMS is enabled, otherwise simply yields the json encoded bytes as is. +// Similarly, ObjectOptions value depends on KMS' status. +func (config *TierConfigMgr) configReader(ctx context.Context) (*PutObjReader, *ObjectOptions, error) { + b, err := config.Bytes() + if err != nil { + return nil, nil, err + } + + payloadSize := int64(len(b)) + br := bytes.NewReader(b) + hr, err := hash.NewReader(ctx, br, payloadSize, "", "", payloadSize) + if err != nil { + return nil, nil, err + } + if GlobalKMS == nil { + return NewPutObjReader(hr), &ObjectOptions{MaxParity: true}, nil + } + + // Note: Local variables with names ek, oek, etc are named inline with + // acronyms defined here - + // https://github.com/minio/minio/blob/master/docs/security/README.md#acronyms + + // Encrypt json encoded tier configurations + metadata := make(map[string]string) + encBr, oek, err := newEncryptReader(context.Background(), hr, crypto.S3, "", nil, minioMetaBucket, tierConfigPath, metadata, kms.Context{}) + if err != nil { + return nil, nil, err + } + + info := ObjectInfo{ + Size: payloadSize, + } + encSize := info.EncryptedSize() + encHr, err := hash.NewReader(ctx, encBr, encSize, "", "", encSize) + if err != nil { + return nil, nil, err + } + + pReader, err := NewPutObjReader(hr).WithEncryption(encHr, &oek) + if err != nil { + return nil, nil, err + } + opts := &ObjectOptions{ + UserDefined: metadata, + MTime: UTCNow(), + MaxParity: true, + } + + return pReader, opts, nil +} + +// Reload updates config by reloading remote tier config from config store. +func (config *TierConfigMgr) Reload(ctx context.Context, objAPI ObjectLayer) error { + newConfig, err := loadTierConfig(ctx, objAPI) + + config.Lock() + defer config.Unlock() + + switch err { + case nil: + break + case errConfigNotFound: // nothing to reload + // To maintain the invariance that lastRefreshedAt records the + // timestamp of last successful refresh + config.lastRefreshedAt = UTCNow() + return nil + default: + return err + } + + // Reset drivercache built using current config + clear(config.drivercache) + // Remove existing tier configs + clear(config.Tiers) + // Copy over the new tier configs + maps.Copy(config.Tiers, newConfig.Tiers) + config.lastRefreshedAt = UTCNow() + return nil +} + +// Save saves tier configuration onto objAPI +func (config *TierConfigMgr) Save(ctx context.Context, objAPI ObjectLayer) error { + if objAPI == nil { + return errServerNotInitialized + } + + pr, opts, err := globalTierConfigMgr.configReader(ctx) + if err != nil { + return err + } + + _, err = objAPI.PutObject(ctx, minioMetaBucket, tierConfigPath, pr, *opts) + return err +} + +// NewTierConfigMgr - creates new tier configuration manager, +func NewTierConfigMgr() *TierConfigMgr { + return &TierConfigMgr{ + drivercache: make(map[string]WarmBackend), + Tiers: make(map[string]madmin.TierConfig), + } +} + +func (config *TierConfigMgr) refreshTierConfig(ctx context.Context, objAPI ObjectLayer) { + const tierCfgRefresh = 15 * time.Minute + r := rand.New(rand.NewSource(time.Now().UnixNano())) + randInterval := func() time.Duration { + return time.Duration(r.Float64() * 5 * float64(time.Second)) + } + + // To avoid all MinIO nodes reading the tier config object at the same + // time. + t := time.NewTimer(tierCfgRefresh + randInterval()) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + err := config.Reload(ctx, objAPI) + if err != nil { + tierLogIf(ctx, err) + } + } + t.Reset(tierCfgRefresh + randInterval()) + } +} + +// loadTierConfig loads remote tier configuration from objAPI. +func loadTierConfig(ctx context.Context, objAPI ObjectLayer) (*TierConfigMgr, error) { + if objAPI == nil { + return nil, errServerNotInitialized + } + + data, err := readConfig(ctx, objAPI, tierConfigPath) + if err != nil { + return nil, err + } + + if len(data) <= 4 { + return nil, errors.New("tierConfigInit: no data") + } + + // Read header + switch format := binary.LittleEndian.Uint16(data[0:2]); format { + case tierConfigFormat: + default: + return nil, fmt.Errorf("tierConfigInit: unknown format: %d", format) + } + + cfg := NewTierConfigMgr() + switch version := binary.LittleEndian.Uint16(data[2:4]); version { + case tierConfigV1, tierConfigVersion: + if _, decErr := cfg.UnmarshalMsg(data[4:]); decErr != nil { + return nil, decErr + } + default: + return nil, fmt.Errorf("tierConfigInit: unknown version: %d", version) + } + + return cfg, nil +} + +// Init initializes tier configuration reading from objAPI +func (config *TierConfigMgr) Init(ctx context.Context, objAPI ObjectLayer) error { + err := config.Reload(ctx, objAPI) + if globalIsDistErasure { + go config.refreshTierConfig(ctx, objAPI) + } + return err +} diff --git a/cmd/tier_gen.go b/cmd/tier_gen.go new file mode 100644 index 00000000..38efbcad --- /dev/null +++ b/cmd/tier_gen.go @@ -0,0 +1,181 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/minio/madmin-go/v3" + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *TierConfigMgr) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Tiers": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Tiers") + return + } + if z.Tiers == nil { + z.Tiers = make(map[string]madmin.TierConfig, zb0002) + } else if len(z.Tiers) > 0 { + clear(z.Tiers) + } + for zb0002 > 0 { + zb0002-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Tiers") + return + } + var za0002 madmin.TierConfig + err = za0002.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001) + return + } + z.Tiers[za0001] = za0002 + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *TierConfigMgr) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "Tiers" + err = en.Append(0x81, 0xa5, 0x54, 0x69, 0x65, 0x72, 0x73) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.Tiers))) + if err != nil { + err = msgp.WrapError(err, "Tiers") + return + } + for za0001, za0002 := range z.Tiers { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "Tiers") + return + } + err = za0002.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *TierConfigMgr) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "Tiers" + o = append(o, 0x81, 0xa5, 0x54, 0x69, 0x65, 0x72, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.Tiers))) + for za0001, za0002 := range z.Tiers { + o = msgp.AppendString(o, za0001) + o, err = za0002.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001) + return + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *TierConfigMgr) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Tiers": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tiers") + return + } + if z.Tiers == nil { + z.Tiers = make(map[string]madmin.TierConfig, zb0002) + } else if len(z.Tiers) > 0 { + clear(z.Tiers) + } + for zb0002 > 0 { + var za0002 madmin.TierConfig + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Tiers") + return + } + bts, err = za0002.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Tiers", za0001) + return + } + z.Tiers[za0001] = za0002 + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *TierConfigMgr) Msgsize() (s int) { + s = 1 + 6 + msgp.MapHeaderSize + if z.Tiers != nil { + for za0001, za0002 := range z.Tiers { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + za0002.Msgsize() + } + } + return +} diff --git a/cmd/tier_gen_test.go b/cmd/tier_gen_test.go new file mode 100644 index 00000000..f6bdf5cf --- /dev/null +++ b/cmd/tier_gen_test.go @@ -0,0 +1,123 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalTierConfigMgr(t *testing.T) { + v := TierConfigMgr{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgTierConfigMgr(b *testing.B) { + v := TierConfigMgr{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgTierConfigMgr(b *testing.B) { + v := TierConfigMgr{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalTierConfigMgr(b *testing.B) { + v := TierConfigMgr{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeTierConfigMgr(t *testing.T) { + v := TierConfigMgr{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeTierConfigMgr Msgsize() is inaccurate") + } + + vn := TierConfigMgr{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeTierConfigMgr(b *testing.B) { + v := TierConfigMgr{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeTierConfigMgr(b *testing.B) { + v := TierConfigMgr{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/tier_test.go b/cmd/tier_test.go new file mode 100644 index 00000000..2b2345c2 --- /dev/null +++ b/cmd/tier_test.go @@ -0,0 +1,52 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "testing" + "time" +) + +func TestTierMetrics(t *testing.T) { + tier := "WARM-1" + globalTierMetrics.Observe(tier, 200*time.Millisecond) + expSuccess := 10 + expFailure := 5 + for range expSuccess { + globalTierMetrics.logSuccess(tier) + } + for range expFailure { + globalTierMetrics.logFailure(tier) + } + metrics := globalTierMetrics.Report() + var succ, fail float64 + for _, metric := range metrics { + switch metric.Description.Name { + case tierRequestsSuccess: + succ += metric.Value + case tierRequestsFailure: + fail += metric.Value + } + } + if int(succ) != expSuccess { + t.Fatalf("Expected %d successes but got %f", expSuccess, succ) + } + if int(fail) != expFailure { + t.Fatalf("Expected %d failures but got %f", expFailure, fail) + } +} diff --git a/cmd/typed-errors.go b/cmd/typed-errors.go new file mode 100644 index 00000000..58f2590e --- /dev/null +++ b/cmd/typed-errors.go @@ -0,0 +1,133 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "errors" +) + +// errInvalidArgument means that input argument is invalid. +var errInvalidArgument = errors.New("Invalid arguments specified") + +// errMethodNotAllowed means that method is not allowed. +var errMethodNotAllowed = errors.New("Method not allowed") + +// errSignatureMismatch means signature did not match. +var errSignatureMismatch = errors.New("Signature does not match") + +// When upload object size is greater than 5G in a single PUT/POST operation. +var errDataTooLarge = errors.New("Object size larger than allowed limit") + +// When upload object size is less than what was expected. +var errDataTooSmall = errors.New("Object size smaller than expected") + +// errServerNotInitialized - server not initialized. +var errServerNotInitialized = errors.New("Server not initialized, please try again") + +// errRPCAPIVersionUnsupported - unsupported rpc API version. +var errRPCAPIVersionUnsupported = errors.New("Unsupported rpc API version") + +// errServerTimeMismatch - server times are too far apart. +var errServerTimeMismatch = errors.New("Server times are too far apart") + +// errInvalidRange - returned when given range value is not valid. +var errInvalidRange = errors.New("Invalid range") + +// errInvalidRangeSource - returned when given range value exceeds +// the source object size. +var errInvalidRangeSource = errors.New("Range specified exceeds source object size") + +// error returned by disks which are to be initialized are waiting for the +// first server to initialize them in distributed set to initialize them. +var errNotFirstDisk = errors.New("Not first drive") + +// error returned by first disk waiting to initialize other servers. +var errFirstDiskWait = errors.New("Waiting on other drives") + +// error returned for a negative actual size. +var errInvalidDecompressedSize = errors.New("Invalid Decompressed Size") + +// error returned in IAM subsystem when user doesn't exist. +var errNoSuchUser = errors.New("Specified user does not exist") + +// error returned by IAM when a use a builtin IDP command when they could mean +// to use a LDAP command. +var errNoSuchUserLDAPWarn = errors.New("Specified user does not exist. If you meant a user in LDAP please use command under `mc idp ldap`") + +// error returned when service account is not found +var errNoSuchServiceAccount = errors.New("Specified service account does not exist") + +// error returned when temporary account is not found +var errNoSuchTempAccount = errors.New("Specified temporary account does not exist") + +// error returned when access key is not found +var errNoSuchAccessKey = errors.New("Specified access key does not exist") + +// error returned in IAM subsystem when an account doesn't exist. +var errNoSuchAccount = errors.New("Specified account does not exist") + +// error returned in IAM subsystem when groups doesn't exist. +var errNoSuchGroup = errors.New("Specified group does not exist") + +// error returned in IAM subsystem when a policy attach/detach request has no +// net effect, i.e. it is already applied. +var errNoPolicyToAttachOrDetach = errors.New("Specified policy update has no net effect") + +// error returned in IAM subsystem when a non-empty group needs to be +// deleted. +var errGroupNotEmpty = errors.New("Specified group is not empty - cannot remove it") + +// error returned in IAM subsystem when a group is disabled +var errGroupDisabled = errors.New("Specified group is disabled") + +// error returned in IAM subsystem when policy doesn't exist. +var errNoSuchPolicy = errors.New("Specified canned policy does not exist") + +// error returned when policy to be deleted is in use. +var errPolicyInUse = errors.New("Specified policy is in use and cannot be deleted.") + +// error returned when more than a single policy is specified when only one is +// expected. +var errTooManyPolicies = errors.New("Only a single policy may be specified here.") + +// error returned in IAM subsystem when an external users systems is configured. +var errIAMActionNotAllowed = errors.New("Specified IAM action is not allowed") + +// error returned in IAM service account +var errIAMServiceAccountNotAllowed = errors.New("Specified service account action is not allowed") + +// error returned in IAM subsystem when IAM sub-system is still being initialized. +var errIAMNotInitialized = errors.New("IAM sub-system is being initialized, please try again") + +// error returned when upload id not found +var errUploadIDNotFound = errors.New("Specified Upload ID is not found") + +// error returned when PartNumber is greater than the maximum allowed 10000 parts +var errInvalidMaxParts = errors.New("Part number is greater than the maximum allowed 10000 parts") + +// error returned for session policies > 2048 +var errSessionPolicyTooLarge = errors.New("Session policy should not exceed 2048 characters") + +// error returned in SFTP when user used public key without certificate +var errSftpPublicKeyWithoutCert = errors.New("public key authentication without certificate is not accepted") + +// error returned in SFTP when user used certificate which does not contain principal(s) +var errSftpCertWithoutPrincipals = errors.New("certificates without principal(s) are not accepted") + +// error returned when group name contains reserved characters +var errGroupNameContainsReservedChars = errors.New("Group name contains reserved characters '=' or ','") diff --git a/cmd/untar.go b/cmd/untar.go new file mode 100644 index 00000000..0f8c428a --- /dev/null +++ b/cmd/untar.go @@ -0,0 +1,278 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "archive/tar" + "bufio" + "bytes" + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path" + "runtime" + "sync" + "time" + + "github.com/cosnicolaou/pbzip2" + "github.com/klauspost/compress/s2" + "github.com/klauspost/compress/zstd" + gzip "github.com/klauspost/pgzip" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/pierrec/lz4/v4" +) + +// Max bzip2 concurrency across calls. 50% of GOMAXPROCS. +var bz2Limiter = pbzip2.CreateConcurrencyPool((runtime.GOMAXPROCS(0) + 1) / 2) + +func detect(r *bufio.Reader) format { + z, err := r.Peek(4) + if err != nil { + return formatUnknown + } + for _, f := range magicHeaders { + if bytes.Equal(f.header, z[:len(f.header)]) { + return f.f + } + } + return formatUnknown +} + +//go:generate stringer -type=format -trimprefix=format $GOFILE +type format int + +const ( + formatUnknown format = iota + formatGzip + formatZstd + formatLZ4 + formatS2 + formatBZ2 +) + +var magicHeaders = []struct { + header []byte + f format +}{ + { + header: []byte{0x1f, 0x8b, 8}, + f: formatGzip, + }, + { + // Zstd default header. + header: []byte{0x28, 0xb5, 0x2f, 0xfd}, + f: formatZstd, + }, + { + // Zstd skippable frame header. + header: []byte{0x2a, 0x4d, 0x18}, + f: formatZstd, + }, + { + // LZ4 + header: []byte{0x4, 0x22, 0x4d, 0x18}, + f: formatLZ4, + }, + { + // Snappy/S2 stream + header: []byte{0xff, 0x06, 0x00, 0x00}, + f: formatS2, + }, + { + header: []byte{0x42, 0x5a, 'h'}, + f: formatBZ2, + }, +} + +type untarOptions struct { + ignoreDirs bool + ignoreErrs bool + prefixAll string +} + +// disconnectReader will ensure that no reads can take place on +// the upstream reader after close has been called. +type disconnectReader struct { + r io.Reader + mu sync.Mutex +} + +func (d *disconnectReader) Read(p []byte) (n int, err error) { + d.mu.Lock() + defer d.mu.Unlock() + if d.r != nil { + return d.r.Read(p) + } + return 0, errors.New("reader closed") +} + +func (d *disconnectReader) Close() error { + d.mu.Lock() + d.r = nil + d.mu.Unlock() + return nil +} + +func untar(ctx context.Context, r io.Reader, putObject func(reader io.Reader, info os.FileInfo, name string) error, o untarOptions) error { + bf := bufio.NewReader(r) + switch f := detect(bf); f { + case formatGzip: + gz, err := gzip.NewReader(bf) + if err != nil { + return err + } + defer gz.Close() + r = gz + case formatS2: + r = s2.NewReader(bf) + case formatZstd: + // Limit to 16 MiB per stream. + dec, err := zstd.NewReader(bf, zstd.WithDecoderMaxWindow(16<<20)) + if err != nil { + return err + } + defer dec.Close() + r = dec + case formatBZ2: + ctx, cancel := context.WithCancel(ctx) + defer cancel() + r = pbzip2.NewReader(ctx, bf, pbzip2.DecompressionOptions( + pbzip2.BZConcurrency((runtime.GOMAXPROCS(0)+1)/2), + pbzip2.BZConcurrencyPool(bz2Limiter))) + case formatLZ4: + r = lz4.NewReader(bf) + case formatUnknown: + r = bf + default: + return fmt.Errorf("Unsupported format %s", f) + } + tarReader := tar.NewReader(r) + n := 0 + asyncWriters := make(chan struct{}, 16) + var wg sync.WaitGroup + + var asyncErr error + var asyncErrMu sync.Mutex + for { + if !o.ignoreErrs { + asyncErrMu.Lock() + err := asyncErr + asyncErrMu.Unlock() + if err != nil { + return err + } + } + + header, err := tarReader.Next() + switch { + // if no more files are found return + case err == io.EOF: + wg.Wait() + return asyncErr + + // return any other error + case err != nil: + wg.Wait() + extra := "" + if n > 0 { + extra = fmt.Sprintf(" after %d successful object(s)", n) + } + return fmt.Errorf("tar file error: %w%s", err, extra) + + // if the header is nil, just skip it (not sure how this happens) + case header == nil: + continue + } + + name := header.Name + switch path.Clean(name) { + case ".", slashSeparator: + continue + } + + switch header.Typeflag { + case tar.TypeDir: // = directory + if o.ignoreDirs { + continue + } + name = trimLeadingSlash(pathJoin(name, slashSeparator)) + case tar.TypeReg, tar.TypeChar, tar.TypeBlock, tar.TypeFifo, tar.TypeGNUSparse: // = regular + name = trimLeadingSlash(path.Clean(name)) + default: + // ignore symlink'ed + continue + } + if o.prefixAll != "" { + name = pathJoin(o.prefixAll, name) + } + + // Do small files async + n++ + if header.Size <= xioutil.MediumBlock { + asyncWriters <- struct{}{} + bufp := xioutil.ODirectPoolMedium.Get() + b := (*bufp)[:header.Size] + if _, err := io.ReadFull(tarReader, b); err != nil { + return err + } + wg.Add(1) + go func(name string, fi fs.FileInfo, b []byte) { + rc := disconnectReader{r: bytes.NewReader(b)} + defer func() { + rc.Close() + <-asyncWriters + wg.Done() + xioutil.ODirectPoolMedium.Put(bufp) + }() + if err := putObject(&rc, fi, name); err != nil { + if o.ignoreErrs { + s3LogIf(ctx, err) + return + } + asyncErrMu.Lock() + if asyncErr == nil { + asyncErr = err + } + asyncErrMu.Unlock() + } + }(name, header.FileInfo(), b) + continue + } + + // If zero or earlier modtime, set to current. + // Otherwise the resulting objects will be invalid. + if header.ModTime.UnixNano() <= 0 { + header.ModTime = time.Now() + } + + // Sync upload. + rc := disconnectReader{r: tarReader} + if err := putObject(&rc, header.FileInfo(), name); err != nil { + rc.Close() + if o.ignoreErrs { + s3LogIf(ctx, err) + continue + } + return err + } + rc.Close() + } +} diff --git a/cmd/update-notifier.go b/cmd/update-notifier.go new file mode 100644 index 00000000..20c169f9 --- /dev/null +++ b/cmd/update-notifier.go @@ -0,0 +1,103 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "runtime" + "strings" + "time" + + "github.com/cheggaaa/pb" + humanize "github.com/dustin/go-humanize" + "github.com/minio/minio/internal/color" +) + +// prepareUpdateMessage - prepares the update message, only if a +// newer version is available. +func prepareUpdateMessage(downloadURL string, older time.Duration) string { + if downloadURL == "" || older <= 0 { + return "" + } + + // Compute friendly duration string to indicate time + // difference between newer and current release. + t := time.Time{} + newerThan := humanize.RelTime(t, t.Add(older), "before the latest release", "") + + if globalServerCtxt.JSON { + return fmt.Sprintf("You are running an older version of MinIO released %s, update: %s", newerThan, downloadURL) + } + + // Return the nicely colored and formatted update message. + return colorizeUpdateMessage(downloadURL, newerThan) +} + +// colorizeUpdateMessage - inspired from Yeoman project npm package https://github.com/yeoman/update-notifier +func colorizeUpdateMessage(updateString string, newerThan string) string { + msgLine1Fmt := " You are running an older version of MinIO released %s " + msgLine2Fmt := " Update: %s " + + // Calculate length *without* color coding: with ANSI terminal + // color characters, the result is incorrect. + line1Length := len(fmt.Sprintf(msgLine1Fmt, newerThan)) + line2Length := len(fmt.Sprintf(msgLine2Fmt, updateString)) + + // Populate lines with color coding. + line1InColor := fmt.Sprintf(msgLine1Fmt, color.YellowBold(newerThan)) + line2InColor := fmt.Sprintf(msgLine2Fmt, color.CyanBold(updateString)) + + // calculate the rectangular box size. + maxContentWidth := max(line1Length, line2Length) + + // termWidth is set to a default one to use when we are + // not able to calculate terminal width via OS syscalls + termWidth := 25 + if width, err := pb.GetTerminalWidth(); err == nil { + termWidth = width + } + + // Box cannot be printed if terminal width is small than maxContentWidth + if maxContentWidth > termWidth { + return "\n" + line1InColor + "\n" + line2InColor + "\n\n" + } + + topLeftChar := "┏" + topRightChar := "┓" + bottomLeftChar := "┗" + bottomRightChar := "┛" + horizBarChar := "━" + vertBarChar := "┃" + // on windows terminal turn off unicode characters. + if runtime.GOOS == globalWindowsOSName { + topLeftChar = "+" + topRightChar = "+" + bottomLeftChar = "+" + bottomRightChar = "+" + horizBarChar = "-" + vertBarChar = "|" + } + + lines := []string{ + color.YellowBold(topLeftChar + strings.Repeat(horizBarChar, maxContentWidth) + topRightChar), + color.YellowBold(vertBarChar) + line1InColor + strings.Repeat(" ", maxContentWidth-line1Length) + color.YellowBold(vertBarChar), + color.YellowBold(vertBarChar) + line2InColor + strings.Repeat(" ", maxContentWidth-line2Length) + color.YellowBold(vertBarChar), + color.YellowBold(bottomLeftChar + strings.Repeat(horizBarChar, maxContentWidth) + bottomRightChar), + } + return "\n" + strings.Join(lines, "\n") + "\n" +} diff --git a/cmd/update-notifier_test.go b/cmd/update-notifier_test.go new file mode 100644 index 00000000..64083356 --- /dev/null +++ b/cmd/update-notifier_test.go @@ -0,0 +1,93 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/minio/minio/internal/color" +) + +// Tests update notifier string builder. +func TestPrepareUpdateMessage(t *testing.T) { + testCases := []struct { + older time.Duration + dlURL string + + expectedSubStr string + }{ + // Testcase index 0 + {72 * time.Hour, "my_download_url", "3 days before the latest release"}, + {3 * time.Hour, "https://my_download_url_is_huge/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa", "3 hours before the latest release"}, + {-72 * time.Hour, "another_update_url", ""}, + {0, "another_update_url", ""}, + {time.Hour, "", ""}, + {0 * time.Second, "my_download_url", "now"}, + {1 * time.Second, "my_download_url", "1 second before the latest release"}, + {37 * time.Second, "my_download_url", "37 seconds before the latest release"}, + {60 * time.Second, "my_download_url", "1 minute before the latest release"}, + {61 * time.Second, "my_download_url", "1 minute before the latest release"}, + + // Testcase index 10 + {37 * time.Minute, "my_download_url", "37 minutes before the latest release"}, + {1 * time.Hour, "my_download_url", "1 hour before the latest release"}, + {61 * time.Minute, "my_download_url", "1 hour before the latest release"}, + {122 * time.Minute, "my_download_url", "2 hours before the latest release"}, + {24 * time.Hour, "my_download_url", "1 day before the latest release"}, + {25 * time.Hour, "my_download_url", "1 day before the latest release"}, + {49 * time.Hour, "my_download_url", "2 days before the latest release"}, + {7 * 24 * time.Hour, "my_download_url", "1 week before the latest release"}, + {8 * 24 * time.Hour, "my_download_url", "1 week before the latest release"}, + {15 * 24 * time.Hour, "my_download_url", "2 weeks before the latest release"}, + + // Testcase index 20 + {30 * 24 * time.Hour, "my_download_url", "1 month before the latest release"}, + {31 * 24 * time.Hour, "my_download_url", "1 month before the latest release"}, + {61 * 24 * time.Hour, "my_download_url", "2 months before the latest release"}, + {360 * 24 * time.Hour, "my_download_url", "1 year before the latest release"}, + {361 * 24 * time.Hour, "my_download_url", "1 year before the latest release"}, + {2 * 365 * 24 * time.Hour, "my_download_url", "2 years before the latest release"}, + } + + plainMsg := "You are running an older version of MinIO released" + + for i, testCase := range testCases { + output := prepareUpdateMessage(testCase.dlURL, testCase.older) + line1 := fmt.Sprintf("%s %s", plainMsg, color.YellowBold(testCase.expectedSubStr)) + line2 := fmt.Sprintf("Update: %s", color.CyanBold(testCase.dlURL)) + // Uncomment below to see message appearance: + // fmt.Println(output) + switch { + case testCase.dlURL == "" && output != "": + t.Errorf("Testcase %d: no newer release available but got an update message: %s", i+1, output) + case output == "" && testCase.dlURL != "" && testCase.older > 0: + t.Errorf("Testcase %d: newer release is available but got empty update message!", i+1) + case output == "" && (testCase.dlURL == "" || testCase.older <= 0): + // Valid no update message case. No further + // validation needed. + continue + case !strings.Contains(output, line1): + t.Errorf("Testcase %d: output '%s' did not contain line 1: '%s'", i+1, output, line1) + case !strings.Contains(output, line2): + t.Errorf("Testcase %d: output '%s' did not contain line 2: '%s'", i+1, output, line2) + } + } +} diff --git a/cmd/update.go b/cmd/update.go new file mode 100644 index 00000000..ff3fcd03 --- /dev/null +++ b/cmd/update.go @@ -0,0 +1,650 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bufio" + "crypto" + "crypto/tls" + "encoding/hex" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "runtime" + "strings" + "sync/atomic" + "time" + + "github.com/klauspost/compress/zstd" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/env" + xnet "github.com/minio/pkg/v3/net" + "github.com/minio/selfupdate" + gopsutilcpu "github.com/shirou/gopsutil/v3/cpu" + "github.com/valyala/bytebufferpool" +) + +const ( + envMinisignPubKey = "MINIO_UPDATE_MINISIGN_PUBKEY" + updateTimeout = 10 * time.Second +) + +var ( + // Newer official download info URLs appear earlier below. + minioReleaseInfoURL = MinioReleaseURL + "minio.sha256sum" + + // For windows our files have .exe additionally. + minioReleaseWindowsInfoURL = MinioReleaseURL + "minio.exe.sha256sum" +) + +// minioVersionToReleaseTime - parses a standard official release +// MinIO version string. +// +// An official binary's version string is the release time formatted +// with RFC3339 (in UTC) - e.g. `2017-09-29T19:16:56Z` +func minioVersionToReleaseTime(version string) (releaseTime time.Time, err error) { + return time.Parse(time.RFC3339, version) +} + +// releaseTimeToReleaseTag - converts a time to a string formatted as +// an official MinIO release tag. +// +// An official minio release tag looks like: +// `RELEASE.2017-09-29T19-16-56Z` +func releaseTimeToReleaseTag(releaseTime time.Time) string { + return "RELEASE." + releaseTime.Format(MinioReleaseTagTimeLayout) +} + +// releaseTagToReleaseTime - reverse of `releaseTimeToReleaseTag()` +func releaseTagToReleaseTime(releaseTag string) (releaseTime time.Time, err error) { + fields := strings.Split(releaseTag, ".") + if len(fields) < 2 || len(fields) > 4 { + return releaseTime, fmt.Errorf("%s is not a valid release tag", releaseTag) + } + if fields[0] != "RELEASE" { + return releaseTime, fmt.Errorf("%s is not a valid release tag", releaseTag) + } + return time.Parse(MinioReleaseTagTimeLayout, fields[1]) +} + +// getModTime - get the file modification time of `path` +func getModTime(path string) (t time.Time, err error) { + // Convert to absolute path + absPath, err := filepath.Abs(path) + if err != nil { + return t, fmt.Errorf("Unable to get absolute path of %s. %w", path, err) + } + + // Version is minio non-standard, we will use minio binary's + // ModTime as release time. + fi, err := Stat(absPath) + if err != nil { + return t, fmt.Errorf("Unable to get ModTime of %s. %w", absPath, err) + } + + // Return the ModTime + return fi.ModTime().UTC(), nil +} + +// GetCurrentReleaseTime - returns this process's release time. If it +// is official minio version, parsed version is returned else minio +// binary's mod time is returned. +func GetCurrentReleaseTime() (releaseTime time.Time, err error) { + if releaseTime, err = minioVersionToReleaseTime(Version); err == nil { + return releaseTime, err + } + + // Looks like version is minio non-standard, we use minio + // binary's ModTime as release time: + return getModTime(os.Args[0]) +} + +// IsDocker - returns if the environment minio is running in docker or +// not. The check is a simple file existence check. +// +// https://github.com/moby/moby/blob/master/daemon/initlayer/setup_unix.go +// https://github.com/containers/podman/blob/master/libpod/runtime.go +// +// "/.dockerenv": "file", +// "/run/.containerenv": "file", +func IsDocker() bool { + var err error + for _, envfile := range []string{ + "/.dockerenv", + "/run/.containerenv", + } { + _, err = os.Stat(envfile) + if err == nil { + return true + } + } + if osIsNotExist(err) { + // if none of the files are present we may be running inside + // CRI-O, Containerd etc.. + // Fallback to our container specific ENVs if they are set. + return env.IsSet("MINIO_ACCESS_KEY_FILE") + } + + // Log error, as we will not propagate it to caller + internalLogIf(GlobalContext, err) + + return err == nil +} + +// IsDCOS returns true if minio is running in DCOS. +func IsDCOS() bool { + // http://mesos.apache.org/documentation/latest/docker-containerizer/ + // Mesos docker containerizer sets this value + return env.Get("MESOS_CONTAINER_NAME", "") != "" +} + +// IsKubernetes returns true if minio is running in kubernetes. +func IsKubernetes() bool { + // Kubernetes env used to validate if we are + // indeed running inside a kubernetes pod + // is KUBERNETES_SERVICE_HOST + // https://github.com/kubernetes/kubernetes/blob/master/pkg/kubelet/kubelet_pods.go#L541 + return env.Get("KUBERNETES_SERVICE_HOST", "") != "" +} + +// IsBOSH returns true if minio is deployed from a bosh package +func IsBOSH() bool { + // "/var/vcap/bosh" exists in BOSH deployed instance. + _, err := os.Stat("/var/vcap/bosh") + if osIsNotExist(err) { + return false + } + + // Log error, as we will not propagate it to caller + internalLogIf(GlobalContext, err) + + return err == nil +} + +// MinIO Helm chart uses DownwardAPIFile to write pod label info to /podinfo/labels +// More info: https://kubernetes.io/docs/tasks/inject-data-application/downward-api-volume-expose-pod-information/#store-pod-fields +// Check if this is Helm package installation and report helm chart version +func getHelmVersion(helmInfoFilePath string) string { + // Read the file exists. + helmInfoFile, err := Open(helmInfoFilePath) + if err != nil { + // Log errors and return "" as MinIO can be deployed + // without Helm charts as well. + if !osIsNotExist(err) { + reqInfo := (&logger.ReqInfo{}).AppendTags("helmInfoFilePath", helmInfoFilePath) + ctx := logger.SetReqInfo(GlobalContext, reqInfo) + internalLogIf(ctx, err) + } + return "" + } + defer helmInfoFile.Close() + scanner := bufio.NewScanner(helmInfoFile) + for scanner.Scan() { + if strings.Contains(scanner.Text(), "chart=") { + helmChartVersion := strings.TrimPrefix(scanner.Text(), "chart=") + // remove quotes from the chart version + return strings.Trim(helmChartVersion, `"`) + } + } + + return "" +} + +// IsSourceBuild - returns if this binary is a non-official build from +// source code. +func IsSourceBuild() bool { + _, err := minioVersionToReleaseTime(Version) + return err != nil +} + +// IsPCFTile returns if server is running in PCF +func IsPCFTile() bool { + return env.Get("MINIO_PCF_TILE_VERSION", "") != "" +} + +// DO NOT CHANGE USER AGENT STYLE. +// The style should be +// +// MinIO (; [; ][; dcos][; kubernetes][; docker][; source]) MinIO/ MinIO/ MinIO/ [MinIO/universe-] [MinIO/helm-] +// +// Any change here should be discussed by opening an issue at +// https://github.com/minio/minio/issues. +func getUserAgent(mode string) string { + userAgentParts := []string{} + // Helper function to concisely append a pair of strings to a + // the user-agent slice. + uaAppend := func(p, q string) { + userAgentParts = append(userAgentParts, p, q) + } + uaAppend(MinioUAName, " (") + uaAppend("", runtime.GOOS) + uaAppend("; ", runtime.GOARCH) + if mode != "" { + uaAppend("; ", mode) + } + if IsDCOS() { + uaAppend("; ", "dcos") + } + if IsKubernetes() { + uaAppend("; ", "kubernetes") + } + if IsDocker() { + uaAppend("; ", "docker") + } + if IsBOSH() { + uaAppend("; ", "bosh") + } + if IsSourceBuild() { + uaAppend("; ", "source") + } + + uaAppend(" ", Version) + uaAppend(" ", ReleaseTag) + uaAppend(" ", CommitID) + if IsDCOS() { + universePkgVersion := env.Get("MARATHON_APP_LABEL_DCOS_PACKAGE_VERSION", "") + // On DC/OS environment try to the get universe package version. + if universePkgVersion != "" { + uaAppend(" universe-", universePkgVersion) + } + } + + if IsKubernetes() { + // In Kubernetes environment, try to fetch the helm package version + helmChartVersion := getHelmVersion("/podinfo/labels") + if helmChartVersion != "" { + uaAppend(" helm-", helmChartVersion) + } + // In Kubernetes environment, try to fetch the Operator, VSPHERE plugin version + opVersion := env.Get("MINIO_OPERATOR_VERSION", "") + if opVersion != "" { + uaAppend(" operator-", opVersion) + } + vsphereVersion := env.Get("MINIO_VSPHERE_PLUGIN_VERSION", "") + if vsphereVersion != "" { + uaAppend(" vsphere-plugin-", vsphereVersion) + } + } + + if IsPCFTile() { + pcfTileVersion := env.Get("MINIO_PCF_TILE_VERSION", "") + if pcfTileVersion != "" { + uaAppend(" pcf-tile-", pcfTileVersion) + } + } + uaAppend("; ", "") + + if cpus, err := gopsutilcpu.Info(); err == nil && len(cpus) > 0 { + cpuMap := make(map[string]struct{}, len(cpus)) + coreMap := make(map[string]struct{}, len(cpus)) + for i := range cpus { + cpuMap[cpus[i].PhysicalID] = struct{}{} + coreMap[cpus[i].CoreID] = struct{}{} + } + cpu := cpus[0] + uaAppend(" CPU ", fmt.Sprintf("(total_cpus:%d, total_cores:%d; vendor:%s; family:%s; model:%s; stepping:%d; model_name:%s)", + len(cpuMap), len(coreMap), cpu.VendorID, cpu.Family, cpu.Model, cpu.Stepping, cpu.ModelName)) + } + uaAppend(")", "") + + return strings.Join(userAgentParts, "") +} + +func downloadReleaseURL(u *url.URL, timeout time.Duration, mode string) (content string, err error) { + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return content, AdminError{ + Code: AdminUpdateUnexpectedFailure, + Message: err.Error(), + StatusCode: http.StatusInternalServerError, + } + } + req.Header.Set("User-Agent", getUserAgent(mode)) + + client := &http.Client{Transport: getUpdateTransport(timeout)} + resp, err := client.Do(req) + if err != nil { + if xnet.IsNetworkOrHostDown(err, false) { + return content, AdminError{ + Code: AdminUpdateURLNotReachable, + Message: err.Error(), + StatusCode: http.StatusServiceUnavailable, + } + } + return content, AdminError{ + Code: AdminUpdateUnexpectedFailure, + Message: err.Error(), + StatusCode: http.StatusInternalServerError, + } + } + if resp == nil { + return content, AdminError{ + Code: AdminUpdateUnexpectedFailure, + Message: fmt.Sprintf("No response from server to download URL %s", u), + StatusCode: http.StatusInternalServerError, + } + } + defer xhttp.DrainBody(resp.Body) + + if resp.StatusCode != http.StatusOK { + return content, AdminError{ + Code: AdminUpdateUnexpectedFailure, + Message: fmt.Sprintf("Error downloading URL %s. Response: %v", u, resp.Status), + StatusCode: resp.StatusCode, + } + } + + contentBytes, err := io.ReadAll(resp.Body) + if err != nil { + return content, AdminError{ + Code: AdminUpdateUnexpectedFailure, + Message: fmt.Sprintf("Error reading response. %s", err), + StatusCode: http.StatusInternalServerError, + } + } + + return string(contentBytes), nil +} + +func releaseInfoToReleaseTime(releaseInfo string) (releaseTime time.Time, err error) { + // Split release of style minio.RELEASE.2019-08-21T19-40-07Z. + nfields := strings.SplitN(releaseInfo, ".", 2) + if len(nfields) != 2 { + err = fmt.Errorf("Unknown release information `%s`", releaseInfo) + return releaseTime, err + } + if nfields[0] != "minio" { + err = fmt.Errorf("Unknown release `%s`", releaseInfo) + return releaseTime, err + } + + releaseTime, err = releaseTagToReleaseTime(nfields[1]) + if err != nil { + err = fmt.Errorf("Unknown release tag format. %w", err) + } + return releaseTime, err +} + +// parseReleaseData - parses release info file content fetched from +// official minio download server. +// +// The expected format is a single line with two words like: +// +// fbe246edbd382902db9a4035df7dce8cb441357d minio.RELEASE.2016-10-07T01-16-39Z. +// +// The second word must be `minio.` appended to a standard release tag. +func parseReleaseData(data string) (sha256Sum []byte, releaseTime time.Time, releaseInfo string, err error) { + defer func() { + if err != nil { + err = AdminError{ + Code: AdminUpdateUnexpectedFailure, + Message: err.Error(), + StatusCode: http.StatusInternalServerError, + } + } + }() + + fields := strings.Fields(data) + if len(fields) != 2 { + err = fmt.Errorf("Unknown release data `%s`", data) + return sha256Sum, releaseTime, releaseInfo, err + } + + sha256Sum, err = hex.DecodeString(fields[0]) + if err != nil { + return sha256Sum, releaseTime, releaseInfo, err + } + + releaseInfo = fields[1] + + releaseTime, err = releaseInfoToReleaseTime(releaseInfo) + return sha256Sum, releaseTime, releaseInfo, err +} + +func getUpdateTransport(timeout time.Duration) http.RoundTripper { + var updateTransport http.RoundTripper = &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: xhttp.NewInternodeDialContext(timeout, globalTCPOptions), + IdleConnTimeout: timeout, + TLSHandshakeTimeout: timeout, + ExpectContinueTimeout: timeout, + TLSClientConfig: &tls.Config{ + RootCAs: globalRootCAs, + ClientSessionCache: tls.NewLRUClientSessionCache(tlsClientSessionCacheSize), + }, + DisableCompression: true, + } + return updateTransport +} + +func getLatestReleaseTime(u *url.URL, timeout time.Duration, mode string) (sha256Sum []byte, releaseTime time.Time, err error) { + data, err := downloadReleaseURL(u, timeout, mode) + if err != nil { + return sha256Sum, releaseTime, err + } + + sha256Sum, releaseTime, _, err = parseReleaseData(data) + return sha256Sum, releaseTime, err +} + +const ( + // Kubernetes deployment doc link. + kubernetesDeploymentDoc = "https://docs.min.io/community/minio-object-store/operations/deployments/kubernetes.html" + + // Mesos deployment doc link. + mesosDeploymentDoc = "https://docs.min.io/community/minio-object-store/operations/deployments/kubernetes.html" +) + +func getDownloadURL(releaseTag string) (downloadURL string) { + // Check if we are in DCOS environment, return + // deployment guide for update procedures. + if IsDCOS() { + return mesosDeploymentDoc + } + + // Check if we are in kubernetes environment, return + // deployment guide for update procedures. + if IsKubernetes() { + return kubernetesDeploymentDoc + } + + // Check if we are docker environment, return docker update command + if IsDocker() { + // Construct release tag name. + return fmt.Sprintf("podman pull quay.io/minio/minio:%s", releaseTag) + } + + // For binary only installations, we return link to the latest binary. + if runtime.GOOS == "windows" { + return MinioReleaseURL + "minio.exe" + } + + return MinioReleaseURL + "minio" +} + +func getUpdateReaderFromURL(u *url.URL, transport http.RoundTripper, mode string) (io.ReadCloser, error) { + clnt := &http.Client{ + Transport: transport, + } + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return nil, AdminError{ + Code: AdminUpdateUnexpectedFailure, + Message: err.Error(), + StatusCode: http.StatusInternalServerError, + } + } + + req.Header.Set("User-Agent", getUserAgent(mode)) + + resp, err := clnt.Do(req) + if err != nil { + if xnet.IsNetworkOrHostDown(err, false) { + return nil, AdminError{ + Code: AdminUpdateURLNotReachable, + Message: err.Error(), + StatusCode: http.StatusServiceUnavailable, + } + } + return nil, AdminError{ + Code: AdminUpdateUnexpectedFailure, + Message: err.Error(), + StatusCode: http.StatusInternalServerError, + } + } + return resp.Body, nil +} + +var updateInProgress atomic.Uint32 + +// Function to get the reader from an architecture +func downloadBinary(u *url.URL, mode string) (binCompressed []byte, bin []byte, err error) { + transport := getUpdateTransport(30 * time.Second) + var reader io.ReadCloser + if u.Scheme == "https" || u.Scheme == "http" { + reader, err = getUpdateReaderFromURL(u, transport, mode) + if err != nil { + return nil, nil, err + } + } else { + return nil, nil, fmt.Errorf("unsupported protocol scheme: %s", u.Scheme) + } + defer xhttp.DrainBody(reader) + + b := bytebufferpool.Get() + bc := bytebufferpool.Get() + defer func() { + b.Reset() + bc.Reset() + + bytebufferpool.Put(b) + bytebufferpool.Put(bc) + }() + + w, err := zstd.NewWriter(bc) + if err != nil { + return nil, nil, err + } + + if _, err = io.Copy(w, io.TeeReader(reader, b)); err != nil { + return nil, nil, err + } + + w.Close() + return bc.Bytes(), b.Bytes(), nil +} + +const ( + // Update this whenever the official minisign pubkey is rotated. + defaultMinisignPubkey = "RWTx5Zr1tiHQLwG9keckT0c45M3AGeHD6IvimQHpyRywVWGbP1aVSGav" +) + +func verifyBinary(u *url.URL, sha256Sum []byte, releaseInfo, mode string, reader io.Reader) (err error) { + if !updateInProgress.CompareAndSwap(0, 1) { + return errors.New("update already in progress") + } + defer updateInProgress.Store(0) + + transport := getUpdateTransport(30 * time.Second) + opts := selfupdate.Options{ + Hash: crypto.SHA256, + Checksum: sha256Sum, + } + + if err := opts.CheckPermissions(); err != nil { + return AdminError{ + Code: AdminUpdateApplyFailure, + Message: fmt.Sprintf("server update failed with: %s, do not restart the servers yet", err), + StatusCode: http.StatusInternalServerError, + } + } + + minisignPubkey := env.Get(envMinisignPubKey, defaultMinisignPubkey) + if minisignPubkey != "" { + v := selfupdate.NewVerifier() + u.Path = path.Dir(u.Path) + slashSeparator + releaseInfo + ".minisig" + if err = v.LoadFromURL(u.String(), minisignPubkey, transport); err != nil { + return AdminError{ + Code: AdminUpdateApplyFailure, + Message: fmt.Sprintf("signature loading failed for %v with %v", u, err), + StatusCode: http.StatusInternalServerError, + } + } + opts.Verifier = v + } + + if err = selfupdate.PrepareAndCheckBinary(reader, opts); err != nil { + var pathErr *os.PathError + if errors.As(err, &pathErr) { + return AdminError{ + Code: AdminUpdateApplyFailure, + Message: fmt.Sprintf("Unable to update the binary at %s: %v", + filepath.Dir(pathErr.Path), pathErr.Err), + StatusCode: http.StatusForbidden, + } + } + return AdminError{ + Code: AdminUpdateApplyFailure, + Message: err.Error(), + StatusCode: http.StatusInternalServerError, + } + } + + return nil +} + +func commitBinary() (err error) { + if !updateInProgress.CompareAndSwap(0, 1) { + return errors.New("update already in progress") + } + defer updateInProgress.Store(0) + + opts := selfupdate.Options{} + + if err = selfupdate.CommitBinary(opts); err != nil { + if rerr := selfupdate.RollbackError(err); rerr != nil { + return AdminError{ + Code: AdminUpdateApplyFailure, + Message: fmt.Sprintf("Failed to rollback from bad update: %v", rerr), + StatusCode: http.StatusInternalServerError, + } + } + var pathErr *os.PathError + if errors.As(err, &pathErr) { + return AdminError{ + Code: AdminUpdateApplyFailure, + Message: fmt.Sprintf("Unable to update the binary at %s: %v", + filepath.Dir(pathErr.Path), pathErr.Err), + StatusCode: http.StatusForbidden, + } + } + return AdminError{ + Code: AdminUpdateApplyFailure, + Message: err.Error(), + StatusCode: http.StatusInternalServerError, + } + } + + return nil +} diff --git a/cmd/update_test.go b/cmd/update_test.go new file mode 100644 index 00000000..e1af9e38 --- /dev/null +++ b/cmd/update_test.go @@ -0,0 +1,343 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "encoding/hex" + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "os" + "runtime" + "strings" + "testing" + "time" +) + +func TestMinioVersionToReleaseTime(t *testing.T) { + testCases := []struct { + version string + isOfficial bool + }{ + {"2017-09-29T19:16:56Z", true}, + {"RELEASE.2017-09-29T19-16-56Z", false}, + {"DEVELOPMENT.GOGET", false}, + } + for i, testCase := range testCases { + _, err := minioVersionToReleaseTime(testCase.version) + if (err == nil) != testCase.isOfficial { + t.Errorf("Test %d: Expected %v but got %v", + i+1, testCase.isOfficial, err == nil) + } + } +} + +func TestReleaseTagToNFromTimeConversion(t *testing.T) { + utcLoc, _ := time.LoadLocation("") + testCases := []struct { + t time.Time + tag string + errStr string + }{ + { + time.Date(2017, time.September, 29, 19, 16, 56, 0, utcLoc), + "RELEASE.2017-09-29T19-16-56Z", "", + }, + { + time.Date(2017, time.August, 5, 0, 0, 53, 0, utcLoc), + "RELEASE.2017-08-05T00-00-53Z", "", + }, + { + time.Now().UTC(), "2017-09-29T19:16:56Z", + "2017-09-29T19:16:56Z is not a valid release tag", + }, + { + time.Now().UTC(), "DEVELOPMENT.GOGET", + "DEVELOPMENT.GOGET is not a valid release tag", + }, + { + time.Date(2017, time.August, 5, 0, 0, 53, 0, utcLoc), + "RELEASE.2017-08-05T00-00-53Z.hotfix", "", + }, + { + time.Date(2017, time.August, 5, 0, 0, 53, 0, utcLoc), + "RELEASE.2017-08-05T00-00-53Z.hotfix.aaaa", "", + }, + } + for i, testCase := range testCases { + if testCase.errStr != "" { + got := releaseTimeToReleaseTag(testCase.t) + if got != testCase.tag && testCase.errStr == "" { + t.Errorf("Test %d: Expected %v but got %v", i+1, testCase.tag, got) + } + } + tagTime, err := releaseTagToReleaseTime(testCase.tag) + if err != nil && err.Error() != testCase.errStr { + t.Errorf("Test %d: Expected %v but got %v", i+1, testCase.errStr, err.Error()) + } + if err == nil && !tagTime.Equal(testCase.t) { + t.Errorf("Test %d: Expected %v but got %v", i+1, testCase.t, tagTime) + } + } +} + +func TestDownloadURL(t *testing.T) { + minioVersion1 := releaseTimeToReleaseTag(UTCNow()) + durl := getDownloadURL(minioVersion1) + if IsDocker() { + if durl != "podman pull quay.io/minio/minio:"+minioVersion1 { + t.Errorf("Expected %s, got %s", "podman pull quay.io/minio/minio:"+minioVersion1, durl) + } + } else { + if runtime.GOOS == "windows" { + if durl != MinioReleaseURL+"minio.exe" { + t.Errorf("Expected %s, got %s", MinioReleaseURL+"minio.exe", durl) + } + } else { + if durl != MinioReleaseURL+"minio" { + t.Errorf("Expected %s, got %s", MinioReleaseURL+"minio", durl) + } + } + } + + t.Setenv("KUBERNETES_SERVICE_HOST", "10.11.148.5") + durl = getDownloadURL(minioVersion1) + if durl != kubernetesDeploymentDoc { + t.Errorf("Expected %s, got %s", kubernetesDeploymentDoc, durl) + } + + t.Setenv("MESOS_CONTAINER_NAME", "mesos-1111") + durl = getDownloadURL(minioVersion1) + if durl != mesosDeploymentDoc { + t.Errorf("Expected %s, got %s", mesosDeploymentDoc, durl) + } +} + +// Tests user agent string. +func TestUserAgent(t *testing.T) { + testCases := []struct { + envName string + envValue string + mode string + expectedStr string + }{ + { + envName: "", + envValue: "", + mode: globalMinioModeFS, + expectedStr: fmt.Sprintf("MinIO (%s; %s; %s; source DEVELOPMENT.GOGET DEVELOPMENT.GOGET DEVELOPMENT.GOGET", runtime.GOOS, runtime.GOARCH, globalMinioModeFS), + }, + { + envName: "MESOS_CONTAINER_NAME", + envValue: "mesos-11111", + mode: globalMinioModeErasure, + expectedStr: fmt.Sprintf("MinIO (%s; %s; %s; %s; source DEVELOPMENT.GOGET DEVELOPMENT.GOGET DEVELOPMENT.GOGET universe-%s", runtime.GOOS, runtime.GOARCH, globalMinioModeErasure, "dcos", "mesos-1111"), + }, + { + envName: "KUBERNETES_SERVICE_HOST", + envValue: "10.11.148.5", + mode: globalMinioModeErasure, + expectedStr: fmt.Sprintf("MinIO (%s; %s; %s; %s; source DEVELOPMENT.GOGET DEVELOPMENT.GOGET DEVELOPMENT.GOGET", runtime.GOOS, runtime.GOARCH, globalMinioModeErasure, "kubernetes"), + }, + } + + for i, testCase := range testCases { + if testCase.envName != "" { + t.Setenv(testCase.envName, testCase.envValue) + if testCase.envName == "MESOS_CONTAINER_NAME" { + t.Setenv("MARATHON_APP_LABEL_DCOS_PACKAGE_VERSION", "mesos-1111") + } + } + + str := getUserAgent(testCase.mode) + expectedStr := testCase.expectedStr + if IsDocker() { + expectedStr = strings.ReplaceAll(expectedStr, "; source", "; docker; source") + } + if !strings.Contains(str, expectedStr) { + t.Errorf("Test %d: expected: %s, got: %s", i+1, expectedStr, str) + } + os.Unsetenv("MARATHON_APP_LABEL_DCOS_PACKAGE_VERSION") + os.Unsetenv(testCase.envName) + } +} + +// Tests if the environment we are running is in DCOS. +func TestIsDCOS(t *testing.T) { + t.Setenv("MESOS_CONTAINER_NAME", "mesos-1111") + dcos := IsDCOS() + if !dcos { + t.Fatalf("Expected %t, got %t", true, dcos) + } + os.Unsetenv("MESOS_CONTAINER_NAME") + dcos = IsDCOS() + if dcos { + t.Fatalf("Expected %t, got %t", false, dcos) + } +} + +// Tests if the environment we are running is in kubernetes. +func TestIsKubernetes(t *testing.T) { + t.Setenv("KUBERNETES_SERVICE_HOST", "10.11.148.5") + kubernetes := IsKubernetes() + if !kubernetes { + t.Fatalf("Expected %t, got %t", true, kubernetes) + } + os.Unsetenv("KUBERNETES_SERVICE_HOST") + + kubernetes = IsKubernetes() + if kubernetes { + t.Fatalf("Expected %t, got %t", false, kubernetes) + } +} + +// Tests if the environment we are running is Helm chart. +func TestGetHelmVersion(t *testing.T) { + createTempFile := func(content string) string { + tmpfile, err := os.CreateTemp(t.TempDir(), "helm-testfile-") + if err != nil { + t.Fatalf("Unable to create temporary file. %s", err) + } + if _, err = tmpfile.WriteString(content); err != nil { + t.Fatalf("Unable to create temporary file. %s", err) + } + if err = tmpfile.Close(); err != nil { + t.Fatalf("Unable to create temporary file. %s", err) + } + return tmpfile.Name() + } + + filename := createTempFile( + `app="virtuous-rat-minio" +chart="minio-0.1.3" +heritage="Tiller" +pod-template-hash="818089471"`) + + defer os.Remove(filename) + + testCases := []struct { + filename string + expectedResult string + }{ + {"", ""}, + {"/tmp/non-existing-file", ""}, + {filename, "minio-0.1.3"}, + } + + for _, testCase := range testCases { + result := getHelmVersion(testCase.filename) + + if testCase.expectedResult != result { + t.Fatalf("result: expected: %v, got: %v", testCase.expectedResult, result) + } + } +} + +func TestDownloadReleaseData(t *testing.T) { + httpServer1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {})) + defer httpServer1.Close() + httpServer2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintln(w, "fbe246edbd382902db9a4035df7dce8cb441357d minio.RELEASE.2016-10-07T01-16-39Z") + })) + defer httpServer2.Close() + httpServer3 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + http.Error(w, "", http.StatusNotFound) + })) + defer httpServer3.Close() + + testCases := []struct { + releaseChecksumURL string + expectedResult string + expectedErr error + }{ + {httpServer1.URL, "", nil}, + {httpServer2.URL, "fbe246edbd382902db9a4035df7dce8cb441357d minio.RELEASE.2016-10-07T01-16-39Z\n", nil}, + {httpServer3.URL, "", fmt.Errorf("Error downloading URL %s. Response: 404 Not Found", httpServer3.URL)}, + } + + for _, testCase := range testCases { + u, err := url.Parse(testCase.releaseChecksumURL) + if err != nil { + t.Fatal(err) + } + + result, err := downloadReleaseURL(u, 1*time.Second, "") + switch { + case testCase.expectedErr == nil: + if err != nil { + t.Fatalf("error: expected: %v, got: %v", testCase.expectedErr, err) + } + case err == nil: + t.Fatalf("error: expected: %v, got: %v", testCase.expectedErr, err) + case testCase.expectedErr.Error() != err.Error(): + t.Fatalf("error: expected: %v, got: %v", testCase.expectedErr, err) + } + + if testCase.expectedResult != result { + t.Fatalf("result: expected: %v, got: %v", testCase.expectedResult, result) + } + } +} + +func TestParseReleaseData(t *testing.T) { + releaseTime, _ := releaseTagToReleaseTime("RELEASE.2016-10-07T01-16-39Z") + testCases := []struct { + data string + expectedResult time.Time + expectedSha256hex string + expectedReleaseInfo string + expectedErr bool + }{ + {"more than two fields", time.Time{}, "", "", true}, + {"more than", time.Time{}, "", "", true}, + {"more than.two.fields", time.Time{}, "", "", true}, + {"more minio.RELEASE.fields", time.Time{}, "", "", true}, + {"more minio.RELEASE.2016-10-07T01-16-39Z", time.Time{}, "", "", true}, + { + "fbe246edbd382902db9a4035df7dce8cb441357d minio.RELEASE.2016-10-07T01-16-39Z\n", releaseTime, "fbe246edbd382902db9a4035df7dce8cb441357d", + "minio.RELEASE.2016-10-07T01-16-39Z", false, + }, + { + "fbe246edbd382902db9a4035df7dce8cb441357d minio.RELEASE.2016-10-07T01-16-39Z.customer-hotfix\n", releaseTime, "fbe246edbd382902db9a4035df7dce8cb441357d", + "minio.RELEASE.2016-10-07T01-16-39Z.customer-hotfix", false, + }, + } + + for i, testCase := range testCases { + sha256Sum, result, releaseInfo, err := parseReleaseData(testCase.data) + if !testCase.expectedErr { + if err != nil { + t.Errorf("error case %d: expected no error, got: %v", i+1, err) + } + } else if err == nil { + t.Errorf("error case %d: expected error got: %v", i+1, err) + } + if err == nil { + if hex.EncodeToString(sha256Sum) != testCase.expectedSha256hex { + t.Errorf("case %d: result: expected: %v, got: %x", i+1, testCase.expectedSha256hex, sha256Sum) + } + if !testCase.expectedResult.Equal(result) { + t.Errorf("case %d: result: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + if testCase.expectedReleaseInfo != releaseInfo { + t.Errorf("case %d: result: expected: %v, got: %v", i+1, testCase.expectedReleaseInfo, releaseInfo) + } + } + } +} diff --git a/cmd/url_test.go b/cmd/url_test.go new file mode 100644 index 00000000..c5fd7781 --- /dev/null +++ b/cmd/url_test.go @@ -0,0 +1,70 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "net/http" + "testing" +) + +func BenchmarkURLQueryForm(b *testing.B) { + req, err := http.NewRequest(http.MethodGet, "http://localhost:9000/bucket/name?uploadId=upload&partNumber=1", http.NoBody) + if err != nil { + b.Fatal(err) + } + + // benchmark utility which helps obtain number of allocations and bytes allocated per ops. + b.ReportAllocs() + // the actual benchmark for PutObject starts here. Reset the benchmark timer. + b.ResetTimer() + + if err := req.ParseForm(); err != nil { + b.Fatal(err) + } + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + req.Form.Get("uploadId") + } + }) + + // Benchmark ends here. Stop timer. + b.StopTimer() +} + +// BenchmarkURLQuery - benchmark URL memory allocations +func BenchmarkURLQuery(b *testing.B) { + req, err := http.NewRequest(http.MethodGet, "http://localhost:9000/bucket/name?uploadId=upload&partNumber=1", http.NoBody) + if err != nil { + b.Fatal(err) + } + + // benchmark utility which helps obtain number of allocations and bytes allocated per ops. + b.ReportAllocs() + // the actual benchmark for PutObject starts here. Reset the benchmark timer. + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + req.URL.Query().Get("uploadId") + } + }) + + // Benchmark ends here. Stop timer. + b.StopTimer() +} diff --git a/cmd/user-provider-utils.go b/cmd/user-provider-utils.go new file mode 100644 index 00000000..086b8de8 --- /dev/null +++ b/cmd/user-provider-utils.go @@ -0,0 +1,141 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "strings" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/auth" +) + +// getUserWithProvider - returns the appropriate internal username based on the user provider. +// if validate is true, an error is returned if the user does not exist. +func getUserWithProvider(ctx context.Context, userProvider, user string, validate bool) (string, error) { + switch userProvider { + case madmin.BuiltinProvider: + if validate { + if _, ok := globalIAMSys.GetUser(ctx, user); !ok { + return "", errNoSuchUser + } + } + return user, nil + case madmin.LDAPProvider: + if globalIAMSys.GetUsersSysType() != LDAPUsersSysType { + return "", errIAMActionNotAllowed + } + res, err := globalIAMSys.LDAPConfig.GetValidatedDNForUsername(user) + if res == nil { + err = errNoSuchUser + } + if err != nil { + if validate { + return "", err + } + if !globalIAMSys.LDAPConfig.ParsesAsDN(user) { + return "", errNoSuchUser + } + } + return res.NormDN, nil + default: + return "", errIAMActionNotAllowed + } +} + +// guessUserProvider - guesses the user provider based on the access key and claims. +func guessUserProvider(credentials auth.Credentials) string { + if !credentials.IsServiceAccount() && !credentials.IsTemp() { + return madmin.BuiltinProvider // regular users are always internal + } + + claims := credentials.Claims + if _, ok := claims[ldapUser]; ok { + return madmin.LDAPProvider // ldap users + } + + if _, ok := claims[subClaim]; ok { + providerPrefix, _, found := strings.Cut(credentials.ParentUser, getKeySeparator()) + if found { + return providerPrefix // this is true for certificate and custom providers + } + return madmin.OpenIDProvider // openid users are already hashed, so no separator + } + + return madmin.BuiltinProvider // default to internal +} + +// getProviderInfoFromClaims - returns the provider info from the claims. +func populateProviderInfoFromClaims(claims map[string]any, provider string, resp *madmin.InfoAccessKeyResp) { + resp.UserProvider = provider + switch provider { + case madmin.LDAPProvider: + resp.LDAPSpecificInfo = getLDAPInfoFromClaims(claims) + case madmin.OpenIDProvider: + resp.OpenIDSpecificInfo = getOpenIDInfoFromClaims(claims) + } +} + +func getOpenIDCfgNameFromClaims(claims map[string]any) (string, bool) { + roleArn := claims[roleArnClaim] + + s := globalServerConfig.Clone() + configs, err := globalIAMSys.OpenIDConfig.GetConfigList(s) + if err != nil { + return "", false + } + for _, cfg := range configs { + if cfg.RoleARN == roleArn { + return cfg.Name, true + } + } + return "", false +} + +func getOpenIDInfoFromClaims(claims map[string]any) madmin.OpenIDSpecificAccessKeyInfo { + info := madmin.OpenIDSpecificAccessKeyInfo{} + + cfgName, ok := getOpenIDCfgNameFromClaims(claims) + if !ok { + return info + } + + info.ConfigName = cfgName + if displayNameClaim := globalIAMSys.OpenIDConfig.GetUserReadableClaim(cfgName); displayNameClaim != "" { + name, _ := claims[displayNameClaim].(string) + info.DisplayName = name + info.DisplayNameClaim = displayNameClaim + } + if idClaim := globalIAMSys.OpenIDConfig.GetUserIDClaim(cfgName); idClaim != "" { + id, _ := claims[idClaim].(string) + info.UserID = id + info.UserIDClaim = idClaim + } + + return info +} + +func getLDAPInfoFromClaims(claims map[string]any) madmin.LDAPSpecificAccessKeyInfo { + info := madmin.LDAPSpecificAccessKeyInfo{} + + if name, ok := claims[ldapUser].(string); ok { + info.Username = name + } + + return info +} diff --git a/cmd/utils.go b/cmd/utils.go new file mode 100644 index 00000000..21fe5c61 --- /dev/null +++ b/cmd/utils.go @@ -0,0 +1,1189 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/md5" + "crypto/tls" + "encoding/base64" + "encoding/json" + "encoding/xml" + "errors" + "fmt" + "io" + "maps" + "net/http" + "net/url" + "os" + "path" + "path/filepath" + "runtime" + "runtime/pprof" + "runtime/trace" + "slices" + "strings" + "sync" + "time" + + "github.com/coreos/go-oidc/v3/oidc" + "github.com/dustin/go-humanize" + "github.com/felixge/fgprof" + "github.com/minio/madmin-go/v3" + xaudit "github.com/minio/madmin-go/v3/logger/audit" + "github.com/minio/minio-go/v7" + miniogopolicy "github.com/minio/minio-go/v7/pkg/policy" + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/config/api" + xtls "github.com/minio/minio/internal/config/identity/tls" + "github.com/minio/minio/internal/config/storageclass" + "github.com/minio/minio/internal/crypto" + "github.com/minio/minio/internal/handlers" + "github.com/minio/minio/internal/hash" + xhttp "github.com/minio/minio/internal/http" + ioutilx "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/logger/message/audit" + "github.com/minio/minio/internal/rest" + "github.com/minio/mux" + "github.com/minio/pkg/v3/certs" + "github.com/minio/pkg/v3/env" + xnet "github.com/minio/pkg/v3/net" + "golang.org/x/oauth2" +) + +const ( + slashSeparator = "/" +) + +// BucketAccessPolicy - Collection of canned bucket policy at a given prefix. +type BucketAccessPolicy struct { + Bucket string `json:"bucket"` + Prefix string `json:"prefix"` + Policy miniogopolicy.BucketPolicy `json:"policy"` +} + +// IsErrIgnored returns whether given error is ignored or not. +func IsErrIgnored(err error, ignoredErrs ...error) bool { + return IsErr(err, ignoredErrs...) +} + +// IsErr returns whether given error is exact error. +func IsErr(err error, errs ...error) bool { + for _, exactErr := range errs { + if errors.Is(err, exactErr) { + return true + } + } + return false +} + +// ErrorRespToObjectError converts MinIO errors to minio object layer errors. +func ErrorRespToObjectError(err error, params ...string) error { + if err == nil { + return nil + } + + bucket := "" + object := "" + versionID := "" + if len(params) >= 1 { + bucket = params[0] + } + if len(params) >= 2 { + object = params[1] + } + if len(params) >= 3 { + versionID = params[2] + } + + if xnet.IsNetworkOrHostDown(err, false) { + return BackendDown{Err: err.Error()} + } + + minioErr, ok := err.(minio.ErrorResponse) + if !ok { + // We don't interpret non MinIO errors. As minio errors will + // have StatusCode to help to convert to object errors. + return err + } + + switch minioErr.Code { + case "SlowDownWrite": + err = InsufficientWriteQuorum{Bucket: bucket, Object: object} + case "SlowDownRead": + err = InsufficientReadQuorum{Bucket: bucket, Object: object} + case "PreconditionFailed": + err = PreConditionFailed{} + case "InvalidRange": + err = InvalidRange{} + case "BucketAlreadyOwnedByYou": + err = BucketAlreadyOwnedByYou{} + case "BucketNotEmpty": + err = BucketNotEmpty{} + case "NoSuchBucketPolicy": + err = BucketPolicyNotFound{} + case "NoSuchLifecycleConfiguration": + err = BucketLifecycleNotFound{} + case "InvalidBucketName": + err = BucketNameInvalid{Bucket: bucket} + case "InvalidPart": + err = InvalidPart{} + case "NoSuchBucket": + err = BucketNotFound{Bucket: bucket} + case "NoSuchKey": + if object != "" { + err = ObjectNotFound{Bucket: bucket, Object: object} + } else { + err = BucketNotFound{Bucket: bucket} + } + case "NoSuchVersion": + if object != "" { + err = ObjectNotFound{Bucket: bucket, Object: object, VersionID: versionID} + } else { + err = BucketNotFound{Bucket: bucket} + } + case "XMinioInvalidObjectName": + err = ObjectNameInvalid{} + case "AccessDenied": + err = PrefixAccessDenied{ + Bucket: bucket, + Object: object, + } + case "XAmzContentSHA256Mismatch": + err = hash.SHA256Mismatch{} + case "NoSuchUpload": + err = InvalidUploadID{} + case "EntityTooSmall": + err = PartTooSmall{} + case "ReplicationPermissionCheck": + err = ReplicationPermissionCheck{} + } + + if minioErr.StatusCode == http.StatusMethodNotAllowed { + err = toObjectErr(errMethodNotAllowed, bucket, object) + } + return err +} + +// returns 'true' if either string has space in the +// - beginning of a string +// OR +// - end of a string +func hasSpaceBE(s string) bool { + return strings.TrimSpace(s) != s +} + +func request2BucketObjectName(r *http.Request) (bucketName, objectName string) { + path, err := getResource(r.URL.Path, r.Host, globalDomainNames) + if err != nil { + logger.CriticalIf(GlobalContext, err) + } + + return path2BucketObject(path) +} + +// path2BucketObjectWithBasePath returns bucket and prefix, if any, +// of a 'path'. basePath is trimmed from the front of the 'path'. +func path2BucketObjectWithBasePath(basePath, path string) (bucket, prefix string) { + path = strings.TrimPrefix(path, basePath) + path = strings.TrimPrefix(path, SlashSeparator) + m := strings.Index(path, SlashSeparator) + if m < 0 { + return path, "" + } + return path[:m], path[m+len(SlashSeparator):] +} + +func path2BucketObject(s string) (bucket, prefix string) { + return path2BucketObjectWithBasePath("", s) +} + +// cloneMSS will clone a map[string]string. +// If input is nil an empty map is returned, not nil. +func cloneMSS(v map[string]string) map[string]string { + r := make(map[string]string, len(v)) + maps.Copy(r, v) + return r +} + +// URI scheme constants. +const ( + httpScheme = "http" + httpsScheme = "https" +) + +// nopCharsetConverter is a dummy charset convert which just copies input to output, +// it is used to ignore custom encoding charset in S3 XML body. +func nopCharsetConverter(label string, input io.Reader) (io.Reader, error) { + return input, nil +} + +// xmlDecoder provide decoded value in xml. +func xmlDecoder(body io.Reader, v any, size int64) error { + var lbody io.Reader + if size > 0 { + lbody = io.LimitReader(body, size) + } else { + lbody = body + } + d := xml.NewDecoder(lbody) + // Ignore any encoding set in the XML body + d.CharsetReader = nopCharsetConverter + err := d.Decode(v) + if errors.Is(err, io.EOF) { + err = &xml.SyntaxError{ + Line: 0, + Msg: err.Error(), + } + } + return err +} + +// validateLengthAndChecksum returns if a content checksum is set, +// and will replace r.Body with a reader that checks the provided checksum +func validateLengthAndChecksum(r *http.Request) bool { + if mdFive := r.Header.Get(xhttp.ContentMD5); mdFive != "" { + want, err := base64.StdEncoding.DecodeString(mdFive) + if err != nil { + return false + } + r.Body = hash.NewChecker(r.Body, md5.New(), want, r.ContentLength) + return true + } + cs, err := hash.GetContentChecksum(r.Header) + if err != nil { + return false + } + if cs == nil || !cs.Type.IsSet() { + return false + } + if cs.Valid() && !cs.Type.Trailing() { + r.Body = hash.NewChecker(r.Body, cs.Type.Hasher(), cs.Raw, r.ContentLength) + } + return true +} + +// http://docs.aws.amazon.com/AmazonS3/latest/dev/UploadingObjects.html +const ( + // Maximum object size per PUT request is 5TB. + // This is a divergence from S3 limit on purpose to support + // use cases where users are going to upload large files + // using 'curl' and presigned URL. + globalMaxObjectSize = 5 * humanize.TiByte + + // Minimum Part size for multipart upload is 5MiB + globalMinPartSize = 5 * humanize.MiByte + + // Maximum Part ID for multipart upload is 10000 + // (Acceptable values range from 1 to 10000 inclusive) + globalMaxPartID = 10000 +) + +// isMaxObjectSize - verify if max object size +func isMaxObjectSize(size int64) bool { + return size > globalMaxObjectSize +} + +// Check if part size is more than or equal to minimum allowed size. +func isMinAllowedPartSize(size int64) bool { + return size >= globalMinPartSize +} + +// isMaxPartNumber - Check if part ID is greater than the maximum allowed ID. +func isMaxPartID(partID int) bool { + return partID > globalMaxPartID +} + +// profilerWrapper is created because pkg/profiler doesn't +// provide any API to calculate the profiler file path in the +// disk since the name of this latter is randomly generated. +type profilerWrapper struct { + // Profile recorded at start of benchmark. + records map[string][]byte + stopFn func() ([]byte, error) + ext string +} + +// record will record the profile and store it as the base. +func (p *profilerWrapper) record(profileType string, debug int, recordName string) { + var buf bytes.Buffer + if p.records == nil { + p.records = make(map[string][]byte) + } + err := pprof.Lookup(profileType).WriteTo(&buf, debug) + if err != nil { + return + } + p.records[recordName] = buf.Bytes() +} + +// Records returns the recorded profiling if any. +func (p profilerWrapper) Records() map[string][]byte { + return p.records +} + +// Stop the currently running benchmark. +func (p profilerWrapper) Stop() ([]byte, error) { + return p.stopFn() +} + +// Extension returns the extension without dot prefix. +func (p profilerWrapper) Extension() string { + return p.ext +} + +// Returns current profile data, returns error if there is no active +// profiling in progress. Stops an active profile. +func getProfileData() (map[string][]byte, error) { + globalProfilerMu.Lock() + defer globalProfilerMu.Unlock() + + if len(globalProfiler) == 0 { + return nil, errors.New("profiler not enabled") + } + + dst := make(map[string][]byte, len(globalProfiler)) + for typ, prof := range globalProfiler { + // Stop the profiler + var err error + buf, err := prof.Stop() + delete(globalProfiler, typ) + if err == nil { + dst[typ+"."+prof.Extension()] = buf + } + for name, buf := range prof.Records() { + if len(buf) > 0 { + dst[typ+"-"+name+"."+prof.Extension()] = buf + } + } + } + return dst, nil +} + +func setDefaultProfilerRates() { + runtime.MemProfileRate = 128 << 10 // 512KB -> 128K - Must be constant throughout application lifetime. + runtime.SetMutexProfileFraction(0) // Disable until needed + runtime.SetBlockProfileRate(0) // Disable until needed +} + +// Starts a profiler returns nil if profiler is not enabled, caller needs to handle this. +func startProfiler(profilerType string) (minioProfiler, error) { + var prof profilerWrapper + prof.ext = "pprof" + // Enable profiler and set the name of the file that pkg/pprof + // library creates to store profiling data. + switch madmin.ProfilerType(profilerType) { + case madmin.ProfilerCPU: + dirPath, err := os.MkdirTemp("", "profile") + if err != nil { + return nil, err + } + fn := filepath.Join(dirPath, "cpu.out") + f, err := Create(fn) + if err != nil { + return nil, err + } + err = pprof.StartCPUProfile(f) + if err != nil { + return nil, err + } + prof.stopFn = func() ([]byte, error) { + pprof.StopCPUProfile() + err := f.Close() + if err != nil { + return nil, err + } + defer RemoveAll(dirPath) + return ioutilx.ReadFile(fn) + } + case madmin.ProfilerCPUIO: + // at 10k or more goroutines fgprof is likely to become + // unable to maintain its sampling rate and to significantly + // degrade the performance of your application + // https://github.com/felixge/fgprof#fgprof + if n := runtime.NumGoroutine(); n > 10000 && !globalIsCICD { + return nil, fmt.Errorf("unable to perform CPU IO profile with %d goroutines", n) + } + dirPath, err := os.MkdirTemp("", "profile") + if err != nil { + return nil, err + } + fn := filepath.Join(dirPath, "cpuio.out") + f, err := Create(fn) + if err != nil { + return nil, err + } + stop := fgprof.Start(f, fgprof.FormatPprof) + startedAt := time.Now() + prof.stopFn = func() ([]byte, error) { + if elapsed := time.Since(startedAt); elapsed < 100*time.Millisecond { + // Light hack around https://github.com/felixge/fgprof/pull/34 + time.Sleep(100*time.Millisecond - elapsed) + } + err := stop() + if err != nil { + return nil, err + } + err = f.Close() + if err != nil { + return nil, err + } + defer RemoveAll(dirPath) + return ioutilx.ReadFile(fn) + } + case madmin.ProfilerMEM: + runtime.GC() + prof.record("heap", 0, "before") + prof.stopFn = func() ([]byte, error) { + runtime.GC() + var buf bytes.Buffer + err := pprof.Lookup("heap").WriteTo(&buf, 0) + return buf.Bytes(), err + } + case madmin.ProfilerBlock: + runtime.SetBlockProfileRate(100) + prof.stopFn = func() ([]byte, error) { + var buf bytes.Buffer + err := pprof.Lookup("block").WriteTo(&buf, 0) + runtime.SetBlockProfileRate(0) + return buf.Bytes(), err + } + case madmin.ProfilerMutex: + prof.record("mutex", 0, "before") + runtime.SetMutexProfileFraction(1) + prof.stopFn = func() ([]byte, error) { + var buf bytes.Buffer + err := pprof.Lookup("mutex").WriteTo(&buf, 0) + runtime.SetMutexProfileFraction(0) + return buf.Bytes(), err + } + case madmin.ProfilerThreads: + prof.record("threadcreate", 0, "before") + prof.stopFn = func() ([]byte, error) { + var buf bytes.Buffer + err := pprof.Lookup("threadcreate").WriteTo(&buf, 0) + return buf.Bytes(), err + } + case madmin.ProfilerGoroutines: + prof.ext = "txt" + prof.record("goroutine", 1, "before") + prof.record("goroutine", 2, "before,debug=2") + prof.stopFn = func() ([]byte, error) { + var buf bytes.Buffer + err := pprof.Lookup("goroutine").WriteTo(&buf, 1) + return buf.Bytes(), err + } + case madmin.ProfilerTrace: + dirPath, err := os.MkdirTemp("", "profile") + if err != nil { + return nil, err + } + fn := filepath.Join(dirPath, "trace.out") + f, err := Create(fn) + if err != nil { + return nil, err + } + err = trace.Start(f) + if err != nil { + return nil, err + } + prof.ext = "trace" + prof.stopFn = func() ([]byte, error) { + trace.Stop() + err := f.Close() + if err != nil { + return nil, err + } + defer RemoveAll(dirPath) + return ioutilx.ReadFile(fn) + } + default: + return nil, errors.New("profiler type unknown") + } + + return prof, nil +} + +// minioProfiler - minio profiler interface. +type minioProfiler interface { + // Return recorded profiles, each profile associated with a distinct generic name. + Records() map[string][]byte + // Stop the profiler + Stop() ([]byte, error) + // Return extension of profile + Extension() string +} + +// Global profiler to be used by service go-routine. +var ( + globalProfiler map[string]minioProfiler + globalProfilerMu sync.Mutex +) + +// dump the request into a string in JSON format. +func dumpRequest(r *http.Request) string { + header := r.Header.Clone() + header.Set("Host", r.Host) + // Replace all '%' to '%%' so that printer format parser + // to ignore URL encoded values. + rawURI := strings.ReplaceAll(r.RequestURI, "%", "%%") + req := struct { + Method string `json:"method"` + RequestURI string `json:"reqURI"` + Header http.Header `json:"header"` + }{r.Method, rawURI, header} + + var buffer bytes.Buffer + enc := json.NewEncoder(&buffer) + enc.SetEscapeHTML(false) + if err := enc.Encode(&req); err != nil { + // Upon error just return Go-syntax representation of the value + return fmt.Sprintf("%#v", req) + } + + // Formatted string. + return strings.TrimSpace(buffer.String()) +} + +// isFile - returns whether given path is a file or not. +func isFile(path string) bool { + if fi, err := os.Stat(path); err == nil { + return fi.Mode().IsRegular() + } + + return false +} + +// UTCNow - returns current UTC time. +func UTCNow() time.Time { + return time.Now().UTC() +} + +// GenETag - generate UUID based ETag +func GenETag() string { + return ToS3ETag(getMD5Hash([]byte(mustGetUUID()))) +} + +// ToS3ETag - return checksum to ETag +func ToS3ETag(etag string) string { + etag = canonicalizeETag(etag) + + if !strings.HasSuffix(etag, "-1") { + // Tools like s3cmd uses ETag as checksum of data to validate. + // Append "-1" to indicate ETag is not a checksum. + etag += "-1" + } + + return etag +} + +// GetDefaultConnSettings returns default HTTP connection settings. +func GetDefaultConnSettings() xhttp.ConnSettings { + return xhttp.ConnSettings{ + LookupHost: globalDNSCache.LookupHost, + DialTimeout: rest.DefaultTimeout, + RootCAs: globalRootCAs, + TCPOptions: globalTCPOptions, + } +} + +// NewInternodeHTTPTransport returns a transport for internode MinIO +// connections. +func NewInternodeHTTPTransport(maxIdleConnsPerHost int) func() http.RoundTripper { + return xhttp.ConnSettings{ + LookupHost: globalDNSCache.LookupHost, + DialTimeout: rest.DefaultTimeout, + RootCAs: globalRootCAs, + CipherSuites: crypto.TLSCiphers(), + CurvePreferences: crypto.TLSCurveIDs(), + EnableHTTP2: false, + TCPOptions: globalTCPOptions, + }.NewInternodeHTTPTransport(maxIdleConnsPerHost) +} + +// NewHTTPTransportWithClientCerts returns a new http configuration +// used while communicating with the cloud backends. +func NewHTTPTransportWithClientCerts(clientCert, clientKey string) http.RoundTripper { + s := xhttp.ConnSettings{ + LookupHost: globalDNSCache.LookupHost, + DialTimeout: defaultDialTimeout, + RootCAs: globalRootCAs, + CipherSuites: crypto.TLSCiphersBackwardCompatible(), + CurvePreferences: crypto.TLSCurveIDs(), + TCPOptions: globalTCPOptions, + EnableHTTP2: false, + } + + if clientCert != "" && clientKey != "" { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + transport, err := s.NewHTTPTransportWithClientCerts(ctx, clientCert, clientKey) + if err != nil { + internalLogIf(ctx, fmt.Errorf("Unable to load client key and cert, please check your client certificate configuration: %w", err)) + } + if transport == nil { + // Client certs are not readable return default transport. + return s.NewHTTPTransportWithTimeout(1 * time.Minute) + } + return transport + } + + return globalRemoteTargetTransport +} + +// NewHTTPTransport returns a new http configuration +// used while communicating with the cloud backends. +func NewHTTPTransport() *http.Transport { + return NewHTTPTransportWithTimeout(1 * time.Minute) +} + +// Default values for dial timeout +const defaultDialTimeout = 5 * time.Second + +// NewHTTPTransportWithTimeout allows setting a timeout. +func NewHTTPTransportWithTimeout(timeout time.Duration) *http.Transport { + return xhttp.ConnSettings{ + LookupHost: globalDNSCache.LookupHost, + DialTimeout: defaultDialTimeout, + RootCAs: globalRootCAs, + TCPOptions: globalTCPOptions, + CipherSuites: crypto.TLSCiphersBackwardCompatible(), + CurvePreferences: crypto.TLSCurveIDs(), + EnableHTTP2: false, + }.NewHTTPTransportWithTimeout(timeout) +} + +// NewRemoteTargetHTTPTransport returns a new http configuration +// used while communicating with the remote replication targets. +func NewRemoteTargetHTTPTransport(insecure bool) func() *http.Transport { + return xhttp.ConnSettings{ + LookupHost: globalDNSCache.LookupHost, + RootCAs: globalRootCAs, + CipherSuites: crypto.TLSCiphersBackwardCompatible(), + CurvePreferences: crypto.TLSCurveIDs(), + TCPOptions: globalTCPOptions, + EnableHTTP2: false, + }.NewRemoteTargetHTTPTransport(insecure) +} + +// ceilFrac takes a numerator and denominator representing a fraction +// and returns its ceiling. If denominator is 0, it returns 0 instead +// of crashing. +func ceilFrac(numerator, denominator int64) (ceil int64) { + if denominator == 0 { + // do nothing on invalid input + return ceil + } + // Make denominator positive + if denominator < 0 { + numerator = -numerator + denominator = -denominator + } + ceil = numerator / denominator + if numerator > 0 && numerator%denominator != 0 { + ceil++ + } + return ceil +} + +// cleanMinioInternalMetadataKeys removes X-Amz-Meta- prefix from minio internal +// encryption metadata. +func cleanMinioInternalMetadataKeys(metadata map[string]string) map[string]string { + newMeta := make(map[string]string, len(metadata)) + for k, v := range metadata { + if strings.HasPrefix(k, "X-Amz-Meta-X-Minio-Internal-") { + newMeta[strings.TrimPrefix(k, "X-Amz-Meta-")] = v + } else { + newMeta[k] = v + } + } + return newMeta +} + +// pathClean is like path.Clean but does not return "." for +// empty inputs, instead returns "empty" as is. +func pathClean(p string) string { + cp := path.Clean(p) + if cp == "." { + return "" + } + return cp +} + +func trimLeadingSlash(ep string) string { + if len(ep) > 0 && ep[0] == '/' { + // Path ends with '/' preserve it + if ep[len(ep)-1] == '/' && len(ep) > 1 { + ep = path.Clean(ep) + ep += slashSeparator + } else { + ep = path.Clean(ep) + } + ep = ep[1:] + } + return ep +} + +// unescapeGeneric is similar to url.PathUnescape or url.QueryUnescape +// depending on input, additionally also handles situations such as +// `//` are normalized as `/`, also removes any `/` prefix before +// returning. +func unescapeGeneric(p string, escapeFn func(string) (string, error)) (string, error) { + ep, err := escapeFn(p) + if err != nil { + return "", err + } + return trimLeadingSlash(ep), nil +} + +// unescapePath is similar to unescapeGeneric but for specifically +// path unescaping. +func unescapePath(p string) (string, error) { + return unescapeGeneric(p, url.PathUnescape) +} + +// similar to unescapeGeneric but never returns any error if the unescaping +// fails, returns the input as is in such occasion, not meant to be +// used where strict validation is expected. +func likelyUnescapeGeneric(p string, escapeFn func(string) (string, error)) string { + ep, err := unescapeGeneric(p, escapeFn) + if err != nil { + return p + } + return ep +} + +func updateReqContext(ctx context.Context, objects ...ObjectV) context.Context { + req := logger.GetReqInfo(ctx) + if req != nil { + req.Lock() + defer req.Unlock() + req.Objects = make([]logger.ObjectVersion, 0, len(objects)) + for _, ov := range objects { + req.Objects = append(req.Objects, logger.ObjectVersion{ + ObjectName: ov.ObjectName, + VersionID: ov.VersionID, + }) + } + return logger.SetReqInfo(ctx, req) + } + return ctx +} + +// Returns context with ReqInfo details set in the context. +func newContext(r *http.Request, w http.ResponseWriter, api string) context.Context { + reqID := w.Header().Get(xhttp.AmzRequestID) + + vars := mux.Vars(r) + bucket := vars["bucket"] + object := likelyUnescapeGeneric(vars["object"], url.PathUnescape) + reqInfo := &logger.ReqInfo{ + DeploymentID: globalDeploymentID(), + RequestID: reqID, + RemoteHost: handlers.GetSourceIP(r), + Host: getHostName(r), + UserAgent: r.UserAgent(), + API: api, + BucketName: bucket, + ObjectName: object, + VersionID: strings.TrimSpace(r.Form.Get(xhttp.VersionID)), + } + + return logger.SetReqInfo(r.Context(), reqInfo) +} + +// Used for registering with rest handlers (have a look at registerStorageRESTHandlers for usage example) +// If it is passed ["aaaa", "bbbb"], it returns ["aaaa", "{aaaa:.*}", "bbbb", "{bbbb:.*}"] +func restQueries(keys ...string) []string { + var accumulator []string + for _, key := range keys { + accumulator = append(accumulator, key, "{"+key+":.*}") + } + return accumulator +} + +// Suffix returns the longest common suffix of the provided strings +func lcpSuffix(strs []string) string { + return lcp(strs, false) +} + +func lcp(strs []string, pre bool) string { + // short-circuit empty list + if len(strs) == 0 { + return "" + } + xfix := strs[0] + // short-circuit single-element list + if len(strs) == 1 { + return xfix + } + // compare first to rest + for _, str := range strs[1:] { + xfixl := len(xfix) + strl := len(str) + // short-circuit empty strings + if xfixl == 0 || strl == 0 { + return "" + } + // maximum possible length + maxl := min(strl, xfixl) + // compare letters + if pre { + // prefix, iterate left to right + for i := range maxl { + if xfix[i] != str[i] { + xfix = xfix[:i] + break + } + } + } else { + // suffix, iterate right to left + for i := range maxl { + xi := xfixl - i - 1 + si := strl - i - 1 + if xfix[xi] != str[si] { + xfix = xfix[xi+1:] + break + } + } + } + } + return xfix +} + +// Returns the mode in which MinIO is running +func getMinioMode() string { + switch { + case globalIsDistErasure: + return globalMinioModeDistErasure + case globalIsErasure: + return globalMinioModeErasure + case globalIsErasureSD: + return globalMinioModeErasureSD + default: + return globalMinioModeFS + } +} + +func iamPolicyClaimNameOpenID() string { + return globalIAMSys.OpenIDConfig.GetIAMPolicyClaimName() +} + +func iamPolicyClaimNameSA() string { + return "sa-policy" +} + +// On MinIO a directory object is stored as a regular object with "__XLDIR__" suffix. +// For ex. "prefix/" is stored as "prefix__XLDIR__" +func encodeDirObject(object string) string { + if HasSuffix(object, slashSeparator) { + return strings.TrimSuffix(object, slashSeparator) + globalDirSuffix + } + return object +} + +// Reverse process of encodeDirObject() +func decodeDirObject(object string) string { + if HasSuffix(object, globalDirSuffix) { + return strings.TrimSuffix(object, globalDirSuffix) + slashSeparator + } + return object +} + +func isDirObject(object string) bool { + if obj := encodeDirObject(object); obj != object { + object = obj + } + return HasSuffix(object, globalDirSuffix) +} + +// Helper method to return total number of nodes in cluster +func totalNodeCount() int { + totalNodesCount := len(globalEndpoints.Hostnames()) + if totalNodesCount == 0 { + totalNodesCount = 1 // For standalone erasure coding + } + return totalNodesCount +} + +// AuditLogOptions takes options for audit logging subsystem activity +type AuditLogOptions struct { + Event string + APIName string + Status string + Bucket string + Object string + VersionID string + Error string + Tags map[string]string +} + +// sends audit logs for internal subsystem activity +func auditLogInternal(ctx context.Context, opts AuditLogOptions) { + if len(logger.AuditTargets()) == 0 { + return + } + + entry := audit.NewEntry(globalDeploymentID()) + entry.Trigger = opts.Event + entry.Event = opts.Event + entry.Error = opts.Error + entry.API.Name = opts.APIName + entry.API.Bucket = opts.Bucket + entry.API.Objects = []xaudit.ObjectVersion{{ObjectName: opts.Object, VersionID: opts.VersionID}} + entry.API.Status = opts.Status + entry.Tags = make(map[string]any, len(opts.Tags)) + for k, v := range opts.Tags { + entry.Tags[k] = v + } + + // Merge tag information if found - this is currently needed for tags + // set during decommissioning. + if reqInfo := logger.GetReqInfo(ctx); reqInfo != nil { + reqInfo.PopulateTagsMap(opts.Tags) + } + ctx = logger.SetAuditEntry(ctx, &entry) + logger.AuditLog(ctx, nil, nil, nil) +} + +func newTLSConfig(getCert certs.GetCertificateFunc) *tls.Config { + if getCert == nil { + return nil + } + + tlsConfig := &tls.Config{ + PreferServerCipherSuites: true, + MinVersion: tls.VersionTLS12, + NextProtos: []string{"http/1.1", "h2"}, + GetCertificate: getCert, + ClientSessionCache: tls.NewLRUClientSessionCache(tlsClientSessionCacheSize), + } + + tlsClientIdentity := env.Get(xtls.EnvIdentityTLSEnabled, "") == config.EnableOn + if tlsClientIdentity { + tlsConfig.ClientAuth = tls.RequestClientCert + } + + if secureCiphers := env.Get(api.EnvAPISecureCiphers, config.EnableOn) == config.EnableOn; secureCiphers { + tlsConfig.CipherSuites = crypto.TLSCiphers() + } else { + tlsConfig.CipherSuites = crypto.TLSCiphersBackwardCompatible() + } + tlsConfig.CurvePreferences = crypto.TLSCurveIDs() + return tlsConfig +} + +/////////// Types and functions for OpenID IAM testing + +// OpenIDClientAppParams - contains openID client application params, used in +// testing. +type OpenIDClientAppParams struct { + ClientID, ClientSecret, ProviderURL, RedirectURL string +} + +// MockOpenIDTestUserInteraction - tries to login to dex using provided credentials. +// It performs the user's browser interaction to login and retrieves the auth +// code from dex and exchanges it for a JWT. +func MockOpenIDTestUserInteraction(ctx context.Context, pro OpenIDClientAppParams, username, password string) (string, error) { + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + + provider, err := oidc.NewProvider(ctx, pro.ProviderURL) + if err != nil { + return "", fmt.Errorf("unable to create provider: %v", err) + } + + // Configure an OpenID Connect aware OAuth2 client. + oauth2Config := oauth2.Config{ + ClientID: pro.ClientID, + ClientSecret: pro.ClientSecret, + RedirectURL: pro.RedirectURL, + + // Discovery returns the OAuth2 endpoints. + Endpoint: provider.Endpoint(), + + // "openid" is a required scope for OpenID Connect flows. + Scopes: []string{oidc.ScopeOpenID, "groups"}, + } + + state := fmt.Sprintf("x%dx", time.Now().Unix()) + authCodeURL := oauth2Config.AuthCodeURL(state) + // fmt.Printf("authcodeurl: %s\n", authCodeURL) + + var lastReq *http.Request + checkRedirect := func(req *http.Request, via []*http.Request) error { + // fmt.Printf("CheckRedirect:\n") + // fmt.Printf("Upcoming: %s %s\n", req.Method, req.URL.String()) + // for i, c := range via { + // fmt.Printf("Sofar %d: %s %s\n", i, c.Method, c.URL.String()) + // } + // Save the last request in a redirect chain. + lastReq = req + // We do not follow redirect back to client application. + if req.URL.Path == "/oauth_callback" { + return http.ErrUseLastResponse + } + return nil + } + + dexClient := http.Client{ + CheckRedirect: checkRedirect, + } + + u, err := url.Parse(authCodeURL) + if err != nil { + return "", fmt.Errorf("url parse err: %v", err) + } + + // Start the user auth flow. This page would present the login with + // email or LDAP option. + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return "", fmt.Errorf("new request err: %v", err) + } + _, err = dexClient.Do(req) + // fmt.Printf("Do: %#v %#v\n", resp, err) + if err != nil { + return "", fmt.Errorf("auth url request err: %v", err) + } + + // Modify u to choose the ldap option + u.Path += "/ldap" + // fmt.Println(u) + + // Pick the LDAP login option. This would return a form page after + // following some redirects. `lastReq` would be the URL of the form + // page, where we need to POST (submit) the form. + req, err = http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return "", fmt.Errorf("new request err (/ldap): %v", err) + } + _, err = dexClient.Do(req) + // fmt.Printf("Fetch LDAP login page: %#v %#v\n", resp, err) + if err != nil { + return "", fmt.Errorf("request err: %v", err) + } + // { + // bodyBuf, err := io.ReadAll(resp.Body) + // if err != nil { + // return "", fmt.Errorf("Error reading body: %v", err) + // } + // fmt.Printf("bodyBuf (for LDAP login page): %s\n", string(bodyBuf)) + // } + + // Fill the login form with our test creds: + // fmt.Printf("login form url: %s\n", lastReq.URL.String()) + formData := url.Values{} + formData.Set("login", username) + formData.Set("password", password) + req, err = http.NewRequestWithContext(ctx, http.MethodPost, lastReq.URL.String(), strings.NewReader(formData.Encode())) + if err != nil { + return "", fmt.Errorf("new request err (/login): %v", err) + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + _, err = dexClient.Do(req) + if err != nil { + return "", fmt.Errorf("post form err: %v", err) + } + // fmt.Printf("resp: %#v %#v\n", resp.StatusCode, resp.Header) + // bodyBuf, err := io.ReadAll(resp.Body) + // if err != nil { + // return "", fmt.Errorf("Error reading body: %v", err) + // } + // fmt.Printf("resp body: %s\n", string(bodyBuf)) + // fmt.Printf("lastReq: %#v\n", lastReq.URL.String()) + + // On form submission, the last redirect response contains the auth + // code, which we now have in `lastReq`. Exchange it for a JWT id_token. + q := lastReq.URL.Query() + // fmt.Printf("lastReq.URL: %#v q: %#v\n", lastReq.URL, q) + code := q.Get("code") + oauth2Token, err := oauth2Config.Exchange(ctx, code) + if err != nil { + return "", fmt.Errorf("unable to exchange code for id token: %v", err) + } + + rawIDToken, ok := oauth2Token.Extra("id_token").(string) + if !ok { + return "", fmt.Errorf("id_token not found!") + } + + // fmt.Printf("TOKEN: %s\n", rawIDToken) + return rawIDToken, nil +} + +// unwrapAll will unwrap the returned error completely. +func unwrapAll(err error) error { + for { + werr := errors.Unwrap(err) + if werr == nil { + return err + } + err = werr + } +} + +// stringsHasPrefixFold tests whether the string s begins with prefix ignoring case. +func stringsHasPrefixFold(s, prefix string) bool { + // Test match with case first. + return len(s) >= len(prefix) && (s[0:len(prefix)] == prefix || strings.EqualFold(s[0:len(prefix)], prefix)) +} + +func ptr[T any](a T) *T { + return &a +} + +// sleepContext sleeps for d duration or until ctx is done. +func sleepContext(ctx context.Context, d time.Duration) error { + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(d): + } + return nil +} + +// helper type to return either item or error. +type itemOrErr[V any] struct { + Item V + Err error +} + +func filterStorageClass(ctx context.Context, s string) string { + // Veeam 14.0 and later clients are not compatible with custom storage classes. + if globalVeeamForceSC != "" && s != storageclass.STANDARD && s != storageclass.RRS && isVeeamClient(ctx) { + return globalVeeamForceSC + } + return s +} + +type ordered interface { + ~int | ~int8 | ~int16 | ~int32 | ~int64 | ~uint | ~uint8 | ~uint16 | ~uint32 | ~uint64 | ~uintptr | ~float32 | ~float64 | string +} + +// mapKeysSorted returns the map keys as a sorted slice. +func mapKeysSorted[Map ~map[K]V, K ordered, V any](m Map) []K { + res := make([]K, 0, len(m)) + for k := range m { + res = append(res, k) + } + slices.Sort(res) + return res +} diff --git a/cmd/utils_test.go b/cmd/utils_test.go new file mode 100644 index 00000000..6d4e26a3 --- /dev/null +++ b/cmd/utils_test.go @@ -0,0 +1,400 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "reflect" + "strings" + "testing" +) + +// Tests maximum object size. +func TestMaxObjectSize(t *testing.T) { + sizes := []struct { + isMax bool + size int64 + }{ + // Test - 1 - maximum object size. + { + true, + globalMaxObjectSize + 1, + }, + // Test - 2 - not maximum object size. + { + false, + globalMaxObjectSize - 1, + }, + } + for i, s := range sizes { + isMax := isMaxObjectSize(s.size) + if isMax != s.isMax { + t.Errorf("Test %d: Expected %t, got %t", i+1, s.isMax, isMax) + } + } +} + +// Tests minimum allowed part size. +func TestMinAllowedPartSize(t *testing.T) { + sizes := []struct { + isMin bool + size int64 + }{ + // Test - 1 - within minimum part size. + { + true, + globalMinPartSize + 1, + }, + // Test - 2 - smaller than minimum part size. + { + false, + globalMinPartSize - 1, + }, + } + + for i, s := range sizes { + isMin := isMinAllowedPartSize(s.size) + if isMin != s.isMin { + t.Errorf("Test %d: Expected %t, got %t", i+1, s.isMin, isMin) + } + } +} + +// Tests maximum allowed part number. +func TestMaxPartID(t *testing.T) { + sizes := []struct { + isMax bool + partN int + }{ + // Test - 1 part number within max part number. + { + false, + globalMaxPartID - 1, + }, + // Test - 2 part number bigger than max part number. + { + true, + globalMaxPartID + 1, + }, + } + + for i, s := range sizes { + isMax := isMaxPartID(s.partN) + if isMax != s.isMax { + t.Errorf("Test %d: Expected %t, got %t", i+1, s.isMax, isMax) + } + } +} + +// Tests extracting bucket and objectname from various types of paths. +func TestPath2BucketObjectName(t *testing.T) { + testCases := []struct { + path string + bucket, object string + }{ + // Test case 1 normal case. + { + path: "/bucket/object", + bucket: "bucket", + object: "object", + }, + // Test case 2 where url only has separator. + { + path: SlashSeparator, + bucket: "", + object: "", + }, + // Test case 3 only bucket is present. + { + path: "/bucket", + bucket: "bucket", + object: "", + }, + // Test case 4 many separators and object is a directory. + { + path: "/bucket/object/1/", + bucket: "bucket", + object: "object/1/", + }, + // Test case 5 object has many trailing separators. + { + path: "/bucket/object/1///", + bucket: "bucket", + object: "object/1///", + }, + // Test case 6 object has only trailing separators. + { + path: "/bucket/object///////", + bucket: "bucket", + object: "object///////", + }, + // Test case 7 object has preceding separators. + { + path: "/bucket////object////", + bucket: "bucket", + object: "///object////", + }, + // Test case 8 url path is empty. + { + path: "", + bucket: "", + object: "", + }, + } + + // Validate all test cases. + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + bucketName, objectName := path2BucketObject(testCase.path) + if bucketName != testCase.bucket { + t.Errorf("failed expected bucket name \"%s\", got \"%s\"", testCase.bucket, bucketName) + } + if objectName != testCase.object { + t.Errorf("failed expected bucket name \"%s\", got \"%s\"", testCase.object, objectName) + } + }) + } +} + +// Add tests for starting and stopping different profilers. +func TestStartProfiler(t *testing.T) { + _, err := startProfiler("") + if err == nil { + t.Fatal("Expected a non nil error, but nil error returned for invalid profiler.") + } +} + +// checkURL - checks if passed address correspond +func checkURL(urlStr string) (*url.URL, error) { + if urlStr == "" { + return nil, errors.New("Address cannot be empty") + } + u, err := url.Parse(urlStr) + if err != nil { + return nil, fmt.Errorf("`%s` invalid: %s", urlStr, err.Error()) + } + return u, nil +} + +// TestCheckURL tests valid url. +func TestCheckURL(t *testing.T) { + testCases := []struct { + urlStr string + shouldPass bool + }{ + {"", false}, + {":", false}, + {"http://localhost/", true}, + {"http://127.0.0.1/", true}, + {"proto://myhostname/path", true}, + } + + // Validates fetching local address. + for i, testCase := range testCases { + _, err := checkURL(testCase.urlStr) + if testCase.shouldPass && err != nil { + t.Errorf("Test %d: expected to pass but got an error: %v\n", i+1, err) + } + if !testCase.shouldPass && err == nil { + t.Errorf("Test %d: expected to fail but passed.", i+1) + } + } +} + +// Testing dumping request function. +func TestDumpRequest(t *testing.T) { + req, err := http.NewRequest(http.MethodGet, "http://localhost:9000?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=USWUXHGYZQYFYFFIT3RE%2F20170529%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20170529T190139Z&X-Amz-Expires=600&X-Amz-Signature=19b58080999df54b446fc97304eb8dda60d3df1812ae97f3e8783351bfd9781d&X-Amz-SignedHeaders=host&prefix=Hello%2AWorld%2A", nil) + if err != nil { + t.Fatal(err) + } + req.RequestURI = "/?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=USWUXHGYZQYFYFFIT3RE%2F20170529%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20170529T190139Z&X-Amz-Expires=600&X-Amz-Signature=19b58080999df54b446fc97304eb8dda60d3df1812ae97f3e8783351bfd9781d&X-Amz-SignedHeaders=host&prefix=Hello%2AWorld%2A" + req.Header.Set("content-md5", "====test") + jsonReq := dumpRequest(req) + type jsonResult struct { + Method string `json:"method"` + RequestURI string `json:"reqURI"` + Header http.Header `json:"header"` + } + res := jsonResult{} + if err = json.Unmarshal([]byte(strings.ReplaceAll(jsonReq, "%%", "%")), &res); err != nil { + t.Fatal(err) + } + + // Look for expected method. + if res.Method != http.MethodGet { + t.Fatalf("Unexpected method %s, expected 'GET'", res.Method) + } + + // Look for expected query values + expectedQuery := url.Values{} + expectedQuery.Set("prefix", "Hello*World*") + expectedQuery.Set("X-Amz-Algorithm", "AWS4-HMAC-SHA256") + expectedQuery.Set("X-Amz-Credential", "USWUXHGYZQYFYFFIT3RE/20170529/us-east-1/s3/aws4_request") + expectedQuery.Set("X-Amz-Date", "20170529T190139Z") + expectedQuery.Set("X-Amz-Expires", "600") + expectedQuery.Set("X-Amz-SignedHeaders", "host") + expectedQuery.Set("X-Amz-Signature", "19b58080999df54b446fc97304eb8dda60d3df1812ae97f3e8783351bfd9781d") + expectedRequestURI := "/?" + expectedQuery.Encode() + if !reflect.DeepEqual(res.RequestURI, expectedRequestURI) { + t.Fatalf("Expected %#v, got %#v", expectedRequestURI, res.RequestURI) + } + + // Look for expected header. + expectedHeader := http.Header{} + expectedHeader.Set("content-md5", "====test") + expectedHeader.Set("host", "localhost:9000") + if !reflect.DeepEqual(res.Header, expectedHeader) { + t.Fatalf("Expected %#v, got %#v", expectedHeader, res.Header) + } +} + +// Test ToS3ETag() +func TestToS3ETag(t *testing.T) { + testCases := []struct { + etag string + expectedETag string + }{ + {`"8019e762"`, `8019e762-1`}, + {"5d57546eeb86b3eba68967292fba0644", "5d57546eeb86b3eba68967292fba0644-1"}, + {`"8019e762-1"`, `8019e762-1`}, + {"5d57546eeb86b3eba68967292fba0644-1", "5d57546eeb86b3eba68967292fba0644-1"}, + } + for i, testCase := range testCases { + etag := ToS3ETag(testCase.etag) + if etag != testCase.expectedETag { + t.Fatalf("test %v: expected: %v, got: %v", i+1, testCase.expectedETag, etag) + } + } +} + +// Test ceilFrac +func TestCeilFrac(t *testing.T) { + cases := []struct { + numerator, denominator, ceiling int64 + }{ + {0, 1, 0}, + {-1, 2, 0}, + {1, 2, 1}, + {1, 1, 1}, + {3, 2, 2}, + {54, 11, 5}, + {45, 11, 5}, + {-4, 3, -1}, + {4, -3, -1}, + {-4, -3, 2}, + {3, 0, 0}, + } + for i, testCase := range cases { + ceiling := ceilFrac(testCase.numerator, testCase.denominator) + if ceiling != testCase.ceiling { + t.Errorf("Case %d: Unexpected result: %d", i, ceiling) + } + } +} + +// Test if isErrIgnored works correctly. +func TestIsErrIgnored(t *testing.T) { + errIgnored := fmt.Errorf("ignored error") + testCases := []struct { + err error + ignored bool + }{ + { + err: nil, + ignored: false, + }, + { + err: errIgnored, + ignored: true, + }, + { + err: errFaultyDisk, + ignored: true, + }, + } + for i, testCase := range testCases { + if ok := IsErrIgnored(testCase.err, append(baseIgnoredErrs, errIgnored)...); ok != testCase.ignored { + t.Errorf("Test: %d, Expected %t, got %t", i+1, testCase.ignored, ok) + } + } +} + +// Test queries() +func TestQueries(t *testing.T) { + testCases := []struct { + keys []string + keyvalues []string + }{ + { + []string{"aaaa", "bbbb"}, + []string{"aaaa", "{aaaa:.*}", "bbbb", "{bbbb:.*}"}, + }, + } + + for i, test := range testCases { + keyvalues := restQueries(test.keys...) + for j := range keyvalues { + if keyvalues[j] != test.keyvalues[j] { + t.Fatalf("test %d: keyvalues[%d] does not match", i+1, j) + } + } + } +} + +func TestLCP(t *testing.T) { + testCases := []struct { + prefixes []string + commonPrefix string + }{ + {[]string{"", ""}, ""}, + {[]string{"a", "b"}, ""}, + {[]string{"a", "a"}, "a"}, + {[]string{"a/", "a/"}, "a/"}, + {[]string{"abcd/", ""}, ""}, + {[]string{"abcd/foo/", "abcd/bar/"}, "abcd/"}, + {[]string{"abcd/foo/bar/", "abcd/foo/bar/zoo"}, "abcd/foo/bar/"}, + } + + for i, test := range testCases { + foundPrefix := lcp(test.prefixes, true) + if foundPrefix != test.commonPrefix { + t.Fatalf("Test %d: Common prefix found: `%v`, expected: `%v`", i+1, foundPrefix, test.commonPrefix) + } + } +} + +func TestGetMinioMode(t *testing.T) { + testMinioMode := func(expected string) { + if mode := getMinioMode(); mode != expected { + t.Fatalf("Expected %s got %s", expected, mode) + } + } + globalIsDistErasure = true + testMinioMode(globalMinioModeDistErasure) + + globalIsDistErasure = false + globalIsErasure = true + testMinioMode(globalMinioModeErasure) + + globalIsDistErasure, globalIsErasure = false, false + testMinioMode(globalMinioModeFS) +} diff --git a/cmd/veeam-sos-api.go b/cmd/veeam-sos-api.go new file mode 100644 index 00000000..33ff9e13 --- /dev/null +++ b/cmd/veeam-sos-api.go @@ -0,0 +1,226 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "encoding/xml" + "io" + "os" + "strings" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/logger" +) + +// From Veeam-SOSAPI_1.0_Document_v1.02d.pdf +// - SOSAPI Protocol Version +// - Model Name of the vendor plus version for statistical analysis. +// - List of Smart Object Storage protocol capabilities supported by the server. +// Currently, there are three capabilities supported: +// - Capacity Reporting +// - Backup data locality for upload sessions (Veeam Smart Entity) +// - Handover of IAM & STS Endpoints instead of manual definition in Veeam Backup & Replication. This allows Veeam +// Agents to directly backup to object storage. +// +// An object storage system can implement one, multiple, or all functions. +// +// - Optional (mandatory if is true): Set Endpoints for IAM and STS processing. +// +// - Optional: Set server preferences for Backup & Replication parallel sessions, batch size of deletes, and block sizes (before +// compression). This is an optional area; by default, there should be no section in the +// system.xml. Vendors can work with Veeam Product Management and the Alliances team on getting approval to integrate +// specific system recommendations based on current support case statistics and storage performance possibilities. +// Vendors might change the settings based on the configuration and scale out of the solution (more storage nodes => +// higher task limit). +// +// +// +// - Defines how many S3 operations are executed parallel within one Repository Task Slot (and within one backup object +// that gets offloaded). The same registry key setting overwrites the storage-defined setting. +// Optional value, default 64, range: 1-unlimited +// +// - +// Some of the Veeam products use Multi Delete operations. This setting can reduce how many objects are included in one +// multi-delete operation. The same registry key setting overwrites the storage-defined setting. +// Optional value, default 1000, range: 1-unlimited (S3 standard maximum is 1000 and should not be set higher) +// +// - +// Setting reduces the parallel Repository Task slots that offload or write data to object storage. The same user interface +// setting overwrites the storage-defined setting. +// Optional value, default 0, range: 0-unlimited (0 equals unlimited, which means the maximum configured repository task +// slots are used for object offloading or writing) +// +// - +// Veeam Block Size for backup and restore processing before compression is applied. The higher the block size, the more +// backup space is needed for incremental backups. Larger block sizes also mean less performance for random read restore +// methods like Instant Restore, File Level Recovery, and Database/Application restores. Veeam recommends that vendors +// optimize the storage system for the default value of 1MB minus compression object sizes. The setting simultaneously +// affects read from source, block, file, dedup, and object storage backup targets for a specific Veeam Job. When customers +// create a new backup job and select the object storage or a SOBR as a backup target with this setting, the job default +// setting will be set to this value. This setting will be only applied to newly created jobs (manual changes with Active Full +// processing possible from the customer side). +// Optional value, default 1024, allowed values 256,512,1024,4096,8192, value defined in KB size. +// +// - The object should be present in all buckets accessed by Veeam products that want to leverage the SOSAPI functionality. +// +// - The current protocol version is 1.0. +type apiEndpoints struct { + IAMEndpoint string `xml:"IAMEndpoint"` + STSEndpoint string `xml:"STSEndpoint"` +} + +// globalVeeamForceSC is set by the environment variable _MINIO_VEEAM_FORCE_SC +// This will override the storage class returned by the storage backend if it is non-standard +// and we detect a Veeam client by checking the User Agent. +var globalVeeamForceSC = os.Getenv("_MINIO_VEEAM_FORCE_SC") + +type systemInfo struct { + XMLName xml.Name `xml:"SystemInfo" json:"-"` + ProtocolVersion string `xml:"ProtocolVersion"` + ModelName string `xml:"ModelName"` + ProtocolCapabilities struct { + CapacityInfo bool `xml:"CapacityInfo"` + UploadSessions bool `xml:"UploadSessions"` + IAMSTS bool `xml:"IAMSTS"` + } `mxl:"ProtocolCapabilities"` + APIEndpoints *apiEndpoints `xml:"APIEndpoints,omitempty"` + SystemRecommendations struct { + S3ConcurrentTaskLimit int `xml:"S3ConcurrentTaskLimit,omitempty"` + S3MultiObjectDeleteLimit int `xml:"S3MultiObjectDeleteLimit,omitempty"` + StorageCurrentTaskLimit int `xml:"StorageCurrentTaskLimit,omitempty"` + KBBlockSize int `xml:"KbBlockSize"` + } `xml:"SystemRecommendations"` +} + +// This optional functionality allows vendors to report space information to Veeam products, and Veeam will make placement +// decisions based on this information. For example, Veeam Backup & Replication has a Scale-out-Backup-Repository feature where +// multiple buckets can be used together. The placement logic for additional backup files is based on available space. Other values +// will augment the Veeam user interface and statistics, including free space warnings. +type capacityInfo struct { + XMLName xml.Name `xml:"CapacityInfo" json:"-"` + Capacity int64 `xml:"Capacity"` + Available int64 `xml:"Available"` + Used int64 `xml:"Used"` +} + +const ( + systemXMLObject = ".system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/system.xml" + capacityXMLObject = ".system-d26a9498-cb7c-4a87-a44a-8ae204f5ba6c/capacity.xml" + veeamAgentSubstr = "APN/1.0 Veeam/1.0" +) + +func isVeeamSOSAPIObject(object string) bool { + switch object { + case systemXMLObject, capacityXMLObject: + return true + default: + return false + } +} + +// isVeeamClient - returns true if the request is from Veeam client. +func isVeeamClient(ctx context.Context) bool { + ri := logger.GetReqInfo(ctx) + return ri != nil && strings.Contains(ri.UserAgent, veeamAgentSubstr) +} + +func veeamSOSAPIHeadObject(ctx context.Context, bucket, object string, opts ObjectOptions) (ObjectInfo, error) { + gr, err := veeamSOSAPIGetObject(ctx, bucket, object, nil, opts) + if gr != nil { + gr.Close() + return gr.ObjInfo, nil + } + return ObjectInfo{}, err +} + +func veeamSOSAPIGetObject(ctx context.Context, bucket, object string, rs *HTTPRangeSpec, opts ObjectOptions) (gr *GetObjectReader, err error) { + var buf []byte + switch object { + case systemXMLObject: + si := systemInfo{ + ProtocolVersion: `"1.0"`, + ModelName: "\"MinIO " + ReleaseTag + "\"", + } + si.ProtocolCapabilities.CapacityInfo = true + + // Default recommended block size with MinIO + si.SystemRecommendations.KBBlockSize = 4096 + + buf = encodeResponse(&si) + case capacityXMLObject: + objAPI := newObjectLayerFn() + if objAPI == nil { + return nil, errServerNotInitialized + } + + q, _ := globalBucketQuotaSys.Get(ctx, bucket) + binfo := globalBucketQuotaSys.GetBucketUsageInfo(ctx, bucket) + + ci := capacityInfo{ + Used: int64(binfo.Size), + } + + var quotaSize int64 + if q != nil && q.Type == madmin.HardQuota { + if q.Size > 0 { + quotaSize = int64(q.Size) + } else if q.Quota > 0 { + quotaSize = int64(q.Quota) + } + } + + if quotaSize == 0 { + info := objAPI.StorageInfo(ctx, true) + info.Backend = objAPI.BackendInfo() + + ci.Capacity = int64(GetTotalUsableCapacity(info.Disks, info)) + } else { + ci.Capacity = quotaSize + } + ci.Available = ci.Capacity - ci.Used + + buf = encodeResponse(&ci) + default: + return nil, errFileNotFound + } + + etag := getMD5Hash(buf) + r := bytes.NewReader(buf) + + off, length := int64(0), r.Size() + if rs != nil { + off, length, err = rs.GetOffsetLength(r.Size()) + if err != nil { + return nil, err + } + } + r.Seek(off, io.SeekStart) + + return NewGetObjectReaderFromReader(io.LimitReader(r, length), ObjectInfo{ + Bucket: bucket, + Name: object, + Size: r.Size(), + IsLatest: true, + ContentType: string(mimeXML), + NumVersions: 1, + ETag: etag, + ModTime: UTCNow(), + }, opts) +} diff --git a/cmd/version_test.go b/cmd/version_test.go new file mode 100644 index 00000000..ef0ee6bb --- /dev/null +++ b/cmd/version_test.go @@ -0,0 +1,31 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "testing" + "time" +) + +func TestVersion(t *testing.T) { + Version = "2017-05-07T06:37:49Z" + _, err := time.Parse(time.RFC3339, Version) + if err != nil { + t.Fatal(err) + } +} diff --git a/cmd/warm-backend-azure.go b/cmd/warm-backend-azure.go new file mode 100644 index 00000000..bb087e74 --- /dev/null +++ b/cmd/warm-backend-azure.go @@ -0,0 +1,249 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "strings" + + "github.com/Azure/azure-sdk-for-go/sdk/azcore" + "github.com/Azure/azure-sdk-for-go/sdk/azcore/to" + "github.com/Azure/azure-sdk-for-go/sdk/azidentity" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob" + "github.com/Azure/azure-sdk-for-go/sdk/storage/azblob/blob" + "github.com/minio/madmin-go/v3" +) + +type warmBackendAzure struct { + clnt *azblob.Client + Bucket string + Prefix string + StorageClass string +} + +func (az *warmBackendAzure) tier() *blob.AccessTier { + if az.StorageClass == "" { + return nil + } + for _, t := range blob.PossibleAccessTierValues() { + if strings.EqualFold(az.StorageClass, string(t)) { + return &t + } + } + return nil +} + +func (az *warmBackendAzure) getDest(object string) string { + destObj := object + if az.Prefix != "" { + destObj = fmt.Sprintf("%s/%s", az.Prefix, object) + } + return destObj +} + +func (az *warmBackendAzure) PutWithMeta(ctx context.Context, object string, r io.Reader, length int64, meta map[string]string) (remoteVersionID, error) { + azMeta := map[string]*string{} + for k, v := range meta { + azMeta[k] = to.Ptr(v) + } + resp, err := az.clnt.UploadStream(ctx, az.Bucket, az.getDest(object), io.LimitReader(r, length), &azblob.UploadStreamOptions{ + Concurrency: 4, + AccessTier: az.tier(), // set tier if specified + Metadata: azMeta, + }) + if err != nil { + return "", azureToObjectError(err, az.Bucket, az.getDest(object)) + } + vid := "" + if resp.VersionID != nil { + vid = *resp.VersionID + } + return remoteVersionID(vid), nil +} + +func (az *warmBackendAzure) Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) { + return az.PutWithMeta(ctx, object, r, length, map[string]string{}) +} + +func (az *warmBackendAzure) Get(ctx context.Context, object string, rv remoteVersionID, opts WarmBackendGetOpts) (r io.ReadCloser, err error) { + if opts.startOffset < 0 { + return nil, InvalidRange{} + } + resp, err := az.clnt.DownloadStream(ctx, az.Bucket, az.getDest(object), &azblob.DownloadStreamOptions{ + Range: blob.HTTPRange{Offset: opts.startOffset, Count: opts.length}, + }) + if err != nil { + return nil, azureToObjectError(err, az.Bucket, az.getDest(object)) + } + + return resp.Body, nil +} + +func (az *warmBackendAzure) Remove(ctx context.Context, object string, rv remoteVersionID) error { + _, err := az.clnt.DeleteBlob(ctx, az.Bucket, az.getDest(object), &azblob.DeleteBlobOptions{}) + return azureToObjectError(err, az.Bucket, az.getDest(object)) +} + +func (az *warmBackendAzure) InUse(ctx context.Context) (bool, error) { + maxResults := int32(1) + pager := az.clnt.NewListBlobsFlatPager(az.Bucket, &azblob.ListBlobsFlatOptions{ + Prefix: &az.Prefix, + MaxResults: &maxResults, + }) + if !pager.More() { + return false, nil + } + + resp, err := pager.NextPage(ctx) + if err != nil { + if strings.Contains(err.Error(), "no more pages") { + return false, nil + } + return false, azureToObjectError(err, az.Bucket, az.Prefix) + } + + return len(resp.Segment.BlobItems) > 0, nil +} + +type azureConf struct { + madmin.TierAzure +} + +func (conf azureConf) Validate() error { + switch { + case conf.AccountName == "": + return errors.New("the account name is required") + case conf.AccountKey != "" && (conf.SPAuth.TenantID != "" || conf.SPAuth.ClientID != "" || conf.SPAuth.ClientSecret != ""): + return errors.New("multiple authentication mechanisms are provided") + case conf.AccountKey == "" && (conf.SPAuth.TenantID == "" || conf.SPAuth.ClientID == "" || conf.SPAuth.ClientSecret == ""): + return errors.New("no authentication mechanism was provided") + } + + if conf.Bucket == "" { + return errors.New("no bucket name was provided") + } + + return nil +} + +func (conf azureConf) NewClient() (clnt *azblob.Client, clntErr error) { + if err := conf.Validate(); err != nil { + return nil, err + } + + ep := conf.Endpoint + if ep == "" { + ep = fmt.Sprintf("https://%s.blob.core.windows.net", conf.AccountName) + } + + if conf.IsSPEnabled() { + credential, err := azidentity.NewClientSecretCredential(conf.SPAuth.TenantID, conf.SPAuth.ClientID, conf.SPAuth.ClientSecret, &azidentity.ClientSecretCredentialOptions{}) + if err != nil { + return nil, err + } + return azblob.NewClient(ep, credential, &azblob.ClientOptions{}) + } + credential, err := azblob.NewSharedKeyCredential(conf.AccountName, conf.AccountKey) + if err != nil { + return nil, err + } + return azblob.NewClientWithSharedKeyCredential(ep, credential, &azblob.ClientOptions{}) +} + +func newWarmBackendAzure(conf madmin.TierAzure, _ string) (*warmBackendAzure, error) { + clnt, err := azureConf{conf}.NewClient() + if err != nil { + return nil, err + } + + return &warmBackendAzure{ + clnt: clnt, + Bucket: conf.Bucket, + Prefix: strings.TrimSuffix(conf.Prefix, slashSeparator), + StorageClass: conf.StorageClass, + }, nil +} + +// Convert azure errors to minio object layer errors. +func azureToObjectError(err error, params ...string) error { + if err == nil { + return nil + } + + bucket := "" + object := "" + if len(params) >= 1 { + bucket = params[0] + } + if len(params) == 2 { + object = params[1] + } + + azureErr, ok := err.(*azcore.ResponseError) + if !ok { + // We don't interpret non Azure errors. As azure errors will + // have StatusCode to help to convert to object errors. + return err + } + + serviceCode := azureErr.ErrorCode + statusCode := azureErr.StatusCode + + return azureCodesToObjectError(err, serviceCode, statusCode, bucket, object) +} + +func azureCodesToObjectError(err error, serviceCode string, statusCode int, bucket string, object string) error { + switch serviceCode { + case "ContainerNotFound", "ContainerBeingDeleted": + err = BucketNotFound{Bucket: bucket} + case "ContainerAlreadyExists": + err = BucketExists{Bucket: bucket} + case "InvalidResourceName": + err = BucketNameInvalid{Bucket: bucket} + case "RequestBodyTooLarge": + err = PartTooBig{} + case "InvalidMetadata": + err = UnsupportedMetadata{} + case "BlobAccessTierNotSupportedForAccountType": + err = NotImplemented{} + case "OutOfRangeInput": + err = ObjectNameInvalid{ + Bucket: bucket, + Object: object, + } + default: + switch statusCode { + case http.StatusNotFound: + if object != "" { + err = ObjectNotFound{ + Bucket: bucket, + Object: object, + } + } else { + err = BucketNotFound{Bucket: bucket} + } + case http.StatusBadRequest: + err = BucketNameInvalid{Bucket: bucket} + } + } + return err +} diff --git a/cmd/warm-backend-gcs.go b/cmd/warm-backend-gcs.go new file mode 100644 index 00000000..cbfcd422 --- /dev/null +++ b/cmd/warm-backend-gcs.go @@ -0,0 +1,229 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + + "cloud.google.com/go/storage" + "github.com/minio/madmin-go/v3" + "google.golang.org/api/googleapi" + "google.golang.org/api/iterator" + "google.golang.org/api/option" + + xioutil "github.com/minio/minio/internal/ioutil" +) + +type warmBackendGCS struct { + client *storage.Client + Bucket string + Prefix string + StorageClass string +} + +func (gcs *warmBackendGCS) getDest(object string) string { + destObj := object + if gcs.Prefix != "" { + destObj = fmt.Sprintf("%s/%s", gcs.Prefix, object) + } + return destObj +} + +func (gcs *warmBackendGCS) PutWithMeta(ctx context.Context, key string, data io.Reader, length int64, meta map[string]string) (remoteVersionID, error) { + object := gcs.client.Bucket(gcs.Bucket).Object(gcs.getDest(key)) + w := object.NewWriter(ctx) + if gcs.StorageClass != "" { + w.StorageClass = gcs.StorageClass + } + w.Metadata = meta + if _, err := xioutil.Copy(w, data); err != nil { + return "", gcsToObjectError(err, gcs.Bucket, key) + } + + if _, err := xioutil.Copy(w, data); err != nil { + return "", gcsToObjectError(err, gcs.Bucket, key) + } + + return "", w.Close() +} + +// FIXME: add support for remote version ID in GCS remote tier and remove this. +// Currently it's a no-op. +func (gcs *warmBackendGCS) Put(ctx context.Context, key string, data io.Reader, length int64) (remoteVersionID, error) { + return gcs.PutWithMeta(ctx, key, data, length, map[string]string{}) +} + +func (gcs *warmBackendGCS) Get(ctx context.Context, key string, rv remoteVersionID, opts WarmBackendGetOpts) (r io.ReadCloser, err error) { + // GCS storage decompresses a gzipped object by default and returns the data. + // Refer to https://cloud.google.com/storage/docs/transcoding#decompressive_transcoding + // Need to set `Accept-Encoding` header to `gzip` when issuing a GetObject call, to be able + // to download the object in compressed state. + // Calling ReadCompressed with true accomplishes that. + object := gcs.client.Bucket(gcs.Bucket).Object(gcs.getDest(key)).ReadCompressed(true) + + r, err = object.NewRangeReader(ctx, opts.startOffset, opts.length) + if err != nil { + return nil, gcsToObjectError(err, gcs.Bucket, key) + } + return r, nil +} + +func (gcs *warmBackendGCS) Remove(ctx context.Context, key string, rv remoteVersionID) error { + err := gcs.client.Bucket(gcs.Bucket).Object(gcs.getDest(key)).Delete(ctx) + return gcsToObjectError(err, gcs.Bucket, key) +} + +func (gcs *warmBackendGCS) InUse(ctx context.Context) (bool, error) { + it := gcs.client.Bucket(gcs.Bucket).Objects(ctx, &storage.Query{ + Delimiter: "/", + Prefix: gcs.Prefix, + Versions: false, + }) + pager := iterator.NewPager(it, 1, "") + gcsObjects := make([]*storage.ObjectAttrs, 0) + _, err := pager.NextPage(&gcsObjects) + if err != nil { + return false, gcsToObjectError(err, gcs.Bucket, gcs.Prefix) + } + if len(gcsObjects) > 0 { + return true, nil + } + return false, nil +} + +func newWarmBackendGCS(conf madmin.TierGCS, tier string) (*warmBackendGCS, error) { + // Validation code + if conf.Creds == "" { + return nil, errors.New("empty credentials unsupported") + } + + if conf.Bucket == "" { + return nil, errors.New("no bucket name was provided") + } + + credsJSON, err := conf.GetCredentialJSON() + if err != nil { + return nil, err + } + + client, err := storage.NewClient(context.Background(), + option.WithCredentialsJSON(credsJSON), + option.WithScopes(storage.ScopeReadWrite), + option.WithUserAgent(fmt.Sprintf("gcs-tier-%s", tier)+SlashSeparator+ReleaseTag), + ) + if err != nil { + return nil, err + } + return &warmBackendGCS{client, conf.Bucket, conf.Prefix, conf.StorageClass}, nil +} + +// Convert GCS errors to minio object layer errors. +func gcsToObjectError(err error, params ...string) error { + if err == nil { + return nil + } + + bucket := "" + object := "" + uploadID := "" + if len(params) >= 1 { + bucket = params[0] + } + if len(params) == 2 { + object = params[1] + } + if len(params) == 3 { + uploadID = params[2] + } + + // in some cases just a plain error is being returned + switch err.Error() { + case "storage: bucket doesn't exist": + err = BucketNotFound{ + Bucket: bucket, + } + return err + case "storage: object doesn't exist": + if uploadID != "" { + err = InvalidUploadID{ + UploadID: uploadID, + } + } else { + err = ObjectNotFound{ + Bucket: bucket, + Object: object, + } + } + return err + } + + googleAPIErr, ok := err.(*googleapi.Error) + if !ok { + // We don't interpret non MinIO errors. As minio errors will + // have StatusCode to help to convert to object errors. + return err + } + + if len(googleAPIErr.Errors) == 0 { + return err + } + + reason := googleAPIErr.Errors[0].Reason + message := googleAPIErr.Errors[0].Message + + switch reason { + case "required": + // Anonymous users does not have storage.xyz access to project 123. + fallthrough + case "keyInvalid": + fallthrough + case "forbidden": + err = PrefixAccessDenied{ + Bucket: bucket, + Object: object, + } + case "invalid": + err = BucketNameInvalid{ + Bucket: bucket, + } + case "notFound": + if object != "" { + err = ObjectNotFound{ + Bucket: bucket, + Object: object, + } + break + } + err = BucketNotFound{Bucket: bucket} + case "conflict": + if message == "You already own this bucket. Please select another name." { + err = BucketAlreadyOwnedByYou{Bucket: bucket} + break + } + if message == "Sorry, that name is not available. Please try a different one." { + err = BucketAlreadyExists{Bucket: bucket} + break + } + err = BucketNotEmpty{Bucket: bucket} + } + + return err +} diff --git a/cmd/warm-backend-minio.go b/cmd/warm-backend-minio.go new file mode 100644 index 00000000..56ca5120 --- /dev/null +++ b/cmd/warm-backend-minio.go @@ -0,0 +1,136 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "math" + "net/url" + "strings" + + "github.com/minio/madmin-go/v3" + minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type warmBackendMinIO struct { + warmBackendS3 +} + +var _ WarmBackend = (*warmBackendMinIO)(nil) + +const ( + maxMultipartPutObjectSize = 1024 * 1024 * 1024 * 1024 * 5 + maxPartsCount = 10000 + maxPartSize = 1024 * 1024 * 1024 * 5 + minPartSize = 1024 * 1024 * 128 // chosen by us to be optimal for HDDs +) + +// optimalPartInfo - calculate the optimal part info for a given +// object size. +// +// NOTE: Assumption here is that for any object to be uploaded to any S3 compatible +// object storage it will have the following parameters as constants. +// +// maxPartsCount - 10000 +// maxMultipartPutObjectSize - 5TiB +func optimalPartSize(objectSize int64) (partSize int64, err error) { + // object size is '-1' set it to 5TiB. + if objectSize == -1 { + objectSize = maxMultipartPutObjectSize + } + + // object size is larger than supported maximum. + if objectSize > maxMultipartPutObjectSize { + err = errors.New("entity too large") + return partSize, err + } + + configuredPartSize := minPartSize + // Use floats for part size for all calculations to avoid + // overflows during float64 to int64 conversions. + partSizeFlt := float64(objectSize / maxPartsCount) + partSizeFlt = math.Ceil(partSizeFlt/float64(configuredPartSize)) * float64(configuredPartSize) + + // Part size. + partSize = int64(partSizeFlt) + if partSize == 0 { + return minPartSize, nil + } + return partSize, nil +} + +func (m *warmBackendMinIO) PutWithMeta(ctx context.Context, object string, r io.Reader, length int64, meta map[string]string) (remoteVersionID, error) { + partSize, err := optimalPartSize(length) + if err != nil { + return remoteVersionID(""), err + } + res, err := m.client.PutObject(ctx, m.Bucket, m.getDest(object), r, length, minio.PutObjectOptions{ + StorageClass: m.StorageClass, + PartSize: uint64(partSize), + DisableContentSha256: true, + UserMetadata: meta, + }) + return remoteVersionID(res.VersionID), m.ToObjectError(err, object) +} + +func (m *warmBackendMinIO) Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) { + return m.PutWithMeta(ctx, object, r, length, map[string]string{}) +} + +func newWarmBackendMinIO(conf madmin.TierMinIO, tier string) (*warmBackendMinIO, error) { + // Validation of credentials + if conf.AccessKey == "" || conf.SecretKey == "" { + return nil, errors.New("both access and secret keys are required") + } + + if conf.Bucket == "" { + return nil, errors.New("no bucket name was provided") + } + + u, err := url.Parse(conf.Endpoint) + if err != nil { + return nil, err + } + + creds := credentials.NewStaticV4(conf.AccessKey, conf.SecretKey, "") + opts := &minio.Options{ + Creds: creds, + Secure: u.Scheme == "https", + Transport: globalRemoteTargetTransport, + TrailingHeaders: true, + } + client, err := minio.New(u.Host, opts) + if err != nil { + return nil, err + } + client.SetAppInfo(fmt.Sprintf("minio-tier-%s", tier), ReleaseTag) + + core := &minio.Core{Client: client} + return &warmBackendMinIO{ + warmBackendS3{ + client: client, + core: core, + Bucket: conf.Bucket, + Prefix: strings.TrimSuffix(conf.Prefix, slashSeparator), + }, + }, nil +} diff --git a/cmd/warm-backend-s3.go b/cmd/warm-backend-s3.go new file mode 100644 index 00000000..5905923b --- /dev/null +++ b/cmd/warm-backend-s3.go @@ -0,0 +1,182 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +type warmBackendS3 struct { + client *minio.Client + core *minio.Core + Bucket string + Prefix string + StorageClass string +} + +func (s3 *warmBackendS3) ToObjectError(err error, params ...string) error { + object := "" + if len(params) >= 1 { + object = params[0] + } + + return ErrorRespToObjectError(err, s3.Bucket, s3.getDest(object)) +} + +func (s3 *warmBackendS3) getDest(object string) string { + destObj := object + if s3.Prefix != "" { + destObj = fmt.Sprintf("%s/%s", s3.Prefix, object) + } + return destObj +} + +func (s3 *warmBackendS3) PutWithMeta(ctx context.Context, object string, r io.Reader, length int64, meta map[string]string) (remoteVersionID, error) { + res, err := s3.client.PutObject(ctx, s3.Bucket, s3.getDest(object), r, length, minio.PutObjectOptions{ + SendContentMd5: true, + StorageClass: s3.StorageClass, + UserMetadata: meta, + }) + return remoteVersionID(res.VersionID), s3.ToObjectError(err, object) +} + +func (s3 *warmBackendS3) Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) { + return s3.PutWithMeta(ctx, object, r, length, map[string]string{}) +} + +func (s3 *warmBackendS3) Get(ctx context.Context, object string, rv remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) { + gopts := minio.GetObjectOptions{} + + if rv != "" { + gopts.VersionID = string(rv) + } + if opts.startOffset >= 0 && opts.length > 0 { + if err := gopts.SetRange(opts.startOffset, opts.startOffset+opts.length-1); err != nil { + return nil, s3.ToObjectError(err, object) + } + } + c := &minio.Core{Client: s3.client} + // Important to use core primitives here to pass range get options as is. + r, _, _, err := c.GetObject(ctx, s3.Bucket, s3.getDest(object), gopts) + if err != nil { + return nil, s3.ToObjectError(err, object) + } + return r, nil +} + +func (s3 *warmBackendS3) Remove(ctx context.Context, object string, rv remoteVersionID) error { + ropts := minio.RemoveObjectOptions{} + if rv != "" { + ropts.VersionID = string(rv) + } + err := s3.client.RemoveObject(ctx, s3.Bucket, s3.getDest(object), ropts) + return s3.ToObjectError(err, object) +} + +func (s3 *warmBackendS3) InUse(ctx context.Context) (bool, error) { + result, err := s3.core.ListObjectsV2(s3.Bucket, s3.Prefix, "", "", slashSeparator, 1) + if err != nil { + return false, s3.ToObjectError(err) + } + return len(result.CommonPrefixes) > 0 || len(result.Contents) > 0, nil +} + +func newWarmBackendS3(conf madmin.TierS3, tier string) (*warmBackendS3, error) { + u, err := url.Parse(conf.Endpoint) + if err != nil { + return nil, err + } + + // Validation code + switch { + case conf.AWSRoleWebIdentityTokenFile == "" && conf.AWSRoleARN != "" || conf.AWSRoleWebIdentityTokenFile != "" && conf.AWSRoleARN == "": + return nil, errors.New("both the token file and the role ARN are required") + case conf.AccessKey == "" && conf.SecretKey != "" || conf.AccessKey != "" && conf.SecretKey == "": + return nil, errors.New("both the access and secret keys are required") + case conf.AWSRole && (conf.AWSRoleWebIdentityTokenFile != "" || conf.AWSRoleARN != "" || conf.AccessKey != "" || conf.SecretKey != ""): + return nil, errors.New("AWS Role cannot be activated with static credentials or the web identity token file") + case conf.Bucket == "": + return nil, errors.New("no bucket name was provided") + } + + // Credentials initialization + var creds *credentials.Credentials + switch { + case conf.AWSRole: + creds = credentials.New(&credentials.IAM{ + Client: &http.Client{ + Transport: NewHTTPTransport(), + }, + }) + case conf.AWSRoleWebIdentityTokenFile != "" && conf.AWSRoleARN != "": + sessionName := conf.AWSRoleSessionName + if sessionName == "" { + // RoleSessionName has a limited set of characters (https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html) + sessionName = "minio-tier-" + mustGetUUID() + } + s3WebIdentityIAM := credentials.IAM{ + Client: &http.Client{ + Transport: NewHTTPTransport(), + }, + EKSIdentity: struct { + TokenFile string + RoleARN string + RoleSessionName string + }{ + conf.AWSRoleWebIdentityTokenFile, + conf.AWSRoleARN, + sessionName, + }, + } + creds = credentials.New(&s3WebIdentityIAM) + case conf.AccessKey != "" && conf.SecretKey != "": + creds = credentials.NewStaticV4(conf.AccessKey, conf.SecretKey, "") + default: + return nil, errors.New("insufficient parameters for S3 backend authentication") + } + opts := &minio.Options{ + Creds: creds, + Secure: u.Scheme == "https", + Transport: globalRemoteTargetTransport, + Region: conf.Region, + } + client, err := minio.New(u.Host, opts) + if err != nil { + return nil, err + } + client.SetAppInfo(fmt.Sprintf("s3-tier-%s", tier), ReleaseTag) + + core := &minio.Core{Client: client} + return &warmBackendS3{ + client: client, + core: core, + Bucket: conf.Bucket, + Prefix: strings.TrimSuffix(conf.Prefix, slashSeparator), + StorageClass: conf.StorageClass, + }, nil +} diff --git a/cmd/warm-backend.go b/cmd/warm-backend.go new file mode 100644 index 00000000..91a93600 --- /dev/null +++ b/cmd/warm-backend.go @@ -0,0 +1,159 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "strings" + + "github.com/minio/madmin-go/v3" + xhttp "github.com/minio/minio/internal/http" +) + +// WarmBackendGetOpts is used to express byte ranges within an object. The zero +// value represents the entire byte range of an object. +type WarmBackendGetOpts struct { + startOffset int64 + length int64 +} + +// WarmBackend provides interface to be implemented by remote tier backends +type WarmBackend interface { + Put(ctx context.Context, object string, r io.Reader, length int64) (remoteVersionID, error) + PutWithMeta(ctx context.Context, object string, r io.Reader, length int64, meta map[string]string) (remoteVersionID, error) + Get(ctx context.Context, object string, rv remoteVersionID, opts WarmBackendGetOpts) (io.ReadCloser, error) + Remove(ctx context.Context, object string, rv remoteVersionID) error + InUse(ctx context.Context) (bool, error) +} + +const probeObject = "probeobject" + +// checkWarmBackend checks if tier config credentials have sufficient privileges +// to perform all operations defined in the WarmBackend interface. +func checkWarmBackend(ctx context.Context, w WarmBackend) error { + remoteVersionID, err := w.Put(ctx, probeObject, strings.NewReader("MinIO"), 5) + if err != nil { + if _, ok := err.(BackendDown); ok { + return err + } + return tierPermErr{ + Op: tierPut, + Err: err, + } + } + + r, err := w.Get(ctx, probeObject, "", WarmBackendGetOpts{}) + xhttp.DrainBody(r) + if err != nil { + if _, ok := err.(BackendDown); ok { + return err + } + switch { + case isErrBucketNotFound(err): + return errTierBucketNotFound + case isErrSignatureDoesNotMatch(err): + return errTierInvalidCredentials + default: + return tierPermErr{ + Op: tierGet, + Err: err, + } + } + } + if err = w.Remove(ctx, probeObject, remoteVersionID); err != nil { + if _, ok := err.(BackendDown); ok { + return err + } + return tierPermErr{ + Op: tierDelete, + Err: err, + } + } + return err +} + +type tierOp uint8 + +const ( + _ tierOp = iota + tierGet + tierPut + tierDelete +) + +func (op tierOp) String() string { + switch op { + case tierGet: + return "GET" + case tierPut: + return "PUT" + case tierDelete: + return "DELETE" + } + return "UNKNOWN" +} + +type tierPermErr struct { + Op tierOp + Err error +} + +func (te tierPermErr) Error() string { + return fmt.Sprintf("failed to perform %s: %v", te.Op, te.Err) +} + +func errIsTierPermError(err error) bool { + var tpErr tierPermErr + return errors.As(err, &tpErr) +} + +// remoteVersionID represents the version id of an object in the remote tier. +// Its usage is remote tier cloud implementation specific. +type remoteVersionID string + +// newWarmBackend instantiates the tier type specific WarmBackend, runs +// checkWarmBackend on it. +func newWarmBackend(ctx context.Context, tier madmin.TierConfig, probe bool) (d WarmBackend, err error) { + switch tier.Type { + case madmin.S3: + d, err = newWarmBackendS3(*tier.S3, tier.Name) + case madmin.Azure: + d, err = newWarmBackendAzure(*tier.Azure, tier.Name) + case madmin.GCS: + d, err = newWarmBackendGCS(*tier.GCS, tier.Name) + case madmin.MinIO: + d, err = newWarmBackendMinIO(*tier.MinIO, tier.Name) + default: + return nil, errTierTypeUnsupported + } + if err != nil { + tierLogIf(ctx, err) + return nil, errTierInvalidConfig + } + + if probe { + if err = checkWarmBackend(ctx, d); err != nil { + return nil, err + } + } + + return d, nil +} diff --git a/cmd/xl-storage-disk-id-check.go b/cmd/xl-storage-disk-id-check.go new file mode 100644 index 00000000..89fac549 --- /dev/null +++ b/cmd/xl-storage-disk-id-check.go @@ -0,0 +1,1166 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "context" + "errors" + "fmt" + "io" + "math/rand" + "path" + "runtime" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/cachevalue" + "github.com/minio/minio/internal/grid" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/logger" +) + +//go:generate stringer -type=storageMetric -trimprefix=storageMetric $GOFILE + +type storageMetric uint8 + +const ( + storageMetricMakeVolBulk storageMetric = iota + storageMetricMakeVol + storageMetricListVols + storageMetricStatVol + storageMetricDeleteVol + storageMetricWalkDir + storageMetricListDir + storageMetricReadFile + storageMetricAppendFile + storageMetricCreateFile + storageMetricReadFileStream + storageMetricRenameFile + storageMetricRenameData + storageMetricCheckParts + storageMetricDelete + storageMetricDeleteVersions + storageMetricVerifyFile + storageMetricWriteAll + storageMetricDeleteVersion + storageMetricWriteMetadata + storageMetricUpdateMetadata + storageMetricReadVersion + storageMetricReadXL + storageMetricReadAll + storageMetricStatInfoFile + storageMetricReadMultiple + storageMetricDeleteAbandonedParts + storageMetricDiskInfo + storageMetricDeleteBulk + storageMetricRenamePart + storageMetricReadParts + + // .... add more + + storageMetricLast +) + +// Detects change in underlying disk. +type xlStorageDiskIDCheck struct { + totalWrites atomic.Uint64 + totalDeletes atomic.Uint64 + totalErrsAvailability atomic.Uint64 // Captures all data availability errors such as faulty disk, timeout errors. + totalErrsTimeout atomic.Uint64 // Captures all timeout only errors + + // apiCalls should be placed first so alignment is guaranteed for atomic operations. + apiCalls [storageMetricLast]uint64 + apiLatencies [storageMetricLast]*lockedLastMinuteLatency + diskID atomic.Pointer[string] + storage *xlStorage + health *diskHealthTracker + healthCheck bool + + metricsCache *cachevalue.Cache[DiskMetrics] + diskCtx context.Context + diskCancel context.CancelFunc +} + +func (p *xlStorageDiskIDCheck) getMetrics() DiskMetrics { + p.metricsCache.InitOnce(5*time.Second, + cachevalue.Opts{}, + func(ctx context.Context) (DiskMetrics, error) { + diskMetric := DiskMetrics{ + LastMinute: make(map[string]AccElem, len(p.apiLatencies)), + APICalls: make(map[string]uint64, len(p.apiCalls)), + } + for i, v := range p.apiLatencies { + diskMetric.LastMinute[storageMetric(i).String()] = v.total() + } + for i := range p.apiCalls { + diskMetric.APICalls[storageMetric(i).String()] = atomic.LoadUint64(&p.apiCalls[i]) + } + return diskMetric, nil + }, + ) + + diskMetric, _ := p.metricsCache.GetWithCtx(context.Background()) + // Do not need this value to be cached. + diskMetric.TotalErrorsTimeout = p.totalErrsTimeout.Load() + diskMetric.TotalErrorsAvailability = p.totalErrsAvailability.Load() + + return diskMetric +} + +// lockedLastMinuteLatency accumulates totals lockless for each second. +type lockedLastMinuteLatency struct { + cachedSec int64 + cached atomic.Pointer[AccElem] + mu sync.Mutex + init sync.Once + lastMinuteLatency +} + +func (e *lockedLastMinuteLatency) add(value time.Duration) { + e.addSize(value, 0) +} + +// addSize will add a duration and size. +func (e *lockedLastMinuteLatency) addSize(value time.Duration, sz int64) { + // alloc on every call, so we have a clean entry to swap in. + t := time.Now().Unix() + e.init.Do(func() { + e.cached.Store(&AccElem{}) + atomic.StoreInt64(&e.cachedSec, t) + }) + acc := e.cached.Load() + if lastT := atomic.LoadInt64(&e.cachedSec); lastT != t { + // Check if lastT was changed by someone else. + if atomic.CompareAndSwapInt64(&e.cachedSec, lastT, t) { + // Now we swap in a new. + newAcc := &AccElem{} + old := e.cached.Swap(newAcc) + var a AccElem + a.Size = atomic.LoadInt64(&old.Size) + a.Total = atomic.LoadInt64(&old.Total) + a.N = atomic.LoadInt64(&old.N) + e.mu.Lock() + e.addAll(t-1, a) + e.mu.Unlock() + acc = newAcc + } else { + // We may be able to grab the new accumulator by yielding. + runtime.Gosched() + acc = e.cached.Load() + } + } + atomic.AddInt64(&acc.N, 1) + atomic.AddInt64(&acc.Total, int64(value)) + atomic.AddInt64(&acc.Size, sz) +} + +// total returns the total call count and latency for the last minute. +func (e *lockedLastMinuteLatency) total() AccElem { + e.mu.Lock() + defer e.mu.Unlock() + return e.getTotal() +} + +func newXLStorageDiskIDCheck(storage *xlStorage, healthCheck bool) *xlStorageDiskIDCheck { + xl := xlStorageDiskIDCheck{ + storage: storage, + health: newDiskHealthTracker(), + healthCheck: healthCheck && globalDriveMonitoring, + metricsCache: cachevalue.New[DiskMetrics](), + } + xl.SetDiskID(emptyDiskID) + + xl.totalWrites.Store(xl.storage.getWriteAttribute()) + xl.totalDeletes.Store(xl.storage.getDeleteAttribute()) + xl.diskCtx, xl.diskCancel = context.WithCancel(context.TODO()) + for i := range xl.apiLatencies[:] { + xl.apiLatencies[i] = &lockedLastMinuteLatency{} + } + if xl.healthCheck { + go xl.monitorDiskWritable(xl.diskCtx) + } + return &xl +} + +func (p *xlStorageDiskIDCheck) String() string { + return p.storage.String() +} + +func (p *xlStorageDiskIDCheck) IsOnline() bool { + storedDiskID, err := p.storage.GetDiskID() + if err != nil { + return false + } + return storedDiskID == *p.diskID.Load() +} + +func (p *xlStorageDiskIDCheck) LastConn() time.Time { + return p.storage.LastConn() +} + +func (p *xlStorageDiskIDCheck) IsLocal() bool { + return p.storage.IsLocal() +} + +func (p *xlStorageDiskIDCheck) Endpoint() Endpoint { + return p.storage.Endpoint() +} + +func (p *xlStorageDiskIDCheck) Hostname() string { + return p.storage.Hostname() +} + +func (p *xlStorageDiskIDCheck) Healing() *healingTracker { + return p.storage.Healing() +} + +func (p *xlStorageDiskIDCheck) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry, scanMode madmin.HealScanMode, _ func() bool) (dataUsageCache, error) { + if contextCanceled(ctx) { + xioutil.SafeClose(updates) + return dataUsageCache{}, ctx.Err() + } + + if err := p.checkDiskStale(); err != nil { + xioutil.SafeClose(updates) + return dataUsageCache{}, err + } + + weSleep := func() bool { + return scannerIdleMode.Load() == 0 + } + + return p.storage.NSScanner(ctx, cache, updates, scanMode, weSleep) +} + +func (p *xlStorageDiskIDCheck) GetDiskLoc() (poolIdx, setIdx, diskIdx int) { + return p.storage.GetDiskLoc() +} + +func (p *xlStorageDiskIDCheck) Close() error { + p.diskCancel() + return p.storage.Close() +} + +func (p *xlStorageDiskIDCheck) GetDiskID() (string, error) { + return p.storage.GetDiskID() +} + +func (p *xlStorageDiskIDCheck) SetDiskID(id string) { + p.diskID.Store(&id) +} + +func (p *xlStorageDiskIDCheck) checkDiskStale() error { + if *p.diskID.Load() == emptyDiskID { + // For empty disk-id we allow the call as the server might be + // coming up and trying to read format.json or create format.json + return nil + } + storedDiskID, err := p.storage.GetDiskID() + if err != nil { + // return any error generated while reading `format.json` + return err + } + if err == nil && *p.diskID.Load() == storedDiskID { + return nil + } + // not the same disk we remember, take it offline. + return errDiskNotFound +} + +func (p *xlStorageDiskIDCheck) DiskInfo(ctx context.Context, opts DiskInfoOptions) (info DiskInfo, err error) { + if contextCanceled(ctx) { + return DiskInfo{}, ctx.Err() + } + + si := p.updateStorageMetrics(storageMetricDiskInfo) + defer si(0, &err) + + if opts.NoOp { + if opts.Metrics { + info.Metrics = p.getMetrics() + } + info.Metrics.TotalWrites = p.totalWrites.Load() + info.Metrics.TotalDeletes = p.totalDeletes.Load() + info.Metrics.TotalWaiting = uint32(p.health.waiting.Load()) + info.Metrics.TotalErrorsTimeout = p.totalErrsTimeout.Load() + info.Metrics.TotalErrorsAvailability = p.totalErrsAvailability.Load() + if p.health.isFaulty() { + // if disk is already faulty return faulty for 'mc admin info' output and prometheus alerts. + return info, errFaultyDisk + } + return info, nil + } + + defer func() { + if opts.Metrics { + info.Metrics = p.getMetrics() + } + info.Metrics.TotalWrites = p.totalWrites.Load() + info.Metrics.TotalDeletes = p.totalDeletes.Load() + info.Metrics.TotalWaiting = uint32(p.health.waiting.Load()) + info.Metrics.TotalErrorsTimeout = p.totalErrsTimeout.Load() + info.Metrics.TotalErrorsAvailability = p.totalErrsAvailability.Load() + }() + + if p.health.isFaulty() { + // if disk is already faulty return faulty for 'mc admin info' output and prometheus alerts. + return info, errFaultyDisk + } + + info, err = p.storage.DiskInfo(ctx, opts) + if err != nil { + return info, err + } + + // check cached diskID against backend + // only if its non-empty. + cachedID := *p.diskID.Load() + if cachedID != "" && cachedID != info.ID { + return info, errDiskNotFound + } + return info, nil +} + +func (p *xlStorageDiskIDCheck) MakeVolBulk(ctx context.Context, volumes ...string) (err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricMakeVolBulk, volumes...) + if err != nil { + return err + } + defer done(0, &err) + + w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) + return w.Run(func() error { return p.storage.MakeVolBulk(ctx, volumes...) }) +} + +func (p *xlStorageDiskIDCheck) MakeVol(ctx context.Context, volume string) (err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricMakeVol, volume) + if err != nil { + return err + } + defer done(0, &err) + + w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) + return w.Run(func() error { return p.storage.MakeVol(ctx, volume) }) +} + +func (p *xlStorageDiskIDCheck) ListVols(ctx context.Context) (vi []VolInfo, err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricListVols, "/") + if err != nil { + return nil, err + } + defer done(0, &err) + + return p.storage.ListVols(ctx) +} + +func (p *xlStorageDiskIDCheck) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricStatVol, volume) + if err != nil { + return vol, err + } + defer done(0, &err) + + return xioutil.WithDeadline[VolInfo](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (result VolInfo, err error) { + return p.storage.StatVol(ctx, volume) + }) +} + +func (p *xlStorageDiskIDCheck) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricDeleteVol, volume) + if err != nil { + return err + } + defer done(0, &err) + + w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) + return w.Run(func() error { return p.storage.DeleteVol(ctx, volume, forceDelete) }) +} + +func (p *xlStorageDiskIDCheck) ListDir(ctx context.Context, origvolume, volume, dirPath string, count int) (s []string, err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricListDir, volume, dirPath) + if err != nil { + return nil, err + } + defer done(0, &err) + + return p.storage.ListDir(ctx, origvolume, volume, dirPath, count) +} + +// Legacy API - does not have any deadlines +func (p *xlStorageDiskIDCheck) ReadFile(ctx context.Context, volume string, path string, offset int64, buf []byte, verifier *BitrotVerifier) (n int64, err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadFile, volume, path) + if err != nil { + return 0, err + } + defer func() { + done(n, &err) + }() + + return xioutil.WithDeadline[int64](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (result int64, err error) { + return p.storage.ReadFile(ctx, volume, path, offset, buf, verifier) + }) +} + +// Legacy API - does not have any deadlines +func (p *xlStorageDiskIDCheck) AppendFile(ctx context.Context, volume string, path string, buf []byte) (err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricAppendFile, volume, path) + if err != nil { + return err + } + defer done(int64(len(buf)), &err) + + w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) + return w.Run(func() error { + return p.storage.AppendFile(ctx, volume, path, buf) + }) +} + +func (p *xlStorageDiskIDCheck) CreateFile(ctx context.Context, origvolume, volume, path string, size int64, reader io.Reader) (err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricCreateFile, volume, path) + if err != nil { + return err + } + defer done(size, &err) + + return p.storage.CreateFile(ctx, origvolume, volume, path, size, io.NopCloser(reader)) +} + +func (p *xlStorageDiskIDCheck) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadFileStream, volume, path) + if err != nil { + return nil, err + } + defer done(length, &err) + + return xioutil.WithDeadline[io.ReadCloser](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (result io.ReadCloser, err error) { + return p.storage.ReadFileStream(ctx, volume, path, offset, length) + }) +} + +func (p *xlStorageDiskIDCheck) RenamePart(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string, meta []byte, skipParent string) (err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricRenamePart, srcVolume, srcPath, dstVolume, dstPath) + if err != nil { + return err + } + defer done(0, &err) + + w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) + return w.Run(func() error { + return p.storage.RenamePart(ctx, srcVolume, srcPath, dstVolume, dstPath, meta, skipParent) + }) +} + +func (p *xlStorageDiskIDCheck) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) (err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricRenameFile, srcVolume, srcPath, dstVolume, dstPath) + if err != nil { + return err + } + defer done(0, &err) + + w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) + return w.Run(func() error { return p.storage.RenameFile(ctx, srcVolume, srcPath, dstVolume, dstPath) }) +} + +func (p *xlStorageDiskIDCheck) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (res RenameDataResp, err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricRenameData, srcPath, fi.DataDir, dstVolume, dstPath) + if err != nil { + return res, err + } + defer func() { + if err == nil && !skipAccessChecks(dstVolume) { + p.storage.setWriteAttribute(p.totalWrites.Add(1)) + } + done(0, &err) + }() + + // Copy inline data to a new buffer to function with deadlines. + if len(fi.Data) > 0 { + fi.Data = append(grid.GetByteBufferCap(len(fi.Data))[:0], fi.Data...) + } + return xioutil.WithDeadline[RenameDataResp](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (res RenameDataResp, err error) { + if len(fi.Data) > 0 { + defer grid.PutByteBuffer(fi.Data) + } + return p.storage.RenameData(ctx, srcVolume, srcPath, fi, dstVolume, dstPath, opts) + }) +} + +func (p *xlStorageDiskIDCheck) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (*CheckPartsResp, error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricCheckParts, volume, path) + if err != nil { + return nil, err + } + defer done(0, &err) + + return p.storage.CheckParts(ctx, volume, path, fi) +} + +func (p *xlStorageDiskIDCheck) DeleteBulk(ctx context.Context, volume string, paths ...string) (err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricDeleteBulk, append([]string{volume}, paths...)...) + if err != nil { + return err + } + defer done(0, &err) + + return p.storage.DeleteBulk(ctx, volume, paths...) +} + +func (p *xlStorageDiskIDCheck) Delete(ctx context.Context, volume string, path string, deleteOpts DeleteOptions) (err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricDelete, volume, path) + if err != nil { + return err + } + defer done(0, &err) + + w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) + return w.Run(func() error { return p.storage.Delete(ctx, volume, path, deleteOpts) }) +} + +// DeleteVersions deletes slice of versions, it can be same object +// or multiple objects. +func (p *xlStorageDiskIDCheck) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions, opts DeleteOptions) (errs []error) { + // Merely for tracing storage + path := "" + if len(versions) > 0 { + path = versions[0].Name + } + errs = make([]error, len(versions)) + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricDeleteVersions, volume, path) + if err != nil { + for i := range errs { + errs[i] = ctx.Err() + } + return errs + } + defer func() { + if !skipAccessChecks(volume) { + var permanentDeletes uint64 + var deleteMarkers uint64 + + for i, nerr := range errs { + if nerr != nil { + continue + } + for _, fi := range versions[i].Versions { + if fi.Deleted { + // Delete markers are a write operation not a permanent delete. + deleteMarkers++ + continue + } + permanentDeletes++ + } + } + if deleteMarkers > 0 { + p.storage.setWriteAttribute(p.totalWrites.Add(deleteMarkers)) + } + if permanentDeletes > 0 { + p.storage.setDeleteAttribute(p.totalDeletes.Add(permanentDeletes)) + } + } + done(0, &err) + }() + + errs = p.storage.DeleteVersions(ctx, volume, versions, opts) + for i := range errs { + if errs[i] != nil { + err = errs[i] + break + } + } + + return errs +} + +func (p *xlStorageDiskIDCheck) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricVerifyFile, volume, path) + if err != nil { + return nil, err + } + defer done(0, &err) + + return p.storage.VerifyFile(ctx, volume, path, fi) +} + +func (p *xlStorageDiskIDCheck) WriteAll(ctx context.Context, volume string, path string, b []byte) (err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricWriteAll, volume, path) + if err != nil { + return err + } + defer done(int64(len(b)), &err) + + w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) + return w.Run(func() error { return p.storage.WriteAll(ctx, volume, path, b) }) +} + +func (p *xlStorageDiskIDCheck) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool, opts DeleteOptions) (err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricDeleteVersion, volume, path) + if err != nil { + return err + } + defer func() { + defer done(0, &err) + + if err == nil && !skipAccessChecks(volume) { + if opts.UndoWrite { + p.storage.setWriteAttribute(p.totalWrites.Add(^uint64(0))) + return + } + + if fi.Deleted { + // Delete markers are a write operation not a permanent delete. + p.storage.setWriteAttribute(p.totalWrites.Add(1)) + return + } + + p.storage.setDeleteAttribute(p.totalDeletes.Add(1)) + } + }() + + w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) + return w.Run(func() error { return p.storage.DeleteVersion(ctx, volume, path, fi, forceDelMarker, opts) }) +} + +func (p *xlStorageDiskIDCheck) UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo, opts UpdateMetadataOpts) (err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricUpdateMetadata, volume, path) + if err != nil { + return err + } + defer done(0, &err) + + w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) + return w.Run(func() error { return p.storage.UpdateMetadata(ctx, volume, path, fi, opts) }) +} + +func (p *xlStorageDiskIDCheck) WriteMetadata(ctx context.Context, origvolume, volume, path string, fi FileInfo) (err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricWriteMetadata, volume, path) + if err != nil { + return err + } + defer done(0, &err) + + w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) + return w.Run(func() error { return p.storage.WriteMetadata(ctx, origvolume, volume, path, fi) }) +} + +func (p *xlStorageDiskIDCheck) ReadVersion(ctx context.Context, origvolume, volume, path, versionID string, opts ReadOptions) (fi FileInfo, err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadVersion, volume, path) + if err != nil { + return fi, err + } + defer done(0, &err) + + return xioutil.WithDeadline[FileInfo](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (result FileInfo, err error) { + return p.storage.ReadVersion(ctx, origvolume, volume, path, versionID, opts) + }) +} + +func (p *xlStorageDiskIDCheck) ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadAll, volume, path) + if err != nil { + return nil, err + } + var sz int + defer func() { + sz = len(buf) + done(int64(sz), &err) + }() + + return xioutil.WithDeadline[[]byte](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (result []byte, err error) { + return p.storage.ReadAll(ctx, volume, path) + }) +} + +func (p *xlStorageDiskIDCheck) ReadXL(ctx context.Context, volume string, path string, readData bool) (rf RawFileInfo, err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadXL, volume, path) + if err != nil { + return RawFileInfo{}, err + } + defer func() { + done(int64(len(rf.Buf)), &err) + }() + + return xioutil.WithDeadline[RawFileInfo](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (result RawFileInfo, err error) { + return p.storage.ReadXL(ctx, volume, path, readData) + }) +} + +func (p *xlStorageDiskIDCheck) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricStatInfoFile, volume, path) + if err != nil { + return nil, err + } + defer done(0, &err) + + return p.storage.StatInfoFile(ctx, volume, path, glob) +} + +func (p *xlStorageDiskIDCheck) ReadParts(ctx context.Context, volume string, partMetaPaths ...string) ([]*ObjectPartInfo, error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadParts, volume, path.Dir(partMetaPaths[0])) + if err != nil { + return nil, err + } + defer done(0, &err) + + return p.storage.ReadParts(ctx, volume, partMetaPaths...) +} + +// ReadMultiple will read multiple files and send each files as response. +// Files are read and returned in the given order. +// The resp channel is closed before the call returns. +// Only a canceled context will return an error. +func (p *xlStorageDiskIDCheck) ReadMultiple(ctx context.Context, req ReadMultipleReq, resp chan<- ReadMultipleResp) (err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricReadMultiple, req.Bucket, req.Prefix) + if err != nil { + xioutil.SafeClose(resp) + return err + } + defer done(0, &err) + + return p.storage.ReadMultiple(ctx, req, resp) +} + +// CleanAbandonedData will read metadata of the object on disk +// and delete any data directories and inline data that isn't referenced in metadata. +func (p *xlStorageDiskIDCheck) CleanAbandonedData(ctx context.Context, volume string, path string) (err error) { + ctx, done, err := p.TrackDiskHealth(ctx, storageMetricDeleteAbandonedParts, volume, path) + if err != nil { + return err + } + defer done(0, &err) + + w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) + return w.Run(func() error { return p.storage.CleanAbandonedData(ctx, volume, path) }) +} + +func storageTrace(s storageMetric, startTime time.Time, duration time.Duration, path string, size int64, err string, custom map[string]string) madmin.TraceInfo { + return madmin.TraceInfo{ + TraceType: madmin.TraceStorage, + Time: startTime, + NodeName: globalLocalNodeName, + FuncName: "storage." + s.String(), + Duration: duration, + Bytes: size, + Path: path, + Error: err, + Custom: custom, + } +} + +func scannerTrace(s scannerMetric, startTime time.Time, duration time.Duration, path string, custom map[string]string) madmin.TraceInfo { + return madmin.TraceInfo{ + TraceType: madmin.TraceScanner, + Time: startTime, + NodeName: globalLocalNodeName, + FuncName: "scanner." + s.String(), + Duration: duration, + Path: path, + Custom: custom, + } +} + +// Update storage metrics +func (p *xlStorageDiskIDCheck) updateStorageMetrics(s storageMetric, paths ...string) func(sz int64, err *error) { + startTime := time.Now() + trace := globalTrace.NumSubscribers(madmin.TraceStorage) > 0 + return func(sz int64, errp *error) { + duration := time.Since(startTime) + + var err error + if errp != nil && *errp != nil { + err = *errp + } + + atomic.AddUint64(&p.apiCalls[s], 1) + if IsErr(err, []error{ + errFaultyDisk, + errFaultyRemoteDisk, + context.DeadlineExceeded, + }...) { + p.totalErrsAvailability.Add(1) + if errors.Is(err, context.DeadlineExceeded) { + p.totalErrsTimeout.Add(1) + } + } + + p.apiLatencies[s].add(duration) + + if trace { + custom := make(map[string]string, 2) + paths = append([]string{p.String()}, paths...) + var errStr string + if err != nil { + errStr = err.Error() + } + custom["total-errs-timeout"] = strconv.FormatUint(p.totalErrsTimeout.Load(), 10) + custom["total-errs-availability"] = strconv.FormatUint(p.totalErrsAvailability.Load(), 10) + globalTrace.Publish(storageTrace(s, startTime, duration, strings.Join(paths, " "), sz, errStr, custom)) + } + } +} + +const ( + diskHealthOK int32 = iota + diskHealthFaulty +) + +type diskHealthTracker struct { + // atomic time of last success + lastSuccess int64 + + // atomic time of last time a token was grabbed. + lastStarted int64 + + // Atomic status of disk. + status atomic.Int32 + + // Atomic number indicates if a disk is hung + waiting atomic.Int32 +} + +// newDiskHealthTracker creates a new disk health tracker. +func newDiskHealthTracker() *diskHealthTracker { + d := diskHealthTracker{ + lastSuccess: time.Now().UnixNano(), + lastStarted: time.Now().UnixNano(), + } + d.status.Store(diskHealthOK) + return &d +} + +// logSuccess will update the last successful operation time. +func (d *diskHealthTracker) logSuccess() { + atomic.StoreInt64(&d.lastSuccess, time.Now().UnixNano()) +} + +func (d *diskHealthTracker) isFaulty() bool { + return d.status.Load() == diskHealthFaulty +} + +type ( + healthDiskCtxKey struct{} + healthDiskCtxValue struct { + lastSuccess *int64 + } +) + +// logSuccess will update the last successful operation time. +func (h *healthDiskCtxValue) logSuccess() { + atomic.StoreInt64(h.lastSuccess, time.Now().UnixNano()) +} + +// noopDoneFunc is a no-op done func. +// Can be reused. +var noopDoneFunc = func(_ int64, _ *error) {} + +// TrackDiskHealth for this request. +// When a non-nil error is returned 'done' MUST be called +// with the status of the response, if it corresponds to disk health. +// If the pointer sent to done is non-nil AND the error +// is either nil or io.EOF the disk is considered good. +// So if unsure if the disk status is ok, return nil as a parameter to done. +// Shadowing will work as long as return error is named: https://go.dev/play/p/sauq86SsTN2 +func (p *xlStorageDiskIDCheck) TrackDiskHealth(ctx context.Context, s storageMetric, paths ...string) (c context.Context, done func(int64, *error), err error) { + done = noopDoneFunc + if contextCanceled(ctx) { + return ctx, done, ctx.Err() + } + + if p.health.status.Load() != diskHealthOK { + return ctx, done, errFaultyDisk + } + + // Verify if the disk is not stale + // - missing format.json (unformatted drive) + // - format.json is valid but invalid 'uuid' + if err = p.checkDiskStale(); err != nil { + return ctx, done, err + } + + // Disallow recursive tracking to avoid deadlocks. + if ctx.Value(healthDiskCtxKey{}) != nil { + done = p.updateStorageMetrics(s, paths...) + return ctx, done, nil + } + + if contextCanceled(ctx) { + return ctx, done, ctx.Err() + } + + atomic.StoreInt64(&p.health.lastStarted, time.Now().UnixNano()) + p.health.waiting.Add(1) + + ctx = context.WithValue(ctx, healthDiskCtxKey{}, &healthDiskCtxValue{lastSuccess: &p.health.lastSuccess}) + si := p.updateStorageMetrics(s, paths...) + var once sync.Once + return ctx, func(sz int64, errp *error) { + p.health.waiting.Add(-1) + once.Do(func() { + if errp != nil { + err := *errp + if err == nil || errors.Is(err, io.EOF) { + p.health.logSuccess() + } + } + si(sz, errp) + }) + }, nil +} + +var toWrite = []byte{2048: 42} + +// monitorDiskStatus should be called once when a drive has been marked offline. +// Once the disk has been deemed ok, it will return to online status. +func (p *xlStorageDiskIDCheck) monitorDiskStatus(spent time.Duration, fn string) { + t := time.NewTicker(5 * time.Second) + defer t.Stop() + + for range t.C { + if contextCanceled(p.diskCtx) { + return + } + + err := p.storage.WriteAll(context.Background(), minioMetaTmpBucket, fn, toWrite) + if err != nil { + continue + } + + b, err := p.storage.ReadAll(context.Background(), minioMetaTmpBucket, fn) + if err != nil || len(b) != len(toWrite) { + continue + } + + err = p.storage.Delete(context.Background(), minioMetaTmpBucket, fn, DeleteOptions{ + Recursive: false, + Immediate: false, + }) + + if err == nil { + logger.Event(context.Background(), "healthcheck", + "node(%s): Read/Write/Delete successful, bringing drive %s online", globalLocalNodeName, p.storage.String()) + p.health.status.Store(diskHealthOK) + p.health.waiting.Add(-1) + return + } + } +} + +// monitorDiskStatus should be called once when a drive has been marked offline. +// Once the disk has been deemed ok, it will return to online status. +func (p *xlStorageDiskIDCheck) monitorDiskWritable(ctx context.Context) { + var ( + // We check every 15 seconds if the disk is writable and we can read back. + checkEvery = 15 * time.Second + + // If the disk has completed an operation successfully within last 5 seconds, don't check it. + skipIfSuccessBefore = 5 * time.Second + ) + + // if disk max timeout is smaller than checkEvery window + // reduce checks by a second. + if globalDriveConfig.GetMaxTimeout() <= checkEvery { + checkEvery = globalDriveConfig.GetMaxTimeout() - time.Second + if checkEvery <= 0 { + checkEvery = globalDriveConfig.GetMaxTimeout() + } + } + + // if disk max timeout is smaller than skipIfSuccessBefore window + // reduce the skipIfSuccessBefore by a second. + if globalDriveConfig.GetMaxTimeout() <= skipIfSuccessBefore { + skipIfSuccessBefore = globalDriveConfig.GetMaxTimeout() - time.Second + if skipIfSuccessBefore <= 0 { + skipIfSuccessBefore = globalDriveConfig.GetMaxTimeout() + } + } + + t := time.NewTicker(checkEvery) + defer t.Stop() + fn := mustGetUUID() + + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + + monitor := func() bool { + if contextCanceled(ctx) { + return false + } + + if p.health.status.Load() != diskHealthOK { + return true + } + + if time.Since(time.Unix(0, atomic.LoadInt64(&p.health.lastSuccess))) < skipIfSuccessBefore { + // We recently saw a success - no need to check. + return true + } + + goOffline := func(err error, spent time.Duration) { + if p.health.status.CompareAndSwap(diskHealthOK, diskHealthFaulty) { + storageLogAlwaysIf(ctx, fmt.Errorf("node(%s): taking drive %s offline: %v", globalLocalNodeName, p.storage.String(), err)) + p.health.waiting.Add(1) + go p.monitorDiskStatus(spent, fn) + } + } + + // Offset checks a bit. + time.Sleep(time.Duration(rng.Int63n(int64(1 * time.Second)))) + + dctx, dcancel := context.WithCancel(ctx) + started := time.Now() + go func() { + timeout := time.NewTimer(globalDriveConfig.GetMaxTimeout()) + select { + case <-dctx.Done(): + if !timeout.Stop() { + <-timeout.C + } + case <-timeout.C: + spent := time.Since(started) + goOffline(fmt.Errorf("unable to write+read for %v", spent.Round(time.Millisecond)), spent) + } + }() + + func() { + defer dcancel() + + err := p.storage.WriteAll(ctx, minioMetaTmpBucket, fn, toWrite) + if err != nil { + if osErrToFileErr(err) == errFaultyDisk { + goOffline(fmt.Errorf("unable to write: %w", err), 0) + } + return + } + b, err := p.storage.ReadAll(context.Background(), minioMetaTmpBucket, fn) + if err != nil || len(b) != len(toWrite) { + if osErrToFileErr(err) == errFaultyDisk { + goOffline(fmt.Errorf("unable to read: %w", err), 0) + } + return + } + }() + + // Continue to monitor + return true + } + + for { + select { + case <-ctx.Done(): + return + case <-t.C: + if !monitor() { + return + } + } + } +} + +// checkID will check if the disk ID matches the provided ID. +func (p *xlStorageDiskIDCheck) checkID(wantID string) (err error) { + if wantID == "" { + return nil + } + id, err := p.storage.GetDiskID() + if err != nil { + return err + } + if id != wantID { + return fmt.Errorf("disk ID %s does not match. disk reports %s", wantID, id) + } + return nil +} + +// diskHealthCheckOK will check if the provided error is nil +// and update disk status if good. +// For convenience a bool is returned to indicate any error state +// that is not io.EOF. +func diskHealthCheckOK(ctx context.Context, err error) bool { + // Check if context has a disk health check. + tracker, ok := ctx.Value(healthDiskCtxKey{}).(*healthDiskCtxValue) + if !ok { + // No tracker, return + return err == nil || errors.Is(err, io.EOF) + } + if err == nil || errors.Is(err, io.EOF) { + tracker.logSuccess() + return true + } + return false +} + +// diskHealthWrapper provides either a io.Reader or io.Writer +// that updates status of the provided tracker. +// Use through diskHealthReader or diskHealthWriter. +type diskHealthWrapper struct { + tracker *healthDiskCtxValue + r io.Reader + w io.Writer +} + +func (d *diskHealthWrapper) Read(p []byte) (int, error) { + if d.r == nil { + return 0, fmt.Errorf("diskHealthWrapper: Read with no reader") + } + n, err := d.r.Read(p) + if err == nil || err == io.EOF && n > 0 { + d.tracker.logSuccess() + } + return n, err +} + +func (d *diskHealthWrapper) Write(p []byte) (int, error) { + if d.w == nil { + return 0, fmt.Errorf("diskHealthWrapper: Write with no writer") + } + n, err := d.w.Write(p) + if err == nil && n == len(p) { + d.tracker.logSuccess() + } + return n, err +} + +// diskHealthReader provides a wrapper that will update disk health on +// ctx, on every successful read. +// This should only be used directly at the os/syscall level, +// otherwise buffered operations may return false health checks. +func diskHealthReader(ctx context.Context, r io.Reader) io.Reader { + // Check if context has a disk health check. + tracker, ok := ctx.Value(healthDiskCtxKey{}).(*healthDiskCtxValue) + if !ok { + // No need to wrap + return r + } + return &diskHealthWrapper{r: r, tracker: tracker} +} + +// diskHealthWriter provides a wrapper that will update disk health on +// ctx, on every successful write. +// This should only be used directly at the os/syscall level, +// otherwise buffered operations may return false health checks. +func diskHealthWriter(ctx context.Context, w io.Writer) io.Writer { + // Check if context has a disk health check. + tracker, ok := ctx.Value(healthDiskCtxKey{}).(*healthDiskCtxValue) + if !ok { + // No need to wrap + return w + } + return &diskHealthWrapper{w: w, tracker: tracker} +} diff --git a/cmd/xl-storage-errors.go b/cmd/xl-storage-errors.go new file mode 100644 index 00000000..86e7097f --- /dev/null +++ b/cmd/xl-storage-errors.go @@ -0,0 +1,141 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "errors" + "os" + "runtime" + "syscall" +) + +// No space left on device error +func isSysErrNoSpace(err error) bool { + return errors.Is(err, syscall.ENOSPC) +} + +// Invalid argument, unsupported flags such as O_DIRECT +func isSysErrInvalidArg(err error) bool { + return errors.Is(err, syscall.EINVAL) +} + +// Input/output error +func isSysErrIO(err error) bool { + return errors.Is(err, syscall.EIO) +} + +// Check if the given error corresponds to EISDIR (is a directory). +func isSysErrIsDir(err error) bool { + return errors.Is(err, syscall.EISDIR) +} + +// Check if the given error corresponds to ENOTDIR (is not a directory). +func isSysErrNotDir(err error) bool { + return errors.Is(err, syscall.ENOTDIR) +} + +// Check if the given error corresponds to the ENAMETOOLONG (name too long). +func isSysErrTooLong(err error) bool { + return errors.Is(err, syscall.ENAMETOOLONG) +} + +// Check if the given error corresponds to the ELOOP (too many symlinks). +func isSysErrTooManySymlinks(err error) bool { + return errors.Is(err, syscall.ELOOP) +} + +// Check if the given error corresponds to ENOTEMPTY for unix, +// EEXIST for solaris variants, +// and ERROR_DIR_NOT_EMPTY for windows (directory not empty). +func isSysErrNotEmpty(err error) bool { + if errors.Is(err, syscall.ENOTEMPTY) { + return true + } + if errors.Is(err, syscall.EEXIST) && runtime.GOOS == "solaris" { + return true + } + var pathErr *os.PathError + if errors.As(err, &pathErr) { + if runtime.GOOS == globalWindowsOSName { + var errno syscall.Errno + if errors.As(pathErr.Err, &errno) { + // ERROR_DIR_NOT_EMPTY + return errno == 0x91 + } + } + } + return false +} + +// Check if the given error corresponds to the specific ERROR_PATH_NOT_FOUND for windows +func isSysErrPathNotFound(err error) bool { + if runtime.GOOS != globalWindowsOSName { + var pathErr *os.PathError + if errors.As(err, &pathErr) { + return pathErr.Err == syscall.ENOENT + } + return false + } + var pathErr *os.PathError + if errors.As(err, &pathErr) { + var errno syscall.Errno + if errors.As(pathErr.Err, &errno) { + // ERROR_PATH_NOT_FOUND + return errno == 0x03 + } + } + return false +} + +// Check if the given error corresponds to the specific ERROR_INVALID_HANDLE for windows +func isSysErrHandleInvalid(err error) bool { + if runtime.GOOS != globalWindowsOSName { + return false + } + // Check if err contains ERROR_INVALID_HANDLE errno + var pathErr *os.PathError + if errors.As(err, &pathErr) { + var errno syscall.Errno + if errors.As(pathErr.Err, &errno) { + // ERROR_PATH_NOT_FOUND + return errno == 0x6 + } + } + return false +} + +func isSysErrCrossDevice(err error) bool { + return errors.Is(err, syscall.EXDEV) +} + +// Check if given error corresponds to too many open files +func isSysErrTooManyFiles(err error) bool { + return errors.Is(err, syscall.ENFILE) || errors.Is(err, syscall.EMFILE) +} + +func osIsNotExist(err error) bool { + return errors.Is(err, os.ErrNotExist) +} + +func osIsPermission(err error) bool { + return errors.Is(err, os.ErrPermission) || errors.Is(err, syscall.EROFS) +} + +func osIsExist(err error) bool { + return errors.Is(err, os.ErrExist) +} diff --git a/cmd/xl-storage-errors_test.go b/cmd/xl-storage-errors_test.go new file mode 100644 index 00000000..0a475776 --- /dev/null +++ b/cmd/xl-storage-errors_test.go @@ -0,0 +1,58 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "os" + "runtime" + "syscall" + "testing" +) + +func TestSysErrors(t *testing.T) { + pathErr := &os.PathError{Err: syscall.ENAMETOOLONG} + ok := isSysErrTooLong(pathErr) + if !ok { + t.Fatalf("Unexpected error expecting %s", syscall.ENAMETOOLONG) + } + pathErr = &os.PathError{Err: syscall.ENOTDIR} + ok = isSysErrNotDir(pathErr) + if !ok { + t.Fatalf("Unexpected error expecting %s", syscall.ENOTDIR) + } + if runtime.GOOS != globalWindowsOSName { + pathErr = &os.PathError{Err: syscall.ENOTEMPTY} + ok = isSysErrNotEmpty(pathErr) + if !ok { + t.Fatalf("Unexpected error expecting %s", syscall.ENOTEMPTY) + } + } else { + pathErr = &os.PathError{Err: syscall.Errno(0x91)} + ok = isSysErrNotEmpty(pathErr) + if !ok { + t.Fatal("Unexpected error expecting 0x91") + } + } + if runtime.GOOS == globalWindowsOSName { + pathErr = &os.PathError{Err: syscall.Errno(0x03)} + ok = isSysErrPathNotFound(pathErr) + if !ok { + t.Fatal("Unexpected error expecting 0x03") + } + } +} diff --git a/cmd/xl-storage-format-utils.go b/cmd/xl-storage-format-utils.go new file mode 100644 index 00000000..13c6836e --- /dev/null +++ b/cmd/xl-storage-format-utils.go @@ -0,0 +1,190 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "errors" + + "github.com/zeebo/xxh3" +) + +// getFileInfoVersions partitions this object's versions such that, +// - fivs.Versions has all the non-free versions +// - fivs.FreeVersions has all the free versions +// +// if inclFreeVersions is true all the versions are in fivs.Versions, free and non-free versions alike. +// +// Note: Only the scanner requires fivs.Versions to have exclusively non-free versions. This is used while enforcing NewerNoncurrentVersions lifecycle element. +func getFileInfoVersions(xlMetaBuf []byte, volume, path string, inclFreeVersions bool) (FileInfoVersions, error) { + fivs, err := getAllFileInfoVersions(xlMetaBuf, volume, path, true) + if err != nil { + return fivs, err + } + + // If inclFreeVersions is false, partition the versions in fivs.Versions + // such that finally fivs.Versions has + // all the non-free versions and fivs.FreeVersions has all the free + // versions. + n := 0 + for _, fi := range fivs.Versions { + // filter our tier object delete marker + if fi.TierFreeVersion() { + if !inclFreeVersions { + fivs.FreeVersions = append(fivs.FreeVersions, fi) + } + } else { + if !inclFreeVersions { + fivs.Versions[n] = fi + } + n++ + } + } + if !inclFreeVersions { + fivs.Versions = fivs.Versions[:n] + } + // Update numversions + for i := range fivs.Versions { + fivs.Versions[i].NumVersions = n + } + return fivs, nil +} + +func getAllFileInfoVersions(xlMetaBuf []byte, volume, path string, allParts bool) (FileInfoVersions, error) { + var versions []FileInfo + var err error + + if buf, _, e := isIndexedMetaV2(xlMetaBuf); e != nil { + return FileInfoVersions{}, e + } else if buf != nil { + versions, err = buf.ListVersions(volume, path, allParts) + } else { + var xlMeta xlMetaV2 + if err := xlMeta.LoadOrConvert(xlMetaBuf); err != nil { + return FileInfoVersions{}, err + } + versions, err = xlMeta.ListVersions(volume, path, allParts) + } + if err == nil && len(versions) == 0 { + // This special case is needed to handle len(xlMeta.versions) == 0 + versions = []FileInfo{ + { + Volume: volume, + Name: path, + Deleted: true, + IsLatest: true, + ModTime: timeSentinel1970, + }, + } + } + if err != nil { + return FileInfoVersions{}, err + } + + return FileInfoVersions{ + Volume: volume, + Name: path, + Versions: versions, + LatestModTime: versions[0].ModTime, + }, nil +} + +type fileInfoOpts struct { + InclFreeVersions bool + Data bool +} + +func getFileInfo(xlMetaBuf []byte, volume, path, versionID string, opts fileInfoOpts) (FileInfo, error) { + var fi FileInfo + var err error + var inData xlMetaInlineData + if buf, data, e := isIndexedMetaV2(xlMetaBuf); e != nil { + return FileInfo{}, e + } else if buf != nil { + inData = data + fi, err = buf.ToFileInfo(volume, path, versionID, true) + if len(buf) != 0 && errors.Is(err, errFileNotFound) { + // This special case is needed to handle len(xlMeta.versions) == 0 + return FileInfo{ + Volume: volume, + Name: path, + VersionID: versionID, + Deleted: true, + IsLatest: true, + ModTime: timeSentinel1970, + }, nil + } + } else { + var xlMeta xlMetaV2 + if err := xlMeta.LoadOrConvert(xlMetaBuf); err != nil { + return FileInfo{}, err + } + if len(xlMeta.versions) == 0 { + // This special case is needed to handle len(xlMeta.versions) == 0 + return FileInfo{ + Volume: volume, + Name: path, + VersionID: versionID, + Deleted: true, + IsLatest: true, + ModTime: timeSentinel1970, + }, nil + } + inData = xlMeta.data + fi, err = xlMeta.ToFileInfo(volume, path, versionID, opts.InclFreeVersions, true) + } + if !opts.Data || err != nil { + return fi, err + } + versionID = fi.VersionID + if versionID == "" { + versionID = nullVersionID + } + + fi.Data = inData.find(versionID) + if len(fi.Data) == 0 { + // PR #11758 used DataDir, preserve it + // for users who might have used master + // branch + fi.Data = inData.find(fi.DataDir) + } + return fi, nil +} + +// hashDeterministicString will return a deterministic hash for the map values. +// Trivial collisions are avoided, but this is by no means a strong hash. +func hashDeterministicString(m map[string]string) uint64 { + // Seed (random) + crc := uint64(0xc2b40bbac11a7295) + // Xor each value to make order independent + for k, v := range m { + // Separate key and value with an individual xor with a random number. + // Add values of each, so they cannot be trivially collided. + crc ^= (xxh3.HashString(k) ^ 0x4ee3bbaf7ab2506b) + (xxh3.HashString(v) ^ 0x8da4c8da66194257) + } + return crc +} + +// hashDeterministicBytes will return a deterministic (weak) hash for the map values. +// Trivial collisions are avoided, but this is by no means a strong hash. +func hashDeterministicBytes(m map[string][]byte) uint64 { + crc := uint64(0x1bbc7e1dde654743) + for k, v := range m { + crc ^= (xxh3.HashString(k) ^ 0x4ee3bbaf7ab2506b) + (xxh3.Hash(v) ^ 0x8da4c8da66194257) + } + return crc +} diff --git a/cmd/xl-storage-format-utils_test.go b/cmd/xl-storage-format-utils_test.go new file mode 100644 index 00000000..12f36315 --- /dev/null +++ b/cmd/xl-storage-format-utils_test.go @@ -0,0 +1,237 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "slices" + "sort" + "testing" + "time" + + "github.com/minio/minio/internal/bucket/lifecycle" + xhttp "github.com/minio/minio/internal/http" +) + +func Test_hashDeterministicString(t *testing.T) { + tests := []struct { + name string + arg map[string]string + }{ + { + name: "zero", + arg: map[string]string{}, + }, + { + name: "nil", + arg: nil, + }, + { + name: "one", + arg: map[string]string{"key": "value"}, + }, + { + name: "several", + arg: map[string]string{ + xhttp.AmzRestore: "FAILED", + xhttp.ContentMD5: mustGetUUID(), + xhttp.AmzBucketReplicationStatus: "PENDING", + xhttp.ContentType: "application/json", + }, + }, + { + name: "someempty", + arg: map[string]string{ + xhttp.AmzRestore: "", + xhttp.ContentMD5: mustGetUUID(), + xhttp.AmzBucketReplicationStatus: "", + xhttp.ContentType: "application/json", + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + const n = 100 + want := hashDeterministicString(tt.arg) + m := tt.arg + for range n { + if got := hashDeterministicString(m); got != want { + t.Errorf("hashDeterministicString() = %v, want %v", got, want) + } + } + // Check casual collisions + if m == nil { + m = make(map[string]string) + } + m["12312312"] = "" + if got := hashDeterministicString(m); got == want { + t.Errorf("hashDeterministicString() = %v, does not want %v", got, want) + } + want = hashDeterministicString(m) + delete(m, "12312312") + m["another"] = "" + + if got := hashDeterministicString(m); got == want { + t.Errorf("hashDeterministicString() = %v, does not want %v", got, want) + } + + want = hashDeterministicString(m) + m["another"] = "hashDeterministicString" + if got := hashDeterministicString(m); got == want { + t.Errorf("hashDeterministicString() = %v, does not want %v", got, want) + } + + want = hashDeterministicString(m) + m["another"] = "hashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicString" + if got := hashDeterministicString(m); got == want { + t.Errorf("hashDeterministicString() = %v, does not want %v", got, want) + } + + // Flip key/value + want = hashDeterministicString(m) + delete(m, "another") + m["hashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicStringhashDeterministicString"] = "another" + if got := hashDeterministicString(m); got == want { + t.Errorf("hashDeterministicString() = %v, does not want %v", got, want) + } + }) + } +} + +func TestGetFileInfoVersions(t *testing.T) { + basefi := FileInfo{ + Volume: "volume", + Name: "object-name", + VersionID: "756100c6-b393-4981-928a-d49bbc164741", + IsLatest: true, + Deleted: false, + TransitionStatus: "", + DataDir: "bffea160-ca7f-465f-98bc-9b4f1c3ba1ef", + XLV1: false, + ModTime: time.Now().UTC(), + Size: 0, + Mode: 0, + Metadata: nil, + Parts: nil, + Erasure: ErasureInfo{ + Algorithm: ReedSolomon.String(), + DataBlocks: 4, + ParityBlocks: 2, + BlockSize: 10000, + Index: 1, + Distribution: []int{1, 2, 3, 4, 5, 6, 7, 8}, + Checksums: []ChecksumInfo{{ + PartNumber: 1, + Algorithm: HighwayHash256S, + Hash: nil, + }}, + }, + MarkDeleted: false, + NumVersions: 1, + SuccessorModTime: time.Time{}, + } + xl := xlMetaV2{} + var versions []FileInfo + var allVersionIDs, freeVersionIDs []string + for i := range 5 { + fi := basefi + fi.VersionID = mustGetUUID() + fi.DataDir = mustGetUUID() + fi.ModTime = basefi.ModTime.Add(time.Duration(i) * time.Second) + if err := xl.AddVersion(fi); err != nil { + t.Fatalf("%d: Failed to add version %v", i+1, err) + } + + if i > 3 { + // Simulate transition of a version + transfi := fi + transfi.TransitionStatus = lifecycle.TransitionComplete + transfi.TransitionTier = "MINIO-TIER" + transfi.TransitionedObjName = mustGetUUID() + xl.DeleteVersion(transfi) + + fi.SetTierFreeVersionID(mustGetUUID()) + // delete this version leading to a free version + xl.DeleteVersion(fi) + freeVersionIDs = append(freeVersionIDs, fi.TierFreeVersionID()) + allVersionIDs = append(allVersionIDs, fi.TierFreeVersionID()) + } else { + versions = append(versions, fi) + allVersionIDs = append(allVersionIDs, fi.VersionID) + } + } + buf, err := xl.AppendTo(nil) + if err != nil { + t.Fatalf("Failed to serialize xlmeta %v", err) + } + fivs, err := getFileInfoVersions(buf, basefi.Volume, basefi.Name, false) + if err != nil { + t.Fatalf("getFileInfoVersions failed: %v", err) + } + chkNumVersions := func(fis []FileInfo) bool { + for i := 0; i < len(fis)-1; i++ { + if fis[i].NumVersions != fis[i+1].NumVersions { + return false + } + } + return true + } + if !chkNumVersions(fivs.Versions) { + t.Fatalf("Expected all versions to have the same NumVersions") + } + + sort.Slice(versions, func(i, j int) bool { + if versions[i].IsLatest { + return true + } + if versions[j].IsLatest { + return false + } + return versions[i].ModTime.After(versions[j].ModTime) + }) + + for i, fi := range fivs.Versions { + if fi.VersionID != versions[i].VersionID { + t.Fatalf("getFileInfoVersions: versions don't match at %d, version id expected %s but got %s", i, fi.VersionID, versions[i].VersionID) + } + if fi.NumVersions != len(fivs.Versions) { + t.Fatalf("getFileInfoVersions: version with %s version id expected to have %d as NumVersions but got %d", fi.VersionID, len(fivs.Versions), fi.NumVersions) + } + } + + for i, free := range fivs.FreeVersions { + if free.VersionID != freeVersionIDs[i] { + t.Fatalf("getFileInfoVersions: free versions don't match at %d, version id expected %s but got %s", i, free.VersionID, freeVersionIDs[i]) + } + } + + // versions are stored in xl-meta sorted in descending order of their ModTime + slices.Reverse(allVersionIDs) + + fivs, err = getFileInfoVersions(buf, basefi.Volume, basefi.Name, true) + if err != nil { + t.Fatalf("getFileInfoVersions failed: %v", err) + } + if !chkNumVersions(fivs.Versions) { + t.Fatalf("Expected all versions to have the same NumVersions") + } + for i, fi := range fivs.Versions { + if fi.VersionID != allVersionIDs[i] { + t.Fatalf("getFileInfoVersions: all versions don't match at %d expected %s but got %s", i, allVersionIDs[i], fi.VersionID) + } + } +} diff --git a/cmd/xl-storage-format-v1.go b/cmd/xl-storage-format-v1.go new file mode 100644 index 00000000..ae9ed631 --- /dev/null +++ b/cmd/xl-storage-format-v1.go @@ -0,0 +1,279 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "encoding/binary" + "encoding/hex" + "encoding/json" + "fmt" + "time" + + "github.com/cespare/xxhash/v2" + jsoniter "github.com/json-iterator/go" +) + +// XL constants. +const ( + // XL metadata file carries per object metadata. + xlStorageFormatFileV1 = "xl.json" +) + +// Valid - tells us if the format is sane by validating +// format version and erasure coding information. +func (m *xlMetaV1Object) valid() bool { + return isXLMetaFormatValid(m.Version, m.Format) && + isXLMetaErasureInfoValid(m.Erasure.DataBlocks, m.Erasure.ParityBlocks) +} + +// Verifies if the backend format metadata is sane by validating +// the version string and format style. +func isXLMetaFormatValid(version, format string) bool { + return ((version == xlMetaVersion101 || + version == xlMetaVersion100) && + format == xlMetaFormat) +} + +// Verifies if the backend format metadata is sane by validating +// the ErasureInfo, i.e. data and parity blocks. +func isXLMetaErasureInfoValid(data, parity int) bool { + return ((data >= parity) && (data > 0) && (parity >= 0)) +} + +//msgp:clearomitted + +//go:generate msgp -file=$GOFILE -unexported + +// A xlMetaV1Object represents `xl.meta` metadata header. +type xlMetaV1Object struct { + Version string `json:"version"` // Version of the current `xl.meta`. + Format string `json:"format"` // Format of the current `xl.meta`. + Stat StatInfo `json:"stat"` // Stat of the current object `xl.meta`. + // Erasure coded info for the current object `xl.meta`. + Erasure ErasureInfo `json:"erasure"` + // MinIO release tag for current object `xl.meta`. + Minio struct { + Release string `json:"release"` + } `json:"minio"` + // Metadata map for current object `xl.meta`. + Meta map[string]string `json:"meta,omitempty"` + // Captures all the individual object `xl.meta`. + Parts []ObjectPartInfo `json:"parts,omitempty"` + + // Dummy values used for legacy use cases. + VersionID string `json:"versionId,omitempty"` + DataDir string `json:"dataDir,omitempty"` // always points to "legacy" +} + +// StatInfo - carries stat information of the object. +type StatInfo struct { + Size int64 `json:"size"` // Size of the object `xl.meta`. + ModTime time.Time `json:"modTime"` // ModTime of the object `xl.meta`. + Name string `json:"name"` + Dir bool `json:"dir"` + Mode uint32 `json:"mode"` +} + +// ErasureInfo holds erasure coding and bitrot related information. +type ErasureInfo struct { + // Algorithm is the string representation of erasure-coding-algorithm + Algorithm string `json:"algorithm"` + // DataBlocks is the number of data blocks for erasure-coding + DataBlocks int `json:"data"` + // ParityBlocks is the number of parity blocks for erasure-coding + ParityBlocks int `json:"parity"` + // BlockSize is the size of one erasure-coded block + BlockSize int64 `json:"blockSize"` + // Index is the index of the current disk + Index int `json:"index"` + // Distribution is the distribution of the data and parity blocks + Distribution []int `json:"distribution"` + // Checksums holds all bitrot checksums of all erasure encoded blocks + Checksums []ChecksumInfo `json:"checksum,omitempty"` +} + +// Equal equates current erasure info with newer erasure info. +// returns false if one of the following check fails +// - erasure algorithm is different +// - data blocks are different +// - parity blocks are different +// - block size is different +// - distribution array size is different +// - distribution indexes are different +func (ei ErasureInfo) Equal(nei ErasureInfo) bool { + if ei.Algorithm != nei.Algorithm { + return false + } + if ei.DataBlocks != nei.DataBlocks { + return false + } + if ei.ParityBlocks != nei.ParityBlocks { + return false + } + if ei.BlockSize != nei.BlockSize { + return false + } + if len(ei.Distribution) != len(nei.Distribution) { + return false + } + for i, ecindex := range ei.Distribution { + if ecindex != nei.Distribution[i] { + return false + } + } + return true +} + +// BitrotAlgorithm specifies a algorithm used for bitrot protection. +type BitrotAlgorithm uint + +const ( + // SHA256 represents the SHA-256 hash function + SHA256 BitrotAlgorithm = 1 + iota + // HighwayHash256 represents the HighwayHash-256 hash function + HighwayHash256 + // HighwayHash256S represents the Streaming HighwayHash-256 hash function + HighwayHash256S + // BLAKE2b512 represents the BLAKE2b-512 hash function + BLAKE2b512 +) + +// DefaultBitrotAlgorithm is the default algorithm used for bitrot protection. +const ( + DefaultBitrotAlgorithm = HighwayHash256S +) + +// ObjectPartInfo Info of each part kept in the multipart metadata +// file after CompleteMultipartUpload() is called. +type ObjectPartInfo struct { + ETag string `json:"etag,omitempty" msg:"e"` + Number int `json:"number" msg:"n"` + Size int64 `json:"size" msg:"s"` // Size of the part on the disk. + ActualSize int64 `json:"actualSize" msg:"as"` // Original size of the part without compression or encryption bytes. + ModTime time.Time `json:"modTime" msg:"mt"` // Date and time at which the part was uploaded. + Index []byte `json:"index,omitempty" msg:"i,omitempty"` + Checksums map[string]string `json:"crc,omitempty" msg:"crc,omitempty"` // Content Checksums + Error string `json:"error,omitempty" msg:"err,omitempty"` // only set while reading part meta from drive. +} + +// ChecksumInfo - carries checksums of individual scattered parts per disk. +type ChecksumInfo struct { + PartNumber int + Algorithm BitrotAlgorithm + Hash []byte +} + +type checksumInfoJSON struct { + Name string `json:"name"` + Algorithm string `json:"algorithm"` + Hash string `json:"hash,omitempty"` +} + +// MarshalJSON marshals the ChecksumInfo struct +func (c ChecksumInfo) MarshalJSON() ([]byte, error) { + info := checksumInfoJSON{ + Name: fmt.Sprintf("part.%d", c.PartNumber), + Algorithm: c.Algorithm.String(), + Hash: hex.EncodeToString(c.Hash), + } + return json.Marshal(info) +} + +// UnmarshalJSON - custom checksum info unmarshaller +func (c *ChecksumInfo) UnmarshalJSON(data []byte) error { + var info checksumInfoJSON + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err := json.Unmarshal(data, &info); err != nil { + return err + } + sum, err := hex.DecodeString(info.Hash) + if err != nil { + return err + } + c.Algorithm = BitrotAlgorithmFromString(info.Algorithm) + c.Hash = sum + if _, err = fmt.Sscanf(info.Name, "part.%d", &c.PartNumber); err != nil { + return err + } + + if !c.Algorithm.Available() { + internalLogIf(GlobalContext, errBitrotHashAlgoInvalid) + return errBitrotHashAlgoInvalid + } + return nil +} + +// constant and shouldn't be changed. +const ( + legacyDataDir = "legacy" +) + +func (m *xlMetaV1Object) ToFileInfo(volume, path string) (FileInfo, error) { + if !m.valid() { + return FileInfo{}, errFileCorrupt + } + + fi := FileInfo{ + Volume: volume, + Name: path, + ModTime: m.Stat.ModTime, + Size: m.Stat.Size, + Metadata: m.Meta, + Parts: m.Parts, + Erasure: m.Erasure, + VersionID: m.VersionID, + DataDir: m.DataDir, + XLV1: true, + NumVersions: 1, + } + + return fi, nil +} + +// Signature will return a signature that is expected to be the same across all disks. +func (m *xlMetaV1Object) Signature() [4]byte { + // Shallow copy + c := *m + // Zero unimportant fields + c.Erasure.Index = 0 + c.Minio.Release = "" + crc := hashDeterministicString(c.Meta) + c.Meta = nil + + if bts, err := c.MarshalMsg(metaDataPoolGet()); err == nil { + crc ^= xxhash.Sum64(bts) + metaDataPoolPut(bts) + } + + // Combine upper and lower part + var tmp [4]byte + binary.LittleEndian.PutUint32(tmp[:], uint32(crc^(crc>>32))) + return tmp +} + +// XL metadata constants. +const ( + // XL meta version. + xlMetaVersion101 = "1.0.1" + + // XL meta version. + xlMetaVersion100 = "1.0.0" + + // XL meta format string. + xlMetaFormat = "xl" +) diff --git a/cmd/xl-storage-format-v1_gen.go b/cmd/xl-storage-format-v1_gen.go new file mode 100644 index 00000000..02d48425 --- /dev/null +++ b/cmd/xl-storage-format-v1_gen.go @@ -0,0 +1,1821 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *BitrotAlgorithm) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 uint + zb0001, err = dc.ReadUint() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = BitrotAlgorithm(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z BitrotAlgorithm) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteUint(uint(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z BitrotAlgorithm) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendUint(o, uint(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BitrotAlgorithm) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 uint + zb0001, bts, err = msgp.ReadUintBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = BitrotAlgorithm(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z BitrotAlgorithm) Msgsize() (s int) { + s = msgp.UintSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ChecksumInfo) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "PartNumber": + z.PartNumber, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "PartNumber") + return + } + case "Algorithm": + { + var zb0002 uint + zb0002, err = dc.ReadUint() + if err != nil { + err = msgp.WrapError(err, "Algorithm") + return + } + z.Algorithm = BitrotAlgorithm(zb0002) + } + case "Hash": + z.Hash, err = dc.ReadBytes(z.Hash) + if err != nil { + err = msgp.WrapError(err, "Hash") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *ChecksumInfo) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "PartNumber" + err = en.Append(0x83, 0xaa, 0x50, 0x61, 0x72, 0x74, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72) + if err != nil { + return + } + err = en.WriteInt(z.PartNumber) + if err != nil { + err = msgp.WrapError(err, "PartNumber") + return + } + // write "Algorithm" + err = en.Append(0xa9, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d) + if err != nil { + return + } + err = en.WriteUint(uint(z.Algorithm)) + if err != nil { + err = msgp.WrapError(err, "Algorithm") + return + } + // write "Hash" + err = en.Append(0xa4, 0x48, 0x61, 0x73, 0x68) + if err != nil { + return + } + err = en.WriteBytes(z.Hash) + if err != nil { + err = msgp.WrapError(err, "Hash") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ChecksumInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "PartNumber" + o = append(o, 0x83, 0xaa, 0x50, 0x61, 0x72, 0x74, 0x4e, 0x75, 0x6d, 0x62, 0x65, 0x72) + o = msgp.AppendInt(o, z.PartNumber) + // string "Algorithm" + o = append(o, 0xa9, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d) + o = msgp.AppendUint(o, uint(z.Algorithm)) + // string "Hash" + o = append(o, 0xa4, 0x48, 0x61, 0x73, 0x68) + o = msgp.AppendBytes(o, z.Hash) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ChecksumInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "PartNumber": + z.PartNumber, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "PartNumber") + return + } + case "Algorithm": + { + var zb0002 uint + zb0002, bts, err = msgp.ReadUintBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Algorithm") + return + } + z.Algorithm = BitrotAlgorithm(zb0002) + } + case "Hash": + z.Hash, bts, err = msgp.ReadBytesBytes(bts, z.Hash) + if err != nil { + err = msgp.WrapError(err, "Hash") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ChecksumInfo) Msgsize() (s int) { + s = 1 + 11 + msgp.IntSize + 10 + msgp.UintSize + 5 + msgp.BytesPrefixSize + len(z.Hash) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ErasureInfo) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Algorithm": + z.Algorithm, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Algorithm") + return + } + case "DataBlocks": + z.DataBlocks, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "DataBlocks") + return + } + case "ParityBlocks": + z.ParityBlocks, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "ParityBlocks") + return + } + case "BlockSize": + z.BlockSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "BlockSize") + return + } + case "Index": + z.Index, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Index") + return + } + case "Distribution": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Distribution") + return + } + if cap(z.Distribution) >= int(zb0002) { + z.Distribution = (z.Distribution)[:zb0002] + } else { + z.Distribution = make([]int, zb0002) + } + for za0001 := range z.Distribution { + z.Distribution[za0001], err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Distribution", za0001) + return + } + } + case "Checksums": + var zb0003 uint32 + zb0003, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Checksums") + return + } + if cap(z.Checksums) >= int(zb0003) { + z.Checksums = (z.Checksums)[:zb0003] + } else { + z.Checksums = make([]ChecksumInfo, zb0003) + } + for za0002 := range z.Checksums { + err = z.Checksums[za0002].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Checksums", za0002) + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *ErasureInfo) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 7 + // write "Algorithm" + err = en.Append(0x87, 0xa9, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d) + if err != nil { + return + } + err = en.WriteString(z.Algorithm) + if err != nil { + err = msgp.WrapError(err, "Algorithm") + return + } + // write "DataBlocks" + err = en.Append(0xaa, 0x44, 0x61, 0x74, 0x61, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x73) + if err != nil { + return + } + err = en.WriteInt(z.DataBlocks) + if err != nil { + err = msgp.WrapError(err, "DataBlocks") + return + } + // write "ParityBlocks" + err = en.Append(0xac, 0x50, 0x61, 0x72, 0x69, 0x74, 0x79, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x73) + if err != nil { + return + } + err = en.WriteInt(z.ParityBlocks) + if err != nil { + err = msgp.WrapError(err, "ParityBlocks") + return + } + // write "BlockSize" + err = en.Append(0xa9, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.BlockSize) + if err != nil { + err = msgp.WrapError(err, "BlockSize") + return + } + // write "Index" + err = en.Append(0xa5, 0x49, 0x6e, 0x64, 0x65, 0x78) + if err != nil { + return + } + err = en.WriteInt(z.Index) + if err != nil { + err = msgp.WrapError(err, "Index") + return + } + // write "Distribution" + err = en.Append(0xac, 0x44, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Distribution))) + if err != nil { + err = msgp.WrapError(err, "Distribution") + return + } + for za0001 := range z.Distribution { + err = en.WriteInt(z.Distribution[za0001]) + if err != nil { + err = msgp.WrapError(err, "Distribution", za0001) + return + } + } + // write "Checksums" + err = en.Append(0xa9, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Checksums))) + if err != nil { + err = msgp.WrapError(err, "Checksums") + return + } + for za0002 := range z.Checksums { + err = z.Checksums[za0002].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Checksums", za0002) + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ErasureInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 7 + // string "Algorithm" + o = append(o, 0x87, 0xa9, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d) + o = msgp.AppendString(o, z.Algorithm) + // string "DataBlocks" + o = append(o, 0xaa, 0x44, 0x61, 0x74, 0x61, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x73) + o = msgp.AppendInt(o, z.DataBlocks) + // string "ParityBlocks" + o = append(o, 0xac, 0x50, 0x61, 0x72, 0x69, 0x74, 0x79, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x73) + o = msgp.AppendInt(o, z.ParityBlocks) + // string "BlockSize" + o = append(o, 0xa9, 0x42, 0x6c, 0x6f, 0x63, 0x6b, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.BlockSize) + // string "Index" + o = append(o, 0xa5, 0x49, 0x6e, 0x64, 0x65, 0x78) + o = msgp.AppendInt(o, z.Index) + // string "Distribution" + o = append(o, 0xac, 0x44, 0x69, 0x73, 0x74, 0x72, 0x69, 0x62, 0x75, 0x74, 0x69, 0x6f, 0x6e) + o = msgp.AppendArrayHeader(o, uint32(len(z.Distribution))) + for za0001 := range z.Distribution { + o = msgp.AppendInt(o, z.Distribution[za0001]) + } + // string "Checksums" + o = append(o, 0xa9, 0x43, 0x68, 0x65, 0x63, 0x6b, 0x73, 0x75, 0x6d, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Checksums))) + for za0002 := range z.Checksums { + o, err = z.Checksums[za0002].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Checksums", za0002) + return + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ErasureInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Algorithm": + z.Algorithm, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Algorithm") + return + } + case "DataBlocks": + z.DataBlocks, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DataBlocks") + return + } + case "ParityBlocks": + z.ParityBlocks, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ParityBlocks") + return + } + case "BlockSize": + z.BlockSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "BlockSize") + return + } + case "Index": + z.Index, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Index") + return + } + case "Distribution": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Distribution") + return + } + if cap(z.Distribution) >= int(zb0002) { + z.Distribution = (z.Distribution)[:zb0002] + } else { + z.Distribution = make([]int, zb0002) + } + for za0001 := range z.Distribution { + z.Distribution[za0001], bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Distribution", za0001) + return + } + } + case "Checksums": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Checksums") + return + } + if cap(z.Checksums) >= int(zb0003) { + z.Checksums = (z.Checksums)[:zb0003] + } else { + z.Checksums = make([]ChecksumInfo, zb0003) + } + for za0002 := range z.Checksums { + bts, err = z.Checksums[za0002].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Checksums", za0002) + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ErasureInfo) Msgsize() (s int) { + s = 1 + 10 + msgp.StringPrefixSize + len(z.Algorithm) + 11 + msgp.IntSize + 13 + msgp.IntSize + 10 + msgp.Int64Size + 6 + msgp.IntSize + 13 + msgp.ArrayHeaderSize + (len(z.Distribution) * (msgp.IntSize)) + 10 + msgp.ArrayHeaderSize + for za0002 := range z.Checksums { + s += z.Checksums[za0002].Msgsize() + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ObjectPartInfo) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 3 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "e": + z.ETag, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "ETag") + return + } + case "n": + z.Number, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Number") + return + } + case "s": + z.Size, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + case "as": + z.ActualSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ActualSize") + return + } + case "mt": + z.ModTime, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + case "i": + z.Index, err = dc.ReadBytes(z.Index) + if err != nil { + err = msgp.WrapError(err, "Index") + return + } + zb0001Mask |= 0x1 + case "crc": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Checksums") + return + } + if z.Checksums == nil { + z.Checksums = make(map[string]string, zb0002) + } else if len(z.Checksums) > 0 { + clear(z.Checksums) + } + for zb0002 > 0 { + zb0002-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Checksums") + return + } + var za0002 string + za0002, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Checksums", za0001) + return + } + z.Checksums[za0001] = za0002 + } + zb0001Mask |= 0x2 + case "err": + z.Error, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Error") + return + } + zb0001Mask |= 0x4 + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if zb0001Mask != 0x7 { + if (zb0001Mask & 0x1) == 0 { + z.Index = nil + } + if (zb0001Mask & 0x2) == 0 { + z.Checksums = nil + } + if (zb0001Mask & 0x4) == 0 { + z.Error = "" + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *ObjectPartInfo) EncodeMsg(en *msgp.Writer) (err error) { + // check for omitted fields + zb0001Len := uint32(8) + var zb0001Mask uint8 /* 8 bits */ + _ = zb0001Mask + if z.Index == nil { + zb0001Len-- + zb0001Mask |= 0x20 + } + if z.Checksums == nil { + zb0001Len-- + zb0001Mask |= 0x40 + } + if z.Error == "" { + zb0001Len-- + zb0001Mask |= 0x80 + } + // variable map header, size zb0001Len + err = en.Append(0x80 | uint8(zb0001Len)) + if err != nil { + return + } + + // skip if no fields are to be emitted + if zb0001Len != 0 { + // write "e" + err = en.Append(0xa1, 0x65) + if err != nil { + return + } + err = en.WriteString(z.ETag) + if err != nil { + err = msgp.WrapError(err, "ETag") + return + } + // write "n" + err = en.Append(0xa1, 0x6e) + if err != nil { + return + } + err = en.WriteInt(z.Number) + if err != nil { + err = msgp.WrapError(err, "Number") + return + } + // write "s" + err = en.Append(0xa1, 0x73) + if err != nil { + return + } + err = en.WriteInt64(z.Size) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + // write "as" + err = en.Append(0xa2, 0x61, 0x73) + if err != nil { + return + } + err = en.WriteInt64(z.ActualSize) + if err != nil { + err = msgp.WrapError(err, "ActualSize") + return + } + // write "mt" + err = en.Append(0xa2, 0x6d, 0x74) + if err != nil { + return + } + err = en.WriteTime(z.ModTime) + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + if (zb0001Mask & 0x20) == 0 { // if not omitted + // write "i" + err = en.Append(0xa1, 0x69) + if err != nil { + return + } + err = en.WriteBytes(z.Index) + if err != nil { + err = msgp.WrapError(err, "Index") + return + } + } + if (zb0001Mask & 0x40) == 0 { // if not omitted + // write "crc" + err = en.Append(0xa3, 0x63, 0x72, 0x63) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.Checksums))) + if err != nil { + err = msgp.WrapError(err, "Checksums") + return + } + for za0001, za0002 := range z.Checksums { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "Checksums") + return + } + err = en.WriteString(za0002) + if err != nil { + err = msgp.WrapError(err, "Checksums", za0001) + return + } + } + } + if (zb0001Mask & 0x80) == 0 { // if not omitted + // write "err" + err = en.Append(0xa3, 0x65, 0x72, 0x72) + if err != nil { + return + } + err = en.WriteString(z.Error) + if err != nil { + err = msgp.WrapError(err, "Error") + return + } + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *ObjectPartInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // check for omitted fields + zb0001Len := uint32(8) + var zb0001Mask uint8 /* 8 bits */ + _ = zb0001Mask + if z.Index == nil { + zb0001Len-- + zb0001Mask |= 0x20 + } + if z.Checksums == nil { + zb0001Len-- + zb0001Mask |= 0x40 + } + if z.Error == "" { + zb0001Len-- + zb0001Mask |= 0x80 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + + // skip if no fields are to be emitted + if zb0001Len != 0 { + // string "e" + o = append(o, 0xa1, 0x65) + o = msgp.AppendString(o, z.ETag) + // string "n" + o = append(o, 0xa1, 0x6e) + o = msgp.AppendInt(o, z.Number) + // string "s" + o = append(o, 0xa1, 0x73) + o = msgp.AppendInt64(o, z.Size) + // string "as" + o = append(o, 0xa2, 0x61, 0x73) + o = msgp.AppendInt64(o, z.ActualSize) + // string "mt" + o = append(o, 0xa2, 0x6d, 0x74) + o = msgp.AppendTime(o, z.ModTime) + if (zb0001Mask & 0x20) == 0 { // if not omitted + // string "i" + o = append(o, 0xa1, 0x69) + o = msgp.AppendBytes(o, z.Index) + } + if (zb0001Mask & 0x40) == 0 { // if not omitted + // string "crc" + o = append(o, 0xa3, 0x63, 0x72, 0x63) + o = msgp.AppendMapHeader(o, uint32(len(z.Checksums))) + for za0001, za0002 := range z.Checksums { + o = msgp.AppendString(o, za0001) + o = msgp.AppendString(o, za0002) + } + } + if (zb0001Mask & 0x80) == 0 { // if not omitted + // string "err" + o = append(o, 0xa3, 0x65, 0x72, 0x72) + o = msgp.AppendString(o, z.Error) + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ObjectPartInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 3 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "e": + z.ETag, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ETag") + return + } + case "n": + z.Number, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Number") + return + } + case "s": + z.Size, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + case "as": + z.ActualSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ActualSize") + return + } + case "mt": + z.ModTime, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + case "i": + z.Index, bts, err = msgp.ReadBytesBytes(bts, z.Index) + if err != nil { + err = msgp.WrapError(err, "Index") + return + } + zb0001Mask |= 0x1 + case "crc": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Checksums") + return + } + if z.Checksums == nil { + z.Checksums = make(map[string]string, zb0002) + } else if len(z.Checksums) > 0 { + clear(z.Checksums) + } + for zb0002 > 0 { + var za0002 string + zb0002-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Checksums") + return + } + za0002, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Checksums", za0001) + return + } + z.Checksums[za0001] = za0002 + } + zb0001Mask |= 0x2 + case "err": + z.Error, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Error") + return + } + zb0001Mask |= 0x4 + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if zb0001Mask != 0x7 { + if (zb0001Mask & 0x1) == 0 { + z.Index = nil + } + if (zb0001Mask & 0x2) == 0 { + z.Checksums = nil + } + if (zb0001Mask & 0x4) == 0 { + z.Error = "" + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *ObjectPartInfo) Msgsize() (s int) { + s = 1 + 2 + msgp.StringPrefixSize + len(z.ETag) + 2 + msgp.IntSize + 2 + msgp.Int64Size + 3 + msgp.Int64Size + 3 + msgp.TimeSize + 2 + msgp.BytesPrefixSize + len(z.Index) + 4 + msgp.MapHeaderSize + if z.Checksums != nil { + for za0001, za0002 := range z.Checksums { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + msgp.StringPrefixSize + len(za0002) + } + } + s += 4 + msgp.StringPrefixSize + len(z.Error) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *StatInfo) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Size": + z.Size, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + case "ModTime": + z.ModTime, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + case "Name": + z.Name, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "Dir": + z.Dir, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Dir") + return + } + case "Mode": + z.Mode, err = dc.ReadUint32() + if err != nil { + err = msgp.WrapError(err, "Mode") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *StatInfo) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 5 + // write "Size" + err = en.Append(0x85, 0xa4, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.Size) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + // write "ModTime" + err = en.Append(0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteTime(z.ModTime) + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + // write "Name" + err = en.Append(0xa4, 0x4e, 0x61, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteString(z.Name) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + // write "Dir" + err = en.Append(0xa3, 0x44, 0x69, 0x72) + if err != nil { + return + } + err = en.WriteBool(z.Dir) + if err != nil { + err = msgp.WrapError(err, "Dir") + return + } + // write "Mode" + err = en.Append(0xa4, 0x4d, 0x6f, 0x64, 0x65) + if err != nil { + return + } + err = en.WriteUint32(z.Mode) + if err != nil { + err = msgp.WrapError(err, "Mode") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *StatInfo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 5 + // string "Size" + o = append(o, 0x85, 0xa4, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.Size) + // string "ModTime" + o = append(o, 0xa7, 0x4d, 0x6f, 0x64, 0x54, 0x69, 0x6d, 0x65) + o = msgp.AppendTime(o, z.ModTime) + // string "Name" + o = append(o, 0xa4, 0x4e, 0x61, 0x6d, 0x65) + o = msgp.AppendString(o, z.Name) + // string "Dir" + o = append(o, 0xa3, 0x44, 0x69, 0x72) + o = msgp.AppendBool(o, z.Dir) + // string "Mode" + o = append(o, 0xa4, 0x4d, 0x6f, 0x64, 0x65) + o = msgp.AppendUint32(o, z.Mode) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *StatInfo) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Size": + z.Size, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + case "ModTime": + z.ModTime, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + case "Name": + z.Name, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "Dir": + z.Dir, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Dir") + return + } + case "Mode": + z.Mode, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Mode") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *StatInfo) Msgsize() (s int) { + s = 1 + 5 + msgp.Int64Size + 8 + msgp.TimeSize + 5 + msgp.StringPrefixSize + len(z.Name) + 4 + msgp.BoolSize + 5 + msgp.Uint32Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *checksumInfoJSON) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Name": + z.Name, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "Algorithm": + z.Algorithm, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Algorithm") + return + } + case "Hash": + z.Hash, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Hash") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z checksumInfoJSON) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "Name" + err = en.Append(0x83, 0xa4, 0x4e, 0x61, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteString(z.Name) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + // write "Algorithm" + err = en.Append(0xa9, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d) + if err != nil { + return + } + err = en.WriteString(z.Algorithm) + if err != nil { + err = msgp.WrapError(err, "Algorithm") + return + } + // write "Hash" + err = en.Append(0xa4, 0x48, 0x61, 0x73, 0x68) + if err != nil { + return + } + err = en.WriteString(z.Hash) + if err != nil { + err = msgp.WrapError(err, "Hash") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z checksumInfoJSON) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "Name" + o = append(o, 0x83, 0xa4, 0x4e, 0x61, 0x6d, 0x65) + o = msgp.AppendString(o, z.Name) + // string "Algorithm" + o = append(o, 0xa9, 0x41, 0x6c, 0x67, 0x6f, 0x72, 0x69, 0x74, 0x68, 0x6d) + o = msgp.AppendString(o, z.Algorithm) + // string "Hash" + o = append(o, 0xa4, 0x48, 0x61, 0x73, 0x68) + o = msgp.AppendString(o, z.Hash) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *checksumInfoJSON) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Name": + z.Name, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Name") + return + } + case "Algorithm": + z.Algorithm, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Algorithm") + return + } + case "Hash": + z.Hash, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Hash") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z checksumInfoJSON) Msgsize() (s int) { + s = 1 + 5 + msgp.StringPrefixSize + len(z.Name) + 10 + msgp.StringPrefixSize + len(z.Algorithm) + 5 + msgp.StringPrefixSize + len(z.Hash) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *xlMetaV1Object) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Version": + z.Version, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Version") + return + } + case "Format": + z.Format, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Format") + return + } + case "Stat": + err = z.Stat.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Stat") + return + } + case "Erasure": + err = z.Erasure.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Erasure") + return + } + case "Minio": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Minio") + return + } + for zb0002 > 0 { + zb0002-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "Minio") + return + } + switch msgp.UnsafeString(field) { + case "Release": + z.Minio.Release, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Minio", "Release") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "Minio") + return + } + } + } + case "Meta": + var zb0003 uint32 + zb0003, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Meta") + return + } + if z.Meta == nil { + z.Meta = make(map[string]string, zb0003) + } else if len(z.Meta) > 0 { + clear(z.Meta) + } + for zb0003 > 0 { + zb0003-- + var za0001 string + za0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Meta") + return + } + var za0002 string + za0002, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Meta", za0001) + return + } + z.Meta[za0001] = za0002 + } + case "Parts": + var zb0004 uint32 + zb0004, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Parts") + return + } + if cap(z.Parts) >= int(zb0004) { + z.Parts = (z.Parts)[:zb0004] + } else { + z.Parts = make([]ObjectPartInfo, zb0004) + } + for za0003 := range z.Parts { + err = z.Parts[za0003].DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "Parts", za0003) + return + } + } + case "VersionID": + z.VersionID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + case "DataDir": + z.DataDir, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "DataDir") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *xlMetaV1Object) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 9 + // write "Version" + err = en.Append(0x89, 0xa7, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e) + if err != nil { + return + } + err = en.WriteString(z.Version) + if err != nil { + err = msgp.WrapError(err, "Version") + return + } + // write "Format" + err = en.Append(0xa6, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Format) + if err != nil { + err = msgp.WrapError(err, "Format") + return + } + // write "Stat" + err = en.Append(0xa4, 0x53, 0x74, 0x61, 0x74) + if err != nil { + return + } + err = z.Stat.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Stat") + return + } + // write "Erasure" + err = en.Append(0xa7, 0x45, 0x72, 0x61, 0x73, 0x75, 0x72, 0x65) + if err != nil { + return + } + err = z.Erasure.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Erasure") + return + } + // write "Minio" + err = en.Append(0xa5, 0x4d, 0x69, 0x6e, 0x69, 0x6f) + if err != nil { + return + } + // map header, size 1 + // write "Release" + err = en.Append(0x81, 0xa7, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65) + if err != nil { + return + } + err = en.WriteString(z.Minio.Release) + if err != nil { + err = msgp.WrapError(err, "Minio", "Release") + return + } + // write "Meta" + err = en.Append(0xa4, 0x4d, 0x65, 0x74, 0x61) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.Meta))) + if err != nil { + err = msgp.WrapError(err, "Meta") + return + } + for za0001, za0002 := range z.Meta { + err = en.WriteString(za0001) + if err != nil { + err = msgp.WrapError(err, "Meta") + return + } + err = en.WriteString(za0002) + if err != nil { + err = msgp.WrapError(err, "Meta", za0001) + return + } + } + // write "Parts" + err = en.Append(0xa5, 0x50, 0x61, 0x72, 0x74, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Parts))) + if err != nil { + err = msgp.WrapError(err, "Parts") + return + } + for za0003 := range z.Parts { + err = z.Parts[za0003].EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "Parts", za0003) + return + } + } + // write "VersionID" + err = en.Append(0xa9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x44) + if err != nil { + return + } + err = en.WriteString(z.VersionID) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + // write "DataDir" + err = en.Append(0xa7, 0x44, 0x61, 0x74, 0x61, 0x44, 0x69, 0x72) + if err != nil { + return + } + err = en.WriteString(z.DataDir) + if err != nil { + err = msgp.WrapError(err, "DataDir") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *xlMetaV1Object) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 9 + // string "Version" + o = append(o, 0x89, 0xa7, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e) + o = msgp.AppendString(o, z.Version) + // string "Format" + o = append(o, 0xa6, 0x46, 0x6f, 0x72, 0x6d, 0x61, 0x74) + o = msgp.AppendString(o, z.Format) + // string "Stat" + o = append(o, 0xa4, 0x53, 0x74, 0x61, 0x74) + o, err = z.Stat.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Stat") + return + } + // string "Erasure" + o = append(o, 0xa7, 0x45, 0x72, 0x61, 0x73, 0x75, 0x72, 0x65) + o, err = z.Erasure.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Erasure") + return + } + // string "Minio" + o = append(o, 0xa5, 0x4d, 0x69, 0x6e, 0x69, 0x6f) + // map header, size 1 + // string "Release" + o = append(o, 0x81, 0xa7, 0x52, 0x65, 0x6c, 0x65, 0x61, 0x73, 0x65) + o = msgp.AppendString(o, z.Minio.Release) + // string "Meta" + o = append(o, 0xa4, 0x4d, 0x65, 0x74, 0x61) + o = msgp.AppendMapHeader(o, uint32(len(z.Meta))) + for za0001, za0002 := range z.Meta { + o = msgp.AppendString(o, za0001) + o = msgp.AppendString(o, za0002) + } + // string "Parts" + o = append(o, 0xa5, 0x50, 0x61, 0x72, 0x74, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Parts))) + for za0003 := range z.Parts { + o, err = z.Parts[za0003].MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "Parts", za0003) + return + } + } + // string "VersionID" + o = append(o, 0xa9, 0x56, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x49, 0x44) + o = msgp.AppendString(o, z.VersionID) + // string "DataDir" + o = append(o, 0xa7, 0x44, 0x61, 0x74, 0x61, 0x44, 0x69, 0x72) + o = msgp.AppendString(o, z.DataDir) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *xlMetaV1Object) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Version": + z.Version, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Version") + return + } + case "Format": + z.Format, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Format") + return + } + case "Stat": + bts, err = z.Stat.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Stat") + return + } + case "Erasure": + bts, err = z.Erasure.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Erasure") + return + } + case "Minio": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Minio") + return + } + for zb0002 > 0 { + zb0002-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "Minio") + return + } + switch msgp.UnsafeString(field) { + case "Release": + z.Minio.Release, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Minio", "Release") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "Minio") + return + } + } + } + case "Meta": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Meta") + return + } + if z.Meta == nil { + z.Meta = make(map[string]string, zb0003) + } else if len(z.Meta) > 0 { + clear(z.Meta) + } + for zb0003 > 0 { + var za0002 string + zb0003-- + var za0001 string + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Meta") + return + } + za0002, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Meta", za0001) + return + } + z.Meta[za0001] = za0002 + } + case "Parts": + var zb0004 uint32 + zb0004, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Parts") + return + } + if cap(z.Parts) >= int(zb0004) { + z.Parts = (z.Parts)[:zb0004] + } else { + z.Parts = make([]ObjectPartInfo, zb0004) + } + for za0003 := range z.Parts { + bts, err = z.Parts[za0003].UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "Parts", za0003) + return + } + } + case "VersionID": + z.VersionID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + case "DataDir": + z.DataDir, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "DataDir") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *xlMetaV1Object) Msgsize() (s int) { + s = 1 + 8 + msgp.StringPrefixSize + len(z.Version) + 7 + msgp.StringPrefixSize + len(z.Format) + 5 + z.Stat.Msgsize() + 8 + z.Erasure.Msgsize() + 6 + 1 + 8 + msgp.StringPrefixSize + len(z.Minio.Release) + 5 + msgp.MapHeaderSize + if z.Meta != nil { + for za0001, za0002 := range z.Meta { + _ = za0002 + s += msgp.StringPrefixSize + len(za0001) + msgp.StringPrefixSize + len(za0002) + } + } + s += 6 + msgp.ArrayHeaderSize + for za0003 := range z.Parts { + s += z.Parts[za0003].Msgsize() + } + s += 10 + msgp.StringPrefixSize + len(z.VersionID) + 8 + msgp.StringPrefixSize + len(z.DataDir) + return +} diff --git a/cmd/xl-storage-format-v1_gen_test.go b/cmd/xl-storage-format-v1_gen_test.go new file mode 100644 index 00000000..fba5a025 --- /dev/null +++ b/cmd/xl-storage-format-v1_gen_test.go @@ -0,0 +1,688 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalChecksumInfo(t *testing.T) { + v := ChecksumInfo{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgChecksumInfo(b *testing.B) { + v := ChecksumInfo{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgChecksumInfo(b *testing.B) { + v := ChecksumInfo{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalChecksumInfo(b *testing.B) { + v := ChecksumInfo{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeChecksumInfo(t *testing.T) { + v := ChecksumInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeChecksumInfo Msgsize() is inaccurate") + } + + vn := ChecksumInfo{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeChecksumInfo(b *testing.B) { + v := ChecksumInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeChecksumInfo(b *testing.B) { + v := ChecksumInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalErasureInfo(t *testing.T) { + v := ErasureInfo{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgErasureInfo(b *testing.B) { + v := ErasureInfo{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgErasureInfo(b *testing.B) { + v := ErasureInfo{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalErasureInfo(b *testing.B) { + v := ErasureInfo{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeErasureInfo(t *testing.T) { + v := ErasureInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeErasureInfo Msgsize() is inaccurate") + } + + vn := ErasureInfo{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeErasureInfo(b *testing.B) { + v := ErasureInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeErasureInfo(b *testing.B) { + v := ErasureInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalObjectPartInfo(t *testing.T) { + v := ObjectPartInfo{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgObjectPartInfo(b *testing.B) { + v := ObjectPartInfo{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgObjectPartInfo(b *testing.B) { + v := ObjectPartInfo{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalObjectPartInfo(b *testing.B) { + v := ObjectPartInfo{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeObjectPartInfo(t *testing.T) { + v := ObjectPartInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeObjectPartInfo Msgsize() is inaccurate") + } + + vn := ObjectPartInfo{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeObjectPartInfo(b *testing.B) { + v := ObjectPartInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeObjectPartInfo(b *testing.B) { + v := ObjectPartInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalStatInfo(t *testing.T) { + v := StatInfo{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgStatInfo(b *testing.B) { + v := StatInfo{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgStatInfo(b *testing.B) { + v := StatInfo{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalStatInfo(b *testing.B) { + v := StatInfo{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeStatInfo(t *testing.T) { + v := StatInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeStatInfo Msgsize() is inaccurate") + } + + vn := StatInfo{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeStatInfo(b *testing.B) { + v := StatInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeStatInfo(b *testing.B) { + v := StatInfo{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalchecksumInfoJSON(t *testing.T) { + v := checksumInfoJSON{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgchecksumInfoJSON(b *testing.B) { + v := checksumInfoJSON{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgchecksumInfoJSON(b *testing.B) { + v := checksumInfoJSON{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalchecksumInfoJSON(b *testing.B) { + v := checksumInfoJSON{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodechecksumInfoJSON(t *testing.T) { + v := checksumInfoJSON{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodechecksumInfoJSON Msgsize() is inaccurate") + } + + vn := checksumInfoJSON{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodechecksumInfoJSON(b *testing.B) { + v := checksumInfoJSON{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodechecksumInfoJSON(b *testing.B) { + v := checksumInfoJSON{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalxlMetaV1Object(t *testing.T) { + v := xlMetaV1Object{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgxlMetaV1Object(b *testing.B) { + v := xlMetaV1Object{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgxlMetaV1Object(b *testing.B) { + v := xlMetaV1Object{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalxlMetaV1Object(b *testing.B) { + v := xlMetaV1Object{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodexlMetaV1Object(t *testing.T) { + v := xlMetaV1Object{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodexlMetaV1Object Msgsize() is inaccurate") + } + + vn := xlMetaV1Object{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodexlMetaV1Object(b *testing.B) { + v := xlMetaV1Object{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodexlMetaV1Object(b *testing.B) { + v := xlMetaV1Object{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/xl-storage-format-v2-legacy.go b/cmd/xl-storage-format-v2-legacy.go new file mode 100644 index 00000000..ec213227 --- /dev/null +++ b/cmd/xl-storage-format-v2-legacy.go @@ -0,0 +1,232 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "fmt" + "time" + + "github.com/tinylib/msgp/msgp" +) + +// unmarshalV unmarshals with a specific header version. +func (x *xlMetaV2VersionHeader) unmarshalV(v uint8, bts []byte) (o []byte, err error) { + switch v { + case 1: + return x.unmarshalV1(bts) + case 2: + x2 := xlMetaV2VersionHeaderV2{xlMetaV2VersionHeader: x} + return x2.UnmarshalMsg(bts) + case xlHeaderVersion: + return x.UnmarshalMsg(bts) + } + return bts, fmt.Errorf("unknown xlHeaderVersion: %d", v) +} + +// unmarshalV1 decodes version 1, never released. +func (x *xlMetaV2VersionHeader) unmarshalV1(bts []byte) (o []byte, err error) { + var zb0001 uint32 + zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return o, err + } + if zb0001 != 4 { + err = msgp.ArrayError{Wanted: 4, Got: zb0001} + return o, err + } + bts, err = msgp.ReadExactBytes(bts, (x.VersionID)[:]) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return o, err + } + x.ModTime, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ModTime") + return o, err + } + { + var zb0002 uint8 + zb0002, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Type") + return o, err + } + x.Type = VersionType(zb0002) + } + { + var zb0003 uint8 + zb0003, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Flags") + return o, err + } + x.Flags = xlFlags(zb0003) + } + o = bts + return o, err +} + +// unmarshalV unmarshals with a specific metadata version. +func (j *xlMetaV2Version) unmarshalV(v uint8, bts []byte) (o []byte, err error) { + if v > xlMetaVersion { + return bts, fmt.Errorf("unknown xlMetaVersion: %d", v) + } + + // Clear omitempty fields: + if j.ObjectV2 != nil && len(j.ObjectV2.PartIndices) > 0 { + j.ObjectV2.PartIndices = j.ObjectV2.PartIndices[:0] + } + o, err = j.UnmarshalMsg(bts) + + // Fix inconsistent x-minio-internal-replication-timestamp by converting to UTC. + // Fixed in version 2 or later + if err == nil && j.Type == DeleteType && v < 2 { + if val, ok := j.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicationTimestamp]; ok { + tm, err := time.Parse(time.RFC3339Nano, string(val)) + if err == nil { + j.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicationTimestamp] = []byte(tm.UTC().Format(time.RFC3339Nano)) + } + } + if val, ok := j.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicaTimestamp]; ok { + tm, err := time.Parse(time.RFC3339Nano, string(val)) + if err == nil { + j.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicaTimestamp] = []byte(tm.UTC().Format(time.RFC3339Nano)) + } + } + } + + // Clean up PartEtags on v1 + if j.ObjectV2 != nil { + allEmpty := true + for _, tag := range j.ObjectV2.PartETags { + if len(tag) != 0 { + allEmpty = false + break + } + } + if allEmpty { + j.ObjectV2.PartETags = nil + } + } + return o, err +} + +// xlMetaV2VersionHeaderV2 is a version 2 of xlMetaV2VersionHeader before EcN and EcM were added. +type xlMetaV2VersionHeaderV2 struct { + *xlMetaV2VersionHeader +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *xlMetaV2VersionHeaderV2) UnmarshalMsg(bts []byte) (o []byte, err error) { + z.EcN, z.EcN = 0, 0 + var zb0001 uint32 + zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return o, err + } + if zb0001 != 5 { + err = msgp.ArrayError{Wanted: 5, Got: zb0001} + return o, err + } + bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:]) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return o, err + } + z.ModTime, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ModTime") + return o, err + } + bts, err = msgp.ReadExactBytes(bts, (z.Signature)[:]) + if err != nil { + err = msgp.WrapError(err, "Signature") + return o, err + } + { + var zb0002 uint8 + zb0002, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Type") + return o, err + } + z.Type = VersionType(zb0002) + } + { + var zb0003 uint8 + zb0003, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Flags") + return o, err + } + z.Flags = xlFlags(zb0003) + } + o = bts + return o, err +} + +// DecodeMsg implements msgp.Decodable +func (z *xlMetaV2VersionHeaderV2) DecodeMsg(dc *msgp.Reader) (err error) { + z.EcN, z.EcN = 0, 0 + var zb0001 uint32 + zb0001, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return err + } + if zb0001 != 5 { + err = msgp.ArrayError{Wanted: 5, Got: zb0001} + return err + } + err = dc.ReadExactBytes((z.VersionID)[:]) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return err + } + z.ModTime, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ModTime") + return err + } + err = dc.ReadExactBytes((z.Signature)[:]) + if err != nil { + err = msgp.WrapError(err, "Signature") + return err + } + { + var zb0002 uint8 + zb0002, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err, "Type") + return err + } + z.Type = VersionType(zb0002) + } + { + var zb0003 uint8 + zb0003, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err, "Flags") + return err + } + z.Flags = xlFlags(zb0003) + } + return err +} diff --git a/cmd/xl-storage-format-v2.go b/cmd/xl-storage-format-v2.go new file mode 100644 index 00000000..7d26b13e --- /dev/null +++ b/cmd/xl-storage-format-v2.go @@ -0,0 +1,2268 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "encoding/binary" + "encoding/hex" + "errors" + "fmt" + "io" + "sort" + "strings" + "time" + + "github.com/cespare/xxhash/v2" + "github.com/google/uuid" + jsoniter "github.com/json-iterator/go" + "github.com/minio/minio/internal/bpool" + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/config/storageclass" + xhttp "github.com/minio/minio/internal/http" + "github.com/tinylib/msgp/msgp" +) + +var ( + // XL header specifies the format + xlHeader = [4]byte{'X', 'L', '2', ' '} + + // Current version being written. + xlVersionCurrent [4]byte +) + +//msgp:clearomitted + +//go:generate msgp -file=$GOFILE -unexported +//go:generate stringer -type VersionType,ErasureAlgo -output=xl-storage-format-v2_string.go $GOFILE + +const ( + // Breaking changes. + // Newer versions cannot be read by older software. + // This will prevent downgrades to incompatible versions. + xlVersionMajor = 1 + + // Non breaking changes. + // Bumping this is informational, but should be done + // if any change is made to the data stored, bumping this + // will allow to detect the exact version later. + xlVersionMinor = 3 +) + +func init() { + binary.LittleEndian.PutUint16(xlVersionCurrent[0:2], xlVersionMajor) + binary.LittleEndian.PutUint16(xlVersionCurrent[2:4], xlVersionMinor) +} + +// The []journal contains all the different versions of the object. +// +// This array can have 3 kinds of objects: +// +// ``object``: If the object is uploaded the usual way: putobject, multipart-put, copyobject +// +// ``delete``: This is the delete-marker +// +// ``legacyObject``: This is the legacy object in xlV1 format, preserved until its overwritten +// +// The most recently updated element in the array is considered the latest version. + +// In addition to these we have a special kind called free-version. This is represented +// using a delete-marker and MetaSys entries. It's used to track tiered content of a +// deleted/overwritten version. This version is visible _only_to the scanner routine, for subsequent deletion. +// This kind of tracking is necessary since a version's tiered content is deleted asynchronously. + +// Backend directory tree structure: +// disk1/ +// └── bucket +// └── object +// ├── a192c1d5-9bd5-41fd-9a90-ab10e165398d +// │ └── part.1 +// ├── c06e0436-f813-447e-ae5e-f2564df9dfd4 +// │ └── part.1 +// ├── df433928-2dcf-47b1-a786-43efa0f6b424 +// │ └── part.1 +// ├── legacy +// │ └── part.1 +// └── xl.meta + +// VersionType defines the type of journal type of the current entry. +type VersionType uint8 + +// List of different types of journal type +const ( + invalidVersionType VersionType = 0 + ObjectType VersionType = 1 + DeleteType VersionType = 2 + LegacyType VersionType = 3 + lastVersionType VersionType = 4 +) + +func (e VersionType) valid() bool { + return e > invalidVersionType && e < lastVersionType +} + +// ErasureAlgo defines common type of different erasure algorithms +type ErasureAlgo uint8 + +// List of currently supported erasure coding algorithms +const ( + invalidErasureAlgo ErasureAlgo = 0 + ReedSolomon ErasureAlgo = 1 + lastErasureAlgo ErasureAlgo = 2 +) + +func (e ErasureAlgo) valid() bool { + return e > invalidErasureAlgo && e < lastErasureAlgo +} + +// ChecksumAlgo defines common type of different checksum algorithms +type ChecksumAlgo uint8 + +// List of currently supported checksum algorithms +const ( + invalidChecksumAlgo ChecksumAlgo = 0 + HighwayHash ChecksumAlgo = 1 + lastChecksumAlgo ChecksumAlgo = 2 +) + +func (e ChecksumAlgo) valid() bool { + return e > invalidChecksumAlgo && e < lastChecksumAlgo +} + +// xlMetaV2DeleteMarker defines the data struct for the delete marker journal type +type xlMetaV2DeleteMarker struct { + VersionID [16]byte `json:"ID" msg:"ID"` // Version ID for delete marker + ModTime int64 `json:"MTime" msg:"MTime"` // Object delete marker modified time + MetaSys map[string][]byte `json:"MetaSys,omitempty" msg:"MetaSys,omitempty"` // Delete marker internal metadata +} + +// xlMetaV2Object defines the data struct for object journal type +type xlMetaV2Object struct { + VersionID [16]byte `json:"ID" msg:"ID"` // Version ID + DataDir [16]byte `json:"DDir" msg:"DDir"` // Data dir ID + ErasureAlgorithm ErasureAlgo `json:"EcAlgo" msg:"EcAlgo"` // Erasure coding algorithm + ErasureM int `json:"EcM" msg:"EcM"` // Erasure data blocks + ErasureN int `json:"EcN" msg:"EcN"` // Erasure parity blocks + ErasureBlockSize int64 `json:"EcBSize" msg:"EcBSize"` // Erasure block size + ErasureIndex int `json:"EcIndex" msg:"EcIndex"` // Erasure disk index + ErasureDist []uint8 `json:"EcDist" msg:"EcDist"` // Erasure distribution + BitrotChecksumAlgo ChecksumAlgo `json:"CSumAlgo" msg:"CSumAlgo"` // Bitrot checksum algo + PartNumbers []int `json:"PartNums" msg:"PartNums"` // Part Numbers + PartETags []string `json:"PartETags" msg:"PartETags,allownil"` // Part ETags + PartSizes []int64 `json:"PartSizes" msg:"PartSizes"` // Part Sizes + PartActualSizes []int64 `json:"PartASizes,omitempty" msg:"PartASizes,allownil"` // Part ActualSizes (compression) + PartIndices [][]byte `json:"PartIndices,omitempty" msg:"PartIdx,omitempty"` // Part Indexes (compression) + Size int64 `json:"Size" msg:"Size"` // Object version size + ModTime int64 `json:"MTime" msg:"MTime"` // Object version modified time + MetaSys map[string][]byte `json:"MetaSys,omitempty" msg:"MetaSys,allownil"` // Object version internal metadata + MetaUser map[string]string `json:"MetaUsr,omitempty" msg:"MetaUsr,allownil"` // Object version metadata set by user +} + +// xlMetaV2Version describes the journal entry, Type defines +// the current journal entry type other types might be nil based +// on what Type field carries, it is imperative for the caller +// to verify which journal type first before accessing rest of the fields. +type xlMetaV2Version struct { + Type VersionType `json:"Type" msg:"Type"` + ObjectV1 *xlMetaV1Object `json:"V1Obj,omitempty" msg:"V1Obj,omitempty"` + ObjectV2 *xlMetaV2Object `json:"V2Obj,omitempty" msg:"V2Obj,omitempty"` + DeleteMarker *xlMetaV2DeleteMarker `json:"DelObj,omitempty" msg:"DelObj,omitempty"` + WrittenByVersion uint64 `msg:"v"` // Tracks written by MinIO version +} + +// xlFlags contains flags on the object. +// This can be extended up to 64 bits without breaking compatibility. +type xlFlags uint8 + +const ( + xlFlagFreeVersion xlFlags = 1 << iota + xlFlagUsesDataDir + xlFlagInlineData +) + +func (x xlFlags) String() string { + var s strings.Builder + if x&xlFlagFreeVersion != 0 { + s.WriteString("FreeVersion") + } + if x&xlFlagUsesDataDir != 0 { + if s.Len() > 0 { + s.WriteByte(',') + } + s.WriteString("UsesDD") + } + if x&xlFlagInlineData != 0 { + if s.Len() > 0 { + s.WriteByte(',') + } + s.WriteString("Inline") + } + return s.String() +} + +// checkXL2V1 will check if the metadata has correct header and is a known major version. +// The remaining payload and versions are returned. +func checkXL2V1(buf []byte) (payload []byte, major, minor uint16, err error) { + if len(buf) <= 8 { + return payload, 0, 0, fmt.Errorf("xlMeta: no data") + } + + if !bytes.Equal(buf[:4], xlHeader[:]) { + return payload, 0, 0, fmt.Errorf("xlMeta: unknown XLv2 header, expected %v, got %v", xlHeader[:4], buf[:4]) + } + + if bytes.Equal(buf[4:8], []byte("1 ")) { + // Set as 1,0. + major, minor = 1, 0 + } else { + major, minor = binary.LittleEndian.Uint16(buf[4:6]), binary.LittleEndian.Uint16(buf[6:8]) + } + if major > xlVersionMajor { + return buf[8:], major, minor, fmt.Errorf("xlMeta: unknown major version %d found", major) + } + + return buf[8:], major, minor, nil +} + +func isXL2V1Format(buf []byte) bool { + _, _, _, err := checkXL2V1(buf) + return err == nil +} + +//msgp:tuple xlMetaV2VersionHeader +type xlMetaV2VersionHeader struct { + VersionID [16]byte + ModTime int64 + Signature [4]byte + Type VersionType + Flags xlFlags + EcN, EcM uint8 // Note that these will be 0/0 for non-v2 objects and older xl.meta +} + +func (x xlMetaV2VersionHeader) String() string { + return fmt.Sprintf("Type: %s, VersionID: %s, Signature: %s, ModTime: %s, Flags: %s, N: %d, M: %d", + x.Type.String(), + hex.EncodeToString(x.VersionID[:]), + hex.EncodeToString(x.Signature[:]), + time.Unix(0, x.ModTime), + x.Flags.String(), + x.EcN, x.EcM, + ) +} + +// matchesNotStrict returns whether x and o have both have non-zero version, +// their versions match and their type match. +// If they have zero version, modtime must match. +func (x xlMetaV2VersionHeader) matchesNotStrict(o xlMetaV2VersionHeader) (ok bool) { + ok = x.VersionID == o.VersionID && x.Type == o.Type && x.matchesEC(o) + if x.VersionID == [16]byte{} { + ok = ok && o.ModTime == x.ModTime + } + return ok +} + +func (x xlMetaV2VersionHeader) matchesEC(o xlMetaV2VersionHeader) bool { + if x.hasEC() && o.hasEC() { + return x.EcN == o.EcN && x.EcM == o.EcM + } // if no EC header this is an older object + return true +} + +// hasEC will return true if the version has erasure coding information. +func (x xlMetaV2VersionHeader) hasEC() bool { + return x.EcM > 0 && x.EcN > 0 +} + +// sortsBefore can be used as a tiebreaker for stable sorting/selecting. +// Returns false on ties. +func (x xlMetaV2VersionHeader) sortsBefore(o xlMetaV2VersionHeader) bool { + if x == o { + return false + } + // Prefer newest modtime. + if x.ModTime != o.ModTime { + return x.ModTime > o.ModTime + } + + // The following doesn't make too much sense, but we want sort to be consistent nonetheless. + // Prefer lower types + if x.Type != o.Type { + return x.Type < o.Type + } + // Consistent sort on signature + if v := bytes.Compare(x.Signature[:], o.Signature[:]); v != 0 { + return v > 0 + } + // On ID mismatch + if v := bytes.Compare(x.VersionID[:], o.VersionID[:]); v != 0 { + return v > 0 + } + // Flags + if x.Flags != o.Flags { + return x.Flags > o.Flags + } + return false +} + +func (j xlMetaV2Version) getDataDir() string { + if j.Valid() { + switch j.Type { + case LegacyType: + return j.ObjectV1.DataDir + case ObjectType: + return uuid.UUID(j.ObjectV2.DataDir).String() + } + } + return "" +} + +// Valid xl meta xlMetaV2Version is valid +func (j xlMetaV2Version) Valid() bool { + if !j.Type.valid() { + return false + } + switch j.Type { + case LegacyType: + return j.ObjectV1 != nil && + j.ObjectV1.valid() + case ObjectType: + return j.ObjectV2 != nil && + j.ObjectV2.ErasureAlgorithm.valid() && + j.ObjectV2.BitrotChecksumAlgo.valid() && + isXLMetaErasureInfoValid(j.ObjectV2.ErasureM, j.ObjectV2.ErasureN) && + j.ObjectV2.ModTime > 0 + case DeleteType: + return j.DeleteMarker != nil && + j.DeleteMarker.ModTime > 0 + } + return false +} + +// header will return a shallow header of the version. +func (j *xlMetaV2Version) header() xlMetaV2VersionHeader { + var flags xlFlags + if j.FreeVersion() { + flags |= xlFlagFreeVersion + } + if j.Type == ObjectType && j.ObjectV2.UsesDataDir() { + flags |= xlFlagUsesDataDir + } + if j.Type == ObjectType && j.ObjectV2.InlineData() { + flags |= xlFlagInlineData + } + var ecM, ecN uint8 + if j.Type == ObjectType && j.ObjectV2 != nil { + ecM, ecN = uint8(j.ObjectV2.ErasureM), uint8(j.ObjectV2.ErasureN) + } + return xlMetaV2VersionHeader{ + VersionID: j.getVersionID(), + ModTime: j.getModTime().UnixNano(), + Signature: j.getSignature(), + Type: j.Type, + Flags: flags, + EcN: ecN, + EcM: ecM, + } +} + +// FreeVersion returns true if x represents a free-version, false otherwise. +func (x xlMetaV2VersionHeader) FreeVersion() bool { + return x.Flags&xlFlagFreeVersion != 0 +} + +// UsesDataDir returns true if this object version uses its data directory for +// its contents and false otherwise. +func (x xlMetaV2VersionHeader) UsesDataDir() bool { + return x.Flags&xlFlagUsesDataDir != 0 +} + +// InlineData returns whether inline data has been set. +// Note that false does not mean there is no inline data, +// only that it is unlikely. +func (x xlMetaV2VersionHeader) InlineData() bool { + return x.Flags&xlFlagInlineData != 0 +} + +// signatureErr is a signature returned when an error occurs. +var signatureErr = [4]byte{'e', 'r', 'r', 0} + +// getSignature will return a signature that is expected to be the same across all disks. +func (j xlMetaV2Version) getSignature() [4]byte { + switch j.Type { + case ObjectType: + return j.ObjectV2.Signature() + case DeleteType: + return j.DeleteMarker.Signature() + case LegacyType: + return j.ObjectV1.Signature() + } + return signatureErr +} + +// getModTime will return the ModTime of the underlying version. +func (j xlMetaV2Version) getModTime() time.Time { + switch j.Type { + case ObjectType: + return time.Unix(0, j.ObjectV2.ModTime) + case DeleteType: + return time.Unix(0, j.DeleteMarker.ModTime) + case LegacyType: + return j.ObjectV1.Stat.ModTime + } + return time.Time{} +} + +// getVersionID will return the versionID of the underlying version. +func (j xlMetaV2Version) getVersionID() [16]byte { + switch j.Type { + case ObjectType: + return j.ObjectV2.VersionID + case DeleteType: + return j.DeleteMarker.VersionID + case LegacyType: + return [16]byte{} + } + return [16]byte{} +} + +// ToFileInfo returns FileInfo of the underlying type. +func (j *xlMetaV2Version) ToFileInfo(volume, path string, allParts bool) (fi FileInfo, err error) { + if j == nil { + return fi, errFileNotFound + } + switch j.Type { + case ObjectType: + fi, err = j.ObjectV2.ToFileInfo(volume, path, allParts) + case DeleteType: + fi, err = j.DeleteMarker.ToFileInfo(volume, path) + case LegacyType: + fi, err = j.ObjectV1.ToFileInfo(volume, path) + default: + return fi, errFileNotFound + } + fi.WrittenByVersion = j.WrittenByVersion + return fi, err +} + +const ( + xlHeaderVersion = 3 + xlMetaVersion = 3 +) + +func (j xlMetaV2DeleteMarker) ToFileInfo(volume, path string) (FileInfo, error) { + versionID := "" + var uv uuid.UUID + // check if the version is not "null" + if j.VersionID != uv { + versionID = uuid.UUID(j.VersionID).String() + } + fi := FileInfo{ + Volume: volume, + Name: path, + ModTime: time.Unix(0, j.ModTime).UTC(), + VersionID: versionID, + Deleted: true, + } + fi.Metadata = make(map[string]string, len(j.MetaSys)) + for k, v := range j.MetaSys { + fi.Metadata[k] = string(v) + } + + fi.ReplicationState = GetInternalReplicationState(j.MetaSys) + if j.FreeVersion() { + fi.SetTierFreeVersion() + fi.TransitionTier = string(j.MetaSys[metaTierName]) + fi.TransitionedObjName = string(j.MetaSys[metaTierObjName]) + fi.TransitionVersionID = string(j.MetaSys[metaTierVersionID]) + } + + return fi, nil +} + +// Signature will return a signature that is expected to be the same across all disks. +func (j *xlMetaV2DeleteMarker) Signature() [4]byte { + // Shallow copy + c := *j + + // Marshal metadata + crc := hashDeterministicBytes(c.MetaSys) + c.MetaSys = nil + if bts, err := c.MarshalMsg(metaDataPoolGet()); err == nil { + crc ^= xxhash.Sum64(bts) + metaDataPoolPut(bts) + } + + // Combine upper and lower part + var tmp [4]byte + binary.LittleEndian.PutUint32(tmp[:], uint32(crc^(crc>>32))) + return tmp +} + +// UsesDataDir returns true if this object version uses its data directory for +// its contents and false otherwise. +func (j xlMetaV2Object) UsesDataDir() bool { + // Skip if this version is not transitioned, i.e it uses its data directory. + if !bytes.Equal(j.MetaSys[metaTierStatus], []byte(lifecycle.TransitionComplete)) { + return true + } + + // Check if this transitioned object has been restored on disk. + return isRestoredObjectOnDisk(j.MetaUser) +} + +// InlineData returns whether inline data has been set. +// Note that false does not mean there is no inline data, +// only that it is unlikely. +func (j xlMetaV2Object) InlineData() bool { + _, ok := j.MetaSys[ReservedMetadataPrefixLower+"inline-data"] + return ok +} + +func (j *xlMetaV2Object) ResetInlineData() { + delete(j.MetaSys, ReservedMetadataPrefixLower+"inline-data") +} + +const ( + metaTierStatus = ReservedMetadataPrefixLower + TransitionStatus + metaTierObjName = ReservedMetadataPrefixLower + TransitionedObjectName + metaTierVersionID = ReservedMetadataPrefixLower + TransitionedVersionID + metaTierName = ReservedMetadataPrefixLower + TransitionTier +) + +func (j *xlMetaV2Object) SetTransition(fi FileInfo) { + j.MetaSys[metaTierStatus] = []byte(fi.TransitionStatus) + j.MetaSys[metaTierObjName] = []byte(fi.TransitionedObjName) + j.MetaSys[metaTierVersionID] = []byte(fi.TransitionVersionID) + j.MetaSys[metaTierName] = []byte(fi.TransitionTier) +} + +func (j *xlMetaV2Object) RemoveRestoreHdrs() { + delete(j.MetaUser, xhttp.AmzRestore) + delete(j.MetaUser, xhttp.AmzRestoreExpiryDays) + delete(j.MetaUser, xhttp.AmzRestoreRequestDate) +} + +// Signature will return a signature that is expected to be the same across all disks. +func (j *xlMetaV2Object) Signature() [4]byte { + // Shallow copy + c := *j + // Zero fields that will vary across disks + c.ErasureIndex = 0 + + // Nil 0 size allownil, so we don't differentiate between nil and 0 len. + allEmpty := true + for _, tag := range c.PartETags { + if len(tag) != 0 { + allEmpty = false + break + } + } + if allEmpty { + c.PartETags = nil + } + if len(c.PartActualSizes) == 0 { + c.PartActualSizes = nil + } + + // Get a 64 bit CRC + crc := hashDeterministicString(c.MetaUser) + crc ^= hashDeterministicBytes(c.MetaSys) + + // Nil fields. + c.MetaSys = nil + c.MetaUser = nil + + if bts, err := c.MarshalMsg(metaDataPoolGet()); err == nil { + crc ^= xxhash.Sum64(bts) + metaDataPoolPut(bts) + } + + // Combine upper and lower part + var tmp [4]byte + binary.LittleEndian.PutUint32(tmp[:], uint32(crc^(crc>>32))) + return tmp +} + +func (j xlMetaV2Object) ToFileInfo(volume, path string, allParts bool) (FileInfo, error) { + versionID := "" + var uv uuid.UUID + // check if the version is not "null" + if j.VersionID != uv { + versionID = uuid.UUID(j.VersionID).String() + } + fi := FileInfo{ + Volume: volume, + Name: path, + Size: j.Size, + ModTime: time.Unix(0, j.ModTime).UTC(), + VersionID: versionID, + } + if allParts { + fi.Parts = make([]ObjectPartInfo, len(j.PartNumbers)) + for i := range fi.Parts { + fi.Parts[i].Number = j.PartNumbers[i] + fi.Parts[i].Size = j.PartSizes[i] + if len(j.PartETags) == len(fi.Parts) { + fi.Parts[i].ETag = j.PartETags[i] + } + fi.Parts[i].ActualSize = j.PartActualSizes[i] + if len(j.PartIndices) == len(fi.Parts) { + fi.Parts[i].Index = j.PartIndices[i] + } + } + } + + // fi.Erasure.Checksums - is left empty since we do not have any + // whole checksums for many years now, no need to allocate. + + fi.Metadata = make(map[string]string, len(j.MetaUser)+len(j.MetaSys)) + for k, v := range j.MetaUser { + // https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w + if equals(k, xhttp.AmzMetaUnencryptedContentLength, xhttp.AmzMetaUnencryptedContentMD5) { + continue + } + if equals(k, "x-amz-storage-class") && v == storageclass.STANDARD { + continue + } + + fi.Metadata[k] = v + } + + tierFVIDKey := ReservedMetadataPrefixLower + tierFVID + tierFVMarkerKey := ReservedMetadataPrefixLower + tierFVMarker + for k, v := range j.MetaSys { + // Make sure we skip free-version-id, similar to AddVersion() + if len(k) > len(ReservedMetadataPrefixLower) && strings.EqualFold(k[:len(ReservedMetadataPrefixLower)], ReservedMetadataPrefixLower) { + // Skip tierFVID, tierFVMarker keys; it's used + // only for creating free-version. + switch k { + case tierFVIDKey, tierFVMarkerKey: + continue + } + } + if equals(k, "x-amz-storage-class") && string(v) == storageclass.STANDARD { + continue + } + switch { + case strings.HasPrefix(strings.ToLower(k), ReservedMetadataPrefixLower), equals(k, VersionPurgeStatusKey): + fi.Metadata[k] = string(v) + } + } + fi.ReplicationState = getInternalReplicationState(fi.Metadata) + fi.Deleted = !fi.VersionPurgeStatus().Empty() + replStatus := fi.ReplicationState.CompositeReplicationStatus() + if replStatus != "" { + fi.Metadata[xhttp.AmzBucketReplicationStatus] = string(replStatus) + } + fi.Erasure.Algorithm = j.ErasureAlgorithm.String() + fi.Erasure.Index = j.ErasureIndex + fi.Erasure.BlockSize = j.ErasureBlockSize + fi.Erasure.DataBlocks = j.ErasureM + fi.Erasure.ParityBlocks = j.ErasureN + fi.Erasure.Distribution = make([]int, len(j.ErasureDist)) + for i := range j.ErasureDist { + fi.Erasure.Distribution[i] = int(j.ErasureDist[i]) + } + fi.DataDir = uuid.UUID(j.DataDir).String() + + if st, ok := j.MetaSys[metaTierStatus]; ok { + fi.TransitionStatus = string(st) + } + if o, ok := j.MetaSys[metaTierObjName]; ok { + fi.TransitionedObjName = string(o) + } + if rv, ok := j.MetaSys[metaTierVersionID]; ok { + fi.TransitionVersionID = string(rv) + } + if sc, ok := j.MetaSys[metaTierName]; ok { + fi.TransitionTier = string(sc) + } + if crcs := j.MetaSys[ReservedMetadataPrefixLower+"crc"]; len(crcs) > 0 { + fi.Checksum = crcs + } + return fi, nil +} + +// Read at most this much on initial read. +const metaDataReadDefault = 4 << 10 + +// Return used metadata byte slices here. +var metaDataPool = bpool.Pool[[]byte]{New: func() []byte { return make([]byte, 0, metaDataReadDefault) }} + +// metaDataPoolGet will return a byte slice with capacity at least metaDataReadDefault. +// It will be length 0. +func metaDataPoolGet() []byte { + return metaDataPool.Get()[:0] +} + +// metaDataPoolPut will put an unused small buffer back into the pool. +func metaDataPoolPut(buf []byte) { + if cap(buf) >= metaDataReadDefault && cap(buf) < metaDataReadDefault*4 { + metaDataPool.Put(buf) + } +} + +// readXLMetaNoData will load the metadata, but skip data segments. +// This should only be used when data is never interesting. +// If data is not xlv2, it is returned in full. +func readXLMetaNoData(r io.Reader, size int64) ([]byte, error) { + initial := size + hasFull := true + if initial > metaDataReadDefault { + initial = metaDataReadDefault + hasFull = false + } + + buf := metaDataPoolGet()[:initial] + _, err := io.ReadFull(r, buf) + if err != nil { + return nil, fmt.Errorf("readXLMetaNoData(io.ReadFull): %w", err) + } + readMore := func(n int64) error { + has := int64(len(buf)) + if has >= n { + return nil + } + if hasFull || n > size { + return io.ErrUnexpectedEOF + } + extra := n - has + if int64(cap(buf)) >= n { + // Extend since we have enough space. + buf = buf[:n] + } else { + buf = append(buf, make([]byte, extra)...) + } + _, err := io.ReadFull(r, buf[has:]) + if err != nil { + if errors.Is(err, io.EOF) { + // Returned if we read nothing. + err = io.ErrUnexpectedEOF + } + return fmt.Errorf("readXLMetaNoData(readMore): %w", err) + } + return nil + } + tmp, major, minor, err := checkXL2V1(buf) + if err != nil { + err = readMore(size) + return buf, err + } + switch major { + case 1: + switch minor { + case 0: + err = readMore(size) + return buf, err + case 1, 2, 3: + sz, tmp, err := msgp.ReadBytesHeader(tmp) + if err != nil { + return nil, fmt.Errorf("readXLMetaNoData(read_meta): unknown metadata version %w", err) + } + want := int64(sz) + int64(len(buf)-len(tmp)) + + // v1.1 does not have CRC. + if minor < 2 { + if err := readMore(want); err != nil { + return nil, err + } + return buf[:want], nil + } + + // CRC is variable length, so we need to truncate exactly that. + wantMax := min(want+msgp.Uint32Size, size) + if err := readMore(wantMax); err != nil { + return nil, err + } + + if int64(len(buf)) < want { + return nil, fmt.Errorf("buffer shorter than expected (buflen: %d, want: %d): %w", len(buf), want, errFileCorrupt) + } + + tmp = buf[want:] + _, after, err := msgp.ReadUint32Bytes(tmp) + if err != nil { + return nil, fmt.Errorf("readXLMetaNoData(read_meta): unknown metadata version %w", err) + } + want += int64(len(tmp) - len(after)) + + return buf[:want], err + + default: + return nil, errors.New("unknown minor metadata version") + } + default: + return nil, errors.New("unknown major metadata version") + } +} + +func decodeXLHeaders(buf []byte) (versions int, headerV, metaV uint8, b []byte, err error) { + hdrVer, buf, err := msgp.ReadUint8Bytes(buf) + if err != nil { + return 0, 0, 0, buf, err + } + metaVer, buf, err := msgp.ReadUint8Bytes(buf) + if err != nil { + return 0, 0, 0, buf, err + } + if hdrVer > xlHeaderVersion { + return 0, 0, 0, buf, fmt.Errorf("decodeXLHeaders: Unknown xl header version %d", hdrVer) + } + if metaVer > xlMetaVersion { + return 0, 0, 0, buf, fmt.Errorf("decodeXLHeaders: Unknown xl meta version %d", metaVer) + } + versions, buf, err = msgp.ReadIntBytes(buf) + if err != nil { + return 0, 0, 0, buf, err + } + if versions < 0 { + return 0, 0, 0, buf, fmt.Errorf("decodeXLHeaders: Negative version count %d", versions) + } + return versions, hdrVer, metaVer, buf, nil +} + +// decodeVersions will decode a number of versions from a buffer +// and perform a callback for each version in order, newest first. +// Return errDoneForNow to stop processing and return nil. +// Any non-nil error is returned. +func decodeVersions(buf []byte, versions int, fn func(idx int, hdr, meta []byte) error) (err error) { + var tHdr, tMeta []byte // Zero copy bytes + for i := range versions { + tHdr, buf, err = msgp.ReadBytesZC(buf) + if err != nil { + return err + } + tMeta, buf, err = msgp.ReadBytesZC(buf) + if err != nil { + return err + } + if err = fn(i, tHdr, tMeta); err != nil { + if err == errDoneForNow { + err = nil + } + return err + } + } + return nil +} + +// isIndexedMetaV2 returns non-nil result if metadata is indexed. +// Returns 3x nil if not XLV2 or not indexed. +// If indexed and unable to parse an error will be returned. +func isIndexedMetaV2(buf []byte) (meta xlMetaBuf, data xlMetaInlineData, err error) { + buf, major, minor, err := checkXL2V1(buf) + if err != nil || major != 1 || minor < 3 { + return nil, nil, nil + } + meta, buf, err = msgp.ReadBytesZC(buf) + if err != nil { + return nil, nil, err + } + if crc, nbuf, err := msgp.ReadUint32Bytes(buf); err == nil { + // Read metadata CRC + buf = nbuf + if got := uint32(xxhash.Sum64(meta)); got != crc { + return nil, nil, fmt.Errorf("xlMetaV2.Load version(%d), CRC mismatch, want 0x%x, got 0x%x", minor, crc, got) + } + } else { + return nil, nil, err + } + data = buf + if data.validate() != nil { + data.repair() + } + + return meta, data, nil +} + +type xlMetaV2ShallowVersion struct { + header xlMetaV2VersionHeader + meta []byte +} + +//msgp:ignore xlMetaV2 xlMetaV2ShallowVersion + +type xlMetaV2 struct { + versions []xlMetaV2ShallowVersion + + // data will contain raw data if any. + // data will be one or more versions indexed by versionID. + // To remove all data set to nil. + data xlMetaInlineData + + // metadata version. + metaV uint8 +} + +// LoadOrConvert will load the metadata in the buffer. +// If this is a legacy format, it will automatically be converted to XLV2. +func (x *xlMetaV2) LoadOrConvert(buf []byte) error { + if isXL2V1Format(buf) { + return x.Load(buf) + } + + xlMeta := &xlMetaV1Object{} + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err := json.Unmarshal(buf, xlMeta); err != nil { + return errFileCorrupt + } + if len(x.versions) > 0 { + x.versions = x.versions[:0] + } + x.data = nil + x.metaV = xlMetaVersion + return x.AddLegacy(xlMeta) +} + +// Load all versions of the stored data. +// Note that references to the incoming buffer will be kept. +func (x *xlMetaV2) Load(buf []byte) error { + if meta, data, err := isIndexedMetaV2(buf); err != nil { + return err + } else if meta != nil { + return x.loadIndexed(meta, data) + } + // Convert older format. + return x.loadLegacy(buf) +} + +func (x *xlMetaV2) loadIndexed(buf xlMetaBuf, data xlMetaInlineData) error { + versions, headerV, metaV, buf, err := decodeXLHeaders(buf) + if err != nil { + return err + } + if cap(x.versions) < versions { + x.versions = make([]xlMetaV2ShallowVersion, 0, versions+1) + } + x.versions = x.versions[:versions] + x.data = data + x.metaV = metaV + if err = x.data.validate(); err != nil { + x.data.repair() + storageLogIf(GlobalContext, fmt.Errorf("xlMetaV2.loadIndexed: data validation failed: %v. %d entries after repair", err, x.data.entries())) + } + return decodeVersions(buf, versions, func(i int, hdr, meta []byte) error { + ver := &x.versions[i] + _, err = ver.header.unmarshalV(headerV, hdr) + if err != nil { + return err + } + ver.meta = meta + + // Fix inconsistent compression index due to https://github.com/minio/minio/pull/20575 + // First search marshaled content for encoded values. + // We have bumped metaV to make this check cheaper. + if metaV < 3 && ver.header.Type == ObjectType && bytes.Contains(meta, []byte("\xa7PartIdx")) && + bytes.Contains(meta, []byte("\xbcX-Minio-Internal-compression\xc4\x15klauspost/compress/s2")) { + // Likely candidate... + version, err := x.getIdx(i) + if err == nil { + // Check write date... + // RELEASE.2023-12-02T10-51-33Z -> RELEASE.2024-10-29T16-01-48Z + const dateStart = 1701471618 + const dateEnd = 1730156418 + if version.WrittenByVersion > dateStart && version.WrittenByVersion < dateEnd && + version.ObjectV2 != nil && len(version.ObjectV2.PartIndices) > 0 { + var changed bool + clearField := true + for i, sz := range version.ObjectV2.PartActualSizes { + if len(version.ObjectV2.PartIndices) > i { + // 8<<20 is current 'compMinIndexSize', but we detach it in case it should change in the future. + if sz <= 8<<20 && len(version.ObjectV2.PartIndices[i]) > 0 { + changed = true + version.ObjectV2.PartIndices[i] = nil + } + clearField = clearField && len(version.ObjectV2.PartIndices[i]) == 0 + } + } + if changed { + // All empty, clear. + if clearField { + version.ObjectV2.PartIndices = nil + } + + // Reindex since it was changed. + meta, err := version.MarshalMsg(make([]byte, 0, len(ver.meta)+10)) + if err == nil { + // Override both if fine. + ver.header = version.header() + ver.meta = meta + } + } + } + } + } + + // Fix inconsistent x-minio-internal-replication-timestamp by loading and reindexing. + if metaV < 2 && ver.header.Type == DeleteType { + // load (and convert) version. + version, err := x.getIdx(i) + if err == nil { + // Only reindex if set. + _, ok1 := version.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicationTimestamp] + _, ok2 := version.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicaTimestamp] + if ok1 || ok2 { + meta, err := version.MarshalMsg(make([]byte, 0, len(ver.meta)+10)) + if err == nil { + // Override both if fine. + ver.header = version.header() + ver.meta = meta + } + } + } + } + return nil + }) +} + +// loadLegacy will load content prior to v1.3 +// Note that references to the incoming buffer will be kept. +func (x *xlMetaV2) loadLegacy(buf []byte) error { + buf, major, minor, err := checkXL2V1(buf) + if err != nil { + return fmt.Errorf("xlMetaV2.Load %w", err) + } + var allMeta []byte + switch major { + case 1: + switch minor { + case 0: + allMeta = buf + case 1, 2: + v, buf, err := msgp.ReadBytesZC(buf) + if err != nil { + return fmt.Errorf("xlMetaV2.Load version(%d), bufLen(%d) %w", minor, len(buf), err) + } + if minor >= 2 { + if crc, nbuf, err := msgp.ReadUint32Bytes(buf); err == nil { + // Read metadata CRC (added in v2) + buf = nbuf + if got := uint32(xxhash.Sum64(v)); got != crc { + return fmt.Errorf("xlMetaV2.Load version(%d), CRC mismatch, want 0x%x, got 0x%x", minor, crc, got) + } + } else { + return fmt.Errorf("xlMetaV2.Load version(%d), loading CRC: %w", minor, err) + } + } + + allMeta = v + // Add remaining data. + x.data = buf + if err = x.data.validate(); err != nil { + x.data.repair() + storageLogIf(GlobalContext, fmt.Errorf("xlMetaV2.Load: data validation failed: %v. %d entries after repair", err, x.data.entries())) + } + default: + return errors.New("unknown minor metadata version") + } + default: + return errors.New("unknown major metadata version") + } + if allMeta == nil { + return errFileCorrupt + } + // bts will shrink as we decode. + bts := allMeta + var field []byte + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + return msgp.WrapError(err, "loadLegacy.ReadMapHeader") + } + + var tmp xlMetaV2Version + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + return msgp.WrapError(err, "loadLegacy.ReadMapKey") + } + switch msgp.UnsafeString(field) { + case "Versions": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + return msgp.WrapError(err, "Versions") + } + if cap(x.versions) >= int(zb0002) { + x.versions = (x.versions)[:zb0002] + } else { + x.versions = make([]xlMetaV2ShallowVersion, zb0002, zb0002+1) + } + for za0001 := range x.versions { + start := len(allMeta) - len(bts) + bts, err = tmp.unmarshalV(1, bts) + if err != nil { + return msgp.WrapError(err, "Versions", za0001) + } + end := len(allMeta) - len(bts) + // We reference the marshaled data, so we don't have to re-marshal. + x.versions[za0001] = xlMetaV2ShallowVersion{ + header: tmp.header(), + meta: allMeta[start:end], + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + return msgp.WrapError(err, "loadLegacy.Skip") + } + } + } + x.metaV = 1 // Fixed for legacy conversions. + x.sortByModTime() + return nil +} + +// latestModtime returns the modtime of the latest version. +func (x *xlMetaV2) latestModtime() time.Time { + if x == nil || len(x.versions) == 0 { + return time.Time{} + } + return time.Unix(0, x.versions[0].header.ModTime) +} + +func (x *xlMetaV2) addVersion(ver xlMetaV2Version) error { + modTime := ver.getModTime().UnixNano() + if !ver.Valid() { + return errors.New("attempted to add invalid version") + } + encoded, err := ver.MarshalMsg(nil) + if err != nil { + return err + } + + // returns error if we have exceeded configured object max versions + if int64(len(x.versions)+1) > globalAPIConfig.getObjectMaxVersions() { + return errMaxVersionsExceeded + } + + // Add space at the end. + // Will have -1 modtime, so it will be inserted there. + x.versions = append(x.versions, xlMetaV2ShallowVersion{header: xlMetaV2VersionHeader{ModTime: -1}}) + + // Linear search, we likely have to insert at front. + for i, existing := range x.versions { + if existing.header.ModTime <= modTime { + // Insert at current idx. First move current back. + copy(x.versions[i+1:], x.versions[i:]) + x.versions[i] = xlMetaV2ShallowVersion{ + header: ver.header(), + meta: encoded, + } + return nil + } + } + return fmt.Errorf("addVersion: Internal error, unable to add version") +} + +// AppendTo will marshal the data in z and append it to the provided slice. +func (x *xlMetaV2) AppendTo(dst []byte) ([]byte, error) { + // Header... + sz := len(xlHeader) + len(xlVersionCurrent) + msgp.ArrayHeaderSize + len(dst) + 3*msgp.Uint32Size + // Existing + Inline data + sz += len(dst) + len(x.data) + // Versions... + for _, ver := range x.versions { + sz += 32 + len(ver.meta) + } + if cap(dst) < sz { + buf := make([]byte, len(dst), sz) + copy(buf, dst) + dst = buf + } + if err := x.data.validate(); err != nil { + return nil, err + } + + dst = append(dst, xlHeader[:]...) + dst = append(dst, xlVersionCurrent[:]...) + // Add "bin 32" type header to always have enough space. + // We will fill out the correct size when we know it. + dst = append(dst, 0xc6, 0, 0, 0, 0) + dataOffset := len(dst) + + dst = msgp.AppendUint(dst, xlHeaderVersion) + dst = msgp.AppendUint(dst, xlMetaVersion) + dst = msgp.AppendInt(dst, len(x.versions)) + + tmp := metaDataPoolGet() + defer metaDataPoolPut(tmp) + for _, ver := range x.versions { + var err error + + // Add header + tmp, err = ver.header.MarshalMsg(tmp[:0]) + if err != nil { + return nil, err + } + dst = msgp.AppendBytes(dst, tmp) + + // Add full meta + dst = msgp.AppendBytes(dst, ver.meta) + } + + // Update size... + binary.BigEndian.PutUint32(dst[dataOffset-4:dataOffset], uint32(len(dst)-dataOffset)) + + // Add CRC of metadata as fixed size (5 bytes) + // Prior to v1.3 this was variable sized. + tmp = tmp[:5] + tmp[0] = 0xce // muint32 + binary.BigEndian.PutUint32(tmp[1:], uint32(xxhash.Sum64(dst[dataOffset:]))) + dst = append(dst, tmp[:5]...) + return append(dst, x.data...), nil +} + +const emptyUUID = "00000000-0000-0000-0000-000000000000" + +func (x *xlMetaV2) findVersionStr(key string) (idx int, ver *xlMetaV2Version, err error) { + if key == nullVersionID { + key = "" + } + var u uuid.UUID + if key != "" { + u, err = uuid.Parse(key) + if err != nil { + return -1, nil, errFileVersionNotFound + } + } + return x.findVersion(u) +} + +func (x *xlMetaV2) findVersion(key [16]byte) (idx int, ver *xlMetaV2Version, err error) { + for i, ver := range x.versions { + if key == ver.header.VersionID { + obj, err := x.getIdx(i) + return i, obj, err + } + } + return -1, nil, errFileVersionNotFound +} + +func (x *xlMetaV2) getIdx(idx int) (ver *xlMetaV2Version, err error) { + if idx < 0 || idx >= len(x.versions) { + return nil, errFileNotFound + } + var dst xlMetaV2Version + _, err = dst.unmarshalV(x.metaV, x.versions[idx].meta) + if false { + if err == nil && x.versions[idx].header.VersionID != dst.getVersionID() { + panic(fmt.Sprintf("header: %x != object id: %x", x.versions[idx].header.VersionID, dst.getVersionID())) + } + } + return &dst, err +} + +// setIdx will replace a version at a given index. +// Note that versions may become re-sorted if modtime changes. +func (x *xlMetaV2) setIdx(idx int, ver xlMetaV2Version) (err error) { + if idx < 0 || idx >= len(x.versions) { + return errFileNotFound + } + update := &x.versions[idx] + prevMod := update.header.ModTime + update.meta, err = ver.MarshalMsg(update.meta[:0:len(update.meta)]) + if err != nil { + update.meta = nil + return err + } + update.header = ver.header() + if prevMod != update.header.ModTime { + x.sortByModTime() + } + return nil +} + +// getDataDirs will return all data directories in the metadata +// as well as all version ids used for inline data. +func (x *xlMetaV2) getDataDirs() ([]string, error) { + dds := make([]string, 0, len(x.versions)*2) + for i, ver := range x.versions { + if ver.header.Type == DeleteType { + continue + } + + obj, err := x.getIdx(i) + if err != nil { + return nil, err + } + switch ver.header.Type { + case ObjectType: + if obj.ObjectV2 == nil { + return nil, errors.New("obj.ObjectV2 unexpectedly nil") + } + dds = append(dds, uuid.UUID(obj.ObjectV2.DataDir).String()) + if obj.ObjectV2.VersionID == [16]byte{} { + dds = append(dds, nullVersionID) + } else { + dds = append(dds, uuid.UUID(obj.ObjectV2.VersionID).String()) + } + case LegacyType: + if obj.ObjectV1 == nil { + return nil, errors.New("obj.ObjectV1 unexpectedly nil") + } + dds = append(dds, obj.ObjectV1.DataDir) + } + } + return dds, nil +} + +// sortByModTime will sort versions by modtime in descending order, +// meaning index 0 will be latest version. +func (x *xlMetaV2) sortByModTime() { + // Quick check + if len(x.versions) <= 1 || sort.SliceIsSorted(x.versions, func(i, j int) bool { + return x.versions[i].header.sortsBefore(x.versions[j].header) + }) { + return + } + + // We should sort. + sort.Slice(x.versions, func(i, j int) bool { + return x.versions[i].header.sortsBefore(x.versions[j].header) + }) +} + +// DeleteVersion deletes the version specified by version id. +// returns to the caller which dataDir to delete, also +// indicates if this is the last version. +func (x *xlMetaV2) DeleteVersion(fi FileInfo) (string, error) { + // This is a situation where versionId is explicitly + // specified as "null", as we do not save "null" + // string it is considered empty. But empty also + // means the version which matches will be purged. + if fi.VersionID == nullVersionID { + fi.VersionID = "" + } + + var uv uuid.UUID + var err error + if fi.VersionID != "" { + uv, err = uuid.Parse(fi.VersionID) + if err != nil { + return "", errFileVersionNotFound + } + } + + var ventry xlMetaV2Version + if fi.Deleted { + ventry = xlMetaV2Version{ + Type: DeleteType, + DeleteMarker: &xlMetaV2DeleteMarker{ + VersionID: uv, + ModTime: fi.ModTime.UnixNano(), + MetaSys: make(map[string][]byte), + }, + WrittenByVersion: globalVersionUnix, + } + if !ventry.Valid() { + return "", errors.New("internal error: invalid version entry generated") + } + } + updateVersion := false + if fi.VersionPurgeStatus().Empty() && (fi.DeleteMarkerReplicationStatus() == "REPLICA" || fi.DeleteMarkerReplicationStatus().Empty()) { + updateVersion = fi.MarkDeleted + } else { + // for replication scenario + if fi.Deleted && fi.VersionPurgeStatus() != replication.VersionPurgeComplete { + if !fi.VersionPurgeStatus().Empty() || fi.DeleteMarkerReplicationStatus().Empty() { + updateVersion = true + } + } + // object or delete-marker versioned delete is not complete + if !fi.VersionPurgeStatus().Empty() && fi.VersionPurgeStatus() != replication.VersionPurgeComplete { + updateVersion = true + } + } + + if fi.Deleted { + if !fi.DeleteMarkerReplicationStatus().Empty() { + switch fi.DeleteMarkerReplicationStatus() { + case replication.Replica: + ventry.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicaStatus] = []byte(fi.ReplicationState.ReplicaStatus) + ventry.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicaTimestamp] = []byte(fi.ReplicationState.ReplicaTimeStamp.UTC().Format(time.RFC3339Nano)) + default: + ventry.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicationStatus] = []byte(fi.ReplicationState.ReplicationStatusInternal) + ventry.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicationTimestamp] = []byte(fi.ReplicationState.ReplicationTimeStamp.UTC().Format(time.RFC3339Nano)) + } + } + if !fi.VersionPurgeStatus().Empty() { + ventry.DeleteMarker.MetaSys[VersionPurgeStatusKey] = []byte(fi.ReplicationState.VersionPurgeStatusInternal) + } + for k, v := range fi.ReplicationState.ResetStatusesMap { + ventry.DeleteMarker.MetaSys[k] = []byte(v) + } + } + + for i, ver := range x.versions { + if ver.header.VersionID != uv { + continue + } + switch ver.header.Type { + case LegacyType: + ver, err := x.getIdx(i) + if err != nil { + return "", err + } + x.versions = append(x.versions[:i], x.versions[i+1:]...) + if fi.Deleted { + err = x.addVersion(ventry) + } + return ver.ObjectV1.DataDir, err + case DeleteType: + if updateVersion { + ver, err := x.getIdx(i) + if err != nil { + return "", err + } + if len(ver.DeleteMarker.MetaSys) == 0 { + ver.DeleteMarker.MetaSys = make(map[string][]byte) + } + if !fi.DeleteMarkerReplicationStatus().Empty() { + switch fi.DeleteMarkerReplicationStatus() { + case replication.Replica: + ver.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicaStatus] = []byte(fi.ReplicationState.ReplicaStatus) + ver.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicaTimestamp] = []byte(fi.ReplicationState.ReplicaTimeStamp.UTC().Format(time.RFC3339Nano)) + default: + ver.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicationStatus] = []byte(fi.ReplicationState.ReplicationStatusInternal) + ver.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicationTimestamp] = []byte(fi.ReplicationState.ReplicationTimeStamp.UTC().Format(time.RFC3339Nano)) + } + } + if !fi.VersionPurgeStatus().Empty() { + ver.DeleteMarker.MetaSys[VersionPurgeStatusKey] = []byte(fi.ReplicationState.VersionPurgeStatusInternal) + } + for k, v := range fi.ReplicationState.ResetStatusesMap { + ver.DeleteMarker.MetaSys[k] = []byte(v) + } + err = x.setIdx(i, *ver) + return "", err + } + x.versions = append(x.versions[:i], x.versions[i+1:]...) + if fi.MarkDeleted && (fi.VersionPurgeStatus().Empty() || (fi.VersionPurgeStatus() != replication.VersionPurgeComplete)) { + err = x.addVersion(ventry) + } else if fi.Deleted && uv.String() == emptyUUID { + return "", x.addVersion(ventry) + } + return "", err + case ObjectType: + if updateVersion && !fi.Deleted { + ver, err := x.getIdx(i) + if err != nil { + return "", err + } + ver.ObjectV2.MetaSys[VersionPurgeStatusKey] = []byte(fi.ReplicationState.VersionPurgeStatusInternal) + for k, v := range fi.ReplicationState.ResetStatusesMap { + ver.ObjectV2.MetaSys[k] = []byte(v) + } + err = x.setIdx(i, *ver) + return uuid.UUID(ver.ObjectV2.DataDir).String(), err + } + } + } + + for i, version := range x.versions { + if version.header.Type != ObjectType || version.header.VersionID != uv { + continue + } + ver, err := x.getIdx(i) + if err != nil { + return "", err + } + switch { + case fi.ExpireRestored: + ver.ObjectV2.RemoveRestoreHdrs() + err = x.setIdx(i, *ver) + case fi.TransitionStatus == lifecycle.TransitionComplete: + ver.ObjectV2.SetTransition(fi) + ver.ObjectV2.ResetInlineData() + err = x.setIdx(i, *ver) + default: + x.versions = append(x.versions[:i], x.versions[i+1:]...) + // if uv has tiered content we add a + // free-version to track it for + // asynchronous deletion via scanner. + if freeVersion, toFree := ver.ObjectV2.InitFreeVersion(fi); toFree { + err = x.addVersion(freeVersion) + } + } + + if fi.Deleted { + err = x.addVersion(ventry) + } + if x.SharedDataDirCount(ver.ObjectV2.VersionID, ver.ObjectV2.DataDir) > 0 { + // Found that another version references the same dataDir + // we shouldn't remove it, and only remove the version instead + return "", nil + } + return uuid.UUID(ver.ObjectV2.DataDir).String(), err + } + + if fi.Deleted { + err = x.addVersion(ventry) + return "", err + } + return "", errFileVersionNotFound +} + +// xlMetaDataDirDecoder is a shallow decoder for decoding object datadir only. +type xlMetaDataDirDecoder struct { + ObjectV2 *struct { + DataDir [16]byte `msg:"DDir"` // Data dir ID + } `msg:"V2Obj,omitempty"` +} + +// UpdateObjectVersion updates metadata and modTime for a given +// versionID, NOTE: versionID must be valid and should exist - +// and must not be a DeleteMarker or legacy object, if no +// versionID is specified 'null' versionID is updated instead. +// +// It is callers responsibility to set correct versionID, this +// function shouldn't be further extended to update immutable +// values such as ErasureInfo, ChecksumInfo. +// +// Metadata is only updated to new values, existing values +// stay as is, if you wish to update all values you should +// update all metadata freshly before calling this function +// in-case you wish to clear existing metadata. +func (x *xlMetaV2) UpdateObjectVersion(fi FileInfo) error { + if fi.VersionID == "" { + // this means versioning is not yet + // enabled or suspend i.e all versions + // are basically default value i.e "null" + fi.VersionID = nullVersionID + } + + var uv uuid.UUID + var err error + if fi.VersionID != "" && fi.VersionID != nullVersionID { + uv, err = uuid.Parse(fi.VersionID) + if err != nil { + return err + } + } + + for i, version := range x.versions { + switch version.header.Type { + case LegacyType, DeleteType: + if version.header.VersionID == uv { + return errMethodNotAllowed + } + case ObjectType: + if version.header.VersionID == uv { + ver, err := x.getIdx(i) + if err != nil { + return err + } + for k, v := range fi.Metadata { + if len(k) > len(ReservedMetadataPrefixLower) && strings.EqualFold(k[:len(ReservedMetadataPrefixLower)], ReservedMetadataPrefixLower) { + ver.ObjectV2.MetaSys[k] = []byte(v) + } else { + ver.ObjectV2.MetaUser[k] = v + } + } + if !fi.ModTime.IsZero() { + ver.ObjectV2.ModTime = fi.ModTime.UnixNano() + } + return x.setIdx(i, *ver) + } + } + } + + return errFileVersionNotFound +} + +// AddVersion adds a new version +func (x *xlMetaV2) AddVersion(fi FileInfo) error { + if fi.VersionID == "" { + // this means versioning is not yet + // enabled or suspend i.e all versions + // are basically default value i.e "null" + fi.VersionID = nullVersionID + } + + var uv uuid.UUID + var err error + if fi.VersionID != "" && fi.VersionID != nullVersionID { + uv, err = uuid.Parse(fi.VersionID) + if err != nil { + return err + } + } + + var dd uuid.UUID + if fi.DataDir != "" { + dd, err = uuid.Parse(fi.DataDir) + if err != nil { + return err + } + } + + ventry := xlMetaV2Version{ + WrittenByVersion: globalVersionUnix, + } + + if fi.Deleted { + ventry.Type = DeleteType + ventry.DeleteMarker = &xlMetaV2DeleteMarker{ + VersionID: uv, + ModTime: fi.ModTime.UnixNano(), + MetaSys: make(map[string][]byte), + } + } else { + ventry.Type = ObjectType + ventry.ObjectV2 = &xlMetaV2Object{ + VersionID: uv, + DataDir: dd, + Size: fi.Size, + ModTime: fi.ModTime.UnixNano(), + ErasureAlgorithm: ReedSolomon, + ErasureM: fi.Erasure.DataBlocks, + ErasureN: fi.Erasure.ParityBlocks, + ErasureBlockSize: fi.Erasure.BlockSize, + ErasureIndex: fi.Erasure.Index, + BitrotChecksumAlgo: HighwayHash, + ErasureDist: make([]uint8, len(fi.Erasure.Distribution)), + PartNumbers: make([]int, len(fi.Parts)), + PartETags: nil, + PartSizes: make([]int64, len(fi.Parts)), + PartActualSizes: make([]int64, len(fi.Parts)), + MetaSys: make(map[string][]byte), + MetaUser: make(map[string]string, len(fi.Metadata)), + } + for i := range fi.Parts { + // Only add etags if any. + if fi.Parts[i].ETag != "" { + ventry.ObjectV2.PartETags = make([]string, len(fi.Parts)) + break + } + } + for i := range fi.Parts { + // Only add indices if any. + if len(fi.Parts[i].Index) > 0 { + ventry.ObjectV2.PartIndices = make([][]byte, len(fi.Parts)) + break + } + } + for i := range fi.Erasure.Distribution { + ventry.ObjectV2.ErasureDist[i] = uint8(fi.Erasure.Distribution[i]) + } + + for i := range fi.Parts { + ventry.ObjectV2.PartSizes[i] = fi.Parts[i].Size + if len(ventry.ObjectV2.PartETags) > 0 && fi.Parts[i].ETag != "" { + ventry.ObjectV2.PartETags[i] = fi.Parts[i].ETag + } + ventry.ObjectV2.PartNumbers[i] = fi.Parts[i].Number + ventry.ObjectV2.PartActualSizes[i] = fi.Parts[i].ActualSize + if len(ventry.ObjectV2.PartIndices) > i { + ventry.ObjectV2.PartIndices[i] = fi.Parts[i].Index + } + } + + tierFVIDKey := ReservedMetadataPrefixLower + tierFVID + tierFVMarkerKey := ReservedMetadataPrefixLower + tierFVMarker + for k, v := range fi.Metadata { + if len(k) > len(ReservedMetadataPrefixLower) && strings.EqualFold(k[:len(ReservedMetadataPrefixLower)], ReservedMetadataPrefixLower) { + // Skip tierFVID, tierFVMarker keys; it's used + // only for creating free-version. + // Also skip xMinIOHealing, xMinIODataMov as used only in RenameData + switch k { + case tierFVIDKey, tierFVMarkerKey, xMinIOHealing, xMinIODataMov: + continue + } + + ventry.ObjectV2.MetaSys[k] = []byte(v) + } else { + ventry.ObjectV2.MetaUser[k] = v + } + } + + // If asked to save data. + if len(fi.Data) > 0 || fi.Size == 0 { + x.data.replace(fi.VersionID, fi.Data) + } + + if fi.TransitionStatus != "" { + ventry.ObjectV2.MetaSys[metaTierStatus] = []byte(fi.TransitionStatus) + } + if fi.TransitionedObjName != "" { + ventry.ObjectV2.MetaSys[metaTierObjName] = []byte(fi.TransitionedObjName) + } + if fi.TransitionVersionID != "" { + ventry.ObjectV2.MetaSys[metaTierVersionID] = []byte(fi.TransitionVersionID) + } + if fi.TransitionTier != "" { + ventry.ObjectV2.MetaSys[metaTierName] = []byte(fi.TransitionTier) + } + if len(fi.Checksum) > 0 { + ventry.ObjectV2.MetaSys[ReservedMetadataPrefixLower+"crc"] = fi.Checksum + } + } + + if !ventry.Valid() { + return errors.New("internal error: invalid version entry generated") + } + + // Check if we should replace first. + for i := range x.versions { + if x.versions[i].header.VersionID != uv { + continue + } + switch x.versions[i].header.Type { + case LegacyType: + // This would convert legacy type into new ObjectType + // this means that we are basically purging the `null` + // version of the object. + return x.setIdx(i, ventry) + case ObjectType: + return x.setIdx(i, ventry) + case DeleteType: + // Allowing delete marker to replaced with proper + // object data type as well, this is not S3 complaint + // behavior but kept here for future flexibility. + return x.setIdx(i, ventry) + } + } + + // We did not find it, add it. + return x.addVersion(ventry) +} + +func (x *xlMetaV2) SharedDataDirCount(versionID [16]byte, dataDir [16]byte) int { + // v2 object is inlined, if it is skip dataDir share check. + if x.data.entries() > 0 && x.data.find(uuid.UUID(versionID).String()) != nil { + return 0 + } + var sameDataDirCount int + var decoded xlMetaDataDirDecoder + for _, version := range x.versions { + if version.header.Type != ObjectType || version.header.VersionID == versionID || !version.header.UsesDataDir() { + continue + } + _, err := decoded.UnmarshalMsg(version.meta) + if err != nil || decoded.ObjectV2 == nil || decoded.ObjectV2.DataDir != dataDir { + continue + } + sameDataDirCount++ + } + return sameDataDirCount +} + +func (x *xlMetaV2) SharedDataDirCountStr(versionID, dataDir string) int { + var ( + uv uuid.UUID + ddir uuid.UUID + err error + ) + if versionID == nullVersionID { + versionID = "" + } + if versionID != "" { + uv, err = uuid.Parse(versionID) + if err != nil { + return 0 + } + } + ddir, err = uuid.Parse(dataDir) + if err != nil { + return 0 + } + return x.SharedDataDirCount(uv, ddir) +} + +// AddLegacy adds a legacy version, is only called when no prior +// versions exist, safe to use it by only one function in xl-storage(RenameData) +func (x *xlMetaV2) AddLegacy(m *xlMetaV1Object) error { + if !m.valid() { + return errFileCorrupt + } + m.VersionID = nullVersionID + + return x.addVersion(xlMetaV2Version{ObjectV1: m, Type: LegacyType, WrittenByVersion: globalVersionUnix}) +} + +// ToFileInfo converts xlMetaV2 into a common FileInfo datastructure +// for consumption across callers. +func (x xlMetaV2) ToFileInfo(volume, path, versionID string, inclFreeVers, allParts bool) (fi FileInfo, err error) { + var uv uuid.UUID + if versionID != "" && versionID != nullVersionID { + uv, err = uuid.Parse(versionID) + if err != nil { + storageLogIf(GlobalContext, fmt.Errorf("invalid versionID specified %s", versionID)) + return fi, errFileVersionNotFound + } + } + var succModTime int64 + isLatest := true + nonFreeVersions := len(x.versions) + + var ( + freeFi FileInfo + freeFound bool + ) + found := false + for _, ver := range x.versions { + header := &ver.header + // skip listing free-version unless explicitly requested via versionID + if header.FreeVersion() { + nonFreeVersions-- + // remember the latest free version; will return this FileInfo if no non-free version remain + var freeVersion xlMetaV2Version + if inclFreeVers && !freeFound { + // ignore unmarshalling errors, will return errFileNotFound in that case + if _, err := freeVersion.unmarshalV(x.metaV, ver.meta); err == nil { + if freeFi, err = freeVersion.ToFileInfo(volume, path, allParts); err == nil { + freeFi.IsLatest = true // when this is returned, it would be the latest free version remaining. + freeFound = true + } + } + } + + if header.VersionID != uv { + continue + } + } + if found { + continue + } + + // We need a specific version, skip... + if versionID != "" && uv != header.VersionID { + isLatest = false + succModTime = header.ModTime + continue + } + + // We found what we need. + found = true + var version xlMetaV2Version + if _, err := version.unmarshalV(x.metaV, ver.meta); err != nil { + return fi, err + } + if fi, err = version.ToFileInfo(volume, path, allParts); err != nil { + return fi, err + } + fi.IsLatest = isLatest + if succModTime != 0 { + fi.SuccessorModTime = time.Unix(0, succModTime) + } + } + if !found { + if versionID == "" { + if inclFreeVers && nonFreeVersions == 0 { + if freeFound { + return freeFi, nil + } + } + return FileInfo{}, errFileNotFound + } + + return FileInfo{}, errFileVersionNotFound + } + fi.NumVersions = nonFreeVersions + return fi, err +} + +// ListVersions lists current versions, and current deleted +// versions returns error for unexpected entries. +// showPendingDeletes is set to true if ListVersions needs to list objects marked deleted +// but waiting to be replicated +func (x xlMetaV2) ListVersions(volume, path string, allParts bool) ([]FileInfo, error) { + versions := make([]FileInfo, 0, len(x.versions)) + var err error + + var dst xlMetaV2Version + for _, version := range x.versions { + _, err = dst.unmarshalV(x.metaV, version.meta) + if err != nil { + return versions, err + } + fi, err := dst.ToFileInfo(volume, path, allParts) + if err != nil { + return versions, err + } + fi.NumVersions = len(x.versions) + versions = append(versions, fi) + } + + for i := range versions { + versions[i].NumVersions = len(versions) + if i > 0 { + versions[i].SuccessorModTime = versions[i-1].ModTime + } + } + if len(versions) > 0 { + versions[0].IsLatest = true + } + return versions, nil +} + +// mergeXLV2Versions will merge all versions, typically from different disks +// that have at least quorum entries in all metas. +// Each version slice should be sorted. +// Quorum must be the minimum number of matching metadata files. +// Quorum should be > 1 and <= len(versions). +// If strict is set to false, entries that match type +func mergeXLV2Versions(quorum int, strict bool, requestedVersions int, versions ...[]xlMetaV2ShallowVersion) (merged []xlMetaV2ShallowVersion) { + if quorum <= 0 { + quorum = 1 + } + if len(versions) < quorum || len(versions) == 0 { + return nil + } + if len(versions) == 1 { + return versions[0] + } + if quorum == 1 { + // No need for non-strict checks if quorum is 1. + strict = true + } + // Shallow copy input + versions = append(make([][]xlMetaV2ShallowVersion, 0, len(versions)), versions...) + + var nVersions int // captures all non-free versions + + // Our result + merged = make([]xlMetaV2ShallowVersion, 0, len(versions[0])) + tops := make([]xlMetaV2ShallowVersion, len(versions)) + for { + // Step 1 create slice with all top versions. + tops = tops[:0] + var topSig xlMetaV2VersionHeader + consistent := true // Are all signatures consistent (shortcut) + for _, vers := range versions { + if len(vers) == 0 { + consistent = false + continue + } + ver := vers[0] + if len(tops) == 0 { + consistent = true + topSig = ver.header + } else { + consistent = consistent && ver.header == topSig + } + tops = append(tops, vers[0]) + } + + // Check if done... + if len(tops) < quorum { + // We couldn't gather enough for quorum + break + } + + var latest xlMetaV2ShallowVersion + if consistent { + // All had the same signature, easy. + latest = tops[0] + merged = append(merged, latest) + + // Calculate latest 'n' non-free versions. + if !latest.header.FreeVersion() { + nVersions++ + } + } else { + // Find latest. + var latestCount int + for i, ver := range tops { + if ver.header == latest.header { + latestCount++ + continue + } + if i == 0 || ver.header.sortsBefore(latest.header) { + switch { + case i == 0 || latestCount == 0: + latestCount = 1 + case !strict && ver.header.matchesNotStrict(latest.header): + latestCount++ + default: + latestCount = 1 + } + latest = ver + continue + } + + // Mismatch, but older. + if latestCount > 0 && !strict && ver.header.matchesNotStrict(latest.header) { + latestCount++ + continue + } + if latestCount > 0 && ver.header.VersionID == latest.header.VersionID { + // Version IDs match, but otherwise unable to resolve. + // We are either strict, or don't have enough information to match. + // Switch to a pure counting algo. + x := make(map[xlMetaV2VersionHeader]int, len(tops)) + for _, a := range tops { + if a.header.VersionID != ver.header.VersionID { + continue + } + if !strict { + // we must match EC, when we are not strict. + if !a.header.matchesEC(ver.header) { + continue + } + + a.header.Signature = [4]byte{} + } + x[a.header]++ + } + latestCount = 0 + for k, v := range x { + if v < latestCount { + continue + } + if v == latestCount && latest.header.sortsBefore(k) { + // Tiebreak, use sort. + continue + } + for _, a := range tops { + hdr := a.header + if !strict { + hdr.Signature = [4]byte{} + } + if hdr == k { + latest = a + } + } + latestCount = v + } + break + } + } + if latestCount >= quorum { + merged = append(merged, latest) + + // Calculate latest 'n' non-free versions. + if !latest.header.FreeVersion() { + nVersions++ + } + } + } + + // Remove from all streams up until latest modtime or if selected. + for i, vers := range versions { + for _, ver := range vers { + // Truncate later modtimes, not selected. + if ver.header.ModTime > latest.header.ModTime { + versions[i] = versions[i][1:] + continue + } + // Truncate matches + if ver.header == latest.header { + versions[i] = versions[i][1:] + continue + } + + // Truncate non-empty version and type matches + if latest.header.VersionID == ver.header.VersionID { + versions[i] = versions[i][1:] + continue + } + // Skip versions with version id we already emitted. + for _, mergedV := range merged { + if ver.header.VersionID == mergedV.header.VersionID { + versions[i] = versions[i][1:] + continue + } + } + // Keep top entry (and remaining)... + break + } + } + + if requestedVersions > 0 && requestedVersions == nVersions { + merged = append(merged, versions[0]...) + break + } + } + + // Sanity check. Enable if duplicates show up. + if false { + found := make(map[[16]byte]struct{}) + for _, ver := range merged { + if _, ok := found[ver.header.VersionID]; ok { + panic("found dupe") + } + found[ver.header.VersionID] = struct{}{} + } + } + return merged +} + +type xlMetaBuf []byte + +// ToFileInfo converts xlMetaV2 into a common FileInfo datastructure +// for consumption across callers. +func (x xlMetaBuf) ToFileInfo(volume, path, versionID string, allParts bool) (fi FileInfo, err error) { + var uv uuid.UUID + if versionID != "" && versionID != nullVersionID { + uv, err = uuid.Parse(versionID) + if err != nil { + storageLogIf(GlobalContext, fmt.Errorf("invalid versionID specified %s", versionID)) + return fi, errFileVersionNotFound + } + } + versions, headerV, metaV, buf, err := decodeXLHeaders(x) + if err != nil { + return fi, err + } + var header xlMetaV2VersionHeader + var succModTime int64 + isLatest := true + nonFreeVersions := versions + found := false + err = decodeVersions(buf, versions, func(idx int, hdr, meta []byte) error { + if _, err := header.unmarshalV(headerV, hdr); err != nil { + return err + } + + // skip listing free-version unless explicitly requested via versionID + if header.FreeVersion() { + nonFreeVersions-- + if header.VersionID != uv { + return nil + } + } + if found { + return nil + } + + // We need a specific version, skip... + if versionID != "" && uv != header.VersionID { + isLatest = false + succModTime = header.ModTime + return nil + } + + // We found what we need. + found = true + var version xlMetaV2Version + if _, err := version.unmarshalV(metaV, meta); err != nil { + return err + } + if fi, err = version.ToFileInfo(volume, path, allParts); err != nil { + return err + } + fi.IsLatest = isLatest + if succModTime != 0 { + fi.SuccessorModTime = time.Unix(0, succModTime) + } + return nil + }) + if !found { + if versionID == "" { + return FileInfo{}, errFileNotFound + } + + return FileInfo{}, errFileVersionNotFound + } + fi.NumVersions = nonFreeVersions + return fi, err +} + +// ListVersions lists current versions, and current deleted +// versions returns error for unexpected entries. +// showPendingDeletes is set to true if ListVersions needs to list objects marked deleted +// but waiting to be replicated +func (x xlMetaBuf) ListVersions(volume, path string, allParts bool) ([]FileInfo, error) { + vers, _, metaV, buf, err := decodeXLHeaders(x) + if err != nil { + return nil, err + } + var succModTime time.Time + isLatest := true + dst := make([]FileInfo, 0, vers) + var xl xlMetaV2Version + err = decodeVersions(buf, vers, func(idx int, hdr, meta []byte) error { + if _, err := xl.unmarshalV(metaV, meta); err != nil { + return err + } + if !xl.Valid() { + return errFileCorrupt + } + fi, err := xl.ToFileInfo(volume, path, allParts) + if err != nil { + return err + } + fi.IsLatest = isLatest + fi.SuccessorModTime = succModTime + fi.NumVersions = vers + isLatest = false + succModTime = xl.getModTime() + + dst = append(dst, fi) + return nil + }) + return dst, err +} + +// IsLatestDeleteMarker returns true if latest version is a deletemarker or there are no versions. +// If any error occurs false is returned. +func (x xlMetaBuf) IsLatestDeleteMarker() bool { + vers, headerV, _, buf, err := decodeXLHeaders(x) + if err != nil { + return false + } + if vers == 0 { + return true + } + isDeleteMarker := false + + _ = decodeVersions(buf, vers, func(idx int, hdr, _ []byte) error { + var xl xlMetaV2VersionHeader + if _, err := xl.unmarshalV(headerV, hdr); err != nil { + return errDoneForNow + } + isDeleteMarker = xl.Type == DeleteType + return errDoneForNow + }) + return isDeleteMarker +} + +// AllHidden returns true are no versions that would show up in a listing (ie all free markers) +// Optionally also return early if top is a delete marker. +func (x xlMetaBuf) AllHidden(topDeleteMarker bool) bool { + vers, headerV, _, buf, err := decodeXLHeaders(x) + if err != nil { + return false + } + if vers == 0 { + return true + } + hidden := true + + var xl xlMetaV2VersionHeader + _ = decodeVersions(buf, vers, func(idx int, hdr, _ []byte) error { + if _, err := xl.unmarshalV(headerV, hdr); err != nil { + return errDoneForNow + } + if topDeleteMarker && idx == 0 && xl.Type == DeleteType { + hidden = true + return errDoneForNow + } + if !xl.FreeVersion() { + hidden = false + return errDoneForNow + } + // Check next version + return nil + }) + return hidden +} diff --git a/cmd/xl-storage-format-v2_gen.go b/cmd/xl-storage-format-v2_gen.go new file mode 100644 index 00000000..e9d00cd0 --- /dev/null +++ b/cmd/xl-storage-format-v2_gen.go @@ -0,0 +1,2444 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *ChecksumAlgo) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 uint8 + zb0001, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = ChecksumAlgo(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z ChecksumAlgo) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteUint8(uint8(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z ChecksumAlgo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendUint8(o, uint8(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ChecksumAlgo) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 uint8 + zb0001, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = ChecksumAlgo(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z ChecksumAlgo) Msgsize() (s int) { + s = msgp.Uint8Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ErasureAlgo) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 uint8 + zb0001, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = ErasureAlgo(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z ErasureAlgo) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteUint8(uint8(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z ErasureAlgo) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendUint8(o, uint8(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ErasureAlgo) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 uint8 + zb0001, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = ErasureAlgo(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z ErasureAlgo) Msgsize() (s int) { + s = msgp.Uint8Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *VersionType) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 uint8 + zb0001, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = VersionType(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z VersionType) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteUint8(uint8(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z VersionType) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendUint8(o, uint8(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *VersionType) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 uint8 + zb0001, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = VersionType(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z VersionType) Msgsize() (s int) { + s = msgp.Uint8Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *xlFlags) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 uint8 + zb0001, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = xlFlags(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z xlFlags) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteUint8(uint8(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z xlFlags) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendUint8(o, uint8(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *xlFlags) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 uint8 + zb0001, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = xlFlags(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z xlFlags) Msgsize() (s int) { + s = msgp.Uint8Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *xlMetaBuf) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 []byte + zb0001, err = dc.ReadBytes([]byte((*z))) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = xlMetaBuf(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z xlMetaBuf) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteBytes([]byte(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z xlMetaBuf) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendBytes(o, []byte(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *xlMetaBuf) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 []byte + zb0001, bts, err = msgp.ReadBytesBytes(bts, []byte((*z))) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = xlMetaBuf(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z xlMetaBuf) Msgsize() (s int) { + s = msgp.BytesPrefixSize + len([]byte(z)) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *xlMetaDataDirDecoder) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 1 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "V2Obj": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "ObjectV2") + return + } + z.ObjectV2 = nil + } else { + if z.ObjectV2 == nil { + z.ObjectV2 = new(struct { + DataDir [16]byte `msg:"DDir"` + }) + } + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "ObjectV2") + return + } + for zb0002 > 0 { + zb0002-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "ObjectV2") + return + } + switch msgp.UnsafeString(field) { + case "DDir": + err = dc.ReadExactBytes((z.ObjectV2.DataDir)[:]) + if err != nil { + err = msgp.WrapError(err, "ObjectV2", "DataDir") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "ObjectV2") + return + } + } + } + } + zb0001Mask |= 0x1 + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if (zb0001Mask & 0x1) == 0 { + z.ObjectV2 = nil + } + + return +} + +// EncodeMsg implements msgp.Encodable +func (z *xlMetaDataDirDecoder) EncodeMsg(en *msgp.Writer) (err error) { + // check for omitted fields + zb0001Len := uint32(1) + var zb0001Mask uint8 /* 1 bits */ + _ = zb0001Mask + if z.ObjectV2 == nil { + zb0001Len-- + zb0001Mask |= 0x1 + } + // variable map header, size zb0001Len + err = en.Append(0x80 | uint8(zb0001Len)) + if err != nil { + return + } + if (zb0001Mask & 0x1) == 0 { // if not omitted + // write "V2Obj" + err = en.Append(0xa5, 0x56, 0x32, 0x4f, 0x62, 0x6a) + if err != nil { + return + } + if z.ObjectV2 == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + // map header, size 1 + // write "DDir" + err = en.Append(0x81, 0xa4, 0x44, 0x44, 0x69, 0x72) + if err != nil { + return + } + err = en.WriteBytes((z.ObjectV2.DataDir)[:]) + if err != nil { + err = msgp.WrapError(err, "ObjectV2", "DataDir") + return + } + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *xlMetaDataDirDecoder) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // check for omitted fields + zb0001Len := uint32(1) + var zb0001Mask uint8 /* 1 bits */ + _ = zb0001Mask + if z.ObjectV2 == nil { + zb0001Len-- + zb0001Mask |= 0x1 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + if (zb0001Mask & 0x1) == 0 { // if not omitted + // string "V2Obj" + o = append(o, 0xa5, 0x56, 0x32, 0x4f, 0x62, 0x6a) + if z.ObjectV2 == nil { + o = msgp.AppendNil(o) + } else { + // map header, size 1 + // string "DDir" + o = append(o, 0x81, 0xa4, 0x44, 0x44, 0x69, 0x72) + o = msgp.AppendBytes(o, (z.ObjectV2.DataDir)[:]) + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *xlMetaDataDirDecoder) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 1 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "V2Obj": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.ObjectV2 = nil + } else { + if z.ObjectV2 == nil { + z.ObjectV2 = new(struct { + DataDir [16]byte `msg:"DDir"` + }) + } + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ObjectV2") + return + } + for zb0002 > 0 { + zb0002-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "ObjectV2") + return + } + switch msgp.UnsafeString(field) { + case "DDir": + bts, err = msgp.ReadExactBytes(bts, (z.ObjectV2.DataDir)[:]) + if err != nil { + err = msgp.WrapError(err, "ObjectV2", "DataDir") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "ObjectV2") + return + } + } + } + } + zb0001Mask |= 0x1 + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if (zb0001Mask & 0x1) == 0 { + z.ObjectV2 = nil + } + + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *xlMetaDataDirDecoder) Msgsize() (s int) { + s = 1 + 6 + if z.ObjectV2 == nil { + s += msgp.NilSize + } else { + s += 1 + 5 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *xlMetaV2DeleteMarker) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 1 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ID": + err = dc.ReadExactBytes((z.VersionID)[:]) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + case "MTime": + z.ModTime, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + case "MetaSys": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "MetaSys") + return + } + if z.MetaSys == nil { + z.MetaSys = make(map[string][]byte, zb0002) + } else if len(z.MetaSys) > 0 { + clear(z.MetaSys) + } + for zb0002 > 0 { + zb0002-- + var za0002 string + za0002, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "MetaSys") + return + } + var za0003 []byte + za0003, err = dc.ReadBytes(za0003) + if err != nil { + err = msgp.WrapError(err, "MetaSys", za0002) + return + } + z.MetaSys[za0002] = za0003 + } + zb0001Mask |= 0x1 + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if (zb0001Mask & 0x1) == 0 { + z.MetaSys = nil + } + + return +} + +// EncodeMsg implements msgp.Encodable +func (z *xlMetaV2DeleteMarker) EncodeMsg(en *msgp.Writer) (err error) { + // check for omitted fields + zb0001Len := uint32(3) + var zb0001Mask uint8 /* 3 bits */ + _ = zb0001Mask + if z.MetaSys == nil { + zb0001Len-- + zb0001Mask |= 0x4 + } + // variable map header, size zb0001Len + err = en.Append(0x80 | uint8(zb0001Len)) + if err != nil { + return + } + + // skip if no fields are to be emitted + if zb0001Len != 0 { + // write "ID" + err = en.Append(0xa2, 0x49, 0x44) + if err != nil { + return + } + err = en.WriteBytes((z.VersionID)[:]) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + // write "MTime" + err = en.Append(0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.ModTime) + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + if (zb0001Mask & 0x4) == 0 { // if not omitted + // write "MetaSys" + err = en.Append(0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73) + if err != nil { + return + } + err = en.WriteMapHeader(uint32(len(z.MetaSys))) + if err != nil { + err = msgp.WrapError(err, "MetaSys") + return + } + for za0002, za0003 := range z.MetaSys { + err = en.WriteString(za0002) + if err != nil { + err = msgp.WrapError(err, "MetaSys") + return + } + err = en.WriteBytes(za0003) + if err != nil { + err = msgp.WrapError(err, "MetaSys", za0002) + return + } + } + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *xlMetaV2DeleteMarker) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // check for omitted fields + zb0001Len := uint32(3) + var zb0001Mask uint8 /* 3 bits */ + _ = zb0001Mask + if z.MetaSys == nil { + zb0001Len-- + zb0001Mask |= 0x4 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + + // skip if no fields are to be emitted + if zb0001Len != 0 { + // string "ID" + o = append(o, 0xa2, 0x49, 0x44) + o = msgp.AppendBytes(o, (z.VersionID)[:]) + // string "MTime" + o = append(o, 0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65) + o = msgp.AppendInt64(o, z.ModTime) + if (zb0001Mask & 0x4) == 0 { // if not omitted + // string "MetaSys" + o = append(o, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73) + o = msgp.AppendMapHeader(o, uint32(len(z.MetaSys))) + for za0002, za0003 := range z.MetaSys { + o = msgp.AppendString(o, za0002) + o = msgp.AppendBytes(o, za0003) + } + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *xlMetaV2DeleteMarker) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 1 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ID": + bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:]) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + case "MTime": + z.ModTime, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + case "MetaSys": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "MetaSys") + return + } + if z.MetaSys == nil { + z.MetaSys = make(map[string][]byte, zb0002) + } else if len(z.MetaSys) > 0 { + clear(z.MetaSys) + } + for zb0002 > 0 { + var za0003 []byte + zb0002-- + var za0002 string + za0002, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "MetaSys") + return + } + za0003, bts, err = msgp.ReadBytesBytes(bts, za0003) + if err != nil { + err = msgp.WrapError(err, "MetaSys", za0002) + return + } + z.MetaSys[za0002] = za0003 + } + zb0001Mask |= 0x1 + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if (zb0001Mask & 0x1) == 0 { + z.MetaSys = nil + } + + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *xlMetaV2DeleteMarker) Msgsize() (s int) { + s = 1 + 3 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + 6 + msgp.Int64Size + 8 + msgp.MapHeaderSize + if z.MetaSys != nil { + for za0002, za0003 := range z.MetaSys { + _ = za0003 + s += msgp.StringPrefixSize + len(za0002) + msgp.BytesPrefixSize + len(za0003) + } + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *xlMetaV2Object) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 1 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ID": + err = dc.ReadExactBytes((z.VersionID)[:]) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + case "DDir": + err = dc.ReadExactBytes((z.DataDir)[:]) + if err != nil { + err = msgp.WrapError(err, "DataDir") + return + } + case "EcAlgo": + { + var zb0002 uint8 + zb0002, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err, "ErasureAlgorithm") + return + } + z.ErasureAlgorithm = ErasureAlgo(zb0002) + } + case "EcM": + z.ErasureM, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "ErasureM") + return + } + case "EcN": + z.ErasureN, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "ErasureN") + return + } + case "EcBSize": + z.ErasureBlockSize, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ErasureBlockSize") + return + } + case "EcIndex": + z.ErasureIndex, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "ErasureIndex") + return + } + case "EcDist": + var zb0003 uint32 + zb0003, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "ErasureDist") + return + } + if cap(z.ErasureDist) >= int(zb0003) { + z.ErasureDist = (z.ErasureDist)[:zb0003] + } else { + z.ErasureDist = make([]uint8, zb0003) + } + for za0003 := range z.ErasureDist { + z.ErasureDist[za0003], err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err, "ErasureDist", za0003) + return + } + } + case "CSumAlgo": + { + var zb0004 uint8 + zb0004, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err, "BitrotChecksumAlgo") + return + } + z.BitrotChecksumAlgo = ChecksumAlgo(zb0004) + } + case "PartNums": + var zb0005 uint32 + zb0005, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "PartNumbers") + return + } + if cap(z.PartNumbers) >= int(zb0005) { + z.PartNumbers = (z.PartNumbers)[:zb0005] + } else { + z.PartNumbers = make([]int, zb0005) + } + for za0004 := range z.PartNumbers { + z.PartNumbers[za0004], err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "PartNumbers", za0004) + return + } + } + case "PartETags": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "PartETags") + return + } + z.PartETags = nil + } else { + var zb0006 uint32 + zb0006, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "PartETags") + return + } + if z.PartETags != nil && cap(z.PartETags) >= int(zb0006) { + z.PartETags = (z.PartETags)[:zb0006] + } else { + z.PartETags = make([]string, zb0006) + } + for za0005 := range z.PartETags { + z.PartETags[za0005], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "PartETags", za0005) + return + } + } + } + case "PartSizes": + var zb0007 uint32 + zb0007, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "PartSizes") + return + } + if cap(z.PartSizes) >= int(zb0007) { + z.PartSizes = (z.PartSizes)[:zb0007] + } else { + z.PartSizes = make([]int64, zb0007) + } + for za0006 := range z.PartSizes { + z.PartSizes[za0006], err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "PartSizes", za0006) + return + } + } + case "PartASizes": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "PartActualSizes") + return + } + z.PartActualSizes = nil + } else { + var zb0008 uint32 + zb0008, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "PartActualSizes") + return + } + if z.PartActualSizes != nil && cap(z.PartActualSizes) >= int(zb0008) { + z.PartActualSizes = (z.PartActualSizes)[:zb0008] + } else { + z.PartActualSizes = make([]int64, zb0008) + } + for za0007 := range z.PartActualSizes { + z.PartActualSizes[za0007], err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "PartActualSizes", za0007) + return + } + } + } + case "PartIdx": + var zb0009 uint32 + zb0009, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "PartIndices") + return + } + if cap(z.PartIndices) >= int(zb0009) { + z.PartIndices = (z.PartIndices)[:zb0009] + } else { + z.PartIndices = make([][]byte, zb0009) + } + for za0008 := range z.PartIndices { + z.PartIndices[za0008], err = dc.ReadBytes(z.PartIndices[za0008]) + if err != nil { + err = msgp.WrapError(err, "PartIndices", za0008) + return + } + } + zb0001Mask |= 0x1 + case "Size": + z.Size, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + case "MTime": + z.ModTime, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + case "MetaSys": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "MetaSys") + return + } + z.MetaSys = nil + } else { + var zb0010 uint32 + zb0010, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "MetaSys") + return + } + if z.MetaSys == nil { + z.MetaSys = make(map[string][]byte, zb0010) + } else if len(z.MetaSys) > 0 { + clear(z.MetaSys) + } + for zb0010 > 0 { + zb0010-- + var za0009 string + za0009, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "MetaSys") + return + } + var za0010 []byte + za0010, err = dc.ReadBytes(za0010) + if err != nil { + err = msgp.WrapError(err, "MetaSys", za0009) + return + } + z.MetaSys[za0009] = za0010 + } + } + case "MetaUsr": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "MetaUser") + return + } + z.MetaUser = nil + } else { + var zb0011 uint32 + zb0011, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "MetaUser") + return + } + if z.MetaUser == nil { + z.MetaUser = make(map[string]string, zb0011) + } else if len(z.MetaUser) > 0 { + clear(z.MetaUser) + } + for zb0011 > 0 { + zb0011-- + var za0011 string + za0011, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "MetaUser") + return + } + var za0012 string + za0012, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "MetaUser", za0011) + return + } + z.MetaUser[za0011] = za0012 + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if (zb0001Mask & 0x1) == 0 { + z.PartIndices = nil + } + + return +} + +// EncodeMsg implements msgp.Encodable +func (z *xlMetaV2Object) EncodeMsg(en *msgp.Writer) (err error) { + // check for omitted fields + zb0001Len := uint32(18) + var zb0001Mask uint32 /* 18 bits */ + _ = zb0001Mask + if z.PartIndices == nil { + zb0001Len-- + zb0001Mask |= 0x2000 + } + // variable map header, size zb0001Len + err = en.WriteMapHeader(zb0001Len) + if err != nil { + return + } + + // skip if no fields are to be emitted + if zb0001Len != 0 { + // write "ID" + err = en.Append(0xa2, 0x49, 0x44) + if err != nil { + return + } + err = en.WriteBytes((z.VersionID)[:]) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + // write "DDir" + err = en.Append(0xa4, 0x44, 0x44, 0x69, 0x72) + if err != nil { + return + } + err = en.WriteBytes((z.DataDir)[:]) + if err != nil { + err = msgp.WrapError(err, "DataDir") + return + } + // write "EcAlgo" + err = en.Append(0xa6, 0x45, 0x63, 0x41, 0x6c, 0x67, 0x6f) + if err != nil { + return + } + err = en.WriteUint8(uint8(z.ErasureAlgorithm)) + if err != nil { + err = msgp.WrapError(err, "ErasureAlgorithm") + return + } + // write "EcM" + err = en.Append(0xa3, 0x45, 0x63, 0x4d) + if err != nil { + return + } + err = en.WriteInt(z.ErasureM) + if err != nil { + err = msgp.WrapError(err, "ErasureM") + return + } + // write "EcN" + err = en.Append(0xa3, 0x45, 0x63, 0x4e) + if err != nil { + return + } + err = en.WriteInt(z.ErasureN) + if err != nil { + err = msgp.WrapError(err, "ErasureN") + return + } + // write "EcBSize" + err = en.Append(0xa7, 0x45, 0x63, 0x42, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.ErasureBlockSize) + if err != nil { + err = msgp.WrapError(err, "ErasureBlockSize") + return + } + // write "EcIndex" + err = en.Append(0xa7, 0x45, 0x63, 0x49, 0x6e, 0x64, 0x65, 0x78) + if err != nil { + return + } + err = en.WriteInt(z.ErasureIndex) + if err != nil { + err = msgp.WrapError(err, "ErasureIndex") + return + } + // write "EcDist" + err = en.Append(0xa6, 0x45, 0x63, 0x44, 0x69, 0x73, 0x74) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.ErasureDist))) + if err != nil { + err = msgp.WrapError(err, "ErasureDist") + return + } + for za0003 := range z.ErasureDist { + err = en.WriteUint8(z.ErasureDist[za0003]) + if err != nil { + err = msgp.WrapError(err, "ErasureDist", za0003) + return + } + } + // write "CSumAlgo" + err = en.Append(0xa8, 0x43, 0x53, 0x75, 0x6d, 0x41, 0x6c, 0x67, 0x6f) + if err != nil { + return + } + err = en.WriteUint8(uint8(z.BitrotChecksumAlgo)) + if err != nil { + err = msgp.WrapError(err, "BitrotChecksumAlgo") + return + } + // write "PartNums" + err = en.Append(0xa8, 0x50, 0x61, 0x72, 0x74, 0x4e, 0x75, 0x6d, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.PartNumbers))) + if err != nil { + err = msgp.WrapError(err, "PartNumbers") + return + } + for za0004 := range z.PartNumbers { + err = en.WriteInt(z.PartNumbers[za0004]) + if err != nil { + err = msgp.WrapError(err, "PartNumbers", za0004) + return + } + } + // write "PartETags" + err = en.Append(0xa9, 0x50, 0x61, 0x72, 0x74, 0x45, 0x54, 0x61, 0x67, 0x73) + if err != nil { + return + } + if z.PartETags == nil { // allownil: if nil + err = en.WriteNil() + if err != nil { + return + } + } else { + err = en.WriteArrayHeader(uint32(len(z.PartETags))) + if err != nil { + err = msgp.WrapError(err, "PartETags") + return + } + for za0005 := range z.PartETags { + err = en.WriteString(z.PartETags[za0005]) + if err != nil { + err = msgp.WrapError(err, "PartETags", za0005) + return + } + } + } + // write "PartSizes" + err = en.Append(0xa9, 0x50, 0x61, 0x72, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.PartSizes))) + if err != nil { + err = msgp.WrapError(err, "PartSizes") + return + } + for za0006 := range z.PartSizes { + err = en.WriteInt64(z.PartSizes[za0006]) + if err != nil { + err = msgp.WrapError(err, "PartSizes", za0006) + return + } + } + // write "PartASizes" + err = en.Append(0xaa, 0x50, 0x61, 0x72, 0x74, 0x41, 0x53, 0x69, 0x7a, 0x65, 0x73) + if err != nil { + return + } + if z.PartActualSizes == nil { // allownil: if nil + err = en.WriteNil() + if err != nil { + return + } + } else { + err = en.WriteArrayHeader(uint32(len(z.PartActualSizes))) + if err != nil { + err = msgp.WrapError(err, "PartActualSizes") + return + } + for za0007 := range z.PartActualSizes { + err = en.WriteInt64(z.PartActualSizes[za0007]) + if err != nil { + err = msgp.WrapError(err, "PartActualSizes", za0007) + return + } + } + } + if (zb0001Mask & 0x2000) == 0 { // if not omitted + // write "PartIdx" + err = en.Append(0xa7, 0x50, 0x61, 0x72, 0x74, 0x49, 0x64, 0x78) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.PartIndices))) + if err != nil { + err = msgp.WrapError(err, "PartIndices") + return + } + for za0008 := range z.PartIndices { + err = en.WriteBytes(z.PartIndices[za0008]) + if err != nil { + err = msgp.WrapError(err, "PartIndices", za0008) + return + } + } + } + // write "Size" + err = en.Append(0xa4, 0x53, 0x69, 0x7a, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.Size) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + // write "MTime" + err = en.Append(0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteInt64(z.ModTime) + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + // write "MetaSys" + err = en.Append(0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73) + if err != nil { + return + } + if z.MetaSys == nil { // allownil: if nil + err = en.WriteNil() + if err != nil { + return + } + } else { + err = en.WriteMapHeader(uint32(len(z.MetaSys))) + if err != nil { + err = msgp.WrapError(err, "MetaSys") + return + } + for za0009, za0010 := range z.MetaSys { + err = en.WriteString(za0009) + if err != nil { + err = msgp.WrapError(err, "MetaSys") + return + } + err = en.WriteBytes(za0010) + if err != nil { + err = msgp.WrapError(err, "MetaSys", za0009) + return + } + } + } + // write "MetaUsr" + err = en.Append(0xa7, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x73, 0x72) + if err != nil { + return + } + if z.MetaUser == nil { // allownil: if nil + err = en.WriteNil() + if err != nil { + return + } + } else { + err = en.WriteMapHeader(uint32(len(z.MetaUser))) + if err != nil { + err = msgp.WrapError(err, "MetaUser") + return + } + for za0011, za0012 := range z.MetaUser { + err = en.WriteString(za0011) + if err != nil { + err = msgp.WrapError(err, "MetaUser") + return + } + err = en.WriteString(za0012) + if err != nil { + err = msgp.WrapError(err, "MetaUser", za0011) + return + } + } + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *xlMetaV2Object) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // check for omitted fields + zb0001Len := uint32(18) + var zb0001Mask uint32 /* 18 bits */ + _ = zb0001Mask + if z.PartIndices == nil { + zb0001Len-- + zb0001Mask |= 0x2000 + } + // variable map header, size zb0001Len + o = msgp.AppendMapHeader(o, zb0001Len) + + // skip if no fields are to be emitted + if zb0001Len != 0 { + // string "ID" + o = append(o, 0xa2, 0x49, 0x44) + o = msgp.AppendBytes(o, (z.VersionID)[:]) + // string "DDir" + o = append(o, 0xa4, 0x44, 0x44, 0x69, 0x72) + o = msgp.AppendBytes(o, (z.DataDir)[:]) + // string "EcAlgo" + o = append(o, 0xa6, 0x45, 0x63, 0x41, 0x6c, 0x67, 0x6f) + o = msgp.AppendUint8(o, uint8(z.ErasureAlgorithm)) + // string "EcM" + o = append(o, 0xa3, 0x45, 0x63, 0x4d) + o = msgp.AppendInt(o, z.ErasureM) + // string "EcN" + o = append(o, 0xa3, 0x45, 0x63, 0x4e) + o = msgp.AppendInt(o, z.ErasureN) + // string "EcBSize" + o = append(o, 0xa7, 0x45, 0x63, 0x42, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.ErasureBlockSize) + // string "EcIndex" + o = append(o, 0xa7, 0x45, 0x63, 0x49, 0x6e, 0x64, 0x65, 0x78) + o = msgp.AppendInt(o, z.ErasureIndex) + // string "EcDist" + o = append(o, 0xa6, 0x45, 0x63, 0x44, 0x69, 0x73, 0x74) + o = msgp.AppendArrayHeader(o, uint32(len(z.ErasureDist))) + for za0003 := range z.ErasureDist { + o = msgp.AppendUint8(o, z.ErasureDist[za0003]) + } + // string "CSumAlgo" + o = append(o, 0xa8, 0x43, 0x53, 0x75, 0x6d, 0x41, 0x6c, 0x67, 0x6f) + o = msgp.AppendUint8(o, uint8(z.BitrotChecksumAlgo)) + // string "PartNums" + o = append(o, 0xa8, 0x50, 0x61, 0x72, 0x74, 0x4e, 0x75, 0x6d, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.PartNumbers))) + for za0004 := range z.PartNumbers { + o = msgp.AppendInt(o, z.PartNumbers[za0004]) + } + // string "PartETags" + o = append(o, 0xa9, 0x50, 0x61, 0x72, 0x74, 0x45, 0x54, 0x61, 0x67, 0x73) + if z.PartETags == nil { // allownil: if nil + o = msgp.AppendNil(o) + } else { + o = msgp.AppendArrayHeader(o, uint32(len(z.PartETags))) + for za0005 := range z.PartETags { + o = msgp.AppendString(o, z.PartETags[za0005]) + } + } + // string "PartSizes" + o = append(o, 0xa9, 0x50, 0x61, 0x72, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.PartSizes))) + for za0006 := range z.PartSizes { + o = msgp.AppendInt64(o, z.PartSizes[za0006]) + } + // string "PartASizes" + o = append(o, 0xaa, 0x50, 0x61, 0x72, 0x74, 0x41, 0x53, 0x69, 0x7a, 0x65, 0x73) + if z.PartActualSizes == nil { // allownil: if nil + o = msgp.AppendNil(o) + } else { + o = msgp.AppendArrayHeader(o, uint32(len(z.PartActualSizes))) + for za0007 := range z.PartActualSizes { + o = msgp.AppendInt64(o, z.PartActualSizes[za0007]) + } + } + if (zb0001Mask & 0x2000) == 0 { // if not omitted + // string "PartIdx" + o = append(o, 0xa7, 0x50, 0x61, 0x72, 0x74, 0x49, 0x64, 0x78) + o = msgp.AppendArrayHeader(o, uint32(len(z.PartIndices))) + for za0008 := range z.PartIndices { + o = msgp.AppendBytes(o, z.PartIndices[za0008]) + } + } + // string "Size" + o = append(o, 0xa4, 0x53, 0x69, 0x7a, 0x65) + o = msgp.AppendInt64(o, z.Size) + // string "MTime" + o = append(o, 0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65) + o = msgp.AppendInt64(o, z.ModTime) + // string "MetaSys" + o = append(o, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73) + if z.MetaSys == nil { // allownil: if nil + o = msgp.AppendNil(o) + } else { + o = msgp.AppendMapHeader(o, uint32(len(z.MetaSys))) + for za0009, za0010 := range z.MetaSys { + o = msgp.AppendString(o, za0009) + o = msgp.AppendBytes(o, za0010) + } + } + // string "MetaUsr" + o = append(o, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x73, 0x72) + if z.MetaUser == nil { // allownil: if nil + o = msgp.AppendNil(o) + } else { + o = msgp.AppendMapHeader(o, uint32(len(z.MetaUser))) + for za0011, za0012 := range z.MetaUser { + o = msgp.AppendString(o, za0011) + o = msgp.AppendString(o, za0012) + } + } + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *xlMetaV2Object) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 1 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ID": + bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:]) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + case "DDir": + bts, err = msgp.ReadExactBytes(bts, (z.DataDir)[:]) + if err != nil { + err = msgp.WrapError(err, "DataDir") + return + } + case "EcAlgo": + { + var zb0002 uint8 + zb0002, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ErasureAlgorithm") + return + } + z.ErasureAlgorithm = ErasureAlgo(zb0002) + } + case "EcM": + z.ErasureM, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ErasureM") + return + } + case "EcN": + z.ErasureN, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ErasureN") + return + } + case "EcBSize": + z.ErasureBlockSize, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ErasureBlockSize") + return + } + case "EcIndex": + z.ErasureIndex, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ErasureIndex") + return + } + case "EcDist": + var zb0003 uint32 + zb0003, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "ErasureDist") + return + } + if cap(z.ErasureDist) >= int(zb0003) { + z.ErasureDist = (z.ErasureDist)[:zb0003] + } else { + z.ErasureDist = make([]uint8, zb0003) + } + for za0003 := range z.ErasureDist { + z.ErasureDist[za0003], bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ErasureDist", za0003) + return + } + } + case "CSumAlgo": + { + var zb0004 uint8 + zb0004, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "BitrotChecksumAlgo") + return + } + z.BitrotChecksumAlgo = ChecksumAlgo(zb0004) + } + case "PartNums": + var zb0005 uint32 + zb0005, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "PartNumbers") + return + } + if cap(z.PartNumbers) >= int(zb0005) { + z.PartNumbers = (z.PartNumbers)[:zb0005] + } else { + z.PartNumbers = make([]int, zb0005) + } + for za0004 := range z.PartNumbers { + z.PartNumbers[za0004], bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "PartNumbers", za0004) + return + } + } + case "PartETags": + if msgp.IsNil(bts) { + bts = bts[1:] + z.PartETags = nil + } else { + var zb0006 uint32 + zb0006, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "PartETags") + return + } + if z.PartETags != nil && cap(z.PartETags) >= int(zb0006) { + z.PartETags = (z.PartETags)[:zb0006] + } else { + z.PartETags = make([]string, zb0006) + } + for za0005 := range z.PartETags { + z.PartETags[za0005], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "PartETags", za0005) + return + } + } + } + case "PartSizes": + var zb0007 uint32 + zb0007, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "PartSizes") + return + } + if cap(z.PartSizes) >= int(zb0007) { + z.PartSizes = (z.PartSizes)[:zb0007] + } else { + z.PartSizes = make([]int64, zb0007) + } + for za0006 := range z.PartSizes { + z.PartSizes[za0006], bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "PartSizes", za0006) + return + } + } + case "PartASizes": + if msgp.IsNil(bts) { + bts = bts[1:] + z.PartActualSizes = nil + } else { + var zb0008 uint32 + zb0008, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "PartActualSizes") + return + } + if z.PartActualSizes != nil && cap(z.PartActualSizes) >= int(zb0008) { + z.PartActualSizes = (z.PartActualSizes)[:zb0008] + } else { + z.PartActualSizes = make([]int64, zb0008) + } + for za0007 := range z.PartActualSizes { + z.PartActualSizes[za0007], bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "PartActualSizes", za0007) + return + } + } + } + case "PartIdx": + var zb0009 uint32 + zb0009, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "PartIndices") + return + } + if cap(z.PartIndices) >= int(zb0009) { + z.PartIndices = (z.PartIndices)[:zb0009] + } else { + z.PartIndices = make([][]byte, zb0009) + } + for za0008 := range z.PartIndices { + z.PartIndices[za0008], bts, err = msgp.ReadBytesBytes(bts, z.PartIndices[za0008]) + if err != nil { + err = msgp.WrapError(err, "PartIndices", za0008) + return + } + } + zb0001Mask |= 0x1 + case "Size": + z.Size, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Size") + return + } + case "MTime": + z.ModTime, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + case "MetaSys": + if msgp.IsNil(bts) { + bts = bts[1:] + z.MetaSys = nil + } else { + var zb0010 uint32 + zb0010, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "MetaSys") + return + } + if z.MetaSys == nil { + z.MetaSys = make(map[string][]byte, zb0010) + } else if len(z.MetaSys) > 0 { + clear(z.MetaSys) + } + for zb0010 > 0 { + var za0010 []byte + zb0010-- + var za0009 string + za0009, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "MetaSys") + return + } + za0010, bts, err = msgp.ReadBytesBytes(bts, za0010) + if err != nil { + err = msgp.WrapError(err, "MetaSys", za0009) + return + } + z.MetaSys[za0009] = za0010 + } + } + case "MetaUsr": + if msgp.IsNil(bts) { + bts = bts[1:] + z.MetaUser = nil + } else { + var zb0011 uint32 + zb0011, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "MetaUser") + return + } + if z.MetaUser == nil { + z.MetaUser = make(map[string]string, zb0011) + } else if len(z.MetaUser) > 0 { + clear(z.MetaUser) + } + for zb0011 > 0 { + var za0012 string + zb0011-- + var za0011 string + za0011, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "MetaUser") + return + } + za0012, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "MetaUser", za0011) + return + } + z.MetaUser[za0011] = za0012 + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if (zb0001Mask & 0x1) == 0 { + z.PartIndices = nil + } + + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *xlMetaV2Object) Msgsize() (s int) { + s = 3 + 3 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + 5 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + 7 + msgp.Uint8Size + 4 + msgp.IntSize + 4 + msgp.IntSize + 8 + msgp.Int64Size + 8 + msgp.IntSize + 7 + msgp.ArrayHeaderSize + (len(z.ErasureDist) * (msgp.Uint8Size)) + 9 + msgp.Uint8Size + 9 + msgp.ArrayHeaderSize + (len(z.PartNumbers) * (msgp.IntSize)) + 10 + msgp.ArrayHeaderSize + for za0005 := range z.PartETags { + s += msgp.StringPrefixSize + len(z.PartETags[za0005]) + } + s += 10 + msgp.ArrayHeaderSize + (len(z.PartSizes) * (msgp.Int64Size)) + 11 + msgp.ArrayHeaderSize + (len(z.PartActualSizes) * (msgp.Int64Size)) + 8 + msgp.ArrayHeaderSize + for za0008 := range z.PartIndices { + s += msgp.BytesPrefixSize + len(z.PartIndices[za0008]) + } + s += 5 + msgp.Int64Size + 6 + msgp.Int64Size + 8 + msgp.MapHeaderSize + if z.MetaSys != nil { + for za0009, za0010 := range z.MetaSys { + _ = za0010 + s += msgp.StringPrefixSize + len(za0009) + msgp.BytesPrefixSize + len(za0010) + } + } + s += 8 + msgp.MapHeaderSize + if z.MetaUser != nil { + for za0011, za0012 := range z.MetaUser { + _ = za0012 + s += msgp.StringPrefixSize + len(za0011) + msgp.StringPrefixSize + len(za0012) + } + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *xlMetaV2Version) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 3 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Type": + { + var zb0002 uint8 + zb0002, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + z.Type = VersionType(zb0002) + } + case "V1Obj": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "ObjectV1") + return + } + z.ObjectV1 = nil + } else { + if z.ObjectV1 == nil { + z.ObjectV1 = new(xlMetaV1Object) + } + err = z.ObjectV1.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "ObjectV1") + return + } + } + zb0001Mask |= 0x1 + case "V2Obj": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "ObjectV2") + return + } + z.ObjectV2 = nil + } else { + if z.ObjectV2 == nil { + z.ObjectV2 = new(xlMetaV2Object) + } + err = z.ObjectV2.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "ObjectV2") + return + } + } + zb0001Mask |= 0x2 + case "DelObj": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "DeleteMarker") + return + } + z.DeleteMarker = nil + } else { + if z.DeleteMarker == nil { + z.DeleteMarker = new(xlMetaV2DeleteMarker) + } + err = z.DeleteMarker.DecodeMsg(dc) + if err != nil { + err = msgp.WrapError(err, "DeleteMarker") + return + } + } + zb0001Mask |= 0x4 + case "v": + z.WrittenByVersion, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "WrittenByVersion") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if zb0001Mask != 0x7 { + if (zb0001Mask & 0x1) == 0 { + z.ObjectV1 = nil + } + if (zb0001Mask & 0x2) == 0 { + z.ObjectV2 = nil + } + if (zb0001Mask & 0x4) == 0 { + z.DeleteMarker = nil + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *xlMetaV2Version) EncodeMsg(en *msgp.Writer) (err error) { + // check for omitted fields + zb0001Len := uint32(5) + var zb0001Mask uint8 /* 5 bits */ + _ = zb0001Mask + if z.ObjectV1 == nil { + zb0001Len-- + zb0001Mask |= 0x2 + } + if z.ObjectV2 == nil { + zb0001Len-- + zb0001Mask |= 0x4 + } + if z.DeleteMarker == nil { + zb0001Len-- + zb0001Mask |= 0x8 + } + // variable map header, size zb0001Len + err = en.Append(0x80 | uint8(zb0001Len)) + if err != nil { + return + } + + // skip if no fields are to be emitted + if zb0001Len != 0 { + // write "Type" + err = en.Append(0xa4, 0x54, 0x79, 0x70, 0x65) + if err != nil { + return + } + err = en.WriteUint8(uint8(z.Type)) + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + if (zb0001Mask & 0x2) == 0 { // if not omitted + // write "V1Obj" + err = en.Append(0xa5, 0x56, 0x31, 0x4f, 0x62, 0x6a) + if err != nil { + return + } + if z.ObjectV1 == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.ObjectV1.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "ObjectV1") + return + } + } + } + if (zb0001Mask & 0x4) == 0 { // if not omitted + // write "V2Obj" + err = en.Append(0xa5, 0x56, 0x32, 0x4f, 0x62, 0x6a) + if err != nil { + return + } + if z.ObjectV2 == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.ObjectV2.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "ObjectV2") + return + } + } + } + if (zb0001Mask & 0x8) == 0 { // if not omitted + // write "DelObj" + err = en.Append(0xa6, 0x44, 0x65, 0x6c, 0x4f, 0x62, 0x6a) + if err != nil { + return + } + if z.DeleteMarker == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = z.DeleteMarker.EncodeMsg(en) + if err != nil { + err = msgp.WrapError(err, "DeleteMarker") + return + } + } + } + // write "v" + err = en.Append(0xa1, 0x76) + if err != nil { + return + } + err = en.WriteUint64(z.WrittenByVersion) + if err != nil { + err = msgp.WrapError(err, "WrittenByVersion") + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *xlMetaV2Version) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // check for omitted fields + zb0001Len := uint32(5) + var zb0001Mask uint8 /* 5 bits */ + _ = zb0001Mask + if z.ObjectV1 == nil { + zb0001Len-- + zb0001Mask |= 0x2 + } + if z.ObjectV2 == nil { + zb0001Len-- + zb0001Mask |= 0x4 + } + if z.DeleteMarker == nil { + zb0001Len-- + zb0001Mask |= 0x8 + } + // variable map header, size zb0001Len + o = append(o, 0x80|uint8(zb0001Len)) + + // skip if no fields are to be emitted + if zb0001Len != 0 { + // string "Type" + o = append(o, 0xa4, 0x54, 0x79, 0x70, 0x65) + o = msgp.AppendUint8(o, uint8(z.Type)) + if (zb0001Mask & 0x2) == 0 { // if not omitted + // string "V1Obj" + o = append(o, 0xa5, 0x56, 0x31, 0x4f, 0x62, 0x6a) + if z.ObjectV1 == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.ObjectV1.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "ObjectV1") + return + } + } + } + if (zb0001Mask & 0x4) == 0 { // if not omitted + // string "V2Obj" + o = append(o, 0xa5, 0x56, 0x32, 0x4f, 0x62, 0x6a) + if z.ObjectV2 == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.ObjectV2.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "ObjectV2") + return + } + } + } + if (zb0001Mask & 0x8) == 0 { // if not omitted + // string "DelObj" + o = append(o, 0xa6, 0x44, 0x65, 0x6c, 0x4f, 0x62, 0x6a) + if z.DeleteMarker == nil { + o = msgp.AppendNil(o) + } else { + o, err = z.DeleteMarker.MarshalMsg(o) + if err != nil { + err = msgp.WrapError(err, "DeleteMarker") + return + } + } + } + // string "v" + o = append(o, 0xa1, 0x76) + o = msgp.AppendUint64(o, z.WrittenByVersion) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *xlMetaV2Version) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + var zb0001Mask uint8 /* 3 bits */ + _ = zb0001Mask + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Type": + { + var zb0002 uint8 + zb0002, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + z.Type = VersionType(zb0002) + } + case "V1Obj": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.ObjectV1 = nil + } else { + if z.ObjectV1 == nil { + z.ObjectV1 = new(xlMetaV1Object) + } + bts, err = z.ObjectV1.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ObjectV1") + return + } + } + zb0001Mask |= 0x1 + case "V2Obj": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.ObjectV2 = nil + } else { + if z.ObjectV2 == nil { + z.ObjectV2 = new(xlMetaV2Object) + } + bts, err = z.ObjectV2.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "ObjectV2") + return + } + } + zb0001Mask |= 0x2 + case "DelObj": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.DeleteMarker = nil + } else { + if z.DeleteMarker == nil { + z.DeleteMarker = new(xlMetaV2DeleteMarker) + } + bts, err = z.DeleteMarker.UnmarshalMsg(bts) + if err != nil { + err = msgp.WrapError(err, "DeleteMarker") + return + } + } + zb0001Mask |= 0x4 + case "v": + z.WrittenByVersion, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "WrittenByVersion") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + // Clear omitted fields. + if zb0001Mask != 0x7 { + if (zb0001Mask & 0x1) == 0 { + z.ObjectV1 = nil + } + if (zb0001Mask & 0x2) == 0 { + z.ObjectV2 = nil + } + if (zb0001Mask & 0x4) == 0 { + z.DeleteMarker = nil + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *xlMetaV2Version) Msgsize() (s int) { + s = 1 + 5 + msgp.Uint8Size + 6 + if z.ObjectV1 == nil { + s += msgp.NilSize + } else { + s += z.ObjectV1.Msgsize() + } + s += 6 + if z.ObjectV2 == nil { + s += msgp.NilSize + } else { + s += z.ObjectV2.Msgsize() + } + s += 7 + if z.DeleteMarker == nil { + s += msgp.NilSize + } else { + s += z.DeleteMarker.Msgsize() + } + s += 2 + msgp.Uint64Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *xlMetaV2VersionHeader) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0001 uint32 + zb0001, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 7 { + err = msgp.ArrayError{Wanted: 7, Got: zb0001} + return + } + err = dc.ReadExactBytes((z.VersionID)[:]) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + z.ModTime, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + err = dc.ReadExactBytes((z.Signature)[:]) + if err != nil { + err = msgp.WrapError(err, "Signature") + return + } + { + var zb0002 uint8 + zb0002, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + z.Type = VersionType(zb0002) + } + { + var zb0003 uint8 + zb0003, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + z.Flags = xlFlags(zb0003) + } + z.EcN, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err, "EcN") + return + } + z.EcM, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err, "EcM") + return + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *xlMetaV2VersionHeader) EncodeMsg(en *msgp.Writer) (err error) { + // array header, size 7 + err = en.Append(0x97) + if err != nil { + return + } + err = en.WriteBytes((z.VersionID)[:]) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + err = en.WriteInt64(z.ModTime) + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + err = en.WriteBytes((z.Signature)[:]) + if err != nil { + err = msgp.WrapError(err, "Signature") + return + } + err = en.WriteUint8(uint8(z.Type)) + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + err = en.WriteUint8(uint8(z.Flags)) + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + err = en.WriteUint8(z.EcN) + if err != nil { + err = msgp.WrapError(err, "EcN") + return + } + err = en.WriteUint8(z.EcM) + if err != nil { + err = msgp.WrapError(err, "EcM") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *xlMetaV2VersionHeader) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // array header, size 7 + o = append(o, 0x97) + o = msgp.AppendBytes(o, (z.VersionID)[:]) + o = msgp.AppendInt64(o, z.ModTime) + o = msgp.AppendBytes(o, (z.Signature)[:]) + o = msgp.AppendUint8(o, uint8(z.Type)) + o = msgp.AppendUint8(o, uint8(z.Flags)) + o = msgp.AppendUint8(o, z.EcN) + o = msgp.AppendUint8(o, z.EcM) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *xlMetaV2VersionHeader) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0001 uint32 + zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 7 { + err = msgp.ArrayError{Wanted: 7, Got: zb0001} + return + } + bts, err = msgp.ReadExactBytes(bts, (z.VersionID)[:]) + if err != nil { + err = msgp.WrapError(err, "VersionID") + return + } + z.ModTime, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "ModTime") + return + } + bts, err = msgp.ReadExactBytes(bts, (z.Signature)[:]) + if err != nil { + err = msgp.WrapError(err, "Signature") + return + } + { + var zb0002 uint8 + zb0002, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Type") + return + } + z.Type = VersionType(zb0002) + } + { + var zb0003 uint8 + zb0003, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + z.Flags = xlFlags(zb0003) + } + z.EcN, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "EcN") + return + } + z.EcM, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "EcM") + return + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *xlMetaV2VersionHeader) Msgsize() (s int) { + s = 1 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + msgp.Int64Size + msgp.ArrayHeaderSize + (4 * (msgp.ByteSize)) + msgp.Uint8Size + msgp.Uint8Size + msgp.Uint8Size + msgp.Uint8Size + return +} diff --git a/cmd/xl-storage-format-v2_gen_test.go b/cmd/xl-storage-format-v2_gen_test.go new file mode 100644 index 00000000..fa7a6670 --- /dev/null +++ b/cmd/xl-storage-format-v2_gen_test.go @@ -0,0 +1,575 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package cmd + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalxlMetaDataDirDecoder(t *testing.T) { + v := xlMetaDataDirDecoder{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgxlMetaDataDirDecoder(b *testing.B) { + v := xlMetaDataDirDecoder{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgxlMetaDataDirDecoder(b *testing.B) { + v := xlMetaDataDirDecoder{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalxlMetaDataDirDecoder(b *testing.B) { + v := xlMetaDataDirDecoder{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodexlMetaDataDirDecoder(t *testing.T) { + v := xlMetaDataDirDecoder{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodexlMetaDataDirDecoder Msgsize() is inaccurate") + } + + vn := xlMetaDataDirDecoder{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodexlMetaDataDirDecoder(b *testing.B) { + v := xlMetaDataDirDecoder{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodexlMetaDataDirDecoder(b *testing.B) { + v := xlMetaDataDirDecoder{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalxlMetaV2DeleteMarker(t *testing.T) { + v := xlMetaV2DeleteMarker{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgxlMetaV2DeleteMarker(b *testing.B) { + v := xlMetaV2DeleteMarker{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgxlMetaV2DeleteMarker(b *testing.B) { + v := xlMetaV2DeleteMarker{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalxlMetaV2DeleteMarker(b *testing.B) { + v := xlMetaV2DeleteMarker{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodexlMetaV2DeleteMarker(t *testing.T) { + v := xlMetaV2DeleteMarker{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodexlMetaV2DeleteMarker Msgsize() is inaccurate") + } + + vn := xlMetaV2DeleteMarker{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodexlMetaV2DeleteMarker(b *testing.B) { + v := xlMetaV2DeleteMarker{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodexlMetaV2DeleteMarker(b *testing.B) { + v := xlMetaV2DeleteMarker{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalxlMetaV2Object(t *testing.T) { + v := xlMetaV2Object{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgxlMetaV2Object(b *testing.B) { + v := xlMetaV2Object{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgxlMetaV2Object(b *testing.B) { + v := xlMetaV2Object{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalxlMetaV2Object(b *testing.B) { + v := xlMetaV2Object{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodexlMetaV2Object(t *testing.T) { + v := xlMetaV2Object{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodexlMetaV2Object Msgsize() is inaccurate") + } + + vn := xlMetaV2Object{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodexlMetaV2Object(b *testing.B) { + v := xlMetaV2Object{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodexlMetaV2Object(b *testing.B) { + v := xlMetaV2Object{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalxlMetaV2Version(t *testing.T) { + v := xlMetaV2Version{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgxlMetaV2Version(b *testing.B) { + v := xlMetaV2Version{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgxlMetaV2Version(b *testing.B) { + v := xlMetaV2Version{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalxlMetaV2Version(b *testing.B) { + v := xlMetaV2Version{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodexlMetaV2Version(t *testing.T) { + v := xlMetaV2Version{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodexlMetaV2Version Msgsize() is inaccurate") + } + + vn := xlMetaV2Version{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodexlMetaV2Version(b *testing.B) { + v := xlMetaV2Version{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodexlMetaV2Version(b *testing.B) { + v := xlMetaV2Version{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalxlMetaV2VersionHeader(t *testing.T) { + v := xlMetaV2VersionHeader{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgxlMetaV2VersionHeader(b *testing.B) { + v := xlMetaV2VersionHeader{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgxlMetaV2VersionHeader(b *testing.B) { + v := xlMetaV2VersionHeader{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalxlMetaV2VersionHeader(b *testing.B) { + v := xlMetaV2VersionHeader{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodexlMetaV2VersionHeader(t *testing.T) { + v := xlMetaV2VersionHeader{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodexlMetaV2VersionHeader Msgsize() is inaccurate") + } + + vn := xlMetaV2VersionHeader{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodexlMetaV2VersionHeader(b *testing.B) { + v := xlMetaV2VersionHeader{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodexlMetaV2VersionHeader(b *testing.B) { + v := xlMetaV2VersionHeader{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/cmd/xl-storage-format-v2_string.go b/cmd/xl-storage-format-v2_string.go new file mode 100644 index 00000000..61959f85 --- /dev/null +++ b/cmd/xl-storage-format-v2_string.go @@ -0,0 +1,46 @@ +// Code generated by "stringer -type VersionType,ErasureAlgo -output=xl-storage-format-v2_string.go xl-storage-format-v2.go"; DO NOT EDIT. + +package cmd + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[invalidVersionType-0] + _ = x[ObjectType-1] + _ = x[DeleteType-2] + _ = x[LegacyType-3] + _ = x[lastVersionType-4] +} + +const _VersionType_name = "invalidVersionTypeObjectTypeDeleteTypeLegacyTypelastVersionType" + +var _VersionType_index = [...]uint8{0, 18, 28, 38, 48, 63} + +func (i VersionType) String() string { + if i >= VersionType(len(_VersionType_index)-1) { + return "VersionType(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _VersionType_name[_VersionType_index[i]:_VersionType_index[i+1]] +} +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[invalidErasureAlgo-0] + _ = x[ReedSolomon-1] + _ = x[lastErasureAlgo-2] +} + +const _ErasureAlgo_name = "invalidErasureAlgoReedSolomonlastErasureAlgo" + +var _ErasureAlgo_index = [...]uint8{0, 18, 29, 44} + +func (i ErasureAlgo) String() string { + if i >= ErasureAlgo(len(_ErasureAlgo_index)-1) { + return "ErasureAlgo(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _ErasureAlgo_name[_ErasureAlgo_index[i]:_ErasureAlgo_index[i+1]] +} diff --git a/cmd/xl-storage-format-v2_test.go b/cmd/xl-storage-format-v2_test.go new file mode 100644 index 00000000..6a9b15a3 --- /dev/null +++ b/cmd/xl-storage-format-v2_test.go @@ -0,0 +1,1192 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bufio" + "bytes" + "compress/gzip" + "encoding/base64" + "encoding/binary" + "encoding/json" + "fmt" + "io" + "math/rand" + "os" + "reflect" + "sort" + "testing" + "time" + + "github.com/google/uuid" + "github.com/klauspost/compress/zip" + "github.com/klauspost/compress/zstd" + "github.com/minio/minio/internal/bucket/lifecycle" + xhttp "github.com/minio/minio/internal/http" +) + +func TestReadXLMetaNoData(t *testing.T) { + f, err := os.Open("testdata/xl.meta-corrupt.gz") + if err != nil { + t.Fatal(err) + } + defer f.Close() + + gz, err := gzip.NewReader(bufio.NewReader(f)) + if err != nil { + t.Fatal(err) + } + + buf, err := io.ReadAll(gz) + if err != nil { + t.Fatal(err) + } + + _, err = readXLMetaNoData(bytes.NewReader(buf), int64(len(buf))) + if err == nil { + t.Fatal("expected error but returned success") + } +} + +func TestXLV2FormatData(t *testing.T) { + failOnErr := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + data := []byte("some object data") + data2 := []byte("some other object data") + + xl := xlMetaV2{} + fi := FileInfo{ + Volume: "volume", + Name: "object-name", + VersionID: "756100c6-b393-4981-928a-d49bbc164741", + IsLatest: true, + Deleted: false, + TransitionStatus: "", + DataDir: "bffea160-ca7f-465f-98bc-9b4f1c3ba1ef", + XLV1: false, + ModTime: time.Now(), + Size: 0, + Mode: 0, + Metadata: nil, + Parts: nil, + Erasure: ErasureInfo{ + Algorithm: ReedSolomon.String(), + DataBlocks: 4, + ParityBlocks: 2, + BlockSize: 10000, + Index: 1, + Distribution: []int{1, 2, 3, 4, 5, 6, 7, 8}, + Checksums: []ChecksumInfo{{ + PartNumber: 1, + Algorithm: HighwayHash256S, + Hash: nil, + }}, + }, + MarkDeleted: false, + Data: data, + NumVersions: 1, + SuccessorModTime: time.Time{}, + } + + failOnErr(xl.AddVersion(fi)) + + fi.VersionID = mustGetUUID() + fi.DataDir = mustGetUUID() + fi.Data = data2 + failOnErr(xl.AddVersion(fi)) + + serialized, err := xl.AppendTo(nil) + failOnErr(err) + // Roundtrip data + var xl2 xlMetaV2 + failOnErr(xl2.Load(serialized)) + + // We should have one data entry + list, err := xl2.data.list() + failOnErr(err) + if len(list) != 2 { + t.Fatalf("want 1 entry, got %d", len(list)) + } + + if !bytes.Equal(xl2.data.find("756100c6-b393-4981-928a-d49bbc164741"), data) { + t.Fatal("Find data returned", xl2.data.find("756100c6-b393-4981-928a-d49bbc164741")) + } + if !bytes.Equal(xl2.data.find(fi.VersionID), data2) { + t.Fatal("Find data returned", xl2.data.find(fi.VersionID)) + } + + // Remove entry + xl2.data.remove(fi.VersionID) + failOnErr(xl2.data.validate()) + if xl2.data.find(fi.VersionID) != nil { + t.Fatal("Data was not removed:", xl2.data.find(fi.VersionID)) + } + if xl2.data.entries() != 1 { + t.Fatal("want 1 entry, got", xl2.data.entries()) + } + // Re-add + xl2.data.replace(fi.VersionID, fi.Data) + failOnErr(xl2.data.validate()) + if xl2.data.entries() != 2 { + t.Fatal("want 2 entries, got", xl2.data.entries()) + } + + // Replace entry + xl2.data.replace("756100c6-b393-4981-928a-d49bbc164741", data2) + failOnErr(xl2.data.validate()) + if xl2.data.entries() != 2 { + t.Fatal("want 2 entries, got", xl2.data.entries()) + } + if !bytes.Equal(xl2.data.find("756100c6-b393-4981-928a-d49bbc164741"), data2) { + t.Fatal("Find data returned", xl2.data.find("756100c6-b393-4981-928a-d49bbc164741")) + } + + if !xl2.data.rename("756100c6-b393-4981-928a-d49bbc164741", "new-key") { + t.Fatal("old key was not found") + } + failOnErr(xl2.data.validate()) + if !bytes.Equal(xl2.data.find("new-key"), data2) { + t.Fatal("Find data returned", xl2.data.find("756100c6-b393-4981-928a-d49bbc164741")) + } + if xl2.data.entries() != 2 { + t.Fatal("want 2 entries, got", xl2.data.entries()) + } + if !bytes.Equal(xl2.data.find(fi.VersionID), data2) { + t.Fatal("Find data returned", xl2.data.find(fi.DataDir)) + } + + // Test trimmed + xl2 = xlMetaV2{} + trimmed := xlMetaV2TrimData(serialized) + failOnErr(xl2.Load(trimmed)) + if len(xl2.data) != 0 { + t.Fatal("data, was not trimmed, bytes left:", len(xl2.data)) + } + // Corrupt metadata, last 5 bytes is the checksum, so go a bit further back. + trimmed[len(trimmed)-10] += 10 + if err := xl2.Load(trimmed); err == nil { + t.Fatal("metadata corruption not detected") + } +} + +// TestUsesDataDir tests xlMetaV2.UsesDataDir +func TestUsesDataDir(t *testing.T) { + vID := uuid.New() + dataDir := uuid.New() + transitioned := make(map[string][]byte) + transitioned[ReservedMetadataPrefixLower+TransitionStatus] = []byte(lifecycle.TransitionComplete) + + toBeRestored := make(map[string]string) + toBeRestored[xhttp.AmzRestore] = ongoingRestoreObj().String() + + restored := make(map[string]string) + restored[xhttp.AmzRestore] = completedRestoreObj(time.Now().UTC().Add(time.Hour)).String() + + restoredExpired := make(map[string]string) + restoredExpired[xhttp.AmzRestore] = completedRestoreObj(time.Now().UTC().Add(-time.Hour)).String() + + testCases := []struct { + xlmeta xlMetaV2Object + uses bool + }{ + { // transitioned object version + xlmeta: xlMetaV2Object{ + VersionID: vID, + DataDir: dataDir, + MetaSys: transitioned, + }, + uses: false, + }, + { // to be restored (requires object version to be transitioned) + xlmeta: xlMetaV2Object{ + VersionID: vID, + DataDir: dataDir, + MetaSys: transitioned, + MetaUser: toBeRestored, + }, + uses: false, + }, + { // restored object version (requires object version to be transitioned) + xlmeta: xlMetaV2Object{ + VersionID: vID, + DataDir: dataDir, + MetaSys: transitioned, + MetaUser: restored, + }, + uses: true, + }, + { // restored object version expired an hour back (requires object version to be transitioned) + xlmeta: xlMetaV2Object{ + VersionID: vID, + DataDir: dataDir, + MetaSys: transitioned, + MetaUser: restoredExpired, + }, + uses: false, + }, + { // object version with no ILM applied + xlmeta: xlMetaV2Object{ + VersionID: vID, + DataDir: dataDir, + }, + uses: true, + }, + } + for i, tc := range testCases { + if got := tc.xlmeta.UsesDataDir(); got != tc.uses { + t.Fatalf("Test %d: Expected %v but got %v for %v", i+1, tc.uses, got, tc.xlmeta) + } + } +} + +func TestDeleteVersionWithSharedDataDir(t *testing.T) { + failOnErr := func(i int, err error) { + t.Helper() + if err != nil { + t.Fatalf("Test %d: failed with %v", i, err) + } + } + + data := []byte("some object data") + data2 := []byte("some other object data") + + xl := xlMetaV2{} + fi := FileInfo{ + Volume: "volume", + Name: "object-name", + VersionID: "756100c6-b393-4981-928a-d49bbc164741", + IsLatest: true, + Deleted: false, + TransitionStatus: "", + DataDir: "bffea160-ca7f-465f-98bc-9b4f1c3ba1ef", + XLV1: false, + ModTime: time.Now(), + Size: 0, + Mode: 0, + Metadata: nil, + Parts: nil, + Erasure: ErasureInfo{ + Algorithm: ReedSolomon.String(), + DataBlocks: 4, + ParityBlocks: 2, + BlockSize: 10000, + Index: 1, + Distribution: []int{1, 2, 3, 4, 5, 6, 7, 8}, + Checksums: []ChecksumInfo{{ + PartNumber: 1, + Algorithm: HighwayHash256S, + Hash: nil, + }}, + }, + MarkDeleted: false, + Data: data, + NumVersions: 1, + SuccessorModTime: time.Time{}, + } + + d0, d1, d2 := mustGetUUID(), mustGetUUID(), mustGetUUID() + testCases := []struct { + versionID string + dataDir string + data []byte + shares int + transitionStatus string + restoreObjStatus string + expireRestored bool + expectedDataDir string + }{ + { // object versions with inlined data don't count towards shared data directory + versionID: mustGetUUID(), + dataDir: d0, + data: data, + shares: 0, + }, + { // object versions with inlined data don't count towards shared data directory + versionID: mustGetUUID(), + dataDir: d1, + data: data2, + shares: 0, + }, + { // transitioned object version don't count towards shared data directory + versionID: mustGetUUID(), + dataDir: d2, + shares: 3, + transitionStatus: lifecycle.TransitionComplete, + }, + { // transitioned object version with an ongoing restore-object request. + versionID: mustGetUUID(), + dataDir: d2, + shares: 3, + transitionStatus: lifecycle.TransitionComplete, + restoreObjStatus: ongoingRestoreObj().String(), + }, + // The following versions are on-disk. + { // restored object version expiring 10 hours from now. + versionID: mustGetUUID(), + dataDir: d2, + shares: 2, + transitionStatus: lifecycle.TransitionComplete, + restoreObjStatus: completedRestoreObj(time.Now().Add(10 * time.Hour)).String(), + expireRestored: true, + }, + { + versionID: mustGetUUID(), + dataDir: d2, + shares: 2, + }, + { + versionID: mustGetUUID(), + dataDir: d2, + shares: 2, + expectedDataDir: d2, + }, + } + + var fileInfos []FileInfo + for i, tc := range testCases { + fi := fi + fi.VersionID = tc.versionID + fi.DataDir = tc.dataDir + fi.Data = tc.data + if tc.data == nil { + fi.Size = 42 // to prevent inlining of data + } + if tc.restoreObjStatus != "" { + fi.Metadata = map[string]string{ + xhttp.AmzRestore: tc.restoreObjStatus, + } + } + fi.TransitionStatus = tc.transitionStatus + fi.ModTime = fi.ModTime.Add(time.Duration(i) * time.Second) + failOnErr(i+1, xl.AddVersion(fi)) + fi.ExpireRestored = tc.expireRestored + fileInfos = append(fileInfos, fi) + } + + for i, tc := range testCases { + _, version, err := xl.findVersion(uuid.MustParse(tc.versionID)) + failOnErr(i+1, err) + if got := xl.SharedDataDirCount(version.getVersionID(), version.ObjectV2.DataDir); got != tc.shares { + t.Fatalf("Test %d: For %#v, expected sharers of data directory %d got %d", i+1, version.ObjectV2.VersionID, tc.shares, got) + } + } + + // Deleting fileInfos[4].VersionID, fileInfos[5].VersionID should return empty data dir; there are other object version sharing the data dir. + // Subsequently deleting fileInfos[6].versionID should return fileInfos[6].dataDir since there are no other object versions sharing this data dir. + count := len(testCases) + for i := 4; i < len(testCases); i++ { + tc := testCases[i] + dataDir, err := xl.DeleteVersion(fileInfos[i]) + failOnErr(count+1, err) + if dataDir != tc.expectedDataDir { + t.Fatalf("Expected %s but got %s", tc.expectedDataDir, dataDir) + } + count++ + } +} + +func Benchmark_mergeXLV2Versions(b *testing.B) { + data, err := os.ReadFile("testdata/xl.meta-v1.2.zst") + if err != nil { + b.Fatal(err) + } + dec, _ := zstd.NewReader(nil) + data, err = dec.DecodeAll(data, nil) + if err != nil { + b.Fatal(err) + } + + var xl xlMetaV2 + if err = xl.LoadOrConvert(data); err != nil { + b.Fatal(err) + } + + vers := make([][]xlMetaV2ShallowVersion, 16) + for i := range vers { + vers[i] = xl.versions + } + + b.Run("requested-none", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + b.SetBytes(855) // number of versions... + for b.Loop() { + mergeXLV2Versions(8, false, 0, vers...) + } + }) + + b.Run("requested-v1", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + b.SetBytes(855) // number of versions... + for b.Loop() { + mergeXLV2Versions(8, false, 1, vers...) + } + }) + + b.Run("requested-v2", func(b *testing.B) { + b.ReportAllocs() + b.ResetTimer() + b.SetBytes(855) // number of versions... + for b.Loop() { + mergeXLV2Versions(8, false, 1, vers...) + } + }) +} + +func Benchmark_xlMetaV2Shallow_Load(b *testing.B) { + data, err := os.ReadFile("testdata/xl.meta-v1.2.zst") + if err != nil { + b.Fatal(err) + } + dec, _ := zstd.NewReader(nil) + data, err = dec.DecodeAll(data, nil) + if err != nil { + b.Fatal(err) + } + + b.Run("legacy", func(b *testing.B) { + var xl xlMetaV2 + b.ReportAllocs() + b.ResetTimer() + b.SetBytes(855) // number of versions... + for b.Loop() { + err = xl.Load(data) + if err != nil { + b.Fatal(err) + } + } + }) + + b.Run("indexed", func(b *testing.B) { + var xl xlMetaV2 + err = xl.Load(data) + if err != nil { + b.Fatal(err) + } + data, err := xl.AppendTo(nil) + if err != nil { + b.Fatal(err) + } + b.ReportAllocs() + b.ResetTimer() + b.SetBytes(855) // number of versions... + for b.Loop() { + err = xl.Load(data) + if err != nil { + b.Fatal(err) + } + } + }) +} + +func Test_xlMetaV2Shallow_Load(t *testing.T) { + // Load Legacy + data, err := os.ReadFile("testdata/xl.meta-v1.2.zst") + if err != nil { + t.Fatal(err) + } + dec, _ := zstd.NewReader(nil) + data, err = dec.DecodeAll(data, nil) + if err != nil { + t.Fatal(err) + } + test := func(t *testing.T, xl *xlMetaV2) { + if len(xl.versions) != 855 { + t.Errorf("want %d versions, got %d", 855, len(xl.versions)) + } + xl.sortByModTime() + if !sort.SliceIsSorted(xl.versions, func(i, j int) bool { + return xl.versions[i].header.ModTime > xl.versions[j].header.ModTime + }) { + t.Errorf("Contents not sorted") + } + for i := range xl.versions { + hdr := xl.versions[i].header + ver, err := xl.getIdx(i) + if err != nil { + t.Error(err) + continue + } + gotHdr := ver.header() + if hdr != gotHdr { + t.Errorf("Header does not match, index: %+v != meta: %+v", hdr, gotHdr) + } + } + } + t.Run("load-legacy", func(t *testing.T) { + var xl xlMetaV2 + err = xl.Load(data) + if err != nil { + t.Fatal(err) + } + test(t, &xl) + }) + t.Run("roundtrip", func(t *testing.T) { + var xl xlMetaV2 + err = xl.Load(data) + if err != nil { + t.Fatal(err) + } + data, err = xl.AppendTo(nil) + if err != nil { + t.Fatal(err) + } + xl = xlMetaV2{} + err = xl.Load(data) + if err != nil { + t.Fatal(err) + } + test(t, &xl) + }) + t.Run("write-timestamp", func(t *testing.T) { + var xl xlMetaV2 + err = xl.Load(data) + if err != nil { + t.Fatal(err) + } + ventry := xlMetaV2Version{ + Type: DeleteType, + DeleteMarker: &xlMetaV2DeleteMarker{ + VersionID: uuid.New(), + ModTime: time.Now().UnixNano(), + MetaSys: map[string][]byte{ReservedMetadataPrefixLower + ReplicationTimestamp: []byte("2022-10-27T15:40:53.195813291+08:00"), ReservedMetadataPrefixLower + ReplicaTimestamp: []byte("2022-10-27T15:40:53.195813291+08:00")}, + }, + WrittenByVersion: globalVersionUnix, + } + xl.data = nil + xl.versions = xl.versions[:2] + xl.addVersion(ventry) + + data, err = xl.AppendTo(nil) + if err != nil { + t.Fatal(err) + } + // t.Logf("data := %#v\n", data) + }) + // Test compressed index consistency fix + t.Run("comp-index", func(t *testing.T) { + // This file has a compressed index, due to https://github.com/minio/minio/pull/20575 + // We ensure it is rewritten without an index. + // We compare this against the signature of the files stored without a version. + data, err := base64.StdEncoding.DecodeString(`WEwyIAEAAwDGAAACKgMCAcQml8QQAAAAAAAAAAAAAAAAAAAAANMYGu+UIK7akcQEofwXhAECCAjFAfyDpFR5cGUBpVYyT2Jq3gASoklExBAAAAAAAAAAAAAAAAAAAAAApEREaXLEEFTyKFqhkkXVoWn+8R1Lr2ymRWNBbGdvAaNFY00Io0VjTginRWNCU2l6ZdIAEAAAp0VjSW5kZXgBpkVjRGlzdNwAEAECAwQFBgcICQoLDA0ODxCoQ1N1bUFsZ28BqFBhcnROdW1zkgECqVBhcnRFVGFnc8CpUGFydFNpemVzktIAFtgq0gAGvb+qUGFydEFTaXplc5LSAFKb69IAGZg0p1BhcnRJZHiSxFqKm+4h9J7JCYCAgAFEABSPlBzH5g6z9gah3wOPnwLDlAGeD+os0xbjFd8O8w+TBoM8rz6bHO0KzQWtBu4GwgGSBocH6QPUSu8J5A/8gwSWtQPOtgL0euoMmAPEAKRTaXpl0gAdlemlTVRpbWXTGBrvlCCu2pGnTWV0YVN5c4K8WC1NaW5pby1JbnRlcm5hbC1hY3R1YWwtc2l6ZcQHNzA5MTIzMbxYLU1pbmlvLUludGVybmFsLWNvbXByZXNzaW9uxBVrbGF1c3Bvc3QvY29tcHJlc3MvczKnTWV0YVVzcoKsY29udGVudC10eXBlqHRleHQvY3N2pGV0YWfZIjEzYmYyMDU0NGVjN2VmY2YxNzhiYWRmNjc4NzNjODg2LTKhds5mYYMqzv8Vdtk=`) + if err != nil { + t.Fatal(err) + } + var xl xlMetaV2 + err = xl.Load(data) + if err != nil { + t.Fatal(err) + } + for _, v := range xl.versions { + // Signature should match + if binary.BigEndian.Uint32(v.header.Signature[:]) != 0x8e5a6406 { + t.Log(v.header.String()) + t.Fatalf("invalid signature 0x%x", binary.BigEndian.Uint32(v.header.Signature[:])) + } + } + }) +} + +func Test_xlMetaV2Shallow_LoadTimeStamp(t *testing.T) { + // v0 Saved with + // ReservedMetadataPrefixLower + ReplicationTimestamp: []byte("2022-10-27T15:40:53.195813291+08:00") + // ReservedMetadataPrefixLower + ReplicaTimestamp: []byte("2022-10-27T15:40:53.195813291+08:00") + data := []byte{0x58, 0x4c, 0x32, 0x20, 0x1, 0x0, 0x3, 0x0, 0xc6, 0x0, 0x0, 0x3, 0xe8, 0x2, 0x1, 0x3, 0xc4, 0x24, 0x95, 0xc4, 0x10, 0x6a, 0x33, 0x51, 0xe3, 0x48, 0x0, 0x4e, 0xed, 0xab, 0x9, 0x21, 0xeb, 0x83, 0x5a, 0xc6, 0x45, 0xd3, 0x17, 0x21, 0xe7, 0xa1, 0x98, 0x52, 0x88, 0xc, 0xc4, 0x4, 0xfe, 0x54, 0xbc, 0x2f, 0x2, 0x0, 0xc4, 0xd5, 0x83, 0xa4, 0x54, 0x79, 0x70, 0x65, 0x2, 0xa6, 0x44, 0x65, 0x6c, 0x4f, 0x62, 0x6a, 0x83, 0xa2, 0x49, 0x44, 0xc4, 0x10, 0x6a, 0x33, 0x51, 0xe3, 0x48, 0x0, 0x4e, 0xed, 0xab, 0x9, 0x21, 0xeb, 0x83, 0x5a, 0xc6, 0x45, 0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65, 0xd3, 0x17, 0x21, 0xe7, 0xa1, 0x98, 0x52, 0x88, 0xc, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73, 0x82, 0xd9, 0x26, 0x78, 0x2d, 0x6d, 0x69, 0x6e, 0x69, 0x6f, 0x2d, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2d, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2d, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0xc4, 0x23, 0x32, 0x30, 0x32, 0x32, 0x2d, 0x31, 0x30, 0x2d, 0x32, 0x37, 0x54, 0x31, 0x35, 0x3a, 0x34, 0x30, 0x3a, 0x35, 0x33, 0x2e, 0x31, 0x39, 0x35, 0x38, 0x31, 0x33, 0x32, 0x39, 0x31, 0x2b, 0x30, 0x38, 0x3a, 0x30, 0x30, 0xd9, 0x22, 0x78, 0x2d, 0x6d, 0x69, 0x6e, 0x69, 0x6f, 0x2d, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2d, 0x72, 0x65, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x2d, 0x74, 0x69, 0x6d, 0x65, 0x73, 0x74, 0x61, 0x6d, 0x70, 0xc4, 0x23, 0x32, 0x30, 0x32, 0x32, 0x2d, 0x31, 0x30, 0x2d, 0x32, 0x37, 0x54, 0x31, 0x35, 0x3a, 0x34, 0x30, 0x3a, 0x35, 0x33, 0x2e, 0x31, 0x39, 0x35, 0x38, 0x31, 0x33, 0x32, 0x39, 0x31, 0x2b, 0x30, 0x38, 0x3a, 0x30, 0x30, 0xa1, 0x76, 0x0, 0xc4, 0x24, 0x95, 0xc4, 0x10, 0xdb, 0xaf, 0x9a, 0xe8, 0xda, 0xe0, 0x40, 0xa6, 0x80, 0xf2, 0x1c, 0x39, 0xe8, 0x7, 0x38, 0x2c, 0xd3, 0x16, 0xb2, 0x7f, 0xfe, 0x45, 0x1c, 0xf, 0x98, 0xc4, 0x4, 0xea, 0x49, 0x78, 0x27, 0x1, 0x6, 0xc5, 0x1, 0x4b, 0x82, 0xa4, 0x54, 0x79, 0x70, 0x65, 0x1, 0xa5, 0x56, 0x32, 0x4f, 0x62, 0x6a, 0xde, 0x0, 0x11, 0xa2, 0x49, 0x44, 0xc4, 0x10, 0xdb, 0xaf, 0x9a, 0xe8, 0xda, 0xe0, 0x40, 0xa6, 0x80, 0xf2, 0x1c, 0x39, 0xe8, 0x7, 0x38, 0x2c, 0xa4, 0x44, 0x44, 0x69, 0x72, 0xc4, 0x10, 0x4f, 0xe7, 0xd7, 0xf3, 0x15, 0x2d, 0x4c, 0xcd, 0x96, 0x65, 0x24, 0x86, 0xdc, 0x2, 0x1b, 0xed, 0xa6, 0x45, 0x63, 0x41, 0x6c, 0x67, 0x6f, 0x1, 0xa3, 0x45, 0x63, 0x4d, 0x4, 0xa3, 0x45, 0x63, 0x4e, 0x4, 0xa7, 0x45, 0x63, 0x42, 0x53, 0x69, 0x7a, 0x65, 0xd2, 0x0, 0x10, 0x0, 0x0, 0xa7, 0x45, 0x63, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x7, 0xa6, 0x45, 0x63, 0x44, 0x69, 0x73, 0x74, 0x98, 0x7, 0x8, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0xa8, 0x43, 0x53, 0x75, 0x6d, 0x41, 0x6c, 0x67, 0x6f, 0x1, 0xa8, 0x50, 0x61, 0x72, 0x74, 0x4e, 0x75, 0x6d, 0x73, 0x91, 0x1, 0xa9, 0x50, 0x61, 0x72, 0x74, 0x45, 0x54, 0x61, 0x67, 0x73, 0x91, 0xa0, 0xa9, 0x50, 0x61, 0x72, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x73, 0x91, 0xd1, 0x1, 0x0, 0xaa, 0x50, 0x61, 0x72, 0x74, 0x41, 0x53, 0x69, 0x7a, 0x65, 0x73, 0x91, 0xd1, 0x1, 0x0, 0xa4, 0x53, 0x69, 0x7a, 0x65, 0xd1, 0x1, 0x0, 0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65, 0xd3, 0x16, 0xb2, 0x7f, 0xfe, 0x45, 0x1c, 0xf, 0x98, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73, 0x81, 0xbc, 0x78, 0x2d, 0x6d, 0x69, 0x6e, 0x69, 0x6f, 0x2d, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2d, 0x69, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x2d, 0x64, 0x61, 0x74, 0x61, 0xc4, 0x4, 0x74, 0x72, 0x75, 0x65, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x73, 0x72, 0x82, 0xac, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x74, 0x79, 0x70, 0x65, 0xb8, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6f, 0x63, 0x74, 0x65, 0x74, 0x2d, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0xa4, 0x65, 0x74, 0x61, 0x67, 0xd9, 0x20, 0x62, 0x65, 0x65, 0x62, 0x32, 0x32, 0x30, 0x66, 0x61, 0x39, 0x64, 0x65, 0x36, 0x31, 0x36, 0x36, 0x62, 0x64, 0x31, 0x66, 0x66, 0x37, 0x32, 0x33, 0x66, 0x35, 0x62, 0x31, 0x66, 0x31, 0x39, 0x66, 0xc4, 0x24, 0x95, 0xc4, 0x10, 0xe6, 0xb4, 0xa8, 0xe0, 0xc6, 0x3d, 0x40, 0x7a, 0x87, 0x61, 0xf2, 0x43, 0x41, 0xdf, 0xcb, 0x93, 0xd3, 0x16, 0xb2, 0x7f, 0xfe, 0x3e, 0x43, 0xf2, 0xf8, 0xc4, 0x4, 0x91, 0xd9, 0xee, 0x2e, 0x1, 0x6, 0xc5, 0x1, 0x4b, 0x82, 0xa4, 0x54, 0x79, 0x70, 0x65, 0x1, 0xa5, 0x56, 0x32, 0x4f, 0x62, 0x6a, 0xde, 0x0, 0x11, 0xa2, 0x49, 0x44, 0xc4, 0x10, 0xe6, 0xb4, 0xa8, 0xe0, 0xc6, 0x3d, 0x40, 0x7a, 0x87, 0x61, 0xf2, 0x43, 0x41, 0xdf, 0xcb, 0x93, 0xa4, 0x44, 0x44, 0x69, 0x72, 0xc4, 0x10, 0xfd, 0x76, 0xe, 0x16, 0xfc, 0x17, 0x4d, 0x38, 0xbc, 0xac, 0x18, 0xde, 0x36, 0xae, 0xd7, 0x48, 0xa6, 0x45, 0x63, 0x41, 0x6c, 0x67, 0x6f, 0x1, 0xa3, 0x45, 0x63, 0x4d, 0x4, 0xa3, 0x45, 0x63, 0x4e, 0x4, 0xa7, 0x45, 0x63, 0x42, 0x53, 0x69, 0x7a, 0x65, 0xd2, 0x0, 0x10, 0x0, 0x0, 0xa7, 0x45, 0x63, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x7, 0xa6, 0x45, 0x63, 0x44, 0x69, 0x73, 0x74, 0x98, 0x7, 0x8, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0xa8, 0x43, 0x53, 0x75, 0x6d, 0x41, 0x6c, 0x67, 0x6f, 0x1, 0xa8, 0x50, 0x61, 0x72, 0x74, 0x4e, 0x75, 0x6d, 0x73, 0x91, 0x1, 0xa9, 0x50, 0x61, 0x72, 0x74, 0x45, 0x54, 0x61, 0x67, 0x73, 0x91, 0xa0, 0xa9, 0x50, 0x61, 0x72, 0x74, 0x53, 0x69, 0x7a, 0x65, 0x73, 0x91, 0xd1, 0x1, 0x0, 0xaa, 0x50, 0x61, 0x72, 0x74, 0x41, 0x53, 0x69, 0x7a, 0x65, 0x73, 0x91, 0xd1, 0x1, 0x0, 0xa4, 0x53, 0x69, 0x7a, 0x65, 0xd1, 0x1, 0x0, 0xa5, 0x4d, 0x54, 0x69, 0x6d, 0x65, 0xd3, 0x16, 0xb2, 0x7f, 0xfe, 0x3e, 0x43, 0xf2, 0xf8, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x53, 0x79, 0x73, 0x81, 0xbc, 0x78, 0x2d, 0x6d, 0x69, 0x6e, 0x69, 0x6f, 0x2d, 0x69, 0x6e, 0x74, 0x65, 0x72, 0x6e, 0x61, 0x6c, 0x2d, 0x69, 0x6e, 0x6c, 0x69, 0x6e, 0x65, 0x2d, 0x64, 0x61, 0x74, 0x61, 0xc4, 0x4, 0x74, 0x72, 0x75, 0x65, 0xa7, 0x4d, 0x65, 0x74, 0x61, 0x55, 0x73, 0x72, 0x82, 0xac, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x2d, 0x74, 0x79, 0x70, 0x65, 0xb8, 0x61, 0x70, 0x70, 0x6c, 0x69, 0x63, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6f, 0x63, 0x74, 0x65, 0x74, 0x2d, 0x73, 0x74, 0x72, 0x65, 0x61, 0x6d, 0xa4, 0x65, 0x74, 0x61, 0x67, 0xd9, 0x20, 0x36, 0x64, 0x31, 0x31, 0x61, 0x34, 0x35, 0x31, 0x62, 0x30, 0x62, 0x32, 0x34, 0x35, 0x65, 0x32, 0x61, 0x34, 0x35, 0x65, 0x34, 0x64, 0x31, 0x36, 0x66, 0x63, 0x33, 0x39, 0x62, 0x62, 0x61, 0x61, 0xce, 0x78, 0x57, 0x89, 0xc4} + var xl xlMetaV2 + err := xl.Load(data) + if err != nil { + t.Fatal(err) + } + v0 := xl.versions[0] + + // Saved with signature 0xfe, 0x54, 0xbc, 0x2f + // Signature must be converted after load. + wantSig := [4]byte{0x1e, 0x5f, 0xba, 0x4a} + if v0.header.Signature != wantSig { + t.Errorf("Wrong signature, want %#v, got %#v", wantSig, v0.header.Signature) + } + v, err := xl.getIdx(0) + if err != nil { + t.Fatal(err) + } + wantTimeStamp := "2022-10-27T07:40:53.195813291Z" + got := string(v.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicationTimestamp]) + if wantTimeStamp != got { + t.Errorf("Wrong timestamp, want %v, got %v", wantTimeStamp, got) + } + got = string(v.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+ReplicaTimestamp]) + if wantTimeStamp != got { + t.Errorf("Wrong timestamp, want %v, got %v", wantTimeStamp, got) + } +} + +func Test_mergeXLV2Versions(t *testing.T) { + dataZ, err := os.ReadFile("testdata/xl-meta-consist.zip") + if err != nil { + t.Fatal(err) + } + var vers [][]xlMetaV2ShallowVersion + zr, err := zip.NewReader(bytes.NewReader(dataZ), int64(len(dataZ))) + if err != nil { + t.Fatal(err) + } + for _, file := range zr.File { + if file.UncompressedSize64 == 0 { + continue + } + in, err := file.Open() + if err != nil { + t.Fatal(err) + } + defer in.Close() + buf, err := io.ReadAll(in) + if err != nil { + t.Fatal(err) + } + var xl xlMetaV2 + err = xl.LoadOrConvert(buf) + if err != nil { + t.Fatal(err) + } + vers = append(vers, xl.versions) + } + for _, v2 := range vers { + for _, ver := range v2 { + b, _ := json.Marshal(ver.header) + t.Log(string(b)) + var x xlMetaV2Version + _, _ = x.unmarshalV(0, ver.meta) + b, _ = json.Marshal(x) + t.Log(string(b), x.getSignature()) + } + } + + for i := range vers { + t.Run(fmt.Sprintf("non-strict-q%d", i), func(t *testing.T) { + merged := mergeXLV2Versions(i, false, 0, vers...) + if len(merged) == 0 { + t.Error("Did not get any results") + return + } + for _, ver := range merged { + if ver.header.Type == invalidVersionType { + t.Errorf("Invalid result returned: %v", ver.header) + } + } + }) + t.Run(fmt.Sprintf("strict-q%d", i), func(t *testing.T) { + merged := mergeXLV2Versions(i, true, 0, vers...) + if len(merged) == 0 { + t.Error("Did not get any results") + return + } + for _, ver := range merged { + if ver.header.Type == invalidVersionType { + t.Errorf("Invalid result returned: %v", ver.header) + } + } + }) + t.Run(fmt.Sprintf("signature-q%d", i), func(t *testing.T) { + // Mutate signature, non strict + vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers)) + for i, ver := range vers { + newVers := make([]xlMetaV2ShallowVersion, 0, len(ver)) + for _, v := range ver { + v.header.Signature = [4]byte{byte(i + 10), 0, 0, 0} + newVers = append(newVers, v) + } + vMod = append(vMod, newVers) + } + merged := mergeXLV2Versions(i, false, 0, vMod...) + if len(merged) == 0 { + t.Error("Did not get any results") + return + } + for _, ver := range merged { + if ver.header.Type == invalidVersionType { + t.Errorf("Invalid result returned: %v", ver.header) + } + } + }) + t.Run(fmt.Sprintf("modtime-q%d", i), func(t *testing.T) { + // Mutate modtime, but rest is consistent. + vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers)) + for i, ver := range vers { + newVers := make([]xlMetaV2ShallowVersion, 0, len(ver)) + for _, v := range ver { + v.header.ModTime += int64(i) + newVers = append(newVers, v) + } + vMod = append(vMod, newVers) + } + merged := mergeXLV2Versions(i, false, 0, vMod...) + if len(merged) == 0 && i < 2 { + t.Error("Did not get any results") + return + } + if len(merged) > 0 && i >= 2 { + t.Error("Got unexpected results") + return + } + for _, ver := range merged { + if ver.header.Type == invalidVersionType { + t.Errorf("Invalid result returned: %v", ver.header) + } + } + }) + t.Run(fmt.Sprintf("flags-q%d", i), func(t *testing.T) { + // Mutate signature, non strict + vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers)) + for i, ver := range vers { + newVers := make([]xlMetaV2ShallowVersion, 0, len(ver)) + for _, v := range ver { + v.header.Flags += xlFlags(i) + newVers = append(newVers, v) + } + vMod = append(vMod, newVers) + } + merged := mergeXLV2Versions(i, false, 0, vMod...) + if len(merged) == 0 { + t.Error("Did not get any results") + return + } + for _, ver := range merged { + if ver.header.Type == invalidVersionType { + t.Errorf("Invalid result returned: %v", ver.header) + } + } + }) + t.Run(fmt.Sprintf("versionid-q%d", i), func(t *testing.T) { + // Mutate signature, non strict + vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers)) + for i, ver := range vers { + newVers := make([]xlMetaV2ShallowVersion, 0, len(ver)) + for _, v := range ver { + v.header.VersionID[0] += byte(i) + newVers = append(newVers, v) + } + vMod = append(vMod, newVers) + } + merged := mergeXLV2Versions(i, false, 0, vMod...) + if len(merged) == 0 && i < 2 { + t.Error("Did not get any results") + return + } + if len(merged) > 0 && i >= 2 { + t.Error("Got unexpected results") + return + } + for _, ver := range merged { + if ver.header.Type == invalidVersionType { + t.Errorf("Invalid result returned: %v", ver.header) + } + } + }) + t.Run(fmt.Sprintf("strict-signature-q%d", i), func(t *testing.T) { + // Mutate signature, non strict + vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers)) + for i, ver := range vers { + newVers := make([]xlMetaV2ShallowVersion, 0, len(ver)) + for _, v := range ver { + v.header.Signature = [4]byte{byte(i + 10), 0, 0, 0} + newVers = append(newVers, v) + } + vMod = append(vMod, newVers) + } + merged := mergeXLV2Versions(i, true, 0, vMod...) + if len(merged) == 0 && i < 2 { + t.Error("Did not get any results") + return + } + if len(merged) > 0 && i >= 2 { + t.Error("Got unexpected results") + return + } + for _, ver := range merged { + if ver.header.Type == invalidVersionType { + t.Errorf("Invalid result returned: %v", ver.header) + } + } + }) + t.Run(fmt.Sprintf("strict-modtime-q%d", i), func(t *testing.T) { + // Mutate signature, non strict + vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers)) + for i, ver := range vers { + newVers := make([]xlMetaV2ShallowVersion, 0, len(ver)) + for _, v := range ver { + v.header.ModTime += int64(i + 10) + newVers = append(newVers, v) + } + vMod = append(vMod, newVers) + } + merged := mergeXLV2Versions(i, true, 0, vMod...) + if len(merged) == 0 && i < 2 { + t.Error("Did not get any results") + return + } + if len(merged) > 0 && i >= 2 { + t.Error("Got unexpected results", len(merged), merged[0].header) + return + } + for _, ver := range merged { + if ver.header.Type == invalidVersionType { + t.Errorf("Invalid result returned: %v", ver.header) + } + } + }) + t.Run(fmt.Sprintf("strict-flags-q%d", i), func(t *testing.T) { + // Mutate signature, non strict + vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers)) + for i, ver := range vers { + newVers := make([]xlMetaV2ShallowVersion, 0, len(ver)) + for _, v := range ver { + v.header.Flags += xlFlags(i + 10) + newVers = append(newVers, v) + } + vMod = append(vMod, newVers) + } + merged := mergeXLV2Versions(i, true, 0, vMod...) + if len(merged) == 0 && i < 2 { + t.Error("Did not get any results") + return + } + if len(merged) > 0 && i >= 2 { + t.Error("Got unexpected results", len(merged)) + return + } + for _, ver := range merged { + if ver.header.Type == invalidVersionType { + t.Errorf("Invalid result returned: %v", ver.header) + } + } + }) + t.Run(fmt.Sprintf("strict-type-q%d", i), func(t *testing.T) { + // Mutate signature, non strict + vMod := make([][]xlMetaV2ShallowVersion, 0, len(vers)) + for i, ver := range vers { + newVers := make([]xlMetaV2ShallowVersion, 0, len(ver)) + for _, v := range ver { + v.header.Type += VersionType(i + 10) + newVers = append(newVers, v) + } + vMod = append(vMod, newVers) + } + merged := mergeXLV2Versions(i, true, 0, vMod...) + if len(merged) == 0 && i < 2 { + t.Error("Did not get any results") + return + } + if len(merged) > 0 && i >= 2 { + t.Error("Got unexpected results", len(merged)) + return + } + for _, ver := range merged { + if ver.header.Type == invalidVersionType { + t.Errorf("Invalid result returned: %v", ver.header) + } + } + }) + } +} + +func Test_mergeXLV2Versions2(t *testing.T) { + vDelMarker := xlMetaV2ShallowVersion{header: xlMetaV2VersionHeader{ + VersionID: [16]byte{2}, + ModTime: 1500, + Signature: [4]byte{5, 6, 7, 8}, + Type: DeleteType, + Flags: 0, + }} + vDelMarker.meta, _ = base64.StdEncoding.DecodeString("gqRUeXBlAqZEZWxPYmqDoklExBCvwGEaY+BAO4B4vyG5ERorpU1UaW1l0xbgJlsWE9IHp01ldGFTeXOA") + + vObj := xlMetaV2ShallowVersion{header: xlMetaV2VersionHeader{ + VersionID: [16]byte{1}, + ModTime: 1000, + Signature: [4]byte{1, 2, 3, 4}, + Type: ObjectType, + Flags: xlFlagUsesDataDir | xlFlagInlineData, + }} + vObj.meta, _ = base64.StdEncoding.DecodeString("gqRUeXBlAaVWMk9iat4AEaJJRMQQEkaOteYCSrWB3nqppSIKTqRERGlyxBAO8fXSJ5RI+YEtsp8KneVVpkVjQWxnbwGjRWNNDKNFY04Ep0VjQlNpemXSABAAAKdFY0luZGV4BaZFY0Rpc3TcABAFBgcICQoLDA0ODxABAgMEqENTdW1BbGdvAahQYXJ0TnVtc5EBqVBhcnRFVGFnc8CpUGFydFNpemVzkdEBL6pQYXJ0QVNpemVzkdEBL6RTaXpl0QEvpU1UaW1l0xbgJhIa6ABvp01ldGFTeXOBvHgtbWluaW8taW50ZXJuYWwtaW5saW5lLWRhdGHEBHRydWWnTWV0YVVzcoKsY29udGVudC10eXBluGFwcGxpY2F0aW9uL29jdGV0LXN0cmVhbaRldGFn2SBlYTIxMDE2MmVlYjRhZGMzMWZmOTg0Y2I3NDRkNmFmNg==") + + testCases := []struct { + name string + input [][]xlMetaV2ShallowVersion + quorum int + reqVersions int + want []xlMetaV2ShallowVersion + }{ + { + name: "obj-on-one", + input: [][]xlMetaV2ShallowVersion{ + 0: {vDelMarker, vObj}, // disk 0 + 1: {vDelMarker}, // disk 1 + 2: {vDelMarker}, // disk 2 + }, + quorum: 2, + reqVersions: 0, + want: []xlMetaV2ShallowVersion{vDelMarker}, + }, + { + name: "obj-on-two", + input: [][]xlMetaV2ShallowVersion{ + 0: {vDelMarker, vObj}, // disk 0 + 1: {vDelMarker, vObj}, // disk 1 + 2: {vDelMarker}, // disk 2 + }, + quorum: 2, + reqVersions: 0, + want: []xlMetaV2ShallowVersion{vDelMarker, vObj}, + }, + { + name: "obj-on-all", + input: [][]xlMetaV2ShallowVersion{ + 0: {vDelMarker, vObj}, // disk 0 + 1: {vDelMarker, vObj}, // disk 1 + 2: {vDelMarker, vObj}, // disk 2 + }, + quorum: 2, + reqVersions: 0, + want: []xlMetaV2ShallowVersion{vDelMarker, vObj}, + }, + { + name: "del-on-one", + input: [][]xlMetaV2ShallowVersion{ + 0: {vDelMarker, vObj}, // disk 0 + 1: {vObj}, // disk 1 + 2: {vObj}, // disk 2 + }, + quorum: 2, + reqVersions: 0, + want: []xlMetaV2ShallowVersion{vObj}, + }, + { + name: "del-on-two", + input: [][]xlMetaV2ShallowVersion{ + 0: {vDelMarker, vObj}, // disk 0 + 1: {vDelMarker, vObj}, // disk 1 + 2: {vObj}, // disk 2 + }, + quorum: 2, + reqVersions: 0, + want: []xlMetaV2ShallowVersion{vDelMarker, vObj}, + }, + { + name: "del-on-two-16stripe", + input: [][]xlMetaV2ShallowVersion{ + 0: {vObj}, // disk 0 + 1: {vDelMarker, vObj}, // disk 1 + 2: {vDelMarker, vObj}, // disk 2 + 3: {vDelMarker, vObj}, // disk 3 + 4: {vDelMarker, vObj}, // disk 4 + 5: {vDelMarker, vObj}, // disk 5 + 6: {vDelMarker, vObj}, // disk 6 + 7: {vDelMarker, vObj}, // disk 7 + 8: {vDelMarker, vObj}, // disk 8 + 9: {vDelMarker, vObj}, // disk 9 + 10: {vObj}, // disk 10 + 11: {vDelMarker, vObj}, // disk 11 + 12: {vDelMarker, vObj}, // disk 12 + 13: {vDelMarker, vObj}, // disk 13 + 14: {vDelMarker, vObj}, // disk 14 + 15: {vDelMarker, vObj}, // disk 15 + }, + quorum: 7, + reqVersions: 0, + want: []xlMetaV2ShallowVersion{vDelMarker, vObj}, + }, + } + for _, test := range testCases { + t.Run(test.name, func(t *testing.T) { + // Run multiple times, shuffling the input order. + for i := range int64(50) { + t.Run(fmt.Sprint(i), func(t *testing.T) { + rng := rand.New(rand.NewSource(i)) + rng.Shuffle(len(test.input), func(i, j int) { + test.input[i], test.input[j] = test.input[j], test.input[i] + }) + got := mergeXLV2Versions(test.quorum, true, 0, test.input...) + if !reflect.DeepEqual(test.want, got) { + t.Errorf("want %v != got %v", test.want, got) + } + }) + } + }) + } +} + +func Test_mergeEntryChannels(t *testing.T) { + dataZ, err := os.ReadFile("testdata/xl-meta-merge.zip") + if err != nil { + t.Fatal(err) + } + var vers []metaCacheEntry + zr, err := zip.NewReader(bytes.NewReader(dataZ), int64(len(dataZ))) + if err != nil { + t.Fatal(err) + } + + for _, file := range zr.File { + if file.UncompressedSize64 == 0 { + continue + } + in, err := file.Open() + if err != nil { + t.Fatal(err) + } + defer in.Close() + buf, err := io.ReadAll(in) + if err != nil { + t.Fatal(err) + } + buf = xlMetaV2TrimData(buf) + + vers = append(vers, metaCacheEntry{ + name: "a", + metadata: buf, + }) + } + + // Shuffle... + for i := range 100 { + rng := rand.New(rand.NewSource(int64(i))) + rng.Shuffle(len(vers), func(i, j int) { + vers[i], vers[j] = vers[j], vers[i] + }) + var entries []chan metaCacheEntry + for _, v := range vers { + v.cached = nil + ch := make(chan metaCacheEntry, 1) + ch <- v + close(ch) + entries = append(entries, ch) + } + out := make(chan metaCacheEntry, 1) + err := mergeEntryChannels(t.Context(), entries, out, 1) + if err != nil { + t.Fatal(err) + } + got, ok := <-out + if !ok { + t.Fatal("Got no result") + } + + xl, err := got.xlmeta() + if err != nil { + t.Fatal(err) + } + if len(xl.versions) != 3 { + t.Fatal("Got wrong number of versions, want 3, got", len(xl.versions)) + } + if !sort.SliceIsSorted(xl.versions, func(i, j int) bool { + return xl.versions[i].header.sortsBefore(xl.versions[j].header) + }) { + t.Errorf("Got unsorted result") + } + } +} + +func TestXMinIOHealingSkip(t *testing.T) { + xl := xlMetaV2{} + failOnErr := func(err error) { + t.Helper() + if err != nil { + t.Fatalf("Test failed with %v", err) + } + } + + fi := FileInfo{ + Volume: "volume", + Name: "object-name", + VersionID: "756100c6-b393-4981-928a-d49bbc164741", + IsLatest: true, + Deleted: false, + ModTime: time.Now(), + Size: 1 << 10, + Mode: 0, + Erasure: ErasureInfo{ + Algorithm: ReedSolomon.String(), + DataBlocks: 4, + ParityBlocks: 2, + BlockSize: 10000, + Index: 1, + Distribution: []int{1, 2, 3, 4, 5, 6, 7, 8}, + Checksums: []ChecksumInfo{{ + PartNumber: 1, + Algorithm: HighwayHash256S, + Hash: nil, + }}, + }, + NumVersions: 1, + } + + fi.SetHealing() + failOnErr(xl.AddVersion(fi)) + + var err error + fi, err = xl.ToFileInfo(fi.Volume, fi.Name, fi.VersionID, false, true) + if err != nil { + t.Fatalf("xl.ToFileInfo failed with %v", err) + } + + if fi.Healing() { + t.Fatal("Expected fi.Healing to be false") + } +} + +func benchmarkManyPartsOptionally(b *testing.B, allParts bool) { + f, err := os.Open("testdata/xl-many-parts.meta") + if err != nil { + b.Fatal(err) + } + defer f.Close() + + data, err := io.ReadAll(f) + if err != nil { + b.Fatal(err) + } + + buf, _, _ := isIndexedMetaV2(data) + if buf == nil { + b.Fatal("buf == nil") + } + + b.Run("ToFileInfo", func(b *testing.B) { + b.ResetTimer() + b.ReportAllocs() + + for b.Loop() { + _, err = buf.ToFileInfo("volume", "path", "", allParts) + if err != nil { + b.Fatal(err) + } + } + }) +} + +func BenchmarkToFileInfoNoParts(b *testing.B) { + benchmarkManyPartsOptionally(b, false) +} + +func BenchmarkToFileInfoWithParts(b *testing.B) { + benchmarkManyPartsOptionally(b, true) +} diff --git a/cmd/xl-storage-format_test.go b/cmd/xl-storage-format_test.go new file mode 100644 index 00000000..12f4b052 --- /dev/null +++ b/cmd/xl-storage-format_test.go @@ -0,0 +1,543 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "fmt" + "math/rand" + "testing" + "time" + + "github.com/dustin/go-humanize" + jsoniter "github.com/json-iterator/go" + xhttp "github.com/minio/minio/internal/http" +) + +func TestIsXLMetaFormatValid(t *testing.T) { + tests := []struct { + name int + version string + format string + want bool + }{ + {1, "123", "fs", false}, + {2, "123", xlMetaFormat, false}, + {3, xlMetaVersion100, "test", false}, + {4, xlMetaVersion101, "hello", false}, + {5, xlMetaVersion100, xlMetaFormat, true}, + {6, xlMetaVersion101, xlMetaFormat, true}, + } + for _, tt := range tests { + if got := isXLMetaFormatValid(tt.version, tt.format); got != tt.want { + t.Errorf("Test %d: Expected %v but received %v", tt.name, got, tt.want) + } + } +} + +func TestIsXLMetaErasureInfoValid(t *testing.T) { + tests := []struct { + name int + data int + parity int + want bool + }{ + {1, 5, 6, false}, + {2, 5, 5, true}, + {3, 0, 5, false}, + {3, -1, 5, false}, + {4, 5, -1, false}, + {5, 5, 0, true}, + {6, 5, 0, true}, + {7, 5, 4, true}, + } + for _, tt := range tests { + if got := isXLMetaErasureInfoValid(tt.data, tt.parity); got != tt.want { + t.Errorf("Test %d: Expected %v but received %v -> %#v", tt.name, got, tt.want, tt) + } + } +} + +// newTestXLMetaV1 - initializes new xlMetaV1Object, adds version, allocates a fresh erasure info and metadata. +func newTestXLMetaV1() xlMetaV1Object { + xlMeta := xlMetaV1Object{} + xlMeta.Version = xlMetaVersion101 + xlMeta.Format = xlMetaFormat + xlMeta.Minio.Release = "test" + xlMeta.Erasure = ErasureInfo{ + Algorithm: "klauspost/reedsolomon/vandermonde", + DataBlocks: 5, + ParityBlocks: 5, + BlockSize: 10485760, + Index: 10, + Distribution: []int{9, 10, 1, 2, 3, 4, 5, 6, 7, 8}, + } + xlMeta.Stat = StatInfo{ + Size: int64(20), + ModTime: UTCNow(), + } + // Set meta data. + xlMeta.Meta = make(map[string]string) + xlMeta.Meta["testKey1"] = "val1" + xlMeta.Meta["testKey2"] = "val2" + return xlMeta +} + +func (m *xlMetaV1Object) AddTestObjectCheckSum(partNumber int, algorithm BitrotAlgorithm, hash string) { + checksum, err := hex.DecodeString(hash) + if err != nil { + panic(err) + } + m.Erasure.Checksums[partNumber-1] = ChecksumInfo{partNumber, algorithm, checksum} +} + +// AddTestObjectPart - add a new object part in order. +func (m *xlMetaV1Object) AddTestObjectPart(partNumber int, partSize int64) { + partInfo := ObjectPartInfo{ + Number: partNumber, + Size: partSize, + } + + // Proceed to include new part info. + m.Parts[partNumber-1] = partInfo +} + +// Constructs xlMetaV1Object{} for given number of parts and converts it into bytes. +func getXLMetaBytes(totalParts int) []byte { + xlSampleMeta := getSampleXLMeta(totalParts) + xlMetaBytes, err := json.Marshal(xlSampleMeta) + if err != nil { + panic(err) + } + return xlMetaBytes +} + +// Returns sample xlMetaV1Object{} for number of parts. +func getSampleXLMeta(totalParts int) xlMetaV1Object { + xlMeta := newTestXLMetaV1() + // Number of checksum info == total parts. + xlMeta.Erasure.Checksums = make([]ChecksumInfo, totalParts) + // total number of parts. + xlMeta.Parts = make([]ObjectPartInfo, totalParts) + for i := range totalParts { + // hard coding hash and algo value for the checksum, Since we are benchmarking the parsing of xl.meta the magnitude doesn't affect the test, + // The magnitude doesn't make a difference, only the size does. + xlMeta.AddTestObjectCheckSum(i+1, BLAKE2b512, "a23f5eff248c4372badd9f3b2455a285cd4ca86c3d9a570b091d3fc5cd7ca6d9484bbea3f8c5d8d4f84daae96874419eda578fd736455334afbac2c924b3915a") + xlMeta.AddTestObjectPart(i+1, 67108864) + } + return xlMeta +} + +// Compare the unmarshaled XLMetaV1 with the one obtained from jsoniter parsing. +func compareXLMetaV1(t *testing.T, unMarshalXLMeta, jsoniterXLMeta xlMetaV1Object) { + // Start comparing the fields of xlMetaV1Object obtained from jsoniter parsing with one parsed using json unmarshalling. + if unMarshalXLMeta.Version != jsoniterXLMeta.Version { + t.Errorf("Expected the Version to be \"%s\", but got \"%s\".", unMarshalXLMeta.Version, jsoniterXLMeta.Version) + } + if unMarshalXLMeta.Format != jsoniterXLMeta.Format { + t.Errorf("Expected the format to be \"%s\", but got \"%s\".", unMarshalXLMeta.Format, jsoniterXLMeta.Format) + } + if unMarshalXLMeta.Stat.Size != jsoniterXLMeta.Stat.Size { + t.Errorf("Expected the stat size to be %v, but got %v.", unMarshalXLMeta.Stat.Size, jsoniterXLMeta.Stat.Size) + } + if !unMarshalXLMeta.Stat.ModTime.Equal(jsoniterXLMeta.Stat.ModTime) { + t.Errorf("Expected the modTime to be \"%v\", but got \"%v\".", unMarshalXLMeta.Stat.ModTime, jsoniterXLMeta.Stat.ModTime) + } + if unMarshalXLMeta.Erasure.Algorithm != jsoniterXLMeta.Erasure.Algorithm { + t.Errorf("Expected the erasure algorithm to be \"%v\", but got \"%v\".", unMarshalXLMeta.Erasure.Algorithm, jsoniterXLMeta.Erasure.Algorithm) + } + if unMarshalXLMeta.Erasure.DataBlocks != jsoniterXLMeta.Erasure.DataBlocks { + t.Errorf("Expected the erasure data blocks to be %v, but got %v.", unMarshalXLMeta.Erasure.DataBlocks, jsoniterXLMeta.Erasure.DataBlocks) + } + if unMarshalXLMeta.Erasure.ParityBlocks != jsoniterXLMeta.Erasure.ParityBlocks { + t.Errorf("Expected the erasure parity blocks to be %v, but got %v.", unMarshalXLMeta.Erasure.ParityBlocks, jsoniterXLMeta.Erasure.ParityBlocks) + } + if unMarshalXLMeta.Erasure.BlockSize != jsoniterXLMeta.Erasure.BlockSize { + t.Errorf("Expected the erasure block size to be %v, but got %v.", unMarshalXLMeta.Erasure.BlockSize, jsoniterXLMeta.Erasure.BlockSize) + } + if unMarshalXLMeta.Erasure.Index != jsoniterXLMeta.Erasure.Index { + t.Errorf("Expected the erasure index to be %v, but got %v.", unMarshalXLMeta.Erasure.Index, jsoniterXLMeta.Erasure.Index) + } + if len(unMarshalXLMeta.Erasure.Distribution) != len(jsoniterXLMeta.Erasure.Distribution) { + t.Errorf("Expected the size of Erasure Distribution to be %d, but got %d.", len(unMarshalXLMeta.Erasure.Distribution), len(jsoniterXLMeta.Erasure.Distribution)) + } else { + for i := 0; i < len(unMarshalXLMeta.Erasure.Distribution); i++ { + if unMarshalXLMeta.Erasure.Distribution[i] != jsoniterXLMeta.Erasure.Distribution[i] { + t.Errorf("Expected the Erasure Distribution to be %d, got %d.", unMarshalXLMeta.Erasure.Distribution[i], jsoniterXLMeta.Erasure.Distribution[i]) + } + } + } + + if len(unMarshalXLMeta.Erasure.Checksums) != len(jsoniterXLMeta.Erasure.Checksums) { + t.Errorf("Expected the size of Erasure Checksums to be %d, but got %d.", len(unMarshalXLMeta.Erasure.Checksums), len(jsoniterXLMeta.Erasure.Checksums)) + } else { + for i := 0; i < len(unMarshalXLMeta.Erasure.Checksums); i++ { + if unMarshalXLMeta.Erasure.Checksums[i].PartNumber != jsoniterXLMeta.Erasure.Checksums[i].PartNumber { + t.Errorf("Expected the Erasure Checksum PartNumber to be \"%d\", got \"%d\".", unMarshalXLMeta.Erasure.Checksums[i].PartNumber, jsoniterXLMeta.Erasure.Checksums[i].PartNumber) + } + if unMarshalXLMeta.Erasure.Checksums[i].Algorithm != jsoniterXLMeta.Erasure.Checksums[i].Algorithm { + t.Errorf("Expected the Erasure Checksum Algorithm to be \"%s\", got \"%s\".", unMarshalXLMeta.Erasure.Checksums[i].Algorithm, jsoniterXLMeta.Erasure.Checksums[i].Algorithm) + } + if !bytes.Equal(unMarshalXLMeta.Erasure.Checksums[i].Hash, jsoniterXLMeta.Erasure.Checksums[i].Hash) { + t.Errorf("Expected the Erasure Checksum Hash to be \"%s\", got \"%s\".", unMarshalXLMeta.Erasure.Checksums[i].Hash, jsoniterXLMeta.Erasure.Checksums[i].Hash) + } + } + } + + if unMarshalXLMeta.Minio.Release != jsoniterXLMeta.Minio.Release { + t.Errorf("Expected the Release string to be \"%s\", but got \"%s\".", unMarshalXLMeta.Minio.Release, jsoniterXLMeta.Minio.Release) + } + if len(unMarshalXLMeta.Parts) != len(jsoniterXLMeta.Parts) { + t.Errorf("Expected info of %d parts to be present, but got %d instead.", len(unMarshalXLMeta.Parts), len(jsoniterXLMeta.Parts)) + } else { + for i := 0; i < len(unMarshalXLMeta.Parts); i++ { + if unMarshalXLMeta.Parts[i].Number != jsoniterXLMeta.Parts[i].Number { + t.Errorf("Expected the number of part %d to be \"%d\", got \"%d\".", i+1, unMarshalXLMeta.Parts[i].Number, jsoniterXLMeta.Parts[i].Number) + } + if unMarshalXLMeta.Parts[i].Size != jsoniterXLMeta.Parts[i].Size { + t.Errorf("Expected the size of part %d to be %v, got %v.", i+1, unMarshalXLMeta.Parts[i].Size, jsoniterXLMeta.Parts[i].Size) + } + } + } + + for key, val := range unMarshalXLMeta.Meta { + jsoniterVal, exists := jsoniterXLMeta.Meta[key] + if !exists { + t.Errorf("No meta data entry for Key \"%s\" exists.", key) + } + if val != jsoniterVal { + t.Errorf("Expected the value for Meta data key \"%s\" to be \"%s\", but got \"%s\".", key, val, jsoniterVal) + } + } +} + +// Tests the correctness of constructing XLMetaV1 using jsoniter lib. +// The result will be compared with the result obtained from json.unMarshal of the byte data. +func TestGetXLMetaV1Jsoniter1(t *testing.T) { + xlMetaJSON := getXLMetaBytes(1) + + var unMarshalXLMeta xlMetaV1Object + if err := json.Unmarshal(xlMetaJSON, &unMarshalXLMeta); err != nil { + t.Errorf("Unmarshalling failed: %v", err) + } + + var jsoniterXLMeta xlMetaV1Object + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err := json.Unmarshal(xlMetaJSON, &jsoniterXLMeta); err != nil { + t.Errorf("jsoniter parsing of XLMeta failed: %v", err) + } + compareXLMetaV1(t, unMarshalXLMeta, jsoniterXLMeta) +} + +// Tests the correctness of constructing XLMetaV1 using jsoniter lib for XLMetaV1 of size 10 parts. +// The result will be compared with the result obtained from json.unMarshal of the byte data. +func TestGetXLMetaV1Jsoniter10(t *testing.T) { + xlMetaJSON := getXLMetaBytes(10) + + var unMarshalXLMeta xlMetaV1Object + if err := json.Unmarshal(xlMetaJSON, &unMarshalXLMeta); err != nil { + t.Errorf("Unmarshalling failed: %v", err) + } + + var jsoniterXLMeta xlMetaV1Object + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err := json.Unmarshal(xlMetaJSON, &jsoniterXLMeta); err != nil { + t.Errorf("jsoniter parsing of XLMeta failed: %v", err) + } + + compareXLMetaV1(t, unMarshalXLMeta, jsoniterXLMeta) +} + +// Test the predicted part size from the part index +func TestGetPartSizeFromIdx(t *testing.T) { + // Create test cases + testCases := []struct { + totalSize int64 + partSize int64 + partIndex int + expectedSize int64 + }{ + // Total size is zero + {0, 10, 1, 0}, + // part size 2MiB, total size 4MiB + {4 * humanize.MiByte, 2 * humanize.MiByte, 1, 2 * humanize.MiByte}, + {4 * humanize.MiByte, 2 * humanize.MiByte, 2, 2 * humanize.MiByte}, + {4 * humanize.MiByte, 2 * humanize.MiByte, 3, 0}, + // part size 2MiB, total size 5MiB + {5 * humanize.MiByte, 2 * humanize.MiByte, 1, 2 * humanize.MiByte}, + {5 * humanize.MiByte, 2 * humanize.MiByte, 2, 2 * humanize.MiByte}, + {5 * humanize.MiByte, 2 * humanize.MiByte, 3, 1 * humanize.MiByte}, + {5 * humanize.MiByte, 2 * humanize.MiByte, 4, 0}, + } + + for i, testCase := range testCases { + s, err := calculatePartSizeFromIdx(GlobalContext, testCase.totalSize, testCase.partSize, testCase.partIndex) + if err != nil { + t.Errorf("Test %d: Expected to pass but failed. %s", i+1, err) + } + if err == nil && s != testCase.expectedSize { + t.Errorf("Test %d: The calculated part size is incorrect: expected = %d, found = %d\n", i+1, testCase.expectedSize, s) + } + } + + testCasesFailure := []struct { + totalSize int64 + partSize int64 + partIndex int + err error + }{ + // partSize is 0, returns error. + {10, 0, 1, errPartSizeZero}, + // partIndex is 0, returns error. + {10, 1, 0, errPartSizeIndex}, + // Total size is -1, returns error. + {-2, 10, 1, errInvalidArgument}, + } + + for i, testCaseFailure := range testCasesFailure { + _, err := calculatePartSizeFromIdx(GlobalContext, testCaseFailure.totalSize, testCaseFailure.partSize, testCaseFailure.partIndex) + if err == nil { + t.Errorf("Test %d: Expected to failed but passed. %s", i+1, err) + } + if err != nil && err != testCaseFailure.err { + t.Errorf("Test %d: Expected err %s, but got %s", i+1, testCaseFailure.err, err) + } + } +} + +func BenchmarkXlMetaV2Shallow(b *testing.B) { + fi := FileInfo{ + Volume: "volume", + Name: "object-name", + VersionID: "756100c6-b393-4981-928a-d49bbc164741", + IsLatest: true, + Deleted: false, + TransitionStatus: "PENDING", + DataDir: "bffea160-ca7f-465f-98bc-9b4f1c3ba1ef", + XLV1: false, + ModTime: time.Now(), + Size: 1234456, + Mode: 0, + Metadata: map[string]string{ + xhttp.AmzRestore: "FAILED", + xhttp.ContentMD5: mustGetUUID(), + xhttp.AmzBucketReplicationStatus: "PENDING", + xhttp.ContentType: "application/json", + }, + Parts: []ObjectPartInfo{ + { + Number: 1, + Size: 1234345, + ActualSize: 1234345, + }, + { + Number: 2, + Size: 1234345, + ActualSize: 1234345, + }, + }, + Erasure: ErasureInfo{ + Algorithm: ReedSolomon.String(), + DataBlocks: 4, + ParityBlocks: 2, + BlockSize: 10000, + Index: 1, + Distribution: []int{1, 2, 3, 4, 5, 6, 7, 8}, + Checksums: []ChecksumInfo{ + { + PartNumber: 1, + Algorithm: HighwayHash256S, + Hash: nil, + }, + { + PartNumber: 2, + Algorithm: HighwayHash256S, + Hash: nil, + }, + }, + }, + } + for _, size := range []int{1, 10, 1000, 100_000} { + b.Run(fmt.Sprint(size, "-versions"), func(b *testing.B) { + var xl xlMetaV2 + ids := make([]string, size) + for i := range size { + fi.VersionID = mustGetUUID() + fi.DataDir = mustGetUUID() + ids[i] = fi.VersionID + fi.ModTime = fi.ModTime.Add(-time.Second) + xl.AddVersion(fi) + } + // Encode all. This is used for benchmarking. + enc, err := xl.AppendTo(nil) + if err != nil { + b.Fatal(err) + } + b.Logf("Serialized size: %d bytes", len(enc)) + rng := rand.New(rand.NewSource(0)) + dump := make([]byte, len(enc)) + b.Run("UpdateObjectVersion", func(b *testing.B) { + b.SetBytes(int64(size)) + b.ResetTimer() + b.ReportAllocs() + for b.Loop() { + // Load... + xl = xlMetaV2{} + err := xl.Load(enc) + if err != nil { + b.Fatal(err) + } + // Update modtime for resorting... + fi.ModTime = fi.ModTime.Add(-time.Second) + // Update a random version. + fi.VersionID = ids[rng.Intn(size)] + // Update... + err = xl.UpdateObjectVersion(fi) + if err != nil { + b.Fatal(err) + } + // Save... + dump, err = xl.AppendTo(dump[:0]) + if err != nil { + b.Fatal(err) + } + } + }) + b.Run("DeleteVersion", func(b *testing.B) { + b.SetBytes(int64(size)) + b.ResetTimer() + b.ReportAllocs() + for b.Loop() { + // Load... + xl = xlMetaV2{} + err := xl.Load(enc) + if err != nil { + b.Fatal(err) + } + // Update a random version. + fi.VersionID = ids[rng.Intn(size)] + // Delete... + _, err = xl.DeleteVersion(fi) + if err != nil { + b.Fatal(err) + } + // Save... + dump, err = xl.AppendTo(dump[:0]) + if err != nil { + b.Fatal(err) + } + } + }) + b.Run("AddVersion", func(b *testing.B) { + b.SetBytes(int64(size)) + b.ResetTimer() + b.ReportAllocs() + for b.Loop() { + // Load... + xl = xlMetaV2{} + err := xl.Load(enc) + if err != nil { + b.Fatal(err) + } + // Update modtime for resorting... + fi.ModTime = fi.ModTime.Add(-time.Second) + // Update a random version. + fi.VersionID = mustGetUUID() + // Add... + err = xl.AddVersion(fi) + if err != nil { + b.Fatal(err) + } + // Save... + dump, err = xl.AppendTo(dump[:0]) + if err != nil { + b.Fatal(err) + } + } + }) + b.Run("ToFileInfo", func(b *testing.B) { + b.SetBytes(int64(size)) + b.ResetTimer() + b.ReportAllocs() + for b.Loop() { + // Load... + xl = xlMetaV2{} + err := xl.Load(enc) + if err != nil { + b.Fatal(err) + } + // List... + _, err = xl.ToFileInfo("volume", "path", ids[rng.Intn(size)], false, true) + if err != nil { + b.Fatal(err) + } + } + }) + b.Run("ListVersions", func(b *testing.B) { + b.SetBytes(int64(size)) + b.ResetTimer() + b.ReportAllocs() + for b.Loop() { + // Load... + xl = xlMetaV2{} + err := xl.Load(enc) + if err != nil { + b.Fatal(err) + } + // List... + _, err = xl.ListVersions("volume", "path", true) + if err != nil { + b.Fatal(err) + } + } + }) + b.Run("ToFileInfoNew", func(b *testing.B) { + b.SetBytes(int64(size)) + b.ResetTimer() + b.ReportAllocs() + for b.Loop() { + buf, _, _ := isIndexedMetaV2(enc) + if buf == nil { + b.Fatal("buf == nil") + } + _, err = buf.ToFileInfo("volume", "path", ids[rng.Intn(size)], true) + if err != nil { + b.Fatal(err) + } + } + }) + b.Run("ListVersionsNew", func(b *testing.B) { + b.SetBytes(int64(size)) + b.ResetTimer() + b.ReportAllocs() + for b.Loop() { + buf, _, _ := isIndexedMetaV2(enc) + if buf == nil { + b.Fatal("buf == nil") + } + _, err = buf.ListVersions("volume", "path", true) + if err != nil { + b.Fatal(err) + } + } + }) + }) + } +} diff --git a/cmd/xl-storage-free-version.go b/cmd/xl-storage-free-version.go new file mode 100644 index 00000000..abf2b5af --- /dev/null +++ b/cmd/xl-storage-free-version.go @@ -0,0 +1,110 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "fmt" + + "github.com/google/uuid" + "github.com/minio/minio/internal/bucket/lifecycle" +) + +const freeVersion = "free-version" + +// InitFreeVersion creates a free-version to track the tiered-content of j. If j has +// no tiered content, it returns false. +func (j xlMetaV2Object) InitFreeVersion(fi FileInfo) (xlMetaV2Version, bool) { + if fi.SkipTierFreeVersion() { + return xlMetaV2Version{}, false + } + if status, ok := j.MetaSys[ReservedMetadataPrefixLower+TransitionStatus]; ok && bytes.Equal(status, []byte(lifecycle.TransitionComplete)) { + vID, err := uuid.Parse(fi.TierFreeVersionID()) + if err != nil { + panic(fmt.Errorf("Invalid Tier Object delete marker versionId %s %v", fi.TierFreeVersionID(), err)) + } + freeEntry := xlMetaV2Version{Type: DeleteType, WrittenByVersion: globalVersionUnix} + freeEntry.DeleteMarker = &xlMetaV2DeleteMarker{ + VersionID: vID, + ModTime: j.ModTime, // fi.ModTime may be empty + MetaSys: make(map[string][]byte), + } + + freeEntry.DeleteMarker.MetaSys[ReservedMetadataPrefixLower+freeVersion] = []byte{} + tierKey := ReservedMetadataPrefixLower + TransitionTier + tierObjKey := ReservedMetadataPrefixLower + TransitionedObjectName + tierObjVIDKey := ReservedMetadataPrefixLower + TransitionedVersionID + + for k, v := range j.MetaSys { + switch k { + case tierKey, tierObjKey, tierObjVIDKey: + freeEntry.DeleteMarker.MetaSys[k] = v + } + } + return freeEntry, true + } + return xlMetaV2Version{}, false +} + +// FreeVersion returns true if j represents a free-version, false otherwise. +func (j xlMetaV2DeleteMarker) FreeVersion() bool { + _, ok := j.MetaSys[ReservedMetadataPrefixLower+freeVersion] + return ok +} + +// FreeVersion returns true if j represents a free-version, false otherwise. +func (j xlMetaV2Version) FreeVersion() bool { + if j.Type == DeleteType { + return j.DeleteMarker.FreeVersion() + } + return false +} + +// AddFreeVersion adds a free-version if needed for fi.VersionID version. +// Free-version will be added if fi.VersionID has transitioned. +func (x *xlMetaV2) AddFreeVersion(fi FileInfo) error { + var uv uuid.UUID + var err error + switch fi.VersionID { + case "", nullVersionID: + default: + uv, err = uuid.Parse(fi.VersionID) + if err != nil { + return err + } + } + + for i, version := range x.versions { + if version.header.VersionID != uv || version.header.Type != ObjectType { + continue + } + // if uv has tiered content we add a + // free-version to track it for asynchronous + // deletion via scanner. + ver, err := x.getIdx(i) + if err != nil { + return err + } + + if freeVersion, toFree := ver.ObjectV2.InitFreeVersion(fi); toFree { + return x.addVersion(freeVersion) + } + return nil + } + return nil +} diff --git a/cmd/xl-storage-free-version_test.go b/cmd/xl-storage-free-version_test.go new file mode 100644 index 00000000..b7205160 --- /dev/null +++ b/cmd/xl-storage-free-version_test.go @@ -0,0 +1,281 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "errors" + "testing" + "time" + + "github.com/google/uuid" + "github.com/minio/minio/internal/bucket/lifecycle" +) + +func (x xlMetaV2) listFreeVersions(volume, path string) ([]FileInfo, error) { + fivs, err := x.ListVersions(volume, path, true) + if err != nil { + return nil, err + } + n := 0 + for _, fiv := range fivs { + if fiv.TierFreeVersion() { + fivs[n] = fiv + n++ + } + } + fivs = fivs[:n] + return fivs, nil +} + +func TestFreeVersion(t *testing.T) { + fatalErr := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + + // Add a version with tiered content, one with local content + xl := xlMetaV2{} + counter := 1 + report := func() { + t.Helper() + // t.Logf("versions (%d): len = %d", counter, len(xl.versions)) + counter++ + } + fi := FileInfo{ + Volume: "volume", + Name: "object-name", + VersionID: "00000000-0000-0000-0000-000000000001", + IsLatest: true, + Deleted: false, + TransitionStatus: "", + DataDir: "bffea160-ca7f-465f-98bc-9b4f1c3ba1ef", + XLV1: false, + ModTime: time.Now(), + Size: 0, + Mode: 0, + Metadata: nil, + Parts: nil, + Erasure: ErasureInfo{ + Algorithm: ReedSolomon.String(), + DataBlocks: 4, + ParityBlocks: 2, + BlockSize: 10000, + Index: 1, + Distribution: []int{1, 2, 3, 4, 5, 6, 7, 8}, + Checksums: []ChecksumInfo{{ + PartNumber: 1, + Algorithm: HighwayHash256S, + Hash: nil, + }}, + }, + MarkDeleted: false, + NumVersions: 1, + SuccessorModTime: time.Time{}, + } + // Add a version with local content + fatalErr(xl.AddVersion(fi)) + report() + + // Add null version with tiered content + tierfi := fi + tierfi.VersionID = "" + fatalErr(xl.AddVersion(tierfi)) + report() + tierfi.TransitionStatus = lifecycle.TransitionComplete + tierfi.TransitionedObjName = mustGetUUID() + tierfi.TransitionTier = "MINIOTIER-1" + var err error + _, err = xl.DeleteVersion(tierfi) + fatalErr(err) + report() + + fvIDs := []string{ + "00000000-0000-0000-0000-0000000000f1", + "00000000-0000-0000-0000-0000000000f2", + } + // Simulate overwrite of null version + newtierfi := tierfi + newtierfi.SetTierFreeVersionID(fvIDs[0]) + fatalErr(xl.AddFreeVersion(newtierfi)) + report() + fatalErr(xl.AddVersion(newtierfi)) + report() + + // Simulate removal of null version + newtierfi.TransitionTier = "" + newtierfi.TransitionedObjName = "" + newtierfi.TransitionStatus = "" + newtierfi.SetTierFreeVersionID(fvIDs[1]) + report() + _, err = xl.DeleteVersion(newtierfi) + report() + fatalErr(err) + + // At this point the version stack must look as below, + // v3 --> free version 00000000-0000-0000-0000-0000000000f2 (from removal of null version) + // v2 --> free version 00000000-0000-0000-0000-0000000000f1 (from overwriting of null version ) + // v1 --> non-free version 00000000-0000-0000-0000-000000000001 + + // Check number of free-versions + freeVersions, err := xl.listFreeVersions(newtierfi.Volume, newtierfi.Name) + if err != nil { + t.Fatalf("failed to list free versions %v", err) + } + if len(freeVersions) != 2 { + t.Fatalf("Expected two free versions but got %d", len(freeVersions)) + } + + freeVersionsTests := []struct { + vol string + name string + inclFreeVers bool + afterFn func(fi FileInfo) (string, error) + expectedFree bool + expectedErr error + }{ + // ToFileInfo with 'inclFreeVers = true' should return the latest + // non-free version if one is present + { + vol: newtierfi.Volume, + name: newtierfi.Name, + inclFreeVers: true, + afterFn: xl.DeleteVersion, + expectedFree: false, + }, + // ToFileInfo with 'inclFreeVers = true' must return the latest free + // version when no non-free versions are present. + { + vol: newtierfi.Volume, + name: newtierfi.Name, + inclFreeVers: true, + expectedFree: true, + }, + // ToFileInfo with 'inclFreeVers = false' must return errFileNotFound + // when no non-free version exist. + { + vol: newtierfi.Volume, + name: newtierfi.Name, + inclFreeVers: false, + expectedErr: errFileNotFound, + }, + } + + for _, ft := range freeVersionsTests { + fi, err := xl.ToFileInfo(ft.vol, ft.name, "", ft.inclFreeVers, true) + if err != nil && !errors.Is(err, ft.expectedErr) { + t.Fatalf("ToFileInfo failed due to %v", err) + } + if got := fi.TierFreeVersion(); got != ft.expectedFree { + t.Fatalf("Expected free-version=%v but got free-version=%v", ft.expectedFree, got) + } + if ft.afterFn != nil { + _, err = ft.afterFn(fi) + if err != nil { + t.Fatalf("ft.afterFn failed with err %v", err) + } + } + } + + // Simulate scanner removing free-version + freefi := newtierfi + for _, fvID := range fvIDs { + freefi.VersionID = fvID + _, err = xl.DeleteVersion(freefi) + fatalErr(err) + } + report() + + // Check number of free-versions + freeVersions, err = xl.listFreeVersions(newtierfi.Volume, newtierfi.Name) + if err != nil { + t.Fatalf("failed to list free versions %v", err) + } + if len(freeVersions) != 0 { + t.Fatalf("Expected zero free version but got %d", len(freeVersions)) + } + report() + + // Adding a free version to a version with no tiered content. + newfi := fi + newfi.SetTierFreeVersionID("00000000-0000-0000-0000-0000000000f3") + fatalErr(xl.AddFreeVersion(newfi)) // this shouldn't add a free-version + report() + + // Check number of free-versions + freeVersions, err = xl.listFreeVersions(newtierfi.Volume, newtierfi.Name) + if err != nil { + t.Fatalf("failed to list free versions %v", err) + } + if len(freeVersions) != 0 { + t.Fatalf("Expected zero free version but got %d", len(freeVersions)) + } +} + +func TestSkipFreeVersion(t *testing.T) { + fi := FileInfo{ + Volume: "volume", + Name: "object-name", + VersionID: "00000000-0000-0000-0000-000000000001", + IsLatest: true, + Deleted: false, + TransitionStatus: "", + DataDir: "bffea160-ca7f-465f-98bc-9b4f1c3ba1ef", + XLV1: false, + ModTime: time.Now(), + Size: 0, + Mode: 0, + Metadata: nil, + Parts: nil, + Erasure: ErasureInfo{ + Algorithm: ReedSolomon.String(), + DataBlocks: 4, + ParityBlocks: 2, + BlockSize: 10000, + Index: 1, + Distribution: []int{1, 2, 3, 4, 5, 6, 7, 8}, + Checksums: []ChecksumInfo{{ + PartNumber: 1, + Algorithm: HighwayHash256S, + Hash: nil, + }}, + }, + MarkDeleted: false, + // DeleteMarkerReplicationStatus: "", + // VersionPurgeStatus: "", + NumVersions: 1, + SuccessorModTime: time.Time{}, + } + fi.SetTierFreeVersionID(uuid.New().String()) + // Test if free version is created when SkipTier wasn't set on fi + j := xlMetaV2Object{} + j.MetaSys = make(map[string][]byte) + j.MetaSys[metaTierName] = []byte("WARM-1") + j.MetaSys[metaTierStatus] = []byte(lifecycle.TransitionComplete) + j.MetaSys[metaTierObjName] = []byte("obj-1") + if _, ok := j.InitFreeVersion(fi); !ok { + t.Fatal("Expected a free version to be created") + } + + // Test if we skip creating a free version if SkipTier was set on fi + fi.SetSkipTierFreeVersion() + if _, ok := j.InitFreeVersion(fi); ok { + t.Fatal("Expected no free version to be created") + } +} diff --git a/cmd/xl-storage-meta-inline.go b/cmd/xl-storage-meta-inline.go new file mode 100644 index 00000000..06ed65e8 --- /dev/null +++ b/cmd/xl-storage-meta-inline.go @@ -0,0 +1,403 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "errors" + "fmt" + "slices" + + "github.com/tinylib/msgp/msgp" +) + +// xlMetaInlineData is serialized data in [string][]byte pairs. +type xlMetaInlineData []byte + +// xlMetaInlineDataVer indicates the version of the inline data structure. +const xlMetaInlineDataVer = 1 + +// versionOK returns whether the version is ok. +func (x xlMetaInlineData) versionOK() bool { + if len(x) == 0 { + return true + } + return x[0] > 0 && x[0] <= xlMetaInlineDataVer +} + +// afterVersion returns the payload after the version, if any. +func (x xlMetaInlineData) afterVersion() []byte { + if len(x) == 0 { + return x + } + return x[1:] +} + +// find the data with key s. +// Returns nil if not for or an error occurs. +func (x xlMetaInlineData) find(key string) []byte { + if len(x) == 0 || !x.versionOK() { + return nil + } + sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion()) + if err != nil || sz == 0 { + return nil + } + for range sz { + var found []byte + found, buf, err = msgp.ReadMapKeyZC(buf) + if err != nil || sz == 0 { + return nil + } + if string(found) == key { + val, _, _ := msgp.ReadBytesZC(buf) + return val + } + // Skip it + _, buf, err = msgp.ReadBytesZC(buf) + if err != nil { + return nil + } + } + return nil +} + +// validate checks if the data is valid. +// It does not check integrity of the stored data. +func (x xlMetaInlineData) validate() error { + if len(x) == 0 { + return nil + } + + if !x.versionOK() { + return fmt.Errorf("xlMetaInlineData: unknown version 0x%x", x[0]) + } + + sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion()) + if err != nil { + return fmt.Errorf("xlMetaInlineData: %w", err) + } + + for i := range sz { + var key []byte + key, buf, err = msgp.ReadMapKeyZC(buf) + if err != nil { + return fmt.Errorf("xlMetaInlineData: %w", err) + } + if len(key) == 0 { + return fmt.Errorf("xlMetaInlineData: key %d is length 0", i) + } + _, buf, err = msgp.ReadBytesZC(buf) + if err != nil { + return fmt.Errorf("xlMetaInlineData: %w", err) + } + } + + return nil +} + +// repair will copy all seemingly valid data entries from a corrupted set. +// This does not ensure that data is correct, but will allow all operations to complete. +func (x *xlMetaInlineData) repair() { + data := *x + if len(data) == 0 { + return + } + + if !data.versionOK() { + *x = nil + return + } + + sz, buf, err := msgp.ReadMapHeaderBytes(data.afterVersion()) + if err != nil { + *x = nil + return + } + + // Remove all current data + keys := make([][]byte, 0, sz) + vals := make([][]byte, 0, sz) + for range sz { + var key, val []byte + key, buf, err = msgp.ReadMapKeyZC(buf) + if err != nil { + break + } + if len(key) == 0 { + break + } + val, buf, err = msgp.ReadBytesZC(buf) + if err != nil { + break + } + keys = append(keys, key) + vals = append(vals, val) + } + x.serialize(-1, keys, vals) +} + +// validate checks if the data is valid. +// It does not check integrity of the stored data. +func (x xlMetaInlineData) list() ([]string, error) { + if len(x) == 0 { + return nil, nil + } + if !x.versionOK() { + return nil, errors.New("xlMetaInlineData: unknown version") + } + + sz, buf, err := msgp.ReadMapHeaderBytes(x.afterVersion()) + if err != nil { + return nil, err + } + keys := make([]string, 0, sz) + for i := range sz { + var key []byte + key, buf, err = msgp.ReadMapKeyZC(buf) + if err != nil { + return keys, err + } + if len(key) == 0 { + return keys, fmt.Errorf("xlMetaInlineData: key %d is length 0", i) + } + keys = append(keys, string(key)) + // Skip data... + _, buf, err = msgp.ReadBytesZC(buf) + if err != nil { + return keys, err + } + } + return keys, nil +} + +// serialize will serialize the provided keys and values. +// The function will panic if keys/value slices aren't of equal length. +// Payload size can give an indication of expected payload size. +// If plSize is <= 0 it will be calculated. +func (x *xlMetaInlineData) serialize(plSize int, keys [][]byte, vals [][]byte) { + if len(keys) != len(vals) { + panic(fmt.Errorf("xlMetaInlineData.serialize: keys/value number mismatch")) + } + if len(keys) == 0 { + *x = nil + return + } + if plSize <= 0 { + plSize = 1 + msgp.MapHeaderSize + for i := range keys { + plSize += len(keys[i]) + len(vals[i]) + msgp.StringPrefixSize + msgp.ArrayHeaderSize + } + } + payload := make([]byte, 1, plSize) + payload[0] = xlMetaInlineDataVer + payload = msgp.AppendMapHeader(payload, uint32(len(keys))) + for i := range keys { + payload = msgp.AppendStringFromBytes(payload, keys[i]) + payload = msgp.AppendBytes(payload, vals[i]) + } + *x = payload +} + +// entries returns the number of entries in the data. +func (x xlMetaInlineData) entries() int { + if len(x) == 0 || !x.versionOK() { + return 0 + } + sz, _, _ := msgp.ReadMapHeaderBytes(x.afterVersion()) + return int(sz) +} + +// replace will add or replace a key/value pair. +func (x *xlMetaInlineData) replace(key string, value []byte) { + in := x.afterVersion() + sz, buf, _ := msgp.ReadMapHeaderBytes(in) + keys := make([][]byte, 0, sz+1) + vals := make([][]byte, 0, sz+1) + + // Version plus header... + plSize := 1 + msgp.MapHeaderSize + replaced := false + for range sz { + var found, foundVal []byte + var err error + found, buf, err = msgp.ReadMapKeyZC(buf) + if err != nil { + break + } + foundVal, buf, err = msgp.ReadBytesZC(buf) + if err != nil { + break + } + plSize += len(found) + msgp.StringPrefixSize + msgp.ArrayHeaderSize + keys = append(keys, found) + if string(found) == key { + vals = append(vals, value) + plSize += len(value) + replaced = true + } else { + vals = append(vals, foundVal) + plSize += len(foundVal) + } + } + + // Add one more. + if !replaced { + keys = append(keys, []byte(key)) + vals = append(vals, value) + plSize += len(key) + len(value) + msgp.StringPrefixSize + msgp.ArrayHeaderSize + } + + // Reserialize... + x.serialize(plSize, keys, vals) +} + +// rename will rename a key. +// Returns whether the key was found. +func (x *xlMetaInlineData) rename(oldKey, newKey string) bool { + in := x.afterVersion() + sz, buf, _ := msgp.ReadMapHeaderBytes(in) + keys := make([][]byte, 0, sz) + vals := make([][]byte, 0, sz) + + // Version plus header... + plSize := 1 + msgp.MapHeaderSize + found := false + for range sz { + var foundKey, foundVal []byte + var err error + foundKey, buf, err = msgp.ReadMapKeyZC(buf) + if err != nil { + break + } + foundVal, buf, err = msgp.ReadBytesZC(buf) + if err != nil { + break + } + plSize += len(foundVal) + msgp.StringPrefixSize + msgp.ArrayHeaderSize + vals = append(vals, foundVal) + if string(foundKey) != oldKey { + keys = append(keys, foundKey) + plSize += len(foundKey) + } else { + keys = append(keys, []byte(newKey)) + plSize += len(newKey) + found = true + } + } + // If not found, just return. + if !found { + return false + } + + // Reserialize... + x.serialize(plSize, keys, vals) + return true +} + +// remove will remove one or more keys. +// Returns true if any key was found. +func (x *xlMetaInlineData) remove(keys ...string) bool { + in := x.afterVersion() + sz, buf, _ := msgp.ReadMapHeaderBytes(in) + newKeys := make([][]byte, 0, sz) + newVals := make([][]byte, 0, sz) + var removeKey func(s []byte) bool + + // Copy if big number of compares... + if len(keys) > 5 && sz > 5 { + mKeys := make(map[string]struct{}, len(keys)) + for _, key := range keys { + mKeys[key] = struct{}{} + } + removeKey = func(s []byte) bool { + _, ok := mKeys[string(s)] + return ok + } + } else { + removeKey = func(s []byte) bool { + return slices.Contains(keys, string(s)) + } + } + + // Version plus header... + plSize := 1 + msgp.MapHeaderSize + found := false + for range sz { + var foundKey, foundVal []byte + var err error + foundKey, buf, err = msgp.ReadMapKeyZC(buf) + if err != nil { + break + } + foundVal, buf, err = msgp.ReadBytesZC(buf) + if err != nil { + break + } + if !removeKey(foundKey) { + plSize += msgp.StringPrefixSize + msgp.ArrayHeaderSize + len(foundKey) + len(foundVal) + newKeys = append(newKeys, foundKey) + newVals = append(newVals, foundVal) + } else { + found = true + } + } + // If not found, just return. + if !found { + return false + } + // If none left... + if len(newKeys) == 0 { + *x = nil + return true + } + + // Reserialize... + x.serialize(plSize, newKeys, newVals) + return true +} + +// xlMetaV2TrimData will trim any data from the metadata without unmarshalling it. +// If any error occurs the unmodified data is returned. +func xlMetaV2TrimData(buf []byte) []byte { + metaBuf, maj, minor, err := checkXL2V1(buf) + if err != nil { + return buf + } + if maj == 1 && minor < 1 { + // First version to carry data. + return buf + } + // Skip header + _, metaBuf, err = msgp.ReadBytesZC(metaBuf) + if err != nil { + storageLogIf(GlobalContext, err) + return buf + } + // Skip CRC + if maj > 1 || minor >= 2 { + _, metaBuf, err = msgp.ReadUint32Bytes(metaBuf) + storageLogIf(GlobalContext, err) + } + // = input - current pos + ends := len(buf) - len(metaBuf) + if ends > len(buf) { + return buf + } + + return buf[:ends] +} diff --git a/cmd/xl-storage.go b/cmd/xl-storage.go new file mode 100644 index 00000000..35d7ac73 --- /dev/null +++ b/cmd/xl-storage.go @@ -0,0 +1,3423 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "io" + "os" + pathutil "path" + "path/filepath" + "runtime" + "slices" + "strconv" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" + + "github.com/dustin/go-humanize" + "github.com/google/uuid" + jsoniter "github.com/json-iterator/go" + "github.com/klauspost/filepathx" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/bucket/lifecycle" + "github.com/minio/minio/internal/bucket/replication" + "github.com/minio/minio/internal/cachevalue" + "github.com/minio/minio/internal/config/storageclass" + + "github.com/minio/minio/internal/disk" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/logger" + "github.com/pkg/xattr" +) + +const ( + nullVersionID = "null" + + // Small file threshold below which data accompanies metadata from storage layer. + smallFileThreshold = 128 * humanize.KiByte // Optimized for NVMe/SSDs + + // For hardrives it is possible to set this to a lower value to avoid any + // spike in latency. But currently we are simply keeping it optimal for SSDs. + + // bigFileThreshold is the point where we add readahead to put operations. + bigFileThreshold = 128 * humanize.MiByte + + // XL metadata file carries per object metadata. + xlStorageFormatFile = "xl.meta" + + // XL metadata file backup file carries previous per object metadata. + xlStorageFormatFileBackup = "xl.meta.bkp" +) + +var alignedBuf []byte + +func init() { + alignedBuf = disk.AlignedBlock(xioutil.DirectioAlignSize) + _, _ = rand.Read(alignedBuf) +} + +// isValidVolname verifies a volname name in accordance with object +// layer requirements. +func isValidVolname(volname string) bool { + if len(volname) < 3 { + return false + } + + if runtime.GOOS == "windows" { + // Volname shouldn't have reserved characters in Windows. + return !strings.ContainsAny(volname, `\:*?\"<>|`) + } + + return true +} + +// xlStorage - implements StorageAPI interface. +type xlStorage struct { + // Indicate of NSScanner is in progress in this disk + scanning int32 + + drivePath string + endpoint Endpoint + + globalSync bool + oDirect bool // indicates if this disk supports ODirect + + diskID string + + formatFileInfo os.FileInfo + formatFile string + formatLegacy bool + formatLastCheck time.Time + + diskInfoCache *cachevalue.Cache[DiskInfo] + sync.RWMutex + formatData []byte + + nrRequests uint64 + major, minor uint32 + fsType string + + immediatePurge chan string + immediatePurgeCancel context.CancelFunc + + // mutex to prevent concurrent read operations overloading walks. + rotational bool + walkMu *sync.Mutex + walkReadMu *sync.Mutex +} + +// checkPathLength - returns error if given path name length more than 255 +func checkPathLength(pathName string) error { + // Apple OS X path length is limited to 1016 + if runtime.GOOS == "darwin" && len(pathName) > 1016 { + return errFileNameTooLong + } + + // Disallow more than 1024 characters on windows, there + // are no known name_max limits on Windows. + if runtime.GOOS == "windows" && len(pathName) > 1024 { + return errFileNameTooLong + } + + // On Unix we reject paths if they are just '.', '..' or '/' + if pathName == "." || pathName == ".." || pathName == slashSeparator { + return errFileAccessDenied + } + + // Check each path segment length is > 255 on all Unix + // platforms, look for this value as NAME_MAX in + // /usr/include/linux/limits.h + var count int64 + for _, p := range pathName { + switch p { + case '/': + count = 0 // Reset + case '\\': + if runtime.GOOS == globalWindowsOSName { + count = 0 + } + default: + count++ + if count > 255 { + return errFileNameTooLong + } + } + } // Success. + return nil +} + +func getValidPath(path string) (string, error) { + if path == "" { + return path, errInvalidArgument + } + + var err error + // Disallow relative paths, figure out absolute paths. + path, err = filepath.Abs(path) + if err != nil { + return path, err + } + + fi, err := Lstat(path) + if err != nil && !osIsNotExist(err) { + return path, err + } + if osIsNotExist(err) { + // Disk not found create it. + if err = mkdirAll(path, 0o777, ""); err != nil { + return path, err + } + } + if fi != nil && !fi.IsDir() { + return path, errDiskNotDir + } + + return path, nil +} + +// Make Erasure backend meta volumes. +func makeFormatErasureMetaVolumes(disk StorageAPI) error { + if disk == nil { + return errDiskNotFound + } + volumes := []string{ + minioMetaTmpDeletedBucket, // creates .minio.sys/tmp as well as .minio.sys/tmp/.trash + minioMetaMultipartBucket, // creates .minio.sys/multipart + dataUsageBucket, // creates .minio.sys/buckets + minioConfigBucket, // creates .minio.sys/config + } + // Attempt to create MinIO internal buckets. + return disk.MakeVolBulk(context.TODO(), volumes...) +} + +// Initialize a new storage disk. +func newXLStorage(ep Endpoint, cleanUp bool) (s *xlStorage, err error) { + immediatePurgeQueue := 100000 + if globalIsTesting || globalIsCICD { + immediatePurgeQueue = 1 + } + + ctx, cancel := context.WithCancel(GlobalContext) + + s = &xlStorage{ + drivePath: ep.Path, + endpoint: ep, + globalSync: globalFSOSync, + diskInfoCache: cachevalue.New[DiskInfo](), + immediatePurge: make(chan string, immediatePurgeQueue), + immediatePurgeCancel: cancel, + } + + defer func() { + if cleanUp && err == nil { + go s.cleanupTrashImmediateCallers(ctx) + } + }() + + s.drivePath, err = getValidPath(ep.Path) + if err != nil { + s.drivePath = ep.Path + return s, err + } + + info, rootDrive, err := getDiskInfo(s.drivePath) + if err != nil { + return s, err + } + + s.major = info.Major + s.minor = info.Minor + s.fsType = info.FSType + + if rootDrive { + return s, errDriveIsRoot + } + + // Sanitize before setting it + if info.NRRequests > 0 { + s.nrRequests = info.NRRequests + } + + // We stagger listings only on HDDs. + if info.Rotational == nil || *info.Rotational { + s.rotational = true + s.walkMu = &sync.Mutex{} + s.walkReadMu = &sync.Mutex{} + } + + if cleanUp { + bgFormatErasureCleanupTmp(s.drivePath) // cleanup any old data. + } + + formatData, formatFi, err := formatErasureMigrate(s.drivePath) + if err != nil && !errors.Is(err, os.ErrNotExist) { + if os.IsPermission(err) { + return s, errDiskAccessDenied + } else if isSysErrIO(err) { + return s, errFaultyDisk + } + return s, err + } + s.formatData = formatData + s.formatFileInfo = formatFi + s.formatFile = pathJoin(s.drivePath, minioMetaBucket, formatConfigFile) + + // Create all necessary bucket folders if possible. + if err = makeFormatErasureMetaVolumes(s); err != nil { + return s, err + } + + if len(s.formatData) > 0 { + format := &formatErasureV3{} + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err = json.Unmarshal(s.formatData, &format); err != nil { + return s, errCorruptedFormat + } + m, n, err := findDiskIndexByDiskID(format, format.Erasure.This) + if err != nil { + return s, err + } + diskID := format.Erasure.This + if m != ep.SetIdx || n != ep.DiskIdx { + storageLogOnceIf(context.Background(), + fmt.Errorf("unexpected drive ordering on pool: %s: found drive at (set=%s, drive=%s), expected at (set=%s, drive=%s): %s(%s): %w", + humanize.Ordinal(ep.PoolIdx+1), humanize.Ordinal(m+1), humanize.Ordinal(n+1), humanize.Ordinal(ep.SetIdx+1), humanize.Ordinal(ep.DiskIdx+1), + s, s.diskID, errInconsistentDisk), "drive-order-format-json") + return s, errInconsistentDisk + } + s.diskID = diskID + s.formatLastCheck = time.Now() + s.formatLegacy = format.Erasure.DistributionAlgo == formatErasureVersionV2DistributionAlgoV1 + } + + // Return an error if ODirect is not supported. Single disk will have + // oDirect off. + if globalIsErasureSD || !disk.ODirectPlatform { + s.oDirect = false + } else if err := s.checkODirectDiskSupport(info.FSType); err == nil { + s.oDirect = true + } else { + return s, err + } + + // Initialize DiskInfo cache + s.diskInfoCache.InitOnce(time.Second, cachevalue.Opts{}, + func(ctx context.Context) (DiskInfo, error) { + dcinfo := DiskInfo{} + di, root, err := getDiskInfo(s.drivePath) + if err != nil { + return dcinfo, err + } + dcinfo.RootDisk = root + dcinfo.Major = di.Major + dcinfo.Minor = di.Minor + dcinfo.Total = di.Total + dcinfo.Free = di.Free + dcinfo.Used = di.Used + dcinfo.UsedInodes = di.Files - di.Ffree + dcinfo.FreeInodes = di.Ffree + dcinfo.FSType = di.FSType + if root { + return dcinfo, errDriveIsRoot + } + + diskID, err := s.GetDiskID() + // Healing is 'true' when + // - if we found an unformatted disk (no 'format.json') + // - if we found healing tracker 'healing.bin' + dcinfo.Healing = errors.Is(err, errUnformattedDisk) + if !dcinfo.Healing { + if hi := s.Healing(); hi != nil && !hi.Finished { + dcinfo.Healing = true + } + } + + dcinfo.ID = diskID + return dcinfo, err + }, + ) + + // Success. + return s, nil +} + +// getDiskInfo returns given disk information. +func getDiskInfo(drivePath string) (di disk.Info, rootDrive bool, err error) { + if err = checkPathLength(drivePath); err == nil { + di, err = disk.GetInfo(drivePath, false) + + if !globalIsCICD && !globalIsErasureSD { + if globalRootDiskThreshold > 0 { + // Use MINIO_ROOTDISK_THRESHOLD_SIZE to figure out if + // this disk is a root disk. treat those disks with + // size less than or equal to the threshold as rootDrives. + rootDrive = di.Total <= globalRootDiskThreshold + } else { + rootDrive, err = disk.IsRootDisk(drivePath, SlashSeparator) + } + } + } + + switch { + case osIsNotExist(err): + err = errDiskNotFound + case isSysErrTooLong(err): + err = errFileNameTooLong + case isSysErrIO(err): + err = errFaultyDisk + } + + return di, rootDrive, err +} + +// Implements stringer compatible interface. +func (s *xlStorage) String() string { + return s.drivePath +} + +func (s *xlStorage) Hostname() string { + return s.endpoint.Host +} + +func (s *xlStorage) Endpoint() Endpoint { + return s.endpoint +} + +func (s *xlStorage) Close() error { + s.immediatePurgeCancel() + return nil +} + +func (s *xlStorage) IsOnline() bool { + return true +} + +func (s *xlStorage) LastConn() time.Time { + return time.Time{} +} + +func (s *xlStorage) IsLocal() bool { + return true +} + +// Retrieve location indexes. +func (s *xlStorage) GetDiskLoc() (poolIdx, setIdx, diskIdx int) { + return s.endpoint.PoolIdx, s.endpoint.SetIdx, s.endpoint.DiskIdx +} + +func (s *xlStorage) Healing() *healingTracker { + healingFile := pathJoin(s.drivePath, minioMetaBucket, + bucketMetaPrefix, healingTrackerFilename) + b, err := os.ReadFile(healingFile) + if err != nil { + if !errors.Is(err, os.ErrNotExist) { + internalLogIf(GlobalContext, fmt.Errorf("unable to read %s: %w", healingFile, err)) + } + return nil + } + if len(b) == 0 { + internalLogIf(GlobalContext, fmt.Errorf("%s is empty", healingFile)) + // 'healing.bin' might be truncated + return nil + } + h := newHealingTracker() + _, err = h.UnmarshalMsg(b) + internalLogIf(GlobalContext, err) + return h +} + +// checkODirectDiskSupport asks the disk to write some data +// with O_DIRECT support, return an error if any and return +// errUnsupportedDisk if there is no O_DIRECT support +func (s *xlStorage) checkODirectDiskSupport(fsType string) error { + if !disk.ODirectPlatform { + return errUnsupportedDisk + } + + // We know XFS already supports O_DIRECT no need to check. + if fsType == "XFS" { + return nil + } + + // For all other FS pay the price of not using our recommended filesystem. + + // Check if backend is writable and supports O_DIRECT + uuid := mustGetUUID() + filePath := pathJoin(s.drivePath, minioMetaTmpDeletedBucket, ".writable-check-"+uuid+".tmp") + + // Create top level directories if they don't exist. + // with mode 0o777 mkdir honors system umask. + mkdirAll(pathutil.Dir(filePath), 0o777, s.drivePath) // don't need to fail here + + w, err := s.openFileDirect(filePath, os.O_CREATE|os.O_WRONLY|os.O_EXCL) + if err != nil { + return err + } + _, err = w.Write(alignedBuf) + w.Close() + if err != nil { + if isSysErrInvalidArg(err) { + err = errUnsupportedDisk + } + } + return err +} + +// readsMetadata and returns disk mTime information for xl.meta +func (s *xlStorage) readMetadataWithDMTime(ctx context.Context, itemPath string) ([]byte, time.Time, error) { + if contextCanceled(ctx) { + return nil, time.Time{}, ctx.Err() + } + + if err := checkPathLength(itemPath); err != nil { + return nil, time.Time{}, err + } + + f, err := OpenFile(itemPath, readMode, 0o666) + if err != nil { + return nil, time.Time{}, err + } + defer f.Close() + stat, err := f.Stat() + if err != nil { + return nil, time.Time{}, err + } + if stat.IsDir() { + return nil, time.Time{}, &os.PathError{ + Op: "open", + Path: itemPath, + Err: syscall.EISDIR, + } + } + buf, err := readXLMetaNoData(f, stat.Size()) + if err != nil { + return nil, stat.ModTime().UTC(), fmt.Errorf("%w -> %s", err, itemPath) + } + return buf, stat.ModTime().UTC(), err +} + +func (s *xlStorage) readMetadata(ctx context.Context, itemPath string) ([]byte, error) { + return xioutil.WithDeadline[[]byte](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) ([]byte, error) { + buf, _, err := s.readMetadataWithDMTime(ctx, itemPath) + return buf, err + }) +} + +func (s *xlStorage) NSScanner(ctx context.Context, cache dataUsageCache, updates chan<- dataUsageEntry, scanMode madmin.HealScanMode, weSleep func() bool) (dataUsageCache, error) { + atomic.AddInt32(&s.scanning, 1) + defer atomic.AddInt32(&s.scanning, -1) + + var err error + stopFn := globalScannerMetrics.log(scannerMetricScanBucketDrive, s.drivePath, cache.Info.Name) + defer func() { + res := make(map[string]string) + if err != nil { + res["err"] = err.Error() + } + stopFn(res) + }() + + // Updates must be closed before we return. + defer xioutil.SafeClose(updates) + var lc *lifecycle.Lifecycle + + // Check if the current bucket has a configured lifecycle policy + if globalLifecycleSys != nil { + lc, err = globalLifecycleSys.Get(cache.Info.Name) + if err == nil && lc.HasActiveRules("") { + cache.Info.lifeCycle = lc + } + } + + // Check if the current bucket has replication configuration + var rcfg *replication.Config + if rcfg, _, err = globalBucketMetadataSys.GetReplicationConfig(ctx, cache.Info.Name); err == nil { + if rcfg.HasActiveRules("", true) { + tgts, err := globalBucketTargetSys.ListBucketTargets(ctx, cache.Info.Name) + if err == nil { + cache.Info.replication = replicationConfig{ + Config: rcfg, + remotes: tgts, + } + } + } + } + + // Check if bucket is object locked. + lr, err := globalBucketObjectLockSys.Get(cache.Info.Name) + if err != nil { + scannerLogOnceIf(ctx, err, cache.Info.Name) + return cache, err + } + + vcfg, _ := globalBucketVersioningSys.Get(cache.Info.Name) + + // return initialized object layer + objAPI := newObjectLayerFn() + // object layer not initialized, return. + if objAPI == nil { + return cache, errServerNotInitialized + } + + poolIdx, setIdx, _ := s.GetDiskLoc() + + disks, err := objAPI.GetDisks(poolIdx, setIdx) + if err != nil { + return cache, err + } + + cache.Info.updates = updates + + dataUsageInfo, err := scanDataFolder(ctx, disks, s, cache, func(item scannerItem) (sizeSummary, error) { + // Look for `xl.meta/xl.json' at the leaf. + if !strings.HasSuffix(item.Path, SlashSeparator+xlStorageFormatFile) && + !strings.HasSuffix(item.Path, SlashSeparator+xlStorageFormatFileV1) { + // if no xl.meta/xl.json found, skip the file. + return sizeSummary{}, errSkipFile + } + stopFn := globalScannerMetrics.log(scannerMetricScanObject, s.drivePath, pathJoin(item.bucket, item.objectPath())) + res := make(map[string]string, 8) + defer func() { + stopFn(res) + }() + + doneSz := globalScannerMetrics.timeSize(scannerMetricReadMetadata) + buf, err := s.readMetadata(ctx, item.Path) + doneSz(len(buf)) + res["metasize"] = strconv.Itoa(len(buf)) + if err != nil { + res["err"] = err.Error() + return sizeSummary{}, errSkipFile + } + + // Remove filename which is the meta file. + item.transformMetaDir() + + fivs, err := getFileInfoVersions(buf, item.bucket, item.objectPath(), false) + metaDataPoolPut(buf) + if err != nil { + res["err"] = err.Error() + return sizeSummary{}, errSkipFile + } + + versioned := vcfg != nil && vcfg.Versioned(item.objectPath()) + objInfos := make([]ObjectInfo, len(fivs.Versions)) + for i, fi := range fivs.Versions { + objInfos[i] = fi.ToObjectInfo(item.bucket, item.objectPath(), versioned) + } + sizeS := sizeSummary{} + for _, tier := range globalTierConfigMgr.ListTiers() { + if sizeS.tiers == nil { + sizeS.tiers = make(map[string]tierStats) + } + sizeS.tiers[tier.Name] = tierStats{} + } + if sizeS.tiers != nil { + sizeS.tiers[storageclass.STANDARD] = tierStats{} + sizeS.tiers[storageclass.RRS] = tierStats{} + } + + if err != nil { + res["err"] = err.Error() + return sizeSummary{}, errSkipFile + } + + var objPresent bool + item.applyActions(ctx, objAPI, objInfos, lr, &sizeS, func(oi ObjectInfo, sz, actualSz int64, sizeS *sizeSummary) { + objPresent = true + if oi.DeleteMarker { + sizeS.deleteMarkers++ + } + if oi.VersionID != "" && sz == actualSz { + sizeS.versions++ + } + sizeS.totalSize += sz + + // Skip tier accounting if object version is a delete-marker or a free-version + // tracking deleted transitioned objects + switch { + case oi.DeleteMarker, oi.TransitionedObject.FreeVersion: + return + } + tier := oi.StorageClass + if tier == "" { + tier = storageclass.STANDARD // no SC means "STANDARD" + } + if oi.TransitionedObject.Status == lifecycle.TransitionComplete { + tier = oi.TransitionedObject.Tier + } + if sizeS.tiers != nil { + if st, ok := sizeS.tiers[tier]; ok { + sizeS.tiers[tier] = st.add(oi.tierStats()) + } + } + }) + + // apply tier sweep action on free versions + for _, freeVersion := range fivs.FreeVersions { + oi := freeVersion.ToObjectInfo(item.bucket, item.objectPath(), versioned) + done := globalScannerMetrics.time(scannerMetricTierObjSweep) + globalExpiryState.enqueueFreeVersion(oi) + done() + } + + // These are rather expensive. Skip if nobody listens. + if globalTrace.NumSubscribers(madmin.TraceScanner) > 0 { + if len(fivs.FreeVersions) > 0 { + res["free-versions"] = strconv.Itoa(len(fivs.FreeVersions)) + } + + if sizeS.versions > 0 { + res["versions"] = strconv.FormatUint(sizeS.versions, 10) + } + res["size"] = strconv.FormatInt(sizeS.totalSize, 10) + for name, tier := range sizeS.tiers { + res["tier-size-"+name] = strconv.FormatUint(tier.TotalSize, 10) + res["tier-versions-"+name] = strconv.Itoa(tier.NumVersions) + } + if sizeS.failedCount > 0 { + res["repl-failed"] = fmt.Sprintf("%d versions, %d bytes", sizeS.failedCount, sizeS.failedSize) + } + if sizeS.pendingCount > 0 { + res["repl-pending"] = fmt.Sprintf("%d versions, %d bytes", sizeS.pendingCount, sizeS.pendingSize) + } + for tgt, st := range sizeS.replTargetStats { + res["repl-size-"+tgt] = strconv.FormatInt(st.replicatedSize, 10) + res["repl-count-"+tgt] = strconv.FormatInt(st.replicatedCount, 10) + if st.failedCount > 0 { + res["repl-failed-"+tgt] = fmt.Sprintf("%d versions, %d bytes", st.failedCount, st.failedSize) + } + if st.pendingCount > 0 { + res["repl-pending-"+tgt] = fmt.Sprintf("%d versions, %d bytes", st.pendingCount, st.pendingSize) + } + } + } + if !objPresent { + // we return errIgnoreFileContrib to signal this function's + // callers to skip this object's contribution towards + // usage. + return sizeSummary{}, errIgnoreFileContrib + } + return sizeS, nil + }, scanMode, weSleep) + if err != nil { + return dataUsageInfo, err + } + + dataUsageInfo.Info.LastUpdate = time.Now() + return dataUsageInfo, nil +} + +func (s *xlStorage) getDeleteAttribute() uint64 { + attr := "user.total_deletes" + buf, err := xattr.LGet(s.formatFile, attr) + if err != nil { + // We start off with '0' if we can read the attributes + return 0 + } + return binary.LittleEndian.Uint64(buf[:8]) +} + +func (s *xlStorage) getWriteAttribute() uint64 { + attr := "user.total_writes" + buf, err := xattr.LGet(s.formatFile, attr) + if err != nil { + // We start off with '0' if we can read the attributes + return 0 + } + + return binary.LittleEndian.Uint64(buf[:8]) +} + +func (s *xlStorage) setDeleteAttribute(deleteCount uint64) error { + attr := "user.total_deletes" + + data := make([]byte, 8) + binary.LittleEndian.PutUint64(data, deleteCount) + return xattr.LSet(s.formatFile, attr, data) +} + +func (s *xlStorage) setWriteAttribute(writeCount uint64) error { + attr := "user.total_writes" + + data := make([]byte, 8) + binary.LittleEndian.PutUint64(data, writeCount) + return xattr.LSet(s.formatFile, attr, data) +} + +// DiskInfo provides current information about disk space usage, +// total free inodes and underlying filesystem. +func (s *xlStorage) DiskInfo(ctx context.Context, _ DiskInfoOptions) (info DiskInfo, err error) { + info, err = s.diskInfoCache.GetWithCtx(ctx) + info.NRRequests = s.nrRequests + info.Rotational = s.rotational + info.MountPath = s.drivePath + info.Endpoint = s.endpoint.String() + info.Scanning = atomic.LoadInt32(&s.scanning) == 1 + return info, err +} + +// getVolDir - will convert incoming volume names to +// corresponding valid volume names on the backend in a platform +// compatible way for all operating systems. If volume is not found +// an error is generated. +func (s *xlStorage) getVolDir(volume string) (string, error) { + if volume == "" || volume == "." || volume == ".." { + return "", errVolumeNotFound + } + volumeDir := pathJoin(s.drivePath, volume) + return volumeDir, nil +} + +func (s *xlStorage) checkFormatJSON() (os.FileInfo, error) { + fi, err := Lstat(s.formatFile) + if err != nil { + // If the disk is still not initialized. + if osIsNotExist(err) { + if err = Access(s.drivePath); err == nil { + // Disk is present but missing `format.json` + return nil, errUnformattedDisk + } + if osIsNotExist(err) { + return nil, errDiskNotFound + } else if osIsPermission(err) { + return nil, errDiskAccessDenied + } + storageLogOnceIf(GlobalContext, err, "check-format-json") // log unexpected errors + return nil, errCorruptedBackend + } else if osIsPermission(err) { + return nil, errDiskAccessDenied + } + storageLogOnceIf(GlobalContext, err, "check-format-json") // log unexpected errors + return nil, errCorruptedBackend + } + return fi, nil +} + +// GetDiskID - returns the cached disk uuid +func (s *xlStorage) GetDiskID() (string, error) { + s.RLock() + diskID := s.diskID + fileInfo := s.formatFileInfo + lastCheck := s.formatLastCheck + + // check if we have a valid disk ID that is less than 1 seconds old. + if fileInfo != nil && diskID != "" && time.Since(lastCheck) <= 1*time.Second { + s.RUnlock() + return diskID, nil + } + s.RUnlock() + + fi, err := s.checkFormatJSON() + if err != nil { + return "", err + } + + if xioutil.SameFile(fi, fileInfo) && diskID != "" { + s.Lock() + // If the file has not changed, just return the cached diskID information. + s.formatLastCheck = time.Now() + s.Unlock() + return diskID, nil + } + + b, err := os.ReadFile(s.formatFile) + if err != nil { + // If the disk is still not initialized. + if osIsNotExist(err) { + if err = Access(s.drivePath); err == nil { + // Disk is present but missing `format.json` + return "", errUnformattedDisk + } + if osIsNotExist(err) { + return "", errDiskNotFound + } else if osIsPermission(err) { + return "", errDiskAccessDenied + } + storageLogOnceIf(GlobalContext, err, "check-format-json") // log unexpected errors + return "", errCorruptedBackend + } else if osIsPermission(err) { + return "", errDiskAccessDenied + } + storageLogOnceIf(GlobalContext, err, "check-format-json") // log unexpected errors + return "", errCorruptedBackend + } + + format := &formatErasureV3{} + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err = json.Unmarshal(b, &format); err != nil { + bugLogIf(GlobalContext, err) // log unexpected errors + return "", errCorruptedFormat + } + + m, n, err := findDiskIndexByDiskID(format, format.Erasure.This) + if err != nil { + return "", err + } + + diskID = format.Erasure.This + ep := s.endpoint + if m != ep.SetIdx || n != ep.DiskIdx { + storageLogOnceIf(GlobalContext, + fmt.Errorf("unexpected drive ordering on pool: %s: found drive at (set=%s, drive=%s), expected at (set=%s, drive=%s): %s(%s): %w", + humanize.Ordinal(ep.PoolIdx+1), humanize.Ordinal(m+1), humanize.Ordinal(n+1), humanize.Ordinal(ep.SetIdx+1), humanize.Ordinal(ep.DiskIdx+1), + s, s.diskID, errInconsistentDisk), "drive-order-format-json") + return "", errInconsistentDisk + } + s.Lock() + s.diskID = diskID + s.formatLegacy = format.Erasure.DistributionAlgo == formatErasureVersionV2DistributionAlgoV1 + s.formatFileInfo = fi + s.formatData = b + s.formatLastCheck = time.Now() + s.Unlock() + return diskID, nil +} + +// Make a volume entry. +func (s *xlStorage) SetDiskID(id string) { + // NO-OP for xlStorage as it is handled either by xlStorageDiskIDCheck{} for local disks or + // storage rest server for remote disks. +} + +func (s *xlStorage) MakeVolBulk(ctx context.Context, volumes ...string) error { + for _, volume := range volumes { + err := s.MakeVol(ctx, volume) + if err != nil && !errors.Is(err, errVolumeExists) { + return err + } + diskHealthCheckOK(ctx, err) + } + return nil +} + +// Make a volume entry. +func (s *xlStorage) MakeVol(ctx context.Context, volume string) error { + if !isValidVolname(volume) { + return errInvalidArgument + } + + volumeDir, err := s.getVolDir(volume) + if err != nil { + return err + } + + if err = Access(volumeDir); err != nil { + // Volume does not exist we proceed to create. + if osIsNotExist(err) { + // Make a volume entry, with mode 0777 mkdir honors system umask. + err = mkdirAll(volumeDir, 0o777, s.drivePath) + } + if osIsPermission(err) { + return errDiskAccessDenied + } else if isSysErrIO(err) { + return errFaultyDisk + } + return err + } + + // Stat succeeds we return errVolumeExists. + return errVolumeExists +} + +// ListVols - list volumes. +func (s *xlStorage) ListVols(ctx context.Context) (volsInfo []VolInfo, err error) { + return listVols(ctx, s.drivePath) +} + +// List all the volumes from drivePath. +func listVols(ctx context.Context, dirPath string) ([]VolInfo, error) { + if err := checkPathLength(dirPath); err != nil { + return nil, err + } + entries, err := readDir(dirPath) + if err != nil { + if errors.Is(err, errFileAccessDenied) { + return nil, errDiskAccessDenied + } else if errors.Is(err, errFileNotFound) { + return nil, errDiskNotFound + } + return nil, err + } + volsInfo := make([]VolInfo, 0, len(entries)) + for _, entry := range entries { + if !HasSuffix(entry, SlashSeparator) || !isValidVolname(pathutil.Clean(entry)) { + // Skip if entry is neither a directory not a valid volume name. + continue + } + volsInfo = append(volsInfo, VolInfo{ + Name: pathutil.Clean(entry), + }) + } + return volsInfo, nil +} + +// StatVol - get volume info. +func (s *xlStorage) StatVol(ctx context.Context, volume string) (vol VolInfo, err error) { + // Verify if volume is valid and it exists. + volumeDir, err := s.getVolDir(volume) + if err != nil { + return VolInfo{}, err + } + + // Stat a volume entry. + var st os.FileInfo + st, err = Lstat(volumeDir) + if err != nil { + switch { + case osIsNotExist(err): + return VolInfo{}, errVolumeNotFound + case osIsPermission(err): + return VolInfo{}, errDiskAccessDenied + case isSysErrIO(err): + return VolInfo{}, errFaultyDisk + default: + return VolInfo{}, err + } + } + // As os.Lstat() doesn't carry other than ModTime(), use ModTime() + // as CreatedTime. + createdTime := st.ModTime() + return VolInfo{ + Name: volume, + Created: createdTime, + }, nil +} + +// DeleteVol - delete a volume. +func (s *xlStorage) DeleteVol(ctx context.Context, volume string, forceDelete bool) (err error) { + // Verify if volume is valid and it exists. + volumeDir, err := s.getVolDir(volume) + if err != nil { + return err + } + + if forceDelete { + err = s.moveToTrash(volumeDir, true, true) + } else { + err = Remove(volumeDir) + } + + if err != nil { + switch { + case errors.Is(err, errFileNotFound): + return errVolumeNotFound + case osIsNotExist(err): + return errVolumeNotFound + case isSysErrNotEmpty(err): + return errVolumeNotEmpty + case osIsPermission(err): + return errDiskAccessDenied + case isSysErrIO(err): + return errFaultyDisk + default: + return err + } + } + return nil +} + +// ListDir - return all the entries at the given directory path. +// If an entry is a directory it will be returned with a trailing SlashSeparator. +func (s *xlStorage) ListDir(ctx context.Context, origvolume, volume, dirPath string, count int) (entries []string, err error) { + if contextCanceled(ctx) { + return nil, ctx.Err() + } + + if origvolume != "" { + if !skipAccessChecks(origvolume) { + origvolumeDir, err := s.getVolDir(origvolume) + if err != nil { + return nil, err + } + if err = Access(origvolumeDir); err != nil { + return nil, convertAccessError(err, errVolumeAccessDenied) + } + } + } + + // Verify if volume is valid and it exists. + volumeDir, err := s.getVolDir(volume) + if err != nil { + return nil, err + } + + dirPathAbs := pathJoin(volumeDir, dirPath) + if count > 0 { + entries, err = readDirN(dirPathAbs, count) + } else { + entries, err = readDir(dirPathAbs) + } + if err != nil { + if errors.Is(err, errFileNotFound) && !skipAccessChecks(volume) { + if ierr := Access(volumeDir); ierr != nil { + return nil, convertAccessError(ierr, errVolumeAccessDenied) + } + } + return nil, err + } + + return entries, nil +} + +func (s *xlStorage) deleteVersions(ctx context.Context, volume, path string, fis ...FileInfo) error { + volumeDir, err := s.getVolDir(volume) + if err != nil { + return err + } + + s.RLock() + legacy := s.formatLegacy + s.RUnlock() + + var legacyJSON bool + buf, err := xioutil.WithDeadline[[]byte](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) ([]byte, error) { + buf, _, err := s.readAllDataWithDMTime(ctx, volume, volumeDir, pathJoin(volumeDir, path, xlStorageFormatFile)) + if err != nil && !errors.Is(err, errFileNotFound) { + return nil, err + } + + if errors.Is(err, errFileNotFound) && legacy { + buf, _, err = s.readAllDataWithDMTime(ctx, volume, volumeDir, pathJoin(volumeDir, path, xlStorageFormatFileV1)) + if err != nil { + return nil, err + } + legacyJSON = true + } + + if len(buf) == 0 { + if errors.Is(err, errFileNotFound) && !skipAccessChecks(volume) { + if aerr := Access(volumeDir); aerr != nil && osIsNotExist(aerr) { + return nil, errVolumeNotFound + } + return nil, errFileNotFound + } + } + return buf, nil + }) + if err != nil { + return err + } + + if legacyJSON { + // Delete the meta file, if there are no more versions the + // top level parent is automatically removed. + return s.deleteFile(volumeDir, pathJoin(volumeDir, path), true, false) + } + + var xlMeta xlMetaV2 + if err = xlMeta.LoadOrConvert(buf); err != nil { + return err + } + + for _, fi := range fis { + dataDir, err := xlMeta.DeleteVersion(fi) + if err != nil { + if !fi.Deleted && (err == errFileNotFound || err == errFileVersionNotFound) { + // Ignore these since they do not exist + continue + } + return err + } + if dataDir != "" { + versionID := fi.VersionID + if versionID == "" { + versionID = nullVersionID + } + + // PR #11758 used DataDir, preserve it + // for users who might have used master + // branch + xlMeta.data.remove(versionID, dataDir) + + // We need to attempt delete "dataDir" on the disk + // due to a CopyObject() bug where it might have + // inlined the data incorrectly, to avoid a situation + // where we potentially leave "DataDir" + filePath := pathJoin(volumeDir, path, dataDir) + if err = checkPathLength(filePath); err != nil { + return err + } + if err = s.moveToTrash(filePath, true, false); err != nil { + if err != errFileNotFound { + return err + } + } + } + } + + lastVersion := len(xlMeta.versions) == 0 + if !lastVersion { + buf, err = xlMeta.AppendTo(metaDataPoolGet()) + defer metaDataPoolPut(buf) + if err != nil { + return err + } + + return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), buf) + } + + return s.deleteFile(volumeDir, pathJoin(volumeDir, path, xlStorageFormatFile), true, false) +} + +// DeleteVersions deletes slice of versions, it can be same object +// or multiple objects. +func (s *xlStorage) DeleteVersions(ctx context.Context, volume string, versions []FileInfoVersions, opts DeleteOptions) []error { + errs := make([]error, len(versions)) + + for i, fiv := range versions { + if contextCanceled(ctx) { + errs[i] = ctx.Err() + continue + } + errs[i] = s.deleteVersions(ctx, volume, fiv.Name, fiv.Versions...) + diskHealthCheckOK(ctx, errs[i]) + } + + return errs +} + +func (s *xlStorage) cleanupTrashImmediateCallers(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case entry := <-s.immediatePurge: + // Add deadlines such that immediate purge is not + // perpetually hung here. + w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) + w.Run(func() error { + return removeAll(entry) + }) + } + } +} + +const almostFilledPercent = 0.05 + +func (s *xlStorage) diskAlmostFilled() bool { + info, err := s.diskInfoCache.Get() + if err != nil { + return false + } + if info.Used == 0 || info.UsedInodes == 0 { + return false + } + return (float64(info.Free)/float64(info.Used)) < almostFilledPercent || (float64(info.FreeInodes)/float64(info.UsedInodes)) < almostFilledPercent +} + +func (s *xlStorage) moveToTrashNoDeadline(filePath string, recursive, immediatePurge bool) (err error) { + pathUUID := mustGetUUID() + targetPath := pathutil.Join(s.drivePath, minioMetaTmpDeletedBucket, pathUUID) + + if recursive { + err = renameAll(filePath, targetPath, pathutil.Join(s.drivePath, minioMetaBucket)) + } else { + err = Rename(filePath, targetPath) + } + + var targetPath2 string + if immediatePurge && HasSuffix(filePath, SlashSeparator) { + // With immediate purge also attempt deleting for `__XL_DIR__` folder/directory objects. + targetPath2 = pathutil.Join(s.drivePath, minioMetaTmpDeletedBucket, mustGetUUID()) + renameAll(encodeDirObject(filePath), targetPath2, pathutil.Join(s.drivePath, minioMetaBucket)) + } + + // ENOSPC is a valid error from rename(); remove instead of rename in that case + if errors.Is(err, errDiskFull) || isSysErrNoSpace(err) { + if recursive { + err = removeAll(filePath) + } else { + err = Remove(filePath) + } + return err // Avoid the immediate purge since not needed + } + + if err != nil { + return err + } + + if !immediatePurge && s.diskAlmostFilled() { + immediatePurge = true + } + + // immediately purge the target + if immediatePurge { + for _, target := range []string{ + targetPath, + targetPath2, + } { + if target == "" { + continue + } + select { + case s.immediatePurge <- target: + default: + // Too much back pressure, we will perform the delete + // blocking at this point we need to serialize operations. + removeAll(target) + } + } + } + return nil +} + +func (s *xlStorage) readAllData(ctx context.Context, volume, volumeDir string, filePath string) (buf []byte, err error) { + return xioutil.WithDeadline[[]byte](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) ([]byte, error) { + data, _, err := s.readAllDataWithDMTime(ctx, volume, volumeDir, filePath) + return data, err + }) +} + +func (s *xlStorage) moveToTrash(filePath string, recursive, immediatePurge bool) (err error) { + w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) + return w.Run(func() (err error) { + return s.moveToTrashNoDeadline(filePath, recursive, immediatePurge) + }) +} + +// DeleteVersion - deletes FileInfo metadata for path at `xl.meta`. forceDelMarker +// will force creating a new `xl.meta` to create a new delete marker +func (s *xlStorage) DeleteVersion(ctx context.Context, volume, path string, fi FileInfo, forceDelMarker bool, opts DeleteOptions) (err error) { + if HasSuffix(path, SlashSeparator) { + return s.Delete(ctx, volume, path, DeleteOptions{ + Recursive: false, + Immediate: false, + }) + } + + volumeDir, err := s.getVolDir(volume) + if err != nil { + return err + } + + // Validate file path length, before reading. + filePath := pathJoin(volumeDir, path) + if err = checkPathLength(filePath); err != nil { + return err + } + + var legacyJSON bool + buf, err := s.readAllData(ctx, volume, volumeDir, pathJoin(filePath, xlStorageFormatFile)) + if err != nil { + if !errors.Is(err, errFileNotFound) { + return err + } + metaDataPoolPut(buf) // Never used, return it + if fi.Deleted && forceDelMarker { + // Create a new xl.meta with a delete marker in it + return s.WriteMetadata(ctx, "", volume, path, fi) + } + + s.RLock() + legacy := s.formatLegacy + s.RUnlock() + if legacy { + buf, err = s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFileV1)) + if err != nil { + if errors.Is(err, errFileNotFound) && fi.VersionID != "" { + return errFileVersionNotFound + } + return err + } + legacyJSON = true + } + } + + if len(buf) == 0 { + if fi.VersionID != "" { + return errFileVersionNotFound + } + return errFileNotFound + } + + if legacyJSON { + // Delete the meta file, if there are no more versions the + // top level parent is automatically removed. + return s.deleteFile(volumeDir, pathJoin(volumeDir, path), true, false) + } + + var xlMeta xlMetaV2 + if err = xlMeta.LoadOrConvert(buf); err != nil { + return err + } + + dataDir, err := xlMeta.DeleteVersion(fi) + if err != nil { + return err + } + if dataDir != "" { + versionID := fi.VersionID + if versionID == "" { + versionID = nullVersionID + } + // PR #11758 used DataDir, preserve it + // for users who might have used master + // branch + xlMeta.data.remove(versionID, dataDir) + + // We need to attempt delete "dataDir" on the disk + // due to a CopyObject() bug where it might have + // inlined the data incorrectly, to avoid a situation + // where we potentially leave "DataDir" + filePath := pathJoin(volumeDir, path, dataDir) + if err = checkPathLength(filePath); err != nil { + return err + } + if err = s.moveToTrash(filePath, true, false); err != nil { + if err != errFileNotFound { + return err + } + } + } + + if len(xlMeta.versions) != 0 { + // xl.meta must still exist for other versions, dataDir is purged. + buf, err = xlMeta.AppendTo(metaDataPoolGet()) + defer metaDataPoolPut(buf) + if err != nil { + return err + } + + return s.writeAllMeta(ctx, volume, pathJoin(path, xlStorageFormatFile), buf, true) + } + + if opts.UndoWrite && opts.OldDataDir != "" { + return renameAll(pathJoin(filePath, opts.OldDataDir, xlStorageFormatFileBackup), pathJoin(filePath, xlStorageFormatFile), filePath) + } + + return s.deleteFile(volumeDir, pathJoin(volumeDir, path, xlStorageFormatFile), true, false) +} + +// Updates only metadata for a given version. +func (s *xlStorage) UpdateMetadata(ctx context.Context, volume, path string, fi FileInfo, opts UpdateMetadataOpts) (err error) { + if len(fi.Metadata) == 0 { + return errInvalidArgument + } + + volumeDir, err := s.getVolDir(volume) + if err != nil { + return err + } + + // Validate file path length, before reading. + filePath := pathJoin(volumeDir, path) + if err = checkPathLength(filePath); err != nil { + return err + } + + buf, err := s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile)) + if err != nil { + if err == errFileNotFound && fi.VersionID != "" { + return errFileVersionNotFound + } + return err + } + defer metaDataPoolPut(buf) + + if !isXL2V1Format(buf) { + return errFileVersionNotFound + } + + var xlMeta xlMetaV2 + if err = xlMeta.Load(buf); err != nil { + return err + } + + if err = xlMeta.UpdateObjectVersion(fi); err != nil { + return err + } + + wbuf, err := xlMeta.AppendTo(metaDataPoolGet()) + if err != nil { + return err + } + defer metaDataPoolPut(wbuf) + + return s.writeAllMeta(ctx, volume, pathJoin(path, xlStorageFormatFile), wbuf, !opts.NoPersistence) +} + +// WriteMetadata - writes FileInfo metadata for path at `xl.meta` +func (s *xlStorage) WriteMetadata(ctx context.Context, origvolume, volume, path string, fi FileInfo) (err error) { + if fi.Fresh { + if origvolume != "" { + origvolumeDir, err := s.getVolDir(origvolume) + if err != nil { + return err + } + + if !skipAccessChecks(origvolume) { + // Stat a volume entry. + if err = Access(origvolumeDir); err != nil { + return convertAccessError(err, errVolumeAccessDenied) + } + } + } + + var xlMeta xlMetaV2 + if err := xlMeta.AddVersion(fi); err != nil { + return err + } + buf, err := xlMeta.AppendTo(metaDataPoolGet()) + defer metaDataPoolPut(buf) + if err != nil { + return err + } + // First writes for special situations do not write to stable storage. + // this is currently used by + // - emphemeral objects such as objects created during listObjects() calls + ok := volume == minioMetaMultipartBucket // - newMultipartUpload() call must be synced to drives. + return s.writeAll(ctx, volume, pathJoin(path, xlStorageFormatFile), buf, ok, "") + } + + buf, err := s.ReadAll(ctx, volume, pathJoin(path, xlStorageFormatFile)) + if err != nil && err != errFileNotFound { + return err + } + defer metaDataPoolPut(buf) + + var xlMeta xlMetaV2 + if !isXL2V1Format(buf) { + // This is both legacy and without proper version. + if err = xlMeta.AddVersion(fi); err != nil { + return err + } + + buf, err = xlMeta.AppendTo(metaDataPoolGet()) + defer metaDataPoolPut(buf) + if err != nil { + return err + } + } else { + if err = xlMeta.Load(buf); err != nil { + // Corrupted data, reset and write. + xlMeta = xlMetaV2{} + } + + if err = xlMeta.AddVersion(fi); err != nil { + return err + } + + buf, err = xlMeta.AppendTo(metaDataPoolGet()) + defer metaDataPoolPut(buf) + if err != nil { + return err + } + } + + return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), buf) +} + +func (s *xlStorage) renameLegacyMetadata(volumeDir, path string) (err error) { + s.RLock() + legacy := s.formatLegacy + s.RUnlock() + if !legacy { + // if its not a legacy backend then this function is + // a no-op always returns errFileNotFound + return errFileNotFound + } + + // Validate file path length, before reading. + filePath := pathJoin(volumeDir, path) + if err = checkPathLength(filePath); err != nil { + return err + } + + srcFilePath := pathJoin(filePath, xlStorageFormatFileV1) + dstFilePath := pathJoin(filePath, xlStorageFormatFile) + + // Renaming xl.json to xl.meta should be fully synced to disk. + defer func() { + if err == nil && s.globalSync { + // Sync to disk only upon success. + globalSync() + } + }() + + if err = Rename(srcFilePath, dstFilePath); err != nil { + switch { + case isSysErrNotDir(err): + return errFileNotFound + case isSysErrPathNotFound(err): + return errFileNotFound + case isSysErrCrossDevice(err): + return fmt.Errorf("%w (%s)->(%s)", errCrossDeviceLink, srcFilePath, dstFilePath) + case osIsNotExist(err): + return errFileNotFound + case osIsExist(err): + // This is returned only when destination is a directory and we + // are attempting a rename from file to directory. + return errIsNotRegular + default: + return err + } + } + return nil +} + +func (s *xlStorage) readRaw(ctx context.Context, volume, volumeDir, filePath string, readData bool) (buf []byte, dmTime time.Time, err error) { + if filePath == "" { + return nil, dmTime, errFileNotFound + } + + xlPath := pathJoin(filePath, xlStorageFormatFile) + if readData { + buf, dmTime, err = s.readAllDataWithDMTime(ctx, volume, volumeDir, xlPath) + } else { + buf, dmTime, err = s.readMetadataWithDMTime(ctx, xlPath) + if err != nil { + if osIsNotExist(err) { + if !skipAccessChecks(volume) { + if aerr := Access(volumeDir); aerr != nil && osIsNotExist(aerr) { + return nil, time.Time{}, errVolumeNotFound + } + } + } + err = osErrToFileErr(err) + } + } + + s.RLock() + legacy := s.formatLegacy + s.RUnlock() + + if err != nil && errors.Is(err, errFileNotFound) && legacy { + buf, dmTime, err = s.readAllDataWithDMTime(ctx, volume, volumeDir, pathJoin(filePath, xlStorageFormatFileV1)) + if err != nil { + return nil, time.Time{}, err + } + } + + if len(buf) == 0 { + if err != nil { + return nil, time.Time{}, err + } + return nil, time.Time{}, errFileNotFound + } + + return buf, dmTime, nil +} + +// ReadXL reads from path/xl.meta, does not interpret the data it read. This +// is a raw call equivalent of ReadVersion(). +func (s *xlStorage) ReadXL(ctx context.Context, volume, path string, readData bool) (RawFileInfo, error) { + volumeDir, err := s.getVolDir(volume) + if err != nil { + return RawFileInfo{}, err + } + + // Validate file path length, before reading. + filePath := pathJoin(volumeDir, path) + if err = checkPathLength(filePath); err != nil { + return RawFileInfo{}, err + } + + buf, _, err := s.readRaw(ctx, volume, volumeDir, filePath, readData) + return RawFileInfo{ + Buf: buf, + }, err +} + +// ReadOptions optional inputs for ReadVersion +type ReadOptions struct { + InclFreeVersions bool + ReadData bool + Healing bool +} + +// ReadVersion - reads metadata and returns FileInfo at path `xl.meta` +// for all objects less than `32KiB` this call returns data as well +// along with metadata. +func (s *xlStorage) ReadVersion(ctx context.Context, origvolume, volume, path, versionID string, opts ReadOptions) (fi FileInfo, err error) { + if origvolume != "" { + origvolumeDir, err := s.getVolDir(origvolume) + if err != nil { + return fi, err + } + + if !skipAccessChecks(origvolume) { + // Stat a volume entry. + if err = Access(origvolumeDir); err != nil { + return fi, convertAccessError(err, errVolumeAccessDenied) + } + } + } + + volumeDir, err := s.getVolDir(volume) + if err != nil { + return fi, err + } + + // Validate file path length, before reading. + filePath := pathJoin(volumeDir, path) + if err = checkPathLength(filePath); err != nil { + return fi, err + } + + readData := opts.ReadData + + buf, _, err := s.readRaw(ctx, volume, volumeDir, filePath, readData) + if err != nil { + if err == errFileNotFound { + if versionID != "" { + return fi, errFileVersionNotFound + } + } + return fi, err + } + + fi, err = getFileInfo(buf, volume, path, versionID, fileInfoOpts{ + Data: opts.ReadData, + InclFreeVersions: opts.InclFreeVersions, + }) + if err != nil { + return fi, err + } + + if len(fi.Data) == 0 { + // We did not read inline data, so we have no references. + defer metaDataPoolPut(buf) + } + + if readData { + if len(fi.Data) > 0 || fi.Size == 0 { + if fi.InlineData() { + // If written with header we are fine. + return fi, nil + } + if fi.Size == 0 || (fi.VersionID == "" || fi.VersionID == nullVersionID) { + // If versioned we have no conflicts. + fi.SetInlineData() + return fi, nil + } + + // For overwritten objects without header we might have a + // conflict with data written later. Check the data path + // if there is a part with data. + partPath := fmt.Sprintf("part.%d", fi.Parts[0].Number) + dataPath := pathJoin(path, fi.DataDir, partPath) + _, lerr := Lstat(pathJoin(volumeDir, dataPath)) + if lerr != nil { + // Set the inline header, our inlined data is fine. + fi.SetInlineData() + return fi, nil + } + // Data exists on disk, remove the version from metadata. + fi.Data = nil + } + + attemptInline := fi.TransitionStatus == "" && fi.DataDir != "" && len(fi.Parts) == 1 + // Reading data for small objects when + // - object has not yet transitioned + // - object has maximum of 1 parts + if attemptInline { + inlineBlock := globalStorageClass.InlineBlock() + if inlineBlock <= 0 { + inlineBlock = 128 * humanize.KiByte + } + + canInline := fi.ShardFileSize(fi.Parts[0].ActualSize) <= inlineBlock + if canInline { + dataPath := pathJoin(volumeDir, path, fi.DataDir, fmt.Sprintf("part.%d", fi.Parts[0].Number)) + fi.Data, err = s.readAllData(ctx, volume, volumeDir, dataPath) + if err != nil { + return FileInfo{}, err + } + } + } + } + + return fi, nil +} + +func (s *xlStorage) readAllDataWithDMTime(ctx context.Context, volume, volumeDir string, filePath string) (buf []byte, dmTime time.Time, err error) { + if filePath == "" { + return nil, dmTime, errFileNotFound + } + + if contextCanceled(ctx) { + return nil, time.Time{}, ctx.Err() + } + + f, err := OpenFile(filePath, readMode, 0o666) + if err != nil { + switch { + case osIsNotExist(err): + // Check if the object doesn't exist because its bucket + // is missing in order to return the correct error. + if !skipAccessChecks(volume) { + if err = Access(volumeDir); err != nil && osIsNotExist(err) { + return nil, dmTime, errVolumeNotFound + } + } + return nil, dmTime, errFileNotFound + case osIsPermission(err): + return nil, dmTime, errFileAccessDenied + case isSysErrNotDir(err) || isSysErrIsDir(err): + return nil, dmTime, errFileNotFound + case isSysErrHandleInvalid(err): + // This case is special and needs to be handled for windows. + return nil, dmTime, errFileNotFound + case isSysErrIO(err): + return nil, dmTime, errFaultyDisk + case isSysErrTooManyFiles(err): + return nil, dmTime, errTooManyOpenFiles + case isSysErrInvalidArg(err): + st, _ := Lstat(filePath) + if st != nil && st.IsDir() { + // Linux returns InvalidArg for directory O_DIRECT + // we need to keep this fallback code to return correct + // errors upwards. + return nil, dmTime, errFileNotFound + } + return nil, dmTime, errUnsupportedDisk + } + return nil, dmTime, err + } + defer f.Close() + + // Get size for precise allocation. + stat, err := f.Stat() + if err != nil { + buf, err = io.ReadAll(f) + return buf, dmTime, osErrToFileErr(err) + } + if stat.IsDir() { + return nil, dmTime, errFileNotFound + } + + // Read into appropriate buffer. + sz := stat.Size() + if sz <= metaDataReadDefault { + buf = metaDataPoolGet() + buf = buf[:sz] + } else { + buf = make([]byte, sz) + } + + // Read file... + _, err = io.ReadFull(f, buf) + + return buf, stat.ModTime().UTC(), osErrToFileErr(err) +} + +// ReadAll is a raw call, reads content at any path and returns the buffer. +func (s *xlStorage) ReadAll(ctx context.Context, volume string, path string) (buf []byte, err error) { + // Specific optimization to avoid re-read from the drives for `format.json` + // in-case the caller is a network operation. + if volume == minioMetaBucket && path == formatConfigFile { + s.RLock() + formatData := make([]byte, len(s.formatData)) + copy(formatData, s.formatData) + s.RUnlock() + if len(formatData) > 0 { + return formatData, nil + } + } + volumeDir, err := s.getVolDir(volume) + if err != nil { + return nil, err + } + + // Validate file path length, before reading. + filePath := pathJoin(volumeDir, path) + if err = checkPathLength(filePath); err != nil { + return nil, err + } + + return s.readAllData(ctx, volume, volumeDir, filePath) +} + +// ReadFile reads exactly len(buf) bytes into buf. It returns the +// number of bytes copied. The error is EOF only if no bytes were +// read. On return, n == len(buf) if and only if err == nil. n == 0 +// for io.EOF. +// +// If an EOF happens after reading some but not all the bytes, +// ReadFile returns ErrUnexpectedEOF. +// +// If the BitrotVerifier is not nil or not verified ReadFile +// tries to verify whether the disk has bitrot. +// +// Additionally ReadFile also starts reading from an offset. ReadFile +// semantics are same as io.ReadFull. +func (s *xlStorage) ReadFile(ctx context.Context, volume string, path string, offset int64, buffer []byte, verifier *BitrotVerifier) (int64, error) { + if offset < 0 { + return 0, errInvalidArgument + } + + volumeDir, err := s.getVolDir(volume) + if err != nil { + return 0, err + } + + var n int + + if !skipAccessChecks(volume) { + // Stat a volume entry. + if err = Access(volumeDir); err != nil { + return 0, convertAccessError(err, errFileAccessDenied) + } + } + + // Validate effective path length before reading. + filePath := pathJoin(volumeDir, path) + if err = checkPathLength(filePath); err != nil { + return 0, err + } + + // Open the file for reading. + file, err := OpenFile(filePath, readMode, 0o666) + if err != nil { + switch { + case osIsNotExist(err): + return 0, errFileNotFound + case osIsPermission(err): + return 0, errFileAccessDenied + case isSysErrNotDir(err): + return 0, errFileAccessDenied + case isSysErrIO(err): + return 0, errFaultyDisk + case isSysErrTooManyFiles(err): + return 0, errTooManyOpenFiles + default: + return 0, err + } + } + + // Close the file descriptor. + defer file.Close() + + st, err := file.Stat() + if err != nil { + return 0, err + } + + // Verify it is a regular file, otherwise subsequent Seek is + // undefined. + if !st.Mode().IsRegular() { + return 0, errIsNotRegular + } + + if verifier == nil { + n, err = file.ReadAt(buffer, offset) + return int64(n), err + } + + h := verifier.algorithm.New() + if _, err = io.Copy(h, io.LimitReader(file, offset)); err != nil { + return 0, err + } + + if n, err = io.ReadFull(file, buffer); err != nil { + return int64(n), err + } + + if _, err = h.Write(buffer); err != nil { + return 0, err + } + + if _, err = io.Copy(h, file); err != nil { + return 0, err + } + + if !bytes.Equal(h.Sum(nil), verifier.sum) { + return 0, errFileCorrupt + } + + return int64(len(buffer)), nil +} + +func (s *xlStorage) openFileDirect(path string, mode int) (f *os.File, err error) { + w, err := OpenFileDirectIO(path, mode, 0o666) + if err != nil { + switch { + case isSysErrInvalidArg(err): + return nil, errUnsupportedDisk + case osIsPermission(err): + return nil, errDiskAccessDenied + case isSysErrIO(err): + return nil, errFaultyDisk + case isSysErrNotDir(err): + return nil, errDiskNotDir + case os.IsNotExist(err): + return nil, errDiskNotFound + } + } + + return w, nil +} + +func (s *xlStorage) openFileSync(filePath string, mode int, skipParent string) (f *os.File, err error) { + return s.openFile(filePath, mode|writeMode, skipParent) +} + +func (s *xlStorage) openFile(filePath string, mode int, skipParent string) (f *os.File, err error) { + if skipParent == "" { + skipParent = s.drivePath + } + // Create top level directories if they don't exist. + // with mode 0777 mkdir honors system umask. + if err = mkdirAll(pathutil.Dir(filePath), 0o777, skipParent); err != nil { + return nil, osErrToFileErr(err) + } + + w, err := OpenFile(filePath, mode, 0o666) + if err != nil { + // File path cannot be verified since one of the parents is a file. + switch { + case isSysErrIsDir(err): + return nil, errIsNotRegular + case osIsPermission(err): + return nil, errFileAccessDenied + case isSysErrNotDir(err): + return nil, errFileAccessDenied + case isSysErrIO(err): + return nil, errFaultyDisk + case isSysErrTooManyFiles(err): + return nil, errTooManyOpenFiles + default: + return nil, err + } + } + + return w, nil +} + +type sendFileReader struct { + io.Reader + io.Closer +} + +// ReadFileStream - Returns the read stream of the file. +func (s *xlStorage) ReadFileStream(ctx context.Context, volume, path string, offset, length int64) (io.ReadCloser, error) { + if offset < 0 { + return nil, errInvalidArgument + } + + volumeDir, err := s.getVolDir(volume) + if err != nil { + return nil, err + } + + // Validate effective path length before reading. + filePath := pathJoin(volumeDir, path) + if err = checkPathLength(filePath); err != nil { + return nil, err + } + + file, err := OpenFile(filePath, readMode, 0o666) + if err != nil { + switch { + case osIsNotExist(err): + if !skipAccessChecks(volume) { + if err = Access(volumeDir); err != nil && osIsNotExist(err) { + return nil, errVolumeNotFound + } + } + return nil, errFileNotFound + case osIsPermission(err): + return nil, errFileAccessDenied + case isSysErrNotDir(err): + return nil, errFileAccessDenied + case isSysErrIO(err): + return nil, errFaultyDisk + case isSysErrTooManyFiles(err): + return nil, errTooManyOpenFiles + case isSysErrInvalidArg(err): + return nil, errUnsupportedDisk + default: + return nil, err + } + } + + if length < 0 { + return file, nil + } + + st, err := file.Stat() + if err != nil { + file.Close() + return nil, err + } + + // Verify it is a regular file, otherwise subsequent Seek is + // undefined. + if !st.Mode().IsRegular() { + file.Close() + return nil, errIsNotRegular + } + + if st.Size() < offset+length { + // Expected size cannot be satisfied for + // requested offset and length + file.Close() + return nil, errFileCorrupt + } + + if offset > 0 { + if _, err = file.Seek(offset, io.SeekStart); err != nil { + file.Close() + return nil, err + } + } + return &sendFileReader{Reader: io.LimitReader(file, length), Closer: file}, nil +} + +// CreateFile - creates the file. +func (s *xlStorage) CreateFile(ctx context.Context, origvolume, volume, path string, fileSize int64, r io.Reader) (err error) { + if origvolume != "" { + origvolumeDir, err := s.getVolDir(origvolume) + if err != nil { + return err + } + + if !skipAccessChecks(origvolume) { + // Stat a volume entry. + if err = Access(origvolumeDir); err != nil { + return convertAccessError(err, errVolumeAccessDenied) + } + } + } + + volumeDir, err := s.getVolDir(volume) + if err != nil { + return err + } + + filePath := pathJoin(volumeDir, path) + if err = checkPathLength(filePath); err != nil { + return err + } + + parentFilePath := pathutil.Dir(filePath) + defer func() { + if err != nil { + if volume == minioMetaTmpBucket { + // only cleanup parent path if the + // parent volume name is minioMetaTmpBucket + removeAll(parentFilePath) + } + } + }() + + return s.writeAllDirect(ctx, filePath, fileSize, r, os.O_CREATE|os.O_WRONLY|os.O_EXCL, volumeDir, false) +} + +func (s *xlStorage) writeAllDirect(ctx context.Context, filePath string, fileSize int64, r io.Reader, flags int, skipParent string, truncate bool) (err error) { + if contextCanceled(ctx) { + return ctx.Err() + } + + if skipParent == "" { + skipParent = s.drivePath + } + + // Create top level directories if they don't exist. + // with mode 0777 mkdir honors system umask. + parentFilePath := pathutil.Dir(filePath) + if err = mkdirAll(parentFilePath, 0o777, skipParent); err != nil { + return osErrToFileErr(err) + } + + odirectEnabled := globalAPIConfig.odirectEnabled() && s.oDirect && fileSize > 0 + + var w *os.File + if odirectEnabled { + w, err = OpenFileDirectIO(filePath, flags, 0o666) + } else { + w, err = OpenFile(filePath, flags, 0o666) + } + if err != nil { + return osErrToFileErr(err) + } + + var bufp *[]byte + switch { + case fileSize <= xioutil.SmallBlock: + bufp = xioutil.ODirectPoolSmall.Get() + defer xioutil.ODirectPoolSmall.Put(bufp) + default: + bufp = xioutil.ODirectPoolLarge.Get() + defer xioutil.ODirectPoolLarge.Put(bufp) + } + + var written int64 + if odirectEnabled { + written, err = xioutil.CopyAligned(diskHealthWriter(ctx, w), r, *bufp, fileSize, w) + } else { + written, err = io.CopyBuffer(diskHealthWriter(ctx, w), r, *bufp) + } + if err != nil { + w.Close() + return err + } + + if written < fileSize && fileSize >= 0 { + if truncate { + w.Truncate(0) // zero-in the file size to indicate that its unreadable + } + w.Close() + return errLessData + } else if written > fileSize && fileSize >= 0 { + if truncate { + w.Truncate(0) // zero-in the file size to indicate that its unreadable + } + w.Close() + return errMoreData + } + + // Only interested in flushing the size_t not mtime/atime + if err = Fdatasync(w); err != nil { + w.Close() + return err + } + + // Dealing with error returns from close() - 'man 2 close' + // + // A careful programmer will check the return value of close(), since it is quite possible that + // errors on a previous write(2) operation are reported only on the final close() that releases + // the open file descriptor. + // + // Failing to check the return value when closing a file may lead to silent loss of data. + // This can especially be observed with NFS and with disk quota. + return w.Close() +} + +// writeAllMeta - writes all metadata to a temp file and then links it to the final destination. +func (s *xlStorage) writeAllMeta(ctx context.Context, volume string, path string, b []byte, sync bool) (err error) { + if contextCanceled(ctx) { + return ctx.Err() + } + + volumeDir, err := s.getVolDir(volume) + if err != nil { + return err + } + + filePath := pathJoin(volumeDir, path) + if err = checkPathLength(filePath); err != nil { + return err + } + + tmpVolumeDir, err := s.getVolDir(minioMetaTmpBucket) + if err != nil { + return err + } + + tmpFilePath := pathJoin(tmpVolumeDir, mustGetUUID()) + defer func() { + if err != nil { + Remove(tmpFilePath) + } + }() + + if err = s.writeAllInternal(ctx, tmpFilePath, b, sync, tmpVolumeDir); err != nil { + return err + } + + return renameAll(tmpFilePath, filePath, volumeDir) +} + +// Create or truncate an existing file before writing +func (s *xlStorage) writeAllInternal(ctx context.Context, filePath string, b []byte, sync bool, skipParent string) (err error) { + flags := os.O_CREATE | os.O_WRONLY | os.O_TRUNC + + var w *os.File + if sync { + // Perform DirectIO along with fdatasync for larger xl.meta, mostly when + // xl.meta has "inlined data" we prefer writing O_DIRECT and then doing + // fdatasync() at the end instead of opening the file with O_DSYNC. + // + // This is an optimization mainly to ensure faster I/O. + if len(b) > xioutil.DirectioAlignSize { + r := bytes.NewReader(b) + return s.writeAllDirect(ctx, filePath, r.Size(), r, flags, skipParent, true) + } + w, err = s.openFileSync(filePath, flags, skipParent) + } else { + w, err = s.openFile(filePath, flags, skipParent) + } + if err != nil { + return err + } + + _, err = w.Write(b) + if err != nil { + w.Truncate(0) // to indicate that we did partial write. + w.Close() + return err + } + + // Dealing with error returns from close() - 'man 2 close' + // + // A careful programmer will check the return value of close(), since it is quite possible that + // errors on a previous write(2) operation are reported only on the final close() that releases + // the open file descriptor. + // + // Failing to check the return value when closing a file may lead to silent loss of data. + // This can especially be observed with NFS and with disk quota. + return w.Close() +} + +func (s *xlStorage) writeAll(ctx context.Context, volume string, path string, b []byte, sync bool, skipParent string) (err error) { + if contextCanceled(ctx) { + return ctx.Err() + } + + volumeDir, err := s.getVolDir(volume) + if err != nil { + return err + } + + filePath := pathJoin(volumeDir, path) + if err = checkPathLength(filePath); err != nil { + return err + } + + return s.writeAllInternal(ctx, filePath, b, sync, skipParent) +} + +func (s *xlStorage) WriteAll(ctx context.Context, volume string, path string, b []byte) (err error) { + // Specific optimization to avoid re-read from the drives for `format.json` + // in-case the caller is a network operation. + if volume == minioMetaBucket && path == formatConfigFile { + s.Lock() + s.formatData = b + s.Unlock() + } + + volumeDir, err := s.getVolDir(volume) + if err != nil { + return err + } + + return s.writeAll(ctx, volume, path, b, true, volumeDir) +} + +// AppendFile - append a byte array at path, if file doesn't exist at +// path this call explicitly creates it. +func (s *xlStorage) AppendFile(ctx context.Context, volume string, path string, buf []byte) (err error) { + volumeDir, err := s.getVolDir(volume) + if err != nil { + return err + } + + if !skipAccessChecks(volume) { + // Stat a volume entry. + if err = Access(volumeDir); err != nil { + return convertAccessError(err, errVolumeAccessDenied) + } + } + + filePath := pathJoin(volumeDir, path) + if err = checkPathLength(filePath); err != nil { + return err + } + + var w *os.File + // Create file if not found. Not doing O_DIRECT here to avoid the code that does buffer aligned writes. + // AppendFile() is only used by healing code to heal objects written in old format. + w, err = s.openFileSync(filePath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, volumeDir) + if err != nil { + return err + } + defer w.Close() + + n, err := w.Write(buf) + if err != nil { + return err + } + + if n != len(buf) { + return io.ErrShortWrite + } + + return nil +} + +// checkPart is a light check of an existing and size of a part, without doing a bitrot operation +// For any unexpected error, return checkPartUnknown (zero) +func (s *xlStorage) checkPart(volumeDir, path, dataDir string, partNum int, expectedSize int64, skipAccessCheck bool) (resp int) { + partPath := pathJoin(path, dataDir, fmt.Sprintf("part.%d", partNum)) + filePath := pathJoin(volumeDir, partPath) + st, err := Lstat(filePath) + if err != nil { + if osIsNotExist(err) { + if !skipAccessCheck { + // Stat a volume entry. + if verr := Access(volumeDir); verr != nil { + if osIsNotExist(verr) { + resp = checkPartVolumeNotFound + } + return resp + } + } + } + if osErrToFileErr(err) == errFileNotFound { + resp = checkPartFileNotFound + } + return resp + } + if st.Mode().IsDir() { + resp = checkPartFileNotFound + return resp + } + // Check if shard is truncated. + if st.Size() < expectedSize { + resp = checkPartFileCorrupt + return resp + } + return checkPartSuccess +} + +// CheckParts check if path has necessary parts available. +func (s *xlStorage) CheckParts(ctx context.Context, volume string, path string, fi FileInfo) (*CheckPartsResp, error) { + volumeDir, err := s.getVolDir(volume) + if err != nil { + return nil, err + } + + err = checkPathLength(pathJoin(volumeDir, path)) + if err != nil { + return nil, err + } + + resp := CheckPartsResp{ + // By default, all results have an unknown status + Results: make([]int, len(fi.Parts)), + } + + for i, part := range fi.Parts { + resp.Results[i], err = xioutil.WithDeadline[int](ctx, globalDriveConfig.GetMaxTimeout(), func(ctx context.Context) (int, error) { + return s.checkPart(volumeDir, path, fi.DataDir, part.Number, fi.Erasure.ShardFileSize(part.Size), skipAccessChecks(volume)), nil + }) + if err != nil { + return nil, err + } + } + + return &resp, nil +} + +// deleteFile deletes a file or a directory if its empty unless recursive +// is set to true. If the target is successfully deleted, it will recursively +// move up the tree, deleting empty parent directories until it finds one +// with files in it. Returns nil for a non-empty directory even when +// recursive is set to false. +func (s *xlStorage) deleteFile(basePath, deletePath string, recursive, immediate bool) error { + if basePath == "" || deletePath == "" { + return nil + } + + bp := pathutil.Clean(basePath) // do not override basepath / or deletePath / + dp := pathutil.Clean(deletePath) + if !strings.HasPrefix(dp, bp) || dp == bp { + return nil + } + + var err error + if recursive { + err = s.moveToTrash(deletePath, true, immediate) + } else { + err = Remove(deletePath) + } + if err != nil { + switch { + case isSysErrNotEmpty(err): + // if object is a directory, but if its not empty + // return FileNotFound to indicate its an empty prefix. + if HasSuffix(deletePath, SlashSeparator) { + return errFileNotFound + } + // if we have .DS_Store only on macOS + if runtime.GOOS == globalMacOSName { + storeFilePath := pathJoin(deletePath, ".DS_Store") + _, err := Stat(storeFilePath) + // .DS_Store exists + if err == nil { + // delete first + Remove(storeFilePath) + // try again + Remove(deletePath) + } + } + // Ignore errors if the directory is not empty. The server relies on + // this functionality, and sometimes uses recursion that should not + // error on parent directories. + return nil + case osIsNotExist(err): + return nil + case errors.Is(err, errFileNotFound): + return nil + case osIsPermission(err): + return errFileAccessDenied + case isSysErrIO(err): + return errFaultyDisk + default: + return err + } + } + + // Delete parent directory obviously not recursively. Errors for + // parent directories shouldn't trickle down. + s.deleteFile(basePath, pathutil.Dir(pathutil.Clean(deletePath)), false, false) + + return nil +} + +// DeleteBulk - delete many files in bulk to trash. +// this delete does not recursively delete empty +// parents, if you need empty parent delete support +// please use Delete() instead. This API is meant as +// an optimization for Multipart operations. +func (s *xlStorage) DeleteBulk(ctx context.Context, volume string, paths ...string) (err error) { + volumeDir, err := s.getVolDir(volume) + if err != nil { + return err + } + + if !skipAccessChecks(volume) { + // Stat a volume entry. + if err = Access(volumeDir); err != nil { + return convertAccessError(err, errVolumeAccessDenied) + } + } + + for _, fp := range paths { + // Following code is needed so that we retain SlashSeparator suffix if any in + // path argument. + filePath := pathJoin(volumeDir, fp) + if err = checkPathLength(filePath); err != nil { + return err + } + + if err = s.moveToTrash(filePath, false, false); err != nil { + return err + } + } + + return nil +} + +// Delete - delete a file at path. +func (s *xlStorage) Delete(ctx context.Context, volume string, path string, deleteOpts DeleteOptions) (err error) { + volumeDir, err := s.getVolDir(volume) + if err != nil { + return err + } + + if !skipAccessChecks(volume) { + // Stat a volume entry. + if err = Access(volumeDir); err != nil { + return convertAccessError(err, errVolumeAccessDenied) + } + } + + // Following code is needed so that we retain SlashSeparator suffix if any in + // path argument. + filePath := pathJoin(volumeDir, path) + if err = checkPathLength(filePath); err != nil { + return err + } + + // Delete file and delete parent directory as well if it's empty. + return s.deleteFile(volumeDir, filePath, deleteOpts.Recursive, deleteOpts.Immediate) +} + +func skipAccessChecks(volume string) (ok bool) { + return strings.HasPrefix(volume, minioMetaBucket) +} + +// RenameData - rename source path to destination path atomically, metadata and data directory. +func (s *xlStorage) RenameData(ctx context.Context, srcVolume, srcPath string, fi FileInfo, dstVolume, dstPath string, opts RenameOptions) (res RenameDataResp, err error) { + defer func() { + ignoredErrs := []error{ + errFileNotFound, + errVolumeNotFound, + errFileVersionNotFound, + errDiskNotFound, + errUnformattedDisk, + errMaxVersionsExceeded, + errFileAccessDenied, + } + if err != nil && !IsErr(err, ignoredErrs...) && !contextCanceled(ctx) { + // Only log these errors if context is not yet canceled. + storageLogOnceIf(ctx, fmt.Errorf("drive:%s, srcVolume: %s, srcPath: %s, dstVolume: %s:, dstPath: %s - error %v", + s.drivePath, + srcVolume, srcPath, + dstVolume, dstPath, + err), "xl-storage-rename-data-"+dstVolume) + } + if s.globalSync { + globalSync() + } + }() + + srcVolumeDir, err := s.getVolDir(srcVolume) + if err != nil { + return res, err + } + + dstVolumeDir, err := s.getVolDir(dstVolume) + if err != nil { + return res, err + } + + if !skipAccessChecks(srcVolume) { + // Stat a volume entry. + if err = Access(srcVolumeDir); err != nil { + return res, convertAccessError(err, errVolumeAccessDenied) + } + } + + if !skipAccessChecks(dstVolume) { + if err = Access(dstVolumeDir); err != nil { + return res, convertAccessError(err, errVolumeAccessDenied) + } + } + + srcFilePath := pathutil.Join(srcVolumeDir, pathJoin(srcPath, xlStorageFormatFile)) + dstFilePath := pathutil.Join(dstVolumeDir, pathJoin(dstPath, xlStorageFormatFile)) + + var srcDataPath string + var dstDataPath string + var dataDir string + if !fi.IsRemote() { + dataDir = retainSlash(fi.DataDir) + } + if dataDir != "" { + srcDataPath = retainSlash(pathJoin(srcVolumeDir, srcPath, dataDir)) + // make sure to always use path.Join here, do not use pathJoin as + // it would additionally add `/` at the end and it comes in the + // way of renameAll(), parentDir creation. + dstDataPath = pathutil.Join(dstVolumeDir, dstPath, dataDir) + } + + if err = checkPathLength(srcFilePath); err != nil { + return res, err + } + + if err = checkPathLength(dstFilePath); err != nil { + return res, err + } + + s.RLock() + formatLegacy := s.formatLegacy + s.RUnlock() + + dstBuf, err := xioutil.ReadFile(dstFilePath) + if err != nil { + // handle situations when dstFilePath is 'file' + // for example such as someone is trying to + // upload an object such as `prefix/object/xl.meta` + // where `prefix/object` is already an object + if isSysErrNotDir(err) && runtime.GOOS != globalWindowsOSName { + // NOTE: On windows the error happens at + // next line and returns appropriate error. + return res, errFileAccessDenied + } + if !osIsNotExist(err) { + return res, osErrToFileErr(err) + } + if formatLegacy { + // errFileNotFound comes here. + err = s.renameLegacyMetadata(dstVolumeDir, dstPath) + if err != nil && err != errFileNotFound { + return res, err + } + if err == nil { + dstBuf, err = xioutil.ReadFile(dstFilePath) + if err != nil && !osIsNotExist(err) { + return res, osErrToFileErr(err) + } + } + } + } + + // Preserve all the legacy data, could be slow, but at max there can be 10,000 parts. + currentDataPath := pathJoin(dstVolumeDir, dstPath) + + var xlMeta xlMetaV2 + var legacyPreserved bool + var legacyEntries []string + if len(dstBuf) > 0 { + if isXL2V1Format(dstBuf) { + if err = xlMeta.Load(dstBuf); err != nil { + // Data appears corrupt. Drop data. + xlMeta = xlMetaV2{} + } + } else { + // This code-path is to preserve the legacy data. + xlMetaLegacy := &xlMetaV1Object{} + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err := json.Unmarshal(dstBuf, xlMetaLegacy); err != nil { + storageLogOnceIf(ctx, err, "read-data-unmarshal-"+dstFilePath) + // Data appears corrupt. Drop data. + } else { + xlMetaLegacy.DataDir = legacyDataDir + if err = xlMeta.AddLegacy(xlMetaLegacy); err != nil { + storageLogOnceIf(ctx, err, "read-data-add-legacy-"+dstFilePath) + } + legacyPreserved = true + } + } + } else { + // It is possible that some drives may not have `xl.meta` file + // in such scenarios verify if at least `part.1` files exist + // to verify for legacy version. + if formatLegacy { + // We only need this code if we are moving + // from `xl.json` to `xl.meta`, we can avoid + // one extra readdir operation here for all + // new deployments. + entries, err := readDir(currentDataPath) + if err != nil && err != errFileNotFound { + return res, osErrToFileErr(err) + } + for _, entry := range entries { + if entry == xlStorageFormatFile || strings.HasSuffix(entry, slashSeparator) { + continue + } + if strings.HasPrefix(entry, "part.") { + legacyPreserved = true + legacyEntries = entries + break + } + } + } + } + + var legacyDataPath string + if formatLegacy { + legacyDataPath = pathJoin(dstVolumeDir, dstPath, legacyDataDir) + if legacyPreserved { + if contextCanceled(ctx) { + return res, ctx.Err() + } + + if len(legacyEntries) > 0 { + // legacy data dir means its old content, honor system umask. + if err = mkdirAll(legacyDataPath, 0o777, dstVolumeDir); err != nil { + // any failed mkdir-calls delete them. + s.deleteFile(dstVolumeDir, legacyDataPath, true, false) + return res, osErrToFileErr(err) + } + for _, entry := range legacyEntries { + // Skip xl.meta renames further, also ignore any directories such as `legacyDataDir` + if entry == xlStorageFormatFile || strings.HasSuffix(entry, slashSeparator) { + continue + } + + if err = Rename(pathJoin(currentDataPath, entry), pathJoin(legacyDataPath, entry)); err != nil { + // Any failed rename calls un-roll previous transaction. + s.deleteFile(dstVolumeDir, legacyDataPath, true, false) + + return res, osErrToFileErr(err) + } + } + } + } + } + + // Set skipParent to skip mkdirAll() calls for deeply nested objects + // - if its an overwrite + // - if its a versioned object + // + // This can potentiall reduce syscalls by strings.Split(path, "/") + // times relative to the object name. + skipParent := dstVolumeDir + if len(dstBuf) > 0 { + skipParent = pathutil.Dir(dstFilePath) + } + + var reqVID string + if fi.VersionID == "" { + reqVID = nullVersionID + } else { + reqVID = fi.VersionID + } + + // Empty fi.VersionID indicates that versioning is either + // suspended or disabled on this bucket. RenameData will replace + // the 'null' version. We add a free-version to track its tiered + // content for asynchronous deletion. + // + // Note: RestoreObject and HealObject requests don't end up replacing the + // null version and therefore don't require the free-version to track + // anything + if fi.VersionID == "" && !fi.IsRestoreObjReq() && !fi.Healing() { + // Note: Restore object request reuses PutObject/Multipart + // upload to copy back its data from the remote tier. This + // doesn't replace the existing version, so we don't need to add + // a free-version. + xlMeta.AddFreeVersion(fi) + } + + // indicates if RenameData() is called by healing. + healing := fi.Healing() + + // Replace the data of null version or any other existing version-id + _, ver, err := xlMeta.findVersionStr(reqVID) + if err == nil { + dataDir := ver.getDataDir() + if dataDir != "" && (xlMeta.SharedDataDirCountStr(reqVID, dataDir) == 0) { + // Purge the destination path as we are not preserving anything + // versioned object was not requested. + res.OldDataDir = dataDir + if healing { + // if old destination path is same as new destination path + // there is nothing to purge, this is true in case of healing + // avoid setting OldDataDir at that point. + res.OldDataDir = "" + } else { + xlMeta.data.remove(reqVID, dataDir) + } + } + } + + if err = xlMeta.AddVersion(fi); err != nil { + if legacyPreserved { + // Any failed rename calls un-roll previous transaction. + s.deleteFile(dstVolumeDir, legacyDataPath, true, false) + } + return res, err + } + + if len(xlMeta.versions) <= 10 { + // any number of versions beyond this is excessive + // avoid healing such objects in this manner, let + // it heal during the regular scanner cycle. + dst := []byte{} + for _, ver := range xlMeta.versions { + dst = slices.Grow(dst, 16) + copy(dst[len(dst):], ver.header.VersionID[:]) + } + res.Sign = dst + } + + newDstBuf, err := xlMeta.AppendTo(metaDataPoolGet()) + defer metaDataPoolPut(newDstBuf) + if err != nil { + if legacyPreserved { + s.deleteFile(dstVolumeDir, legacyDataPath, true, false) + } + return res, errFileCorrupt + } + + if contextCanceled(ctx) { + return res, ctx.Err() + } + + if err = s.WriteAll(ctx, srcVolume, pathJoin(srcPath, xlStorageFormatFile), newDstBuf); err != nil { + if legacyPreserved { + s.deleteFile(dstVolumeDir, legacyDataPath, true, false) + } + return res, osErrToFileErr(err) + } + diskHealthCheckOK(ctx, err) + + notInline := srcDataPath != "" && len(fi.Data) == 0 && fi.Size > 0 + if notInline { + if healing { + // renameAll only for objects that have xl.meta not saved inline. + // this must be done in healing only, otherwise it is expected + // that for fresh PutObject() call dstDataPath can never exist. + // if its an overwrite then the caller deletes the DataDir + // in a separate RPC call. + s.moveToTrash(dstDataPath, true, false) + + // If we are healing we should purge any legacyDataPath content, + // that was previously preserved during PutObject() call + // on a versioned bucket. + s.moveToTrash(legacyDataPath, true, false) + } + if contextCanceled(ctx) { + return res, ctx.Err() + } + if err = renameAll(srcDataPath, dstDataPath, skipParent); err != nil { + if legacyPreserved { + // Any failed rename calls un-roll previous transaction. + s.deleteFile(dstVolumeDir, legacyDataPath, true, false) + } + // if its a partial rename() do not attempt to delete recursively. + s.deleteFile(dstVolumeDir, dstDataPath, false, false) + return res, osErrToFileErr(err) + } + diskHealthCheckOK(ctx, err) + } + + // If we have oldDataDir then we must preserve current xl.meta + // as backup, in-case needing renames(). + if res.OldDataDir != "" { + if contextCanceled(ctx) { + return res, ctx.Err() + } + + // preserve current xl.meta inside the oldDataDir. + if err = s.writeAll(ctx, dstVolume, pathJoin(dstPath, res.OldDataDir, xlStorageFormatFileBackup), dstBuf, true, skipParent); err != nil { + if legacyPreserved { + s.deleteFile(dstVolumeDir, legacyDataPath, true, false) + } + return res, osErrToFileErr(err) + } + diskHealthCheckOK(ctx, err) + } + + if contextCanceled(ctx) { + return res, ctx.Err() + } + + // Commit meta-file + if err = renameAll(srcFilePath, dstFilePath, skipParent); err != nil { + if legacyPreserved { + // Any failed rename calls un-roll previous transaction. + s.deleteFile(dstVolumeDir, legacyDataPath, true, false) + } + // if its a partial rename() do not attempt to delete recursively. + // this can be healed since all parts are available. + s.deleteFile(dstVolumeDir, dstDataPath, false, false) + return res, osErrToFileErr(err) + } + + if srcVolume != minioMetaMultipartBucket { + // srcFilePath is some-times minioMetaTmpBucket, an attempt to + // remove the temporary folder is enough since at this point + // ideally all transaction should be complete. + Remove(pathutil.Dir(srcFilePath)) + } else { + s.deleteFile(srcVolumeDir, pathutil.Dir(srcFilePath), true, false) + } + return res, nil +} + +// RenamePart - rename part path to destination path atomically, this is meant to be used +// only with multipart API +func (s *xlStorage) RenamePart(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string, meta []byte, skipParent string) (err error) { + srcVolumeDir, err := s.getVolDir(srcVolume) + if err != nil { + return err + } + dstVolumeDir, err := s.getVolDir(dstVolume) + if err != nil { + return err + } + if !skipAccessChecks(srcVolume) { + // Stat a volume entry. + if err = Access(srcVolumeDir); err != nil { + if osIsNotExist(err) { + return errVolumeNotFound + } else if isSysErrIO(err) { + return errFaultyDisk + } + return err + } + } + if !skipAccessChecks(dstVolume) { + if err = Access(dstVolumeDir); err != nil { + if osIsNotExist(err) { + return errVolumeNotFound + } else if isSysErrIO(err) { + return errFaultyDisk + } + return err + } + } + + srcIsDir := HasSuffix(srcPath, SlashSeparator) + dstIsDir := HasSuffix(dstPath, SlashSeparator) + // either source or destination is a directory return error. + if srcIsDir || dstIsDir { + return errFileAccessDenied + } + + srcFilePath := pathutil.Join(srcVolumeDir, srcPath) + if err = checkPathLength(srcFilePath); err != nil { + return err + } + + dstFilePath := pathutil.Join(dstVolumeDir, dstPath) + if err = checkPathLength(dstFilePath); err != nil { + return err + } + // when skipParent is from rpc. it’s ok for not adding another rpc HandlerID like HandlerRenamePart2 + // For this case, skipParent is empty, destBaseDir is equal to dstVolumeDir, that behavior is the same as the previous one + destBaseDir := pathutil.Join(dstVolumeDir, skipParent) + if err = checkPathLength(destBaseDir); err != nil { + return err + } + + if err = renameAll(srcFilePath, dstFilePath, destBaseDir); err != nil { + if isSysErrNotEmpty(err) || isSysErrNotDir(err) { + return errFileAccessDenied + } + err = osErrToFileErr(err) + if errors.Is(err, errFileNotFound) || errors.Is(err, errFileAccessDenied) { + return errUploadIDNotFound + } + return err + } + + if err = s.WriteAll(ctx, dstVolume, dstPath+".meta", meta); err != nil { + return osErrToFileErr(err) + } + + // Remove parent dir of the source file if empty + parentDir := pathutil.Dir(srcFilePath) + s.deleteFile(srcVolumeDir, parentDir, false, false) + + return nil +} + +// RenameFile - rename source path to destination path atomically. +func (s *xlStorage) RenameFile(ctx context.Context, srcVolume, srcPath, dstVolume, dstPath string) (err error) { + srcVolumeDir, err := s.getVolDir(srcVolume) + if err != nil { + return err + } + dstVolumeDir, err := s.getVolDir(dstVolume) + if err != nil { + return err + } + if !skipAccessChecks(srcVolume) { + // Stat a volume entry. + if err = Access(srcVolumeDir); err != nil { + if osIsNotExist(err) { + return errVolumeNotFound + } else if isSysErrIO(err) { + return errFaultyDisk + } + return err + } + } + if !skipAccessChecks(dstVolume) { + if err = Access(dstVolumeDir); err != nil { + if osIsNotExist(err) { + return errVolumeNotFound + } else if isSysErrIO(err) { + return errFaultyDisk + } + return err + } + } + srcIsDir := HasSuffix(srcPath, SlashSeparator) + dstIsDir := HasSuffix(dstPath, SlashSeparator) + // Either src and dst have to be directories or files, else return error. + if (!srcIsDir || !dstIsDir) && (srcIsDir || dstIsDir) { + return errFileAccessDenied + } + srcFilePath := pathutil.Join(srcVolumeDir, srcPath) + if err = checkPathLength(srcFilePath); err != nil { + return err + } + dstFilePath := pathutil.Join(dstVolumeDir, dstPath) + if err = checkPathLength(dstFilePath); err != nil { + return err + } + if srcIsDir { + // If source is a directory, we expect the destination to be non-existent but we + // we still need to allow overwriting an empty directory since it represents + // an object empty directory. + dirInfo, err := Lstat(dstFilePath) + if isSysErrIO(err) { + return errFaultyDisk + } + if err != nil { + if !osIsNotExist(err) { + return err + } + } else { + if !dirInfo.IsDir() { + return errFileAccessDenied + } + if err = Remove(dstFilePath); err != nil { + if isSysErrNotEmpty(err) || isSysErrNotDir(err) { + return errFileAccessDenied + } else if isSysErrIO(err) { + return errFaultyDisk + } + return err + } + } + } + + if err = renameAll(srcFilePath, dstFilePath, dstVolumeDir); err != nil { + if isSysErrNotEmpty(err) || isSysErrNotDir(err) { + return errFileAccessDenied + } + return osErrToFileErr(err) + } + + // Remove parent dir of the source file if empty + parentDir := pathutil.Dir(srcFilePath) + s.deleteFile(srcVolumeDir, parentDir, false, false) + + return nil +} + +func (s *xlStorage) bitrotVerify(ctx context.Context, partPath string, partSize int64, algo BitrotAlgorithm, sum []byte, shardSize int64) error { + // Open the file for reading. + file, err := OpenFile(partPath, readMode, 0o666) + if err != nil { + return osErrToFileErr(err) + } + + // Close the file descriptor. + defer file.Close() + fi, err := file.Stat() + if err != nil { + // Unable to stat on the file, return an expected error + // for healing code to fix this file. + return err + } + return bitrotVerify(diskHealthReader(ctx, file), fi.Size(), partSize, algo, sum, shardSize) +} + +func (s *xlStorage) VerifyFile(ctx context.Context, volume, path string, fi FileInfo) (*CheckPartsResp, error) { + volumeDir, err := s.getVolDir(volume) + if err != nil { + return nil, err + } + + if !skipAccessChecks(volume) { + // Stat a volume entry. + if err = Access(volumeDir); err != nil { + return nil, convertAccessError(err, errVolumeAccessDenied) + } + } + + resp := CheckPartsResp{ + // By default, the result is unknown per part + Results: make([]int, len(fi.Parts)), + } + + erasure := fi.Erasure + for i, part := range fi.Parts { + checksumInfo := erasure.GetChecksumInfo(part.Number) + partPath := pathJoin(volumeDir, path, fi.DataDir, fmt.Sprintf("part.%d", part.Number)) + err := s.bitrotVerify(ctx, partPath, + erasure.ShardFileSize(part.Size), + checksumInfo.Algorithm, + checksumInfo.Hash, erasure.ShardSize()) + + resp.Results[i] = convPartErrToInt(err) + + // Only log unknown errors + if resp.Results[i] == checkPartUnknown && err != errFileAccessDenied { + logger.GetReqInfo(ctx).AppendTags("disk", s.String()) + storageLogOnceIf(ctx, err, partPath) + } + } + + return &resp, nil +} + +func (s *xlStorage) ReadParts(ctx context.Context, volume string, partMetaPaths ...string) ([]*ObjectPartInfo, error) { + volumeDir, err := s.getVolDir(volume) + if err != nil { + return nil, err + } + + parts := make([]*ObjectPartInfo, len(partMetaPaths)) + for idx, partMetaPath := range partMetaPaths { + var partNumber int + fmt.Sscanf(pathutil.Base(partMetaPath), "part.%d.meta", &partNumber) + + if contextCanceled(ctx) { + parts[idx] = &ObjectPartInfo{ + Error: ctx.Err().Error(), + Number: partNumber, + } + continue + } + + if err := Access(pathJoin(volumeDir, pathutil.Dir(partMetaPath), fmt.Sprintf("part.%d", partNumber))); err != nil { + parts[idx] = &ObjectPartInfo{ + Error: err.Error(), + Number: partNumber, + } + continue + } + + data, err := s.readAllData(ctx, volume, volumeDir, pathJoin(volumeDir, partMetaPath)) + if err != nil { + parts[idx] = &ObjectPartInfo{ + Error: err.Error(), + Number: partNumber, + } + continue + } + + pinfo := &ObjectPartInfo{} + if _, err = pinfo.UnmarshalMsg(data); err != nil { + parts[idx] = &ObjectPartInfo{ + Error: err.Error(), + Number: partNumber, + } + continue + } + + parts[idx] = pinfo + } + diskHealthCheckOK(ctx, nil) + return parts, nil +} + +// ReadMultiple will read multiple files and send each back as response. +// Files are read and returned in the given order. +// The resp channel is closed before the call returns. +// Only a canceled context will return an error. +func (s *xlStorage) ReadMultiple(ctx context.Context, req ReadMultipleReq, resp chan<- ReadMultipleResp) error { + defer xioutil.SafeClose(resp) + + volumeDir := pathJoin(s.drivePath, req.Bucket) + found := 0 + for _, f := range req.Files { + if contextCanceled(ctx) { + return ctx.Err() + } + r := ReadMultipleResp{ + Bucket: req.Bucket, + Prefix: req.Prefix, + File: f, + } + + var data []byte + var mt time.Time + + fullPath := pathJoin(volumeDir, req.Prefix, f) + w := xioutil.NewDeadlineWorker(globalDriveConfig.GetMaxTimeout()) + if err := w.Run(func() (err error) { + if req.MetadataOnly { + data, mt, err = s.readMetadataWithDMTime(ctx, fullPath) + } else { + data, mt, err = s.readAllDataWithDMTime(ctx, req.Bucket, volumeDir, fullPath) + } + return err + }); err != nil { + if !IsErr(err, errFileNotFound, errVolumeNotFound) { + r.Exists = true + r.Error = err.Error() + } + select { + case <-ctx.Done(): + return ctx.Err() + case resp <- r: + } + if req.AbortOn404 && !r.Exists { + // We stop at first file not found. + // We have already reported the error, return nil. + return nil + } + continue + } + diskHealthCheckOK(ctx, nil) + if req.MaxSize > 0 && int64(len(data)) > req.MaxSize { + r.Exists = true + r.Error = fmt.Sprintf("max size (%d) exceeded: %d", req.MaxSize, len(data)) + select { + case <-ctx.Done(): + return ctx.Err() + case resp <- r: + continue + } + } + found++ + r.Exists = true + r.Data = data + r.Modtime = mt + select { + case <-ctx.Done(): + return ctx.Err() + case resp <- r: + } + if req.MaxResults > 0 && found >= req.MaxResults { + return nil + } + } + return nil +} + +func (s *xlStorage) StatInfoFile(ctx context.Context, volume, path string, glob bool) (stat []StatInfo, err error) { + volumeDir, err := s.getVolDir(volume) + if err != nil { + return stat, err + } + + files := []string{pathJoin(volumeDir, path)} + if glob { + files, err = filepathx.Glob(filepath.Join(volumeDir, path)) + if err != nil { + return nil, err + } + } + for _, filePath := range files { + if err := checkPathLength(filePath); err != nil { + return stat, err + } + st, _ := Lstat(filePath) + if st == nil { + if !skipAccessChecks(volume) { + // Stat a volume entry. + if verr := Access(volumeDir); verr != nil { + return stat, convertAccessError(verr, errVolumeAccessDenied) + } + } + return stat, errPathNotFound + } + name, err := filepath.Rel(volumeDir, filePath) + if err != nil { + name = filePath + } + stat = append(stat, StatInfo{ + Name: filepath.ToSlash(name), + Size: st.Size(), + Dir: st.IsDir(), + Mode: uint32(st.Mode()), + ModTime: st.ModTime(), + }) + } + return stat, nil +} + +// CleanAbandonedData will read metadata of the object on disk +// and delete any data directories and inline data that isn't referenced in metadata. +// Metadata itself is not modified, only inline data. +func (s *xlStorage) CleanAbandonedData(ctx context.Context, volume string, path string) error { + if volume == "" || path == "" { + return nil // Ignore + } + + volumeDir, err := s.getVolDir(volume) + if err != nil { + return err + } + baseDir := pathJoin(volumeDir, path+slashSeparator) + metaPath := pathutil.Join(baseDir, xlStorageFormatFile) + buf, err := s.readAllData(ctx, volume, volumeDir, metaPath) + if err != nil { + return err + } + defer metaDataPoolPut(buf) + + if !isXL2V1Format(buf) { + return nil + } + var xl xlMetaV2 + err = xl.LoadOrConvert(buf) + if err != nil { + return err + } + foundDirs := make(map[string]struct{}, len(xl.versions)) + err = readDirFn(baseDir, func(name string, typ os.FileMode) error { + if !typ.IsDir() { + return nil + } + // See if directory has a UUID name. + base := filepath.Base(name) + _, err := uuid.Parse(base) + if err == nil { + foundDirs[base] = struct{}{} + } + return nil + }) + if err != nil { + return err + } + wantDirs, err := xl.getDataDirs() + if err != nil { + return err + } + + // Delete all directories we expect to be there. + for _, dir := range wantDirs { + delete(foundDirs, dir) + } + + // Delete excessive directories. + // Do not abort on context errors. + for dir := range foundDirs { + toRemove := pathJoin(volumeDir, path, dir+SlashSeparator) + err = s.deleteFile(volumeDir, toRemove, true, true) + diskHealthCheckOK(ctx, err) + if err != nil { + return err + } + } + + // Do the same for inline data + dirs, err := xl.data.list() + if err != nil { + return err + } + + // Clear and repopulate + clear(foundDirs) + + // Populate into map + for _, k := range dirs { + foundDirs[k] = struct{}{} + } + + // Delete all directories we expect to be there. + for _, dir := range wantDirs { + delete(foundDirs, dir) + } + + // Nothing to delete + if len(foundDirs) == 0 { + return nil + } + + // Delete excessive inline entries. + // Convert to slice. + dirs = dirs[:0] + for dir := range foundDirs { + dirs = append(dirs, dir) + } + if xl.data.remove(dirs...) { + newBuf, err := xl.AppendTo(metaDataPoolGet()) + if err == nil { + defer metaDataPoolPut(newBuf) + return s.WriteAll(ctx, volume, pathJoin(path, xlStorageFormatFile), buf) + } + } + return nil +} + +func convertAccessError(err, permErr error) error { + switch { + case osIsNotExist(err): + return errVolumeNotFound + case isSysErrIO(err): + return errFaultyDisk + case osIsPermission(err): + return permErr + default: + return err + } +} diff --git a/cmd/xl-storage_noatime_notsupported.go b/cmd/xl-storage_noatime_notsupported.go new file mode 100644 index 00000000..c9c91d62 --- /dev/null +++ b/cmd/xl-storage_noatime_notsupported.go @@ -0,0 +1,32 @@ +//go:build !unix || darwin || freebsd + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "os" +) + +var ( + // No special option for reads on windows + readMode = os.O_RDONLY + + // Write with sync no buffering only used only for `xl.meta` writes + writeMode = os.O_SYNC +) diff --git a/cmd/xl-storage_noatime_supported.go b/cmd/xl-storage_noatime_supported.go new file mode 100644 index 00000000..efa75ff0 --- /dev/null +++ b/cmd/xl-storage_noatime_supported.go @@ -0,0 +1,34 @@ +//go:build unix && !darwin && !freebsd + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "os" + "syscall" +) + +var ( + // Disallow updating access times + // Add non-block to avoid syscall to attempt to set epoll on files. + readMode = os.O_RDONLY | 0x40000 | syscall.O_NONBLOCK // O_NOATIME + + // Write with data sync only used only for `xl.meta` writes + writeMode = 0x1000 | syscall.O_NONBLOCK // O_DSYNC +) diff --git a/cmd/xl-storage_test.go b/cmd/xl-storage_test.go new file mode 100644 index 00000000..c27647f2 --- /dev/null +++ b/cmd/xl-storage_test.go @@ -0,0 +1,1943 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "crypto/rand" + "io" + "net/url" + "os" + slashpath "path" + "runtime" + "strings" + "syscall" + "testing" + + "github.com/google/uuid" +) + +func TestCheckPathLength(t *testing.T) { + // Check path length restrictions are not same on windows/darwin + if runtime.GOOS == "windows" || runtime.GOOS == "darwin" { + t.Skip() + } + + testCases := []struct { + path string + expectedErr error + }{ + {".", errFileAccessDenied}, + {"/", errFileAccessDenied}, + {"..", errFileAccessDenied}, + {"data/G_792/srv-tse/c/users/denis/documents/gestion!20locative/heritier/propri!E9taire/20190101_a2.03!20-!20m.!20heritier!20re!B4mi!20-!20proce!60s-verbal!20de!20livraison!20et!20de!20remise!20des!20cle!B4s!20acque!B4reurs!20-!204-!20livraison!20-!20lp!20promotion!20toulouse!20-!20encre!20et!20plume!20-!205!20de!B4c.!202019!20a!60!2012-49.pdf.ecc", errFileNameTooLong}, + {"data/G_792/srv-tse/c/users/denis/documents/gestionlocative.txt", nil}, + } + + for _, testCase := range testCases { + gotErr := checkPathLength(testCase.path) + t.Run("", func(t *testing.T) { + if gotErr != testCase.expectedErr { + t.Errorf("Expected %s, got %s", testCase.expectedErr, gotErr) + } + }) + } +} + +// Tests validate volume name. +func TestIsValidVolname(t *testing.T) { + testCases := []struct { + volName string + shouldPass bool + }{ + // Cases which should pass the test. + // passing in valid bucket names. + {"lol", true}, + {"1-this-is-valid", true}, + {"1-this-too-is-valid-1", true}, + {"this.works.too.1", true}, + {"1234567", true}, + {"123", true}, + {"s3-eu-west-1.amazonaws.com", true}, + {"ideas-are-more-powerful-than-guns", true}, + {"testbucket", true}, + {"1bucket", true}, + {"bucket1", true}, + {"$this-is-not-valid-too", true}, + {"contains-$-dollar", true}, + {"contains-^-carrot", true}, + {"contains-$-dollar", true}, + {"contains-$-dollar", true}, + {".starts-with-a-dot", true}, + {"ends-with-a-dot.", true}, + {"ends-with-a-dash-", true}, + {"-starts-with-a-dash", true}, + {"THIS-BEINGS-WITH-UPPERCASe", true}, + {"tHIS-ENDS-WITH-UPPERCASE", true}, + {"ThisBeginsAndEndsWithUpperCase", true}, + {"una ñina", true}, + {"lalalallalallalalalallalallalala-theString-size-is-greater-than-64", true}, + // cases for which test should fail. + // passing invalid bucket names. + {"", false}, + {SlashSeparator, false}, + {"a", false}, + {"ab", false}, + {"ab/", true}, + {"......", true}, + } + + for i, testCase := range testCases { + isValidVolname := isValidVolname(testCase.volName) + if testCase.shouldPass && !isValidVolname { + t.Errorf("Test case %d: Expected \"%s\" to be a valid bucket name", i+1, testCase.volName) + } + if !testCase.shouldPass && isValidVolname { + t.Errorf("Test case %d: Expected bucket name \"%s\" to be invalid", i+1, testCase.volName) + } + } +} + +func newLocalXLStorage(path string) (*xlStorage, error) { + return newLocalXLStorageWithDiskIdx(path, 0) +} + +// Initialize a new storage disk. +func newLocalXLStorageWithDiskIdx(path string, diskIdx int) (*xlStorage, error) { + u := url.URL{Path: path} + return newXLStorage(Endpoint{ + URL: &u, + IsLocal: true, + PoolIdx: 0, + SetIdx: 0, + DiskIdx: diskIdx, + }, true) +} + +// creates a temp dir and sets up xlStorage layer. +// returns xlStorage layer, temp dir path to be used for the purpose of tests. +func newXLStorageTestSetup(tb testing.TB) (*xlStorageDiskIDCheck, string, error) { + diskPath := tb.TempDir() + + // Initialize a new xlStorage layer. + storage, err := newLocalXLStorageWithDiskIdx(diskPath, 3) + if err != nil { + return nil, "", err + } + + // Create a sample format.json file + if err = storage.WriteAll(tb.Context(), minioMetaBucket, formatConfigFile, []byte(`{"version":"1","format":"xl","id":"592a41c2-b7cc-4130-b883-c4b5cb15965b","xl":{"version":"3","this":"da017d62-70e3-45f1-8a1a-587707e69ad1","sets":[["e07285a6-8c73-4962-89c6-047fb939f803","33b8d431-482d-4376-b63c-626d229f0a29","cff6513a-4439-4dc1-bcaa-56c9e880c352","da017d62-70e3-45f1-8a1a-587707e69ad1","9c9f21d5-1f15-4737-bce6-835faa0d9626","0a59b346-1424-4fc2-9fa2-a2e80541d0c1","7924a3dc-b69a-4971-9a2e-014966d6aebb","4d2b8dd9-4e48-444b-bdca-c89194b26042"]],"distributionAlgo":"CRCMOD"}}`)); err != nil { + return nil, "", err + } + + disk := newXLStorageDiskIDCheck(storage, false) + disk.SetDiskID("da017d62-70e3-45f1-8a1a-587707e69ad1") + return disk, diskPath, nil +} + +// createPermDeniedFile - creates temporary directory and file with path '/mybucket/myobject' +func createPermDeniedFile(t *testing.T) (permDeniedDir string) { + var err error + permDeniedDir = t.TempDir() + + if err = os.Mkdir(slashpath.Join(permDeniedDir, "mybucket"), 0o775); err != nil { + t.Fatalf("Unable to create temporary directory %v. %v", slashpath.Join(permDeniedDir, "mybucket"), err) + } + + if err = os.WriteFile(slashpath.Join(permDeniedDir, "mybucket", "myobject"), []byte(""), 0o400); err != nil { + t.Fatalf("Unable to create file %v. %v", slashpath.Join(permDeniedDir, "mybucket", "myobject"), err) + } + + if err = os.Chmod(slashpath.Join(permDeniedDir, "mybucket"), 0o400); err != nil { + t.Fatalf("Unable to change permission to temporary directory %v. %v", slashpath.Join(permDeniedDir, "mybucket"), err) + } + t.Cleanup(func() { + os.Chmod(slashpath.Join(permDeniedDir, "mybucket"), 0o775) + }) + + if err = os.Chmod(permDeniedDir, 0o400); err != nil { + t.Fatalf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err) + } + t.Cleanup(func() { + os.Chmod(permDeniedDir, 0o775) + }) + + return permDeniedDir +} + +// TestXLStorages xlStorage.getDiskInfo() +func TestXLStorageGetDiskInfo(t *testing.T) { + path := t.TempDir() + + testCases := []struct { + diskPath string + expectedErr error + }{ + {path, nil}, + {"/nonexistent-dir", errDiskNotFound}, + } + + // Check test cases. + for _, testCase := range testCases { + if _, _, err := getDiskInfo(testCase.diskPath); err != testCase.expectedErr { + t.Fatalf("expected: %s, got: %s", testCase.expectedErr, err) + } + } +} + +func TestXLStorageIsDirEmpty(t *testing.T) { + tmp := t.TempDir() + + // Should give false on non-existent directory. + dir1 := slashpath.Join(tmp, "non-existent-directory") + if isDirEmpty(dir1, true) { + t.Error("expected false for non-existent directory, got true") + } + + // Should give false for not-a-directory. + dir2 := slashpath.Join(tmp, "file") + err := os.WriteFile(dir2, []byte("hello"), 0o777) + if err != nil { + t.Fatal(err) + } + + if isDirEmpty(dir2, true) { + t.Error("expected false for a file, got true") + } + + // Should give true for a real empty directory. + dir3 := slashpath.Join(tmp, "empty") + err = os.Mkdir(dir3, 0o777) + if err != nil { + t.Fatal(err) + } + + if !isDirEmpty(dir3, true) { + t.Error("expected true for empty dir, got false") + } +} + +func TestXLStorageReadVersionLegacy(t *testing.T) { + const legacyJSON = `{"version":"1.0.1","format":"xl","stat":{"size":2016,"modTime":"2021-10-11T23:40:34.914361617Z"},"erasure":{"algorithm":"klauspost/reedsolomon/vandermonde","data":2,"parity":2,"blockSize":10485760,"index":2,"distribution":[2,3,4,1],"checksum":[{"name":"part.1","algorithm":"highwayhash256S"}]},"minio":{"release":"RELEASE.2019-12-30T05-45-39Z"},"meta":{"X-Minio-Internal-Server-Side-Encryption-Iv":"kInsJB/0yxyz/40ZI+lmQYJfZacDYqZsGh2wEiv+N50=","X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id":"my-minio-key","X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key":"eyJhZWFkIjoiQUVTLTI1Ni1HQ00tSE1BQy1TSEEtMjU2IiwiaWQiOiJjMzEwNDVjODFmMTA2MWU5NTI4ODcxZmNhMmRkYzA3YyIsIml2IjoiOWQ5cUxGMFhSaFBXbEVqT2JDMmo0QT09Iiwibm9uY2UiOiJYaERsemlCU1cwSENuK2RDIiwiYnl0ZXMiOiJUM0lmY1haQ1dtMWpLeWxBWmFUUnczbDVoYldLWW95dm5iNTZVaWJEbE5LOFZVU2tuQmx3NytIMG8yZnRzZ1UrIn0=","X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key":"IAAfANqt801MT+wwzQRkfFhTrndmhfNiN0alKwDS4AQ1dznNADRQgoq6I4pPVfRsbDp5rQawlripQZvPWUSNJA==","X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm":"DAREv2-HMAC-SHA256","content-type":"application/octet-stream","etag":"20000f00cf5e68d3d6b60e44fcd8b9e8-1"},"parts":[{"number":1,"name":"part.1","etag":"","size":2016,"actualSize":1984}]}` + + // create xlStorage test setup + xlStorage, _, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to cfgreate xlStorage test setup, %s", err) + } + + // Create files for the test cases. + if err = xlStorage.MakeVol(t.Context(), "exists-legacy"); err != nil { + t.Fatalf("Unable to create a volume \"exists-legacy\", %s", err) + } + + if err = xlStorage.AppendFile(t.Context(), "exists-legacy", "as-file/xl.json", []byte(legacyJSON)); err != nil { + t.Fatalf("Unable to create a file \"as-file\", %s", err) + } + + fi, err := xlStorage.ReadVersion(t.Context(), "", "exists-legacy", "as-file", "", ReadOptions{}) + if err != nil { + t.Fatalf("Unable to read older 'xl.json' content: %s", err) + } + + if !fi.XLV1 { + t.Fatal("Unexpected 'xl.json' content should be correctly interpreted as legacy content") + } +} + +// TestXLStorageReadVersion - TestXLStorages the functionality implemented by xlStorage ReadVersion storage API. +func TestXLStorageReadVersion(t *testing.T) { + // create xlStorage test setup + xlStorage, _, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to cfgreate xlStorage test setup, %s", err) + } + + xlMeta, _ := os.ReadFile("testdata/xl.meta") + fi, _ := getFileInfo(xlMeta, "exists", "as-file", "", fileInfoOpts{Data: false}) + + // Create files for the test cases. + if err = xlStorage.MakeVol(t.Context(), "exists"); err != nil { + t.Fatalf("Unable to create a volume \"exists\", %s", err) + } + if err = xlStorage.AppendFile(t.Context(), "exists", "as-directory/as-file/xl.meta", xlMeta); err != nil { + t.Fatalf("Unable to create a file \"as-directory/as-file\", %s", err) + } + if err = xlStorage.AppendFile(t.Context(), "exists", "as-file/xl.meta", xlMeta); err != nil { + t.Fatalf("Unable to create a file \"as-file\", %s", err) + } + if err = xlStorage.AppendFile(t.Context(), "exists", "as-file-parent/xl.meta", xlMeta); err != nil { + t.Fatalf("Unable to create a file \"as-file-parent\", %s", err) + } + if err = xlStorage.MakeVol(t.Context(), "exists/as-file/"+fi.DataDir); err != nil { + t.Fatalf("Unable to create a dataDir %s, %s", fi.DataDir, err) + } + + // TestXLStoragecases to validate different conditions for ReadVersion API. + testCases := []struct { + volume string + path string + err error + }{ + // TestXLStorage case - 1. + // Validate volume does not exist. + { + volume: "i-dont-exist", + path: "", + err: errVolumeNotFound, + }, + // TestXLStorage case - 2. + // Validate bad condition file does not exist. + { + volume: "exists", + path: "as-file-not-found", + err: errFileNotFound, + }, + // TestXLStorage case - 3. + // Validate bad condition file exists as prefix/directory and + // we are attempting to read it. + { + volume: "exists", + path: "as-directory", + err: errFileNotFound, + }, + // TestXLStorage case - 4. + { + volume: "exists", + path: "as-file-parent/as-file", + err: errFileNotFound, + }, + // TestXLStorage case - 5. + // Validate the good condition file exists and we are able to read it. + { + volume: "exists", + path: "as-file", + err: nil, + }, + // TestXLStorage case - 6. + // TestXLStorage case with invalid volume name. + { + volume: "ab", + path: "as-file", + err: errVolumeNotFound, + }, + } + + // Run through all the test cases and validate for ReadVersion. + for i, testCase := range testCases { + _, err = xlStorage.ReadVersion(t.Context(), "", testCase.volume, testCase.path, "", ReadOptions{}) + if err != testCase.err { + t.Fatalf("TestXLStorage %d: Expected err \"%s\", got err \"%s\"", i+1, testCase.err, err) + } + } +} + +// TestXLStorageReadAll - TestXLStorages the functionality implemented by xlStorage ReadAll storage API. +func TestXLStorageReadAll(t *testing.T) { + // create xlStorage test setup + xlStorage, _, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to create xlStorage test setup, %s", err) + } + + // Create files for the test cases. + if err = xlStorage.MakeVol(t.Context(), "exists"); err != nil { + t.Fatalf("Unable to create a volume \"exists\", %s", err) + } + if err = xlStorage.AppendFile(t.Context(), "exists", "as-directory/as-file", []byte("Hello, World")); err != nil { + t.Fatalf("Unable to create a file \"as-directory/as-file\", %s", err) + } + if err = xlStorage.AppendFile(t.Context(), "exists", "as-file", []byte("Hello, World")); err != nil { + t.Fatalf("Unable to create a file \"as-file\", %s", err) + } + if err = xlStorage.AppendFile(t.Context(), "exists", "as-file-parent", []byte("Hello, World")); err != nil { + t.Fatalf("Unable to create a file \"as-file-parent\", %s", err) + } + + // TestXLStoragecases to validate different conditions for ReadAll API. + testCases := []struct { + volume string + path string + err error + }{ + // TestXLStorage case - 1. + // Validate volume does not exist. + { + volume: "i-dont-exist", + path: "", + err: errVolumeNotFound, + }, + // TestXLStorage case - 2. + // Validate bad condition file does not exist. + { + volume: "exists", + path: "as-file-not-found", + err: errFileNotFound, + }, + // TestXLStorage case - 3. + // Validate bad condition file exists as prefix/directory and + // we are attempting to read it. + { + volume: "exists", + path: "as-directory", + err: errFileNotFound, + }, + // TestXLStorage case - 4. + { + volume: "exists", + path: "as-file-parent/as-file", + err: errFileNotFound, + }, + // TestXLStorage case - 5. + // Validate the good condition file exists and we are able to read it. + { + volume: "exists", + path: "as-file", + err: nil, + }, + // TestXLStorage case - 6. + // TestXLStorage case with invalid volume name. + { + volume: "ab", + path: "as-file", + err: errVolumeNotFound, + }, + } + + var dataRead []byte + // Run through all the test cases and validate for ReadAll. + for i, testCase := range testCases { + dataRead, err = xlStorage.ReadAll(t.Context(), testCase.volume, testCase.path) + if err != testCase.err { + t.Errorf("TestXLStorage %d: Expected err \"%v\", got err \"%v\"", i+1, testCase.err, err) + continue + } + if err == nil { + if !bytes.Equal(dataRead, []byte("Hello, World")) { + t.Errorf("TestXLStorage %d: Expected the data read to be \"%s\", but instead got \"%s\"", i+1, "Hello, World", string(dataRead)) + } + } + } +} + +// TestNewXLStorage all the cases handled in xlStorage storage layer initialization. +func TestNewXLStorage(t *testing.T) { + // Temporary dir name. + tmpDirName := globalTestTmpDir + SlashSeparator + "minio-" + nextSuffix() + // Temporary file name. + tmpFileName := globalTestTmpDir + SlashSeparator + "minio-" + nextSuffix() + f, _ := os.Create(tmpFileName) + f.Close() + defer os.Remove(tmpFileName) + + // List of all tests for xlStorage initialization. + testCases := []struct { + name string + err error + }{ + // Validates input argument cannot be empty. + { + "", + errInvalidArgument, + }, + // Validates if the directory does not exist and + // gets automatically created. + { + tmpDirName, + nil, + }, + // Validates if the disk exists as file and returns error + // not a directory. + { + tmpFileName, + errDiskNotDir, + }, + } + + // Validate all test cases. + for i, testCase := range testCases { + // Initialize a new xlStorage layer. + _, err := newLocalXLStorage(testCase.name) + if err != testCase.err { + t.Fatalf("TestXLStorage %d failed wanted: %s, got: %s", i+1, err, testCase.err) + } + } +} + +// TestXLStorageMakeVol - TestXLStorage validate the logic for creation of new xlStorage volume. +// Asserts the failures too against the expected failures. +func TestXLStorageMakeVol(t *testing.T) { + // create xlStorage test setup + xlStorage, path, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to create xlStorage test setup, %s", err) + } + + // Setup test environment. + // Create a file. + if err := os.WriteFile(slashpath.Join(path, "vol-as-file"), []byte{}, os.ModePerm); err != nil { + t.Fatalf("Unable to create file, %s", err) + } + // Create a directory. + if err := os.Mkdir(slashpath.Join(path, "existing-vol"), 0o777); err != nil { + t.Fatalf("Unable to create directory, %s", err) + } + + testCases := []struct { + volName string + expectedErr error + }{ + // TestXLStorage case - 1. + // A valid case, volume creation is expected to succeed. + { + volName: "success-vol", + expectedErr: nil, + }, + // TestXLStorage case - 2. + // Case where a file exists by the name of the volume to be created. + { + volName: "vol-as-file", + expectedErr: errVolumeExists, + }, + // TestXLStorage case - 3. + { + volName: "existing-vol", + expectedErr: errVolumeExists, + }, + // TestXLStorage case - 5. + // TestXLStorage case with invalid volume name. + { + volName: "ab", + expectedErr: errInvalidArgument, + }, + } + + for i, testCase := range testCases { + if err := xlStorage.MakeVol(t.Context(), testCase.volName); err != testCase.expectedErr { + t.Fatalf("TestXLStorage %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) + } + } + + // TestXLStorage for permission denied. + if runtime.GOOS != globalWindowsOSName { + permDeniedDir := createPermDeniedFile(t) + if err = os.Chmod(permDeniedDir, 0o400); err != nil { + t.Fatalf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err) + } + + // Initialize xlStorage storage layer for permission denied error. + _, err = newLocalXLStorage(permDeniedDir) + if err != nil && err != errDiskAccessDenied { + t.Fatalf("Unable to initialize xlStorage, %s", err) + } + + if err = os.Chmod(permDeniedDir, 0o755); err != nil { + t.Fatalf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err) + } + + xlStorageNew, err := newLocalXLStorage(permDeniedDir) + if err != nil { + t.Fatalf("Unable to initialize xlStorage, %s", err) + } + + // change backend permissions for MakeVol error. + if err = os.Chmod(permDeniedDir, 0o400); err != nil { + t.Fatalf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err) + } + + if err := xlStorageNew.MakeVol(t.Context(), "test-vol"); err != errDiskAccessDenied { + t.Fatalf("expected: %s, got: %s", errDiskAccessDenied, err) + } + } +} + +// TestXLStorageDeleteVol - Validates the expected behavior of xlStorage.DeleteVol for various cases. +func TestXLStorageDeleteVol(t *testing.T) { + // create xlStorage test setup + xlStorage, path, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to create xlStorage test setup, %s", err) + } + + // Setup test environment. + if err = xlStorage.MakeVol(t.Context(), "success-vol"); err != nil { + t.Fatalf("Unable to create volume, %s", err) + } + + // TestXLStorage failure cases. + vol := slashpath.Join(path, "nonempty-vol") + if err = os.Mkdir(vol, 0o777); err != nil { + t.Fatalf("Unable to create directory, %s", err) + } + if err = os.WriteFile(slashpath.Join(vol, "test-file"), []byte{}, os.ModePerm); err != nil { + t.Fatalf("Unable to create file, %s", err) + } + + testCases := []struct { + volName string + expectedErr error + }{ + // TestXLStorage case - 1. + // A valida case. Empty vol, should be possible to delete. + { + volName: "success-vol", + expectedErr: nil, + }, + // TestXLStorage case - 2. + // volume is non-existent. + { + volName: "nonexistent-vol", + expectedErr: errVolumeNotFound, + }, + // TestXLStorage case - 3. + // It shouldn't be possible to delete an non-empty volume, validating the same. + { + volName: "nonempty-vol", + expectedErr: errVolumeNotEmpty, + }, + // TestXLStorage case - 5. + // Invalid volume name. + { + volName: "ab", + expectedErr: errVolumeNotFound, + }, + } + + for i, testCase := range testCases { + if err = xlStorage.DeleteVol(t.Context(), testCase.volName, false); err != testCase.expectedErr { + t.Fatalf("TestXLStorage: %d, expected: %s, got: %s", i+1, testCase.expectedErr, err) + } + } + + // TestXLStorage for permission denied. + if runtime.GOOS != globalWindowsOSName { + permDeniedDir := t.TempDir() + if err = os.Mkdir(slashpath.Join(permDeniedDir, "mybucket"), 0o400); err != nil { + t.Fatalf("Unable to create temporary directory %v. %v", slashpath.Join(permDeniedDir, "mybucket"), err) + } + t.Cleanup(func() { + os.Chmod(slashpath.Join(permDeniedDir, "mybucket"), 0o775) + }) + + if err = os.Chmod(permDeniedDir, 0o400); err != nil { + t.Fatalf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err) + } + t.Cleanup(func() { + os.Chmod(permDeniedDir, 0o775) + }) + + // Initialize xlStorage storage layer for permission denied error. + _, err = newLocalXLStorage(permDeniedDir) + if err != nil && err != errDiskAccessDenied { + t.Fatalf("Unable to initialize xlStorage, %s", err) + } + + if err = os.Chmod(permDeniedDir, 0o755); err != nil { + t.Fatalf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err) + } + + xlStorageNew, err := newLocalXLStorage(permDeniedDir) + if err != nil { + t.Fatalf("Unable to initialize xlStorage, %s", err) + } + + // change backend permissions for MakeVol error. + if err = os.Chmod(permDeniedDir, 0o400); err != nil { + t.Fatalf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err) + } + + if err = xlStorageNew.DeleteVol(t.Context(), "mybucket", false); err != errDiskAccessDenied { + t.Fatalf("expected: Permission error, got: %s", err) + } + } + + xlStorageDeletedStorage, diskPath, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to create xlStorage test setup, %s", err) + } + // removing the disk, used to recreate disk not found error. + os.RemoveAll(diskPath) + + // TestXLStorage for delete on an removed disk. + // should fail with disk not found. + err = xlStorageDeletedStorage.DeleteVol(t.Context(), "Del-Vol", false) + if err != errDiskNotFound { + t.Errorf("Expected: \"Drive not found\", got \"%s\"", err) + } +} + +// TestXLStorageStatVol - TestXLStorages validate the volume info returned by xlStorage.StatVol() for various inputs. +func TestXLStorageStatVol(t *testing.T) { + // create xlStorage test setup + xlStorage, _, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to create xlStorage test setup, %s", err) + } + + // Setup test environment. + if err = xlStorage.MakeVol(t.Context(), "success-vol"); err != nil { + t.Fatalf("Unable to create volume, %s", err) + } + + testCases := []struct { + volName string + expectedErr error + }{ + // TestXLStorage case - 1. + { + volName: "success-vol", + expectedErr: nil, + }, + // TestXLStorage case - 2. + { + volName: "nonexistent-vol", + expectedErr: errVolumeNotFound, + }, + // TestXLStorage case - 3. + { + volName: "ab", + expectedErr: errVolumeNotFound, + }, + } + + for i, testCase := range testCases { + var volInfo VolInfo + volInfo, err = xlStorage.StatVol(t.Context(), testCase.volName) + if err != testCase.expectedErr { + t.Fatalf("TestXLStorage case : %d, Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) + } + + if err == nil { + if volInfo.Name != testCase.volName { + t.Errorf("TestXLStorage case %d: Expected the volume name to be \"%s\", instead found \"%s\"", + i+1, volInfo.Name, testCase.volName) + } + } + } + + xlStorageDeletedStorage, diskPath, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to create xlStorage test setup, %s", err) + } + // removing the disk, used to recreate disk not found error. + os.RemoveAll(diskPath) + + // TestXLStorage for delete on an removed disk. + // should fail with disk not found. + _, err = xlStorageDeletedStorage.StatVol(t.Context(), "Stat vol") + if err != errDiskNotFound { + t.Errorf("Expected: \"Drive not found\", got \"%s\"", err) + } +} + +// TestXLStorageListVols - Validates the result and the error output for xlStorage volume listing functionality xlStorage.ListVols(). +func TestXLStorageListVols(t *testing.T) { + // create xlStorage test setup + xlStorage, path, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to create xlStorage test setup, %s", err) + } + + var volInfos []VolInfo + // TestXLStorage empty list vols. + if volInfos, err = xlStorage.ListVols(t.Context()); err != nil { + t.Fatalf("expected: , got: %s", err) + } else if len(volInfos) != 1 { + t.Fatalf("expected: one entry, got: %v", volInfos) + } + + // TestXLStorage non-empty list vols. + if err = xlStorage.MakeVol(t.Context(), "success-vol"); err != nil { + t.Fatalf("Unable to create volume, %s", err) + } + + volInfos, err = xlStorage.ListVols(t.Context()) + if err != nil { + t.Fatalf("expected: , got: %s", err) + } + if len(volInfos) != 2 { + t.Fatalf("expected: 2, got: %d", len(volInfos)) + } + volFound := false + for _, info := range volInfos { + if info.Name == "success-vol" { + volFound = true + break + } + } + if !volFound { + t.Errorf("expected: success-vol to be created") + } + + // removing the path and simulating disk failure + os.RemoveAll(path) + // should fail with errDiskNotFound. + if _, err = xlStorage.ListVols(t.Context()); err != errDiskNotFound { + t.Errorf("Expected to fail with \"%s\", but instead failed with \"%s\"", errDiskNotFound, err) + } +} + +// TestXLStorageListDir - TestXLStorages validate the directory listing functionality provided by xlStorage.ListDir . +func TestXLStorageListDir(t *testing.T) { + // create xlStorage test setup + xlStorage, _, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to create xlStorage test setup, %s", err) + } + + // create xlStorage test setup. + xlStorageDeletedStorage, diskPath, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to create xlStorage test setup, %s", err) + } + // removing the disk, used to recreate disk not found error. + os.RemoveAll(diskPath) + // Setup test environment. + if err = xlStorage.MakeVol(t.Context(), "success-vol"); err != nil { + t.Fatalf("Unable to create volume, %s", err) + } + if err = xlStorage.AppendFile(t.Context(), "success-vol", "abc/def/ghi/success-file", []byte("Hello, world")); err != nil { + t.Fatalf("Unable to create file, %s", err) + } + if err = xlStorage.AppendFile(t.Context(), "success-vol", "abc/xyz/ghi/success-file", []byte("Hello, world")); err != nil { + t.Fatalf("Unable to create file, %s", err) + } + + testCases := []struct { + srcVol string + srcPath string + // expected result. + expectedListDir []string + expectedErr error + }{ + // TestXLStorage case - 1. + // valid case with existing volume and file to delete. + { + srcVol: "success-vol", + srcPath: "abc", + expectedListDir: []string{"def/", "xyz/"}, + expectedErr: nil, + }, + // TestXLStorage case - 1. + // valid case with existing volume and file to delete. + { + srcVol: "success-vol", + srcPath: "abc/def", + expectedListDir: []string{"ghi/"}, + expectedErr: nil, + }, + // TestXLStorage case - 1. + // valid case with existing volume and file to delete. + { + srcVol: "success-vol", + srcPath: "abc/def/ghi", + expectedListDir: []string{"success-file"}, + expectedErr: nil, + }, + // TestXLStorage case - 2. + { + srcVol: "success-vol", + srcPath: "abcdef", + expectedErr: errFileNotFound, + }, + // TestXLStorage case - 3. + // TestXLStorage case with invalid volume name. + { + srcVol: "ab", + srcPath: "success-file", + expectedErr: errVolumeNotFound, + }, + // TestXLStorage case - 4. + // TestXLStorage case with non existent volume. + { + srcVol: "non-existent-vol", + srcPath: "success-file", + expectedErr: errVolumeNotFound, + }, + } + + for i, testCase := range testCases { + var dirList []string + dirList, err = xlStorage.ListDir(t.Context(), "", testCase.srcVol, testCase.srcPath, -1) + if err != testCase.expectedErr { + t.Errorf("TestXLStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) + } + if err == nil { + for _, expected := range testCase.expectedListDir { + if !strings.Contains(strings.Join(dirList, ","), expected) { + t.Errorf("TestXLStorage case %d: Expected the directory listing to be \"%v\", but got \"%v\"", i+1, testCase.expectedListDir, dirList) + } + } + } + } + + // TestXLStorage for permission denied. + if runtime.GOOS != globalWindowsOSName { + permDeniedDir := createPermDeniedFile(t) + + // Initialize xlStorage storage layer for permission denied error. + _, err = newLocalXLStorage(permDeniedDir) + if err != nil && err != errDiskAccessDenied { + t.Fatalf("Unable to initialize xlStorage, %s", err) + } + + if err = os.Chmod(permDeniedDir, 0o755); err != nil { + t.Fatalf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err) + } + + xlStorageNew, err := newLocalXLStorage(permDeniedDir) + if err != nil { + t.Fatalf("Unable to initialize xlStorage, %s", err) + } + + if err = xlStorageNew.Delete(t.Context(), "mybucket", "myobject", DeleteOptions{ + Recursive: false, + Immediate: false, + }); err != errFileAccessDenied { + t.Errorf("expected: %s, got: %s", errFileAccessDenied, err) + } + } + + // TestXLStorage for delete on an removed disk. + // should fail with disk not found. + err = xlStorageDeletedStorage.Delete(t.Context(), "del-vol", "my-file", DeleteOptions{ + Recursive: false, + Immediate: false, + }) + if err != errDiskNotFound { + t.Errorf("Expected: \"Drive not found\", got \"%s\"", err) + } +} + +// TestXLStorageDeleteFile - Series of test cases construct valid and invalid input data and validates the result and the error response. +func TestXLStorageDeleteFile(t *testing.T) { + if runtime.GOOS == globalWindowsOSName { + t.Skip() + } + + // create xlStorage test setup + xlStorage, path, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to create xlStorage test setup, %s", err) + } + + // Setup test environment. + if err = xlStorage.MakeVol(t.Context(), "success-vol"); err != nil { + t.Fatalf("Unable to create volume, %s", err) + } + if err = xlStorage.AppendFile(t.Context(), "success-vol", "success-file", []byte("Hello, world")); err != nil { + t.Fatalf("Unable to create file, %s", err) + } + + if err = xlStorage.MakeVol(t.Context(), "no-permissions"); err != nil { + t.Fatalf("Unable to create volume, %s", err.Error()) + } + if err = xlStorage.AppendFile(t.Context(), "no-permissions", "dir/file", []byte("Hello, world")); err != nil { + t.Fatalf("Unable to create file, %s", err.Error()) + } + // Parent directory must have write permissions, this is read + execute. + if err = os.Chmod(pathJoin(path, "no-permissions"), 0o555); err != nil { + t.Fatalf("Unable to chmod directory, %s", err.Error()) + } + t.Cleanup(func() { + os.Chmod(pathJoin(path, "no-permissions"), 0o775) + }) + + testCases := []struct { + srcVol string + srcPath string + expectedErr error + }{ + // TestXLStorage case - 1. + // valid case with existing volume and file to delete. + { + srcVol: "success-vol", + srcPath: "success-file", + expectedErr: nil, + }, + // TestXLStorage case - 2. + // The file was deleted in the last case, so Delete should not fail. + { + srcVol: "success-vol", + srcPath: "success-file", + expectedErr: nil, + }, + // TestXLStorage case - 3. + // TestXLStorage case with segment of the volume name > 255. + { + srcVol: "my", + srcPath: "success-file", + expectedErr: errVolumeNotFound, + }, + // TestXLStorage case - 4. + // TestXLStorage case with non-existent volume. + { + srcVol: "non-existent-vol", + srcPath: "success-file", + expectedErr: errVolumeNotFound, + }, + // TestXLStorage case - 5. + // TestXLStorage case with src path segment > 255. + { + srcVol: "success-vol", + srcPath: "my-obj-del-0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", + expectedErr: errFileNameTooLong, + }, + } + + for i, testCase := range testCases { + if err = xlStorage.Delete(t.Context(), testCase.srcVol, testCase.srcPath, DeleteOptions{ + Recursive: false, + Immediate: false, + }); err != testCase.expectedErr { + t.Errorf("TestXLStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) + } + } + + // TestXLStorage for permission denied. + if runtime.GOOS != globalWindowsOSName { + permDeniedDir := createPermDeniedFile(t) + + // Initialize xlStorage storage layer for permission denied error. + _, err = newLocalXLStorage(permDeniedDir) + if err != nil && err != errDiskAccessDenied { + t.Fatalf("Unable to initialize xlStorage, %s", err) + } + + if err = os.Chmod(permDeniedDir, 0o755); err != nil { + t.Fatalf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err) + } + + xlStorageNew, err := newLocalXLStorage(permDeniedDir) + if err != nil { + t.Fatalf("Unable to initialize xlStorage, %s", err) + } + + if err = xlStorageNew.Delete(t.Context(), "mybucket", "myobject", DeleteOptions{ + Recursive: false, + Immediate: false, + }); err != errFileAccessDenied { + t.Errorf("expected: %s, got: %s", errFileAccessDenied, err) + } + } + + // create xlStorage test setup + xlStorageDeletedStorage, diskPath, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to create xlStorage test setup, %s", err) + } + // removing the disk, used to recreate disk not found error. + err = os.RemoveAll(diskPath) + if err != nil { + t.Fatalf("Unable to remoe xlStorage diskpath, %s", err) + } + + // TestXLStorage for delete on an removed disk. + // should fail with disk not found. + err = xlStorageDeletedStorage.Delete(t.Context(), "del-vol", "my-file", DeleteOptions{ + Recursive: false, + Immediate: false, + }) + if err != errDiskNotFound { + t.Errorf("Expected: \"Drive not found\", got \"%s\"", err) + } +} + +// TestXLStorageReadFile - TestXLStorages xlStorage.ReadFile with wide range of cases and asserts the result and error response. +func TestXLStorageReadFile(t *testing.T) { + // create xlStorage test setup + xlStorage, path, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to create xlStorage test setup, %s", err) + } + + volume := "success-vol" + // Setup test environment. + if err = xlStorage.MakeVol(t.Context(), volume); err != nil { + t.Fatalf("Unable to create volume, %s", err) + } + + // Create directory to make errIsNotRegular + if err = os.Mkdir(slashpath.Join(path, "success-vol", "object-as-dir"), 0o777); err != nil { + t.Fatalf("Unable to create directory, %s", err) + } + + testCases := []struct { + volume string + fileName string + offset int64 + bufSize int + expectedBuf []byte + expectedErr error + }{ + // Successful read at offset 0 and proper buffer size. - 1 + { + volume, "myobject", 0, 5, + []byte("hello"), nil, + }, + // Success read at hierarchy. - 2 + { + volume, "path/to/my/object", 0, 5, + []byte("hello"), nil, + }, + // Object is a directory. - 3 + { + volume, "object-as-dir", + 0, 5, nil, errIsNotRegular, + }, + // One path segment length is > 255 chars long. - 4 + { + volume, "path/to/my/object0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", + 0, 5, nil, errFileNameTooLong, + }, + // Path length is > 1024 chars long. - 5 + { + volume, "level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001/level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002/level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003/object000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", + 0, 5, nil, errFileNameTooLong, + }, + // Buffer size greater than object size. - 6 + { + volume, "myobject", 0, 16, + []byte("hello, world"), + io.ErrUnexpectedEOF, + }, + // Reading from an offset success. - 7 + { + volume, "myobject", 7, 5, + []byte("world"), nil, + }, + // Reading from an object but buffer size greater. - 8 + { + volume, "myobject", + 7, 8, + []byte("world"), + io.ErrUnexpectedEOF, + }, + // Seeking ahead returns io.EOF. - 9 + { + volume, "myobject", 14, 1, nil, io.EOF, + }, + // Empty volume name. - 10 + { + "", "myobject", 14, 1, nil, errVolumeNotFound, + }, + // Empty filename name. - 11 + { + volume, "", 14, 1, nil, errIsNotRegular, + }, + // Non existent volume name - 12 + { + "abcd", "", 14, 1, nil, errVolumeNotFound, + }, + // Non existent filename - 13 + { + volume, "abcd", 14, 1, nil, errFileNotFound, + }, + } + + // Create all files needed during testing. + appendFiles := testCases[:4] + v := NewBitrotVerifier(SHA256, getSHA256Sum([]byte("hello, world"))) + // Create test files for further reading. + for i, appendFile := range appendFiles { + err = xlStorage.AppendFile(t.Context(), volume, appendFile.fileName, []byte("hello, world")) + if err != appendFile.expectedErr { + t.Fatalf("Creating file failed: %d %#v, expected: %s, got: %s", i+1, appendFile, appendFile.expectedErr, err) + } + } + + { + buf := make([]byte, 5) + // Test for negative offset. + if _, err = xlStorage.ReadFile(t.Context(), volume, "myobject", -1, buf, v); err == nil { + t.Fatalf("expected: error, got: ") + } + } + + for range 2 { + // Following block validates all ReadFile test cases. + for i, testCase := range testCases { + var n int64 + // Common read buffer. + buf := make([]byte, testCase.bufSize) + n, err = xlStorage.ReadFile(t.Context(), testCase.volume, testCase.fileName, testCase.offset, buf, v) + if err != nil && testCase.expectedErr != nil { + // Validate if the type string of the errors are an exact match. + if err.Error() != testCase.expectedErr.Error() { + if runtime.GOOS != globalWindowsOSName { + t.Errorf("Case: %d %#v, expected: %s, got: %s", i+1, testCase, testCase.expectedErr, err) + } else { + var resultErrno, expectErrno uintptr + if pathErr, ok := err.(*os.PathError); ok { + if errno, pok := pathErr.Err.(syscall.Errno); pok { + resultErrno = uintptr(errno) + } + } + if pathErr, ok := testCase.expectedErr.(*os.PathError); ok { + if errno, pok := pathErr.Err.(syscall.Errno); pok { + expectErrno = uintptr(errno) + } + } + if expectErrno == 0 || resultErrno == 0 || expectErrno != resultErrno { + t.Errorf("Case: %d %#v, expected: %s, got: %s", i+1, testCase, testCase.expectedErr, err) + } + } + } + // Err unexpected EOF special case, where we verify we have provided a larger + // buffer than the data itself, but the results are in-fact valid. So we validate + // this error condition specifically treating it as a good condition with valid + // results. In this scenario return 'n' is always lesser than the input buffer. + if err == io.ErrUnexpectedEOF { + if !bytes.Equal(testCase.expectedBuf, buf[:n]) { + t.Errorf("Case: %d %#v, expected: \"%s\", got: \"%s\"", i+1, testCase, string(testCase.expectedBuf), string(buf[:n])) + } + if n > int64(len(buf)) { + t.Errorf("Case: %d %#v, expected: %d, got: %d", i+1, testCase, testCase.bufSize, n) + } + } + } + // ReadFile has returned success, but our expected error is non 'nil'. + if err == nil && err != testCase.expectedErr { + t.Errorf("Case: %d %#v, expected: %s, got :%s", i+1, testCase, testCase.expectedErr, err) + } + // Expected error returned, proceed further to validate the returned results. + if err != nil && testCase.expectedErr == nil { + t.Errorf("Case: %d %#v, expected: %s, got :%s", i+1, testCase, testCase.expectedErr, err) + } + if err == nil { + if !bytes.Equal(testCase.expectedBuf, buf) { + t.Errorf("Case: %d %#v, expected: \"%s\", got: \"%s\"", i+1, testCase, string(testCase.expectedBuf), string(buf[:testCase.bufSize])) + } + if n != int64(testCase.bufSize) { + t.Errorf("Case: %d %#v, expected: %d, got: %d", i+1, testCase, testCase.bufSize, n) + } + } + } + } + + // TestXLStorage for permission denied. + if runtime.GOOS != globalWindowsOSName { + permDeniedDir := createPermDeniedFile(t) + + // Initialize xlStorage storage layer for permission denied error. + _, err = newLocalXLStorage(permDeniedDir) + if err != nil && err != errDiskAccessDenied { + t.Fatalf("Unable to initialize xlStorage, %s", err) + } + + if err = os.Chmod(permDeniedDir, 0o755); err != nil { + t.Fatalf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err) + } + + xlStoragePermStorage, err := newLocalXLStorage(permDeniedDir) + if err != nil { + t.Fatalf("Unable to initialize xlStorage, %s", err) + } + + // Common read buffer. + buf := make([]byte, 10) + if _, err = xlStoragePermStorage.ReadFile(t.Context(), "mybucket", "myobject", 0, buf, v); err != errFileAccessDenied { + t.Errorf("expected: %s, got: %s", errFileAccessDenied, err) + } + } +} + +var xlStorageReadFileWithVerifyTests = []struct { + file string + offset int + length int + algorithm BitrotAlgorithm + expError error +}{ + {file: "myobject", offset: 0, length: 100, algorithm: SHA256, expError: nil}, // 0 + {file: "myobject", offset: 25, length: 74, algorithm: SHA256, expError: nil}, // 1 + {file: "myobject", offset: 29, length: 70, algorithm: SHA256, expError: nil}, // 2 + {file: "myobject", offset: 100, length: 0, algorithm: SHA256, expError: nil}, // 3 + {file: "myobject", offset: 1, length: 120, algorithm: SHA256, expError: errFileCorrupt}, // 4 + {file: "myobject", offset: 3, length: 1100, algorithm: SHA256, expError: nil}, // 5 + {file: "myobject", offset: 2, length: 100, algorithm: SHA256, expError: errFileCorrupt}, // 6 + {file: "myobject", offset: 1000, length: 1001, algorithm: SHA256, expError: nil}, // 7 + {file: "myobject", offset: 0, length: 100, algorithm: BLAKE2b512, expError: errFileCorrupt}, // 8 + {file: "myobject", offset: 25, length: 74, algorithm: BLAKE2b512, expError: nil}, // 9 + {file: "myobject", offset: 29, length: 70, algorithm: BLAKE2b512, expError: errFileCorrupt}, // 10 + {file: "myobject", offset: 100, length: 0, algorithm: BLAKE2b512, expError: nil}, // 11 + {file: "myobject", offset: 1, length: 120, algorithm: BLAKE2b512, expError: nil}, // 12 + {file: "myobject", offset: 3, length: 1100, algorithm: BLAKE2b512, expError: nil}, // 13 + {file: "myobject", offset: 2, length: 100, algorithm: BLAKE2b512, expError: nil}, // 14 + {file: "myobject", offset: 1000, length: 1001, algorithm: BLAKE2b512, expError: nil}, // 15 +} + +// TestXLStorageReadFile with bitrot verification - tests the xlStorage level +// ReadFile API with a BitrotVerifier. Only tests hashing related +// functionality. Other functionality is tested with +// TestXLStorageReadFile. +func TestXLStorageReadFileWithVerify(t *testing.T) { + volume, object := "test-vol", "myobject" + xlStorage, _, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to create xlStorage test setup, %s", err) + } + if err = xlStorage.MakeVol(t.Context(), volume); err != nil { + t.Fatalf("Unable to create volume %s: %v", volume, err) + } + data := make([]byte, 8*1024) + if _, err = io.ReadFull(rand.Reader, data); err != nil { + t.Fatalf("Unable to create generate random data: %v", err) + } + if err = xlStorage.AppendFile(t.Context(), volume, object, data); err != nil { + t.Fatalf("Unable to create object: %v", err) + } + + for i, test := range xlStorageReadFileWithVerifyTests { + h := test.algorithm.New() + h.Write(data) + if test.expError != nil { + h.Write([]byte{0}) + } + + buffer := make([]byte, test.length) + n, err := xlStorage.ReadFile(t.Context(), volume, test.file, int64(test.offset), buffer, NewBitrotVerifier(test.algorithm, h.Sum(nil))) + + switch { + case err == nil && test.expError != nil: + t.Errorf("Test %d: Expected error %v but got none.", i, test.expError) + case err == nil && n != int64(test.length): + t.Errorf("Test %d: %d bytes were expected, but %d were written", i, test.length, n) + case err == nil && !bytes.Equal(data[test.offset:test.offset+test.length], buffer): + t.Errorf("Test %d: Expected bytes: %v, but got: %v", i, data[test.offset:test.offset+test.length], buffer) + case err != nil && err != test.expError: + t.Errorf("Test %d: Expected error: %v, but got: %v", i, test.expError, err) + } + } +} + +// TestXLStorageFormatFileChange - to test if changing the diskID makes the calls fail. +func TestXLStorageFormatFileChange(t *testing.T) { + volume := "fail-vol" + xlStorage, _, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to create xlStorage test setup, %s", err) + } + + if err = xlStorage.MakeVol(t.Context(), volume); err != nil { + t.Fatalf("MakeVol failed with %s", err) + } + + // Change the format.json such that "this" is changed to "randomid". + if err = os.WriteFile(pathJoin(xlStorage.String(), minioMetaBucket, formatConfigFile), []byte(`{"version":"1","format":"xl","id":"592a41c2-b7cc-4130-b883-c4b5cb15965b","xl":{"version":"3","this":"randomid","sets":[["e07285a6-8c73-4962-89c6-047fb939f803","33b8d431-482d-4376-b63c-626d229f0a29","cff6513a-4439-4dc1-bcaa-56c9e880c352","randomid","9c9f21d5-1f15-4737-bce6-835faa0d9626","0a59b346-1424-4fc2-9fa2-a2e80541d0c1","7924a3dc-b69a-4971-9a2e-014966d6aebb","4d2b8dd9-4e48-444b-bdca-c89194b26042"]],"distributionAlgo":"CRCMOD"}}`), 0o644); err != nil { + t.Fatalf("ioutil.WriteFile failed with %s", err) + } + + err = xlStorage.MakeVol(t.Context(), volume) + if err != errVolumeExists { + t.Fatalf("MakeVol expected to fail with errDiskNotFound but failed with %s", err) + } +} + +// TestXLStorage xlStorage.AppendFile() +func TestXLStorageAppendFile(t *testing.T) { + // create xlStorage test setup + xlStorage, path, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to create xlStorage test setup, %s", err) + } + + // Setup test environment. + if err = xlStorage.MakeVol(t.Context(), "success-vol"); err != nil { + t.Fatalf("Unable to create volume, %s", err) + } + + // Create directory to make errIsNotRegular + if err = os.Mkdir(slashpath.Join(path, "success-vol", "object-as-dir"), 0o777); err != nil { + t.Fatalf("Unable to create directory, %s", err) + } + + testCases := []struct { + fileName string + expectedErr error + }{ + {"myobject", nil}, + {"path/to/my/object", nil}, + // TestXLStorage to append to previously created file. + {"myobject", nil}, + // TestXLStorage to use same path of previously created file. + {"path/to/my/testobject", nil}, + // TestXLStorage to use object is a directory now. + {"object-as-dir", errIsNotRegular}, + // path segment uses previously uploaded object. + {"myobject/testobject", errFileAccessDenied}, + // One path segment length is > 255 chars long. + {"path/to/my/object0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", errFileNameTooLong}, + // path length is > 1024 chars long. + {"level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001/level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000002/level0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000003/object000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", errFileNameTooLong}, + } + + for i, testCase := range testCases { + if err = xlStorage.AppendFile(t.Context(), "success-vol", testCase.fileName, []byte("hello, world")); err != testCase.expectedErr { + t.Errorf("Case: %d, expected: %s, got: %s", i+1, testCase.expectedErr, err) + } + } + + // TestXLStorage for permission denied. + if runtime.GOOS != globalWindowsOSName { + permDeniedDir := createPermDeniedFile(t) + + var xlStoragePermStorage StorageAPI + // Initialize xlStorage storage layer for permission denied error. + _, err = newLocalXLStorage(permDeniedDir) + if err != nil && err != errDiskAccessDenied { + t.Fatalf("Unable to initialize xlStorage, %s", err) + } + + if err = os.Chmod(permDeniedDir, 0o755); err != nil { + t.Fatalf("Unable to change permission to temporary directory %v. %v", permDeniedDir, err) + } + + xlStoragePermStorage, err = newLocalXLStorage(permDeniedDir) + if err != nil { + t.Fatalf("Unable to initialize xlStorage, %s", err) + } + + if err = xlStoragePermStorage.AppendFile(t.Context(), "mybucket", "myobject", []byte("hello, world")); err != errFileAccessDenied { + t.Fatalf("expected: errFileAccessDenied error, got: %s", err) + } + } + + // TestXLStorage case with invalid volume name. + // A valid volume name should be at least of size 3. + err = xlStorage.AppendFile(t.Context(), "bn", "yes", []byte("hello, world")) + if err != errVolumeNotFound { + t.Fatalf("expected: \"Invalid argument error\", got: \"%s\"", err) + } +} + +// TestXLStorage xlStorage.RenameFile() +func TestXLStorageRenameFile(t *testing.T) { + // create xlStorage test setup + xlStorage, _, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to create xlStorage test setup, %s", err) + } + + // Setup test environment. + if err := xlStorage.MakeVol(t.Context(), "src-vol"); err != nil { + t.Fatalf("Unable to create volume, %s", err) + } + + if err := xlStorage.MakeVol(t.Context(), "dest-vol"); err != nil { + t.Fatalf("Unable to create volume, %s", err) + } + + if err := xlStorage.AppendFile(t.Context(), "src-vol", "file1", []byte("Hello, world")); err != nil { + t.Fatalf("Unable to create file, %s", err) + } + + if err := xlStorage.AppendFile(t.Context(), "src-vol", "file2", []byte("Hello, world")); err != nil { + t.Fatalf("Unable to create file, %s", err) + } + if err := xlStorage.AppendFile(t.Context(), "src-vol", "file3", []byte("Hello, world")); err != nil { + t.Fatalf("Unable to create file, %s", err) + } + if err := xlStorage.AppendFile(t.Context(), "src-vol", "file4", []byte("Hello, world")); err != nil { + t.Fatalf("Unable to create file, %s", err) + } + + if err := xlStorage.AppendFile(t.Context(), "src-vol", "file5", []byte("Hello, world")); err != nil { + t.Fatalf("Unable to create file, %s", err) + } + if err := xlStorage.AppendFile(t.Context(), "src-vol", "path/to/file1", []byte("Hello, world")); err != nil { + t.Fatalf("Unable to create file, %s", err) + } + + testCases := []struct { + srcVol string + destVol string + srcPath string + destPath string + expectedErr error + }{ + // TestXLStorage case - 1. + { + srcVol: "src-vol", + destVol: "dest-vol", + srcPath: "file1", + destPath: "file-one", + expectedErr: nil, + }, + // TestXLStorage case - 2. + { + srcVol: "src-vol", + destVol: "dest-vol", + srcPath: "path/", + destPath: "new-path/", + expectedErr: nil, + }, + // TestXLStorage case - 3. + // TestXLStorage to overwrite destination file. + { + srcVol: "src-vol", + destVol: "dest-vol", + srcPath: "file2", + destPath: "file-one", + expectedErr: nil, + }, + // TestXLStorage case - 4. + // TestXLStorage case with io error count set to 1. + // expected not to fail. + { + srcVol: "src-vol", + destVol: "dest-vol", + srcPath: "file3", + destPath: "file-two", + expectedErr: nil, + }, + // TestXLStorage case - 5. + // TestXLStorage case with io error count set to maximum allowed count. + // expected not to fail. + { + srcVol: "src-vol", + destVol: "dest-vol", + srcPath: "file4", + destPath: "file-three", + expectedErr: nil, + }, + // TestXLStorage case - 6. + // TestXLStorage case with non-existent source file. + { + srcVol: "src-vol", + destVol: "dest-vol", + srcPath: "non-existent-file", + destPath: "file-three", + expectedErr: errFileNotFound, + }, + // TestXLStorage case - 7. + // TestXLStorage to check failure of source and destination are not same type. + { + srcVol: "src-vol", + destVol: "dest-vol", + srcPath: "path/", + destPath: "file-one", + expectedErr: errFileAccessDenied, + }, + // TestXLStorage case - 8. + // TestXLStorage to check failure of destination directory exists. + { + srcVol: "src-vol", + destVol: "dest-vol", + srcPath: "path/", + destPath: "new-path/", + expectedErr: errFileAccessDenied, + }, + // TestXLStorage case - 9. + // TestXLStorage case with source being a file and destination being a directory. + // Either both have to be files or directories. + // Expecting to fail with `errFileAccessDenied`. + { + srcVol: "src-vol", + destVol: "dest-vol", + srcPath: "file4", + destPath: "new-path/", + expectedErr: errFileAccessDenied, + }, + // TestXLStorage case - 10. + // TestXLStorage case with non-existent source volume. + // Expecting to fail with `errVolumeNotFound`. + { + srcVol: "src-vol-non-existent", + destVol: "dest-vol", + srcPath: "file4", + destPath: "new-path/", + expectedErr: errVolumeNotFound, + }, + // TestXLStorage case - 11. + // TestXLStorage case with non-existent destination volume. + // Expecting to fail with `errVolumeNotFound`. + { + srcVol: "src-vol", + destVol: "dest-vol-non-existent", + srcPath: "file4", + destPath: "new-path/", + expectedErr: errVolumeNotFound, + }, + // TestXLStorage case - 12. + // TestXLStorage case with invalid src volume name. Length should be at least 3. + // Expecting to fail with `errInvalidArgument`. + { + srcVol: "ab", + destVol: "dest-vol-non-existent", + srcPath: "file4", + destPath: "new-path/", + expectedErr: errVolumeNotFound, + }, + // TestXLStorage case - 13. + // TestXLStorage case with invalid destination volume name. Length should be at least 3. + // Expecting to fail with `errInvalidArgument`. + { + srcVol: "abcd", + destVol: "ef", + srcPath: "file4", + destPath: "new-path/", + expectedErr: errVolumeNotFound, + }, + // TestXLStorage case - 14. + // TestXLStorage case with invalid destination volume name. Length should be at least 3. + // Expecting to fail with `errInvalidArgument`. + { + srcVol: "abcd", + destVol: "ef", + srcPath: "file4", + destPath: "new-path/", + expectedErr: errVolumeNotFound, + }, + // TestXLStorage case - 15. + // TestXLStorage case with the parent of the destination being a file. + // expected to fail with `errFileAccessDenied`. + { + srcVol: "src-vol", + destVol: "dest-vol", + srcPath: "file5", + destPath: "file-one/parent-is-file", + expectedErr: errFileAccessDenied, + }, + // TestXLStorage case - 16. + // TestXLStorage case with segment of source file name more than 255. + // expected not to fail. + { + srcVol: "src-vol", + destVol: "dest-vol", + srcPath: "path/to/my/object0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", + destPath: "file-six", + expectedErr: errFileNameTooLong, + }, + // TestXLStorage case - 17. + // TestXLStorage case with segment of destination file name more than 255. + // expected not to fail. + { + srcVol: "src-vol", + destVol: "dest-vol", + srcPath: "file6", + destPath: "path/to/my/object0000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001", + expectedErr: errFileNameTooLong, + }, + } + + for i, testCase := range testCases { + if err := xlStorage.RenameFile(t.Context(), testCase.srcVol, testCase.srcPath, testCase.destVol, testCase.destPath); err != testCase.expectedErr { + t.Fatalf("TestXLStorage %d: Expected the error to be : \"%v\", got: \"%v\".", i+1, testCase.expectedErr, err) + } + } +} + +// TestXLStorageDeleteVersion will test if version deletes and bulk deletes work as expected. +func TestXLStorageDeleteVersion(t *testing.T) { + // create xlStorage test setup + xl, _, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to create xlStorage test setup, %s", err) + } + ctx := t.Context() + + volume := "myvol-vol" + object := "my-object" + if err := xl.MakeVol(ctx, volume); err != nil { + t.Fatalf("Unable to create volume, %s", err) + } + var versions [50]string + for i := range versions { + versions[i] = uuid.New().String() + fi := FileInfo{ + Name: object, Volume: volume, VersionID: versions[i], ModTime: UTCNow(), DataDir: "", Size: 10000, + Erasure: ErasureInfo{ + Algorithm: erasureAlgorithm, + DataBlocks: 4, + ParityBlocks: 4, + BlockSize: blockSizeV2, + Index: 1, + Distribution: []int{0, 1, 2, 3, 4, 5, 6, 7}, + Checksums: nil, + }, + } + if err := xl.WriteMetadata(ctx, "", volume, object, fi); err != nil { + t.Fatalf("Unable to create object, %s", err) + } + } + var deleted [len(versions)]bool + checkVerExist := func(t testing.TB) { + t.Helper() + for i := range versions { + shouldExist := !deleted[i] + fi, err := xl.ReadVersion(ctx, "", volume, object, versions[i], ReadOptions{}) + if shouldExist { + if err != nil { + t.Fatalf("Version %s should exist, but got err %v", versions[i], err) + } + return + } + if err != errFileVersionNotFound { + t.Fatalf("Version %s should not exist, but returned: %#v", versions[i], fi) + } + } + } + + // Delete version 0... + checkVerExist(t) + err = xl.DeleteVersion(ctx, volume, object, FileInfo{Name: object, Volume: volume, VersionID: versions[0]}, false, DeleteOptions{}) + if err != nil { + t.Fatal(err) + } + deleted[0] = true + checkVerExist(t) + + // Delete 10 in bulk, including a non-existing. + fis := []FileInfoVersions{{Name: object, Volume: volume}} + for i := range versions[:10] { + fis[0].Versions = append(fis[0].Versions, FileInfo{Name: object, Volume: volume, VersionID: versions[i]}) + deleted[i] = true + } + errs := xl.DeleteVersions(ctx, volume, fis, DeleteOptions{}) + if errs[0] != nil { + t.Fatalf("expected nil error, got %v", errs[0]) + } + checkVerExist(t) + + // Delete them all... (some again) + fis[0].Versions = nil + for i := range versions[:] { + fis[0].Versions = append(fis[0].Versions, FileInfo{Name: object, Volume: volume, VersionID: versions[i]}) + deleted[i] = true + } + errs = xl.DeleteVersions(ctx, volume, fis, DeleteOptions{}) + if errs[0] != nil { + t.Fatalf("expected nil error, got %v", errs[0]) + } + checkVerExist(t) + + // Meta should be deleted now... + fi, err := xl.ReadVersion(ctx, "", volume, object, "", ReadOptions{}) + if err != errFileNotFound { + t.Fatalf("Object %s should not exist, but returned: %#v", object, fi) + } +} + +// TestXLStorage xlStorage.StatInfoFile() +func TestXLStorageStatInfoFile(t *testing.T) { + // create xlStorage test setup + xlStorage, _, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to create xlStorage test setup, %s", err) + } + + // Setup test environment. + if err := xlStorage.MakeVol(t.Context(), "success-vol"); err != nil { + t.Fatalf("Unable to create volume, %s", err) + } + + if err := xlStorage.AppendFile(t.Context(), "success-vol", pathJoin("success-file", xlStorageFormatFile), []byte("Hello, world")); err != nil { + t.Fatalf("Unable to create file, %s", err) + } + + if err := xlStorage.AppendFile(t.Context(), "success-vol", pathJoin("path/to/success-file", xlStorageFormatFile), []byte("Hello, world")); err != nil { + t.Fatalf("Unable to create file, %s", err) + } + + if err := xlStorage.MakeVol(t.Context(), "success-vol/path/to/"+xlStorageFormatFile); err != nil { + t.Fatalf("Unable to create path, %s", err) + } + + testCases := []struct { + srcVol string + srcPath string + expectedErr error + }{ + // TestXLStorage case - 1. + // TestXLStorage case with valid inputs, expected to pass. + { + srcVol: "success-vol", + srcPath: "success-file", + expectedErr: nil, + }, + // TestXLStorage case - 2. + // TestXLStorage case with valid inputs, expected to pass. + { + srcVol: "success-vol", + srcPath: "path/to/success-file", + expectedErr: nil, + }, + // TestXLStorage case - 3. + // TestXLStorage case with non-existent file. + { + srcVol: "success-vol", + srcPath: "nonexistent-file", + expectedErr: errPathNotFound, + }, + // TestXLStorage case - 4. + // TestXLStorage case with non-existent file path. + { + srcVol: "success-vol", + srcPath: "path/2/success-file", + expectedErr: errPathNotFound, + }, + // TestXLStorage case - 5. + // TestXLStorage case with path being a directory. + { + srcVol: "success-vol", + srcPath: "path", + expectedErr: errPathNotFound, + }, + // TestXLStorage case - 6. + // TestXLStorage case with non existent volume. + { + srcVol: "non-existent-vol", + srcPath: "success-file", + expectedErr: errVolumeNotFound, + }, + // TestXLStorage case - 7. + // TestXLStorage case with file with directory. + { + srcVol: "success-vol", + srcPath: "path/to", + expectedErr: nil, + }, + } + + for i, testCase := range testCases { + _, err := xlStorage.StatInfoFile(t.Context(), testCase.srcVol, testCase.srcPath+"/"+xlStorageFormatFile, false) + if err != testCase.expectedErr { + t.Errorf("TestXLStorage case %d: Expected: \"%s\", got: \"%s\"", i+1, testCase.expectedErr, err) + } + } +} + +// Test xlStorage.VerifyFile() +func TestXLStorageVerifyFile(t *testing.T) { + // We test 4 cases: + // 1) Whole-file bitrot check on proper file + // 2) Whole-file bitrot check on corrupted file + // 3) Streaming bitrot check on proper file + // 4) Streaming bitrot check on corrupted file + + // create xlStorage test setup + storage, path, err := newXLStorageTestSetup(t) + if err != nil { + t.Fatalf("Unable to create xlStorage test setup, %s", err) + } + + volName := "testvol" + fileName := "testfile" + if err := storage.MakeVol(t.Context(), volName); err != nil { + t.Fatal(err) + } + + // 1) Whole-file bitrot check on proper file + size := int64(4*1024*1024 + 100*1024) // 4.1 MB + data := make([]byte, size) + if _, err := rand.Read(data); err != nil { + t.Fatal(err) + } + algo := HighwayHash256 + h := algo.New() + h.Write(data) + hashBytes := h.Sum(nil) + if err := storage.WriteAll(t.Context(), volName, fileName, data); err != nil { + t.Fatal(err) + } + if err := storage.storage.bitrotVerify(t.Context(), pathJoin(path, volName, fileName), size, algo, hashBytes, 0); err != nil { + t.Fatal(err) + } + + // 2) Whole-file bitrot check on corrupted file + if err := storage.AppendFile(t.Context(), volName, fileName, []byte("a")); err != nil { + t.Fatal(err) + } + + // Check if VerifyFile reports the incorrect file length (the correct length is `size+1`) + if err := storage.storage.bitrotVerify(t.Context(), pathJoin(path, volName, fileName), size, algo, hashBytes, 0); err == nil { + t.Fatal("expected to fail bitrot check") + } + + // Check if bitrot fails + if err := storage.storage.bitrotVerify(t.Context(), pathJoin(path, volName, fileName), size+1, algo, hashBytes, 0); err == nil { + t.Fatal("expected to fail bitrot check") + } + + if err := storage.Delete(t.Context(), volName, fileName, DeleteOptions{ + Recursive: false, + Immediate: false, + }); err != nil { + t.Fatal(err) + } + + // 3) Streaming bitrot check on proper file + algo = HighwayHash256S + shardSize := int64(1024 * 1024) + shard := make([]byte, shardSize) + w := newStreamingBitrotWriter(storage, "", volName, fileName, size, algo, shardSize) + reader := bytes.NewReader(data) + for { + // Using io.Copy instead of this loop will not work for us as io.Copy + // will use bytes.Reader.WriteTo() which will not do shardSize'ed writes + // causing error. + n, err := reader.Read(shard) + w.Write(shard[:n]) + if err == nil { + continue + } + if err == io.EOF { + break + } + t.Fatal(err) + } + w.(io.Closer).Close() + if err := storage.storage.bitrotVerify(t.Context(), pathJoin(path, volName, fileName), size, algo, nil, shardSize); err != nil { + t.Fatal(err) + } + + // 4) Streaming bitrot check on corrupted file + filePath := pathJoin(storage.String(), volName, fileName) + f, err := os.OpenFile(filePath, os.O_WRONLY|os.O_SYNC, 0o644) + if err != nil { + t.Fatal(err) + } + // Replace first 256 with 'a'. + if _, err := f.WriteString(strings.Repeat("a", 256)); err != nil { + t.Fatal(err) + } + f.Close() + if err := storage.storage.bitrotVerify(t.Context(), pathJoin(path, volName, fileName), size, algo, nil, shardSize); err == nil { + t.Fatal("expected to fail bitrot check") + } + if err := storage.storage.bitrotVerify(t.Context(), pathJoin(path, volName, fileName), size+1, algo, nil, shardSize); err == nil { + t.Fatal("expected to fail bitrot check") + } +} + +// TestXLStorageReadMetadata tests readMetadata +func TestXLStorageReadMetadata(t *testing.T) { + volume, object := "test-vol", "AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" + tmpDir := t.TempDir() + + disk, err := newLocalXLStorage(tmpDir) + if err != nil { + t.Fatal(err) + } + + disk.MakeVol(t.Context(), volume) + if _, err := disk.readMetadata(t.Context(), pathJoin(tmpDir, volume, object)); err != errFileNameTooLong { + t.Fatalf("Unexpected error from readMetadata - expect %v: got %v", errFileNameTooLong, err) + } +} diff --git a/cmd/xl-storage_unix_test.go b/cmd/xl-storage_unix_test.go new file mode 100644 index 00000000..cec6990b --- /dev/null +++ b/cmd/xl-storage_unix_test.go @@ -0,0 +1,109 @@ +//go:build linux || darwin || dragonfly || freebsd || netbsd || openbsd +// +build linux darwin dragonfly freebsd netbsd openbsd + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "os" + "path" + "syscall" + "testing" +) + +// Based on `man getumask` a vaporware GNU extension to glibc. +// returns file mode creation mask. +func getUmask() int { + mask := syscall.Umask(0) + syscall.Umask(mask) + return mask +} + +// Tests if the directory and file creations happen with proper umask. +func TestIsValidUmaskVol(t *testing.T) { + tmpPath := t.TempDir() + testCases := []struct { + volName string + expectedUmask int + }{ + {"is-this-valid", getUmask()}, + } + testCase := testCases[0] + + // Initialize a new xlStorage layer. + disk, err := newLocalXLStorage(tmpPath) + if err != nil { + t.Fatalf("Initializing xlStorage failed with %s.", err) + } + + // Attempt to create a volume to verify the permissions later. + // MakeVol creates 0777. + if err = disk.MakeVol(t.Context(), testCase.volName); err != nil { + t.Fatalf("Creating a volume failed with %s expected to pass.", err) + } + + // Stat to get permissions bits. + st, err := os.Stat(path.Join(tmpPath, testCase.volName)) + if err != nil { + t.Fatalf("Stat failed with %s expected to pass.", err) + } + + // Get umask of the bits stored. + currentUmask := 0o777 - uint32(st.Mode().Perm()) + + // Verify if umask is correct. + if int(currentUmask) != testCase.expectedUmask { + t.Fatalf("Umask check failed expected %d, got %d", testCase.expectedUmask, currentUmask) + } +} + +// Tests if the file creations happen with proper umask. +func TestIsValidUmaskFile(t *testing.T) { + tmpPath := t.TempDir() + testCases := []struct { + volName string + expectedUmask int + }{ + {"is-this-valid", getUmask()}, + } + testCase := testCases[0] + + // Initialize a new xlStorage layer. + disk, err := newLocalXLStorage(tmpPath) + if err != nil { + t.Fatalf("Initializing xlStorage failed with %s.", err) + } + + // Attempt to create a volume to verify the permissions later. + // MakeVol creates directory with 0777 perms. + if err = disk.MakeVol(t.Context(), testCase.volName); err != nil { + t.Fatalf("Creating a volume failed with %s expected to pass.", err) + } + + // Attempt to create a file to verify the permissions later. + // AppendFile creates file with 0666 perms. + if err = disk.AppendFile(t.Context(), testCase.volName, pathJoin("hello-world.txt", xlStorageFormatFile), []byte("Hello World")); err != nil { + t.Fatalf("Create a file `test` failed with %s expected to pass.", err) + } + + // CheckFile - stat the file. + if _, err := disk.StatInfoFile(t.Context(), testCase.volName, "hello-world.txt/"+xlStorageFormatFile, false); err != nil { + t.Fatalf("Stat failed with %s expected to pass.", err) + } +} diff --git a/cmd/xl-storage_windows_test.go b/cmd/xl-storage_windows_test.go new file mode 100644 index 00000000..7bb5466e --- /dev/null +++ b/cmd/xl-storage_windows_test.go @@ -0,0 +1,100 @@ +//go:build windows +// +build windows + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cmd + +import ( + "bytes" + "context" + "fmt" + "testing" +) + +// Test if various paths work as expected when converted to UNC form +func TestUNCPaths(t *testing.T) { + testCases := []struct { + objName string + pass bool + }{ + {"/abcdef", true}, + {"/a/b/c/d/e/f/g", true}, + {string(bytes.Repeat([]byte("界"), 85)), true}, + // Each path component must be <= 255 bytes long. + {string(bytes.Repeat([]byte("界"), 280)), false}, + {`/p/q/r/s/t`, true}, + } + dir := t.TempDir() + + // Instantiate posix object to manage a disk + fs, err := newLocalXLStorage(dir) + if err != nil { + t.Fatal(err) + } + + // Create volume to use in conjunction with other StorageAPI's file API(s) + err = fs.MakeVol(context.Background(), "voldir") + if err != nil { + t.Fatal(err) + } + + for i, test := range testCases { + t.Run(fmt.Sprint(i), func(t *testing.T) { + err = fs.AppendFile(context.Background(), "voldir", test.objName, []byte("hello")) + if err != nil && test.pass { + t.Error(err) + } else if err == nil && !test.pass { + t.Error(err) + } + fs.Delete(context.Background(), "voldir", test.objName, DeleteOptions{ + Recursive: false, + Immediate: false, + }) + }) + } +} + +// Test to validate xlStorage behavior on windows when a non-final path component is a file. +func TestUNCPathENOTDIR(t *testing.T) { + // Instantiate posix object to manage a disk + dir := t.TempDir() + + fs, err := newLocalXLStorage(dir) + if err != nil { + t.Fatal(err) + } + + // Create volume to use in conjunction with other StorageAPI's file API(s) + err = fs.MakeVol(context.Background(), "voldir") + if err != nil { + t.Fatal(err) + } + + err = fs.AppendFile(context.Background(), "voldir", "/file", []byte("hello")) + if err != nil { + t.Fatal(err) + } + + // Try to create a file that includes a file in its path components. + // In *nix, this returns syscall.ENOTDIR while in windows we receive the following error. + err = fs.AppendFile(context.Background(), "voldir", "/file/obj1", []byte("hello")) + if err != errFileAccessDenied { + t.Errorf("expected: %s, got: %s", errFileAccessDenied, err) + } +} diff --git a/code_of_conduct.md b/code_of_conduct.md new file mode 100644 index 00000000..cb232c3c --- /dev/null +++ b/code_of_conduct.md @@ -0,0 +1,80 @@ +# Contributor Covenant Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of experience, +nationality, personal appearance, race, religion, or sexual identity and +orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior, in compliance with the +licensing terms applying to the Project developments. + +Project maintainers have the right and responsibility to remove, edit, or +reject comments, commits, code, wiki edits, issues, and other contributions +that are not aligned to this Code of Conduct, or to ban temporarily or +permanently any contributor for other behaviors that they deem inappropriate, +threatening, offensive, or harmful. However, these actions shall respect the +licensing terms of the Project Developments that will always supersede such +Code of Conduct. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +## Enforcement + +Instances of abusive, harassing, or otherwise unacceptable behavior may be +reported by contacting the project team at dev@min.io. The project team +will review and investigate all complaints, and will respond in a way that it deems +appropriate to the circumstances. The project team is obligated to maintain +confidentiality with regard to the reporter of an incident. +Further details of specific enforcement policies may be posted separately. + +Project maintainers who do not follow or enforce the Code of Conduct in good +faith may face temporary or permanent repercussions as determined by other +members of the project's leadership. + +## Attribution + +This Code of Conduct is adapted from the [Contributor Covenant][homepage], version 1.4, +available at [http://contributor-covenant.org/version/1/4][version] + +This version includes a clarification to ensure that the code of conduct is in +compliance with the free software licensing terms of the project. + +[homepage]: http://contributor-covenant.org +[version]: http://contributor-covenant.org/version/1/4/ diff --git a/docker-buildx.sh b/docker-buildx.sh new file mode 100755 index 00000000..d77dd618 --- /dev/null +++ b/docker-buildx.sh @@ -0,0 +1,37 @@ +#!/bin/bash + +sudo sysctl net.ipv6.conf.all.disable_ipv6=0 + +remote=$(git remote get-url upstream) +if test "$remote" != "git@github.com:minio/minio.git"; then + echo "Script requires that the 'upstream' remote is set to git@github.com:minio/minio.git" + exit 1 +fi + +git remote update upstream && git checkout master && git rebase upstream/master + +release=$(git describe --abbrev=0 --tags) + +docker buildx build --push --no-cache \ + --build-arg RELEASE="${release}" \ + -t "minio/minio:latest" \ + -t "minio/minio:latest-cicd" \ + -t "quay.io/minio/minio:latest" \ + -t "quay.io/minio/minio:latest-cicd" \ + -t "minio/minio:${release}" \ + -t "quay.io/minio/minio:${release}" \ + --platform=linux/arm64,linux/amd64,linux/ppc64le \ + -f Dockerfile.release . + +docker buildx prune -f + +docker buildx build --push --no-cache \ + --build-arg RELEASE="${release}" \ + -t "minio/minio:${release}-cpuv1" \ + -t "quay.io/minio/minio:${release}-cpuv1" \ + --platform=linux/arm64,linux/amd64,linux/ppc64le \ + -f Dockerfile.release.old_cpu . + +docker buildx prune -f + +sudo sysctl net.ipv6.conf.all.disable_ipv6=0 diff --git a/dockerscripts/docker-entrypoint.sh b/dockerscripts/docker-entrypoint.sh new file mode 100755 index 00000000..bd617cb8 --- /dev/null +++ b/dockerscripts/docker-entrypoint.sh @@ -0,0 +1,26 @@ +#!/bin/sh +# + +# If command starts with an option, prepend minio. +if [ "${1}" != "minio" ]; then + if [ -n "${1}" ]; then + set -- minio "$@" + fi +fi + +docker_switch_user() { + if [ -n "${MINIO_USERNAME}" ] && [ -n "${MINIO_GROUPNAME}" ]; then + if [ -n "${MINIO_UID}" ] && [ -n "${MINIO_GID}" ]; then + chroot --userspec=${MINIO_UID}:${MINIO_GID} / "$@" + else + echo "${MINIO_USERNAME}:x:1000:1000:${MINIO_USERNAME}:/:/sbin/nologin" >>/etc/passwd + echo "${MINIO_GROUPNAME}:x:1000" >>/etc/group + chroot --userspec=${MINIO_USERNAME}:${MINIO_GROUPNAME} / "$@" + fi + else + exec "$@" + fi +} + +## DEPRECATED and unsupported - switch to user if applicable. +docker_switch_user "$@" diff --git a/dockerscripts/download-static-curl.sh b/dockerscripts/download-static-curl.sh new file mode 100644 index 00000000..0f12464c --- /dev/null +++ b/dockerscripts/download-static-curl.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +function download_arch_specific_executable { + curl -f -L -s -q \ + https://github.com/moparisthebest/static-curl/releases/latest/download/curl-$1 \ + -o /go/bin/curl || exit 1 + chmod +x /go/bin/curl +} + +case $TARGETARCH in +"arm64") + download_arch_specific_executable aarch64 + ;; +"s390x") + echo "Not downloading static cURL because it does not exist for the $TARGETARCH architecture." + ;; +*) + download_arch_specific_executable "$TARGETARCH" + ;; +esac diff --git a/docs/LICENSE b/docs/LICENSE new file mode 100644 index 00000000..2f244ac8 --- /dev/null +++ b/docs/LICENSE @@ -0,0 +1,395 @@ +Attribution 4.0 International + +======================================================================= + +Creative Commons Corporation ("Creative Commons") is not a law firm and +does not provide legal services or legal advice. Distribution of +Creative Commons public licenses does not create a lawyer-client or +other relationship. Creative Commons makes its licenses and related +information available on an "as-is" basis. Creative Commons gives no +warranties regarding its licenses, any material licensed under their +terms and conditions, or any related information. Creative Commons +disclaims all liability for damages resulting from their use to the +fullest extent possible. + +Using Creative Commons Public Licenses + +Creative Commons public licenses provide a standard set of terms and +conditions that creators and other rights holders may use to share +original works of authorship and other material subject to copyright +and certain other rights specified in the public license below. The +following considerations are for informational purposes only, are not +exhaustive, and do not form part of our licenses. + + Considerations for licensors: Our public licenses are + intended for use by those authorized to give the public + permission to use material in ways otherwise restricted by + copyright and certain other rights. Our licenses are + irrevocable. Licensors should read and understand the terms + and conditions of the license they choose before applying it. + Licensors should also secure all rights necessary before + applying our licenses so that the public can reuse the + material as expected. Licensors should clearly mark any + material not subject to the license. This includes other CC- + licensed material, or material used under an exception or + limitation to copyright. More considerations for licensors: + wiki.creativecommons.org/Considerations_for_licensors + + Considerations for the public: By using one of our public + licenses, a licensor grants the public permission to use the + licensed material under specified terms and conditions. If + the licensor's permission is not necessary for any reason--for + example, because of any applicable exception or limitation to + copyright--then that use is not regulated by the license. Our + licenses grant only permissions under copyright and certain + other rights that a licensor has authority to grant. Use of + the licensed material may still be restricted for other + reasons, including because others have copyright or other + rights in the material. A licensor may make special requests, + such as asking that all changes be marked or described. + Although not required by our licenses, you are encouraged to + respect those requests where reasonable. More_considerations + for the public: + wiki.creativecommons.org/Considerations_for_licensees + +======================================================================= + +Creative Commons Attribution 4.0 International Public License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution 4.0 International Public License ("Public License"). To the +extent this Public License may be interpreted as a contract, You are +granted the Licensed Rights in consideration of Your acceptance of +these terms and conditions, and the Licensor grants You such rights in +consideration of benefits the Licensor receives from making the +Licensed Material available under these terms and conditions. + + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + + b. Adapter's License means the license You apply to Your Copyright + and Similar Rights in Your contributions to Adapted Material in + accordance with the terms and conditions of this Public License. + + c. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + + d. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + e. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + f. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + g. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + h. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + i. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + j. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + k. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part; and + + b. produce, reproduce, and Share Adapted Material. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties. + + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + 4. If You Share Adapted Material You produce, the Adapter's + License You apply must not prevent recipients of the Adapted + Material from complying with this Public License. + + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material; and + + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + + +======================================================================= + +Creative Commons is not a party to its public +licenses. Notwithstanding, Creative Commons may elect to apply one of +its public licenses to material it publishes and in those instances +will be considered the “Licensor.” The text of the Creative Commons +public licenses is dedicated to the public domain under the CC0 Public +Domain Dedication. Except for the limited purpose of indicating that +material is shared under a Creative Commons public license or as +otherwise permitted by the Creative Commons policies published at +creativecommons.org/policies, Creative Commons does not authorize the +use of the trademark "Creative Commons" or any other trademark or logo +of Creative Commons without its prior written consent including, +without limitation, in connection with any unauthorized modifications +to any of its public licenses or any other arrangements, +understandings, or agreements concerning use of licensed material. For +the avoidance of doubt, this paragraph does not form part of the +public licenses. + +Creative Commons may be contacted at creativecommons.org. diff --git a/docs/auditlog/auditlog-echo.go b/docs/auditlog/auditlog-echo.go new file mode 100644 index 00000000..1fb89489 --- /dev/null +++ b/docs/auditlog/auditlog-echo.go @@ -0,0 +1,62 @@ +//go:build ignore +// +build ignore + +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" +) + +var port int + +func init() { + flag.IntVar(&port, "port", 8080, "Port to listen on") +} + +func mainHandler(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + defer r.Body.Close() + if err != nil { + log.Printf("Error reading request body: %v", err) + w.WriteHeader(http.StatusBadRequest) + return + } + + log.Printf(">>> %s %s\n", r.Method, r.URL.Path) + var out bytes.Buffer + json.Indent(&out, body, "", " ") + log.Printf("%s\n", out.String()) + + w.WriteHeader(http.StatusOK) +} + +func main() { + flag.Parse() + http.HandleFunc("/", mainHandler) + + log.Printf("Listening on :%d\n", port) + log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", port), nil)) +} diff --git a/docs/auditlog/auditlog-echo.md b/docs/auditlog/auditlog-echo.md new file mode 100644 index 00000000..4d0ab326 --- /dev/null +++ b/docs/auditlog/auditlog-echo.md @@ -0,0 +1,17 @@ +# `auditlog-echo`: A tool to view MinIO Audit logs on the console + +1. Run the tool with: + +``` +go run docs/auditlog/auditlog-echo.go +``` + +The listen port has a default value (8080), but can be set with the `-port` flag. + +2. Configure audit logging in MinIO with for example: + +``` +mc admin config set myminio audit_webhook enable=on endpoint=http://localhost:8080 +``` + +3. Make any requests to MinIO and see audit logs printed to the tool's console. diff --git a/docs/batch-jobs/README.md b/docs/batch-jobs/README.md new file mode 100644 index 00000000..ccb69043 --- /dev/null +++ b/docs/batch-jobs/README.md @@ -0,0 +1,152 @@ +# MinIO Batch Job +MinIO Batch jobs is an MinIO object management feature that lets you manage objects at scale. Jobs currently supported by MinIO + +- Replicate objects between buckets on multiple sites + +Upcoming Jobs + +- Copy objects from NAS to MinIO +- Copy objects from HDFS to MinIO + +## Replication Job +To perform replication via batch jobs, you create a job. The job consists of a job description YAML that describes + +- Source location from where the objects must be copied from +- Target location from where the objects must be copied to +- Fine grained filtering is available to pick relevant objects from source to copy from + +MinIO batch jobs framework also provides + +- Retrying a failed job automatically driven by user input +- Monitoring job progress in real-time +- Send notifications upon completion or failure to user configured target + +Following YAML describes the structure of a replication job, each value is documented and self-describing. + +```yaml +replicate: + apiVersion: v1 + # source of the objects to be replicated + source: + type: TYPE # valid values are "minio" + bucket: BUCKET + prefix: PREFIX + # NOTE: if source is remote then target must be "local" + # endpoint: ENDPOINT + # credentials: + # accessKey: ACCESS-KEY + # secretKey: SECRET-KEY + # sessionToken: SESSION-TOKEN # Available when rotating credentials are used + + # target where the objects must be replicated + target: + type: TYPE # valid values are "minio" + bucket: BUCKET + prefix: PREFIX + # NOTE: if target is remote then source must be "local" + # endpoint: ENDPOINT + # credentials: + # accessKey: ACCESS-KEY + # secretKey: SECRET-KEY + # sessionToken: SESSION-TOKEN # Available when rotating credentials are used + + # optional flags based filtering criteria + # for all source objects + flags: + filter: + newerThan: "7d" # match objects newer than this value (e.g. 7d10h31s) + olderThan: "7d" # match objects older than this value (e.g. 7d10h31s) + createdAfter: "date" # match objects created after "date" + createdBefore: "date" # match objects created before "date" + + ## NOTE: tags are not supported when "source" is remote. + # tags: + # - key: "name" + # value: "pick*" # match objects with tag 'name', with all values starting with 'pick' + + ## NOTE: metadata filter not supported when "source" is non MinIO. + # metadata: + # - key: "content-type" + # value: "image/*" # match objects with 'content-type', with all values starting with 'image/' + + notify: + endpoint: "https://notify.endpoint" # notification endpoint to receive job status events + token: "Bearer xxxxx" # optional authentication token for the notification endpoint + + retry: + attempts: 10 # number of retries for the job before giving up + delay: "500ms" # least amount of delay between each retry +``` + +You can create and run multiple 'replication' jobs at a time there are no predefined limits set. + +## Batch Jobs Terminology + +### Job +A job is the basic unit of work for MinIO Batch Job. A job is a self describing YAML, once this YAML is submitted and evaluated - MinIO performs the requested actions on each of the objects obtained under the described criteria in job YAML file. + +### Type +Type describes the job type, such as replicating objects between MinIO sites. Each job performs a single type of operation across all objects that match the job description criteria. + +## Batch Jobs via Commandline +[mc](http://github.com/minio/mc) provides 'mc batch' command to create, start and manage submitted jobs. + +``` +NAME: + mc batch - manage batch jobs + +USAGE: + mc batch COMMAND [COMMAND FLAGS | -h] [ARGUMENTS...] + +COMMANDS: + generate generate a new batch job definition + start start a new batch job + list, ls list all current batch jobs + status summarize job events on MinIO server in real-time + describe describe job definition for a job +``` + +### Generate a job yaml +``` +mc batch generate alias/ replicate +``` + +### Start the batch job (returns back the JID) +``` +mc batch start alias/ ./replicate.yaml +Successfully start 'replicate' job `E24HH4nNMcgY5taynaPfxu` on '2022-09-26 17:19:06.296974771 -0700 PDT' +``` + +### List all batch jobs +``` +mc batch list alias/ +ID TYPE USER STARTED +E24HH4nNMcgY5taynaPfxu replicate minioadmin 1 minute ago +``` + +### List all 'replicate' batch jobs +``` +mc batch list alias/ --type replicate +ID TYPE USER STARTED +E24HH4nNMcgY5taynaPfxu replicate minioadmin 1 minute ago +``` + +### Real-time 'status' for a batch job +``` +mc batch status myminio/ E24HH4nNMcgY5taynaPfxu +●∙∙ +Objects: 28766 +Versions: 28766 +Throughput: 3.0 MiB/s +Transferred: 406 MiB +Elapsed: 2m14.227222868s +CurrObjName: share/doc/xml-core/examples/foo.xmlcatalogs +``` + +### 'describe' the batch job yaml. +``` +mc batch describe myminio/ E24HH4nNMcgY5taynaPfxu +replicate: + apiVersion: v1 +... +``` diff --git a/docs/bigdata/README.md b/docs/bigdata/README.md new file mode 100644 index 00000000..39a7efe5 --- /dev/null +++ b/docs/bigdata/README.md @@ -0,0 +1,306 @@ +# **Disaggregated HDP Spark and Hive with MinIO** + +## **1. Cloud-native Architecture** + +![cloud-native](https://github.com/minio/minio/blob/master/docs/bigdata/images/image1.png?raw=true "cloud native architecture") + +Kubernetes manages stateless Spark and Hive containers elastically on the compute nodes. Spark has native scheduler integration with Kubernetes. Hive, for legacy reasons, uses YARN scheduler on top of Kubernetes. + +All access to MinIO object storage is via S3/SQL SELECT API. In addition to the compute nodes, MinIO containers are also managed by Kubernetes as stateful containers with local storage (JBOD/JBOF) mapped as persistent local volumes. This architecture enables multi-tenant MinIO, allowing isolation of data between customers. + +MinIO also supports multi-cluster, multi-site federation similar to AWS regions and tiers. Using MinIO Information Lifecycle Management (ILM), you can configure data to be tiered between NVMe based hot storage, and HDD based warm storage. All data is encrypted with per-object key. Access Control and Identity Management between the tenants are managed by MinIO using OpenID Connect or Kerberos/LDAP/AD. + +## **2. Prerequisites** + +- Install Hortonworks Distribution using this [guide.](https://docs.hortonworks.com/HDPDocuments/Ambari-2.7.1.0/bk_ambari-installation/content/ch_Installing_Ambari.html) + - [Setup Ambari](https://docs.hortonworks.com/HDPDocuments/Ambari-2.7.1.0/bk_ambari-installation/content/set_up_the_ambari_server.html) which automatically sets up YARN + - [Installing Spark](https://docs.hortonworks.com/HDPDocuments/HDP3/HDP-3.0.1/installing-spark/content/installing_spark.html) +- Install MinIO Distributed Server using one of the guides below. + - [Deployment based on Kubernetes](https://docs.min.io/community/minio-object-store/operations/deployments/kubernetes.html) + - [Deployment based on MinIO Helm Chart](https://github.com/helm/charts/tree/master/stable/minio) + +## **3. Configure Hadoop, Spark, Hive to use MinIO** + +After successful installation navigate to the Ambari UI `http://:8080/` and login using the default credentials: [**_username: admin, password: admin_**] + +![ambari-login](https://github.com/minio/minio/blob/master/docs/bigdata/images/image3.png?raw=true "ambari login") + +### **3.1 Configure Hadoop** + +Navigate to **Services** -> **HDFS** -> **CONFIGS** -> **ADVANCED** as shown below + +![hdfs-configs](https://github.com/minio/minio/blob/master/docs/bigdata/images/image2.png?raw=true "hdfs advanced configs") + +Navigate to **Custom core-site** to configure MinIO parameters for `_s3a_` connector + +![s3a-config](https://github.com/minio/minio/blob/master/docs/bigdata/images/image5.png?raw=true "custom core-site") + +``` +sudo pip install yq +alias kv-pairify='yq ".configuration[]" | jq ".[]" | jq -r ".name + \"=\" + .value"' +``` + +Let's take for example a set of 12 compute nodes with an aggregate memory of _1.2TiB_, we need to do following settings for optimal results. Add the following optimal entries for _core-site.xml_ to configure _s3a_ with **MinIO**. Most important options here are + +``` +cat ${HADOOP_CONF_DIR}/core-site.xml | kv-pairify | grep "mapred" + +mapred.maxthreads.generate.mapoutput=2 # Num threads to write map outputs +mapred.maxthreads.partition.closer=0 # Asynchronous map flushers +mapreduce.fileoutputcommitter.algorithm.version=2 # Use the latest committer version +mapreduce.job.reduce.slowstart.completedmaps=0.99 # 99% map, then reduce +mapreduce.reduce.shuffle.input.buffer.percent=0.9 # Min % buffer in RAM +mapreduce.reduce.shuffle.merge.percent=0.9 # Minimum % merges in RAM +mapreduce.reduce.speculative=false # Disable speculation for reducing +mapreduce.task.io.sort.factor=999 # Threshold before writing to disk +mapreduce.task.sort.spill.percent=0.9 # Minimum % before spilling to disk +``` + +S3A is the connector to use S3 and other S3-compatible object stores such as MinIO. MapReduce workloads typically interact with object stores in the same way they do with HDFS. These workloads rely on HDFS atomic rename functionality to complete writing data to the datastore. Object storage operations are atomic by nature and they do not require/implement rename API. The default S3A committer emulates renames through copy and delete APIs. This interaction pattern causes significant loss of performance because of the write amplification. _Netflix_, for example, developed two new staging committers - the Directory staging committer and the Partitioned staging committer - to take full advantage of native object storage operations. These committers do not require rename operation. The two staging committers were evaluated, along with another new addition called the Magic committer for benchmarking. + +It was found that the directory staging committer was the fastest among the three, S3A connector should be configured with the following parameters for optimal results: + +``` +cat ${HADOOP_CONF_DIR}/core-site.xml | kv-pairify | grep "s3a" + +fs.s3a.access.key=minio +fs.s3a.secret.key=minio123 +fs.s3a.path.style.access=true +fs.s3a.block.size=512M +fs.s3a.buffer.dir=${hadoop.tmp.dir}/s3a +fs.s3a.committer.magic.enabled=false +fs.s3a.committer.name=directory +fs.s3a.committer.staging.abort.pending.uploads=true +fs.s3a.committer.staging.conflict-mode=append +fs.s3a.committer.staging.tmp.path=/tmp/staging +fs.s3a.committer.staging.unique-filenames=true +fs.s3a.connection.establish.timeout=5000 +fs.s3a.connection.ssl.enabled=false +fs.s3a.connection.timeout=200000 +fs.s3a.endpoint=http://minio:9000 +fs.s3a.impl=org.apache.hadoop.fs.s3a.S3AFileSystem + +fs.s3a.committer.threads=2048 # Number of threads writing to MinIO +fs.s3a.connection.maximum=8192 # Maximum number of concurrent conns +fs.s3a.fast.upload.active.blocks=2048 # Number of parallel uploads +fs.s3a.fast.upload.buffer=disk # Use disk as the buffer for uploads +fs.s3a.fast.upload=true # Turn on fast upload mode +fs.s3a.max.total.tasks=2048 # Maximum number of parallel tasks +fs.s3a.multipart.size=512M # Size of each multipart chunk +fs.s3a.multipart.threshold=512M # Size before using multipart uploads +fs.s3a.socket.recv.buffer=65536 # Read socket buffer hint +fs.s3a.socket.send.buffer=65536 # Write socket buffer hint +fs.s3a.threads.max=2048 # Maximum number of threads for S3A +``` + +The rest of the other optimization options are discussed in the links below + +- [https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/index.html](https://hadoop.apache.org/docs/current/hadoop-aws/tools/hadoop-aws/index.html) +- [https://hadoop.apache.org/docs/r3.1.1/hadoop-aws/tools/hadoop-aws/committers.html](https://hadoop.apache.org/docs/r3.1.1/hadoop-aws/tools/hadoop-aws/committers.html) + +Once the config changes are applied, proceed to restart **Hadoop** services. + +![hdfs-services](https://github.com/minio/minio/blob/master/docs/bigdata/images/image7.png?raw=true "hdfs restart services") + +### **3.2 Configure Spark2** + +Navigate to **Services** -> **Spark2** -> **CONFIGS** as shown below + +![spark-config](https://github.com/minio/minio/blob/master/docs/bigdata/images/image6.png?raw=true "spark config") + +Navigate to “**Custom spark-defaults**” to configure MinIO parameters for `_s3a_` connector + +![spark-config](https://github.com/minio/minio/blob/master/docs/bigdata/images/image9.png?raw=true "spark defaults") + +Add the following optimal entries for _spark-defaults.conf_ to configure Spark with **MinIO**. + +``` +spark.hadoop.fs.s3a.access.key minio +spark.hadoop.fs.s3a.secret.key minio123 +spark.hadoop.fs.s3a.path.style.access true +spark.hadoop.fs.s3a.block.size 512M +spark.hadoop.fs.s3a.buffer.dir ${hadoop.tmp.dir}/s3a +spark.hadoop.fs.s3a.committer.magic.enabled false +spark.hadoop.fs.s3a.committer.name directory +spark.hadoop.fs.s3a.committer.staging.abort.pending.uploads true +spark.hadoop.fs.s3a.committer.staging.conflict-mode append +spark.hadoop.fs.s3a.committer.staging.tmp.path /tmp/staging +spark.hadoop.fs.s3a.committer.staging.unique-filenames true +spark.hadoop.fs.s3a.committer.threads 2048 # number of threads writing to MinIO +spark.hadoop.fs.s3a.connection.establish.timeout 5000 +spark.hadoop.fs.s3a.connection.maximum 8192 # maximum number of concurrent conns +spark.hadoop.fs.s3a.connection.ssl.enabled false +spark.hadoop.fs.s3a.connection.timeout 200000 +spark.hadoop.fs.s3a.endpoint http://minio:9000 +spark.hadoop.fs.s3a.fast.upload.active.blocks 2048 # number of parallel uploads +spark.hadoop.fs.s3a.fast.upload.buffer disk # use disk as the buffer for uploads +spark.hadoop.fs.s3a.fast.upload true # turn on fast upload mode +spark.hadoop.fs.s3a.impl org.apache.hadoop.spark.hadoop.fs.s3a.S3AFileSystem +spark.hadoop.fs.s3a.max.total.tasks 2048 # maximum number of parallel tasks +spark.hadoop.fs.s3a.multipart.size 512M # size of each multipart chunk +spark.hadoop.fs.s3a.multipart.threshold 512M # size before using multipart uploads +spark.hadoop.fs.s3a.socket.recv.buffer 65536 # read socket buffer hint +spark.hadoop.fs.s3a.socket.send.buffer 65536 # write socket buffer hint +spark.hadoop.fs.s3a.threads.max 2048 # maximum number of threads for S3A +``` + +Once the config changes are applied, proceed to restart **Spark** services. + +![spark-config](https://github.com/minio/minio/blob/master/docs/bigdata/images/image12.png?raw=true "spark restart services") + +### **3.3 Configure Hive** + +Navigate to **Services** -> **Hive** -> **CONFIGS**-> **ADVANCED** as shown below + +![hive-config](https://github.com/minio/minio/blob/master/docs/bigdata/images/image10.png?raw=true "hive advanced config") + +Navigate to “**Custom hive-site**” to configure MinIO parameters for `_s3a_` connector + +![hive-config](https://github.com/minio/minio/blob/master/docs/bigdata/images/image11.png?raw=true "hive advanced config") + +Add the following optimal entries for `hive-site.xml` to configure Hive with **MinIO**. + +``` +hive.blobstore.use.blobstore.as.scratchdir=true +hive.exec.input.listing.max.threads=50 +hive.load.dynamic.partitions.thread=25 +hive.metastore.fshandler.threads=50 +hive.mv.files.threads=40 +mapreduce.input.fileinputformat.list-status.num-threads=50 +``` + +For more information about these options please visit [https://www.cloudera.com/documentation/enterprise/5-11-x/topics/admin_hive_on_s3_tuning.html](https://www.cloudera.com/documentation/enterprise/5-11-x/topics/admin_hive_on_s3_tuning.html) + +![hive-config](https://github.com/minio/minio/blob/master/docs/bigdata/images/image13.png?raw=true "hive advanced custom config") + +Once the config changes are applied, proceed to restart all Hive services. + +![hive-config](https://github.com/minio/minio/blob/master/docs/bigdata/images/image14.png?raw=true "restart hive services") + +## **4. Run Sample Applications** + +After installing Hive, Hadoop and Spark successfully, we can now proceed to run some sample applications to see if they are configured appropriately. We can use Spark Pi and Spark WordCount programs to validate our Spark installation. We can also explore how to run Spark jobs from the command line and Spark shell. + +### **4.1 Spark Pi** + +Test the Spark installation by running the following compute intensive example, which calculates pi by “throwing darts” at a circle. The program generates points in the unit square ((0,0) to (1,1)) and counts how many points fall within the unit circle within the square. The result approximates pi. + +Follow these steps to run the Spark Pi example: + +- Login as user **‘spark’**. +- When the job runs, the library can now use **MinIO** during intermediate processing. +- Navigate to a node with the Spark client and access the spark2-client directory: + +``` +cd /usr/hdp/current/spark2-client +su spark +``` + +- Run the Apache Spark Pi job in yarn-client mode, using code from **org.apache.spark**: + +``` +./bin/spark-submit --class org.apache.spark.examples.SparkPi \ + --master yarn-client \ + --num-executors 1 \ + --driver-memory 512m \ + --executor-memory 512m \ + --executor-cores 1 \ + examples/jars/spark-examples*.jar 10 +``` + +The job should produce an output as shown below. Note the value of pi in the output. + +``` +17/03/22 23:21:10 INFO DAGScheduler: Job 0 finished: reduce at SparkPi.scala:38, took 1.302805 s +Pi is roughly 3.1445191445191445 +``` + +Job status can also be viewed in a browser by navigating to the YARN ResourceManager Web UI and clicking on job history server information. + +### **4.2 WordCount** + +WordCount is a simple program that counts how often a word occurs in a text file. The code builds a dataset of (String, Int) pairs called counts, and saves the dataset to a file. + +The following example submits WordCount code to the Scala shell. Select an input file for the Spark WordCount example. We can use any text file as input. + +- Login as user **‘spark’**. +- When the job runs, the library can now use **MinIO** during intermediate processing. +- Navigate to a node with Spark client and access the spark2-client directory: + +``` +cd /usr/hdp/current/spark2-client +su spark +``` + +The following example uses _log4j.properties_ as the input file: + +#### **4.2.1 Upload the input file to HDFS:** + +``` +hadoop fs -copyFromLocal /etc/hadoop/conf/log4j.properties + s3a://testbucket/testdata +``` + +#### **4.2.2 Run the Spark shell:** + +``` +./bin/spark-shell --master yarn-client --driver-memory 512m --executor-memory 512m +``` + +The command should produce an output as shown below. (with additional status messages): + +``` +Spark context Web UI available at http://172.26.236.247:4041 +Spark context available as 'sc' (master = yarn, app id = application_1490217230866_0002). +Spark session available as 'spark'. +Welcome to + + + ____ __ + / __/__ ___ _____/ /__ + _\ \/ _ \/ _ `/ __/ '_/ + /___/ .__/\_,_/_/ /_/\_\ version 2.1.0.2.6.0.0-598 + /_/ + +Using Scala version 2.11.8 (Java HotSpot(TM) 64-Bit Server VM, Java 1.8.0_112) +Type in expressions to have them evaluated. +Type :help for more information. + +scala> +``` + +- At the _scala>_ prompt, submit the job by typing the following commands, Replace node names, file name, and file location with your values: + +``` +scala> val file = sc.textFile("s3a://testbucket/testdata") +file: org.apache.spark.rdd.RDD[String] = s3a://testbucket/testdata MapPartitionsRDD[1] at textFile at :24 + +scala> val counts = file.flatMap(line => line.split(" ")).map(word => (word, 1)).reduceByKey(_ + _) +counts: org.apache.spark.rdd.RDD[(String, Int)] = ShuffledRDD[4] at reduceByKey at :25 + +scala> counts.saveAsTextFile("s3a://testbucket/wordcount") +``` + +Use one of the following approaches to view job output: + +View output in the Scala shell: + +``` +scala> counts.count() +364 +``` + +To view the output from MinIO exit the Scala shell. View WordCount job status: + +``` +hadoop fs -ls s3a://testbucket/wordcount +``` + +The output should be similar to the following: + +``` +Found 3 items +-rw-rw-rw- 1 spark spark 0 2019-05-04 01:36 s3a://testbucket/wordcount/_SUCCESS +-rw-rw-rw- 1 spark spark 4956 2019-05-04 01:36 s3a://testbucket/wordcount/part-00000 +-rw-rw-rw- 1 spark spark 5616 2019-05-04 01:36 s3a://testbucket/wordcount/part-00001 +``` diff --git a/docs/bigdata/images/image1.png b/docs/bigdata/images/image1.png new file mode 100644 index 00000000..007edb2d Binary files /dev/null and b/docs/bigdata/images/image1.png differ diff --git a/docs/bigdata/images/image10.png b/docs/bigdata/images/image10.png new file mode 100644 index 00000000..92521a10 Binary files /dev/null and b/docs/bigdata/images/image10.png differ diff --git a/docs/bigdata/images/image11.png b/docs/bigdata/images/image11.png new file mode 100644 index 00000000..ad434593 Binary files /dev/null and b/docs/bigdata/images/image11.png differ diff --git a/docs/bigdata/images/image12.png b/docs/bigdata/images/image12.png new file mode 100644 index 00000000..da96937e Binary files /dev/null and b/docs/bigdata/images/image12.png differ diff --git a/docs/bigdata/images/image13.png b/docs/bigdata/images/image13.png new file mode 100644 index 00000000..9776a970 Binary files /dev/null and b/docs/bigdata/images/image13.png differ diff --git a/docs/bigdata/images/image14.png b/docs/bigdata/images/image14.png new file mode 100644 index 00000000..4a4693fb Binary files /dev/null and b/docs/bigdata/images/image14.png differ diff --git a/docs/bigdata/images/image2.png b/docs/bigdata/images/image2.png new file mode 100644 index 00000000..0870d7a0 Binary files /dev/null and b/docs/bigdata/images/image2.png differ diff --git a/docs/bigdata/images/image3.png b/docs/bigdata/images/image3.png new file mode 100644 index 00000000..ba12e29e Binary files /dev/null and b/docs/bigdata/images/image3.png differ diff --git a/docs/bigdata/images/image4.png b/docs/bigdata/images/image4.png new file mode 100644 index 00000000..e82a5b4b Binary files /dev/null and b/docs/bigdata/images/image4.png differ diff --git a/docs/bigdata/images/image5.png b/docs/bigdata/images/image5.png new file mode 100644 index 00000000..6221e29a Binary files /dev/null and b/docs/bigdata/images/image5.png differ diff --git a/docs/bigdata/images/image6.png b/docs/bigdata/images/image6.png new file mode 100644 index 00000000..0bbae0f4 Binary files /dev/null and b/docs/bigdata/images/image6.png differ diff --git a/docs/bigdata/images/image7.png b/docs/bigdata/images/image7.png new file mode 100644 index 00000000..6d232d55 Binary files /dev/null and b/docs/bigdata/images/image7.png differ diff --git a/docs/bigdata/images/image8.png b/docs/bigdata/images/image8.png new file mode 100644 index 00000000..d0aee63b Binary files /dev/null and b/docs/bigdata/images/image8.png differ diff --git a/docs/bigdata/images/image9.png b/docs/bigdata/images/image9.png new file mode 100644 index 00000000..c9f38861 Binary files /dev/null and b/docs/bigdata/images/image9.png differ diff --git a/docs/bucket/lifecycle/DESIGN.md b/docs/bucket/lifecycle/DESIGN.md new file mode 100644 index 00000000..f534023a --- /dev/null +++ b/docs/bucket/lifecycle/DESIGN.md @@ -0,0 +1,55 @@ +# ILM Tiering Design [![slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) + +Lifecycle transition functionality provided in [bucket lifecycle guide](https://github.com/minio/minio/master/docs/bucket/lifecycle/README.md) allows tiering of content from MinIO object store to public clouds or other MinIO clusters. + +Transition tiers can be added to MinIO using `mc admin tier add` command to associate a `gcs`, `s3` or `azure` bucket or prefix path on a bucket to the tier name. +Lifecycle transition rules can be applied to buckets (both versioned and un-versioned) by specifying the tier name defined above as the transition storage class for the lifecycle rule. + +## Implementation + +ILM tiering takes place when a object placed in the bucket meets lifecycle transition rules and becomes eligible for tiering. MinIO scanner (which runs at one minute intervals, each time scanning one sixteenth of the namespace), picks up the object for tiering. The data is moved to the remote tier in entirety, leaving only the object metadata on MinIO. + +The data on the backend is stored under the `bucket/prefix` specified in the tier configuration with a custom name derived from a randomly generated uuid - e.g. `0b/c4/0bc4fab7-2daf-4d2f-8e39-5c6c6fb7e2d3`. The first two prefixes are characters 1-2,3-4 from the uuid. This format allows tiering to any cloud irrespective of whether the cloud in question supports versioning. The reference to the transitioned object name and transitioned tier is stored as part of the internal metadata for the object (or its version) on MinIO. + +Extra metadata maintained internally in `xl.meta` for a transitioned object + +``` +... + "MetaSys": { + "x-minio-internal-transition-status": "Y29tcGxldGU=", + "x-minio-internal-transition-tier": "R0NTVElFUjE=", + "x-minio-internal-transitioned-object": "ZDIvN2MvZDI3Y2MwYWMtZGIzNC00ZGM1LWIxNDUtYjI5MGNjZjU1MjY5" + }, +``` + +When a transitioned object is restored temporarily to local MinIO instance via PostRestoreObject API, the object data is copied back from the remote tier, and additional metadata for the restored object is maintained as referenced below. Once the restore period expires, the local copy of the object is removed by the scanner during its periodic runs. + +``` +... + "MetaUsr": { + "X-Amz-Restore-Expiry-Days": "4", + "X-Amz-Restore-Request-Date": "Mon, 22 Feb 2021 21:10:09 GMT", + "x-amz-restore": "ongoing-request=false, expiry-date=Sat, 27 Feb 2021 00:00:00 GMT", +... +``` + +### Encrypted/Object locked objects + +For objects under SSE-S3 or SSE-C encryption, the encrypted content from MinIO cluster is copied as is to the remote tier without any decryption. The content is decrypted as it is streamed from remote tier on `GET/HEAD`. Objects under retention are protected because the metadata present on MinIO server ensures that the object (version) is not deleted until retention period is over. Administrators need to ensure that the remote tier bucket is under proper access control. + +### Transition Status + +MinIO specific extension header `X-Minio-Transition` is displayed on `HEAD/GET` to predict expected transition date on the object. Once object is transitioned to the remote tier,`x-amz-storage-class` shows the tier name to which object transitioned. Additional headers such as "X-Amz-Restore-Expiry-Days", "x-amz-restore", and "X-Amz-Restore-Request-Date" are displayed when a object is under restore/has been restored to local MinIO cluster. + +### Expiry or removal events + +An object that is in transition tier will be deleted once the object hits expiry date or if removed via `mc rm` (`mc rm --vid` in the case of delete of a specific object version). Other rules specific to legal hold and object locking precede any lifecycle rules. + +### Additional notes + +Tiering and lifecycle transition are applicable only to erasure/distributed MinIO. + +## Explore Further + +- [MinIO | Golang Client API Reference](https://docs.min.io/community/minio-object-store/developers/go/API.html#SetBucketLifecycle) +- [Object Lifecycle Management](https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html) diff --git a/docs/bucket/lifecycle/README.md b/docs/bucket/lifecycle/README.md new file mode 100644 index 00000000..6888838f --- /dev/null +++ b/docs/bucket/lifecycle/README.md @@ -0,0 +1,232 @@ +# Bucket Lifecycle Configuration Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) + +Enable object lifecycle configuration on buckets to setup automatic deletion of objects after a specified number of days or a specified date. + +## 1. Prerequisites + +- Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html). +- Install `mc` - [mc Quickstart Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart) + +## 2. Enable bucket lifecycle configuration + +- Create a bucket lifecycle configuration which expires the objects under the prefix `old/` on `2020-01-01T00:00:00.000Z` date and the objects under `temp/` after 7 days. +- Enable bucket lifecycle configuration using `mc`: + +```sh +$ mc ilm import play/testbucket < NOTE: If the latest object is a delete marker then filtering based on `Filter.Tags` is ignored and +> if the DELETE marker modTime satisfies the `Expiration.Days` then all versions of the object are +> immediately purged. + +``` +{ + "Rules": [ + { + "ID": "Purge all versions of an expired object", + "Status": "Enabled", + "Filter": { + "Prefix": "users-uploads/" + }, + "Expiration": { + "Days": 7, + "ExpiredObjectAllVersions": true + } + } + ] +} +``` + +### 3.3 Automatic removal of delete markers with no other versions + +When an object has only one version as a delete marker, the latter can be automatically removed after a certain number of days using the following configuration: + +``` +{ + "Rules": [ + { + "ID": "Removing all delete markers", + "Expiration": { + "ExpiredObjectDeleteMarker": true + }, + "Status": "Enabled" + } + ] +} +``` + +## 4. Enable ILM transition feature + +In Erasure mode, MinIO supports tiering to public cloud providers such as GCS, AWS and Azure as well as to other MinIO clusters via the ILM transition feature. This will allow transitioning of older objects to a different cluster or the public cloud by setting up transition rules in the bucket lifecycle configuration. This feature enables applications to optimize storage costs by moving less frequently accessed data to a cheaper storage without compromising accessibility of data. + +To transition objects in a bucket to a destination bucket on a different cluster, applications need to specify a transition tier defined on MinIO instead of storage class while setting up the ILM lifecycle rule. + +> To create a transition tier for transitioning objects to a prefix `testprefix` in `azurebucket` on Azure blob using `mc`: + +``` + mc admin tier add azure source AZURETIER --endpoint https://blob.core.windows.net --access-key AZURE_ACCOUNT_NAME --secret-key AZURE_ACCOUNT_KEY --bucket azurebucket --prefix testprefix1/ +``` + +> The admin user running this command needs the "admin:SetTier" and "admin:ListTier" permissions if not running as root. + +Using above tier, set up a lifecycle rule with transition: + +``` + mc ilm add --expiry-days 365 --transition-days 45 --storage-class "AZURETIER" myminio/srcbucket +``` + +Note: In the case of S3, it is possible to create a tier from MinIO running in EC2 to S3 using AWS role attached to EC2 as credentials instead of accesskey/secretkey: + +``` +mc admin tier add s3 source S3TIER --bucket s3bucket --prefix testprefix/ --use-aws-role +``` + +Once transitioned, GET or HEAD on the object will stream the content from the transitioned tier. In the event that the object needs to be restored temporarily to the local cluster, the AWS [RestoreObject API](https://docs.aws.amazon.com/AmazonS3/latest/API/API_RestoreObject.html) can be utilized. + +``` +aws s3api restore-object --bucket srcbucket \ +--key object \ +--restore-request Days=3 +``` + +### 4.1 Monitoring transition events + +`s3:ObjectTransition:Complete` and `s3:ObjectTransition:Failed` events can be used to monitor transition events between the source cluster and transition tier. To watch lifecycle events, you can enable bucket notification on the source bucket with `mc event add` and specify `--event ilm` flag. + +Note that transition event notification is a MinIO extension. + +## Explore Further + +- [MinIO | Golang Client API Reference](https://docs.min.io/community/minio-object-store/developers/go/API.html) +- [Object Lifecycle Management](https://docs.aws.amazon.com/AmazonS3/latest/dev/object-lifecycle-mgmt.html) diff --git a/docs/bucket/lifecycle/setup_ilm_transition.sh b/docs/bucket/lifecycle/setup_ilm_transition.sh new file mode 100755 index 00000000..975ea81f --- /dev/null +++ b/docs/bucket/lifecycle/setup_ilm_transition.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env bash + +set -x + +trap 'catch $LINENO' ERR + +# shellcheck disable=SC2120 +catch() { + if [ $# -ne 0 ]; then + echo "error on line $1" + for site in sitea siteb; do + echo "$site server logs =========" + cat "/tmp/${site}_1.log" + echo "===========================" + cat "/tmp/${site}_2.log" + done + fi + + echo "Cleaning up instances of MinIO" + pkill minio + pkill -9 minio + rm -rf /tmp/multisitea + rm -rf /tmp/multisiteb + if [ $# -ne 0 ]; then + exit $# + fi +} + +catch + +export MINIO_CI_CD=1 +export MINIO_BROWSER=off +export MINIO_KMS_AUTO_ENCRYPTION=off +export MINIO_PROMETHEUS_AUTH_TYPE=public +export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= +unset MINIO_KMS_KES_CERT_FILE +unset MINIO_KMS_KES_KEY_FILE +unset MINIO_KMS_KES_ENDPOINT +unset MINIO_KMS_KES_KEY_NAME + +if [ ! -f ./mc ]; then + wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc +fi + +minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 & +minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 & + +minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 & +minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 & + +# Wait to make sure all MinIO instances are up + +export MC_HOST_sitea=http://minioadmin:minioadmin@127.0.0.1:9001 +export MC_HOST_siteb=http://minioadmin:minioadmin@127.0.0.1:9004 + +./mc ready sitea +./mc ready siteb + +./mc mb --ignore-existing sitea/bucket +./mc mb --ignore-existing siteb/bucket + +sleep 10s + +## Add warm tier +./mc ilm tier add minio sitea WARM-TIER --endpoint http://localhost:9004 --access-key minioadmin --secret-key minioadmin --bucket bucket + +## Add ILM rules +./mc ilm add sitea/bucket --transition-days 0 --transition-tier WARM-TIER +./mc ilm rule list sitea/bucket + +./mc cp README.md sitea/bucket/README.md + +until $(./mc stat sitea/bucket/README.md --json | jq -r '.metadata."X-Amz-Storage-Class"' | grep -q WARM-TIER); do + echo "waiting until the object is tiered to run heal" + sleep 1s +done +./mc stat sitea/bucket/README.md + +success=$(./mc admin heal -r sitea/bucket/README.md --json --force | jq -r 'select((.name == "bucket/README.md") and (.after.color == "green")) | .after.color == "green"') +if [ "${success}" != "true" ]; then + echo "Found bug expected transitioned object to report 'green'" + exit 1 +fi + +catch diff --git a/docs/bucket/notifications/README.md b/docs/bucket/notifications/README.md new file mode 100644 index 00000000..062f75fe --- /dev/null +++ b/docs/bucket/notifications/README.md @@ -0,0 +1,1443 @@ +# MinIO Bucket Notification Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +Events occurring on objects in a bucket can be monitored using bucket event notifications. + +Various event types supported by MinIO server are + +| Supported Object Event Types | | | +| :---------------------- | ------------------------------------------ | ------------------------------------- | +| `s3:ObjectCreated:Put` | `s3:ObjectCreated:CompleteMultipartUpload` | `s3:ObjectAccessed:Head` | +| `s3:ObjectCreated:Post` | `s3:ObjectRemoved:Delete` | `s3:ObjectRemoved:DeleteMarkerCreated` | +| `s3:ObjectCreated:Copy` | `s3:ObjectAccessed:Get` | | +| `s3:ObjectCreated:PutRetention` | `s3:ObjectCreated:PutLegalHold` | | +| `s3:ObjectAccessed:GetRetention` | `s3:ObjectAccessed:GetLegalHold` | | + +| Supported Replication Event Types | +| :------------ | +| `s3:Replication:OperationFailedReplication` | +| `s3:Replication:OperationCompletedReplication` | +| `s3:Replication:OperationNotTracked` | +| `s3:Replication:OperationMissedThreshold` | +| `s3:Replication:OperationReplicatedAfterThreshold` | + +| Supported ILM Transition Event Types | +| :----- | +| `s3:ObjectRestore:Post` | +| `s3:ObjectRestore:Completed` | + +| Supported Global Event Types (Only supported through ListenNotification API) | +| :----- | +| `s3:BucketCreated` | +| `s3:BucketRemoved` | + +Use client tools like `mc` to set and listen for event notifications using the [`event` sub-command](https://docs.min.io/community/minio-object-store/reference/minio-mc/mc-event-add.html). MinIO SDK's [`BucketNotification` APIs](https://docs.min.io/community/minio-object-store/developers/go/API.html#setbucketnotification-ctx-context-context-bucketname-string-config-notification-configuration-error) can also be used. The notification message MinIO sends to publish an event is a JSON message with the following [structure](https://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html). + +Bucket events can be published to the following targets: + +| Supported Notification Targets | | | +| :-------------------------------- | --------------------------- | ------------------------------- | +| [`AMQP`](#AMQP) | [`Redis`](#Redis) | [`MySQL`](#MySQL) | +| [`MQTT`](#MQTT) | [`NATS`](#NATS) | [`Apache Kafka`](#apache-kafka) | +| [`Elasticsearch`](#Elasticsearch) | [`PostgreSQL`](#PostgreSQL) | [`Webhooks`](#webhooks) | +| [`NSQ`](#NSQ) | | | + +## Prerequisites + +- Install and configure MinIO Server from [here](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html#procedure). +- Install and configure MinIO Client from [here](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart). + +``` +$ mc admin config get myminio | grep notify +notify_webhook publish bucket notifications to webhook endpoints +notify_amqp publish bucket notifications to AMQP endpoints +notify_kafka publish bucket notifications to Kafka endpoints +notify_mqtt publish bucket notifications to MQTT endpoints +notify_nats publish bucket notifications to NATS endpoints +notify_nsq publish bucket notifications to NSQ endpoints +notify_mysql publish bucket notifications to MySQL databases +notify_postgres publish bucket notifications to Postgres databases +notify_elasticsearch publish bucket notifications to Elasticsearch endpoints +notify_redis publish bucket notifications to Redis datastores +``` + +> NOTE: +> +> - '\*' at the end of arg means its mandatory. +> - '\*' at the end of the values, means its the default value for the arg. +> - When configured using environment variables, the `:name` can be specified using this format `MINIO_NOTIFY_WEBHOOK_ENABLE_`. + +## Publish MinIO events via AMQP + +Install RabbitMQ from [here](https://www.rabbitmq.com/). + +### Step 1: Add AMQP endpoint to MinIO + +The AMQP configuration is located under the sub-system `notify_amqp` top-level key. Create a configuration key-value pair here for your AMQP instance. The key is a name for your AMQP endpoint, and the value is a collection of key-value parameters described in the table below. + +``` +KEY: +notify_amqp[:name] publish bucket notifications to AMQP endpoints + +ARGS: +url* (url) AMQP server endpoint e.g. `amqp://myuser:mypassword@localhost:5672` +exchange (string) name of the AMQP exchange +exchange_type (string) AMQP exchange type +routing_key (string) routing key for publishing +mandatory (on|off) quietly ignore undelivered messages when set to 'off', default is 'on' +durable (on|off) persist queue across broker restarts when set to 'on', default is 'off' +no_wait (on|off) non-blocking message delivery when set to 'on', default is 'off' +internal (on|off) set to 'on' for exchange to be not used directly by publishers, but only when bound to other exchanges +auto_deleted (on|off) auto delete queue when set to 'on', when there are no consumers +delivery_mode (number) set to '1' for non-persistent or '2' for persistent queue +queue_dir (path) staging dir for undelivered messages e.g. '/home/events' +queue_limit (number) maximum limit for undelivered messages, defaults to '100000' +comment (sentence) optionally add a comment to this setting +``` + +Or environment variables + +``` +KEY: +notify_amqp[:name] publish bucket notifications to AMQP endpoints + +ARGS: +MINIO_NOTIFY_AMQP_ENABLE* (on|off) enable notify_amqp target, default is 'off' +MINIO_NOTIFY_AMQP_URL* (url) AMQP server endpoint e.g. `amqp://myuser:mypassword@localhost:5672` +MINIO_NOTIFY_AMQP_EXCHANGE (string) name of the AMQP exchange +MINIO_NOTIFY_AMQP_EXCHANGE_TYPE (string) AMQP exchange type +MINIO_NOTIFY_AMQP_ROUTING_KEY (string) routing key for publishing +MINIO_NOTIFY_AMQP_MANDATORY (on|off) quietly ignore undelivered messages when set to 'off', default is 'on' +MINIO_NOTIFY_AMQP_DURABLE (on|off) persist queue across broker restarts when set to 'on', default is 'off' +MINIO_NOTIFY_AMQP_NO_WAIT (on|off) non-blocking message delivery when set to 'on', default is 'off' +MINIO_NOTIFY_AMQP_INTERNAL (on|off) set to 'on' for exchange to be not used directly by publishers, but only when bound to other exchanges +MINIO_NOTIFY_AMQP_AUTO_DELETED (on|off) auto delete queue when set to 'on', when there are no consumers +MINIO_NOTIFY_AMQP_DELIVERY_MODE (number) set to '1' for non-persistent or '2' for persistent queue +MINIO_NOTIFY_AMQP_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' +MINIO_NOTIFY_AMQP_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' +MINIO_NOTIFY_AMQP_COMMENT (sentence) optionally add a comment to this setting +``` + +MinIO supports persistent event store. The persistent store will backup events when the AMQP broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 100000. + +To update the configuration, use `mc admin config get notify_amqp` command to get the current configuration for `notify_amqp`. + +```sh +$ mc admin config get myminio/ notify_amqp +notify_amqp:1 delivery_mode="0" exchange_type="" no_wait="off" queue_dir="" queue_limit="0" url="" auto_deleted="off" durable="off" exchange="" internal="off" mandatory="off" routing_key="" +``` + +Use `mc admin config set` command to update the configuration for the deployment.Restart the MinIO server to put the changes into effect. The server will print a line like `SQS ARNs: arn:minio:sqs::1:amqp` at start-up if there were no errors. + +An example configuration for RabbitMQ is shown below: + +```sh +mc admin config set myminio/ notify_amqp:1 exchange="bucketevents" exchange_type="fanout" mandatory="off" no_wait="off" url="amqp://myuser:mypassword@localhost:5672" auto_deleted="off" delivery_mode="0" durable="off" internal="off" routing_key="bucketlogs" +``` + +MinIO supports all the exchanges available in [RabbitMQ](https://www.rabbitmq.com/). For this setup, we are using `fanout` exchange. + +MinIO also sends with the notifications two headers: `minio-bucket` and `minio-event`. An exchange using the type "headers" can use this information to route the notifications to proper queues. + +Note that, you can add as many AMQP server endpoint configurations as needed by providing an identifier (like "1" in the example above) for the AMQP instance and an object of per-server configuration parameters. + +### Step 2: Enable RabbitMQ bucket notification using MinIO client + +We will enable bucket event notification to trigger whenever a JPEG image is uploaded or deleted `images` bucket on `myminio` server. Here ARN value is `arn:minio:sqs::1:amqp`. To understand more about ARN please follow [AWS ARN](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) documentation. + +``` +mc mb myminio/images +mc event add myminio/images arn:minio:sqs::1:amqp --suffix .jpg +mc event list myminio/images +arn:minio:sqs::1:amqp s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg” +``` + +### Step 3: Test on RabbitMQ + +The python program below waits on the queue exchange `bucketevents` and prints event notifications on the console. We use [Pika Python Client](https://www.rabbitmq.com/tutorials/tutorial-three-python.html) library to do this. + +```py +#!/usr/bin/env python +import pika + +connection = pika.BlockingConnection(pika.ConnectionParameters( + host='localhost')) +channel = connection.channel() + +channel.exchange_declare(exchange='bucketevents', + exchange_type='fanout') + +result = channel.queue_declare(exclusive=False) +queue_name = result.method.queue + +channel.queue_bind(exchange='bucketevents', + queue=queue_name) + +print(' [*] Waiting for logs. To exit press CTRL+C') + +def callback(ch, method, properties, body): + print(" [x] %r" % body) + +channel.basic_consume(callback, + queue=queue_name, + no_ack=False) + +channel.start_consuming() +``` + +Execute this example python program to watch for RabbitMQ events on the console. + +```py +python rabbit.py +``` + +Open another terminal and upload a JPEG image into `images` bucket. + +``` +mc cp myphoto.jpg myminio/images +``` + +You should receive the following event notification via RabbitMQ once the upload completes. + +```py +python rabbit.py +'{"Records":[{"eventVersion":"2.0","eventSource":"aws:s3","awsRegion":"","eventTime":"2016–09–08T22:34:38.226Z","eventName":"s3:ObjectCreated:Put","userIdentity":{"principalId":"minio"},"requestParameters":{"sourceIPAddress":"10.1.10.150:44576"},"responseElements":{},"s3":{"s3SchemaVersion":"1.0","configurationId":"Config","bucket":{"name":"images","ownerIdentity":{"principalId":"minio"},"arn":"arn:aws:s3:::images"},"object":{"key":"myphoto.jpg","size":200436,"sequencer":"147279EAF9F40933"}}}],"level":"info","msg":"","time":"2016–09–08T15:34:38–07:00"}' +``` + +## Publish MinIO events MQTT + +Install an MQTT Broker from [here](https://mosquitto.org/). + +### Step 1: Add MQTT endpoint to MinIO + +The MQTT configuration is located as `notify_mqtt` key. Create a configuration key-value pair here for your MQTT instance. The key is a name for your MQTT endpoint, and the value is a collection of key-value parameters described in the table below. + +``` +KEY: +notify_mqtt[:name] publish bucket notifications to MQTT endpoints + +ARGS: +broker* (uri) MQTT server endpoint e.g. `tcp://localhost:1883` +topic* (string) name of the MQTT topic to publish +username (string) MQTT username +password (string) MQTT password +qos (number) set the quality of service priority, defaults to '0' +keep_alive_interval (duration) keep-alive interval for MQTT connections in s,m,h,d +reconnect_interval (duration) reconnect interval for MQTT connections in s,m,h,d +queue_dir (path) staging dir for undelivered messages e.g. '/home/events' +queue_limit (number) maximum limit for undelivered messages, defaults to '100000' +comment (sentence) optionally add a comment to this setting +``` + +or environment variables + +``` +KEY: +notify_mqtt[:name] publish bucket notifications to MQTT endpoints + +ARGS: +MINIO_NOTIFY_MQTT_ENABLE* (on|off) enable notify_mqtt target, default is 'off' +MINIO_NOTIFY_MQTT_BROKER* (uri) MQTT server endpoint e.g. `tcp://localhost:1883` +MINIO_NOTIFY_MQTT_TOPIC* (string) name of the MQTT topic to publish +MINIO_NOTIFY_MQTT_USERNAME (string) MQTT username +MINIO_NOTIFY_MQTT_PASSWORD (string) MQTT password +MINIO_NOTIFY_MQTT_QOS (number) set the quality of service priority, defaults to '0' +MINIO_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL (duration) keep-alive interval for MQTT connections in s,m,h,d +MINIO_NOTIFY_MQTT_RECONNECT_INTERVAL (duration) reconnect interval for MQTT connections in s,m,h,d +MINIO_NOTIFY_MQTT_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' +MINIO_NOTIFY_MQTT_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' +MINIO_NOTIFY_MQTT_COMMENT (sentence) optionally add a comment to this setting +``` + +MinIO supports persistent event store. The persistent store will backup events when the MQTT broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 100000. + +To update the configuration, use `mc admin config get` command to get the current configuration. + +```sh +$ mc admin config get myminio/ notify_mqtt +notify_mqtt:1 broker="" password="" queue_dir="" queue_limit="0" reconnect_interval="0s" keep_alive_interval="0s" qos="0" topic="" username="" +``` + +Use `mc admin config set` command to update the configuration for the deployment. Restart the MinIO server to put the changes into effect. The server will print a line like `SQS ARNs: arn:minio:sqs::1:mqtt` at start-up if there were no errors. + +```sh +mc admin config set myminio notify_mqtt:1 broker="tcp://localhost:1883" password="" queue_dir="" queue_limit="0" reconnect_interval="0s" keep_alive_interval="0s" qos="1" topic="minio" username="" +``` + +MinIO supports any MQTT server that supports MQTT 3.1 or 3.1.1 and can connect to them over TCP, TLS, or a Websocket connection using `tcp://`, `tls://`, or `ws://` respectively as the scheme for the broker url. See the [Go Client](http://www.eclipse.org/paho/clients/golang/) documentation for more information. + +Note that, you can add as many MQTT server endpoint configurations as needed by providing an identifier (like "1" in the example above) for the MQTT instance and an object of per-server configuration parameters. + +### Step 2: Enable MQTT bucket notification using MinIO client + +We will enable bucket event notification to trigger whenever a JPEG image is uploaded or deleted `images` bucket on `myminio` server. Here ARN value is `arn:minio:sqs::1:mqtt`. + +``` +mc mb myminio/images +mc event add myminio/images arn:minio:sqs::1:mqtt --suffix .jpg +mc event list myminio/images +arn:minio:sqs::1:amqp s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg” +``` + +### Step 3: Test on MQTT + +The python program below waits on mqtt topic `/minio` and prints event notifications on the console. We use [paho-mqtt](https://pypi.python.org/pypi/paho-mqtt/) library to do this. + +```py +#!/usr/bin/env python3 +from __future__ import print_function +import paho.mqtt.client as mqtt + +# This is the Subscriber + +def on_connect(client, userdata, flags, rc): + print("Connected with result code "+str(rc)) + # qos level is set to 1 + client.subscribe("minio", 1) + +def on_message(client, userdata, msg): + print(msg.payload) + +# client_id is a randomly generated unique ID for the mqtt broker to identify the connection. +client = mqtt.Client(client_id="myclientid",clean_session=False) + +client.on_connect = on_connect +client.on_message = on_message + +client.connect("localhost",1883,60) +client.loop_forever() +``` + +Execute this example python program to watch for MQTT events on the console. + +```py +python mqtt.py +``` + +Open another terminal and upload a JPEG image into `images` bucket. + +``` +mc cp myphoto.jpg myminio/images +``` + +You should receive the following event notification via MQTT once the upload completes. + +```py +python mqtt.py +{“Records”:[{“eventVersion”:”2.0",”eventSource”:”aws:s3",”awsRegion”:”",”eventTime”:”2016–09–08T22:34:38.226Z”,”eventName”:”s3:ObjectCreated:Put”,”userIdentity”:{“principalId”:”minio”},”requestParameters”:{“sourceIPAddress”:”10.1.10.150:44576"},”responseElements”:{},”s3":{“s3SchemaVersion”:”1.0",”configurationId”:”Config”,”bucket”:{“name”:”images”,”ownerIdentity”:{“principalId”:”minio”},”arn”:”arn:aws:s3:::images”},”object”:{“key”:”myphoto.jpg”,”size”:200436,”sequencer”:”147279EAF9F40933"}}}],”level”:”info”,”msg”:””,”time”:”2016–09–08T15:34:38–07:00"} +``` + +## Publish MinIO events via Elasticsearch + +Install [Elasticsearch](https://www.elastic.co/downloads/elasticsearch) server. + +This notification target supports two formats: _namespace_ and _access_. + +When the _namespace_ format is used, MinIO synchronizes objects in the bucket with documents in the index. For each event in the MinIO, the server creates a document with the bucket and object name from the event as the document ID. Other details of the event are stored in the body of the document. Thus if an existing object is over-written in MinIO, the corresponding document in the Elasticsearch index is updated. If an object is deleted, the corresponding document is deleted from the index. + +When the _access_ format is used, MinIO appends events as documents in an Elasticsearch index. For each event, a document with the event details, with the timestamp of document set to the event's timestamp is appended to an index. The ID of the documented is randomly generated by Elasticsearch. No documents are deleted or modified in this format. + +The steps below show how to use this notification target in `namespace` format. The other format is very similar and is omitted for brevity. + +### Step 1: Ensure Elasticsearch minimum requirements are met + +MinIO requires a 5.x series version of Elasticsearch. This is the latest major release series. Elasticsearch provides version upgrade migration guidelines [here](https://www.elastic.co/guide/en/elasticsearch/reference/current/setup-upgrade.html). + +### Step 2: Add Elasticsearch endpoint to MinIO + +The Elasticsearch configuration is located in the `notify_elasticsearch` key. Create a configuration key-value pair here for your Elasticsearch instance. The key is a name for your Elasticsearch endpoint, and the value is a collection of key-value parameters described in the table below. + +``` +KEY: +notify_elasticsearch[:name] publish bucket notifications to Elasticsearch endpoints + +ARGS: +url* (url) Elasticsearch server's address, with optional authentication info +index* (string) Elasticsearch index to store/update events, index is auto-created +format* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' +queue_dir (path) staging dir for undelivered messages e.g. '/home/events' +queue_limit (number) maximum limit for undelivered messages, defaults to '100000' +username (string) username for Elasticsearch basic-auth +password (string) password for Elasticsearch basic-auth +comment (sentence) optionally add a comment to this setting +``` + +or environment variables + +``` +KEY: +notify_elasticsearch[:name] publish bucket notifications to Elasticsearch endpoints + +ARGS: +MINIO_NOTIFY_ELASTICSEARCH_ENABLE* (on|off) enable notify_elasticsearch target, default is 'off' +MINIO_NOTIFY_ELASTICSEARCH_URL* (url) Elasticsearch server's address, with optional authentication info +MINIO_NOTIFY_ELASTICSEARCH_INDEX* (string) Elasticsearch index to store/update events, index is auto-created +MINIO_NOTIFY_ELASTICSEARCH_FORMAT* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' +MINIO_NOTIFY_ELASTICSEARCH_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' +MINIO_NOTIFY_ELASTICSEARCH_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' +MINIO_NOTIFY_ELASTICSEARCH_USERNAME (string) username for Elasticsearch basic-auth +MINIO_NOTIFY_ELASTICSEARCH_PASSWORD (string) password for Elasticsearch basic-auth +MINIO_NOTIFY_ELASTICSEARCH_COMMENT (sentence) optionally add a comment to this setting +``` + +For example: `http://localhost:9200` or with authentication info `http://elastic:MagicWord@127.0.0.1:9200`. + +MinIO supports persistent event store. The persistent store will backup events when the Elasticsearch broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 100000. + +If Elasticsearch has authentication enabled, the credentials can be supplied to MinIO via the `url` parameter formatted as `PROTO://USERNAME:PASSWORD@ELASTICSEARCH_HOST:PORT`. + +To update the configuration, use `mc admin config get` command to get the current configuration. + +```sh +$ mc admin config get myminio/ notify_elasticsearch +notify_elasticsearch:1 queue_limit="0" url="" format="namespace" index="" queue_dir="" +``` + +Use `mc admin config set` command to update the configuration for the deployment. Restart the MinIO server to put the changes into effect. The server will print a line like `SQS ARNs: arn:minio:sqs::1:elasticsearch` at start-up if there were no errors. + +```sh +mc admin config set myminio notify_elasticsearch:1 queue_limit="0" url="http://127.0.0.1:9200" format="namespace" index="minio_events" queue_dir="" username="" password="" +``` + +Note that, you can add as many Elasticsearch server endpoint configurations as needed by providing an identifier (like "1" in the example above) for the Elasticsearch instance and an object of per-server configuration parameters. + +### Step 3: Enable Elastichsearch bucket notification using MinIO client + +We will now enable bucket event notifications on a bucket named `images`. Whenever a JPEG image is created/overwritten, a new document is added or an existing document is updated in the Elasticsearch index configured above. When an existing object is deleted, the corresponding document is deleted from the index. Thus, the rows in the Elasticsearch index, reflect the `.jpg` objects in the `images` bucket. + +To configure this bucket notification, we need the ARN printed by MinIO in the previous step. Additional information about ARN is available [here](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). + +With the `mc` tool, the configuration is very simple to add. Let us say that the MinIO server is aliased as `myminio` in our mc configuration. Execute the following: + +``` +mc mb myminio/images +mc event add myminio/images arn:minio:sqs::1:elasticsearch --suffix .jpg +mc event list myminio/images +arn:minio:sqs::1:elasticsearch s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg” +``` + +### Step 4: Test on Elasticsearch + +Upload a JPEG image into `images` bucket. + +``` +mc cp myphoto.jpg myminio/images +``` + +Use curl to view contents of `minio_events` index. + +``` +$ curl "http://localhost:9200/minio_events/_search?pretty=true" +{ + "took" : 40, + "timed_out" : false, + "_shards" : { + "total" : 5, + "successful" : 5, + "failed" : 0 + }, + "hits" : { + "total" : 1, + "max_score" : 1.0, + "hits" : [ + { + "_index" : "minio_events", + "_type" : "event", + "_id" : "images/myphoto.jpg", + "_score" : 1.0, + "_source" : { + "Records" : [ + { + "eventVersion" : "2.0", + "eventSource" : "minio:s3", + "awsRegion" : "", + "eventTime" : "2017-03-30T08:00:41Z", + "eventName" : "s3:ObjectCreated:Put", + "userIdentity" : { + "principalId" : "minio" + }, + "requestParameters" : { + "sourceIPAddress" : "127.0.0.1:38062" + }, + "responseElements" : { + "x-amz-request-id" : "14B09A09703FC47B", + "x-minio-origin-endpoint" : "http://192.168.86.115:9000" + }, + "s3" : { + "s3SchemaVersion" : "1.0", + "configurationId" : "Config", + "bucket" : { + "name" : "images", + "ownerIdentity" : { + "principalId" : "minio" + }, + "arn" : "arn:aws:s3:::images" + }, + "object" : { + "key" : "myphoto.jpg", + "size" : 6474, + "eTag" : "a3410f4f8788b510d6f19c5067e60a90", + "sequencer" : "14B09A09703FC47B" + } + }, + "source" : { + "host" : "127.0.0.1", + "port" : "38062", + "userAgent" : "MinIO (linux; amd64) minio-go/2.0.3 mc/2017-02-15T17:57:25Z" + } + } + ] + } + } + ] + } +} +``` + +This output shows that a document has been created for the event in Elasticsearch. + +Here we see that the document ID is the bucket and object name. In case `access` format was used, the document ID would be automatically generated by Elasticsearch. + +## Publish MinIO events via Redis + +Install [Redis](http://redis.io/download) server. For illustrative purposes, we have set the database password as "yoursecret". + +This notification target supports two formats: _namespace_ and _access_. + +When the _namespace_ format is used, MinIO synchronizes objects in the bucket with entries in a hash. For each entry, the key is formatted as "bucketName/objectName" for an object that exists in the bucket, and the value is the JSON-encoded event data about the operation that created/replaced the object in MinIO. When objects are updated or deleted, the corresponding entry in the hash is also updated or deleted. + +When the _access_ format is used, MinIO appends events to a list using [RPUSH](https://redis.io/commands/rpush). Each item in the list is a JSON encoded list with two items, where the first item is a timestamp string, and the second item is a JSON object containing event data about the operation that happened in the bucket. No entries appended to the list are updated or deleted by MinIO in this format. + +The steps below show how to use this notification target in `namespace` and `access` format. + +### Step 1: Add Redis endpoint to MinIO + +The MinIO server configuration file is stored on the backend in json format.The Redis configuration is located in the `redis` key under the `notify` top-level key. Create a configuration key-value pair here for your Redis instance. The key is a name for your Redis endpoint, and the value is a collection of key-value parameters described in the table below. + +``` +KEY: +notify_redis[:name] publish bucket notifications to Redis datastores + +ARGS: +address* (address) Redis server's address. For example: `localhost:6379` +key* (string) Redis key to store/update events, key is auto-created +format* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' +password (string) Redis server password +queue_dir (path) staging dir for undelivered messages e.g. '/home/events' +queue_limit (number) maximum limit for undelivered messages, defaults to '100000' +comment (sentence) optionally add a comment to this setting +``` + +or environment variables + +``` +KEY: +notify_redis[:name] publish bucket notifications to Redis datastores + +ARGS: +MINIO_NOTIFY_REDIS_ENABLE* (on|off) enable notify_redis target, default is 'off' +MINIO_NOTIFY_REDIS_KEY* (string) Redis key to store/update events, key is auto-created +MINIO_NOTIFY_REDIS_FORMAT* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' +MINIO_NOTIFY_REDIS_PASSWORD (string) Redis server password +MINIO_NOTIFY_REDIS_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' +MINIO_NOTIFY_REDIS_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' +MINIO_NOTIFY_REDIS_COMMENT (sentence) optionally add a comment to this setting +``` + +MinIO supports persistent event store. The persistent store will backup events when the Redis broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 100000. + +To update the configuration, use `mc admin config get` command to get the current configuration. + +```sh +$ mc admin config get myminio/ notify_redis +notify_redis:1 address="" format="namespace" key="" password="" queue_dir="" queue_limit="0" +``` + +Use `mc admin config set` command to update the configuration for the deployment.Restart the MinIO server to put the changes into effect. The server will print a line like `SQS ARNs: arn:minio:sqs::1:redis` at start-up if there were no errors. + +```sh +mc admin config set myminio/ notify_redis:1 address="127.0.0.1:6379" format="namespace" key="bucketevents" password="yoursecret" queue_dir="" queue_limit="0" +``` + +Note that, you can add as many Redis server endpoint configurations as needed by providing an identifier (like "1" in the example above) for the Redis instance and an object of per-server configuration parameters. + +### Step 2: Enable Redis bucket notification using MinIO client + +We will now enable bucket event notifications on a bucket named `images`. Whenever a JPEG image is created/overwritten, a new key is added or an existing key is updated in the Redis hash configured above. When an existing object is deleted, the corresponding key is deleted from the Redis hash. Thus, the rows in the Redis hash, reflect the `.jpg` objects in the `images` bucket. + +To configure this bucket notification, we need the ARN printed by MinIO in the previous step. Additional information about ARN is available [here](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). + +With the `mc` tool, the configuration is very simple to add. Let us say that the MinIO server is aliased as `myminio` in our mc configuration. Execute the following: + +``` +mc mb myminio/images +mc event add myminio/images arn:minio:sqs::1:redis --suffix .jpg +mc event list myminio/images +arn:minio:sqs::1:redis s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg” +``` + +### Step 3: Test on Redis + +Start the `redis-cli` Redis client program to inspect the contents in Redis. Run the `monitor` Redis command. This prints each operation performed on Redis as it occurs. + +``` +redis-cli -a yoursecret +127.0.0.1:6379> monitor +OK +``` + +Open another terminal and upload a JPEG image into `images` bucket. + +``` +mc cp myphoto.jpg myminio/images +``` + +In the previous terminal, you will now see the operation that MinIO performs on Redis: + +``` +127.0.0.1:6379> monitor +OK +1490686879.650649 [0 172.17.0.1:44710] "PING" +1490686879.651061 [0 172.17.0.1:44710] "HSET" "minio_events" "images/myphoto.jpg" "{\"Records\":[{\"eventVersion\":\"2.0\",\"eventSource\":\"minio:s3\",\"awsRegion\":\"\",\"eventTime\":\"2017-03-28T07:41:19Z\",\"eventName\":\"s3:ObjectCreated:Put\",\"userIdentity\":{\"principalId\":\"minio\"},\"requestParameters\":{\"sourceIPAddress\":\"127.0.0.1:52234\"},\"responseElements\":{\"x-amz-request-id\":\"14AFFBD1ACE5F632\",\"x-minio-origin-endpoint\":\"http://192.168.86.115:9000\"},\"s3\":{\"s3SchemaVersion\":\"1.0\",\"configurationId\":\"Config\",\"bucket\":{\"name\":\"images\",\"ownerIdentity\":{\"principalId\":\"minio\"},\"arn\":\"arn:aws:s3:::images\"},\"object\":{\"key\":\"myphoto.jpg\",\"size\":2586,\"eTag\":\"5d284463f9da279f060f0ea4d11af098\",\"sequencer\":\"14AFFBD1ACE5F632\"}},\"source\":{\"host\":\"127.0.0.1\",\"port\":\"52234\",\"userAgent\":\"MinIO (linux; amd64) minio-go/2.0.3 mc/2017-02-15T17:57:25Z\"}}]}" +``` + +Here we see that MinIO performed `HSET` on `minio_events` key. + +In case, `access` format was used, then `minio_events` would be a list, and the MinIO server would have performed an `RPUSH` to append to the list. A consumer of this list would ideally use `BLPOP` to remove list items from the left-end of the list. + +## Publish MinIO events via NATS + +Install NATS from [here](http://nats.io/). + +### Step 1: Add NATS endpoint to MinIO + +MinIO supports persistent event store. The persistent store will backup events when the NATS broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 100000. + +``` +KEY: +notify_nats[:name] publish bucket notifications to NATS endpoints + +ARGS: +address* (address) NATS server address e.g. '0.0.0.0:4222' +subject* (string) NATS subscription subject +username (string) NATS username +password (string) NATS password +token (string) NATS token +tls (on|off) set to 'on' to enable TLS +tls_skip_verify (on|off) trust server TLS without verification, defaults to "on" (verify) +ping_interval (duration) client ping commands interval in s,m,h,d. Disabled by default +streaming (on|off) set to 'on', to use streaming NATS server +streaming_async (on|off) set to 'on', to enable asynchronous publish +streaming_max_pub_acks_in_flight (number) number of messages to publish without waiting for ACKs +streaming_cluster_id (string) unique ID for NATS streaming cluster +cert_authority (string) path to certificate chain of the target NATS server +client_cert (string) client cert for NATS mTLS auth +client_key (string) client cert key for NATS mTLS auth +queue_dir (path) staging dir for undelivered messages e.g. '/home/events' +queue_limit (number) maximum limit for undelivered messages, defaults to '100000' +comment (sentence) optionally add a comment to this setting +``` + +or environment variables + +``` +KEY: +notify_nats[:name] publish bucket notifications to NATS endpoints + +ARGS: +MINIO_NOTIFY_NATS_ENABLE* (on|off) enable notify_nats target, default is 'off' +MINIO_NOTIFY_NATS_ADDRESS* (address) NATS server address e.g. '0.0.0.0:4222' +MINIO_NOTIFY_NATS_SUBJECT* (string) NATS subscription subject +MINIO_NOTIFY_NATS_USERNAME (string) NATS username +MINIO_NOTIFY_NATS_PASSWORD (string) NATS password +MINIO_NOTIFY_NATS_TOKEN (string) NATS token +MINIO_NOTIFY_NATS_TLS (on|off) set to 'on' to enable TLS +MINIO_NOTIFY_NATS_TLS_SKIP_VERIFY (on|off) trust server TLS without verification, defaults to "on" (verify) +MINIO_NOTIFY_NATS_PING_INTERVAL (duration) client ping commands interval in s,m,h,d. Disabled by default +MINIO_NOTIFY_NATS_STREAMING (on|off) set to 'on', to use streaming NATS server +MINIO_NOTIFY_NATS_STREAMING_ASYNC (on|off) set to 'on', to enable asynchronous publish +MINIO_NOTIFY_NATS_STREAMING_MAX_PUB_ACKS_IN_FLIGHT (number) number of messages to publish without waiting for ACKs +MINIO_NOTIFY_NATS_STREAMING_CLUSTER_ID (string) unique ID for NATS streaming cluster +MINIO_NOTIFY_NATS_CERT_AUTHORITY (string) path to certificate chain of the target NATS server +MINIO_NOTIFY_NATS_CLIENT_CERT (string) client cert for NATS mTLS auth +MINIO_NOTIFY_NATS_CLIENT_KEY (string) client cert key for NATS mTLS auth +MINIO_NOTIFY_NATS_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' +MINIO_NOTIFY_NATS_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' +MINIO_NOTIFY_NATS_COMMENT (sentence) optionally add a comment to this setting +``` + +To update the configuration, use `mc admin config get` command to get the current configuration file for the minio deployment. + +```sh +$ mc admin config get myminio/ notify_nats +notify_nats:1 password="yoursecret" streaming_max_pub_acks_in_flight="10" subject="" address="0.0.0.0:4222" token="" username="yourusername" ping_interval="0" queue_limit="0" tls="off" tls_skip_verify="off" streaming_async="on" queue_dir="" streaming_cluster_id="test-cluster" streaming_enable="on" +``` + +Use `mc admin config set` command to update the configuration for the deployment.Restart MinIO server to reflect config changes. `bucketevents` is the subject used by NATS in this example. + +```sh +mc admin config set myminio notify_nats:1 password="yoursecret" streaming_max_pub_acks_in_flight="10" subject="" address="0.0.0.0:4222" token="" username="yourusername" ping_interval="0" queue_limit="0" tls="off" streaming_async="on" queue_dir="" streaming_cluster_id="test-cluster" streaming_enable="on" +``` + +MinIO server also supports [NATS Streaming mode](http://nats.io/documentation/streaming/nats-streaming-intro/) that offers additional functionality like `At-least-once-delivery`, and `Publisher rate limiting`. To configure MinIO server to send notifications to NATS Streaming server, update the MinIO server configuration file as follows: + +Read more about sections `cluster_id`, `client_id` on [NATS documentation](https://github.com/nats-io/nats-streaming-server/blob/master/README.md). Section `maxPubAcksInflight` is explained [here](https://github.com/nats-io/stan.go#publisher-rate-limiting). + +### Step 2: Enable NATS bucket notification using MinIO client + +We will enable bucket event notification to trigger whenever a JPEG image is uploaded or deleted from `images` bucket on `myminio` server. Here ARN value is `arn:minio:sqs::1:nats`. To understand more about ARN please follow [AWS ARN](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) documentation. + +``` +mc mb myminio/images +mc event add myminio/images arn:minio:sqs::1:nats --suffix .jpg +mc event list myminio/images +arn:minio:sqs::1:nats s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg” +``` + +### Step 3: Test on NATS + +If you use NATS server, check out this sample program below to log the bucket notification added to NATS. + +```go +package main + +// Import Go and NATS packages +import ( + "log" + "runtime" + + "github.com/nats-io/nats.go" +) + +func main() { + + // Create server connection + natsConnection, _ := nats.Connect("nats://yourusername:yoursecret@localhost:4222") + log.Println("Connected") + + // Subscribe to subject + log.Printf("Subscribing to subject 'bucketevents'\n") + natsConnection.Subscribe("bucketevents", func(msg *nats.Msg) { + + // Handle the message + log.Printf("Received message '%s\n", string(msg.Data)+"'") + }) + + // Keep the connection alive + runtime.Goexit() +} +``` + +``` +go run nats.go +2016/10/12 06:39:18 Connected +2016/10/12 06:39:18 Subscribing to subject 'bucketevents' +``` + +Open another terminal and upload a JPEG image into `images` bucket. + +``` +mc cp myphoto.jpg myminio/images +``` + +The example `nats.go` program prints event notification to console. + +``` +go run nats.go +2016/10/12 06:51:26 Connected +2016/10/12 06:51:26 Subscribing to subject 'bucketevents' +2016/10/12 06:51:33 Received message '{"EventType":"s3:ObjectCreated:Put","Key":"images/myphoto.jpg","Records":[{"eventVersion":"2.0","eventSource":"aws:s3","awsRegion":"","eventTime":"2016-10-12T13:51:33Z","eventName":"s3:ObjectCreated:Put","userIdentity":{"principalId":"minio"},"requestParameters":{"sourceIPAddress":"[::1]:57106"},"responseElements":{},"s3":{"s3SchemaVersion":"1.0","configurationId":"Config","bucket":{"name":"images","ownerIdentity":{"principalId":"minio"},"arn":"arn:aws:s3:::images"},"object":{"key":"myphoto.jpg","size":56060,"eTag":"1d97bf45ecb37f7a7b699418070df08f","sequencer":"147CCD1AE054BFD0"}}}],"level":"info","msg":"","time":"2016-10-12T06:51:33-07:00"} +``` + +If you use NATS Streaming server, check out this sample program below to log the bucket notification added to NATS. + +```go +package main + +// Import Go and NATS packages +import ( + "fmt" + "runtime" + + "github.com/nats-io/stan.go" +) + +func main() { + + var stanConnection stan.Conn + + subscribe := func() { + fmt.Printf("Subscribing to subject 'bucketevents'\n") + stanConnection.Subscribe("bucketevents", func(m *stan.Msg) { + + // Handle the message + fmt.Printf("Received a message: %s\n", string(m.Data)) + }) + } + + + stanConnection, _ = stan.Connect("test-cluster", "test-client", stan.NatsURL("nats://yourusername:yoursecret@0.0.0.0:4222"), stan.SetConnectionLostHandler(func(c stan.Conn, _ error) { + go func() { + for { + // Reconnect if the connection is lost. + if stanConnection == nil || stanConnection.NatsConn() == nil || !stanConnection.NatsConn().IsConnected() { + stanConnection, _ = stan.Connect("test-cluster", "test-client", stan.NatsURL("nats://yourusername:yoursecret@0.0.0.0:4222"), stan.SetConnectionLostHandler(func(c stan.Conn, _ error) { + if c.NatsConn() != nil { + c.NatsConn().Close() + } + _ = c.Close() + })) + if stanConnection != nil { + subscribe() + } + + } + } + + }() + })) + + // Subscribe to subject + subscribe() + + // Keep the connection alive + runtime.Goexit() +} + +``` + +``` +go run nats.go +2017/07/07 11:47:40 Connected +2017/07/07 11:47:40 Subscribing to subject 'bucketevents' +``` + +Open another terminal and upload a JPEG image into `images` bucket. + +``` +mc cp myphoto.jpg myminio/images +``` + +The example `nats.go` program prints event notification to console. + +``` +Received a message: {"EventType":"s3:ObjectCreated:Put","Key":"images/myphoto.jpg","Records":[{"eventVersion":"2.0","eventSource":"minio:s3","awsRegion":"","eventTime":"2017-07-07T18:46:37Z","eventName":"s3:ObjectCreated:Put","userIdentity":{"principalId":"minio"},"requestParameters":{"sourceIPAddress":"192.168.1.80:55328"},"responseElements":{"x-amz-request-id":"14CF20BD1EFD5B93","x-minio-origin-endpoint":"http://127.0.0.1:9000"},"s3":{"s3SchemaVersion":"1.0","configurationId":"Config","bucket":{"name":"images","ownerIdentity":{"principalId":"minio"},"arn":"arn:aws:s3:::images"},"object":{"key":"myphoto.jpg","size":248682,"eTag":"f1671feacb8bbf7b0397c6e9364e8c92","contentType":"image/jpeg","userDefined":{"content-type":"image/jpeg"},"versionId":"1","sequencer":"14CF20BD1EFD5B93"}},"source":{"host":"192.168.1.80","port":"55328","userAgent":"MinIO (linux; amd64) minio-go/2.0.4 mc/DEVELOPMENT.GOGET"}}],"level":"info","msg":"","time":"2017-07-07T11:46:37-07:00"} +``` + +## Publish MinIO events via PostgreSQL + +> NOTE: Until release RELEASE.2020-04-10T03-34-42Z PostgreSQL notification used to support following options: +> +> ``` +> host (hostname) Postgres server hostname (used only if `connection_string` is empty) +> port (port) Postgres server port, defaults to `5432` (used only if `connection_string` is empty) +> username (string) database username (used only if `connection_string` is empty) +> password (string) database password (used only if `connection_string` is empty) +> database (string) database name (used only if `connection_string` is empty) +> ``` +> +> These are now deprecated, if you plan to upgrade to any releases after _RELEASE.2020-04-10T03-34-42Z_ make sure +> to migrate to only using _connection_string_ option. To migrate, once you have upgraded all the servers use the +> following command to update the existing notification targets. +> +> ``` +> mc admin config set myminio/ notify_postgres[:name] connection_string="host=hostname port=2832 username=psqluser password=psqlpass database=bucketevents" +> ``` +> +> Please make sure this step is carried out, without this step PostgreSQL notification targets will not work, +> an error message will be shown on the console upon server upgrade/restart, make sure to follow the above +> instructions appropriately. For further questions please join our + +Install [PostgreSQL](https://www.postgresql.org/) database server. For illustrative purposes, we have set the "postgres" user password as `password` and created a database called `minio_events` to store the events. + +This notification target supports two formats: _namespace_ and _access_. + +When the _namespace_ format is used, MinIO synchronizes objects in the bucket with rows in the table. It creates rows with two columns: key and value. The key is the bucket and object name of an object that exists in MinIO. The value is JSON encoded event data about the operation that created/replaced the object in MinIO. When objects are updated or deleted, the corresponding row from this table is updated or deleted respectively. + +When the _access_ format is used, MinIO appends events to a table. It creates rows with two columns: event_time and event_data. The event_time is the time at which the event occurred in the MinIO server. The event_data is the JSON encoded event data about the operation on an object. No rows are deleted or modified in this format. + +The steps below show how to use this notification target in `namespace` format. The other format is very similar and is omitted for brevity. + +### Step 1: Ensure postgresql minimum requirements are met + +MinIO requires PostgreSQL version 9.5 or above. MinIO uses the [`INSERT ON CONFLICT`](https://www.postgresql.org/docs/9.5/static/sql-insert.html#SQL-ON-CONFLICT) (aka UPSERT) feature, introduced in version 9.5 and the [JSONB](https://www.postgresql.org/docs/9.4/static/datatype-json.html) data-type introduced in version 9.4. + +### Step 2: Add PostgreSQL endpoint to MinIO + +The PostgreSQL configuration is located in the `notify_postgresql` key. Create a configuration key-value pair here for your PostgreSQL instance. The key is a name for your PostgreSQL endpoint, and the value is a collection of key-value parameters described in the table below. + +``` +KEY: +notify_postgres[:name] publish bucket notifications to Postgres databases + +ARGS: +connection_string* (string) Postgres server connection-string e.g. "host=localhost port=5432 dbname=minio_events user=postgres password=password sslmode=disable" +table* (string) DB table name to store/update events, table is auto-created +format* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' +queue_dir (path) staging dir for undelivered messages e.g. '/home/events' +queue_limit (number) maximum limit for undelivered messages, defaults to '100000' +max_open_connections (number) maximum number of open connections to the database, defaults to '2' +comment (sentence) optionally add a comment to this setting +``` + +or environment variables + +``` +KEY: +notify_postgres[:name] publish bucket notifications to Postgres databases + +ARGS: +MINIO_NOTIFY_POSTGRES_ENABLE* (on|off) enable notify_postgres target, default is 'off' +MINIO_NOTIFY_POSTGRES_CONNECTION_STRING* (string) Postgres server connection-string e.g. "host=localhost port=5432 dbname=minio_events user=postgres password=password sslmode=disable" +MINIO_NOTIFY_POSTGRES_TABLE* (string) DB table name to store/update events, table is auto-created +MINIO_NOTIFY_POSTGRES_FORMAT* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' +MINIO_NOTIFY_POSTGRES_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' +MINIO_NOTIFY_POSTGRES_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' +MINIO_NOTIFY_POSTGRES_COMMENT (sentence) optionally add a comment to this setting +MINIO_NOTIFY_POSTGRES_MAX_OPEN_CONNECTIONS (number) maximum number of open connections to the database, defaults to '2' +``` + +> NOTE: If the `max_open_connections` key or the environment variable `MINIO_NOTIFY_POSTGRES_MAX_OPEN_CONNECTIONS` is set to `0`, There will be no limit set on the number of +> open connections to the database. This setting is generally NOT recommended as the behavior may be inconsistent during recursive deletes in `namespace` format. + +MinIO supports persistent event store. The persistent store will backup events when the PostgreSQL connection goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 100000. + +Note that for illustration here, we have disabled SSL. In the interest of security, for production this is not recommended. +To update the configuration, use `mc admin config get` command to get the current configuration. + +```sh +$ mc admin config get myminio notify_postgres +notify_postgres:1 queue_dir="" connection_string="" queue_limit="0" table="" format="namespace" +``` + +Use `mc admin config set` command to update the configuration for the deployment. Restart the MinIO server to put the changes into effect. The server will print a line like `SQS ARNs: arn:minio:sqs::1:postgresql` at start-up if there were no errors. + +```sh +mc admin config set myminio notify_postgres:1 connection_string="host=localhost port=5432 dbname=minio_events user=postgres password=password sslmode=disable" table="bucketevents" format="namespace" +``` + +Note that, you can add as many PostgreSQL server endpoint configurations as needed by providing an identifier (like "1" in the example above) for the PostgreSQL instance and an object of per-server configuration parameters. + +### Step 3: Enable PostgreSQL bucket notification using MinIO client + +We will now enable bucket event notifications on a bucket named `images`. Whenever a JPEG image is created/overwritten, a new row is added or an existing row is updated in the PostgreSQL configured above. When an existing object is deleted, the corresponding row is deleted from the PostgreSQL table. Thus, the rows in the PostgreSQL table, reflect the `.jpg` objects in the `images` bucket. + +To configure this bucket notification, we need the ARN printed by MinIO in the previous step. Additional information about ARN is available [here](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). + +With the `mc` tool, the configuration is very simple to add. Let us say that the MinIO server is aliased as `myminio` in our mc configuration. Execute the following: + +``` +# Create bucket named `images` in myminio +mc mb myminio/images +# Add notification configuration on the `images` bucket using the MySQL ARN. The --suffix argument filters events. +mc event add myminio/images arn:minio:sqs::1:postgresql --suffix .jpg +# Print out the notification configuration on the `images` bucket. +mc event list myminio/images +mc event list myminio/images +arn:minio:sqs::1:postgresql s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg” +``` + +### Step 4: Test on PostgreSQL + +Open another terminal and upload a JPEG image into `images` bucket. + +``` +mc cp myphoto.jpg myminio/images +``` + +Open PostgreSQL terminal to list the rows in the `bucketevents` table. + +``` +$ psql -h 127.0.0.1 -U postgres -d minio_events +minio_events=# select * from bucketevents; + +key | value +--------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- + images/myphoto.jpg | {"Records": [{"s3": {"bucket": {"arn": "arn:aws:s3:::images", "name": "images", "ownerIdentity": {"principalId": "minio"}}, "object": {"key": "myphoto.jpg", "eTag": "1d97bf45ecb37f7a7b699418070df08f", "size": 56060, "sequencer": "147CE57C70B31931"}, "configurationId": "Config", "s3SchemaVersion": "1.0"}, "awsRegion": "", "eventName": "s3:ObjectCreated:Put", "eventTime": "2016-10-12T21:18:20Z", "eventSource": "aws:s3", "eventVersion": "2.0", "userIdentity": {"principalId": "minio"}, "responseElements": {}, "requestParameters": {"sourceIPAddress": "[::1]:39706"}}]} +(1 row) +``` + +## Publish MinIO events via MySQL + +> NOTE: Until release RELEASE.2020-04-10T03-34-42Z MySQL notification used to support following options: +> +> ``` +> host (hostname) MySQL server hostname (used only if `dsn_string` is empty) +> port (port) MySQL server port (used only if `dsn_string` is empty) +> username (string) database username (used only if `dsn_string` is empty) +> password (string) database password (used only if `dsn_string` is empty) +> database (string) database name (used only if `dsn_string` is empty) +> ``` +> +> These are now deprecated, if you plan to upgrade to any releases after _RELEASE.2020-04-10T03-34-42Z_ make sure +> to migrate to only using _dsn_string_ option. To migrate, once you have upgraded all the servers use the +> following command to update the existing notification targets. +> +> ``` +> mc admin config set myminio/ notify_mysql[:name] dsn_string="mysqluser:mysqlpass@tcp(localhost:2832)/bucketevents" +> ``` +> +> Please make sure this step is carried out, without this step MySQL notification targets will not work, +> an error message will be shown on the console upon server upgrade/restart, make sure to follow the above +> instructions appropriately. For further questions please join our + +Install MySQL from [here](https://dev.mysql.com/downloads/mysql/). For illustrative purposes, we have set the root password as `password` and created a database called `miniodb` to store the events. + +This notification target supports two formats: _namespace_ and _access_. + +When the _namespace_ format is used, MinIO synchronizes objects in the bucket with rows in the table. It creates rows with two columns: key_name and value. The key_name is the bucket and object name of an object that exists in MinIO. The value is JSON encoded event data about the operation that created/replaced the object in MinIO. When objects are updated or deleted, the corresponding row from this table is updated or deleted respectively. + +When the _access_ format is used, MinIO appends events to a table. It creates rows with two columns: event_time and event_data. The event_time is the time at which the event occurred in the MinIO server. The event_data is the JSON encoded event data about the operation on an object. No rows are deleted or modified in this format. + +The steps below show how to use this notification target in `namespace` format. The other format is very similar and is omitted for brevity. + +### Step 1: Ensure MySQL minimum requirements are met + +MinIO requires MySQL version 5.7.8 or above. MinIO uses the [JSON](https://dev.mysql.com/doc/refman/5.7/en/json.html) data-type introduced in version 5.7.8. We tested this setup on MySQL 5.7.17. + +### Step 2: Add MySQL server endpoint configuration to MinIO + +The MySQL configuration is located in the `notify_mysql` key. Create a configuration key-value pair here for your MySQL instance. The key is a name for your MySQL endpoint, and the value is a collection of key-value parameters described in the table below. + +``` +KEY: +notify_mysql[:name] publish bucket notifications to MySQL databases. When multiple MySQL server endpoints are needed, a user specified "name" can be added for each configuration, (e.g."notify_mysql:myinstance"). + +ARGS: +dsn_string* (string) MySQL data-source-name connection string e.g. ":@tcp(:)/" +table* (string) DB table name to store/update events, table is auto-created +format* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' +queue_dir (path) staging dir for undelivered messages e.g. '/home/events' +queue_limit (number) maximum limit for undelivered messages, defaults to '100000' +max_open_connections (number) maximum number of open connections to the database, defaults to '2' +comment (sentence) optionally add a comment to this setting +``` + +or environment variables + +``` +KEY: +notify_mysql[:name] publish bucket notifications to MySQL databases + +ARGS: +MINIO_NOTIFY_MYSQL_ENABLE* (on|off) enable notify_mysql target, default is 'off' +MINIO_NOTIFY_MYSQL_DSN_STRING* (string) MySQL data-source-name connection string e.g. ":@tcp(:)/" +MINIO_NOTIFY_MYSQL_TABLE* (string) DB table name to store/update events, table is auto-created +MINIO_NOTIFY_MYSQL_FORMAT* (namespace*|access) 'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace' +MINIO_NOTIFY_MYSQL_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' +MINIO_NOTIFY_MYSQL_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' +MINIO_NOTIFY_MYSQL_MAX_OPEN_CONNECTIONS (number) maximum number of open connections to the database, defaults to '2' +MINIO_NOTIFY_MYSQL_COMMENT (sentence) optionally add a comment to this setting +``` + +> NOTE: If the `max_open_connections` key or the environment variable `MINIO_NOTIFY_MYSQL_MAX_OPEN_CONNECTIONS` is set to `0`, There will be no limit set on the number of +> open connections to the database. This setting is generally NOT recommended as the behavior may be inconsistent during recursive deletes in `namespace` format. + +`dsn_string` is required and is of form `":@tcp(:)/"` + +MinIO supports persistent event store. The persistent store will backup events if MySQL connection goes offline and then replays the stored events when the broken connection comes back up. The event store can be configured by setting a directory path in `queue_dir` field, and the maximum number of events, which can be stored in a `queue_dir`, in `queue_limit` field. For example, `queue_dir` can be set to `/home/events` and `queue_limit` can be set to `1000`. By default, the `queue_limit` is set to `100000`. + +Before updating the configuration, let's start with `mc admin config get` command to get the current configuration. + +```sh +$ mc admin config get myminio/ notify_mysql +notify_mysql:myinstance enable=off format=namespace host= port= username= password= database= dsn_string= table= queue_dir= queue_limit=0 +``` + +Use `mc admin config set` command to update MySQL notification configuration for the deployment with `dsn_string` parameter: + +```sh +mc admin config set myminio notify_mysql:myinstance table="minio_images" dsn_string="root:xxxx@tcp(172.17.0.1:3306)/miniodb" +``` + +Note that, you can add as many MySQL server endpoint configurations as needed by providing an identifier (like "myinstance" in the example above) for each MySQL instance desired. + +Restart the MinIO server to put the changes into effect. The server will print a line like `SQS ARNs: arn:minio:sqs::myinstance:mysql` at start-up, if there are no errors. + +### Step 3: Enable MySQL bucket notification using MinIO client + +We will now setup bucket notifications on a bucket named `images`. Whenever a JPEG image object is created/overwritten, a new row is added or an existing row is updated in the MySQL table configured above. When an existing object is deleted, the corresponding row is deleted from the MySQL table. Thus, the rows in the MySQL table, reflect the `.jpg` objects in the `images` bucket. + +To configure this bucket notification, we need the ARN printed by MinIO in the previous step. Additional information about ARN is available [here](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html). + +With the `mc` tool, the configuration is very simple to add. Let us say that the MinIO server is aliased as `myminio` in our mc configuration. Execute the following: + +``` +# Create bucket named `images` in myminio +mc mb myminio/images +# Add notification configuration on the `images` bucket using the MySQL ARN. The --suffix argument filters events. +mc event add myminio/images arn:minio:sqs::myinstance:mysql --suffix .jpg +# Print out the notification configuration on the `images` bucket. +mc event list myminio/images +arn:minio:sqs::myinstance:mysql s3:ObjectCreated:*,s3:ObjectRemoved:*,s3:ObjectAccessed:* Filter: suffix=”.jpg” +``` + +### Step 4: Test on MySQL + +Open another terminal and upload a JPEG image into `images` bucket: + +``` +mc cp myphoto.jpg myminio/images +``` + +Open MySQL terminal and list the rows in the `minio_images` table. + +``` +$ mysql -h 172.17.0.1 -P 3306 -u root -p miniodb +mysql> select * from minio_images; ++--------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| key_name | value | ++--------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +| images/myphoto.jpg | {"Records": [{"s3": {"bucket": {"arn": "arn:aws:s3:::images", "name": "images", "ownerIdentity": {"principalId": "minio"}}, "object": {"key": "myphoto.jpg", "eTag": "467886be95c8ecfd71a2900e3f461b4f", "size": 26, "sequencer": "14AC59476F809FD3"}, "configurationId": "Config", "s3SchemaVersion": "1.0"}, "awsRegion": "", "eventName": "s3:ObjectCreated:Put", "eventTime": "2017-03-16T11:29:00Z", "eventSource": "aws:s3", "eventVersion": "2.0", "userIdentity": {"principalId": "minio"}, "responseElements": {"x-amz-request-id": "14AC59476F809FD3", "x-minio-origin-endpoint": "http://192.168.86.110:9000"}, "requestParameters": {"sourceIPAddress": "127.0.0.1:38260"}}]} | ++--------------------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+ +1 row in set (0.01 sec) + +``` + +## Publish MinIO events via Kafka + +Install Apache Kafka from [here](http://kafka.apache.org/). + +### Step 1: Ensure minimum requirements are met + +MinIO requires Kafka version 0.10 or 0.9. Internally MinIO uses the [Shopify/sarama](https://github.com/Shopify/sarama/) library and so has the same version compatibility as provided by this library. + +### Step 2: Add Kafka endpoint to MinIO + +MinIO supports persistent event store. The persistent store will backup events when the kafka broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 100000. + +``` +KEY: +notify_kafka[:name] publish bucket notifications to Kafka endpoints + +ARGS: +brokers* (csv) comma separated list of Kafka broker addresses +topic (string) Kafka topic used for bucket notifications +sasl_username (string) username for SASL/PLAIN or SASL/SCRAM authentication +sasl_password (string) password for SASL/PLAIN or SASL/SCRAM authentication +sasl_mechanism (string) sasl authentication mechanism, default 'PLAIN' +tls_client_auth (string) clientAuth determines the Kafka server's policy for TLS client auth +sasl (on|off) set to 'on' to enable SASL authentication +tls (on|off) set to 'on' to enable TLS +tls_skip_verify (on|off) trust server TLS without verification, defaults to "on" (verify) +client_tls_cert (path) path to client certificate for mTLS auth +client_tls_key (path) path to client key for mTLS auth +queue_dir (path) staging dir for undelivered messages e.g. '/home/events' +queue_limit (number) maximum limit for undelivered messages, defaults to '100000' +version (string) specify the version of the Kafka cluster e.g '2.2.0' +comment (sentence) optionally add a comment to this setting +``` + +or environment variables + +``` +KEY: +notify_kafka[:name] publish bucket notifications to Kafka endpoints + +ARGS: +MINIO_NOTIFY_KAFKA_ENABLE* (on|off) enable notify_kafka target, default is 'off' +MINIO_NOTIFY_KAFKA_BROKERS* (csv) comma separated list of Kafka broker addresses +MINIO_NOTIFY_KAFKA_TOPIC (string) Kafka topic used for bucket notifications +MINIO_NOTIFY_KAFKA_SASL_USERNAME (string) username for SASL/PLAIN or SASL/SCRAM authentication +MINIO_NOTIFY_KAFKA_SASL_PASSWORD (string) password for SASL/PLAIN or SASL/SCRAM authentication +MINIO_NOTIFY_KAFKA_SASL_MECHANISM (plain*|sha256|sha512) sasl authentication mechanism, default 'plain' +MINIO_NOTIFY_KAFKA_TLS_CLIENT_AUTH (string) clientAuth determines the Kafka server's policy for TLS client auth +MINIO_NOTIFY_KAFKA_SASL (on|off) set to 'on' to enable SASL authentication +MINIO_NOTIFY_KAFKA_TLS (on|off) set to 'on' to enable TLS +MINIO_NOTIFY_KAFKA_TLS_SKIP_VERIFY (on|off) trust server TLS without verification, defaults to "on" (verify) +MINIO_NOTIFY_KAFKA_CLIENT_TLS_CERT (path) path to client certificate for mTLS auth +MINIO_NOTIFY_KAFKA_CLIENT_TLS_KEY (path) path to client key for mTLS auth +MINIO_NOTIFY_KAFKA_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' +MINIO_NOTIFY_KAFKA_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' +MINIO_NOTIFY_KAFKA_COMMENT (sentence) optionally add a comment to this setting +MINIO_NOTIFY_KAFKA_VERSION (string) specify the version of the Kafka cluster e.g. '2.2.0' +MINIO_NOTIFY_KAFKA_PRODUCER_COMPRESSION_CODEC (none|snappy|gzip|lz4|zstd) compression codec for producer messages +MINIO_NOTIFY_KAFKA_PRODUCER_COMPRESSION_LEVEL (number) compression level for producer messages, defaults to '0' +``` + +To update the configuration, use `mc admin config get` command to get the current configuration. + +```sh +$ mc admin config get myminio/ notify_kafka +notify_kafka:1 tls_skip_verify="off" queue_dir="" queue_limit="0" sasl="off" sasl_password="" sasl_username="" tls_client_auth="0" tls="off" brokers="" topic="" client_tls_cert="" client_tls_key="" version="" +``` + +Use `mc admin config set` command to update the configuration for the deployment. Restart the MinIO server to put the changes into effect. The server will print a line like `SQS ARNs: arn:minio:sqs::1:kafka` at start-up if there were no errors.`bucketevents` is the topic used by kafka in this example. + +```sh +mc admin config set myminio notify_kafka:1 tls_skip_verify="off" queue_dir="" queue_limit="0" sasl="off" sasl_password="" sasl_username="" tls_client_auth="0" tls="off" client_tls_cert="" client_tls_key="" brokers="localhost:9092,localhost:9093" topic="bucketevents" version="" +``` + +### Step 3: Enable Kafka bucket notification using MinIO client + +We will enable bucket event notification to trigger whenever a JPEG image is uploaded or deleted from `images` bucket on `myminio` server. Here ARN value is `arn:minio:sqs::1:kafka`. To understand more about ARN please follow [AWS ARN](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) documentation. + +``` +mc mb myminio/images +mc event add myminio/images arn:minio:sqs::1:kafka --suffix .jpg +mc event list myminio/images +arn:minio:sqs::1:kafka s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg” +``` + +### Step 4: Test on Kafka + +We used [kafkacat](https://github.com/edenhill/kafkacat) to print all notifications on the console. + +``` +kafkacat -C -b localhost:9092 -t bucketevents +``` + +Open another terminal and upload a JPEG image into `images` bucket. + +``` +mc cp myphoto.jpg myminio/images +``` + +`kafkacat` prints the event notification to the console. + +``` +kafkacat -b localhost:9092 -t bucketevents +{ + "EventName": "s3:ObjectCreated:Put", + "Key": "images/myphoto.jpg", + "Records": [ + { + "eventVersion": "2.0", + "eventSource": "minio:s3", + "awsRegion": "", + "eventTime": "2019-09-10T17:41:54Z", + "eventName": "s3:ObjectCreated:Put", + "userIdentity": { + "principalId": "AKIAIOSFODNN7EXAMPLE" + }, + "requestParameters": { + "accessKey": "AKIAIOSFODNN7EXAMPLE", + "region": "", + "sourceIPAddress": "192.168.56.192" + }, + "responseElements": { + "x-amz-request-id": "15C3249451E12784", + "x-minio-deployment-id": "751a8ba6-acb2-42f6-a297-4cdf1cf1fa4f", + "x-minio-origin-endpoint": "http://192.168.97.83:9000" + }, + "s3": { + "s3SchemaVersion": "1.0", + "configurationId": "Config", + "bucket": { + "name": "images", + "ownerIdentity": { + "principalId": "AKIAIOSFODNN7EXAMPLE" + }, + "arn": "arn:aws:s3:::images" + }, + "object": { + "key": "myphoto.jpg", + "size": 6474, + "eTag": "430f89010c77aa34fc8760696da62d08-1", + "contentType": "image/jpeg", + "userMetadata": { + "content-type": "image/jpeg" + }, + "versionId": "1", + "sequencer": "15C32494527B46C5" + } + }, + "source": { + "host": "192.168.56.192", + "port": "", + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:69.0) Gecko/20100101 Firefox/69.0" + } + } + ] +} +``` + +## Publish MinIO events via Webhooks + +[Webhooks](https://en.wikipedia.org/wiki/Webhook) are a way to receive information when it happens, rather than continually polling for that data. + +### Step 1: Add Webhook endpoint to MinIO + +MinIO supports persistent event store. The persistent store will backup events when the webhook goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 100000. + +``` +KEY: +notify_webhook[:name] publish bucket notifications to webhook endpoints + +ARGS: +endpoint* (url) webhook server endpoint e.g. http://localhost:8080/minio/events +auth_token (string) opaque string or JWT authorization token +queue_dir (path) staging dir for undelivered messages e.g. '/home/events' +queue_limit (number) maximum limit for undelivered messages, defaults to '100000' +client_cert (string) client cert for Webhook mTLS auth +client_key (string) client cert key for Webhook mTLS auth +comment (sentence) optionally add a comment to this setting +``` + +or environment variables + +``` +KEY: +notify_webhook[:name] publish bucket notifications to webhook endpoints + +ARGS: +MINIO_NOTIFY_WEBHOOK_ENABLE* (on|off) enable notify_webhook target, default is 'off' +MINIO_NOTIFY_WEBHOOK_ENDPOINT* (url) webhook server endpoint e.g. http://localhost:8080/minio/events +MINIO_NOTIFY_WEBHOOK_AUTH_TOKEN (string) opaque string or JWT authorization token +MINIO_NOTIFY_WEBHOOK_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' +MINIO_NOTIFY_WEBHOOK_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' +MINIO_NOTIFY_WEBHOOK_COMMENT (sentence) optionally add a comment to this setting +MINIO_NOTIFY_WEBHOOK_CLIENT_CERT (string) client cert for Webhook mTLS auth +MINIO_NOTIFY_WEBHOOK_CLIENT_KEY (string) client cert key for Webhook mTLS auth +``` + +```sh +$ mc admin config get myminio/ notify_webhook +notify_webhook:1 endpoint="" auth_token="" queue_limit="0" queue_dir="" client_cert="" client_key="" +``` + +Use `mc admin config set` command to update the configuration for the deployment. Here the endpoint is the server listening for webhook notifications. Save the settings and restart the MinIO server for changes to take effect. Note that the endpoint needs to be live and reachable when you restart your MinIO server. + +```sh +mc admin config set myminio notify_webhook:1 queue_limit="0" endpoint="http://localhost:3000" queue_dir="" +``` + +### Step 2: Enable Webhook bucket notification using MinIO client + +We will enable bucket event notification to trigger whenever a JPEG image is uploaded to `images` bucket on `myminio` server. Here ARN value is `arn:minio:sqs::1:webhook`. To learn more about ARN please follow [AWS ARN](http://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html) documentation. + +``` +mc mb myminio/images +mc mb myminio/images-thumbnail +mc event add myminio/images arn:minio:sqs::1:webhook --event put --suffix .jpg +``` + +Check if event notification is successfully configured by + +``` +mc event list myminio/images +``` + +You should get a response like this + +``` +arn:minio:sqs::1:webhook s3:ObjectCreated:* Filter: suffix=".jpg" +``` + +### Step 3: Test with Thumbnailer + +We used [Thumbnailer](https://github.com/minio/thumbnailer) to listen for MinIO notifications when a new JPEG file is uploaded (HTTP PUT). Triggered by a notification, Thumbnailer uploads a thumbnail of new image to MinIO server. To start with, download and install Thumbnailer. + +``` +git clone https://github.com/minio/thumbnailer/ +npm install +``` + +Then open the Thumbnailer config file at `config/webhook.json` and add the configuration for your MinIO server and then start Thumbnailer by + +``` +NODE_ENV=webhook node thumbnail-webhook.js +``` + +Thumbnailer starts running at `http://localhost:3000/`. Next, configure the MinIO server to send notifications to this URL (as mentioned in step 1) and use `mc` to set up bucket notifications (as mentioned in step 2). Then upload a JPEG image to MinIO server by + +``` +mc cp ~/images.jpg myminio/images +.../images.jpg: 8.31 KB / 8.31 KB ┃▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓┃ 100.00% 59.42 KB/s 0s +``` + +Wait a few moments, then check the bucket’s contents with mc ls — you will see a thumbnail appear. + +``` +mc ls myminio/images-thumbnail +[2017-02-08 11:39:40 IST] 992B images-thumbnail.jpg +``` + +## Publish MinIO events to NSQ + +Install an NSQ Daemon from [here](https://nsq.io/). Or use the following Docker +command for starting an nsq daemon: + +``` +podman run --rm -p 4150-4151:4150-4151 nsqio/nsq /nsqd +``` + +### Step 1: Add NSQ endpoint to MinIO + +MinIO supports persistent event store. The persistent store will backup events when the NSQ broker goes offline and replays it when the broker comes back online. The event store can be configured by setting the directory path in `queue_dir` field and the maximum limit of events in the queue_dir in `queue_limit` field. For eg, the `queue_dir` can be `/home/events` and `queue_limit` can be `1000`. By default, the `queue_limit` is set to 100000. + +To update the configuration, use `mc admin config get` command to get the current configuration for `notify_nsq`. + +``` +KEY: +notify_nsq[:name] publish bucket notifications to NSQ endpoints + +ARGS: +nsqd_address* (address) NSQ server address e.g. '127.0.0.1:4150' +topic* (string) NSQ topic +tls (on|off) set to 'on' to enable TLS +tls_skip_verify (on|off) trust server TLS without verification, defaults to "on" (verify) +queue_dir (path) staging dir for undelivered messages e.g. '/home/events' +queue_limit (number) maximum limit for undelivered messages, defaults to '100000' +comment (sentence) optionally add a comment to this setting +``` + +or environment variables + +``` +KEY: +notify_nsq[:name] publish bucket notifications to NSQ endpoints + +ARGS: +MINIO_NOTIFY_NSQ_ENABLE* (on|off) enable notify_nsq target, default is 'off' +MINIO_NOTIFY_NSQ_NSQD_ADDRESS* (address) NSQ server address e.g. '127.0.0.1:4150' +MINIO_NOTIFY_NSQ_TOPIC* (string) NSQ topic +MINIO_NOTIFY_NSQ_TLS (on|off) set to 'on' to enable TLS +MINIO_NOTIFY_NSQ_TLS_SKIP_VERIFY (on|off) trust server TLS without verification, defaults to "on" (verify) +MINIO_NOTIFY_NSQ_QUEUE_DIR (path) staging dir for undelivered messages e.g. '/home/events' +MINIO_NOTIFY_NSQ_QUEUE_LIMIT (number) maximum limit for undelivered messages, defaults to '100000' +MINIO_NOTIFY_NSQ_COMMENT (sentence) optionally add a comment to this setting +``` + +```sh +$ mc admin config get myminio/ notify_nsq +notify_nsq:1 nsqd_address="" queue_dir="" queue_limit="0" tls="off" tls_skip_verify="off" topic="" +``` + +Use `mc admin config set` command to update the configuration for the deployment. Restart the MinIO server to put the changes into effect. The server will print a line like `SQS ARNs: arn:minio:sqs::1:nsq` at start-up if there were no errors. + +```sh +mc admin config set myminio notify_nsq:1 nsqd_address="127.0.0.1:4150" queue_dir="" queue_limit="0" tls="off" tls_skip_verify="on" topic="minio" +``` + +Note that, you can add as many NSQ daemon endpoint configurations as needed by providing an identifier (like "1" in the example above) for the NSQ instance and an object of per-server configuration parameters. + +### Step 2: Enable NSQ bucket notification using MinIO client + +We will enable bucket event notification to trigger whenever a JPEG image is uploaded or deleted `images` bucket on `myminio` server. Here ARN value is `arn:minio:sqs::1:nsq`. + +``` +mc mb myminio/images +mc event add myminio/images arn:minio:sqs::1:nsq --suffix .jpg +mc event list myminio/images +arn:minio:sqs::1:nsq s3:ObjectCreated:*,s3:ObjectRemoved:* Filter: suffix=”.jpg” +``` + +### Step 3: Test on NSQ + +The simplest test is to download `nsq_tail` from [nsq github](https://github.com/nsqio/nsq/releases) + +``` +./nsq_tail -nsqd-tcp-address 127.0.0.1:4150 -topic minio +``` + +Open another terminal and upload a JPEG image into `images` bucket. + +``` +mc cp gopher.jpg myminio/images +``` + +You should receive the following event notification via NSQ once the upload completes. + +``` +{"EventName":"s3:ObjectCreated:Put","Key":"images/gopher.jpg","Records":[{"eventVersion":"2.0","eventSource":"minio:s3","awsRegion":"","eventTime":"2018-10-31T09:31:11Z","eventName":"s3:ObjectCreated:Put","userIdentity":{"principalId":"21EJ9HYV110O8NVX2VMS"},"requestParameters":{"sourceIPAddress":"10.1.1.1"},"responseElements":{"x-amz-request-id":"1562A792DAA53426","x-minio-origin-endpoint":"http://10.0.3.1:9000"},"s3":{"s3SchemaVersion":"1.0","configurationId":"Config","bucket":{"name":"images","ownerIdentity":{"principalId":"21EJ9HYV110O8NVX2VMS"},"arn":"arn:aws:s3:::images"},"object":{"key":"gopher.jpg","size":162023,"eTag":"5337769ffa594e742408ad3f30713cd7","contentType":"image/jpeg","userMetadata":{"content-type":"image/jpeg"},"versionId":"1","sequencer":"1562A792DAA53426"}},"source":{"host":"","port":"","userAgent":"MinIO (linux; amd64) minio-go/v6.0.8 mc/DEVELOPMENT.GOGET"}}]} +``` diff --git a/docs/bucket/quota/README.md b/docs/bucket/quota/README.md new file mode 100644 index 00000000..643a65cc --- /dev/null +++ b/docs/bucket/quota/README.md @@ -0,0 +1,30 @@ +# Bucket Quota Configuration Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) + +![quota](https://raw.githubusercontent.com/minio/minio/master/docs/bucket/quota/bucketquota.png) + +Buckets can be configured to have `Hard` quota - it disallows writes to the bucket after configured quota limit is reached. + +## Prerequisites + +- Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html#procedure). +- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart) + +## Set bucket quota configuration + +### Set a hard quota of 1GB for a bucket `mybucket` on MinIO object storage + +```sh +mc admin bucket quota myminio/mybucket --hard 1gb +``` + +### Verify the quota configured on `mybucket` on MinIO + +```sh +mc admin bucket quota myminio/mybucket +``` + +### Clear bucket quota configuration for `mybucket` on MinIO + +```sh +mc admin bucket quota myminio/mybucket --clear +``` diff --git a/docs/bucket/quota/bucketquota.png b/docs/bucket/quota/bucketquota.png new file mode 100644 index 00000000..0d24baab Binary files /dev/null and b/docs/bucket/quota/bucketquota.png differ diff --git a/docs/bucket/replication/DELETE_bucket_replication.png b/docs/bucket/replication/DELETE_bucket_replication.png new file mode 100644 index 00000000..19cdebe6 Binary files /dev/null and b/docs/bucket/replication/DELETE_bucket_replication.png differ diff --git a/docs/bucket/replication/DESIGN.md b/docs/bucket/replication/DESIGN.md new file mode 100644 index 00000000..2f3d55f0 --- /dev/null +++ b/docs/bucket/replication/DESIGN.md @@ -0,0 +1,160 @@ +# Bucket Replication Design [![slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) + +This document explains the design approach of server side bucket replication. If you're looking to get started with replication, we suggest you go through the [Bucket replication guide](https://github.com/minio/minio/blob/master/docs/bucket/replication/README.md) first. + +## Overview + +Replication relies on immutability provided by versioning to sync objects between the configured source and replication target. Replication results in the object data, metadata, last modification time and version ID all being identical between the source and target. Thus version ordering is automatically guaranteed on the source and target clusters. + +### Replication of object version and metadata + +If an object meets replication rules as set in the replication configuration, `X-Amz-Replication-Status` is first set to `PENDING` as the PUT operation completes and replication is queued (unless synchronous replication is in place). After replication is performed, the metadata on the source object version changes to `COMPLETED` or `FAILED` depending on whether replication succeeded. The object version on the target shows `X-Amz-Replication-Status` of `REPLICA` + +All replication failures are picked up by the scanner which runs at a one minute frequency, each time scanning up to a sixteenth of the namespace. Object versions marked `PENDING` or `FAILED` are re-queued for replication. + +Replication speed depends on the cluster load, number of objects in the object store as well as storage speed. In addition, any bandwidth limits set via `mc admin bucket remote add` could also contribute to replication speed. The number of workers used for replication defaults to 100. Based on network bandwidth and system load, the number of workers used in replication can be configured using `mc admin config set alias api` to set the `replication_workers`. The prometheus metrics exposed by MinIO can be used to plan resource allocation and bandwidth management to optimize replication speed. + +If synchronous replication is configured above, replication is attempted right away prior to returning the PUT object response. In the event that the replication target is down, the `X-Amz-Replication-Status` is marked as `FAILED` and resynced with target when the scanner runs again. + +Any metadata changes on the source object version, such as metadata updates via PutObjectTagging, PutObjectRetention, PutObjectLegalHold and COPY api are replicated in a similar manner to target version, with the `X-Amz-Replication-Status` again cycling through the same states. + +The description above details one way replication from source to target w.r.t incoming object uploads and metadata changes to source object version. If active-active replication is configured, any incoming uploads and metadata changes to versions created on the target, will sync back to the source and be marked as `REPLICA` on the source. AWS, as well as MinIO do not by default sync metadata changes on a object version marked `REPLICA` back to source. This requires a setting in the replication configuration called [replica modification sync](https://aws.amazon.com/about-aws/whats-new/2020/12/amazon-s3-replication-adds-support-two-way-replication/). + +For active-active replication, automatic failover occurs on `GET/HEAD` operations if object or object version requested qualifies for replication and is missing on one site, but present on the other. This allows the applications to take full advantage of two-way replication even before the two sites get fully synced. + +In the case of multi destination replication, the replication status shows `COMPLETED` only after the replication operation succeeds on each of the targets specified in the replication configuration. If multiple targets are configured to use active-active replication and multi destination replication, the administrator should ensure that the replication features enabled (such as replica metadata sync, delete marker replication etc) are identical to avoid asymmetric state. This is because all replication activity is inherently a one-way operation from source to target, irrespective of the number of targets. + +### Replication of DeleteMarker and versioned Delete + +MinIO allows DeleteMarker replication and versioned delete replication by setting `--replicate delete,delete-marker` while setting up replication configuration using `mc replicate add`. The MinIO implementation is based on V2 configuration, however it has been extended to allow both DeleteMarker replication and replication of versioned deletes with the `DeleteMarkerReplication` and `DeleteReplication` fields in the replication configuration. By default, this is set to `Disabled` unless the user specifies it while adding a replication rule. + +Similar to object version replication, DeleteMarker replication also cycles through `PENDING` to `COMPLETED` or `FAILED` states for the `X-Amz-Replication-Status` on the source when a delete marker is set (i.e. performing `mc rm` on an object without specifying a version).After replication syncs the delete marker on the target, the DeleteMarker on the target shows `X-Amz-Replication-Status` of `REPLICA`. The status of DeleteMarker replication is returned by `X-Minio-Replication-DeleteMarker-Status` header on `HEAD/GET` calls for the delete marker version in question - i.e with `mc stat --version-id dm-version-id` + +It must be noted that if active-active replication is set up with delete marker replication, there is potential for duplicate delete markers to be created if both source and target concurrently set a Delete Marker or if one/both of the clusters went down at tandem before the replication event was synced.This is an unavoidable side-effect in active-active replication caused by allowing delete markers set on a object version with `REPLICA` status back to source. + +In the case of versioned deletes a.k.a permanent delete of a version by doing a `mc rm --version-id` on a object, replication implementation marks a object version permanently deleted as `PENDING` purge and deletes the version from source after syncing to the target and ensuring target version is deleted. The delete marker being deleted or object version being deleted will still be visible on listing with `mc ls --versions` until the sync is completed. Objects marked as deleted will not be accessible via `GET` or `HEAD` requests and would return a http response code of `405`. The status of versioned delete replication on the source can be queried by `HEAD` request on the delete marker versionID or object versionID in question. +An additional header `X-Minio-Replication-Delete-Status` is returned which would show `PENDING` or `FAILED` status if the replication is still not caught up. + +Note that synchronous replication, i.e. when remote target is configured with --sync mode in `mc admin bucket remote add` does not apply to `DELETE` operations. The version being deleted on the source cluster needs to maintain state and ensure that the operation is mirrored to the target cluster prior to completing on the source object version. Since this needs to account for the target cluster availability and the need to serialize concurrent DELETE operations on different versions of the same object during multi DELETE operations, the current implementation queues the `DELETE` operations in both sync and async modes. + +### Existing object replication + +Existing object replication works similar to regular replication. Objects qualifying for existing object replication are detected when scanner runs, and will be replicated if existing object replication is enabled and applicable replication rules are satisfied. Because replication depends on the immutability of versions, only pre-existing objects created while versioning was enabled can be replicated. Even if replication rules are disabled and re-enabled later, the objects created during the interim will be synced as the scanner queues them. For saving iops, objects qualifying for +existing object replication are not marked as `PENDING` prior to replication. + +Note that objects with `null` versions, i.e. objects created prior to enabling versioning break the immutability guarantees provided by versioning. When existing object replication is enabled, these objects will be replicated as `null` versions to the remote targets provided they are not present on the target or if `null` version of object on source is newer than the `null` version of object on target. + +If the remote site is fully lost and objects previously replicated need to be re-synced, the `mc replicate resync start` command with optional flag of `--older-than` needs to be used to trigger re-syncing of previously replicated objects. This command generates a ResetID which is a unique UUID saved to the remote target config along with the applicable date(defaults to time of initiating the reset). All objects created prior to this date are eligible for re-replication if existing object replication is enabled for the replication rule the object satisfies. At the time of completion of replication, `x-minio-internal-replication-reset-arn:` is set in the metadata with the timestamp of replication and ResetID. For saving iops, the objects which are re-replicated are not first set to `PENDING` state. + +This is a slower operation that does not use replication queues and is designed to walk the namespace and replicate objects one at a time so as not to impede server load. Ideally, resync should not be initiated for multiple buckets simultaneously - progress of the syncing can be monitored by looking at `mc replicate resync status alias/bucket --remote-bucket `. In the event that resync operation failed to replicate some versions, they would be picked up by the healing mechanism in-built as part of the scanner. If the resync operation reports a failed status or in the event of a cluster restart while resync is in progress, a fresh `resync start` can be issued - this will replicate previously unsynced content at the cost of additional overhead in additional metadata updates. + +### Multi destination replication + +The replication design for multiple sites works in a similar manner as described above for two site scenario. However there are some +important exceptions. + +Replication status on the source cluster will be marked as `COMPLETED` only after replication is completed on all targets. If one or more targets failed replication, the replication status is reflected as `PENDING`. + +If 3 or more targets are participating in active-active replication, the replication configuration for replica metadata sync, delete marker replication and delete replication should match to avoid inconsistent picture between the clusters. It is not recommended to turn on asymmetric replication - for e.g. if three sites A,B,C are participating in replication, it would be better to avoid replication setups like A -> [B, C], B -> A. In this particular example, an object uploaded to A will be replicated to B,C. If replica metadata sync is turned on in site B, any metadata updates on a replica version made in B would reflect in A, but not in C. + +### Internal metadata for replication + +`xl.meta` that is in use for [versioning](https://github.com/minio/minio/blob/master/docs/bucket/versioning/DESIGN.md) has additional metadata for replication of objects,delete markers and versioned deletes. + +### Metadata for object replication - on source + +``` +... + "MetaSys": { + "x-minio-internal-inline-data": "dHJ1ZQ==", + "x-minio-internal-replication-status": "YXJuOm1pbmlvOnJlcGxpY2F0aW9uOjo2YjdmYzFlMS0wNmU4LTQxMTUtYjYxNy00YTgzZGIyODhmNTM6YnVja2V0PUNPTVBMRVRFRDthcm46bWluaW86cmVwbGljYXRpb246OmI5MGYxZWEzLWMzYWQtNDEyMy1iYWE2LWZjMDZhYmEyMjA2MjpidWNrZXQ9Q09NUExFVEVEOw==", + "x-minio-internal-replication-timestamp": "MjAyMS0wOS0xN1QwMTo0MzozOC40MDQwMDA0ODNa", + "x-minio-internal-tier-free-versionID": "OWZlZjk5N2QtMjMzZi00N2U3LTlkZmMtNWYxNzc3NzdlZTM2" + }, + "MetaUsr": { + "X-Amz-Replication-Status": "COMPLETED", + "content-type": "application/octet-stream", + "etag": "8315e643ed6a5d7c9962fc0a8ef9c11f" + }, +... +``` + +### Metadata for object replication - on target + +``` +... + "MetaSys": { + "x-minio-internal-inline-data": "dHJ1ZQ==", + "x-minio-internal-replica-status": "UkVQTElDQQ==", + "x-minio-internal-replica-timestamp": "MjAyMS0wOS0xN1QwMTo0MzozOC4zODg5ODU4ODRa" + }, + "MetaUsr": { + "X-Amz-Replication-Status": "REPLICA", + "content-type": "application/octet-stream", + "etag": "8315e643ed6a5d7c9962fc0a8ef9c11f", + "x-amz-storage-class": "STANDARD" + }, +... +``` + +### Additional replication metadata for DeleteMarker + +``` +... + { + "DelObj": { + "ID": "u8H5pYQFRMKgkIgkpSKIkQ==", + "MTime": 1631843124147668389, + "MetaSys": { + "x-minio-internal-replication-status": "YXJuOm1pbmlvOnJlcGxpY2F0aW9uOjpiOTBmMWVhMy1jM2FkLTQxMjMtYmFhNi1mYzA2YWJhMjIwNjI6YnVja2V0PUNPTVBMRVRFRDthcm46bWluaW86cmVwbGljYXRpb246OjZiN2ZjMWUxLTA2ZTgtNDExNS1iNjE3LTRhODNkYjI4OGY1MzpidWNrZXQ9Q09NUExFVEVEOw==", + "x-minio-internal-replication-timestamp": "U3VuLCAzMSBEZWMgMDAwMCAxOTowMzo1OCBHTVQ=" + } + }, + "Type": 2 +} +``` + +### Additional replication metadata for versioned delete + +``` +{ + "DelObj": { + "ID": "u8H5pYQFRMKgkIgkpSKIkQ==", + "MTime": 1631843124147668389, + "MetaSys": { + "purgestatus": "YXJuOm1pbmlvOnJlcGxpY2F0aW9uOjpiOTBmMWVhMy1jM2FkLTQxMjMtYmFhNi1mYzA2YWJhMjIwNjI6YnVja2V0PUNPTVBMRVRFO2FybjptaW5pbzpyZXBsaWNhdGlvbjo6NmI3ZmMxZTEtMDZlOC00MTE1LWI2MTctNGE4M2RiMjg4ZjUzOmJ1Y2tldD1GQUlMRUQ7", + "x-minio-internal-replication-status": "YXJuOm1pbmlvOnJlcGxpY2F0aW9uOjpiOTBmMWVhMy1jM2FkLTQxMjMtYmFhNi1mYzA2YWJhMjIwNjI6YnVja2V0PTthcm46bWluaW86cmVwbGljYXRpb246OjZiN2ZjMWUxLTA2ZTgtNDExNS1iNjE3LTRhODNkYjI4OGY1MzpidWNrZXQ9Ow==", + "x-minio-internal-replication-timestamp": "U3VuLCAzMSBEZWMgMDAwMCAxOTowMzo1OCBHTVQ=" + } + }, + "Type": 2 +} +``` + +### Additional Metadata for object replication resync - on source + +``` +... + "MetaSys": { + ... + "x-minio-internal-replication-reset-arn:minio:replication::af470089-d354-4473-934c-9e1f52f6da89:bucket": "TW9uLCAwNyBGZWIgMjAyMiAyMDowMzo0MCBHTVQ7ZGMxMWQzNDgtMTAwMS00ODA3LWFhNjEtOGY2MmFiNWQ5ZjU2", + ... + }, +... +``` + +### Additional Metadata for resync replication of delete-markers - on source + +``` +... + "MetaSys": { + "x-minio-internal-replication-reset-arn:minio:replication::af470089-d354-4473-934c-9e1f52f6da89:bucket": "TW9uLCAwNyBGZWIgMjAyMiAyMDowMzo0MCBHTVQ7ZGMxMWQzNDgtMTAwMS00ODA3LWFhNjEtOGY2MmFiNWQ5ZjU2", + ... + } +... +``` + +## Explore Further + +- [MinIO Bucket Versioning Implementation](https://docs.min.io/community/minio-object-store/administration/object-management/object-versioning.html) +- [MinIO Client Quickstart Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart) diff --git a/docs/bucket/replication/HEAD_bucket_replication.png b/docs/bucket/replication/HEAD_bucket_replication.png new file mode 100644 index 00000000..d544fdec Binary files /dev/null and b/docs/bucket/replication/HEAD_bucket_replication.png differ diff --git a/docs/bucket/replication/PUT_bucket_replication.png b/docs/bucket/replication/PUT_bucket_replication.png new file mode 100644 index 00000000..d662cc79 Binary files /dev/null and b/docs/bucket/replication/PUT_bucket_replication.png differ diff --git a/docs/bucket/replication/README.md b/docs/bucket/replication/README.md new file mode 100644 index 00000000..829d98f0 --- /dev/null +++ b/docs/bucket/replication/README.md @@ -0,0 +1,281 @@ +# Bucket Replication Guide [![slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) + +Bucket replication is designed to replicate selected objects in a bucket to a destination bucket. + +The contents of this page have been migrated to the new [MinIO Documentation: Bucket Replication](https://docs.min.io/community/minio-object-store/administration/bucket-replication.html) page. The [Bucket Replication](https://docs.min.io/community/minio-object-store/administration/bucket-replication/bucket-replication-requirements.html) page references dedicated tutorials for configuring one-way "Active-Passive" and two-way "Active-Active" bucket replication. + +To replicate objects in a bucket to a destination bucket on a target site either in the same cluster or a different cluster, start by enabling [versioning](https://docs.min.io/community/minio-object-store/administration/object-management/object-versioning.html) for both source and destination buckets. Finally, the target site and the destination bucket need to be configured on the source MinIO server. + +## Highlights + +- Supports source and destination buckets to have the same name unlike AWS S3, addresses variety of use-cases such as *Splunk*, *Veeam* site to site DR. +- Supports object locking/retention across source and destination buckets natively out of the box, unlike AWS S3. +- Simpler implementation than [AWS S3 Bucket Replication Config](https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html) with requirements such as IAM Role, AccessControlTranslation, Metrics and SourceSelectionCriteria are not needed with MinIO. +- Active-Active replication +- Multi destination replication + +## How to use? + +Ensure that versioning is enabled on the source and target buckets with `mc version` command. If object locking is required, the buckets should have been created with `mc mb --with-lock` + +The user setting up replication needs *s3:GetReplicationConfiguration* and *s3:GetBucketVersioning* permission on the source cluster. We do not recommend running root credentials/super admin with replication, instead create a dedicated user. The access credentials used at the destination requires *s3:ReplicateObject* permission. + +The following minimal permission policy is needed by admin user setting up replication on the `source`: + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "admin:SetBucketTarget", + "admin:GetBucketTarget" + ], + "Effect": "Allow", + "Sid": "" + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetReplicationConfiguration", + "s3:PutReplicationConfiguration", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:GetBucketLocation", + "s3:GetBucketVersioning" + ], + "Resource": [ + "arn:aws:s3:::srcbucket" + ] + } + ] +} +``` + +The access key provided for the replication *target* cluster should have these minimal permissions: + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:GetReplicationConfiguration", + "s3:ListBucket", + "s3:ListBucketMultipartUploads", + "s3:GetBucketLocation", + "s3:GetBucketVersioning", + "s3:GetBucketObjectLockConfiguration" + ], + "Resource": [ + "arn:aws:s3:::destbucket" + ] + }, + { + "Effect": "Allow", + "Action": [ + "s3:GetReplicationConfiguration", + "s3:ReplicateTags", + "s3:AbortMultipartUpload", + "s3:GetObject", + "s3:GetObjectVersion", + "s3:GetObjectVersionTagging", + "s3:PutObject", + "s3:DeleteObject", + "s3:ReplicateObject", + "s3:ReplicateDelete" + ], + "Resource": [ + "arn:aws:s3:::destbucket/*" + ] + } + ] +} +``` + +Please note that the permissions required by the admin user on the target cluster can be more fine grained to exclude permissions like "s3:ReplicateDelete", "s3:GetBucketObjectLockConfiguration" etc depending on whether delete replication rules are set up or if object locking is disabled on `destbucket`. The above policies assume that replication of objects, tags and delete marker replication are all enabled on object lock enabled buckets. A sample script to setup replication is provided [here](https://github.com/minio/minio/blob/master/docs/bucket/replication/setup_replication.sh) + +To set up replication from a source bucket `srcbucket` on myminio cluster to a bucket `destbucket` on the target minio cluster with endpoint https://replica-endpoint:9000, use: +``` +mc replicate add myminio/srcbucket --priority 1 --remote-bucket https://accessKey:secretKey@replica-endpoint:9000/destbucket +Replication configuration applied successfully to myminio/srcbucket. +``` +Internally, this creates an ARN for the remote target associating the remote bucket as a replication target to the srcbucket on myminio.By default, if --replicate flag is not specified, replication of delete marker, permanent deletes, existing object replication and replica modification sync are all enabled. If you are using older mc versions, the ARN needs to be generated as a separate step before adding a replication rule. + +> NOTE: If you are using a mc version below `RELEASE.2022-12-24T15-21-38Z`, the --remote-bucket flag needs an ARN generated by `mc admin bucket remote add` command. For mc versions RELEASE.2021-09-02T09-21-27Z and older, the remote target ARN needs to be passed in the --arn flag and actual remote bucket name in --remote-bucket flag of `mc replicate add`. For example, in older releases of mc replication configuration used to be added with: + +``` +mc admin bucket remote add myminio/srcbucket https://accessKey:secretKey@replica-endpoint:9000/destbucket --service replication --region us-east-1 +Remote ARN = 'arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket' + +mc replicate add myminio/srcbucket/Tax --priority 1 --remote-bucket destbucket --remote-bucket "arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket" --tags "Year=2019&Company=AcmeCorp" --storage-class "STANDARD" --replicate "delete,delete-marker" +Replication configuration applied successfully to myminio/srcbucket. +``` + +The replication configuration generated has the following format and can be exported with `mc replicate export` command: + +```json +{ + "Role" :"", + "Rules": [ + { + "Status": "Enabled", + "Priority": 1, + "DeleteMarkerReplication": { "Status": "Disabled" }, + "DeleteReplication": { "Status": "Disabled" }, + "Filter" : { + "And": { + "Prefix": "Tax", + "Tags": [ + { + "Key": "Year", + "Value": "2019" + }, + { + "Key": "Company", + "Value": "AcmeCorp" + } + ] + } + }, + "Destination": { + "Bucket": "arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket", + "StorageClass": "STANDARD" + }, + "SourceSelectionCriteria": { + "ReplicaModifications": { + "Status": "Enabled" + } + } + } + ] +} +``` + +The replication configuration follows [AWS S3 Spec](https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html). Any objects uploaded to the source bucket that meet replication criteria will now be automatically replicated by the MinIO server to the remote destination bucket. Replication can be disabled at any time by disabling specific rules in the configuration or deleting the replication configuration entirely. + +When object locking is used in conjunction with replication, both source and destination buckets needs to have [object locking](https://docs.min.io/community/minio-object-store/administration/object-management/object-retention.html) enabled. Similarly objects encrypted on the server side, will be replicated if destination also supports encryption. + +Replication status can be seen in the metadata on the source and destination objects. On the source side, the `X-Amz-Replication-Status` changes from `PENDING` to `COMPLETED` or `FAILED` after replication attempt either succeeded or failed respectively. On the destination side, a `X-Amz-Replication-Status` status of `REPLICA` indicates that the object was replicated successfully. Any replication failures are automatically re-attempted during a periodic disk scanner cycle. + +To perform bi-directional replication, repeat the above process on the target site - this time setting the source bucket as the replication target. It is recommended that replication be run in a system with at least two CPU's available to the process, so that replication can run in its own thread. + +![put](https://raw.githubusercontent.com/minio/minio/master/docs/bucket/replication/PUT_bucket_replication.png) + +![head](https://raw.githubusercontent.com/minio/minio/master/docs/bucket/replication/HEAD_bucket_replication.png) + +## Replica Modification sync + +If bi-directional replication is set up between two clusters, any metadata update on the REPLICA object is by default reflected back in the source object when `ReplicaModifications` status in the `SourceSelectionCriteria` is `Enabled`. In MinIO, this is enabled by default. If a metadata update is performed on the "REPLICA" object, its `X-Amz-Replication-Status` will change from `PENDING` to `COMPLETE` or `FAILED`, and the source object version will show `X-Amz-Replication-Status` of `REPLICA` once the replication operation is complete. + +The replication configuration in use on a bucket can be viewed using the `mc replicate export alias/bucket` command. + +To disable replica metadata modification syncing, use `mc replicate edit` with the --replicate flag. + +``` +mc replicate edit alias/bucket --id xyz.id --replicate "delete,delete-marker" +``` + +To re-enable replica metadata modification syncing, + +``` +mc replicate edit alias/bucket --id xyz.id --replicate "delete,delete-marker,replica-metadata-sync" +``` + +## MinIO Extension + +### Replicating Deletes + +Delete marker replication is allowed in [AWS V1 Configuration](https://aws.amazon.com/blogs/storage/managing-delete-marker-replication-in-amazon-s3/) but not in V2 configuration. The MinIO implementation above is based on V2 configuration, however it has been extended to allow both DeleteMarker replication and replication of versioned deletes with the `DeleteMarkerReplication` and `DeleteReplication` fields in the replication configuration above. By default, this is set to `Disabled` unless the user specifies it while adding a replication rule. + +When an object is deleted from the source bucket, the corresponding replica version will be marked deleted if delete marker replication is enabled in the replication configuration. Replication of deletes that specify a version id (a.k.a hard deletes) can be enabled by setting the `DeleteReplication` status to enabled in the replication configuration. This is a MinIO specific extension that can be enabled using the `mc replicate add` or `mc replicate edit` command with the --replicate "delete" flag. + +Note that due to this extension behavior, AWS SDK's may not support the extension functionality pertaining to replicating versioned deletes. + +Note that just like with [AWS](https://docs.aws.amazon.com/AmazonS3/latest/userguide/delete-marker-replication.html), Delete marker replication is disallowed in MinIO when the replication rule has tags. + +To add a replication rule allowing both delete marker replication, versioned delete replication or both specify the --replicate flag with comma separated values as in the example below. + +Additional permission of "s3:ReplicateDelete" action would need to be specified on the access key configured for the target cluster if Delete Marker replication or versioned delete replication is enabled. + +``` +mc replicate add myminio/srcbucket/Tax --priority 1 --remote-bucket `remote-target` --tags "Year=2019&Company=AcmeCorp" --storage-class "STANDARD" --replicate "delete,delete-marker" +Replication configuration applied successfully to myminio/srcbucket. +``` + +> NOTE: In mc versions `RELEASE.2022-12-24T15-21-38Z` and above `remote-target` should be of the format `https://accessKey:secretKey@replica-endpoint:9000/destbucket` which earlier used to be set during `mc admin bucket remote add`. For older releases, use the arn generated with `mc admin bucket remote add` command - e.g."arn:minio:replication:us-east-1:c5be6b16-769d-432a-9ef1-4567081f3566:destbucket" as the `remote-target`. + +Also note that `mc` version `RELEASE.2021-09-02T09-21-27Z` or older supports only a single remote target per bucket. To take advantage of multiple destination replication, use the latest version of `mc` + +Status of delete marker replication can be viewed by doing a GET/HEAD on the object version - it will return a `X-Minio-Replication-DeleteMarker-Status` header and http response code of `405`. In the case of permanent deletes, if the delete replication is pending or failed to propagate to the target cluster, GET/HEAD will return additional `X-Minio-Replication-Delete-Status` header and a http response code of `405`. + +![delete](https://raw.githubusercontent.com/minio/minio/master/docs/bucket/replication/DELETE_bucket_replication.png) + +The status of replication can be monitored by configuring event notifications on the source and target buckets using `mc event add`.On the source side, the `s3:PutObject`, `s3:Replication:OperationCompletedReplication` and `s3:Replication:OperationFailedReplication` events show the status of replication in the `X-Amz-Replication-Status` metadata. + +On the target bucket, `s3:PutObject` event shows `X-Amz-Replication-Status` status of `REPLICA` in the metadata. Additional metrics to monitor backlog state for the purpose of bandwidth management and resource allocation are exposed via Prometheus - see for more details. + +### Sync/Async Replication + +By default, replication is completed asynchronously. If synchronous replication is desired, set the --sync flag while adding a +remote replication target using the `mc admin bucket remote add` command. For mc releases on or after `RELEASE.2022-12-24T15-21-38Z`, the +--sync, --health-check and --bandwidth flags can be specified in `mc replicate add|update` command + +``` + mc admin bucket remote add myminio/srcbucket https://accessKey:secretKey@replica-endpoint:9000/destbucket --service replication --region us-east-1 --sync --healthcheck-seconds 100 +``` + +### Existing object replication + +Existing object replication as detailed [here](https://aws.amazon.com/blogs/storage/replicating-existing-objects-between-s3-buckets/) can be enabled by passing `existing-objects` as a value to `--replicate` flag while adding or editing a replication rule. + +Once existing object replication is enabled, all objects or object prefixes that satisfy the replication rules and were created prior to adding replication configuration OR while replication rules were disabled will be synced to the target cluster. Depending on the number of previously existing objects, the existing objects that are now eligible to be replicated will eventually be synced to the target cluster as the scanner schedules them. This may be slower depending on the load on the cluster, latency and size of the namespace. + +In the rare event that target DR site is entirely lost and previously replicated objects to the DR cluster need to be re-replicated, `mc replicate resync start alias/bucket --remote-bucket ` can be used to initiate a reset. This would initiate a re-sync between the two clusters by walking the bucket namespace and replicating eligible objects that satisfy the existing objects replication rule specified in the replication config. The status of the resync operation can be viewed with `mc replicate resync status alias/bucket --remote-bucket `. + +Note that ExistingObjectReplication needs to be enabled in the config via `mc replicate [add|edit]` by passing `existing-objects` as one of the values to `--replicate` flag. Only those objects meeting replication rules and having existing object replication enabled will be re-synced. + +### Multi destination replication + +Replication from a source bucket to multiple destination buckets is supported. For each of the targets, repeat the steps to configure a remote target ARN and add replication rules to the source bucket's replication config. + +Note that on the source side, the `X-Amz-Replication-Status` changes from `PENDING` to `COMPLETED` after replication succeeds to each of the targets. On the destination side, a `X-Amz-Replication-Status` status of `REPLICA` indicates that the object was replicated successfully. Any replication failures are automatically re-attempted during a periodic disk scanner cycle. + +### Interaction with extended Bucket Versioning configuration + +When Bucket Versioning with excluded prefixes are configured objects matching these prefixes are excluded from being versioned and replicated. + +``` + + Enabled + true + + + app1-jobs/*/_temporary/ + + + app2-jobs/*/_magic/ + + + + +``` + +In the above sample config, objects under prefixes matching any of the `ExcludedPrefixes` glob patterns will neither be versioned nor replicated. + +### SSE-C Encryption + +MinIO does not support SSE-C encrypted objects on replicated buckets, any application uploading SSE-C encrypted objects will be rejected with an error on replicated buckets. + +#### Rationale + +- SSE-C requires application to remember the keys for all GET/PUT operations, any unfortunate loss of keys would automatically mean the objects cannot be accessed anymore. +- SSE-C is hardly adopted by most widely used applications, applications prefer server to manage the keys via SSE-KMS or SSE-S3. +- MinIO recommends applications to use SSE-KMS, SSE-S3 for simpler, safer and robust encryption mechanism for replicated buckets. + +## Explore Further + +- [MinIO Bucket Replication Design](https://github.com/minio/minio/blob/master/docs/bucket/replication/DESIGN.md) +- [MinIO Bucket Versioning Implementation](https://docs.min.io/community/minio-object-store/administration/object-management/object-retention.html) +- [MinIO Client Quickstart Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart) diff --git a/docs/bucket/replication/delete-replication.sh b/docs/bucket/replication/delete-replication.sh new file mode 100755 index 00000000..c91c70f2 --- /dev/null +++ b/docs/bucket/replication/delete-replication.sh @@ -0,0 +1,150 @@ +#!/usr/bin/env bash + +echo "Running $0" + +if [ -n "$TEST_DEBUG" ]; then + set -x +fi + +trap 'catch $LINENO' ERR + +# shellcheck disable=SC2120 +catch() { + if [ $# -ne 0 ]; then + echo "error on line $1" + echo "dc1 server logs =========" + cat /tmp/dc1.log + echo "dc2 server logs =========" + cat /tmp/dc2.log + fi + + echo "Cleaning up instances of MinIO" + set +e + pkill minio + pkill mc + rm -rf /tmp/xl/ + if [ $# -ne 0 ]; then + exit $# + fi +} + +catch + +set -e +export MINIO_CI_CD=1 +export MINIO_BROWSER=off +export MINIO_ROOT_USER="minio" +export MINIO_ROOT_PASSWORD="minio123" +export MINIO_KMS_AUTO_ENCRYPTION=off +export MINIO_PROMETHEUS_AUTH_TYPE=public +export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= +unset MINIO_KMS_KES_CERT_FILE +unset MINIO_KMS_KES_KEY_FILE +unset MINIO_KMS_KES_ENDPOINT +unset MINIO_KMS_KES_KEY_NAME + +if [ ! -f ./mc ]; then + wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc +fi + +mkdir -p /tmp/xl/1/ /tmp/xl/2/ + +export MINIO_KMS_SECRET_KEY="my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw=" +export MINIO_ROOT_USER="minioadmin" +export MINIO_ROOT_PASSWORD="minioadmin" + +./minio server --address ":9001" /tmp/xl/1/{1...4}/ 2>&1 >/tmp/dc1.log & +pid1=$! +./minio server --address ":9002" /tmp/xl/2/{1...4}/ 2>&1 >/tmp/dc2.log & +pid2=$! + +sleep 3 + +export MC_HOST_myminio1=http://minioadmin:minioadmin@localhost:9001 +export MC_HOST_myminio2=http://minioadmin:minioadmin@localhost:9002 + +./mc ready myminio1 +./mc ready myminio2 + +./mc mb myminio1/testbucket/ +./mc version enable myminio1/testbucket/ +./mc mb myminio2/testbucket/ +./mc version enable myminio2/testbucket/ + +./mc replicate add myminio1/testbucket --remote-bucket http://minioadmin:minioadmin@localhost:9002/testbucket/ --priority 1 + +# Test replication of delete markers and permanent deletes + +./mc cp README.md myminio1/testbucket/dir/file +./mc cp README.md myminio1/testbucket/dir/file + +sleep 1s + +echo "=== myminio1" +./mc ls --versions myminio1/testbucket/dir/file + +echo "=== myminio2" +./mc ls --versions myminio2/testbucket/dir/file + +versionId="$(./mc ls --json --versions myminio1/testbucket/dir/ | tail -n1 | jq -r .versionId)" + +export AWS_ACCESS_KEY_ID=minioadmin +export AWS_SECRET_ACCESS_KEY=minioadmin +export AWS_REGION=us-east-1 + +aws s3api --endpoint-url http://localhost:9001 delete-object --bucket testbucket --key dir/file --version-id "$versionId" + +./mc ls -r --versions myminio1/testbucket >/tmp/myminio1.txt +./mc ls -r --versions myminio2/testbucket >/tmp/myminio2.txt + +out=$(diff -qpruN /tmp/myminio1.txt /tmp/myminio2.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after replication: $out" + exit 1 +fi + +./mc rm myminio1/testbucket/dir/file +sleep 1s + +./mc ls -r --versions myminio1/testbucket >/tmp/myminio1.txt +./mc ls -r --versions myminio2/testbucket >/tmp/myminio2.txt + +out=$(diff -qpruN /tmp/myminio1.txt /tmp/myminio2.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after replication: $out" + exit 1 +fi + +# Test listing of non replicated permanent deletes + +set -x + +./mc mb myminio1/foobucket/ myminio2/foobucket/ --with-versioning +./mc replicate add myminio1/foobucket --remote-bucket http://minioadmin:minioadmin@localhost:9002/foobucket/ --priority 1 +./mc cp README.md myminio1/foobucket/dir/file + +versionId="$(./mc ls --json --versions myminio1/foobucket/dir/ | jq -r .versionId)" + +kill ${pid2} && wait ${pid2} || true + +aws s3api --endpoint-url http://localhost:9001 delete-object --bucket foobucket --key dir/file --version-id "$versionId" + +out="$(./mc ls myminio1/foobucket/dir/)" +if [ "$out" != "" ]; then + echo "BUG: non versioned listing should not show pending/failed replicated delete:" + echo "$out" + exit 1 +fi + +out="$(./mc ls --versions myminio1/foobucket/dir/)" +if [ "$out" != "" ]; then + echo "BUG: versioned listing should not show pending/failed replicated deletes:" + echo "$out" + exit 1 +fi + +echo "Success" +catch diff --git a/docs/bucket/replication/setup_2site_existing_replication.sh b/docs/bucket/replication/setup_2site_existing_replication.sh new file mode 100755 index 00000000..d7146201 --- /dev/null +++ b/docs/bucket/replication/setup_2site_existing_replication.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash + +echo "Running $0" + +set -x + +trap 'catch $LINENO' ERR + +# shellcheck disable=SC2120 +catch() { + if [ $# -ne 0 ]; then + echo "error on line $1" + for site in sitea siteb; do + echo "$site server logs =========" + cat "/tmp/${site}_1.log" + echo "===========================" + cat "/tmp/${site}_2.log" + done + fi + + echo "Cleaning up instances of MinIO" + pkill minio + pkill -9 minio + rm -rf /tmp/multisitea + rm -rf /tmp/multisiteb + rm -rf /tmp/data + if [ $# -ne 0 ]; then + exit $# + fi +} + +catch + +set -e +export MINIO_CI_CD=1 +export MINIO_BROWSER=off +export MINIO_ROOT_USER="minio" +export MINIO_ROOT_PASSWORD="minio123" +export MINIO_KMS_AUTO_ENCRYPTION=off +export MINIO_PROMETHEUS_AUTH_TYPE=public +export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= +unset MINIO_KMS_KES_CERT_FILE +unset MINIO_KMS_KES_KEY_FILE +unset MINIO_KMS_KES_ENDPOINT +unset MINIO_KMS_KES_KEY_NAME + +if [ ! -f ./mc ]; then + wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc +fi + +minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 & +minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 & + +minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 & +minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 & + +export MC_HOST_sitea=http://minio:minio123@127.0.0.1:9001 +export MC_HOST_siteb=http://minio:minio123@127.0.0.1:9004 + +./mc ready sitea +./mc ready siteb + +./mc mb sitea/bucket + +## Create 100 files +mkdir -p /tmp/data +for i in $(seq 1 10); do + echo "T" >/tmp/data/file_${i}.txt +done + +./mc mirror /tmp/data sitea/bucket/ +./mc version enable sitea/bucket + +./mc cp /tmp/data/file_1.txt sitea/bucket/marker +./mc rm sitea/bucket/marker + +./mc mb siteb/bucket/ +./mc version enable siteb/bucket/ + +echo "adding replication rule for site a -> site b" +./mc replicate add sitea/bucket/ \ + --remote-bucket http://minio:minio123@127.0.0.1:9004/bucket + +remote_arn=$(./mc replicate ls sitea/bucket --json | jq -r .rule.Destination.Bucket) +sleep 1 + +./mc replicate resync start sitea/bucket/ --remote-bucket "${remote_arn}" +sleep 30s ## sleep for 30s idea is that we give 300ms per object. + +./mc ls -r --versions sitea/bucket >/tmp/sitea.txt +./mc ls -r --versions siteb/bucket >/tmp/siteb.txt + +out=$(diff -qpruN /tmp/sitea.txt /tmp/siteb.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after replication: $out" + exit 1 +fi + +./mc cp /tmp/data/file_1.txt sitea/bucket/marker_new +./mc rm sitea/bucket/marker_new + +sleep 12s ## sleep for 12s idea is that we give 100ms per object. + +./mc ls -r --versions sitea/bucket >/tmp/sitea.txt +./mc ls -r --versions siteb/bucket >/tmp/siteb.txt + +out=$(diff -qpruN /tmp/sitea.txt /tmp/siteb.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no 'diff' after replication: $out" + exit 1 +fi + +./mc rm -r --force --versions sitea/bucket/marker +sleep 14s ## sleep for 14s idea is that we give 100ms per object. + +./mc ls -r --versions sitea/bucket >/tmp/sitea.txt +./mc ls -r --versions siteb/bucket >/tmp/siteb.txt + +out=$(diff -qpruN /tmp/sitea.txt /tmp/siteb.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no 'diff' after replication: $out" + exit 1 +fi + +./mc mb sitea/bucket-version/ +./mc mb siteb/bucket-version + +./mc version enable sitea/bucket-version/ +./mc version enable siteb/bucket-version/ + +echo "adding replication rule for site a -> site b" +./mc replicate add sitea/bucket-version/ \ + --remote-bucket http://minio:minio123@127.0.0.1:9004/bucket-version + +./mc mb sitea/bucket-version/directory/ + +sleep 2s + +./mc ls -r --versions sitea/bucket-version/ >/tmp/sitea_dirs.txt +./mc ls -r --versions siteb/bucket-version/ >/tmp/siteb_dirs.txt + +out=$(diff -qpruN /tmp/sitea_dirs.txt /tmp/siteb_dirs.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no 'diff' after replication: $out" + exit 1 +fi + +./mc rm -r --versions --force sitea/bucket-version/ + +sleep 2s + +./mc ls -r --versions sitea/bucket-version/ >/tmp/sitea_dirs.txt +./mc ls -r --versions siteb/bucket-version/ >/tmp/siteb_dirs.txt + +out=$(diff -qpruN /tmp/sitea_dirs.txt /tmp/siteb_dirs.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no 'diff' after replication: $out" + exit 1 +fi + +## check if we don't create delete markers on the directory objects, its always permanent delete. +./mc mb sitea/bucket-version/directory/ + +sleep 2s + +./mc rm -r --force sitea/bucket-version/ + +sleep 2s + +./mc ls -r --versions sitea/bucket-version/ >/tmp/sitea_dirs.txt +./mc ls -r --versions siteb/bucket-version/ >/tmp/siteb_dirs.txt + +out=$(diff -qpruN /tmp/sitea_dirs.txt /tmp/siteb_dirs.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no 'diff' after replication: $out" + exit 1 +fi + +sitea_count=$(cat /tmp/sitea_dirs.txt | wc -l) # need to do it this way to avoid filename in the output +siteb_count=$(cat /tmp/siteb_dirs.txt | wc -l) # need to do it this way to avoid filename in the output +sitea_out=$(cat /tmp/sitea_dirs.txt) +siteb_out=$(cat /tmp/siteb_dirs.txt) + +if [ $sitea_count -ne 0 ]; then + echo "BUG: expected no 'directory objects' left after deletion: ${sitea_out}" + exit 1 +fi + +if [ $siteb_count -ne 0 ]; then + echo "BUG: expected no 'directory objects' left after deletion: ${siteb_out}" + exit 1 +fi + +catch diff --git a/docs/bucket/replication/setup_3site_replication.sh b/docs/bucket/replication/setup_3site_replication.sh new file mode 100755 index 00000000..db8fc714 --- /dev/null +++ b/docs/bucket/replication/setup_3site_replication.sh @@ -0,0 +1,244 @@ +#!/usr/bin/env bash + +echo "Running $0" + +if [ -n "$TEST_DEBUG" ]; then + set -x +fi + +trap 'catch $LINENO' ERR + +# shellcheck disable=SC2120 +catch() { + if [ $# -ne 0 ]; then + echo "error on line $1" + for site in sitea siteb sitec; do + echo "$site server logs =========" + cat "/tmp/${site}_1.log" + echo "===========================" + cat "/tmp/${site}_2.log" + done + fi + + echo "Cleaning up instances of MinIO" + pkill minio + pkill -9 minio + rm -rf /tmp/multisitea + rm -rf /tmp/multisiteb + rm -rf /tmp/multisitec + if [ $# -ne 0 ]; then + exit $# + fi +} + +catch + +set -e +export MINIO_CI_CD=1 +export MINIO_BROWSER=off +export MINIO_ROOT_USER="minio" +export MINIO_ROOT_PASSWORD="minio123" +export MINIO_KMS_AUTO_ENCRYPTION=off +export MINIO_PROMETHEUS_AUTH_TYPE=public +export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= +unset MINIO_KMS_KES_CERT_FILE +unset MINIO_KMS_KES_KEY_FILE +unset MINIO_KMS_KES_ENDPOINT +unset MINIO_KMS_KES_KEY_NAME + +go install -v github.com/minio/mc@master +cp -a $(go env GOPATH)/bin/mc ./mc + +if [ ! -f mc.RELEASE.2021-03-12T03-36-59Z ]; then + wget -q -O mc.RELEASE.2021-03-12T03-36-59Z https://dl.minio.io/client/mc/release/linux-amd64/archive/mc.RELEASE.2021-03-12T03-36-59Z && + chmod +x mc.RELEASE.2021-03-12T03-36-59Z +fi + +minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 & +minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 & + +minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 & +minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 & + +minio server --address 127.0.0.1:9005 "http://127.0.0.1:9005/tmp/multisitec/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9006/tmp/multisitec/data/disterasure/xl{5...8}" >/tmp/sitec_1.log 2>&1 & +minio server --address 127.0.0.1:9006 "http://127.0.0.1:9005/tmp/multisitec/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9006/tmp/multisitec/data/disterasure/xl{5...8}" >/tmp/sitec_2.log 2>&1 & + +export MC_HOST_sitea=http://minio:minio123@127.0.0.1:9001 +export MC_HOST_siteb=http://minio:minio123@127.0.0.1:9004 +export MC_HOST_sitec=http://minio:minio123@127.0.0.1:9006 + +./mc ready sitea +./mc ready siteb +./mc ready sitec + +./mc mb sitea/bucket +./mc version enable sitea/bucket +./mc mb -l sitea/olockbucket + +./mc mb siteb/bucket/ +./mc version enable siteb/bucket/ +./mc mb -l siteb/olockbucket/ + +./mc mb sitec/bucket/ +./mc version enable sitec/bucket/ +./mc mb -l sitec/olockbucket + +echo "adding replication rule for a -> b : ${remote_arn}" +sleep 1 +./mc replicate add sitea/bucket/ \ + --remote-bucket http://minio:minio123@127.0.0.1:9004/bucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" +sleep 1 + +echo "adding replication rule for b -> a : ${remote_arn}" +./mc replicate add siteb/bucket/ \ + --remote-bucket http://minio:minio123@127.0.0.1:9001/bucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" +sleep 1 + +echo "adding replication rule for a -> c : ${remote_arn}" +./mc replicate add sitea/bucket/ \ + --remote-bucket http://minio:minio123@127.0.0.1:9006/bucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2 +sleep 1 + +echo "adding replication rule for c -> a : ${remote_arn}" +./mc replicate add sitec/bucket/ \ + --remote-bucket http://minio:minio123@127.0.0.1:9001/bucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2 +sleep 1 + +echo "adding replication rule for b -> c : ${remote_arn}" +./mc replicate add siteb/bucket/ \ + --remote-bucket http://minio:minio123@127.0.0.1:9006/bucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3 +sleep 1 + +echo "adding replication rule for c -> b : ${remote_arn}" +./mc replicate add sitec/bucket/ \ + --remote-bucket http://minio:minio123@127.0.0.1:9004/bucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3 +sleep 1 + +echo "adding replication rule for olockbucket a -> b : ${remote_arn}" +./mc replicate add sitea/olockbucket/ \ + --remote-bucket http://minio:minio123@127.0.0.1:9004/olockbucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" +sleep 1 + +echo "adding replication rule for olockbucket b -> a : ${remote_arn}" +./mc replicate add siteb/olockbucket/ \ + --remote-bucket http://minio:minio123@127.0.0.1:9001/olockbucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" +sleep 1 + +echo "adding replication rule for olockbucket a -> c : ${remote_arn}" +./mc replicate add sitea/olockbucket/ \ + --remote-bucket http://minio:minio123@127.0.0.1:9006/olockbucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2 +sleep 1 + +echo "adding replication rule for olockbucket c -> a : ${remote_arn}" +./mc replicate add sitec/olockbucket/ \ + --remote-bucket http://minio:minio123@127.0.0.1:9001/olockbucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 2 +sleep 1 + +echo "adding replication rule for olockbucket b -> c : ${remote_arn}" +./mc replicate add siteb/olockbucket/ \ + --remote-bucket http://minio:minio123@127.0.0.1:9006/olockbucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3 +sleep 1 + +echo "adding replication rule for olockbucket c -> b : ${remote_arn}" +./mc replicate add sitec/olockbucket/ \ + --remote-bucket http://minio:minio123@127.0.0.1:9004/olockbucket \ + --replicate "existing-objects,delete,delete-marker,replica-metadata-sync" --priority 3 +sleep 1 + +echo "Set default governance retention 30d" +./mc retention set --default governance 30d sitea/olockbucket + +echo "Copying data to source sitea/bucket" +./mc cp --enc-s3 "sitea/" --quiet /etc/hosts sitea/bucket +sleep 1 + +echo "Copying data to source sitea/olockbucket" +./mc cp --quiet /etc/hosts sitea/olockbucket +sleep 1 + +echo "Verifying the metadata difference between source and target" +if diff -pruN <(./mc stat --no-list --json sitea/bucket/hosts | jq .) <(./mc stat --no-list --json siteb/bucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then + echo "verified sitea-> COMPLETED, siteb-> REPLICA" +fi + +if diff -pruN <(./mc stat --no-list --json sitea/bucket/hosts | jq .) <(./mc stat --no-list --json sitec/bucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then + echo "verified sitea-> COMPLETED, sitec-> REPLICA" +fi + +echo "Verifying the metadata difference between source and target" +if diff -pruN <(./mc stat --no-list --json sitea/olockbucket/hosts | jq .) <(./mc stat --no-list --json siteb/olockbucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then + echo "verified sitea-> COMPLETED, siteb-> REPLICA" +fi + +if diff -pruN <(./mc stat --no-list --json sitea/olockbucket/hosts | jq .) <(./mc stat --no-list --json sitec/olockbucket/hosts | jq .) | grep -q 'COMPLETED\|REPLICA'; then + echo "verified sitea-> COMPLETED, sitec-> REPLICA" +fi + +sleep 5 + +head -c 221227088 200M +./mc.RELEASE.2021-03-12T03-36-59Z cp --config-dir ~/.mc --encrypt "sitea" --quiet 200M "sitea/bucket/200M-enc-v1" +./mc.RELEASE.2021-03-12T03-36-59Z cp --config-dir ~/.mc --quiet 200M "sitea/bucket/200M-v1" + +./mc cp --enc-s3 "sitea" --quiet 200M "sitea/bucket/200M-enc-v2" +./mc cp --quiet 200M "sitea/bucket/200M-v2" + +sleep 10 + +echo "Verifying ETag for all objects" +./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9001/ -bucket bucket +./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9002/ -bucket bucket +./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9003/ -bucket bucket +./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9004/ -bucket bucket +./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9005/ -bucket bucket +./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9006/ -bucket bucket + +./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9001/ -bucket olockbucket +./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9002/ -bucket olockbucket +./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9003/ -bucket olockbucket +./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9004/ -bucket olockbucket +./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9005/ -bucket olockbucket +./s3-check-md5 -versions -access-key minio -secret-key minio123 -endpoint http://127.0.0.1:9006/ -bucket olockbucket + +# additional tests for encryption object alignment +go install -v github.com/minio/multipart-debug@latest + +upload_id=$(multipart-debug --endpoint 127.0.0.1:9001 --accesskey minio --secretkey minio123 multipart new --bucket bucket --object new-test-encrypted-object --encrypt) + +dd if=/dev/urandom bs=1 count=7048531 of=/tmp/7048531.txt +dd if=/dev/urandom bs=1 count=2847391 of=/tmp/2847391.txt + +sudo apt install jq -y + +etag_1=$(multipart-debug --endpoint 127.0.0.1:9002 --accesskey minio --secretkey minio123 multipart upload --bucket bucket --object new-test-encrypted-object --uploadid ${upload_id} --file /tmp/7048531.txt --number 1 | jq -r .ETag) +etag_2=$(multipart-debug --endpoint 127.0.0.1:9001 --accesskey minio --secretkey minio123 multipart upload --bucket bucket --object new-test-encrypted-object --uploadid ${upload_id} --file /tmp/2847391.txt --number 2 | jq -r .ETag) +multipart-debug --endpoint 127.0.0.1:9002 --accesskey minio --secretkey minio123 multipart complete --bucket bucket --object new-test-encrypted-object --uploadid ${upload_id} 1.${etag_1} 2.${etag_2} + +sleep 10 + +./mc stat --no-list sitea/bucket/new-test-encrypted-object +./mc stat --no-list siteb/bucket/new-test-encrypted-object +./mc stat --no-list sitec/bucket/new-test-encrypted-object + +./mc ls -r sitea/bucket/ +./mc ls -r siteb/bucket/ +./mc ls -r sitec/bucket/ + +catch diff --git a/docs/bucket/replication/setup_ilm_expiry_replication.sh b/docs/bucket/replication/setup_ilm_expiry_replication.sh new file mode 100755 index 00000000..b7fca927 --- /dev/null +++ b/docs/bucket/replication/setup_ilm_expiry_replication.sh @@ -0,0 +1,314 @@ +#!/usr/bin/env bash + +set -x + +trap 'catch $LINENO' ERR + +# shellcheck disable=SC2120 +catch() { + if [ $# -ne 0 ]; then + echo "error on line $1" + for site in sitea siteb sitec sited; do + echo "$site server logs =========" + cat "/tmp/${site}_1.log" + echo "===========================" + cat "/tmp/${site}_2.log" + done + fi + + echo "Cleaning up instances of MinIO" + pkill minio + pkill -9 minio + rm -rf /tmp/multisitea + rm -rf /tmp/multisiteb + rm -rf /tmp/multisitec + rm -rf /tmp/multisited + rm -rf /tmp/data + if [ $# -ne 0 ]; then + exit $# + fi +} + +catch + +set -e +export MINIO_CI_CD=1 +export MINIO_BROWSER=off +export MINIO_ROOT_USER="minio" +export MINIO_ROOT_PASSWORD="minio123" +export MINIO_KMS_AUTO_ENCRYPTION=off +export MINIO_PROMETHEUS_AUTH_TYPE=public +export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= +unset MINIO_KMS_KES_CERT_FILE +unset MINIO_KMS_KES_KEY_FILE +unset MINIO_KMS_KES_ENDPOINT +unset MINIO_KMS_KES_KEY_NAME + +if [ ! -f ./mc ]; then + wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc +fi + +minio server --address 127.0.0.1:9001 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 & +minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 & + +minio server --address 127.0.0.1:9003 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 & +minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/multisiteb/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9004/tmp/multisiteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 & + +minio server --address 127.0.0.1:9005 "http://127.0.0.1:9005/tmp/multisitec/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9006/tmp/multisitec/data/disterasure/xl{5...8}" >/tmp/sitec_1.log 2>&1 & +minio server --address 127.0.0.1:9006 "http://127.0.0.1:9005/tmp/multisitec/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9006/tmp/multisitec/data/disterasure/xl{5...8}" >/tmp/sitec_2.log 2>&1 & + +minio server --address 127.0.0.1:9007 "http://127.0.0.1:9007/tmp/multisited/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9008/tmp/multisited/data/disterasure/xl{5...8}" >/tmp/sited_1.log 2>&1 & +minio server --address 127.0.0.1:9008 "http://127.0.0.1:9007/tmp/multisited/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9008/tmp/multisited/data/disterasure/xl{5...8}" >/tmp/sited_2.log 2>&1 & + +# Wait to make sure all MinIO instances are up + +export MC_HOST_sitea=http://minio:minio123@127.0.0.1:9001 +export MC_HOST_siteb=http://minio:minio123@127.0.0.1:9004 +export MC_HOST_sitec=http://minio:minio123@127.0.0.1:9006 +export MC_HOST_sited=http://minio:minio123@127.0.0.1:9008 + +./mc ready sitea +./mc ready siteb +./mc ready sitec +./mc ready sited + +./mc mb sitea/bucket +./mc mb sitec/bucket + +## Setup site replication +./mc admin replicate add sitea siteb --replicate-ilm-expiry + +sleep 10s + +## Add warm tier +./mc ilm tier add minio sitea WARM-TIER --endpoint http://localhost:9006 --access-key minio --secret-key minio123 --bucket bucket + +## Add ILM rules +./mc ilm add sitea/bucket --transition-days 0 --transition-tier WARM-TIER --transition-days 0 --noncurrent-expire-days 2 --expire-days 3 --prefix "myprefix" --tags "tag1=val1&tag2=val2" +./mc ilm rule list sitea/bucket + +## Check ilm expiry flag +./mc admin replicate info sitea --json +flag1=$(./mc admin replicate info sitea --json | jq '.sites[0]."replicate-ilm-expiry"') +flag2=$(./mc admin replicate info sitea --json | jq '.sites[1]."replicate-ilm-expiry"') +if [ "$flag1" != "true" ]; then + echo "BUG: Expected ILM expiry replication not set for 'sitea'" + exit 1 +fi +if [ "$flag2" != "true" ]; then + echo "BUG: Expected ILM expiry replication not set for 'siteb'" + exit 1 +fi + +## Check if ILM expiry rules replicated +sleep 30s + +./mc ilm rule list siteb/bucket +count=$(./mc ilm rule list siteb/bucket --json | jq '.config.Rules | length') +if [ $count -ne 1 ]; then + echo "BUG: ILM expiry rules not replicated to 'siteb'" + exit 1 +fi + +## Check replication of rules content +expDays=$(./mc ilm rule list siteb/bucket --json | jq '.config.Rules[0].Expiration.Days') +noncurrentDays=$(./mc ilm rule list siteb/bucket --json | jq '.config.Rules[0].NoncurrentVersionExpiration.NoncurrentDays') +if [ $expDays -ne 3 ]; then + echo "BUG: Incorrect expiry days '${expDays}' set for 'siteb'" + exit 1 +fi +if [ $noncurrentDays -ne 2 ]; then + echo "BUG: Incorrect non current expiry days '${noncurrentDays}' set for siteb" + exit 1 +fi + +## Make sure transition rule not replicated to siteb +tranDays=$(./mc ilm rule list siteb/bucket --json | jq '.config.Rules[0].Transition.Days') +if [ "${tranDays}" != "null" ]; then + echo "BUG: Transition rules as well copied to siteb" + exit 1 +fi + +## Check replication of rules prefix and tags +prefix=$(./mc ilm rule list siteb/bucket --json | jq '.config.Rules[0].Filter.And.Prefix' | sed 's/"//g') +tagName1=$(./mc ilm rule list siteb/bucket --json | jq '.config.Rules[0].Filter.And.Tags[0].Key' | sed 's/"//g') +tagVal1=$(./mc ilm rule list siteb/bucket --json | jq '.config.Rules[0].Filter.And.Tags[0].Value' | sed 's/"//g') +tagName2=$(./mc ilm rule list siteb/bucket --json | jq '.config.Rules[0].Filter.And.Tags[1].Key' | sed 's/"//g') +tagVal2=$(./mc ilm rule list siteb/bucket --json | jq '.config.Rules[0].Filter.And.Tags[1].Value' | sed 's/"//g') +if [ "${prefix}" != "myprefix" ]; then + echo "BUG: ILM expiry rules prefix not replicated to 'siteb'" + exit 1 +fi +if [ "${tagName1}" != "tag1" ] || [ "${tagVal1}" != "val1" ] || [ "${tagName2}" != "tag2" ] || [ "${tagVal2}" != "val2" ]; then + echo "BUG: ILM expiry rules tags not replicated to 'siteb'" + exit 1 +fi + +## Check edit of ILM expiry rule and its replication +id=$(./mc ilm rule list sitea/bucket --json | jq '.config.Rules[] | select(.Expiration.Days==3) | .ID' | sed 's/"//g') +./mc ilm edit --id "${id}" --expire-days "100" sitea/bucket +sleep 30s + +count1=$(./mc ilm rule list sitea/bucket --json | jq '.config.Rules[0].Expiration.Days') +count2=$(./mc ilm rule list siteb/bucket --json | jq '.config.Rules[0].Expiration.Days') +if [ $count1 -ne 100 ]; then + echo "BUG: Expiration days not changed on 'sitea'" + exit 1 +fi +if [ $count2 -ne 100 ]; then + echo "BUG: Modified ILM expiry rule not replicated to 'siteb'" + exit 1 +fi + +## Check disabling of ILM expiry rules replication +./mc admin replicate update sitea --disable-ilm-expiry-replication +flag=$(./mc admin replicate info sitea --json | jq '.sites[] | select (.name=="sitea") | ."replicate-ilm-expiry"') +if [ "$flag" != "false" ]; then + echo "BUG: ILM expiry replication not disabled for 'sitea'" + exit 1 +fi +flag=$(./mc admin replicate info siteb --json | jq '.sites[] | select (.name=="sitea") | ."replicate-ilm-expiry"') +if [ "$flag" != "false" ]; then + echo "BUG: ILM expiry replication not disabled for 'siteb'" + exit 1 +fi + +## Perform individual updates of rules to sites +./mc ilm edit --id "${id}" --expire-days "999" sitea/bucket +sleep 5s + +./mc ilm edit --id "${id}" --expire-days "888" siteb/bucket # when ilm expiry re-enabled, this should win + +## Check re-enabling of ILM expiry rules replication +./mc admin replicate update sitea --enable-ilm-expiry-replication +flag=$(./mc admin replicate info sitea --json | jq '.sites[] | select (.name=="sitea") | ."replicate-ilm-expiry"') +if [ "$flag" != "true" ]; then + echo "BUG: ILM expiry replication not enabled for 'sitea'" + exit 1 +fi +flag=$(./mc admin replicate info siteb --json | jq '.sites[] | select (.name=="sitea") | ."replicate-ilm-expiry"') +if [ "$flag" != "true" ]; then + echo "BUG: ILM expiry replication not enabled for 'siteb'" + exit 1 +fi + +## Check if latest updated rules get replicated to all sites post re-enable of ILM expiry rules replication +sleep 30s +count1=$(./mc ilm rule list sitea/bucket --json | jq '.config.Rules[0].Expiration.Days') +count2=$(./mc ilm rule list siteb/bucket --json | jq '.config.Rules[0].Expiration.Days') +if [ $count1 -ne 888 ]; then + echo "BUG: Latest expiration days not updated on 'sitea'" + exit 1 +fi +if [ $count2 -ne 888 ]; then + echo "BUG: Latest expiration days not updated on 'siteb'" + exit 1 +fi + +## Check to make sure sitea transition rule is not overwritten +transDays=$(./mc ilm rule list sitea/bucket --json | jq '.config.Rules[0].Transition.Days') +if [ $transDays -ne 0 ] || [ "${transDays}" == "null" ]; then + echo "BUG: Transition rule on sitea seems to be overwritten" + exit 1 +fi + +## Check replication of edit of prefix, tags and status of ILM Expiry Rules +./mc ilm rule edit --id "${id}" --prefix "newprefix" --tags "ntag1=nval1&ntag2=nval2" --disable sitea/bucket +sleep 30s + +nprefix=$(./mc ilm rule list siteb/bucket --json | jq '.config.Rules[0].Filter.And.Prefix' | sed 's/"//g') +ntagName1=$(./mc ilm rule list siteb/bucket --json | jq '.config.Rules[0].Filter.And.Tags[0].Key' | sed 's/"//g') +ntagVal1=$(./mc ilm rule list siteb/bucket --json | jq '.config.Rules[0].Filter.And.Tags[0].Value' | sed 's/"//g') +ntagName2=$(./mc ilm rule list siteb/bucket --json | jq '.config.Rules[0].Filter.And.Tags[1].Key' | sed 's/"//g') +ntagVal2=$(./mc ilm rule list siteb/bucket --json | jq '.config.Rules[0].Filter.And.Tags[1].Value' | sed 's/"//g') +st=$(./mc ilm rule list siteb/bucket --json | jq '.config.Rules[0].Status' | sed 's/"//g') +if [ "${nprefix}" != "newprefix" ]; then + echo "BUG: ILM expiry rules prefix not replicated to 'siteb'" + exit 1 +fi +if [ "${ntagName1}" != "ntag1" ] || [ "${ntagVal1}" != "nval1" ] || [ "${ntagName2}" != "ntag2" ] || [ "${ntagVal2}" != "nval2" ]; then + echo "BUG: ILM expiry rules tags not replicated to 'siteb'" + exit 1 +fi +if [ "${st}" != "Disabled" ]; then + echo "BUG: ILM expiry rules status not replicated to 'siteb'" + exit 1 +fi + +## Check replication of deleted ILM expiry rules +./mc ilm rule remove --id "${id}" sitea/bucket +sleep 30s + +# should error as rule doesn't exist +error=$(./mc ilm rule list siteb/bucket --json | jq '.error.cause.message' | sed 's/"//g') +if [ "$error" != "The lifecycle configuration does not exist" ]; then + echo "BUG: Removed ILM expiry rule not replicated to 'siteb'" + exit 1 +fi + +## Check addition of new replication site to existing site replication setup +# Add rules again as previous tests removed all +./mc ilm add sitea/bucket --transition-days 0 --transition-tier WARM-TIER --transition-days 0 --noncurrent-expire-days 2 --expire-days 3 --prefix "myprefix" --tags "tag1=val1&tag2=val2" +./mc admin replicate add sitea siteb sited +sleep 30s + +# Check site replication info and status for new site +sitesCount=$(mc admin replicate info sited --json | jq '.sites | length') +if [ ${sitesCount} -ne 3 ]; then + echo "BUG: New site 'sited' not appearing in site replication info" + exit 1 +fi +flag3=$(./mc admin replicate info sited --json | jq '.sites[2]."replicate-ilm-expiry"') +if [ "${flag3}" != "true" ]; then + echo "BUG: ILM expiry replication not enabled for 'sited'" + exit 1 +fi +rulesCount=$(./mc ilm rule list sited/bucket --json | jq '.config.Rules | length') +if [ ${rulesCount} -ne 1 ]; then + echo "BUG: ILM expiry rules not replicated to 'sited'" + exit 1 +fi +prefix=$(./mc ilm rule list sited/bucket --json | jq '.config.Rules[0].Filter.And.Prefix' | sed 's/"//g') +tagName1=$(./mc ilm rule list sited/bucket --json | jq '.config.Rules[0].Filter.And.Tags[0].Key' | sed 's/"//g') +tagVal1=$(./mc ilm rule list sited/bucket --json | jq '.config.Rules[0].Filter.And.Tags[0].Value' | sed 's/"//g') +tagName2=$(./mc ilm rule list sited/bucket --json | jq '.config.Rules[0].Filter.And.Tags[1].Key' | sed 's/"//g') +tagVal2=$(./mc ilm rule list sited/bucket --json | jq '.config.Rules[0].Filter.And.Tags[1].Value' | sed 's/"//g') +if [ "${prefix}" != "myprefix" ]; then + echo "BUG: ILM expiry rules prefix not replicated to 'sited'" + exit 1 +fi +if [ "${tagName1}" != "tag1" ] || [ "${tagVal1}" != "val1" ] || [ "${tagName2}" != "tag2" ] || [ "${tagVal2}" != "val2" ]; then + echo "BUG: ILM expiry rules tags not replicated to 'sited'" + exit 1 +fi + +## Check replication of deleted ILM expiry rules when target has transition part as well +## Only the expiry part of rules should get removed as part if replication of removal from +## other site +id=$(./mc ilm rule list siteb/bucket --json | jq '.config.Rules[] | select(.Expiration.Days==3) | .ID' | sed 's/"//g') +# Remove rule from siteb +./mc ilm rule remove --id "${id}" siteb/bucket +sleep 30s # allow to replicate + +# sitea should still contain the transition portion of rule +transitionRuleDays=$(./mc ilm rule list sitea/bucket --json | jq '.config.Rules[0].Transition.Days') +expirationRuleDet=$(./mc ilm rule list sitea/bucket --json | jq '.config.Rules[0].Expiration') +if [ ${transitionRuleDays} -ne 0 ]; then + echo "BUG: Transition rules not retained as part of replication of deleted ILM expiry rules on 'sitea'" + exit 1 +fi +if [ ${expirationRuleDet} != null ]; then + echo "BUG: removed ILM expiry rule not replicated to 'sitea'" + exit 1 +fi + +catch diff --git a/docs/bucket/replication/setup_replication.sh b/docs/bucket/replication/setup_replication.sh new file mode 100755 index 00000000..1b3774a8 --- /dev/null +++ b/docs/bucket/replication/setup_replication.sh @@ -0,0 +1,101 @@ +#!/bin/sh + +# Create buckets with versioning and object locking enabled. +mc mb -l source/bucket +mc mb -l dest/bucket + +#### Create a replication admin on source alias +# create a replication admin user : repladmin +mc admin user add source repladmin repladmin123 + +# create a replication policy for repladmin +cat >repladmin-policy-source.json <replpolicy.json </tmp/dir_1.txt +./mc ls -r --versions myminio2/testbucket/dir/ >/tmp/dir_2.txt + +out=$(diff -qpruN /tmp/dir_1.txt /tmp/dir_2.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no 'diff' after replication: $out" + exit 1 +fi diff --git a/docs/bucket/replication/test_del_marker_proxying.sh b/docs/bucket/replication/test_del_marker_proxying.sh new file mode 100755 index 00000000..8d7521b4 --- /dev/null +++ b/docs/bucket/replication/test_del_marker_proxying.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2120 +exit_1() { + cleanup + + for site in sitea siteb; do + echo "$site server logs =========" + cat "/tmp/${site}_1.log" + echo "===========================" + cat "/tmp/${site}_2.log" + done + + exit 1 +} + +cleanup() { + echo -n "Cleaning up instances of MinIO ..." + pkill -9 minio || sudo pkill -9 minio + rm -rf /tmp/sitea + rm -rf /tmp/siteb + echo "done" +} + +cleanup + +export MINIO_CI_CD=1 +export MINIO_BROWSER=off + +make install-race + +# Start MinIO instances +echo -n "Starting MinIO instances ..." +minio server --address 127.0.0.1:9001 --console-address ":10000" "http://127.0.0.1:9001/tmp/sitea/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9002/tmp/sitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 & +minio server --address 127.0.0.1:9002 "http://127.0.0.1:9001/tmp/sitea/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9002/tmp/sitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 & + +minio server --address 127.0.0.1:9003 --console-address ":10001" "http://127.0.0.1:9003/tmp/siteb/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9004/tmp/siteb/data/disterasure/xl{5...8}" >/tmp/siteb_1.log 2>&1 & +minio server --address 127.0.0.1:9004 "http://127.0.0.1:9003/tmp/siteb/data/disterasure/xl{1...4}" \ + "http://127.0.0.1:9004/tmp/siteb/data/disterasure/xl{5...8}" >/tmp/siteb_2.log 2>&1 & + +echo "done" + +if [ ! -f ./mc ]; then + wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc +fi + +export MC_HOST_sitea=http://minioadmin:minioadmin@127.0.0.1:9001 +export MC_HOST_siteb=http://minioadmin:minioadmin@127.0.0.1:9004 + +./mc ready sitea +./mc ready siteb + +./mc mb sitea/bucket +./mc version enable sitea/bucket +./mc mb siteb/bucket +./mc version enable siteb/bucket + +# Set bucket replication +./mc replicate add sitea/bucket --remote-bucket siteb/bucket + +# Run the test to make sure proxying of DEL marker doesn't happen +loop_count=0 +while true; do + if [ $loop_count -eq 1000 ]; then + break + fi + echo "Hello World" | ./mc pipe sitea/bucket/obj$loop_count + ./mc rm sitea/bucket/obj$loop_count + RESULT=$({ ./mc stat --no-list sitea/bucket/obj$loop_count; } 2>&1) + if [[ ${RESULT} != *"Object does not exist"* ]]; then + echo "BUG: stat should fail. succeeded." + exit_1 + fi + loop_count=$((loop_count + 1)) +done + +cleanup diff --git a/docs/bucket/retention/README.md b/docs/bucket/retention/README.md new file mode 100644 index 00000000..c5087b22 --- /dev/null +++ b/docs/bucket/retention/README.md @@ -0,0 +1,59 @@ +# Object Lock and Immutability Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +MinIO server allows WORM for specific objects or by configuring a bucket with default object lock configuration that applies default retention mode and retention duration to all objects. This makes objects in the bucket immutable i.e. delete of the version are not allowed until an expiry specified in the bucket's object lock configuration or object retention. + +Object locking requires locking to be enabled on a bucket at the time of bucket creation refer to `mc mb --with-lock`, object locking enables versioning on the bucket and cannot be disabled. + +A default retention period and retention mode can be configured on a bucket to be applied to objects created in that bucket. Independent of retention, an object can also be under legal hold. This effectively disallows all deletes of an object under legal hold until the legal hold is removed by an API call. + +## Get Started + +### 1. Prerequisites + +- Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html) +- Install `awscli` - [Installing AWS Command Line Interface](https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html) + +### 2. Set bucket WORM configuration + +WORM on a bucket is enabled by setting object lock configuration. This configuration is applied to all the objects in the bucket. Below is an example to set `Governance` mode and one day retention time on `mybucket`. + +```sh +awscli s3api put-object-lock-configuration --bucket mybucket --object-lock-configuration 'ObjectLockEnabled=\"Enabled\",Rule={DefaultRetention={Mode=\"GOVERNANCE\",Days=1}}' +``` + +### Set object lock + +PutObject API allows setting per object retention mode and retention duration using `x-amz-object-lock-mode` and `x-amz-object-lock-retain-until-date` headers. This takes precedence over any bucket object lock configuration w.r.t retention. + +```sh +aws s3api put-object --bucket testbucket --key lockme --object-lock-mode GOVERNANCE --object-lock-retain-until-date "2019-11-20" --body /etc/issue +``` + +See for AWS S3 spec on object locking and permissions required for object retention and governance bypass overrides. + +### Set legal hold on an object + +PutObject API allows setting legal hold using `x-amz-object-lock-legal-hold` header. + +```sh +aws s3api put-object --bucket testbucket --key legalhold --object-lock-legal-hold-status ON --body /etc/issue +``` + +See for AWS S3 spec on object locking and permissions required for specifying legal hold. + +## Concepts + +- If an object is under legal hold, it cannot be deleted unless the legal hold is explicitly removed for the respective version id. DeleteObjectVersion() would fail otherwise. +- In `Compliance` mode, objects cannot be deleted by anyone until retention period has expired for the given version id. If user has governance bypass permissions, an object's retention date can be extended in `Compliance` mode. +- Once object lock configuration is set to a bucket + - New objects inherit the retention settings of the bucket object lock configuration automatically + - Retention headers can be optionally set while uploading objects + - Once objects are uploaded PutObjectRetention API can be called to change retention settings +- *MINIO_NTP_SERVER* environment variable can be set to remote NTP server endpoint if system time is not desired for setting retention dates. + +## Explore Further + +- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart) +- [Use `aws-cli` with MinIO Server](https://docs.min.io/community/minio-object-store/integrations/aws-cli-with-minio.html) +- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/go/minio-go.html) +- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html) diff --git a/docs/bucket/versioning/DESIGN.md b/docs/bucket/versioning/DESIGN.md new file mode 100644 index 00000000..a6e865a4 --- /dev/null +++ b/docs/bucket/versioning/DESIGN.md @@ -0,0 +1,138 @@ +# Bucket Versioning Design Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) + +## Description of `xl.meta` + +`xl.meta` is a new self describing backend format used by MinIO to support AWS S3 compatible versioning. +This file is the source of truth for each `version` at rest. `xl.meta` is a msgpack file serialized from a +well defined data structure. To understand `xl.meta` here are the few things to start with + +`xl.meta` carries first 8 bytes an XL header which describes the current format and the format version, +allowing the unmarshaller's to automatically use the right data structures to parse the subsequent content in the stream. + +### v1.0 + +| Entry | Encoding | Content +| ----------|-------------|---------------------------------------- +| xlHeader | [4]byte | `'X', 'L', '2', ' '` +| xlVersion | [4]byte | `'1', ' ', ' ', ' '` +| xlMetaV2 | msgp object | All versions as single messagepack object +| [EOF] | | + +### v1.1+ + +Version 1.1 added inline data, which will be placed after the metadata. + +Therefore, the metadata is wrapped as a binary array for easy skipping. + +| Entry | Encoding | Content +| ---------------|----------------|---------------------------------------- +| xlHeader | [4]byte | `'X', 'L', '2', ' '` +| xlVersionMajor | uint16 | Major xl-meta version. +| xlVersionMinor | uint16 | Minor xl-meta version. +| xlMetaV2 | msgp bin array | Bin array with serialized metadata +| crc | msgp uint | Lower 32 bits of 64 bit xxhash of previous array contents (v1.2+ only) +| inline data | binary | Inline data if any, see Inline Data section for encoding. +| [EOF] | | + +## v1.0-v1.2 Versions + +`xl.meta` carries three types of object entries which designate the type of version object stored. + +- ObjectType (default) +- LegacyObjectType (preserves existing deployments and older xl.json format) +- DeleteMarker (a versionId to capture the DELETE sequences implemented primarily for AWS spec compatibility) + +A sample msgpack-JSON `xl.meta`, you can debug the content inside `xl.meta` using [xl-meta.go](https://github.com/minio/minio/tree/master/docs/debugging#decoding-metadata) program. + +```json +{ + "Versions": [ + { + "Type": 1, + "V2Obj": { + "ID": "KWUs8S+8RZq4Vp5TWy6KFg==", + "DDir": "X3pDAFu8Rjyft7QD6t7W5g==", + "EcAlgo": 1, + "EcM": 2, + "EcN": 2, + "EcBSize": 10485760, + "EcIndex": 3, + "EcDist": [3, 4, 1, 2], + "CSumAlgo": 1, + "PartNums": [1], + "PartETags": [""], + "PartSizes": [314], + "PartASizes": [282], + "Size": 314, + "MTime": 1591820730, + "MetaSys": { + "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id": "bXktbWluaW8ta2V5", + "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key": "ZXlKaFpXRmtJam9pUVVWVExUSTFOaTFIUTAwdFNFMUJReTFUU0VFdE1qVTJJaXdpYVhZaU9pSkJMMVZzZFVnelZYVjZSR2N6UkhGWUwycEViRmRCUFQwaUxDSnViMjVqWlNJNklpdE9lbkJXVWtseFlWSlNVa2t2UVhNaUxDSmllWFJsY3lJNklrNDBabVZsZG5WU1NWVnRLMFoyUWpBMVlYTk9aMU41YVhoU1RrNUpkMDlhTkdKa2RuaGpLMjFuVDNnMFFYbFJhbE15V0hkU1pEZzNRMk54ZUN0SFFuSWlmUT09", + "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "REFSRXYyLUhNQUMtU0hBMjU2", + "X-Minio-Internal-Server-Side-Encryption-Iv": "bW5YRDhRUGczMVhkc2pJT1V1UVlnbWJBcndIQVhpTUN1dnVBS0QwNUVpaz0=", + "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key": "SUFBZkFPeUo5ZHVVSEkxYXFLU0NSRkJTTnM0QkVJNk9JWU1QcFVTSXFhK2dHVThXeE9oSHJCZWwwdnRvTldUNE8zS1BtcWluR0cydmlNNFRWa0N0Mmc9PQ==" + }, + "MetaUsr": { + "content-type": "application/octet-stream", + "etag": "20000f00f58c508b40720270929bd90e9f07b9bd78fb605e5432a67635fc34722e4fc53b1d5fab9ff8400eb9ded4fba2" + } + } + } + ] +} +``` + +### v1.3+ versions + +Version 1.3 introduces changes to help with [faster metadata reads and updates](https://blog.min.io/minio-versioning-metadata-deep-dive/) + +| Entry | Encoding | Content +| ----------------|-----------------------------|---------------------------------------- +| xlHeaderVersion | msgp uint | header version identifier +| xlMetaVersion | msgp uint | metadata version identifier +| versions | msgp int | Number of versions following +| header_1 | msgp bin array | Header of version 1 +| metadata_1 | msgp bin array | Metadata of version 1 +| ...header_n | msgp bin array | Header of last version +| ...metadata_n | msgp bin array | Metadata of last version + +Each header contains a mspg array (tuple) encoded object: + +xlHeaderVersion version == 1: + +``` +//msgp:tuple xlMetaV2VersionHeader +type xlMetaV2VersionHeader struct { + VersionID [16]byte // Version UUID, raw. + ModTime int64 // Unix nanoseconds. + Signature [4]byte // Signature of metadata. + Type uint8 // Type if the version + Flags uint8 +} +``` + +The following flags are defined: + +``` +const ( + FreeVersion = 1 << 0 + UsesDataDir = 1 << 1 + InlineData = 1 << 2 +) +``` + +The "Metadata" section contains a single version, encoded in similar fashion as each version in the `Versions` array +of the previous version. + +## Inline Data + +Inline data is optional. If no inline data is present, it is encoded as 0 bytes. + +| Entry | Encoding | Content +| --------------------|-----------------------------|---------------------------------------- +| xlMetaInlineDataVer | byte | version identifier +| id -> data | msgp `map[string][]byte` | Map of string id -> byte content + +Currently only xlMetaInlineDataVer == 1 exists. + +The ID is the string encoded Version ID of which the data corresponds. diff --git a/docs/bucket/versioning/README.md b/docs/bucket/versioning/README.md new file mode 100644 index 00000000..aeb9aa00 --- /dev/null +++ b/docs/bucket/versioning/README.md @@ -0,0 +1,217 @@ +# Bucket Versioning Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) + +MinIO versioning is designed to keep multiple versions of an object in one bucket. For example, you could store `spark.csv` (version `ede336f2`) and `spark.csv` (version `fae684da`) in a single bucket. Versioning protects you from unintended overwrites, deletions, protect objects with retention policies. + +To control data retention and storage usage, use object versioning with [object lifecycle management](https://github.com/minio/minio/blob/master/docs/bucket/lifecycle/README.md). If you have an object expiration lifecycle policy in your non-versioned bucket and you want to maintain the same permanent delete behavior when on versioning-enabled bucket, you must add a noncurrent expiration policy. The noncurrent expiration lifecycle policy will manage the deletes of the noncurrent object versions in the versioning-enabled bucket. (A version-enabled bucket maintains one current and zero or more noncurrent object versions.) + +Versioning must be explicitly enabled on a bucket, versioning is not enabled by default. Object locking enabled buckets have versioning enabled automatically. Enabling and suspending versioning is done at the bucket level. + +Only MinIO generates version IDs, and they can't be edited. Version IDs are simply of `DCE 1.1 v4 UUID 4` (random data based), UUIDs are 128 bit numbers which are intended to have a high likelihood of uniqueness over space and time and are computationally difficult to guess. They are globally unique identifiers which can be locally generated without contacting a global registration authority. UUIDs are intended as unique identifiers for both mass tagging objects with an extremely short lifetime and to reliably identifying very persistent objects across a network. + +When you PUT an object in a versioning-enabled bucket, the noncurrent version is not overwritten. The following figure shows that when a new version of `spark.csv` is PUT into a bucket that already contains an object with the same name, the original object (ID = `ede336f2`) remains in the bucket, MinIO generates a new version (ID = `fae684da`), and adds the newer version to the bucket. + +![put](https://raw.githubusercontent.com/minio/minio/master/docs/bucket/versioning/versioning_PUT_versionEnabled.png) + +This protects against accidental overwrites or deletes of objects, allows previous versions to be retrieved. + +When you DELETE an object, all versions remain in the bucket and MinIO adds a delete marker, as shown below: + +![delete](https://raw.githubusercontent.com/minio/minio/master/docs/bucket/versioning/versioning_DELETE_versionEnabled.png) + +Now the delete marker becomes the current version of the object. GET requests by default always retrieve the latest stored version. So performing a simple GET object request when the current version is a delete marker would return `404` `The specified key does not exist` as shown below: + +![get](https://raw.githubusercontent.com/minio/minio/master/docs/bucket/versioning/versioning_GET_versionEnabled.png) + +GET requests by specifying a version ID as shown below, you can retrieve the specific object version `fae684da`. + +![get_version_id](https://raw.githubusercontent.com/minio/minio/master/docs/bucket/versioning/versioning_GET_versionEnabled_id.png) + +To permanently delete an object you need to specify the version you want to delete, only the user with appropriate permissions can permanently delete a version. As shown below DELETE request called with a specific version id permanently deletes an object from a bucket. Delete marker is not added for DELETE requests with version id. + +![delete_version_id](https://raw.githubusercontent.com/minio/minio/master/docs/bucket/versioning/versioning_DELETE_versionEnabled_id.png) + +## Concepts + +- All Buckets on MinIO are always in one of the following states: unversioned (the default) and all other existing deployments, versioning-enabled, or versioning-suspended. +- Versioning state applies to all of the objects in the versioning enabled bucket. The first time you enable a bucket for versioning, objects in the bucket are thereafter always versioned and given a unique version ID. +- Existing or newer buckets can be created with versioning enabled and eventually can be suspended as well. Existing versions of objects stay as is and can still be accessed using the version ID. +- All versions, including delete-markers should be deleted before deleting a bucket. +- **Versioning feature is only available in erasure coded and distributed erasure coded setups**. + +## How to configure versioning on a bucket + +Each bucket created has a versioning configuration associated with it. By default bucket is unversioned as shown below + +``` + + +``` + +To enable versioning, you send a request to MinIO with a versioning configuration with Status set to `Enabled`. + +``` + + Enabled + +``` + +Similarly to suspend versioning set the configuration with Status set to `Suspended`. + +``` + + Suspended + +``` + +## MinIO extension to Bucket Versioning + +### Idempotent versions on directory objects + +All directory objects such as objects that end with `/`, will only have one versionId (i.e `null`). A delete marker will never be created on these directory objects, instead a DELETE will delete the directory objects. This is done to ensure that directory objects even with multiple overwrites - do not ever need multiple versions in the first place. All overwrite calls on these directory objects are idempotent. + +> NOTE: Server side replication is supported for idempotent versions on directory objects. + +### Idempotent versions on delete markers + +Duplicate delete markers are not created on MinIO buckets with versioning, if an application performs a soft delete on an object repeatedly - that object will only ever have a single DELETE marker for all such successive attempts. This is done to ensure that repeated soft deletes do not ever need multiple versions in the first place. + +> NOTE: Server side replication is supported for idempotent versions on delete marked objects. + +### Motivation + +**PLEASE READ: This feature is meant for advanced use-cases only where the setup is using bucket versioning or with replicated buckets, use this feature to optimize versioning behavior for some specific applications. MinIO experts will evaluate and guide on the benefits for your application, please reach out to us on .** + +Spark/Hadoop workloads which use Hadoop MR Committer v1/v2 algorithm upload objects to a temporary prefix in a bucket. These objects are 'renamed' to a different prefix on Job commit. Object storage admins are forced to configure separate ILM policies to expire these objects and their versions to reclaim space. + +### Solution + +To exclude objects under a list of prefix (glob) patterns from being versioned, you can send the following versioning configuration with Status set to `Enabled`. + +``` + + Enabled + + */_temporary + + + */__magic + + + */_staging + + + + +``` + +### Features + +- Objects matching these prefixes will behave as though versioning were suspended. These objects **will not** be replicated if bucket has replication configured. +- Objects matching these prefixes will also not leave `null` delete markers, dramatically reduces namespace pollution while keeping the benefits of replication. +- Users with explicit permissions or the root credential can configure the versioning state of any bucket. + +## Examples of enabling bucket versioning using MinIO Java SDK + +### EnableVersioning() API + +``` +import io.minio.EnableVersioningArgs; +import io.minio.MinioClient; +import io.minio.errors.MinioException; +import java.io.IOException; +import java.security.InvalidKeyException; +import java.security.NoSuchAlgorithmException; + +public class EnableVersioning { + /** MinioClient.enableVersioning() example. */ + public static void main(String[] args) + throws IOException, NoSuchAlgorithmException, InvalidKeyException { + try { + /* play.min.io for test and development. */ + MinioClient minioClient = + MinioClient.builder() + .endpoint("https://play.min.io") + .credentials("Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG") + .build(); + + /* Amazon S3: */ + // MinioClient minioClient = + // MinioClient.builder() + // .endpoint("https://s3.amazonaws.com") + // .credentials("YOUR-ACCESSKEY", "YOUR-SECRETACCESSKEY") + // .build(); + + // Enable versioning on 'my-bucketname'. + minioClient.enableVersioning(EnableVersioningArgs.builder().bucket("my-bucketname").build()); + + System.out.println("Bucket versioning is enabled successfully"); + + } catch (MinioException e) { + System.out.println("Error occurred: " + e); + } + } +} +``` + +### isVersioningEnabled() API + +``` +public class IsVersioningEnabled { + /** MinioClient.isVersioningEnabled() example. */ + public static void main(String[] args) + throws IOException, NoSuchAlgorithmException, InvalidKeyException { + try { + /* play.min.io for test and development. */ + MinioClient minioClient = + MinioClient.builder() + .endpoint("https://play.min.io") + .credentials("Q3AM3UQ867SPQQA43P2F", "zuf+tfteSlswRu7BJ86wekitnifILbZam1KYY3TG") + .build(); + + /* Amazon S3: */ + // MinioClient minioClient = + // MinioClient.builder() + // .endpoint("https://s3.amazonaws.com") + // .credentials("YOUR-ACCESSKEY", "YOUR-SECRETACCESSKEY") + // .build(); + + // Create bucket 'my-bucketname' if it doesn`t exist. + if (!minioClient.bucketExists(BucketExistsArgs.builder().bucket("my-bucketname").build())) { + minioClient.makeBucket(MakeBucketArgs.builder().bucket("my-bucketname").build()); + System.out.println("my-bucketname is created successfully"); + } + + boolean isVersioningEnabled = + minioClient.isVersioningEnabled( + IsVersioningEnabledArgs.builder().bucket("my-bucketname").build()); + if (isVersioningEnabled) { + System.out.println("Bucket versioning is enabled"); + } else { + System.out.println("Bucket versioning is disabled"); + } + // Enable versioning on 'my-bucketname'. + minioClient.enableVersioning(EnableVersioningArgs.builder().bucket("my-bucketname").build()); + System.out.println("Bucket versioning is enabled successfully"); + + isVersioningEnabled = + minioClient.isVersioningEnabled( + IsVersioningEnabledArgs.builder().bucket("my-bucketname").build()); + if (isVersioningEnabled) { + System.out.println("Bucket versioning is enabled"); + } else { + System.out.println("Bucket versioning is disabled"); + } + + } catch (MinioException e) { + System.out.println("Error occurred: " + e); + } + } +} +``` + +## Explore Further + +- [Use `minio-java` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/java/minio-java.html) +- [Object Lock and Immutability Guide](https://docs.min.io/community/minio-object-store/administration/object-management/object-retention.html) +- [MinIO Admin Complete Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc-admin.html) +- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html) diff --git a/docs/bucket/versioning/versioning-tests.sh b/docs/bucket/versioning/versioning-tests.sh new file mode 100755 index 00000000..84fbec6f --- /dev/null +++ b/docs/bucket/versioning/versioning-tests.sh @@ -0,0 +1,99 @@ +#!/usr/bin/env bash + +if [ -n "$TEST_DEBUG" ]; then + set -x +fi + +trap 'catch $LINENO' ERR + +# shellcheck disable=SC2120 +catch() { + if [ $# -ne 0 ]; then + echo "error on line $1" + echo "server logs =========" + cat "/tmp/sitea_1.log" + echo "===========================" + cat "/tmp/sitea_2.log" + fi + + echo "Cleaning up instances of MinIO" + pkill minio + pkill -9 minio + rm -rf /tmp/multisitea + if [ $# -ne 0 ]; then + exit $# + fi +} + +catch + +set -e +export MINIO_CI_CD=1 +export MINIO_BROWSER=off +export MINIO_KMS_AUTO_ENCRYPTION=off +export MINIO_PROMETHEUS_AUTH_TYPE=public +export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= +unset MINIO_KMS_KES_CERT_FILE +unset MINIO_KMS_KES_KEY_FILE +unset MINIO_KMS_KES_ENDPOINT +unset MINIO_KMS_KES_KEY_NAME + +if [ ! -f ./mc ]; then + wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc +fi + +minio server -S /tmp/no-certs --address ":9001" "http://localhost:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ + "http://localhost:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_1.log 2>&1 & + +minio server -S /tmp/no-certs --address ":9002" "http://localhost:9001/tmp/multisitea/data/disterasure/xl{1...4}" \ + "http://localhost:9002/tmp/multisitea/data/disterasure/xl{5...8}" >/tmp/sitea_2.log 2>&1 & + +export MC_HOST_sitea=http://minioadmin:minioadmin@localhost:9002 + +./mc ready sitea + +./mc mb sitea/delissue --insecure + +./mc version enable sitea/delissue --insecure + +echo hello | ./mc pipe sitea/delissue/hello --insecure + +./mc version suspend sitea/delissue --insecure + +./mc rm sitea/delissue/hello --insecure + +./mc version enable sitea/delissue --insecure + +echo hello | ./mc pipe sitea/delissue/hello --insecure + +./mc version suspend sitea/delissue --insecure + +./mc rm sitea/delissue/hello --insecure + +count=$(./mc ls --versions sitea/delissue --insecure | wc -l) + +if [ ${count} -ne 3 ]; then + echo "BUG: expected number of versions to be '3' found ${count}" + echo "===== DEBUG =====" + ./mc ls --versions sitea/delissue +fi + +./mc mb sitea/testbucket + +./mc version enable sitea/testbucket + +./mc put --quiet README.md sitea/testbucket/file +etag1=$(./mc cat sitea/testbucket/file | md5sum --tag | awk {'print $4'}) + +./mc cp --quiet --storage-class "STANDARD" sitea/testbucket/file sitea/testbucket/file +etag2=$(./mc cat sitea/testbucket/file | md5sum --tag | awk {'print $4'}) +if [ $etag1 != $etag2 ]; then + echo "expected $etag1, got $etag2" + exit 1 +fi + +echo "SUCCESS:" +./mc ls --versions sitea/delissue --insecure + +catch diff --git a/docs/bucket/versioning/versioning_DELETE_versionEnabled.png b/docs/bucket/versioning/versioning_DELETE_versionEnabled.png new file mode 100644 index 00000000..18dde122 Binary files /dev/null and b/docs/bucket/versioning/versioning_DELETE_versionEnabled.png differ diff --git a/docs/bucket/versioning/versioning_DELETE_versionEnabled_id.png b/docs/bucket/versioning/versioning_DELETE_versionEnabled_id.png new file mode 100644 index 00000000..99afdd4f Binary files /dev/null and b/docs/bucket/versioning/versioning_DELETE_versionEnabled_id.png differ diff --git a/docs/bucket/versioning/versioning_GET_versionEnabled.png b/docs/bucket/versioning/versioning_GET_versionEnabled.png new file mode 100644 index 00000000..aacbeb37 Binary files /dev/null and b/docs/bucket/versioning/versioning_GET_versionEnabled.png differ diff --git a/docs/bucket/versioning/versioning_GET_versionEnabled_id.png b/docs/bucket/versioning/versioning_GET_versionEnabled_id.png new file mode 100644 index 00000000..44ad109a Binary files /dev/null and b/docs/bucket/versioning/versioning_GET_versionEnabled_id.png differ diff --git a/docs/bucket/versioning/versioning_PUT_versionEnabled.png b/docs/bucket/versioning/versioning_PUT_versionEnabled.png new file mode 100644 index 00000000..fe13b2c0 Binary files /dev/null and b/docs/bucket/versioning/versioning_PUT_versionEnabled.png differ diff --git a/docs/chroot/README.md b/docs/chroot/README.md new file mode 100644 index 00000000..1adb019f --- /dev/null +++ b/docs/chroot/README.md @@ -0,0 +1,58 @@ +# Deploy MinIO on Chrooted Environment [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) + +Chroot allows user based namespace isolation on many standard Linux deployments. + +## 1. Prerequisites + +- Familiarity with [chroot](http://man7.org/linux/man-pages/man2/chroot.2.html) +- Chroot installed on your machine. + +## 2. Install MinIO in Chroot + +> **Note:** MinIO community edition is now distributed as source code only. Pre-compiled binaries are no longer provided for new releases. + +Build MinIO from source and install it in the chroot directory: + +```sh +# Build MinIO from source +go install github.com/minio/minio@latest + +# Create the bin directory in your chroot +mkdir -p /mnt/export/${USER}/bin + +# Copy the built binary to the chroot directory +cp $(go env GOPATH)/bin/minio /mnt/export/${USER}/bin/minio +chmod +x /mnt/export/${USER}/bin/minio +``` + +Alternatively, if you have an existing legacy binary, you can still use it, but note that it will not receive updates. + +Bind your `proc` mount to the target chroot directory + +``` +sudo mount --bind /proc /mnt/export/${USER}/proc +``` + +## 3. Run Standalone MinIO in Chroot + +### GNU/Linux + +```sh +sudo chroot --userspec username:group /mnt/export/${USER} /bin/minio --config-dir=/.minio server /data + +Endpoint: http://192.168.1.92:9000 http://65.19.167.92:9000 +AccessKey: MVPSPBW4NP2CMV1W3TXD +SecretKey: X3RKxEeFOI8InuNWoPsbG+XEVoaJVCqbvxe+PTOa +... +... +``` + +Instance is now accessible on the host at port 9000, proceed to access the Web browser at + +## Explore Further + +- [MinIO Erasure Code Overview](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html) +- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html) +- [Use `aws-cli` with MinIO Server](https://docs.min.io/community/minio-object-store/integrations/aws-cli-with-minio.html) +- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/go/minio-go.html) +- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html) diff --git a/docs/compression/README.md b/docs/compression/README.md new file mode 100644 index 00000000..cc7bac66 --- /dev/null +++ b/docs/compression/README.md @@ -0,0 +1,137 @@ +# Compression Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +MinIO server allows streaming compression to ensure efficient disk space usage. +Compression happens inflight, i.e objects are compressed before being written to disk(s). +MinIO uses [`klauspost/compress/s2`](https://github.com/klauspost/compress/tree/master/s2) +streaming compression due to its stability and performance. + +This algorithm is specifically optimized for machine generated content. +Write throughput is typically at least 500MB/s per CPU core, +and scales with the number of available CPU cores. +Decompression speed is typically at least 1GB/s. + +This means that in cases where raw IO is below these numbers +compression will not only reduce disk usage but also help increase system throughput. +Typically, enabling compression on spinning disk systems +will increase speed when the content can be compressed. + +## Get Started + +### 1. Prerequisites + +Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html). + +### 2. Run MinIO with compression + +Compression can be enabled by updating the `compress` config settings for MinIO server config. +Config `compress` settings take extensions and mime-types to be compressed. + +```bash +~ mc admin config get myminio compression +compression extensions=".txt,.log,.csv,.json,.tar,.xml,.bin" mime_types="text/*,application/json,application/xml" +``` + +Default config includes most common highly compressible content extensions and mime-types. + +```bash +~ mc admin config set myminio compression extensions=".pdf" mime_types="application/pdf" +``` + +To show help on setting compression config values. + +```bash +~ mc admin config set myminio compression +``` + +To enable compression for all content, no matter the extension and content type +(except for the default excluded types) set BOTH extensions and mime types to empty. + +```bash +~ mc admin config set myminio compression enable="on" extensions="" mime_types="" +``` + +The compression settings may also be set through environment variables. +When set, environment variables override the defined `compress` config settings in the server config. + +```bash +export MINIO_COMPRESSION_ENABLE="on" +export MINIO_COMPRESSION_EXTENSIONS=".txt,.log,.csv,.json,.tar,.xml,.bin" +export MINIO_COMPRESSION_MIME_TYPES="text/*,application/json,application/xml" +``` + +> [!NOTE] +> To enable compression for all content when using environment variables, set either or both of the extensions and MIME types to `*` instead of an empty string: +> ```bash +> export MINIO_COMPRESSION_ENABLE="on" +> export MINIO_COMPRESSION_EXTENSIONS="*" +> export MINIO_COMPRESSION_MIME_TYPES="*" +> ``` + +### 3. Compression + Encryption + +Combining encryption and compression is not safe in all setups. +This is particularly so if the compression ratio of your content reveals information about it. +See [CRIME TLS](https://en.wikipedia.org/wiki/CRIME) as an example of this. + +Therefore, compression is disabled when encrypting by default, and must be enabled separately. + +Consult our security experts on [SUBNET](https://min.io/pricing) to help you evaluate if +your setup can use this feature combination safely. + +To enable compression+encryption use: + +```bash +~ mc admin config set myminio compression allow_encryption=on +``` + +Or alternatively through the environment variable `MINIO_COMPRESSION_ALLOW_ENCRYPTION=on`. + +### 4. Excluded Types + +- Already compressed objects are not fit for compression since they do not have compressible patterns. +Such objects do not produce efficient [`LZ compression`](https://en.wikipedia.org/wiki/LZ77_and_LZ78) +which is a fitness factor for a lossless data compression. + +Pre-compressed input typically compresses in excess of 2GiB/s per core, +so performance impact should be minimal even if precompressed data is re-compressed. +Decompressing incompressible data has no significant performance impact. + +Below is a list of common files and content-types which are typically not suitable for compression. + +- Extensions + + | `gz` | (GZIP) | + | `bz2` | (BZIP2) | + | `rar` | (WinRAR) | + | `zip` | (ZIP) | + | `7z` | (7-Zip) | + | `xz` | (LZMA) | + | `mp4` | (MP4) | + | `mkv` | (MKV media) | + | `mov` | (MOV) | + +- Content-Types + + | `video/*` | + | `audio/*` | + | `application/zip` | + | `application/x-gzip` | + | `application/zip` | + | `application/x-bz2` | + | `application/x-compress` | + | `application/x-xz` | + +All files with these extensions and mime types are excluded from compression, +even if compression is enabled for all types. + +## To test the setup + +To test this setup, practice put calls to the server using `mc` and use `mc ls` on +the data directory to view the size of the object. + +## Explore Further + +- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html) +- [Use `aws-cli` with MinIO Server](https://docs.min.io/community/minio-object-store/integrations/aws-cli-with-minio.html) +- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/go/minio-go.html) +- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html) diff --git a/docs/config/README.md b/docs/config/README.md new file mode 100644 index 00000000..94347527 --- /dev/null +++ b/docs/config/README.md @@ -0,0 +1,340 @@ +# MinIO Server Config Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) + +## Configuration Directory + +MinIO stores all its config as part of the server deployment, config is erasure coded on MinIO. On a fresh deployment MinIO automatically generates a new `config` and this config is available to be configured via `mc admin config` command. MinIO also encrypts all the config, IAM and policies content if KMS is configured. Please refer to how to encrypt your config and IAM credentials [here](https://github.com/minio/minio/blob/master/docs/kms/IAM.md). + +### Certificate Directory + +TLS certificates by default are expected to be stored under ``${HOME}/.minio/certs`` directory. You need to place certificates here to enable `HTTPS` based access. Read more about [How to secure access to MinIO server with TLS](https://docs.min.io/community/minio-object-store/operations/network-encryption.html). + +Following is a sample directory structure for MinIO server with TLS certificates. + +```sh +$ mc tree --files ~/.minio +/home/user1/.minio +└─ certs + ├─ CAs + ├─ private.key + └─ public.crt +``` + +You can provide a custom certs directory using `--certs-dir` command line option. + +#### Credentials + +On MinIO admin credentials or root credentials are only allowed to be changed using ENVs namely `MINIO_ROOT_USER` and `MINIO_ROOT_PASSWORD`. + +```sh +export MINIO_ROOT_USER=minio +export MINIO_ROOT_PASSWORD=minio13 +minio server /data +``` + +#### Site + +``` +KEY: +site label the server and its location + +ARGS: +name (string) name for the site e.g. "cal-rack0" +region (string) name of the location of the server e.g. "us-west-1" +comment (sentence) optionally add a comment to this setting +``` + +or environment variables + +``` +KEY: +site label the server and its location + +ARGS: +MINIO_SITE_NAME (string) name for the site e.g. "cal-rack0" +MINIO_SITE_REGION (string) name of the location of the server e.g. "us-west-1" +MINIO_SITE_COMMENT (sentence) optionally add a comment to this setting +``` + +Example: + +```sh +export MINIO_SITE_REGION="us-west-0" +export MINIO_SITE_NAME="sfo-rack-1" +minio server /data +``` + +### Storage Class + +By default, parity for objects with standard storage class is set to `N/2`, and parity for objects with reduced redundancy storage class objects is set to `2`. Read more about storage class support in MinIO server [here](https://github.com/minio/minio/blob/master/docs/erasure/storage-class/README.md). + +``` +KEY: +storage_class define object level redundancy + +ARGS: +standard (string) set the parity count for default standard storage class e.g. "EC:4" +rrs (string) set the parity count for reduced redundancy storage class e.g. "EC:2" +comment (sentence) optionally add a comment to this setting +``` + +or environment variables + +``` +KEY: +storage_class define object level redundancy + +ARGS: +MINIO_STORAGE_CLASS_STANDARD (string) set the parity count for default standard storage class e.g. "EC:4" +MINIO_STORAGE_CLASS_RRS (string) set the parity count for reduced redundancy storage class e.g. "EC:2" +MINIO_STORAGE_CLASS_COMMENT (sentence) optionally add a comment to this setting +``` + +#### Etcd + +MinIO supports storing encrypted IAM assets in etcd, if KMS is configured. Please refer to how to encrypt your config and IAM credentials [here](https://github.com/minio/minio/blob/master/docs/kms/IAM.md). + +> NOTE: if *path_prefix* is set then MinIO will not federate your buckets, namespaced IAM assets are assumed as isolated tenants, only buckets are considered globally unique but performing a lookup with a *bucket* which belongs to a different tenant will fail unlike federated setups where MinIO would port-forward and route the request to relevant cluster accordingly. This is a special feature, federated deployments should not need to set *path_prefix*. + +``` +KEY: +etcd federate multiple clusters for IAM and Bucket DNS + +ARGS: +endpoints* (csv) comma separated list of etcd endpoints e.g. "http://localhost:2379" +path_prefix (path) namespace prefix to isolate tenants e.g. "customer1/" +coredns_path (path) shared bucket DNS records, default is "/skydns" +client_cert (path) client cert for mTLS authentication +client_cert_key (path) client cert key for mTLS authentication +comment (sentence) optionally add a comment to this setting +``` + +or environment variables + +``` +KEY: +etcd federate multiple clusters for IAM and Bucket DNS + +ARGS: +MINIO_ETCD_ENDPOINTS* (csv) comma separated list of etcd endpoints e.g. "http://localhost:2379" +MINIO_ETCD_PATH_PREFIX (path) namespace prefix to isolate tenants e.g. "customer1/" +MINIO_ETCD_COREDNS_PATH (path) shared bucket DNS records, default is "/skydns" +MINIO_ETCD_CLIENT_CERT (path) client cert for mTLS authentication +MINIO_ETCD_CLIENT_CERT_KEY (path) client cert key for mTLS authentication +MINIO_ETCD_COMMENT (sentence) optionally add a comment to this setting +``` + +### API + +By default, there is no limitation on the number of concurrent requests that a server/cluster processes at the same time. However, it is possible to impose such limitation using the API subsystem. Read more about throttling limitation in MinIO server [here](https://github.com/minio/minio/blob/master/docs/throttle/README.md). + +``` +KEY: +api manage global HTTP API call specific features, such as throttling, authentication types, etc. + +ARGS: +requests_max (number) set the maximum number of concurrent requests (default: 'auto') +cluster_deadline (duration) set the deadline for cluster readiness check (default: '10s') +cors_allow_origin (csv) set comma separated list of origins allowed for CORS requests (default: '*') +remote_transport_deadline (duration) set the deadline for API requests on remote transports while proxying between federated instances e.g. "2h" (default: '2h') +list_quorum (string) set the acceptable quorum expected for list operations e.g. "optimal", "reduced", "disk", "strict", "auto" (default: 'strict') +replication_priority (string) set replication priority (default: 'auto') +replication_max_workers (number) set the maximum number of replication workers (default: '500') +replication_max_lrg_workers (number) set the maximum number of replication workers MinIO uses to replicate large objects between sites. (default: '10') +transition_workers (number) set the number of transition workers (default: '100') +stale_uploads_expiry (duration) set to expire stale multipart uploads older than this values (default: '24h') +stale_uploads_cleanup_interval (duration) set to change intervals when stale multipart uploads are expired (default: '6h') +delete_cleanup_interval (duration) set to change intervals when deleted objects are permanently deleted from ".trash" folder (default: '5m') +odirect (boolean) set to enable or disable O_DIRECT for writes under special conditions. NOTE: do not disable O_DIRECT without prior testing (default: 'on') +root_access (boolean) turn 'off' root credential access for all API calls including s3, admin operations (default: 'on') +sync_events (boolean) set to enable synchronous bucket notifications (default: 'off') +object_max_versions (number) set max allowed number of versions per object (default: '9223372036854775807') +``` + +or environment variables + +``` +MINIO_API_REQUESTS_MAX (number) set the maximum number of concurrent requests (default: 'auto') +MINIO_API_CLUSTER_DEADLINE (duration) set the deadline for cluster readiness check (default: '10s') +MINIO_API_CORS_ALLOW_ORIGIN (csv) set comma separated list of origins allowed for CORS requests (default: '*') +MINIO_API_REMOTE_TRANSPORT_DEADLINE (duration) set the deadline for API requests on remote transports while proxying between federated instances e.g. "2h" (default: '2h') +MINIO_API_LIST_QUORUM (string) set the acceptable quorum expected for list operations e.g. "optimal", "reduced", "disk", "strict", "auto" (default: 'strict') +MINIO_API_REPLICATION_PRIORITY (string) set replication priority (default: 'auto') +MINIO_API_REPLICATION_MAX_WORKERS (number) set the maximum number of replication workers (default: '500') +MINIO_API_TRANSITION_WORKERS (number) set the number of transition workers (default: '100') +MINIO_API_STALE_UPLOADS_EXPIRY (duration) set to expire stale multipart uploads older than this values (default: '24h') +MINIO_API_STALE_UPLOADS_CLEANUP_INTERVAL (duration) set to change intervals when stale multipart uploads are expired (default: '6h') +MINIO_API_DELETE_CLEANUP_INTERVAL (duration) set to change intervals when deleted objects are permanently deleted from ".trash" folder (default: '5m') +MINIO_API_ODIRECT (boolean) set to enable or disable O_DIRECT for writes under special conditions. NOTE: do not disable O_DIRECT without prior testing (default: 'on') +MINIO_API_ROOT_ACCESS (boolean) turn 'off' root credential access for all API calls including s3, admin operations (default: 'on') +MINIO_API_SYNC_EVENTS (boolean) set to enable synchronous bucket notifications (default: 'off') +MINIO_API_OBJECT_MAX_VERSIONS (number) set max allowed number of versions per object (default: '9223372036854775807') +``` + +#### Notifications + +Notification targets supported by MinIO are in the following list. To configure individual targets please refer to more detailed documentation [here](https://docs.min.io/community/minio-object-store/administration/monitoring.html#bucket-notifications). + +``` +notify_webhook publish bucket notifications to webhook endpoints +notify_amqp publish bucket notifications to AMQP endpoints +notify_kafka publish bucket notifications to Kafka endpoints +notify_mqtt publish bucket notifications to MQTT endpoints +notify_nats publish bucket notifications to NATS endpoints +notify_nsq publish bucket notifications to NSQ endpoints +notify_mysql publish bucket notifications to MySQL databases +notify_postgres publish bucket notifications to Postgres databases +notify_elasticsearch publish bucket notifications to Elasticsearch endpoints +notify_redis publish bucket notifications to Redis datastores +``` + +### Accessing configuration + +All configuration changes can be made using [`mc admin config` get/set/reset/export/import commands](https://github.com/minio/mc/blob/master/docs/minio-admin-complete-guide.md). + +#### List all config keys available + +``` +~ mc admin config set myminio/ +``` + +#### Obtain help for each key + +``` +~ mc admin config set myminio/ +``` + +e.g: `mc admin config set myminio/ etcd` returns available `etcd` config args + +``` +~ mc admin config set play/ etcd +KEY: +etcd federate multiple clusters for IAM and Bucket DNS + +ARGS: +endpoints* (csv) comma separated list of etcd endpoints e.g. "http://localhost:2379" +path_prefix (path) namespace prefix to isolate tenants e.g. "customer1/" +coredns_path (path) shared bucket DNS records, default is "/skydns" +client_cert (path) client cert for mTLS authentication +client_cert_key (path) client cert key for mTLS authentication +comment (sentence) optionally add a comment to this setting +``` + +To get ENV equivalent for each config args use `--env` flag + +``` +~ mc admin config set play/ etcd --env +KEY: +etcd federate multiple clusters for IAM and Bucket DNS + +ARGS: +MINIO_ETCD_ENDPOINTS* (csv) comma separated list of etcd endpoints e.g. "http://localhost:2379" +MINIO_ETCD_PATH_PREFIX (path) namespace prefix to isolate tenants e.g. "customer1/" +MINIO_ETCD_COREDNS_PATH (path) shared bucket DNS records, default is "/skydns" +MINIO_ETCD_CLIENT_CERT (path) client cert for mTLS authentication +MINIO_ETCD_CLIENT_CERT_KEY (path) client cert key for mTLS authentication +MINIO_ETCD_COMMENT (sentence) optionally add a comment to this setting +``` + +This behavior is consistent across all keys; each key self-documents itself with valid examples. + +## Dynamic systems without restarting server + +The following sub-systems are dynamic i.e., configuration parameters for each sub-systems can be changed while the server is running without any restarts. + +``` +api manage global HTTP API call specific features, such as throttling, authentication types, etc. +heal manage object healing frequency and bitrot verification checks +scanner manage namespace scanning for usage calculation, lifecycle, healing and more +``` + +> NOTE: if you set any of the following sub-system configuration using ENVs, dynamic behavior is not supported. + +### Usage scanner + +Data usage scanner is enabled by default. The following configuration settings allow for more staggered delay in terms of usage calculation. The scanner adapts to the system speed and completely pauses when the system is under load. It is possible to adjust the speed of the scanner and thereby the latency of updates being reflected. The delays between each operation of the scanner can be adjusted by the `mc admin config set alias/ delay=15.0`. By default the value is `10.0`. This means the scanner will sleep *10x* the time each operation takes. + +In most setups this will keep the scanner slow enough to not impact overall system performance. Setting the `delay` key to a *lower* value will make the scanner faster and setting it to 0 will make the scanner run at full speed (not recommended in production). Setting it to a higher value will make the scanner slower, consuming less resources with the trade off of not collecting metrics for operations like healing and disk usage as fast. + +``` +~ mc admin config set alias/ scanner +KEY: +scanner manage namespace scanning for usage calculation, lifecycle, healing and more + +ARGS: +delay (float) scanner delay multiplier, defaults to '10.0' +max_wait (duration) maximum wait time between operations, defaults to '15s' +cycle (duration) time duration between scanner cycles +``` + +Example: the following setting will decrease the scanner speed by a factor of 3, reducing the system resource use, but increasing the latency of updates being reflected. + +```sh +~ mc admin config set alias/ scanner delay=30.0 +``` + +Once set the scanner settings are automatically applied without the need for server restarts. + +### Healing + +Healing is enabled by default. The following configuration settings allow for more staggered delay in terms of healing. The healing system by default adapts to the system speed and pauses up to '250ms' per object when the system has `max_io` number of concurrent requests. It is possible to adjust the `max_sleep` and `max_io` values thereby increasing the healing speed. The delays between each operation of the healer can be adjusted by the `mc admin config set alias/ heal max_sleep=1s` and maximum concurrent requests allowed before we start slowing things down can be configured with `mc admin config set alias/ heal max_io=30` . By default the wait delay is `250ms` beyond 100 concurrent operations. This means the healer will sleep *250 milliseconds* at max for each heal operation if there are more than *100* concurrent client requests. + +In most setups this is sufficient to heal the content after drive replacements. Setting `max_sleep` to a *lower* value and setting `max_io` to a *higher* value would make heal go faster. + +Each node is responsible of healing its local drives; Each drive will have multiple heal workers which is the quarter of the number of CPU cores of the node or the quarter of the configured nr_requests of the drive (https://www.kernel.org/doc/Documentation/block/queue-sysfs.txt). It is also possible to provide a custom number of workers by using this command: `mc admin config set alias/ heal drive_workers=100` . + + +``` +~ mc admin config set alias/ heal +KEY: +heal manage object healing frequency and bitrot verification checks + +ARGS: +bitrotscan (on|off) perform bitrot scan on drives when checking objects during scanner +max_sleep (duration) maximum sleep duration between objects to slow down heal operation. eg. 2s +max_io (int) maximum IO requests allowed between objects to slow down heal operation. eg. 3 +drive_workers (int) the number of workers per drive to heal a new disk replacement. +``` + +Example: The following settings will increase the heal operation speed by allowing healing operation to run without delay up to `100` concurrent requests, and the maximum delay between each heal operation is set to `300ms`. + +```sh +~ mc admin config set alias/ heal max_sleep=300ms max_io=100 +``` + +Once set the healer settings are automatically applied without the need for server restarts. + +## Environment only settings (not in config) + +### Browser + +Enable or disable access to console web UI. By default it is set to `on`. You may override this field with `MINIO_BROWSER` environment variable. + +Example: + +```sh +export MINIO_BROWSER=off +minio server /data +``` + +### Domain + +By default, MinIO supports path-style requests that are of the format . `MINIO_DOMAIN` environment variable is used to enable virtual-host-style requests. If the request `Host` header matches with `(.+).mydomain.com` then the matched pattern `$1` is used as bucket and the path is used as object. Read more about path-style and virtual-host-style [here](http://docs.aws.amazon.com/AmazonS3/latest/dev/RESTAPI.html). + +Example: + +```sh +export MINIO_DOMAIN=mydomain.com +minio server /data +``` + +For advanced use cases `MINIO_DOMAIN` environment variable supports multiple-domains with comma separated values. + +```sh +export MINIO_DOMAIN=sub1.mydomain.com,sub2.mydomain.com +minio server /data +``` + +## Explore Further + +* [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html) +* [Configure MinIO Server with TLS](https://docs.min.io/community/minio-object-store/operations/network-encryption.html) diff --git a/docs/debugging/README.md b/docs/debugging/README.md new file mode 100644 index 00000000..df01c6e3 --- /dev/null +++ b/docs/debugging/README.md @@ -0,0 +1,138 @@ +# MinIO Server Debugging Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) + +## HTTP Trace + +HTTP tracing can be enabled by using [`mc admin trace`](https://docs.min.io/community/minio-object-store/reference/minio-mc-admin/mc-admin-trace.html) command. + +Example: + +```sh +minio server /data +``` + +Default trace is succinct only to indicate the API operations being called and the HTTP response status. + +```sh +mc admin trace myminio +``` + +To trace entire HTTP request + +```sh +mc admin trace --verbose myminio +``` + +To trace entire HTTP request and also internode communication + +```sh +mc admin trace --all --verbose myminio +``` + +## Subnet Health + +Subnet Health diagnostics help ensure that the underlying infrastructure that runs MinIO is configured correctly, and is functioning properly. This test is one-shot long running one, that is recommended to be run as soon as the cluster is first provisioned, and each time a failure scenario is encountered. Note that the test incurs majority of the available resources on the system. Care must be taken when using this to debug failure scenario, so as to prevent larger outages. Health tests can be triggered using `mc support diagnostics` command. + +Example: + +```sh +minio server /data{1...4} +``` + +The command takes no flags + +```sh +mc support diagnostics myminio/ +``` + +The output printed will be of the form + +```sh +● Admin Info ... ✔ +● CPU ... ✔ +● Disk Hardware ... ✔ +● Os Info ... ✔ +● Mem Info ... ✔ +● Process Info ... ✔ +● Config ... ✔ +● Drive ... ✔ +● Net ... ✔ +********************************************************************************* + WARNING!! + ** THIS FILE MAY CONTAIN SENSITIVE INFORMATION ABOUT YOUR ENVIRONMENT ** + ** PLEASE INSPECT CONTENTS BEFORE SHARING IT ON ANY PUBLIC FORUM ** +********************************************************************************* +mc: Health data saved to dc-11-health_20200321053323.json.gz +``` + +The gzipped output contains debugging information for your system + +## Decoding Metadata + +Metadata is stored in `xl.meta` files for erasure coded objects. Each disk in the set containing the object has this file. The file format is a binary format and therefore requires tools to view values. + +### Installing xl-meta + +To install, [Go](https://golang.org/dl/) must be installed. Once installed, execute this to install the binary: + +```bash +go install github.com/minio/minio/docs/debugging/xl-meta@latest +``` + +### Using xl-meta + +Executing `xl-meta` will look for an `xl.meta` in the current folder and decode it to JSON. It is also possible to specify multiple files or wildcards, for example `xl-meta ./**/xl.meta` will output decoded metadata recursively. It is possible to view what inline data is stored inline in the metadata using `--data` parameter `xl-meta -data xl.json` will display an id -> data size. To export inline data to a file use the `--export` option. + +### Remotely Inspecting backend data + +`mc support inspect` allows collecting files based on *path* from all backend drives. Matching files will be collected in a zip file with their respective host+drive+path. A MinIO host from October 2021 or later is required for full functionality. Syntax is `mc support inspect ALIAS/path/to/files`. This can for example be used to collect `xl.meta` from objects that are misbehaving. To collect `xl.meta` from a specific object, for example placed at `ALIAS/bucket/path/to/file.txt` append `/xl.meta`, for instance `mc support inspect ALIAS/bucket/path/to/file.txt/xl.meta`. All files can be collected, so this can also be used to retrieve `part.*` files, etc. + +Wildcards can be used, for example `mc support inspect ALIAS/bucket/path/**/xl.meta` will collect all `xl.meta` recursively. `mc support inspect ALIAS/bucket/path/to/file.txt/*/part.*` will collect parts for all versions for the object located at `bucket/path/to/file.txt`. + +`xl-meta` accepts zip files as input and will output all `xl.meta` files found within the archive. For example: + +``` +$ mc support inspect play/test123/test*/xl.meta +mc: File data successfully downloaded as inspect.6f96b336.zip +$ xl-meta inspect.6f96b336.zip +{ + "bf6178f9-4014-4008-9699-86f2fac62226/test123/testw3c.pdf/xl.meta": {"Versions":[{"Type":1,"V2Obj":{"ID":"aGEA/ZUOR4ueRIZsAgfDqA==","DDir":"9MMwM47bS+K6KvQqN3hlDw==","EcAlgo":1,"EcM":2,"EcN":2,"EcBSize":1048576,"EcIndex":4,"EcDist":[4,1,2,3],"CSumAlgo":1,"PartNums":[1],"PartETags":[""],"PartSizes":[101974],"PartASizes":[176837],"Size":101974,"MTime":1634106631319256439,"MetaSys":{"X-Minio-Internal-compression":"a2xhdXNwb3N0L2NvbXByZXNzL3My","X-Minio-Internal-actual-size":"MTc2ODM3","x-minio-internal-objectlock-legalhold-timestamp":"MjAyMS0xMC0xOVQyMjozNTo0Ni4zNTE4MDU3NTda"},"MetaUsr":{"x-amz-object-lock-mode":"COMPLIANCE","x-amz-object-lock-retain-until-date":"2022-10-13T06:30:31.319Z","etag":"67ed8f49b7137cb957858ce468f2e79e","content-type":"application/pdf","x-amz-object-lock-legal-hold":"OFF"}}}]}, + "fe012443-6ba9-4ef2-bb94-b729d2060c78/test123/testw3c.pdf/xl.meta": {"Versions":[{"Type":1,"V2Obj":{"ID":"aGEA/ZUOR4ueRIZsAgfDqA==","DDir":"9MMwM47bS+K6KvQqN3hlDw==","EcAlgo":1,"EcM":2,"EcN":2,"EcBSize":1048576,"EcIndex":1,"EcDist":[4,1,2,3],"CSumAlgo":1,"PartNums":[1],"PartETags":[""],"PartSizes":[101974],"PartASizes":[176837],"Size":101974,"MTime":1634106631319256439,"MetaSys":{"X-Minio-Internal-compression":"a2xhdXNwb3N0L2NvbXByZXNzL3My","X-Minio-Internal-actual-size":"MTc2ODM3","x-minio-internal-objectlock-legalhold-timestamp":"MjAyMS0xMC0xOVQyMjozNTo0Ni4zNTE4MDU3NTda"},"MetaUsr":{"content-type":"application/pdf","x-amz-object-lock-legal-hold":"OFF","x-amz-object-lock-mode":"COMPLIANCE","x-amz-object-lock-retain-until-date":"2022-10-13T06:30:31.319Z","etag":"67ed8f49b7137cb957858ce468f2e79e"}}}]}, + "5dcb9f38-08ea-4728-bb64-5cecc7102436/test123/testw3c.pdf/xl.meta": {"Versions":[{"Type":1,"V2Obj":{"ID":"aGEA/ZUOR4ueRIZsAgfDqA==","DDir":"9MMwM47bS+K6KvQqN3hlDw==","EcAlgo":1,"EcM":2,"EcN":2,"EcBSize":1048576,"EcIndex":2,"EcDist":[4,1,2,3],"CSumAlgo":1,"PartNums":[1],"PartETags":[""],"PartSizes":[101974],"PartASizes":[176837],"Size":101974,"MTime":1634106631319256439,"MetaSys":{"X-Minio-Internal-compression":"a2xhdXNwb3N0L2NvbXByZXNzL3My","X-Minio-Internal-actual-size":"MTc2ODM3","x-minio-internal-objectlock-legalhold-timestamp":"MjAyMS0xMC0xOVQyMjozNTo0Ni4zNTE4MDU3NTda"},"MetaUsr":{"content-type":"application/pdf","x-amz-object-lock-legal-hold":"OFF","x-amz-object-lock-mode":"COMPLIANCE","x-amz-object-lock-retain-until-date":"2022-10-13T06:30:31.319Z","etag":"67ed8f49b7137cb957858ce468f2e79e"}}}]}, + "48beacc7-4be0-4660-9026-4eceaf147504/test123/testw3c.pdf/xl.meta": {"Versions":[{"Type":1,"V2Obj":{"ID":"aGEA/ZUOR4ueRIZsAgfDqA==","DDir":"9MMwM47bS+K6KvQqN3hlDw==","EcAlgo":1,"EcM":2,"EcN":2,"EcBSize":1048576,"EcIndex":3,"EcDist":[4,1,2,3],"CSumAlgo":1,"PartNums":[1],"PartETags":[""],"PartSizes":[101974],"PartASizes":[176837],"Size":101974,"MTime":1634106631319256439,"MetaSys":{"X-Minio-Internal-compression":"a2xhdXNwb3N0L2NvbXByZXNzL3My","X-Minio-Internal-actual-size":"MTc2ODM3","x-minio-internal-objectlock-legalhold-timestamp":"MjAyMS0xMC0xOVQyMjozNTo0Ni4zNTE4MDU3NTda"},"MetaUsr":{"x-amz-object-lock-retain-until-date":"2022-10-13T06:30:31.319Z","x-amz-object-lock-legal-hold":"OFF","etag":"67ed8f49b7137cb957858ce468f2e79e","content-type":"application/pdf","x-amz-object-lock-mode":"COMPLIANCE"}}}]} +} +``` + +Optionally `--encrypt` can be specified. This will output an encrypted file and a decryption key: + +``` +$ mc support inspect --encrypt play/test123/test*/*/part.* +mc: Encrypted file data successfully downloaded as inspect.ad2b43d8.enc +mc: Decryption key: ad2b43d847fdb14e54c5836200177f7158b3f745433525f5d23c0e0208e50c9948540b54 + +mc: The decryption key will ONLY be shown here. It cannot be recovered. +mc: The encrypted file can safely be shared without the decryption key. +mc: Even with the decryption key, data stored with encryption cannot be accessed. +``` + +This file can be decrypted using the decryption tool below: + +### Installing decryption tool + +To install, [Go](https://golang.org/dl/) must be installed. + +Once installed, execute this to install the binary: + +```bash +go install github.com/minio/minio/docs/debugging/inspect@latest +``` + +### Usage + +To decrypt the file above: + +``` +$ inspect -key=ad2b43d847fdb14e54c5836200177f7158b3f745433525f5d23c0e0208e50c9948540b54 inspect.ad2b43d8.enc +Output decrypted to inspect.ad2b43d8.zip +``` + +If `--key` is not specified an interactive prompt will ask for it. The file name will contain the beginning of the key. This can be used to verify that the key is for the encrypted file. diff --git a/docs/debugging/build.sh b/docs/debugging/build.sh new file mode 100755 index 00000000..204e0ea1 --- /dev/null +++ b/docs/debugging/build.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +export CGO_ENABLED=0 +for dir in docs/debugging/*/; do + bin=$(basename ${dir}) + go build -C ${dir} -o ${PWD}/${bin} +done diff --git a/docs/distributed/CONFIG.md b/docs/distributed/CONFIG.md new file mode 100644 index 00000000..bb029dda --- /dev/null +++ b/docs/distributed/CONFIG.md @@ -0,0 +1,91 @@ +## MinIO configuration YAML + +MinIO now supports starting the server arguments and configuration via a YAML configuration file. This YAML configuration describes everything that can be configured in a MinIO setup, such as '--address', '--console-address' and command line arguments for the MinIO server. + +Historically everything to MinIO was provided via command arguments for the hostnames and the drives via an ellipses syntax such as `minio server http://host{1...4}/disk{1...4}` this requirement added an additional burden to have sequential hostnames for us to make sure that we can provide horizontal distribution, however we have come across situations where sometimes this is not feasible and there are no easier alternatives without modifying /etc/hosts on the host system as root user. Many times in airgapped deployments this is not allowed or requires audits and approvals. + +MinIO server configuration file allows users to provide topology that allows for heterogeneous hostnames, allowing MinIO to deployed in pre-existing environments without any further OS level configurations. + +### Usage + +``` +minio server --config config.yaml +``` + +Lets you start MinIO server with all inputs to start MinIO server provided via this configuration file, once the configuration file is provided all other pre-existing values on disk for configuration are overridden by the new values set in this configuration file. + +Following is an example YAML configuration structure. +```yaml +version: v2 +address: ":9000" +rootUser: "minioadmin" +rootPassword: "minioadmin" +console-address: ":9001" +certs-dir: "/home/user/.minio/certs/" +pools: # Specify the nodes and drives with pools + - args: + - "https://server-example-pool1:9000/mnt/disk{1...4}/" + - "https://server{1...2}-pool1:9000/mnt/disk{1...4}/" + - "https://server3-pool1:9000/mnt/disk{1...4}/" + - "https://server4-pool1:9000/mnt/disk{1...4}/" + - args: + - "https://server-example-pool2:9000/mnt/disk{1...4}/" + - "https://server{1...2}-pool2:9000/mnt/disk{1...4}/" + - "https://server3-pool2:9000/mnt/disk{1...4}/" + - "https://server4-pool2:9000/mnt/disk{1...4}/" + # more args + +options: + ftp: # settings for MinIO to act as an ftp server + address: ":8021" + passive-port-range: "30000-40000" + sftp: # settings for MinIO to act as an sftp server + address: ":8022" + ssh-private-key: "/home/user/.ssh/id_rsa" +``` + +If you are using the config `v1` YAML you should migrate your `pools:` field values to the following format + +`v1` format +```yaml +pools: # Specify the nodes and drives with pools + - + - "https://server-example-pool1:9000/mnt/disk{1...4}/" + - "https://server{1...2}-pool1:9000/mnt/disk{1...4}/" + - "https://server3-pool1:9000/mnt/disk{1...4}/" + - "https://server4-pool1:9000/mnt/disk{1...4}/" +``` + +to `v2` format + +```yaml +pools: + - args: + - "https://server-example-pool1:9000/mnt/disk{1...4}/" + - "https://server{1...2}-pool1:9000/mnt/disk{1...4}/" + - "https://server3-pool1:9000/mnt/disk{1...4}/" + - "https://server4-pool1:9000/mnt/disk{1...4}/" + set-drive-count: 4 # Advanced option, must be used under guidance from MinIO team. +``` + +### Things to know + +- Fields such as `version` and `pools` are mandatory, however all other fields are optional. +- Each pool expects a minimum of 2 nodes per pool, and unique non-repeating hosts for each argument. +- Each pool expects each host in this pool has the same number of drives specified as any other host. +- Mixing `local-path` and `distributed-path` is not allowed, doing so would cause MinIO to refuse starting the server. +- Ellipses and bracket notation (e.g. `{1...10}`) are allowed. + +> NOTE: MinIO environmental variables still take precedence over the `config.yaml` file, however `config.yaml` is preferred over MinIO internal config KV settings via `mc admin config set alias/ `. + +### TODO + +In subsequent releases we are planning to extend this to provide things like + +- Reload() of MinIO server arguments without fully restarting the process. + +- Expanding 1 node at a time by automating the process of creating a new pool + and decommissioning to provide a functionality that smaller deployments + care about. + +- Fully allow bracket notation (e.g. `{a,c,f}`) to have multiple entries on one line. \ No newline at end of file diff --git a/docs/distributed/DECOMMISSION.md b/docs/distributed/DECOMMISSION.md new file mode 100644 index 00000000..177b9c1a --- /dev/null +++ b/docs/distributed/DECOMMISSION.md @@ -0,0 +1,122 @@ +# Decommissioning + +Decommissiong is a mechanism in MinIO to drain older pools (usually with old hardware) and migrate the content from such pools to a newer pools (usually better hardware). Decommissioning spreads the data across all pools - for example, if you decommission `pool1`, all the data from `pool1` spreads across `pool2` and `pool3`. + +## Features + +- A pool in decommission still allows READ access to all its contents, newer WRITEs will automatically be scheduled to only pools not in decommission status. +- All versioned buckets maintain the same order for "versions" for each object after being decommissioned to the other pools. +- A pool interrupted during the decommission process, such as for a cluster restart, resumes from where it left off. + +## How to decommission a pool + +``` +λ mc admin decommission start alias/ http://minio{1...2}/data{1...4} +``` + +## Status decommissioning a pool + +### Decommissioning without args lists all pools + +``` +λ mc admin decommission status alias/ +┌─────┬─────────────────────────────────┬──────────────────────────────────┬────────┐ +│ ID │ Pools │ Capacity │ Status │ +│ 1st │ http://minio{1...2}/data{1...4} │ 439 GiB (used) / 561 GiB (total) │ Active │ +│ 2nd │ http://minio{3...4}/data{1...4} │ 329 GiB (used) / 421 GiB (total) │ Active │ +└─────┴─────────────────────────────────┴──────────────────────────────────┴────────┘ +``` + +### Decommissioning status + +``` +λ mc admin decommission status alias/ http://minio{1...2}/data{1...4} +Decommissioning rate at 36 MiB/sec [4 TiB/50 TiB] +Started: 1 minute ago +``` + +Once it is **Complete** + +``` +λ mc admin decommission status alias/ http://minio{1...2}/data{1...4} +Decommission of pool http://minio{1...2}/data{1...4} is complete, you may now remove it from server command line +``` + +### A pool not under decommissioning will throw an error + +``` +λ mc admin decommission status alias/ http://minio{1...2}/data{1...4} +ERROR: This pool is not scheduled for decommissioning currently. +``` + +## Canceling a decommission + +Stop an on-going decommission in progress, mainly used in situations when the load may be too high and you may want to schedule the decommission at a later point in time. + +`mc admin decommission cancel` without an argument, lists out any on-going decommission in progress. + +``` +λ mc admin decommission cancel alias/ +┌─────┬─────────────────────────────────┬──────────────────────────────────┬──────────┐ +│ ID │ Pools │ Capacity │ Status │ +│ 1st │ http://minio{1...2}/data{1...4} │ 439 GiB (used) / 561 GiB (total) │ Draining │ +└─────┴─────────────────────────────────┴──────────────────────────────────┴──────────┘ +``` + +> NOTE: Canceled decommission will not make the pool active again, since we might have potentially partial namespace on the other pools, to avoid this scenario be absolutely sure to make decommissioning a planned well thought activity. This is not to be run on a daily basis. + +``` +λ mc admin decommission cancel alias/ http://minio{1...2}/data{1...4} +┌─────┬─────────────────────────────────┬──────────────────────────────────┬────────────────────┐ +│ ID │ Pools │ Capacity │ Status │ +│ 1st │ http://minio{1...2}/data{1...4} │ 439 GiB (used) / 561 GiB (total) │ Draining(Canceled) │ +└─────┴─────────────────────────────────┴──────────────────────────────────┴────────────────────┘ +``` + +If the decommission process fails for any reason, the status indicates failed. + +``` +λ mc admin decommission status alias/ +┌─────┬─────────────────────────────────┬──────────────────────────────────┬──────────────────┐ +│ ID │ Pools │ Capacity │ Status │ +│ 1st │ http://minio{1...2}/data{1...4} │ 439 GiB (used) / 561 GiB (total) │ Draining(Failed) │ +│ 2nd │ http://minio{3...4}/data{1...4} │ 329 GiB (used) / 421 GiB (total) │ Active │ +└─────┴─────────────────────────────────┴──────────────────────────────────┴──────────────────┘ +``` + +## Restart a canceled or failed decommission + +``` +λ mc admin decommission start alias/ http://minio{1...2}/data{1...4} +``` + +## When decommission is 'Complete' + +Once decommission is complete, it will be indicated with *Complete* status. *Complete* means that now you can now safely remove the first pool argument from the MinIO command line. + +``` +λ mc admin decommission status alias/ +┌─────┬─────────────────────────────────┬──────────────────────────────────┬──────────┐ +│ ID │ Pools │ Capacity │ Status │ +│ 1st │ http://minio{1...2}/data{1...4} │ 439 GiB (used) / 561 GiB (total) │ Complete │ +│ 2nd │ http://minio{3...4}/data{1...4} │ 329 GiB (used) / 421 GiB (total) │ Active │ +└─────┴─────────────────────────────────┴──────────────────────────────────┴──────────┘ +``` + +- On baremetal setups, if you have `MINIO_VOLUMES="http://minio{1...2}/data{1...4} http://minio{3...4}/data{1...4}"`, you can remove the first argument `http://minio{1...2}/data{1...4}` to update your `MINIO_VOLUMES` setting, then restart all the servers in the setup in parallel using `systemctl restart minio`. + +- On Kubernetes setups, the statefulset specification needs to be modified by changing the command line input for the MinIO container. Once the relevant changes are done, proceed to execute `kubectl apply -f statefulset.yaml`. + +- On Operator based MinIO deployments, you need to modify the `tenant.yaml` specification and modify the `pools:` section from two entries to a single entry. After making relevant changes, proceed to execute `kubectl apply -f tenant.yaml`. + +> Without a 'Complete' status any 'Active' or 'Draining' pool(s) are not allowed to be removed once configured. + +## NOTE + +- Empty delete markers (such as for objects with no other successor versions) do not transition to the new pool to avoid creating empty metadata on the other pool(s). If you believe transitioning empty delete markers is required, open a GitHub issue. + +## TODO + +- Richer progress UI is not present at the moment, this will be addressed in subsequent releases. Currently however a RATE of data transfer and usage increase is displayed via `mc`. +- Transitioned Hot Tier's as pooled setups are not currently supported, attempting to decommission buckets with ILM Transition will be rejected by the server. This will be supported in future releases. +- Embedded Console UI does not support Decommissioning through the UI yet. This will be supported in future releases. diff --git a/docs/distributed/DESIGN.md b/docs/distributed/DESIGN.md new file mode 100644 index 00000000..4c663d4b --- /dev/null +++ b/docs/distributed/DESIGN.md @@ -0,0 +1,166 @@ +# Distributed Server Design Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +This document explains the design, architecture and advanced use cases of the MinIO distributed server. + +## Command-line + +``` +NAME: + minio server - start object storage server + +USAGE: + minio server [FLAGS] DIR1 [DIR2..] + minio server [FLAGS] DIR{1...64} + minio server [FLAGS] DIR{1...64} DIR{65...128} + +DIR: + DIR points to a directory on a filesystem. When you want to combine + multiple drives into a single large system, pass one directory per + filesystem separated by space. You may also use a '...' convention + to abbreviate the directory arguments. Remote directories in a + distributed setup are encoded as HTTP(s) URIs. +``` + +## Common usage + +Standalone erasure coded configuration with 4 sets with 16 drives each. + +``` +minio server dir{1...64} +``` + +Distributed erasure coded configuration with 64 sets with 16 drives each. + +``` +minio server http://host{1...16}/export{1...64} +``` + +## Architecture + +Expansion of ellipses and choice of erasure sets based on this expansion is an automated process in MinIO. Here are some of the details of our underlying erasure coding behavior. + +- Erasure coding used by MinIO is [Reed-Solomon](https://github.com/klauspost/reedsolomon) erasure coding scheme, which has a total shard maximum of 256 i.e 128 data and 128 parity. MinIO design goes beyond this limitation by doing some practical architecture choices. + +- Erasure set is a single erasure coding unit within a MinIO deployment. An object is sharded within an erasure set. Erasure set size is automatically calculated based on the number of drives. MinIO supports unlimited number of drives but each erasure set can be up to 16 drives and a minimum of 2 drives. + +- We limited the number of drives to 16 for erasure set because, erasure code shards more than 16 can become chatty and do not have any performance advantages. Additionally since 16 drive erasure set gives you tolerance of 8 drives per object by default which is plenty in any practical scenario. + +- Choice of erasure set size is automatic based on the number of drives available, let's say for example if there are 32 servers and 32 drives which is a total of 1024 drives. In this scenario 16 becomes the erasure set size. This is decided based on the greatest common divisor (GCD) of acceptable erasure set sizes ranging from *4 to 16*. + +- *If total drives has many common divisors the algorithm chooses the minimum amounts of erasure sets possible for a erasure set size of any N*. In the example with 1024 drives - 4, 8, 16 are GCD factors. With 16 drives we get a total of 64 possible sets, with 8 drives we get a total of 128 possible sets, with 4 drives we get a total of 256 possible sets. So algorithm automatically chooses 64 sets, which is *16* 64 = 1024* drives in total. + +- *If total number of nodes are of odd number then GCD algorithm provides affinity towards odd number erasure sets to provide for uniform distribution across nodes*. This is to ensure that same number of drives are pariticipating in any erasure set. For example if you have 2 nodes with 180 drives then GCD is 15 but this would lead to uneven distribution, one of the nodes would participate more drives. To avoid this the affinity is given towards nodes which leads to next best GCD factor of 12 which provides uniform distribution. + +- In this algorithm, we also make sure that we spread the drives out evenly. MinIO server expands ellipses passed as arguments. Here is a sample expansion to demonstrate the process. + +``` +minio server http://host{1...2}/export{1...8} +``` + +Expected expansion + +``` +> http://host1/export1 +> http://host2/export1 +> http://host1/export2 +> http://host2/export2 +> http://host1/export3 +> http://host2/export3 +> http://host1/export4 +> http://host2/export4 +> http://host1/export5 +> http://host2/export5 +> http://host1/export6 +> http://host2/export6 +> http://host1/export7 +> http://host2/export7 +> http://host1/export8 +> http://host2/export8 +``` + +*A noticeable trait of this expansion is that it chooses unique hosts such the setup provides maximum protection and availability.* + +- Choosing an erasure set for the object is decided during `PutObject()`, object names are used to find the right erasure set using the following pseudo code. + +```go +// hashes the key returning an integer. +func sipHashMod(key string, cardinality int, id [16]byte) int { + if cardinality <= 0 { + return -1 + } + sip := siphash.New(id[:]) + sip.Write([]byte(key)) + return int(sip.Sum64() % uint64(cardinality)) +} +``` + +Input for the key is the object name specified in `PutObject()`, returns a unique index. This index is one of the erasure sets where the object will reside. This function is a consistent hash for a given object name i.e for a given object name the index returned is always the same. + +- Write and Read quorum are required to be satisfied only across the erasure set for an object. Healing is also done per object within the erasure set which contains the object. + +- MinIO does erasure coding at the object level not at the volume level, unlike other object storage vendors. This allows applications to choose different storage class by setting `x-amz-storage-class=STANDARD/REDUCED_REDUNDANCY` for each object uploads so effectively utilizing the capacity of the cluster. Additionally these can also be enforced using IAM policies to make sure the client uploads with correct HTTP headers. + +- MinIO also supports expansion of existing clusters in server pools. Each pool is a self contained entity with same SLA's (read/write quorum) for each object as original cluster. By using the existing namespace for lookup validation MinIO ensures conflicting objects are not created. When no such object exists then MinIO simply uses the least used pool to place new objects. + +### There are no limits on how many server pools can be combined + +``` +minio server http://host{1...32}/export{1...32} http://host{1...12}/export{1...12} +``` + +In above example there are two server pools + +- 32 * 32 = 1024 drives pool1 +- 12 * 12 = 144 drives pool2 + +> Notice the requirement of common SLA here original cluster had 1024 drives with 16 drives per erasure set with default parity of '4', second pool is expected to have a minimum of 8 drives per erasure set to match the original cluster SLA (parity count) of '4'. '12' drives stripe per erasure set in the second pool satisfies the original pool's parity count. + +Refer to the sizing guide with details on the default parity count chosen for different erasure stripe sizes [here](https://github.com/minio/minio/blob/master/docs/distributed/SIZING.md) + +MinIO places new objects in server pools based on proportionate free space, per pool. Following pseudo code demonstrates this behavior. + +```go +func getAvailablePoolIdx(ctx context.Context) int { + serverPools := z.getServerPoolsAvailableSpace(ctx) + total := serverPools.TotalAvailable() + // choose when we reach this many + choose := rand.Uint64() % total + atTotal := uint64(0) + for _, pool := range serverPools { + atTotal += pool.Available + if atTotal > choose && pool.Available > 0 { + return pool.Index + } + } + // Should not happen, but print values just in case. + panic(fmt.Errorf("reached end of serverPools (total: %v, atTotal: %v, choose: %v)", total, atTotal, choose)) +} +``` + +## Other usages + +### Advanced use cases with multiple ellipses + +Standalone erasure coded configuration with 4 sets with 16 drives each, which spawns drives across controllers. + +``` +minio server /mnt/controller{1...4}/data{1...16} +``` + +Standalone erasure coded configuration with 16 sets, 16 drives per set, across mounts and controllers. + +``` +minio server /mnt{1...4}/controller{1...4}/data{1...16} +``` + +Distributed erasure coded configuration with 2 sets, 16 drives per set across hosts. + +``` +minio server http://host{1...32}/disk1 +``` + +Distributed erasure coded configuration with rack level redundancy 32 sets in total, 16 drives per set. + +``` +minio server http://rack{1...4}-host{1...8}.example.net/export{1...16} +``` diff --git a/docs/distributed/README.md b/docs/distributed/README.md new file mode 100644 index 00000000..0ebf6cfc --- /dev/null +++ b/docs/distributed/README.md @@ -0,0 +1,109 @@ +# Distributed MinIO Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) + +MinIO in distributed mode lets you pool multiple drives (even on different machines) into a single object storage server. As drives are distributed across several nodes, distributed MinIO can withstand multiple node failures and yet ensure full data protection. + +## Why distributed MinIO? + +MinIO in distributed mode can help you setup a highly-available storage system with a single object storage deployment. With distributed MinIO, you can optimally use storage devices, irrespective of their location in a network. + +### Data protection + +Distributed MinIO provides protection against multiple node/drive failures and [bit rot](https://github.com/minio/minio/blob/master/docs/erasure/README.md#what-is-bit-rot-protection) using [erasure code](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html). As the minimum drives required for distributed MinIO is 2 (same as minimum drives required for erasure coding), erasure code automatically kicks in as you launch distributed MinIO. + +If one or more drives are offline at the start of a PutObject or NewMultipartUpload operation the object will have additional data protection bits added automatically to provide additional safety for these objects. + +### High availability + +A stand-alone MinIO server would go down if the server hosting the drives goes offline. In contrast, a distributed MinIO setup with _m_ servers and _n_ drives will have your data safe as long as _m/2_ servers or _m*n_/2 or more drives are online. + +For example, an 16-server distributed setup with 200 drives per node would continue serving files, up to 4 servers can be offline in default configuration i.e around 800 drives down MinIO would continue to read and write objects. + +Refer to sizing guide for more understanding on default values chosen depending on your erasure stripe size [here](https://github.com/minio/minio/blob/master/docs/distributed/SIZING.md). Parity settings can be changed using [storage classes](https://github.com/minio/minio/tree/master/docs/erasure/storage-class). + +### Consistency Guarantees + +MinIO follows strict **read-after-write** and **list-after-write** consistency model for all i/o operations both in distributed and standalone modes. This consistency model is only guaranteed if you use disk filesystems such as xfs, zfs or btrfs etc.. for distributed setup. + +**In our tests we also found ext4 does not honor POSIX O_DIRECT/Fdatasync semantics, ext4 trades performance for consistency guarantees. Please avoid ext4 in your setup.** + +**If MinIO distributed setup is using NFS volumes underneath it is not guaranteed MinIO will provide these consistency guarantees since NFS is not strictly consistent (If you must use NFS we recommend that you at least use NFSv4 instead of NFSv3 for relatively better outcomes).** + +## Get started + +If you're aware of stand-alone MinIO set up, the process remains largely the same. MinIO server automatically switches to stand-alone or distributed mode, depending on the command line parameters. + +### 1. Prerequisites + +Install MinIO either on Kubernetes or Distributed Linux. + +Install MinIO on Kubernetes: + +- [MinIO Quickstart Guide for Kubernetes](https://docs.min.io/community/minio-object-store/operations/deployments/kubernetes.html). +- [Deploy a Tenant from the MinIO Operator](https://docs.min.io/community/minio-object-store/operations/deployments/k8s-deploy-minio-tenant-on-kubernetes.html) + +Install Distributed MinIO on Linux: +- [Deploy Distributed MinIO on Linux](https://docs.min.io/community/minio-object-store/operations/install-deploy-manage/deploy-minio-multi-node-multi-drive.html#deploy-distributed-minio) + +### 2. Run distributed MinIO + +To start a distributed MinIO instance, you just need to pass drive locations as parameters to the minio server command. Then, you’ll need to run the same command on all the participating nodes. + +**NOTE:** + +- All the nodes running distributed MinIO should share a common root credentials, for the nodes to connect and trust each other. To achieve this, it is **recommended** to export root user and root password as environment variables, `MINIO_ROOT_USER` and `MINIO_ROOT_PASSWORD`, on all the nodes before executing MinIO server command. If not exported, default `minioadmin/minioadmin` credentials shall be used. +- **MinIO creates erasure-coding sets of _2_ to _16_ drives per set. The number of drives you provide in total must be a multiple of one of those numbers.** +- **MinIO chooses the largest EC set size which divides into the total number of drives or total number of nodes given - making sure to keep the uniform distribution i.e each node participates equal number of drives per set**. +- **Each object is written to a single EC set, and therefore is spread over no more than 16 drives.** +- **All the nodes running distributed MinIO setup are recommended to be homogeneous, i.e. same operating system, same number of drives and same network interconnects.** +- MinIO distributed mode requires **fresh directories**. If required, the drives can be shared with other applications. You can do this by using a sub-directory exclusive to MinIO. For example, if you have mounted your volume under `/export`, pass `/export/data` as arguments to MinIO server. +- The IP addresses and drive paths below are for demonstration purposes only, you need to replace these with the actual IP addresses and drive paths/folders. +- Servers running distributed MinIO instances should be less than 15 minutes apart. You can enable [NTP](http://www.ntp.org/) service as a best practice to ensure same times across servers. +- `MINIO_DOMAIN` environment variable should be defined and exported for bucket DNS style support. +- Running Distributed MinIO on **Windows** operating system is considered **experimental**. Please proceed with caution. + +Example 1: Start distributed MinIO instance on n nodes with m drives each mounted at `/export1` to `/exportm` (pictured below), by running this command on all the n nodes: + +![Distributed MinIO, n nodes with m drives each](https://github.com/minio/minio/blob/master/docs/screenshots/Architecture-diagram_distributed_nm.png?raw=true) + +### GNU/Linux and macOS + +```sh +export MINIO_ROOT_USER= +export MINIO_ROOT_PASSWORD= +minio server http://host{1...n}/export{1...m} +``` + +> **NOTE:** In above example `n` and `m` represent positive integers, _do not copy paste and expect it work make the changes according to local deployment and setup_. +> **NOTE:** `{1...n}` shown have 3 dots! Using only 2 dots `{1..n}` will be interpreted by your shell and won't be passed to MinIO server, affecting the erasure coding order, which would impact performance and high availability. **Always use ellipses syntax `{1...n}` (3 dots!) for optimal erasure-code distribution** + +### Expanding existing distributed setup + +MinIO supports expanding distributed erasure coded clusters by specifying new set of clusters on the command-line as shown below: + +```sh +export MINIO_ROOT_USER= +export MINIO_ROOT_PASSWORD= +minio server http://host{1...n}/export{1...m} http://host{o...z}/export{1...m} +``` + +For example: + +``` +minio server http://host{1...4}/export{1...16} http://host{5...12}/export{1...16} +``` + +Now the server has expanded total storage by _(newly_added_servers\*m)_ more drives, taking the total count to _(existing_servers\*m)+(newly_added_servers\*m)_ drives. New object upload requests automatically start using the least used cluster. This expansion strategy works endlessly, so you can perpetually expand your clusters as needed. When you restart, it is immediate and non-disruptive to the applications. Each group of servers in the command-line is called a pool. There are 2 server pools in this example. New objects are placed in server pools in proportion to the amount of free space in each pool. Within each pool, the location of the erasure-set of drives is determined based on a deterministic hashing algorithm. + +> **NOTE:** **Each pool you add must have the same erasure coding parity configuration as the original pool, so the same data redundancy SLA is maintained.** + +## 3. Test your setup + +To test this setup, access the MinIO server via browser or [`mc`](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart). + +## Explore Further + +- [MinIO Erasure Code QuickStart Guide](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html) +- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html) +- [Use `aws-cli` with MinIO Server](https://docs.min.io/community/minio-object-store/integrations/aws-cli-with-minio.html) +- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/go/minio-go.html) +- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html) diff --git a/docs/distributed/SIZING.md b/docs/distributed/SIZING.md new file mode 100644 index 00000000..0ed55ad4 --- /dev/null +++ b/docs/distributed/SIZING.md @@ -0,0 +1,39 @@ +# Erasure code sizing guide + +## Toy Setups + +Capacity constrained environments, MinIO will work but not recommended for production. + +| servers | drives (per node) | stripe_size | parity chosen (default) | tolerance for reads (servers) | tolerance for writes (servers) | +|--------:|------------------:|------------:|------------------------:|------------------------------:|-------------------------------:| +| 1 | 1 | 1 | 0 | 0 | 0 | +| 1 | 4 | 4 | 2 | 0 | 0 | +| 4 | 1 | 4 | 2 | 2 | 1 | +| 5 | 1 | 5 | 2 | 2 | 2 | +| 6 | 1 | 6 | 3 | 3 | 2 | +| 7 | 1 | 7 | 3 | 3 | 3 | + +## Minimum System Configuration for Production + +| servers | drives (per node) | stripe_size | parity chosen (default) | tolerance for reads (servers) | tolerance for writes (servers) | +|--------:|------------------:|------------:|------------------------:|------------------------------:|-------------------------------:| +| 4 | 2 | 8 | 4 | 2 | 1 | +| 5 | 2 | 10 | 4 | 2 | 2 | +| 6 | 2 | 12 | 4 | 2 | 2 | +| 7 | 2 | 14 | 4 | 2 | 2 | +| 8 | 1 | 8 | 4 | 4 | 3 | +| 8 | 2 | 16 | 4 | 2 | 2 | +| 9 | 2 | 9 | 4 | 4 | 4 | +| 10 | 2 | 10 | 4 | 4 | 4 | +| 11 | 2 | 11 | 4 | 4 | 4 | +| 12 | 2 | 12 | 4 | 4 | 4 | +| 13 | 2 | 13 | 4 | 4 | 4 | +| 14 | 2 | 14 | 4 | 4 | 4 | +| 15 | 2 | 15 | 4 | 4 | 4 | +| 16 | 2 | 16 | 4 | 4 | 4 | + +If one or more drives are offline at the start of a PutObject or NewMultipartUpload operation the object will have additional data +protection bits added automatically to provide the regular safety for these objects up to 50% of the number of drives. +This will allow normal write operations to take place on systems that exceed the write tolerance. + +This means that in the examples above the system will always write 4 parity shards at the expense of slightly higher disk usage. diff --git a/docs/distributed/decom-compressed-sse-s3.sh b/docs/distributed/decom-compressed-sse-s3.sh new file mode 100755 index 00000000..f8aba098 --- /dev/null +++ b/docs/distributed/decom-compressed-sse-s3.sh @@ -0,0 +1,153 @@ +#!/bin/bash + +if [ -n "$TEST_DEBUG" ]; then + set -x +fi + +pkill minio +rm -rf /tmp/xl + +if [ ! -f ./mc ]; then + wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc +fi + +export CI=true +export MINIO_COMPRESSION_ENABLE="on" +export MINIO_COMPRESSION_EXTENSIONS=".go" +export MINIO_COMPRESSION_MIME_TYPES="application/*" +export MINIO_COMPRESSION_ALLOW_ENCRYPTION="on" +export MINIO_KMS_AUTO_ENCRYPTION=on +export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= +export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9000/" + +(minio server http://localhost:9000/tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null) & +pid=$! + +./mc ready myminio + +./mc admin user add myminio/ minio123 minio123 +./mc admin user add myminio/ minio12345 minio12345 + +./mc admin policy create myminio/ rw ./docs/distributed/rw.json +./mc admin policy create myminio/ lake ./docs/distributed/rw.json + +./mc admin policy attach myminio/ rw --user=minio123 +./mc admin policy attach myminio/ lake --user=minio12345 + +./mc mb -l myminio/versioned + +./mc mirror internal myminio/versioned/ --quiet >/dev/null + +## Soft delete (creates delete markers) +./mc rm -r --force myminio/versioned >/dev/null + +## mirror again to create another set of version on top +./mc mirror internal myminio/versioned/ --quiet >/dev/null + +expected_checksum=$(./mc cat internal/dsync/drwmutex.go | md5sum) + +user_count=$(./mc admin user list myminio/ | wc -l) +policy_count=$(./mc admin policy list myminio/ | wc -l) + +kill $pid + +(minio server http://localhost:9000/tmp/xl/{1...10}/disk{0...1} http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded_1.log) & +pid_1=$! + +(minio server --address ":9001" http://localhost:9000/tmp/xl/{1...10}/disk{0...1} http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded_2.log) & +pid_2=$! + +sleep 30 +./mc ready myminio + +expanded_user_count=$(./mc admin user list myminio/ | wc -l) +expanded_policy_count=$(./mc admin policy list myminio/ | wc -l) + +if [ $user_count -ne $expanded_user_count ]; then + echo "BUG: original user count differs from expanded setup" + exit 1 +fi + +if [ $policy_count -ne $expanded_policy_count ]; then + echo "BUG: original policy count differs from expanded setup" + exit 1 +fi + +./mc version info myminio/versioned | grep -q "versioning is enabled" +ret=$? +if [ $ret -ne 0 ]; then + echo "expected versioning enabled after expansion" + exit 1 +fi + +./mc mirror cmd myminio/versioned/ --quiet >/dev/null + +./mc ls -r myminio/versioned/ >expanded_ns.txt +./mc ls -r --versions myminio/versioned/ >expanded_ns_versions.txt + +./mc admin decom start myminio/ http://localhost:9000/tmp/xl/{1...10}/disk{0...1} + +until $(./mc admin decom status myminio/ | grep -q Complete); do + echo "waiting for decom to finish..." + sleep 1 +done + +kill $pid_1 +kill $pid_2 + +sleep 5 + +(minio server --address ":9001" http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/removed.log) & +pid=$! + +export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9001/" + +./mc ready myminio + +decom_user_count=$(./mc admin user list myminio/ | wc -l) +decom_policy_count=$(./mc admin policy list myminio/ | wc -l) + +if [ $user_count -ne $decom_user_count ]; then + echo "BUG: original user count differs after decommission" + exit 1 +fi + +if [ $policy_count -ne $decom_policy_count ]; then + echo "BUG: original policy count differs after decommission" + exit 1 +fi + +./mc version info myminio/versioned | grep -q "versioning is enabled" +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected versioning enabled after decommission" + exit 1 +fi + +got_checksum=$(./mc cat myminio/versioned/dsync/drwmutex.go | md5sum) +if [ "${expected_checksum}" != "${got_checksum}" ]; then + echo "BUG: decommission failed on encrypted objects: expected ${expected_checksum} got ${got_checksum}" + exit 1 +fi + +./mc ls -r myminio/versioned >decommissioned_ns.txt +./mc ls -r --versions myminio/versioned >decommissioned_ns_versions.txt + +out=$(diff -qpruN expanded_ns.txt decommissioned_ns.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after decommission: $out" + exit 1 +fi + +out=$(diff -qpruN expanded_ns_versions.txt decommissioned_ns_versions.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after decommission: $out" + exit 1 +fi + +./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://127.0.0.1:9001/ -bucket versioned + +kill $pid diff --git a/docs/distributed/decom-encrypted-kes.sh b/docs/distributed/decom-encrypted-kes.sh new file mode 100755 index 00000000..836a22b5 --- /dev/null +++ b/docs/distributed/decom-encrypted-kes.sh @@ -0,0 +1,245 @@ +#!/bin/bash + +if [ -n "$TEST_DEBUG" ]; then + set -x +fi + +pkill minio +pkill kes +rm -rf /tmp/xl + +if [ ! -f ./mc ]; then + wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc +fi + +if [ ! -f ./kes ]; then + wget --quiet -O kes https://github.com/minio/kes/releases/latest/download/kes-linux-amd64 && + chmod +x kes +fi + +if ! openssl version &>/dev/null; then + apt install openssl || sudo apt install opensssl +fi + +# Start KES Server +(./kes server --dev 2>&1 >kes-server.log) & +kes_pid=$! +sleep 5s +API_KEY=$(grep "API Key" /dev/null 1>public.crt) + +export CI=true +export MINIO_KMS_KES_ENDPOINT=https://127.0.0.1:7373 +export MINIO_KMS_KES_API_KEY="${API_KEY}" +export MINIO_KMS_KES_KEY_NAME=minio-default-key +export MINIO_KMS_KES_CAPATH=public.crt +export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9000/" + +(minio server http://localhost:9000/tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null) & +pid=$! + +./mc ready myminio + +./mc admin user add myminio/ minio123 minio123 +./mc admin user add myminio/ minio12345 minio12345 + +./mc admin policy create myminio/ rw ./docs/distributed/rw.json +./mc admin policy create myminio/ lake ./docs/distributed/rw.json + +./mc admin policy attach myminio/ rw --user=minio123 +./mc admin policy attach myminio/ lake --user=minio12345 + +./mc mb -l myminio/versioned +./mc mb -l myminio/versioned-1 + +./mc encrypt set sse-s3 myminio/versioned +./mc encrypt set sse-kms minio-default-key myminio/versioned-1 + +./mc mirror internal myminio/versioned/ --quiet >/dev/null +./mc mirror internal myminio/versioned-1/ --quiet >/dev/null + +## Soft delete (creates delete markers) +./mc rm -r --force myminio/versioned >/dev/null +./mc rm -r --force myminio/versioned-1 >/dev/null + +## mirror again to create another set of version on top +./mc mirror internal myminio/versioned/ --quiet >/dev/null +./mc mirror internal myminio/versioned-1/ --quiet >/dev/null + +expected_checksum=$(./mc cat internal/dsync/drwmutex.go | md5sum) + +user_count=$(./mc admin user list myminio/ | wc -l) +policy_count=$(./mc admin policy list myminio/ | wc -l) + +kill $pid + +(minio server http://localhost:9000/tmp/xl/{1...10}/disk{0...1} http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded_1.log) & +pid_1=$! + +(minio server --address ":9001" http://localhost:9000/tmp/xl/{1...10}/disk{0...1} http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded_2.log) & +pid_2=$! + +./mc ready myminio + +expanded_user_count=$(./mc admin user list myminio/ | wc -l) +expanded_policy_count=$(./mc admin policy list myminio/ | wc -l) + +if [ "$user_count" -ne "$expanded_user_count" ]; then + echo "BUG: original user count differs from expanded setup" + exit 1 +fi + +if [ "$policy_count" -ne "$expanded_policy_count" ]; then + echo "BUG: original policy count differs from expanded setup" + exit 1 +fi + +./mc version info myminio/versioned | grep -q "versioning is enabled" +ret=$? +if [ $ret -ne 0 ]; then + echo "expected versioning enabled after expansion" + exit 1 +fi + +./mc encrypt info myminio/versioned | grep -q "Auto encryption 'sse-s3' is enabled" +ret=$? +if [ $ret -ne 0 ]; then + echo "expected encryption enabled after expansion" + exit 1 +fi + +./mc version info myminio/versioned-1 | grep -q "versioning is enabled" +ret=$? +if [ $ret -ne 0 ]; then + echo "expected versioning enabled after expansion" + exit 1 +fi + +./mc encrypt info myminio/versioned-1 | grep -q "Auto encryption 'sse-kms' is enabled" +ret=$? +if [ $ret -ne 0 ]; then + echo "expected encryption enabled after expansion" + exit 1 +fi + +./mc mirror cmd myminio/versioned/ --quiet >/dev/null +./mc mirror cmd myminio/versioned-1/ --quiet >/dev/null + +./mc ls -r myminio/versioned/ >expanded_ns.txt +./mc ls -r --versions myminio/versioned/ >expanded_ns_versions.txt +./mc ls -r myminio/versioned-1/ >expanded_ns_1.txt +./mc ls -r --versions myminio/versioned-1/ >expanded_ns_versions_1.txt + +./mc admin decom start myminio/ http://localhost:9000/tmp/xl/{1...10}/disk{0...1} + +until $(./mc admin decom status myminio/ | grep -q Complete); do + echo "waiting for decom to finish..." + sleep 1s +done + +kill $pid_1 +kill $pid_2 + +sleep 5s + +(minio server --address ":9001" http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/removed.log) & +pid=$! + +sleep 30s + +export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9001/" + +./mc ready myminio + +decom_user_count=$(./mc admin user list myminio/ | wc -l) +decom_policy_count=$(./mc admin policy list myminio/ | wc -l) + +if [ "$user_count" -ne "$decom_user_count" ]; then + echo "BUG: original user count differs after decommission" + exit 1 +fi + +if [ "$policy_count" -ne "$decom_policy_count" ]; then + echo "BUG: original policy count differs after decommission" + exit 1 +fi + +./mc version info myminio/versioned | grep -q "versioning is enabled" +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected versioning enabled after decommission" + exit 1 +fi + +./mc encrypt info myminio/versioned | grep -q "Auto encryption 'sse-s3' is enabled" +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected encryption enabled after expansion" + exit 1 +fi + +./mc version info myminio/versioned-1 | grep -q "versioning is enabled" +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected versioning enabled after decommission" + exit 1 +fi + +./mc encrypt info myminio/versioned-1 | grep -q "Auto encryption 'sse-kms' is enabled" +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected encryption enabled after expansion" + exit 1 +fi + +got_checksum=$(./mc cat myminio/versioned/dsync/drwmutex.go | md5sum) +if [ "${expected_checksum}" != "${got_checksum}" ]; then + echo "BUG: decommission failed on encrypted objects: expected ${expected_checksum} got ${got_checksum}" + exit 1 +fi + +got_checksum_1=$(./mc cat myminio/versioned-1/dsync/drwmutex.go | md5sum) +if [ "${expected_checksum}" != "${got_checksum_1}" ]; then + echo "BUG: decommission failed on encrypted objects: expected ${expected_checksum} got ${got_checksum_1}" + exit 1 +fi + +./mc ls -r myminio/versioned >decommissioned_ns.txt +./mc ls -r --versions myminio/versioned >decommissioned_ns_versions.txt +./mc ls -r myminio/versioned-1 >decommissioned_ns_1.txt +./mc ls -r --versions myminio/versioned-1 >decommissioned_ns_versions_1.txt + +out=$(diff -qpruN expanded_ns.txt decommissioned_ns.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after decommission: $out" + exit 1 +fi + +out=$(diff -qpruN expanded_ns_versions.txt decommissioned_ns_versions.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after decommission: $out" + exit 1 +fi + +out1=$(diff -qpruN expanded_ns_1.txt decommissioned_ns_1.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after decommission: $out1" + exit 1 +fi + +out1=$(diff -qpruN expanded_ns_versions_1.txt decommissioned_ns_versions_1.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after decommission: $out1" + exit 1 +fi + +./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://127.0.0.1:9001/ -bucket versioned +./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://127.0.0.1:9001/ -bucket versioned-1 + +kill $pid +kill $kes_pid diff --git a/docs/distributed/decom-encrypted-sse-s3.sh b/docs/distributed/decom-encrypted-sse-s3.sh new file mode 100755 index 00000000..65180348 --- /dev/null +++ b/docs/distributed/decom-encrypted-sse-s3.sh @@ -0,0 +1,163 @@ +#!/bin/bash + +if [ -n "$TEST_DEBUG" ]; then + set -x +fi + +pkill minio +rm -rf /tmp/xl + +if [ ! -f ./mc ]; then + wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc +fi + +export CI=true +export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= +export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9000/" + +(minio server http://localhost:9000/tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null) & +pid=$! + +./mc ready myminio + +./mc admin user add myminio/ minio123 minio123 +./mc admin user add myminio/ minio12345 minio12345 + +./mc admin policy create myminio/ rw ./docs/distributed/rw.json +./mc admin policy create myminio/ lake ./docs/distributed/rw.json + +./mc admin policy attach myminio/ rw --user=minio123 +./mc admin policy attach myminio/ lake --user=minio12345 + +./mc mb -l myminio/versioned + +./mc encrypt set sse-s3 myminio/versioned + +./mc mirror internal myminio/versioned/ --quiet >/dev/null + +## Soft delete (creates delete markers) +./mc rm -r --force myminio/versioned >/dev/null + +## mirror again to create another set of version on top +./mc mirror internal myminio/versioned/ --quiet >/dev/null + +expected_checksum=$(./mc cat internal/dsync/drwmutex.go | md5sum) + +user_count=$(./mc admin user list myminio/ | wc -l) +policy_count=$(./mc admin policy list myminio/ | wc -l) + +kill $pid + +(minio server http://localhost:9000/tmp/xl/{1...10}/disk{0...1} http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded_1.log) & +pid_1=$! + +(minio server --address ":9001" http://localhost:9000/tmp/xl/{1...10}/disk{0...1} http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded_2.log) & +pid_2=$! + +./mc ready myminio + +expanded_user_count=$(./mc admin user list myminio/ | wc -l) +expanded_policy_count=$(./mc admin policy list myminio/ | wc -l) + +if [ $user_count -ne $expanded_user_count ]; then + echo "BUG: original user count differs from expanded setup" + exit 1 +fi + +if [ $policy_count -ne $expanded_policy_count ]; then + echo "BUG: original policy count differs from expanded setup" + exit 1 +fi + +./mc version info myminio/versioned | grep -q "versioning is enabled" +ret=$? +if [ $ret -ne 0 ]; then + echo "expected versioning enabled after expansion" + exit 1 +fi + +./mc encrypt info myminio/versioned | grep -q "Auto encryption 'sse-s3' is enabled" +ret=$? +if [ $ret -ne 0 ]; then + echo "expected encryption enabled after expansion" + exit 1 +fi + +./mc mirror cmd myminio/versioned/ --quiet >/dev/null + +./mc ls -r myminio/versioned/ >expanded_ns.txt +./mc ls -r --versions myminio/versioned/ >expanded_ns_versions.txt + +./mc admin decom start myminio/ http://localhost:9000/tmp/xl/{1...10}/disk{0...1} + +until $(./mc admin decom status myminio/ | grep -q Complete); do + echo "waiting for decom to finish..." + sleep 1 +done + +kill $pid_1 +kill $pid_2 + +sleep 5 + +(minio server --address ":9001" http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/removed.log) & +pid=$! + +export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9001/" + +./mc ready myminio + +decom_user_count=$(./mc admin user list myminio/ | wc -l) +decom_policy_count=$(./mc admin policy list myminio/ | wc -l) + +if [ $user_count -ne $decom_user_count ]; then + echo "BUG: original user count differs after decommission" + exit 1 +fi + +if [ $policy_count -ne $decom_policy_count ]; then + echo "BUG: original policy count differs after decommission" + exit 1 +fi + +./mc version info myminio/versioned | grep -q "versioning is enabled" +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected versioning enabled after decommission" + exit 1 +fi + +./mc encrypt info myminio/versioned | grep -q "Auto encryption 'sse-s3' is enabled" +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected encryption enabled after expansion" + exit 1 +fi + +got_checksum=$(./mc cat myminio/versioned/dsync/drwmutex.go | md5sum) +if [ "${expected_checksum}" != "${got_checksum}" ]; then + echo "BUG: decommission failed on encrypted objects: expected ${expected_checksum} got ${got_checksum}" + exit 1 +fi + +./mc ls -r myminio/versioned >decommissioned_ns.txt +./mc ls -r --versions myminio/versioned >decommissioned_ns_versions.txt + +out=$(diff -qpruN expanded_ns.txt decommissioned_ns.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after decommission: $out" + exit 1 +fi + +out=$(diff -qpruN expanded_ns_versions.txt decommissioned_ns_versions.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after decommission: $out" + exit 1 +fi + +./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://127.0.0.1:9001/ -bucket versioned + +kill $pid diff --git a/docs/distributed/decom-encrypted.sh b/docs/distributed/decom-encrypted.sh new file mode 100755 index 00000000..6d47537b --- /dev/null +++ b/docs/distributed/decom-encrypted.sh @@ -0,0 +1,149 @@ +#!/bin/bash + +if [ -n "$TEST_DEBUG" ]; then + set -x +fi + +pkill minio +rm -rf /tmp/xl + +if [ ! -f ./mc ]; then + wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc +fi + +export CI=true +export MINIO_KMS_AUTO_ENCRYPTION=on +export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= + +(minio server http://localhost:9000/tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null) & +pid=$! + +export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9000/" + +./mc ready myminio + +./mc admin user add myminio/ minio123 minio123 +./mc admin user add myminio/ minio12345 minio12345 + +./mc admin policy create myminio/ rw ./docs/distributed/rw.json +./mc admin policy create myminio/ lake ./docs/distributed/rw.json + +./mc admin policy attach myminio/ rw --user=minio123 +./mc admin policy attach myminio/ lake --user=minio12345 + +./mc mb -l myminio/versioned + +./mc mirror internal myminio/versioned/ --quiet >/dev/null + +## Soft delete (creates delete markers) +./mc rm -r --force myminio/versioned >/dev/null + +## mirror again to create another set of version on top +./mc mirror internal myminio/versioned/ --quiet >/dev/null + +expected_checksum=$(./mc cat internal/dsync/drwmutex.go | md5sum) + +user_count=$(./mc admin user list myminio/ | wc -l) +policy_count=$(./mc admin policy list myminio/ | wc -l) + +kill $pid + +(minio server http://localhost:9000/tmp/xl/{1...10}/disk{0...1} http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded_1.log) & +pid_1=$! + +(minio server --address ":9001" http://localhost:9000/tmp/xl/{1...10}/disk{0...1} http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded_2.log) & +pid_2=$! + +./mc ready myminio + +expanded_user_count=$(./mc admin user list myminio/ | wc -l) +expanded_policy_count=$(./mc admin policy list myminio/ | wc -l) + +if [ $user_count -ne $expanded_user_count ]; then + echo "BUG: original user count differs from expanded setup" + exit 1 +fi + +if [ $policy_count -ne $expanded_policy_count ]; then + echo "BUG: original policy count differs from expanded setup" + exit 1 +fi + +./mc version info myminio/versioned | grep -q "versioning is enabled" +ret=$? +if [ $ret -ne 0 ]; then + echo "expected versioning enabled after expansion" + exit 1 +fi + +./mc mirror cmd myminio/versioned/ --quiet >/dev/null + +./mc ls -r myminio/versioned/ >expanded_ns.txt +./mc ls -r --versions myminio/versioned/ >expanded_ns_versions.txt + +./mc admin decom start myminio/ http://localhost:9000/tmp/xl/{1...10}/disk{0...1} + +until $(./mc admin decom status myminio/ | grep -q Complete); do + echo "waiting for decom to finish..." + sleep 1 +done + +kill $pid_1 +kill $pid_2 + +sleep 5 + +(minio server --address ":9001" http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/removed.log) & +pid=$! + +export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9001/" + +./mc ready myminio + +decom_user_count=$(./mc admin user list myminio/ | wc -l) +decom_policy_count=$(./mc admin policy list myminio/ | wc -l) + +if [ $user_count -ne $decom_user_count ]; then + echo "BUG: original user count differs after decommission" + exit 1 +fi + +if [ $policy_count -ne $decom_policy_count ]; then + echo "BUG: original policy count differs after decommission" + exit 1 +fi + +./mc version info myminio/versioned | grep -q "versioning is enabled" +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected versioning enabled after decommission" + exit 1 +fi + +./mc ls -r myminio/versioned >decommissioned_ns.txt +./mc ls -r --versions myminio/versioned >decommissioned_ns_versions.txt + +out=$(diff -qpruN expanded_ns.txt decommissioned_ns.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after decommission: $out" + exit 1 +fi + +out=$(diff -qpruN expanded_ns_versions.txt decommissioned_ns_versions.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after decommission: $out" + exit 1 +fi + +got_checksum=$(./mc cat myminio/versioned/dsync/drwmutex.go | md5sum) +if [ "${expected_checksum}" != "${got_checksum}" ]; then + echo "BUG: decommission failed on encrypted objects: expected ${expected_checksum} got ${got_checksum}" + exit 1 +fi + +./s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://127.0.0.1:9001/ -bucket versioned + +kill $pid diff --git a/docs/distributed/decom.sh b/docs/distributed/decom.sh new file mode 100755 index 00000000..18d9ca4f --- /dev/null +++ b/docs/distributed/decom.sh @@ -0,0 +1,218 @@ +#!/bin/bash + +if [ -n "$TEST_DEBUG" ]; then + set -x +fi + +pkill minio +rm -rf /tmp/xl +rm -rf /tmp/xltier + +if [ ! -f ./mc ]; then + wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc +fi + +export CI=true +export MINIO_SCANNER_SPEED=fastest + +(minio server http://localhost:9000/tmp/xl/{1...10}/disk{0...1} 2>&1 >/tmp/decom.log) & +pid=$! + +export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9000/" + +./mc ready myminio + +./mc admin user add myminio/ minio123 minio123 +./mc admin user add myminio/ minio12345 minio12345 + +./mc admin policy create myminio/ rw ./docs/distributed/rw.json +./mc admin policy create myminio/ lake ./docs/distributed/rw.json + +./mc admin policy attach myminio/ rw --user=minio123 +./mc admin policy attach myminio/ lake --user=minio12345 + +./mc mb -l myminio/versioned + +./mc mirror internal myminio/versioned/ --quiet >/dev/null + +## Soft delete (creates delete markers) +./mc rm -r --force myminio/versioned >/dev/null + +## mirror again to create another set of version on top +./mc mirror internal myminio/versioned/ --quiet >/dev/null + +expected_checksum=$(./mc cat internal/dsync/drwmutex.go | md5sum) + +user_count=$(./mc admin user list myminio/ | wc -l) +policy_count=$(./mc admin policy list myminio/ | wc -l) + +## create a warm tier instance +(minio server /tmp/xltier/{1...4}/disk{0...1} --address :9002 2>&1 >/dev/null) & + +export MC_HOST_mytier="http://minioadmin:minioadmin@localhost:9002/" + +./mc ready myminio + +./mc mb -l myminio/bucket2 +./mc mb -l mytier/tiered + +## create a tier and set up ilm policy to tier immediately +./mc admin tier add minio myminio TIER1 --endpoint http://localhost:9002 --access-key minioadmin --secret-key minioadmin --bucket tiered --prefix prefix5/ +./mc ilm add myminio/bucket2 --transition-days 0 --transition-tier TIER1 --transition-days 0 + +## mirror some content to bucket2 and capture versions tiered +./mc mirror internal myminio/bucket2/ --quiet >/dev/null +./mc ls -r myminio/bucket2/ >bucket2_ns.txt +./mc ls -r --versions myminio/bucket2/ >bucket2_ns_versions.txt + +sleep 30 + +./mc ls -r --versions mytier/tiered/ >tiered_ns_versions.txt + +kill $pid + +(minio server http://localhost:9000/tmp/xl/{1...10}/disk{0...1} http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded_1.log) & +pid_1=$! + +(minio server --address ":9001" http://localhost:9000/tmp/xl/{1...10}/disk{0...1} http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/expanded_2.log) & +pid_2=$! + +./mc ready myminio + +expanded_user_count=$(./mc admin user list myminio/ | wc -l) +expanded_policy_count=$(./mc admin policy list myminio/ | wc -l) + +if [ $user_count -ne $expanded_user_count ]; then + echo "BUG: original user count differs from expanded setup" + exit 1 +fi + +if [ $policy_count -ne $expanded_policy_count ]; then + echo "BUG: original policy count differs from expanded setup" + exit 1 +fi + +./mc version info myminio/versioned | grep -q "versioning is enabled" +ret=$? +if [ $ret -ne 0 ]; then + echo "expected versioning enabled after expansion" + exit 1 +fi + +./mc mirror cmd myminio/versioned/ --quiet >/dev/null + +./mc ls -r myminio/versioned/ >expanded_ns.txt +./mc ls -r --versions myminio/versioned/ >expanded_ns_versions.txt + +./mc admin decom start myminio/ http://localhost:9000/tmp/xl/{1...10}/disk{0...1} + +count=0 +until $(./mc admin decom status myminio/ | grep -q Complete); do + echo "waiting for decom to finish..." + count=$((count + 1)) + if [ ${count} -eq 120 ]; then + ./mc cat /tmp/expanded_*.log + fi + sleep 1 +done + +kill $pid_1 +kill $pid_2 + +sleep 5 + +(minio server --address ":9001" http://localhost:9001/tmp/xl/{11...30}/disk{0...3} 2>&1 >/tmp/removed.log) & +pid=$! + +sleep 5 +export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9001/" + +./mc ready myminio + +decom_user_count=$(./mc admin user list myminio/ | wc -l) +decom_policy_count=$(./mc admin policy list myminio/ | wc -l) + +if [ $user_count -ne $decom_user_count ]; then + echo "BUG: original user count differs after decommission" + exit 1 +fi + +if [ $policy_count -ne $decom_policy_count ]; then + echo "BUG: original policy count differs after decommission" + exit 1 +fi + +./mc version info myminio/versioned | grep -q "versioning is enabled" +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected versioning enabled after decommission" + exit 1 +fi + +./mc ls -r myminio/versioned >decommissioned_ns.txt +./mc ls -r --versions myminio/versioned >decommissioned_ns_versions.txt + +out=$(diff -qpruN expanded_ns.txt decommissioned_ns.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after decommission: $out" + exit 1 +fi + +out=$(diff -qpruN expanded_ns_versions.txt decommissioned_ns_versions.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after decommission: $out" + exit 1 +fi + +got_checksum=$(./mc cat myminio/versioned/dsync/drwmutex.go | md5sum) +if [ "${expected_checksum}" != "${got_checksum}" ]; then + echo "BUG: decommission failed on encrypted objects: expected ${expected_checksum} got ${got_checksum}" + exit 1 +fi + +# after decommissioning, compare listings in bucket2 and tiered +./mc version info myminio/bucket2 | grep -q "versioning is enabled" +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected versioning enabled after decommission on bucket2" + exit 1 +fi + +./mc ls -r myminio/bucket2 >decommissioned_bucket2_ns.txt +./mc ls -r --versions myminio/bucket2 >decommissioned_bucket2_ns_versions.txt +./mc ls -r --versions mytier/tiered/ >tiered_ns_versions2.txt + +out=$(diff -qpruN bucket2_ns.txt decommissioned_bucket2_ns.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after decommission in bucket2: $out" + exit 1 +fi + +out=$(diff -qpruN bucket2_ns_versions.txt decommissioned_bucket2_ns_versions.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after decommission in bucket2x: $out" + exit 1 +fi + +out=$(diff -qpruN tiered_ns_versions.txt tiered_ns_versions2.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after decommission in warm tier: $out" + exit 1 +fi + +got_checksum=$(./mc cat myminio/bucket2/dsync/drwmutex.go | md5sum) +if [ "${expected_checksum}" != "${got_checksum}" ]; then + echo "BUG: decommission failed on encrypted objects with tiering: expected ${expected_checksum} got ${got_checksum}" + exit 1 +fi + +s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://127.0.0.1:9001/ -bucket bucket2 +s3-check-md5 -versions -access-key minioadmin -secret-key minioadmin -endpoint http://127.0.0.1:9001/ -bucket versioned + +kill $pid diff --git a/docs/distributed/distributed-from-config-file.sh b/docs/distributed/distributed-from-config-file.sh new file mode 100755 index 00000000..cea17172 --- /dev/null +++ b/docs/distributed/distributed-from-config-file.sh @@ -0,0 +1,109 @@ +#!/usr/bin/env bash + +set -e + +cleanup() { + echo "Cleaning up instances of MinIO" + pkill minio || true + pkill -9 minio || true + rm -rf /tmp/xl/ || true + rm -rf /tmp/minio.configfile.{1,2,3,4} || true +} + +cleanup + +unset MINIO_KMS_KES_CERT_FILE +unset MINIO_KMS_KES_KEY_FILE +unset MINIO_KMS_KES_ENDPOINT +unset MINIO_KMS_KES_KEY_NAME + +export MINIO_CI_CD=1 + +if [ ! -f ./mc ]; then + os="$(uname -s)" + arch="$(uname -m)" + case "${arch}" in + "x86_64") + arch="amd64" + ;; + esac + + wget -O mc https://dl.minio.io/client/mc/release/${os,,}-${arch,,}/mc && + chmod +x mc +fi + +for i in $(seq 1 4); do + s3Port="$((9000 + i))" + consolePort="$((s3Port + 1000))" + + cat </tmp/minio.configfile.$i +version: v1 +address: ':${s3Port}' +console-address: ':${consolePort}' +rootUser: 'minr0otUS2r' +rootPassword: 'pBU94AGAY85e' +pools: # Specify the nodes and drives with pools + - + - 'http://localhost:9001/tmp/xl/node9001/mnt/disk{1...4}/' + - 'http://localhost:9002/tmp/xl/node9002/mnt/disk{1,2,3,4}/' + - + - 'http://localhost:9003/tmp/xl/node9003/mnt/disk{1...4}/' + - 'http://localhost:9004/tmp/xl/node9004/mnt/disk1/' + - 'http://localhost:9004/tmp/xl/node9004/mnt/disk2/' + - 'http://localhost:9004/tmp/xl/node9004/mnt/disk3/' + - 'http://localhost:9004/tmp/xl/node9004/mnt/disk4/' +EOF +done + +minio server --config /tmp/minio.configfile.1 >/tmp/minio1_1.log 2>&1 & +site1_pid=$! +minio server --config /tmp/minio.configfile.2 >/tmp/minio2_1.log 2>&1 & +site2_pid=$! +minio server --config /tmp/minio.configfile.3 >/tmp/minio3_1.log 2>&1 & +site3_pid=$! +minio server --config /tmp/minio.configfile.4 >/tmp/minio4_1.log 2>&1 & +site4_pid=$! + +export MC_HOST_minio1=http://minr0otUS2r:pBU94AGAY85e@localhost:9001 +export MC_HOST_minio3=http://minr0otUS2r:pBU94AGAY85e@localhost:9003 + +./mc ready minio1 +./mc ready minio3 + +./mc mb minio1/testbucket +# copy large upload to newbucket on minio1 +truncate -s 17M lrgfile +expected_checksum=$(cat ./lrgfile | md5sum) + +./mc cp ./lrgfile minio1/testbucket + +actual_checksum=$(./mc cat minio3/testbucket/lrgfile | md5sum) + +if [ "${expected_checksum}" != "${actual_checksum}" ]; then + echo "unexpected object checksum, expected: ${expected_checksum} got: ${actual_checksum}" + exit +fi + +# Compare the difference of the list of disks and their location, with the below expected output +diff <(./mc admin info minio1 --json | jq -r '.info.servers[].drives[] | "\(.pool_index),\(.set_index),\(.disk_index) \(.endpoint)"' | sort) <( + cat < minio-iam-testing/ldap/50-bootstrap.ldif" +cp docs/distributed/samples/bootstrap-complete.ldif minio-iam-testing/ldap/50-bootstrap.ldif || exit 1 +cd ./minio-iam-testing +make docker-images +make docker-run +cd - + +export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:22000" +export MC_HOST_myminio1="http://minioadmin:minioadmin@localhost:24000" + +# Start MinIO instance +export CI=true +(minio server --address :22000 --console-address :10000 http://localhost:22000/tmp/ldap{1...4} 2>&1 >/dev/null) & +sleep 30 +./mc ready myminio + +./mc idp ldap add myminio server_addr=localhost:389 server_insecure=on \ + lookup_bind_dn=cn=admin,dc=min,dc=io lookup_bind_password=admin \ + user_dn_search_base_dn=dc=min,dc=io user_dn_search_filter="(uid=%s)" \ + group_search_base_dn=ou=swengg,dc=min,dc=io group_search_filter="(&(objectclass=groupOfNames)(member=%d))" + +./mc admin service restart myminio --json +./mc ready myminio +./mc admin cluster iam import myminio docs/distributed/samples/myminio-iam-info.zip +sleep 10 + +# Verify the list of users and service accounts from the import +./mc admin user list myminio +USER_COUNT=$(./mc admin user list myminio | wc -l) +if [ "${USER_COUNT}" -ne 2 ]; then + echo "BUG: Expected no of users: 2 Found: ${USER_COUNT}" + exit 1 +fi +./mc admin user svcacct list myminio "uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io" --json +SVCACCT_COUNT_1=$(./mc admin user svcacct list myminio "uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io" --json | jq '.accessKey' | wc -l) +if [ "${SVCACCT_COUNT_1}" -ne 2 ]; then + echo "BUG: Expected svcacct count for 'uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io': 2. Found: ${SVCACCT_COUNT_1}" + exit 1 +fi +./mc admin user svcacct list myminio "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" --json +SVCACCT_COUNT_2=$(./mc admin user svcacct list myminio "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" --json | jq '.accessKey' | wc -l) +if [ "${SVCACCT_COUNT_2}" -ne 2 ]; then + echo "BUG: Expected svcacct count for 'uid=dillon,ou=people,ou=swengg,dc=min,dc=io': 2. Found: ${SVCACCT_COUNT_2}" + exit 1 +fi + +# Kill MinIO and LDAP to start afresh with missing groups/DN +pkill minio +docker rm -f $(docker ps -aq) +rm -rf /tmp/ldap{1..4} + +# Deploy the LDAP config witg missing groups/DN +echo "Copying docs/distributed/samples/bootstrap-partial.ldif => minio-iam-testing/ldap/50-bootstrap.ldif" +cp docs/distributed/samples/bootstrap-partial.ldif minio-iam-testing/ldap/50-bootstrap.ldif || exit 1 +cd ./minio-iam-testing +make docker-images +make docker-run +cd - + +(minio server --address ":24000" --console-address :10000 http://localhost:24000/tmp/ldap1{1...4} 2>&1 >/dev/null) & +sleep 30 +./mc ready myminio1 + +./mc idp ldap add myminio1 server_addr=localhost:389 server_insecure=on \ + lookup_bind_dn=cn=admin,dc=min,dc=io lookup_bind_password=admin \ + user_dn_search_base_dn=dc=min,dc=io user_dn_search_filter="(uid=%s)" \ + group_search_base_dn=ou=hwengg,dc=min,dc=io group_search_filter="(&(objectclass=groupOfNames)(member=%d))" + +./mc admin service restart myminio1 --json +./mc ready myminio1 +./mc admin cluster iam import myminio1 docs/distributed/samples/myminio-iam-info.zip +sleep 10 + +# Verify the list of users and service accounts from the import +./mc admin user list myminio1 +USER_COUNT=$(./mc admin user list myminio1 | wc -l) +if [ "${USER_COUNT}" -ne 1 ]; then + echo "BUG: Expected no of users: 1 Found: ${USER_COUNT}" + exit 1 +fi +./mc admin user svcacct list myminio1 "uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io" --json +SVCACCT_COUNT_1=$(./mc admin user svcacct list myminio1 "uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io" --json | jq '.accessKey' | wc -l) +if [ "${SVCACCT_COUNT_1}" -ne 2 ]; then + echo "BUG: Expected svcacct count for 'uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io': 2. Found: ${SVCACCT_COUNT_1}" + exit 1 +fi +./mc admin user svcacct list myminio1 "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" --json +SVCACCT_COUNT_2=$(./mc admin user svcacct list myminio1 "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" --json | jq '.accessKey' | wc -l) +if [ "${SVCACCT_COUNT_2}" -ne 0 ]; then + echo "BUG: Expected svcacct count for 'uid=dillon,ou=people,ou=swengg,dc=min,dc=io': 0. Found: ${SVCACCT_COUNT_2}" + exit 1 +fi + +# Finally kill running processes +pkill minio +docker rm -f $(docker ps -aq) diff --git a/docs/distributed/iam-import-with-openid.sh b/docs/distributed/iam-import-with-openid.sh new file mode 100755 index 00000000..ca703aa5 --- /dev/null +++ b/docs/distributed/iam-import-with-openid.sh @@ -0,0 +1,82 @@ +#!/bin/bash + +if [ -n "$TEST_DEBUG" ]; then + set -x +fi + +pkill minio +docker rm -f $(docker ps -aq) +rm -rf /tmp/openid{1..4} + +export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:22000" +# The service account used below is already present in iam configuration getting imported +export MC_HOST_myminio1="http://dillon-service-2:dillon-service-2@localhost:22000" + +# Start MinIO instance +export CI=true + +if [ ! -f ./mc ]; then + wget --quiet -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc +fi + +mc -v + +# Start openid server +( + cd ./minio-iam-testing + make docker-images + make docker-run + cd - +) + +(minio server --address :22000 --console-address :10000 http://localhost:22000/tmp/openid{1...4} 2>&1 >/tmp/server.log) & +./mc ready myminio +./mc mb myminio/test-bucket +./mc cp /etc/hosts myminio/test-bucket + +./mc idp openid add myminio \ + config_url="http://localhost:5556/dex/.well-known/openid-configuration" \ + client_id="minio-client-app" \ + client_secret="minio-client-app-secret" \ + scopes="openid,groups,email,profile" \ + redirect_uri="http://127.0.0.1:10000/oauth_callback" \ + display_name="Login via dex1" \ + role_policy="consoleAdmin" + +./mc admin service restart myminio --json +./mc ready myminio +./mc admin cluster iam import myminio docs/distributed/samples/myminio-iam-info-openid.zip + +# Verify if buckets / objects accessible using service account +echo "Verifying buckets and objects access for the imported service account" + +./mc ls myminio1/ --json +BKT_COUNT=$(./mc ls myminio1/ --json | jq '.key' | wc -l) +if [ "${BKT_COUNT}" -ne 1 ]; then + echo "BUG: Expected no of bucket: 1, Found: ${BKT_COUNT}" + exit 1 +fi + +BKT_NAME=$(./mc ls myminio1/ --json | jq '.key' | sed 's/"//g' | sed 's\/\\g') +if [[ ${BKT_NAME} != "test-bucket" ]]; then + echo "BUG: Expected bucket: test-bucket, Found: ${BKT_NAME}" + exit 1 +fi + +./mc ls myminio1/test-bucket +OBJ_COUNT=$(./mc ls myminio1/test-bucket --json | jq '.key' | wc -l) +if [ "${OBJ_COUNT}" -ne 1 ]; then + echo "BUG: Expected no of objects: 1, Found: ${OBJ_COUNT}" + exit 1 +fi + +OBJ_NAME=$(./mc ls myminio1/test-bucket --json | jq '.key' | sed 's/"//g') +if [[ ${OBJ_NAME} != "hosts" ]]; then + echo "BUG: Expected object: hosts, Found: ${BKT_NAME}" + exit 1 +fi + +# Finally kill running processes +pkill minio +docker rm -f $(docker ps -aq) diff --git a/docs/distributed/rw.json b/docs/distributed/rw.json new file mode 100644 index 00000000..66171dc9 --- /dev/null +++ b/docs/distributed/rw.json @@ -0,0 +1,14 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:*" + ], + "Resource": [ + "arn:aws:s3:::*" + ] + } + ] + } \ No newline at end of file diff --git a/docs/distributed/samples/bootstrap-complete.ldif b/docs/distributed/samples/bootstrap-complete.ldif new file mode 100644 index 00000000..6f4f4571 --- /dev/null +++ b/docs/distributed/samples/bootstrap-complete.ldif @@ -0,0 +1,123 @@ +# Create hardware engg org unit +dn: ou=hwengg,dc=min,dc=io +objectClass: organizationalUnit +ou: hwengg + +# Create people sub-org +dn: ou=people,ou=hwengg,dc=min,dc=io +objectClass: organizationalUnit +ou: people + +# Create Alice, Bob and Cody in hwengg +dn: uid=alice1,ou=people,ou=hwengg,dc=min,dc=io +objectClass: inetOrgPerson +cn: Alice Smith +sn: Smith +uid: alice1 +mail: alice@example.io +userPassword: {SSHA}Yeh2/IV/q/HjG2yzN3YdE9CAF3EJFCLu + +dn: uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io +objectClass: inetOrgPerson +cn: Robert Fisher +sn: Fisher +uid: bobfisher +mail: bob@example.io +userPassword: {SSHA}LktfbhK5oXSdDWCNzauJ9JA+Poxinl3y + +dn: uid=cody3,ou=people,ou=hwengg,dc=min,dc=io +objectClass: inetOrgPerson +cn: Cody Thomas +sn: Thomas +uid: cody3 +mail: cody@example.io +userPassword: {SSHA}H8B0gaOd4bRklK3fXj9ltHvJXWQFXW5Q + +# Create groups ou for hwengg +dn: ou=groups,ou=hwengg,dc=min,dc=io +objectclass: organizationalUnit +ou: groups +description: groups branch + +# Create project groups + +dn: cn=projectx,ou=groups,ou=hwengg,dc=min,dc=io +objectclass: groupofnames +cn: projectx +description: Project X group members +member: uid=alice1,ou=people,ou=hwengg,dc=min,dc=io +member: uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io + +dn: cn=projecty,ou=groups,ou=hwengg,dc=min,dc=io +objectclass: groupofnames +cn: projecty +description: Project Y group members +member: uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io +member: uid=cody3,ou=people,ou=hwengg,dc=min,dc=io + +# Create software engg org unit +dn: ou=swengg,dc=min,dc=io +objectClass: organizationalUnit +ou: swengg + +# Create people sub-org +dn: ou=people,ou=swengg,dc=min,dc=io +objectClass: organizationalUnit +ou: people + +# Create Dillon, Elizabeth and Fahim in swengg +dn: uid=dillon,ou=people,ou=swengg,dc=min,dc=io +objectClass: inetOrgPerson +cn: Dillon Harper +sn: Harper +uid: dillon +mail: dillon@example.io +userPassword: {SSHA}UH+LmoEhWWW6s9rjgdpqHPI0qCMouY8+ + +dn: uid=liza,ou=people,ou=swengg,dc=min,dc=io +objectClass: inetOrgPerson +cn: Elizabeth Jones +sn: Jones +uid: liza +mail: ejones@example.io +userPassword: {SSHA}feVkKkafHtsu2Io7n0tQP4Cnh8/Oy1PK + +dn: uid=fahim,ou=people,ou=swengg,dc=min,dc=io +objectClass: inetOrgPerson +cn: Fahim Ahmed +sn: Ahmed +uid: fahim +mail: fahmed@example.io +userPassword: {SSHA}lRNH+PHooRaruiEb+CBEA21EZLMkAmcc + +# Add a user with special chars. The password = example here. +dn: uid=Пользователь,OU=people,OU=swengg,DC=min,DC=io +objectClass: inetOrgPerson +cn: Special Charsman +sn: Charsman +uid: Пользователь +mail: scharsman@example.io +userPassword: {SSHA}XQSZqLPvYgm30wR7pk67a1GW+q+DDvSj + +# Creates groups ou for swengg +dn: ou=groups,ou=swengg,dc=min,dc=io +objectclass: organizationalUnit +ou: groups +description: groups branch + +# Create project groups + +dn: cn=projecta,ou=groups,ou=swengg,dc=min,dc=io +objectclass: groupofnames +cn: projecta +description: Project A group members +member: uid=dillon,ou=people,ou=swengg,dc=min,dc=io + +dn: cn=projectb,ou=groups,ou=swengg,dc=min,dc=io +objectclass: groupofnames +cn: projectb +description: Project B group members +member: uid=dillon,ou=people,ou=swengg,dc=min,dc=io +member: uid=liza,ou=people,ou=swengg,dc=min,dc=io +member: uid=fahim,ou=people,ou=swengg,dc=min,dc=io +member: uid=Пользователь,OU=people,OU=swengg,DC=min,DC=io diff --git a/docs/distributed/samples/bootstrap-partial.ldif b/docs/distributed/samples/bootstrap-partial.ldif new file mode 100644 index 00000000..02cbb832 --- /dev/null +++ b/docs/distributed/samples/bootstrap-partial.ldif @@ -0,0 +1,56 @@ +# Create hardware engg org unit +dn: ou=hwengg,dc=min,dc=io +objectClass: organizationalUnit +ou: hwengg + +# Create people sub-org +dn: ou=people,ou=hwengg,dc=min,dc=io +objectClass: organizationalUnit +ou: people + +# Create Alice, Bob and Cody in hwengg +dn: uid=alice1,ou=people,ou=hwengg,dc=min,dc=io +objectClass: inetOrgPerson +cn: Alice Smith +sn: Smith +uid: alice1 +mail: alice@example.io +userPassword: {SSHA}Yeh2/IV/q/HjG2yzN3YdE9CAF3EJFCLu + +dn: uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io +objectClass: inetOrgPerson +cn: Robert Fisher +sn: Fisher +uid: bobfisher +mail: bob@example.io +userPassword: {SSHA}LktfbhK5oXSdDWCNzauJ9JA+Poxinl3y + +dn: uid=cody3,ou=people,ou=hwengg,dc=min,dc=io +objectClass: inetOrgPerson +cn: Cody Thomas +sn: Thomas +uid: cody3 +mail: cody@example.io +userPassword: {SSHA}H8B0gaOd4bRklK3fXj9ltHvJXWQFXW5Q + +# Create groups ou for hwengg +dn: ou=groups,ou=hwengg,dc=min,dc=io +objectclass: organizationalUnit +ou: groups +description: groups branch + +# Create project groups + +dn: cn=projectx,ou=groups,ou=hwengg,dc=min,dc=io +objectclass: groupofnames +cn: projectx +description: Project X group members +member: uid=alice1,ou=people,ou=hwengg,dc=min,dc=io +member: uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io + +dn: cn=projecty,ou=groups,ou=hwengg,dc=min,dc=io +objectclass: groupofnames +cn: projecty +description: Project Y group members +member: uid=bobfisher,ou=people,ou=hwengg,dc=min,dc=io +member: uid=cody3,ou=people,ou=hwengg,dc=min,dc=io diff --git a/docs/distributed/samples/myminio-iam-info-openid.zip b/docs/distributed/samples/myminio-iam-info-openid.zip new file mode 100644 index 00000000..aec4ca70 Binary files /dev/null and b/docs/distributed/samples/myminio-iam-info-openid.zip differ diff --git a/docs/distributed/samples/myminio-iam-info.zip b/docs/distributed/samples/myminio-iam-info.zip new file mode 100644 index 00000000..cd1d7ec1 Binary files /dev/null and b/docs/distributed/samples/myminio-iam-info.zip differ diff --git a/docs/docker/README.md b/docs/docker/README.md new file mode 100644 index 00000000..ccc9031f --- /dev/null +++ b/docs/docker/README.md @@ -0,0 +1,215 @@ +# MinIO Docker Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) + +See our web documentation on [Deploying MinIO in Standalone Mode](Deploy Standalone MinIO in a Container) for a more structured tutorial on deploying MinIO in a container. + +## Prerequisites + +Docker installed on your machine. Download the relevant installer from [here](https://www.docker.com/community-edition#/download). + +## Run Standalone MinIO on Docker + +*Note*: Standalone MinIO is intended for early development and evaluation. For production clusters, deploy a [Distributed](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-as-a-container.html) MinIO deployment. + +MinIO needs a persistent volume to store configuration and application data. For testing purposes, you can launch MinIO by simply passing a directory (`/data` in the example below). This directory gets created in the container filesystem at the time of container start. But all the data is lost after container exits. + +```sh +docker run \ + -p 9000:9000 \ + -p 9001:9001 \ + -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \ + -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \ + quay.io/minio/minio server /data --console-address ":9001" +``` + +To create a MinIO container with persistent storage, you need to map local persistent directories from the host OS to virtual config. To do this, run the below commands + +### GNU/Linux and macOS + +```sh +mkdir -p ~/minio/data + +docker run \ + -p 9000:9000 \ + -p 9001:9001 \ + --name minio1 \ + -v ~/minio/data:/data \ + -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \ + -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \ + quay.io/minio/minio server /data --console-address ":9001" +``` + +The command creates a new local directory `~/minio/data` in your user home directory. It then starts the MinIO container with the `-v` argument to map the local path (`~/minio/data`) to the specified virtual container directory (`/data`). When MinIO writes data to `/data`, that data is actually written to the local path `~/minio/data` where it can persist between container restarts. + +### Windows + +```sh +docker run \ + -p 9000:9000 \ + -p 9001:9001 \ + --name minio1 \ + -v D:\data:/data \ + -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \ + -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \ + quay.io/minio/minio server /data --console-address ":9001" +``` + +## Run Distributed MinIO on Containers + +We recommend kubernetes based deployment for production level deployment . + +See the [Kubernetes documentation](https://docs.min.io/community/minio-object-store/operations/deployments/kubernetes.html) for more information. + +## MinIO Docker Tips + +### MinIO Custom Access and Secret Keys + +To override MinIO's auto-generated keys, you may pass secret and access keys explicitly as environment variables. MinIO server also allows regular strings as access and secret keys. + +#### GNU/Linux and macOS (custom access and secret keys) + +```sh +docker run \ + -p 9000:9000 \ + -p 9001:9001 \ + --name minio1 \ + -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \ + -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \ + -v /mnt/data:/data \ + quay.io/minio/minio server /data --console-address ":9001" +``` + +#### Windows (custom access and secret keys) + +```powershell +docker run \ + -p 9000:9000 \ + -p 9001:9001 \ + --name minio1 \ + -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \ + -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" \ + -v D:\data:/data \ + quay.io/minio/minio server /data --console-address ":9001" +``` + +### Run MinIO Docker as a regular user + +Docker provides standardized mechanisms to run docker containers as non-root users. + +#### GNU/Linux and macOS (regular user) + +On Linux and macOS you can use `--user` to run the container as regular user. + +> NOTE: make sure --user has write permission to *${HOME}/data* prior to using `--user`. + +```sh +mkdir -p ${HOME}/data +docker run \ + -p 9000:9000 \ + -p 9001:9001 \ + --user $(id -u):$(id -g) \ + --name minio1 \ + -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \ + -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY" \ + -v ${HOME}/data:/data \ + quay.io/minio/minio server /data --console-address ":9001" +``` + +#### Windows (regular user) + +On windows you would need to use [Docker integrated windows authentication](https://success.docker.com/article/modernizing-traditional-dot-net-applications#integratedwindowsauthentication) and [Create a container with Active Directory Support](https://blogs.msdn.microsoft.com/containerstuff/2017/01/30/create-a-container-with-active-directory-support/) + +> NOTE: make sure your AD/Windows user has write permissions to *D:\data* prior to using `credentialspec=`. + +```powershell +docker run \ + -p 9000:9000 \ + -p 9001:9001 \ + --name minio1 \ + --security-opt "credentialspec=file://myuser.json" + -e "MINIO_ROOT_USER=AKIAIOSFODNN7EXAMPLE" \ + -e "MINIO_ROOT_PASSWORD=wJalrXUtnFEMIK7MDENGbPxRfiCYEXAMPLEKEY" \ + -v D:\data:/data \ + quay.io/minio/minio server /data --console-address ":9001" +``` + +### MinIO Custom Access and Secret Keys using Docker secrets + +To override MinIO's auto-generated keys, you may pass secret and access keys explicitly by creating access and secret keys as [Docker secrets](https://docs.docker.com/engine/swarm/secrets/). MinIO server also allows regular strings as access and secret keys. + +``` +echo "AKIAIOSFODNN7EXAMPLE" | docker secret create access_key - +echo "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" | docker secret create secret_key - +``` + +Create a MinIO service using `docker service` to read from Docker secrets. + +``` +docker service create --name="minio-service" --secret="access_key" --secret="secret_key" quay.io/minio/minio server /data +``` + +Read more about `docker service` [here](https://docs.docker.com/engine/swarm/how-swarm-mode-works/services/) + +#### MinIO Custom Access and Secret Key files + +To use other secret names follow the instructions above and replace `access_key` and `secret_key` with your custom names (e.g. `my_secret_key`,`my_custom_key`). Run your service with + +``` +docker service create --name="minio-service" \ + --secret="my_access_key" \ + --secret="my_secret_key" \ + --env="MINIO_ROOT_USER_FILE=my_access_key" \ + --env="MINIO_ROOT_PASSWORD_FILE=my_secret_key" \ + quay.io/minio/minio server /data +``` + +`MINIO_ROOT_USER_FILE` and `MINIO_ROOT_PASSWORD_FILE` also support custom absolute paths, in case Docker secrets are mounted to custom locations or other tools are used to mount secrets into the container. For example, HashiCorp Vault injects secrets to `/vault/secrets`. With the custom names above, set the environment variables to + +``` +MINIO_ROOT_USER_FILE=/vault/secrets/my_access_key +MINIO_ROOT_PASSWORD_FILE=/vault/secrets/my_secret_key +``` + +### Retrieving Container ID + +To use Docker commands on a specific container, you need to know the `Container ID` for that container. To get the `Container ID`, run + +```sh +docker ps -a +``` + +`-a` flag makes sure you get all the containers (Created, Running, Exited). Then identify the `Container ID` from the output. + +### Starting and Stopping Containers + +To start a stopped container, you can use the [`docker start`](https://docs.docker.com/engine/reference/commandline/start/) command. + +```sh +docker start +``` + +To stop a running container, you can use the [`docker stop`](https://docs.docker.com/engine/reference/commandline/stop/) command. + +```sh +docker stop +``` + +### MinIO container logs + +To access MinIO logs, you can use the [`docker logs`](https://docs.docker.com/engine/reference/commandline/logs/) command. + +```sh +docker logs +``` + +### Monitor MinIO Docker Container + +To monitor the resources used by MinIO container, you can use the [`docker stats`](https://docs.docker.com/engine/reference/commandline/stats/) command. + +```sh +docker stats +``` + +## Explore Further + +* [MinIO in a Container Installation Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-as-a-container.html) +* [MinIO Erasure Code QuickStart Guide](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html) diff --git a/docs/erasure/README.md b/docs/erasure/README.md new file mode 100644 index 00000000..48b3cb6b --- /dev/null +++ b/docs/erasure/README.md @@ -0,0 +1,67 @@ +# MinIO Erasure Code Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +MinIO protects data against hardware failures and silent data corruption using erasure code and checksums. With the highest level of redundancy, you may lose up to half (N/2) of the total drives and still be able to recover the data. + +## What is Erasure Code? + +Erasure code is a mathematical algorithm to reconstruct missing or corrupted data. MinIO uses Reed-Solomon code to shard objects into variable data and parity blocks. For example, in a 12 drive setup, an object can be sharded to a variable number of data and parity blocks across all the drives - ranging from six data and six parity blocks to ten data and two parity blocks. + +By default, MinIO shards the objects across N/2 data and N/2 parity drives. Though, you can use [storage classes](https://github.com/minio/minio/tree/master/docs/erasure/storage-class) to use a custom configuration. We recommend N/2 data and parity blocks, as it ensures the best protection from drive failures. + +In 12 drive example above, with MinIO server running in the default configuration, you can lose any of the six drives and still reconstruct the data reliably from the remaining drives. + +## Why is Erasure Code useful? + +Erasure code protects data from multiple drives failure, unlike RAID or replication. For example, RAID6 can protect against two drive failure whereas in MinIO erasure code you can lose as many as half of drives and still the data remains safe. Further, MinIO's erasure code is at the object level and can heal one object at a time. For RAID, healing can be done only at the volume level which translates into high downtime. As MinIO encodes each object individually, it can heal objects incrementally. Storage servers once deployed should not require drive replacement or healing for the lifetime of the server. MinIO's erasure coded backend is designed for operational efficiency and takes full advantage of hardware acceleration whenever available. + +![Erasure](https://github.com/minio/minio/blob/master/docs/screenshots/erasure-code.jpg?raw=true) + +## What is Bit Rot protection? + +Bit Rot, also known as data rot or silent data corruption is a data loss issue faced by disk drives today. Data on the drive may silently get corrupted without signaling an error has occurred, making bit rot more dangerous than a permanent hard drive failure. + +MinIO's erasure coded backend uses high speed [HighwayHash](https://github.com/minio/highwayhash) checksums to protect against Bit Rot. + +## How are drives used for Erasure Code? + +MinIO divides the drives you provide into erasure-coding sets of *2 to 16* drives. Therefore, the number of drives you present must be a multiple of one of these numbers. Each object is written to a single erasure-coding set. + +Minio uses the largest possible EC set size which divides into the number of drives given. For example, *18 drives* are configured as *2 sets of 9 drives*, and *24 drives* are configured as *2 sets of 12 drives*. This is true for scenarios when running MinIO as a standalone erasure coded deployment. In [distributed setup however node (affinity) based](https://docs.min.io/community/minio-object-store/operations/install-deploy-manage/deploy-minio-multi-node-multi-drive.html) erasure stripe sizes are chosen. + +The drives should all be of approximately the same size. + +## Get Started with MinIO in Erasure Code + +### 1. Prerequisites + +Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html) + +### 2. Run MinIO Server with Erasure Code + +Example: Start MinIO server in a 12 drives setup, using MinIO binary. + +```sh +minio server /data{1...12} +``` + +Example: Start MinIO server in a 8 drives setup, using MinIO Docker image. + +```sh +podman run \ + -p 9000:9000 \ + -p 9001:9001 \ + --name minio \ + -v /mnt/data1:/data1 \ + -v /mnt/data2:/data2 \ + -v /mnt/data3:/data3 \ + -v /mnt/data4:/data4 \ + -v /mnt/data5:/data5 \ + -v /mnt/data6:/data6 \ + -v /mnt/data7:/data7 \ + -v /mnt/data8:/data8 \ + quay.io/minio/minio server /data{1...8} --console-address ":9001" +``` + +### 3. Test your setup + +You may unplug drives randomly and continue to perform I/O on the system. diff --git a/docs/erasure/storage-class/README.md b/docs/erasure/storage-class/README.md new file mode 100644 index 00000000..15190a65 --- /dev/null +++ b/docs/erasure/storage-class/README.md @@ -0,0 +1,119 @@ +# MinIO Storage Class Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +MinIO server supports storage class in erasure coding mode. This allows configurable data and parity drives per object. + +This page is intended as a summary of MinIO Erasure Coding. For a more complete explanation, see . + +## Overview + +MinIO supports two storage classes, Reduced Redundancy class and Standard class. These classes can be defined using environment variables +set before starting MinIO server. After the data and parity drives for each storage class are defined using environment variables, +you can set the storage class of an object via request metadata field `x-amz-storage-class`. MinIO server then honors the storage class by +saving the object in specific number of data and parity drives. + +## Storage usage + +The selection of varying data and parity drives has a direct impact on the drive space usage. With storage class, you can optimize for high +redundancy or better drive space utilization. + +To get an idea of how various combinations of data and parity drives affect the storage usage, let’s take an example of a 100 MiB file stored +on 16 drive MinIO deployment. If you use eight data and eight parity drives, the file space usage will be approximately twice, i.e. 100 MiB +file will take 200 MiB space. But, if you use ten data and six parity drives, same 100 MiB file takes around 160 MiB. If you use 14 data and +two parity drives, 100 MiB file takes only approximately 114 MiB. + +Below is a list of data/parity drives and corresponding _approximate_ storage space usage on a 16 drive MinIO deployment. The field _storage +usage ratio_ is simply the drive space used by the file after erasure-encoding, divided by actual file size. + +| Total Drives (N) | Data Drives (D) | Parity Drives (P) | Storage Usage Ratio | +|------------------|-----------------|-------------------|---------------------| +| 16 | 8 | 8 | 2.00 | +| 16 | 9 | 7 | 1.79 | +| 16 | 10 | 6 | 1.60 | +| 16 | 11 | 5 | 1.45 | +| 16 | 12 | 4 | 1.34 | +| 16 | 13 | 3 | 1.23 | +| 16 | 14 | 2 | 1.14 | + +You can calculate _approximate_ storage usage ratio using the formula - total drives (N) / data drives (D). + +### Allowed values for STANDARD storage class + +`STANDARD` storage class implies more parity than `REDUCED_REDUNDANCY` class. So, `STANDARD` parity drives should be + +- Greater than or equal to 2, if `REDUCED_REDUNDANCY` parity is not set. +- Greater than `REDUCED_REDUNDANCY` parity, if it is set. + +Parity blocks can not be higher than data blocks, so `STANDARD` storage class parity can not be higher than N/2. (N being total number of drives) + +The default value for the `STANDARD` storage class depends on the number of volumes in the erasure set: + +| Erasure Set Size | Default Parity (EC:N) | +|------------------|-----------------------| +| 5 or fewer | EC:2 | +| 6-7 | EC:3 | +| 8 or more | EC:4 | + +For more complete documentation on Erasure Set sizing, see the [MinIO Documentation on Erasure Sets](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html#erasure-sets). + +### Allowed values for REDUCED_REDUNDANCY storage class + +`REDUCED_REDUNDANCY` implies lesser parity than `STANDARD` class. So,`REDUCED_REDUNDANCY` parity drives should be + +- Less than N/2, if `STANDARD` parity is not set. +- Less than `STANDARD` Parity, if it is set. + +Default value for `REDUCED_REDUNDANCY` storage class is `1`. + +## Get started with Storage Class + +### Set storage class + +The format to set storage class environment variables is as follows + +`MINIO_STORAGE_CLASS_STANDARD=EC:parity` +`MINIO_STORAGE_CLASS_RRS=EC:parity` + +For example, set `MINIO_STORAGE_CLASS_RRS` parity 2 and `MINIO_STORAGE_CLASS_STANDARD` parity 3 + +```sh +export MINIO_STORAGE_CLASS_STANDARD=EC:3 +export MINIO_STORAGE_CLASS_RRS=EC:2 +``` + +Storage class can also be set via `mc admin config` get/set commands to update the configuration. Refer [storage class](https://github.com/minio/minio/tree/master/docs/config#storage-class) for +more details. + +#### Note + +- If `STANDARD` storage class is set via environment variables or `mc admin config` get/set commands, and `x-amz-storage-class` is not present in request metadata, MinIO server will +apply `STANDARD` storage class to the object. This means the data and parity drives will be used as set in `STANDARD` storage class. + +- If storage class is not defined before starting MinIO server, and subsequent PutObject metadata field has `x-amz-storage-class` present +with values `REDUCED_REDUNDANCY` or `STANDARD`, MinIO server uses default parity values. + +### Set metadata + +In below example `minio-go` is used to set the storage class to `REDUCED_REDUNDANCY`. This means this object will be split across 6 data drives and 2 parity drives (as per the storage class set in previous step). + +```go +s3Client, err := minio.New("localhost:9000", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true) +if err != nil { + log.Fatalln(err) +} + +object, err := os.Open("my-testfile") +if err != nil { + log.Fatalln(err) +} +defer object.Close() +objectStat, err := object.Stat() +if err != nil { + log.Fatalln(err) +} + +n, err := s3Client.PutObject("my-bucketname", "my-objectname", object, objectStat.Size(), minio.PutObjectOptions{ContentType: "application/octet-stream", StorageClass: "REDUCED_REDUNDANCY"}) +if err != nil { + log.Fatalln(err) +} +log.Println("Uploaded", "my-objectname", " of size: ", n, "Successfully.") +``` diff --git a/docs/extensions/fan-out/README.md b/docs/extensions/fan-out/README.md new file mode 100644 index 00000000..2f828e7d --- /dev/null +++ b/docs/extensions/fan-out/README.md @@ -0,0 +1,18 @@ +# Fan-Out Uploads [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) + +## Overview + +MinIO implements an S3 extension to perform multiple concurrent fan-out upload operations. A perfect use case scenario for performing fan-out operations of incoming TSB (Time Shift Buffer's). TSBs are a method of facilitating time-shifted playback of television signaling, and media content. + +MinIO implements an S3 extension to the [PostUpload](https://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectPOST.html) where in a special fan-out list is sent along with the TSB's for MinIO make multiple uploads from a single source stream. Optionally supports custom metadata, tags and other retention settings. All objects are also readable independently once upload is completed via the regular S3 [GetObject](https://docs.aws.amazon.com/AmazonS3/latest/API/API_GetObject.html) API. + +## How to enable Fan-Out Uploads ? + +Fan-Out uploads are automatically enabled if `x-minio-fanout-list` form-field is provided with the PostUpload API, to keep things simple higher level APIs are provided in our SDKs for example in `minio-go` SDK: + +``` +PutObjectFanOut(ctx context.Context, bucket string, fanOutContent io.Reader, fanOutReq minio.PutObjectFanOutRequest) ([]minio.PutObjectFanOutResponse, error) +``` + + + diff --git a/docs/extensions/s3zip/README.md b/docs/extensions/s3zip/README.md new file mode 100644 index 00000000..7db6bdec --- /dev/null +++ b/docs/extensions/s3zip/README.md @@ -0,0 +1,45 @@ +# Perform S3 operations in a ZIP content[![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) + +## Overview + +MinIO implements an S3 extension to list, stat and download files inside a ZIP file stored in any bucket. A perfect use case scenario is when you have a lot of small files archived in multiple ZIP files. Uploading them is faster than uploading small files individually. Besides, your S3 applications will be able to access to the data with little performance overhead. + +The main limitation is that to update or delete content of a file inside a ZIP file the entire ZIP file must be replaced. + +## How to enable S3 ZIP behavior ? + +Ensure to set the following header `x-minio-extract` to `true` in your S3 requests. + +## How to access to files inside a ZIP archive + +Accessing to contents inside an archive can be done using regular S3 API with a modified request path. You just need to append the path of the content inside the archive to the path of the archive itself. + +e.g.: +To download `2021/taxes.csv` archived in `financial.zip` and stored under a bucket named `company-data`, you can issue a GET request using the following path 'company-data/financial.zip/2021/taxes.csv` + +## Contents properties + +All properties except the file size are tied to the zip file. This means that modification date, headers, tags, etc. can only be set for the zip file as a whole. In similar fashion, replication will replicate the zip file as a whole and not individual files. + +## Code Examples + +[Using minio-go library](https://github.com/minio/minio/blob/master/docs/extensions/s3zip/examples/minio-go/main.go) +[Using AWS JS SDK v2](https://github.com/minio/minio/blob/master/docs/extensions/s3zip/examples/aws-js/main.js) +[Using boto3](https://github.com/minio/minio/blob/master/docs/extensions/s3zip/examples/boto3/main.py) + +## Requirements and limits + +- ListObjectsV2 can only list the most recent ZIP archive version of your object, applicable only for versioned buckets. +- ListObjectsV2 API calls must be used to list zip file content. +- Range requests for GetObject/HeadObject for individual files from zip is not supported. +- Names inside ZIP files are kept unmodified, but some may lead to invalid paths. See [Object key naming guidelines](https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-keys.html) on safe names. +- This API behavior is limited for following **read** operations on files inside a zip archive: + - `HeadObject` + - `GetObject` + - `ListObjectsV2` +- If the ZIP file directory isn't located within the last 100MB the file will not be parsed. +- A maximum of 100M inside a single zip is allowed. However, a reasonable limit of 100,000 files inside a single ZIP archive is recommended for best performance and memory usage trade-off. + +## Content-Type + +The Content-Type of the response will be determined by the extension and the following: https://pkg.go.dev/mime#TypeByExtension \ No newline at end of file diff --git a/docs/extensions/s3zip/examples/aws-js/main.js b/docs/extensions/s3zip/examples/aws-js/main.js new file mode 100644 index 00000000..02b0571f --- /dev/null +++ b/docs/extensions/s3zip/examples/aws-js/main.js @@ -0,0 +1,31 @@ + +var AWS = require('aws-sdk'); + +var s3 = new AWS.S3({ + accessKeyId: 'YOUR-ACCESSKEYID' , + secretAccessKey: 'YOUR-SECRETACCESSKEY' , + endpoint: 'http://127.0.0.1:9000' , + s3ForcePathStyle: true, + signatureVersion: 'v4' +}); + +// List all contents stored in the zip archive +s3.listObjectsV2({Bucket : 'your-bucket', Prefix: 'path/to/file.zip/'}). + on('build', function(req) { req.httpRequest.headers['X-Minio-Extract'] = 'true'; }). + send(function(err, data) { + if (err) { + console.log("Error", err); + } else { + console.log("Success", data); + } + }); + + +// Download a file in the archive and store it in /tmp/data.csv +var file = require('fs').createWriteStream('/tmp/data.csv'); +s3.getObject({Bucket: 'your-bucket', Key: 'path/to/file.zip/data.csv'}). + on('build', function(req) { req.httpRequest.headers['X-Minio-Extract'] = 'true'; }). + on('httpData', function(chunk) { file.write(chunk); }). + on('httpDone', function() { file.end(); }). + send(); + diff --git a/docs/extensions/s3zip/examples/aws-js/package.json b/docs/extensions/s3zip/examples/aws-js/package.json new file mode 100644 index 00000000..29f2bd2d --- /dev/null +++ b/docs/extensions/s3zip/examples/aws-js/package.json @@ -0,0 +1,8 @@ +{ + "name": "s3-zip-example", + "version": "1.0.0", + "main": "main.js", + "dependencies": { + "aws-sdk": "^2.924.0" + } +} diff --git a/docs/extensions/s3zip/examples/boto3/main.py b/docs/extensions/s3zip/examples/boto3/main.py new file mode 100644 index 00000000..bb3d58af --- /dev/null +++ b/docs/extensions/s3zip/examples/boto3/main.py @@ -0,0 +1,25 @@ +#!/usr/bin/env/python + +import boto3 +from botocore.client import Config + +s3 = boto3.client('s3', + endpoint_url='http://localhost:9000', + aws_access_key_id='YOUR-ACCESSKEYID', + aws_secret_access_key='YOUR-SECRETACCESSKEY', + config=Config(signature_version='s3v4'), + region_name='us-east-1') + + +def _add_header(request, **kwargs): + request.headers.add_header('x-minio-extract', 'true') +event_system = s3.meta.events +event_system.register_first('before-sign.s3.*', _add_header) + +# List zip contents +response = s3.list_objects_v2(Bucket="your-bucket", Prefix="path/to/file.zip/") +print(response) + +# Download data.csv stored in the zip file +s3.download_file(Bucket='your-bucket', Key='path/to/file.zip/data.csv', Filename='/tmp/data.csv') + diff --git a/docs/extensions/s3zip/examples/minio-go/main.go b/docs/extensions/s3zip/examples/minio-go/main.go new file mode 100644 index 00000000..7b572ae2 --- /dev/null +++ b/docs/extensions/s3zip/examples/minio-go/main.go @@ -0,0 +1,35 @@ +package main + +import ( + "context" + "io" + "log" + "os" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +func main() { + s3Client, err := minio.New("minio-server-address:9000", &minio.Options{ + Creds: credentials.NewStaticV4("access-key", "secret-key", ""), + }) + if err != nil { + log.Fatalln(err) + } + + var opts minio.GetObjectOptions + + // Add extract header to request: + opts.Set("x-minio-extract", "true") + + // Download API.md from the archive + rd, err := s3Client.GetObject(context.Background(), "your-bucket", "path/to/file.zip/data.csv", opts) + if err != nil { + log.Fatalln(err) + } + _, err = io.Copy(os.Stdout, rd) + if err != nil { + log.Fatalln(err) + } +} diff --git a/docs/federation/lookup/Corefile.example b/docs/federation/lookup/Corefile.example new file mode 100644 index 00000000..b44a848a --- /dev/null +++ b/docs/federation/lookup/Corefile.example @@ -0,0 +1,11 @@ +. { + etcd churchofminio.com { + endpoint http://localhost:2379 http://localhost:4001 + upstream /etc/resolv.conf + } + debug + prometheus + cache 160 mydomain.com + loadbalance + forward . /etc/resolv.conf +} \ No newline at end of file diff --git a/docs/federation/lookup/README.md b/docs/federation/lookup/README.md new file mode 100644 index 00000000..b2cebd63 --- /dev/null +++ b/docs/federation/lookup/README.md @@ -0,0 +1,86 @@ +# Federation Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) *Federation feature is deprecated and should be avoided for future deployments* + +This document explains how to configure MinIO with `Bucket lookup from DNS` style federation. + +## Get started + +### 1. Prerequisites + +Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html). + +### 2. Run MinIO in federated mode + +Bucket lookup from DNS federation requires two dependencies + +- etcd (for bucket DNS service records) +- CoreDNS (for DNS management based on populated bucket DNS service records, optional) + +## Architecture + +![bucket-lookup](https://github.com/minio/minio/blob/master/docs/federation/lookup/bucket-lookup.png?raw=true) + +### Environment variables + +#### MINIO_ETCD_ENDPOINTS + +This is comma separated list of etcd servers that you want to use as the MinIO federation back-end. This should +be same across the federated deployment, i.e. all the MinIO instances within a federated deployment should use same +etcd back-end. + +#### MINIO_DOMAIN + +This is the top level domain name used for the federated setup. This domain name should ideally resolve to a load-balancer +running in front of all the federated MinIO instances. The domain name is used to create sub domain entries to etcd. For +example, if the domain is set to `domain.com`, the buckets `bucket1`, `bucket2` will be accessible as `bucket1.domain.com` +and `bucket2.domain.com`. + +#### MINIO_PUBLIC_IPS + +This is comma separated list of IP addresses to which buckets created on this MinIO instance will resolve to. For example, +a bucket `bucket1` created on current MinIO instance will be accessible as `bucket1.domain.com`, and the DNS entry for +`bucket1.domain.com` will point to IP address set in `MINIO_PUBLIC_IPS`. + +- This field is mandatory for standalone and erasure code MinIO server deployments, to enable federated mode. +- This field is optional for distributed deployments. If you don't set this field in a federated setup, we use the IP addresses of +hosts passed to the MinIO server startup and use them for DNS entries. + +### Run Multiple Clusters + +> cluster1 + +```sh +export MINIO_ETCD_ENDPOINTS="http://remote-etcd1:2379,http://remote-etcd2:4001" +export MINIO_DOMAIN=domain.com +export MINIO_PUBLIC_IPS=44.35.2.1,44.35.2.2,44.35.2.3,44.35.2.4 +minio server http://rack{1...4}.host{1...4}.domain.com/mnt/export{1...32} +``` + +> cluster2 + +```sh +export MINIO_ETCD_ENDPOINTS="http://remote-etcd1:2379,http://remote-etcd2:4001" +export MINIO_DOMAIN=domain.com +export MINIO_PUBLIC_IPS=44.35.1.1,44.35.1.2,44.35.1.3,44.35.1.4 +minio server http://rack{5...8}.host{5...8}.domain.com/mnt/export{1...32} +``` + +In this configuration you can see `MINIO_ETCD_ENDPOINTS` points to the etcd backend which manages MinIO's +`config.json` and bucket DNS SRV records. `MINIO_DOMAIN` indicates the domain suffix for the bucket which +will be used to resolve bucket through DNS. For example if you have a bucket such as `mybucket`, the +client can use now `mybucket.domain.com` to directly resolve itself to the right cluster. `MINIO_PUBLIC_IPS` +points to the public IP address where each cluster might be accessible, this is unique for each cluster. + +NOTE: `mybucket` only exists on one cluster either `cluster1` or `cluster2` this is random and +is decided by how `domain.com` gets resolved, if there is a round-robin DNS on `domain.com` then +it is randomized which cluster might provision the bucket. + +### 3. Test your setup + +To test this setup, access the MinIO server via browser or [`mc`](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart). You’ll see the uploaded files are accessible from the all the MinIO endpoints. + +## Explore Further + +- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html) +- [Use `aws-cli` with MinIO Server](https://docs.min.io/community/minio-object-store/integrations/aws-cli-with-minio.html) +- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/go/minio-go.html) +- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html) diff --git a/docs/federation/lookup/bucket-lookup.png b/docs/federation/lookup/bucket-lookup.png new file mode 100644 index 00000000..b2b60d1b Binary files /dev/null and b/docs/federation/lookup/bucket-lookup.png differ diff --git a/docs/ftp/README.md b/docs/ftp/README.md new file mode 100644 index 00000000..1fb6e579 --- /dev/null +++ b/docs/ftp/README.md @@ -0,0 +1,257 @@ +# MinIO FTP/SFTP Server + +MinIO natively supports FTP/SFTP protocol, this allows any ftp/sftp client to upload and download files. + +Currently supported `FTP/SFTP` operations are as follows: + +| ftp-client commands | supported | +|:-------------------:|:----------| +| get | yes | +| put | yes | +| ls | yes | +| mkdir | yes | +| rmdir | yes | +| delete | yes | +| append | no | +| rename | no | + +MinIO supports following FTP/SFTP based protocols to access and manage data. + +- Secure File Transfer Protocol (SFTP) – Defined by the Internet Engineering Task Force (IETF) as an + extended version of SSH 2.0, allowing file transfer over SSH and for use with Transport Layer + Security (TLS) and VPN applications. + +- File Transfer Protocol over SSL/TLS (FTPS) – Encrypted FTP communication via TLS certificates. + +- File Transfer Protocol (FTP) – Defined by RFC114 originally, and replaced by RFC765 and RFC959 + unencrypted FTP communication (Not-recommended) + +## Scope + +- All IAM Credentials are allowed access excluding rotating credentials, rotating credentials + are not allowed to login via FTP/SFTP ports, you must use S3 API port for if you are using + rotating credentials. + +- Access to bucket(s) and object(s) are governed via IAM policies associated with the incoming + login credentials. + +- Allows authentication and access for all + - Built-in IDP users and their respective service accounts + - LDAP/AD users and their respective service accounts + - OpenID/OIDC service accounts + +- On versioned buckets, FTP/SFTP only operates on latest objects, if you need to retrieve + an older version you must use an `S3 API client` such as [`mc`](https://github.com/minio/mc). + +- All features currently used by your buckets will work as is without any changes + - SSE (Server Side Encryption) + - Replication (Server Side Replication) + +## Prerequisites + +- It is assumed you have users created and configured with relevant access policies, to start with + use basic "readwrite" canned policy to test all the operations before you finalize on what level + of restrictions are needed for a user. + +- No "admin:*" operations are needed for FTP/SFTP access to the bucket(s) and object(s), so you may + skip them for restrictions. + +## Usage + +Start MinIO in a distributed setup, with 'ftp/sftp' enabled. + +``` +minio server http://server{1...4}/disk{1...4} + --ftp="address=:8021" --ftp="passive-port-range=30000-40000" \ + --sftp="address=:8022" --sftp="ssh-private-key=/home/miniouser/.ssh/id_rsa" +... +... +``` + +Following example shows connecting via ftp client using `minioadmin` credentials, and list a bucket named `runner`: + +``` +ftp localhost -P 8021 +Connected to localhost. +220 Welcome to MinIO FTP Server +Name (localhost:user): minioadmin +331 User name ok, password required +Password: +230 Password ok, continue +Remote system type is UNIX. +Using binary mode to transfer files. +ftp> ls runner/ +229 Entering Extended Passive Mode (|||39155|) +150 Opening ASCII mode data connection for file list +drwxrwxrwx 1 nobody nobody 0 Jan 1 00:00 chunkdocs/ +drwxrwxrwx 1 nobody nobody 0 Jan 1 00:00 testdir/ +... +``` + +Following example shows how to list an object and download it locally via `ftp` client: + +``` +ftp> ls runner/chunkdocs/metadata +229 Entering Extended Passive Mode (|||44269|) +150 Opening ASCII mode data connection for file list +-rwxrwxrwx 1 nobody nobody 45 Apr 1 06:13 chunkdocs/metadata +226 Closing data connection, sent 75 bytes +ftp> get +(remote-file) runner/chunkdocs/metadata +(local-file) test +local: test remote: runner/chunkdocs/metadata +229 Entering Extended Passive Mode (|||37785|) +150 Data transfer starting 45 bytes + 45 3.58 KiB/s +226 Closing data connection, sent 45 bytes +45 bytes received in 00:00 (3.55 KiB/s) +... +``` + + +Following example shows connecting via sftp client using `minioadmin` credentials, and list a bucket named `runner`: + +``` +sftp -P 8022 minioadmin@localhost +minioadmin@localhost's password: +Connected to localhost. +sftp> ls runner/ +chunkdocs testdir +``` + +Following example shows how to download an object locally via `sftp` client: + +``` +sftp> get runner/chunkdocs/metadata metadata +Fetching /runner/chunkdocs/metadata to metadata +metadata 100% 226 16.6KB/s 00:00 +sftp> +``` + +## Advanced options + +### Change default FTP port + +Default port '8021' can be changed via + +``` +--ftp="address=:3021" +``` + +### Change FTP passive port range + +By default FTP requests OS to give a free port automatically, however you may want to restrict +this to specific ports in certain restricted environments via + +``` +--ftp="passive-port-range=30000-40000" +``` + +### Change default SFTP port + +Default port '8022' can be changed via + +``` +--sftp="address=:3022" +``` + +### TLS (FTP) + +Unlike SFTP server, FTP server is insecure by default. To operate under TLS mode, you need to provide certificates via + +``` +--ftp="tls-private-key=path/to/private.key" --ftp="tls-public-cert=path/to/public.crt" +``` + +> NOTE: if MinIO distributed setup is already configured to run under TLS, FTP will automatically use the relevant +> certs from the server certificate chain, this is mainly to add simplicity of setup. However if you wish to terminate +> TLS certificates via a different domain for your FTP servers you may choose the above command line options. + + +### Custom Algorithms (SFTP) + +Custom algorithms can be specified via command line parameters. +Algorithms are comma separated. +Note that valid values does not in all cases represent default values. + +`--sftp=pub-key-algos=...` specifies the supported client public key +authentication algorithms. Note that this doesn't include certificate types +since those use the underlying algorithm. This list is sent to the client if +it supports the server-sig-algs extension. Order is irrelevant. + +Valid values +``` +ssh-ed25519 +sk-ssh-ed25519@openssh.com +sk-ecdsa-sha2-nistp256@openssh.com +ecdsa-sha2-nistp256 +ecdsa-sha2-nistp384 +ecdsa-sha2-nistp521 +rsa-sha2-256 +rsa-sha2-512 +ssh-rsa +ssh-dss +``` + +`--sftp=kex-algos=...` specifies the supported key-exchange algorithms in preference order. + +Valid values: + +``` +curve25519-sha256 +curve25519-sha256@libssh.org +ecdh-sha2-nistp256 +ecdh-sha2-nistp384 +ecdh-sha2-nistp521 +diffie-hellman-group14-sha256 +diffie-hellman-group16-sha512 +diffie-hellman-group14-sha1 +diffie-hellman-group1-sha1 +``` + +`--sftp=cipher-algos=...` specifies the allowed cipher algorithms. +If unspecified then a sensible default is used. + +Valid values: +``` +aes128-ctr +aes192-ctr +aes256-ctr +aes128-gcm@openssh.com +aes256-gcm@openssh.com +chacha20-poly1305@openssh.com +arcfour256 +arcfour128 +arcfour +aes128-cbc +3des-cbc +``` + +`--sftp=mac-algos=...` specifies a default set of MAC algorithms in preference order. +This is based on RFC 4253, section 6.4, but with hmac-md5 variants removed because they have +reached the end of their useful life. + +Valid values: + +``` +hmac-sha2-256-etm@openssh.com +hmac-sha2-512-etm@openssh.com +hmac-sha2-256 +hmac-sha2-512 +hmac-sha1 +hmac-sha1-96 +``` + +### Certificate-based authentication + +`--sftp=trusted-user-ca-key=...` specifies a file containing public key of certificate authority that is trusted +to sign user certificates for authentication. + +Implementation is identical with "TrustedUserCAKeys" setting in OpenSSH server with exception that only one CA +key can be defined. + +If a certificate is presented for authentication and has its signing CA key is in this file, then it may be +used for authentication for any user listed in the certificate's principals list. + +Note that certificates that lack a list of principals will not be permitted for authentication using trusted-user-ca-key. +For more details on certificates, see the CERTIFICATES section in ssh-keygen(1). diff --git a/docs/iam/access-management-plugin.md b/docs/iam/access-management-plugin.md new file mode 100644 index 00000000..f18b1e11 --- /dev/null +++ b/docs/iam/access-management-plugin.md @@ -0,0 +1,161 @@ +# Access Management Plugin Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) + +MinIO now includes support for using an Access Management Plugin. This is to allow object storage access control to be managed externally via a webhook. + +When configured, MinIO sends request and credential details for every API call to an external HTTP(S) endpoint and expects an allow/deny response. MinIO is thus able to delegate access management to an external system, and users are able to use a custom solution instead of S3 standard IAM policies. + +Latency sensitive applications may notice an increased latency due to a request to the external plugin upon every authenticated request to MinIO. User are advised to provision their infrastructure such that latency and performance is acceptable. + +## Quickstart + +To easily try out the feature, run the included demo Access Management Plugin program in this directory: + +```sh +go run access-manager-plugin.go +``` + +This program, lets the admin user perform any action and prevents all other users from performing `s3:Put*` operations. + +In another terminal start MinIO: + +```sh +export MINIO_CI_CD=1 +export MINIO_ROOT_USER=minio +export MINIO_ROOT_PASSWORD=minio123 +export MINIO_POLICY_PLUGIN_URL=http://localhost:8080/ +minio server /tmp/disk{1...4} +``` + +Now, let's test it out with `mc`: + +```sh +mc alias set myminio http://localhost:9000 minio minio123 +mc ls myminio +mc mb myminio/test +mc cp /etc/issue myminio/test +mc admin user add myminio foo foobar123 +export MC_HOST_foo=http://foo:foobar123@localhost:9000 +mc ls foo +mc cp /etc/issue myminio/test/issue2 +``` + +Only the last operation would fail with a permissions error. + +## Configuration + +Access Management Plugin can be configured with environment variables: + +```sh +$ mc admin config set myminio policy_plugin --env +KEY: +policy_plugin enable Access Management Plugin for policy enforcement + +ARGS: +MINIO_POLICY_PLUGIN_URL* (url) plugin hook endpoint (HTTP(S)) e.g. "http://localhost:8181/v1/data/httpapi/authz/allow" +MINIO_POLICY_PLUGIN_AUTH_TOKEN (string) authorization header for plugin hook endpoint +MINIO_POLICY_PLUGIN_ENABLE_HTTP2 (bool) Enable experimental HTTP2 support to connect to plugin service (default: 'off') +MINIO_POLICY_PLUGIN_COMMENT (sentence) optionally add a comment to this setting +``` + +By default this plugin uses HTTP 1.x. To enable HTTP2 use the `MINIO_POLICY_PLUGIN_ENABLE_HTTP2` environment variable. + +## Request and Response + +MinIO will make a `POST` request with a JSON body to the given plugin URL. If the auth token parameter is set, it will be sent as an authorization header. + +The JSON body structure can be seen from this sample: + +
Request Body Sample + +```json +{ + "input": { + "account": "minio", + "groups": null, + "action": "s3:ListBucket", + "bucket": "test", + "conditions": { + "Authorization": [ + "AWS4-HMAC-SHA256 Credential=minio/20220507/us-east-1/s3/aws4_request, SignedHeaders=host;x-amz-content-sha256;x-amz-date, Signature=62012db6c47d697620cf6c68f0f45f6e34894589a53ab1faf6dc94338468c78a" + ], + "CurrentTime": [ + "2022-05-07T18:31:41Z" + ], + "Delimiter": [ + "/" + ], + "EpochTime": [ + "1651948301" + ], + "Prefix": [ + "" + ], + "Referer": [ + "" + ], + "SecureTransport": [ + "false" + ], + "SourceIp": [ + "127.0.0.1" + ], + "User-Agent": [ + "MinIO (linux; amd64) minio-go/v7.0.24 mc/DEVELOPMENT.2022-04-20T23-07-53Z" + ], + "UserAgent": [ + "MinIO (linux; amd64) minio-go/v7.0.24 mc/DEVELOPMENT.2022-04-20T23-07-53Z" + ], + "X-Amz-Content-Sha256": [ + "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" + ], + "X-Amz-Date": [ + "20220507T183141Z" + ], + "authType": [ + "REST-HEADER" + ], + "principaltype": [ + "Account" + ], + "signatureversion": [ + "AWS4-HMAC-SHA256" + ], + "userid": [ + "minio" + ], + "username": [ + "minio" + ], + "versionid": [ + "" + ] + }, + "owner": true, + "object": "", + "claims": {}, + "denyOnly": false + } +} +``` + +
+ +The response expected by MinIO, is a JSON body with a boolean: + +```json +{ + "result": true +} +``` + +The following structure is also accepted: + +```json +{ + "result": { + "allow": true + } +} +``` + +Any unmentioned JSON object keys in the above are ignored. diff --git a/docs/iam/access-manager-plugin.go b/docs/iam/access-manager-plugin.go new file mode 100644 index 00000000..218083a3 --- /dev/null +++ b/docs/iam/access-manager-plugin.go @@ -0,0 +1,112 @@ +//go:build ignore +// +build ignore + +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package main + +import ( + "bytes" + "encoding/json" + "flag" + "fmt" + "io" + "log" + "net/http" + "strings" +) + +var ( + keyFile string + certFile string +) + +func init() { + flag.StringVar(&keyFile, "key-file", "", "Path to TLS cert key file") + flag.StringVar(&certFile, "cert-file", "", "Path to TLS cert file") +} + +func writeErrorResponse(w http.ResponseWriter, err error) { + w.WriteHeader(http.StatusBadRequest) + json.NewEncoder(w).Encode(map[string]string{ + "error": fmt.Sprintf("%v", err), + }) +} + +type Result struct { + Result bool `json:"result"` +} + +func mainHandler(w http.ResponseWriter, r *http.Request) { + body, err := io.ReadAll(r.Body) + if err != nil { + writeErrorResponse(w, err) + return + } + + var out bytes.Buffer + json.Indent(&out, body, "", " ") + fmt.Printf("Received JSON payload:\n%s\n", out.String()) + + reqMap := make(map[string]interface{}) + err = json.Unmarshal(body, &reqMap) + if err != nil { + writeErrorResponse(w, err) + return + } + + m := reqMap["input"].(map[string]interface{}) + accountValue := m["account"].(string) + actionValue := m["action"].(string) + + // Allow user `minio` to perform any action. + var res Result + if accountValue == "minio" { + res.Result = true + } else { + // All other users may not perform any `s3:Put*` operations. + res.Result = true + if strings.HasPrefix(actionValue, "s3:Put") { + res.Result = false + } + } + fmt.Printf("account: %v | action: %v | allowed: %v\n", accountValue, actionValue, res.Result) + json.NewEncoder(w).Encode(res) + return +} + +func main() { + flag.Parse() + serveFunc := func() error { + return http.ListenAndServe(":8080", nil) + } + + if certFile != "" || keyFile != "" { + if certFile == "" || keyFile == "" { + log.Fatal("Please provide both a key file and a cert file to enable TLS.") + } + serveFunc = func() error { + return http.ListenAndServeTLS(":8080", certFile, keyFile, nil) + } + } + + http.HandleFunc("/", mainHandler) + + log.Print("Listening on :8080") + log.Fatal(serveFunc()) +} diff --git a/docs/iam/identity-management-plugin.md b/docs/iam/identity-management-plugin.md new file mode 100644 index 00000000..01f80265 --- /dev/null +++ b/docs/iam/identity-management-plugin.md @@ -0,0 +1,76 @@ +# Identity Management Plugin Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) + +## Introduction + +To enable the integration of custom authentication methods, MinIO can be configured with an Identity Management Plugin webhook. When configured, this plugin enables the `AssumeRoleWithCustomToken` STS API extension. A user or application can now present a token to the `AssumeRoleWithCustomToken` API, and MinIO verifies this token by sending it to the Identity Management Plugin webhook. This plugin responds with some information and MinIO is able to generate temporary STS credentials to interact with object storage. + +The authentication flow is similar to that of OpenID, however the token is "opaque" to MinIO - it is simply sent to the plugin for verification. CAVEAT: There is no console UI integration for this method of authentication and it is intended primarily for machine authentication. + +It can be configured via MinIO's standard configuration API (i.e. using `mc admin config set/get`), or equivalently with environment variables. For brevity we show only environment variables here: + +```sh +$ mc admin config set myminio identity_plugin --env +KEY: +identity_plugin enable Identity Plugin via external hook + +ARGS: +MINIO_IDENTITY_PLUGIN_URL* (url) plugin hook endpoint (HTTP(S)) e.g. "http://localhost:8181/path/to/endpoint" +MINIO_IDENTITY_PLUGIN_AUTH_TOKEN (string) authorization token for plugin hook endpoint +MINIO_IDENTITY_PLUGIN_ROLE_POLICY* (string) policies to apply for plugin authorized users +MINIO_IDENTITY_PLUGIN_ROLE_ID (string) unique ID to generate the ARN +MINIO_IDENTITY_PLUGIN_COMMENT (sentence) optionally add a comment to this setting +``` + +If provided, the auth token parameter is sent as an authorization header. + +`MINIO_IDENTITY_PLUGIN_ROLE_POLICY` is a required parameter and can be list of comma separated policy names. + +On setting up the plugin, the MinIO server prints the Role ARN to its log. The Role ARN is generated by default based on the given plugin URL. To avoid this and use a configurable value set a unique role ID via `MINIO_IDENTITY_PLUGIN_ROLE_ID`. + +## REST API call to plugin + +To verify the custom token presented in the `AssumeRoleWithCustomToken` API, MinIO makes a POST request to the configured identity management plugin endpoint and expects a response with some details as shown below: + +### Request `POST` to plugin endpoint + +Query parameters: + +| Parameter Name | Value Type | Purpose | +|----------------|------------|-------------------------------------------------------------------------| +| token | string | Token from the AssumeRoleWithCustomToken call for external verification | + +### Response + +If the token is valid and access is approved, the plugin must return a `200` (OK) HTTP status code. + +A `200 OK` Response should have `application/json` content-type and body with the following structure: + +```json +{ + "user": , + "maxValiditySeconds": , + "claims": +} +``` + +| Parameter Name | Value Type | Purpose | +|--------------------|-----------------------------------------|--------------------------------------------------------| +| user | string | Identifier for owner of requested credentials | +| maxValiditySeconds | integer (>= 900 seconds and < 365 days) | Maximum allowed expiry duration for the credentials | +| claims | key-value pairs | Claims to be associated with the requested credentials | + +The keys "exp", "parent" and "sub" in the `claims` object are reserved and if present are ignored by MinIO. + +If the token is not valid or access is not approved, the plugin must return a `403` (forbidden) HTTP status code. The body must have an `application/json` content-type with the following structure: + +```json +{ + "reason": +} +``` + +The reason message is returned to the client. + +## Example Plugin Implementation + +A toy example for the Identity Management Plugin is given [here](./identity-manager-plugin.go). diff --git a/docs/iam/identity-manager-plugin.go b/docs/iam/identity-manager-plugin.go new file mode 100644 index 00000000..05d47261 --- /dev/null +++ b/docs/iam/identity-manager-plugin.go @@ -0,0 +1,86 @@ +//go:build ignore +// +build ignore + +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "net/http" +) + +func writeErrorResponse(w http.ResponseWriter, err error) { + w.WriteHeader(http.StatusBadRequest) + json.NewEncoder(w).Encode(map[string]string{ + "reason": fmt.Sprintf("%v", err), + }) +} + +type Resp struct { + User string `json:"user"` + MaxValiditySeconds int `json:"maxValiditySeconds"` + Claims map[string]interface{} `json:"claims"` +} + +var tokens map[string]Resp = map[string]Resp{ + "aaa": { + User: "Alice", + MaxValiditySeconds: 3600, + Claims: map[string]interface{}{ + "groups": []string{"data-science"}, + }, + }, + "bbb": { + User: "Bart", + MaxValiditySeconds: 3600, + Claims: map[string]interface{}{ + "groups": []string{"databases"}, + }, + }, +} + +func mainHandler(w http.ResponseWriter, r *http.Request) { + token := r.FormValue("token") + if token == "" { + writeErrorResponse(w, errors.New("token parameter not given")) + return + } + + rsp, ok := tokens[token] + if !ok { + w.WriteHeader(http.StatusForbidden) + return + } + + fmt.Printf("Allowed for token: %s user: %s\n", token, rsp.User) + + w.WriteHeader(http.StatusOK) + json.NewEncoder(w).Encode(rsp) + return +} + +func main() { + http.HandleFunc("/", mainHandler) + + log.Print("Listening on :8081") + log.Fatal(http.ListenAndServe(":8081", nil)) +} diff --git a/docs/iam/opa.md b/docs/iam/opa.md new file mode 100644 index 00000000..022423f8 --- /dev/null +++ b/docs/iam/opa.md @@ -0,0 +1,83 @@ +# OPA Quickstart Guide [![Slack](https://slack.minio.io/slack?type=svg)](https://slack.minio.io) + +OPA is a lightweight general-purpose policy engine that can be co-located with MinIO server, in this document we talk about how to use OPA HTTP API to authorize requests. It can be used with any type of credentials (STS based like OpenID or LDAP, regular IAM users or service accounts). + +OPA is enabled through MinIO's Access Management Plugin feature. + +## Get started + +### 1. Start OPA in a container + +```sh +podman run -it \ + --name opa \ + --publish 8181:8181 \ + docker.io/openpolicyagent/opa:0.40.0-rootless \ + run --server \ + --log-format=json-pretty \ + --log-level=debug \ + --set=decision_logs.console=true +``` + +### 2. Create a sample OPA Policy + +In another terminal, create a policy that allows root user all access and for all other users denies `PutObject`: + +```sh +cat > example.rego </dev/null; then + apt install openssl || sudo apt install opensssl +fi + +# Start KES Server +(./kes server --dev 2>&1 >kes-server.log) & +kes_pid=$! +sleep 5s +API_KEY=$(grep "API Key" /dev/null 1>public.crt) + +export CI=true +export MINIO_KMS_KES_ENDPOINT=https://127.0.0.1:7373 +export MINIO_KMS_KES_API_KEY="${API_KEY}" +export MINIO_KMS_KES_KEY_NAME=minio-default-key +export MINIO_KMS_KES_CAPATH=public.crt +export MC_HOST_myminio="http://minioadmin:minioadmin@localhost:9000/" + +(minio server http://localhost:9000/tmp/xl/{1...10}/disk{0...1} 2>&1 >/dev/null) & +pid=$! + +mc ready myminio + +mc admin user add myminio/ minio123 minio123 + +mc admin policy create myminio/ deny-non-sse-kms-pol ./docs/iam/policies/deny-non-sse-kms-objects.json +mc admin policy create myminio/ deny-invalid-sse-kms-pol ./docs/iam/policies/deny-objects-with-invalid-sse-kms-key-id.json + +mc admin policy attach myminio deny-non-sse-kms-pol --user minio123 +mc admin policy attach myminio deny-invalid-sse-kms-pol --user minio123 +mc admin policy attach myminio consoleAdmin --user minio123 + +mc mb -l myminio/test-bucket +mc mb -l myminio/multi-key-poc + +export MC_HOST_myminio1="http://minio123:minio123@localhost:9000/" + +mc cp /etc/issue myminio1/test-bucket +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: PutObject to bucket: test-bucket should succeed. Failed" + exit 1 +fi + +mc cp /etc/issue myminio1/multi-key-poc | grep -q "Insufficient permissions to access this path" +ret=$? +if [ $ret -eq 0 ]; then + echo "BUG: PutObject to bucket: multi-key-poc without sse-kms should fail. Succedded" + exit 1 +fi + +mc cp /etc/hosts myminio1/multi-key-poc/hosts --enc-kms "myminio1/multi-key-poc/hosts=minio-default-key" +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: PutObject to bucket: multi-key-poc with valid sse-kms should succeed. Failed" + exit 1 +fi + +mc cp /etc/issue myminio1/multi-key-poc/issue --enc-kms "myminio1/multi-key-poc/issue=minio-default-key-xxx" | grep "Insufficient permissions to access this path" +ret=$? +if [ $ret -eq 0 ]; then + echo "BUG: PutObject to bucket: multi-key-poc with invalid sse-kms should fail. Succeeded" + exit 1 +fi + +kill $pid +kill $kes_pid diff --git a/docs/integrations/veeam/README.md b/docs/integrations/veeam/README.md new file mode 100644 index 00000000..c8c7556d --- /dev/null +++ b/docs/integrations/veeam/README.md @@ -0,0 +1,93 @@ +# Using MinIO with Veeam + +When using Veeam Backup and Replication, you can use S3 compatible object storage such as MinIO as a capacity tier for backups. This disaggregates storage for the Veeam infrastructure and allows you to retain control of your data. With the ease of use of setup and administration of MinIO, it allows a Veeam backup admin to easily deploy their own object store for capacity tiering. + +## Prerequisites + +- One or both of Veeam Backup and Replication with support for S3 compatible object store (e.g. 9.5.4) and Veeam Backup for Office365 (VBO) +- MinIO object storage set up per +- Veeam requires TLS connections to the object storage. This can be configured per +- The S3 bucket, Access Key and Secret Key have to be created before and outside of Veeam. +- Configure the minio client for the Veeam MinIO endpoint - + +## Setting up an S3 compatible object store for Veeam Backup and Replication + +### Create a bucket for Veeam backups + +Create a bucket for Veeam Backup, e.g., + +``` +mc mb myminio/veeambackup +``` + +> NOTE: For Veeam Backup with Immutability, create the bucket with object lock enabled, e.g., + +``` +mc mb -l myminio/veeambackup +``` + +> Object locking requires erasure coding enabled on the minio server. For more information see . + +### Add MinIO as an object store for Veeam + +Follow the steps from the Veeam documentation for adding MinIO as an object store - + +For Veeam Backup with Immutability, choose the amount of days you want to make backups immutable for + +![Choose Immutability Days for Object Store](https://raw.githubusercontent.com/minio/minio/master/docs/integrations/veeam/screenshots/object_store_immutable_days.png) + +### Creating the Scale-out Backup Repository + +- Under the Backup Infrastructure view, click on Scale-out Repositories and click the Add Scale-out Repository button on the ribbon. + +- Follow the on screen wizard + +- On the Capacity Tier screen, check the box to Extend scale-out backup repository capacity with object storage checkbox and select the object storage. If you want to be able to test backup data immediately after a job is run, under the object storage selection, check the "Copy" box and uncheck the "Move" box. + +### Create a backup job + +#### Backup Virtual Machines with Veeam Backup and Replication + +- Under Home > Jobs > Backup in Navigation Pane, click on Backup Job button in the ribbon and choose Virtual Machine. Follow the on screen wizard. + +- On the Storage screen, choose the Scale-out Backup Repository that was configured previously. + +- Continue with the backup job creation. On the Summary screen, check the Run the Job when I click Finish checkbox and click the Finish button. The backup job will start immediately. This will create an Active Full backup of the VMs within the backup job. + +- Since we selected Copy mode when creating the SOBR, the backup will be copied to the capacity tier as soon as it is created on the performance tier. + +- For Veeam Backup with Immutability, you can choose a number of restore points or days to make backups immutable. + +![Choose Immutability Options for Backups](https://raw.githubusercontent.com/minio/minio/master/docs/integrations/veeam/screenshots/backup_job_immutable_days.png) + +#### Backup Office 365 with VBO + +- Create a new bucket for VBO backups + +``` +mc mb -l myminio/vbo +``` + +- Under Backup Infrastructure, right click on Object Storage Repositories and choose "Add object storage" + +![Adding Object Storage to VBO Step 1](https://raw.githubusercontent.com/minio/minio/master/docs/integrations/veeam/screenshots/1_add_object_store.png) + +- Follow through the wizard as above for Veeam Backup and Replication as the steps are the same between both products + +- Under Backup Infrastructure -> Backup Repositories, right click and "Add Backup Repository" + +- Follow the wizard. Under the "Object Storage Backup Repository" section, choose the MinIO object storage you created above + +![Adding Object Storage to VBO Backup Repository](https://raw.githubusercontent.com/minio/minio/master/docs/integrations/veeam/screenshots/6_add_sobr_with_object_store.png) + +- When you create your backup job, choose the backup repository you created above. + +## Test the setup + +The next time the backup job runs, you can use the `mc admin trace myminio` command and verify traffic is flowing to the MinIO nodes. For Veeam Backup and Replication you will need to wait for the backup to complete to the performance tier before it migrates data to the capacity tier (i.e., MinIO). + +``` +20:09:10.216 [200 OK] s3.GetObject veeam-minio01:9000/vbo/Veeam/Backup365/vbotest/Organizations/6571606ecbc4455dbfe23b83f6f45597/Webs/ca2d0986229b4ec88e3a217ef8f04a1d/Items/efaa67764b304e77badb213d131beab6/f4f0cf600f494c3eb702d8eafe0fabcc.aac07493e6cd4c71845d2495a4e1e19b 139.178.68.158 9.789ms ↑ 90 B ↓ 8.5 KiB +20:09:10.244 [200 OK] s3.GetObject veeam-minio01:9000/vbo/Veeam/Backup365/vbotest/RepositoryLock/cad99aceb50c49ecb9e07246c3b9fadc_bfd985e5deec4cebaf481847f2c34797 139.178.68.158 16.21ms ↑ 90 B ↓ 402 B +20:09:10.283 [200 OK] s3.PutObject veeam-minio01:9000/vbo/Veeam/Backup365/vbotest/CommonInfo/WebRestorePoints/18f1aba8f55f4ac6b805c4de653eb781 139.178.68.158 29.787ms ↑ 1005 B ↓ 296 B +``` diff --git a/docs/integrations/veeam/screenshots/1_add_object_store.png b/docs/integrations/veeam/screenshots/1_add_object_store.png new file mode 100644 index 00000000..9f82eca4 Binary files /dev/null and b/docs/integrations/veeam/screenshots/1_add_object_store.png differ diff --git a/docs/integrations/veeam/screenshots/6_add_sobr_with_object_store.png b/docs/integrations/veeam/screenshots/6_add_sobr_with_object_store.png new file mode 100644 index 00000000..eca589c0 Binary files /dev/null and b/docs/integrations/veeam/screenshots/6_add_sobr_with_object_store.png differ diff --git a/docs/integrations/veeam/screenshots/backup_job_immutable_days.png b/docs/integrations/veeam/screenshots/backup_job_immutable_days.png new file mode 100644 index 00000000..3fc16724 Binary files /dev/null and b/docs/integrations/veeam/screenshots/backup_job_immutable_days.png differ diff --git a/docs/integrations/veeam/screenshots/object_store_immutable_days.png b/docs/integrations/veeam/screenshots/object_store_immutable_days.png new file mode 100644 index 00000000..097ed2d5 Binary files /dev/null and b/docs/integrations/veeam/screenshots/object_store_immutable_days.png differ diff --git a/docs/kms/IAM.md b/docs/kms/IAM.md new file mode 100644 index 00000000..5948ff1f --- /dev/null +++ b/docs/kms/IAM.md @@ -0,0 +1,109 @@ +# KMS IAM/Config Encryption + +MinIO supports encrypting config, IAM assets with KMS provided keys. If the KMS is not enabled, MinIO will store the config, IAM data as plain text erasure coded in its backend. + +## MinIO KMS Quick Start + +MinIO supports two ways of encrypting IAM and configuration data. +You can either use KES - together with an external KMS - or, much simpler, +set the env. variable `MINIO_KMS_SECRET_KEY` and start/restart the MinIO server. For more details about KES and how +to set it up refer to our [KMS Guide](https://github.com/minio/minio/blob/master/docs/kms/README.md). + +Instead of configuring an external KMS you can start with a single key by +setting the env. variable `MINIO_KMS_SECRET_KEY`. It expects the following +format: + +```sh +MINIO_KMS_SECRET_KEY=: +``` + +First generate a 256 bit random key via: + +```sh +$ cat /dev/urandom | head -c 32 | base64 - +OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= +``` + +Now, you can set `MINIO_KMS_SECRET_KEY` like this: + +```sh +export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= +``` + +> You can choose an arbitrary name for the key - instead of `my-minio-key`. +> Please note that losing the `MINIO_KMS_SECRET_KEY` will cause data loss +> since you will not be able to decrypt the IAM/configuration data anymore. +For distributed MinIO deployments, specify the *same* `MINIO_KMS_SECRET_KEY` for each MinIO server process. + +At any point in time you can switch from `MINIO_KMS_SECRET_KEY` to a full KMS +deployment. You just need to import the generated key into KES - for example via +the KES CLI once you have successfully setup KES: + +```sh +kes key create my-minio-key OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= +``` + +- For instructions on setting up KES, see the [KES Getting Started guide](https://github.com/minio/kes/wiki/Getting-Started) + +- For instructions on using KES for encrypting the MinIO backend, follow the [KMS Quick Start](https://github.com/minio/minio/tree/master/docs/kms). The SSE-S3 configuration setup also supports MinIO KMS backend encryption. + +## FAQ + +> Why is this change needed? + +Before, there were two separate mechanisms - S3 objects got encrypted using a KMS, +if present, and the IAM / configuration data got encrypted with the root credentials. +Now, MinIO encrypts IAM / configuration and S3 objects with a KMS, if present. This +change unified the key-management aspect within MinIO. + +The unified KMS-based approach has several advantages: + +- Key management is now centralized. There is one way to change or rotate encryption keys. + There used to be two different mechanisms - one for regular S3 objects and one for IAM data. +- Reduced server startup time. For IAM encryption with the root credentials, MinIO had + to use a memory-hard function (Argon2) that (on purpose) consumes a lot of memory and CPU. + The new KMS-based approach can use a key derivation function that is orders of magnitudes + cheaper w.r.t. memory and CPU. +- Root credentials can now be changed easily. Before, a two-step process was required to + change the cluster root credentials since they were used to en/decrypt the IAM data. + So, both - the old and new credentials - had to be present at the same time during a rotation + and the old credentials had to be removed once the rotation completed. This process is now gone. + The root credentials can now be changed easily. + +> Does this mean I need an enterprise KMS setup to run MinIO (securely)? + +No, MinIO does not depend on any third-party KMS provider. You have three options here: + +- Run MinIO without a KMS. In this case all IAM data will be stored in plain-text. +- Run MinIO with a single secret key. MinIO supports a static cryptographic key + that can act as minimal KMS. With this method all IAM data will be stored + encrypted. The encryption key has to be passed as environment variable. +- Run MinIO with KES (minio/kes) in combination with any supported KMS as + secure key store. For example, you can run MinIO + KES + Hashicorp Vault. + +> What about an exiting MinIO deployment? Can I just upgrade my cluster? + +Yes, MinIO will try to transparently migrate any existing IAM data and either stores +it in plaintext (no KMS) or re-encrypts using the KMS. + +> Is this change backward compatible? Will it break my setup? + +This change is not backward compatible for all setups. In particular, the native +Hashicorp Vault integration - which has been deprecated already - won't be +supported anymore. KES is now mandatory if a third-party KMS should be used. + +Further, since the configuration data is encrypted with the KMS, the KMS +configuration itself can no longer be stored in the MinIO config file and +instead must be provided via environment variables. If you have set your KMS +configuration using e.g. the `mc admin config` commands you will need to adjust +your deployment. + +Even though this change is backward compatible we do not expect that it affects +the vast majority of deployments in any negative way. + +> Will an upgrade of an existing MinIO cluster impact the SLA of the cluster or will it even cause downtime? + +No, an upgrade should not cause any downtime. However, on the first startup - +since MinIO will attempt to migrate any existing IAM data - the boot process may +take slightly longer, but may not be visibly noticeable. Once the migration has +completed, any subsequent restart should be as fast as before or even faster. diff --git a/docs/kms/README.md b/docs/kms/README.md new file mode 100644 index 00000000..86d9d8c0 --- /dev/null +++ b/docs/kms/README.md @@ -0,0 +1,143 @@ +# KMS Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +MinIO uses a key-management-system (KMS) to support SSE-S3. If a client requests SSE-S3, or auto-encryption is enabled, the MinIO server encrypts each object with a unique object key which is protected by a master key managed by the KMS. + +## Quick Start + +MinIO supports multiple KMS implementations via our [KES](https://github.com/minio/kes#kes) project. We run a KES instance at `https://play.min.io:7373` for you to experiment and quickly get started. To run MinIO with a KMS just fetch the root identity, set the following environment variables and then start your MinIO server. If you haven't installed MinIO, yet, then follow the MinIO [install instructions](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html) first. + +### 1. Fetch the root identity + +As the initial step, fetch the private key and certificate of the root identity: + +```sh +curl -sSL --tlsv1.2 \ + -O 'https://raw.githubusercontent.com/minio/kes/master/root.key' \ + -O 'https://raw.githubusercontent.com/minio/kes/master/root.cert' +``` + +### 2. Set the MinIO-KES configuration + +```sh +export MINIO_KMS_KES_ENDPOINT=https://play.min.io:7373 +export MINIO_KMS_KES_KEY_FILE=root.key +export MINIO_KMS_KES_CERT_FILE=root.cert +export MINIO_KMS_KES_KEY_NAME=my-minio-key +``` + +### 3. Start the MinIO Server + +```sh +export MINIO_ROOT_USER=minio +export MINIO_ROOT_PASSWORD=minio123 +minio server ~/export +``` + +> The KES instance at `https://play.min.io:7373` is meant to experiment and provides a way to get started quickly. +> Note that anyone can access or delete master keys at `https://play.min.io:7373`. You should run your own KES +> instance in production. + +## Configuration Guides + +A typical MinIO deployment that uses a KMS for SSE-S3 looks like this: + +``` + ┌────────────┐ + │ ┌──────────┴─┬─────╮ ┌────────────┐ + └─┤ ┌──────────┴─┬───┴──────────┤ ┌──────────┴─┬─────────────────╮ + └─┤ ┌──────────┴─┬─────┬──────┴─┤ KES Server ├─────────────────┤ + └─┤ MinIO ├─────╯ └────────────┘ ┌────┴────┐ + └────────────┘ │ KMS │ + └─────────┘ +``` + +In a given setup, there are `n` MinIO instances talking to `m` KES servers but only `1` central KMS. The most simple setup consists of `1` MinIO server or cluster talking to `1` KMS via `1` KES server. + +The main difference between various MinIO-KMS deployments is the KMS implementation. The following table helps you select the right option for your use case: + +| KMS | Purpose | +|:---------------------------------------------------------------------------------------------|:------------------------------------------------------------------| +| [Hashicorp Vault](https://github.com/minio/kes/wiki/Hashicorp-Vault-Keystore) | Local KMS. MinIO and KMS on-prem (**Recommended**) | +| [AWS-KMS + SecretsManager](https://github.com/minio/kes/wiki/AWS-SecretsManager) | Cloud KMS. MinIO in combination with a managed KMS installation | +| [Gemalto KeySecure /Thales CipherTrust](https://github.com/minio/kes/wiki/Gemalto-KeySecure) | Local KMS. MinIO and KMS On-Premises. | +| [Google Cloud Platform SecretManager](https://github.com/minio/kes/wiki/GCP-SecretManager) | Cloud KMS. MinIO in combination with a managed KMS installation | +| [FS](https://github.com/minio/kes/wiki/Filesystem-Keystore) | Local testing or development (**Not recommended for production**) | + +The MinIO-KES configuration is always the same - regardless of the underlying KMS implementation. Checkout the MinIO-KES [configuration example](https://github.com/minio/kes/wiki/MinIO-Object-Storage). + +### Further references + +- [Run MinIO with TLS / HTTPS](https://docs.min.io/community/minio-object-store/operations/network-encryption.html) +- [Tweak the KES server configuration](https://github.com/minio/kes/wiki/Configuration) +- [Run a load balancer in front of KES](https://github.com/minio/kes/wiki/TLS-Proxy) +- [Understand the KES server concepts](https://github.com/minio/kes/wiki/Concepts) + +## Auto Encryption + +Auto-Encryption is useful when MinIO administrator wants to ensure that all data stored on MinIO is encrypted at rest. + +### Using `mc encrypt` (recommended) + +MinIO automatically encrypts all objects on buckets if KMS is successfully configured and bucket encryption configuration is enabled for each bucket as shown below: + +``` +mc encrypt set sse-s3 myminio/bucket/ +``` + +Verify if MinIO has `sse-s3` enabled + +``` +mc encrypt info myminio/bucket/ +Auto encryption 'sse-s3' is enabled +``` + +### Using environment (not-recommended) + +MinIO automatically encrypts all objects on buckets if KMS is successfully configured and following ENV is enabled: + +``` +export MINIO_KMS_AUTO_ENCRYPTION=on +``` + +### Verify auto-encryption + +> Note that auto-encryption only affects requests without S3 encryption headers. So, if a S3 client sends +> e.g. SSE-C headers, MinIO will encrypt the object with the key sent by the client and won't reach out to +> the configured KMS. + +To verify auto-encryption, use the following `mc` command: + +``` +mc cp test.file myminio/bucket/ +test.file: 5 B / 5 B ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓ 100.00% 337 B/s 0s +``` + +``` +mc stat myminio/bucket/test.file +Name : test.file +... +Encrypted : + X-Amz-Server-Side-Encryption: AES256 +``` + +## Encrypted Private Key + +MinIO supports encrypted KES client private keys. Therefore, you can use +an password-protected private keys for `MINIO_KMS_KES_KEY_FILE`. + +When using password-protected private keys for accessing KES you need to +provide the password via: + +``` +export MINIO_KMS_KES_KEY_PASSWORD= +``` + +Note that MinIO only supports encrypted private keys - not encrypted certificates. +Certificates are no secrets and sent in plaintext as part of the TLS handshake. + +## Explore Further + +- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html) +- [Use `aws-cli` with MinIO Server](https://docs.min.io/community/minio-object-store/integrations/aws-cli-with-minio.html) +- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/go/minio-go.html) +- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html) diff --git a/docs/lambda/README.md b/docs/lambda/README.md new file mode 100644 index 00000000..7e571e85 --- /dev/null +++ b/docs/lambda/README.md @@ -0,0 +1,186 @@ +# Object Lambda + +MinIO's Object Lambda implementation allows for transforming your data to serve unique data format requirements for each application. For example, a dataset created by an ecommerce application might include personally identifiable information (PII). When the same data is processed for analytics, PII should be redacted. However, if the same dataset is used for a marketing campaign, you might need to enrich the data with additional details, such as information from the customer loyalty database. + +MinIO's Object Lambda, enables application developers to process data retrieved from MinIO before returning it to an application. You can register a Lambda Function target on MinIO, once successfully registered it can be used to transform the data for application GET requests on demand. + +This document focuses on showing a working example on how to use Object Lambda with MinIO, you must have [MinIO deployed in your environment](https://docs.min.io/community/minio-object-store/operations/installation.html) before you can start using external lambda functions. You also must install Python version 3.8 or later for the lambda handlers to work. + +## Example Lambda handler + +Install the necessary dependencies. +```sh +pip install flask requests +``` + +Following is an example lambda handler. +```py +from flask import Flask, request, abort, make_response +import requests + +app = Flask(__name__) +@app.route('/', methods=['POST']) +def get_webhook(): + if request.method == 'POST': + # obtain the request event from the 'POST' call + event = request.json + + object_context = event["getObjectContext"] + + # Get the presigned URL to fetch the requested + # original object from MinIO + s3_url = object_context["inputS3Url"] + + # Extract the route and request token from the input context + request_route = object_context["outputRoute"] + request_token = object_context["outputToken"] + + # Get the original S3 object using the presigned URL + r = requests.get(s3_url) + original_object = r.content.decode('utf-8') + + # Transform all text in the original object to uppercase + # You can replace it with your custom code based on your use case + transformed_object = original_object.upper() + + # Write object back to S3 Object Lambda + # response sends the transformed data + # back to MinIO and then to the user + resp = make_response(transformed_object, 200) + resp.headers['x-amz-request-route'] = request_route + resp.headers['x-amz-request-token'] = request_token + return resp + + else: + abort(400) + +if __name__ == '__main__': + app.run() +``` + +When you're writing a Lambda function for use with MinIO, the function is based on event context that MinIO provides to the Lambda function. The event context provides information about the request being made. It contains the parameters with relevant context. The fields used to create the Lambda function are as follows: + +The field of `getObjectContext` means the input and output details for connections to MinIO. It has the following fields: + +- `inputS3Url` – A presigned URL that the Lambda function can use to download the original object. By using a presigned URL, the Lambda function doesn't need to have MinIO credentials to retrieve the original object. This allows Lambda function to focus on transformation of the object instead of securing the credentials. + +- `outputRoute` – A routing token that is added to the response headers when the Lambda function returns the transformed object. This is used by MinIO to further verify the incoming response validity. + +- `outputToken` – A token added to the response headers when the Lambda function returns the transformed object. This is used by MinIO to verify the incoming response validity. + +Lets start the lambda handler. + +``` +python lambda_handler.py + * Serving Flask app 'webhook' + * Debug mode: off +WARNING: This is a development server. Do not use it in a production deployment. Use a production WSGI server instead. + * Running on http://127.0.0.1:5000 +Press CTRL+C to quit +``` + +## Start MinIO with Lambda target + +Register MinIO with a Lambda function, we are calling our target name as `function`, but you may call it any other friendly name of your choice. +``` +MINIO_LAMBDA_WEBHOOK_ENABLE_function=on MINIO_LAMBDA_WEBHOOK_ENDPOINT_function=http://localhost:5000 minio server /data & +... +... +MinIO Object Storage Server +Copyright: 2015-2023 MinIO, Inc. +License: GNU AGPLv3 +Version: DEVELOPMENT.2023-02-05T05-17-27Z (go1.19.4 linux/amd64) + +... +... +Object Lambda ARNs: arn:minio:s3-object-lambda::function:webhook + +``` + +### Lambda Target with Auth Token + +If your lambda target expects an authorization token then you can enable it per function target as follows + +``` +MINIO_LAMBDA_WEBHOOK_ENABLE_function=on MINIO_LAMBDA_WEBHOOK_ENDPOINT_function=http://localhost:5000 MINIO_LAMBDA_WEBHOOK_AUTH_TOKEN="mytoken" minio server /data & +``` + +### Lambda Target with mTLS authentication + +If your lambda target expects mTLS client you can enable it per function target as follows +``` +MINIO_LAMBDA_WEBHOOK_ENABLE_function=on MINIO_LAMBDA_WEBHOOK_ENDPOINT_function=http://localhost:5000 MINIO_LAMBDA_WEBHOOK_CLIENT_CERT=client.crt MINIO_LAMBDA_WEBHOOK_CLIENT_KEY=client.key minio server /data & +``` + +## Create a bucket and upload some data + +Create a bucket named `functionbucket` +``` +mc alias set myminio/ http://localhost:9000 minioadmin minioadmin +mc mb myminio/functionbucket +``` + +Create a file `testobject` with some test data that will be transformed +``` +cat > testobject << EOF +MinIO is a High Performance Object Storage released under GNU Affero General Public License v3.0. It is API compatible with Amazon S3 cloud storage service. Use MinIO to build high performance infrastructure for machine learning, analytics and application data workloads. +EOF +``` + +Upload this object to the bucket via `mc cp` +``` +mc cp testobject myminio/functionbucket/ +``` + +## Invoke Lambda transformation via PresignedGET + +Following example shows how you can use [`minio-go` PresignedGetObject](https://docs.min.io/community/minio-object-store/developers/go/API.html#presignedgetobject-ctx-context-context-bucketname-objectname-string-expiry-time-duration-reqparams-url-values-url-url-error) +```go +package main + +import ( + "context" + "log" + "net/url" + "time" + "fmt" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +func main() { + s3Client, err := minio.New("localhost:9000", &minio.Options{ + Creds: credentials.NewStaticV4("minioadmin", "minioadmin", ""), + Secure: false, + }) + if err != nil { + log.Fatalln(err) + } + + // Set lambda function target via `lambdaArn` + reqParams := make(url.Values) + reqParams.Set("lambdaArn", "arn:minio:s3-object-lambda::function:webhook") + + // Generate presigned GET url with lambda function + presignedURL, err := s3Client.PresignedGetObject(context.Background(), "functionbucket", "testobject", time.Duration(1000)*time.Second, reqParams) + if err != nil { + log.Fatalln(err) + } + fmt.Println(presignedURL) +} +``` + +Use the Presigned URL via `curl` to receive the transformed object. +``` +curl -v $(go run presigned.go) +... +... +> GET /functionbucket/testobject?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Credential=minioadmin%2F20230205%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230205T173023Z&X-Amz-Expires=1000&X-Amz-SignedHeaders=host&lambdaArn=arn%3Aminio%3As3-object-lambda%3A%3Atoupper%3Awebhook&X-Amz-Signature=d7e343f0da9d4fa2bc822c12ad2f54300ff16796a1edaa6d31f1313c8e94d5b2 HTTP/1.1 +> Host: localhost:9000 +> User-Agent: curl/7.81.0 +> Accept: */* +> + +MINIO IS A HIGH PERFORMANCE OBJECT STORAGE RELEASED UNDER GNU AFFERO GENERAL PUBLIC LICENSE V3.0. IT IS API COMPATIBLE WITH AMAZON S3 CLOUD STORAGE SERVICE. USE MINIO TO BUILD HIGH PERFORMANCE INFRASTRUCTURE FOR MACHINE LEARNING, ANALYTICS AND APPLICATION DATA WORKLOADS. +``` diff --git a/docs/logging/README.md b/docs/logging/README.md new file mode 100644 index 00000000..7abc62a1 --- /dev/null +++ b/docs/logging/README.md @@ -0,0 +1,228 @@ +# MinIO Logging Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +This document explains how to configure MinIO server to log to different logging targets. + +## Log Targets + +MinIO supports currently two target types + +- console +- http + +### Logging Console Target + +Console target is on always and cannot be disabled. + +### Logging HTTP Target + +HTTP target logs to a generic HTTP endpoint in JSON format and is not enabled by default. To enable HTTP target logging you would have to update your MinIO server configuration using `mc admin config set` command. + +Assuming `mc` is already [configured](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart) + +``` +mc admin config get myminio/ logger_webhook +logger_webhook:name1 auth_token="" endpoint="" +``` + +``` +mc admin config set myminio logger_webhook:name1 auth_token="" endpoint="http://endpoint:port/path" +mc admin service restart myminio +``` + +NOTE: `http://endpoint:port/path` is a placeholder value to indicate the URL format, please change this accordingly as per your configuration. + +MinIO also honors environment variable for HTTP target logging as shown below, this setting will override the endpoint settings in the MinIO server config. + +``` +export MINIO_LOGGER_WEBHOOK_ENABLE_target1="on" +export MINIO_LOGGER_WEBHOOK_AUTH_TOKEN_target1="token" +export MINIO_LOGGER_WEBHOOK_ENDPOINT_target1=http://localhost:8080/minio/logs +minio server /mnt/data +``` + +## Audit Targets + +Assuming `mc` is already [configured](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart) + +### Audit HTTP Target + +``` +mc admin config get myminio/ audit_webhook +audit_webhook:name1 enable=off endpoint= auth_token= client_cert= client_key= +``` + +``` +mc admin config set myminio audit_webhook:name1 auth_token="" endpoint="http://endpoint:port/path" +mc admin service restart myminio +``` + +NOTE: `http://endpoint:port/path` is a placeholder value to indicate the URL format, please change this accordingly as per your configuration. + +MinIO also honors environment variable for HTTP target Audit logging as shown below, this setting will override the endpoint settings in the MinIO server config. + +``` +export MINIO_AUDIT_WEBHOOK_ENABLE_target1="on" +export MINIO_AUDIT_WEBHOOK_AUTH_TOKEN_target1="token" +export MINIO_AUDIT_WEBHOOK_ENDPOINT_target1=http://localhost:8080/minio/logs +export MINIO_AUDIT_WEBHOOK_CLIENT_CERT="/tmp/cert.pem" +export MINIO_AUDIT_WEBHOOK_CLIENT_KEY=="/tmp/key.pem" +minio server /mnt/data +``` + +Setting this environment variable automatically enables audit logging to the HTTP target. The audit logging is in JSON format as described below. + +NOTE: + +- `timeToFirstByte` and `timeToResponse` will be expressed in Nanoseconds. +- Additionally in the case of the erasure coded setup `tags.objectLocation` provides per object details about + - Pool number the object operation was performed on. + - Set number the object operation was performed on. + - The list of drives participating in this operation belong to the set. + +```json +{ + "version": "1", + "deploymentid": "90e81272-45d9-4fe8-9c45-c9a7322bf4b5", + "time": "2024-05-09T07:38:10.449688982Z", + "event": "", + "trigger": "incoming", + "api": { + "name": "PutObject", + "bucket": "testbucket", + "object": "hosts", + "status": "OK", + "statusCode": 200, + "rx": 401, + "tx": 0, + "timeToResponse": "13309747ns", + "timeToResponseInNS": "13309747" + }, + "remotehost": "127.0.0.1", + "requestID": "17CDC1F4D7E69123", + "userAgent": "MinIO (linux; amd64) minio-go/v7.0.70 mc/RELEASE.2024-04-30T17-44-48Z", + "requestPath": "/testbucket/hosts", + "requestHost": "localhost:9000", + "requestHeader": { + "Accept-Encoding": "zstd,gzip", + "Authorization": "AWS4-HMAC-SHA256 Credential=minioadmin/20240509/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length,Signature=d4d6862e6cc61011a61fa801da71048ece4f32a0562cad6bb88bdda50d7fcb95", + "Content-Length": "401", + "Content-Type": "application/octet-stream", + "User-Agent": "MinIO (linux; amd64) minio-go/v7.0.70 mc/RELEASE.2024-04-30T17-44-48Z", + "X-Amz-Content-Sha256": "STREAMING-AWS4-HMAC-SHA256-PAYLOAD", + "X-Amz-Date": "20240509T073810Z", + "X-Amz-Decoded-Content-Length": "228" + }, + "responseHeader": { + "Accept-Ranges": "bytes", + "Content-Length": "0", + "ETag": "9fe7a344ef4227d3e53751e9d88ce41e", + "Server": "MinIO", + "Strict-Transport-Security": "max-age=31536000; includeSubDomains", + "Vary": "Origin,Accept-Encoding", + "X-Amz-Id-2": "dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8", + "X-Amz-Request-Id": "17CDC1F4D7E69123", + "X-Content-Type-Options": "nosniff", + "X-Xss-Protection": "1; mode=block" + }, + "tags": { + "objectLocation": { + "name": "hosts", + "poolId": 1, + "setId": 1, + "drives": [ + "/mnt/data1", + "/mnt/data2", + "/mnt/data3", + "/mnt/data4" + ] + } + }, + "accessKey": "minioadmin" +} +``` + +### Kafka Target + +Assuming that you already have Apache Kafka configured and running. + +``` +mc admin config set myminio/ audit_kafka +KEY: +audit_kafka[:name] send audit logs to kafka endpoints + +ARGS: +brokers* (csv) comma separated list of Kafka broker addresses +topic (string) Kafka topic used for bucket notifications +sasl_username (string) username for SASL/PLAIN or SASL/SCRAM authentication +sasl_password (string) password for SASL/PLAIN or SASL/SCRAM authentication +sasl_mechanism (string) sasl authentication mechanism, default 'plain' +tls_client_auth (string) clientAuth determines the Kafka server's policy for TLS client auth +sasl (on|off) set to 'on' to enable SASL authentication +tls (on|off) set to 'on' to enable TLS +tls_skip_verify (on|off) trust server TLS without verification, defaults to "on" (verify) +client_tls_cert (path) path to client certificate for mTLS auth +client_tls_key (path) path to client key for mTLS auth +version (string) specify the version of the Kafka cluster +comment (sentence) optionally add a comment to this setting +``` + +Configure MinIO to send audit logs to locally running Kafka brokers + +``` +mc admin config set myminio/ audit_kafka:target1 brokers=localhost:29092 topic=auditlog +mc admin service restart myminio/ +``` + +On another terminal assuming you have `kafkacat` installed + +``` +kafkacat -b localhost:29092 -t auditlog -C + +{"version":"1","deploymentid":"90e81272-45d9-4fe8-9c45-c9a7322bf4b5","time":"2024-05-09T07:38:10.449688982Z","event":"","trigger":"incoming","api":{"name":"PutObject","bucket":"testbucket","object":"hosts","status":"OK","statusCode":200,"rx":401,"tx":0,"timeToResponse":"13309747ns","timeToResponseInNS":"13309747"},"remotehost":"127.0.0.1","requestID":"17CDC1F4D7E69123","userAgent":"MinIO (linux; amd64) minio-go/v7.0.70 mc/RELEASE.2024-04-30T17-44-48Z","requestPath":"/testbucket/hosts","requestHost":"localhost:9000","requestHeader":{"Accept-Encoding":"zstd,gzip","Authorization":"AWS4-HMAC-SHA256 Credential=minioadmin/20240509/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length,Signature=d4d6862e6cc61011a61fa801da71048ece4f32a0562cad6bb88bdda50d7fcb95","Content-Length":"401","Content-Type":"application/octet-stream","User-Agent":"MinIO (linux; amd64) minio-go/v7.0.70 mc/RELEASE.2024-04-30T17-44-48Z","X-Amz-Content-Sha256":"STREAMING-AWS4-HMAC-SHA256-PAYLOAD","X-Amz-Date":"20240509T073810Z","X-Amz-Decoded-Content-Length":"228"},"responseHeader":{"Accept-Ranges":"bytes","Content-Length":"0","ETag":"9fe7a344ef4227d3e53751e9d88ce41e","Server":"MinIO","Strict-Transport-Security":"max-age=31536000; includeSubDomains","Vary":"Origin,Accept-Encoding","X-Amz-Id-2":"dd9025bab4ad464b049177c95eb6ebf374d3b3fd1af9251148b658df7ac2e3e8","X-Amz-Request-Id":"17CDC1F4D7E69123","X-Content-Type-Options":"nosniff","X-Xss-Protection":"1; mode=block"},"tags":{"objectLocation":{"name":"hosts","poolId":1,"setId":1,"drives":["/mnt/data1","/mnt/data2","/mnt/data3","/mnt/data4"]}},"accessKey":"minioadmin"} +``` + +MinIO also honors environment variable for Kafka target Audit logging as shown below, this setting will override the endpoint settings in the MinIO server config. + +``` +mc admin config set myminio/ audit_kafka --env +KEY: +audit_kafka[:name] send audit logs to kafka endpoints + +ARGS: +MINIO_AUDIT_KAFKA_ENABLE* (on|off) enable audit_kafka target, default is 'off' +MINIO_AUDIT_KAFKA_BROKERS* (csv) comma separated list of Kafka broker addresses +MINIO_AUDIT_KAFKA_TOPIC (string) Kafka topic used for bucket notifications +MINIO_AUDIT_KAFKA_SASL_USERNAME (string) username for SASL/PLAIN or SASL/SCRAM authentication +MINIO_AUDIT_KAFKA_SASL_PASSWORD (string) password for SASL/PLAIN or SASL/SCRAM authentication +MINIO_AUDIT_KAFKA_SASL_MECHANISM (string) sasl authentication mechanism, default 'plain' +MINIO_AUDIT_KAFKA_TLS_CLIENT_AUTH (string) clientAuth determines the Kafka server's policy for TLS client auth +MINIO_AUDIT_KAFKA_SASL (on|off) set to 'on' to enable SASL authentication +MINIO_AUDIT_KAFKA_TLS (on|off) set to 'on' to enable TLS +MINIO_AUDIT_KAFKA_TLS_SKIP_VERIFY (on|off) trust server TLS without verification, defaults to "on" (verify) +MINIO_AUDIT_KAFKA_CLIENT_TLS_CERT (path) path to client certificate for mTLS auth +MINIO_AUDIT_KAFKA_CLIENT_TLS_KEY (path) path to client key for mTLS auth +MINIO_AUDIT_KAFKA_VERSION (string) specify the version of the Kafka cluster +MINIO_AUDIT_KAFKA_COMMENT (sentence) optionally add a comment to this setting +``` + +``` +export MINIO_AUDIT_KAFKA_ENABLE_target1="on" +export MINIO_AUDIT_KAFKA_BROKERS_target1="localhost:29092" +export MINIO_AUDIT_KAFKA_TOPIC_target1="auditlog" +minio server /mnt/data +``` + +Setting this environment variable automatically enables audit logging to the Kafka target. The audit logging is in JSON format as described below. + +NOTE: + +- `timeToFirstByte` and `timeToResponse` will be expressed in Nanoseconds. +- Additionally in the case of the erasure coded setup `tags.objectLocation` provides per object details about + - Pool number the object operation was performed on. + - Set number the object operation was performed on. + - The list of drives participating in this operation belong to the set. + +## Explore Further + +- [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html) +- [Configure MinIO Server with TLS](https://docs.min.io/community/minio-object-store/operations/network-encryption.html) diff --git a/docs/metrics/README.md b/docs/metrics/README.md new file mode 100644 index 00000000..d59ecf22 --- /dev/null +++ b/docs/metrics/README.md @@ -0,0 +1,32 @@ +# MinIO Monitoring Guide + +MinIO server exposes monitoring data over endpoints. Monitoring tools can pick the data from these endpoints. This document lists the monitoring endpoints and relevant documentation. + +## Healthcheck Probe + +MinIO server has two healthcheck related un-authenticated endpoints, a liveness probe to indicate if server is responding, cluster probe to check if server can be taken down for maintenance. + +- Liveness probe available at `/minio/health/live` +- Cluster probe available at `/minio/health/cluster` + +Read more on how to use these endpoints in [MinIO healthcheck guide](https://github.com/minio/minio/blob/master/docs/metrics/healthcheck/README.md). + +## Prometheus Probe + +MinIO allows reading metrics for the entire cluster from any single node. This allows for metrics collection for a MinIO instance across all servers. Thus, metrics collection for instances behind a load balancer can be done without any knowledge of the individual node addresses. The cluster wide metrics can be read at +`
/minio/v2/metrics/cluster`. + +The additional node specific metrics which include additional go metrics or process metrics are exposed at +`
/minio/v2/metrics/node`. + +The additional bucket specific metrics which include additional go metrics or process metrics are exposed at +`
/minio/v2/metrics/bucket`. + +The additional resource specific metrics which include additional go metrics or process metrics are exposed at +`
/minio/v2/metrics/resource`. + +To use this endpoint, setup Prometheus to scrape data from this endpoint. Read more on how to configure and use Prometheus to monitor MinIO server in [How to monitor MinIO server with Prometheus](https://github.com/minio/minio/blob/master/docs/metrics/prometheus/README.md). + +### **Deprecated metrics monitoring** + +- Prometheus' data available at `/minio/prometheus/metrics` is deprecated diff --git a/docs/metrics/healthcheck/README.md b/docs/metrics/healthcheck/README.md new file mode 100644 index 00000000..c74fd2e5 --- /dev/null +++ b/docs/metrics/healthcheck/README.md @@ -0,0 +1,93 @@ +# MinIO Healthcheck + +MinIO server exposes three un-authenticated, healthcheck endpoints liveness probe and a cluster probe at `/minio/health/live` and `/minio/health/cluster` respectively. + +## Liveness probe + +This probe always responds with '200 OK'. Only fails if 'etcd' is configured and unreachable. When liveness probe fails, Kubernetes like platforms restart the container. + +``` +livenessProbe: + httpGet: + path: /minio/health/live + port: 9000 + scheme: HTTP + initialDelaySeconds: 120 + periodSeconds: 30 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 3 +``` + +## Readiness probe + +This probe always responds with '200 OK'. Only fails if 'etcd' is configured and unreachable. When readiness probe fails, Kubernetes like platforms turn-off routing to the container. + +``` +readinessProbe: + httpGet: + path: /minio/health/ready + port: 9000 + scheme: HTTP + initialDelaySeconds: 120 + periodSeconds: 15 + timeoutSeconds: 10 + successThreshold: 1 + failureThreshold: 3 +``` + +## Cluster probe + +### Cluster-writeable probe + +The reply is '200 OK' if cluster has write quorum if not it returns '503 Service Unavailable'. + +``` +curl http://minio1:9001/minio/health/cluster +HTTP/1.1 503 Service Unavailable +Accept-Ranges: bytes +Content-Length: 0 +Server: MinIO +Vary: Origin +X-Amz-Bucket-Region: us-east-1 +X-Minio-Write-Quorum: 3 +X-Amz-Request-Id: 16239D6AB80EBECF +X-Xss-Protection: 1; mode=block +Date: Tue, 21 Jul 2020 00:36:14 GMT +``` + +### Cluster-readable probe + +The reply is '200 OK' if cluster has read quorum if not it returns '503 Service Unavailable'. + +``` +curl http://minio1:9001/minio/health/cluster/read +HTTP/1.1 503 Service Unavailable +Accept-Ranges: bytes +Content-Length: 0 +Server: MinIO +Vary: Origin +X-Amz-Bucket-Region: us-east-1 +X-Minio-Write-Quorum: 3 +X-Amz-Request-Id: 16239D6AB80EBECF +X-Xss-Protection: 1; mode=block +Date: Tue, 21 Jul 2020 00:36:14 GMT +``` + +### Checking cluster health for maintenance + +You may query the cluster probe endpoint to check if the node which received the request can be taken down for maintenance, if the server replies back '412 Precondition Failed' this means you will lose HA. '200 OK' means you are okay to proceed. + +``` +curl http://minio1:9001/minio/health/cluster?maintenance=true +HTTP/1.1 412 Precondition Failed +Accept-Ranges: bytes +Content-Length: 0 +Server: MinIO +Vary: Origin +X-Amz-Bucket-Region: us-east-1 +X-Amz-Request-Id: 16239D63820C6E76 +X-Xss-Protection: 1; mode=block +X-Minio-Write-Quorum: 3 +Date: Tue, 21 Jul 2020 00:35:43 GMT +``` diff --git a/docs/metrics/prometheus/README.md b/docs/metrics/prometheus/README.md new file mode 100644 index 00000000..3cfb94e4 --- /dev/null +++ b/docs/metrics/prometheus/README.md @@ -0,0 +1,195 @@ +# How to monitor MinIO server with Prometheus? [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +[Prometheus](https://prometheus.io) is a cloud-native monitoring platform. Prometheus offers a multi-dimensional data model with time series data identified by metric name and key/value pairs. The data collection happens via a pull model over HTTP/HTTPS. Users looking to monitor their MinIO instances can point Prometheus configuration to scrape data from following endpoints. + +- MinIO exports Prometheus compatible data by default as an authorized endpoint at `/minio/v2/metrics/cluster`. +- MinIO exports Prometheus compatible data by default which is bucket centric as an authorized endpoint at `/minio/v2/metrics/bucket`. + +This document explains how to setup Prometheus and configure it to scrape data from MinIO servers. + +## Prerequisites + +To get started with MinIO, refer [MinIO QuickStart Document](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html). +Follow below steps to get started with MinIO monitoring using Prometheus. + +### 1. Download Prometheus + +[Download the latest release](https://prometheus.io/download) of Prometheus for your platform, then extract it + +```sh +tar xvfz prometheus-*.tar.gz +cd prometheus-* +``` + +Prometheus server is a single binary called `prometheus` (or `prometheus.exe` on Microsoft Windows). Run the binary and pass `--help` flag to see available options + +```sh +./prometheus --help +usage: prometheus [] + +The Prometheus monitoring server + +. . . +``` + +Refer [Prometheus documentation](https://prometheus.io/docs/introduction/first_steps/) for more details. + +### 2. Configure authentication type for Prometheus metrics + +MinIO supports two authentication modes for Prometheus either `jwt` or `public`, by default MinIO runs in `jwt` mode. To allow public access without authentication for prometheus metrics set environment as follows. + +``` +export MINIO_PROMETHEUS_AUTH_TYPE="public" +minio server ~/test +``` + +### 3. Configuring Prometheus + +#### 3.1 Authenticated Prometheus config + +> If MinIO is configured to expose metrics without authentication, you don't need to use `mc` to generate prometheus config. You can skip reading further and move to 3.2 section. + +The Prometheus endpoint in MinIO requires authentication by default. Prometheus supports a bearer token approach to authenticate prometheus scrape requests, override the default Prometheus config with the one generated using mc. To generate a Prometheus config for an alias, use [mc](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart) as follows `mc admin prometheus generate [METRIC-TYPE]`. The valid values for METRIC-TYPE are `cluster`, `node`, `bucket` and `resource` and if not mentioned, it defaults to `cluster`. + +The command will generate the `scrape_configs` section of the prometheus.yml as follows: + +##### Cluster + +```yaml +scrape_configs: +- job_name: minio-job + bearer_token: + metrics_path: /minio/v2/metrics/cluster + scheme: http + static_configs: + - targets: ['localhost:9000'] +``` + +##### Bucket centric + +```yaml +- job_name: minio-job-bucket + bearer_token: + metrics_path: /minio/v2/metrics/bucket + scheme: http + static_configs: + - targets: ['localhost:9000'] +``` + +##### Node centric (optional) + +```yaml +- job_name: minio-job-node + bearer_token: + metrics_path: /minio/v2/metrics/node + scheme: http + static_configs: + - targets: ['localhost:9000'] +``` + +##### Resource centric (optional) + +```yaml +- job_name: minio-job-resource + bearer_token: + metrics_path: /minio/v2/metrics/resource + scheme: http + static_configs: + - targets: ['localhost:9000'] +``` + +#### 3.2 Public Prometheus config + +If Prometheus endpoint authentication type is set to `public`. Following prometheus config is sufficient to start scraping metrics data from MinIO. +This can be collected from any server once per collection. + +##### Cluster + +```yaml +scrape_configs: +- job_name: minio-job + metrics_path: /minio/v2/metrics/cluster + scheme: http + static_configs: + - targets: ['localhost:9000'] +``` + +#### Bucket centric + +```yaml +scrape_configs: +- job_name: minio-job-bucket + metrics_path: /minio/v2/metrics/bucket + scheme: http + static_configs: + - targets: ['localhost:9000'] +``` + +##### Node (optional) + +Optionally you can also collect per node metrics. This needs to be done on a per server instance. +The scrape configurations should use all the servers under `targets` so that graphing systems like +grafana can visualize them for all the nodes + +```yaml +scrape_configs: +- job_name: minio-job + metrics_path: /minio/v2/metrics/node + scheme: http + static_configs: + - targets: ['server1:9000','server2:9000','server3:9000','server4:9000'] +``` + +##### Resource (optional) + +Optionally you can also collect resource metrics. + +```yaml +scrape_configs: +- job_name: minio-job + metrics_path: /minio/v2/metrics/resource + scheme: http + static_configs: + - targets: ['localhost:9000'] +``` + +### 4. Update `scrape_configs` section in prometheus.yml + +To authorize every scrape request, copy and paste the generated `scrape_configs` section in the prometheus.yml and restart the Prometheus service. + +### 5. Start Prometheus + +Start (or) Restart Prometheus service by running + +```sh +./prometheus --config.file=prometheus.yml +``` + +Here `prometheus.yml` is the name of configuration file. You can now see MinIO metrics in Prometheus dashboard. By default Prometheus dashboard is accessible at `http://localhost:9090`. + +Prometheus sets the `Host` header to `domain:port` as part of HTTP operations against the MinIO metrics endpoint. For MinIO deployments behind a load balancer, reverse proxy, or other control plane (HAProxy, nginx, pfsense, opnsense, etc.), ensure the network service supports routing these requests to the deployment. + +### 6. Configure Grafana + +After Prometheus is configured, you can use Grafana to visualize MinIO metrics. Refer the [document here to setup Grafana with MinIO prometheus metrics](https://github.com/minio/minio/blob/master/docs/metrics/prometheus/grafana/README.md). + +## List of metrics exposed by MinIO + +- MinIO exports Prometheus compatible data by default as an authorized endpoint at `/minio/v2/metrics/cluster`. +- MinIO exports Prometheus compatible data by default which is bucket centric as an authorized endpoint at `/minio/v2/metrics/bucket`. +- MinIO exports Prometheus compatible data by default which is node centric as an authorized endpoint at `/minio/v2/metrics/node`. +- MinIO exports Prometheus compatible data by default which is resource centric as an authorized endpoint at `/minio/v2/metrics/resource`. + +All of these can be accessed via Prometheus dashboard. A sample list of exposed metrics along with their definition is available on our public demo server at + +```sh +curl https://play.min.io/minio/v2/metrics/cluster +``` + +### List of metrics reported Cluster and Bucket level + +[The list of metrics reported can be here](https://github.com/minio/minio/blob/master/docs/metrics/prometheus/list.md) + +### Configure Alerts for Prometheus + +[The Prometheus AlertManager and alerts can be configured following this](https://github.com/minio/minio/blob/master/docs/metrics/prometheus/alerts.md) diff --git a/docs/metrics/prometheus/alerts.md b/docs/metrics/prometheus/alerts.md new file mode 100644 index 00000000..06794d36 --- /dev/null +++ b/docs/metrics/prometheus/alerts.md @@ -0,0 +1,120 @@ +# How to configure Prometheus AlertManager + +Alerting with prometheus is two step process. First we setup alerts in Prometheus server and then we need to send alerts to the AlertManager. +Prometheus AlertManager is the component that manages sending, inhibition and silencing of the alerts generated from Prometheus. The AlertManager can be configured to send alerts to variety of receivers. Refer [Prometheus AlertManager receivers](https://prometheus.io/docs/alerting/latest/configuration/#receiver) for more details. + +Follow below steps to enable and use AlertManager. + +## Deploy and start AlertManager +Install Prometheus AlertManager from https://prometheus.io/download/ and create configuration as below + +```yaml +route: + group_by: ['alertname'] + group_wait: 30s + group_interval: 5m + repeat_interval: 1h + receiver: 'web.hook' +receivers: + - name: 'web.hook' + webhook_configs: + - url: 'http://127.0.0.1:8010/webhook' +inhibit_rules: + - source_match: + severity: 'critical' + target_match: + severity: 'warning' + equal: ['alertname', 'dev', 'instance'] +``` + +This sample configuration uses a `webhook` at http://127.0.0.1:8010/webhook to post the alerts. +Start the AlertManager and it listens on port `9093` by default. Make sure your webhook is up and listening for the alerts. + +## Configure Prometheus to use AlertManager + +Add below section to your `prometheus.yml` +```yaml +alerting: + alertmanagers: + - static_configs: + - targets: ['localhost:9093'] +rule_files: + - rules.yml +``` +Here `rules.yml` is the file which should contain the alerting rules defined. + +## Add rules for your deployment +Below is a sample alerting rules configuration for MinIO. Refer https://prometheus.io/docs/prometheus/latest/configuration/alerting_rules/ for more instructions on writing alerting rules for Prometheus. + +```yaml +groups: +- name: example + rules: + - alert: MinIOClusterTolerance + expr: minio_cluster_health_erasure_set_status < 1 + for: 5m + labels: + severity: critical + annotations: + summary: "Instance {{ $labels.server }} has lost quorum on pool {{ $labels.pool }} on set {{ $labels.set }}" + description: "MinIO instance {{ $labels.server }} of job {{ $labels.job }} has lost quorum on pool {{ $labels.pool }} on set {{ $labels.set }} for more than 5 minutes." +``` + +## Verify the configuration and alerts +To verify the above sample alert follow below steps + +1. Start a distributed MinIO instance (4 nodes setup) +2. Start Prometheus server and AlertManager +3. Bring down couple of MinIO instances to bring down the Erasure Set tolerance to -1 and verify the same with `mc admin prometheus metrics ALIAS | grep minio_cluster_health_erasure_set_status` +4. Wait for 5 mins (as alert is configured to be firing after 5 mins), and verify that you see an entry in webhook for the alert as well as in Prometheus console as shown below + +```json +{ + "receiver": "web\\.hook", + "status": "firing", + "alerts": [ + { + "status": "firing", + "labels": { + "alertname": "MinIOClusterTolerance", + "instance": "localhost:9000", + "job": "minio-job-node", + "pool": "0", + "server": "127.0.0.1:9000", + "set": "0", + "severity": "critical" + }, + "annotations": { + "description": "MinIO instance 127.0.0.1:9000 of job minio-job has tolerance <=0 for more than 5 minutes.", + "summary": "Instance 127.0.0.1:9000 unable to tolerate node failures" + }, + "startsAt": "2023-11-18T06:20:09.456Z", + "endsAt": "0001-01-01T00:00:00Z", + "generatorURL": "http://fedora-minio:9090/graph?g0.expr=minio_cluster_health_erasure_set_tolerance+%3C%3D+0&g0.tab=1", + "fingerprint": "2255608b0da28ca3" + } + ], + "groupLabels": { + "alertname": "MinIOClusterTolerance" + }, + "commonLabels": { + "alertname": "MinIOClusterTolerance", + "instance": "localhost:9000", + "job": "minio-job-node", + "pool": "0", + "server": "127.0.0.1:9000", + "set": "0", + "severity": "critical" + }, + "commonAnnotations": { + "description": "MinIO instance 127.0.0.1:9000 of job minio-job has lost quorum on pool 0 on set 0 for more than 5 minutes.", + "summary": "Instance 127.0.0.1:9000 has lost quorum on pool 0 on set 0" + }, + "externalURL": "http://fedora-minio:9093", + "version": "4", + "groupKey": "{}:{alertname=\"MinIOClusterTolerance\"}", + "truncatedAlerts": 0 +} +``` + +![Prometheus](https://raw.githubusercontent.com/minio/minio/master/docs/metrics/prometheus/minio-es-tolerance-alert.png) diff --git a/docs/metrics/prometheus/grafana/README.md b/docs/metrics/prometheus/grafana/README.md new file mode 100644 index 00000000..0b307f2a --- /dev/null +++ b/docs/metrics/prometheus/grafana/README.md @@ -0,0 +1,34 @@ +# How to monitor MinIO server with Grafana [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +[Grafana](https://grafana.com/) allows you to query, visualize, alert on and understand your metrics no matter where they are stored. Create, explore, and share dashboards with your team and foster a data driven culture. + +## Prerequisites + +- Prometheus and MinIO configured as explained in [document here](https://github.com/minio/minio/blob/master/docs/metrics/prometheus/README.md). +- Grafana installed as explained [here](https://grafana.com/grafana/download). + +## MinIO Grafana Dashboard + +Visualize MinIO metrics with our official Grafana dashboard available on the [Grafana dashboard portal](https://grafana.com/grafana/dashboards/13502). + +Refer to the dashboard [json file here](https://raw.githubusercontent.com/minio/minio/master/docs/metrics/prometheus/grafana/minio-dashboard.json). + +![Grafana](https://raw.githubusercontent.com/minio/minio/master/docs/metrics/prometheus/grafana/grafana-minio.png) + +Node level Replication metrics can be viewed in the Grafana dashboard using [json file here](https://raw.githubusercontent.com/minio/minio/master/docs/metrics/prometheus/grafana/replication/minio-replication-node.json) + +![Grafana](https://raw.githubusercontent.com/minio/minio/master/docs/metrics/prometheus/grafana/replication/grafana-replication-node.png) + +Cluster level Replication metrics can be viewed in the Grafana dashboard using [json file here](https://raw.githubusercontent.com/minio/minio/master/docs/metrics/prometheus/grafana/replication/minio-replication-cluster.json) + +![Grafana](https://raw.githubusercontent.com/minio/minio/master/docs/metrics/prometheus/grafana/replication/grafana-replication-cluster.png) + +Bucket metrics can be viewed in the Grafana dashboard using [json file here](https://raw.githubusercontent.com/minio/minio/master/docs/metrics/prometheus/grafana/bucket/minio-bucket.json) + +![Grafana](https://raw.githubusercontent.com/minio/minio/master/docs/metrics/prometheus/grafana/bucket/grafana-bucket.png) + +Node metrics can be viewed in the Grafana dashboard using [json file here](https://raw.githubusercontent.com/minio/minio/master/docs/metrics/prometheus/grafana/node/minio-node.json) + +![Grafana](https://raw.githubusercontent.com/minio/minio/master/docs/metrics/prometheus/grafana/node/grafana-node.png) + +Note: All these dashboards are provided as an example and need basis they should be customized as well as new graphs should be added. diff --git a/docs/metrics/prometheus/grafana/bucket/grafana-bucket.png b/docs/metrics/prometheus/grafana/bucket/grafana-bucket.png new file mode 100644 index 00000000..55c25707 Binary files /dev/null and b/docs/metrics/prometheus/grafana/bucket/grafana-bucket.png differ diff --git a/docs/metrics/prometheus/grafana/bucket/minio-bucket.json b/docs/metrics/prometheus/grafana/bucket/minio-bucket.json new file mode 100644 index 00000000..5ce85790 --- /dev/null +++ b/docs/metrics/prometheus/grafana/bucket/minio-bucket.json @@ -0,0 +1,4214 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "MinIO Grafana Dashboard - https://min.io/", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 15306, + "graphTooltip": 0, + "id": 296, + "links": [ + { + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "minio" + ], + "type": "dashboards" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "semi-dark-red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 0 + }, + "id": 52, + "options": { + "displayMode": "basic", + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": false, + "sizing": "auto", + "text": {}, + "valueMode": "color" + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (bucket,range) (minio_bucket_objects_size_distribution{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{bucket,range}}", + "refId": "A", + "step": 300 + } + ], + "title": "Object Size Distribution", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "semi-dark-red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 0 + }, + "id": 53, + "options": { + "displayMode": "basic", + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": false, + "sizing": "auto", + "text": {}, + "valueMode": "color" + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (bucket,range) (minio_bucket_objects_version_distribution{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{bucket,range}}", + "refId": "A", + "step": 300 + } + ], + "title": "Object Version Distribution", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "semi-dark-red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 59, + "options": { + "displayMode": "basic", + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": false, + "sizing": "auto", + "text": {}, + "valueMode": "color" + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (bucket,le,api) (minio_bucket_requests_ttfb_seconds_distribution{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{bucket,le,api}}", + "refId": "A", + "step": 300 + } + ], + "title": "TTFB Distribution", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 6 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket,api) (increase(minio_bucket_requests_4xx_errors_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{bucket,api}}", + "refId": "A" + } + ], + "title": "S3 API Request 4xx Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 6 + }, + "id": 61, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket,api) (minio_bucket_requests_inflight_total{job=~\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{bucket,api}}", + "refId": "A" + } + ], + "title": "Inflight Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 12 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket,api) (increase(minio_bucket_requests_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{bucket,api}}", + "refId": "A" + } + ], + "title": "Total Requests Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 12 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket) (rate(minio_bucket_traffic_sent_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Data Sent [{{bucket}}]", + "refId": "A" + } + ], + "title": "Total Data Sent Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 12 + }, + "id": 64, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket) (rate(minio_bucket_usage_total_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Usage [{{bucket}}]", + "refId": "A" + } + ], + "title": "Usage Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 12 + }, + "id": 65, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket) (minio_bucket_usage_object_total{job=~\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{bucket}}", + "refId": "A" + } + ], + "title": "Objects", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 18 + }, + "id": 66, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket) (minio_bucket_usage_version_total{job=~\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{bucket}}", + "refId": "A" + } + ], + "title": "Versions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 18 + }, + "id": 67, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket) (minio_bucket_usage_deletemarker_total{job=~\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{bucket}}", + "refId": "A" + } + ], + "title": "Delete Markers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 18 + }, + "id": 68, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "Prometheus" + }, + "exemplar": true, + "expr": "minio_usage_last_activity_nano_seconds{job=~\"$scrape_jobs\"}", + "interval": "1m", + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "Time Elapsed Since Last Scan (nanos)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 18 + }, + "id": 69, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket) (rate(minio_bucket_traffic_received_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Data Received [{{bucket}}]", + "refId": "A" + } + ], + "title": "Total Data Received Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 24 + }, + "id": 54, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket) (minio_bucket_replication_received_bytes{job=~\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Data Received [{{bucket}}]", + "refId": "A" + } + ], + "title": "Replication Data Received", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 24 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket) (minio_bucket_replication_sent_bytes{job=~\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Replication Data Sent [{{bucket}}]", + "refId": "A" + } + ], + "title": "Replication Data Sent", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 24 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket) (minio_bucket_replication_total_failed_bytes{job=~\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Replication Failed [{{bucket}}]", + "refId": "A" + } + ], + "title": "Replication Data Failed", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 24 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket) (minio_bucket_replication_total_failed_count{job=~\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Replication Failed Objects [{{bucket}}]", + "refId": "A" + } + ], + "title": "Replication Failed Objects", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 30 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket) (minio_bucket_replication_received_count{job=~\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Replicated In Objects [{{bucket}}]", + "refId": "A" + } + ], + "title": "Replicated In Objects", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 30 + }, + "id": 71, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket) (minio_bucket_replication_sent_count{job=~\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Replicated Out Objects [{{bucket}}]", + "refId": "A" + } + ], + "title": "Replicated Out Objects", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 30 + }, + "id": 72, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket) (rate(minio_bucket_replication_last_hour_failed_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Last Hour Failed Size [{{bucket}}]", + "refId": "A" + } + ], + "title": "Last Hour Failed Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 30 + }, + "id": 73, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket) (minio_bucket_replication_last_hour_failed_count{job=~\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Last Hour Failed Objects [{{bucket}}]", + "refId": "A" + } + ], + "title": "Last Hour Failed Objects", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 36 + }, + "id": 74, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket) (rate(minio_bucket_replication_last_minute_failed_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Last Minute Failed Size [{{bucket}}]", + "refId": "A" + } + ], + "title": "Last Minute Failed Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 36 + }, + "id": 75, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket) (minio_bucket_replication_last_minute_failed_count{job=~\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Last Minute Failed Objects [{{bucket}}]", + "refId": "A" + } + ], + "title": "Last Minute Failed Objects", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "semi-dark-red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 36 + }, + "id": 58, + "options": { + "displayMode": "basic", + "maxVizHeight": 300, + "minVizHeight": 10, + "minVizWidth": 0, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showUnfilled": false, + "sizing": "auto", + "text": {}, + "valueMode": "color" + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "sum by (bucket,range,operation) (minio_bucket_replication_latency_ms{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{bucket,range,operation}}", + "refId": "A", + "step": 300 + } + ], + "title": "Replication Latency (millis)", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 42 + }, + "id": 76, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket) (minio_bucket_replication_proxied_head_requests_total{job=~\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{bucket}}", + "refId": "A" + } + ], + "title": "Total Proxied Head Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 42 + }, + "id": 77, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_bucket_replication_proxied_head_requests_failures{job=~\"$scrape_jobs\"}", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Total Failed Proxied Head Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 42 + }, + "id": 78, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket) (minio_bucket_replication_proxied_put_tagging_requests_total{job=~\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{bucket}}", + "refId": "A" + } + ], + "title": "Total Proxied Put Tag Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 42 + }, + "id": 79, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_bucket_replication_proxied_put_tagging_requests_failures{job=~\"$scrape_jobs\"}", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Total Failed Proxied Put Tag Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 48 + }, + "id": 80, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket) (minio_bucket_replication_proxied_get_tagging_requests_total{job=~\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{bucket}}", + "refId": "A" + } + ], + "title": "Total Proxied Get Tag Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 48 + }, + "id": 81, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_bucket_replication_proxied_get_tagging_requests_failures{job=~\"$scrape_jobs\"}", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Total Failed Proxied Get Tag Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 48 + }, + "id": 82, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket) (minio_bucket_replication_proxied_delete_tagging_requests_total{job=~\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{bucket}}", + "refId": "A" + } + ], + "title": "Total Proxied Delete Tag Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 48 + }, + "id": 83, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_bucket_replication_proxied_delete_tagging_requests_failures{job=~\"$scrape_jobs\"}", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Total Failed Proxied Delete Tag Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 54 + }, + "id": 84, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (bucket) (minio_bucket_replication_proxied_get_requests_total{job=~\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{bucket}}", + "refId": "A" + } + ], + "title": "Total Proxied Get Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 54 + }, + "id": 85, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_bucket_replication_proxied_get_requests_failures{job=~\"$scrape_jobs\"}", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "refId": "A" + } + ], + "title": "Total Failed Proxied Get Requests", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "minio" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "label": "Data source", + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(job)", + "includeAll": false, + "name": "scrape_jobs", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "MinIO Bucket Dashboard", + "uid": "TgmJnqnnk2", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/docs/metrics/prometheus/grafana/grafana-minio.png b/docs/metrics/prometheus/grafana/grafana-minio.png new file mode 100644 index 00000000..6e48f6ef Binary files /dev/null and b/docs/metrics/prometheus/grafana/grafana-minio.png differ diff --git a/docs/metrics/prometheus/grafana/minio-dashboard.json b/docs/metrics/prometheus/grafana/minio-dashboard.json new file mode 100644 index 00000000..8976eb7b --- /dev/null +++ b/docs/metrics/prometheus/grafana/minio-dashboard.json @@ -0,0 +1,3868 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "MinIO Grafana Dashboard - https://min.io/", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 13502, + "graphTooltip": 0, + "id": 292, + "links": [ + { + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "minio" + ], + "type": "dashboards" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "dtdurations" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 3, + "x": 0, + "y": 0 + }, + "id": 1, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "time() - max(minio_node_process_starttime_seconds{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 0 + }, + "id": 65, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (instance) (minio_s3_traffic_received_bytes{job=~\"$scrape_jobs\"})", + "format": "table", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Total S3 Ingress", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Free" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Used" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "orange", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 4, + "x": 6, + "y": 0 + }, + "id": 50, + "interval": "1m", + "maxDataPoints": 100, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent" + ] + }, + "pieType": "donut", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "topk(1, sum(minio_cluster_capacity_usable_total_bytes{job=~\"$scrape_jobs\"}) by (instance)) - topk(1, sum(minio_cluster_capacity_usable_free_bytes{job=~\"$scrape_jobs\"}) by (instance))", + "format": "time_series", + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "Used", + "refId": "A", + "step": 300 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "topk(1, sum(minio_cluster_capacity_usable_free_bytes{job=~\"$scrape_jobs\"}) by (instance)) ", + "hide": false, + "interval": "1m", + "legendFormat": "Free", + "refId": "B" + } + ], + "title": "Capacity", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": true, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Objects" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + }, + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "Usage" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 10, + "y": 0 + }, + "id": 68, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "8.2.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(minio_cluster_usage_total_bytes{job=~\"$scrape_jobs\"})", + "interval": "", + "legendFormat": "Usage", + "range": true, + "refId": "A" + } + ], + "title": "Data Usage Growth", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 5, + "x": 16, + "y": 0 + }, + "id": 52, + "options": { + "displayMode": "lcd", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": true, + "expr": "minio_cluster_objects_size_distribution{job=~\"$scrape_jobs\"}", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{range}}", + "refId": "A", + "step": 300, + "useBackend": false + } + ], + "title": "Object Size Distribution", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 0 + }, + "id": 61, + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "minio_node_file_descriptor_open_total{job=~\"$scrape_jobs\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{server}}", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Open FDs ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 3, + "y": 3 + }, + "id": 64, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (instance) (minio_s3_traffic_sent_bytes{job=~\"$scrape_jobs\"})", + "format": "table", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Total S3 Egress", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 21, + "y": 3 + }, + "id": 62, + "maxDataPoints": 100, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "minio_node_go_routine_total{job=~\"$scrape_jobs\"}", + "format": "time_series", + "hide": false, + "instant": false, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{server}}", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Goroutines", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bool_on_off" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 6 + }, + "id": 94, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": true, + "expr": "minio_cluster_health_status{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "includeNullMetadata": true, + "interval": "", + "legendFormat": "Pool: {{pool}} Set: {{set}}", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Cluster Health Status", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 3, + "x": 3, + "y": 6 + }, + "id": 78, + "maxDataPoints": 100, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(minio_cluster_drive_online_total{job=~\"$scrape_jobs\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": ".", + "metric": "process_start_time_seconds", + "range": false, + "refId": "A", + "step": 60 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(minio_cluster_drive_offline_total{job=~\"$scrape_jobs\"})", + "format": "time_series", + "hide": false, + "instant": true, + "legendFormat": ".", + "range": false, + "refId": "B" + } + ], + "title": "Total Online/Offline Drives", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "dark-yellow", + "value": 75000000 + }, + { + "color": "dark-red", + "value": 100000000 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 6, + "y": 6 + }, + "id": 66, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(minio_cluster_bucket_total{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Number of Buckets", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 9, + "y": 6 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (rate(minio_s3_traffic_received_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Data Received [{{server}}]", + "refId": "A" + } + ], + "title": "S3 API Ingress Rate ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "binBps" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (rate(minio_s3_traffic_sent_bytes{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "Data Sent [{{server}}]", + "refId": "A" + } + ], + "title": "S3 API Egress Rate ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 8 + }, + "id": 53, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "max(minio_cluster_nodes_online_total{job=~\"$scrape_jobs\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Total Online Servers", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "dark-yellow", + "value": 75000000 + }, + { + "color": "dark-red", + "value": 100000000 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 3, + "w": 3, + "x": 6, + "y": 9 + }, + "id": 44, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "max(minio_cluster_usage_object_total{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": false, + "interval": "1m", + "intervalFactor": 1, + "legendFormat": "", + "refId": "A" + } + ], + "title": "Number of Objects", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ns" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 0, + "y": 10 + }, + "id": 80, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "max(minio_heal_time_last_activity_nano_seconds{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Time Since Last Heal", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "ns" + }, + "overrides": [] + }, + "gridPos": { + "h": 2, + "w": 3, + "x": 3, + "y": 10 + }, + "id": 81, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "last" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "max(minio_usage_last_activity_nano_seconds{job=~\"$scrape_jobs\"})", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Time Since Last Scan", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 9, + "x": 0, + "y": 12 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server,api) (increase(minio_s3_requests_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server,api}}", + "refId": "A" + } + ], + "title": "S3 API Request Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 7, + "x": 9, + "y": 12 + }, + "id": 88, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server,api) (increase(minio_s3_requests_4xx_errors_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server,api}}", + "refId": "A" + } + ], + "title": "S3 API Request Error Rate (4xx)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 8, + "x": 16, + "y": 12 + }, + "id": 86, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server,api) (increase(minio_s3_requests_5xx_errors_total{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server,api}}", + "refId": "A" + } + ], + "title": "S3 API Request Error Rate (5xx)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "fixed" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 18 + }, + "id": 99, + "options": { + "displayMode": "lcd", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_online_drives{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Online Drives", + "range": true, + "refId": "A", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_read_quorum{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Read Quorum", + "range": true, + "refId": "B", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_write_quorum{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Write Quorum", + "range": true, + "refId": "C", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_healing_drives{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Healing Drives", + "range": true, + "refId": "D", + "useBackend": false + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "disableTextWrap": false, + "editorMode": "builder", + "expr": "minio_cluster_health_erasure_set_status{job=~\"$scrape_jobs\"}", + "fullMetaSearch": false, + "hide": false, + "includeNullMetadata": true, + "instant": false, + "legendFormat": "Pool {{pool}} / Set {{set}} - Status", + "range": true, + "refId": "E", + "useBackend": false + } + ], + "title": "Health Breakdown", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "continuous-GrYlRd" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 18 + }, + "id": 76, + "options": { + "displayMode": "lcd", + "maxVizHeight": 300, + "minVizHeight": 16, + "minVizWidth": 8, + "namePlacement": "auto", + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true, + "sizing": "auto", + "valueMode": "color" + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "minio_node_process_resident_memory_bytes{job=~\"$scrape_jobs\"}", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{server}}", + "range": true, + "refId": "A" + } + ], + "title": "Memory Usage ", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 73, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_node_io_rchar_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "Node RChar [{{server}}]", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_node_io_wchar_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Node WChar [{{server}}]", + "refId": "B" + } + ], + "title": "Read, Write I/O", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "red", + "value": 85 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 77, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(minio_node_process_cpu_total_seconds{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "{{server}}", + "range": true, + "refId": "A" + } + ], + "title": "CPU Usage", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Total number of bytes received and sent on MinIO cluster", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 25, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 33 + }, + "id": 17, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "10.3.1", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "rate(minio_inter_node_traffic_sent_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Internode Bytes Received [{{server}}]", + "metric": "minio_http_requests_duration_seconds_count", + "range": true, + "refId": "A", + "step": 4 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_inter_node_traffic_received_bytes{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Internode Bytes Sent [{{server}}]", + "refId": "B" + } + ], + "title": "Internode Traffic", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "available 10.13.1.25:9000" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used 10.13.1.25:9000" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 33 + }, + "id": 8, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_node_file_descriptor_open_total{job=~\"$scrape_jobs\"}", + "interval": "", + "legendFormat": "Open FDs [{{server}}]", + "refId": "B" + } + ], + "title": "File Descriptors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "Number of online drives per MinIO Server", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 100, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Offline 10.13.1.25:9000" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "dark-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Total 10.13.1.25:9000" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 40 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_node_syscall_read_total{job=~\"$scrape_jobs\"}[$__rate_interval])", + "format": "time_series", + "interval": "", + "intervalFactor": 2, + "legendFormat": "Read Syscalls [{{server}}]", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_node_syscall_write_total{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "", + "legendFormat": "Write Syscalls [{{server}}]", + "refId": "B" + } + ], + "title": "Syscalls", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 40 + }, + "id": 95, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_node_scanner_objects_scanned{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "title": "Scanned Objects", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 75, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_node_scanner_versions_scanned{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "title": "Scanned Versions", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 96, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_node_scanner_directories_scanned{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "title": "Scanned Directories", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "dtdurations" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 0, + "y": 54 + }, + "id": 89, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_cluster_kms_uptime{job=~\"$scrape_jobs\"}", + "format": "time_series", + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "{{instance}}", + "metric": "minio_cluster_kms_uptime", + "refId": "A", + "step": 60 + } + ], + "title": "KMS Uptime", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 4, + "y": 54 + }, + "id": 91, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (increase(minio_cluster_kms_request_error{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "KMS Request 4xx Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bool_on_off" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 4, + "x": 8, + "y": 54 + }, + "id": 90, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (minio_cluster_kms_online{job=~\"$scrape_jobs\"})", + "interval": "1m", + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "KMS Online(1)/Offline(0)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 12, + "x": 12, + "y": 54 + }, + "id": 98, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_node_scanner_bucket_scans_finished{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "title": "Bucket Scans Finished", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 58 + }, + "id": 92, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (increase(minio_cluster_kms_request_failure{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "KMS Request 5xx Error Rate", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 58 + }, + "id": 93, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (rate(minio_cluster_kms_request_success{job=~\"$scrape_jobs\"}[$__rate_interval]))", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "KMS Request Success [{{server}}]", + "refId": "A" + } + ], + "title": "KMS Request Success Rate ", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 12, + "x": 12, + "y": 61 + }, + "id": 97, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "rate(minio_node_scanner_bucket_scans_started{job=~\"$scrape_jobs\"}[$__rate_interval])", + "interval": "1m", + "legendFormat": "[{{server}}]", + "refId": "A" + } + ], + "title": "Bucket Scans Started", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "minio" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "label": "Data source", + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(job)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "scrape_jobs", + "options": [], + "query": { + "query": "label_values(job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "MinIO Dashboard", + "uid": "TgmJnqnnk", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/docs/metrics/prometheus/grafana/node/grafana-node.png b/docs/metrics/prometheus/grafana/node/grafana-node.png new file mode 100644 index 00000000..4b12d541 Binary files /dev/null and b/docs/metrics/prometheus/grafana/node/grafana-node.png differ diff --git a/docs/metrics/prometheus/grafana/node/minio-node.json b/docs/metrics/prometheus/grafana/node/minio-node.json new file mode 100644 index 00000000..783190dd --- /dev/null +++ b/docs/metrics/prometheus/grafana/node/minio-node.json @@ -0,0 +1,955 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "MinIO Nodes Grafana Dashboard - https://min.io/", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 15306, + "graphTooltip": 0, + "id": 267, + "links": [ + { + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "minio" + ], + "type": "dashboards" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [ + { + "options": { + "match": "null", + "result": { + "text": "N/A" + } + }, + "type": "special" + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 21, + "maxDataPoints": 100, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "mean" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "text": {}, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "max(minio_node_drive_total{job=~\"$scrape_jobs\",server=\"$server\"})", + "format": "table", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": "", + "metric": "process_start_time_seconds", + "refId": "A", + "step": 60 + } + ], + "title": "Total Drives", + "type": "stat" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "mappings": [], + "thresholds": { + "mode": "percentage", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 22, + "maxDataPoints": 100, + "options": { + "minVizHeight": 75, + "minVizWidth": 75, + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showThresholdLabels": false, + "showThresholdMarkers": true, + "sizing": "auto" + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(minio_node_drive_online_total{job=~\"$scrape_jobs\",server=\"$server\"})", + "format": "time_series", + "hide": false, + "instant": true, + "interval": "", + "intervalFactor": 1, + "legendFormat": ".", + "metric": "process_start_time_seconds", + "range": false, + "refId": "A", + "step": 60 + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "max(minio_node_drive_offline_total{job=~\"$scrape_jobs\",server=\"$server\"})", + "format": "time_series", + "hide": false, + "instant": true, + "legendFormat": ".", + "range": false, + "refId": "B" + } + ], + "title": "Total Online/Offline Drives", + "type": "gauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_node_drive_total_bytes{job=~\"$scrape_jobs\",server=\"$server\"}", + "interval": "", + "legendFormat": "Total [{{drive}}]", + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_node_drive_used_bytes{job=~\"$scrape_jobs\",server=\"$server\"}", + "interval": "", + "legendFormat": "Used [{{drive}}]", + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_node_drive_free_bytes{job=~\"$scrape_jobs\",server=\"$server\"}", + "interval": "", + "legendFormat": "Free [{{drive}}]", + "refId": "C" + } + ], + "title": "Drive Usage", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_node_drive_free_inodes{job=~\"$scrape_jobs\",server=\"$server\"}", + "interval": "", + "legendFormat": "[{{drive}}]", + "refId": "B" + } + ], + "title": "Free Inodes", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 0, + "y": 5 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_node_drive_latency_us{job=~\"$scrape_jobs\",server=\"$server\"}", + "interval": "", + "legendFormat": "[{{drive}}:{{api}}]", + "refId": "B" + } + ], + "title": "Drive Latency (micro sec)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 6, + "y": 5 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_node_drive_errors_availability{job=~\"$scrape_jobs\",server=\"$server\"}", + "interval": "", + "legendFormat": "[{{drive}}]", + "refId": "B" + } + ], + "title": "Drive Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "available 10.13.1.25:9000" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "green", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "used 10.13.1.25:9000" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "blue", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 12, + "y": 5 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_node_drive_errors_timeout{job=~\"$scrape_jobs\",server=\"$server\"}", + "interval": "", + "legendFormat": "[{{drive}}]", + "refId": "B" + } + ], + "title": "Drive Timeout Errors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "links": [], + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 5, + "w": 6, + "x": 18, + "y": 5 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_node_drive_io_waiting{job=~\"$scrape_jobs\",server=\"$server\"}", + "interval": "", + "legendFormat": "[{{drive}}]", + "refId": "B" + } + ], + "title": "IO Operations Waiting", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "minio" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "label": "Data source", + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(job)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "scrape_jobs", + "options": [], + "query": { + "query": "label_values(job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(minio_node_drive_total,server)", + "hide": 0, + "includeAll": false, + "label": "Server", + "multi": false, + "name": "server", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(minio_node_drive_total,server)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "MinIO Node Dashboard", + "uid": "TgmJnnqn2k", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/docs/metrics/prometheus/grafana/replication/grafana-replication-cluster.png b/docs/metrics/prometheus/grafana/replication/grafana-replication-cluster.png new file mode 100644 index 00000000..0899664c Binary files /dev/null and b/docs/metrics/prometheus/grafana/replication/grafana-replication-cluster.png differ diff --git a/docs/metrics/prometheus/grafana/replication/grafana-replication-node.png b/docs/metrics/prometheus/grafana/replication/grafana-replication-node.png new file mode 100644 index 00000000..2758631b Binary files /dev/null and b/docs/metrics/prometheus/grafana/replication/grafana-replication-node.png differ diff --git a/docs/metrics/prometheus/grafana/replication/minio-replication-cluster.json b/docs/metrics/prometheus/grafana/replication/minio-replication-cluster.json new file mode 100644 index 00000000..7e34467d --- /dev/null +++ b/docs/metrics/prometheus/grafana/replication/minio-replication-cluster.json @@ -0,0 +1,2952 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "MinIO Grafana Dashboard - https://min.io/", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 15306, + "graphTooltip": 0, + "id": 285, + "links": [ + { + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "minio" + ], + "type": "dashboards" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (minio_cluster_replication_received_bytes{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "Received Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 58, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (minio_cluster_replication_received_count{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "Received Objects", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (endpoint,server) (minio_cluster_replication_sent_bytes{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{endpoint}},{{server}}", + "refId": "A" + } + ], + "title": "Sent Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (endpoint,server) (minio_cluster_replication_sent_count{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{endpoint}},{{server}}", + "refId": "A" + } + ], + "title": "Sent Objects", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 6 + }, + "id": 61, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (minio_cluster_replication_total_failed_bytes{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "Failed Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 6 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (minio_cluster_replication_total_failed_count{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "Failed Objects", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 6 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (endpoint,server) (minio_cluster_replication_total_failed_bytes{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{endpoint}},{{server}}", + "refId": "A" + } + ], + "title": "Total Failed Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 6 + }, + "id": 64, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (endpoint,server) (minio_cluster_replication_total_failed_count{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{endpoint}},{{server}}", + "refId": "A" + } + ], + "title": "Total Failed Objects", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 12 + }, + "id": 65, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (endpoint,server) (minio_cluster_replication_last_hour_failed_bytes{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{endpoint}},{{server}}", + "refId": "A" + } + ], + "title": "Last Hour Failed Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 12 + }, + "id": 66, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (endpoint,server) (minio_cluster_replication_last_hour_failed_count{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{endpoint}},{{server}}", + "refId": "A" + } + ], + "title": "Last Hour Failed Objects", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 12 + }, + "id": 67, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (endpoint,server) (minio_cluster_replication_last_minute_failed_bytes{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{endpoint}},{{server}}", + "refId": "A" + } + ], + "title": "Last Minute Failed Data", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 12 + }, + "id": 68, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (endpoint,server) (minio_cluster_replication_last_minute_failed_count{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{endpoint}},{{server}}", + "refId": "A" + } + ], + "title": "Last Minute Failed Objects", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 18 + }, + "id": 69, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "minio_cluster_replication_proxied_head_requests_total{job=\"$scrape_jobs\"}", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Total Proxied Head Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 18 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "minio_cluster_replication_proxied_head_requests_failures{job=\"$scrape_jobs\"}", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Total Failed Proxied Head Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 18 + }, + "id": 71, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "minio_cluster_replication_proxied_put_tagging_requests_total{job=\"$scrape_jobs\"}", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Total Proxied Put Tag Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 18 + }, + "id": 72, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "minio_cluster_replication_proxied_put_tagging_requests_failures{job=\"$scrape_jobs\"}", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Total Failed Proxied Put Tag Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 24 + }, + "id": 73, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "minio_cluster_replication_proxied_get_tagging_requests_total{job=\"$scrape_jobs\"}", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Total Proxied Get Tag Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 24 + }, + "id": 74, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "minio_cluster_replication_proxied_get_tagging_requests_failures{job=\"$scrape_jobs\"}", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Total Failed Proxied Get Tag Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 24 + }, + "id": 75, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "minio_cluster_replication_proxied_delete_tagging_requests_total{job=\"$scrape_jobs\"}", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Total Proxied Delete Tag Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 24 + }, + "id": 76, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "minio_cluster_replication_proxied_delete_tagging_requests_failures{job=\"$scrape_jobs\"}", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Total Failed Proxied Delete Tag Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 77, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "minio_cluster_replication_proxied_get_requests_total{job=\"$scrape_jobs\"}", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Total Proxied Get Requests", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 30 + }, + "id": 78, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": true, + "expr": "minio_cluster_replication_proxied_get_requests_failures{job=\"$scrape_jobs\"}", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{instance}}", + "range": true, + "refId": "A" + } + ], + "title": "Total Failed Proxied Get Requests", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "minio" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "label": "Data source", + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(job)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "scrape_jobs", + "options": [], + "query": { + "query": "label_values(job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "MinIO Cluster Replication Dashboard", + "uid": "TgmJnnqnk3", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/docs/metrics/prometheus/grafana/replication/minio-replication-node.json b/docs/metrics/prometheus/grafana/replication/minio-replication-node.json new file mode 100644 index 00000000..6e0c29b4 --- /dev/null +++ b/docs/metrics/prometheus/grafana/replication/minio-replication-node.json @@ -0,0 +1,2397 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "description": "MinIO Grafana Dashboard - https://min.io/", + "editable": true, + "fiscalYearStartMonth": 0, + "gnetId": 15306, + "graphTooltip": 0, + "id": 283, + "links": [ + { + "icon": "external link", + "includeVars": true, + "keepTime": true, + "tags": [ + "minio" + ], + "type": "dashboards" + } + ], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 0 + }, + "id": 55, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (minio_node_replication_average_active_workers{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "Avg. Active Workers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 0 + }, + "id": 56, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server, endpoint) (minio_node_replication_average_link_latency_ms{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server,endpoint}}", + "refId": "A" + } + ], + "title": "Avg. Link Latency (millis)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 0 + }, + "id": 57, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (minio_node_replication_average_queued_bytes{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "Avg. Queued Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 0 + }, + "id": 58, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (minio_node_replication_average_queued_count{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "Avg. Queued Objects", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 6 + }, + "id": 59, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (minio_node_replication_average_transfer_rate{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "Avg. Transfer Rate (bytes/s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 6 + }, + "id": 60, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (minio_node_replication_current_active_workers{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "Active Workers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 6 + }, + "id": 61, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server,endpoint) (minio_node_replication_current_link_latency_ms{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "Current Link Latency (millis)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "Replication Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "Replication Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 6 + }, + "id": 62, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (minio_node_replication_current_transfer_rate{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "Current Transfer Rate (bytes/s)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 12 + }, + "id": 63, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (minio_node_replication_last_minute_queued_bytes{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "Last Minute Queued", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 12 + }, + "id": 64, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server) (minio_node_replication_last_minute_queued_count{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{bucket}}", + "refId": "A" + } + ], + "title": "Last Minute Queued Objects", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "S3 Errors" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-red", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "S3 Requests" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "light-green", + "mode": "fixed" + } + } + ] + } + ] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 12 + }, + "id": 65, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server,endpoint) (minio_node_replication_link_downtime_duration_seconds{job=\"$scrape_jobs\"})", + "interval": "1m", + "intervalFactor": 2, + "legendFormat": "{{server,endpoint}}", + "refId": "A" + } + ], + "title": "Link Downtime Duration (sec)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 12 + }, + "id": 66, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server,endpoint) (minio_node_replication_link_offline_duration_seconds{job=\"$scrape_jobs\"})", + "interval": "1m", + "legendFormat": "{{server,endpoint}}", + "refId": "A" + } + ], + "title": "Link Offline Duration (sec)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 18 + }, + "id": 67, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_node_replication_max_active_workers{job=\"$scrape_jobs\"}", + "interval": "1m", + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "Max Active Workers", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 18 + }, + "id": 68, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server,endpoibt) (minio_node_replication_max_link_latency_ms{job=\"$scrape_jobs\"})", + "interval": "1m", + "legendFormat": "{{server,endpoint}}", + "refId": "A" + } + ], + "title": "Max Link Latency (millis)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 18 + }, + "id": 70, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_node_replication_max_queued_bytes{job=\"$scrape_jobs\"}", + "interval": "1m", + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "Max Queued Size", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 18 + }, + "id": 71, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_node_replication_max_queued_count{job=\"$scrape_jobs\"}", + "interval": "1m", + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "Max Queued Objects", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 0, + "y": 24 + }, + "id": 72, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_node_replication_max_transfer_rate{job=\"$scrape_jobs\"}", + "interval": "1m", + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "Max Transfer Rate (per sec)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 6, + "y": 24 + }, + "id": 73, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_node_replication_recent_backlog_count{job=\"$scrape_jobs\"}", + "interval": "1m", + "legendFormat": "{{server}}", + "refId": "A" + } + ], + "title": "Backlog (last 5 mins)", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 12, + "y": 24 + }, + "id": 74, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "sum by (server,endpoint) (minio_node_replication_link_online{job=\"$scrape_jobs\"})", + "interval": "1m", + "legendFormat": "{{endpoint}}", + "refId": "A" + } + ], + "title": "Link Online/Offline", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 6, + "x": 18, + "y": 24 + }, + "id": 75, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "pluginVersion": "10.4.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "exemplar": true, + "expr": "minio_node_replication_link_offline_duration_seconds{job=\"$scrape_jobs\"}", + "interval": "1m", + "legendFormat": "{{endpoint}}", + "refId": "A" + } + ], + "title": "Replication Link Offline Duration (sec)", + "type": "timeseries" + } + ], + "refresh": "", + "schemaVersion": 39, + "tags": [ + "minio" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "All", + "value": "$__all" + }, + "label": "Data source", + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + }, + { + "current": {}, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(job)", + "hide": 0, + "includeAll": false, + "multi": false, + "name": "scrape_jobs", + "options": [], + "query": { + "query": "label_values(job)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-1h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": [ + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] + }, + "timezone": "", + "title": "MinIO Node Replication Dashboard", + "uid": "gmTJnqnnk3", + "version": 1, + "weekStart": "" +} \ No newline at end of file diff --git a/docs/metrics/prometheus/list.md b/docs/metrics/prometheus/list.md new file mode 100644 index 00000000..70500064 --- /dev/null +++ b/docs/metrics/prometheus/list.md @@ -0,0 +1,471 @@ +# Cluster Metrics + +MinIO collects the following metrics at the cluster level. +Metrics may include one or more labels, such as the server that calculated that metric. + +These metrics can be obtained from any MinIO server once per collection by using the following URL: + +```shell +https://HOSTNAME:PORT/minio/v2/metrics/cluster +``` + +Replace ``HOSTNAME:PORT`` with the hostname of your MinIO deployment. +For deployments behind a load balancer, use the load balancer hostname instead of a single node hostname. + +## Audit Metrics + +| Name | Description | +|:----------------------------------|:----------------------------------------------------------| +| `minio_audit_failed_messages` | Total number of messages that failed to send since start. | +| `minio_audit_target_queue_length` | Number of unsent messages in queue for target. | +| `minio_audit_total_messages` | Total number of messages sent since start. | + +## Cluster Capacity Metrics + +| Name | Description | +|:---------------------------------------------|:---------------------------------------------------------------| +| `minio_cluster_capacity_raw_free_bytes` | Total free capacity online in the cluster. | +| `minio_cluster_capacity_raw_total_bytes` | Total capacity online in the cluster. | +| `minio_cluster_capacity_usable_free_bytes` | Total free usable capacity online in the cluster. | +| `minio_cluster_capacity_usable_total_bytes` | Total usable capacity online in the cluster. | +| `minio_cluster_objects_size_distribution` | Distribution of object sizes across a cluster | +| `minio_cluster_objects_version_distribution` | Distribution of object versions across a cluster | +| `minio_cluster_usage_object_total` | Total number of objects in a cluster | +| `minio_cluster_usage_total_bytes` | Total cluster usage in bytes | +| `minio_cluster_usage_version_total` | Total number of versions (includes delete marker) in a cluster | +| `minio_cluster_usage_deletemarker_total` | Total number of delete markers in a cluster | +| `minio_cluster_bucket_total` | Total number of buckets in the cluster | + +## Cluster Drive Metrics + +| Name | Description | +|:------------------------------------|:--------------------------------------| +| `minio_cluster_drive_offline_total` | Total drives offline in this cluster. | +| `minio_cluster_drive_online_total` | Total drives online in this cluster. | +| `minio_cluster_drive_total` | Total drives in this cluster. | + +## Cluster ILM Metrics + +| Name | Description | +|:------------------------------------------|:-------------------------------------------------| +| `minio_cluster_ilm_transitioned_bytes` | Total bytes transitioned to a tier. | +| `minio_cluster_ilm_transitioned_objects` | Total number of objects transitioned to a tier. | +| `minio_cluster_ilm_transitioned_versions` | Total number of versions transitioned to a tier. | + +## Cluster KMS Metrics + +| Name | Description | +|:------------------------------------|:-----------------------------------------------------------------------------------------| +| `minio_cluster_kms_online` | Reports whether the KMS is online (1) or offline (0). | +| `minio_cluster_kms_request_error` | Number of KMS requests that failed due to some error. (HTTP 4xx status code). | +| `minio_cluster_kms_request_failure` | Number of KMS requests that failed due to some internal failure. (HTTP 5xx status code). | +| `minio_cluster_kms_request_success` | Number of KMS requests that succeeded. | +| `minio_cluster_kms_uptime` | The time the KMS has been up and running in seconds. | + +## Cluster Health Metrics + +| Name | Description | +|:--------------------------------------------------|:-----------------------------------------------| +| `minio_cluster_nodes_offline_total` | Total number of MinIO nodes offline. | +| `minio_cluster_nodes_online_total` | Total number of MinIO nodes online. | +| `minio_cluster_write_quorum` | Maximum write quorum across all pools and sets | +| `minio_cluster_health_status` | Get current cluster health status | +| `minio_cluster_health_erasure_set_healing_drives` | Count of healing drives in the erasure set | +| `minio_cluster_health_erasure_set_online_drives` | Count of online drives in the erasure set | +| `minio_cluster_health_erasure_set_read_quorum` | Get read quorum of the erasure set | +| `minio_cluster_health_erasure_set_write_quorum` | Get write quorum of the erasure set | +| `minio_cluster_health_erasure_set_status` | Get current health status of the erasure set | + +## Cluster Replication Metrics + +Metrics marked as ``Site Replication Only`` only populate on deployments with [Site Replication](https://docs.min.io/community/minio-object-store/operations/install-deploy-manage/multi-site-replication.html) configurations. +For deployments with [bucket](https://docs.min.io/community/minio-object-store/administration/bucket-replication.html) or [batch](https://docs.min.io/community/minio-object-store/administration/batch-framework.html#replicate) configurations, these metrics populate instead under the [Bucket Metrics](#bucket-metrics) endpoint. + +| Name | Description +|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------| +| `minio_cluster_replication_last_hour_failed_bytes` | (_Site Replication Only_) Total number of bytes failed at least once to replicate in the last full hour. | +| `minio_cluster_replication_last_hour_failed_count` | (_Site Replication Only_) Total number of objects which failed replication in the last full hour. | +| `minio_cluster_replication_last_minute_failed_bytes` | Total number of bytes failed at least once to replicate in the last full minute. | +| `minio_cluster_replication_last_minute_failed_count` | Total number of objects which failed replication in the last full minute. | +| `minio_cluster_replication_total_failed_bytes` | (_Site Replication Only_) Total number of bytes failed at least once to replicate since server start. | +| `minio_cluster_replication_total_failed_count` | (_Site Replication Only_) Total number of objects which failed replication since server start. | +| `minio_cluster_replication_received_bytes` | (_Site Replication Only_) Total number of bytes replicated to this cluster from another source cluster. | +| `minio_cluster_replication_received_count` | (_Site Replication Only_) Total number of objects received by this cluster from another source cluster. | +| `minio_cluster_replication_sent_bytes` | (_Site Replication Only_) Total number of bytes replicated to the target cluster. | +| `minio_cluster_replication_sent_count` | (_Site Replication Only_) Total number of objects replicated to the target cluster. | +| `minio_cluster_replication_credential_errors` | (_Site Replication Only_) Total number of replication credential errors since server start | +| `minio_cluster_replication_proxied_get_requests_total` | (_Site Replication Only_)Number of GET requests proxied to replication target | +| `minio_cluster_replication_proxied_head_requests_total` | (_Site Replication Only_)Number of HEAD requests proxied to replication target | +| `minio_cluster_replication_proxied_delete_tagging_requests_total` | (_Site Replication Only_)Number of DELETE tagging requests proxied to replication target | +| `minio_cluster_replication_proxied_get_tagging_requests_total` | (_Site Replication Only_)Number of GET tagging requests proxied to replication target | +| `minio_cluster_replication_proxied_put_tagging_requests_total` | (_Site Replication Only_)Number of PUT tagging requests proxied to replication target | +| `minio_cluster_replication_proxied_get_requests_failures` | (_Site Replication Only_)Number of failures in GET requests proxied to replication target | +| `minio_cluster_replication_proxied_head_requests_failures` | (_Site Replication Only_)Number of failures in HEAD requests proxied to replication target | +| `minio_cluster_replication_proxied_delete_tagging_requests_failures` | (_Site Replication Only_)Number of failures proxying DELETE tagging requests to replication target | +| `minio_cluster_replication_proxied_get_tagging_requests_failures` | (_Site Replication Only_)Number of failures proxying GET tagging requests to replication target | +| `minio_cluster_replication_proxied_put_tagging_requests_failures` | (_Site Replication Only_)Number of failures proxying PUT tagging requests to replication target | + + +## Node Replication Metrics + +Metrics marked as ``Site Replication Only`` only populate on deployments with [Site Replication](https://docs.min.io/community/minio-object-store/operations/install-deploy-manage/multi-site-replication.html) configurations. +For deployments with [bucket](https://docs.min.io/community/minio-object-store/administration/bucket-replication.html) or [batch](https://docs.min.io/community/minio-object-store/administration/batch-framework.html#replicate) configurations, these metrics populate instead under the [Bucket Metrics](#bucket-metrics) endpoint. + +| Name | Description +|:-----------------------------------------------------------|:---------------------------------------------------------------------------------------------------------| +| `minio_node_replication_current_active_workers` | Total number of active replication workers | +| `minio_node_replication_average_active_workers` | Average number of active replication workers | +| `minio_node_replication_max_active_workers` | Maximum number of active replication workers seen since server start | +| `minio_node_replication_link_online` | Reports whether the replication link is online (1) or offline (0). | +| `minio_node_replication_link_offline_duration_seconds` | Total duration of replication link being offline in seconds since last offline event | +| `minio_node_replication_link_downtime_duration_seconds` | Total downtime of replication link in seconds since server start | +| `minio_node_replication_average_link_latency_ms` | Average replication link latency in milliseconds | +| `minio_node_replication_max_link_latency_ms` | Maximum replication link latency in milliseconds seen since server start | +| `minio_node_replication_current_link_latency_ms` | Current replication link latency in milliseconds | +| `minio_node_replication_current_transfer_rate` | Current replication transfer rate in bytes/sec | +| `minio_node_replication_average_transfer_rate` | Average replication transfer rate in bytes/sec | +| `minio_node_replication_max_transfer_rate` | Maximum replication transfer rate in bytes/sec seen since server start | +| `minio_node_replication_last_minute_queued_count` | Total number of objects queued for replication in the last full minute | +| `minio_node_replication_last_minute_queued_bytes` | Total number of bytes queued for replication in the last full minute | +| `minio_node_replication_average_queued_count` | Average number of objects queued for replication since server start | +| `minio_node_replication_average_queued_bytes` | Average number of bytes queued for replication since server start | +| `minio_node_replication_max_queued_bytes` | Maximum number of bytes queued for replication seen since server start | +| `minio_node_replication_max_queued_count` | Maximum number of objects queued for replication seen since server start | +| `minio_node_replication_recent_backlog_count` | Total number of objects seen in replication backlog in the last 5 minutes | + +## Healing Metrics + +| Name | Description | +|:---------------------------------------------|:-----------------------------------------------------------------| +| `minio_heal_objects_errors_total` | Objects for which healing failed in current self healing run. | +| `minio_heal_objects_heal_total` | Objects healed in current self healing run. | +| `minio_heal_objects_total` | Objects scanned in current self healing run. | +| `minio_heal_time_last_activity_nano_seconds` | Time elapsed (in nano seconds) since last self healing activity. | + +## Inter Node Metrics + +| Name | Description | +|:------------------------------------------|:--------------------------------------------------------| +| `minio_inter_node_traffic_dial_avg_time` | Average time of internodes TCP dial calls. | +| `minio_inter_node_traffic_dial_errors` | Total number of internode TCP dial timeouts and errors. | +| `minio_inter_node_traffic_errors_total` | Total number of failed internode calls. | +| `minio_inter_node_traffic_received_bytes` | Total number of bytes received from other peer nodes. | +| `minio_inter_node_traffic_sent_bytes` | Total number of bytes sent to the other peer nodes. | + +## Bucket Notification Metrics + +| Name | Description | +|:-----------------------------------------------|:--------------------------------------------------------------------------------------------------------------------------------------------| +| `minio_notify_current_send_in_progress` | Number of concurrent async Send calls active to all targets (deprecated, please use `minio_notify_target_current_send_in_progress` instead) | +| `minio_notify_events_errors_total` | Events that were failed to be sent to the targets (deprecated, please use `minio_notify_target_failed_events` instead) | +| `minio_notify_events_sent_total` | Total number of events sent to the targets (deprecated, please use `minio_notify_target_total_events` instead) | +| `minio_notify_events_skipped_total` | Events that were skipped to be sent to the targets due to the in-memory queue being full | +| `minio_notify_target_current_send_in_progress` | Number of concurrent async Send calls active to the target | +| `minio_notify_target_queue_length` | Number of events currently staged in the queue_dir configured for the target. | +| `minio_notify_target_total_events` | Total number of events sent (or) queued to the target | + +## S3 API Request Metrics + +| Name | Description | +|:----------------------------------------------|:---------------------------------------------------------| +| `minio_s3_requests_4xx_errors_total` | Total number S3 requests with (4xx) errors. | +| `minio_s3_requests_5xx_errors_total` | Total number S3 requests with (5xx) errors. | +| `minio_s3_requests_canceled_total` | Total number S3 requests canceled by the client. | +| `minio_s3_requests_errors_total` | Total number S3 requests with (4xx and 5xx) errors. | +| `minio_s3_requests_incoming_total` | Volatile number of total incoming S3 requests. | +| `minio_s3_requests_inflight_total` | Total number of S3 requests currently in flight. | +| `minio_s3_requests_rejected_auth_total` | Total number S3 requests rejected for auth failure. | +| `minio_s3_requests_rejected_header_total` | Total number S3 requests rejected for invalid header. | +| `minio_s3_requests_rejected_invalid_total` | Total number S3 invalid requests. | +| `minio_s3_requests_rejected_timestamp_total` | Total number S3 requests rejected for invalid timestamp. | +| `minio_s3_requests_total` | Total number S3 requests. | +| `minio_s3_requests_waiting_total` | Number of S3 requests in the waiting queue. | +| `minio_s3_requests_ttfb_seconds_distribution` | Distribution of the time to first byte across API calls. | +| `minio_s3_traffic_received_bytes` | Total number of s3 bytes received. | +| `minio_s3_traffic_sent_bytes` | Total number of s3 bytes sent. | + +## Software Metrics + +| Name | Description | +|:------------------------------|:---------------------------------------| +| `minio_software_commit_info` | Git commit hash for the MinIO release. | +| `minio_software_version_info` | MinIO Release tag for the server. | + +## Drive Metrics + +| Name | Description | +|:---------------------------------------|:--------------------------------------------------------------------| +| `minio_node_drive_free_bytes` | Total storage available on a drive. | +| `minio_node_drive_free_inodes` | Total free inodes. | +| `minio_node_drive_latency_us` | Average last minute latency in µs for drive API storage operations. | +| `minio_node_drive_offline_total` | Total drives offline in this node. | +| `minio_node_drive_online_total` | Total drives online in this node. | +| `minio_node_drive_total` | Total drives in this node. | +| `minio_node_drive_total_bytes` | Total storage on a drive. | +| `minio_node_drive_used_bytes` | Total storage used on a drive. | +| `minio_node_drive_errors_timeout` | Total number of drive timeout errors since server start | +| `minio_node_drive_errors_ioerror` | Total number of drive I/O errors since server start | +| `minio_node_drive_errors_availability` | Total number of drive I/O errors, timeouts since server start | +| `minio_node_drive_io_waiting` | Total number I/O operations waiting on drive | + +## Identity and Access Management (IAM) Metrics + +| Name | Description | +|:-------------------------------------------|:------------------------------------------------------------| +| `minio_node_iam_last_sync_duration_millis` | Last successful IAM data sync duration in milliseconds. | +| `minio_node_iam_since_last_sync_millis` | Time (in milliseconds) since last successful IAM data sync. | +| `minio_node_iam_sync_failures` | Number of failed IAM data syncs since server start. | +| `minio_node_iam_sync_successes` | Number of successful IAM data syncs since server start. | + +## Information Lifecycle Management (ILM) Metrics + +| Name | Description | +|:-------------------------------------------------------------|:-----------------------------------------------------------------------------------------------------------| +| `minio_node_ilm_expiry_pending_tasks` | Number of pending ILM expiry tasks in the queue. | +| `minio_node_ilm_transition_active_tasks` | Number of active ILM transition tasks. | +| `minio_node_ilm_transition_pending_tasks` | Number of pending ILM transition tasks in the queue. | +| `minio_node_ilm_transition_missed_immediate_tasks` | Number of missed immediate ILM transition tasks. | +| `minio_node_ilm_versions_scanned` | Total number of object versions checked for ilm actions since server start. | +| `minio_node_ilm_action_count_delete_action` | Total action outcome of lifecycle checks since server start for deleting object | +| `minio_node_ilm_action_count_delete_version_action` | Total action outcome of lifecycle checks since server start for deleting a version | +| `minio_node_ilm_action_count_transition_action` | Total action outcome of lifecycle checks since server start for transition of an object | +| `minio_node_ilm_action_count_transition_version_action` | Total action outcome of lifecycle checks since server start for transition of a particular object version | +| `minio_node_ilm_action_count_delete_restored_action` | Total action outcome of lifecycle checks since server start for deletion of temporarily restored object | +| `minio_node_ilm_action_count_delete_restored_version_action` | Total action outcome of lifecycle checks since server start for deletion of a temporarily restored version | +| `minio_node_ilm_action_count_delete_all_versions_action` | Total action outcome of lifecycle checks since server start for deletion of all versions | + +## Tier Metrics + +| Name | Description | +|:---------------------------------------------------|:----------------------------------------------------------------------------| +| `minio_node_tier_tier_ttlb_seconds_distribution` | Distribution of time to last byte for objects downloaded from warm tier | +| `minio_node_tier_requests_success` | Number of requests to download object from warm tier that were successful | +| `minio_node_tier_requests_failure` | Number of requests to download object from warm tier that were failure | + +## System Metrics + +| Name | Description | +|:-------------------------------------------|:----------------------------------------------------------------------------------------------------------------| +| `minio_node_file_descriptor_limit_total` | Limit on total number of open file descriptors for the MinIO Server process. | +| `minio_node_file_descriptor_open_total` | Total number of open file descriptors by the MinIO Server process. | +| `minio_node_go_routine_total` | Total number of go routines running. | +| `minio_node_io_rchar_bytes` | Total bytes read by the process from the underlying storage system including cache, /proc/[pid]/io rchar. | +| `minio_node_io_read_bytes` | Total bytes read by the process from the underlying storage system, /proc/[pid]/io read_bytes. | +| `minio_node_io_wchar_bytes` | Total bytes written by the process to the underlying storage system including page cache, /proc/[pid]/io wchar. | +| `minio_node_io_write_bytes` | Total bytes written by the process to the underlying storage system, /proc/[pid]/io write_bytes. | +| `minio_node_process_cpu_total_seconds` | Total user and system CPU time spent in seconds by the process. | +| `minio_node_process_resident_memory_bytes` | Resident memory size in bytes. | +| `minio_node_process_virtual_memory_bytes` | Virtual memory size in bytes. | +| `minio_node_process_starttime_seconds` | Start time for MinIO process per node, time in seconds since Unix epoc. | +| `minio_node_process_uptime_seconds` | Uptime for MinIO process per node in seconds. | + +## Scanner Metrics + +| Name | Description | +|:-------------------------------------------|:------------------------------------------------------------| +| `minio_node_scanner_bucket_scans_finished` | Total number of bucket scans finished since server start. | +| `minio_node_scanner_bucket_scans_started` | Total number of bucket scans started since server start. | +| `minio_node_scanner_directories_scanned` | Total number of directories scanned since server start. | +| `minio_node_scanner_objects_scanned` | Total number of unique objects scanned since server start. | +| `minio_node_scanner_versions_scanned` | Total number of object versions scanned since server start. | +| `minio_node_syscall_read_total` | Total read SysCalls to the kernel. /proc/[pid]/io syscr. | +| `minio_node_syscall_write_total` | Total write SysCalls to the kernel. /proc/[pid]/io syscw. | +| `minio_usage_last_activity_nano_seconds` | Time elapsed (in nano seconds) since last scan activity. | + +# Bucket Metrics + +MinIO collects the following metrics at the bucket level. +Each metric includes the ``bucket`` label to identify the corresponding bucket. +Metrics may include one or more additional labels, such as the server that calculated that metric. + +These metrics can be obtained from any MinIO server once per collection by using the following URL: + +```shell +https://HOSTNAME:PORT/minio/v2/metrics/bucket +``` + +Replace ``HOSTNAME:PORT`` with the hostname of your MinIO deployment. +For deployments behind a load balancer, use the load balancer hostname instead of a single node hostname. + +## Distribution Metrics + +| Name | Description | +|:--------------------------------------------|:--------------------------------------------------------------------------------| +| `minio_bucket_objects_size_distribution` | Distribution of object sizes in the bucket, includes label for the bucket name. | +| `minio_bucket_objects_version_distribution` | Distribution of object sizes in a bucket, by number of versions | + +## Replication Metrics + +These metrics only populate on deployments with [Bucket Replication](https://docs.min.io/community/minio-object-store/administration/bucket-replication.html) or [Batch Replication](https://docs.min.io/community/minio-object-store/administration/batch-framework.html) configurations. +For deployments with [Site Replication](https://docs.min.io/community/minio-object-store/operations/install-deploy-manage/multi-site-replication.html) configured, select metrics populate under the [Cluster Metrics](#cluster-metrics) endpoint. + +| Name | Description | +|:----------------------------------------------------|:---------------------------------------------------------------------------------| +| `minio_bucket_replication_last_minute_failed_bytes` | Total number of bytes failed at least once to replicate in the last full minute. | +| `minio_bucket_replication_last_minute_failed_count` | Total number of objects which failed replication in the last full minute. | +| `minio_bucket_replication_last_hour_failed_bytes` | Total number of bytes failed at least once to replicate in the last full hour. | +| `minio_bucket_replication_last_hour_failed_count` | Total number of objects which failed replication in the last full hour. | +| `minio_bucket_replication_total_failed_bytes` | Total number of bytes failed at least once to replicate since server start. | +| `minio_bucket_replication_total_failed_count` | Total number of objects which failed replication since server start. | +| `minio_bucket_replication_latency_ms` | Replication latency in milliseconds. | +| `minio_bucket_replication_received_bytes` | Total number of bytes replicated to this bucket from another source bucket. | +| `minio_bucket_replication_received_count` | Total number of objects received by this bucket from another source bucket. | +| `minio_bucket_replication_sent_bytes` | Total number of bytes replicated to the target bucket. | +| `minio_bucket_replication_sent_count` | Total number of objects replicated to the target bucket. | +| `minio_bucket_replication_credential_errors` | Total number of replication credential errors since server start | +| `minio_bucket_replication_proxied_get_requests_total` | Number of GET requests proxied to replication target | +| `minio_bucket_replication_proxied_head_requests_total` | Number of HEAD requests proxied to replication target | +| `minio_bucket_replication_proxied_delete_tagging_requests_total` | Number of DELETE tagging requests proxied to replication target | +| `minio_bucket_replication_proxied_get_tagging_requests_total` | Number of GET tagging requests proxied to replication target | +| `minio_bucket_replication_proxied_put_tagging_requests_total` | Number of PUT tagging requests proxied to replication target | +| `minio_bucket_replication_proxied_get_requests_failures` | Number of failures in GET requests proxied to replication target | +| `minio_bucket_replication_proxied_head_requests_failures` | Number of failures in HEAD requests proxied to replication target | +| `minio_bucket_replication_proxied_delete_tagging_requests_failures` | Number of failures in DELETE tagging proxy requests to replication target | +| `minio_bucket_replication_proxied_get_tagging_requests_failures` |Number of failures in GET tagging proxy requests to replication target | +| `minio_bucket_replication_proxied_put_tagging_requests_failures` | Number of failures in PUT tagging proxy requests to replication target | + +## Traffic Metrics + +| Name | Description | +|:--------------------------------------|:---------------------------------------------------| +| `minio_bucket_traffic_received_bytes` | Total number of S3 bytes received for this bucket. | +| `minio_bucket_traffic_sent_bytes` | Total number of S3 bytes sent for this bucket. | + +## Usage Metrics + +| Name | Description | +|:----------------------------------------|:--------------------------------------------------| +| `minio_bucket_usage_object_total` | Total number of objects. | +| `minio_bucket_usage_version_total` | Total number of versions (includes delete marker) | +| `minio_bucket_usage_deletemarker_total` | Total number of delete markers. | +| `minio_bucket_usage_total_bytes` | Total bucket size in bytes. | +| `minio_bucket_quota_total_bytes` | Total bucket quota size in bytes. | + +## Requests Metrics + +| Name | Description | +|:--------------------------------------------------|:----------------------------------------------------------------| +| `minio_bucket_requests_4xx_errors_total` | Total number of S3 requests with (4xx) errors on a bucket. | +| `minio_bucket_requests_5xx_errors_total` | Total number of S3 requests with (5xx) errors on a bucket. | +| `minio_bucket_requests_inflight_total` | Total number of S3 requests currently in flight on a bucket. | +| `minio_bucket_requests_total` | Total number of S3 requests on a bucket. | +| `minio_bucket_requests_canceled_total` | Total number S3 requests canceled by the client. | +| `minio_bucket_requests_ttfb_seconds_distribution` | Distribution of time to first byte across API calls per bucket. | + +# Resource Metrics + +MinIO collects the following resource metrics at the node level. +Each metric includes the `server` label to identify the corresponding node. +Metrics may include one or more additional labels, such as the drive path, interface name, etc. + +These metrics can be obtained from any MinIO server once per collection by using the following URL: + +```shell +https://HOSTNAME:PORT/minio/v2/metrics/resource +``` + +Replace `HOSTNAME:PORT` with the hostname of your MinIO deployment. +For deployments behind a load balancer, use the load balancer hostname instead of a single node hostname. + +## Drive Resource Metrics + +| Name | Description | +|:-------------------------------------|:---------------------------------------------------------| +| `minio_node_drive_total_bytes` | Total bytes on a drive. | +| `minio_node_drive_used_bytes` | Used bytes on a drive. | +| `minio_node_drive_total_inodes` | Total inodes on a drive. | +| `minio_node_drive_used_inodes` | Total inodes used on a drive. | +| `minio_node_drive_reads_per_sec` | Reads per second on a drive. | +| `minio_node_drive_reads_kb_per_sec` | Kilobytes read per second on a drive. | +| `minio_node_drive_reads_await` | Average time for read requests to be served on a drive. | +| `minio_node_drive_writes_per_sec` | Writes per second on a drive. | +| `minio_node_drive_writes_kb_per_sec` | Kilobytes written per second on a drive. | +| `minio_node_drive_writes_await` | Average time for write requests to be served on a drive. | +| `minio_node_drive_perc_util` | Percentage of time the disk was busy since uptime. | + +## Network Interface Metrics + +| Name | Description | +|:------------------------------|:-----------------------------------------------------------| +| `minio_node_if_rx_bytes` | Bytes received on the interface in 60s. | +| `minio_node_if_rx_bytes_avg` | Bytes received on the interface in 60s (avg) since uptime. | +| `minio_node_if_rx_bytes_max` | Bytes received on the interface in 60s (max) since uptime. | +| `minio_node_if_rx_errors` | Receive errors in 60s. | +| `minio_node_if_rx_errors_avg` | Receive errors in 60s (avg). | +| `minio_node_if_rx_errors_max` | Receive errors in 60s (max). | +| `minio_node_if_tx_bytes` | Bytes transmitted in 60s. | +| `minio_node_if_tx_bytes_avg` | Bytes transmitted in 60s (avg). | +| `minio_node_if_tx_bytes_max` | Bytes transmitted in 60s (max). | +| `minio_node_if_tx_errors` | Transmit errors in 60s. | +| `minio_node_if_tx_errors_avg` | Transmit errors in 60s (avg). | +| `minio_node_if_tx_errors_max` | Transmit errors in 60s (max). | + +## CPU Metrics + +| Name | Description | +|:-------------------------------------|:-------------------------------------------| +| `minio_node_cpu_avg_user` | CPU user time. | +| `minio_node_cpu_avg_user_avg` | CPU user time (avg). | +| `minio_node_cpu_avg_user_max` | CPU user time (max). | +| `minio_node_cpu_avg_system` | CPU system time. | +| `minio_node_cpu_avg_system_avg` | CPU system time (avg). | +| `minio_node_cpu_avg_system_max` | CPU system time (max). | +| `minio_node_cpu_avg_idle` | CPU idle time. | +| `minio_node_cpu_avg_idle_avg` | CPU idle time (avg). | +| `minio_node_cpu_avg_idle_max` | CPU idle time (max). | +| `minio_node_cpu_avg_iowait` | CPU ioWait time. | +| `minio_node_cpu_avg_iowait_avg` | CPU ioWait time (avg). | +| `minio_node_cpu_avg_iowait_max` | CPU ioWait time (max). | +| `minio_node_cpu_avg_nice` | CPU nice time. | +| `minio_node_cpu_avg_nice_avg` | CPU nice time (avg). | +| `minio_node_cpu_avg_nice_max` | CPU nice time (max). | +| `minio_node_cpu_avg_steal` | CPU steam time. | +| `minio_node_cpu_avg_steal_avg` | CPU steam time (avg). | +| `minio_node_cpu_avg_steal_max` | CPU steam time (max). | +| `minio_node_cpu_avg_load1` | CPU load average 1min. | +| `minio_node_cpu_avg_load1_avg` | CPU load average 1min (avg). | +| `minio_node_cpu_avg_load1_max` | CPU load average 1min (max). | +| `minio_node_cpu_avg_load1_perc` | CPU load average 1min (percentage). | +| `minio_node_cpu_avg_load1_perc_avg` | CPU load average 1min (percentage) (avg). | +| `minio_node_cpu_avg_load1_perc_max` | CPU load average 1min (percentage) (max). | +| `minio_node_cpu_avg_load5` | CPU load average 5min. | +| `minio_node_cpu_avg_load5_avg` | CPU load average 5min (avg). | +| `minio_node_cpu_avg_load5_max` | CPU load average 5min (max). | +| `minio_node_cpu_avg_load5_perc` | CPU load average 5min (percentage). | +| `minio_node_cpu_avg_load5_perc_avg` | CPU load average 5min (percentage) (avg). | +| `minio_node_cpu_avg_load5_perc_max` | CPU load average 5min (percentage) (max). | +| `minio_node_cpu_avg_load15` | CPU load average 15min. | +| `minio_node_cpu_avg_load15_avg` | CPU load average 15min (avg). | +| `minio_node_cpu_avg_load15_max` | CPU load average 15min (max). | +| `minio_node_cpu_avg_load15_perc` | CPU load average 15min (percentage). | +| `minio_node_cpu_avg_load15_perc_avg` | CPU load average 15min (percentage) (avg). | +| `minio_node_cpu_avg_load15_perc_max` | CPU load average 15min (percentage) (max). | + +## Memory Metrics + +| Name | Description | +|:-------------------------------|:------------------------------------------| +| `minio_node_mem_available` | Available memory on the node. | +| `minio_node_mem_available_avg` | Available memory on the node (avg). | +| `minio_node_mem_available_max` | Available memory on the node (max). | +| `minio_node_mem_buffers` | Buffers memory on the node. | +| `minio_node_mem_buffers_avg` | Buffers memory on the node (avg). | +| `minio_node_mem_buffers_max` | Buffers memory on the node (max). | +| `minio_node_mem_cache` | Cache memory on the node. | +| `minio_node_mem_cache_avg` | Cache memory on the node (avg). | +| `minio_node_mem_cache_max` | Cache memory on the node (max). | +| `minio_node_mem_free` | Free memory on the node. | +| `minio_node_mem_free_avg` | Free memory on the node (avg). | +| `minio_node_mem_free_max` | Free memory on the node (max). | +| `minio_node_mem_shared` | Shared memory on the node. | +| `minio_node_mem_shared_avg` | Shared memory on the node (avg). | +| `minio_node_mem_shared_max` | Shared memory on the node (max). | +| `minio_node_mem_total` | Total memory on the node. | +| `minio_node_mem_total_avg` | Total memory on the node (avg). | +| `minio_node_mem_total_max` | Total memory on the node (max). | +| `minio_node_mem_used` | Used memory on the node. | +| `minio_node_mem_used_avg` | Used memory on the node (avg). | +| `minio_node_mem_used_max` | Used memory on the node (max). | +| `minio_node_mem_used_perc` | Used memory percentage on the node. | +| `minio_node_mem_used_perc_avg` | Used memory percentage on the node (avg). | +| `minio_node_mem_used_perc_max` | Used memory percentage on the node (max). | diff --git a/docs/metrics/prometheus/minio-es-tolerance-alert.png b/docs/metrics/prometheus/minio-es-tolerance-alert.png new file mode 100644 index 00000000..0994dd7a Binary files /dev/null and b/docs/metrics/prometheus/minio-es-tolerance-alert.png differ diff --git a/docs/metrics/v3.md b/docs/metrics/v3.md new file mode 100644 index 00000000..050bf092 --- /dev/null +++ b/docs/metrics/v3.md @@ -0,0 +1,417 @@ +# Metrics Version 3 + +In metrics version 3, all metrics are available under the following base endpoint: + +``` +/minio/metrics/v3 +``` + +To query metrics of a specific type, append the appropriate path to the base endpoint. +Querying the base endpoint returns "404 Not Found." + +Metrics are organized into groups at paths **relative** to the top-level endpoint above. + +Metrics are also available using the [MinIO Admin Client](https://docs.min.io/community/minio-object-store/reference/minio-mc-admin.html) and the `mc admin prometheus metrics` command. For more information, see [Metrics and Alerts](https://docs.min.io/community/minio-object-store/operations/monitoring/metrics-and-alerts.html) in the MinIO Documentation. + +## Metrics Request Handling + +Each endpoint can be queried as needed via a scrape configuration in Prometheus or a compatible metrics collection tool. You should schedule scrape operations so a prior scrape completes before the next one begins. + +For ease of configuration, each (non-empty) parent of the path serves all the metric endpoints at its child paths. For example, to query all system metrics scrape `/minio/metrics/v3/system/`. + +Each metric endpoint may support multiple child endpoints. For example, the `/v3/system/` metric has multiple child groups as `/v3/system/`. Querying the parent endpoint returns metrics for all child groups. Querying a child group returns only metrics for that child. + +### Per-bucket Metrics + +Metrics with a `/bucket` component in the path return results for each specified bucket in the deployment. These endpoints **require** providing a list of buckets as a `bucket` query parameter. The endpoint then returns only metrics for the given buckets, with the bucket name in a `bucket` label. + +For example, to query API metrics for buckets `test1` and `test2`, make a scrape request to `/minio/metrics/v3/api/bucket?buckets=test1,test2`. + +### List Available Metrics + +Instead of a metrics scrape, you can list the metrics that would be returned by a path by adding a `list` query parameter. The MinIO server then lists all available metrics that could be returned. Note that during an actual metrics scrape only metrics with available _values_ are returned. Metrics with null values are omitted from the scrape results. + +To set the output format, set the request `Content-Type` to the desired format. Accepted values are `application/json` for JSON output or `text/plain` for a Markdown-formatted table. The default is Markdown. + +For example, the following returns a list of all available bucket metrics: +``` +/minio/metrics/v3/api/bucket?list +``` + +## Metric Categories + +At a high level, metrics are grouped into categories as described in the following sections. The path in each of the tables is relative to the top-level endpoint. Note that the standard GoCollector metrics are not shown. + +### Request metrics + +Metrics about requests served by the current node. + +| Path | Description | +|-----------------|-----------------------------------------------| +| `/api/requests` | Metrics over all requests. | +| `/bucket/api` | Metrics over all requests for a given bucket. | + +#### `/api/requests` + +| Name | Description | Labels | +|------------------------------------------------|--------------------------------------------------------------------------------|----------------------------------------------| +| `minio_api_requests_rejected_auth_total` | Total number of requests rejected for auth failure.

Type: counter | `type`, `pool_index`, `server` | +| `minio_api_requests_rejected_header_total` | Total number of requests rejected for invalid header.

Type: counter | `type`, `pool_index`, `server` | +| `minio_api_requests_rejected_timestamp_total` | Total number of requests rejected for invalid timestamp.

Type: counter | `type`, `pool_index`, `server` | +| `minio_api_requests_rejected_invalid_total` | Total number of invalid requests.

Type: counter | `type`, `pool_index`, `server` | +| `minio_api_requests_waiting_total` | Total number of requests in the waiting queue.

Type: gauge | `type`, `pool_index`, `server` | +| `minio_api_requests_incoming_total` | Total number of incoming requests.

Type: gauge | `type`, `pool_index`, `server` | +| `minio_api_requests_inflight_total` | Total number of requests currently in flight.

Type: gauge | `name`, `type`, `pool_index`, `server` | +| `minio_api_requests_total` | Total number of requests.

Type: counter | `name`, `type`, `pool_index`, `server` | +| `minio_api_requests_errors_total` | Total number of requests with 4xx or 5xx errors.

Type: counter | `name`, `type`, `pool_index`, `server` | +| `minio_api_requests_5xx_errors_total` | Total number of requests with 5xx errors.

Type: counter | `name`, `type`, `pool_index`, `server` | +| `minio_api_requests_4xx_errors_total` | Total number of requests with 4xx errors.

Type: counter | `name`, `type`, `pool_index`, `server` | +| `minio_api_requests_canceled_total` | Total number of requests canceled by the client.

Type: counter | `name`, `type`, `pool_index`, `server` | +| `minio_api_requests_ttfb_seconds_distribution` | Distribution of time to first byte across API calls.

Type: counter | `name`, `type`, `le`, `pool_index`, `server` | +| `minio_api_requests_traffic_sent_bytes` | Total number of bytes sent.

Type: counter | `type`, `pool_index`, `server` | +| `minio_api_requests_traffic_received_bytes` | Total number of bytes received.

Type: counter | `type`, `pool_index`, `server` | + +#### `/bucket/api` + +| Name | Description | Labels | +|----------------------------------------------|-----------------------------------------------------------------------------------------|--------------------------------------------------------| +| `minio_bucket_api_traffic_received_bytes` | Total number of bytes sent for a bucket.

Type: counter | `bucket`, `type`, `server`, `pool_index` | +| `minio_bucket_api_traffic_sent_bytes` | Total number of bytes received for a bucket.

Type: counter | `bucket`, `type`, `server`, `pool_index` | +| `minio_bucket_api_inflight_total` | Total number of requests currently in flight for a bucket.

Type: gauge | `bucket`, `name`, `type`, `server`, `pool_index` | +| `minio_bucket_api_total` | Total number of requests for a bucket.

Type: counter | `bucket`, `name`, `type`, `server`, `pool_index` | +| `minio_bucket_api_canceled_total` | Total number of requests canceled by the client for a bucket.

Type: counter | `bucket`, `name`, `type`, `server`, `pool_index` | +| `minio_bucket_api_4xx_errors_total` | Total number of requests with 4xx errors for a bucket.

Type: counter | `bucket`, `name`, `type`, `server`, `pool_index` | +| `minio_bucket_api_5xx_errors_total` | Total number of requests with 5xx errors for a bucket.

Type: counter | `bucket`, `name`, `type`, `server`, `pool_index` | +| `minio_bucket_api_ttfb_seconds_distribution` | Distribution of time to first byte across API calls for a bucket.

Type: counter | `bucket`, `name`, `le`, `type`, `server`, `pool_index` | + +### Audit metrics + +Metrics about the MinIO audit functionality. + +| Path | Description | +|----------|-----------------------------------------| +| `/audit` | Metrics related to audit functionality. | + +#### `/audit` + +| Name | Description | Labels | +|-----------------------------------|---------------------------------------------------------------------------------|-----------------------| +| `minio_audit_failed_messages` | Total number of messages that failed to send since start.

Type: counter | `target_id`, `server` | +| `minio_audit_target_queue_length` | Number of unsent messages in queue for target.

Type: gauge | `target_id`, `server` | +| `minio_audit_total_messages` | Total number of messages sent since start.

Type: counter | `target_id`, `server` | + +### Cluster metrics + +Metrics about an entire MinIO cluster. + +| Path | Description | +|--------------------------|--------------------------------| +| `/cluster/config` | Cluster configuration metrics. | +| `/cluster/erasure-set` | Erasure set metrics. | +| `/cluster/health` | Cluster health metrics. | +| `/cluster/iam` | Cluster iam metrics. | +| `/cluster/usage/buckets` | Object statistics by bucket. | +| `/cluster/usage/objects` | Object statistics. | + +#### `/cluster/config` + +| Name | Description | Labels | +|----------------------------------------|--------------------------------------------------------------|--------| +| `minio_cluster_config_rrs_parity` | Reduced redundancy storage class parity.

Type: gauge | | +| `minio_cluster_config_standard_parity` | Standard storage class parity.

Type: gauge | | + +#### `/cluster/erasure-set` + +| Name | Description | Labels | +|--------------------------------------------------|---------------------------------------------------------------------------------------------------------|---------------------| +| `minio_cluster_erasure_set_overall_write_quorum` | Overall write quorum across pools and sets.

Type: gauge | | +| `minio_cluster_erasure_set_overall_health` | Overall health across pools and sets (1=healthy, 0=unhealthy).

Type: gauge | | +| `minio_cluster_erasure_set_read_quorum` | Read quorum for the erasure set in a pool.

Type: gauge | `pool_id`, `set_id` | +| `minio_cluster_erasure_set_write_quorum` | Write quorum for the erasure set in a pool.

Type: gauge | `pool_id`, `set_id` | +| `minio_cluster_erasure_set_online_drives_count` | Count of online drives in the erasure set in a pool.

Type: gauge | `pool_id`, `set_id` | +| `minio_cluster_erasure_set_healing_drives_count` | Count of healing drives in the erasure set in a pool.

Type: gauge | `pool_id`, `set_id` | +| `minio_cluster_erasure_set_health` | Health of the erasure set in a pool (1=healthy, 0=unhealthy).

Type: gauge | `pool_id`, `set_id` | +| `minio_cluster_erasure_set_read_tolerance` | Number of drive failures that can be tolerated without disrupting read operations.

Type: gauge | `pool_id`, `set_id` | +| `minio_cluster_erasure_set_write_tolerance` | Number of drive failures that can be tolerated without disrupting write operations.

Type: gauge | `pool_id`, `set_id` | +| `minio_cluster_erasure_set_read_health` | Health of the erasure set in a pool for read operations (1=healthy, 0=unhealthy).

Type: gauge | `pool_id`, `set_id` | +| `minio_cluster_erasure_set_write_health` | Health of the erasure set in a pool for write operations (1=healthy, 0=unhealthy).

Type: gauge | `pool_id`, `set_id` | + +#### `/cluster/health` + +| Name | Description | Labels | +|----------------------------------------------------|---------------------------------------------------------------------|--------| +| `minio_cluster_health_drives_offline_count` | Count of offline drives in the cluster.

Type: gauge | | +| `minio_cluster_health_drives_online_count` | Count of online drives in the cluster.

Type: gauge | | +| `minio_cluster_health_drives_count` | Count of all drives in the cluster.

Type: gauge | | +| `minio_cluster_health_nodes_offline_count` | Count of offline nodes in the cluster.

Type: gauge | | +| `minio_cluster_health_nodes_online_count` | Count of online nodes in the cluster.

Type: gauge | | +| `minio_cluster_health_capacity_raw_total_bytes` | Total cluster raw storage capacity in bytes.

Type: gauge | | +| `minio_cluster_health_capacity_raw_free_bytes` | Total cluster raw storage free in bytes.

Type: gauge | | +| `minio_cluster_health_capacity_usable_total_bytes` | Total cluster usable storage capacity in bytes.

Type: gauge | | +| `minio_cluster_health_capacity_usable_free_bytes` | Total cluster usable storage free in bytes.

Type: gauge | | + +#### `/cluster/iam` + +| Name | Description | Labels | +|-----------------------------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------|--------| +| `minio_cluster_iam_last_sync_duration_millis` | Last successful IAM data sync duration in milliseconds.

Type: counter | | +| `minio_cluster_iam_plugin_authn_service_failed_requests_minute` | When plugin authentication is configured, returns failed requests count in the last full minute.

Type: counter | | +| `minio_cluster_iam_plugin_authn_service_last_fail_seconds` | When plugin authentication is configured, returns time (in seconds) since the last failed request to the service.

Type: counter | | +| `minio_cluster_iam_plugin_authn_service_last_succ_seconds` | When plugin authentication is configured, returns time (in seconds) since the last successful request to the service.

Type: counter | | +| `minio_cluster_iam_plugin_authn_service_succ_avg_rtt_ms_minute` | When plugin authentication is configured, returns average round-trip time of successful requests in the last full minute.

Type: counter | | +| `minio_cluster_iam_plugin_authn_service_succ_max_rtt_ms_minute` | When plugin authentication is configured, returns maximum round-trip time of successful requests in the last full minute.

Type: counter | | +| `minio_cluster_iam_plugin_authn_service_total_requests_minute` | When plugin authentication is configured, returns total requests count in the last full minute.

Type: counter | | +| `minio_cluster_iam_since_last_sync_millis` | Time (in milliseconds) since last successful IAM data sync.

Type: counter | | +| `minio_cluster_iam_sync_failures` | Number of failed IAM data syncs since server start.

Type: counter | | +| `minio_cluster_iam_sync_successes` | Number of successful IAM data syncs since server start.

Type: counter | | + +#### `/cluster/usage/buckets` + +| Name | Description | Labels | +|-----------------------------------------------------------------|--------------------------------------------------------------------------------------|-------------------| +| `minio_cluster_usage_buckets_since_last_update_seconds` | Time since last update of usage metrics in seconds.

Type: gauge | | +| `minio_cluster_usage_buckets_total_bytes` | Total bucket size in bytes.

Type: gauge | `bucket` | +| `minio_cluster_usage_buckets_objects_count` | Total object count in bucket.

Type: gauge | `bucket` | +| `minio_cluster_usage_buckets_versions_count` | Total object versions count in bucket, including delete markers.

Type: gauge | `bucket` | +| `minio_cluster_usage_buckets_delete_markers_count` | Total delete markers count in bucket.

Type: gauge | `bucket` | +| `minio_cluster_usage_buckets_quota_total_bytes` | Total bucket quota in bytes.

Type: gauge | `bucket` | +| `minio_cluster_usage_buckets_object_size_distribution` | Bucket object size distribution.

Type: gauge | `range`, `bucket` | +| `minio_cluster_usage_buckets_object_version_count_distribution` | Bucket object version count distribution.

Type: gauge | `range`, `bucket` | + +#### `/cluster/usage/objects` + +| Name | Description | Labels | +|----------------------------------------------------------|------------------------------------------------------------------------------------|---------| +| `minio_cluster_usage_objects_since_last_update_seconds` | Time since last update of usage metrics in seconds.

Type: gauge | | +| `minio_cluster_usage_objects_total_bytes` | Total cluster usage in bytes.

Type: gauge | | +| `minio_cluster_usage_objects_count` | Total cluster objects count.

Type: gauge | | +| `minio_cluster_usage_objects_versions_count` | Total cluster object versions count, including delete markers.

Type: gauge | | +| `minio_cluster_usage_objects_delete_markers_count` | Total cluster delete markers count.

Type: gauge | | +| `minio_cluster_usage_objects_buckets_count` | Total cluster buckets count.

Type: gauge | | +| `minio_cluster_usage_objects_size_distribution` | Cluster object size distribution.

Type: gauge | `range` | +| `minio_cluster_usage_objects_version_count_distribution` | Cluster object version count distribution.

Type: gauge | `range` | + +### Debug metrics + +Standard Go runtime metrics from the [Prometheus Go Client base collector](https://github.com/prometheus/client_golang). + +| Path | Description | +|-------------|---------------------| +| `/debug/go` | Go runtime metrics. | + +### ILM metrics + +Metrics about the MinIO ILM functionality. + +| Path | Description | +|--------|---------------------------------------| +| `/ilm` | Metrics related to ILM functionality. | + +#### `/ilm` + +| Name | Description | Labels | +|-------------------------------------------------------|---------------------------------------------------------------------------------------------------|----------| +| `minio_cluster_ilm_expiry_pending_tasks` | Number of pending ILM expiry tasks in the queue.

Type: gauge | `server` | +| `minio_cluster_ilm_transition_active_tasks` | Number of active ILM transition tasks.

Type: gauge | `server` | +| `minio_cluster_ilm_transition_pending_tasks` | Number of pending ILM transition tasks in the queue.

Type: gauge | `server` | +| `minio_cluster_ilm_transition_missed_immediate_tasks` | Number of missed immediate ILM transition tasks.

Type: counter | `server` | +| `minio_cluster_ilm_versions_scanned` | Total number of object versions checked for ILM actions since server start.

Type: counter | `server` | + +### Logger webhook metrics + +Metrics about MinIO logger webhooks. + +| Path | Description | +|-------------------|-------------------------------------| +| `/logger/webhook` | Metrics related to logger webhooks. | + +#### `/logger/webhook` + +| Name | Description | Labels | +|-----------------------------------------|---------------------------------------------------------------------|------------------------------| +| `minio_logger_webhook_failed_messages` | Number of messages that failed to send.

Type: counter | `server`, `name`, `endpoint` | +| `minio_logger_webhook_queue_length` | Webhook queue length.

Type: gauge | `server`, `name`, `endpoint` | +| `minio_logger_webhook_total_message` | Total number of messages sent to this target.

Type: counter | `server`, `name`, `endpoint` | + +### Notification metrics + +Metrics about the MinIO notification functionality. + +| Path | Description | +|-----------------|------------------------------------------------| +| `/notification` | Metrics related to notification functionality. | + +#### `/notification` + +| Name | Description | Labels | +|-----------------------------------------------|-------------------------------------------------------------------------------------------------------|----------| +| `minio_notification_current_send_in_progress` | Number of concurrent async Send calls active to all targets.

Type: counter | `server` | +| `minio_notification_events_errors_total` | Total number of events that failed to send to the targets.

Type: counter | `server` | +| `minio_notification_events_sent_total` | Total number of events sent to the targets.

Type: counter | `server` | +| `minio_notification_events_skipped_total` | Number of events not sent to the targets due to the in-memory queue being full.

Type: counter | `server` | + +### Replication metrics + +Metrics about MinIO site and bucket replication. + +| Path | Description | +|-----------------------|----------------------------------------| +| `/bucket/replication` | Metrics related to bucket replication. | +| `/replication` | Metrics related to site replication. | + +#### `/replication` + +| Name | Description | Labels | +|---------------------------------------------------|---------------------------------------------------------------------------------------------|----------| +| `minio_replication_average_active_workers` | Average number of active replication workers.

Type: gauge | `server` | +| `minio_replication_average_queued_bytes` | Average number of bytes queued for replication since server start.

Type: gauge | `server` | +| `minio_replication_average_queued_count` | Average number of objects queued for replication since server start.

Type: gauge | `server` | +| `minio_replication_average_data_transfer_rate` | Average replication data transfer rate in bytes/sec.

Type: gauge | `server` | +| `minio_replication_current_active_workers` | Total number of active replication workers.

Type: gauge | `server` | +| `minio_replication_current_data_transfer_rate` | Current replication data transfer rate in bytes/sec.

Type: gauge | `server` | +| `minio_replication_last_minute_queued_bytes` | Number of bytes queued for replication in the last full minute.

Type: gauge | `server` | +| `minio_replication_last_minute_queued_count` | Number of objects queued for replication in the last full minute.

Type: gauge | `server` | +| `minio_replication_max_active_workers` | Maximum number of active replication workers seen since server start.

Type: gauge | `server` | +| `minio_replication_max_queued_bytes` | Maximum number of bytes queued for replication since server start.

Type: gauge | `server` | +| `minio_replication_max_queued_count` | Maximum number of objects queued for replication since server start.

Type: gauge | `server` | +| `minio_replication_max_data_transfer_rate` | Maximum replication data transfer rate in bytes/sec since server start.

Type: gauge | `server` | +| `minio_replication_recent_backlog_count` | Total number of objects seen in replication backlog in the last 5 minutes

Type: gauge | `server` | +#### `/bucket/replication` + +| Name | Description | Labels | +|---------------------------------------------------------------------|-----------------------------------------------------------------------------------------------------------------|-------------------------------------------------------| +| `minio_bucket_replication_last_hour_failed_bytes` | Total number of bytes on a bucket which failed to replicate at least once in the last hour.

Type: gauge | `bucket`, `server` | +| `minio_bucket_replication_last_hour_failed_count` | Total number of objects on a bucket which failed to replicate in the last hour.

Type: gauge | `bucket`, `server` | +| `minio_bucket_replication_last_minute_failed_bytes` | Total number of bytes on a bucket which failed at least once in the last full minute.

Type: gauge | `bucket`, `server` | +| `minio_bucket_replication_last_minute_failed_count` | Total number of objects on a bucket which failed to replicate in the last full minute.

Type: gauge | `bucket`, `server` | +| `minio_bucket_replication_latency_ms` | Replication latency on a bucket in milliseconds.

Type: gauge | `bucket`, `operation`, `range`, `targetArn`, `server` | +| `minio_bucket_replication_proxied_delete_tagging_requests_total` | Number of DELETE tagging requests proxied to replication target.

Type: counter | `bucket`, `server` | +| `minio_bucket_replication_proxied_get_requests_failures` | Number of failures in GET requests proxied to replication target.

Type: counter | `bucket`, `server` | +| `minio_bucket_replication_proxied_get_requests_total` | Number of GET requests proxied to replication target.

Type: counter | `bucket`, `server` | +| `minio_bucket_replication_proxied_get_tagging_requests_failures` | Number of failures in GET tagging requests proxied to replication target.

Type: counter | `bucket`, `server` | +| `minio_bucket_replication_proxied_get_tagging_requests_total` | Number of GET tagging requests proxied to replication target.

Type: counter | `bucket`, `server` | +| `minio_bucket_replication_proxied_head_requests_failures` | Number of failures in HEAD requests proxied to replication target.

Type: counter | `bucket`, `server` | +| `minio_bucket_replication_proxied_head_requests_total` | Number of HEAD requests proxied to replication target.

Type: counter | `bucket`, `server` | +| `minio_bucket_replication_proxied_put_tagging_requests_failures` | Number of failures in PUT tagging requests proxied to replication target.

Type: counter | `bucket`, `server` | +| `minio_bucket_replication_proxied_put_tagging_requests_total` | Number of PUT tagging requests proxied to replication target.

Type: counter | `bucket`, `server` | +| `minio_bucket_replication_sent_bytes` | Total number of bytes replicated to the target.

Type: counter | `bucket`, `server` | +| `minio_bucket_replication_sent_count` | Total number of objects replicated to the target.

Type: counter | `bucket`, `server` | +| `minio_bucket_replication_total_failed_bytes` | Total number of bytes failed to replicate at least once since server start.

Type: counter | `bucket`, `server` | +| `minio_bucket_replication_total_failed_count` | Total number of objects that failed to replicate since server start.

Type: counter | `bucket`, `server` | +| `minio_bucket_replication_proxied_delete_tagging_requests_failures` | Number of failures in DELETE tagging requests proxied to replication target.

Type: counter | `bucket`, `server` | + +### Scanner metrics + +Metrics about the MinIO scanner. + +| Path | Description | +|------------|---------------------------------------| +| `/scanner` | Metrics related to the MinIO scanner. | + +#### `/scanner` + +| Name | Description | Labels | +|--------------------------------------------|-----------------------------------------------------------------------------------|----------| +| `minio_scanner_bucket_scans_finished` | Total number of bucket scans completed since server start.

Type: counter | `server` | +| `minio_scanner_bucket_scans_started` | Total number of bucket scans started since server start.

Type: counter | `server` | +| `minio_scanner_directories_scanned` | Total number of directories scanned since server start.

Type: counter | `server` | +| `minio_scanner_last_activity_seconds` | Time elapsed (in seconds) since last scan activity.

Type: gauge | `server` | +| `minio_scanner_objects_scanned` | Total number of unique objects scanned since server start.

Type: counter | `server` | +| `minio_scanner_versions_scanned` | Total number of object versions scanned since server start.

Type: counter | `server` | + +### System metrics + +Metrics about the MinIO process and the node. + +| Path | Description | +|-----------------------------|----------------------------------------------------| +| `/system/cpu` | Metrics about CPUs on the system. | +| `/system/drive` | Metrics about drives on the system. | +| `/system/network/internode` | Metrics about internode requests made by the node. | +| `/system/memory` | Metrics about memory on the system. | +| `/system/process` | Standard process metrics. | + +#### `/system/drive` + +| Name | Description | Labels | +|------------------------------------------------|-----------------------------------------------------------------------------------------|--------------------------------------------------------------------| +| `minio_system_drive_used_bytes` | Total storage used on a drive in bytes.

Type: gauge | `drive`, `set_index`, `drive_index`, `pool_index`, `server` | +| `minio_system_drive_free_bytes` | Total storage free on a drive in bytes.

Type: gauge | `drive`, `set_index`, `drive_index`, `pool_index`, `server` | +| `minio_system_drive_total_bytes` | Total storage available on a drive in bytes.

Type: gauge | `drive`, `set_index`, `drive_index`, `pool_index`, `server` | +| `minio_system_drive_used_inodes` | Total used inodes on a drive.

Type: gauge | `drive`, `set_index`, `drive_index`, `pool_index`, `server` | +| `minio_system_drive_free_inodes` | Total free inodes on a drive.

Type: gauge | `drive`, `set_index`, `drive_index`, `pool_index`, `server` | +| `minio_system_drive_total_inodes` | Total inodes available on a drive.

Type: gauge | `drive`, `set_index`, `drive_index`, `pool_index`, `server` | +| `minio_system_drive_timeout_errors_total` | Total timeout errors on a drive.

Type: counter | `drive`, `set_index`, `drive_index`, `pool_index`, `server` | +| `minio_system_drive_io_errors_total` | Total I/O errors on a drive.

Type: counter | `drive`, `set_index`, `drive_index`, `pool_index`, `server` | +| `minio_system_drive_availability_errors_total` | Total availability errors (I/O errors, timeouts) on a drive.

Type: counter | `drive`, `set_index`, `drive_index`, `pool_index`, `server` | +| `minio_system_drive_waiting_io` | Total waiting I/O operations on a drive.

Type: gauge | `drive`, `set_index`, `drive_index`, `pool_index`, `server` | +| `minio_system_drive_api_latency_micros` | Average last minute latency in µs for drive API storage operations.

Type: gauge | `drive`, `api`, `set_index`, `drive_index`, `pool_index`, `server` | +| `minio_system_drive_offline_count` | Count of offline drives.

Type: gauge | `pool_index`, `server` | +| `minio_system_drive_online_count` | Count of online drives.

Type: gauge | `pool_index`, `server` | +| `minio_system_drive_count` | Count of all drives.

Type: gauge | `pool_index`, `server` | +| `minio_system_drive_health` | Drive health (0 = offline, 1 = healthy, 2 = healing).

Type: gauge | `drive`, `set_index`, `drive_index`, `pool_index`, `server` | +| `minio_system_drive_reads_per_sec` | Reads per second on a drive.

Type: gauge | `drive`, `set_index`, `drive_index`, `pool_index`, `server` | +| `minio_system_drive_reads_kb_per_sec` | Kilobytes read per second on a drive.

Type: gauge | `drive`, `set_index`, `drive_index`, `pool_index`, `server` | +| `minio_system_drive_reads_await` | Average time for read requests served on a drive.

Type: gauge | `drive`, `set_index`, `drive_index`, `pool_index`, `server` | +| `minio_system_drive_writes_per_sec` | Writes per second on a drive.

Type: gauge | `drive`, `set_index`, `drive_index`, `pool_index`, `server` | +| `minio_system_drive_writes_kb_per_sec` | Kilobytes written per second on a drive.

Type: gauge | `drive`, `set_index`, `drive_index`, `pool_index`, `server` | +| `minio_system_drive_writes_await` | Average time for write requests served on a drive.

Type: gauge | `drive`, `set_index`, `drive_index`, `pool_index`, `server` | +| `minio_system_drive_perc_util` | Percentage of time the disk was busy.

Type: gauge | `drive`, `set_index`, `drive_index`, `pool_index`, `server` | + +#### `/system/memory` + +| Name | Description | Labels | +|----------------------------------|---------------------------------------------------------|----------| +| `minio_system_memory_used` | Used memory on the node.

Type: gauge | `server` | +| `minio_system_memory_used_perc` | Used memory percentage on the node.

Type: gauge | `server` | +| `minio_system_memory_free` | Free memory on the node.

Type: gauge | `server` | +| `minio_system_memory_total` | Total memory on the node.

Type: gauge | `server` | +| `minio_system_memory_buffers` | Buffers memory on the node.

Type: gauge | `server` | +| `minio_system_memory_cache` | Cache memory on the node.

Type: gauge | `server` | +| `minio_system_memory_shared` | Shared memory on the node.

Type: gauge | `server` | +| `minio_system_memory_available` | Available memory on the node.

Type: gauge | `server` | + +#### `/system/cpu` + +| Name | Description | Labels | +|-------------------------------|---------------------------------------------------------|----------| +| `minio_system_cpu_avg_idle` | Average CPU idle time.

Type: gauge | `server` | +| `minio_system_cpu_avg_iowait` | Average CPU IOWait time.

Type: gauge | `server` | +| `minio_system_cpu_load` | CPU load average 1min.

Type: gauge | `server` | +| `minio_system_cpu_load_perc` | CPU load average 1min (percentage).

Type: gauge | `server` | +| `minio_system_cpu_nice` | CPU nice time.

Type: gauge | `server` | +| `minio_system_cpu_steal` | CPU steal time.

Type: gauge | `server` | +| `minio_system_cpu_system` | CPU system time.

Type: gauge | `server` | +| `minio_system_cpu_user` | CPU user time.

Type: gauge | `server` | + +#### `/system/network/internode` + +| Name | Description | Labels | +|------------------------------------------------------|-------------------------------------------------------------------------------|------------------------| +| `minio_system_network_internode_errors_total` | Total number of failed internode calls.

Type: counter | `server`, `pool_index` | +| `minio_system_network_internode_dial_errors_total` | Total number of internode TCP dial timeouts and errors.

Type: counter | `server`, `pool_index` | +| `minio_system_network_internode_dial_avg_time_nanos` | Average dial time of internodes TCP calls in nanoseconds.

Type: gauge | `server`, `pool_index` | +| `minio_system_network_internode_sent_bytes_total` | Total number of bytes sent to other peer nodes.

Type: counter | `server`, `pool_index` | +| `minio_system_network_internode_recv_bytes_total` | Total number of bytes received from other peer nodes.

Type: counter | `server`, `pool_index` | + +#### `/system/process` + +| Name | Description | Labels | +|----------------------------------------------------|---------------------------------------------------------------------------------------------------------------------------------------|----------| +| `minio_system_process_locks_read_total` | Number of current READ locks on this peer.

Type: gauge | `server` | +| `minio_system_process_locks_write_total` | Number of current WRITE locks on this peer.

Type: gauge | `server` | +| `minio_system_process_cpu_total_seconds` | Total user and system CPU time spent in seconds.

Type: counter | `server` | +| `minio_system_process_go_routine_total` | Total number of go routines running.

Type: gauge | `server` | +| `minio_system_process_io_rchar_bytes` | Total bytes read by the process from the underlying storage system including cache, /proc/[pid]/io rchar.

Type: counter | `server` | +| `minio_system_process_io_read_bytes` | Total bytes read by the process from the underlying storage system, /proc/[pid]/io read_bytes.

Type: counter | `server` | +| `minio_system_process_io_wchar_bytes` | Total bytes written by the process to the underlying storage system including page cache, /proc/[pid]/io wchar.

Type: counter | `server` | +| `minio_system_process_io_write_bytes` | Total bytes written by the process to the underlying storage system, /proc/[pid]/io write_bytes.

Type: counter | `server` | +| `minio_system_process_start_time_seconds` | Start time for MinIO process in seconds since Unix epoch.

Type: gauge | `server` | +| `minio_system_process_uptime_seconds` | Uptime for MinIO process in seconds.

Type: gauge | `server` | +| `minio_system_process_file_descriptor_limit_total` | Limit on total number of open file descriptors for the MinIO Server process.

Type: gauge | `server` | +| `minio_system_process_file_descriptor_open_total` | Total number of open file descriptors by the MinIO Server process.

Type: gauge | `server` | +| `minio_system_process_syscall_read_total` | Total read SysCalls to the kernel. /proc/[pid]/io syscr.

Type: counter | `server` | +| `minio_system_process_syscall_write_total` | Total write SysCalls to the kernel. /proc/[pid]/io syscw.

Type: counter | `server` | +| `minio_system_process_resident_memory_bytes` | Resident memory size in bytes.

Type: gauge | `server` | +| `minio_system_process_virtual_memory_bytes` | Virtual memory size in bytes.

Type: gauge | `server` | +| `minio_system_process_virtual_memory_max_bytes` | Maximum virtual memory size in bytes.

Type: gauge | `server` | diff --git a/docs/minio-limits.md b/docs/minio-limits.md new file mode 100644 index 00000000..e7295d19 --- /dev/null +++ b/docs/minio-limits.md @@ -0,0 +1,72 @@ +# MinIO Server Limits Per Tenant + +For optimal production setup MinIO recommends Linux kernel version 4.x and later. + +## Erasure Code (Multiple Drives / Servers) + +| Item | Specification | +|:----------------------------------------------------------------|:--------------| +| Maximum number of servers per cluster | no-limit | +| Minimum number of servers | 02 | +| Minimum number of drives per server when server count is 1 | 02 | +| Minimum number of drives per server when server count is 2 or 3 | 01 | +| Minimum number of drives per server when server count is 4 | 01 | +| Maximum number of drives per server | no-limit | +| Read quorum | N/2 | +| Write quorum | N/2+1 | + +## Limits of S3 API + +| Item | Specification | +|:--------------------------------------------------------------------------------|:--------------------------------------------------------------------------------| +| Maximum number of buckets | unlimited (we recommend not beyond 500000 buckets) - see NOTE: | +| Maximum number of objects per bucket | no-limit | +| Maximum object size | 50 TiB | +| Minimum object size | 0 B | +| Maximum object size per PUT operation | 5 TiB | +| Maximum number of parts per upload | 10,000 | +| Part size range | 5 MiB to 5 TiB. Last part can be 0 B to 5 TiB | +| Maximum number of parts returned per list parts request | 10000 | +| Maximum number of objects returned per list objects request | 1000 | +| Maximum number of multipart uploads returned per list multipart uploads request | 1000 | +| Maximum length for bucket names | 63 | +| Maximum length for object names | 1024 | +| Maximum length for '/' separated object name segment | 255 | +| Maximum number of versions per object | 10000 (can be configured to higher values but we do not recommend beyond 10000) | + +> NOTE: While MinIO does not implement an upper boundary on buckets, your cluster's hardware has natural limits that depend on the workload and its scaling patterns. We strongly recommend [MinIO SUBNET](https://min.io/pricing) for architecture and sizing guidance for your production use case. + +## List of Amazon S3 APIs not supported on MinIO + +We found the following APIs to be redundant or less useful outside of AWS S3. If you have a different view on any of the APIs we missed, please consider opening a [GitHub issue](https://github.com/minio/minio/issues) with relevant details on why MinIO must implement them. + +### List of Amazon S3 Bucket APIs not supported on MinIO + +- BucketACL (Use [bucket policies](https://docs.min.io/community/minio-object-store/administration/identity-access-management/policy-based-access-control.html) instead) +- BucketCORS (CORS enabled by default on all buckets for all HTTP verbs, you can optionally restrict the CORS domains) +- BucketWebsite (Use [`caddy`](https://github.com/caddyserver/caddy) or [`nginx`](https://www.nginx.com/resources/wiki/)) +- BucketAnalytics, BucketMetrics, BucketLogging (Use [bucket notification](https://docs.min.io/community/minio-object-store/administration/monitoring/bucket-notifications.html) APIs) + +### List of Amazon S3 Object APIs not supported on MinIO + +- ObjectACL (Use [bucket policies](https://docs.min.io/community/minio-object-store/administration/identity-access-management/policy-based-access-control.html) instead) + +## Object name restrictions on MinIO + +- Object name restrictions on MinIO are governed by OS and filesystem limitations. For example object names that contain characters `^*|\/&";` are unsupported on Windows platform or any other file systems that do not support filenames with special characters. + +> **This list is non exhaustive, it depends on the operating system and filesystem under use - please consult your operating system vendor for a more comprehensive list of special characters**. + +MinIO recommends using Linux operating system for production workloads. + +- Objects must not have conflicting objects as parent objects, applications using this behavior should change their behavior and use non-conflicting unique keys, for example situations such as following conflicting key patterns are not supported. + +``` +PUT /a/b/1.txt +PUT /a/b +``` + +``` +PUT /a/b +PUT /a/b/1.txt +``` diff --git a/docs/multi-tenancy/README.md b/docs/multi-tenancy/README.md new file mode 100644 index 00000000..055535e7 --- /dev/null +++ b/docs/multi-tenancy/README.md @@ -0,0 +1,67 @@ +# MinIO Multi-Tenant Deployment Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) + +This topic provides commands to set up different configurations of hosts, nodes, and drives. The examples provided here can be used as a starting point for other configurations. + +1. [Standalone Deployment](#standalone-deployment) +2. [Distributed Deployment](#distributed-deployment) +3. [Cloud Scale Deployment](#cloud-scale-deployment) + +## 1. Standalone Deployment + +To host multiple tenants on a single machine, run one MinIO Server per tenant with a dedicated HTTPS port, configuration, and data directory. + +### 1.1 Host Multiple Tenants on a Single Drive + +Use the following commands to host 3 tenants on a single drive: + +```sh +minio server --address :9001 /data/tenant1 +minio server --address :9002 /data/tenant2 +minio server --address :9003 /data/tenant3 +``` + +![Example-1](https://github.com/minio/minio/blob/master/docs/screenshots/Example-1.jpg?raw=true) + +### 1.2 Host Multiple Tenants on Multiple Drives (Erasure Code) + +Use the following commands to host 3 tenants on multiple drives: + +```sh +minio server --address :9001 /disk{1...4}/data/tenant1 +minio server --address :9002 /disk{1...4}/data/tenant2 +minio server --address :9003 /disk{1...4}/data/tenant3 +``` + +![Example-2](https://github.com/minio/minio/blob/master/docs/screenshots/Example-2.jpg?raw=true) + +## 2. Distributed Deployment + +To host multiple tenants in a distributed environment, run several distributed MinIO Server instances concurrently. + +### 2.1 Host Multiple Tenants on Multiple Drives (Erasure Code) + +Use the following commands to host 3 tenants on a 4-node distributed configuration: + +```sh +export MINIO_ROOT_USER= +export MINIO_ROOT_PASSWORD= +minio server --address :9001 http://192.168.10.1{1...4}/data/tenant1 + +export MINIO_ROOT_USER= +export MINIO_ROOT_PASSWORD= +minio server --address :9002 http://192.168.10.1{1...4}/data/tenant2 + +export MINIO_ROOT_USER= +export MINIO_ROOT_PASSWORD= +minio server --address :9003 http://192.168.10.1{1...4}/data/tenant3 +``` + +**Note:** Execute the commands on all 4 nodes. + +![Example-3](https://github.com/minio/minio/blob/master/docs/screenshots/Example-3.jpg?raw=true) + +**Note**: On distributed systems, root credentials are recommend to be defined by exporting the `MINIO_ROOT_USER` and `MINIO_ROOT_PASSWORD` environment variables. If no value is set MinIO setup will assume `minioadmin/minioadmin` as default credentials. If a domain is required, it must be specified by defining and exporting the `MINIO_DOMAIN` environment variable. + +## Cloud Scale Deployment + +A container orchestration platform (e.g. Kubernetes) is recommended for large-scale, multi-tenant MinIO deployments. See the [MinIO Deployment Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/kubernetes.html) to get started with MinIO on orchestration platforms. diff --git a/docs/multi-user/README.md b/docs/multi-user/README.md new file mode 100644 index 00000000..3dbd6ce1 --- /dev/null +++ b/docs/multi-user/README.md @@ -0,0 +1,278 @@ +# MinIO Multi-user Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +MinIO supports multiple long term users in addition to default user created during server startup. New users can be added after server starts up, and server can be configured to deny or allow access to buckets and resources to each of these users. This document explains how to add/remove users and modify their access rights. + +## Get started + +In this document we will explain in detail on how to configure multiple users. + +### 1. Prerequisites + +- Install mc - [MinIO Client Quickstart Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart) +- Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html) +- Configure etcd - [Etcd V3 Quickstart Guide](https://github.com/minio/minio/blob/master/docs/sts/etcd.md) + +### 2. Create a new user with canned policy + +Use [`mc admin policy`](https://docs.min.io/community/minio-object-store/reference/minio-mc-admin/mc-admin-policy.html) to create canned policies. Server provides a default set of canned policies namely `writeonly`, `readonly` and `readwrite` *(these policies apply to all resources on the server)*. These can be overridden by custom policies using `mc admin policy` command. + +Create new canned policy file `getonly.json`. This policy enables users to download all objects under `my-bucketname`. + +```json +cat > getonly.json << EOF +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "s3:GetObject" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::my-bucketname/*" + ], + "Sid": "" + } + ] +} +EOF +``` + +Create new canned policy by name `getonly` using `getonly.json` policy file. + +``` +mc admin policy create myminio getonly getonly.json +``` + +Create a new user `newuser` on MinIO use `mc admin user`. + +``` +mc admin user add myminio newuser newuser123 +``` + +Once the user is successfully created you can now apply the `getonly` policy for this user. + +``` +mc admin policy attach myminio getonly --user=newuser +``` + +### 3. Create a new group + +``` +mc admin group add myminio newgroup newuser +``` + +Once the group is successfully created you can now apply the `getonly` policy for this group. + +``` +mc admin policy attach myminio getonly --group=newgroup +``` + +### 4. Disable user + +Disable user `newuser`. + +``` +mc admin user disable myminio newuser +``` + +Disable group `newgroup`. + +``` +mc admin group disable myminio newgroup +``` + +### 5. Remove user + +Remove the user `newuser`. + +``` +mc admin user remove myminio newuser +``` + +Remove the user `newuser` from a group. + +``` +mc admin group remove myminio newgroup newuser +``` + +Remove the group `newgroup`. + +``` +mc admin group remove myminio newgroup +``` + +### 6. Change user or group policy + +Change the policy for user `newuser` to `putonly` canned policy. + +``` +mc admin policy attach myminio putonly --user=newuser +``` + +Change the policy for group `newgroup` to `putonly` canned policy. + +``` +mc admin policy attach myminio putonly --group=newgroup +``` + +### 7. List all users or groups + +List all enabled and disabled users. + +``` +mc admin user list myminio +``` + +List all enabled or disabled groups. + +``` +mc admin group list myminio +``` + +### 8. Configure `mc` + +``` +mc alias set myminio-newuser http://localhost:9000 newuser newuser123 --api s3v4 +mc cat myminio-newuser/my-bucketname/my-objectname +``` + +### Policy Variables + +You can use policy variables in the *Resource* element and in string comparisons in the *Condition* element. + +You can use a policy variable in the Resource element, but only in the resource portion of the ARN. This portion of the ARN appears after the 5th colon (:). You can't use a variable to replace parts of the ARN before the 5th colon, such as the service or account. The following policy might be attached to a group. It gives each of the users in the group full programmatic access to a user-specific object (their own "home directory") in MinIO. + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": ["s3:ListBucket"], + "Effect": "Allow", + "Resource": ["arn:aws:s3:::mybucket"], + "Condition": {"StringLike": {"s3:prefix": ["${aws:username}/*"]}} + }, + { + "Action": [ + "s3:GetObject", + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": ["arn:aws:s3:::mybucket/${aws:username}/*"] + } + ] +} +``` + +If the user is authenticating using an STS credential which was authorized from OpenID connect we allow all `jwt:*` variables specified in the JWT specification, custom `jwt:*` or extensions are not supported. List of policy variables for OpenID based STS. + +- `jwt:sub` +- `jwt:iss` +- `jwt:aud` +- `jwt:jti` +- `jwt:upn` +- `jwt:name` +- `jwt:groups` +- `jwt:given_name` +- `jwt:family_name` +- `jwt:middle_name` +- `jwt:nickname` +- `jwt:preferred_username` +- `jwt:profile` +- `jwt:picture` +- `jwt:website` +- `jwt:email` +- `jwt:gender` +- `jwt:birthdate` +- `jwt:phone_number` +- `jwt:address` +- `jwt:scope` +- `jwt:client_id` + +Following example shows OpenID users with full programmatic access to a OpenID user-specific directory (their own "home directory") in MinIO. + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": ["s3:ListBucket"], + "Effect": "Allow", + "Resource": ["arn:aws:s3:::mybucket"], + "Condition": {"StringLike": {"s3:prefix": ["${jwt:preferred_username}/*"]}} + }, + { + "Action": [ + "s3:GetObject", + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": ["arn:aws:s3:::mybucket/${jwt:preferred_username}/*"] + } + ] +} +``` + +If the user is authenticating using an STS credential which was authorized from AD/LDAP we allow `ldap:*` variables. + +Currently supports + +- `ldap:username` +- `ldap:user` +- `ldap:groups` + +Following example shows LDAP users full programmatic access to a LDAP user-specific directory (their own "home directory") in MinIO. + +``` +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": ["s3:ListBucket"], + "Effect": "Allow", + "Resource": ["arn:aws:s3:::mybucket"], + "Condition": {"StringLike": {"s3:prefix": ["${ldap:username}/*"]}} + }, + { + "Action": [ + "s3:GetObject", + "s3:PutObject" + ], + "Effect": "Allow", + "Resource": ["arn:aws:s3:::mybucket/${ldap:username}/*"] + } + ] +} +``` + +#### Common information available in all requests + +- `aws:CurrentTime` - This can be used for conditions that check the date and time. +- `aws:EpochTime` - This is the date in epoch or Unix time, for use with date/time conditions. +- `aws:PrincipalType` - This value indicates whether the principal is an account (Root credential), user (MinIO user), or assumed role (STS) +- `aws:SecureTransport` - This is a Boolean value that represents whether the request was sent over TLS. +- `aws:SourceIp` - This is the requester's IP address, for use with IP address conditions. If running behind Nginx like proxies, MinIO preserve's the source IP. + +``` +{ + "Version": "2012-10-17", + "Statement": { + "Effect": "Allow", + "Action": "s3:ListBucket*", + "Resource": "arn:aws:s3:::mybucket", + "Condition": {"IpAddress": {"aws:SourceIp": "203.0.113.0/24"}} + } +} +``` + +- `aws:UserAgent` - This value is a string that contains information about the requester's client application. This string is generated by the client and can be unreliable. You can only use this context key from `mc` or other MinIO SDKs which standardize the User-Agent string. +- `aws:username` - This is a string containing the friendly name of the current user, this value would point to STS temporary credential in `AssumeRole`ed requests, use `jwt:preferred_username` in case of OpenID connect and `ldap:username` in case of AD/LDAP. *aws:userid* is an alias to *aws:username* in MinIO. +- `aws:groups` - This is an array containing the group names, this value would point to group mappings for the user, use `jwt:groups` in case of OpenID connect and `ldap:groups` in case of AD/LDAP. + +## Explore Further + +- [MinIO Client Complete Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc.html) +- [MinIO STS Quickstart Guide](https://docs.min.io/community/minio-object-store/developers/security-token-service.html) +- [MinIO Admin Complete Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc-admin.html) +- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html) diff --git a/docs/multi-user/admin/README.md b/docs/multi-user/admin/README.md new file mode 100644 index 00000000..ff43cd7b --- /dev/null +++ b/docs/multi-user/admin/README.md @@ -0,0 +1,172 @@ +# MinIO Admin Multi-user Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +MinIO supports multiple admin users in addition to default operator credential created during server startup. New admins can be added after server starts up, and server can be configured to deny or allow access to different admin operations for these users. This document explains how to add/remove admin users and modify their access rights. + +## Get started + +In this document we will explain in detail on how to configure admin users. + +### 1. Prerequisites + +- Install mc - [MinIO Client Quickstart Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart) +- Install MinIO - [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html) + +### 2. Create a new admin user with CreateUser, DeleteUser and ConfigUpdate permissions + +Use [`mc admin policy`](https://docs.min.io/community/minio-object-store/reference/minio-mc-admin/mc-admin-policy.html#command-mc.admin.policy) to create custom admin policies. + +Create new canned policy file `adminManageUser.json`. This policy enables admin user to +manage other users. + +```json +cat > adminManageUser.json << EOF +{ + "Version": "2012-10-17", + "Statement": [ + { + "Action": [ + "admin:CreateUser", + "admin:DeleteUser", + "admin:ConfigUpdate" + ], + "Effect": "Allow", + "Sid": "" + }, + { + "Action": [ + "s3:*" + ], + "Effect": "Allow", + "Resource": [ + "arn:aws:s3:::*" + ], + "Sid": "" + } + ] +} +EOF +``` + +Create new canned policy by name `userManager` using `userManager.json` policy file. + +``` +mc admin policy attach myminio userManager adminManageUser.json +``` + +Create a new admin user `admin1` on MinIO use `mc admin user`. + +``` +mc admin user add myminio admin1 admin123 +``` + +Once the user is successfully created you can now apply the `userManage` policy for this user. + +``` +mc admin policy attach myminio userManager --user=admin1 +``` + +This admin user will then be allowed to perform create/delete user operations via `mc admin user` + +### 3. Configure `mc` and create another user user1 with attached policy user1policy + +``` +mc alias set myminio-admin1 http://localhost:9000 admin1 admin123 --api s3v4 + +mc admin user add myminio-admin1 user1 user123 +mc admin policy attach myminio-admin1 user1policy ~/user1policy.json +mc admin policy attach myminio-admin1 user1policy --user=user1 +``` + +### 4. List of permissions defined for admin operations + +#### Config management permissions + +- admin:ConfigUpdate + +#### User management permissions + +- admin:CreateUser +- admin:DeleteUser +- admin:ListUsers +- admin:EnableUser +- admin:DisableUser +- admin:GetUser + +#### Service management permissions + +- admin:ServerInfo +- admin:ServerUpdate +- admin:StorageInfo +- admin:DataUsageInfo +- admin:TopLocks +- admin:OBDInfo +- admin:Profiling, +- admin:ServerTrace +- admin:ConsoleLog +- admin:KMSKeyStatus +- admin:KMSCreateKey +- admin:ServiceRestart +- admin:ServiceStop +- admin:Prometheus +- admin:ForceUnlock +- admin:TopLocksInfo +- admin:BandwidthMonitor + +#### User/Group management permissions + +- admin:AddUserToGroup +- admin:RemoveUserFromGroup +- admin:GetGroup +- admin:ListGroups +- admin:EnableGroup +- admin:DisableGroup + +#### Policy management permissions + +- admin:CreatePolicy +- admin:DeletePolicy +- admin:GetPolicy +- admin:AttachUserOrGroupPolicy +- admin:ListUserPolicies + +#### Heal management permissions + +- admin:Heal + +#### Service account management permissions + +- admin:CreateServiceAccount +- admin:UpdateServiceAccount +- admin:RemoveServiceAccount +- admin:ListServiceAccounts + +#### Bucket quota management permissions + +- admin:SetBucketQuota +- admin:GetBucketQuota + +#### Bucket target management permissions + +- admin:SetBucketTarget +- admin:GetBucketTarget + +#### Remote tier management permissions + +- admin:SetTier +- admin:ListTier + +#### Give full admin permissions + +- admin:* + +### 5. Using an external IDP for admin users + +Admin users can also be externally managed by an IDP by configuring admin policy with +special permissions listed above. Follow [MinIO STS Quickstart Guide](https://docs.min.io/community/minio-object-store/developers/security-token-service.html) to manage users with an IDP. + +## Explore Further + +- [MinIO Client Complete Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc.html) +- [MinIO STS Quickstart Guide](https://docs.min.io/community/minio-object-store/developers/security-token-service.html) +- [MinIO Admin Complete Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc-admin.html) +- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html) diff --git a/docs/orchestration/README.md b/docs/orchestration/README.md new file mode 100644 index 00000000..327f83b9 --- /dev/null +++ b/docs/orchestration/README.md @@ -0,0 +1,21 @@ +# MinIO Deployment Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) + +MinIO is a cloud-native application designed to scale in a sustainable manner in multi-tenant environments. Orchestration platforms provide perfect launchpad for MinIO to scale. Below is the list of MinIO deployment documents for various orchestration platforms: + +| Orchestration platforms | +|:---------------------------------------------------------------------------------------------------| +| [`Kubernetes`](https://docs.min.io/community/minio-object-store/operations/deployments/kubernetes.html) | + +## Why is MinIO cloud-native? + +The term cloud-native revolves around the idea of applications deployed as micro services, that scale well. It is not about just retrofitting monolithic applications onto modern container based compute environment. A cloud-native application is portable and resilient by design, and can scale horizontally by simply replicating. Modern orchestration platforms like Kubernetes, DC/OS make replicating and managing containers in huge clusters easier than ever. + +While containers provide isolated application execution environment, orchestration platforms allow seamless scaling by helping replicate and manage containers. MinIO extends this by adding isolated storage environment for each tenant. + +MinIO is built ground up on the cloud-native premise. With features like erasure-coding, distributed and shared setup, it focuses only on storage and does it very well. While, it can be scaled by just replicating MinIO instances per tenant via an orchestration platform. + +> In a cloud-native environment, scalability is not a function of the application but the orchestration platform. + +In a typical modern infrastructure deployment, application, database, key-store, etc. already live in containers and are managed by orchestration platforms. MinIO brings robust, scalable, AWS S3 compatible object storage to the lot. + +![Cloud-native](https://github.com/minio/minio/blob/master/docs/screenshots/Minio_Cloud_Native_Arch.jpg?raw=true) diff --git a/docs/orchestration/docker-compose/README.md b/docs/orchestration/docker-compose/README.md new file mode 100644 index 00000000..2e196514 --- /dev/null +++ b/docs/orchestration/docker-compose/README.md @@ -0,0 +1,59 @@ +# Deploy MinIO on Docker Compose [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) + +Docker Compose allows defining and running single host, multi-container Docker applications. + +With Compose, you use a Compose file to configure MinIO services. Then, using a single command, you can create and launch all the Distributed MinIO instances from your configuration. Distributed MinIO instances will be deployed in multiple containers on the same host. This is a great way to set up development, testing, and staging environments, based on Distributed MinIO. + +## 1. Prerequisites + +* Familiarity with [Docker Compose](https://docs.docker.com/compose/overview/). +* Docker installed on your machine. Download the relevant installer from [here](https://www.docker.com/community-edition#/download). + +## 2. Run Distributed MinIO on Docker Compose + +To deploy Distributed MinIO on Docker Compose, please download [docker-compose.yaml](https://github.com/minio/minio/blob/master/docs/orchestration/docker-compose/docker-compose.yaml?raw=true) and [nginx.conf](https://github.com/minio/minio/blob/master/docs/orchestration/docker-compose/nginx.conf?raw=true) to your current working directory. Note that Docker Compose pulls the MinIO Docker image, so there is no need to build MinIO from source when using Docker. For non-Docker deployments, MinIO community edition is now source-only and can be installed via `go install github.com/minio/minio@latest`. Then run one of the below commands + +### GNU/Linux and macOS + +```sh +docker-compose pull +docker-compose up +``` + +or + +```sh +docker stack deploy --compose-file docker-compose.yaml minio +``` + +### Windows + +```sh +docker-compose.exe pull +docker-compose.exe up +``` + +or + +```sh +docker stack deploy --compose-file docker-compose.yaml minio +``` + +Distributed instances are now accessible on the host using the Minio CLI on port 9000 and the Minio Web Console on port 9001. Proceed to access the Web browser at . Here 4 MinIO server instances are reverse proxied through Nginx load balancing. + +### Notes + +* By default the Docker Compose file uses the Docker image for latest MinIO server release. You can change the image tag to pull a specific [MinIO Docker image](https://hub.docker.com/r/minio/minio/). + +* There are 4 minio distributed instances created by default. You can add more MinIO services (up to total 16) to your MinIO Compose deployment. To add a service + * Replicate a service definition and change the name of the new service appropriately. + * Update the command section in each service. + * Add a new MinIO server instance to the upstream directive in the Nginx configuration file. + + Read more about distributed MinIO [here](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-as-a-container.html). + +### Explore Further + +* [Overview of Docker Compose](https://docs.docker.com/compose/overview/) +* [MinIO Docker Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-as-a-container.html) +* [MinIO Erasure Code QuickStart Guide](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html) diff --git a/docs/orchestration/docker-compose/docker-compose.yaml b/docs/orchestration/docker-compose/docker-compose.yaml new file mode 100644 index 00000000..cfd7924a --- /dev/null +++ b/docs/orchestration/docker-compose/docker-compose.yaml @@ -0,0 +1,75 @@ +version: '3.7' + +# Settings and configurations that are common for all containers +x-minio-common: &minio-common + image: quay.io/minio/minio:RELEASE.2025-09-06T17-38-46Z + command: server --console-address ":9001" http://minio{1...4}/data{1...2} + expose: + - "9000" + - "9001" + # environment: + # MINIO_ROOT_USER: minioadmin + # MINIO_ROOT_PASSWORD: minioadmin + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 5s + retries: 5 + +# starts 4 docker containers running minio server instances. +# using nginx reverse proxy, load balancing, you can access +# it through port 9000. +services: + minio1: + <<: *minio-common + hostname: minio1 + volumes: + - data1-1:/data1 + - data1-2:/data2 + + minio2: + <<: *minio-common + hostname: minio2 + volumes: + - data2-1:/data1 + - data2-2:/data2 + + minio3: + <<: *minio-common + hostname: minio3 + volumes: + - data3-1:/data1 + - data3-2:/data2 + + minio4: + <<: *minio-common + hostname: minio4 + volumes: + - data4-1:/data1 + - data4-2:/data2 + + nginx: + image: nginx:1.19.2-alpine + hostname: nginx + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + ports: + - "9000:9000" + - "9001:9001" + depends_on: + - minio1 + - minio2 + - minio3 + - minio4 + +## By default this config uses default local driver, +## For custom volumes replace with volume driver configuration. +volumes: + data1-1: + data1-2: + data2-1: + data2-2: + data3-1: + data3-2: + data4-1: + data4-2: diff --git a/docs/orchestration/docker-compose/nginx.conf b/docs/orchestration/docker-compose/nginx.conf new file mode 100644 index 00000000..cca82f6f --- /dev/null +++ b/docs/orchestration/docker-compose/nginx.conf @@ -0,0 +1,106 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 4096; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + sendfile on; + keepalive_timeout 65; + + # include /etc/nginx/conf.d/*.conf; + + upstream minio { + server minio1:9000; + server minio2:9000; + server minio3:9000; + server minio4:9000; + } + + upstream console { + ip_hash; + server minio1:9001; + server minio2:9001; + server minio3:9001; + server minio4:9001; + } + + server { + listen 9000; + listen [::]:9000; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio; + } + } + + server { + listen 9001; + listen [::]:9001; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-NginX-Proxy true; + + # This is necessary to pass the correct IP to be hashed + real_ip_header X-Real-IP; + + proxy_connect_timeout 300; + + # To support websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + chunked_transfer_encoding off; + + proxy_pass http://console; + } + } +} diff --git a/docs/orchestration/kubernetes/README.md b/docs/orchestration/kubernetes/README.md new file mode 100644 index 00000000..3ea3fb27 --- /dev/null +++ b/docs/orchestration/kubernetes/README.md @@ -0,0 +1,21 @@ +# Deploy MinIO on Kubernetes [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) + +MinIO is a high performance distributed object storage server, designed for large-scale private cloud infrastructure. MinIO is designed in a cloud-native manner to scale sustainably in multi-tenant environments. Orchestration platforms like Kubernetes provide perfect cloud-native environment to deploy and scale MinIO. + +## MinIO Deployment on Kubernetes + +There are multiple options to deploy MinIO on Kubernetes: + +- MinIO-Operator: Operator offers seamless way to create and update highly available distributed MinIO clusters. Refer [MinIO Operator documentation](https://github.com/minio/minio-operator/blob/master/README.md) for more details. + +- Helm Chart: MinIO Helm Chart offers customizable and easy MinIO deployment with a single command. Refer [MinIO Helm Chart documentation](https://github.com/minio/minio/tree/master/helm/minio) for more details. + +## Monitoring MinIO in Kubernetes + +MinIO server exposes un-authenticated liveness endpoints so Kubernetes can natively identify unhealthy MinIO containers. MinIO also exposes Prometheus compatible data on a different endpoint to enable Prometheus users to natively monitor their MinIO deployments. + +## Explore Further + +- [MinIO Erasure Code QuickStart Guide](https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html) +- [Kubernetes Documentation](https://kubernetes.io/docs/home/) +- [Helm package manager for kubernetes](https://helm.sh/) diff --git a/docs/resiliency/docker-compose.yaml b/docs/resiliency/docker-compose.yaml new file mode 100644 index 00000000..842d766a --- /dev/null +++ b/docs/resiliency/docker-compose.yaml @@ -0,0 +1,125 @@ +# Settings and configurations that are common for all containers +x-minio-common: &minio-common + build: + context: ../../. + dockerfile: Dockerfile + command: server --console-address ":9001" http://minio{1...4}/data{1...8} + expose: + - "9000" + - "9001" + environment: + MINIO_CI_CD: 1 + healthcheck: + test: ["CMD", "mc", "ready", "local"] + interval: 5s + timeout: 5s + retries: 5 + +# starts 4 docker containers running minio server instances. +# using nginx reverse proxy, load balancing, you can access +# it through port 9000. +services: + minio1: + <<: *minio-common + hostname: minio1 + volumes: + - data1-1:/data1 + - data1-2:/data2 + - data1-3:/data3 + - data1-4:/data4 + - data1-5:/data5 + - data1-6:/data6 + - data1-7:/data7 + - data1-8:/data8 + + minio2: + <<: *minio-common + hostname: minio2 + volumes: + - data2-1:/data1 + - data2-2:/data2 + - data2-3:/data3 + - data2-4:/data4 + - data2-5:/data5 + - data2-6:/data6 + - data2-7:/data7 + - data2-8:/data8 + + minio3: + <<: *minio-common + hostname: minio3 + volumes: + - data3-1:/data1 + - data3-2:/data2 + - data3-3:/data3 + - data3-4:/data4 + - data3-5:/data5 + - data3-6:/data6 + - data3-7:/data7 + - data3-8:/data8 + + minio4: + <<: *minio-common + hostname: minio4 + volumes: + - data4-1:/data1 + - data4-2:/data2 + - data4-3:/data3 + - data4-4:/data4 + - data4-5:/data5 + - data4-6:/data6 + - data4-7:/data7 + - data4-8:/data8 + + nginx: + image: nginx:1.19.2-alpine + hostname: nginx + volumes: + - ./nginx.conf:/etc/nginx/nginx.conf:ro + ports: + - "9000:9000" + - "9001:9001" + depends_on: + - minio1 + - minio2 + - minio3 + - minio4 + +## By default this config uses default local driver, +## For custom volumes replace with volume driver configuration. +volumes: + data1-1: + data1-2: + data1-3: + data1-4: + data1-5: + data1-6: + data1-7: + data1-8: + + data2-1: + data2-2: + data2-3: + data2-4: + data2-5: + data2-6: + data2-7: + data2-8: + + data3-1: + data3-2: + data3-3: + data3-4: + data3-5: + data3-6: + data3-7: + data3-8: + + data4-1: + data4-2: + data4-3: + data4-4: + data4-5: + data4-6: + data4-7: + data4-8: diff --git a/docs/resiliency/nginx.conf b/docs/resiliency/nginx.conf new file mode 100644 index 00000000..cca82f6f --- /dev/null +++ b/docs/resiliency/nginx.conf @@ -0,0 +1,106 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 4096; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + sendfile on; + keepalive_timeout 65; + + # include /etc/nginx/conf.d/*.conf; + + upstream minio { + server minio1:9000; + server minio2:9000; + server minio3:9000; + server minio4:9000; + } + + upstream console { + ip_hash; + server minio1:9001; + server minio2:9001; + server minio3:9001; + server minio4:9001; + } + + server { + listen 9000; + listen [::]:9000; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + + proxy_connect_timeout 300; + # Default is HTTP/1, keepalive is only enabled in HTTP/1.1 + proxy_http_version 1.1; + proxy_set_header Connection ""; + chunked_transfer_encoding off; + + proxy_pass http://minio; + } + } + + server { + listen 9001; + listen [::]:9001; + server_name localhost; + + # To allow special characters in headers + ignore_invalid_headers off; + # Allow any size file to be uploaded. + # Set to a value such as 1000m; to restrict file size to a specific value + client_max_body_size 0; + # To disable buffering + proxy_buffering off; + proxy_request_buffering off; + + location / { + proxy_set_header Host $http_host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_set_header X-NginX-Proxy true; + + # This is necessary to pass the correct IP to be hashed + real_ip_header X-Real-IP; + + proxy_connect_timeout 300; + + # To support websocket + proxy_http_version 1.1; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + chunked_transfer_encoding off; + + proxy_pass http://console; + } + } +} diff --git a/docs/resiliency/resiliency-initial-script.sh b/docs/resiliency/resiliency-initial-script.sh new file mode 100755 index 00000000..fcdca6ff --- /dev/null +++ b/docs/resiliency/resiliency-initial-script.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# This script will run inside ubuntu-pod that is located at default namespace in the cluster +# This script will not and should not be executed in the self hosted runner + +echo "script failed" >resiliency-initial.log # assume initial state + +echo "sleep to wait for MinIO Server to be ready prior mc commands" +# https://github.com/minio/mc/issues/3599 + +MINIO_SERVER_URL="http://127.0.0.1:9000" +ALIAS_NAME=myminio +BUCKET="test-bucket" +SRC_DIR="/tmp/data" +INLINED_DIR="/tmp/inlined" +DEST_DIR="/tmp/dest" + +TIMEOUT=10 +while true; do + if [[ ${TIMEOUT} -le 0 ]]; then + echo retry: timeout while running: mc alias set + exit 1 + fi + eval ./mc alias set "${ALIAS_NAME}" "${MINIO_SERVER_URL}" minioadmin minioadmin && break + TIMEOUT=$((TIMEOUT - 1)) + sleep 1 +done + +./mc ready "${ALIAS_NAME}" + +./mc mb "${ALIAS_NAME}"/"${BUCKET}" +rm -rf "${SRC_DIR}" "${INLINED_DIR}" "${DEST_DIR}" && mkdir -p "${SRC_DIR}" "${INLINED_DIR}" "${DEST_DIR}" +for idx in {1..10}; do + # generate random nr of blocks + COUNT=$((RANDOM % 100 + 100)) + # generate random content + dd if=/dev/urandom bs=50K count="${COUNT}" of="${SRC_DIR}"/file"$idx" +done + +# create small object that will be inlined into xl.meta +dd if=/dev/urandom bs=50K count=1 of="${INLINED_DIR}"/inlined + +if ./mc cp --quiet --recursive "${SRC_DIR}/" "${ALIAS_NAME}"/"${BUCKET}"/initial-data/; then + if ./mc cp --quiet --recursive "${INLINED_DIR}/" "${ALIAS_NAME}"/"${BUCKET}"/inlined-data/; then + echo "script passed" >resiliency-initial.log + fi +fi diff --git a/docs/resiliency/resiliency-tests.sh b/docs/resiliency/resiliency-tests.sh new file mode 100755 index 00000000..12b093d3 --- /dev/null +++ b/docs/resiliency/resiliency-tests.sh @@ -0,0 +1,433 @@ +#!/usr/bin/env bash + +TESTS_RUN_STATUS=1 + +function cleanup() { + echo "Cleaning up MinIO deployment" + docker compose -f "${DOCKER_COMPOSE_FILE}" down --volumes + for container in $(docker ps -q); do + echo Removing docker $container + docker rm -f $container >/dev/null 2>&1 + docker wait $container + done +} + +function cleanup_and_prune() { + cleanup + docker system prune --volumes --force + docker image prune --all --force +} + +function verify_resiliency() { + docs/resiliency/resiliency-verify-script.sh + RESULT=$(grep "script passed" /dev/null 2>&1 + STATUS=$? + if [ $STATUS -eq 0 ]; then + DATA_DRIVE=1 + fi + + if [ $DATA_DRIVE -eq -1 ]; then + # Check for existence of file in erasure set 2 + docker exec resiliency-minio1-1 /bin/sh -c "stat /data5/test-bucket/$DIR/$FILE/xl.meta" >/dev/null 2>&1 + STATUS=$? + if [ $STATUS -eq 0 ]; then + DATA_DRIVE=5 + fi + fi + echo $DATA_DRIVE +} + +function test_resiliency_healing_missing_xl_metas() { + echo + echo -e "${GREEN}Running test_resiliency_healing_missing_xl_metas ...${NC}" + + DIR="initial-data" + FILE="file1" + DATA_DRIVE=$(find_erasure_set_for_file $FILE $DIR) + STATUS=$? + if [ $STATUS -ne 0 ]; then + echo -e "${RED}Could not find erasure set for file: ${FILE}${NC}" + echo -e "${RED}"${FUNCNAME[0]}" Failed${NC}" + TESTS_RUN_STATUS=$((TESTS_RUN_STATUS & 0)) + return 1 + fi + + # Remove single xl.meta -- status still green + OUTPUT=$(docker exec resiliency-minio1-1 /bin/sh -c "rm /data$((DATA_DRIVE))/test-bucket/initial-data/$FILE/xl.meta") + WANT='{ "before": { "color": "green", "missing": 1, "corrupted": 0 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }' + verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}" + + # Remove two xl.meta's -- status becomes yellow + OUTPUT=$(docker exec resiliency-minio1-1 /bin/sh -c "rm /data$((DATA_DRIVE))/test-bucket/initial-data/$FILE/xl.meta") + OUTPUT=$(docker exec resiliency-minio2-1 /bin/sh -c "rm /data$((DATA_DRIVE + 1))/test-bucket/initial-data/$FILE/xl.meta") + WANT='{ "before": { "color": "yellow", "missing": 2, "corrupted": 0 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }' + verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}" + + # Remove three xl.meta's -- status becomes red (3 missing) + OUTPUT=$(docker exec resiliency-minio1-1 /bin/sh -c "rm /data$((DATA_DRIVE))/test-bucket/initial-data/$FILE/xl.meta") + OUTPUT=$(docker exec resiliency-minio2-1 /bin/sh -c "rm /data$((DATA_DRIVE + 1))/test-bucket/initial-data/$FILE/xl.meta") + OUTPUT=$(docker exec resiliency-minio3-1 /bin/sh -c "rm /data$((DATA_DRIVE + 2))/test-bucket/initial-data/$FILE/xl.meta") + WANT='{ "before": { "color": "red", "missing": 3, "corrupted": 0 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }' + verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}" + + # Remove four xl.meta's -- status becomes red (4 missing) + OUTPUT=$(docker exec resiliency-minio1-1 /bin/sh -c "rm /data$((DATA_DRIVE))/test-bucket/initial-data/$FILE/xl.meta") + OUTPUT=$(docker exec resiliency-minio2-1 /bin/sh -c "rm /data$((DATA_DRIVE + 1))/test-bucket/initial-data/$FILE/xl.meta") + OUTPUT=$(docker exec resiliency-minio3-1 /bin/sh -c "rm /data$((DATA_DRIVE + 2))/test-bucket/initial-data/$FILE/xl.meta") + OUTPUT=$(docker exec resiliency-minio4-1 /bin/sh -c "rm /data$((DATA_DRIVE + 3))/test-bucket/initial-data/$FILE/xl.meta") + WANT='{ "before": { "color": "red", "missing": 4, "corrupted": 0 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }' + verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}" +} + +function test_resiliency_healing_truncated_parts() { + echo + echo -e "${GREEN}Running test_resiliency_healing_truncated_parts ...${NC}" + + DIR="initial-data" + FILE="file2" + DATA_DRIVE=$(find_erasure_set_for_file $FILE $DIR) + STATUS=$? + if [ $STATUS -ne 0 ]; then + echo -e "${RED}Could not find erasure set for file: ${FILE}${NC}" + echo -e "${RED}"${FUNCNAME[0]}" Failed${NC}" + TESTS_RUN_STATUS=$((TESTS_RUN_STATUS & 0)) + return 1 + fi + + # Truncate single part -- status still green + OUTPUT=$(docker exec resiliency-minio1-1 /bin/sh -c "truncate --size=10K /data$((DATA_DRIVE))/test-bucket/initial-data/$FILE/*/part.1") + WANT='{ "before": { "color": "green", "missing": 0, "corrupted": 1 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }' + verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}" + + # Truncate two parts -- status becomes yellow (2 missing) + OUTPUT=$(docker exec resiliency-minio2-1 /bin/sh -c "truncate --size=10K /data{$((DATA_DRIVE))..$((DATA_DRIVE + 1))}/test-bucket/initial-data/$FILE/*/part.1") + WANT='{ "before": { "color": "yellow", "missing": 0, "corrupted": 2 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }' + verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}" + + # Truncate three parts -- status becomes red (3 missing) + OUTPUT=$(docker exec resiliency-minio3-1 /bin/sh -c "truncate --size=10K /data{$((DATA_DRIVE))..$((DATA_DRIVE + 2))}/test-bucket/initial-data/$FILE/*/part.1") + WANT='{ "before": { "color": "red", "missing": 0, "corrupted": 3 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }' + verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}" + + # Truncate four parts -- status becomes red (4 missing) + OUTPUT=$(docker exec resiliency-minio4-1 /bin/sh -c "truncate --size=10K /data{$((DATA_DRIVE))..$((DATA_DRIVE + 3))}/test-bucket/initial-data/$FILE/*/part.1") + WANT='{ "before": { "color": "red", "missing": 0, "corrupted": 4 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }' + verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}" +} + +function induce_bitrot() { + local NODE=$1 + local DIR=$2 + local FILE=$3 + # Figure out the UUID of the directory where the `part.*` files are stored + UUID=$(docker exec resiliency-minio$NODE-1 /bin/sh -c "ls -l $DIR/test-bucket/initial-data/$FILE/*/part.1") + UUID=$(echo $UUID | cut -d " " -f 9 | cut -d "/" -f 6) + + # Determine head and tail size of file where we will introduce bitrot + FILE_SIZE=$(docker exec resiliency-minio$NODE-1 /bin/sh -c "stat --printf="%s" $DIR/test-bucket/initial-data/$FILE/$UUID/part.1") + TAIL_SIZE=$((FILE_SIZE - 32 * 2)) + + # Extract head and tail of file + $(docker exec resiliency-minio$NODE-1 /bin/sh -c "cat $DIR/test-bucket/initial-data/$FILE/$UUID/part.1 | head --bytes 32 > /tmp/head") + $(docker exec resiliency-minio$NODE-1 /bin/sh -c "cat $DIR/test-bucket/initial-data/$FILE/$UUID/part.1 | tail --bytes $TAIL_SIZE > /tmp/tail") + + # Corrupt the part by writing head twice followed by tail + $(docker exec resiliency-minio$NODE-1 /bin/sh -c "cat /tmp/head /tmp/head /tmp/tail > $DIR/test-bucket/initial-data/$FILE/$UUID/part.1") +} + +function test_resiliency_healing_induced_bitrot() { + echo + echo -e "${GREEN}Running test_resiliency_healing_induced_bitrot ...${NC}" + + DIR="initial-data" + FILE="file3" + DATA_DRIVE=$(find_erasure_set_for_file $FILE $DIR) + STATUS=$? + if [ $STATUS -ne 0 ]; then + echo -e "${RED}Could not find erasure set for file: ${FILE}${NC}" + echo -e "${RED}"${FUNCNAME[0]}" Failed${NC}" + TESTS_RUN_STATUS=$((TESTS_RUN_STATUS & 0)) + return 1 + fi + + # Induce bitrot in single part -- status still green + induce_bitrot "2" "/data"$((DATA_DRIVE + 1)) $FILE + WANT='{ "before": { "color": "green", "missing": 0, "corrupted": 1 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'", "deep": true} }' + verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}" + + # Induce bitrot in two parts -- status becomes yellow (2 corrupted) + induce_bitrot "2" "/data"$((DATA_DRIVE)) $FILE + induce_bitrot "1" "/data"$((DATA_DRIVE + 1)) $FILE + WANT='{ "before": { "color": "yellow", "missing": 0, "corrupted": 2 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'", "deep": true} }' + verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}" + + # Induce bitrot in three parts -- status becomes red (3 corrupted) + induce_bitrot "3" "/data"$((DATA_DRIVE)) $FILE + induce_bitrot "2" "/data"$((DATA_DRIVE + 1)) $FILE + induce_bitrot "1" "/data"$((DATA_DRIVE + 2)) $FILE + WANT='{ "before": { "color": "red", "missing": 0, "corrupted": 3 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'", "deep": true} }' + verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}" + + # Induce bitrot in four parts -- status becomes red (4 corrupted) + induce_bitrot "4" "/data"$((DATA_DRIVE)) $FILE + induce_bitrot "3" "/data"$((DATA_DRIVE + 1)) $FILE + induce_bitrot "2" "/data"$((DATA_DRIVE + 2)) $FILE + induce_bitrot "1" "/data"$((DATA_DRIVE + 3)) $FILE + WANT='{ "before": { "color": "red", "missing": 0, "corrupted": 4 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'", "deep": true} }' + verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}" +} + +function induce_bitrot_for_xlmeta() { + local NODE=$1 + local DIR=$2 + local FILE=$3 + + # Determine head and tail size of file where we will introduce bitrot + FILE_SIZE=$(docker exec resiliency-minio$NODE-1 /bin/sh -c "stat --printf="%s" $DIR/test-bucket/inlined-data/$FILE/xl.meta") + HEAD_SIZE=$((FILE_SIZE - 32 * 2)) + + # Extract head and tail of file + $(docker exec resiliency-minio$NODE-1 /bin/sh -c "cat $DIR/test-bucket/inlined-data/$FILE/xl.meta | head --bytes $HEAD_SIZE > /head") + $(docker exec resiliency-minio$NODE-1 /bin/sh -c "cat $DIR/test-bucket/inlined-data/$FILE/xl.meta | tail --bytes 32 > /tail") + + # Corrupt xl.meta by writing head followed by tail twice + $(docker exec resiliency-minio$NODE-1 /bin/sh -c "cat /head /tail tmp/tail > $DIR/test-bucket/inlined-data/$FILE/xl.meta") +} + +function test_resiliency_healing_inlined_metadata() { + echo + echo -e "${GREEN}Running test_resiliency_healing_inlined_metadata ...${NC}" + + DIR="inlined-data" + FILE="inlined" + DATA_DRIVE=$(find_erasure_set_for_file $FILE $DIR) + STATUS=$? + if [ $STATUS -ne 0 ]; then + echo -e "${RED}Could not find erasure set for file: ${FILE}${NC}" + echo -e "${RED}"${FUNCNAME[0]}" Failed${NC}" + TESTS_RUN_STATUS=$((TESTS_RUN_STATUS & 0)) + return 1 + fi + + # Induce bitrot in single inlined xl.meta -- status still green + induce_bitrot_for_xlmeta "2" "/data"$((DATA_DRIVE + 1)) $FILE + WANT='{ "before": { "color": "green", "missing": 0, "corrupted": 1 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }' + verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}" + + # Induce bitrot in two inlined xl.meta's -- status becomes yellow (2 corrupted) + induce_bitrot_for_xlmeta "3" "/data"$((DATA_DRIVE + 1)) $FILE + induce_bitrot_for_xlmeta "3" "/data"$((DATA_DRIVE + 2)) $FILE + WANT='{ "before": { "color": "yellow", "missing": 0, "corrupted": 2 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }' + verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}" + + # Induce bitrot in three inlined xl.meta's -- status becomes red (3 corrupted) + induce_bitrot_for_xlmeta "4" "/data"$((DATA_DRIVE + 1)) $FILE + induce_bitrot_for_xlmeta "4" "/data"$((DATA_DRIVE + 2)) $FILE + induce_bitrot_for_xlmeta "4" "/data"$((DATA_DRIVE + 3)) $FILE + WANT='{ "before": { "color": "red", "missing": 0, "corrupted": 3 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }' + verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}" + + # Induce bitrot in four inlined xl.meta's -- status becomes red (4 corrupted) + induce_bitrot_for_xlmeta "1" "/data"$((DATA_DRIVE)) $FILE + induce_bitrot_for_xlmeta "1" "/data"$((DATA_DRIVE + 1)) $FILE + induce_bitrot_for_xlmeta "1" "/data"$((DATA_DRIVE + 2)) $FILE + induce_bitrot_for_xlmeta "1" "/data"$((DATA_DRIVE + 3)) $FILE + WANT='{ "before": { "color": "red", "missing": 0, "corrupted": 4 }, "after": { "color": "green", "missing": 0, "corrupted": 0 }, "args": {"file": "'${FILE}'", "dir": "'${DIR}'"} }' + verify_resiliency_healing "${FUNCNAME[0]}" "${WANT}" +} + +function main() { + if [ ! -f ./mc ]; then + wget -q https://dl.minio.io/client/mc/release/linux-amd64/mc && chmod +x ./mc + fi + + export MC_HOST_myminio=http://minioadmin:minioadmin@localhost:9000 + + cleanup_and_prune + + # Run resiliency tests against MinIO + docker compose -f "${DOCKER_COMPOSE_FILE}" up -d + + # Initial setup + docs/resiliency/resiliency-initial-script.sh + RESULT=$(grep "script passed" resiliency-verify-failure.log # assume initial state + +ALIAS_NAME=myminio +BUCKET="test-bucket" +DEST_DIR="/tmp/dest" + +OUT=$(./mc cp --quiet --recursive "${ALIAS_NAME}"/"${BUCKET}"/initial-data/ "${DEST_DIR}"/) +RET=${?} +if [ ${RET} -ne 0 ]; then + # It is a success scenario as get objects should fail + echo "GET objects failed as expected" + echo "script passed" >resiliency-verify-failure.log + exit 0 +else + echo "GET objects expected to fail, but succeeded: ${OUT}" +fi diff --git a/docs/resiliency/resiliency-verify-healing-script.sh b/docs/resiliency/resiliency-verify-healing-script.sh new file mode 100755 index 00000000..8127d03d --- /dev/null +++ b/docs/resiliency/resiliency-verify-healing-script.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash + +echo "script failed" >resiliency-verify-healing.log # assume initial state + +# Extract arguments from json object ... +FILE=$(echo $1 | jq -r '.args.file') +DIR=$(echo $1 | jq -r '.args.dir') +DEEP=$(echo $1 | jq -r '.args.deep') +WANT=$(echo $1 | jq 'del(.args)') # ... and remove args from wanted result + +ALIAS_NAME=myminio +BUCKET="test-bucket" +JQUERY='select(.name=="'"${BUCKET}"'/'"${DIR}"'/'"${FILE}"'") | {"before":{"color": .before.color, "missing": .before.missing, "corrupted": .before.corrupted},"after":{"color": .after.color, "missing": .after.missing, "corrupted": .after.corrupted}}' +if [ "$DEEP" = "true" ]; then + SCAN_DEEP="--scan=deep" +fi + +GOT=$(./mc admin heal --json ${SCAN_DEEP} ${ALIAS_NAME}/${BUCKET}/${DIR}/${FILE}) +GOT=$(echo $GOT | jq "${JQUERY}") + +if [ "$(echo "$GOT" | jq -S .)" = "$(echo "$WANT" | jq -S .)" ]; then + echo "script passed" >resiliency-verify-healing.log +else + echo "Error during healing:" + echo "----GOT: "$GOT + echo "---WANT: "$WANT +fi diff --git a/docs/resiliency/resiliency-verify-script.sh b/docs/resiliency/resiliency-verify-script.sh new file mode 100755 index 00000000..50220b0a --- /dev/null +++ b/docs/resiliency/resiliency-verify-script.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +echo "script failed" >resiliency-verify.log # assume initial state + +ALIAS_NAME=myminio +BUCKET="test-bucket" +SRC_DIR="/tmp/data" +DEST_DIR="/tmp/dest" + +./mc admin config set "$ALIAS_NAME" api requests_max=400 + +OBJ_COUNT_AFTER_STOP=$(./mc ls "${ALIAS_NAME}"/"${BUCKET}"/initial-data/ | wc -l) +# Count should match the initial count of 10 +if [ "${OBJ_COUNT_AFTER_STOP}" -ne 10 ]; then + echo "Expected 10 objects; received ${OBJ_COUNT_AFTER_STOP}" + exit 1 +fi + +./mc ready "${ALIAS_NAME}" --json + +OUT=$(./mc cp --quiet "${SRC_DIR}"/* "${ALIAS_NAME}"/"${BUCKET}"/new-data/) +RET=${?} +if [ ${RET} -ne 0 ]; then + echo "Error copying objects to new prefix: ${OUT}" + exit 1 +fi + +OBJ_COUNT_AFTER_COPY=$(./mc ls "${ALIAS_NAME}"/"${BUCKET}"/new-data/ | wc -l) +if [ "${OBJ_COUNT_AFTER_COPY}" -ne "${OBJ_COUNT_AFTER_STOP}" ]; then + echo "Expected ${OBJ_COUNT_AFTER_STOP} objects; received ${OBJ_COUNT_AFTER_COPY}" + exit 1 +fi + +OUT=$(./mc cp --quiet --recursive "${ALIAS_NAME}"/"${BUCKET}"/new-data/ "${DEST_DIR}"/) +RET=${?} +if [ ${RET} -ne 0 ]; then + echo "Get objects failed: ${OUT}" + exit 1 +fi + +# Check if check sums match for source and destination directories +CHECK_SUM_SRC=$(sha384sum <(sha384sum "${SRC_DIR}"/* | cut -d " " -f 1 | sort) | cut -d " " -f 1) +CHECK_SUM_DEST=$(sha384sum <(sha384sum "${DEST_DIR}"/* | cut -d " " -f 1 | sort) | cut -d " " -f 1) +if [ "${CHECK_SUM_SRC}" != "${CHECK_SUM_DEST}" ]; then + echo "Checksum verification of source files and destination files failed" + exit 1 +fi + +echo "script passed" >resiliency-verify.log diff --git a/docs/screenshots/Architecture-diagram_distributed_16.jpg b/docs/screenshots/Architecture-diagram_distributed_16.jpg new file mode 100644 index 00000000..49eb5400 Binary files /dev/null and b/docs/screenshots/Architecture-diagram_distributed_16.jpg differ diff --git a/docs/screenshots/Architecture-diagram_distributed_16.png b/docs/screenshots/Architecture-diagram_distributed_16.png new file mode 100644 index 00000000..dc844595 Binary files /dev/null and b/docs/screenshots/Architecture-diagram_distributed_16.png differ diff --git a/docs/screenshots/Architecture-diagram_distributed_16.svg b/docs/screenshots/Architecture-diagram_distributed_16.svg new file mode 100644 index 00000000..6fbb1ace --- /dev/null +++ b/docs/screenshots/Architecture-diagram_distributed_16.svg @@ -0,0 +1,1063 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + INTERNET + + + 16-PORT10 GIGABITETHERNETSWITCH + + + 4 SERVERS + + + 16 8TBDRIVES + +NODE 1 +NODE 2 +NODE 3 +NODE 4 +192.168.1.11:9000 +192.168.1.12:9000 +192.168.1.13:9000 +192.168.1.14:9000 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + EXPORT 4 + EXPORT 3 + EXPORT 1 + EXPORT 2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + EXPORT 4 + EXPORT 3 + EXPORT 1 + EXPORT 2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + EXPORT 4 + EXPORT 3 + EXPORT 1 + EXPORT 2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + EXPORT 4 + EXPORT 3 + EXPORT 1 + EXPORT 2 + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/screenshots/Architecture-diagram_distributed_8.jpg b/docs/screenshots/Architecture-diagram_distributed_8.jpg new file mode 100644 index 00000000..9dcfcaa1 Binary files /dev/null and b/docs/screenshots/Architecture-diagram_distributed_8.jpg differ diff --git a/docs/screenshots/Architecture-diagram_distributed_8.png b/docs/screenshots/Architecture-diagram_distributed_8.png new file mode 100644 index 00000000..cff2f1df Binary files /dev/null and b/docs/screenshots/Architecture-diagram_distributed_8.png differ diff --git a/docs/screenshots/Architecture-diagram_distributed_8.svg b/docs/screenshots/Architecture-diagram_distributed_8.svg new file mode 100644 index 00000000..7dfbf7f2 --- /dev/null +++ b/docs/screenshots/Architecture-diagram_distributed_8.svg @@ -0,0 +1 @@ +Architecture-diagram_Distributed 8ContainersContainersInfrastructureOrchestrationProvision object storage as containersMINIOMySQLREDISNODE JSDOCKERDOCKERDOCKERDOCKERDOCKER SWARMMESOSPHEREKUBERNETESHOST 1TENANT 1TENANT 2TENANT 3DISK 1MINIO 1MINIO 2MINIO 3900190029003Example 1: 3 tenants on single host, single driveHOST 1TENANT 1TENANT 2TENANT 3DISK 1DISK 2DISK 3DISK 4MINIO 1MINIO 2MINIO 3900190029003Example 2: 3 tenants on single host, 4 drives (erasure code)TENANT 1TENANT 2TENANT 3HOST 1DISK 1MINIO 1MINIO 2MINIO 3HOST 2DISK 1MINIO 1MINIO 2MINIO 3HOST 4DISK 1MINIO 1MINIO 2MINIO 3HOST 3DISK 1MINIO 1MINIO 2MINIO 3900190039003900190029002900190029003900190029003Example 3: 4 node distributed setupCHECKSUMCHECKSUMCHECKSUMCHECKSUMDATA BLOCK1238CHECKSUMCHECKSUMCHECKSUMCHECKSUMPARITY BLOCK1P2P3P8POBJECT ERASURE-CODED OVER 16 DRIVESTolerates up to any 8 disk failuresINTERNET16-PORT10 GIGABITETHERNETSWITCH8 SERVERSNODE 1NODE 2NODE 3NODE 4NODE 5NODE 6NODE 7NODE 8192.168.1.11:9000192.168.1.12:9000192.168.1.13:9000192.168.1.14:9000192.168.1.15:9000192.168.1.16:9000192.168.1.17:9000192.168.1.18:9000EXPORT 1EXPORT 1EXPORT 1EXPORT 1EXPORT 1 EXPORT 1EXPORT 1 8 8TBDRIVESEXPORT 1INTERNET16-PORT10 GIGABITETHERNETSWITCH4 SERVERS16 8TBDRIVESNODE 1NODE 2NODE 3NODE 4192.168.1.11:9000192.168.1.12:9000192.168.1.13:9000192.168.1.14:9000EXPORT 4EXPORT 3 EXPORT 1EXPORT 2EXPORT 4EXPORT 3 EXPORT 1EXPORT 2EXPORT 4EXPORT 3 EXPORT 1EXPORT 2EXPORT 4EXPORT 3 EXPORT 1EXPORT 2INTERNET16-PORT10 GIGABITETHERNETSWITCH8 SERVERSNODE 1NODE 2NODE 3NODE 4NODE 5NODE 6NODE 7NODE 8192.168.1.11:9000192.168.1.12:9000192.168.1.13:9000192.168.1.14:9000192.168.1.15:9000192.168.1.16:9000192.168.1.17:9000192.168.1.18:9000EXPORT 1EXPORT 1EXPORT 1EXPORT 1EXPORT 1 EXPORT 1EXPORT 1 8 8TBDRIVESEXPORT 1 \ No newline at end of file diff --git a/docs/screenshots/Architecture-diagram_distributed_nm.png b/docs/screenshots/Architecture-diagram_distributed_nm.png new file mode 100644 index 00000000..0ba9284e Binary files /dev/null and b/docs/screenshots/Architecture-diagram_distributed_nm.png differ diff --git a/docs/screenshots/Example-1.jpg b/docs/screenshots/Example-1.jpg new file mode 100644 index 00000000..f46a7d21 Binary files /dev/null and b/docs/screenshots/Example-1.jpg differ diff --git a/docs/screenshots/Example-1.png b/docs/screenshots/Example-1.png new file mode 100644 index 00000000..7408107d Binary files /dev/null and b/docs/screenshots/Example-1.png differ diff --git a/docs/screenshots/Example-2.jpg b/docs/screenshots/Example-2.jpg new file mode 100644 index 00000000..838f5c88 Binary files /dev/null and b/docs/screenshots/Example-2.jpg differ diff --git a/docs/screenshots/Example-2.png b/docs/screenshots/Example-2.png new file mode 100644 index 00000000..02e98adf Binary files /dev/null and b/docs/screenshots/Example-2.png differ diff --git a/docs/screenshots/Example-3.jpg b/docs/screenshots/Example-3.jpg new file mode 100644 index 00000000..e0394cff Binary files /dev/null and b/docs/screenshots/Example-3.jpg differ diff --git a/docs/screenshots/Example-3.png b/docs/screenshots/Example-3.png new file mode 100644 index 00000000..581886bb Binary files /dev/null and b/docs/screenshots/Example-3.png differ diff --git a/docs/screenshots/Minio_Cloud_Native_Arch.jpg b/docs/screenshots/Minio_Cloud_Native_Arch.jpg new file mode 100644 index 00000000..c4ca5ceb Binary files /dev/null and b/docs/screenshots/Minio_Cloud_Native_Arch.jpg differ diff --git a/docs/screenshots/Minio_Cloud_Native_Arch.png b/docs/screenshots/Minio_Cloud_Native_Arch.png new file mode 100644 index 00000000..d2946f49 Binary files /dev/null and b/docs/screenshots/Minio_Cloud_Native_Arch.png differ diff --git a/docs/screenshots/Minio_Cloud_Native_Arch.svg b/docs/screenshots/Minio_Cloud_Native_Arch.svg new file mode 100644 index 00000000..c5e075d1 --- /dev/null +++ b/docs/screenshots/Minio_Cloud_Native_Arch.svg @@ -0,0 +1,583 @@ + + + + + + + + + + + + + + + + + + + + + +Containers +Containers +Infrastructure +Orchestration + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Provision object storage as Containers +MINIO +MySQL +REDIS +NODE JS + + DOCKER + + + DOCKER + + + DOCKER + + + DOCKER + + + DOCKER SWARM + + + MESOSPHERE + + + KUBERNETES + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/screenshots/erasure-code.jpg b/docs/screenshots/erasure-code.jpg new file mode 100644 index 00000000..440f307b Binary files /dev/null and b/docs/screenshots/erasure-code.jpg differ diff --git a/docs/screenshots/erasure-code.png b/docs/screenshots/erasure-code.png new file mode 100644 index 00000000..61d8bcbc Binary files /dev/null and b/docs/screenshots/erasure-code.png differ diff --git a/docs/screenshots/erasure-code.svg b/docs/screenshots/erasure-code.svg new file mode 100644 index 00000000..219a4a2d --- /dev/null +++ b/docs/screenshots/erasure-code.svg @@ -0,0 +1,341 @@ + + + + + + + + + + + + + + + + + CHECKSUM + + + CHECKSUM + + + CHECKSUM + + + CHECKSUM + + + + + + + + + + +OBJECT ERASURE-CODED OVER 16 DRIVESTolerates up to any 8 disk failures + + + + + + + DATA BLOCK + + + + + + + + +1 +2 +3 +8 +CHECKSUM +CHECKSUM +CHECKSUM +CHECKSUM + + + PARITY BLOCK + +1P +2P +3P +8P + diff --git a/docs/screenshots/pic1.png b/docs/screenshots/pic1.png new file mode 100644 index 00000000..4ba49cba Binary files /dev/null and b/docs/screenshots/pic1.png differ diff --git a/docs/screenshots/pic2.png b/docs/screenshots/pic2.png new file mode 100644 index 00000000..99065892 Binary files /dev/null and b/docs/screenshots/pic2.png differ diff --git a/docs/security/README.md b/docs/security/README.md new file mode 100644 index 00000000..9d56e4b8 --- /dev/null +++ b/docs/security/README.md @@ -0,0 +1,182 @@ +# MinIO Security Overview [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +## Server-Side Encryption + +MinIO supports two different types of server-side encryption ([SSE](#sse)): + +- **SSE-C**: The MinIO server en/decrypts an object with a secret key provided by the S3 client as part of the HTTP request headers. Therefore, [SSE-C](#ssec) requires TLS/HTTPS. +- **SSE-S3**: The MinIO server en/decrypts an object with a secret key managed by a KMS. Therefore, MinIO requires a valid KMS configuration for [SSE-S3](#sses3). + +### Server-Side Encryption - Preliminaries + +#### Secret Keys + +The MinIO server uses a unique, randomly generated secret key per object also known as, Object Encryption Key ([OEK](#oek)). Neither the client-provided SSE-C key nor the KMS-managed key is directly used to en/decrypt an object. Instead, the OEK is stored as part of the object metadata next to the object in an encrypted form. To en/decrypt the OEK another secret key is needed also known as, Key Encryption Key ([KEK](#kek)). + +The MinIO server runs a key-derivation algorithm to generate the KEK using a pseudo-random function ([PRF](#prf)): +`KEK := PRF(EK, IV, context_values)` where: + +- [EK](#ek): is the external key. In case of SSE-C this is the client-provided key. In case of SSE-S3 this is secret key generated by the KMS. For further details see [SSE-C](#Server-Side-Encryption-with-client-provided-Keys) or [SSE-S3](#Server-Side-Encryption-with-a-KMS). +- [IV](#iv): is a randomly generated initialization vector. It is public and part of the object metadata. +- `context_values`: are values like the bucket and object name and other information which should be cryptographically bound to the KEK. + +To summarize for any encrypted object there exists (at least) three different keys: + +- [OEK](#oek): A secret and unique key used to encrypted the object, stored in an encrypted form as part of the object metadata and only loaded to RAM in plaintext during en/decrypting the object. +- [KEK](#kek): A secret and unique key used to en/decrypt the OEK and never stored anywhere. It is(re-)generated whenever en/decrypting an object using an external secret key and public parameters. +- [EK](#ek): An external secret key - either the SSE-C client-provided key or a secret key generated by the KMS. + +#### Content Encryption + +The MinIO server uses an authenticated encryption scheme ([AEAD](#aead)) to en/decrypt and authenticate the object content. The AEAD is combined with some state to build a *Secure Channel*. A *Secure Channel* is a cryptographic construction that ensures confidentiality and integrity of the processed data. In particular the *Secure Channel* splits the plaintext content into fixed size chunks and en/decrypts each chunk separately using a unique key-nonce combination. + +##### Figure 1 - Secure Channel construction + +``` +plaintext := chunk_0 || chunk_1 || chunk_2 || ... + | | | + | | | + AEAD <- key, nonce + 0 AEAD <- key, nonce + 1 AEAD <- key, nonce + 2 ... + | | | + | | | +ciphertext := sealed_chunk_0 || sealed_chunk_1 || sealed_chunk_2 || ... +``` + +In case of a S3 multi-part operation each part is en/decrypted with the scheme shown in Figure 1. However, for each part a unique secret key is derived from the OEK and the part number using a PRF. So in case of multi-part not the OEK but the output of `PRF(OEK, part_id)` is used as secret key. + +#### Cryptographic Primitives + +The SSE schemes described in [Secret Keys](#Secret-Keys) and [Content Encryption](#Content-Encryption) are generic over the cryptographic primitives. However, the MinIO server uses the following cryptographic primitive implementations: + +- [PRF](#prf): HMAC-SHA-256 +- [AEAD](#aead): AES-256-GCM if the CPU supports AES-NI, ChaCha20-Poly1305 otherwise. More specifically AES-256-GCM is only selected for X86-64 CPUs with AES-NI extension. + +Further any secret key (apart from the KMS-generated ones) is 256 bits long. The KMS-generated keys may be 256 bits but this depends on the KMS capabilities and configuration. + +The *Secure Channel* splits the object content into chunks of a fixed size of `65536` bytes. The last chunk may be smaller to avoid adding additional overhead and is treated specially to prevent truncation attacks. The nonce value is 96 bits long and generated randomly per object / multi-part part. The *Secure Channel* supports plaintexts up to `65536 * 2^32 = 256 TiB`. + +#### Randomness + +The MinIO server generates unique keys and other cryptographic values using a cryptographically secure pseudo-random number generator ([CSPRNG](#csprng)). However, in the context of SSE, the MinIO server does not require that the CSPRNG generates values that are indistinguishable from truly random bit strings. Instead, it is sufficient if the generated values are unique - which is a weaker requirement. Nevertheless other parts - for example the TLS-stack - may require that CSPRNG-generated values are indistinguishable from truly random bit strings. + +### Server-Side Encryption with client-provided Keys + +SSE-C allows an S3 client to en/decrypt an object at the MinIO server. Therefore the S3 client sends a secret key as part of the HTTP request. This secret key is **never** stored by the MinIO server and only resides in RAM during the en/decryption process. + +MinIO does not assume or require that the client-provided key is unique. It may be used for multiple objects or buckets. Especially a single client-provided key may be used for all objects - even though all objects must be treated as compromised if that key is ever compromised. + +#### Key rotation + +S3 clients can change the client-provided key of an existing object. Therefore an S3 client must perform a S3 COPY operation where the copy source and destination are equal. Further the COPY request headers must contain the current and the new client key: + +- `X-Amz-Server-Side-Encryption-Customer-Key`: Base64 encoded new key. +- `X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key`: Base64 encoded current key. + +Such a special COPY request is also known as S3 SSE-C key rotation. + +### Server-Side Encryption with a KMS + +SSE-S3 allows an S3 client to en/decrypt an object at the MinIO server using a KMS. The MinIO +server only assumes that the KMS provides two services: + +- `GenerateKey`: Takes a key ID and generates a new data key from a master key referenced by the key ID. It returns the new data key in two different forms: The plain data key and the data key encrypted using the master key. + +- `DecryptKey`: Takes a key ID and an encrypted data key and returns the plain data key - the decryption of the encrypted data key using the master key referenced by the key ID - on success or an error otherwise. + +More details about supported KMS implementations and configuration can be found at the [KMS guide](https://github.com/minio/minio/blob/master/docs/kms/README.md). + +The MinIO server requests a new data key from the KMS for each uploaded object and uses that data key as EK. Additionally it stores the encrypted form of the data key and the master key ID as part of the object metadata. The plain data only resides in RAM during the en/decryption process. The MinIO server does not store any SSE-related key at the KMS. Instead the KMS is treated as trusted component that performs key sealing/unsealing operations to build a key hierarchy: + +#### Figure 2 - KMS key hierarchy + +``` + CMK (master key) + | + +-----------------------------------+-----------------------------------+ + | | | + +-------+----------------+ +-------+----------------+ ... + | EK_1 | EK_1_encrypted | | EK_2 | EK_2_encrypted | + +---+----------+---------+ +---+----------+---------+ + | | | | + | | | | + +---+---+ | +---+---+ | + | KEK_1 | | | KEK_2 | | + +---+---+ | +---+---+ | + | | | | + | | | | + +---+---+ | +---+---+ | + | OEK_1 | | | OEK_2 | | + +---+---+ | +---+---+ | + | | + | | + | | + +---------+---------+ +---------+---------+ + | object_metadata_1 | | object_metadata_2 | + +-------------------+ +-------------------+ +``` + +#### Key rotation - Basic Operation + +The MinIO server supports key rotation for SSE-S3 encrypted objects. The minio server decrypts the OEK using the current encrypted data key and the master key ID of the object metadata. If this succeeds, the server requests a new data key from the KMS using the master key ID of the **current MinIO KMS configuration** and re-wraps the *OEK* with a new *KEK* derived from the new data key / EK: + +##### Figure 3 - KMS data key rotation + +``` + object metadata KMS + | | + | +----------------+ 1a | +-------+ + |-------------------->| EK_1_encrypted |-----------|->| CMK_1 | + | +----------------+ | +---+---+ + | | | + | +---------------+ +------+ 1b | | + |------------->| OEK_encrypted | | EK_1 |<---|------+ + | +-------+-------+ +------+ | + | \ / | + | \___ 2 ___/ | + | \___/ | + | | | + | +--+--+ | + | | OEK | | +-------+ + | +--+--+ | | CMK_2 | + | | | +---+---+ + | | | | + | 5 +----------------+ |4 +------+ 3a | | + |<------| OEK_encrypted' |<----+-------| EK_2 |<---|------+ + | +----------------+ +------+ | | + | +----------------+ 3b | | + |<-------------------| EK_2_encrypted |<-----------|------+ + | +----------------+ | + | | + + +1a) Send encrypted data key and master key ID to KMS. +1b) Receive decrypted data key. +2) Decrypt encrypted object key with the KEK derived from the data key. +3a) Receive new plain data key from the KMS using the master key ID of the server config. +3b) Receive encrypted form of the data key from the KMS. +4) Derive a new KEK from the new data key and re-encrypt the OEK with it. +5) Store the encrypted OEK encrypted data key and master key ID in object metadata. + ``` + +Only the root/admin user can perform an SSE-S3 key rotation using the Admin-API via [mc](https://github.com/minio/mc). For more details about how to perform key management operations using the CLI refer to [mc admin guide](https://github.com/minio/mc/blob/master/docs/minio-admin-complete-guide.md) or run `mc admin kms key`. + +#### Secure Erasure and Locking + +The MinIO server requires an available KMS to en/decrypt SSE-S3 encrypted objects. Therefore it is possible to erase or lock some or all encrypted objects. For example in case of a detected attack or other emergency situations the following actions can be taken: + +- Seal the KMS such that it cannot be accessed by MinIO server anymore. That will lock **all** SSE-S3 encrypted objects protected by master keys stored on the KMS. All these objects can not be decrypted as long as the KMS is sealed. +- Seal/Unmount one/some master keys. That will lock all SSE-S3 encrypted objects protected by these master keys. All these objects can not be decrypted as long as the key(s) are sealed. +- Delete one/some master keys. From a security standpoint, this is equal to erasing all SSE-S3 encrypted objects protected by these master keys. All these objects are lost forever as they cannot be decrypted. Especially deleting all master keys at the KMS is equivalent to secure erasing all SSE-S3 encrypted objects. + +## Acronyms + +- **AEAD**: Authenticated Encryption with Associated Data +- **CSPRNG**: Cryptographically Secure Pseudo Random Number Generator +- **EK**: External Key +- **IV**: Initialization Vector +- **KEK**: Key Encryption Key +- **OEK**: Object Encryption Key +- **PRF**: Pseudo Random Function +- **SSE**: Server-Side Encryption +- **SSE-C**: Server-Side Encryption with client-provided Keys +- **SSE-S3**: Server-Side Encryption with a KMS diff --git a/docs/select/README.md b/docs/select/README.md new file mode 100644 index 00000000..7aff3553 --- /dev/null +++ b/docs/select/README.md @@ -0,0 +1,131 @@ +# Select API Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +Traditional retrieval of objects is always as whole entities, i.e GetObject for a 5 GiB object, will always return 5 GiB of data. S3 Select API allows us to retrieve a subset of data by using simple SQL expressions. By using Select API to retrieve only the data needed by the application, drastic performance improvements can be achieved. + +You can use the Select API to query objects with following features: + +- Objects must be in CSV, JSON, or Parquet(*) format. +- UTF-8 is the only encoding type the Select API supports. +- GZIP or BZIP2 - CSV and JSON files can be compressed using GZIP, BZIP2, [ZSTD](https://facebook.github.io/zstd/), and streaming formats of [LZ4](https://lz4.github.io/lz4/), [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) and [SNAPPY](http://google.github.io/snappy/). +- Parquet API supports columnar compression for using GZIP, Snappy, LZ4. Whole object compression is not supported for Parquet objects. +- Server-side encryption - The Select API supports querying objects that are protected with server-side encryption. + +Type inference and automatic conversion of values is performed based on the context when the value is un-typed (such as when reading CSV data). If present, the CAST function overrides automatic conversion. + +The [mc sql](https://docs.min.io/community/minio-object-store/reference/minio-mc/mc-sql.html) command can be used for executing queries using the command line. + +(*) Parquet is disabled on the MinIO server by default. See below how to enable it. + +## Enabling Parquet Format + +Parquet is DISABLED by default since hostile crafted input can easily crash the server. + +If you are in a controlled environment where it is safe to assume no hostile content can be uploaded to your cluster you can safely enable Parquet. +To enable Parquet set the environment variable `MINIO_API_SELECT_PARQUET=on`. + +## Example using Python API + +### 1. Prerequisites + +- Install MinIO Server from [here](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html#procedure). +- Familiarity with AWS S3 API. +- Familiarity with Python and installing dependencies. + +### 2. Install boto3 + +Install `aws-sdk-python` from AWS SDK for Python official docs [here](https://aws.amazon.com/sdk-for-python/) + +### 3. Example + +As an example, let us take a gzip compressed CSV file. Without S3 Select, we would need to download, decompress and process the entire CSV to get the data you needed. With Select API, can use a simple SQL expression to return only the data from the CSV you’re interested in, instead of retrieving the entire object. Following Python example shows how to retrieve the first column `Location` from an object containing data in CSV format. + +Please replace ``endpoint_url``,``aws_access_key_id``, ``aws_secret_access_key``, ``Bucket`` and ``Key`` with your local setup in this ``select.py`` file. + +```py +#!/usr/bin/env/env python3 +import boto3 + +s3 = boto3.client('s3', + endpoint_url='http://localhost:9000', + aws_access_key_id='minio', + aws_secret_access_key='minio123', + region_name='us-east-1') + +r = s3.select_object_content( + Bucket='mycsvbucket', + Key='sampledata/TotalPopulation.csv.gz', + ExpressionType='SQL', + Expression="select * from s3object s where s.Location like '%United States%'", + InputSerialization={ + 'CSV': { + "FileHeaderInfo": "USE", + }, + 'CompressionType': 'GZIP', + }, + OutputSerialization={'CSV': {}}, +) + +for event in r['Payload']: + if 'Records' in event: + records = event['Records']['Payload'].decode('utf-8') + print(records) + elif 'Stats' in event: + statsDetails = event['Stats']['Details'] + print("Stats details bytesScanned: ") + print(statsDetails['BytesScanned']) + print("Stats details bytesProcessed: ") + print(statsDetails['BytesProcessed']) +``` + +## 4. Run the Program + +Upload a sample dataset to MinIO using the following commands. + +```sh +curl "https://population.un.org/wpp/Download/Files/1_Indicators%20(Standard)/CSV_FILES/WPP2019_TotalPopulationBySex.csv" > TotalPopulation.csv +mc mb myminio/mycsvbucket +gzip TotalPopulation.csv +mc cp TotalPopulation.csv.gz myminio/mycsvbucket/sampledata/ +``` + +Now let us proceed to run our select example to query for `Location` which matches `United States`. + +```sh +$ python3 select.py +840,United States of America,2,Medium,1950,1950.5,79233.218,79571.179,158804.395 + +840,United States of America,2,Medium,1951,1951.5,80178.933,80726.116,160905.035 + +840,United States of America,2,Medium,1952,1952.5,81305.206,82019.632,163324.851 + +840,United States of America,2,Medium,1953,1953.5,82565.875,83422.307,165988.190 +.... +.... +.... + +Stats details bytesScanned: +6758866 +Stats details bytesProcessed: +25786743 +``` + +For a more detailed SELECT SQL reference, please see [here](https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-select.html) + +## 5. Explore Further + +- [Use `mc` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc.html) +- [Use `mc sql` with MinIO Server](https://docs.min.io/community/minio-object-store/reference/minio-mc/mc-sql.html#command-mc.sql) +- [Use `minio-go` SDK with MinIO Server](https://docs.min.io/community/minio-object-store/developers/go/minio-go.html) +- [Use `aws-cli` with MinIO Server](https://docs.min.io/community/minio-object-store/integrations/aws-cli-with-minio.html) +- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html) + +## 6. Implementation Status + +- Full AWS S3 [SELECT SQL](https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-select.html) syntax is supported. +- All [operators](https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-operators.html) are supported. +- All aggregation, conditional, type-conversion and string functions are supported. +- JSON path expressions such as `FROM S3Object[*].path` are not yet evaluated. +- Large numbers (outside of the signed 64-bit range) are not yet supported. +- The Date [functions](https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-date.html) `DATE_ADD`, `DATE_DIFF`, `EXTRACT` and `UTCNOW` along with type conversion using `CAST` to the `TIMESTAMP` data type are currently supported. +- AWS S3's [reserved keywords](https://docs.aws.amazon.com/AmazonS3/latest/dev/s3-glacier-select-sql-reference-keyword-list.html) list is not yet respected. +- CSV input fields (even quoted) cannot contain newlines even if `RecordDelimiter` is something else. diff --git a/docs/select/select.py b/docs/select/select.py new file mode 100644 index 00000000..3c9a5e85 --- /dev/null +++ b/docs/select/select.py @@ -0,0 +1,33 @@ +#!/usr/bin/env/env python3 +import boto3 + +s3 = boto3.client('s3', + endpoint_url='http://localhost:9000', + aws_access_key_id='minio', + aws_secret_access_key='minio123', + region_name='us-east-1') + +r = s3.select_object_content( + Bucket='mycsvbucket', + Key='sampledata/TotalPopulation.csv.gz', + ExpressionType='SQL', + Expression="select * from s3object s where s.Location like '%United States%'", + InputSerialization={ + 'CSV': { + "FileHeaderInfo": "USE", + }, + 'CompressionType': 'GZIP', + }, + OutputSerialization={'CSV': {}}, +) + +for event in r['Payload']: + if 'Records' in event: + records = event['Records']['Payload'].decode('utf-8') + print(records) + elif 'Stats' in event: + statsDetails = event['Stats']['Details'] + print("Stats details bytesScanned: ") + print(statsDetails['BytesScanned']) + print("Stats details bytesProcessed: ") + print(statsDetails['BytesProcessed']) diff --git a/docs/site-replication/README.md b/docs/site-replication/README.md new file mode 100644 index 00000000..8edcc09d --- /dev/null +++ b/docs/site-replication/README.md @@ -0,0 +1,63 @@ +# Automatic Site Replication + +This feature allows multiple independent MinIO sites (or clusters) that are using the same external IDentity Provider (IDP) to be configured as replicas. In this situation the set of replica sites are referred to as peer sites or just sites. When site-replication is enabled on a set of sites, the following changes are replicated to all other sites: + +- Creation and deletion of buckets and objects +- Creation and deletion of all IAM users, groups, policies and their mappings to users or groups +- Creation of STS credentials +- Creation and deletion of service accounts (except those owned by the root user) +- Changes to Bucket features such as: + - Bucket Policies + - Bucket Tags + - Bucket Object-Lock configurations (including retention and legal hold configuration) + - Bucket Encryption configuration + +> NOTE: Bucket versioning is automatically enabled for all new and existing buckets on all replicated sites. + +The following Bucket features will **not be replicated**, is designed to differ between sites: + +- Bucket notification configuration +- Bucket lifecycle (ILM) configuration + +## Pre-requisites + +- Initially, only **one** of the sites added for replication may have data. After site-replication is successfully configured, this data is replicated to the other (initially empty) sites. Subsequently, objects may be written to any of the sites, and they will be replicated to all other sites. + +- **Removing a site** is not allowed from a set of replicated sites once configured. +- All sites must be using the **same** external IDP(s) if any. +- For [SSE-S3 or SSE-KMS encryption via KMS](https://docs.min.io/community/minio-object-store/operations/server-side-encryption.html "MinIO KMS Guide"), all sites **must** have access to a central KMS deployment. This can be achieved via a central KES server or multiple KES servers (say one per site) connected via a central KMS (Vault) server. + +## Configuring Site Replication + +- Configure an alias in `mc` for each of the sites. For example if you have three MinIO sites, you may run: + +```sh +mc alias set minio1 https://minio1.example.com:9000 adminuser adminpassword +mc alias set minio2 https://minio2.example.com:9000 adminuser adminpassword +mc alias set minio3 https://minio3.example.com:9000 adminuser adminpassword +``` + +or + +```sh +export MC_HOST_minio1=https://adminuser:adminpassword@minio1.example.com +export MC_HOST_minio2=https://adminuser:adminpassword@minio2.example.com +export MC_HOST_minio3=https://adminuser:adminpassword@minio3.example.com +``` + +- Add site replication configuration with: + +```sh +mc admin replicate add minio1 minio2 minio3 +``` + +- Once the above command returns success, you may query site replication configuration with: + +```sh +mc admin replicate info minio1 +``` + +** Note ** +Previously, site replication required the root credentials of peer sites to be identical. This is no longer necessary because STS tokens are now signed with the site replicator service account credentials, thus allowing flexibility in the independent management of root accounts across sites and the ability to disable root accounts eventually. + +However, this means that STS tokens signed previously by root credentials will no longer be valid upon upgrading to the latest version with this change. Please re-generate them as you usually do. Additionally, if site replication is ever removed - the STS tokens will become invalid, regenerate them as you usually do. \ No newline at end of file diff --git a/docs/site-replication/gen-oidc-sts-cred.go b/docs/site-replication/gen-oidc-sts-cred.go new file mode 100644 index 00000000..42798e88 --- /dev/null +++ b/docs/site-replication/gen-oidc-sts-cred.go @@ -0,0 +1,80 @@ +//go:build ignore +// +build ignore + +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package main + +// This programs mocks user interaction against Dex IDP and generates STS +// credentials. It is for MinIO testing purposes only. +// +// Run like: +// +// $ MINIO_ENDPOINT=http://localhost:9000 go run gen-oidc-sts-cred.go + +import ( + "context" + "fmt" + "log" + "net/http" + "os" + + cr "github.com/minio/minio-go/v7/pkg/credentials" + cmd "github.com/minio/minio/cmd" +) + +func main() { + ctx := context.Background() + + endpoint := os.Getenv("MINIO_ENDPOINT") + if endpoint == "" { + log.Fatalf("Please specify a MinIO server endpoint environment variable like:\n\n\texport MINIO_ENDPOINT=http://localhost:9000") + } + + appParams := cmd.OpenIDClientAppParams{ + ClientID: "minio-client-app", + ClientSecret: "minio-client-app-secret", + ProviderURL: "http://127.0.0.1:5556/dex", + RedirectURL: "http://127.0.0.1:10000/oauth_callback", + } + + oidcToken, err := cmd.MockOpenIDTestUserInteraction(ctx, appParams, "dillon@example.io", "dillon") + if err != nil { + log.Fatalf("Failed to generate OIDC token: %v", err) + } + + roleARN := os.Getenv("ROLE_ARN") + webID := cr.STSWebIdentity{ + Client: &http.Client{}, + STSEndpoint: endpoint, + GetWebIDTokenExpiry: func() (*cr.WebIdentityToken, error) { + return &cr.WebIdentityToken{ + Token: oidcToken, + }, nil + }, + RoleARN: roleARN, + } + + value, err := webID.Retrieve() + if err != nil { + log.Fatalf("Expected to generate credentials: %v", err) + } + + // Print credentials separated by colons: + fmt.Printf("%s:%s:%s\n", value.AccessKeyID, value.SecretAccessKey, value.SessionToken) +} diff --git a/docs/site-replication/ldap.yaml b/docs/site-replication/ldap.yaml new file mode 100644 index 00000000..b1e1c309 --- /dev/null +++ b/docs/site-replication/ldap.yaml @@ -0,0 +1,14 @@ +# To run locally an OpenLDAP instance using Docker +# $ docker-compose -f ldap.yaml up -d +version: '3.7' + +services: + openldap: + image: quay.io/minio/openldap + ports: + - "389:389" + - "636:636" + environment: + LDAP_ORGANIZATION: "MinIO Inc" + LDAP_DOMAIN: "min.io" + LDAP_ADMIN_PASSWORD: "admin" diff --git a/docs/site-replication/run-multi-site-ldap.sh b/docs/site-replication/run-multi-site-ldap.sh new file mode 100755 index 00000000..351ca92a --- /dev/null +++ b/docs/site-replication/run-multi-site-ldap.sh @@ -0,0 +1,391 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2120 +exit_1() { + cleanup + + echo "minio1 ============" + cat /tmp/minio1_1.log + cat /tmp/minio1_2.log + echo "minio2 ============" + cat /tmp/minio2_1.log + cat /tmp/minio2_2.log + echo "minio3 ============" + cat /tmp/minio3_1.log + cat /tmp/minio3_2.log + + exit 1 +} + +cleanup() { + echo "Cleaning up instances of MinIO" + pkill minio + pkill -9 minio + rm -rf /tmp/minio-ldap-idp{1,2,3} +} + +cleanup + +unset MINIO_KMS_KES_CERT_FILE +unset MINIO_KMS_KES_KEY_FILE +unset MINIO_KMS_KES_ENDPOINT +unset MINIO_KMS_KES_KEY_NAME + +export MINIO_CI_CD=1 +export MINIO_BROWSER=off +export MINIO_ROOT_USER="minio" +export MINIO_ROOT_PASSWORD="minio123" +export MINIO_KMS_AUTO_ENCRYPTION=off +export MINIO_PROMETHEUS_AUTH_TYPE=public +export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= +export MINIO_IDENTITY_LDAP_SERVER_ADDR="localhost:389" +export MINIO_IDENTITY_LDAP_SERVER_INSECURE="on" +export MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN="cn=admin,dc=min,dc=io" +export MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD="admin" +export MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN="dc=min,dc=io" +export MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER="(uid=%s)" +export MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN="ou=swengg,dc=min,dc=io" +export MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER="(&(objectclass=groupOfNames)(member=%d))" + +if [ ! -f ./mc ]; then + wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc +fi + +minio server --config-dir /tmp/minio-ldap --address ":9001" /tmp/minio-ldap-idp1/{1...4} >/tmp/minio1_1.log 2>&1 & +site1_pid=$! +minio server --config-dir /tmp/minio-ldap --address ":9002" /tmp/minio-ldap-idp2/{1...4} >/tmp/minio2_1.log 2>&1 & +site2_pid=$! +minio server --config-dir /tmp/minio-ldap --address ":9003" /tmp/minio-ldap-idp3/{1...4} >/tmp/minio3_1.log 2>&1 & +site3_pid=$! + +export MC_HOST_minio1=http://minio:minio123@localhost:9001 +export MC_HOST_minio2=http://minio:minio123@localhost:9002 +export MC_HOST_minio3=http://minio:minio123@localhost:9003 + +./mc ready minio1 +./mc ready minio2 +./mc ready minio3 + +./mc admin replicate add minio1 minio2 minio3 + +./mc idp ldap policy attach minio1 consoleAdmin --user="uid=dillon,ou=people,ou=swengg,dc=min,dc=io" +sleep 5 + +./mc admin user info minio2 "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" +./mc admin user info minio3 "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" +./mc admin policy create minio1 rw ./docs/site-replication/rw.json + +sleep 5 +./mc admin policy info minio2 rw >/dev/null 2>&1 +./mc admin policy info minio3 rw >/dev/null 2>&1 + +./mc admin policy remove minio3 rw + +sleep 10 +./mc admin policy info minio1 rw +if [ $? -eq 0 ]; then + echo "expecting the command to fail, exiting.." + exit_1 +fi + +./mc admin policy info minio2 rw +if [ $? -eq 0 ]; then + echo "expecting the command to fail, exiting.." + exit_1 +fi + +./mc admin user info minio1 "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" +if [ $? -ne 0 ]; then + echo "policy mapping missing, exiting.." + exit_1 +fi + +./mc admin user info minio2 "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" +if [ $? -ne 0 ]; then + echo "policy mapping missing, exiting.." + exit_1 +fi + +./mc admin user info minio3 "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" +if [ $? -ne 0 ]; then + echo "policy mapping missing, exiting.." + exit_1 +fi + +# LDAP simple user +./mc admin user svcacct add minio2 dillon --access-key testsvc --secret-key testsvc123 +if [ $? -ne 0 ]; then + echo "adding svc account failed, exiting.." + exit_1 +fi + +sleep 10 + +./mc idp ldap policy entities minio1 +./mc idp ldap policy entities minio2 +./mc idp ldap policy entities minio3 + +./mc admin service restart minio1 +./mc admin service restart minio2 +./mc admin service restart minio3 + +sleep 10 + +./mc idp ldap policy entities minio1 +./mc idp ldap policy entities minio2 +./mc idp ldap policy entities minio3 + +./mc admin user svcacct info minio1 testsvc +if [ $? -ne 0 ]; then + echo "svc account not mirrored, exiting.." + exit_1 +fi + +./mc admin user svcacct info minio2 testsvc +if [ $? -ne 0 ]; then + echo "svc account not mirrored, exiting.." + exit_1 +fi + +./mc admin user svcacct info minio3 testsvc +if [ $? -ne 0 ]; then + echo "svc account not mirrored, exiting.." + exit_1 +fi + +MC_HOST_svc1=http://testsvc:testsvc123@localhost:9001 ./mc ls svc1 +MC_HOST_svc2=http://testsvc:testsvc123@localhost:9002 ./mc ls svc2 +MC_HOST_svc3=http://testsvc:testsvc123@localhost:9003 ./mc ls svc3 + +./mc admin user svcacct rm minio1 testsvc +if [ $? -ne 0 ]; then + echo "removing svc account failed, exiting.." + exit_1 +fi + +./mc admin user info minio1 "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" +if [ $? -ne 0 ]; then + echo "policy mapping missing, exiting.." + exit_1 +fi + +./mc admin user info minio2 "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" +if [ $? -ne 0 ]; then + echo "policy mapping missing, exiting.." + exit_1 +fi + +./mc admin user info minio3 "uid=dillon,ou=people,ou=swengg,dc=min,dc=io" +if [ $? -ne 0 ]; then + echo "policy mapping missing, exiting.." + exit_1 +fi + +sleep 10 + +./mc admin user svcacct info minio2 testsvc +if [ $? -eq 0 ]; then + echo "svc account found after delete, exiting.." + exit_1 +fi + +./mc admin user svcacct info minio3 testsvc +if [ $? -eq 0 ]; then + echo "svc account found after delete, exiting.." + exit_1 +fi + +./mc mb minio1/newbucket +# copy large upload to newbucket on minio1 +truncate -s 17M lrgfile +expected_checksum=$(cat ./lrgfile | md5sum) + +./mc cp ./lrgfile minio1/newbucket + +# create a bucket bucket2 on minio1. +./mc mb minio1/bucket2 + +sleep 5 +./mc stat --no-list minio2/newbucket +if [ $? -ne 0 ]; then + echo "expecting bucket to be present. exiting.." + exit_1 +fi + +./mc stat --no-list minio3/newbucket +if [ $? -ne 0 ]; then + echo "expecting bucket to be present. exiting.." + exit_1 +fi + +./mc cp README.md minio2/newbucket/ + +sleep 5 +./mc stat --no-list minio1/newbucket/README.md +if [ $? -ne 0 ]; then + echo "expecting object to be present. exiting.." + exit_1 +fi + +./mc stat --no-list minio3/newbucket/README.md +if [ $? -ne 0 ]; then + echo "expecting object to be present. exiting.." + exit_1 +fi + +sleep 10 +./mc stat --no-list minio3/newbucket/lrgfile +if [ $? -ne 0 ]; then + echo "expected object to be present, exiting.." + exit_1 +fi +actual_checksum=$(./mc cat minio3/newbucket/lrgfile | md5sum) +if [ "${expected_checksum}" != "${actual_checksum}" ]; then + echo "replication failed on multipart objects expected ${expected_checksum} got ${actual_checksum}" + exit +fi +rm ./lrgfile + +./mc rm -r --versions --force minio1/newbucket/lrgfile +if [ $? -ne 0 ]; then + echo "expected object to be present, exiting.." + exit_1 +fi + +sleep 5 +./mc stat --no-list minio1/newbucket/lrgfile +if [ $? -eq 0 ]; then + echo "expected object to be deleted permanently after replication, exiting.." + exit_1 +fi + +vID=$(./mc stat --no-list minio2/newbucket/README.md --json | jq .versionID) +if [ $? -ne 0 ]; then + echo "expecting object to be present. exiting.." + exit_1 +fi +./mc tag set --version-id "${vID}" minio2/newbucket/README.md "key=val" +if [ $? -ne 0 ]; then + echo "expecting tag set to be successful. exiting.." + exit_1 +fi +sleep 5 + +./mc tag remove --version-id "${vID}" minio2/newbucket/README.md +if [ $? -ne 0 ]; then + echo "expecting tag removal to be successful. exiting.." + exit_1 +fi +sleep 5 + +replStatus_minio2=$(./mc stat --no-list minio2/newbucket/README.md --json | jq -r .replicationStatus) +if [ $? -ne 0 ]; then + echo "expecting object to be present. exiting.." + exit_1 +fi + +if [ ${replStatus_minio2} != "COMPLETED" ]; then + echo "expected tag removal to have replicated, exiting..." + exit_1 +fi + +./mc rm minio3/newbucket/README.md +sleep 5 + +./mc stat --no-list minio2/newbucket/README.md +if [ $? -eq 0 ]; then + echo "expected file to be deleted, exiting.." + exit_1 +fi + +./mc stat --no-list minio1/newbucket/README.md +if [ $? -eq 0 ]; then + echo "expected file to be deleted, exiting.." + exit_1 +fi + +./mc mb --with-lock minio3/newbucket-olock +sleep 5 + +enabled_minio2=$(./mc stat --json minio2/newbucket-olock | jq -r .ObjectLock.enabled) +if [ $? -ne 0 ]; then + echo "expected bucket to be mirrored with object-lock but not present, exiting..." + exit_1 +fi + +if [ "${enabled_minio2}" != "Enabled" ]; then + echo "expected bucket to be mirrored with object-lock enabled, exiting..." + exit_1 +fi + +enabled_minio1=$(./mc stat --json minio1/newbucket-olock | jq -r .ObjectLock.enabled) +if [ $? -ne 0 ]; then + echo "expected bucket to be mirrored with object-lock but not present, exiting..." + exit_1 +fi + +if [ "${enabled_minio1}" != "Enabled" ]; then + echo "expected bucket to be mirrored with object-lock enabled, exiting..." + exit_1 +fi + +# "Test if most recent tag update is replicated" +./mc tag set minio2/newbucket "key=val1" +if [ $? -ne 0 ]; then + echo "expecting tag set to be successful. exiting.." + exit_1 +fi + +sleep 10 + +val=$(./mc tag list minio1/newbucket --json | jq -r .tagset | jq -r .key) +if [ "${val}" != "val1" ]; then + echo "expected bucket tag to have replicated, exiting..." + exit_1 +fi +# stop minio1 +kill -9 ${site1_pid} +# Update tag on minio2/newbucket when minio1 is down +./mc tag set minio2/newbucket "key=val2" +# create a new bucket on minio2. This should replicate to minio1 after it comes online. +./mc mb minio2/newbucket2 +# delete bucket2 on minio2. This should replicate to minio1 after it comes online. +./mc rb minio2/bucket2 + +# Restart minio1 instance +minio server --config-dir /tmp/minio-ldap --address ":9001" /tmp/minio-ldap-idp1/{1...4} >/tmp/minio1_1.log 2>&1 & +sleep 200 + +# Test whether most recent tag update on minio2 is replicated to minio1 +val=$(./mc tag list minio1/newbucket --json | jq -r .tagset | jq -r .key) +if [ "${val}" != "val2" ]; then + echo "expected bucket tag to have replicated, exiting..." + exit_1 +fi + +# Test if bucket created/deleted when minio1 is down healed +diff -q <(./mc ls minio1) <(./mc ls minio2) 1>/dev/null +if [ $? -ne 0 ]; then + echo "expected 'bucket2' delete and 'newbucket2' creation to have replicated, exiting..." + exit_1 +fi + +# force a resync after removing all site replication +./mc admin replicate rm --all --force minio1 +./mc rb minio2 --force --dangerous +./mc admin replicate add minio1 minio2 +./mc admin replicate resync start minio1 minio2 +sleep 30 + +./mc ls -r --versions minio1/newbucket >/tmp/minio1.txt +./mc ls -r --versions minio2/newbucket >/tmp/minio2.txt + +out=$(diff -qpruN /tmp/minio1.txt /tmp/minio2.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after replication resync: $out" + exit 1 +fi + +cleanup diff --git a/docs/site-replication/run-multi-site-minio-idp.sh b/docs/site-replication/run-multi-site-minio-idp.sh new file mode 100755 index 00000000..48df4f24 --- /dev/null +++ b/docs/site-replication/run-multi-site-minio-idp.sh @@ -0,0 +1,458 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2120 +exit_1() { + cleanup + + echo "minio1 ============" + cat /tmp/minio1_1.log + cat /tmp/minio1_2.log + echo "minio2 ============" + cat /tmp/minio2_1.log + cat /tmp/minio2_2.log + echo "minio3 ============" + cat /tmp/minio3_1.log + cat /tmp/minio3_2.log + + exit 1 +} + +cleanup() { + echo "Cleaning up instances of MinIO" + pkill minio + pkill -9 minio + rm -rf /tmp/minio-internal-idp{1,2,3} +} + +cleanup + +unset MINIO_KMS_KES_CERT_FILE +unset MINIO_KMS_KES_KEY_FILE +unset MINIO_KMS_KES_ENDPOINT +unset MINIO_KMS_KES_KEY_NAME + +export MINIO_CI_CD=1 +export MINIO_BROWSER=off +export MINIO_ROOT_USER="minio" +export MINIO_ROOT_PASSWORD="minio123" +export MINIO_KMS_AUTO_ENCRYPTION=off +export MINIO_PROMETHEUS_AUTH_TYPE=public +export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= + +if [ ! -f ./mc ]; then + wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc +fi + +minio server --config-dir /tmp/minio-internal --address ":9001" http://localhost:9001/tmp/minio-internal-idp1/{1...4} http://localhost:9010/tmp/minio-internal-idp1/{5...8} >/tmp/minio1_1.log 2>&1 & +site1_pid1=$! +minio server --config-dir /tmp/minio-internal --address ":9010" http://localhost:9001/tmp/minio-internal-idp1/{1...4} http://localhost:9010/tmp/minio-internal-idp1/{5...8} >/tmp/minio1_2.log 2>&1 & +site1_pid2=$! + +minio server --config-dir /tmp/minio-internal --address ":9002" http://localhost:9002/tmp/minio-internal-idp2/{1...4} http://localhost:9020/tmp/minio-internal-idp2/{5...8} >/tmp/minio2_1.log 2>&1 & +site2_pid1=$! +minio server --config-dir /tmp/minio-internal --address ":9020" http://localhost:9002/tmp/minio-internal-idp2/{1...4} http://localhost:9020/tmp/minio-internal-idp2/{5...8} >/tmp/minio2_2.log 2>&1 & +site2_pid2=$! + +minio server --config-dir /tmp/minio-internal --address ":9003" http://localhost:9003/tmp/minio-internal-idp3/{1...4} http://localhost:9030/tmp/minio-internal-idp3/{5...8} >/tmp/minio3_1.log 2>&1 & +site3_pid1=$! +minio server --config-dir /tmp/minio-internal --address ":9030" http://localhost:9003/tmp/minio-internal-idp3/{1...4} http://localhost:9030/tmp/minio-internal-idp3/{5...8} >/tmp/minio3_2.log 2>&1 & +site3_pid2=$! + +export MC_HOST_minio1=http://minio:minio123@localhost:9001 +export MC_HOST_minio2=http://minio:minio123@localhost:9002 +export MC_HOST_minio3=http://minio:minio123@localhost:9003 + +export MC_HOST_minio10=http://minio:minio123@localhost:9010 +export MC_HOST_minio20=http://minio:minio123@localhost:9020 +export MC_HOST_minio30=http://minio:minio123@localhost:9030 + +./mc ready minio1 +./mc ready minio2 +./mc ready minio3 +./mc ready minio10 +./mc ready minio20 +./mc ready minio30 + +./mc admin replicate add minio1 minio2 + +site_enabled=$(./mc admin replicate info minio1) +site_enabled_peer=$(./mc admin replicate info minio10) + +[[ $site_enabled =~ "is not enabled" ]] && { + echo "expected both peers to have same information" + exit_1 +} + +[[ $site_enabled_peer =~ "is not enabled" ]] && { + echo "expected both peers to have same information" + exit_1 +} + +./mc admin user add minio1 foobar foo12345 + +## add foobar-g group with foobar +./mc admin group add minio2 foobar-g foobar + +./mc admin policy attach minio1 consoleAdmin --user=foobar +sleep 5 + +./mc admin user info minio2 foobar + +./mc admin group info minio1 foobar-g + +./mc admin policy create minio1 rw ./docs/site-replication/rw.json + +sleep 5 +./mc admin policy info minio2 rw >/dev/null 2>&1 + +./mc admin replicate status minio1 + +## Add a new empty site +./mc admin replicate add minio1 minio2 minio3 + +sleep 10 + +./mc admin policy info minio3 rw >/dev/null 2>&1 + +./mc admin policy remove minio3 rw + +./mc admin replicate status minio3 + +sleep 10 + +./mc admin policy info minio1 rw +if [ $? -eq 0 ]; then + echo "expecting the command to fail, exiting.." + exit_1 +fi + +./mc admin policy info minio2 rw +if [ $? -eq 0 ]; then + echo "expecting the command to fail, exiting.." + exit_1 +fi + +./mc admin policy info minio3 rw +if [ $? -eq 0 ]; then + echo "expecting the command to fail, exiting.." + exit_1 +fi + +./mc admin user info minio1 foobar +if [ $? -ne 0 ]; then + echo "policy mapping missing on 'minio1', exiting.." + exit_1 +fi + +./mc admin user info minio2 foobar +if [ $? -ne 0 ]; then + echo "policy mapping missing on 'minio2', exiting.." + exit_1 +fi + +./mc admin user info minio3 foobar +if [ $? -ne 0 ]; then + echo "policy mapping missing on 'minio3', exiting.." + exit_1 +fi + +./mc admin group info minio3 foobar-g +if [ $? -ne 0 ]; then + echo "group mapping missing on 'minio3', exiting.." + exit_1 +fi + +./mc admin user svcacct add minio2 foobar --access-key testsvc --secret-key testsvc123 +if [ $? -ne 0 ]; then + echo "adding svc account failed, exiting.." + exit_1 +fi + +./mc admin user svcacct add minio2 minio --access-key testsvc2 --secret-key testsvc123 +if [ $? -ne 0 ]; then + echo "adding root svc account testsvc2 failed, exiting.." + exit_1 +fi + +sleep 10 + +export MC_HOST_rootsvc=http://testsvc2:testsvc123@localhost:9002 +./mc ls rootsvc +if [ $? -ne 0 ]; then + echo "root service account not inherited root permissions, exiting.." + exit_1 +fi + +./mc admin user svcacct info minio1 testsvc +if [ $? -ne 0 ]; then + echo "svc account not mirrored, exiting.." + exit_1 +fi + +./mc admin user svcacct info minio2 testsvc +if [ $? -ne 0 ]; then + echo "svc account not mirrored, exiting.." + exit_1 +fi + +./mc admin user svcacct rm minio1 testsvc +if [ $? -ne 0 ]; then + echo "removing svc account failed, exiting.." + exit_1 +fi + +sleep 10 +./mc admin user svcacct info minio2 testsvc +if [ $? -eq 0 ]; then + echo "svc account found after delete, exiting.." + exit_1 +fi + +./mc admin user svcacct info minio3 testsvc +if [ $? -eq 0 ]; then + echo "svc account found after delete, exiting.." + exit_1 +fi + +./mc mb minio1/newbucket +# copy large upload to newbucket on minio1 +truncate -s 17M lrgfile +expected_checksum=$(cat ./lrgfile | md5sum) + +./mc cp ./lrgfile minio1/newbucket + +sleep 5 +./mc stat --no-list minio2/newbucket +if [ $? -ne 0 ]; then + echo "expecting bucket to be present. exiting.." + exit_1 +fi + +./mc stat --no-list minio3/newbucket +if [ $? -ne 0 ]; then + echo "expecting bucket to be present. exiting.." + exit_1 +fi + +err_minio2=$(./mc stat --no-list minio2/newbucket/xxx --json | jq -r .error.cause.message) +if [ $? -ne 0 ]; then + echo "expecting object to be missing. exiting.." + exit_1 +fi + +if [ "${err_minio2}" != "Object does not exist" ]; then + echo "expected to see Object does not exist error, exiting..." + exit_1 +fi + +./mc cp README.md minio2/newbucket/ + +sleep 5 +./mc stat --no-list minio1/newbucket/README.md +if [ $? -ne 0 ]; then + echo "expecting object to be present. exiting.." + exit_1 +fi + +./mc stat --no-list minio3/newbucket/README.md +if [ $? -ne 0 ]; then + echo "expecting object to be present. exiting.." + exit_1 +fi + +sleep 10 +./mc stat --no-list minio3/newbucket/lrgfile +if [ $? -ne 0 ]; then + echo "expected object to be present, exiting.." + exit_1 +fi + +actual_checksum=$(./mc cat minio3/newbucket/lrgfile | md5sum) +if [ "${expected_checksum}" != "${actual_checksum}" ]; then + echo "replication failed on multipart objects expected ${expected_checksum} got ${actual_checksum}" + exit +fi +rm ./lrgfile + +./mc rm -r --versions --force minio1/newbucket/lrgfile +if [ $? -ne 0 ]; then + echo "expected object to be present, exiting.." + exit_1 +fi + +sleep 5 +./mc stat --no-list minio1/newbucket/lrgfile +if [ $? -eq 0 ]; then + echo "expected object to be deleted permanently after replication, exiting.." + exit_1 +fi + +vID=$(./mc stat --no-list minio2/newbucket/README.md --json | jq .versionID) +if [ $? -ne 0 ]; then + echo "expecting object to be present. exiting.." + exit_1 +fi +./mc tag set --version-id "${vID}" minio2/newbucket/README.md "key=val" +if [ $? -ne 0 ]; then + echo "expecting tag set to be successful. exiting.." + exit_1 +fi +sleep 5 +val=$(./mc tag list minio1/newbucket/README.md --version-id "${vID}" --json | jq -r .tagset.key) +if [ "${val}" != "val" ]; then + echo "expected bucket tag to have replicated, exiting..." + exit_1 +fi +./mc tag remove --version-id "${vID}" minio2/newbucket/README.md +if [ $? -ne 0 ]; then + echo "expecting tag removal to be successful. exiting.." + exit_1 +fi +sleep 5 + +replStatus_minio2=$(./mc stat --no-list minio2/newbucket/README.md --json | jq -r .replicationStatus) +if [ $? -ne 0 ]; then + echo "expecting object to be present. exiting.." + exit_1 +fi + +if [ ${replStatus_minio2} != "COMPLETED" ]; then + echo "expected tag removal to have replicated, exiting..." + exit_1 +fi + +./mc rm minio3/newbucket/README.md +sleep 5 + +./mc stat --no-list minio2/newbucket/README.md +if [ $? -eq 0 ]; then + echo "expected file to be deleted, exiting.." + exit_1 +fi + +./mc stat --no-list minio1/newbucket/README.md +if [ $? -eq 0 ]; then + echo "expected file to be deleted, exiting.." + exit_1 +fi + +./mc mb --with-lock minio3/newbucket-olock +sleep 5 + +set -x + +enabled_minio2=$(./mc stat --json minio2/newbucket-olock | jq -r .ObjectLock.enabled) +if [ $? -ne 0 ]; then + echo "expected bucket to be mirrored with object-lock but not present, exiting..." + exit_1 +fi + +if [ "${enabled_minio2}" != "Enabled" ]; then + echo "expected bucket to be mirrored with object-lock enabled, exiting..." + exit_1 +fi + +enabled_minio1=$(./mc stat --json minio1/newbucket-olock | jq -r .ObjectLock.enabled) +if [ $? -ne 0 ]; then + echo "expected bucket to be mirrored with object-lock but not present, exiting..." + exit_1 +fi + +if [ "${enabled_minio1}" != "Enabled" ]; then + echo "expected bucket to be mirrored with object-lock enabled, exiting..." + exit_1 +fi + +set +x + +# "Test if most recent tag update is replicated" +./mc tag set minio2/newbucket "key=val1" +if [ $? -ne 0 ]; then + echo "expecting tag set to be successful. exiting.." + exit_1 +fi +sleep 5 + +val=$(./mc tag list minio1/newbucket --json | jq -r .tagset | jq -r .key) +if [ "${val}" != "val1" ]; then + echo "expected bucket tag to have replicated, exiting..." + exit_1 +fi +# Create user with policy consoleAdmin on minio1 +./mc admin user add minio1 foobarx foobar123 +if [ $? -ne 0 ]; then + echo "adding user failed, exiting.." + exit_1 +fi +./mc admin policy attach minio1 consoleAdmin --user=foobarx +if [ $? -ne 0 ]; then + echo "adding policy mapping failed, exiting.." + exit_1 +fi +sleep 10 + +# unset policy for foobarx in minio2 +./mc admin policy detach minio2 consoleAdmin --user=foobarx +if [ $? -ne 0 ]; then + echo "unset policy mapping failed, exiting.." + exit_1 +fi + +# create a bucket bucket2 on minio1. +./mc mb minio1/bucket2 + +sleep 10 + +# Test whether policy detach replicated to minio1 +policy=$(./mc admin user info minio1 foobarx --json | jq -r .policyName) +if [ "${policy}" != "null" ]; then + echo "expected policy detach to have replicated, exiting..." + exit_1 +fi + +kill -9 ${site1_pid1} ${site1_pid2} + +# Update tag on minio2/newbucket when minio1 is down +./mc tag set minio2/newbucket "key=val2" +# create a new bucket on minio2. This should replicate to minio1 after it comes online. +./mc mb minio2/newbucket2 + +# delete bucket2 on minio2. This should replicate to minio1 after it comes online. +./mc rb minio2/bucket2 + +# Restart minio1 instance +minio server --config-dir /tmp/minio-internal --address ":9001" http://localhost:9001/tmp/minio-internal-idp1/{1...4} http://localhost:9010/tmp/minio-internal-idp1/{5...8} >/tmp/minio1_1.log 2>&1 & +minio server --config-dir /tmp/minio-internal --address ":9010" http://localhost:9001/tmp/minio-internal-idp1/{1...4} http://localhost:9010/tmp/minio-internal-idp1/{5...8} >/tmp/minio1_2.log 2>&1 & +sleep 200 + +# Test whether most recent tag update on minio2 is replicated to minio1 +val=$(./mc tag list minio1/newbucket --json | jq -r .tagset | jq -r .key) +if [ "${val}" != "val2" ]; then + echo "expected bucket tag to have replicated, exiting..." + exit_1 +fi + +# Test if bucket created/deleted when minio1 is down healed +diff -q <(./mc ls minio1) <(./mc ls minio2) 1>/dev/null +if [ $? -ne 0 ]; then + echo "expected 'bucket2' delete and 'newbucket2' creation to have replicated, exiting..." + exit_1 +fi + +# force a resync after removing all site replication +./mc admin replicate rm --all --force minio1 +./mc rb minio2 --force --dangerous +./mc admin replicate add minio1 minio2 +./mc admin replicate resync start minio1 minio2 +sleep 30 + +./mc ls -r --versions minio1/newbucket >/tmp/minio1.txt +./mc ls -r --versions minio2/newbucket >/tmp/minio2.txt + +out=$(diff -qpruN /tmp/minio1.txt /tmp/minio2.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after replication resync: $out" + exit 1 +fi diff --git a/docs/site-replication/run-multi-site-oidc.sh b/docs/site-replication/run-multi-site-oidc.sh new file mode 100755 index 00000000..d71a86a4 --- /dev/null +++ b/docs/site-replication/run-multi-site-oidc.sh @@ -0,0 +1,327 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2120 +exit_1() { + cleanup + + echo "minio1 ============" + cat /tmp/minio1_1.log + echo "minio2 ============" + cat /tmp/minio2_1.log + echo "minio3 ============" + cat /tmp/minio3_1.log + + exit 1 +} + +cleanup() { + echo "Cleaning up instances of MinIO" + pkill minio + pkill -9 minio + rm -rf /tmp/minio{1,2,3} +} + +cleanup + +unset MINIO_KMS_KES_CERT_FILE +unset MINIO_KMS_KES_KEY_FILE +unset MINIO_KMS_KES_ENDPOINT +unset MINIO_KMS_KES_KEY_NAME + +export MINIO_CI_CD=1 +export MINIO_BROWSER=off +export MINIO_ROOT_USER="minio" +export MINIO_ROOT_PASSWORD="minio123" +export MINIO_KMS_AUTO_ENCRYPTION=off +export MINIO_PROMETHEUS_AUTH_TYPE=public +export MINIO_KMS_SECRET_KEY=my-minio-key:OSMM+vkKUTCvQs9YL/CVMIMt43HFhkUpqJxTmGl6rYw= +export MINIO_IDENTITY_OPENID_CONFIG_URL="http://localhost:5556/dex/.well-known/openid-configuration" +export MINIO_IDENTITY_OPENID_CLIENT_ID="minio-client-app" +export MINIO_IDENTITY_OPENID_CLIENT_SECRET="minio-client-app-secret" +export MINIO_IDENTITY_OPENID_CLAIM_NAME="groups" +export MINIO_IDENTITY_OPENID_SCOPES="openid,groups" + +export MINIO_IDENTITY_OPENID_REDIRECT_URI="http://127.0.0.1:10000/oauth_callback" +minio server --address ":9001" --console-address ":10000" /tmp/minio1/{1...4} >/tmp/minio1_1.log 2>&1 & +site1_pid=$! +export MINIO_IDENTITY_OPENID_REDIRECT_URI="http://127.0.0.1:11000/oauth_callback" +minio server --address ":9002" --console-address ":11000" /tmp/minio2/{1...4} >/tmp/minio2_1.log 2>&1 & +site2_pid=$! + +export MINIO_IDENTITY_OPENID_REDIRECT_URI="http://127.0.0.1:12000/oauth_callback" +minio server --address ":9003" --console-address ":12000" /tmp/minio3/{1...4} >/tmp/minio3_1.log 2>&1 & +site3_pid=$! + +if [ ! -f ./mc ]; then + wget -O mc https://dl.minio.io/client/mc/release/linux-amd64/mc && + chmod +x mc +fi + +export MC_HOST_minio1=http://minio:minio123@localhost:9001 +export MC_HOST_minio2=http://minio:minio123@localhost:9002 +export MC_HOST_minio3=http://minio:minio123@localhost:9003 + +./mc ready minio1 +./mc ready minio2 +./mc ready minio3 + +./mc admin replicate add minio1 minio2 minio3 + +./mc admin policy create minio1 projecta ./docs/site-replication/rw.json +sleep 5 + +./mc admin policy info minio2 projecta >/dev/null 2>&1 +if [ $? -ne 0 ]; then + echo "expecting the command to succeed, exiting.." + exit_1 +fi +./mc admin policy info minio3 projecta >/dev/null 2>&1 +if [ $? -ne 0 ]; then + echo "expecting the command to succeed, exiting.." + exit_1 +fi + +./mc admin policy remove minio3 projecta + +sleep 10 +./mc admin policy info minio1 projecta +if [ $? -eq 0 ]; then + echo "expecting the command to fail, exiting.." + exit_1 +fi + +./mc admin policy info minio2 projecta +if [ $? -eq 0 ]; then + echo "expecting the command to fail, exiting.." + exit_1 +fi + +./mc admin policy create minio1 projecta ./docs/site-replication/rw.json +sleep 5 + +# Generate STS credential with STS call to minio1 +STS_CRED=$(MINIO_ENDPOINT=http://localhost:9001 go run ./docs/site-replication/gen-oidc-sts-cred.go) + +MC_HOST_foo=http://${STS_CRED}@localhost:9001 ./mc ls foo +if [ $? -ne 0 ]; then + echo "Expected sts credential to work, exiting.." + exit_1 +fi + +sleep 2 + +# Check that the STS credential works on minio2 and minio3. +MC_HOST_foo=http://${STS_CRED}@localhost:9002 ./mc ls foo +if [ $? -ne 0 ]; then + echo "Expected sts credential to work, exiting.." + exit_1 +fi + +MC_HOST_foo=http://${STS_CRED}@localhost:9003 ./mc ls foo +if [ $? -ne 0 ]; then + echo "Expected sts credential to work, exiting.." + exit_1 +fi + +STS_ACCESS_KEY=$(echo ${STS_CRED} | cut -d ':' -f 1) + +# Create service account for STS user +./mc admin user svcacct add minio2 $STS_ACCESS_KEY --access-key testsvc --secret-key testsvc123 +if [ $? -ne 0 ]; then + echo "adding svc account failed, exiting.." + exit_1 +fi + +sleep 10 + +./mc admin user svcacct info minio1 testsvc +if [ $? -ne 0 ]; then + echo "svc account not mirrored, exiting.." + exit_1 +fi + +./mc admin user svcacct info minio2 testsvc +if [ $? -ne 0 ]; then + echo "svc account not mirrored, exiting.." + exit_1 +fi + +./mc admin user svcacct rm minio1 testsvc +if [ $? -ne 0 ]; then + echo "removing svc account failed, exiting.." + exit_1 +fi + +sleep 10 +./mc admin user svcacct info minio2 testsvc +if [ $? -eq 0 ]; then + echo "svc account found after delete, exiting.." + exit_1 +fi + +./mc admin user svcacct info minio3 testsvc +if [ $? -eq 0 ]; then + echo "svc account found after delete, exiting.." + exit_1 +fi + +# create a bucket bucket2 on minio1. +./mc mb minio1/bucket2 + +./mc mb minio1/newbucket + +# copy large upload to newbucket on minio1 +truncate -s 17M lrgfile +expected_checksum=$(cat ./lrgfile | md5sum) + +./mc cp ./lrgfile minio1/newbucket +sleep 5 +./mc stat --no-list minio2/newbucket +if [ $? -ne 0 ]; then + echo "expecting bucket to be present. exiting.." + exit_1 +fi + +./mc stat --no-list minio3/newbucket +if [ $? -ne 0 ]; then + echo "expecting bucket to be present. exiting.." + exit_1 +fi + +./mc cp README.md minio2/newbucket/ + +sleep 5 +./mc stat --no-list minio1/newbucket/README.md +if [ $? -ne 0 ]; then + echo "expecting object to be present. exiting.." + exit_1 +fi + +./mc stat --no-list minio3/newbucket/README.md +if [ $? -ne 0 ]; then + echo "expecting object to be present. exiting.." + exit_1 +fi + +./mc rm minio3/newbucket/README.md +sleep 5 + +./mc stat --no-list minio2/newbucket/README.md +if [ $? -eq 0 ]; then + echo "expected file to be deleted, exiting.." + exit_1 +fi + +./mc stat --no-list minio1/newbucket/README.md +if [ $? -eq 0 ]; then + echo "expected file to be deleted, exiting.." + exit_1 +fi + +sleep 10 +./mc stat --no-list minio3/newbucket/lrgfile +if [ $? -ne 0 ]; then + echo "expected object to be present, exiting.." + exit_1 +fi +actual_checksum=$(./mc cat minio3/newbucket/lrgfile | md5sum) +if [ "${expected_checksum}" != "${actual_checksum}" ]; then + echo "replication failed on multipart objects expected ${expected_checksum} got ${actual_checksum}" + exit +fi +rm ./lrgfile + +./mc rm -r --versions --force minio1/newbucket/lrgfile +if [ $? -ne 0 ]; then + echo "expected object to be present, exiting.." + exit_1 +fi + +sleep 5 +./mc stat --no-list minio1/newbucket/lrgfile +if [ $? -eq 0 ]; then + echo "expected object to be deleted permanently after replication, exiting.." + exit_1 +fi + +./mc mb --with-lock minio3/newbucket-olock +sleep 5 + +enabled_minio2=$(./mc stat --json minio2/newbucket-olock | jq -r .ObjectLock.enabled) +if [ $? -ne 0 ]; then + echo "expected bucket to be mirrored with object-lock but not present, exiting..." + exit_1 +fi + +if [ "${enabled_minio2}" != "Enabled" ]; then + echo "expected bucket to be mirrored with object-lock enabled, exiting..." + exit_1 +fi + +enabled_minio1=$(./mc stat --json minio1/newbucket-olock | jq -r .ObjectLock.enabled) +if [ $? -ne 0 ]; then + echo "expected bucket to be mirrored with object-lock but not present, exiting..." + exit_1 +fi + +if [ "${enabled_minio1}" != "Enabled" ]; then + echo "expected bucket to be mirrored with object-lock enabled, exiting..." + exit_1 +fi + +# "Test if most recent tag update is replicated" +./mc tag set minio2/newbucket "key=val1" +if [ $? -ne 0 ]; then + echo "expecting tag set to be successful. exiting.." + exit_1 +fi + +sleep 10 +val=$(./mc tag list minio1/newbucket --json | jq -r .tagset | jq -r .key) +if [ "${val}" != "val1" ]; then + echo "expected bucket tag to have replicated, exiting..." + exit_1 +fi +# stop minio1 instance +kill -9 ${site1_pid} +# Update tag on minio2/newbucket when minio1 is down +./mc tag set minio2/newbucket "key=val2" +# create a new bucket on minio2. This should replicate to minio1 after it comes online. +./mc mb minio2/newbucket2 +# delete bucket2 on minio2. This should replicate to minio1 after it comes online. +./mc rb minio2/bucket2 + +# Restart minio1 instance +minio server --address ":9001" --console-address ":10000" /tmp/minio1/{1...4} >/tmp/minio1_1.log 2>&1 & +sleep 200 + +# Test whether most recent tag update on minio2 is replicated to minio1 +val=$(./mc tag list minio1/newbucket --json | jq -r .tagset | jq -r .key) +if [ "${val}" != "val2" ]; then + echo "expected bucket tag to have replicated, exiting..." + exit_1 +fi + +# Test if bucket created/deleted when minio1 is down healed +diff -q <(./mc ls minio1) <(./mc ls minio2) 1>/dev/null +if [ $? -ne 0 ]; then + echo "expected 'bucket2' delete and 'newbucket2' creation to have replicated, exiting..." + exit_1 +fi + +# force a resync after removing all site replication +./mc admin replicate rm --all --force minio1 +./mc rb minio2 --force --dangerous +./mc admin replicate add minio1 minio2 +./mc admin replicate resync start minio1 minio2 +sleep 30 + +./mc ls -r --versions minio1/newbucket >/tmp/minio1.txt +./mc ls -r --versions minio2/newbucket >/tmp/minio2.txt + +out=$(diff -qpruN /tmp/minio1.txt /tmp/minio2.txt) +ret=$? +if [ $ret -ne 0 ]; then + echo "BUG: expected no missing entries after replication resync: $out" + exit 1 +fi diff --git a/docs/site-replication/run-replication-with-checksum-header.sh b/docs/site-replication/run-replication-with-checksum-header.sh new file mode 100755 index 00000000..f7bf81a2 --- /dev/null +++ b/docs/site-replication/run-replication-with-checksum-header.sh @@ -0,0 +1,262 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2120 +exit_1() { + cleanup + + echo "minio1 ============" + cat /tmp/minio1_1.log + echo "minio2 ============" + cat /tmp/minio2_1.log + + exit 1 +} + +cleanup() { + echo -n "Cleaning up instances of MinIO ..." + pkill -9 minio || sudo pkill -9 minio + rm -rf /tmp/minio{1,2} + echo "done" +} + +# Function to convert number to corresponding alphabet +num_to_alpha() { + local num=$1 + # ASCII value of 'a' is 97, so we add (num - 1) to 97 to get the corresponding alphabet + local ascii_value=$((96 + num)) + # Convert the ASCII value to the character using printf + printf "\\$(printf '%03o' "$ascii_value")" +} + +cleanup + +export MINIO_CI_CD=1 +export MINIO_BROWSER=off +export MINIO_ROOT_USER="minio" +export MINIO_ROOT_PASSWORD="minio123" + +# Download AWS CLI +echo -n "Download and install AWS CLI" +rm -rf /usr/local/aws-cli || sudo rm -rf /usr/local/aws-cli +curl "https://awscli.amazonaws.com/awscli-exe-linux-x86_64.zip" -o "awscliv2.zip" +unzip -qq awscliv2.zip +./aws/install || sudo ./aws/install +echo "done" + +# Add credentials to ~/.aws/credentials +if ! [ -d ~/.aws ]; then + mkdir -p ~/.aws +fi +cat >~/.aws/credentials </tmp/minio1_1.log 2>&1 & +CI=on MINIO_KMS_SECRET_KEY=minio-default-key:IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g= MINIO_ROOT_USER=minio MINIO_ROOT_PASSWORD=minio123 minio server --certs-dir /tmp/certs --address ":9002" --console-address ":11000" /tmp/minio2/{1...4}/disk{1...4} /tmp/minio2/{5...8}/disk{1...4} >/tmp/minio2_1.log 2>&1 & +echo "done" + +if [ ! -f ./mc ]; then + echo -n "Downloading MinIO client ..." + wget -O mc https://dl.min.io/client/mc/release/linux-amd64/mc && + chmod +x mc + echo "done" +fi + +export MC_HOST_minio1=https://minio:minio123@localhost:9001 +export MC_HOST_minio2=https://minio:minio123@localhost:9002 + +./mc ready minio1 --insecure +./mc ready minio2 --insecure + +# Prepare data for tests +echo -n "Preparing test data ..." +mkdir -p /tmp/data +echo "Hello World" >/tmp/data/obj +touch /tmp/data/mpartobj +shred -s 500M /tmp/data/mpartobj +echo "done" + +# Add replication site +./mc admin replicate add minio1 minio2 --insecure +# sleep for replication to complete +sleep 30 + +# Create bucket in source cluster +echo "Create bucket in source MinIO instance" +./mc mb minio1/test-bucket --insecure + +# Load objects to source site with checksum header +echo "Loading objects to source MinIO instance" +OBJ_CHKSUM=$(openssl dgst -sha256 -binary fileparts.json +jq fileparts.json +jq /tmp/minio1_1.log 2>&1 & +CI=on MINIO_KMS_SECRET_KEY=minio-default-key:IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g= MINIO_ROOT_USER=minio MINIO_ROOT_PASSWORD=minio123 minio server --certs-dir /tmp/certs --address ":9002" --console-address ":11000" /tmp/minio2/{1...4}/disk{1...4} /tmp/minio2/{5...8}/disk{1...4} >/tmp/minio2_1.log 2>&1 & +echo "done" + +if [ ! -f ./mc ]; then + echo -n "Downloading MinIO client ..." + wget -O mc https://dl.min.io/client/mc/release/linux-amd64/mc && + chmod +x mc + echo "done" +fi + +export MC_HOST_minio1=https://minio:minio123@localhost:9001 +export MC_HOST_minio2=https://minio:minio123@localhost:9002 + +./mc ready minio1 --insecure +./mc ready minio2 --insecure + +# Prepare data for tests +echo -n "Preparing test data ..." +mkdir -p /tmp/data +echo "Hello from encrypted world" >/tmp/data/encrypted +touch /tmp/data/mpartobj +shred -s 500M /tmp/data/mpartobj +touch /tmp/data/defpartsize +shred -s 500M /tmp/data/defpartsize +touch /tmp/data/custpartsize +shred -s 500M /tmp/data/custpartsize +echo "done" + +# Add replication site +./mc admin replicate add minio1 minio2 --insecure +# sleep for replication to complete +sleep 30 + +# Create bucket in source cluster +echo "Create bucket in source MinIO instance" +./mc mb minio1/test-bucket --insecure + +# Enable SSE KMS for the bucket +./mc encrypt set sse-kms minio-default-key minio1/test-bucket --insecure + +# Load objects to source site +echo "Loading objects to source MinIO instance" +./mc cp /tmp/data/encrypted minio1/test-bucket --insecure +./mc cp /tmp/data/mpartobj minio1/test-bucket/mpartobj --enc-c "minio1/test-bucket/mpartobj=${TEST_MINIO_ENC_KEY}" --insecure +./mc cp /tmp/data/defpartsize minio1/test-bucket --insecure +./mc put /tmp/data/custpartsize minio1/test-bucket --insecure --part-size 50MiB +sleep 120 + +# List the objects from source site +echo "Objects from source instance" +./mc ls minio1/test-bucket --insecure +count1=$(./mc ls minio1/test-bucket/encrypted --insecure | wc -l) +if [ "${count1}" -ne 1 ]; then + echo "BUG: object minio1/test-bucket/encrypted not found" + exit_1 +fi +count2=$(./mc ls minio1/test-bucket/mpartobj --insecure | wc -l) +if [ "${count2}" -ne 1 ]; then + echo "BUG: object minio1/test-bucket/mpartobj not found" + exit_1 +fi +count3=$(./mc ls minio1/test-bucket/defpartsize --insecure | wc -l) +if [ "${count3}" -ne 1 ]; then + echo "BUG: object minio1/test-bucket/defpartsize not found" + exit_1 +fi +count4=$(./mc ls minio1/test-bucket/custpartsize --insecure | wc -l) +if [ "${count4}" -ne 1 ]; then + echo "BUG: object minio1/test-bucket/custpartsize not found" + exit_1 +fi + +# List the objects from replicated site +echo "Objects from replicated instance" +./mc ls minio2/test-bucket --insecure +repcount1=$(./mc ls minio2/test-bucket/encrypted --insecure | wc -l) +if [ "${repcount1}" -ne 1 ]; then + echo "BUG: object test-bucket/encrypted not replicated" + exit_1 +fi +repcount2=$(./mc ls minio2/test-bucket/mpartobj --insecure | wc -l) +if [ "${repcount2}" -ne 1 ]; then + echo "BUG: object test-bucket/mpartobj not replicated" + exit_1 +fi +repcount3=$(./mc ls minio2/test-bucket/defpartsize --insecure | wc -l) +if [ "${repcount3}" -ne 1 ]; then + echo "BUG: object test-bucket/defpartsize not replicated" + exit_1 +fi +repcount4=$(./mc ls minio2/test-bucket/custpartsize --insecure | wc -l) +if [ "${repcount4}" -ne 1 ]; then + echo "BUG: object test-bucket/custpartsize not replicated" + exit_1 +fi + +# Stat the objects from source site +echo "Stat minio1/test-bucket/encrypted" +./mc stat --no-list minio1/test-bucket/encrypted --insecure --json +stat_out1=$(./mc stat --no-list minio1/test-bucket/encrypted --insecure --json) +src_obj1_algo=$(echo "${stat_out1}" | jq '.metadata."X-Amz-Server-Side-Encryption"') +src_obj1_keyid=$(echo "${stat_out1}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"') +echo "Stat minio1/test-bucket/defpartsize" +./mc stat --no-list minio1/test-bucket/defpartsize --insecure --json +stat_out2=$(./mc stat --no-list minio1/test-bucket/defpartsize --insecure --json) +src_obj2_algo=$(echo "${stat_out2}" | jq '.metadata."X-Amz-Server-Side-Encryption"') +src_obj2_keyid=$(echo "${stat_out2}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"') +echo "Stat minio1/test-bucket/custpartsize" +./mc stat --no-list minio1/test-bucket/custpartsize --insecure --json +stat_out3=$(./mc stat --no-list minio1/test-bucket/custpartsize --insecure --json) +src_obj3_algo=$(echo "${stat_out3}" | jq '.metadata."X-Amz-Server-Side-Encryption"') +src_obj3_keyid=$(echo "${stat_out3}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"') +echo "Stat minio1/test-bucket/mpartobj" +./mc stat --no-list minio1/test-bucket/mpartobj --enc-c "minio1/test-bucket/mpartobj=${TEST_MINIO_ENC_KEY}" --insecure --json +stat_out4=$(./mc stat --no-list minio1/test-bucket/mpartobj --enc-c "minio1/test-bucket/mpartobj=${TEST_MINIO_ENC_KEY}" --insecure --json) +src_obj4_etag=$(echo "${stat_out4}" | jq '.etag') +src_obj4_size=$(echo "${stat_out4}" | jq '.size') +src_obj4_md5=$(echo "${stat_out4}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') + +# Stat the objects from replicated site +echo "Stat minio2/test-bucket/encrypted" +./mc stat --no-list minio2/test-bucket/encrypted --insecure --json +stat_out1_rep=$(./mc stat --no-list minio2/test-bucket/encrypted --insecure --json) +rep_obj1_algo=$(echo "${stat_out1_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption"') +rep_obj1_keyid=$(echo "${stat_out1_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"') +echo "Stat minio2/test-bucket/defpartsize" +./mc stat --no-list minio2/test-bucket/defpartsize --insecure --json +stat_out2_rep=$(./mc stat --no-list minio2/test-bucket/defpartsize --insecure --json) +rep_obj2_algo=$(echo "${stat_out2_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption"') +rep_obj2_keyid=$(echo "${stat_out2_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"') +echo "Stat minio2/test-bucket/custpartsize" +./mc stat --no-list minio2/test-bucket/custpartsize --insecure --json +stat_out3_rep=$(./mc stat --no-list minio2/test-bucket/custpartsize --insecure --json) +rep_obj3_algo=$(echo "${stat_out3_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption"') +rep_obj3_keyid=$(echo "${stat_out3_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id"') +echo "Stat minio2/test-bucket/mpartobj" +./mc stat --no-list minio2/test-bucket/mpartobj --enc-c "minio2/test-bucket/mpartobj=${TEST_MINIO_ENC_KEY}" --insecure --json +stat_out4_rep=$(./mc stat --no-list minio2/test-bucket/mpartobj --enc-c "minio2/test-bucket/mpartobj=${TEST_MINIO_ENC_KEY}" --insecure --json) +rep_obj4_etag=$(echo "${stat_out4}" | jq '.etag') +rep_obj4_size=$(echo "${stat_out4}" | jq '.size') +rep_obj4_md5=$(echo "${stat_out4}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') + +# Check the algo and keyId of replicated objects +if [ "${rep_obj1_algo}" != "${src_obj1_algo}" ]; then + echo "BUG: Algorithm: '${rep_obj1_algo}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_algo}'" + exit_1 +fi +if [ "${rep_obj1_keyid}" != "${src_obj1_keyid}" ]; then + echo "BUG: KeyId: '${rep_obj1_keyid}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_keyid}'" + exit_1 +fi +if [ "${rep_obj2_algo}" != "${src_obj2_algo}" ]; then + echo "BUG: Algorithm: '${rep_obj2_algo}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_algo}'" + exit_1 +fi +if [ "${rep_obj2_keyid}" != "${src_obj2_keyid}" ]; then + echo "BUG: KeyId: '${rep_obj2_keyid}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_keyid}'" + exit_1 +fi +if [ "${rep_obj3_algo}" != "${src_obj3_algo}" ]; then + echo "BUG: Algorithm: '${rep_obj3_algo}' of replicated object: 'minio2/test-bucket/custpartsize' doesn't match with source value: '${src_obj3_algo}'" + exit_1 +fi +if [ "${rep_obj3_keyid}" != "${src_obj3_keyid}" ]; then + echo "BUG: KeyId: '${rep_obj3_keyid}' of replicated object: 'minio2/test-bucket/custpartsize' doesn't match with source value: '${src_obj3_keyid}'" + exit_1 +fi + +# Check the etag, size and md5 of replicated SSEC object +if [ "${rep_obj4_etag}" != "${src_obj4_etag}" ]; then + echo "BUG: Etag: '${rep_obj4_etag}' of replicated object: 'minio2/test-bucket/mpartobj' doesn't match with source value: '${src_obj4_etag}'" + exit_1 +fi +if [ "${rep_obj4_size}" != "${src_obj4_size}" ]; then + echo "BUG: Size: '${rep_obj4_size}' of replicated object: 'minio2/test-bucket/mpartobj' doesn't match with source value: '${src_obj4_size}'" + exit_1 +fi +if [ "${src_obj4_md5}" != "${rep_obj4_md5}" ]; then + echo "BUG: MD5 checksum of object 'minio2/test-bucket/mpartobj' doesn't match with source. Expected: '${src_obj4_md5}', Found: '${rep_obj4_md5}'" + exit_1 +fi + +# Check content of replicated objects +./mc cat minio2/test-bucket/encrypted --insecure +./mc cat minio2/test-bucket/mpartobj --enc-c "minio2/test-bucket/mpartobj=${TEST_MINIO_ENC_KEY}" --insecure >/dev/null || exit_1 +./mc cat minio2/test-bucket/defpartsize --insecure >/dev/null || exit_1 +./mc cat minio2/test-bucket/custpartsize --insecure >/dev/null || exit_1 + +echo -n "Starting MinIO instances with different kms key ..." +CI=on MINIO_KMS_SECRET_KEY=minio3-default-key:IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g= MINIO_ROOT_USER=minio MINIO_ROOT_PASSWORD=minio123 minio server --certs-dir /tmp/certs --address ":9003" --console-address ":10000" /tmp/minio3/disk{1...4} >/tmp/minio3_1.log 2>&1 & +CI=on MINIO_KMS_SECRET_KEY=minio4-default-key:IyqsU3kMFloCNup4BsZtf/rmfHVcTgznO2F25CkEH1g= MINIO_ROOT_USER=minio MINIO_ROOT_PASSWORD=minio123 minio server --certs-dir /tmp/certs --address ":9004" --console-address ":11000" /tmp/minio4/disk{1...4} >/tmp/minio4_1.log 2>&1 & +echo "done" + +export MC_HOST_minio3=https://minio:minio123@localhost:9003 +export MC_HOST_minio4=https://minio:minio123@localhost:9004 + +./mc ready minio3 --insecure +./mc ready minio4 --insecure + +./mc admin replicate add minio3 minio4 --insecure +./mc mb minio3/bucket --insecure +./mc cp --insecure --enc-kms minio3/bucket=minio3-default-key /tmp/data/encrypted minio3/bucket/x +sleep 10 +st=$(./mc stat --json --no-list --insecure minio3/bucket/x | jq -r .replicationStatus) +if [ "${st}" != "FAILED" ]; then + echo "BUG: Replication succeeded when kms key is different" + exit_1 +fi + +cleanup diff --git a/docs/site-replication/run-ssec-object-replication-with-compression.sh b/docs/site-replication/run-ssec-object-replication-with-compression.sh new file mode 100755 index 00000000..4f55f7a1 --- /dev/null +++ b/docs/site-replication/run-ssec-object-replication-with-compression.sh @@ -0,0 +1,196 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2120 +exit_1() { + cleanup + + echo "minio1 ============" + cat /tmp/minio1_1.log + echo "minio2 ============" + cat /tmp/minio2_1.log + + exit 1 +} + +cleanup() { + echo -n "Cleaning up instances of MinIO ..." + pkill minio || sudo pkill minio + pkill -9 minio || sudo pkill -9 minio + rm -rf /tmp/minio{1,2} + echo "done" +} + +cleanup + +export MINIO_CI_CD=1 +export MINIO_BROWSER=off +export MINIO_ROOT_USER="minio" +export MINIO_ROOT_PASSWORD="minio123" +TEST_MINIO_ENC_KEY="MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDA" + +# Create certificates for TLS enabled MinIO +echo -n "Setup certs for MinIO instances ..." +wget -O certgen https://github.com/minio/certgen/releases/latest/download/certgen-linux-amd64 && chmod +x certgen +./certgen --host localhost +mkdir -p /tmp/certs +mv public.crt /tmp/certs || sudo mv public.crt /tmp/certs +mv private.key /tmp/certs || sudo mv private.key /tmp/certs +echo "done" + +# Start MinIO instances +echo -n "Starting MinIO instances ..." +minio server --certs-dir /tmp/certs --address ":9001" --console-address ":10000" /tmp/minio1/{1...4}/disk{1...4} /tmp/minio1/{5...8}/disk{1...4} >/tmp/minio1_1.log 2>&1 & +minio server --certs-dir /tmp/certs --address ":9002" --console-address ":11000" /tmp/minio2/{1...4}/disk{1...4} /tmp/minio2/{5...8}/disk{1...4} >/tmp/minio2_1.log 2>&1 & +echo "done" + +if [ ! -f ./mc ]; then + echo -n "Downloading MinIO client ..." + wget -O mc https://dl.min.io/client/mc/release/linux-amd64/mc && + chmod +x mc + echo "done" +fi + +export MC_HOST_minio1=https://minio:minio123@localhost:9001 +export MC_HOST_minio2=https://minio:minio123@localhost:9002 + +./mc ready minio1 --insecure +./mc ready minio2 --insecure + +# Prepare data for tests +echo -n "Preparing test data ..." +mkdir -p /tmp/data +echo "Hello world" >/tmp/data/plainfile +echo "Hello from encrypted world" >/tmp/data/encrypted +touch /tmp/data/defpartsize +shred -s 500M /tmp/data/defpartsize +touch /tmp/data/mpartobj.txt +shred -s 500M /tmp/data/mpartobj.txt +echo "done" + +# Enable compression for site minio1 +./mc admin config set minio1 compression enable=on extensions=".txt" --insecure +./mc admin config set minio1 compression allow_encryption=off --insecure + +# Create bucket in source cluster +echo "Create bucket in source MinIO instance" +./mc mb minio1/test-bucket --insecure + +# Load objects to source site +echo "Loading objects to source MinIO instance" +./mc cp /tmp/data/plainfile minio1/test-bucket --insecure +./mc cp /tmp/data/encrypted minio1/test-bucket/encrypted --enc-c "minio1/test-bucket/encrypted=${TEST_MINIO_ENC_KEY}" --insecure +./mc cp /tmp/data/defpartsize minio1/test-bucket/defpartsize --enc-c "minio1/test-bucket/defpartsize=${TEST_MINIO_ENC_KEY}" --insecure + +# Below should fail as compression and SSEC used at the same time +# DISABLED: We must check the response header to see if compression was actually applied +#RESULT=$({ ./mc put /tmp/data/mpartobj.txt minio1/test-bucket/mpartobj.txt --enc-c "minio1/test-bucket/mpartobj.txt=${TEST_MINIO_ENC_KEY}" --insecure; } 2>&1) +#if [[ ${RESULT} != *"Server side encryption specified with SSE-C with compression not allowed"* ]]; then +# echo "BUG: Loading an SSE-C object to site with compression should fail. Succeeded though." +# exit_1 +#fi + +# Add replication site +./mc admin replicate add minio1 minio2 --insecure +# sleep for replication to complete +sleep 30 + +# List the objects from source site +echo "Objects from source instance" +./mc ls minio1/test-bucket --insecure +count1=$(./mc ls minio1/test-bucket/plainfile --insecure | wc -l) +if [ "${count1}" -ne 1 ]; then + echo "BUG: object minio1/test-bucket/plainfile not found" + exit_1 +fi +count2=$(./mc ls minio1/test-bucket/encrypted --insecure | wc -l) +if [ "${count2}" -ne 1 ]; then + echo "BUG: object minio1/test-bucket/encrypted not found" + exit_1 +fi +count3=$(./mc ls minio1/test-bucket/defpartsize --insecure | wc -l) +if [ "${count3}" -ne 1 ]; then + echo "BUG: object minio1/test-bucket/defpartsize not found" + exit_1 +fi +sleep 120 + +# List the objects from replicated site +echo "Objects from replicated instance" +./mc ls minio2/test-bucket --insecure +repcount1=$(./mc ls minio2/test-bucket/plainfile --insecure | wc -l) +if [ "${repcount1}" -ne 1 ]; then + echo "BUG: object test-bucket/plainfile not replicated" + exit_1 +fi +repcount2=$(./mc ls minio2/test-bucket/encrypted --insecure | wc -l) +if [ "${repcount2}" -ne 1 ]; then + echo "BUG: object test-bucket/encrypted not replicated" + exit_1 +fi +repcount3=$(./mc ls minio2/test-bucket/defpartsize --insecure | wc -l) +if [ "${repcount3}" -ne 1 ]; then + echo "BUG: object test-bucket/defpartsize not replicated" + exit_1 +fi + +# Stat the SSEC objects from source site +echo "Stat minio1/test-bucket/encrypted" +./mc stat --no-list minio1/test-bucket/encrypted --enc-c "minio1/test-bucket/encrypted=${TEST_MINIO_ENC_KEY}" --insecure --json +stat_out1=$(./mc stat --no-list minio1/test-bucket/encrypted --enc-c "minio1/test-bucket/encrypted=${TEST_MINIO_ENC_KEY}" --insecure --json) +src_obj1_etag=$(echo "${stat_out1}" | jq '.etag') +src_obj1_size=$(echo "${stat_out1}" | jq '.size') +src_obj1_md5=$(echo "${stat_out1}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') +echo "Stat minio1/test-bucket/defpartsize" +./mc stat --no-list minio1/test-bucket/defpartsize --enc-c "minio1/test-bucket/defpartsize=${TEST_MINIO_ENC_KEY}" --insecure --json +stat_out2=$(./mc stat --no-list minio1/test-bucket/defpartsize --enc-c "minio1/test-bucket/defpartsize=${TEST_MINIO_ENC_KEY}" --insecure --json) +src_obj2_etag=$(echo "${stat_out2}" | jq '.etag') +src_obj2_size=$(echo "${stat_out2}" | jq '.size') +src_obj2_md5=$(echo "${stat_out2}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') + +# Stat the SSEC objects from replicated site +echo "Stat minio2/test-bucket/encrypted" +./mc stat --no-list minio2/test-bucket/encrypted --enc-c "minio2/test-bucket/encrypted=${TEST_MINIO_ENC_KEY}" --insecure --json +stat_out1_rep=$(./mc stat --no-list minio2/test-bucket/encrypted --enc-c "minio2/test-bucket/encrypted=${TEST_MINIO_ENC_KEY}" --insecure --json) +rep_obj1_etag=$(echo "${stat_out1_rep}" | jq '.etag') +rep_obj1_size=$(echo "${stat_out1_rep}" | jq '.size') +rep_obj1_md5=$(echo "${stat_out1_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') +echo "Stat minio2/test-bucket/defpartsize" +./mc stat --no-list minio2/test-bucket/defpartsize --enc-c "minio2/test-bucket/defpartsize=${TEST_MINIO_ENC_KEY}" --insecure --json +stat_out2_rep=$(./mc stat --no-list minio2/test-bucket/defpartsize --enc-c "minio2/test-bucket/defpartsize=${TEST_MINIO_ENC_KEY}" --insecure --json) +rep_obj2_etag=$(echo "${stat_out2_rep}" | jq '.etag') +rep_obj2_size=$(echo "${stat_out2_rep}" | jq '.size') +rep_obj2_md5=$(echo "${stat_out2_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') + +# Check the etag and size of replicated SSEC objects +if [ "${rep_obj1_etag}" != "${src_obj1_etag}" ]; then + echo "BUG: Etag: '${rep_obj1_etag}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_etag}'" + exit_1 +fi +if [ "${rep_obj1_size}" != "${src_obj1_size}" ]; then + echo "BUG: Size: '${rep_obj1_size}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_size}'" + exit_1 +fi +if [ "${rep_obj2_etag}" != "${src_obj2_etag}" ]; then + echo "BUG: Etag: '${rep_obj2_etag}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_etag}'" + exit_1 +fi +if [ "${rep_obj2_size}" != "${src_obj2_size}" ]; then + echo "BUG: Size: '${rep_obj2_size}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_size}'" + exit_1 +fi + +# Check content of replicated SSEC objects +./mc cat minio2/test-bucket/encrypted --enc-c "minio2/test-bucket/encrypted=${TEST_MINIO_ENC_KEY}" --insecure +./mc cat minio2/test-bucket/defpartsize --enc-c "minio2/test-bucket/defpartsize=${TEST_MINIO_ENC_KEY}" --insecure >/dev/null || exit_1 + +# Check the MD5 checksums of encrypted objects from source and target +if [ "${src_obj1_md5}" != "${rep_obj1_md5}" ]; then + echo "BUG: MD5 checksum of object 'minio2/test-bucket/encrypted' doesn't match with source. Expected: '${src_obj1_md5}', Found: '${rep_obj1_md5}'" + exit_1 +fi +if [ "${src_obj2_md5}" != "${rep_obj2_md5}" ]; then + echo "BUG: MD5 checksum of object 'minio2/test-bucket/defpartsize' doesn't match with source. Expected: '${src_obj2_md5}', Found: '${rep_obj2_md5}'" + exit_1 +fi + +cleanup diff --git a/docs/site-replication/run-ssec-object-replication.sh b/docs/site-replication/run-ssec-object-replication.sh new file mode 100755 index 00000000..0f50f834 --- /dev/null +++ b/docs/site-replication/run-ssec-object-replication.sh @@ -0,0 +1,223 @@ +#!/usr/bin/env bash + +# shellcheck disable=SC2120 +exit_1() { + cleanup + + echo "minio1 ============" + cat /tmp/minio1_1.log + echo "minio2 ============" + cat /tmp/minio2_1.log + + exit 1 +} + +cleanup() { + echo -n "Cleaning up instances of MinIO ..." + pkill minio || sudo pkill minio + pkill -9 minio || sudo pkill -9 minio + rm -rf /tmp/minio{1,2} + echo "done" +} + +cleanup + +export MINIO_CI_CD=1 +export MINIO_BROWSER=off +export MINIO_ROOT_USER="minio" +export MINIO_ROOT_PASSWORD="minio123" +TEST_MINIO_ENC_KEY="MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDA" + +# Create certificates for TLS enabled MinIO +echo -n "Setup certs for MinIO instances ..." +wget -O certgen https://github.com/minio/certgen/releases/latest/download/certgen-linux-amd64 && chmod +x certgen +./certgen --host localhost +mkdir -p /tmp/certs +mv public.crt /tmp/certs || sudo mv public.crt /tmp/certs +mv private.key /tmp/certs || sudo mv private.key /tmp/certs +echo "done" + +# Start MinIO instances +echo -n "Starting MinIO instances ..." +minio server --certs-dir /tmp/certs --address ":9001" --console-address ":10000" /tmp/minio1/{1...4}/disk{1...4} /tmp/minio1/{5...8}/disk{1...4} >/tmp/minio1_1.log 2>&1 & +minio server --certs-dir /tmp/certs --address ":9002" --console-address ":11000" /tmp/minio2/{1...4}/disk{1...4} /tmp/minio2/{5...8}/disk{1...4} >/tmp/minio2_1.log 2>&1 & +echo "done" + +if [ ! -f ./mc ]; then + echo -n "Downloading MinIO client ..." + wget -O mc https://dl.min.io/client/mc/release/linux-amd64/mc && + chmod +x mc + echo "done" +fi + +export MC_HOST_minio1=https://minio:minio123@localhost:9001 +export MC_HOST_minio2=https://minio:minio123@localhost:9002 + +./mc ready minio1 --insecure +./mc ready minio2 --insecure + +# Prepare data for tests +echo -n "Preparing test data ..." +mkdir -p /tmp/data +echo "Hello world" >/tmp/data/plainfile +echo "Hello from encrypted world" >/tmp/data/encrypted +touch /tmp/data/defpartsize +shred -s 500M /tmp/data/defpartsize +touch /tmp/data/custpartsize +shred -s 500M /tmp/data/custpartsize +echo "done" + +# Add replication site +./mc admin replicate add minio1 minio2 --insecure +# sleep for replication to complete +sleep 30 + +# Create bucket in source cluster +echo "Create bucket in source MinIO instance" +./mc mb minio1/test-bucket --insecure + +# Load objects to source site +echo "Loading objects to source MinIO instance" +set -x +./mc cp /tmp/data/plainfile minio1/test-bucket --insecure +./mc cp /tmp/data/encrypted minio1/test-bucket/encrypted --enc-c "minio1/test-bucket/encrypted=${TEST_MINIO_ENC_KEY}" --insecure +./mc cp /tmp/data/defpartsize minio1/test-bucket/defpartsize --enc-c "minio1/test-bucket/defpartsize=${TEST_MINIO_ENC_KEY}" --insecure +./mc put /tmp/data/custpartsize minio1/test-bucket/custpartsize --enc-c "minio1/test-bucket/custpartsize=${TEST_MINIO_ENC_KEY}" --insecure --part-size 50MiB +set +x +sleep 120 + +# List the objects from source site +echo "Objects from source instance" +./mc ls minio1/test-bucket --insecure +count1=$(./mc ls minio1/test-bucket/plainfile --insecure | wc -l) +if [ "${count1}" -ne 1 ]; then + echo "BUG: object minio1/test-bucket/plainfile not found" + exit_1 +fi +count2=$(./mc ls minio1/test-bucket/encrypted --insecure | wc -l) +if [ "${count2}" -ne 1 ]; then + echo "BUG: object minio1/test-bucket/encrypted not found" + exit_1 +fi +count3=$(./mc ls minio1/test-bucket/defpartsize --insecure | wc -l) +if [ "${count3}" -ne 1 ]; then + echo "BUG: object minio1/test-bucket/defpartsize not found" + exit_1 +fi +count4=$(./mc ls minio1/test-bucket/custpartsize --insecure | wc -l) +if [ "${count4}" -ne 1 ]; then + echo "BUG: object minio1/test-bucket/custpartsize not found" + exit_1 +fi + +# List the objects from replicated site +echo "Objects from replicated instance" +./mc ls minio2/test-bucket --insecure +repcount1=$(./mc ls minio2/test-bucket/plainfile --insecure | wc -l) +if [ "${repcount1}" -ne 1 ]; then + echo "BUG: object test-bucket/plainfile not replicated" + exit_1 +fi +repcount2=$(./mc ls minio2/test-bucket/encrypted --insecure | wc -l) +if [ "${repcount2}" -ne 1 ]; then + echo "BUG: object test-bucket/encrypted not replicated" + exit_1 +fi +repcount3=$(./mc ls minio2/test-bucket/defpartsize --insecure | wc -l) +if [ "${repcount3}" -ne 1 ]; then + echo "BUG: object test-bucket/defpartsize not replicated" + exit_1 +fi + +repcount4=$(./mc ls minio2/test-bucket/custpartsize --insecure | wc -l) +if [ "${repcount4}" -ne 1 ]; then + echo "BUG: object test-bucket/custpartsize not replicated" + exit_1 +fi + +# Stat the SSEC objects from source site +echo "Stat minio1/test-bucket/encrypted" +./mc stat --no-list minio1/test-bucket/encrypted --enc-c "minio1/test-bucket/encrypted=${TEST_MINIO_ENC_KEY}" --insecure --json +stat_out1=$(./mc stat --no-list minio1/test-bucket/encrypted --enc-c "minio1/test-bucket/encrypted=${TEST_MINIO_ENC_KEY}" --insecure --json) +src_obj1_etag=$(echo "${stat_out1}" | jq '.etag') +src_obj1_size=$(echo "${stat_out1}" | jq '.size') +src_obj1_md5=$(echo "${stat_out1}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') +echo "Stat minio1/test-bucket/defpartsize" +./mc stat --no-list minio1/test-bucket/defpartsize --enc-c "minio1/test-bucket/defpartsize=${TEST_MINIO_ENC_KEY}" --insecure --json +stat_out2=$(./mc stat --no-list minio1/test-bucket/defpartsize --enc-c "minio1/test-bucket/defpartsize=${TEST_MINIO_ENC_KEY}" --insecure --json) +src_obj2_etag=$(echo "${stat_out2}" | jq '.etag') +src_obj2_size=$(echo "${stat_out2}" | jq '.size') +src_obj2_md5=$(echo "${stat_out2}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') +echo "Stat minio1/test-bucket/custpartsize" +./mc stat --no-list minio1/test-bucket/custpartsize --enc-c "minio1/test-bucket/custpartsize=${TEST_MINIO_ENC_KEY}" --insecure --json +stat_out3=$(./mc stat --no-list minio1/test-bucket/custpartsize --enc-c "minio1/test-bucket/custpartsize=${TEST_MINIO_ENC_KEY}" --insecure --json) +src_obj3_etag=$(echo "${stat_out3}" | jq '.etag') +src_obj3_size=$(echo "${stat_out3}" | jq '.size') +src_obj3_md5=$(echo "${stat_out3}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') + +# Stat the SSEC objects from replicated site +echo "Stat minio2/test-bucket/encrypted" +./mc stat --no-list minio2/test-bucket/encrypted --enc-c "minio2/test-bucket/encrypted=${TEST_MINIO_ENC_KEY}" --insecure --json +stat_out1_rep=$(./mc stat --no-list minio2/test-bucket/encrypted --enc-c "minio2/test-bucket/encrypted=${TEST_MINIO_ENC_KEY}" --insecure --json) +rep_obj1_etag=$(echo "${stat_out1_rep}" | jq '.etag') +rep_obj1_size=$(echo "${stat_out1_rep}" | jq '.size') +rep_obj1_md5=$(echo "${stat_out1_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') +echo "Stat minio2/test-bucket/defpartsize" +./mc stat --no-list minio2/test-bucket/defpartsize --enc-c "minio2/test-bucket/defpartsize=${TEST_MINIO_ENC_KEY}" --insecure --json +stat_out2_rep=$(./mc stat --no-list minio2/test-bucket/defpartsize --enc-c "minio2/test-bucket/defpartsize=${TEST_MINIO_ENC_KEY}" --insecure --json) +rep_obj2_etag=$(echo "${stat_out2_rep}" | jq '.etag') +rep_obj2_size=$(echo "${stat_out2_rep}" | jq '.size') +rep_obj2_md5=$(echo "${stat_out2_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') +echo "Stat minio2/test-bucket/custpartsize" +./mc stat --no-list minio2/test-bucket/custpartsize --enc-c "minio2/test-bucket/custpartsize=${TEST_MINIO_ENC_KEY}" --insecure --json +stat_out3_rep=$(./mc stat --no-list minio2/test-bucket/custpartsize --enc-c "minio2/test-bucket/custpartsize=${TEST_MINIO_ENC_KEY}" --insecure --json) +rep_obj3_etag=$(echo "${stat_out3_rep}" | jq '.etag') +rep_obj3_size=$(echo "${stat_out3_rep}" | jq '.size') +rep_obj3_md5=$(echo "${stat_out3_rep}" | jq '.metadata."X-Amz-Server-Side-Encryption-Customer-Key-Md5"') + +# Check the etag and size of replicated SSEC objects +if [ "${rep_obj1_etag}" != "${src_obj1_etag}" ]; then + echo "BUG: Etag: '${rep_obj1_etag}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_etag}'" + exit_1 +fi +if [ "${rep_obj1_size}" != "${src_obj1_size}" ]; then + echo "BUG: Size: '${rep_obj1_size}' of replicated object: 'minio2/test-bucket/encrypted' doesn't match with source value: '${src_obj1_size}'" + exit_1 +fi +if [ "${rep_obj2_etag}" != "${src_obj2_etag}" ]; then + echo "BUG: Etag: '${rep_obj2_etag}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_etag}'" + exit_1 +fi +if [ "${rep_obj2_size}" != "${src_obj2_size}" ]; then + echo "BUG: Size: '${rep_obj2_size}' of replicated object: 'minio2/test-bucket/defpartsize' doesn't match with source value: '${src_obj2_size}'" + exit_1 +fi +if [ "${rep_obj3_etag}" != "${src_obj3_etag}" ]; then + echo "BUG: Etag: '${rep_obj3_etag}' of replicated object: 'minio2/test-bucket/custpartsize' doesn't match with source value: '${src_obj3_etag}'" + exit_1 +fi +if [ "${rep_obj3_size}" != "${src_obj3_size}" ]; then + echo "BUG: Size: '${rep_obj3_size}' of replicated object: 'minio2/test-bucket/custpartsize' doesn't match with source value: '${src_obj3_size}'" + exit_1 +fi + +# Check content of replicated SSEC objects +./mc cat minio2/test-bucket/encrypted --enc-c "minio2/test-bucket/encrypted=${TEST_MINIO_ENC_KEY}" --insecure +./mc cat minio2/test-bucket/defpartsize --enc-c "minio2/test-bucket/defpartsize=${TEST_MINIO_ENC_KEY}" --insecure >/dev/null || exit_1 +./mc cat minio2/test-bucket/custpartsize --enc-c "minio2/test-bucket/custpartsize=${TEST_MINIO_ENC_KEY}" --insecure >/dev/null || exit_1 + +# Check the MD5 checksums of encrypted objects from source and target +if [ "${src_obj1_md5}" != "${rep_obj1_md5}" ]; then + echo "BUG: MD5 checksum of object 'minio2/test-bucket/encrypted' doesn't match with source. Expected: '${src_obj1_md5}', Found: '${rep_obj1_md5}'" + exit_1 +fi +if [ "${src_obj2_md5}" != "${rep_obj2_md5}" ]; then + echo "BUG: MD5 checksum of object 'minio2/test-bucket/defpartsize' doesn't match with source. Expected: '${src_obj2_md5}', Found: '${rep_obj2_md5}'" + exit_1 +fi +if [ "${src_obj3_md5}" != "${rep_obj3_md5}" ]; then + echo "BUG: MD5 checksum of object 'minio2/test-bucket/custpartsize' doesn't match with source. Expected: '${src_obj3_md5}', Found: '${rep_obj3_md5}'" + exit_1 +fi + +cleanup diff --git a/docs/site-replication/rw.json b/docs/site-replication/rw.json new file mode 100644 index 00000000..b5f5adc5 --- /dev/null +++ b/docs/site-replication/rw.json @@ -0,0 +1 @@ +{"Version":"2012-10-17","Statement":[{"Effect":"Allow","Action":["admin:*"]},{"Effect":"Allow","Action":["s3:*"],"Resource":["arn:aws:s3:::*"]}]} diff --git a/docs/sts/.gitignore b/docs/sts/.gitignore new file mode 100644 index 00000000..894a44cc --- /dev/null +++ b/docs/sts/.gitignore @@ -0,0 +1,104 @@ +# Byte-compiled / optimized / DLL files +__pycache__/ +*.py[cod] +*$py.class + +# C extensions +*.so + +# Distribution / packaging +.Python +build/ +develop-eggs/ +dist/ +downloads/ +eggs/ +.eggs/ +lib/ +lib64/ +parts/ +sdist/ +var/ +wheels/ +*.egg-info/ +.installed.cfg +*.egg +MANIFEST + +# PyInstaller +# Usually these files are written by a python script from a template +# before PyInstaller builds the exe, so as to inject date/other infos into it. +*.manifest +*.spec + +# Installer logs +pip-log.txt +pip-delete-this-directory.txt + +# Unit test / coverage reports +htmlcov/ +.tox/ +.coverage +.coverage.* +.cache +nosetests.xml +coverage.xml +*.cover +.hypothesis/ +.pytest_cache/ + +# Translations +*.mo +*.pot + +# Django stuff: +*.log +local_settings.py +db.sqlite3 + +# Flask stuff: +instance/ +.webassets-cache + +# Scrapy stuff: +.scrapy + +# Sphinx documentation +docs/_build/ + +# PyBuilder +target/ + +# Jupyter Notebook +.ipynb_checkpoints + +# pyenv +.python-version + +# celery beat schedule file +celerybeat-schedule + +# SageMath parsed files +*.sage.py + +# Environments +.env +.venv +env/ +venv/ +ENV/ +env.bak/ +venv.bak/ + +# Spyder project settings +.spyderproject +.spyproject + +# Rope project settings +.ropeproject + +# mkdocs documentation +/site + +# mypy +.mypy_cache/ diff --git a/docs/sts/README.md b/docs/sts/README.md new file mode 100644 index 00000000..62fa48b1 --- /dev/null +++ b/docs/sts/README.md @@ -0,0 +1,110 @@ +# MinIO STS Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +The MinIO Security Token Service (STS) is an endpoint service that enables clients to request temporary credentials for MinIO resources. Temporary credentials work almost identically to default admin credentials, with some differences: + +- Temporary credentials are short-term, as the name implies. They can be configured to last for anywhere from a few minutes to several hours. After the credentials expire, MinIO no longer recognizes them or allows any kind of access from API requests made with them. +- Temporary credentials do not need to be stored with the application but are generated dynamically and provided to the application when requested. When (or even before) the temporary credentials expire, the application can request new credentials. + +Following are advantages for using temporary credentials: + +- Eliminates the need to embed long-term credentials with an application. +- Eliminates the need to provide access to buckets and objects without having to define static credentials. +- Temporary credentials have a limited lifetime, there is no need to rotate them or explicitly revoke them. Expired temporary credentials cannot be reused. + +## Identity Federation + +| AuthN | Description | +| :---------------------- | ------------------------------------------ | +| [**WebIdentity**](https://github.com/minio/minio/blob/master/docs/sts/web-identity.md) | Let users request temporary credentials using any OpenID(OIDC) compatible web identity providers such as KeyCloak, Dex, Facebook, Google etc. | +| [**AD/LDAP**](https://github.com/minio/minio/blob/master/docs/sts/ldap.md) | Let AD/LDAP users request temporary credentials using AD/LDAP username and password. | +| [**AssumeRole**](https://github.com/minio/minio/blob/master/docs/sts/assume-role.md) | Let MinIO users request temporary credentials using user access and secret keys. | + +### Understanding JWT Claims + +> NOTE: JWT claims are only meant for WebIdentity and ClientGrants. +> AssumeRole or LDAP users can skip the entire portion and directly visit one of the links below. +> +> - [**AssumeRole**](https://github.com/minio/minio/blob/master/docs/sts/assume-role.md) +> - [**AD/LDAP**](https://github.com/minio/minio/blob/master/docs/sts/ldap.md) + +The id_token received is a signed JSON Web Token (JWT). Use a JWT decoder to decode the id_token to access the payload of the token that includes following JWT claims, `policy` claim is mandatory and should be present as part of your JWT claim. Without this claim the generated credentials will not have access to any resources on the server, using these credentials application would receive 'Access Denied' errors. + +| Claim Name | Type | Claim Value | +|:----------:|:-------------------------------------------------:|:------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:| +| policy | _string_ or _[]string_ or _comma_separated_value_ | Canned policy name to be applied for STS credentials. (Mandatory) - This can be configured to any desired value such as `roles` or `groups` by setting the environment variable `MINIO_IDENTITY_OPENID_CLAIM_NAME` | + +## Get started + +In this document we will explain in detail on how to configure all the prerequisites. + +> NOTE: If you are interested in AssumeRole API only, skip to [here](https://github.com/minio/minio/blob/master/docs/sts/assume-role.md) + +### Prerequisites + +- [Configuring keycloak](https://github.com/minio/minio/blob/master/docs/sts/keycloak.md) or [Configuring Casdoor](https://github.com/minio/minio/blob/master/docs/sts/casdoor.md) +- [Configuring etcd](https://github.com/minio/minio/blob/master/docs/sts/etcd.md) + +### Setup MinIO with Identity Provider + +Make sure we have followed the previous step and configured each software independently, once done we can now proceed to use MinIO STS API and MinIO server to use these credentials to perform object API operations. + +#### KeyCloak + +``` +export MINIO_ROOT_USER=minio +export MINIO_ROOT_PASSWORD=minio123 +export MINIO_IDENTITY_OPENID_CONFIG_URL=http://localhost:8080/auth/realms/demo/.well-known/openid-configuration +export MINIO_IDENTITY_OPENID_CLIENT_ID="843351d4-1080-11ea-aa20-271ecba3924a" +minio server /mnt/data +``` + +#### Casdoor + +``` +export MINIO_ROOT_USER=minio +export MINIO_ROOT_PASSWORD=minio123 +export MINIO_IDENTITY_OPENID_CONFIG_URL=http://CASDOOR_ENDPOINT/.well-known/openid-configuration +export MINIO_IDENTITY_OPENID_CLIENT_ID="843351d4-1080-11ea-aa20-271ecba3924a" +minio server /mnt/data +``` + +### Using WebIdentiy API + +On another terminal run `web-identity.go` a sample client application which obtains JWT id_tokens from an identity provider, in our case its Keycloak. Uses the returned id_token response to get new temporary credentials from the MinIO server using the STS API call `AssumeRoleWithWebIdentity`. + +``` +$ go run docs/sts/web-identity.go -cid account -csec 072e7f00-4289-469c-9ab2-bbe843c7f5a8 -config-ep "http://localhost:8080/auth/realms/demo/.well-known/openid-configuration" -port 8888 +2018/12/26 17:49:36 listening on http://localhost:8888/ +``` + +This will open the login page of keycloak, upon successful login, STS credentials along with any buckets discovered using the credentials will be printed on the screen, for example: + +``` +{ + "buckets": [ + "bucket-x" + ], + "credentials": { + "AccessKeyID": "6N2BALX7ELO827DXS3GK", + "SecretAccessKey": "23JKqAD+um8ObHqzfIh+bfqwG9V8qs9tFY6MqeFR+xxx", + "SessionToken": "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiI2TjJCQUxYN0VMTzgyN0RYUzNHSyIsImFjciI6IjAiLCJhdWQiOiJhY2NvdW50IiwiYXV0aF90aW1lIjoxNTY5OTEwNTUyLCJhenAiOiJhY2NvdW50IiwiZW1haWxfdmVyaWZpZWQiOmZhbHNlLCJleHAiOjE1Njk5MTQ1NTQsImlhdCI6MTU2OTkxMDk1NCwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgxL2F1dGgvcmVhbG1zL2RlbW8iLCJqdGkiOiJkOTk4YTBlZS01NDk2LTQ4OWYtYWJlMi00ZWE5MjJiZDlhYWYiLCJuYmYiOjAsInBvbGljeSI6InJlYWR3cml0ZSIsInByZWZlcnJlZF91c2VybmFtZSI6Im5ld3VzZXIxIiwic2Vzc2lvbl9zdGF0ZSI6IjJiYTAyYTI2LWE5MTUtNDUxNC04M2M1LWE0YjgwYjc4ZTgxNyIsInN1YiI6IjY4ZmMzODVhLTA5MjItNGQyMS04N2U5LTZkZTdhYjA3Njc2NSIsInR5cCI6IklEIn0._UG_-ZHgwdRnsp0gFdwChb7VlbPs-Gr_RNUz9EV7TggCD59qjCFAKjNrVHfOSVkKvYEMe0PvwfRKjnJl3A_mBA"", + "SignerType": 1 + } +} +``` + +> NOTE: You can use the `-cscopes` parameter to restrict the requested scopes, for example to `"openid,policy_role_attribute"`, being `policy_role_attribute` a client_scope / client_mapper that maps a role attribute called policy to a `policy` claim returned by Keycloak. + +These credentials can now be used to perform MinIO API operations. + +### Using MinIO Console + +- Open MinIO URL on the browser, lets say +- Click on `Login with SSO` +- User will be redirected to the Keycloak user login page, upon successful login the user will be redirected to MinIO page and logged in automatically, + the user should see now the buckets and objects they have access to. + +## Explore Further + +- [MinIO Admin Complete Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc-admin.html) +- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html) diff --git a/docs/sts/assume-role.go b/docs/sts/assume-role.go new file mode 100644 index 00000000..7c8735ff --- /dev/null +++ b/docs/sts/assume-role.go @@ -0,0 +1,164 @@ +//go:build ignore +// +build ignore + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package main + +import ( + "context" + "flag" + "fmt" + "io" + "log" + "net/url" + "os" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7" + cr "github.com/minio/minio-go/v7/pkg/credentials" +) + +var ( + // Minio endpoint (for STS API) + stsEndpoint string + + // User account credentials + minioUsername string + minioPassword string + + // Display credentials flag + displayCreds bool + + // Credential expiry duration + expiryDuration time.Duration + + // Bucket to list + bucketToList string + + // Session policy file (FIXME: add support in minio-go) + sessionPolicyFile string +) + +func init() { + flag.StringVar(&stsEndpoint, "sts-ep", "http://localhost:9000", "STS endpoint") + flag.StringVar(&minioUsername, "u", "", "MinIO Username") + flag.StringVar(&minioPassword, "p", "", "MinIO Password") + flag.BoolVar(&displayCreds, "d", false, "Only show generated credentials") + flag.DurationVar(&expiryDuration, "e", 0, "Request a duration of validity for the generated credential") + flag.StringVar(&bucketToList, "b", "", "Bucket to list (defaults to username)") + flag.StringVar(&sessionPolicyFile, "s", "", "File containing session policy to apply to the STS request") +} + +func main() { + flag.Parse() + if minioUsername == "" || minioPassword == "" { + flag.PrintDefaults() + return + } + + // The credentials package in minio-go provides an interface to call the + // STS API. + + // Initialize credential options + var stsOpts cr.STSAssumeRoleOptions + stsOpts.AccessKey = minioUsername + stsOpts.SecretKey = minioPassword + + if sessionPolicyFile != "" { + var policy string + if f, err := os.Open(sessionPolicyFile); err != nil { + log.Fatalf("Unable to open session policy file: %v", err) + } else { + defer f.Close() + bs, err := io.ReadAll(f) + if err != nil { + log.Fatalf("Error reading session policy file: %v", err) + } + policy = string(bs) + } + stsOpts.Policy = policy + } + if expiryDuration != 0 { + stsOpts.DurationSeconds = int(expiryDuration.Seconds()) + } + li, err := cr.NewSTSAssumeRole(stsEndpoint, stsOpts) + if err != nil { + log.Fatalf("Error initializing STS Identity: %v", err) + } + + stsEndpointURL, err := url.Parse(stsEndpoint) + if err != nil { + log.Fatalf("Error parsing sts endpoint: %v", err) + } + + opts := &minio.Options{ + Creds: li, + Secure: stsEndpointURL.Scheme == "https", + } + + mopts := &madmin.Options{ + Creds: li, + Secure: stsEndpointURL.Scheme == "https", + } + + v, err := li.Get() + if err != nil { + log.Fatalf("Error retrieving STS credentials: %v", err) + } + + if displayCreds { + fmt.Println("Only displaying credentials:") + fmt.Println("AccessKeyID:", v.AccessKeyID) + fmt.Println("SecretAccessKey:", v.SecretAccessKey) + fmt.Println("SessionToken:", v.SessionToken) + return + } + + // API requests are secure (HTTPS) if secure=true and insecure (HTTP) otherwise. + // New returns an MinIO Admin client object. + madmClnt, err := madmin.NewWithOptions(stsEndpointURL.Host, mopts) + if err != nil { + log.Fatalln(err) + } + + err = madmClnt.ServiceRestart(context.Background()) + if err != nil { + log.Fatalln(err) + } + + // Use generated credentials to authenticate with MinIO server + minioClient, err := minio.New(stsEndpointURL.Host, opts) + if err != nil { + log.Fatalf("Error initializing client: %v", err) + } + + // Use minIO Client object normally like the regular client. + if bucketToList == "" { + bucketToList = minioUsername + } + fmt.Printf("Calling list objects on bucket named `%s` with temp creds:\n===\n", bucketToList) + objCh := minioClient.ListObjects(context.Background(), bucketToList, minio.ListObjectsOptions{}) + for obj := range objCh { + if obj.Err != nil { + log.Fatalf("Listing error: %v", obj.Err) + } + fmt.Printf("Key: %s\nSize: %d\nLast Modified: %s\n===\n", obj.Key, obj.Size, obj.LastModified) + } +} diff --git a/docs/sts/assume-role.md b/docs/sts/assume-role.md new file mode 100644 index 00000000..8b9600f7 --- /dev/null +++ b/docs/sts/assume-role.md @@ -0,0 +1,138 @@ +# AssumeRole [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +## Introduction + +Returns a set of temporary security credentials that you can use to access MinIO resources. AssumeRole requires authorization credentials for an existing user on MinIO. The advantages of this API are + +- To be able to reliably use S3 multipart APIs feature of the SDKs without re-inventing the wheel of pre-signing the each URL in multipart API. This is very tedious to implement with all the scenarios of fault tolerance that's already implemented by the client SDK. The general client SDKs don't support multipart with presigned URLs. +- To be able to easily get the temporary credentials to upload to a prefix. Make it possible for a client to upload a whole folder using the session. The server side applications need not create a presigned URL and serve to the client for each file. Since, the client would have the session it can do it by itself. + +The temporary security credentials returned by this API consists of an access key, a secret key, and a security token. Applications can use these temporary security credentials to sign calls to MinIO API operations. The policy applied to these temporary credentials is inherited from the MinIO user credentials. By default, the temporary security credentials created by AssumeRole last for one hour. However, use the optional DurationSeconds parameter to specify the duration of the credentials. This value varies from 900 seconds (15 minutes) up to the maximum session duration of 365 days. + +## API Request Parameters + +### Version + +Indicates STS API version information, the only supported value is '2011-06-15'. This value is borrowed from AWS STS API documentation for compatibility reasons. + +| Params | Value | +| :-- | :-- | +| *Type* | *String* | +| *Required* | *Yes* | + +### AUTHPARAMS + +Indicates STS API Authorization information. If you are familiar with AWS Signature V4 Authorization header, this STS API supports signature V4 authorization as mentioned [here](https://docs.aws.amazon.com/general/latest/gr/signature-version-4.html) + +### DurationSeconds + +The duration, in seconds. The value can range from 900 seconds (15 minutes) up to 365 days. If value is higher than this setting, then operation fails. By default, the value is set to 3600 seconds. + +| Params | Value | +| :-- | :-- | +| *Type* | *Integer* | +| *Valid Range* | *Minimum value of 900. Maximum value of 31536000.* | +| *Required* | *No* | + +### Policy + +An IAM policy in JSON format that you want to use as an inline session policy. This parameter is optional. Passing policies to this operation returns new temporary credentials. The resulting session's permissions are the intersection of the canned policy name and the policy set here. You cannot use this policy to grant more permissions than those allowed by the canned policy name being assumed. + +| Params | Value | +| :-- | :-- | +| *Type* | *String* | +| *Valid Range* | *Minimum length of 1. Maximum length of 2048.* | +| *Required* | *No* | + +### Response Elements + +XML response for this API is similar to [AWS STS AssumeRole](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html#API_AssumeRole_ResponseElements) + +### Errors + +XML error response for this API is similar to [AWS STS AssumeRole](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRole.html#API_AssumeRole_Errors) + +## Sample `POST` Request + +``` +http://minio:9000/?Action=AssumeRole&DurationSeconds=3600&Version=2011-06-15&Policy={"Version":"2012-10-17","Statement":[{"Sid":"Stmt1","Effect":"Allow","Action":"s3:*","Resource":"arn:aws:s3:::*"}]}&AUTHPARAMS +``` + +## Sample Response + +``` + + + + + + + + + Y4RJU1RNFGK48LGO9I2S + sYLRKS1Z7hSjluf6gEbb9066hnx315wHTiACPAjg + 2019-08-08T20:26:12Z + eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJZNFJKVTFSTkZHSzQ4TEdPOUkyUyIsImF1ZCI6IlBvRWdYUDZ1Vk80NUlzRU5SbmdEWGo1QXU1WWEiLCJhenAiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiZXhwIjoxNTQxODExMDcxLCJpYXQiOjE1NDE4MDc0NzEsImlzcyI6Imh0dHBzOi8vbG9jYWxob3N0Ojk0NDMvb2F1dGgyL3Rva2VuIiwianRpIjoiYTBiMjc2MjktZWUxYS00M2JmLTg3MzktZjMzNzRhNGNkYmMwIn0.ewHqKVFTaP-j_kgZrcOEKroNUjk10GEp8bqQjxBbYVovV0nHO985VnRESFbcT6XMDDKHZiWqN2vi_ETX_u3Q-w + + + + c6104cbe-af31-11e0-8154-cbc7ccf896c7 + + +``` + +## Using AssumeRole API + +``` +export MINIO_ROOT_USER=minio +export MINIO_ROOT_PASSWORD=minio123 +minio server ~/test +``` + +Create new users following the multi-user guide [here](https://docs.min.io/community/minio-object-store/administration/identity-access-management.html) + +### Testing an example with awscli tool + +> Use the same username and password created in the previous steps. + +``` +[foobar] +region = us-east-1 +aws_access_key_id = foobar +aws_secret_access_key = foo12345 +``` + +> NOTE: In the following commands `--role-arn` and `--role-session-name` are not meaningful for MinIO and can be set to any value satisfying the command line requirements. + +``` +$ aws --profile foobar --endpoint-url http://localhost:9000 sts assume-role --policy '{"Version":"2012-10-17","Statement":[{"Sid":"Stmt1","Effect":"Allow","Action":"s3:*","Resource":"arn:aws:s3:::*"}]}' --role-arn arn:xxx:xxx:xxx:xxxx --role-session-name anything +{ + "AssumedRoleUser": { + "Arn": "" + }, + "Credentials": { + "SecretAccessKey": "xbnWUoNKgFxi+uv3RI9UgqP3tULQMdI+Hj+4psd4", + "SessionToken": "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJLOURUSU1VVlpYRVhKTDNBVFVPWSIsImV4cCI6MzYwMDAwMDAwMDAwMCwicG9saWN5IjoidGVzdCJ9.PetK5wWUcnCJkMYv6TEs7HqlA4x_vViykQ8b2T_6hapFGJTO34sfTwqBnHF6lAiWxRoZXco11B0R7y58WAsrQw", + "Expiration": "2019-02-20T19:56:59-08:00", + "AccessKeyId": "K9DTIMUVZXEXJL3ATUOY" + } +} +``` + +### Testing an example with `assume-role.go` + +The included program in this directory can also be used for testing: + +``` shell +$ go run assume-role.go -u foobar -p foo12345 -d +Only displaying credentials: +AccessKeyID: 27YDRYEM0S9B44AJJX9X +SecretAccessKey: LHPdHeaLiYk+pDZ3hgN3sdwXpJC2qbhBfZ8ii9Z3 +SessionToken: eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiIyN1lEUllFTTBTOUI0NEFKSlg5WCIsImV4cCI6MzYwMDAwMDAwMDAwMCwicG9saWN5IjoiY29uc29sZUFkbWluIn0.2d9t0UOm1jQmwe31_5CyN63f6CL-fhqZSO-XhZIp-NH5QteWv9oSMjIrcNWzMgNDblrUfAZ0JSs8a1ciLQF9Ww + +``` + +## Explore Further + +- [MinIO Admin Complete Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc-admin.html) +- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html) diff --git a/docs/sts/casdoor.md b/docs/sts/casdoor.md new file mode 100644 index 00000000..4bdf745b --- /dev/null +++ b/docs/sts/casdoor.md @@ -0,0 +1,116 @@ +# Casdoor Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +Casdoor is a UI-first centralized authentication / Single-Sign-On (SSO) platform supporting OAuth 2.0, OIDC and SAML, integrated with Casbin RBAC and ABAC permission management. This document covers configuring Casdoor identity provider support with MinIO. + +## Prerequisites + +Configure and install casdoor server by following [Casdoor Server Installation](https://casdoor.org/docs/basic/server-installation). +For a quick installation, docker-compose reference configs are also available on the [Casdoor Try with Docker](https://casdoor.org/docs/basic/try-with-docker). + +### Configure Casdoor + +- Go to Applications + - Create or use an existing Casdoor application + - Edit the application + - Copy `Client ID` and `Client secret` + - Add your redirect url (callback url) to `Redirect URLs` + - Save + +- Go to Users + - Edit the user + - Add your MinIO policy (ex: `readwrite`) in `Tag` + - Save + +- Open your favorite browser and visit: **http://`CASDOOR_ENDPOINT`/.well-known/openid-configuration**, you will see the OIDC configure of Casdoor. + +### Configure MinIO + +``` +export MINIO_ROOT_USER=minio +export MINIO_ROOT_PASSWORD=minio123 +minio server /mnt/export +``` + +Here are all the available options to configure OpenID connect + +``` +mc admin config set myminio/ identity_openid + +KEY: +identity_openid enable OpenID SSO support + +ARGS: +config_url* (url) openid discovery document e.g. "https://accounts.google.com/.well-known/openid-configuration" +client_id (string) unique public identifier for apps e.g. "292085223830.apps.googleusercontent.com" +claim_name (string) JWT canned policy claim name, defaults to "policy" +claim_prefix (string) JWT claim namespace prefix e.g. "customer1/" +scopes (csv) Comma separated list of OpenID scopes for server, defaults to advertised scopes from discovery document e.g. "email,admin" +comment (sentence) optionally add a comment to this setting +``` + +and ENV based options + +``` +mc admin config set myminio/ identity_openid --env + +KEY: +identity_openid enable OpenID SSO support + +ARGS: +MINIO_IDENTITY_OPENID_CONFIG_URL* (url) openid discovery document e.g. "https://accounts.google.com/.well-known/openid-configuration" +MINIO_IDENTITY_OPENID_CLIENT_ID (string) unique public identifier for apps e.g. "292085223830.apps.googleusercontent.com" +MINIO_IDENTITY_OPENID_CLAIM_NAME (string) JWT canned policy claim name, defaults to "policy" +MINIO_IDENTITY_OPENID_CLAIM_PREFIX (string) JWT claim namespace prefix e.g. "customer1/" +MINIO_IDENTITY_OPENID_SCOPES (csv) Comma separated list of OpenID scopes for server, defaults to advertised scopes from discovery document e.g. "email,admin" +MINIO_IDENTITY_OPENID_COMMENT (sentence) optionally add a comment to this setting +``` + +Set `identity_openid` config with `config_url`, `client_id` and restart MinIO + +``` +~ mc admin config set myminio identity_openid config_url="http://CASDOOR_ENDPOINT/.well-known/openid-configuration" client_id= client_secret= claim_name="tag" +``` + +> NOTE: As MinIO needs to use a claim attribute in JWT for its policy, you should configure it in casdoor as well. Currently, casdoor uses `tag` as a workaround for configuring MinIO's policy. + +Once successfully set restart the MinIO instance. + +``` +mc admin service restart myminio +``` + +### Using WebIdentiy API + +On another terminal run `web-identity.go` a sample client application which obtains JWT id_tokens from an identity provider, in our case its Keycloak. Uses the returned id_token response to get new temporary credentials from the MinIO server using the STS API call `AssumeRoleWithWebIdentity`. + +``` +$ go run docs/sts/web-identity.go -cid account -csec 072e7f00-4289-469c-9ab2-bbe843c7f5a8 -config-ep "http://CASDOOR_ENDPOINT/.well-known/openid-configuration" -port 8888 +2018/12/26 17:49:36 listening on http://localhost:8888/ +``` + +This will open the login page of Casdoor, upon successful login, STS credentials along with any buckets discovered using the credentials will be printed on the screen, for example: + +``` +{ + buckets: [ ], + credentials: { + AccessKeyID: "EJOLVY3K3G4BF37YD1A0", + SecretAccessKey: "1b+w8LlDqMQOquKxIlZ2ggP+bgE51iwNG7SUVPJJ", + SessionToken: "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJFSk9MVlkzSzNHNEJGMzdZRDFBMCIsImFkZHJlc3MiOltdLCJhZmZpbGlhdGlvbiI6IiIsImFwcGxlIjoiIiwiYXVkIjpbIjI0YTI1ZWEwNzE0ZDkyZTc4NTk1Il0sImF2YXRhciI6Imh0dHBzOi8vY2FzYmluLm9yZy9pbWcvY2FzYmluLnN2ZyIsImF6dXJlYWQiOiIiLCJiaW8iOiIiLCJiaXJ0aGRheSI6IiIsImNyZWF0ZWRJcCI6IiIsImNyZWF0ZWRUaW1lIjoiMjAyMS0xMi0wNlQyMzo1ODo0MyswODowMCIsImRpbmd0YWxrIjoiIiwiZGlzcGxheU5hbWUiOiJjYmMiLCJlZHVjYXRpb24iOiIiLCJlbWFpbCI6IjE5OTkwNjI2LmxvdmVAMTYzLmNvbSIsImV4cCI6MTY0MzIwMjIyMCwiZmFjZWJvb2siOiIiLCJnZW5kZXIiOiIiLCJnaXRlZSI6IiIsImdpdGh1YiI6IiIsImdpdGxhYiI6IiIsImdvb2dsZSI6IiIsImhhc2giOiIiLCJob21lcGFnZSI6IiIsImlhdCI6MTY0MzE5MjEwMSwiaWQiOiIxYzU1NTgxZS01ZmEyLTQ4NTEtOWM2NC04MjNhNjYyZDBkY2IiLCJpZENhcmQiOiIiLCJpZENhcmRUeXBlIjoiIiwiaXNBZG1pbiI6dHJ1ZSwiaXNEZWZhdWx0QXZhdGFyIjpmYWxzZSwiaXNEZWxldGVkIjpmYWxzZSwiaXNGb3JiaWRkZW4iOmZhbHNlLCJpc0dsb2JhbEFkbWluIjp0cnVlLCJpc09ubGluZSI6ZmFsc2UsImlzcyI6Imh0dHA6Ly9sb2NhbGhvc3Q6ODAwMCIsImxhbmd1YWdlIjoiIiwibGFyayI6IiIsImxhc3RTaWduaW5JcCI6IiIsImxhc3RTaWduaW5UaW1lIjoiIiwibGRhcCI6IiIsImxpbmtlZGluIjoiIiwibG9jYXRpb24iOiIiLCJuYW1lIjoiY2JjIiwibmJmIjoxNjQzMTkyMTAxLCJub25jZSI6Im51bGwiLCJvd25lciI6ImJ1aWx0LWluIiwicGFzc3dvcmQiOiIiLCJwYXNzd29yZFNhbHQiOiIiLCJwZXJtYW5lbnRBdmF0YXIiOiIiLCJwaG9uZSI6IjE4ODE3NTgzMjA3IiwicHJlSGFzaCI6IjAwY2JiNGEyOTBjZDBjZDgwZmZkZWMyZjBhOWJlM2E2IiwicHJvcGVydGllcyI6e30sInFxIjoiIiwicmFua2luZyI6MCwicmVnaW9uIjoiIiwic2NvcmUiOjIwMDAsInNpZ251cEFwcGxpY2F0aW9uIjoiYXBwLWJ1aWx0LWluIiwic2xhY2siOiIiLCJzdWIiOiIxYzU1NTgxZS01ZmEyLTQ4NTEtOWM2NC04MjNhNjYyZDBkY2IiLCJ0YWciOiJyZWFkd3JpdGUiLCJ0aXRsZSI6IiIsInR5cGUiOiJub3JtYWwtdXNlciIsInVwZGF0ZWRUaW1lIjoiIiwid2VjaGF0IjoiIiwid2Vjb20iOiIiLCJ3ZWlibyI6IiJ9.C5ZoJrojpRSePg_Ef9O-JTnc9BgoDNC5JX5AxlE9npd2tNl3ftudhny47pG6GgNDeiCMiaxueNyb_HPEPltJTw", + SignerType: 1 + } +} +``` + +### Using MinIO Console + +- Open MinIO URL on the browser, lets say +- Click on `Login with SSO` +- User will be redirected to the Casdoor user login page, upon successful login the user will be redirected to MinIO page and logged in automatically, + the user should see now the buckets and objects they have access to. + +## Explore Further + +- [Casdoor MinIO Integration](https://casdoor.org/docs/integration/minio) +- [MinIO STS Quickstart Guide](https://docs.min.io/community/minio-object-store/developers/security-token-service.html) +- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html) diff --git a/docs/sts/client-grants.go b/docs/sts/client-grants.go new file mode 100644 index 00000000..9a98ee44 --- /dev/null +++ b/docs/sts/client-grants.go @@ -0,0 +1,130 @@ +//go:build ignore +// +build ignore + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package main + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "flag" + "fmt" + "log" + "net/http" + "net/url" + "strings" + + minio "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +// JWTToken - parses the output from IDP id_token. +type JWTToken struct { + AccessToken string `json:"access_token"` + Expiry int `json:"expires_in"` +} + +var ( + stsEndpoint string + idpEndpoint string + clientID string + clientSecret string +) + +func init() { + flag.StringVar(&stsEndpoint, "sts-ep", "http://localhost:9000", "STS endpoint") + flag.StringVar(&idpEndpoint, "idp-ep", "http://localhost:8080/auth/realms/minio/protocol/openid-connect/token", "IDP token endpoint") + flag.StringVar(&clientID, "cid", "", "Client ID") + flag.StringVar(&clientSecret, "csec", "", "Client secret") +} + +func getTokenExpiry() (*credentials.ClientGrantsToken, error) { + data := url.Values{} + data.Set("grant_type", "client_credentials") + req, err := http.NewRequest(http.MethodPost, idpEndpoint, strings.NewReader(data.Encode())) + if err != nil { + return nil, err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + req.SetBasicAuth(clientID, clientSecret) + t := &http.Transport{ + TLSClientConfig: &tls.Config{ + InsecureSkipVerify: true, + }, + } + hclient := http.Client{ + Transport: t, + } + resp, err := hclient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("%s", resp.Status) + } + + var idpToken JWTToken + if err = json.NewDecoder(resp.Body).Decode(&idpToken); err != nil { + return nil, err + } + + return &credentials.ClientGrantsToken{Token: idpToken.AccessToken, Expiry: idpToken.Expiry}, nil +} + +func main() { + flag.Parse() + if clientID == "" || clientSecret == "" { + flag.PrintDefaults() + return + } + + sts, err := credentials.NewSTSClientGrants(stsEndpoint, getTokenExpiry) + if err != nil { + log.Fatal(err) + } + + // Uncomment this to use MinIO API operations by initializing minio + // client with obtained credentials. + + opts := &minio.Options{ + Creds: sts, + BucketLookup: minio.BucketLookupAuto, + } + + u, err := url.Parse(stsEndpoint) + if err != nil { + log.Fatal(err) + } + + clnt, err := minio.New(u.Host, opts) + if err != nil { + log.Fatal(err) + } + + d := bytes.NewReader([]byte("Hello, World")) + n, err := clnt.PutObject(context.Background(), "my-bucketname", "my-objectname", d, d.Size(), minio.PutObjectOptions{}) + if err != nil { + log.Fatalln(err) + } + + log.Println("Uploaded", "my-objectname", " of size: ", n, "Successfully.") +} diff --git a/docs/sts/client-grants.md b/docs/sts/client-grants.md new file mode 100644 index 00000000..1d4132ad --- /dev/null +++ b/docs/sts/client-grants.md @@ -0,0 +1,115 @@ +# AssumeRoleWithClientGrants [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +## Introduction + +Returns a set of temporary security credentials for applications/clients who have been authenticated through client credential grants provided by identity provider. Example providers include KeyCloak, Okta etc. + +Calling AssumeRoleWithClientGrants does not require the use of MinIO default credentials. Therefore, client application can be distributed that requests temporary security credentials without including MinIO default credentials. Instead, the identity of the caller is validated by using a JWT access token from the identity provider. The temporary security credentials returned by this API consists of an access key, a secret key, and a security token. Applications can use these temporary security credentials to sign calls to MinIO API operations. + +By default, the temporary security credentials created by AssumeRoleWithClientGrants last for one hour. However, use the optional DurationSeconds parameter to specify the duration of the credentials. This value varies from 900 seconds (15 minutes) up to the maximum session duration of 365 days. + +## API Request Parameters + +### Token + +The OAuth 2.0 access token that is provided by the identity provider. Application must get this token by authenticating the application using client credential grants before the application makes an AssumeRoleWithClientGrants call. + +| Params | Value | +| :-- | :-- | +| *Type* | *String* | +| *Length Constraints* | *Minimum length of 4. Maximum length of 2048.* | +| *Required* | *Yes* | + +### Version + +Indicates STS API version information, the only supported value is '2011-06-15'. This value is borrowed from AWS STS API documentation for compatibility reasons. + +| Params | Value | +| :-- | :-- | +| *Type* | *String* | +| *Required* | *Yes* | + +### DurationSeconds + +The duration, in seconds. The value can range from 900 seconds (15 minutes) up to 365 days. If value is higher than this setting, then operation fails. By default, the value is set to 3600 seconds. If no *DurationSeconds* is specified expiry seconds is obtained from *Token*. + +| Params | Value | +| :-- | :-- | +| *Type* | *Integer* | +| *Valid Range* | *Minimum value of 900. Maximum value of 31536000.* | +| *Required* | *No* | + +### Policy + +An IAM policy in JSON format that you want to use as an inline session policy. This parameter is optional. Passing policies to this operation returns new temporary credentials. The resulting session's permissions are the intersection of the canned policy name and the policy set here. You cannot use this policy to grant more permissions than those allowed by the canned policy name being assumed. + +| Params | Value | +| :-- | :-- | +| *Type* | *String* | +| *Valid Range* | *Minimum length of 1. Maximum length of 2048.* | +| *Required* | *No* | + +### Response Elements + +XML response for this API is similar to [AWS STS AssumeRoleWithWebIdentity](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html#API_AssumeRoleWithWebIdentity_ResponseElements) + +### Errors + +XML error response for this API is similar to [AWS STS AssumeRoleWithWebIdentity](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html#API_AssumeRoleWithWebIdentity_Errors) + +## Sample `POST` Request + +``` +http://minio.cluster:9000?Action=AssumeRoleWithClientGrants&DurationSeconds=3600&Token=eyJ4NXQiOiJOVEF4Wm1NeE5ETXlaRGczTVRVMVpHTTBNekV6T0RKaFpXSTRORE5sWkRVMU9HRmtOakZpTVEiLCJraWQiOiJOVEF4Wm1NeE5ETXlaRGczTVRVMVpHTTBNekV6T0RKaFpXSTRORE5sWkRVMU9HRmtOakZpTVEiLCJhbGciOiJSUzI1NiJ9.eyJhdWQiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiYXpwIjoiUG9FZ1hQNnVWTzQ1SXNFTlJuZ0RYajVBdTVZYSIsImlzcyI6Imh0dHBzOlwvXC9sb2NhbGhvc3Q6OTQ0M1wvb2F1dGgyXC90b2tlbiIsImV4cCI6MTU0MTgwOTU4MiwiaWF0IjoxNTQxODA1OTgyLCJqdGkiOiI2Y2YyMGIwZS1lNGZmLTQzZmQtYTdiYS1kYTc3YTE3YzM2MzYifQ.Jm29jPliRvrK6Os34nSK3rhzIYLFjE__zdVGNng3uGKXGKzP3We_i6NPnhA0szJXMOKglXzUF1UgSz8MctbaxFS8XDusQPVe4LkB_45hwBm6TmBxzui911nt-1RbBLN_jZIlvl2lPrbTUH5hSn9kEkph6seWanTNQpz9tNEoVa6R_OX3kpJqxe8tLQUWw453A1JTwFNhdHa6-f1K8_Q_eEZ_4gOYINQ9t_fhTibdbkXZkJQFLop-Jwoybi9s4nwQU_dATocgcufq5eCeNItQeleT-23lGxIz0X7CiJrJynYLdd-ER0F77SumqEb5iCxhxuf4H7dovwd1kAmyKzLxpw&Version=2011-06-15 +``` + +## Sample Response + +``` + + + + + + + + + Y4RJU1RNFGK48LGO9I2S + sYLRKS1Z7hSjluf6gEbb9066hnx315wHTiACPAjg + 2019-08-08T20:26:12Z + eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJZNFJKVTFSTkZHSzQ4TEdPOUkyUyIsImF1ZCI6IlBvRWdYUDZ1Vk80NUlzRU5SbmdEWGo1QXU1WWEiLCJhenAiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiZXhwIjoxNTQxODExMDcxLCJpYXQiOjE1NDE4MDc0NzEsImlzcyI6Imh0dHBzOi8vbG9jYWxob3N0Ojk0NDMvb2F1dGgyL3Rva2VuIiwianRpIjoiYTBiMjc2MjktZWUxYS00M2JmLTg3MzktZjMzNzRhNGNkYmMwIn0.ewHqKVFTaP-j_kgZrcOEKroNUjk10GEp8bqQjxBbYVovV0nHO985VnRESFbcT6XMDDKHZiWqN2vi_ETX_u3Q-w + + + + +``` + +## Using ClientGrants API + +``` +export MINIO_ROOT_USER=minio +export MINIO_ROOT_PASSWORD=minio123 +export MINIO_IDENTITY_OPENID_CONFIG_URL=http://localhost:8080/auth/realms/demo/.well-known/openid-configuration +export MINIO_IDENTITY_OPENID_CLIENT_ID="843351d4-1080-11ea-aa20-271ecba3924a" +minio server /mnt/export +``` + +Testing with an example +> Obtaining client ID and secrets follow [Keycloak configuring documentation](https://github.com/minio/minio/blob/master/docs/sts/keycloak.md) + +``` +$ go run client-grants.go -cid PoEgXP6uVO45IsENRngDXj5Au5Ya -csec eKsw6z8CtOJVBtrOWvhRWL4TUCga + +##### Credentials +{ + "accessKey": "NUIBORZYTV2HG2BMRSXR", + "secretKey": "qQlP5O7CFPc5m5IXf1vYhuVTFj7BRVJqh0FqZ86S", + "expiration": "2018-08-21T17:10:29-07:00", + "sessionToken": "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJOVUlCT1JaWVRWMkhHMkJNUlNYUiIsImF1ZCI6IlBvRWdYUDZ1Vk80NUlzRU5SbmdEWGo1QXU1WWEiLCJhenAiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiZXhwIjoxNTM0ODk2NjI5LCJpYXQiOjE1MzQ4OTMwMjksImlzcyI6Imh0dHBzOi8vbG9jYWxob3N0Ojk0NDMvb2F1dGgyL3Rva2VuIiwianRpIjoiNjY2OTZjZTctN2U1Ny00ZjU5LWI0MWQtM2E1YTMzZGZiNjA4In0.eJONnVaSVHypiXKEARSMnSKgr-2mlC2Sr4fEGJitLcJF_at3LeNdTHv0_oHsv6ZZA3zueVGgFlVXMlREgr9LXA" +} +``` + +## Explore Further + +- [MinIO Admin Complete Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc-admin.html) +- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html) diff --git a/docs/sts/client-grants.py b/docs/sts/client-grants.py new file mode 100644 index 00000000..e10db8f4 --- /dev/null +++ b/docs/sts/client-grants.py @@ -0,0 +1,37 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import logging + +import boto3 +from boto3.session import Session +from botocore.session import get_session + +from client_grants import ClientGrantsCredentialProvider + +boto3.set_stream_logger('boto3.resources', logging.DEBUG) + +bc_session = get_session() +bc_session.get_component('credential_provider').insert_before( + 'env', + ClientGrantsCredentialProvider('NZLOOFRSluw9RfIkuHGqfk1HFp4a', + '0Z4VTG8uJBSekn42HE40DK9vQb4a'), +) + +boto3_session = Session(botocore_session=bc_session) +s3 = boto3_session.resource('s3', endpoint_url='http://localhost:9000') + +with open('/etc/hosts', 'rb') as data: + s3.meta.client.upload_fileobj(data, + 'testbucket', + 'hosts', + ExtraArgs={'ServerSideEncryption': 'AES256'}) + +# Upload with server side encryption, using temporary credentials +s3.meta.client.upload_file('/etc/hosts', + 'testbucket', + 'hosts', + ExtraArgs={'ServerSideEncryption': 'AES256'}) + +# Download encrypted object using temporary credentials +s3.meta.client.download_file('testbucket', 'hosts', '/tmp/hosts') diff --git a/docs/sts/client_grants/__init__.py b/docs/sts/client_grants/__init__.py new file mode 100644 index 00000000..d19b15dd --- /dev/null +++ b/docs/sts/client_grants/__init__.py @@ -0,0 +1,132 @@ +# -*- coding: utf-8 -*- +import json +# standard. +import os + +import certifi +# Dependencies +import urllib3 +from botocore.credentials import CredentialProvider, RefreshableCredentials +from botocore.exceptions import CredentialRetrievalError +from dateutil.parser import parse + +from .sts_element import STSElement + + +class ClientGrantsCredentialProvider(CredentialProvider): + """ + ClientGrantsCredentialProvider implements CredentialProvider compatible + implementation to be used with boto_session + """ + METHOD = 'assume-role-client-grants' + CANONICAL_NAME = 'AssumeRoleClientGrants' + + def __init__(self, cid, csec, + idp_ep='http://localhost:8080/auth/realms/minio/protocol/openid-connect/token', + sts_ep='http://localhost:9000'): + self.cid = cid + self.csec = csec + self.idp_ep = idp_ep + self.sts_ep = sts_ep + + # Load CA certificates from SSL_CERT_FILE file if set + ca_certs = os.environ.get('SSL_CERT_FILE') + if not ca_certs: + ca_certs = certifi.where() + + self._http = urllib3.PoolManager( + timeout=urllib3.Timeout.DEFAULT_TIMEOUT, + maxsize=10, + cert_reqs='CERT_NONE', + ca_certs=ca_certs, + retries=urllib3.Retry( + total=5, + backoff_factor=0.2, + status_forcelist=[500, 502, 503, 504] + ) + ) + + def load(self): + """ + Search for credentials with client_grants + """ + if self.cid is not None: + fetcher = self._create_credentials_fetcher() + return RefreshableCredentials.create_from_metadata( + metadata=fetcher(), + refresh_using=fetcher, + method=self.METHOD, + ) + else: + return None + + def _create_credentials_fetcher(self): + method = self.METHOD + + def fetch_credentials(): + # HTTP headers are case insensitive filter out + # all duplicate headers and pick one. + headers = {} + headers['content-type'] = 'application/x-www-form-urlencoded' + headers['authorization'] = urllib3.make_headers( + basic_auth='%s:%s' % (self.cid, self.csec))['authorization'] + + response = self._http.urlopen('POST', self.idp_ep, + body="grant_type=client_credentials", + headers=headers, + preload_content=True, + ) + if response.status != 200: + message = "Credential refresh failed, response: %s" + raise CredentialRetrievalError( + provider=method, + error_msg=message % response.status, + ) + + creds = json.loads(response.data) + + query = {} + query['Action'] = 'AssumeRoleWithClientGrants' + query['Token'] = creds['access_token'] + query['DurationSeconds'] = creds['expires_in'] + query['Version'] = '2011-06-15' + + query_components = [] + for key in query: + if query[key] is not None: + query_components.append("%s=%s" % (key, query[key])) + + query_string = '&'.join(query_components) + sts_ep_url = self.sts_ep + if query_string: + sts_ep_url = self.sts_ep + '?' + query_string + + response = self._http.urlopen( + 'POST', sts_ep_url, preload_content=True) + if response.status != 200: + message = "Credential refresh failed, response: %s" + raise CredentialRetrievalError( + provider=method, + error_msg=message % response.status, + ) + + return parse_grants_response(response.data) + + def parse_grants_response(data): + """ + Parser for AssumeRoleWithClientGrants response + + :param data: Response data for AssumeRoleWithClientGrants request + :return: dict + """ + root = STSElement.fromstring( + 'AssumeRoleWithClientGrantsResponse', data) + result = root.find('AssumeRoleWithClientGrantsResult') + creds = result.find('Credentials') + return dict( + access_key=creds.get_child_text('AccessKeyId'), + secret_key=creds.get_child_text('SecretAccessKey'), + token=creds.get_child_text('SessionToken'), + expiry_time=parse(creds.get_child_text('Expiration')).isoformat()) + + return fetch_credentials diff --git a/docs/sts/client_grants/sts_element.py b/docs/sts/client_grants/sts_element.py new file mode 100644 index 00000000..89eb8d3d --- /dev/null +++ b/docs/sts/client_grants/sts_element.py @@ -0,0 +1,78 @@ +# -*- coding: utf-8 -*- +from xml.etree import cElementTree +from xml.etree.cElementTree import ParseError + +if hasattr(cElementTree, 'ParseError'): + _ETREE_EXCEPTIONS = (ParseError, AttributeError, ValueError, TypeError) +else: + _ETREE_EXCEPTIONS = (SyntaxError, AttributeError, ValueError, TypeError) + +_STS_NS = {'sts': 'https://sts.amazonaws.com/doc/2011-06-15/'} + + +class STSElement(object): + """STS aware XML parsing class. Wraps a root element name and + cElementTree.Element instance. Provides STS namespace aware parsing + functions. + + """ + + def __init__(self, root_name, element): + self.root_name = root_name + self.element = element + + @classmethod + def fromstring(cls, root_name, data): + """Initialize STSElement from name and XML string data. + + :param name: Name for XML data. Used in XML errors. + :param data: string data to be parsed. + :return: Returns an STSElement. + """ + try: + return cls(root_name, cElementTree.fromstring(data)) + except _ETREE_EXCEPTIONS as error: + raise InvalidXMLError( + '"{}" XML is not parsable. Message: {}'.format( + root_name, error.message + ) + ) + + def findall(self, name): + """Similar to ElementTree.Element.findall() + + """ + return [ + STSElement(self.root_name, elem) + for elem in self.element.findall('sts:{}'.format(name), _STS_NS) + ] + + def find(self, name): + """Similar to ElementTree.Element.find() + + """ + elt = self.element.find('sts:{}'.format(name), _STS_NS) + return STSElement(self.root_name, elt) if elt is not None else None + + def get_child_text(self, name, strict=True): + """Extract text of a child element. If strict, and child element is + not present, raises InvalidXMLError and otherwise returns + None. + + """ + if strict: + try: + return self.element.find('sts:{}'.format(name), _STS_NS).text + except _ETREE_EXCEPTIONS as error: + raise InvalidXMLError( + ('Invalid XML provided for "{}" - erroring tag <{}>. ' + 'Message: {}').format(self.root_name, name, error.message) + ) + else: + return self.element.findtext('sts:{}'.format(name), None, _STS_NS) + + def text(self): + """Fetch the current node's text + + """ + return self.element.text diff --git a/docs/sts/custom-token-identity.go b/docs/sts/custom-token-identity.go new file mode 100644 index 00000000..3d2d8d79 --- /dev/null +++ b/docs/sts/custom-token-identity.go @@ -0,0 +1,121 @@ +//go:build ignore +// +build ignore + +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package main + +import ( + "context" + "flag" + "fmt" + "log" + "net/url" + "time" + + "github.com/minio/minio-go/v7" + cr "github.com/minio/minio-go/v7/pkg/credentials" +) + +var ( + // LDAP integrated Minio endpoint + stsEndpoint string + + // token to use with AssumeRoleWithCustomToken + token string + + // Role ARN to use + roleArn string + + // Display credentials flag + displayCreds bool + + // Credential expiry duration + expiryDuration time.Duration + + // Bucket to list + bucketToList string +) + +func init() { + flag.StringVar(&stsEndpoint, "sts-ep", "http://localhost:9000", "STS endpoint") + flag.StringVar(&token, "t", "", "Token to use with AssumeRoleWithCustomToken STS API (required)") + flag.StringVar(&roleArn, "r", "", "RoleARN to use with the request (required)") + flag.BoolVar(&displayCreds, "d", false, "Only show generated credentials") + flag.DurationVar(&expiryDuration, "e", 0, "Request a duration of validity for the generated credential") + flag.StringVar(&bucketToList, "b", "mybucket", "Bucket to list (defaults to mybucket)") +} + +func main() { + flag.Parse() + if token == "" || roleArn == "" { + flag.PrintDefaults() + return + } + + // The credentials package in minio-go provides an interface to call the + // AssumeRoleWithCustomToken STS API. + + var opts []cr.CustomTokenOpt + if expiryDuration != 0 { + opts = append(opts, cr.CustomTokenValidityOpt(expiryDuration)) + } + + // Initialize + li, err := cr.NewCustomTokenCredentials(stsEndpoint, token, roleArn, opts...) + if err != nil { + log.Fatalf("Error initializing CustomToken Identity: %v", err) + } + + v, err := li.Get() + if err != nil { + log.Fatalf("Error retrieving STS credentials: %v", err) + } + + if displayCreds { + fmt.Println("Only displaying credentials:") + fmt.Println("AccessKeyID:", v.AccessKeyID) + fmt.Println("SecretAccessKey:", v.SecretAccessKey) + fmt.Println("SessionToken:", v.SessionToken) + return + } + + // Use generated credentials to authenticate with MinIO server + stsEndpointURL, err := url.Parse(stsEndpoint) + if err != nil { + log.Fatalf("Error parsing sts endpoint: %v", err) + } + copts := &minio.Options{ + Creds: li, + Secure: stsEndpointURL.Scheme == "https", + } + minioClient, err := minio.New(stsEndpointURL.Host, copts) + if err != nil { + log.Fatalf("Error initializing client: ", err) + } + + // Use minIO Client object normally like the regular client. + fmt.Printf("Calling list objects on bucket named `%s` with temp creds:\n===\n", bucketToList) + objCh := minioClient.ListObjects(context.Background(), bucketToList, minio.ListObjectsOptions{}) + for obj := range objCh { + if obj.Err != nil { + log.Fatalf("Listing error: %v", obj.Err) + } + fmt.Printf("Key: %s\nSize: %d\nLast Modified: %s\n===\n", obj.Key, obj.Size, obj.LastModified) + } +} diff --git a/docs/sts/custom-token-identity.md b/docs/sts/custom-token-identity.md new file mode 100644 index 00000000..882a67bc --- /dev/null +++ b/docs/sts/custom-token-identity.md @@ -0,0 +1,53 @@ +# AssumeRoleWithCustomToken [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +## Introduction + +To integrate with custom authentication methods using the [Identity Management Plugin](../iam/identity-management-plugin.md)), MinIO provides an STS API extension called `AssumeRoleWithCustomToken`. + +After configuring the plugin, use the generated Role ARN with `AssumeRoleWithCustomToken` to get temporary credentials to access object storage. + +## API Request + +To make an STS API request with this method, send a POST request to the MinIO endpoint with following query parameters: + +| Parameter | Type | Required | | +|-----------------|---------|----------|----------------------------------------------------------------------| +| Action | String | Yes | Value must be `AssumeRoleWithCustomToken` | +| Version | String | Yes | Value must be `2011-06-15` | +| Token | String | Yes | Token to be authenticated by identity plugin | +| RoleArn | String | Yes | Must match the Role ARN generated for the identity plugin | +| DurationSeconds | Integer | No | Duration of validity of generated credentials. Must be at least 900. | + +The validity duration of the generated STS credentials is the minimum of the `DurationSeconds` parameter (if passed) and the validity duration returned by the Identity Management Plugin. + +## API Response + +XML response for this API is similar to [AWS STS AssumeRoleWithWebIdentity](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html#API_AssumeRoleWithWebIdentity_ResponseElements) + +## Example request and response + +Sample request with `curl`: + +```sh +curl -XPOST 'http://localhost:9001/?Action=AssumeRoleWithCustomToken&Version=2011-06-15&Token=aaa&RoleArn=arn:minio:iam:::role/idmp-vGxBdLkOc8mQPU1-UQbBh-yWWVQ' +``` + +Prettified Response: + +```xml + + + + + 24Y5H9VHE14H47GEOKCX + H+aBfQ9B1AeWWb++84hvp4tlFBo9aP+hUTdLFIeg + 2022-05-25T19:56:34Z + eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiIyNFk1SDlWSEUxNEg0N0dFT0tDWCIsImV4cCI6MTY1MzUwODU5NCwiZ3JvdXBzIjpbImRhdGEtc2NpZW5jZSJdLCJwYXJlbnQiOiJjdXN0b206QWxpY2UiLCJyb2xlQXJuIjoiYXJuOm1pbmlvOmlhbTo6OnJvbGUvaWRtcC14eHgiLCJzdWIiOiJjdXN0b206QWxpY2UifQ.1tO1LmlUNXiy-wl-ZbkJLWTpaPlhaGqHehsi21lNAmAGCImHHsPb-GA4lRq6GkvHAODN5ZYCf_S-OwpOOdxFwA + + custom:Alice + + + 16F26E081E36DE63 + + +``` diff --git a/docs/sts/dex.md b/docs/sts/dex.md new file mode 100644 index 00000000..e7bf3d83 --- /dev/null +++ b/docs/sts/dex.md @@ -0,0 +1,103 @@ +# Dex Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +Dex is an identity service that uses OpenID Connect to drive authentication for apps. Dex acts as a portal to other identity providers through "connectors." This lets dex defer authentication to LDAP servers, SAML providers, or established identity providers like GitHub, Google, and Active Directory. Clients write their authentication logic once to talk to dex, then dex handles the protocols for a given backend. + +## Prerequisites + +Install Dex by following [Dex Getting Started Guide](https://dexidp.io/docs/getting-started/) + +### Start Dex + +``` +~ ./bin/dex serve dex.yaml +time="2020-07-12T20:45:50Z" level=info msg="config issuer: http://127.0.0.1:5556/dex" +time="2020-07-12T20:45:50Z" level=info msg="config storage: sqlite3" +time="2020-07-12T20:45:50Z" level=info msg="config static client: Example App" +time="2020-07-12T20:45:50Z" level=info msg="config connector: mock" +time="2020-07-12T20:45:50Z" level=info msg="config connector: local passwords enabled" +time="2020-07-12T20:45:50Z" level=info msg="config response types accepted: [code token id_token]" +time="2020-07-12T20:45:50Z" level=info msg="config using password grant connector: local" +time="2020-07-12T20:45:50Z" level=info msg="config signing keys expire after: 3h0m0s" +time="2020-07-12T20:45:50Z" level=info msg="config id tokens valid for: 3h0m0s" +time="2020-07-12T20:45:50Z" level=info msg="listening (http) on 0.0.0.0:5556" +``` + +### Configure MinIO server with Dex + +``` +~ export MINIO_IDENTITY_OPENID_CLAIM_NAME=name +~ export MINIO_IDENTITY_OPENID_CONFIG_URL=http://127.0.0.1:5556/dex/.well-known/openid-configuration +~ minio server ~/test +``` + +### Run the `web-identity.go` + +``` +~ go run web-identity.go -cid example-app -csec ZXhhbXBsZS1hcHAtc2VjcmV0 \ + -config-ep http://127.0.0.1:5556/dex/.well-known/openid-configuration \ + -cscopes groups,openid,email,profile +``` + +``` +~ mc admin policy create admin allaccess.json +``` + +Contents of `allaccess.json` + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:*" + ], + "Resource": [ + "arn:aws:s3:::*" + ] + } + ] +} +``` + +### Visit + +You will be redirected to dex login screen - click "Login with email", enter username password +> username: admin@example.com +> password: password + +and then click "Grant access" + +On the browser now you shall see the list of buckets output, along with your temporary credentials obtained from MinIO. + +``` +{ + "buckets": [ + "dl.minio.equipment", + "dl.minio.service-fulfillment", + "testbucket" + ], + "credentials": { + "AccessKeyID": "Q31CVS1PSCJ4OTK2YVEM", + "SecretAccessKey": "rmDEOKARqKYmEyjWGhmhLpzcncyu7Jf8aZ9bjDic", + "SessionToken": "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJRMzFDVlMxUFNDSjRPVEsyWVZFTSIsImF0X2hhc2giOiI4amItZFE2OXRtZEVueUZaMUttNWhnIiwiYXVkIjoiZXhhbXBsZS1hcHAiLCJlbWFpbCI6ImFkbWluQGV4YW1wbGUuY29tIiwiZW1haWxfdmVyaWZpZWQiOnRydWUsImV4cCI6IjE1OTQ2MDAxODIiLCJpYXQiOjE1OTQ1ODkzODQsImlzcyI6Imh0dHA6Ly8xMjcuMC4wLjE6NTU1Ni9kZXgiLCJuYW1lIjoiYWRtaW4iLCJzdWIiOiJDaVF3T0dFNE5qZzBZaTFrWWpnNExUUmlOek10T1RCaE9TMHpZMlF4TmpZeFpqVTBOallTQld4dlkyRnMifQ.nrbzIJz99Om7TvJ04jnSTmhvlM7aR9hMM1Aqjp2ONJ1UKYCvegBLrTu6cYR968_OpmnAGJ8vkd7sIjUjtR4zbw", + "SignerType": 1 + } +} +``` + +Now you have successfully configured Dex IdP with MinIO. + +> NOTE: Dex supports groups with external connectors so you can use `groups` as policy claim instead of `name`. + +``` +export MINIO_IDENTITY_OPENID_CLAIM_NAME=groups +``` + +and add relevant policies on MinIO using `mc admin policy create myminio/ group-access.json` + +## Explore Further + +- [MinIO STS Quickstart Guide](https://docs.min.io/community/minio-object-store/developers/security-token-service.html) +- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html) diff --git a/docs/sts/dex.yaml b/docs/sts/dex.yaml new file mode 100644 index 00000000..20b81896 --- /dev/null +++ b/docs/sts/dex.yaml @@ -0,0 +1,78 @@ +# The base path of dex and the external name of the OpenID Connect service. +# This is the canonical URL that all clients MUST use to refer to dex. If a +# path is provided, dex's HTTP service will listen at a non-root URL. +issuer: http://127.0.0.1:5556/dex + +# The storage configuration determines where dex stores its state. Supported +# options include SQL flavors and Kubernetes third party resources. +# +# See the storage document at Documentation/storage.md for further information. +storage: + type: sqlite3 + config: + file: examples/dex.db + +# Configuration for the HTTP endpoints. +web: + http: 0.0.0.0:5556 + # Uncomment for HTTPS options. + # https: 127.0.0.1:5554 + # tlsCert: /etc/dex/tls.crt + # tlsKey: /etc/dex/tls.key + + # Configuration for telemetry + telemetry: + http: 0.0.0.0:5558 + +# Uncomment this block to enable configuration for the expiration time durations. +expiry: + signingKeys: "3h" + idTokens: "3h" + + # Options for controlling the logger. + logger: + level: "debug" + format: "text" # can also be "json" + +# Default values shown below +oauth2: + # use ["code", "token", "id_token"] to enable implicit flow for web-only clients + responseTypes: [ "code", "token", "id_token" ] # also allowed are "token" and "id_token" + # By default, Dex will ask for approval to share data with application + # (approval for sharing data from connected IdP to Dex is separate process on IdP) + skipApprovalScreen: false + # If only one authentication method is enabled, the default behavior is to + # go directly to it. For connected IdPs, this redirects the browser away + # from application to upstream provider such as the Google login page + alwaysShowLoginScreen: false + # Uncommend the passwordConnector to use a specific connector for password grants + passwordConnector: local + +# Instead of reading from an external storage, use this list of clients. +# +# If this option isn't chosen clients may be added through the gRPC API. +staticClients: + - id: example-app + redirectURIs: + - 'http://localhost:8080/oauth2/callback' + name: 'Example App' + secret: ZXhhbXBsZS1hcHAtc2VjcmV0 + +connectors: + - type: mockCallback + id: mock + name: Example + +# Let dex keep a list of passwords which can be used to login to dex. +enablePasswordDB: true + +# A static list of passwords to login the end user. By identifying here, dex +# won't look in its underlying storage for passwords. +# +# If this option isn't chosen users may be added through the gRPC API. +staticPasswords: + - email: "admin@example.com" + # bcrypt hash of the string "password" + hash: "$2a$10$2b2cU8CPhOTaGrs1HRQuAueS7JTT5ZHsHSzYiFPm1leZck7Mc8T4W" + username: "admin" + userID: "08a8684b-db88-4b73-90a9-3cd1661f5466" diff --git a/docs/sts/etcd.md b/docs/sts/etcd.md new file mode 100644 index 00000000..d9300186 --- /dev/null +++ b/docs/sts/etcd.md @@ -0,0 +1,72 @@ +# etcd V3 Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +etcd is a distributed key value store that provides a reliable way to store data across a cluster of machines. + +## Get started + +### 1. Prerequisites + +- Docker 18.03 or above, refer here for [installation](https://docs.docker.com/install/). + +### 2. Start etcd + +etcd uses [gcr.io/etcd-development/etcd](https://console.cloud.google.com/gcr/images/etcd-development/GLOBAL/etcd) as a primary container registry. + +``` +rm -rf /tmp/etcd-data.tmp && mkdir -p /tmp/etcd-data.tmp && \ + podman rmi gcr.io/etcd-development/etcd:v3.3.9 || true && \ + podman run \ + -p 2379:2379 \ + -p 2380:2380 \ + --mount type=bind,source=/tmp/etcd-data.tmp,destination=/etcd-data \ + --name etcd-gcr-v3.3.9 \ + gcr.io/etcd-development/etcd:v3.3.9 \ + /usr/local/bin/etcd \ + --name s1 \ + --data-dir /etcd-data \ + --listen-client-urls http://0.0.0.0:2379 \ + --advertise-client-urls http://0.0.0.0:2379 \ + --listen-peer-urls http://0.0.0.0:2380 \ + --initial-advertise-peer-urls http://0.0.0.0:2380 \ + --initial-cluster s1=http://0.0.0.0:2380 \ + --initial-cluster-token tkn \ + --initial-cluster-state new +``` + +You may also setup etcd with TLS following this documentation [here](https://coreos.com/etcd/docs/latest/op-guide/security.html) + +### 3. Setup MinIO with etcd + +MinIO server expects environment variable for etcd as `MINIO_ETCD_ENDPOINTS`, this environment variable takes many comma separated entries. + +``` +export MINIO_ETCD_ENDPOINTS=http://localhost:2379 +minio server /data +``` + +NOTE: If `etcd` is configured with `Client-to-server authentication with HTTPS client certificates` then you need to use additional envs such as `MINIO_ETCD_CLIENT_CERT` pointing to path to `etcd-client.crt` and `MINIO_ETCD_CLIENT_CERT_KEY` path to `etcd-client.key` . + +### 4. Test with MinIO STS API + +Once etcd is configured, **any STS configuration** will work including Client Grants, Web Identity or AD/LDAP. + +For example, you can configure STS with Client Grants (KeyCloak) using the guides at [MinIO STS Quickstart Guide](https://docs.min.io/community/minio-object-store/developers/security-token-service.html) and [KeyCloak Configuration Guide](https://github.com/minio/minio/blob/master/docs/sts/keycloak.md). Once this is done, STS credentials can be generated: + +``` +go run client-grants.go -cid PoEgXP6uVO45IsENRngDXj5Au5Ya -csec eKsw6z8CtOJVBtrOWvhRWL4TUCga + +##### Credentials +{ + "accessKey": "IRBLVDGN5QGMDCMO1X8V", + "secretKey": "KzS3UZKE7xqNdtRbKyfcWgxBS6P1G4kwZn4DXKuY", + "expiration": "2018-08-21T15:49:38-07:00", + "sessionToken": "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJJUkJMVkRHTjVRR01EQ01PMVg4ViIsImF1ZCI6IlBvRWdYUDZ1Vk80NUlzRU5SbmdEWGo1QXU1WWEiLCJhenAiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiZXhwIjoxNTM0ODkxNzc4LCJpYXQiOjE1MzQ4ODgxNzgsImlzcyI6Imh0dHBzOi8vbG9jYWxob3N0Ojk0NDMvb2F1dGgyL3Rva2VuIiwianRpIjoiMTg0NDMyOWMtZDY1YS00OGEzLTgyMjgtOWRmNzNmZTgzZDU2In0.4rKsZ8VkZnIS_ALzfTJ9UbEKPFlQVvIyuHw6AWTJcDFDVgQA2ooQHmH9wUDnhXBi1M7o8yWJ47DXP-TLPhwCgQ" +} +``` + +These credentials can now be used to perform MinIO API operations, these credentials automatically expire in 1hr. To understand more about credential expiry duration and client grants STS API read further [here](https://github.com/minio/minio/blob/master/docs/sts/client-grants.md). + +## Explore Further + +- [MinIO STS Quickstart Guide](https://docs.min.io/community/minio-object-store/developers/security-token-service.html) +- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html) diff --git a/docs/sts/keycloak.md b/docs/sts/keycloak.md new file mode 100644 index 00000000..8c60dc55 --- /dev/null +++ b/docs/sts/keycloak.md @@ -0,0 +1,176 @@ +# Keycloak Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +Keycloak is an open source Identity and Access Management solution aimed at modern applications and services, this document covers configuring Keycloak identity provider support with MinIO. + +## Prerequisites + +Configure and install keycloak server by following [Keycloak Installation Guide](https://www.keycloak.org/docs/latest/server_installation/#installing-the-software). +For a quick installation, docker-compose reference configs are also available on the [Keycloak GitHub](https://github.com/keycloak/keycloak-containers/tree/main/docker-compose-examples). + +### Configure Keycloak Realm + +- Go to Clients + - Click on account + - Settings + - Change `Access Type` to `confidential`. + - Save + - Click on credentials tab + - Copy the `Secret` to clipboard. + - This value is needed for `MINIO_IDENTITY_OPENID_CLIENT_SECRET` for MinIO. + +- Go to Users + - Click on the user + - Attribute, add a new attribute `Key` is `policy`, `Value` is name of the `policy` on MinIO (ex: `readwrite`) + - Add and Save + +- Go to Clients + - Click on `account` + - Settings, set `Valid Redirect URIs` to `*`, expand `Advanced Settings` and set `Access Token Lifespan` to `1 Hours` + - Save + +- Go to Clients + - Click on `account` + - Mappers + - Create + - `Name` with any text + - `Mapper Type` is `User Attribute` + - `User Attribute` is `policy` + - `Token Claim Name` is `policy` + - `Claim JSON Type` is `string` + - Save + +- Open to verify OpenID discovery document, verify it has `authorization_endpoint` and `jwks_uri` + +### Enable Keycloak Admin REST API support + +Before being able to authenticate against the Admin REST API using a client_id and a client_secret you need to make sure the client is configured as it follows: + +- `account` client_id is a confidential client that belongs to the realm `{realm}` +- `account` client_id is has **Service Accounts Enabled** option enabled. +- `account` client_id has a custom "Audience" mapper, in the Mappers section. + - Included Client Audience: security-admin-console + +#### Adding 'admin' Role + +- Go to Roles + - Add new Role `admin` with Description `${role_admin}`. + - Add this Role into compositive role named `default-roles-{realm}` - `{realm}` should be replaced with whatever realm you created from `prerequisites` section. This role is automatically trusted in the 'Service Accounts' tab. + +- Check that `account` client_id has the role 'admin' assigned in the "Service Account Roles" tab. + +After that, you will be able to obtain an id_token for the Admin REST API using client_id and client_secret: + +``` +curl \ + -d "client_id=" \ + -d "client_secret=" \ + -d "grant_type=client_credentials" \ + "http://localhost:8080/auth/realms/{realm}/protocol/openid-connect/token" +``` + +The result will be a JSON document. To invoke the API you need to extract the value of the access_token property. You can then invoke the API by including the value in the Authorization header of requests to the API. + +The following example shows how to get the details of the user with `{userid}` from `{realm}` realm: + +``` +curl \ + -H "Authorization: Bearer eyJhbGciOiJSUz..." \ + "http://localhost:8080/auth/admin/realms/{realm}/users/{userid}" +``` + +### Configure MinIO + +``` +export MINIO_ROOT_USER=minio +export MINIO_ROOT_PASSWORD=minio123 +minio server /mnt/export +``` + +Here are all the available options to configure OpenID connect + +``` +mc admin config set myminio/ identity_openid + +KEY: +identity_openid enable OpenID SSO support + +ARGS: +config_url* (url) openid discovery document e.g. "https://accounts.google.com/.well-known/openid-configuration" +client_id (string) unique public identifier for apps e.g. "292085223830.apps.googleusercontent.com" +claim_name (string) JWT canned policy claim name, defaults to "policy" +claim_prefix (string) JWT claim namespace prefix e.g. "customer1/" +scopes (csv) Comma separated list of OpenID scopes for server, defaults to advertised scopes from discovery document e.g. "email,admin" +comment (sentence) optionally add a comment to this setting +``` + +and ENV based options + +``` +mc admin config set myminio/ identity_openid --env + +KEY: +identity_openid enable OpenID SSO support + +ARGS: +MINIO_IDENTITY_OPENID_CONFIG_URL* (url) openid discovery document e.g. "https://accounts.google.com/.well-known/openid-configuration" +MINIO_IDENTITY_OPENID_CLIENT_ID (string) unique public identifier for apps e.g. "292085223830.apps.googleusercontent.com" +MINIO_IDENTITY_OPENID_CLAIM_NAME (string) JWT canned policy claim name, defaults to "policy" +MINIO_IDENTITY_OPENID_CLAIM_PREFIX (string) JWT claim namespace prefix e.g. "customer1/" +MINIO_IDENTITY_OPENID_SCOPES (csv) Comma separated list of OpenID scopes for server, defaults to advertised scopes from discovery document e.g. "email,admin" +MINIO_IDENTITY_OPENID_COMMENT (sentence) optionally add a comment to this setting +``` + +Set `identity_openid` config with `config_url`, `client_id` and restart MinIO + +``` +~ mc admin config set myminio identity_openid config_url="http://localhost:8080/auth/realms/{your-realm-name}/.well-known/openid-configuration" client_id="account" +``` + +> NOTE: You can configure the `scopes` parameter to restrict the OpenID scopes requested by minio to the IdP, for example, `"openid,policy_role_attribute"`, being `policy_role_attribute` a client_scope / client_mapper that maps a role attribute called policy to a `policy` claim returned by Keycloak + +Once successfully set restart the MinIO instance. + +``` +mc admin service restart myminio +``` + +### Using WebIdentiy API + +Client ID can be found by clicking any of the clients listed [here](http://localhost:8080/auth/admin/master/console/#/realms/minio/clients). If you have followed the above steps docs, the default Client ID will be `account`. + +``` +$ go run docs/sts/web-identity.go -cid account -csec 072e7f00-4289-469c-9ab2-bbe843c7f5a8 -config-ep "http://localhost:8080/auth/realms/minio/.well-known/openid-configuration" -port 8888 +2018/12/26 17:49:36 listening on http://localhost:8888/ +``` + +This will open the login page of keycloak, upon successful login, STS credentials along with any buckets discovered using the credentials will be printed on the screen, for example: + +``` +{ + "buckets": [ + "bucket-x" + ], + "credentials": { + "AccessKeyID": "6N2BALX7ELO827DXS3GK", + "SecretAccessKey": "23JKqAD+um8ObHqzfIh+bfqwG9V8qs9tFY6MqeFR+xxx", + "SessionToken": "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiI2TjJCQUxYN0VMTzgyN0RYUzNHSyIsImFjciI6IjAiLCJhdWQiOiJhY2NvdW50IiwiYXV0aF90aW1lIjoxNTY5OTEwNTUyLCJhenAiOiJhY2NvdW50IiwiZW1haWxfdmVyaWZpZWQiOmZhbHNlLCJleHAiOjE1Njk5MTQ1NTQsImlhdCI6MTU2OTkxMDk1NCwiaXNzIjoiaHR0cDovL2xvY2FsaG9zdDo4MDgxL2F1dGgvcmVhbG1zL2RlbW8iLCJqdGkiOiJkOTk4YTBlZS01NDk2LTQ4OWYtYWJlMi00ZWE5MjJiZDlhYWYiLCJuYmYiOjAsInBvbGljeSI6InJlYWR3cml0ZSIsInByZWZlcnJlZF91c2VybmFtZSI6Im5ld3VzZXIxIiwic2Vzc2lvbl9zdGF0ZSI6IjJiYTAyYTI2LWE5MTUtNDUxNC04M2M1LWE0YjgwYjc4ZTgxNyIsInN1YiI6IjY4ZmMzODVhLTA5MjItNGQyMS04N2U5LTZkZTdhYjA3Njc2NSIsInR5cCI6IklEIn0._UG_-ZHgwdRnsp0gFdwChb7VlbPs-Gr_RNUz9EV7TggCD59qjCFAKjNrVHfOSVkKvYEMe0PvwfRKjnJl3A_mBA"", + "SignerType": 1 + } +} +``` + +> NOTE: You can use the `-cscopes` parameter to restrict the requested scopes, for example to `"openid,policy_role_attribute"`, being `policy_role_attribute` a client_scope / client_mapper that maps a role attribute called policy to a `policy` claim returned by Keycloak. + +These credentials can now be used to perform MinIO API operations. + +### Using MinIO Console + +- Open MinIO URL on the browser, lets say +- Click on `Login with SSO` +- User will be redirected to the Keycloak user login page, upon successful login the user will be redirected to MinIO page and logged in automatically, + the user should see now the buckets and objects they have access to. + +## Explore Further + +- [MinIO STS Quickstart Guide](https://docs.min.io/community/minio-object-store/developers/security-token-service.html) +- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html) diff --git a/docs/sts/ldap.go b/docs/sts/ldap.go new file mode 100644 index 00000000..1f753f03 --- /dev/null +++ b/docs/sts/ldap.go @@ -0,0 +1,142 @@ +//go:build ignore +// +build ignore + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package main + +import ( + "context" + "flag" + "fmt" + "io" + "log" + "net/url" + "os" + "time" + + "github.com/minio/minio-go/v7" + cr "github.com/minio/minio-go/v7/pkg/credentials" +) + +var ( + // LDAP integrated Minio endpoint + stsEndpoint string + + // LDAP credentials + ldapUsername string + ldapPassword string + + // Display credentials flag + displayCreds bool + + // Credential expiry duration + expiryDuration time.Duration + + // Bucket to list + bucketToList string + + // Session policy file + sessionPolicyFile string +) + +func init() { + flag.StringVar(&stsEndpoint, "sts-ep", "http://localhost:9000", "STS endpoint") + flag.StringVar(&ldapUsername, "u", "", "AD/LDAP Username") + flag.StringVar(&ldapPassword, "p", "", "AD/LDAP Password") + flag.BoolVar(&displayCreds, "d", false, "Only show generated credentials") + flag.DurationVar(&expiryDuration, "e", 0, "Request a duration of validity for the generated credential") + flag.StringVar(&bucketToList, "b", "", "Bucket to list (defaults to ldap username)") + flag.StringVar(&sessionPolicyFile, "s", "", "File containing session policy to apply to the STS request") +} + +func main() { + flag.Parse() + if ldapUsername == "" || ldapPassword == "" { + flag.PrintDefaults() + return + } + + // The credentials package in minio-go provides an interface to call the + // LDAP STS API. + + // Initialize LDAP credentials + var ldapOpts []cr.LDAPIdentityOpt + if sessionPolicyFile != "" { + var policy string + if f, err := os.Open(sessionPolicyFile); err != nil { + log.Fatalf("Unable to open session policy file %s: %v", sessionPolicyFile, err) + } else { + bs, err := io.ReadAll(f) + if err != nil { + log.Fatalf("Error reading session policy file: %v", err) + } + policy = string(bs) + } + ldapOpts = append(ldapOpts, cr.LDAPIdentityPolicyOpt(policy)) + } + if expiryDuration != 0 { + ldapOpts = append(ldapOpts, cr.LDAPIdentityExpiryOpt(expiryDuration)) + } + li, err := cr.NewLDAPIdentity(stsEndpoint, ldapUsername, ldapPassword, ldapOpts...) + if err != nil { + log.Fatalf("Error initializing LDAP Identity: %v", err) + } + + stsEndpointURL, err := url.Parse(stsEndpoint) + if err != nil { + log.Fatalf("Error parsing sts endpoint: %v", err) + } + + opts := &minio.Options{ + Creds: li, + Secure: stsEndpointURL.Scheme == "https", + } + + v, err := li.Get() + if err != nil { + log.Fatalf("Error retrieving STS credentials: %v", err) + } + + if displayCreds { + fmt.Println("Only displaying credentials:") + fmt.Println("AccessKeyID:", v.AccessKeyID) + fmt.Println("SecretAccessKey:", v.SecretAccessKey) + fmt.Println("SessionToken:", v.SessionToken) + return + } + + // Use generated credentials to authenticate with MinIO server + minioClient, err := minio.New(stsEndpointURL.Host, opts) + if err != nil { + log.Fatalf("Error initializing client: %v", err) + } + + // Use minIO Client object normally like the regular client. + if bucketToList == "" { + bucketToList = ldapUsername + } + fmt.Printf("Calling list objects on bucket named `%s` with temp creds:\n===\n", bucketToList) + objCh := minioClient.ListObjects(context.Background(), bucketToList, minio.ListObjectsOptions{}) + for obj := range objCh { + if obj.Err != nil { + log.Fatalf("Listing error: %v", obj.Err) + } + fmt.Printf("Key: %s\nSize: %d\nLast Modified: %s\n===\n", obj.Key, obj.Size, obj.LastModified) + } +} diff --git a/docs/sts/ldap.md b/docs/sts/ldap.md new file mode 100644 index 00000000..4e59d051 --- /dev/null +++ b/docs/sts/ldap.md @@ -0,0 +1,323 @@ +# AssumeRoleWithLDAPIdentity [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +## Introduction + +MinIO provides a custom STS API that allows integration with LDAP based corporate environments including Microsoft Active Directory. The MinIO server uses a separate LDAP service account to lookup user information. The login flow for a user is as follows: + +- User provides their AD/LDAP username and password to the STS API. +- MinIO looks up the user's information (specifically the user's Distinguished Name) in the LDAP server. +- On finding the user's info, MinIO verifies the login credentials with the AD/LDAP server. +- MinIO optionally queries the AD/LDAP server for a list of groups that the user is a member of. +- MinIO then checks if there are any policies [explicitly associated](#managing-usergroup-access-policy) with the user or their groups. +- On finding at least one associated policy, MinIO generates temporary credentials for the user storing the list of groups in a cryptographically secure session token. The temporary access key, secret key and session token are returned to the user. +- The user can now use these credentials to make requests to the MinIO server. + +The administrator will associate IAM access policies with each group and if required with the user too. The MinIO server then evaluates applicable policies on a user (these are the policies associated with the groups along with the policy on the user if any) to check if the request should be allowed or denied. + +To ensure that changes in the LDAP directory are reflected in object storage access changes, MinIO performs an **Automatic LDAP sync**. MinIO periodically queries the LDAP service to: + +- find accounts (user DNs) that have been removed; any active STS credentials or MinIO service accounts belonging to these users are purged. + +- find accounts whose group memberships have changed; access policies available to a credential are updated to reflect the change, i.e. they will lose any privileges associated with a group they are removed from, and gain any privileges associated with a group they are added to. + +**Please note that when AD/LDAP is configured, MinIO will not support long term users defined internally.** Only AD/LDAP users (and the root user) are allowed. In addition to this, the server will not support operations on users or groups using `mc admin user` or `mc admin group` commands except `mc admin user info` and `mc admin group info` to list set policies for users and groups. This is because users and groups are defined externally in AD/LDAP. + +## Configuring AD/LDAP on MinIO + +LDAP STS configuration can be performed via MinIO's standard configuration API (i.e. using `mc admin config set/get` commands) or equivalently via environment variables. For brevity we refer to environment variables here. + +LDAP is configured via the following environment variables: + +``` +$ mc admin config set myminio identity_ldap --env +KEY: +identity_ldap enable LDAP SSO support + +ARGS: +MINIO_IDENTITY_LDAP_SERVER_ADDR* (address) AD/LDAP server address e.g. "myldap.com" or "myldapserver.com:636" +MINIO_IDENTITY_LDAP_SRV_RECORD_NAME (string) DNS SRV record name for LDAP service, if given, must be one of "ldap", "ldaps" or "on" +MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN (string) DN for LDAP read-only service account used to perform DN and group lookups +MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD (string) Password for LDAP read-only service account used to perform DN and group lookups +MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN (list) ";" separated list of user search base DNs e.g. "dc=myldapserver,dc=com" +MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER (string) Search filter to lookup user DN +MINIO_IDENTITY_LDAP_USER_DN_ATTRIBUTES (list) "," separated list of user DN attributes e.g. "uid,cn,mail,sshPublicKey" +MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER (string) search filter for groups e.g. "(&(objectclass=groupOfNames)(memberUid=%s))" +MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN (list) ";" separated list of group search base DNs e.g. "dc=myldapserver,dc=com" +MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY (on|off) trust server TLS without verification (default: 'off') +MINIO_IDENTITY_LDAP_SERVER_INSECURE (on|off) allow plain text connection to AD/LDAP server (default: 'off') +MINIO_IDENTITY_LDAP_SERVER_STARTTLS (on|off) use StartTLS connection to AD/LDAP server (default: 'off') +MINIO_IDENTITY_LDAP_COMMENT (sentence) optionally add a comment to this setting +``` + +### LDAP server connectivity + +The variables relevant to configuring connectivity to the LDAP service are: + +``` +MINIO_IDENTITY_LDAP_SERVER_ADDR* (address) AD/LDAP server address e.g. "myldap.com" or "myldapserver.com:1686" +MINIO_IDENTITY_LDAP_SRV_RECORD_NAME (string) DNS SRV record name for LDAP service, if given, must be one of ldap, ldaps or on +MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY (on|off) trust server TLS without verification, defaults to "off" (verify) +MINIO_IDENTITY_LDAP_SERVER_INSECURE (on|off) allow plain text connection to AD/LDAP server, defaults to "off" +MINIO_IDENTITY_LDAP_SERVER_STARTTLS (on|off) use StartTLS connection to AD/LDAP server, defaults to "off" +``` + +The server address variable is _required_. TLS is assumed to be on by default. The port in the server address is optional and defaults to 636 if not provided. + +**MinIO sends LDAP credentials to the LDAP server for validation. So we _strongly recommend_ to use MinIO with AD/LDAP server over TLS or StartTLS _only_. Using plain-text connection between MinIO and LDAP server means _credentials can be compromised_ by anyone listening to network traffic.** + +If a self-signed certificate is being used, the certificate can be added to MinIO's certificates directory, so it can be trusted by the server. + +#### DNS SRV Records + +Many Active Directory and other LDAP services are setup with [DNS SRV Records](https://ldap.com/dns-srv-records-for-ldap/) for high-availability of the directory service. To use this to find LDAP servers to connect to, an LDAP client makes a DNS SRV record request to the DNS service on a domain that looks like `_service._proto.example.com`. For LDAP the `proto` value is always `tcp`, and `service` is usually `ldap` or `ldaps`. + +To enable MinIO to use the SRV records, specify the `srv_record_name` config parameter (or equivalently the `MINIO_IDENTITY_LDAP_SRV_RECORD_NAME` environment variable). This parameter can be set to `ldap` or `ldaps` and MinIO will substitute it into the `service` value. For example, when `server_addr=myldapserver.com` and `srv_record_name=ldap`, MinIO will lookup the SRV record for `_ldap._tcp.myldapserver.com` and pick an appropriate target for LDAP requests. + +If the DNS SRV record is at an entirely different place, say `_ldapsrv._tcpish.myldapserver.com`, then set `srv_record_name` to the special value `on` and set `server_addr=_ldapsrv._tcpish.myldapserver.com`. + +When using this feature, do not specify a port in the `server_addr` as the port is picked up automatically from the SRV record. + +With the default (empty) value for `srv_record_name`, MinIO **will not** perform any SRV record request. + +The value of `srv_record_name` does not affect any TLS settings - they must be configured with their own parameters. + +### Lookup-Bind + +A low-privilege read-only LDAP service account is configured in the MinIO server by providing the account's Distinguished Name (DN) and password. This service account is used to perform directory lookups as needed. + +``` +MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN* (string) DN for LDAP read-only service account used to perform DN and group lookups +MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD (string) Password for LDAP read-only service account used to perform DN and group lookups +``` + +If you set an empty lookup bind password, the lookup bind will use the unauthenticated authentication mechanism, as described in [RFC 4513 Section 5.1.2](https://tools.ietf.org/html/rfc4513#section-5.1.2). + +### User lookup + +When a user provides their LDAP credentials, MinIO runs a lookup query to find the user's Distinguished Name (DN). The search filter and base DN used in this lookup query are configured via the following variables: + +``` +MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN* (list) ";" separated list of user search base DNs e.g. "dc=myldapserver,dc=com" +MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER* (string) Search filter to lookup user DN +``` + +The search filter must use the LDAP username to find the user DN. This is done via [variable substitution](#variable-substitution-in-configuration-strings). + +The returned user's DN and their password are then verified with the LDAP server. The user DN may also be associated with an [access policy](#managing-usergroup-access-policy). + +The User DN attributes configuration parameter: +``` +MINIO_IDENTITY_LDAP_USER_DN_ATTRIBUTES (list) "," separated list of user DN attributes e.g. "uid,cn,mail,sshPublicKey" +``` +is optional and can be used to specify additional attributes to lookup on the User DN record in the LDAP server. This is for certain display purposes and may be used for extended functionality that may be added in the future. + +### Group membership search + +MinIO can be optionally configured to find the groups of a user from AD/LDAP by specifying the following variables: + +``` +MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER (string) search filter for groups e.g. "(&(objectclass=groupOfNames)(memberUid=%s))" +MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN (list) ";" separated list of group search base DNs e.g. "dc=myldapserver,dc=com" +``` + +The search filter must use the username or the DN to find the user's groups. This is done via [variable substitution](#variable-substitution-in-configuration-strings). + +A group's DN may be associated with an [access policy](#managing-usergroup-access-policy). + +#### Nested groups usage in LDAP/AD +If you are using Active directory with nested groups you have to add LDAP_MATCHING_RULE_IN_CHAIN: :1.2.840.113556.1.4.1941: to your query. +For example: +```shell +group_search_filter: (&(objectClass=group)(member:1.2.840.113556.1.4.1941:=%d)) +user_dn_search_filter: (&(memberOf:1.2.840.113556.1.4.1941:=CN=group,DC=dc,DC=net)(sAMAccountName=%s)) +``` + +### Sample settings + +Here are some (minimal) sample settings for development or experimentation: + +```shell +export MINIO_IDENTITY_LDAP_SERVER_ADDR=myldapserver.com:636 +export MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN='cn=admin,dc=min,dc=io' +export MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD=admin +export MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN='ou=hwengg,dc=min,dc=io' +export MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER='(uid=%s)' +export MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY=on +``` + +### Variable substitution in configuration strings + +In the configuration variables, `%s` is substituted with the _username_ from the STS request and `%d` is substituted with the _distinguished username (user DN)_ of the LDAP user. Please see the following table for which configuration variables support these substitution variables: + +| Variable | Supported substitutions | +|---------------------------------------------|-------------------------| +| `MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER` | `%s` | +| `MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER` | `%s` and `%d` | + +## Managing User/Group Access Policy + +Access policies may be associated by their name with a group or user directly. Access policies are first defined on the MinIO server using IAM policy JSON syntax. To define a new policy, you can use the [AWS policy generator](https://awspolicygen.s3.amazonaws.com/policygen.html). Copy the policy into a text file `mypolicy.json` and issue the command like so: + +```sh +mc admin policy create myminio mypolicy mypolicy.json +``` + +To associate the policy with an LDAP user or group, use the full DN of the user or group: + +```sh +mc idp ldap policy attach myminio mypolicy --user='uid=james,cn=accounts,dc=myldapserver,dc=com' +``` + +```sh +mc idp ldap policy attach myminio mypolicy ----group='cn=projectx,ou=groups,ou=hwengg,dc=min,dc=io' +``` + +To remove a policy association, use the similar `detach` command: + +```sh +mc idp ldap policy detach myminio mypolicy --user='uid=james,cn=accounts,dc=myldapserver,dc=com' +``` + +```sh +mc idp ldap policy detach myminio mypolicy ----group='cn=projectx,ou=groups,ou=hwengg,dc=min,dc=io' +``` + + +Note that the commands above attempt to validate if the given entity (user or group) exist in the LDAP directory and return an error if they are not found. + +
View **DEPRECATED** older policy association commands + +Please **do not use** these as they may be removed or their behavior may change. + +```sh +mc admin policy attach myminio mypolicy --user='uid=james,cn=accounts,dc=myldapserver,dc=com' +``` + + +```sh +mc admin policy attach myminio mypolicy --group='cn=projectx,ou=groups,ou=hwengg,dc=min,dc=io' +``` + +
+ +**Note that by default no policy is set on a user**. Thus even if they successfully authenticate with AD/LDAP credentials, they have no access to object storage as the default access policy is to deny all access. + +## API Request Parameters + +### LDAPUsername + +Is AD/LDAP username to login. Application must ask user for this value to successfully obtain rotating access credentials from AssumeRoleWithLDAPIdentity. + +| Params | Value | +| :-- | :-- | +| _Type_ | _String_ | +| _Length Constraints_ | _Minimum length of 2. Maximum length of 2048._ | +| _Required_ | _Yes_ | + +### LDAPPassword + +Is AD/LDAP username password to login. Application must ask user for this value to successfully obtain rotating access credentials from AssumeRoleWithLDAPIdentity. + +| Params | Value | +| :-- | :-- | +| _Type_ | _String_ | +| _Length Constraints_ | _Minimum length of 4. Maximum length of 2048._ | +| _Required_ | _Yes_ | + +### Version + +Indicates STS API version information, the only supported value is '2011-06-15'. This value is borrowed from AWS STS API documentation for compatibility reasons. + +| Params | Value | +| :-- | :-- | +| _Type_ | _String_ | +| _Required_ | _Yes_ | + +### DurationSeconds + +The duration, in seconds. The value can range from 900 seconds (15 minutes) up to 365 days. If value is higher than this setting, then operation fails. By default, the value is set to 3600 seconds. + +| Params | Value | +| :-- | :-- | +| _Type_ | _Integer_ | +| _Valid Range_ | _Minimum value of 900. Maximum value of 31536000._ | +| _Required_ | _No_ | + +### Policy + +An IAM policy in JSON format that you want to use as an inline session policy. This parameter is optional. Passing policies to this operation returns new temporary credentials. The resulting session's permissions are the intersection of the canned policy name and the policy set here. You cannot use this policy to grant more permissions than those allowed by the canned policy name being assumed. + +| Params | Value | +| :-- | :-- | +| _Type_ | _String_ | +| _Valid Range_ | _Minimum length of 1. Maximum length of 2048._ | +| _Required_ | _No_ | + +### Response Elements + +XML response for this API is similar to [AWS STS AssumeRoleWithWebIdentity](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html#API_AssumeRoleWithWebIdentity_ResponseElements) + +### Errors + +XML error response for this API is similar to [AWS STS AssumeRoleWithWebIdentity](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html#API_AssumeRoleWithWebIdentity_Errors) + +## Sample `POST` Request + +``` +http://minio.cluster:9000?Action=AssumeRoleWithLDAPIdentity&LDAPUsername=foouser&LDAPPassword=foouserpassword&Version=2011-06-15&DurationSeconds=7200 +``` + +## Sample Response + +``` + + + + + + + + + Y4RJU1RNFGK48LGO9I2S + sYLRKS1Z7hSjluf6gEbb9066hnx315wHTiACPAjg + 2019-08-08T20:26:12Z + eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJZNFJKVTFSTkZHSzQ4TEdPOUkyUyIsImF1ZCI6IlBvRWdYUDZ1Vk80NUlzRU5SbmdEWGo1QXU1WWEiLCJhenAiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiZXhwIjoxNTQxODExMDcxLCJpYXQiOjE1NDE4MDc0NzEsImlzcyI6Imh0dHBzOi8vbG9jYWxob3N0Ojk0NDMvb2F1dGgyL3Rva2VuIiwianRpIjoiYTBiMjc2MjktZWUxYS00M2JmLTg3MzktZjMzNzRhNGNkYmMwIn0.ewHqKVFTaP-j_kgZrcOEKroNUjk10GEp8bqQjxBbYVovV0nHO985VnRESFbcT6XMDDKHZiWqN2vi_ETX_u3Q-w + + + + +``` + +## Using LDAP STS API + +With multiple OU hierarchies for users, and multiple group search base DN's. + +``` +export MINIO_ROOT_USER=minio +export MINIO_ROOT_PASSWORD=minio123 +export MINIO_IDENTITY_LDAP_SERVER_ADDR='my.ldap-active-dir-server.com:636' +export MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN='cn=admin,dc=min,dc=io' +export MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD=admin +export MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN='dc=minioad,dc=local;dc=somedomain,dc=com' +export MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER='(&(objectclass=groupOfNames)(member=%d))' +minio server ~/test +``` + +You can make sure it works appropriately using our [example program](https://raw.githubusercontent.com/minio/minio/master/docs/sts/ldap.go): + +``` +$ go run ldap.go -u foouser -p foopassword + +##### Credentials +{ + "accessKey": "NUIBORZYTV2HG2BMRSXR", + "secretKey": "qQlP5O7CFPc5m5IXf1vYhuVTFj7BRVJqh0FqZ86S", + "expiration": "2018-08-21T17:10:29-07:00", + "sessionToken": "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJOVUlCT1JaWVRWMkhHMkJNUlNYUiIsImF1ZCI6IlBvRWdYUDZ1Vk80NUlzRU5SbmdEWGo1QXU1WWEiLCJhenAiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiZXhwIjoxNTM0ODk2NjI5LCJpYXQiOjE1MzQ4OTMwMjksImlzcyI6Imh0dHBzOi8vbG9jYWxob3N0Ojk0NDMvb2F1dGgyL3Rva2VuIiwianRpIjoiNjY2OTZjZTctN2U1Ny00ZjU5LWI0MWQtM2E1YTMzZGZiNjA4In0.eJONnVaSVHypiXKEARSMnSKgr-2mlC2Sr4fEGJitLcJF_at3LeNdTHv0_oHsv6ZZA3zueVGgFlVXMlREgr9LXA" +} +``` + +## Explore Further + +- [MinIO Admin Complete Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc-admin.html) +- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html) diff --git a/docs/sts/rw-ldap-username.json b/docs/sts/rw-ldap-username.json new file mode 100644 index 00000000..a203afad --- /dev/null +++ b/docs/sts/rw-ldap-username.json @@ -0,0 +1,14 @@ +{ + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Action": [ + "s3:*" + ], + "Resource": [ + "arn:aws:s3:::${ldap:username}/*" + ] + } + ] +} diff --git a/docs/sts/tls.md b/docs/sts/tls.md new file mode 100644 index 00000000..bcc39522 --- /dev/null +++ b/docs/sts/tls.md @@ -0,0 +1,117 @@ +# AssumeRoleWithCertificate [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +## Introduction + +MinIO provides a custom STS API that allows authentication with client X.509 / TLS certificates. + +A major advantage of certificate-based authentication compared to other STS authentication methods, like OpenID Connect or LDAP/AD, is that client authentication works without any additional/external component that must be constantly available. Therefore, certificate-based authentication may provide better availability / lower operational complexity. + +The MinIO TLS STS API can be configured via MinIO's standard configuration API (i.e. using `mc admin config set/get`). Further, it can be configured via the following environment variables: + +``` +mc admin config set myminio identity_tls --env +KEY: +identity_tls enable X.509 TLS certificate SSO support + +ARGS: +MINIO_IDENTITY_TLS_SKIP_VERIFY (on|off) trust client certificates without verification. Defaults to "off" (verify) +``` + +The MinIO TLS STS API is disabled by default. However, it can be *enabled* by setting environment variable: + +``` +export MINIO_IDENTITY_TLS_ENABLE=on +``` + +## Example + +MinIO exposes a custom S3 STS API endpoint as `Action=AssumeRoleWithCertificate`. A client has to send an HTTP `POST` request to `https://:?Action=AssumeRoleWithCertificate&Version=2011-06-15`. Since the authentication and authorization happens via X.509 certificates the client has to send the request over **TLS** and has to provide +a client certificate. + +The following curl example shows how to authenticate to a MinIO server with client certificate and obtain STS access credentials. + +```curl +curl -X POST --key private.key --cert public.crt "https://minio:9000?Action=AssumeRoleWithCertificate&Version=2011-06-15&DurationSeconds=3600" +``` + +```xml + + + + + YC12ZBHUVW588BQAE5BM + Zgl9+zdE0pZ88+hLqtfh0ocLN+WQTJixHouCkZkW + 2021-07-19T20:10:45ZeyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJZQzEyWkJIVVZXNTg4QlFBRTVCTSIsImV4cCI6MTYyNjcyNTQ0NX0.wvMUf3w_x16qpVWgua8WxnV1Sgtv1jOnSu03vbrwOMzV3cI4q3_9WZD9LwlP-34DTsvbsg7gCBGh6YNriMMiQw + + + + 169339CD8B3A6948 + + +``` + +## Authentication Flow + +A client can request temp. S3 credentials via the STS API. It can authenticate via a client certificate and obtain a access/secret key pair as well as a session token. These credentials are associated to an S3 policy at the MinIO server. + +In case of certificate-based authentication, MinIO has to map the client-provided certificate to an S3 policy. MinIO does this via the subject common name field of the X.509 certificate. So, MinIO will associate a certificate with a subject `CN = foobar` to a S3 policy named `foobar`. + +The following self-signed certificate is issued for `consoleAdmin`. So, MinIO would associate it with the pre-defined `consoleAdmin` policy. + +``` +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + 35:ac:60:46:ad:8d:de:18:dc:0b:f6:98:14:ee:89:e8 + Signature Algorithm: ED25519 + Issuer: CN = consoleAdmin + Validity + Not Before: Jul 19 15:08:44 2021 GMT + Not After : Aug 18 15:08:44 2021 GMT + Subject: CN = consoleAdmin + Subject Public Key Info: + Public Key Algorithm: ED25519 + ED25519 Public-Key: + pub: + 5a:91:87:b8:77:fe:d4:af:d9:c7:c7:ce:55:ae:74: + aa:f3:f1:fe:04:63:9b:cb:20:97:61:97:90:94:fa: + 12:8b + X509v3 extensions: + X509v3 Key Usage: critical + Digital Signature + X509v3 Extended Key Usage: + TLS Web Client Authentication + X509v3 Basic Constraints: critical + CA:FALSE + Signature Algorithm: ED25519 + 7e:aa:be:ed:47:4d:b9:2f:fc:ed:7f:5a:fc:6b:c0:05:5b:f5: + a0:31:fe:86:e3:8e:3f:49:af:6d:d5:ac:c7:c4:57:47:ce:97: + 7d:ab:b8:e9:75:ec:b4:39:fb:c8:cf:53:16:5b:1f:15:b6:7f: + 5a:d1:35:2d:fc:31:3a:10:e7:0c +``` + +> Observe the `Subject: CN = consoleAdmin` field. + +Also, note that the certificate has to contain the `Extended Key Usage: TLS Web Client Authentication`. Otherwise, MinIO would not accept the certificate as client certificate. + +Now, the STS certificate-based authentication happens in 4 steps: + +- Client sends HTTP `POST` request over a TLS connection hitting the MinIO TLS STS API. +- MinIO verifies that the client certificate is valid. +- MinIO tries to find a policy that matches the `CN` of the client certificate. +- MinIO returns temp. S3 credentials associated to the found policy. + +The returned credentials expiry after a certain period of time that can be configured via `&DurationSeconds=3600`. By default, the STS credentials are valid for 1 hour. The minimum expiration allowed is 15 minutes. + +Further, the temp. S3 credentials will never out-live the client certificate. For example, if the `MINIO_IDENTITY_TLS_STS_EXPIRY` is 7 days but the certificate itself is only valid for the next 3 days, then MinIO will return S3 credentials that are valid for 3 days only. + +## Caveat + +*Applications that use direct S3 API will work fine, however interactive users uploading content using (when POSTing to the presigned URL an app generates) a popup becomes visible on browser to provide client certs, you would have to manually cancel and continue. This may be annoying to use but there is no workaround for now.* + +## Explore Further + +- [MinIO Admin Complete Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc-admin.html) +- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html) diff --git a/docs/sts/web-identity.go b/docs/sts/web-identity.go new file mode 100644 index 00000000..b39fb73b --- /dev/null +++ b/docs/sts/web-identity.go @@ -0,0 +1,274 @@ +//go:build ignore +// +build ignore + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package main + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/base64" + "encoding/json" + "errors" + "flag" + "fmt" + "log" + "net/http" + "net/url" + "strings" + "time" + + "golang.org/x/oauth2" + + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" +) + +// Returns a base64 encoded random 32 byte string. +func randomState() string { + b := make([]byte, 32) + rand.Read(b) + return base64.RawURLEncoding.EncodeToString(b) +} + +var ( + stsEndpoint string + configEndpoint string + clientID string + clientSec string + clientScopes string + port int +) + +// DiscoveryDoc - parses the output from openid-configuration +// for example http://localhost:8080/auth/realms/minio/.well-known/openid-configuration +type DiscoveryDoc struct { + Issuer string `json:"issuer,omitempty"` + AuthEndpoint string `json:"authorization_endpoint,omitempty"` + TokenEndpoint string `json:"token_endpoint,omitempty"` + UserInfoEndpoint string `json:"userinfo_endpoint,omitempty"` + RevocationEndpoint string `json:"revocation_endpoint,omitempty"` + JwksURI string `json:"jwks_uri,omitempty"` + ResponseTypesSupported []string `json:"response_types_supported,omitempty"` + SubjectTypesSupported []string `json:"subject_types_supported,omitempty"` + IDTokenSigningAlgValuesSupported []string `json:"id_token_signing_alg_values_supported,omitempty"` + ScopesSupported []string `json:"scopes_supported,omitempty"` + TokenEndpointAuthMethods []string `json:"token_endpoint_auth_methods_supported,omitempty"` + ClaimsSupported []string `json:"claims_supported,omitempty"` + CodeChallengeMethodsSupported []string `json:"code_challenge_methods_supported,omitempty"` +} + +func parseDiscoveryDoc(ustr string) (DiscoveryDoc, error) { + d := DiscoveryDoc{} + req, err := http.NewRequest(http.MethodGet, ustr, nil) + if err != nil { + return d, err + } + clnt := http.Client{ + Transport: http.DefaultTransport, + } + resp, err := clnt.Do(req) + if err != nil { + return d, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return d, fmt.Errorf("unexpected error returned by %s : status(%s)", ustr, resp.Status) + } + dec := json.NewDecoder(resp.Body) + if err = dec.Decode(&d); err != nil { + return d, err + } + return d, nil +} + +func init() { + flag.StringVar(&stsEndpoint, "sts-ep", "http://localhost:9000", "STS endpoint") + flag.StringVar(&configEndpoint, "config-ep", + "http://localhost:8080/auth/realms/minio/.well-known/openid-configuration", + "OpenID discovery document endpoint") + flag.StringVar(&clientID, "cid", "", "Client ID") + flag.StringVar(&clientSec, "csec", "", "Client Secret") + flag.StringVar(&clientScopes, "cscopes", "openid", "Client Scopes") + flag.IntVar(&port, "port", 8080, "Port") +} + +func implicitFlowURL(c oauth2.Config, state string) string { + var buf bytes.Buffer + buf.WriteString(c.Endpoint.AuthURL) + v := url.Values{ + "response_type": {"id_token"}, + "response_mode": {"form_post"}, + "client_id": {c.ClientID}, + } + if c.RedirectURL != "" { + v.Set("redirect_uri", c.RedirectURL) + } + if len(c.Scopes) > 0 { + v.Set("scope", strings.Join(c.Scopes, " ")) + } + v.Set("state", state) + v.Set("nonce", state) + if strings.Contains(c.Endpoint.AuthURL, "?") { + buf.WriteByte('&') + } else { + buf.WriteByte('?') + } + buf.WriteString(v.Encode()) + return buf.String() +} + +func main() { + flag.Parse() + if clientID == "" { + flag.PrintDefaults() + return + } + + ddoc, err := parseDiscoveryDoc(configEndpoint) + if err != nil { + log.Println(fmt.Errorf("Failed to parse OIDC discovery document %s", err)) + fmt.Println(err) + return + } + + scopes := ddoc.ScopesSupported + if clientScopes != "" { + scopes = strings.Split(clientScopes, ",") + } + + ctx := context.Background() + + config := oauth2.Config{ + ClientID: clientID, + ClientSecret: clientSec, + Endpoint: oauth2.Endpoint{ + AuthURL: ddoc.AuthEndpoint, + TokenURL: ddoc.TokenEndpoint, + }, + RedirectURL: fmt.Sprintf("http://10.0.0.67:%d/oauth2/callback", port), + Scopes: scopes, + } + + state := randomState() + + http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { + log.Printf("%s %s", r.Method, r.RequestURI) + if r.RequestURI != "/" { + http.NotFound(w, r) + return + } + if clientSec != "" { + http.Redirect(w, r, config.AuthCodeURL(state), http.StatusFound) + } else { + http.Redirect(w, r, implicitFlowURL(config, state), http.StatusFound) + } + }) + + http.HandleFunc("/oauth2/callback", func(w http.ResponseWriter, r *http.Request) { + log.Printf("%s %s", r.Method, r.RequestURI) + + if err := r.ParseForm(); err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + if r.Form.Get("state") != state { + http.Error(w, "state did not match", http.StatusBadRequest) + return + } + + var getWebTokenExpiry func() (*credentials.WebIdentityToken, error) + if clientSec == "" { + getWebTokenExpiry = func() (*credentials.WebIdentityToken, error) { + return &credentials.WebIdentityToken{ + Token: r.Form.Get("id_token"), + }, nil + } + } else { + getWebTokenExpiry = func() (*credentials.WebIdentityToken, error) { + oauth2Token, err := config.Exchange(ctx, r.URL.Query().Get("code")) + if err != nil { + return nil, err + } + if !oauth2Token.Valid() { + return nil, errors.New("invalid token") + } + + return &credentials.WebIdentityToken{ + Token: oauth2Token.Extra("id_token").(string), + Expiry: int(oauth2Token.Expiry.Sub(time.Now().UTC()).Seconds()), + }, nil + } + } + + sts, err := credentials.NewSTSWebIdentity(stsEndpoint, getWebTokenExpiry) + if err != nil { + log.Println(fmt.Errorf("Could not get STS credentials: %s", err)) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + opts := &minio.Options{ + Creds: sts, + BucketLookup: minio.BucketLookupAuto, + } + + u, err := url.Parse(stsEndpoint) + if err != nil { + log.Println(fmt.Errorf("Failed to parse STS Endpoint: %s", err)) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + clnt, err := minio.New(u.Host, opts) + if err != nil { + log.Println(fmt.Errorf("Error while initializing Minio client, %s", err)) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + buckets, err := clnt.ListBuckets(r.Context()) + if err != nil { + log.Println(fmt.Errorf("Error while listing buckets, %s", err)) + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + creds, _ := sts.Get() + + bucketNames := []string{} + + for _, bucket := range buckets { + log.Println(fmt.Sprintf("Bucket discovered: %s", bucket.Name)) + bucketNames = append(bucketNames, bucket.Name) + } + response := make(map[string]interface{}) + response["credentials"] = creds + response["buckets"] = bucketNames + c, err := json.MarshalIndent(response, "", "\t") + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + w.Write(c) + }) + + address := fmt.Sprintf(":%v", port) + log.Printf("listening on http://%s/", address) + log.Fatal(http.ListenAndServe(address, nil)) +} diff --git a/docs/sts/web-identity.md b/docs/sts/web-identity.md new file mode 100644 index 00000000..0818e697 --- /dev/null +++ b/docs/sts/web-identity.md @@ -0,0 +1,275 @@ +# AssumeRoleWithWebIdentity [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +## Introduction + +MinIO supports the standard AssumeRoleWithWebIdentity STS API to enable integration with OIDC/OpenID based identity provider environments. This allows the generation of temporary credentials with pre-defined access policies for applications/users to interact with MinIO object storage. + +Calling AssumeRoleWithWebIdentity does not require the use of MinIO root or IAM credentials. Therefore, you can distribute an application (for example, on mobile devices) that requests temporary security credentials without including MinIO long lasting credentials in the application. Instead, the identity of the caller is validated by using a JWT id_token from the web identity provider. The temporary security credentials returned by this API consists of an access key, a secret key, and a security token. Applications can use these temporary security credentials to sign calls to MinIO API operations. + +By default, the temporary security credentials created by AssumeRoleWithWebIdentity last for one hour. However, the optional DurationSeconds parameter can be used to specify the validity duration of the generated credentials. This value varies from 900 seconds (15 minutes) up to the maximum session duration of 365 days. + +## Configuring OpenID Identity Provider on MinIO + +Configuration can be performed via MinIO's standard configuration API (i.e. using `mc admin config set/get` commands) or equivalently via environment variables. For brevity we show only environment variables here: + +``` +$ mc admin config set myminio identity_openid --env +KEY: +identity_openid[:name] enable OpenID SSO support + +ARGS: +MINIO_IDENTITY_OPENID_ENABLE* (on|off) enable identity_openid target, default is 'off' +MINIO_IDENTITY_OPENID_DISPLAY_NAME (string) Friendly display name for this Provider/App +MINIO_IDENTITY_OPENID_CONFIG_URL* (url) openid discovery document e.g. "https://accounts.google.com/.well-known/openid-configuration" +MINIO_IDENTITY_OPENID_CLIENT_ID* (string) unique public identifier for apps e.g. "292085223830.apps.googleusercontent.com" +MINIO_IDENTITY_OPENID_CLIENT_SECRET* (string) secret for the unique public identifier for apps +MINIO_IDENTITY_OPENID_ROLE_POLICY (string) Set the IAM access policies applicable to this client application and IDP e.g. "app-bucket-write,app-bucket-list" +MINIO_IDENTITY_OPENID_CLAIM_NAME (string) JWT canned policy claim name (default: 'policy') +MINIO_IDENTITY_OPENID_SCOPES (csv) Comma separated list of OpenID scopes for server, defaults to advertised scopes from discovery document e.g. "email,admin" +MINIO_IDENTITY_OPENID_VENDOR (string) Specify vendor type for vendor specific behavior to checking validity of temporary credentials and service accounts on MinIO +MINIO_IDENTITY_OPENID_CLAIM_USERINFO (on|off) Enable fetching claims from UserInfo Endpoint for authenticated user +MINIO_IDENTITY_OPENID_KEYCLOAK_REALM (string) Specify Keycloak 'realm' name, only honored if vendor was set to 'keycloak' as value, if no realm is specified 'master' is default +MINIO_IDENTITY_OPENID_KEYCLOAK_ADMIN_URL (string) Specify Keycloak 'admin' REST API endpoint e.g. http://localhost:8080/auth/admin/ +MINIO_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC (on|off) Enable 'Host' header based dynamic redirect URI (default: 'off') +MINIO_IDENTITY_OPENID_COMMENT (sentence) optionally add a comment to this setting +``` + +### Access Control Configuration Variables + +Either `MINIO_IDENTITY_OPENID_ROLE_POLICY` (recommended) or `MINIO_IDENTITY_OPENID_CLAIM_NAME` must be specified but not both. See the section Access Control Policies to understand the differences between the two. + +**NOTE**: When configuring multiple OpenID based authentication providers on a MinIO cluster, any number of Role Policy based providers may be configured, and at most one JWT Claim based provider may be configured. + +
Example 1: Two role policy providers + +Sample environment variables: + +``` +MINIO_IDENTITY_OPENID_DISPLAY_NAME="my first openid" +MINIO_IDENTITY_OPENID_CONFIG_URL=http://myopenid.com/.well-known/openid-configuration +MINIO_IDENTITY_OPENID_CLIENT_ID="minio-client-app" +MINIO_IDENTITY_OPENID_CLIENT_SECRET="minio-client-app-secret" +MINIO_IDENTITY_OPENID_SCOPES="openid,groups" +MINIO_IDENTITY_OPENID_REDIRECT_URI="http://127.0.0.1:10000/oauth_callback" +MINIO_IDENTITY_OPENID_ROLE_POLICY="consoleAdmin" + +MINIO_IDENTITY_OPENID_DISPLAY_NAME_APP2="another oidc" +MINIO_IDENTITY_OPENID_CONFIG_URL_APP2="http://anotheroidc.com/.well-known/openid-configuration" +MINIO_IDENTITY_OPENID_CLIENT_ID_APP2="minio-client-app-2" +MINIO_IDENTITY_OPENID_CLIENT_SECRET_APP2="minio-client-app-secret-2" +MINIO_IDENTITY_OPENID_SCOPES_APP2="openid,groups" +MINIO_IDENTITY_OPENID_REDIRECT_URI_APP2="http://127.0.0.1:10000/oauth_callback" +MINIO_IDENTITY_OPENID_ROLE_POLICY_APP2="readwrite" + +``` +
+ +
Example 2: Single claim based provider + +Sample environment variables: + +``` +MINIO_IDENTITY_OPENID_DISPLAY_NAME="my openid" +MINIO_IDENTITY_OPENID_CONFIG_URL=http://myopenid.com/.well-known/openid-configuration +MINIO_IDENTITY_OPENID_CLIENT_ID="minio-client-app" +MINIO_IDENTITY_OPENID_CLIENT_SECRET="minio-client-app-secret" +MINIO_IDENTITY_OPENID_SCOPES="openid,groups" +MINIO_IDENTITY_OPENID_REDIRECT_URI="http://127.0.0.1:10000/oauth_callback" +MINIO_IDENTITY_OPENID_CLAIM_NAME="groups" +``` +
+ +### Redirection from OpenID Provider + +To login to MinIO, the user first loads the MinIO console on their browser, and selects the OpenID Provider they wish to use (the `MINIO_IDENTITY_OPENID_DISPLAY_NAME` value is shown here). The user is then redirected to the OpenID provider's login page and performs necessary login actions (e.g. entering credentials, responding to MFA authentication challenges, etc). After successful login, the user is redirected back to the MinIO console. This redirect URL is specified as a parameter by MinIO when the user is redirected to the OpenID Provider in the beginning. For some setups, extra configuration may be required for this step to work correctly. + +For a simple setup where the user/client app accesses MinIO directly (i.e. with no intervening proxies/load-balancers), and each MinIO server (if there are more than one) has a unique domain name, this redirection should work automatically with no further configuration. For example, if the MinIO service is being accessed by the browser at the URL `https://minio-node-1.example.org`, the redirect URL will be `https://minio-node-1.example.org/oauth_callback` and all is well. + +For deployments with a load-balancer (LB), it is required that the LB is configured to send requests from the same user/client-app to the same backend MinIO server (at least for the initial login request and subsequent redirection, as the OpenID auth flow's state parameter is currently local to the MinIO server). For this setup, set the `MINIO_BROWSER_REDIRECT_URL` parameter to the publicly/client-accessible endpoint for the MinIO Console. For example `MINIO_BROWSER_REDIRECT_URL=https://console.minio.example.org`. This will ensure that the redirect URL is set to `https://console.minio.example.org/oauth_callback` and the login process should work correctly. + +For deployments employing DNS round-robin on a single domain to all the MinIO servers, it is possible that after redirection the browser may land on a different MinIO server. For example, the domain `console.minio.example.org` may resolve to `console-X.minio.example.org`, where `X` is `1`, `2`, `3` or `4`. For the login to work, if the user first landed on `console-1.minio.example.org`, they must be redirected back to the same place after logging in at the OpenID provider's web-page. To ensure this, set the `MINIO_IDENTITY_OPENID_REDIRECT_URI_DYNAMIC=on` parameter - this lets MinIO set the redirect URL based on the "Host" header of the (initial login) request. + +The **deprecated** parameter `MINIO_IDENTITY_OPENID_REDIRECT_URI` works similar to the `MINIO_BROWSER_REDIRECT_URL` but needs to include the `/oauth_callback` suffix. Please do not use it, as it is sufficient to the set the `MINIO_BROWSER_REDIRECT_URL` parameter (which is required anyway for most load-balancer based setups to work correctly). This deprecated parameter **will be removed** in a future release. + +## Specifying Access Control with IAM Policies + +The STS API authenticates the user by verifying the JWT provided in the request. However access to object storage resources are controlled via named IAM policies defined in the MinIO instance. Once authenticated via the STS API, the MinIO server applies one or more IAM policies to the generated credentials. MinIO's AssumeRoleWithWebIdentity implementation supports specifying IAM policies in two ways: + +1. Role Policy (Recommended): When specified as part of the OpenID provider configuration, all users authenticating via this provider are authorized to (only) use the specified role policy. The policy to associate with such users is specified via the `role_policy` configuration parameter or the `MINIO_IDENTITY_OPENID_ROLE_POLICY` environment variable. The value is a comma-separated list of IAM access policy names already defined in the server. In this situation, the server prints a role ARN at startup that must be specified as a `RoleArn` API request parameter in the STS AssumeRoleWithWebIdentity API call. When using Role Policies, multiple OpenID providers and/or client applications (with unique client IDs) may be configured with independent role policies. Each configuration is assigned a unique RoleARN by the MinIO server and this is used to select the policies to apply to temporary credentials generated in the AssumeRoleWithWebIdentity call. + +2. `id_token` claims: When the role policy is not configured, MinIO looks for a specific claim in the `id_token` (JWT) returned by the OpenID provider in the STS request. The default claim is `policy` and can be overridden by the `claim_name` configuration parameter or the `MINIO_IDENTITY_OPENID_CLAIM_NAME` environment variable. The claim value can be a string (comma-separated list) or an array of IAM access policy names defined in the server. A `RoleArn` API request parameter *must not* be specified in the STS AssumeRoleWithWebIdentity API call. + +## API Request Parameters + +### WebIdentityToken + +The OAuth 2.0 id_token that is provided by the web identity provider. Application must get this token by authenticating the user who is using your application with a web identity provider before the application makes an AssumeRoleWithWebIdentity call. + +| Params | Value | +| :-- | :-- | +| *Type* | *String* | +| *Length Constraints* | *Minimum length of 4. Maximum length of 2048.* | +| *Required* | *Yes* | + +### WebIdentityAccessToken (MinIO Extension) + +There are situations when identity provider does not provide user claims in `id_token` instead it needs to be retrieved from UserInfo endpoint, this extension is only useful in this scenario. This is rare so use it accordingly depending on your Identity provider implementation. `access_token` is available as part of the OIDC authentication flow similar to `id_token`. + +| Params | Value | +| :-- | :-- | +| *Type* | *String* | +| *Required* | *No* | + +### RoleArn + +The role ARN to use. This must be specified if and only if the web identity provider is configured with a role policy. + +| Params | Value | +| :-- | :-- | +| *Type* | *String* | +| *Required* | *No* | + +### Version + +Indicates STS API version information, the only supported value is '2011-06-15'. This value is borrowed from AWS STS API documentation for compatibility reasons. + +| Params | Value | +| :-- | :-- | +| *Type* | *String* | +| *Required* | *Yes* | + +### DurationSeconds + +The duration, in seconds. The value can range from 900 seconds (15 minutes) up to 365 days. If value is higher than this setting, then operation fails. By default, the value is set to 3600 seconds. If no *DurationSeconds* is specified expiry seconds is obtained from *WebIdentityToken*. + +| Params | Value | +| :-- | :-- | +| *Type* | *Integer* | +| *Valid Range* | *Minimum value of 900. Maximum value of 31536000.* | +| *Required* | *No* | + +### Policy + +An IAM policy in JSON format that you want to use as an inline session policy. This parameter is optional. Passing policies to this operation returns new temporary credentials. The resulting session's permissions are the intersection of the canned policy name and the policy set here. You cannot use this policy to grant more permissions than those allowed by the canned policy name being assumed. + +| Params | Value | +| :-- | :-- | +| *Type* | *String* | +| *Valid Range* | *Minimum length of 1. Maximum length of 2048.* | +| *Required* | *No* | + +### Response Elements + +XML response for this API is similar to [AWS STS AssumeRoleWithWebIdentity](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html#API_AssumeRoleWithWebIdentity_ResponseElements) + +### Errors + +XML error response for this API is similar to [AWS STS AssumeRoleWithWebIdentity](https://docs.aws.amazon.com/STS/latest/APIReference/API_AssumeRoleWithWebIdentity.html#API_AssumeRoleWithWebIdentity_Errors) + +## Sample `POST` Request + +``` +http://minio.cluster:9000?Action=AssumeRoleWithWebIdentity&DurationSeconds=3600&WebIdentityToken=eyJ4NXQiOiJOVEF4Wm1NeE5ETXlaRGczTVRVMVpHTTBNekV6T0RKaFpXSTRORE5sWkRVMU9HRmtOakZpTVEiLCJraWQiOiJOVEF4Wm1NeE5ETXlaRGczTVRVMVpHTTBNekV6T0RKaFpXSTRORE5sWkRVMU9HRmtOakZpTVEiLCJhbGciOiJSUzI1NiJ9.eyJhdWQiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiYXpwIjoiUG9FZ1hQNnVWTzQ1SXNFTlJuZ0RYajVBdTVZYSIsImlzcyI6Imh0dHBzOlwvXC9sb2NhbGhvc3Q6OTQ0M1wvb2F1dGgyXC90b2tlbiIsImV4cCI6MTU0MTgwOTU4MiwiaWF0IjoxNTQxODA1OTgyLCJqdGkiOiI2Y2YyMGIwZS1lNGZmLTQzZmQtYTdiYS1kYTc3YTE3YzM2MzYifQ.Jm29jPliRvrK6Os34nSK3rhzIYLFjE__zdVGNng3uGKXGKzP3We_i6NPnhA0szJXMOKglXzUF1UgSz8MctbaxFS8XDusQPVe4LkB_45hwBm6TmBxzui911nt-1RbBLN_jZIlvl2lPrbTUH5hSn9kEkph6seWanTNQpz9tNEoVa6R_OX3kpJqxe8tLQUWw453A1JTwFNhdHa6-f1K8_Q_eEZ_4gOYINQ9t_fhTibdbkXZkJQFLop-Jwoybi9s4nwQU_dATocgcufq5eCeNItQeleT-23lGxIz0X7CiJrJynYLdd-ER0F77SumqEb5iCxhxuf4H7dovwd1kAmyKzLxpw&Version=2011-06-15 +``` + +## Sample Response + +``` + + + + + + + + + Y4RJU1RNFGK48LGO9I2S + sYLRKS1Z7hSjluf6gEbb9066hnx315wHTiACPAjg + 2019-08-08T20:26:12Z + eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJZNFJKVTFSTkZHSzQ4TEdPOUkyUyIsImF1ZCI6IlBvRWdYUDZ1Vk80NUlzRU5SbmdEWGo1QXU1WWEiLCJhenAiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiZXhwIjoxNTQxODExMDcxLCJpYXQiOjE1NDE4MDc0NzEsImlzcyI6Imh0dHBzOi8vbG9jYWxob3N0Ojk0NDMvb2F1dGgyL3Rva2VuIiwianRpIjoiYTBiMjc2MjktZWUxYS00M2JmLTg3MzktZjMzNzRhNGNkYmMwIn0.ewHqKVFTaP-j_kgZrcOEKroNUjk10GEp8bqQjxBbYVovV0nHO985VnRESFbcT6XMDDKHZiWqN2vi_ETX_u3Q-w + + + + +``` + +## Using WebIdentity API + +``` +export MINIO_ROOT_USER=minio +export MINIO_ROOT_PASSWORD=minio123 +export MINIO_IDENTITY_OPENID_CONFIG_URL=https://accounts.google.com/.well-known/openid-configuration +export MINIO_IDENTITY_OPENID_CLIENT_ID="843351d4-1080-11ea-aa20-271ecba3924a" +# Optional: Allow to specify the requested OpenID scopes (OpenID only requires the `openid` scope) +#export MINIO_IDENTITY_OPENID_SCOPES="openid,profile,email" +minio server /mnt/export +``` + +or using `mc` + +``` +mc admin config get myminio identity_openid +identity_openid config_url=https://accounts.google.com/.well-known/openid-configuration client_id=843351d4-1080-11ea-aa20-271ecba3924a +``` + +Testing with an example +> Visit [Google Developer Console](https://console.cloud.google.com) under Project, APIs, Credentials to get your OAuth2 client credentials. Add `http://localhost:8080/oauth2/callback` as a valid OAuth2 Redirect URL. + +``` +$ go run web-identity.go -cid 204367807228-ok7601k6gj1pgge7m09h7d79co8p35xx.apps.googleusercontent.com -csec XsT_PgPdT1nO9DD45rMLJw7G +2018/12/26 17:49:36 listening on http://localhost:8080/ +``` + +> NOTE: for a reasonable test outcome, make sure the assumed user has at least permission/policy to list all buckets. That policy would look like below: + +``` +{ + "version": "2012-10-17", + "statement": [ + { + "effect": "Allow", + "action": [ + "s3:ListAllMyBuckets" + ], + "resource": [ + "arn:aws:s3:::*" + ] + } + ] +} +``` + +## Authorization Flow + +- Visit , login will direct the user to the Google OAuth2 Auth URL to obtain a permission grant. +- The redirection URI (callback handler) receives the OAuth2 callback, verifies the state parameter, and obtains a Token. +- Using the id_token the callback handler further talks to Google OAuth2 Token URL to obtain an JWT id_token. +- Once obtained the JWT id_token is further sent to STS endpoint i.e MinIO to retrieve temporary credentials. +- Temporary credentials are displayed on the browser upon successful retrieval. + +## Using MinIO Console + +To support WebIdentity based login for MinIO Console, set openid configuration and restart MinIO + +``` +mc admin config set myminio identity_openid config_url="" client_id="" +``` + +``` +mc admin service restart myminio +``` + +Sample URLs for Keycloak are + +`config_url` - `http://localhost:8080/auth/realms/demo/.well-known/openid-configuration` + +JWT token returned by the Identity Provider should include a custom claim for the policy, this is required to create a STS user in MinIO. The name of the custom claim could be either `policy` or `policy`. If there is no namespace then `claim_prefix` can be ignored. For example if the custom claim name is `https://min.io/policy` then, `claim_prefix` should be set as `https://min.io/`. + +- Open MinIO Console and click `Login with SSO` +- The user will be redirected to the Identity Provider login page +- Upon successful login on Identity Provider page the user will be automatically logged into MinIO Console. + +## Explore Further + +- [MinIO Admin Complete Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc-admin.html) +- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html) diff --git a/docs/sts/web-identity.py b/docs/sts/web-identity.py new file mode 100644 index 00000000..45c5f2a0 --- /dev/null +++ b/docs/sts/web-identity.py @@ -0,0 +1,101 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- + +import json +import logging +import urllib +from uuid import uuid4 + +import boto3 +import requests +from botocore.client import Config +from flask import Flask, request + +boto3.set_stream_logger('boto3.resources', logging.DEBUG) + +authorize_url = "http://localhost:8080/auth/realms/minio/protocol/openid-connect/auth" +token_url = "http://localhost:8080/auth/realms/minio/protocol/openid-connect/token" + +# callback url specified when the application was defined +callback_uri = "http://localhost:8000/oauth2/callback" + +# keycloak id and secret +client_id = 'account' +client_secret = 'daaa3008-80f0-40f7-80d7-e15167531ff0' + +sts_client = boto3.client( + 'sts', + region_name='us-east-1', + use_ssl=False, + endpoint_url='http://localhost:9000', +) + +app = Flask(__name__) + + +@app.route('/') +def homepage(): + text = 'Authenticate with keycloak' + return text % make_authorization_url() + + +def make_authorization_url(): + # Generate a random string for the state parameter + # Save it for use later to prevent xsrf attacks + + state = str(uuid4()) + params = {"client_id": client_id, + "response_type": "code", + "state": state, + "redirect_uri": callback_uri, + "scope": "openid"} + + url = authorize_url + "?" + urllib.parse.urlencode(params) + return url + + +@app.route('/oauth2/callback') +def callback(): + error = request.args.get('error', '') + if error: + return "Error: " + error + + authorization_code = request.args.get('code') + + data = {'grant_type': 'authorization_code', + 'code': authorization_code, 'redirect_uri': callback_uri} + id_token_response = requests.post( + token_url, data=data, verify=False, + allow_redirects=False, auth=(client_id, client_secret)) + + print('body: ' + id_token_response.text) + + # we can now use the id_token as much as we want to access protected resources. + tokens = json.loads(id_token_response.text) + id_token = tokens['id_token'] + + response = sts_client.assume_role_with_web_identity( + RoleArn='arn:aws:iam::123456789012:user/svc-internal-api', + RoleSessionName='test', + WebIdentityToken=id_token, + DurationSeconds=3600 + ) + + s3_resource = boto3.resource('s3', + endpoint_url='http://localhost:9000', + aws_access_key_id=response['Credentials']['AccessKeyId'], + aws_secret_access_key=response['Credentials']['SecretAccessKey'], + aws_session_token=response['Credentials']['SessionToken'], + config=Config(signature_version='s3v4'), + region_name='us-east-1') + + bucket = s3_resource.Bucket('testbucket') + + for obj in bucket.objects.all(): + print(obj) + + return "success" + + +if __name__ == '__main__': + app.run(debug=True, port=8000) diff --git a/docs/sts/wso2.md b/docs/sts/wso2.md new file mode 100644 index 00000000..c846159d --- /dev/null +++ b/docs/sts/wso2.md @@ -0,0 +1,109 @@ +# WSO2 Quickstart Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +WSO2 is an Identity Server open source and is released under Apache Software License Version 2.0, this document covers configuring WSO2 to be used as an identity provider for MinIO server STS API. + +## Get started + +### 1. Prerequisites + +- JAVA 1.8 and above installed already and JAVA_HOME points to JAVA 1.8 installation. +- Download WSO2 follow their [installation guide](https://docs.wso2.com/display/IS540/Installation+Guide). + +### 2. Configure WSO2 + +Once WSO2 is up and running, configure WSO2 to generate Self contained id_tokens. In OAuth 2.0 specification there are primarily two ways to provide id_tokens + +1. The id_token is an identifier that is hard to guess. For example, a randomly generated string of sufficient length, that the server handling the protected resource can use to lookup the associated authorization information. +2. The id_token self-contains the authorization information in a manner that can be verified. For example, by encoding authorization information along with a signature into the token. + +WSO2 generates tokens in first style by default, but if to be used with MinIO we should configure WSO2 to provide JWT tokens instead. + +### 3. Generate Self-contained Access Tokens + +By default, a UUID is issued as an id_token in WSO2 Identity Server, which is of the first type above. But, it also can be configured to issue a self-contained id_token (JWT), which is of the second type above. + +- Open the `/repository/conf/identity/identity.xml` file and uncomment the following entry under `` element. + +``` +org.wso2.carbon.identity.oauth2.token.JWTTokenIssuer +``` + +- Restart the server. +- Configure an [OAuth service provider](https://docs.wso2.com/display/IS540/Adding+and+Configuring+a+Service+Provider). +- Initiate an id_token request to the WSO2 Identity Server, over a known [grant type](https://docs.wso2.com/display/IS540/OAuth+2.0+Grant+Types). For example, the following cURL command illustrates the syntax of an id_token request that can be initiated over the [Client Credentials Grant](https://docs.wso2.com/display/IS540/Client+Credentials+Grant) grant type. + - Navigate to service provider section, expand Inbound Authentication Configurations and expand OAuth/OpenID Connect Configuration. + - Copy the OAuth Client Key as the value for ``. + - Copy the OAuth Client Secret as the value for ``. + - By default, `` is localhost. However, if using a public IP, the respective IP address or domain needs to be specified. + - By default, `` has been set to 9443. However, if the port offset has been incremented by n, the default port value needs to be incremented by n. + +Request + +``` +curl -u : -k -d "grant_type=client_credentials" -H "Content-Type:application/x-www-form-urlencoded" https://:/oauth2/token +``` + +Example: + +``` +curl -u PoEgXP6uVO45IsENRngDXj5Au5Ya:eKsw6z8CtOJVBtrOWvhRWL4TUCga -k -d "grant_type=client_credentials" -H "Content-Type:application/x-www-form-urlencoded" https://localhost:9443/oauth2/token +``` + +In response, the self-contained JWT id_token will be returned as shown below. + +``` +{ + "id_token": "eyJ4NXQiOiJOVEF4Wm1NeE5ETXlaRGczTVRVMVpHTTBNekV6T0RKaFpXSTRORE5sWkRVMU9HRmtOakZpTVEiLCJraWQiOiJOVEF4Wm1NeE5ETXlaRGczTVRVMVpHTTBNekV6T0RKaFpXSTRORE5sWkRVMU9HRmtOakZpTVEiLCJhbGciOiJSUzI1NiJ9.eyJhdWQiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiYXpwIjoiUG9FZ1hQNnVWTzQ1SXNFTlJuZ0RYajVBdTVZYSIsImlzcyI6Imh0dHBzOlwvXC9sb2NhbGhvc3Q6OTQ0M1wvb2F1dGgyXC90b2tlbiIsImV4cCI6MTUzNDg5MTc3OCwiaWF0IjoxNTM0ODg4MTc4LCJqdGkiOiIxODQ0MzI5Yy1kNjVhLTQ4YTMtODIyOC05ZGY3M2ZlODNkNTYifQ.ELZ8ujk2Xp9xTGgMqnCa5ehuimaAPXWlSCW5QeBbTJIT4M5OB_2XEVIV6p89kftjUdKu50oiYe4SbfrxmLm6NGSGd2qxkjzJK3SRKqsrmVWEn19juj8fz1neKtUdXVHuSZu6ws_bMDy4f_9hN2Jv9dFnkoyeNT54r4jSTJ4A2FzN2rkiURheVVsc8qlm8O7g64Az-5h4UGryyXU4zsnjDCBKYk9jdbEpcUskrFMYhuUlj1RWSASiGhHHHDU5dTRqHkVLIItfG48k_fb-ehU60T7EFWH1JBdNjOxM9oN_yb0hGwOjLUyCUJO_Y7xcd5F4dZzrBg8LffFmvJ09wzHNtQ", + "token_type": "Bearer", + "expires_in": 3600 +} +``` + +### 4. JWT Claims + +The id_token received is a signed JSON Web Token (JWT). Use a JWT decoder to decode the id_token to access the payload of the token that includes following JWT claims: + +| Claim Name | Type | Claim Value | +|:----------:|:--------------:|:---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------:| +| iss | _string_ | The issuer of the JWT. The '> Identity Provider Entity Id ' value of the OAuth2/OpenID Connect Inbound Authentication configuration of the Resident Identity Provider is returned here. | +| aud | _string array_ | The token audience list. The client identifier of the OAuth clients that the JWT is intended for, is sent herewith. | +| azp | _string_ | The authorized party for which the token is issued to. The client identifier of the OAuth client that the token is issued for, is sent herewith. | +| iat | _integer_ | The token issue time. | +| exp | _integer_ | The token expiration time. | +| jti | _string_ | Unique identifier for the JWT token. | +| policy | _string_ | Canned policy name to be applied for STS credentials. (Recommended) | + +Using the above `id_token` we can perform an STS request to MinIO to get temporary credentials for MinIO API operations. MinIO STS API uses [JSON Web Key Set Endpoint](https://docs.wso2.com/display/IS541/JSON+Web+Key+Set+Endpoint) to validate if JWT is valid and is properly signed. + +**We recommend setting `policy` as a custom claim for the JWT service provider follow [here](https://docs.wso2.com/display/IS550/Configuring+Claims+for+a+Service+Provider) and [here](https://docs.wso2.com/display/IS550/Handling+Custom+Claims+with+the+JWT+Bearer+Grant+Type) for relevant docs on how to configure claims for a service provider.** + +### 5. Setup MinIO with OpenID configuration URL + +MinIO server expects environment variable for OpenID configuration url as `MINIO_IDENTITY_OPENID_CONFIG_URL`, this environment variable takes a single entry. + +``` +export MINIO_IDENTITY_OPENID_CONFIG_URL=https://localhost:9443/oauth2/oidcdiscovery/.well-known/openid-configuration +export MINIO_IDENTITY_OPENID_CLIENT_ID="843351d4-1080-11ea-aa20-271ecba3924a" +minio server /mnt/data +``` + +Assuming that MinIO server is configured to support STS API by following the doc [MinIO STS Quickstart Guide](https://docs.min.io/community/minio-object-store/developers/security-token-service.html), execute the following command to temporary credentials from MinIO server. + +``` +go run client-grants.go -cid PoEgXP6uVO45IsENRngDXj5Au5Ya -csec eKsw6z8CtOJVBtrOWvhRWL4TUCga + +##### Credentials +{ + "accessKey": "IRBLVDGN5QGMDCMO1X8V", + "secretKey": "KzS3UZKE7xqNdtRbKyfcWgxBS6P1G4kwZn4DXKuY", + "expiration": "2018-08-21T15:49:38-07:00", + "sessionToken": "eyJhbGciOiJIUzUxMiIsInR5cCI6IkpXVCJ9.eyJhY2Nlc3NLZXkiOiJJUkJMVkRHTjVRR01EQ01PMVg4ViIsImF1ZCI6IlBvRWdYUDZ1Vk80NUlzRU5SbmdEWGo1QXU1WWEiLCJhenAiOiJQb0VnWFA2dVZPNDVJc0VOUm5nRFhqNUF1NVlhIiwiZXhwIjoxNTM0ODkxNzc4LCJpYXQiOjE1MzQ4ODgxNzgsImlzcyI6Imh0dHBzOi8vbG9jYWxob3N0Ojk0NDMvb2F1dGgyL3Rva2VuIiwianRpIjoiMTg0NDMyOWMtZDY1YS00OGEzLTgyMjgtOWRmNzNmZTgzZDU2In0.4rKsZ8VkZnIS_ALzfTJ9UbEKPFlQVvIyuHw6AWTJcDFDVgQA2ooQHmH9wUDnhXBi1M7o8yWJ47DXP-TLPhwCgQ" +} +``` + +These credentials can now be used to perform MinIO API operations, these credentials automatically expire in 1hr. To understand more about credential expiry duration and client grants STS API read further [here](https://github.com/minio/minio/blob/master/docs/sts/client-grants.md). + +## Explore Further + +- [MinIO STS Quickstart Guide](https://docs.min.io/community/minio-object-store/developers/security-token-service.html) +- [The MinIO documentation website](https://docs.min.io/community/minio-object-store/index.html) diff --git a/docs/throttle/README.md b/docs/throttle/README.md new file mode 100644 index 00000000..b49c0f7b --- /dev/null +++ b/docs/throttle/README.md @@ -0,0 +1,33 @@ +# MinIO Server Throttling Guide [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![Docker Pulls](https://img.shields.io/docker/pulls/minio/minio.svg?maxAge=604800)](https://hub.docker.com/r/minio/minio/) + +MinIO server allows to throttle incoming requests: + +- limit the number of active requests allowed across the cluster +- limit the wait duration for each request in the queue + +These values are enabled using server's configuration or environment variables. + +## Examples + +### Configuring connection limit + +If you have traditional spinning (hdd) drives, some applications with high concurrency might require MinIO cluster to be tuned such that to avoid random I/O on the drives. The way to convert high concurrent I/O into a sequential I/O is by reducing the number of concurrent operations allowed per cluster. This allows MinIO cluster to be operationally resilient to such workloads, while also making sure the drives are at optimal efficiency and responsive. + +Example: Limit a MinIO cluster to accept at max 1600 simultaneous S3 API requests across all nodes of the cluster. + +```sh +export MINIO_API_REQUESTS_MAX=1600 +export MINIO_ROOT_USER=your-access-key +export MINIO_ROOT_PASSWORD=your-secret-key +minio server http://server{1...8}/mnt/hdd{1...16} +``` + +or + +```sh +mc admin config set myminio/ api requests_max=1600 +mc admin service restart myminio/ +``` + +> NOTE: A zero value of `requests_max` means MinIO will automatically calculate requests based on available RAM size and that is the default behavior. + diff --git a/docs/tls/README.md b/docs/tls/README.md new file mode 100644 index 00000000..09ec84e5 --- /dev/null +++ b/docs/tls/README.md @@ -0,0 +1,244 @@ +# How to secure access to MinIO server with TLS [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +This guide explains how to configure MinIO Server with TLS certificates on Linux and Windows platforms. + +1. [Install MinIO Server](#install-minio-server) +2. [Use an Existing Key and Certificate with MinIO](#use-an-existing-key-and-certificate-with-minio) +3. [Generate and use Self-signed Keys and Certificates with MinIO](#generate-use-self-signed-keys-certificates) +4. [Install Certificates from Third-party CAs](#install-certificates-from-third-party-cas) + +## 1. Install MinIO Server + +Install MinIO Server using the instructions in the [MinIO Quickstart Guide](https://docs.min.io/community/minio-object-store/operations/deployments/baremetal-deploy-minio-on-redhat-linux.html). + +## 2. Use an Existing Key and Certificate with MinIO + +This section describes how to use a private key and public certificate that have been obtained from a certificate authority (CA). If these files have not been obtained, skip to [3. Generate Self-signed Certificates](#generate-use-self-signed-keys-certificates) or generate them with [Let's Encrypt](https://letsencrypt.org) using these instructions: [Generate Let's Encrypt certificate using Certbot for MinIO](https://docs.min.io/community/minio-object-store/integrations/generate-lets-encrypt-certificate-using-certbot-for-minio.html). For more about TLS and certificates in MinIO, see the [Network Encryption documentation](https://docs.min.io/community/minio-object-store/operations/network-encryption.html). + +Copy the existing private key and public certificate to the `certs` directory. The default certs directory is: + +* **Linux:** `${HOME}/.minio/certs` +* **Windows:** `%%USERPROFILE%%\.minio\certs` + +**Note:** + +* Location of custom certs directory can be specified using `--certs-dir` command line option. +* Inside the `certs` directory, the private key must by named `private.key` and the public key must be named `public.crt`. +* A certificate signed by a CA contains information about the issued identity (e.g. name, expiry, public key) and any intermediate certificates. The root CA is not included. + +## 3. Generate and use Self-signed Keys and Certificates with MinIO + +This section describes how to generate a self-signed certificate using various tools: + +* 3.1 [Use certgen to Generate a Certificate](#using-go) +* 3.2 [Use OpenSSL to Generate a Certificate](#using-open-ssl) +* 3.3 [Use OpenSSL (with IP address) to Generate a Certificate](#using-open-ssl-with-ip) +* 3.4 [Use GnuTLS (for Windows) to Generate a Certificate](#using-gnu-tls) + +**Note:** + +* MinIO only supports keys and certificates in PEM format on Linux and Windows. +* MinIO doesn't currently support PFX certificates. + +### 3.1 Use `certgen` to Generate a Certificate + +Download [`certgen`](https://github.com/minio/certgen/releases/latest) for your specific operating system and platform. + +`certgen` is a simple *Go* tool to generate self-signed certificates, and provides SAN certificates with DNS and IP entries: + +```sh +./certgen -host "10.10.0.3,10.10.0.4,10.10.0.5" +``` + +A response similar to this one should be displayed: + +``` +2018/11/21 10:16:18 wrote public.crt +2018/11/21 10:16:18 wrote private.key +``` + +### 3.2 Use OpenSSL to Generate a Certificate + +Use one of the following methods to generate a certificate using `openssl`: + +* 3.2.1 [Generate a private key with ECDSA](#generate-private-key-with-ecdsa) +* 3.2.2 [Generate a private key with RSA](#generate-private-key-with-rsa) +* 3.2.3 [Generate a self-signed certificate](#generate-a-self-signed-certificate) + +#### 3.2.1 Generate a private key with ECDSA + +Use the following command to generate a private key with ECDSA: + +```sh +openssl ecparam -genkey -name prime256v1 | openssl ec -out private.key +``` + +A response similar to this one should be displayed: + +``` +read EC key +writing EC key +``` + +Alternatively, use the following command to generate a private ECDSA key protected by a password: + +```sh +openssl ecparam -genkey -name prime256v1 | openssl ec -aes256 -out private.key -passout pass:PASSWORD +``` + +#### 3.2.2 Generate a private key with RSA + +Use the following command to generate a private key with RSA: + +```sh +openssl genrsa -out private.key 2048 +``` + +A response similar to this one should be displayed: + +``` +Generating RSA private key, 2048 bit long modulus +............................................+++ +...........+++ +e is 65537 (0x10001) +``` + +Alternatively, use the following command to generate a private RSA key protected by a password: + +```sh +openssl genrsa -aes256 -passout pass:PASSWORD -out private.key 2048 +``` + +**Note:** When using a password-protected private key, the password must be provided through the environment variable `MINIO_CERT_PASSWD` using the following command: + +```sh +export MINIO_CERT_PASSWD= +``` + +The default OpenSSL format for private encrypted keys is PKCS-8, but MinIO only supports PKCS-1. An RSA key that has been formatted with PKCS-8 can be converted to PKCS-1 using the following command: + +```sh +openssl rsa -in private-pkcs8-key.key -aes256 -passout pass:PASSWORD -out private.key +``` + +#### 3.2.3 Generate a self-signed certificate + +Create a file named `openssl.conf` with the content below. Set `IP.1` and/or `DNS.1` to point to the correct IP/DNS addresses: + +```sh +[req] +distinguished_name = req_distinguished_name +x509_extensions = v3_req +prompt = no + +[req_distinguished_name] +C = US +ST = VA +L = Somewhere +O = MyOrg +OU = MyOU +CN = MyServerName + +[v3_req] +subjectAltName = @alt_names + +[alt_names] +IP.1 = 127.0.0.1 +DNS.1 = localhost +``` + +Run `openssl` by specifying the configuration file and enter a passphrase if prompted: + +```sh +openssl req -new -x509 -nodes -days 730 -keyout private.key -out public.crt -config openssl.conf +``` + +### 3.3 Use GnuTLS (for Windows) to Generate a Certificate + +This section describes how to use GnuTLS on Windows to generate a certificate. + +#### 3.3.1 Install and configure GnuTLS + +Download and decompress the Windows version of GnuTLS from [here](http://www.gnutls.org/download.html). + +Use PowerShell to add the path of the extracted GnuTLS binary to the system path: + +``` +setx path "%path%;C:\Users\MyUser\Downloads\gnutls-3.4.9-w64\bin" +``` + +**Note:** PowerShell may need to be restarted for this change to take effect. + +#### 3.3.2 Generate a private key + +Run the following command to generate a private `.key` file: + +``` +certtool.exe --generate-privkey --outfile private.key +``` + +A response similar to this one should be displayed: + +``` +Generating a 3072 bit RSA private key... +``` + +#### 3.3.3 Generate a public certificate + +Create a file called `cert.cnf` with the content below. This file contains all of the information necessary to generate a certificate using `certtool.exe`: + +``` +# X.509 Certificate options +# +# DN options + +# The organization of the subject. +organization = "Example Inc." + +# The organizational unit of the subject. +#unit = "sleeping dept." + +# The state of the certificate owner. +state = "Example" + +# The country of the subject. Two letter code. +country = "EX" + +# The common name of the certificate owner. +cn = "Sally Certowner" + +# In how many days, counting from today, this certificate will expire. +expiration_days = 365 + +# X.509 v3 extensions + +# DNS name(s) of the server +dns_name = "localhost" + +# (Optional) Server IP address +ip_address = "127.0.0.1" + +# Whether this certificate will be used for a TLS server +tls_www_server +``` + +Run `certtool.exe` and specify the configuration file to generate a certificate: + +``` +certtool.exe --generate-self-signed --load-privkey private.key --template cert.cnf --outfile public.crt +``` + +## 4. Install Certificates from Third-party CAs + +MinIO can connect to other servers, including MinIO nodes or other server types such as NATs and Redis. If these servers use certificates that were not registered with a known CA, add trust for these certificates to MinIO Server by placing these certificates under one of the following MinIO configuration paths: + +* **Linux:** `~/.minio/certs/CAs/` +* **Windows**: `C:\Users\\.minio\certs\CAs` + +## Explore Further + +* [TLS Configuration for MinIO server on Kubernetes](https://github.com/minio/minio/tree/master/docs/tls/kubernetes) +* [MinIO Client Complete Guide](https://docs.min.io/community/minio-object-store/reference/minio-mc.html) +* [MinIO Network Encryption Overview](https://docs.min.io/community/minio-object-store/operations/network-encryption.html) +* [Generate Let's Encrypt Certificate](https://docs.min.io/community/minio-object-store/integrations/generate-lets-encrypt-certificate-using-certbot-for-minio.html) +* [Setup nginx Proxy with MinIO Server](https://docs.min.io/community/minio-object-store/integrations/setup-nginx-proxy-with-minio.html) diff --git a/docs/tls/kubernetes/README.md b/docs/tls/kubernetes/README.md new file mode 100644 index 00000000..952909c8 --- /dev/null +++ b/docs/tls/kubernetes/README.md @@ -0,0 +1,71 @@ +# How to secure access to MinIO on Kubernetes with TLS [![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) + +This document explains how to configure MinIO server with TLS certificates on Kubernetes. + +## 1. Prerequisites + +- Familiarity with [MinIO deployment process on Kubernetes](https://docs.min.io/community/minio-object-store/operations/deployments/kubernetes.html). + +- Kubernetes cluster with `kubectl` configured. + +- Acquire TLS certificates, either from a CA or [create self-signed certificates](https://docs.min.io/community/minio-object-store/operations/network-encryption.html). + +For a [distributed MinIO setup](https://docs.min.io/community/minio-object-store/operations/deployments/kubernetes.html), where there are multiple pods with different domain names expected to run, you will either need wildcard certificates valid for all the domains or have specific certificates for each domain. If you are going to use specific certificates, make sure to create Kubernetes secrets accordingly. + +For testing purposes, here is [how to create self-signed certificates](https://github.com/minio/minio/tree/master/docs/tls#3-generate-self-signed-certificates). + +## 2. Create Kubernetes secret + +[Kubernetes secrets](https://kubernetes.io/docs/concepts/configuration/secret) are intended to hold sensitive information. +We'll use secrets to hold the TLS certificate and key. To create a secret, update the paths to `private.key` and `public.crt` +below. + +Then type + +```sh +kubectl create secret generic tls-ssl-minio --from-file=path/to/private.key --from-file=path/to/public.crt +``` + +Cross check if the secret is created successfully using + +```sh +kubectl get secrets +``` + +You should see a secret named `tls-ssl-minio`. + +## 3. Update deployment yaml file + +Whether you are planning to use Kubernetes StatefulSet or Kubernetes Deployment, the steps remain the same. + +If you're using certificates provided by a CA, add the below section in your yaml file under `spec.volumes[]` + +```yaml + volumes: + - name: secret-volume + secret: + secretName: tls-ssl-minio + items: + - key: public.crt + path: public.crt + - key: private.key + path: private.key + - key: public.crt + path: CAs/public.crt +``` + +Note that the `secretName` should be same as the secret name created in previous step. Then add the below section under +`spec.containers[].volumeMounts[]` + +```yaml + volumeMounts: + - name: secret-volume + mountPath: //.minio/certs +``` + +Here the name of `volumeMount` should match the name of `volume` created previously. Also `mountPath` must be set to the path of +the MinIO server's config sub-directory that is used to store certificates. By default, the location is +`//.minio/certs`. + +*Tip*: In a standard Kubernetes configuration, this will be `/root/.minio/certs`. Kubernetes will mount the secrets volume read-only, +so avoid setting `mountPath` to a path that MinIO server expects to write to. diff --git a/docs/tuning/README.md b/docs/tuning/README.md new file mode 100644 index 00000000..7a0721ee --- /dev/null +++ b/docs/tuning/README.md @@ -0,0 +1,26 @@ +# How to enable 'minio' performance profile with tuned? + +## Prerequisites + +Please make sure the following packages are already installed via `dnf` or `apt`: + +- `tuned` +- `curl` + +### Install `tuned.conf` performance profile + +#### Step 1 - download `tuned.conf` from the referenced link +``` +wget https://raw.githubusercontent.com/minio/minio/master/docs/tuning/tuned.conf +``` + +#### Step 2 - install tuned.conf as supported performance profile on all nodes +``` +sudo mkdir -p /usr/lib/tuned/minio/ +sudo mv tuned.conf /usr/lib/tuned/minio +``` + +#### Step 3 - to enable minio performance profile on all the nodes +``` +sudo tuned-adm profile minio +``` diff --git a/docs/tuning/tuned.conf b/docs/tuning/tuned.conf new file mode 100644 index 00000000..18f5dece --- /dev/null +++ b/docs/tuning/tuned.conf @@ -0,0 +1,83 @@ +[main] +summary=Maximum server performance for MinIO + +[vm] +transparent_hugepage=madvise + +[sysfs] +/sys/kernel/mm/transparent_hugepage/defrag=defer+madvise +/sys/kernel/mm/transparent_hugepage/khugepaged/max_ptes_none=0 + +[cpu] +force_latency=1 +governor=performance +energy_perf_bias=performance +min_perf_pct=100 + +[sysctl] +fs.xfs.xfssyncd_centisecs=72000 +net.core.busy_read=50 +net.core.busy_poll=50 +kernel.numa_balancing=1 + +# Do not use swap at all +vm.swappiness=0 +vm.vfs_cache_pressure=50 + +# Start writeback at 3% memory +vm.dirty_background_ratio=3 +# Force writeback at 10% memory +vm.dirty_ratio=10 + +# Quite a few memory map +# areas may be consumed +vm.max_map_count=524288 + +# Default is 500000 = 0.5ms +kernel.sched_migration_cost_ns=5000000 + +# stalled hdd io threads +kernel.hung_task_timeout_secs=85 + +# network tuning for bigger throughput +net.core.netdev_max_backlog=250000 +net.core.somaxconn=16384 +net.ipv4.tcp_syncookies=0 +net.ipv4.tcp_max_syn_backlog=16384 +net.core.wmem_max=4194304 +net.core.rmem_max=4194304 +net.core.rmem_default=4194304 +net.core.wmem_default=4194304 +net.ipv4.tcp_rmem="4096 87380 4194304" +net.ipv4.tcp_wmem="4096 65536 4194304" + +# Reduce CPU utilization +net.ipv4.tcp_timestamps=0 + +# Increase throughput +net.ipv4.tcp_sack=1 + +# Low latency mode for TCP +net.ipv4.tcp_low_latency=1 + +# The following variable is used to tell the kernel how +# much of the socket buffer space should be used for TCP +# window size, and how much to save for an application buffer. +net.ipv4.tcp_adv_win_scale=1 + +# disable RFC2861 behavior +net.ipv4.tcp_slow_start_after_idle = 0 + +# Fix faulty network setups +net.ipv4.tcp_mtu_probing=1 +net.ipv4.tcp_base_mss=1280 + +# Disable ipv6 +net.ipv6.conf.all.disable_ipv6=1 +net.ipv6.conf.default.disable_ipv6=1 +net.ipv6.conf.lo.disable_ipv6=1 + +[bootloader] +# Avoid firing timers for all CPUs at the same time. This is irrelevant for +# full nohz systems +cmdline=skew_tick=1 \ No newline at end of file diff --git a/go.mod b/go.mod new file mode 100644 index 00000000..a9570852 --- /dev/null +++ b/go.mod @@ -0,0 +1,276 @@ +module github.com/minio/minio + +go 1.24.0 + +toolchain go1.24.8 + +// Install tools using 'go install tool'. +tool ( + github.com/tinylib/msgp + golang.org/x/tools/cmd/stringer +) + +require ( + aead.dev/mtls v0.2.1 + cloud.google.com/go/storage v1.52.0 + github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 + github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 + github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 + github.com/IBM/sarama v1.45.1 + github.com/alecthomas/participle v0.7.1 + github.com/beevik/ntp v1.4.3 + github.com/buger/jsonparser v1.1.1 + github.com/cespare/xxhash/v2 v2.3.0 + github.com/cheggaaa/pb v1.0.29 + github.com/coreos/go-oidc/v3 v3.14.1 + github.com/coreos/go-systemd/v22 v22.5.0 + github.com/cosnicolaou/pbzip2 v1.0.5 + github.com/dchest/siphash v1.2.3 + github.com/dustin/go-humanize v1.0.1 + github.com/eclipse/paho.mqtt.golang v1.5.0 + github.com/elastic/go-elasticsearch/v7 v7.17.10 + github.com/fatih/color v1.18.0 + github.com/felixge/fgprof v0.9.5 + github.com/fraugster/parquet-go v0.12.0 + github.com/go-ldap/ldap/v3 v3.4.11 + github.com/go-openapi/loads v0.22.0 + github.com/go-sql-driver/mysql v1.9.2 + github.com/gobwas/ws v1.4.0 + github.com/golang-jwt/jwt/v4 v4.5.2 + github.com/gomodule/redigo v1.9.2 + github.com/google/uuid v1.6.0 + github.com/inconshreveable/mousetrap v1.1.0 + github.com/json-iterator/go v1.1.12 + github.com/klauspost/compress v1.18.0 + github.com/klauspost/cpuid/v2 v2.2.10 + github.com/klauspost/filepathx v1.1.1 + github.com/klauspost/pgzip v1.2.6 + github.com/klauspost/readahead v1.4.0 + github.com/klauspost/reedsolomon v1.12.4 + github.com/lib/pq v1.10.9 + github.com/lithammer/shortuuid/v4 v4.2.0 + github.com/miekg/dns v1.1.65 + github.com/minio/cli v1.24.2 + github.com/minio/console v1.7.7-0.20250905210349-2017f33b26e1 + github.com/minio/csvparser v1.0.0 + github.com/minio/dnscache v0.1.1 + github.com/minio/dperf v0.6.3 + github.com/minio/highwayhash v1.0.3 + github.com/minio/kms-go/kes v0.3.1 + github.com/minio/kms-go/kms v0.5.1-0.20250225090116-4e64ce8d0f35 + github.com/minio/madmin-go/v3 v3.0.109 + github.com/minio/minio-go/v7 v7.0.91 + github.com/minio/mux v1.9.2 + github.com/minio/pkg/v3 v3.1.3 + github.com/minio/selfupdate v0.6.0 + github.com/minio/simdjson-go v0.4.5 + github.com/minio/sio v0.4.1 + github.com/minio/xxml v0.0.3 + github.com/minio/zipindex v0.4.0 + github.com/mitchellh/go-homedir v1.1.0 + github.com/nats-io/nats-server/v2 v2.11.1 + github.com/nats-io/nats.go v1.41.2 + github.com/nats-io/stan.go v0.10.4 + github.com/ncw/directio v1.0.5 + github.com/nsqio/go-nsq v1.1.0 + github.com/philhofer/fwd v1.2.0 + github.com/pierrec/lz4/v4 v4.1.22 + github.com/pkg/errors v0.9.1 + github.com/pkg/sftp v1.13.9 + github.com/pkg/xattr v0.4.10 + github.com/prometheus/client_golang v1.22.0 + github.com/prometheus/client_model v0.6.2 + github.com/prometheus/common v0.63.0 + github.com/prometheus/procfs v0.16.1 + github.com/puzpuzpuz/xsync/v3 v3.5.1 + github.com/rabbitmq/amqp091-go v1.10.0 + github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 + github.com/rs/cors v1.11.1 + github.com/secure-io/sio-go v0.3.1 + github.com/shirou/gopsutil/v3 v3.24.5 + github.com/tinylib/msgp v1.4.0 + github.com/valyala/bytebufferpool v1.0.0 + github.com/xdg/scram v1.0.5 + github.com/zeebo/xxh3 v1.0.2 + go.etcd.io/etcd/api/v3 v3.5.21 + go.etcd.io/etcd/client/v3 v3.5.21 + go.uber.org/atomic v1.11.0 + go.uber.org/zap v1.27.0 + goftp.io/server/v2 v2.0.1 + golang.org/x/crypto v0.37.0 + golang.org/x/oauth2 v0.29.0 + golang.org/x/sync v0.13.0 + golang.org/x/sys v0.32.0 + golang.org/x/term v0.31.0 + golang.org/x/time v0.11.0 + google.golang.org/api v0.230.0 + gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.1 +) + +require ( + aead.dev/mem v0.2.0 // indirect + aead.dev/minisign v0.3.0 // indirect + cel.dev/expr v0.23.1 // indirect + cloud.google.com/go v0.120.1 // indirect + cloud.google.com/go/auth v0.16.0 // indirect + cloud.google.com/go/auth/oauth2adapt v0.2.8 // indirect + cloud.google.com/go/compute/metadata v0.6.0 // indirect + cloud.google.com/go/iam v1.5.2 // indirect + cloud.google.com/go/monitoring v1.24.2 // indirect + filippo.io/edwards25519 v1.1.0 // indirect + github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 // indirect + github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 // indirect + github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 // indirect + github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 // indirect + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect + github.com/apache/thrift v0.21.0 // indirect + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect + github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/charmbracelet/bubbles v0.20.0 // indirect + github.com/charmbracelet/bubbletea v1.3.4 // indirect + github.com/charmbracelet/lipgloss v1.0.0 // indirect + github.com/charmbracelet/x/ansi v0.8.0 // indirect + github.com/charmbracelet/x/term v0.2.1 // indirect + github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f // indirect + github.com/coreos/go-semver v0.3.1 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 // indirect + github.com/docker/go-units v0.5.0 // indirect + github.com/eapache/go-resiliency v1.7.0 // indirect + github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 // indirect + github.com/eapache/queue v1.1.0 // indirect + github.com/envoyproxy/go-control-plane/envoy v1.32.4 // indirect + github.com/envoyproxy/protoc-gen-validate v1.2.1 // indirect + github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f // indirect + github.com/fatih/structs v1.1.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 // indirect + github.com/go-ini/ini v1.67.0 // indirect + github.com/go-jose/go-jose/v4 v4.1.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.3.0 // indirect + github.com/go-openapi/analysis v0.23.0 // indirect + github.com/go-openapi/errors v0.22.1 // indirect + github.com/go-openapi/jsonpointer v0.21.1 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/runtime v0.28.0 // indirect + github.com/go-openapi/spec v0.21.0 // indirect + github.com/go-openapi/strfmt v0.23.0 // indirect + github.com/go-openapi/swag v0.23.1 // indirect + github.com/go-openapi/validate v0.24.0 // indirect + github.com/gobwas/httphead v0.1.0 // indirect + github.com/gobwas/pool v0.2.1 // indirect + github.com/goccy/go-json v0.10.5 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt/v5 v5.2.2 // indirect + github.com/golang/protobuf v1.5.4 // indirect + github.com/golang/snappy v1.0.0 // indirect + github.com/google/go-tpm v0.9.3 // indirect + github.com/google/pprof v0.0.0-20250422154841-e1f9c1950416 // indirect + github.com/google/s2a-go v0.1.9 // indirect + github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.6 // indirect + github.com/googleapis/gax-go/v2 v2.14.1 // indirect + github.com/gorilla/mux v1.8.1 // indirect + github.com/gorilla/websocket v1.5.3 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/hashicorp/go-multierror v1.1.1 // indirect + github.com/hashicorp/go-uuid v1.0.3 // indirect + github.com/hashicorp/golang-lru v1.0.2 // indirect + github.com/jcmturner/aescts/v2 v2.0.0 // indirect + github.com/jcmturner/dnsutils/v2 v2.0.0 // indirect + github.com/jcmturner/gofork v1.7.6 // indirect + github.com/jcmturner/gokrb5/v8 v8.4.4 // indirect + github.com/jcmturner/rpc/v2 v2.0.3 // indirect + github.com/jedib0t/go-pretty/v6 v6.6.7 // indirect + github.com/jessevdk/go-flags v1.6.1 // indirect + github.com/josharian/intern v1.0.0 // indirect + github.com/juju/ratelimit v1.0.2 // indirect + github.com/kr/fs v0.1.0 // indirect + github.com/kylelemons/godebug v1.1.0 // indirect + github.com/lestrrat-go/blackmagic v1.0.2 // indirect + github.com/lestrrat-go/httpcc v1.0.1 // indirect + github.com/lestrrat-go/httprc v1.0.6 // indirect + github.com/lestrrat-go/iter v1.0.2 // indirect + github.com/lestrrat-go/jwx/v2 v2.1.4 // indirect + github.com/lestrrat-go/option v1.0.1 // indirect + github.com/lucasb-eyer/go-colorful v1.2.0 // indirect + github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 // indirect + github.com/mailru/easyjson v0.9.0 // indirect + github.com/mattn/go-colorable v0.1.14 // indirect + github.com/mattn/go-ieproxy v0.0.12 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/mattn/go-localereader v0.0.1 // indirect + github.com/mattn/go-runewidth v0.0.16 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/minio/colorjson v1.0.8 // indirect + github.com/minio/crc64nvme v1.0.1 // indirect + github.com/minio/filepath v1.0.0 // indirect + github.com/minio/mc v0.0.0-20250313080218-cf909e1063a9 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/websocket v1.6.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 // indirect + github.com/muesli/cancelreader v0.2.2 // indirect + github.com/muesli/reflow v0.3.0 // indirect + github.com/muesli/termenv v0.16.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nats-io/jwt/v2 v2.7.4 // indirect + github.com/nats-io/nats-streaming-server v0.24.6 // indirect + github.com/nats-io/nkeys v0.4.11 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + github.com/oklog/ulid v1.3.1 // indirect + github.com/olekukonko/tablewriter v0.0.5 // indirect + github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect + github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect + github.com/posener/complete v1.2.3 // indirect + github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 // indirect + github.com/prometheus/prom2json v1.4.2 // indirect + github.com/prometheus/prometheus v0.303.0 // indirect + github.com/rivo/uniseg v0.4.7 // indirect + github.com/rjeczalik/notify v0.9.3 // indirect + github.com/rs/xid v1.6.0 // indirect + github.com/safchain/ethtool v0.5.10 // indirect + github.com/segmentio/asm v1.2.0 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/spiffe/go-spiffe/v2 v2.5.0 // indirect + github.com/tidwall/gjson v1.18.0 // indirect + github.com/tidwall/match v1.1.1 // indirect + github.com/tidwall/pretty v1.2.1 // indirect + github.com/tklauser/go-sysconf v0.3.15 // indirect + github.com/tklauser/numcpus v0.10.0 // indirect + github.com/unrolled/secure v1.17.0 // indirect + github.com/vbauerster/mpb/v8 v8.9.3 // indirect + github.com/xdg/stringprep v1.0.3 // indirect + github.com/yusufpapurcu/wmi v1.2.4 // indirect + github.com/zeebo/errs v1.4.0 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.21 // indirect + go.mongodb.org/mongo-driver v1.17.3 // indirect + go.opentelemetry.io/auto/sdk v1.1.0 // indirect + go.opentelemetry.io/contrib/detectors/gcp v1.35.0 // indirect + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 // indirect + go.opentelemetry.io/otel v1.35.0 // indirect + go.opentelemetry.io/otel/metric v1.35.0 // indirect + go.opentelemetry.io/otel/sdk v1.35.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.35.0 // indirect + go.opentelemetry.io/otel/trace v1.35.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + golang.org/x/mod v0.24.0 // indirect + golang.org/x/net v0.39.0 // indirect + golang.org/x/text v0.24.0 // indirect + golang.org/x/tools v0.32.0 // indirect + google.golang.org/genproto v0.0.0-20250422160041-2d3770c4ea7f // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20250422160041-2d3770c4ea7f // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20250422160041-2d3770c4ea7f // indirect + google.golang.org/grpc v1.72.0 // indirect + google.golang.org/protobuf v1.36.6 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 00000000..1ef381cf --- /dev/null +++ b/go.sum @@ -0,0 +1,879 @@ +aead.dev/mem v0.2.0 h1:ufgkESS9+lHV/GUjxgc2ObF43FLZGSemh+W+y27QFMI= +aead.dev/mem v0.2.0/go.mod h1:4qj+sh8fjDhlvne9gm/ZaMRIX9EkmDrKOLwmyDtoMWM= +aead.dev/minisign v0.2.0/go.mod h1:zdq6LdSd9TbuSxchxwhpA9zEb9YXcVGoE8JakuiGaIQ= +aead.dev/minisign v0.3.0 h1:8Xafzy5PEVZqYDNP60yJHARlW1eOQtsKNp/Ph2c0vRA= +aead.dev/minisign v0.3.0/go.mod h1:NLvG3Uoq3skkRMDuc3YHpWUTMTrSExqm+Ij73W13F6Y= +aead.dev/mtls v0.2.1 h1:47NHWciMvrmEhlkpnis8/RGEa9HR9gcbDPfcArG+Yqs= +aead.dev/mtls v0.2.1/go.mod h1:rZvRApIcPkCNu2AgpFoaMxKBee/XVkKs7wEuYgqLI3Q= +cel.dev/expr v0.23.1 h1:K4KOtPCJQjVggkARsjG9RWXP6O4R73aHeJMa/dmCQQg= +cel.dev/expr v0.23.1/go.mod h1:hLPLo1W4QUmuYdA72RBX06QTs6MXw941piREPl3Yfiw= +cloud.google.com/go v0.120.1 h1:Z+5V7yd383+9617XDCyszmK5E4wJRJL+tquMfDj9hLM= +cloud.google.com/go v0.120.1/go.mod h1:56Vs7sf/i2jYM6ZL9NYlC82r04PThNcPS5YgFmb0rp8= +cloud.google.com/go/auth v0.16.0 h1:Pd8P1s9WkcrBE2n/PhAwKsdrR35V3Sg2II9B+ndM3CU= +cloud.google.com/go/auth v0.16.0/go.mod h1:1howDHJ5IETh/LwYs3ZxvlkXF48aSqqJUM+5o02dNOI= +cloud.google.com/go/auth/oauth2adapt v0.2.8 h1:keo8NaayQZ6wimpNSmW5OPc283g65QNIiLpZnkHRbnc= +cloud.google.com/go/auth/oauth2adapt v0.2.8/go.mod h1:XQ9y31RkqZCcwJWNSx2Xvric3RrU88hAYYbjDWYDL+c= +cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I= +cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg= +cloud.google.com/go/iam v1.5.2 h1:qgFRAGEmd8z6dJ/qyEchAuL9jpswyODjA2lS+w234g8= +cloud.google.com/go/iam v1.5.2/go.mod h1:SE1vg0N81zQqLzQEwxL2WI6yhetBdbNQuTvIKCSkUHE= +cloud.google.com/go/logging v1.13.0 h1:7j0HgAp0B94o1YRDqiqm26w4q1rDMH7XNRU34lJXHYc= +cloud.google.com/go/logging v1.13.0/go.mod h1:36CoKh6KA/M0PbhPKMq6/qety2DCAErbhXT62TuXALA= +cloud.google.com/go/longrunning v0.6.7 h1:IGtfDWHhQCgCjwQjV9iiLnUta9LBCo8R9QmAFsS/PrE= +cloud.google.com/go/longrunning v0.6.7/go.mod h1:EAFV3IZAKmM56TyiE6VAP3VoTzhZzySwI/YI1s/nRsY= +cloud.google.com/go/monitoring v1.24.2 h1:5OTsoJ1dXYIiMiuL+sYscLc9BumrL3CarVLL7dd7lHM= +cloud.google.com/go/monitoring v1.24.2/go.mod h1:x7yzPWcgDRnPEv3sI+jJGBkwl5qINf+6qY4eq0I9B4U= +cloud.google.com/go/storage v1.52.0 h1:ROpzMW/IwipKtatA69ikxibdzQSiXJrY9f6IgBa9AlA= +cloud.google.com/go/storage v1.52.0/go.mod h1:4wrBAbAYUvYkbrf19ahGm4I5kDQhESSqN3CGEkMGvOY= +cloud.google.com/go/trace v1.11.6 h1:2O2zjPzqPYAHrn3OKl029qlqG6W8ZdYaOWRyr8NgMT4= +cloud.google.com/go/trace v1.11.6/go.mod h1:GA855OeDEBiBMzcckLPE2kDunIpC72N+Pq8WFieFjnI= +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0 h1:Gt0j3wceWMwPmiazCa8MzMA0MfhmPIz0Qp0FJ6qcM0U= +github.com/Azure/azure-sdk-for-go/sdk/azcore v1.18.0/go.mod h1:Ot/6aikWnKWi4l9QB7qVSwa8iMphQNqkWALMoNT3rzM= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0 h1:OVoM452qUFBrX+URdH3VpR299ma4kfom0yB0URYky9g= +github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.9.0/go.mod h1:kUjrAo8bgEwLeZ/CmHqNl3Z/kPm7y6FKfxxK0izYUg4= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2 h1:yz1bePFlP5Vws5+8ez6T3HWXPmwOK7Yvq8QxDBD3SKY= +github.com/Azure/azure-sdk-for-go/sdk/azidentity/cache v0.3.2/go.mod h1:Pa9ZNPuoNu/GztvBSKk9J1cDJW6vk/n0zLtV4mgd8N8= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1 h1:FPKJS1T+clwv+OLGt13a8UjqeRuh0O4SJ3lUriThc+4= +github.com/Azure/azure-sdk-for-go/sdk/internal v1.11.1/go.mod h1:j2chePtV91HrC22tGoRX3sGY42uF13WzmmV80/OdVAA= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0 h1:LR0kAX9ykz8G4YgLCaRDVJ3+n43R8MneB5dTy2konZo= +github.com/Azure/azure-sdk-for-go/sdk/resourcemanager/storage/armstorage v1.8.0/go.mod h1:DWAciXemNf++PQJLeXUB4HHH5OpsAh12HZnu2wXE1jA= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1 h1:lhZdRq7TIx0GJQvSyX2Si406vrYsov2FXGp/RnSEtcs= +github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.6.1/go.mod h1:8cl44BDmi+effbARHMQjgOKA2AYvcohNm7KEt42mSV8= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358 h1:mFRzDkZVAjdal+s7s0MwaRv9igoPqLRdzOLzw/8Xvq8= +github.com/Azure/go-ntlmssp v0.0.0-20221128193559-754e69321358/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJTmL004Abzc5wDB5VtZG2PJk5ndYDgVacGqfirKxjM= +github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= +github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= +github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 h1:ErKg/3iS1AKcTkf3yixlZ54f9U1rljCkQyEXWUnIUxc= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0/go.mod h1:yAZHSGnqScoU556rBOVkwLze6WP5N+U11RHuWaGVxwY= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 h1:fYE9p3esPxA/C0rQ0AHhP0drtPXDRhaWiwg1DPqO7IU= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0/go.mod h1:BnBReJLvVYx2CS/UHOgVz2BXKXD9wsQPxZug20nZhd0= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0 h1:OqVGm6Ei3x5+yZmSJG1Mh2NwHvpVmZ08CB5qJhT9Nuk= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/cloudmock v0.51.0/go.mod h1:SZiPHWGOOk3bl8tkevxkoiwPgsIl6CwrWcbwjfHZpdM= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0 h1:6/0iUd0xrnX7qt+mLNRwg5c0PGv8wpE8K90ryANQwMI= +github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.51.0/go.mod h1:otE2jQekW/PqXk1Awf5lmfokJx4uwuqcj1ab5SpGeW0= +github.com/IBM/sarama v1.45.1 h1:nY30XqYpqyXOXSNoe2XCgjj9jklGM1Ye94ierUb1jQ0= +github.com/IBM/sarama v1.45.1/go.mod h1:qifDhA3VWSrQ1TjSMyxDl3nYL3oX2C83u+G6L79sq4w= +github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow= +github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= +github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= +github.com/alecthomas/participle v0.7.1 h1:2bN7reTw//5f0cugJcTOnY/NYZcWQOaajW+BwZB5xWs= +github.com/alecthomas/participle v0.7.1/go.mod h1:HfdmEuwvr12HXQN44HPWXR0lHmVolVYe4dyL6lQ3duY= +github.com/alecthomas/repr v0.0.0-20181024024818-d37bc2a10ba1/go.mod h1:xTS7Pm1pD1mvyM075QCDSRqH6qRLXylzS24ZTpRiSzQ= +github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa h1:LHTHcTQiSGT7VVbI0o4wBRNQIgn917usHWOd6VAffYI= +github.com/alexbrainman/sspi v0.0.0-20231016080023-1a75b4708caa/go.mod h1:cEWa1LVoE5KvSD9ONXsZrj0z6KqySlCCNKHlLzbqAt4= +github.com/antithesishq/antithesis-sdk-go v0.4.3-default-no-op h1:+OSa/t11TFhqfrX0EOSqQBDJ0YlpmK0rDSiB19dg9M0= +github.com/antithesishq/antithesis-sdk-go v0.4.3-default-no-op/go.mod h1:IUpT2DPAKh6i/YhSbt6Gl3v2yvUZjmKncl7U91fup7E= +github.com/apache/thrift v0.16.0/go.mod h1:PHK3hniurgQaNMZYaCLEqXKsYK8upmhPbmdP2FXSqgU= +github.com/apache/thrift v0.21.0 h1:tdPmh/ptjE1IJnhbhrcl2++TauVjy242rkV/UzJChnE= +github.com/apache/thrift v0.21.0/go.mod h1:W1H8aR/QRtYNvrPeFXBtobyRkd0/YVhTc6i07XIAgDw= +github.com/araddon/dateparse v0.0.0-20210429162001-6b43995a97de/go.mod h1:DCaWoUhZrYW9p1lxo/cm8EmUOOzAPSEZNGF2DK1dJgw= +github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878/go.mod h1:3AMJUQhVx52RsWOnlkpikZr01T/yAVN2gn0861vByNg= +github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= +github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= +github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= +github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= +github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= +github.com/aymanbagabas/go-udiff v0.2.0/go.mod h1:RE4Ex0qsGkTAJoQdQQCA0uG+nAzJO/pI/QwceO5fgrA= +github.com/beevik/ntp v1.4.3 h1:PlbTvE5NNy4QHmA4Mg57n7mcFTmr1W1j3gcK7L1lqho= +github.com/beevik/ntp v1.4.3/go.mod h1:Unr8Zg+2dRn7d8bHFuehIMSvvUYssHMxW3Q5Nx4RW5Q= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/charmbracelet/bubbles v0.20.0 h1:jSZu6qD8cRQ6k9OMfR1WlM+ruM8fkPWkHvQWD9LIutE= +github.com/charmbracelet/bubbles v0.20.0/go.mod h1:39slydyswPy+uVOHZ5x/GjwVAFkCsV8IIVy+4MhzwwU= +github.com/charmbracelet/bubbletea v1.3.4 h1:kCg7B+jSCFPLYRA52SDZjr51kG/fMUEoPoZrkaDHyoI= +github.com/charmbracelet/bubbletea v1.3.4/go.mod h1:dtcUCyCGEX3g9tosuYiut3MXgY/Jsv9nKVdibKKRRXo= +github.com/charmbracelet/lipgloss v1.0.0 h1:O7VkGDvqEdGi93X+DeqsQ7PKHDgtQfF8j8/O2qFMQNg= +github.com/charmbracelet/lipgloss v1.0.0/go.mod h1:U5fy9Z+C38obMs+T+tJqst9VGzlOYGj4ri9reL3qUlo= +github.com/charmbracelet/x/ansi v0.8.0 h1:9GTq3xq9caJW8ZrBTe0LIe2fvfLR/bYXKTx2llXn7xE= +github.com/charmbracelet/x/ansi v0.8.0/go.mod h1:wdYl/ONOLHLIVmQaxbIYEC/cRKOQyjTkowiI4blgS9Q= +github.com/charmbracelet/x/exp/golden v0.0.0-20240815200342-61de596daa2b h1:MnAMdlwSltxJyULnrYbkZpp4k58Co7Tah3ciKhSNo0Q= +github.com/charmbracelet/x/exp/golden v0.0.0-20240815200342-61de596daa2b/go.mod h1:wDlXFlCrmJ8J+swcL/MnGUuYnqgQdW9rhSD61oNMb6U= +github.com/charmbracelet/x/term v0.2.1 h1:AQeHeLZ1OqSXhrAWpYUtZyX1T3zVxfpZuEQMIQaGIAQ= +github.com/charmbracelet/x/term v0.2.1/go.mod h1:oQ4enTYFV7QN4m0i9mzHrViD7TQKvNEEkHUMCmsxdUg= +github.com/cheggaaa/pb v1.0.29 h1:FckUN5ngEk2LpvuG0fw1GEFx6LtyY2pWI/Z2QgCnEYo= +github.com/cheggaaa/pb v1.0.29/go.mod h1:W40334L7FMC5JKWldsTWbdGjLo0RxUKK73K+TuPxX30= +github.com/chromedp/cdproto v0.0.0-20230802225258-3cf4e6d46a89/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs= +github.com/chromedp/chromedp v0.9.2/go.mod h1:LkSXJKONWTCHAfQasKFUZI+mxqS4tZqhmtGzzhLsnLs= +github.com/chromedp/sysutil v1.0.0/go.mod h1:kgWmDdq8fTzXYcKIBqIYvRRTnYb9aNS9moAV0xufSww= +github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= +github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= +github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f h1:C5bqEmzEPLsHm9Mv73lSE9e9bKV23aB1vxOsmZrkl3k= +github.com/cncf/xds/go v0.0.0-20250326154945-ae57f3c0d45f/go.mod h1:W+zGtBO5Y1IgJhy4+A9GOqVhqLpfZi+vwmdNXUehLA8= +github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= +github.com/coreos/go-oidc/v3 v3.14.1 h1:9ePWwfdwC4QKRlCXsJGou56adA/owXczOzwKdOumLqk= +github.com/coreos/go-oidc/v3 v3.14.1/go.mod h1:HaZ3szPaZ0e4r6ebqvsLWlk2Tn+aejfmrfah6hnSYEU= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-semver v0.3.1 h1:yi21YpKnrx1gt5R+la8n5WgS0kCrsPp33dmEyHReZr4= +github.com/coreos/go-semver v0.3.1/go.mod h1:irMmmIw/7yzSRPWryHsK7EYSg09caPQL03VsM8rvUec= +github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= +github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cosnicolaou/pbzip2 v1.0.5 h1:+PZ8yRBx6bRXncOJWQvEThyFm8XhF9Yb6WUMN6KsgrA= +github.com/cosnicolaou/pbzip2 v1.0.5/go.mod h1:uCNfm0iE2wIKGRlLyq31M4toziFprNhEnvueGmh5u3M= +github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dchest/siphash v1.2.3 h1:QXwFc8cFOR2dSa/gE6o/HokBMWtLUaNDVd+22aKHeEA= +github.com/dchest/siphash v1.2.3/go.mod h1:0NvQU092bT0ipiFN++/rXm69QG9tVxLAlQHIXMPAkHc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0 h1:NMZiJj8QnKe1LgsbDayM4UoHwbvwDRwnI3hwNaAHRnc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.0/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/eapache/go-resiliency v1.7.0 h1:n3NRTnBn5N0Cbi/IeOHuQn9s2UwVUH7Ga0ZWcP+9JTA= +github.com/eapache/go-resiliency v1.7.0/go.mod h1:5yPzW0MIvSe0JDsv0v+DvcjEv2FyD6iZYSs1ZI+iQho= +github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3 h1:Oy0F4ALJ04o5Qqpdz8XLIpNA3WM/iSIXqxtqo7UGVws= +github.com/eapache/go-xerial-snappy v0.0.0-20230731223053-c322873962e3/go.mod h1:YvSRo5mw33fLEx1+DlK6L2VV43tJt5Eyel9n9XBcR+0= +github.com/eapache/queue v1.1.0 h1:YOEu7KNc61ntiQlcEeUIoDTJ2o8mQznoNvUhiigpIqc= +github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= +github.com/eclipse/paho.mqtt.golang v1.5.0 h1:EH+bUVJNgttidWFkLLVKaQPGmkTUfQQqjOsyvMGvD6o= +github.com/eclipse/paho.mqtt.golang v1.5.0/go.mod h1:du/2qNQVqJf/Sqs4MEL77kR8QTqANF7XU7Fk0aOTAgk= +github.com/elastic/go-elasticsearch/v7 v7.17.10 h1:TCQ8i4PmIJuBunvBS6bwT2ybzVFxxUhhltAs3Gyu1yo= +github.com/elastic/go-elasticsearch/v7 v7.17.10/go.mod h1:OJ4wdbtDNk5g503kvlHLyErCgQwwzmDtaFC4XyOxXA4= +github.com/envoyproxy/go-control-plane v0.13.4 h1:zEqyPVyku6IvWCFwux4x9RxkLOMUL+1vC9xUFv5l2/M= +github.com/envoyproxy/go-control-plane v0.13.4/go.mod h1:kDfuBlDVsSj2MjrLEtRWtHlsWIFcGyB2RMO44Dc5GZA= +github.com/envoyproxy/go-control-plane/envoy v1.32.4 h1:jb83lalDRZSpPWW2Z7Mck/8kXZ5CQAFYVjQcdVIr83A= +github.com/envoyproxy/go-control-plane/envoy v1.32.4/go.mod h1:Gzjc5k8JcJswLjAx1Zm+wSYE20UrLtt7JZMWiWQXQEw= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0 h1:/G9QYbddjL25KvtKTv3an9lx6VBE2cnb8wp1vEGNYGI= +github.com/envoyproxy/go-control-plane/ratelimit v0.1.0/go.mod h1:Wk+tMFAFbCXaJPzVVHnPgRKdUdwW/KdbRt94AzgRee4= +github.com/envoyproxy/protoc-gen-validate v1.2.1 h1:DEo3O99U8j4hBFwbJfrz9VtgcDfUKS7KJ7spH3d86P8= +github.com/envoyproxy/protoc-gen-validate v1.2.1/go.mod h1:d/C80l/jxXLdfEIhX1W2TmLfsJ31lvEjwamM4DxlWXU= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f h1:Y/CXytFA4m6baUTXGLOoWe4PQhGxaX0KpnayAqC48p4= +github.com/erikgeiser/coninput v0.0.0-20211004153227-1c3628e74d0f/go.mod h1:vw97MGsxSvLiUE2X8qFplwetxpGLQrlU1Q9AUEIzCaM= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.18.0 h1:S8gINlzdQ840/4pfAwic/ZE0djQEH3wM94VfqLTZcOM= +github.com/fatih/color v1.18.0/go.mod h1:4FelSpRwEGDpQ12mAdzqdOukCy4u8WUtOY6lkT/6HfU= +github.com/fatih/structs v1.1.0 h1:Q7juDM0QtcnhCpeyLGQKyg4TOIghuNXrkL32pHAUMxo= +github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= +github.com/felixge/fgprof v0.9.5 h1:8+vR6yu2vvSKn08urWyEuxx75NWPEvybbkBirEpsbVY= +github.com/felixge/fgprof v0.9.5/go.mod h1:yKl+ERSa++RYOs32d8K6WEXCB4uXdLls4ZaZPpayhMM= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/fraugster/parquet-go v0.12.0 h1:1slnC5y2VWEOUSlzbeXatM0BvSWcLUDsR/EcZsXXCZc= +github.com/fraugster/parquet-go v0.12.0/go.mod h1:dGzUxdNqXsAijatByVgbAWVPlFirnhknQbdazcUIjY0= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667 h1:BP4M0CvQ4S3TGls2FvczZtj5Re/2ZzkV9VwqPHH/3Bo= +github.com/go-asn1-ber/asn1-ber v1.5.8-0.20250403174932-29230038a667/go.mod h1:hEBeB/ic+5LoWskz+yKT7vGhhPYkProFKoKdwZRWMe0= +github.com/go-ini/ini v1.67.0 h1:z6ZrTEZqSWOTyH2FlglNbNgARyHG8oLW9gMELqKr06A= +github.com/go-ini/ini v1.67.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= +github.com/go-jose/go-jose/v4 v4.1.0 h1:cYSYxd3pw5zd2FSXk2vGdn9igQU2PS8MuxrCOCl0FdY= +github.com/go-jose/go-jose/v4 v4.1.0/go.mod h1:GG/vqmYm3Von2nYiB2vGTXzdoNKE5tix5tuc6iAd+sw= +github.com/go-ldap/ldap/v3 v3.4.11 h1:4k0Yxweg+a3OyBLjdYn5OKglv18JNvfDykSoI8bW0gU= +github.com/go-ldap/ldap/v3 v3.4.11/go.mod h1:bY7t0FLK8OAVpp/vV6sSlpz3EQDGcQwc8pF0ujLgKvM= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE= +github.com/go-ole/go-ole v1.3.0/go.mod h1:5LS6F96DhAwUc7C+1HLexzMXY1xGRSryjyPPKW6zv78= +github.com/go-openapi/analysis v0.23.0 h1:aGday7OWupfMs+LbmLZG4k0MYXIANxcuBTYUC03zFCU= +github.com/go-openapi/analysis v0.23.0/go.mod h1:9mz9ZWaSlV8TvjQHLl2mUW2PbZtemkE8yA5v22ohupo= +github.com/go-openapi/errors v0.22.1 h1:kslMRRnK7NCb/CvR1q1VWuEQCEIsBGn5GgKD9e+HYhU= +github.com/go-openapi/errors v0.22.1/go.mod h1:+n/5UdIqdVnLIJ6Q9Se8HNGUXYaY6CN8ImWzfi/Gzp0= +github.com/go-openapi/jsonpointer v0.21.1 h1:whnzv/pNXtK2FbX/W9yJfRmE2gsmkfahjMKB0fZvcic= +github.com/go-openapi/jsonpointer v0.21.1/go.mod h1:50I1STOfbY1ycR8jGz8DaMeLCdXiI6aDteEdRNNzpdk= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/loads v0.22.0 h1:ECPGd4jX1U6NApCGG1We+uEozOAvXvJSF4nnwHZ8Aco= +github.com/go-openapi/loads v0.22.0/go.mod h1:yLsaTCS92mnSAZX5WWoxszLj0u+Ojl+Zs5Stn1oF+rs= +github.com/go-openapi/runtime v0.28.0 h1:gpPPmWSNGo214l6n8hzdXYhPuJcGtziTOgUpvsFWGIQ= +github.com/go-openapi/runtime v0.28.0/go.mod h1:QN7OzcS+XuYmkQLw05akXk0jRH/eZ3kb18+1KwW9gyc= +github.com/go-openapi/spec v0.21.0 h1:LTVzPc3p/RzRnkQqLRndbAzjY0d0BCL72A6j3CdL9ZY= +github.com/go-openapi/spec v0.21.0/go.mod h1:78u6VdPw81XU44qEWGhtr982gJ5BWg2c0I5XwVMotYk= +github.com/go-openapi/strfmt v0.23.0 h1:nlUS6BCqcnAk0pyhi9Y+kdDVZdZMHfEKQiS4HaMgO/c= +github.com/go-openapi/strfmt v0.23.0/go.mod h1:NrtIpfKtWIygRkKVsxh7XQMDQW5HKQl6S5ik2elW+K4= +github.com/go-openapi/swag v0.23.1 h1:lpsStH0n2ittzTnbaSloVZLuB5+fvSY/+hnagBjSNZU= +github.com/go-openapi/swag v0.23.1/go.mod h1:STZs8TbRvEQQKUA+JZNAm3EWlgaOBGpyFDqQnDHMef0= +github.com/go-openapi/validate v0.24.0 h1:LdfDKwNbpB6Vn40xhTdNZAnfLECL81w+VX3BumrGD58= +github.com/go-openapi/validate v0.24.0/go.mod h1:iyeX1sEufmv3nPbBdX3ieNviWnOZaJ1+zquzJEf2BAQ= +github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/go-sql-driver/mysql v1.9.2 h1:4cNKDYQ1I84SXslGddlsrMhc8k4LeDVj6Ad6WRjiHuU= +github.com/go-sql-driver/mysql v1.9.2/go.mod h1:qn46aNg1333BRMNU69Lq93t8du/dwxI64Gl8i5p1WMU= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= +github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= +github.com/gobwas/ws v1.4.0 h1:CTaoG1tojrh4ucGPcoJFiAQUAsEWekEWvLy7GsVNqGs= +github.com/gobwas/ws v1.4.0/go.mod h1:G3gNqMNtPppf5XUz7O4shetPpcZ1VJ7zt18dlUeakrc= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt/v4 v4.5.2 h1:YtQM7lnr8iZ+j5q71MGKkNw9Mn7AjHM68uc9g5fXeUI= +github.com/golang-jwt/jwt/v4 v4.5.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0= +github.com/golang-jwt/jwt/v5 v5.2.2 h1:Rl4B7itRWVtYIHFrSNd7vhTiz9UpLdi6gZhZ3wEeDy8= +github.com/golang-jwt/jwt/v5 v5.2.2/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= +github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v1.9.2 h1:HrutZBLhSIU8abiSfW8pj8mPhOyMYjZT/wcA4/L9L9s= +github.com/gomodule/redigo v1.9.2/go.mod h1:KsU3hiK/Ay8U42qpaJk+kuNa3C+spxapWpM+ywhcgtw= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= +github.com/google/go-tpm v0.9.3 h1:+yx0/anQuGzi+ssRqeD6WpXjW2L/V0dItUayO0i9sRc= +github.com/google/go-tpm v0.9.3/go.mod h1:h9jEsEECg7gtLis0upRBQU+GhYVH6jMjrFxI8u6bVUY= +github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian/v3 v3.3.3 h1:DIhPTQrbPkgs2yJYdXU/eNACCG5DVQjySNRNlflZ9Fc= +github.com/google/martian/v3 v3.3.3/go.mod h1:iEPrYcgCF7jA9OtScMFQyAlZZ4YXTKEtJ1E6RWzmBA0= +github.com/google/pprof v0.0.0-20240227163752-401108e1b7e7/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20250422154841-e1f9c1950416 h1:1/qwHx8P72glDXdyCKesJ+/c40x71SY4q2avOxJ2iYQ= +github.com/google/pprof v0.0.0-20250422154841-e1f9c1950416/go.mod h1:5hDyRhoBCxViHszMt12TnOpEI4VVi+U8Gm9iphldiMA= +github.com/google/s2a-go v0.1.9 h1:LGD7gtMgezd8a/Xak7mEWL0PjoTQFvpRudN895yqKW0= +github.com/google/s2a-go v0.1.9/go.mod h1:YA0Ei2ZQL3acow2O62kdp9UlnvMmU7kA6Eutn0dXayM= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= +github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.6 h1:GW/XbdyBFQ8Qe+YAmFU9uHLo7OnF5tL52HFAgMmyrf4= +github.com/googleapis/enterprise-certificate-proxy v0.3.6/go.mod h1:MkHOF77EYAE7qfSuSS9PU6g4Nt4e11cnsDUowfwewLA= +github.com/googleapis/gax-go/v2 v2.14.1 h1:hb0FFeiPaQskmvakKu5EbCbpntQn48jyHuvrkurSS/Q= +github.com/googleapis/gax-go/v2 v2.14.1/go.mod h1:Hb/NubMaVM88SrNkvl8X/o8XWwDJEPqouaLeN2IUxoA= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= +github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= +github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.9.1/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= +github.com/hashicorp/go-hclog v1.1.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-msgpack v1.1.5 h1:9byZdVjKTe5mce63pRVNP1L7UAmdHOTEMGehn6KvJWs= +github.com/hashicorp/go-msgpack v1.1.5/go.mod h1:gWVc3sv/wbDmR3rQsj1CAktEZzoz1YNK9NfGLXJ69/4= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/raft v1.3.9 h1:9yuo1aR0bFTr1cw7pj3S2Bk6MhJCsnr2NAxvIBrP2x4= +github.com/hashicorp/raft v1.3.9/go.mod h1:4Ak7FSPnuvmb0GV6vgIAJ4vYT4bek9bb6Q+7HVbyzqM= +github.com/ianlancetaylor/demangle v0.0.0-20230524184225-eabc099b10ab/go.mod h1:gx7rwoVhcfuVKG5uya9Hs3Sxj7EIvldVofAWIUtGouw= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/jcmturner/aescts/v2 v2.0.0 h1:9YKLH6ey7H4eDBXW8khjYslgyqG2xZikXP0EQFKrle8= +github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= +github.com/jcmturner/dnsutils/v2 v2.0.0 h1:lltnkeZGL0wILNvrNiVCR6Ro5PGU/SeBvVO/8c/iPbo= +github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= +github.com/jcmturner/gofork v1.7.6 h1:QH0l3hzAU1tfT3rZCnW5zXl+orbkNMMRGJfdJjHVETg= +github.com/jcmturner/gofork v1.7.6/go.mod h1:1622LH6i/EZqLloHfE7IeZ0uEJwMSUyQ/nDd82IeqRo= +github.com/jcmturner/goidentity/v6 v6.0.1 h1:VKnZd2oEIMorCTsFBnJWbExfNN7yZr3EhJAxwOkZg6o= +github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= +github.com/jcmturner/gokrb5/v8 v8.4.4 h1:x1Sv4HaTpepFkXbt2IkL29DXRf8sOfZXo8eRKh687T8= +github.com/jcmturner/gokrb5/v8 v8.4.4/go.mod h1:1btQEpgT6k+unzCwX1KdWMEwPPkkgBtP+F6aCACiMrs= +github.com/jcmturner/rpc/v2 v2.0.3 h1:7FXXj8Ti1IaVFpSAziCZWNzbNuZmnvw/i6CqLNdWfZY= +github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= +github.com/jedib0t/go-pretty/v6 v6.6.7 h1:m+LbHpm0aIAPLzLbMfn8dc3Ht8MW7lsSO4MPItz/Uuo= +github.com/jedib0t/go-pretty/v6 v6.6.7/go.mod h1:YwC5CE4fJ1HFUDeivSV1r//AmANFHyqczZk+U6BDALU= +github.com/jessevdk/go-flags v1.6.1 h1:Cvu5U8UGrLay1rZfv/zP7iLpSHGUZ/Ou68T0iX1bBK4= +github.com/jessevdk/go-flags v1.6.1/go.mod h1:Mk8T1hIAWpOiJiHa9rJASDK2UGWji0EuPGBnNLMooyc= +github.com/jlaffaye/ftp v0.0.0-20190624084859-c1312a7102bf/go.mod h1:lli8NYPQOFy3O++YmYbqVgOcQ1JPCwdOy+5zSjKJ9qY= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= +github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI= +github.com/juju/ratelimit v1.0.2/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= +github.com/keybase/go-keychain v0.0.1 h1:way+bWYa6lDppZoZcgMbYsvC7GxljxrskdNInRtuthU= +github.com/keybase/go-keychain v0.0.1/go.mod h1:PdEILRW3i9D8JcdM+FmY6RwkHGnhHxXwkPPMeUgOK1k= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.14.4/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= +github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= +github.com/klauspost/filepathx v1.1.1 h1:201zvAsL1PhZvmXTP+QLer3AavWrO3U1NILWpniHK4w= +github.com/klauspost/filepathx v1.1.1/go.mod h1:XWxdp8rEw4gupPBrxrV5Q57dL/71xj0OgV1gKt2zTfU= +github.com/klauspost/pgzip v1.2.6 h1:8RXeL5crjEUFnR2/Sn6GJNWtSQ3Dk8pq4CL3jvdDyjU= +github.com/klauspost/pgzip v1.2.6/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= +github.com/klauspost/readahead v1.4.0 h1:w4hQ3BpdLjBnRQkZyNi+nwdHU7eGP9buTexWK9lU7gY= +github.com/klauspost/readahead v1.4.0/go.mod h1:7bolpMKhT5LKskLwYXGSDOyA2TYtMFgdgV0Y8gy7QhA= +github.com/klauspost/reedsolomon v1.12.4 h1:5aDr3ZGoJbgu/8+j45KtUJxzYm8k08JGtB9Wx1VQ4OA= +github.com/klauspost/reedsolomon v1.12.4/go.mod h1:d3CzOMOt0JXGIFZm1StgkyF14EYr3xneR2rNWo7NcMU= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/fs v0.1.0 h1:Jskdu9ieNAYnjxsi0LbQp1ulIKZV1LAFgK1tWhpZgl8= +github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= +github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/ledongthuc/pdf v0.0.0-20220302134840-0c2507a12d80/go.mod h1:imJHygn/1yfhB7XSJJKlFZKl/J+dCPAknuiaGOshXAs= +github.com/lestrrat-go/blackmagic v1.0.2 h1:Cg2gVSc9h7sz9NOByczrbUvLopQmXrfFx//N+AkAr5k= +github.com/lestrrat-go/blackmagic v1.0.2/go.mod h1:UrEqBzIR2U6CnzVyUtfM6oZNMt/7O7Vohk2J0OGSAtU= +github.com/lestrrat-go/httpcc v1.0.1 h1:ydWCStUeJLkpYyjLDHihupbn2tYmZ7m22BGkcvZZrIE= +github.com/lestrrat-go/httpcc v1.0.1/go.mod h1:qiltp3Mt56+55GPVCbTdM9MlqhvzyuL6W/NMDA8vA5E= +github.com/lestrrat-go/httprc v1.0.6 h1:qgmgIRhpvBqexMJjA/PmwSvhNk679oqD1RbovdCGW8k= +github.com/lestrrat-go/httprc v1.0.6/go.mod h1:mwwz3JMTPBjHUkkDv/IGJ39aALInZLrhBp0X7KGUZlo= +github.com/lestrrat-go/iter v1.0.2 h1:gMXo1q4c2pHmC3dn8LzRhJfP1ceCbgSiT9lUydIzltI= +github.com/lestrrat-go/iter v1.0.2/go.mod h1:Momfcq3AnRlRjI5b5O8/G5/BvpzrhoFTZcn06fEOPt4= +github.com/lestrrat-go/jwx/v2 v2.1.4 h1:uBCMmJX8oRZStmKuMMOFb0Yh9xmEMgNJLgjuKKt4/qc= +github.com/lestrrat-go/jwx/v2 v2.1.4/go.mod h1:nWRbDFR1ALG2Z6GJbBXzfQaYyvn751KuuyySN2yR6is= +github.com/lestrrat-go/option v1.0.1 h1:oAzP2fvZGQKWkvHa1/SAcFolBEca1oN+mQ7eooNBEYU= +github.com/lestrrat-go/option v1.0.1/go.mod h1:5ZHFbivi4xwXxhxY9XHDe2FHo6/Z7WWmtT7T5nBBp3I= +github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lithammer/shortuuid/v4 v4.2.0 h1:LMFOzVB3996a7b8aBuEXxqOBflbfPQAiVzkIcHO0h8c= +github.com/lithammer/shortuuid/v4 v4.2.0/go.mod h1:D5noHZ2oFw/YaKCfGy0YxyE7M0wMbezmMjPdhyEFe6Y= +github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY= +github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35 h1:PpXWgLPs+Fqr325bN2FD2ISlRRztXibcX6e8f5FR5Dc= +github.com/lufia/plan9stats v0.0.0-20250317134145-8bc96cf8fc35/go.mod h1:autxFIvghDt3jPTLoqZ9OZ7s9qTGNAWmYCjVFWPX/zg= +github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.9.0 h1:PrnmzHw7262yW8sTBwxi1PdJA3Iw/EKBa8psRf7d9a4= +github.com/mailru/easyjson v0.9.0/go.mod h1:1+xMtQp2MRNVL/V1bOzuP3aP8VNwRW55fQUto+XFtTU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.14 h1:9A9LHSqF/7dyVVX6g0U9cwm9pG3kP9gSzcuIPHPsaIE= +github.com/mattn/go-colorable v0.1.14/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= +github.com/mattn/go-ieproxy v0.0.12 h1:OZkUFJC3ESNZPQ+6LzC3VJIFSnreeFLQyqvBWtvfL2M= +github.com/mattn/go-ieproxy v0.0.12/go.mod h1:Vn+N61199DAnVeTgaF8eoB9PvLO8P3OBnG95ENh7B7c= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-localereader v0.0.1 h1:ygSAOl7ZXTx4RdPYinUpg6W99U8jWvWi9Ye2JC/oIi4= +github.com/mattn/go-localereader v0.0.1/go.mod h1:8fBrzywKY7BI3czFoHkuzRoWE9C+EiG4R1k4Cjx5p88= +github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= +github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= +github.com/mattn/go-runewidth v0.0.16 h1:E5ScNMtiwvlvB5paMFdw9p4kSQzbXFikJ5SQO6TULQc= +github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/dns v1.1.65 h1:0+tIPHzUW0GCge7IiK3guGP57VAw7hoPDfApjkMD1Fc= +github.com/miekg/dns v1.1.65/go.mod h1:Dzw9769uoKVaLuODMDZz9M6ynFU6Em65csPuoi8G0ck= +github.com/minio/cli v1.24.2 h1:J+fCUh9mhPLjN3Lj/YhklXvxj8mnyE/D6FpFduXJ2jg= +github.com/minio/cli v1.24.2/go.mod h1:bYxnK0uS629N3Bq+AOZZ+6lwF77Sodk4+UL9vNuXhOY= +github.com/minio/colorjson v1.0.8 h1:AS6gEQ1dTRYHmC4xuoodPDRILHP/9Wz5wYUGDQfPLpg= +github.com/minio/colorjson v1.0.8/go.mod h1:wrs39G/4kqNlGjwqHvPlAnXuc2tlPszo6JKdSBCLN8w= +github.com/minio/console v1.7.7-0.20250905210349-2017f33b26e1 h1:jOW1ggtITn8sreTzUjcdYE/ZffxeVmWstXNlBLOE6j4= +github.com/minio/console v1.7.7-0.20250905210349-2017f33b26e1/go.mod h1:hKNkzdKBKU84w5wXqMnkH74QocJGHW2zjvFtuGETDsc= +github.com/minio/crc64nvme v1.0.1 h1:DHQPrYPdqK7jQG/Ls5CTBZWeex/2FMS3G5XGkycuFrY= +github.com/minio/crc64nvme v1.0.1/go.mod h1:eVfm2fAzLlxMdUGc0EEBGSMmPwmXD5XiNRpnu9J3bvg= +github.com/minio/csvparser v1.0.0 h1:xJEHcYK8ZAjeW4hNV9Zu30u+/2o4UyPnYgyjWp8b7ZU= +github.com/minio/csvparser v1.0.0/go.mod h1:lKXskSLzPgC5WQyzP7maKH7Sl1cqvANXo9YCto8zbtM= +github.com/minio/dnscache v0.1.1 h1:AMYLqomzskpORiUA1ciN9k7bZT1oB3YZN4cEIi88W5o= +github.com/minio/dnscache v0.1.1/go.mod h1:WCumm6offO4rQ/82oTCSoHnlhTmc81+vXgKpBtSYRbg= +github.com/minio/dperf v0.6.3 h1:+UzGe64Xmb/sXFBH38CCnXJiMEoQKURHydc+baGkysI= +github.com/minio/dperf v0.6.3/go.mod h1:+3BJsm3Jrb1yTRmkoeKCNootOmULPiLoAu+qBP7MaMk= +github.com/minio/filepath v1.0.0 h1:fvkJu1+6X+ECRA6G3+JJETj4QeAYO9sV43I79H8ubDY= +github.com/minio/filepath v1.0.0/go.mod h1:/nRZA2ldl5z6jT9/KQuvZcQlxZIMQoFFQPvEXx9T/Bw= +github.com/minio/highwayhash v1.0.2/go.mod h1:BQskDq+xkJ12lmlUUi7U0M5Swg3EWR+dLTk+kldvVxY= +github.com/minio/highwayhash v1.0.3 h1:kbnuUMoHYyVl7szWjSxJnxw11k2U709jqFPPmIUyD6Q= +github.com/minio/highwayhash v1.0.3/go.mod h1:GGYsuwP/fPD6Y9hMiXuapVvlIUEhFhMTh0rxU3ik1LQ= +github.com/minio/kms-go/kes v0.3.1 h1:K3sPFAvFbJx33XlCTUBnQo8JRmSZyDvT6T2/MQ2iC3A= +github.com/minio/kms-go/kes v0.3.1/go.mod h1:Q9Ct0KUAuN9dH0hSVa0eva45Jg99cahbZpPxeqR9rOQ= +github.com/minio/kms-go/kms v0.5.1-0.20250225090116-4e64ce8d0f35 h1:ISNz42SPD+heeHhpl9bwMRRusPTCsbYKd1YoED265E0= +github.com/minio/kms-go/kms v0.5.1-0.20250225090116-4e64ce8d0f35/go.mod h1:JFQu2srrnWxMn6KcwS5347oTwNKW7nkewgBlrodjF9k= +github.com/minio/madmin-go/v3 v3.0.109 h1:hRHlJ6yaIB3tlIj5mz9L9mGcyLC37S9qL1WtFrRtyQ0= +github.com/minio/madmin-go/v3 v3.0.109/go.mod h1:WOe2kYmYl1OIlY2DSRHVQ8j1v4OItARQ6jGyQqcCud8= +github.com/minio/mc v0.0.0-20250313080218-cf909e1063a9 h1:6RyInOHKL6jz8zxcAar/h6rg/aJCxDP/uFuSNvYSuMI= +github.com/minio/mc v0.0.0-20250313080218-cf909e1063a9/go.mod h1:h5UQZ+5Qfq6XV81E4iZSgStPZ6Hy+gMuHMkLkjq4Gys= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v6 v6.0.46/go.mod h1:qD0lajrGW49lKZLtXKtCB4X/qkMf0a5tBvN2PaZg7Gg= +github.com/minio/minio-go/v7 v7.0.91 h1:tWLZnEfo3OZl5PoXQwcwTAPNNrjyWwOh6cbZitW5JQc= +github.com/minio/minio-go/v7 v7.0.91/go.mod h1:uvMUcGrpgeSAAI6+sD3818508nUyMULw94j2Nxku/Go= +github.com/minio/mux v1.9.2 h1:dQchne49BUBgOlxIHjx5wVe1gl5VXF2sxd4YCXkikTw= +github.com/minio/mux v1.9.2/go.mod h1:OuHAsZsux+e562bcO2P3Zv/P0LMo6fPQ310SmoyG7mQ= +github.com/minio/pkg/v3 v3.1.3 h1:6iBVcTPq7z29suUROciYUBpvLxfzDV3/+Ls0RFDOta8= +github.com/minio/pkg/v3 v3.1.3/go.mod h1:XIUU35+I9lWuTuMf94pwnQjvli6nZfRND6TjZGgqSEE= +github.com/minio/selfupdate v0.6.0 h1:i76PgT0K5xO9+hjzKcacQtO7+MjJ4JKA8Ak8XQ9DDwU= +github.com/minio/selfupdate v0.6.0/go.mod h1:bO02GTIPCMQFTEvE5h4DjYB58bCoZ35XLeBf0buTDdM= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/minio/simdjson-go v0.4.5 h1:r4IQwjRGmWCQ2VeMc7fGiilu1z5du0gJ/I/FsKwgo5A= +github.com/minio/simdjson-go v0.4.5/go.mod h1:eoNz0DcLQRyEDeaPr4Ru6JpjlZPzbA0IodxVJk8lO8E= +github.com/minio/sio v0.4.1 h1:EMe3YBC1nf+sRQia65Rutxi+Z554XPV0dt8BIBA+a/0= +github.com/minio/sio v0.4.1/go.mod h1:oBSjJeGbBdRMZZwna07sX9EFzZy+ywu5aofRiV1g79I= +github.com/minio/websocket v1.6.0 h1:CPvnQvNvlVaQmvw5gtJNyYQhg4+xRmrPNhBbv8BdpAE= +github.com/minio/websocket v1.6.0/go.mod h1:COH1CePZfHT9Ec1O7vZjTlX5uEPpyYnrifPNbu665DM= +github.com/minio/xxml v0.0.3 h1:ZIpPQpfyG5uZQnqqC0LZuWtPk/WT8G/qkxvO6jb7zMU= +github.com/minio/xxml v0.0.3/go.mod h1:wcXErosl6IezQIMEWSK/LYC2VS7LJ1dAkgvuyIN3aH4= +github.com/minio/zipindex v0.4.0 h1:NFPp7OscsUm5Y91+2tJ9Hr4jEG2R20xaz2Wd0ac7uJQ= +github.com/minio/zipindex v0.4.0/go.mod h1:3xib1QhqfYkkxofF881t/50FQMHFH2XvYGyPrd4N948= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= +github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6 h1:ZK8zHtRHOkbHy6Mmr5D264iyp3TiX5OmNcI5cIARiQI= +github.com/muesli/ansi v0.0.0-20230316100256-276c6243b2f6/go.mod h1:CJlz5H+gyd6CUWT45Oy4q24RdLyn7Md9Vj2/ldJBSIo= +github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= +github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/muesli/reflow v0.3.0 h1:IFsN6K9NfGtjeggFP+68I4chLZV2yIKsXJFNZ+eWh6s= +github.com/muesli/reflow v0.3.0/go.mod h1:pbwTDkVPibjO2kyvBQRBxTWEEGDGq0FlB1BIKtnHY/8= +github.com/muesli/termenv v0.16.0 h1:S5AlUN9dENB57rsbnkPyfdGuWIlkmzJjbFf0Tf5FWUc= +github.com/muesli/termenv v0.16.0/go.mod h1:ZRfOIKPFDYQoDFF4Olj7/QJbW60Ol/kL1pU3VfY/Cnk= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nats-io/jwt/v2 v2.2.1-0.20220330180145-442af02fd36a/go.mod h1:0tqz9Hlu6bCBFLWAASKhE5vUA4c24L9KPUUgvwumE/k= +github.com/nats-io/jwt/v2 v2.7.4 h1:jXFuDDxs/GQjGDZGhNgH4tXzSUK6WQi2rsj4xmsNOtI= +github.com/nats-io/jwt/v2 v2.7.4/go.mod h1:me11pOkwObtcBNR8AiMrUbtVOUGkqYjMQZ6jnSdVUIA= +github.com/nats-io/nats-server/v2 v2.8.2/go.mod h1:vIdpKz3OG+DCg4q/xVPdXHoztEyKDWRtykQ4N7hd7C4= +github.com/nats-io/nats-server/v2 v2.11.1 h1:LwdauqMqMNhTxTN3+WFTX6wGDOKntHljgZ+7gL5HCnk= +github.com/nats-io/nats-server/v2 v2.11.1/go.mod h1:leXySghbdtXSUmWem8K9McnJ6xbJOb0t9+NQ5HTRZjI= +github.com/nats-io/nats-streaming-server v0.24.6 h1:iIZXuPSznnYkiy0P3L0AP9zEN9Etp+tITbbX1KKeq4Q= +github.com/nats-io/nats-streaming-server v0.24.6/go.mod h1:tdKXltY3XLeBJ21sHiZiaPl+j8sK3vcCKBWVyxeQs10= +github.com/nats-io/nats.go v1.13.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= +github.com/nats-io/nats.go v1.14.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= +github.com/nats-io/nats.go v1.15.0/go.mod h1:BPko4oXsySz4aSWeFgOHLZs3G4Jq4ZAyE6/zMCxRT6w= +github.com/nats-io/nats.go v1.22.1/go.mod h1:tLqubohF7t4z3du1QDPYJIQQyhb4wl6DhjxEajSI7UA= +github.com/nats-io/nats.go v1.41.2 h1:5UkfLAtu/036s99AhFRlyNDI1Ieylb36qbGjJzHixos= +github.com/nats-io/nats.go v1.41.2/go.mod h1:iRWIPokVIFbVijxuMQq4y9ttaBTMe0SFdlZfMDd+33g= +github.com/nats-io/nkeys v0.3.0/go.mod h1:gvUNGjVcM2IPr5rCsRsC6Wb3Hr2CQAm08dsxtV6A5y4= +github.com/nats-io/nkeys v0.4.11 h1:q44qGV008kYd9W1b1nEBkNzvnWxtRSQ7A8BoqRrcfa0= +github.com/nats-io/nkeys v0.4.11/go.mod h1:szDimtgmfOi9n25JpfIdGw12tZFYXqhGxjhVxsatHVE= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nats-io/stan.go v0.10.2/go.mod h1:vo2ax8K2IxaR3JtEMLZRFKIdoK/3o1/PKueapB7ezX0= +github.com/nats-io/stan.go v0.10.4 h1:19GS/eD1SeQJaVkeM9EkvEYattnvnWrZ3wkSWSw4uXw= +github.com/nats-io/stan.go v0.10.4/go.mod h1:3XJXH8GagrGqajoO/9+HgPyKV5MWsv7S5ccdda+pc6k= +github.com/ncw/directio v1.0.5 h1:JSUBhdjEvVaJvOoyPAbcW0fnd0tvRXD76wEfZ1KcQz4= +github.com/ncw/directio v1.0.5/go.mod h1:rX/pKEYkOXBGOggmcyJeJGloCkleSvphPx2eV3t6ROk= +github.com/nsqio/go-nsq v1.1.0 h1:PQg+xxiUjA7V+TLdXw7nVrJ5Jbl3sN86EhGCQj4+FYE= +github.com/nsqio/go-nsq v1.1.0/go.mod h1:vKq36oyeVXgsS5Q8YEO7WghqidAVXQlcFxzQbQTuDEY= +github.com/oklog/ulid v1.3.1 h1:EGfNDEx6MqHz8B3uNV6QAib1UR2Lm97sHi3ocA6ESJ4= +github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= +github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= +github.com/orisano/pixelmatch v0.0.0-20220722002657-fb0b55479cde/go.mod h1:nZgzbfBr3hhjoZnS66nKrHmduYNpc34ny7RK4z5/HM0= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= +github.com/philhofer/fwd v1.2.0 h1:e6DnBTl7vGY+Gz322/ASL4Gyp1FspeMvx1RNDoToZuM= +github.com/philhofer/fwd v1.2.0/go.mod h1:RqIHx9QI14HlwKwm98g9Re5prTQ6LdeRQn+gXJFxsJM= +github.com/pierrec/lz4/v4 v4.1.22 h1:cKFw6uJDK+/gfw5BcDL0JL5aBsAFdsIT18eRtLj7VIU= +github.com/pierrec/lz4/v4 v4.1.22/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c h1:+mdjkGKdHQG3305AYmdv1U2eRNDiU2ErMBj1gwrq8eQ= +github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c/go.mod h1:7rwL4CYBLnjLxUqIJNnCWiEdr3bn6IUYi15bNlnbCCU= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/sftp v1.13.9 h1:4NGkvGudBL7GteO3m6qnaQ4pC0Kvf0onSVc9gR3EWBw= +github.com/pkg/sftp v1.13.9/go.mod h1:OBN7bVXdstkFFN/gdnHPUb5TE8eb8G1Rp9wCItqjkkA= +github.com/pkg/xattr v0.4.10 h1:Qe0mtiNFHQZ296vRgUjRCoPHPqH7VdTOrZx3g0T+pGA= +github.com/pkg/xattr v0.4.10/go.mod h1:di8WF84zAKk8jzR1UBTEWh9AUlIZZ7M/JNt8e9B6ktU= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 h1:GFCKgmp0tecUJ0sJuv4pzYCqS9+RGSn52M3FUwPs+uo= +github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10/go.mod h1:t/avpk3KcrXxUnYOhZhMXJlSEyie6gQbtLq5NM3loB8= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.2.3 h1:NP0eAhjcjImqslEwo/1hq7gpajME0fTLTezBKDqfXqo= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55 h1:o4JXh1EVt9k/+g42oCprj/FisM4qX9L3sZB3upGN2ZU= +github.com/power-devops/perfstat v0.0.0-20240221224432-82ca36839d55/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_golang v1.22.0 h1:rb93p9lokFEsctTys46VnV1kLCDpVZ0a/Y92Vm0Zc6Q= +github.com/prometheus/client_golang v1.22.0/go.mod h1:R7ljNsLXhuQXYZYtw6GAE9AZg8Y7vEW5scdCXrWRXC0= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.63.0 h1:YR/EIY1o3mEFP/kZCD7iDMnLPlGyuU2Gb3HIcXnA98k= +github.com/prometheus/common v0.63.0/go.mod h1:VVFF/fBIoToEnWRVkYoXEkq3R3paCoxG9PXP74SnV18= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.16.1 h1:hZ15bTNuirocR6u0JZ6BAHHmwS1p8B4P6MRqxtzMyRg= +github.com/prometheus/procfs v0.16.1/go.mod h1:teAbpZRB1iIAJYREa1LsoWUXykVXA1KlTmWl8x/U+Is= +github.com/prometheus/prom2json v1.4.2 h1:PxCTM+Whqi/eykO1MKsEL0p/zMpxp9ybpsmdFamw6po= +github.com/prometheus/prom2json v1.4.2/go.mod h1:zuvPm7u3epZSbXPWHny6G+o8ETgu6eAK3oPr6yFkRWE= +github.com/prometheus/prometheus v0.303.0 h1:wsNNsbd4EycMCphYnTmNY9JASBVbp7NWwJna857cGpA= +github.com/prometheus/prometheus v0.303.0/go.mod h1:8PMRi+Fk1WzopMDeb0/6hbNs9nV6zgySkU/zds5Lu3o= +github.com/puzpuzpuz/xsync/v3 v3.5.1 h1:GJYJZwO6IdxN/IKbneznS6yPkVC+c3zyY/j19c++5Fg= +github.com/puzpuzpuz/xsync/v3 v3.5.1/go.mod h1:VjzYrABPabuM4KyBh1Ftq6u8nhwY5tBPKP9jpmh0nnA= +github.com/rabbitmq/amqp091-go v1.10.0 h1:STpn5XsHlHGcecLmMFCtg7mqq0RnD+zFr4uzukfVhBw= +github.com/rabbitmq/amqp091-go v1.10.0/go.mod h1:Hy4jKW5kQART1u+JkDTF9YYOQUHXqMuhrgxOEeS7G4o= +github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9 h1:bsUq1dX0N8AOIL7EB/X911+m4EHsnWEHeJ0c+3TTBrg= +github.com/rcrowley/go-metrics v0.0.0-20250401214520-65e299d6c5c9/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/redis/go-redis/v9 v9.7.3 h1:YpPyAayJV+XErNsatSElgRZZVCwXX9QzkKYNvO7x0wM= +github.com/redis/go-redis/v9 v9.7.3/go.mod h1:bGUrSggJ9X9GUmZpZNEOQKaANxSGgOEBRltRTZHSvrA= +github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= +github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= +github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rjeczalik/notify v0.9.3 h1:6rJAzHTGKXGj76sbRgDiDcYj/HniypXmSJo1SWakZeY= +github.com/rjeczalik/notify v0.9.3/go.mod h1:gF3zSOrafR9DQEWSE8TjfI9NkooDxbyT4UgRGKZA0lc= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/rs/cors v1.11.1 h1:eU3gRzXLRK57F5rKMGMZURNdIG4EoAmX8k94r9wXWHA= +github.com/rs/cors v1.11.1/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/rs/xid v1.6.0 h1:fV591PaemRlL6JfRxGDEPl69wICngIQ3shQtzfy2gxU= +github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0= +github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/safchain/ethtool v0.5.10 h1:Im294gZtuf4pSGJRAOGKaASNi3wMeFaGaWuSaomedpc= +github.com/safchain/ethtool v0.5.10/go.mod h1:w9jh2Lx7YBR4UwzLkzCmWl85UY0W2uZdd7/DckVE5+c= +github.com/scylladb/termtables v0.0.0-20191203121021-c4c0b6d42ff4/go.mod h1:C1a7PQSMz9NShzorzCiG2fk9+xuCgLkPeCvMHYR2OWg= +github.com/secure-io/sio-go v0.3.1 h1:dNvY9awjabXTYGsTF1PiCySl9Ltofk9GA3VdWlo7rRc= +github.com/secure-io/sio-go v0.3.1/go.mod h1:+xbkjDzPjwh4Axd07pRKSNriS9SCiYksWnZqdnfpQxs= +github.com/segmentio/asm v1.2.0 h1:9BQrFxC+YOHJlTlHGkTrFWf59nbL3XnCoFLTwDCI7ys= +github.com/segmentio/asm v1.2.0/go.mod h1:BqMnlJP91P8d+4ibuonYZw9mfnzI9HfxselHZr5aAcs= +github.com/shirou/gopsutil/v3 v3.24.5 h1:i0t8kL+kQTvpAYToeuiVk3TgDeKOFioZO3Ztz/iZ9pI= +github.com/shirou/gopsutil/v3 v3.24.5/go.mod h1:bsoOS1aStSs9ErQ1WWfxllSeS1K5D+U30r2NfcubMVk= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= +github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= +github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= +github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/spiffe/go-spiffe/v2 v2.5.0 h1:N2I01KCUkv1FAjZXJMwh95KK1ZIQLYbPfhaxw8WS0hE= +github.com/spiffe/go-spiffe/v2 v2.5.0/go.mod h1:P+NxobPc6wXhVtINNtFjNWGBTreew1GBUCwT2wPmb7g= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= +github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= +github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= +github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tinylib/msgp v1.4.0 h1:SYOeDRiydzOw9kSiwdYp9UcBgPFtLU2WDHaJXyHruf8= +github.com/tinylib/msgp v1.4.0/go.mod h1:cvjFkb4RiC8qSBOPMGPSzSAx47nAsfhLVTCZZNuHv5o= +github.com/tklauser/go-sysconf v0.3.15 h1:VE89k0criAymJ/Os65CSn1IXaol+1wrsFHEB8Ol49K4= +github.com/tklauser/go-sysconf v0.3.15/go.mod h1:Dmjwr6tYFIseJw7a3dRLJfsHAMXZ3nEnL/aZY+0IuI4= +github.com/tklauser/numcpus v0.10.0 h1:18njr6LDBk1zuna922MgdjQuJFjrdppsZG60sHGfjso= +github.com/tklauser/numcpus v0.10.0/go.mod h1:BiTKazU708GQTYF4mB+cmlpT2Is1gLk7XVuEeem8LsQ= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/unrolled/secure v1.17.0 h1:Io7ifFgo99Bnh0J7+Q+qcMzWM6kaDPCA5FroFZEdbWU= +github.com/unrolled/secure v1.17.0/go.mod h1:BmF5hyM6tXczk3MpQkFf1hpKSRqCyhqcbiQtiAF7+40= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/vbauerster/mpb/v8 v8.9.3 h1:PnMeF+sMvYv9u23l6DO6Q3+Mdj408mjLRXIzmUmU2Z8= +github.com/vbauerster/mpb/v8 v8.9.3/go.mod h1:hxS8Hz4C6ijnppDSIX6LjG8FYJSoPo9iIOcE53Zik0c= +github.com/xdg/scram v1.0.5 h1:TuS0RFmt5Is5qm9Tm2SoD89OPqe4IRiFtyFY4iwWXsw= +github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= +github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4= +github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.4 h1:zFUKzehAFReQwLys1b/iSMl+JQGSCSjtVqQn9bBrPo0= +github.com/yusufpapurcu/wmi v1.2.4/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/errs v1.4.0 h1:XNdoD/RRMKP7HD0UhJnIzUy74ISdGGxURlYG8HSWSfM= +github.com/zeebo/errs v1.4.0/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= +github.com/zeebo/xxh3 v1.0.2 h1:xZmwmqxHZA8AI603jOQ0tMqmBr9lPeFwGg6d+xy9DC0= +github.com/zeebo/xxh3 v1.0.2/go.mod h1:5NWz9Sef7zIDm2JHfFlcQvNekmcEl9ekUZQQKCYaDcA= +go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/etcd/api/v3 v3.5.21 h1:A6O2/JDb3tvHhiIz3xf9nJ7REHvtEFJJ3veW3FbCnS8= +go.etcd.io/etcd/api/v3 v3.5.21/go.mod h1:c3aH5wcvXv/9dqIw2Y810LDXJfhSYdHQ0vxmP3CCHVY= +go.etcd.io/etcd/client/pkg/v3 v3.5.21 h1:lPBu71Y7osQmzlflM9OfeIV2JlmpBjqBNlLtcoBqUTc= +go.etcd.io/etcd/client/pkg/v3 v3.5.21/go.mod h1:BgqT/IXPjK9NkeSDjbzwsHySX3yIle2+ndz28nVsjUs= +go.etcd.io/etcd/client/v3 v3.5.21 h1:T6b1Ow6fNjOLOtM0xSoKNQt1ASPCLWrF9XMHcH9pEyY= +go.etcd.io/etcd/client/v3 v3.5.21/go.mod h1:mFYy67IOqmbRf/kRUvsHixzo3iG+1OF2W2+jVIQRAnU= +go.mongodb.org/mongo-driver v1.17.3 h1:TQyXhnsWfWtgAhMtOgtYHMTkZIfBTpMTsMnd9ZBeHxQ= +go.mongodb.org/mongo-driver v1.17.3/go.mod h1:Hy04i7O2kC4RS06ZrhPRqj/u4DTYkFDAAccj+rVKqgQ= +go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA= +go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A= +go.opentelemetry.io/contrib/detectors/gcp v1.35.0 h1:bGvFt68+KTiAKFlacHW6AhA56GF2rS0bdD3aJYEnmzA= +go.opentelemetry.io/contrib/detectors/gcp v1.35.0/go.mod h1:qGWP8/+ILwMRIUf9uIVLloR1uo5ZYAslM4O6OqUi1DA= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0 h1:x7wzEgXfnzJcHDwStJT+mxOz4etr2EcexjqhBvmoakw= +go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.60.0/go.mod h1:rg+RlpR5dKwaS95IyyZqj5Wd4E13lk/msnTS0Xl9lJM= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0 h1:sbiXRNDSWJOTobXh5HyQKjq6wUC5tNybqjIqDpAY4CU= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.60.0/go.mod h1:69uWxva0WgAA/4bu2Yy70SLDBwZXuQ6PbBpbsa5iZrQ= +go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= +go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0 h1:PB3Zrjs1sG1GBX51SXyTSoOTqcDglmsk7nT6tkKPb/k= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.35.0/go.mod h1:U2R3XyVPzn0WX7wOIypPuptulsMcPDPs/oiSVOMVnHY= +go.opentelemetry.io/otel/metric v1.35.0 h1:0znxYu2SNyuMSQT4Y9WDWej0VpcsxkuklLa4/siN90M= +go.opentelemetry.io/otel/metric v1.35.0/go.mod h1:nKVFgxBZ2fReX6IlyW28MgZojkoAkJGaE8CpgeAU3oE= +go.opentelemetry.io/otel/sdk v1.35.0 h1:iPctf8iprVySXSKJffSS79eOjl9pvxV9ZqOWT0QejKY= +go.opentelemetry.io/otel/sdk v1.35.0/go.mod h1:+ga1bZliga3DxJ3CQGg3updiaAJoNECOgJREo9KHGQg= +go.opentelemetry.io/otel/sdk/metric v1.35.0 h1:1RriWBmCKgkeHEhM7a2uMjMUfP7MsOF5JpUCaEqEI9o= +go.opentelemetry.io/otel/sdk/metric v1.35.0/go.mod h1:is6XYCUMpcKi+ZsOvfluY5YstFnhW0BidkR+gL+qN+w= +go.opentelemetry.io/otel/trace v1.35.0 h1:dPpEfJu1sDIqruz7BHFG3c7528f6ddfSWfFDVt/xgMs= +go.opentelemetry.io/otel/trace v1.35.0/go.mod h1:WUk7DtFp1Aw2MkvqGdwiXYDZZNvA/1J8o6xRXLrIkyc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= +go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= +goftp.io/server/v2 v2.0.1 h1:H+9UbCX2N206ePDSVNCjBftOKOgil6kQ5RAQNx5hJwE= +goftp.io/server/v2 v2.0.1/go.mod h1:7+H/EIq7tXdfo1Muu5p+l3oQ6rYkDZ8lY7IM5d5kVdQ= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210314154223-e6e6c4f2bb5b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU= +golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.37.0 h1:kJNSjF/Xp7kU0iB2Z+9viTPMW4EqqsrywMXLJOOsXSE= +golang.org/x/crypto v0.37.0/go.mod h1:vg+k43peMZ0pUMhYmVAWysMK35e6ioLh3wB8ZCAfbVc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.15.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.17.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.24.0 h1:ZfthKaKaT4NrhGVZHO1/WDTwGES4De8KtWO0SIbNJMU= +golang.org/x/mod v0.24.0/go.mod h1:IXM97Txy2VM4PJ3gI61r1YEk/gAj6zAHN3AdZt6S9Ww= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM= +golang.org/x/net v0.39.0 h1:ZCu7HMWDxpXpaiKdhzIfaltL9Lp31x/3fCP11bc6/fY= +golang.org/x/net v0.39.0/go.mod h1:X7NRbYVEA+ewNkCNyJ513WmMdQ3BineSwVtN2zD/d+E= +golang.org/x/oauth2 v0.29.0 h1:WdYw2tdTK1S8olAzWHdgeqfy+Mtm9XNhv/xJsY65d98= +golang.org/x/oauth2 v0.29.0/go.mod h1:onh5ek6nERTohokkhCD/y2cV4Do3fxFHFuAejCkRWT8= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.13.0 h1:AauUjRAJ9OSnvULf/ARrrVywoJDy0YS2AwQ98I37610= +golang.org/x/sync v0.13.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA= +golang.org/x/sys v0.0.0-20180926160741-c2ed4eda69e7/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190130150945-aca44879d564/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210228012217-479acdf4ea46/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220111092808-5a964db01320/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220408201424-a24fb2fb8a0f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.32.0 h1:s77OFDvIQeibCmezSnk/q6iAfkdiQaJi4VzroCFrN20= +golang.org/x/sys v0.32.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k= +golang.org/x/telemetry v0.0.0-20240228155512-f48c80bd79b2/go.mod h1:TeRTkGYfJXctD9OcfyVLyj2J3IxLnKwHJR8f4D8a3YE= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/term v0.31.0 h1:erwDkOK1Msy6offm1mOgvspSkslFnIGsFnxOKoufg3o= +golang.org/x/term v0.31.0/go.mod h1:R4BeIy7D95HzImkxGkTW1UQTtP54tio2RyHz7PwK0aw= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= +golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.11.0 h1:/bpjEDfN9tkoN/ryeYHnv5hcMlc8ncjMcM4XBk5NWV0= +golang.org/x/time v0.11.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190424220101-1e8e1cfdf96b/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.32.0 h1:Q7N1vhpkQv7ybVzLFtTjvQya2ewbwNDZzUgfXGqtMWU= +golang.org/x/tools v0.32.0/go.mod h1:ZxrU41P/wAbZD8EDa6dDCa6XfpkhJ7HFMjHJXfBDu8s= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.230.0 h1:2u1hni3E+UXAXrONrrkfWpi/V6cyKVAbfGVeGtC3OxM= +google.golang.org/api v0.230.0/go.mod h1:aqvtoMk7YkiXx+6U12arQFExiRV9D/ekvMCwCd/TksQ= +google.golang.org/genproto v0.0.0-20250422160041-2d3770c4ea7f h1:iZiXS7qm4saaCcdK7S/i1Qx9ZHO2oa16HQqwYc1tPKY= +google.golang.org/genproto v0.0.0-20250422160041-2d3770c4ea7f/go.mod h1:Cej/8iHf9mPl71o/a+R1rrvSFrAAVCUFX9s/sbNttBc= +google.golang.org/genproto/googleapis/api v0.0.0-20250422160041-2d3770c4ea7f h1:tjZsroqekhC63+WMqzmWyW5Twj/ZfR5HAlpd5YQ1Vs0= +google.golang.org/genproto/googleapis/api v0.0.0-20250422160041-2d3770c4ea7f/go.mod h1:Cd8IzgPo5Akum2c9R6FsXNaZbH3Jpa2gpHlW89FqlyQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250422160041-2d3770c4ea7f h1:N/PrbTw4kdkqNRzVfWPrBekzLuarFREcbFOiOLkXon4= +google.golang.org/genproto/googleapis/rpc v0.0.0-20250422160041-2d3770c4ea7f/go.mod h1:qQ0YXyHHx3XkvlzUtpXDkS29lDSafHMZBAZDc03LQ3A= +google.golang.org/grpc v1.72.0 h1:S7UkcVa60b5AAQTaO6ZKamFp1zMZSU0fGDK2WZLbBnM= +google.golang.org/grpc v1.72.0/go.mod h1:wH5Aktxcg25y1I3w7H69nHfXdOG3UiadoBtjh3izSDM= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= +google.golang.org/protobuf v1.36.6/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/urfave/cli.v1 v1.20.0/go.mod h1:vuBzUtMdQeixQj8LVd+/98pzhxNGQoyuPBlsXHOQNO0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/helm-reindex.sh b/helm-reindex.sh new file mode 100755 index 00000000..42453366 --- /dev/null +++ b/helm-reindex.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +helm package helm/minio -d helm-releases/ + +helm repo index --merge index.yaml --url https://charts.min.io . diff --git a/helm-releases/minio-1.0.0.tgz b/helm-releases/minio-1.0.0.tgz new file mode 100644 index 00000000..f88132a6 Binary files /dev/null and b/helm-releases/minio-1.0.0.tgz differ diff --git a/helm-releases/minio-1.0.1.tgz b/helm-releases/minio-1.0.1.tgz new file mode 100644 index 00000000..04ad8e0f Binary files /dev/null and b/helm-releases/minio-1.0.1.tgz differ diff --git a/helm-releases/minio-1.0.2.tgz b/helm-releases/minio-1.0.2.tgz new file mode 100644 index 00000000..c629174d Binary files /dev/null and b/helm-releases/minio-1.0.2.tgz differ diff --git a/helm-releases/minio-1.0.3.tgz b/helm-releases/minio-1.0.3.tgz new file mode 100644 index 00000000..bac16002 Binary files /dev/null and b/helm-releases/minio-1.0.3.tgz differ diff --git a/helm-releases/minio-1.0.4.tgz b/helm-releases/minio-1.0.4.tgz new file mode 100644 index 00000000..4cbbc13b Binary files /dev/null and b/helm-releases/minio-1.0.4.tgz differ diff --git a/helm-releases/minio-1.0.5.tgz b/helm-releases/minio-1.0.5.tgz new file mode 100644 index 00000000..b0c6af74 Binary files /dev/null and b/helm-releases/minio-1.0.5.tgz differ diff --git a/helm-releases/minio-2.0.0.tgz b/helm-releases/minio-2.0.0.tgz new file mode 100644 index 00000000..fcf5e7a0 Binary files /dev/null and b/helm-releases/minio-2.0.0.tgz differ diff --git a/helm-releases/minio-2.0.1.tgz b/helm-releases/minio-2.0.1.tgz new file mode 100644 index 00000000..7b5d106d Binary files /dev/null and b/helm-releases/minio-2.0.1.tgz differ diff --git a/helm-releases/minio-3.0.0.tgz b/helm-releases/minio-3.0.0.tgz new file mode 100644 index 00000000..4722915b Binary files /dev/null and b/helm-releases/minio-3.0.0.tgz differ diff --git a/helm-releases/minio-3.0.1.tgz b/helm-releases/minio-3.0.1.tgz new file mode 100644 index 00000000..27ab069a Binary files /dev/null and b/helm-releases/minio-3.0.1.tgz differ diff --git a/helm-releases/minio-3.0.2.tgz b/helm-releases/minio-3.0.2.tgz new file mode 100644 index 00000000..c37f86d5 Binary files /dev/null and b/helm-releases/minio-3.0.2.tgz differ diff --git a/helm-releases/minio-3.1.0.tgz b/helm-releases/minio-3.1.0.tgz new file mode 100644 index 00000000..6d8941de Binary files /dev/null and b/helm-releases/minio-3.1.0.tgz differ diff --git a/helm-releases/minio-3.1.1.tgz b/helm-releases/minio-3.1.1.tgz new file mode 100644 index 00000000..1b3c49e0 Binary files /dev/null and b/helm-releases/minio-3.1.1.tgz differ diff --git a/helm-releases/minio-3.1.2.tgz b/helm-releases/minio-3.1.2.tgz new file mode 100644 index 00000000..269df4ec Binary files /dev/null and b/helm-releases/minio-3.1.2.tgz differ diff --git a/helm-releases/minio-3.1.3.tgz b/helm-releases/minio-3.1.3.tgz new file mode 100644 index 00000000..e6fde51f Binary files /dev/null and b/helm-releases/minio-3.1.3.tgz differ diff --git a/helm-releases/minio-3.1.4.tgz b/helm-releases/minio-3.1.4.tgz new file mode 100644 index 00000000..186ff7b6 Binary files /dev/null and b/helm-releases/minio-3.1.4.tgz differ diff --git a/helm-releases/minio-3.1.5.tgz b/helm-releases/minio-3.1.5.tgz new file mode 100644 index 00000000..e3af75fd Binary files /dev/null and b/helm-releases/minio-3.1.5.tgz differ diff --git a/helm-releases/minio-3.1.6.tgz b/helm-releases/minio-3.1.6.tgz new file mode 100644 index 00000000..d136d617 Binary files /dev/null and b/helm-releases/minio-3.1.6.tgz differ diff --git a/helm-releases/minio-3.1.7.tgz b/helm-releases/minio-3.1.7.tgz new file mode 100644 index 00000000..0344582d Binary files /dev/null and b/helm-releases/minio-3.1.7.tgz differ diff --git a/helm-releases/minio-3.1.8.tgz b/helm-releases/minio-3.1.8.tgz new file mode 100644 index 00000000..58d127a7 Binary files /dev/null and b/helm-releases/minio-3.1.8.tgz differ diff --git a/helm-releases/minio-3.1.9.tgz b/helm-releases/minio-3.1.9.tgz new file mode 100644 index 00000000..d7926051 Binary files /dev/null and b/helm-releases/minio-3.1.9.tgz differ diff --git a/helm-releases/minio-3.2.0.tgz b/helm-releases/minio-3.2.0.tgz new file mode 100644 index 00000000..cc311916 Binary files /dev/null and b/helm-releases/minio-3.2.0.tgz differ diff --git a/helm-releases/minio-3.3.0.tgz b/helm-releases/minio-3.3.0.tgz new file mode 100644 index 00000000..34db95c8 Binary files /dev/null and b/helm-releases/minio-3.3.0.tgz differ diff --git a/helm-releases/minio-3.3.1.tgz b/helm-releases/minio-3.3.1.tgz new file mode 100644 index 00000000..b4602bb9 Binary files /dev/null and b/helm-releases/minio-3.3.1.tgz differ diff --git a/helm-releases/minio-3.3.2.tgz b/helm-releases/minio-3.3.2.tgz new file mode 100644 index 00000000..aa8389bc Binary files /dev/null and b/helm-releases/minio-3.3.2.tgz differ diff --git a/helm-releases/minio-3.3.3.tgz b/helm-releases/minio-3.3.3.tgz new file mode 100644 index 00000000..ebcab437 Binary files /dev/null and b/helm-releases/minio-3.3.3.tgz differ diff --git a/helm-releases/minio-3.3.4.tgz b/helm-releases/minio-3.3.4.tgz new file mode 100644 index 00000000..009557d6 Binary files /dev/null and b/helm-releases/minio-3.3.4.tgz differ diff --git a/helm-releases/minio-3.4.0.tgz b/helm-releases/minio-3.4.0.tgz new file mode 100644 index 00000000..0520d55e Binary files /dev/null and b/helm-releases/minio-3.4.0.tgz differ diff --git a/helm-releases/minio-3.4.1.tgz b/helm-releases/minio-3.4.1.tgz new file mode 100644 index 00000000..18dff864 Binary files /dev/null and b/helm-releases/minio-3.4.1.tgz differ diff --git a/helm-releases/minio-3.4.2.tgz b/helm-releases/minio-3.4.2.tgz new file mode 100644 index 00000000..496b2bec Binary files /dev/null and b/helm-releases/minio-3.4.2.tgz differ diff --git a/helm-releases/minio-3.4.3.tgz b/helm-releases/minio-3.4.3.tgz new file mode 100644 index 00000000..bd120f50 Binary files /dev/null and b/helm-releases/minio-3.4.3.tgz differ diff --git a/helm-releases/minio-3.4.4.tgz b/helm-releases/minio-3.4.4.tgz new file mode 100644 index 00000000..7a4d9462 Binary files /dev/null and b/helm-releases/minio-3.4.4.tgz differ diff --git a/helm-releases/minio-3.4.5.tgz b/helm-releases/minio-3.4.5.tgz new file mode 100644 index 00000000..7d5a8126 Binary files /dev/null and b/helm-releases/minio-3.4.5.tgz differ diff --git a/helm-releases/minio-3.4.6.tgz b/helm-releases/minio-3.4.6.tgz new file mode 100644 index 00000000..4c8179c1 Binary files /dev/null and b/helm-releases/minio-3.4.6.tgz differ diff --git a/helm-releases/minio-3.4.7.tgz b/helm-releases/minio-3.4.7.tgz new file mode 100644 index 00000000..a33e8791 Binary files /dev/null and b/helm-releases/minio-3.4.7.tgz differ diff --git a/helm-releases/minio-3.4.8.tgz b/helm-releases/minio-3.4.8.tgz new file mode 100644 index 00000000..e3156c67 Binary files /dev/null and b/helm-releases/minio-3.4.8.tgz differ diff --git a/helm-releases/minio-3.5.0.tgz b/helm-releases/minio-3.5.0.tgz new file mode 100644 index 00000000..b606d705 Binary files /dev/null and b/helm-releases/minio-3.5.0.tgz differ diff --git a/helm-releases/minio-3.5.1.tgz b/helm-releases/minio-3.5.1.tgz new file mode 100644 index 00000000..dda1c6b1 Binary files /dev/null and b/helm-releases/minio-3.5.1.tgz differ diff --git a/helm-releases/minio-3.5.2.tgz b/helm-releases/minio-3.5.2.tgz new file mode 100644 index 00000000..502fa30c Binary files /dev/null and b/helm-releases/minio-3.5.2.tgz differ diff --git a/helm-releases/minio-3.5.3.tgz b/helm-releases/minio-3.5.3.tgz new file mode 100644 index 00000000..e0ead3c1 Binary files /dev/null and b/helm-releases/minio-3.5.3.tgz differ diff --git a/helm-releases/minio-3.5.4.tgz b/helm-releases/minio-3.5.4.tgz new file mode 100644 index 00000000..214e705e Binary files /dev/null and b/helm-releases/minio-3.5.4.tgz differ diff --git a/helm-releases/minio-3.5.5.tgz b/helm-releases/minio-3.5.5.tgz new file mode 100644 index 00000000..35f97126 Binary files /dev/null and b/helm-releases/minio-3.5.5.tgz differ diff --git a/helm-releases/minio-3.5.6.tgz b/helm-releases/minio-3.5.6.tgz new file mode 100644 index 00000000..970de3ea Binary files /dev/null and b/helm-releases/minio-3.5.6.tgz differ diff --git a/helm-releases/minio-3.5.7.tgz b/helm-releases/minio-3.5.7.tgz new file mode 100644 index 00000000..05a48404 Binary files /dev/null and b/helm-releases/minio-3.5.7.tgz differ diff --git a/helm-releases/minio-3.5.8.tgz b/helm-releases/minio-3.5.8.tgz new file mode 100644 index 00000000..ec8c7ae9 Binary files /dev/null and b/helm-releases/minio-3.5.8.tgz differ diff --git a/helm-releases/minio-3.5.9.tgz b/helm-releases/minio-3.5.9.tgz new file mode 100644 index 00000000..36838a1a Binary files /dev/null and b/helm-releases/minio-3.5.9.tgz differ diff --git a/helm-releases/minio-3.6.0.tgz b/helm-releases/minio-3.6.0.tgz new file mode 100644 index 00000000..9eb9e69c Binary files /dev/null and b/helm-releases/minio-3.6.0.tgz differ diff --git a/helm-releases/minio-3.6.1.tgz b/helm-releases/minio-3.6.1.tgz new file mode 100644 index 00000000..fca22214 Binary files /dev/null and b/helm-releases/minio-3.6.1.tgz differ diff --git a/helm-releases/minio-3.6.2.tgz b/helm-releases/minio-3.6.2.tgz new file mode 100644 index 00000000..be850146 Binary files /dev/null and b/helm-releases/minio-3.6.2.tgz differ diff --git a/helm-releases/minio-3.6.3.tgz b/helm-releases/minio-3.6.3.tgz new file mode 100644 index 00000000..9829d32c Binary files /dev/null and b/helm-releases/minio-3.6.3.tgz differ diff --git a/helm-releases/minio-3.6.4.tgz b/helm-releases/minio-3.6.4.tgz new file mode 100644 index 00000000..f966f646 Binary files /dev/null and b/helm-releases/minio-3.6.4.tgz differ diff --git a/helm-releases/minio-3.6.5.tgz b/helm-releases/minio-3.6.5.tgz new file mode 100644 index 00000000..d54d40c4 Binary files /dev/null and b/helm-releases/minio-3.6.5.tgz differ diff --git a/helm-releases/minio-3.6.6.tgz b/helm-releases/minio-3.6.6.tgz new file mode 100644 index 00000000..9e04eb3d Binary files /dev/null and b/helm-releases/minio-3.6.6.tgz differ diff --git a/helm-releases/minio-4.0.0.tgz b/helm-releases/minio-4.0.0.tgz new file mode 100644 index 00000000..f231aaf6 Binary files /dev/null and b/helm-releases/minio-4.0.0.tgz differ diff --git a/helm-releases/minio-4.0.1.tgz b/helm-releases/minio-4.0.1.tgz new file mode 100644 index 00000000..3da38fb2 Binary files /dev/null and b/helm-releases/minio-4.0.1.tgz differ diff --git a/helm-releases/minio-4.0.10.tgz b/helm-releases/minio-4.0.10.tgz new file mode 100644 index 00000000..0ca1a8db Binary files /dev/null and b/helm-releases/minio-4.0.10.tgz differ diff --git a/helm-releases/minio-4.0.11.tgz b/helm-releases/minio-4.0.11.tgz new file mode 100644 index 00000000..dec7a7c5 Binary files /dev/null and b/helm-releases/minio-4.0.11.tgz differ diff --git a/helm-releases/minio-4.0.12.tgz b/helm-releases/minio-4.0.12.tgz new file mode 100644 index 00000000..7ab04b43 Binary files /dev/null and b/helm-releases/minio-4.0.12.tgz differ diff --git a/helm-releases/minio-4.0.13.tgz b/helm-releases/minio-4.0.13.tgz new file mode 100644 index 00000000..61276997 Binary files /dev/null and b/helm-releases/minio-4.0.13.tgz differ diff --git a/helm-releases/minio-4.0.14.tgz b/helm-releases/minio-4.0.14.tgz new file mode 100644 index 00000000..10faaba5 Binary files /dev/null and b/helm-releases/minio-4.0.14.tgz differ diff --git a/helm-releases/minio-4.0.15.tgz b/helm-releases/minio-4.0.15.tgz new file mode 100644 index 00000000..985d0bba Binary files /dev/null and b/helm-releases/minio-4.0.15.tgz differ diff --git a/helm-releases/minio-4.0.2.tgz b/helm-releases/minio-4.0.2.tgz new file mode 100644 index 00000000..bb5a205e Binary files /dev/null and b/helm-releases/minio-4.0.2.tgz differ diff --git a/helm-releases/minio-4.0.3.tgz b/helm-releases/minio-4.0.3.tgz new file mode 100644 index 00000000..cc57cc05 Binary files /dev/null and b/helm-releases/minio-4.0.3.tgz differ diff --git a/helm-releases/minio-4.0.4.tgz b/helm-releases/minio-4.0.4.tgz new file mode 100644 index 00000000..35f8c2c6 Binary files /dev/null and b/helm-releases/minio-4.0.4.tgz differ diff --git a/helm-releases/minio-4.0.5.tgz b/helm-releases/minio-4.0.5.tgz new file mode 100644 index 00000000..35ea5112 Binary files /dev/null and b/helm-releases/minio-4.0.5.tgz differ diff --git a/helm-releases/minio-4.0.6.tgz b/helm-releases/minio-4.0.6.tgz new file mode 100644 index 00000000..a6152b6d Binary files /dev/null and b/helm-releases/minio-4.0.6.tgz differ diff --git a/helm-releases/minio-4.0.7.tgz b/helm-releases/minio-4.0.7.tgz new file mode 100644 index 00000000..10b221c8 Binary files /dev/null and b/helm-releases/minio-4.0.7.tgz differ diff --git a/helm-releases/minio-4.0.8.tgz b/helm-releases/minio-4.0.8.tgz new file mode 100644 index 00000000..ca11a590 Binary files /dev/null and b/helm-releases/minio-4.0.8.tgz differ diff --git a/helm-releases/minio-4.0.9.tgz b/helm-releases/minio-4.0.9.tgz new file mode 100644 index 00000000..d28cc762 Binary files /dev/null and b/helm-releases/minio-4.0.9.tgz differ diff --git a/helm-releases/minio-4.1.0.tgz b/helm-releases/minio-4.1.0.tgz new file mode 100644 index 00000000..0911284b Binary files /dev/null and b/helm-releases/minio-4.1.0.tgz differ diff --git a/helm-releases/minio-5.0.0.tgz b/helm-releases/minio-5.0.0.tgz new file mode 100644 index 00000000..5e915a57 Binary files /dev/null and b/helm-releases/minio-5.0.0.tgz differ diff --git a/helm-releases/minio-5.0.1.tgz b/helm-releases/minio-5.0.1.tgz new file mode 100644 index 00000000..fc6ba101 Binary files /dev/null and b/helm-releases/minio-5.0.1.tgz differ diff --git a/helm-releases/minio-5.0.10.tgz b/helm-releases/minio-5.0.10.tgz new file mode 100644 index 00000000..1453f635 Binary files /dev/null and b/helm-releases/minio-5.0.10.tgz differ diff --git a/helm-releases/minio-5.0.11.tgz b/helm-releases/minio-5.0.11.tgz new file mode 100644 index 00000000..1f7dc55d Binary files /dev/null and b/helm-releases/minio-5.0.11.tgz differ diff --git a/helm-releases/minio-5.0.12.tgz b/helm-releases/minio-5.0.12.tgz new file mode 100644 index 00000000..f9f16d2c Binary files /dev/null and b/helm-releases/minio-5.0.12.tgz differ diff --git a/helm-releases/minio-5.0.13.tgz b/helm-releases/minio-5.0.13.tgz new file mode 100644 index 00000000..41ae1179 Binary files /dev/null and b/helm-releases/minio-5.0.13.tgz differ diff --git a/helm-releases/minio-5.0.14.tgz b/helm-releases/minio-5.0.14.tgz new file mode 100644 index 00000000..67cd85aa Binary files /dev/null and b/helm-releases/minio-5.0.14.tgz differ diff --git a/helm-releases/minio-5.0.15.tgz b/helm-releases/minio-5.0.15.tgz new file mode 100644 index 00000000..ee5e9373 Binary files /dev/null and b/helm-releases/minio-5.0.15.tgz differ diff --git a/helm-releases/minio-5.0.2.tgz b/helm-releases/minio-5.0.2.tgz new file mode 100644 index 00000000..26c15289 Binary files /dev/null and b/helm-releases/minio-5.0.2.tgz differ diff --git a/helm-releases/minio-5.0.3.tgz b/helm-releases/minio-5.0.3.tgz new file mode 100644 index 00000000..e0d5419c Binary files /dev/null and b/helm-releases/minio-5.0.3.tgz differ diff --git a/helm-releases/minio-5.0.4.tgz b/helm-releases/minio-5.0.4.tgz new file mode 100644 index 00000000..bc8a77c8 Binary files /dev/null and b/helm-releases/minio-5.0.4.tgz differ diff --git a/helm-releases/minio-5.0.5.tgz b/helm-releases/minio-5.0.5.tgz new file mode 100644 index 00000000..d286ae2c Binary files /dev/null and b/helm-releases/minio-5.0.5.tgz differ diff --git a/helm-releases/minio-5.0.6.tgz b/helm-releases/minio-5.0.6.tgz new file mode 100644 index 00000000..359d2acf Binary files /dev/null and b/helm-releases/minio-5.0.6.tgz differ diff --git a/helm-releases/minio-5.0.7.tgz b/helm-releases/minio-5.0.7.tgz new file mode 100644 index 00000000..b4d81167 Binary files /dev/null and b/helm-releases/minio-5.0.7.tgz differ diff --git a/helm-releases/minio-5.0.8.tgz b/helm-releases/minio-5.0.8.tgz new file mode 100644 index 00000000..02204b7f Binary files /dev/null and b/helm-releases/minio-5.0.8.tgz differ diff --git a/helm-releases/minio-5.0.9.tgz b/helm-releases/minio-5.0.9.tgz new file mode 100644 index 00000000..1e33e5f0 Binary files /dev/null and b/helm-releases/minio-5.0.9.tgz differ diff --git a/helm-releases/minio-5.1.0.tgz b/helm-releases/minio-5.1.0.tgz new file mode 100644 index 00000000..97fe39d3 Binary files /dev/null and b/helm-releases/minio-5.1.0.tgz differ diff --git a/helm-releases/minio-5.2.0.tgz b/helm-releases/minio-5.2.0.tgz new file mode 100644 index 00000000..92d60a60 Binary files /dev/null and b/helm-releases/minio-5.2.0.tgz differ diff --git a/helm-releases/minio-5.3.0.tgz b/helm-releases/minio-5.3.0.tgz new file mode 100644 index 00000000..cac2baa5 Binary files /dev/null and b/helm-releases/minio-5.3.0.tgz differ diff --git a/helm-releases/minio-5.4.0.tgz b/helm-releases/minio-5.4.0.tgz new file mode 100644 index 00000000..22f8d733 Binary files /dev/null and b/helm-releases/minio-5.4.0.tgz differ diff --git a/helm/minio/.helmignore b/helm/minio/.helmignore new file mode 100644 index 00000000..a9fe7278 --- /dev/null +++ b/helm/minio/.helmignore @@ -0,0 +1,23 @@ +# Patterns to ignore when building packages. +# This supports shell glob matching, relative path matching, and +# negation (prefixed with !). Only one pattern per line. +.DS_Store +# Common VCS dirs +.git/ +.gitignore +.bzr/ +.bzrignore +.hg/ +.hgignore +.svn/ +# Common backup files +*.swp +*.bak +*.tmp +*~ +# Various IDEs +.project +.idea/ +*.tmproj +# OWNERS file for Kubernetes +OWNERS \ No newline at end of file diff --git a/helm/minio/Chart.yaml b/helm/minio/Chart.yaml new file mode 100644 index 00000000..6ae6f116 --- /dev/null +++ b/helm/minio/Chart.yaml @@ -0,0 +1,18 @@ +apiVersion: v1 +description: High Performance Object Storage +name: minio +version: 5.4.0 +appVersion: RELEASE.2024-12-18T13-15-44Z +keywords: + - minio + - storage + - object-storage + - s3 + - cluster +home: https://min.io +icon: https://min.io/resources/img/logo/MINIO_wordmark.png +sources: +- https://github.com/minio/minio +maintainers: +- name: MinIO, Inc + email: dev@minio.io diff --git a/helm/minio/README.md b/helm/minio/README.md new file mode 100644 index 00000000..00d1e45d --- /dev/null +++ b/helm/minio/README.md @@ -0,0 +1,264 @@ +# MinIO Community Helm Chart + +[![Slack](https://slack.min.io/slack?type=svg)](https://slack.min.io) [![license](https://img.shields.io/badge/license-AGPL%20V3-blue)](https://github.com/minio/minio/blob/master/LICENSE) + +MinIO is a High Performance Object Storage released under GNU Affero General Public License v3.0. It is API compatible with Amazon S3 cloud storage service. Use MinIO to build high performance infrastructure for machine learning, analytics and application data workloads. + +| IMPORTANT | +| -------------------------- | +| This Helm chart is community built, maintained, and supported. MinIO does not guarantee support for any given bug, feature request, or update referencing this chart.

MinIO publishes a separate [MinIO Kubernetes Operator and Tenant Helm Chart](https://github.com/minio/operator/tree/master/helm) that is officially maintained and supported. MinIO strongly recommends using the MinIO Kubernetes Operator for production deployments. See [Deploy Operator With Helm](https://docs.min.io/community/minio-object-store/operations/deployments/k8s-deploy-operator-helm-on-kubernetes.html?ref=github) for additional documentation. | + +## Introduction + +This chart bootstraps MinIO Cluster on [Kubernetes](http://kubernetes.io) using the [Helm](https://helm.sh) package manager. + +## Prerequisites + +- Helm cli with Kubernetes cluster configured. +- PV provisioner support in the underlying infrastructure. (We recommend using ) +- Use Kubernetes version v1.19 and later for best experience. + +## Configure MinIO Helm repo + +```bash +helm repo add minio https://charts.min.io/ +``` + +### Installing the Chart + +Install this chart using: + +```bash +helm install --namespace minio --set rootUser=rootuser,rootPassword=rootpass123 --generate-name minio/minio +``` + +The command deploys MinIO on the Kubernetes cluster in the default configuration. The [configuration](#configuration) section lists the parameters that can be configured during installation. + +### Installing the Chart (toy-setup) + +Minimal toy setup for testing purposes can be deployed using: + +```bash +helm install --set resources.requests.memory=512Mi --set replicas=1 --set persistence.enabled=false --set mode=standalone --set rootUser=rootuser,rootPassword=rootpass123 --generate-name minio/minio +``` + +### Upgrading the Chart + +You can use Helm to update MinIO version in a live release. Assuming your release is named as `my-release`, get the values using the command: + +```bash +helm get values my-release > old_values.yaml +``` + +Then change the field `image.tag` in `old_values.yaml` file with MinIO image tag you want to use. Now update the chart using + +```bash +helm upgrade -f old_values.yaml my-release minio/minio +``` + +Default upgrade strategies are specified in the `values.yaml` file. Update these fields if you'd like to use a different strategy. + +### Configuration + +Refer the [Values file](./values.yaml) for all the possible config fields. + +You can specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, + +```bash +helm install --name my-release --set persistence.size=1Ti minio/minio +``` + +The above command deploys MinIO server with a 1Ti backing persistent volume. + +Alternately, you can provide a YAML file that specifies parameter values while installing the chart. For example, + +```bash +helm install --name my-release -f values.yaml minio/minio +``` + +### Persistence + +This chart provisions a PersistentVolumeClaim and mounts corresponding persistent volume to default location `/export`. You'll need physical storage available in the Kubernetes cluster for this to work. If you'd rather use `emptyDir`, disable PersistentVolumeClaim by: + +```bash +helm install --set persistence.enabled=false minio/minio +``` + +> *"An emptyDir volume is first created when a Pod is assigned to a Node, and exists as long as that Pod is running on that node. When a Pod is removed from a node for any reason, the data in the emptyDir is deleted forever."* + +### Existing PersistentVolumeClaim + +If a Persistent Volume Claim already exists, specify it during installation. + +1. Create the PersistentVolume +2. Create the PersistentVolumeClaim +3. Install the chart + +```bash +helm install --set persistence.existingClaim=PVC_NAME minio/minio +``` + +### NetworkPolicy + +To enable network policy for MinIO, +install [a networking plugin that implements the Kubernetes +NetworkPolicy spec](https://kubernetes.io/docs/tasks/administer-cluster/declare-network-policy#before-you-begin), +and set `networkPolicy.enabled` to `true`. + +For Kubernetes v1.5 & v1.6, you must also turn on NetworkPolicy by setting +the DefaultDeny namespace annotation. Note: this will enforce policy for *all* pods in the namespace: + +``` +kubectl annotate namespace default "net.beta.kubernetes.io/network-policy={\"ingress\":{\"isolation\":\"DefaultDeny\"}}" +``` + +When using `Cilium` as a CNI in your cluster, please edit the `flavor` field to `cilium`. + +With NetworkPolicy enabled, traffic will be limited to just port 9000. + +For more precise policy, set `networkPolicy.allowExternal=true`. This will +only allow pods with the generated client label to connect to MinIO. +This label will be displayed in the output of a successful install. + +### Existing secret + +Instead of having this chart create the secret for you, you can supply a preexisting secret, much +like an existing PersistentVolumeClaim. + +First, create the secret: + +```bash +kubectl create secret generic my-minio-secret --from-literal=rootUser=foobarbaz --from-literal=rootPassword=foobarbazqux +``` + +Then install the chart, specifying that you want to use an existing secret: + +```bash +helm install --set existingSecret=my-minio-secret minio/minio +``` + +The following fields are expected in the secret: + +| .data.\ in Secret | Corresponding variable | Description | Required | +|:------------------------|:-----------------------|:---------------|:---------| +| `rootUser` | `rootUser` | Root user. | yes | +| `rootPassword` | `rootPassword` | Root password. | yes | + +All corresponding variables will be ignored in values file. + +### Configure TLS + +To enable TLS for MinIO containers, acquire TLS certificates from a CA or create self-signed certificates. While creating / acquiring certificates ensure the corresponding domain names are set as per the standard [DNS naming conventions](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#pod-identity) in a Kubernetes StatefulSet (for a distributed MinIO setup). Then create a secret using + +```bash +kubectl create secret generic tls-ssl-minio --from-file=path/to/private.key --from-file=path/to/public.crt +``` + +Then install the chart, specifying that you want to use the TLS secret: + +```bash +helm install --set tls.enabled=true,tls.certSecret=tls-ssl-minio minio/minio +``` + +### Installing certificates from third party CAs + +MinIO can connect to other servers, including MinIO nodes or other server types such as NATs and Redis. If these servers use certificates that were not registered with a known CA, add trust for these certificates to MinIO Server by bundling these certificates into a Kubernetes secret and providing it to Helm via the `trustedCertsSecret` value. If `.Values.tls.enabled` is `true` and you're installing certificates for third party CAs, remember to include MinIO's own certificate with key `public.crt`, if it also needs to be trusted. + +For instance, given that TLS is enabled and you need to add trust for MinIO's own CA and for the CA of a Keycloak server, a Kubernetes secret can be created from the certificate files using `kubectl`: + +``` +kubectl -n minio create secret generic minio-trusted-certs --from-file=public.crt --from-file=keycloak.crt +``` + +If TLS is not enabled, you would need only the third party CA: + +``` +kubectl -n minio create secret generic minio-trusted-certs --from-file=keycloak.crt +``` + +The name of the generated secret can then be passed to Helm using a values file or the `--set` parameter: + +``` +trustedCertsSecret: "minio-trusted-certs" + +or + +--set trustedCertsSecret=minio-trusted-certs +``` + +### Create buckets after install + +Install the chart, specifying the buckets you want to create after install: + +```bash +helm install --set buckets[0].name=bucket1,buckets[0].policy=none,buckets[0].purge=false minio/minio +``` + +Description of the configuration parameters used above - + +- `buckets[].name` - name of the bucket to create, must be a string with length > 0 +- `buckets[].policy` - can be one of none|download|upload|public +- `buckets[].purge` - purge if bucket exists already + +### Create policies after install + +Install the chart, specifying the policies you want to create after install: + +```bash +helm install --set policies[0].name=mypolicy,policies[0].statements[0].resources[0]='arn:aws:s3:::bucket1',policies[0].statements[0].actions[0]='s3:ListBucket',policies[0].statements[0].actions[1]='s3:GetObject' minio/minio +``` + +Description of the configuration parameters used above - + +- `policies[].name` - name of the policy to create, must be a string with length > 0 +- `policies[].statements[]` - list of statements, includes actions and resources +- `policies[].statements[].resources[]` - list of resources that applies the statement +- `policies[].statements[].actions[]` - list of actions granted + +### Create user after install + +Install the chart, specifying the users you want to create after install: + +```bash +helm install --set users[0].accessKey=accessKey,users[0].secretKey=secretKey,users[0].policy=none,users[1].accessKey=accessKey2,users[1].secretRef=existingSecret,users[1].secretKey=password,users[1].policy=none minio/minio +``` + +Description of the configuration parameters used above - + +- `users[].accessKey` - accessKey of user +- `users[].secretKey` - secretKey of usersecretRef +- `users[].existingSecret` - secret name that contains the secretKey of user +- `users[].existingSecretKey` - data key in existingSecret secret containing the secretKey +- `users[].policy` - name of the policy to assign to user + +### Create service account after install + +Install the chart, specifying the service accounts you want to create after install: + +```bash +helm install --set svcaccts[0].accessKey=accessKey,svcaccts[0].secretKey=secretKey,svcaccts[0].user=parentUser,svcaccts[1].accessKey=accessKey2,svcaccts[1].secretRef=existingSecret,svcaccts[1].secretKey=password,svcaccts[1].user=parentUser2 minio/minio +``` + +Description of the configuration parameters used above - + +- `svcaccts[].accessKey` - accessKey of service account +- `svcaccts[].secretKey` - secretKey of svcacctsecretRef +- `svcaccts[].existingSecret` - secret name that contains the secretKey of service account +- `svcaccts[].existingSecretKey` - data key in existingSecret secret containing the secretKey +- `svcaccts[].user` - name of the parent user to assign to service account + +## Uninstalling the Chart + +Assuming your release is named as `my-release`, delete it using the command: + +```bash +helm delete my-release +``` + +or + +```bash +helm uninstall my-release +``` + +The command removes all the Kubernetes components associated with the chart and deletes the release. diff --git a/helm/minio/templates/NOTES.txt b/helm/minio/templates/NOTES.txt new file mode 100644 index 00000000..73d77e18 --- /dev/null +++ b/helm/minio/templates/NOTES.txt @@ -0,0 +1,43 @@ +{{- if eq .Values.service.type "ClusterIP" "NodePort" }} +MinIO can be accessed via port {{ .Values.service.port }} on the following DNS name from within your cluster: +{{ template "minio.fullname" . }}.{{ .Release.Namespace }}.{{ .Values.clusterDomain }} + +To access MinIO from localhost, run the below commands: + + 1. export POD_NAME=$(kubectl get pods --namespace {{ .Release.Namespace }} -l "release={{ .Release.Name }}" -o jsonpath="{.items[0].metadata.name}") + + 2. kubectl port-forward $POD_NAME 9000 --namespace {{ .Release.Namespace }} + +Read more about port forwarding here: http://kubernetes.io/docs/user-guide/kubectl/kubectl_port-forward/ + +You can now access MinIO server on http://localhost:9000. Follow the below steps to connect to MinIO server with mc client: + + 1. Download the MinIO mc client - https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart + + 2. export MC_HOST_{{ template "minio.fullname" . }}_local=http://$(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "minio.secretName" . }} -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret --namespace {{ .Release.Namespace }} {{ template "minio.secretName" . }} -o jsonpath="{.data.rootPassword}" | base64 --decode)@localhost:{{ .Values.service.port }} + + 3. mc ls {{ template "minio.fullname" . }}_local + +{{- end }} +{{- if eq .Values.service.type "LoadBalancer" }} +MinIO can be accessed via port {{ .Values.service.port }} on an external IP address. Get the service external IP address by: +kubectl get svc --namespace {{ .Release.Namespace }} -l app={{ template "minio.fullname" . }} + +Note that the public IP may take a couple of minutes to be available. + +You can now access MinIO server on http://:9000. Follow the below steps to connect to MinIO server with mc client: + + 1. Download the MinIO mc client - https://docs.min.io/community/minio-object-store/reference/minio-mc.html#quickstart + + 2. export MC_HOST_{{ template "minio.fullname" . }}_local=http://$(kubectl get secret {{ template "minio.secretName" . }} --namespace {{ .Release.Namespace }} -o jsonpath="{.data.rootUser}" | base64 --decode):$(kubectl get secret {{ template "minio.secretName" . }} -o jsonpath="{.data.rootPassword}" | base64 --decode)@:{{ .Values.service.port }} + + 3. mc ls {{ template "minio.fullname" . }} + +Alternately, you can use your browser or the MinIO SDK to access the server - https://docs.min.io/community/minio-object-store/reference/minio-server/minio-server.html +{{- end }} + +{{ if and (.Values.networkPolicy.enabled) (not .Values.networkPolicy.allowExternal) }} +Note: Since NetworkPolicy is enabled, only pods with label +{{ template "minio.fullname" . }}-client=true" +will be able to connect to this minio cluster. +{{- end }} diff --git a/helm/minio/templates/_helper_create_bucket.txt b/helm/minio/templates/_helper_create_bucket.txt new file mode 100644 index 00000000..62b19178 --- /dev/null +++ b/helm/minio/templates/_helper_create_bucket.txt @@ -0,0 +1,123 @@ +#!/bin/sh +set -e # Have script exit in the event of a failed command. + +{{- if .Values.configPathmc }} +MC_CONFIG_DIR="{{ .Values.configPathmc }}" +MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" +{{- else }} +MC="/usr/bin/mc --insecure" +{{- end }} + +# connectToMinio +# Use a check-sleep-check loop to wait for MinIO service to be available +connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 + LIMIT=29 # Allow 30 attempts + set -e # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) + SECRET=$(cat /config/rootPassword) + set +e # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" + $MC_COMMAND + STATUS=$? + until [ $STATUS = 0 ]; do + ATTEMPTS=$(expr $ATTEMPTS + 1) + echo \"Failed attempts: $ATTEMPTS\" + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 + fi + sleep 2 # 1 second intervals between attempts + $MC_COMMAND + STATUS=$? + done + set -e # reset `e` as active + return 0 +} + +# checkBucketExists ($bucket) +# Check if the bucket exists, by using the exit code of `mc ls` +checkBucketExists() { + BUCKET=$1 + CMD=$(${MC} stat myminio/$BUCKET >/dev/null 2>&1) + return $? +} + +# createBucket ($bucket, $policy, $purge) +# Ensure bucket exists, purging if asked to +createBucket() { + BUCKET=$1 + POLICY=$2 + PURGE=$3 + VERSIONING=$4 + OBJECTLOCKING=$5 + + # Purge the bucket, if set & exists + # Since PURGE is user input, check explicitly for `true` + if [ $PURGE = true ]; then + if checkBucketExists $BUCKET; then + echo "Purging bucket '$BUCKET'." + set +e # don't exit if this fails + ${MC} rm -r --force myminio/$BUCKET + set -e # reset `e` as active + else + echo "Bucket '$BUCKET' does not exist, skipping purge." + fi + fi + + # Create the bucket if it does not exist and set objectlocking if enabled (NOTE: versioning will be not changed if OBJECTLOCKING is set because it enables versioning to the Buckets created) + if ! checkBucketExists $BUCKET; then + if [ ! -z $OBJECTLOCKING ]; then + if [ $OBJECTLOCKING = true ]; then + echo "Creating bucket with OBJECTLOCKING '$BUCKET'" + ${MC} mb --with-lock myminio/$BUCKET + elif [ $OBJECTLOCKING = false ]; then + echo "Creating bucket '$BUCKET'" + ${MC} mb myminio/$BUCKET + fi + elif [ -z $OBJECTLOCKING ]; then + echo "Creating bucket '$BUCKET'" + ${MC} mb myminio/$BUCKET + else + echo "Bucket '$BUCKET' already exists." + fi + fi + + # set versioning for bucket if objectlocking is disabled or not set + if [ $OBJECTLOCKING = false ]; then + if [ ! -z $VERSIONING ]; then + if [ $VERSIONING = true ]; then + echo "Enabling versioning for '$BUCKET'" + ${MC} version enable myminio/$BUCKET + elif [ $VERSIONING = false ]; then + echo "Suspending versioning for '$BUCKET'" + ${MC} version suspend myminio/$BUCKET + fi + else + echo "No versioning action for '$BUCKET'" + fi + else + echo "Bucket '$BUCKET' versioning unchanged." + fi + + # At this point, the bucket should exist, skip checking for existence + # Set policy on the bucket + echo "Setting policy of bucket '$BUCKET' to '$POLICY'." + ${MC} anonymous set $POLICY myminio/$BUCKET +} + +# Try connecting to MinIO instance +{{- if .Values.tls.enabled }} +scheme=https +{{- else }} +scheme=http +{{- end }} +connectToMinio $scheme + +{{ if .Values.buckets }} +{{ $global := . }} +# Create the buckets +{{- range .Values.buckets }} +createBucket {{ tpl .name $global }} {{ .policy | default "none" | quote }} {{ .purge | default false }} {{ .versioning | default "" }} {{ .objectlocking | default false }}{{- end }} +{{- end }} diff --git a/helm/minio/templates/_helper_create_policy.txt b/helm/minio/templates/_helper_create_policy.txt new file mode 100644 index 00000000..aa584952 --- /dev/null +++ b/helm/minio/templates/_helper_create_policy.txt @@ -0,0 +1,75 @@ +#!/bin/sh +set -e ; # Have script exit in the event of a failed command. + +{{- if .Values.configPathmc }} +MC_CONFIG_DIR="{{ .Values.configPathmc }}" +MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" +{{- else }} +MC="/usr/bin/mc --insecure" +{{- end }} + +# connectToMinio +# Use a check-sleep-check loop to wait for MinIO service to be available +connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 1 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 +} + +# checkPolicyExists ($policy) +# Check if the policy exists, by using the exit code of `mc admin policy info` +checkPolicyExists() { + POLICY=$1 + CMD=$(${MC} admin policy info myminio $POLICY > /dev/null 2>&1) + return $? +} + +# createPolicy($name, $filename) +createPolicy () { + NAME=$1 + FILENAME=$2 + + # Create the name if it does not exist + echo "Checking policy: $NAME (in /config/$FILENAME.json)" + if ! checkPolicyExists $NAME ; then + echo "Creating policy '$NAME'" + else + echo "Policy '$NAME' already exists." + fi + ${MC} admin policy create myminio $NAME /config/$FILENAME.json + +} + +# Try connecting to MinIO instance +{{- if .Values.tls.enabled }} +scheme=https +{{- else }} +scheme=http +{{- end }} +connectToMinio $scheme + +{{ if .Values.policies }} +# Create the policies +{{- range $idx, $policy := .Values.policies }} +createPolicy {{ $policy.name }} policy_{{ $idx }} +{{- end }} +{{- end }} diff --git a/helm/minio/templates/_helper_create_svcacct.txt b/helm/minio/templates/_helper_create_svcacct.txt new file mode 100644 index 00000000..5c8aec4f --- /dev/null +++ b/helm/minio/templates/_helper_create_svcacct.txt @@ -0,0 +1,106 @@ +#!/bin/sh +set -e ; # Have script exit in the event of a failed command. + +{{- if .Values.configPathmc }} +MC_CONFIG_DIR="{{ .Values.configPathmc }}" +MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" +{{- else }} +MC="/usr/bin/mc --insecure" +{{- end }} + +# AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. +# Special characters for example : ',",<,>,{,} +MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_svcacct_tmp" + +# connectToMinio +# Use a check-sleep-check loop to wait for MinIO service to be available +connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 2 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 +} + +# checkSvcacctExists () +# Check if the svcacct exists, by using the exit code of `mc admin user svcacct info` +checkSvcacctExists() { + CMD=$(${MC} admin user svcacct info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) + return $? +} + +# createSvcacct ($user) +createSvcacct () { + USER=$1 + FILENAME=$2 + #check accessKey_and_secretKey_tmp file + if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then + echo "credentials file does not exist" + return 1 + fi + if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then + echo "credentials file is invalid" + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP + return 1 + fi + SVCACCT=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) + # Create the svcacct if it does not exist + if ! checkSvcacctExists ; then + echo "Creating svcacct '$SVCACCT'" + # Check if policy file is define + if [ -z $FILENAME ]; then + ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) myminio $USER + else + ${MC} admin user svcacct add --access-key $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --secret-key $(tail -n1 $MINIO_ACCESSKEY_SECRETKEY_TMP) --policy /config/$FILENAME.json myminio $USER + fi + else + echo "Svcacct '$SVCACCT' already exists." + fi + #clean up credentials files. + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP +} + +# Try connecting to MinIO instance +{{- if .Values.tls.enabled }} +scheme=https +{{- else }} +scheme=http +{{- end }} +connectToMinio $scheme + +{{ if .Values.svcaccts }} +{{ $global := . }} +# Create the svcaccts +{{- range $idx, $svc := .Values.svcaccts }} +echo {{ tpl .accessKey $global }} > $MINIO_ACCESSKEY_SECRETKEY_TMP +{{- if .existingSecret }} +cat /config/secrets-svc/{{ tpl .existingSecret $global }}/{{ tpl .existingSecretKey $global }} >> $MINIO_ACCESSKEY_SECRETKEY_TMP +# Add a new line if it doesn't exist +echo >> $MINIO_ACCESSKEY_SECRETKEY_TMP +{{ else }} +echo {{ .secretKey }} >> $MINIO_ACCESSKEY_SECRETKEY_TMP +{{- end }} +{{- if $svc.policy}} +createSvcacct {{ .user }} svc_policy_{{ $idx }} +{{ else }} +createSvcacct {{ .user }} +{{- end }} +{{- end }} +{{- end }} diff --git a/helm/minio/templates/_helper_create_user.txt b/helm/minio/templates/_helper_create_user.txt new file mode 100644 index 00000000..bfb79bee --- /dev/null +++ b/helm/minio/templates/_helper_create_user.txt @@ -0,0 +1,107 @@ +#!/bin/sh +set -e ; # Have script exit in the event of a failed command. + +{{- if .Values.configPathmc }} +MC_CONFIG_DIR="{{ .Values.configPathmc }}" +MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" +{{- else }} +MC="/usr/bin/mc --insecure" +{{- end }} + +# AccessKey and secretkey credentials file are added to prevent shell execution errors caused by special characters. +# Special characters for example : ',",<,>,{,} +MINIO_ACCESSKEY_SECRETKEY_TMP="/tmp/accessKey_and_secretKey_tmp" + +# connectToMinio +# Use a check-sleep-check loop to wait for MinIO service to be available +connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 1 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 +} + +# checkUserExists () +# Check if the user exists, by using the exit code of `mc admin user info` +checkUserExists() { + CMD=$(${MC} admin user info myminio $(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) > /dev/null 2>&1) + return $? +} + +# createUser ($policy) +createUser() { + POLICY=$1 + #check accessKey_and_secretKey_tmp file + if [[ ! -f $MINIO_ACCESSKEY_SECRETKEY_TMP ]];then + echo "credentials file does not exist" + return 1 + fi + if [[ $(cat $MINIO_ACCESSKEY_SECRETKEY_TMP|wc -l) -ne 2 ]];then + echo "credentials file is invalid" + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP + return 1 + fi + USER=$(head -1 $MINIO_ACCESSKEY_SECRETKEY_TMP) + # Create the user if it does not exist + if ! checkUserExists ; then + echo "Creating user '$USER'" + cat $MINIO_ACCESSKEY_SECRETKEY_TMP | ${MC} admin user add myminio + else + echo "User '$USER' already exists." + fi + #clean up credentials files. + rm -f $MINIO_ACCESSKEY_SECRETKEY_TMP + + # set policy for user + if [ ! -z $POLICY -a $POLICY != " " ] ; then + echo "Adding policy '$POLICY' for '$USER'" + set +e ; # policy already attach errors out, allow it. + ${MC} admin policy attach myminio $POLICY --user=$USER + set -e + else + echo "User '$USER' has no policy attached." + fi +} + +# Try connecting to MinIO instance +{{- if .Values.tls.enabled }} +scheme=https +{{- else }} +scheme=http +{{- end }} +connectToMinio $scheme + +{{ if .Values.users }} +{{ $global := . }} +# Create the users +{{- range .Values.users }} +echo {{ tpl .accessKey $global }} > $MINIO_ACCESSKEY_SECRETKEY_TMP +{{- if .existingSecret }} +cat /config/secrets/{{ tpl .existingSecret $global }}/{{ tpl .existingSecretKey $global }} >> $MINIO_ACCESSKEY_SECRETKEY_TMP +# Add a new line if it doesn't exist +echo >> $MINIO_ACCESSKEY_SECRETKEY_TMP +createUser {{ .policy }} +{{ else }} +echo {{ .secretKey }} >> $MINIO_ACCESSKEY_SECRETKEY_TMP +createUser {{ .policy }} +{{- end }} +{{- end }} +{{- end }} diff --git a/helm/minio/templates/_helper_custom_command.txt b/helm/minio/templates/_helper_custom_command.txt new file mode 100644 index 00000000..b583a778 --- /dev/null +++ b/helm/minio/templates/_helper_custom_command.txt @@ -0,0 +1,58 @@ +#!/bin/sh +set -e ; # Have script exit in the event of a failed command. + +{{- if .Values.configPathmc }} +MC_CONFIG_DIR="{{ .Values.configPathmc }}" +MC="/usr/bin/mc --insecure --config-dir ${MC_CONFIG_DIR}" +{{- else }} +MC="/usr/bin/mc --insecure" +{{- end }} + +# connectToMinio +# Use a check-sleep-check loop to wait for MinIO service to be available +connectToMinio() { + SCHEME=$1 + ATTEMPTS=0 ; LIMIT=29 ; # Allow 30 attempts + set -e ; # fail if we can't read the keys. + ACCESS=$(cat /config/rootUser) ; SECRET=$(cat /config/rootPassword) ; + set +e ; # The connections to minio are allowed to fail. + echo "Connecting to MinIO server: $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT" ; + MC_COMMAND="${MC} alias set myminio $SCHEME://$MINIO_ENDPOINT:$MINIO_PORT $ACCESS $SECRET" ; + $MC_COMMAND ; + STATUS=$? ; + until [ $STATUS = 0 ] + do + ATTEMPTS=`expr $ATTEMPTS + 1` ; + echo \"Failed attempts: $ATTEMPTS\" ; + if [ $ATTEMPTS -gt $LIMIT ]; then + exit 1 ; + fi ; + sleep 2 ; # 1 second intervals between attempts + $MC_COMMAND ; + STATUS=$? ; + done ; + set -e ; # reset `e` as active + return 0 +} + +# runCommand ($@) +# Run custom mc command +runCommand() { + ${MC} "$@" + return $? +} + +# Try connecting to MinIO instance +{{- if .Values.tls.enabled }} +scheme=https +{{- else }} +scheme=http +{{- end }} +connectToMinio $scheme + +{{ if .Values.customCommands }} +# Run custom commands +{{- range .Values.customCommands }} +runCommand {{ .command }} +{{- end }} +{{- end }} diff --git a/helm/minio/templates/_helper_policy.tpl b/helm/minio/templates/_helper_policy.tpl new file mode 100644 index 00000000..8be998e5 --- /dev/null +++ b/helm/minio/templates/_helper_policy.tpl @@ -0,0 +1,28 @@ +{{- $statements_length := len .statements -}} +{{- $statements_length := sub $statements_length 1 -}} +{ + "Version": "2012-10-17", + "Statement": [ +{{- range $i, $statement := .statements }} + { + "Effect": "{{ $statement.effect | default "Allow" }}", + "Action": [ +"{{ $statement.actions | join "\",\n\"" }}" + ]{{ if $statement.resources }}, + "Resource": [ +"{{ $statement.resources | join "\",\n\"" }}" + ]{{ end }} +{{- if $statement.conditions }} +{{- $condition_len := len $statement.conditions }} +{{- $condition_len := sub $condition_len 1 }} + , + "Condition": { + {{- range $k,$v := $statement.conditions }} + {{- range $operator,$object := $v }} + "{{ $operator }}": { {{ $object }} }{{- if lt $k $condition_len }},{{- end }} + {{- end }}{{- end }} + }{{- end }} + }{{ if lt $i $statements_length }},{{end }} +{{- end }} + ] +} diff --git a/helm/minio/templates/_helpers.tpl b/helm/minio/templates/_helpers.tpl new file mode 100644 index 00000000..64f34aeb --- /dev/null +++ b/helm/minio/templates/_helpers.tpl @@ -0,0 +1,216 @@ +{{/* vim: set filetype=mustache: */}} +{{/* +Expand the name of the chart. +*/}} +{{- define "minio.name" -}} + {{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Create a default fully qualified app name. +We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec). +If release name contains chart name it will be used as a full name. +*/}} +{{- define "minio.fullname" -}} + {{- if .Values.fullnameOverride -}} + {{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}} + {{- else -}} + {{- $name := default .Chart.Name .Values.nameOverride -}} + {{- if contains $name .Release.Name -}} + {{- .Release.Name | trunc 63 | trimSuffix "-" -}} + {{- else -}} + {{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}} + {{- end -}} + {{- end -}} +{{- end -}} + +{{/* +Create chart name and version as used by the chart label. +*/}} +{{- define "minio.chart" -}} + {{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for networkpolicy. +*/}} +{{- define "minio.networkPolicy.apiVersion" -}} + {{- if semverCompare ">=1.4-0, <1.7-0" .Capabilities.KubeVersion.Version -}} + {{- print "extensions/v1beta1" -}} + {{- else if semverCompare ">=1.7-0, <1.16-0" .Capabilities.KubeVersion.Version -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else if semverCompare "^1.16-0" .Capabilities.KubeVersion.Version -}} + {{- print "networking.k8s.io/v1" -}} + {{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for deployment. +*/}} +{{- define "minio.deployment.apiVersion" -}} + {{- if semverCompare "<1.9-0" .Capabilities.KubeVersion.Version -}} + {{- print "apps/v1beta2" -}} + {{- else -}} + {{- print "apps/v1" -}} + {{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for statefulset. +*/}} +{{- define "minio.statefulset.apiVersion" -}} + {{- if semverCompare "<1.16-0" .Capabilities.KubeVersion.Version -}} + {{- print "apps/v1beta2" -}} + {{- else -}} + {{- print "apps/v1" -}} + {{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for ingress. +*/}} +{{- define "minio.ingress.apiVersion" -}} + {{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} + {{- print "extensions/v1beta1" -}} + {{- else if semverCompare "<1.19-0" .Capabilities.KubeVersion.GitVersion -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else -}} + {{- print "networking.k8s.io/v1" -}} + {{- end -}} +{{- end -}} + +{{/* +Return the appropriate apiVersion for console ingress. +*/}} +{{- define "minio.consoleIngress.apiVersion" -}} + {{- if semverCompare "<1.14-0" .Capabilities.KubeVersion.GitVersion -}} + {{- print "extensions/v1beta1" -}} + {{- else if semverCompare "<1.19-0" .Capabilities.KubeVersion.GitVersion -}} + {{- print "networking.k8s.io/v1beta1" -}} + {{- else -}} + {{- print "networking.k8s.io/v1" -}} + {{- end -}} +{{- end -}} + +{{/* +Determine secret name. +*/}} +{{- define "minio.secretName" -}} + {{- if .Values.existingSecret -}} + {{- .Values.existingSecret }} + {{- else -}} + {{- include "minio.fullname" . -}} + {{- end -}} +{{- end -}} + +{{/* +Determine name for scc role and rolebinding +*/}} +{{- define "minio.sccRoleName" -}} + {{- printf "%s-%s" "scc" (include "minio.fullname" .) | trunc 63 | trimSuffix "-" -}} +{{- end -}} + +{{/* +Properly format optional additional arguments to MinIO binary +*/}} +{{- define "minio.extraArgs" -}} +{{- range .Values.extraArgs -}} +{{ " " }}{{ . }} +{{- end -}} +{{- end -}} + +{{/* +Return the proper Docker Image Registry Secret Names +*/}} +{{- define "minio.imagePullSecrets" -}} +{{/* +Helm 2.11 supports the assignment of a value to a variable defined in a different scope, +but Helm 2.9 and 2.10 does not support it, so we need to implement this if-else logic. +Also, we can not use a single if because lazy evaluation is not an option +*/}} +{{- if .Values.global }} +{{- if .Values.global.imagePullSecrets }} +imagePullSecrets: + {{ toYaml .Values.global.imagePullSecrets }} +{{- else if .Values.imagePullSecrets }} +imagePullSecrets: + {{ toYaml .Values.imagePullSecrets }} +{{- end -}} +{{- else if .Values.imagePullSecrets }} +imagePullSecrets: + {{ toYaml .Values.imagePullSecrets }} +{{- end -}} +{{- end -}} + +{{/* +Formats volumeMount for MinIO TLS keys and trusted certs +*/}} +{{- define "minio.tlsKeysVolumeMount" -}} +{{- if .Values.tls.enabled }} +- name: cert-secret-volume + mountPath: {{ .Values.certsPath }} +{{- end }} +{{- if or .Values.tls.enabled (ne .Values.trustedCertsSecret "") }} +{{- $casPath := printf "%s/CAs" .Values.certsPath | clean }} +- name: trusted-cert-secret-volume + mountPath: {{ $casPath }} +{{- end }} +{{- end -}} + +{{/* +Formats volume for MinIO TLS keys and trusted certs +*/}} +{{- define "minio.tlsKeysVolume" -}} +{{- if .Values.tls.enabled }} +- name: cert-secret-volume + secret: + secretName: {{ tpl .Values.tls.certSecret $ }} + items: + - key: {{ .Values.tls.publicCrt }} + path: public.crt + - key: {{ .Values.tls.privateKey }} + path: private.key +{{- end }} +{{- if or .Values.tls.enabled (ne .Values.trustedCertsSecret "") }} +{{- $certSecret := eq .Values.trustedCertsSecret "" | ternary .Values.tls.certSecret .Values.trustedCertsSecret }} +{{- $publicCrt := eq .Values.trustedCertsSecret "" | ternary .Values.tls.publicCrt "" }} +- name: trusted-cert-secret-volume + secret: + secretName: {{ $certSecret }} + {{- if ne $publicCrt "" }} + items: + - key: {{ $publicCrt }} + path: public.crt + {{- end }} +{{- end }} +{{- end -}} + +{{/* +Returns the available value for certain key in an existing secret (if it exists), +otherwise it generates a random value. +*/}} +{{- define "minio.getValueFromSecret" }} + {{- $len := (default 16 .Length) | int -}} + {{- $obj := (lookup "v1" "Secret" .Namespace .Name).data -}} + {{- if $obj }} + {{- index $obj .Key | b64dec -}} + {{- else -}} + {{- randAlphaNum $len -}} + {{- end -}} +{{- end }} + +{{- define "minio.root.username" -}} + {{- if .Values.rootUser }} + {{- .Values.rootUser | toString }} + {{- else }} + {{- include "minio.getValueFromSecret" (dict "Namespace" .Release.Namespace "Name" (include "minio.fullname" .) "Length" 20 "Key" "rootUser") }} + {{- end }} +{{- end -}} + +{{- define "minio.root.password" -}} + {{- if .Values.rootPassword }} + {{- .Values.rootPassword | toString }} + {{- else }} + {{- include "minio.getValueFromSecret" (dict "Namespace" .Release.Namespace "Name" (include "minio.fullname" .) "Length" 40 "Key" "rootPassword") }} + {{- end }} +{{- end -}} diff --git a/helm/minio/templates/ciliumnetworkpolicy.yaml b/helm/minio/templates/ciliumnetworkpolicy.yaml new file mode 100644 index 00000000..1dc91bcf --- /dev/null +++ b/helm/minio/templates/ciliumnetworkpolicy.yaml @@ -0,0 +1,33 @@ +{{- if and (.Values.networkPolicy.enabled) (eq .Values.networkPolicy.flavor "cilium") }} +kind: CiliumNetworkPolicy +apiVersion: cilium.io/v2 +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + endpointSelector: + matchLabels: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} + ingress: + - toPorts: + - ports: + - port: "{{ .Values.minioAPIPort }}" + protocol: TCP + - port: "{{ .Values.minioConsolePort }}" + protocol: TCP + {{- if not .Values.networkPolicy.allowExternal }} + fromEndpoints: + - matchLabels: + {{ template "minio.name" . }}-client: "true" + {{- end }} + egress: + {{- range $entity := .Values.networkPolicy.egressEntities }} + - toEntities: + - {{ $entity }} + {{- end }} +{{- end }} diff --git a/helm/minio/templates/configmap.yaml b/helm/minio/templates/configmap.yaml new file mode 100644 index 00000000..47f64cc2 --- /dev/null +++ b/helm/minio/templates/configmap.yaml @@ -0,0 +1,32 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +data: + initialize: |- + {{- include (print $.Template.BasePath "/_helper_create_bucket.txt") . | nindent 4 }} + add-user: |- + {{- include (print $.Template.BasePath "/_helper_create_user.txt") . | nindent 4 }} + add-policy: |- + {{- include (print $.Template.BasePath "/_helper_create_policy.txt") . | nindent 4 }} + {{- range $idx, $policy := .Values.policies }} + # Policy: {{ $policy.name }} + policy_{{ $idx }}.json: |- + {{- include (print $.Template.BasePath "/_helper_policy.tpl") . | nindent 4 }} + {{ end }} + {{- range $idx, $svc := .Values.svcaccts }} + {{- if $svc.policy }} + # SVC: {{ $svc.accessKey }} + svc_policy_{{ $idx }}.json: |- + {{- include (print $.Template.BasePath "/_helper_policy.tpl") .policy | nindent 4 }} + {{- end }} + {{- end }} + add-svcacct: |- + {{- include (print $.Template.BasePath "/_helper_create_svcacct.txt") . | nindent 4 }} + custom-command: |- + {{- include (print $.Template.BasePath "/_helper_custom_command.txt") . | nindent 4 }} diff --git a/helm/minio/templates/console-ingress.yaml b/helm/minio/templates/console-ingress.yaml new file mode 100644 index 00000000..79a2b1b5 --- /dev/null +++ b/helm/minio/templates/console-ingress.yaml @@ -0,0 +1,55 @@ +{{- if .Values.consoleIngress.enabled -}} +{{- $fullName := printf "%s-console" (include "minio.fullname" .) -}} +{{- $servicePort := .Values.consoleService.port -}} +{{- $ingressPath := .Values.consoleIngress.path -}} +apiVersion: {{ template "minio.consoleIngress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.consoleIngress.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.consoleIngress.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.consoleIngress.ingressClassName }} + ingressClassName: {{ .Values.consoleIngress.ingressClassName }} + {{- end }} + {{- if .Values.consoleIngress.tls }} + tls: + {{- range .Values.consoleIngress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.consoleIngress.hosts }} + - http: + paths: + - path: {{ $ingressPath }} + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + pathType: Prefix + backend: + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + backend: + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- if . }} + host: {{ tpl . $ | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/minio/templates/console-service.yaml b/helm/minio/templates/console-service.yaml new file mode 100644 index 00000000..f09e3f3c --- /dev/null +++ b/helm/minio/templates/console-service.yaml @@ -0,0 +1,45 @@ +{{ $scheme := .Values.tls.enabled | ternary "https" "http" }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "minio.fullname" . }}-console + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.consoleService.annotations }} + annotations: {{- toYaml .Values.consoleService.annotations | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.consoleService.type }} + {{- if and (eq .Values.consoleService.type "ClusterIP") .Values.consoleService.clusterIP }} + clusterIP: {{ .Values.consoleService.clusterIP }} + {{- end }} + {{- if or (eq .Values.consoleService.type "LoadBalancer") (eq .Values.consoleService.type "NodePort") }} + externalTrafficPolicy: {{ .Values.consoleService.externalTrafficPolicy | quote }} + {{- end }} + {{- if and (eq .Values.consoleService.type "LoadBalancer") .Values.consoleService.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{ .Values.consoleService.loadBalancerSourceRanges }} + {{ end }} + {{- if and (eq .Values.consoleService.type "LoadBalancer") (not (empty .Values.consoleService.loadBalancerIP)) }} + loadBalancerIP: {{ .Values.consoleService.loadBalancerIP }} + {{- end }} + ports: + - name: {{ $scheme }} + port: {{ .Values.consoleService.port }} + protocol: TCP + {{- if (and (eq .Values.consoleService.type "NodePort") ( .Values.consoleService.nodePort)) }} + nodePort: {{ .Values.consoleService.nodePort }} + {{- else }} + targetPort: {{ .Values.minioConsolePort }} + {{- end }} + {{- if .Values.consoleService.externalIPs }} + externalIPs: + {{- range $i , $ip := .Values.consoleService.externalIPs }} + - {{ $ip }} + {{- end }} + {{- end }} + selector: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} diff --git a/helm/minio/templates/deployment.yaml b/helm/minio/templates/deployment.yaml new file mode 100644 index 00000000..4c57010f --- /dev/null +++ b/helm/minio/templates/deployment.yaml @@ -0,0 +1,213 @@ +{{- if eq .Values.mode "standalone" }} +{{ $scheme := .Values.tls.enabled | ternary "https" "http" }} +{{ $bucketRoot := or ($.Values.bucketRoot) ($.Values.mountPath) }} +apiVersion: {{ template "minio.deployment.apiVersion" . }} +kind: Deployment +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.additionalLabels }} + {{- toYaml .Values.additionalLabels | nindent 4 }} + {{- end }} + {{- if .Values.additionalAnnotations }} + annotations: {{- toYaml .Values.additionalAnnotations | nindent 4 }} + {{- end }} +spec: + strategy: + type: {{ .Values.deploymentUpdate.type }} + {{- if eq .Values.deploymentUpdate.type "RollingUpdate" }} + rollingUpdate: + maxSurge: {{ .Values.deploymentUpdate.maxSurge }} + maxUnavailable: {{ .Values.deploymentUpdate.maxUnavailable }} + {{- end }} + replicas: 1 + selector: + matchLabels: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} + template: + metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} + {{- if .Values.podLabels }} + {{- toYaml .Values.podLabels | nindent 8 }} + {{- end }} + annotations: + {{- if not .Values.ignoreChartChecksums }} + checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- end }} + {{- if .Values.podAnnotations }} + {{- toYaml .Values.podAnnotations | trimSuffix "\n" | nindent 8 }} + {{- end }} + spec: + {{- if .Values.priorityClassName }} + priorityClassName: "{{ .Values.priorityClassName }}" + {{- end }} + {{- if .Values.runtimeClassName }} + runtimeClassName: "{{ .Values.runtimeClassName }}" + {{- end }} + {{- if and .Values.securityContext.enabled .Values.persistence.enabled }} + securityContext: + {{ omit .Values.securityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + {{ if .Values.serviceAccount.create }} + serviceAccountName: {{ .Values.serviceAccount.name }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}" + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: + - "/bin/sh" + - "-ce" + - "/usr/bin/docker-entrypoint.sh minio server {{ $bucketRoot }} -S {{ .Values.certsPath }} --address :{{ .Values.minioAPIPort }} --console-address :{{ .Values.minioConsolePort }} {{- template "minio.extraArgs" . }}" + volumeMounts: + - name: minio-user + mountPath: "/tmp/credentials" + readOnly: true + - name: export + mountPath: {{ .Values.mountPath }} + {{- if and .Values.persistence.enabled .Values.persistence.subPath }} + subPath: "{{ .Values.persistence.subPath }}" + {{- end }} + {{- if .Values.extraSecret }} + - name: extra-secret + mountPath: "/tmp/minio-config-env" + {{- end }} + {{- include "minio.tlsKeysVolumeMount" . | indent 12 }} + {{- if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + {{- end }} + ports: + - name: {{ $scheme }} + containerPort: {{ .Values.minioAPIPort }} + - name: {{ $scheme }}-console + containerPort: {{ .Values.minioConsolePort }} + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: {{ template "minio.secretName" . }} + key: rootUser + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "minio.secretName" . }} + key: rootPassword + {{- if .Values.extraSecret }} + - name: MINIO_CONFIG_ENV_FILE + value: "/tmp/minio-config-env/config.env" + {{- end }} + {{- if .Values.metrics.serviceMonitor.public }} + - name: MINIO_PROMETHEUS_AUTH_TYPE + value: "public" + {{- end }} + {{- if .Values.oidc.enabled }} + - name: MINIO_IDENTITY_OPENID_CONFIG_URL + value: {{ .Values.oidc.configUrl }} + - name: MINIO_IDENTITY_OPENID_CLIENT_ID + {{- if and .Values.oidc.existingClientSecretName .Values.oidc.existingClientIdKey }} + valueFrom: + secretKeyRef: + name: {{ .Values.oidc.existingClientSecretName }} + key: {{ .Values.oidc.existingClientIdKey }} + {{- else }} + value: {{ .Values.oidc.clientId }} + {{- end }} + - name: MINIO_IDENTITY_OPENID_CLIENT_SECRET + {{- if and .Values.oidc.existingClientSecretName .Values.oidc.existingClientSecretKey }} + valueFrom: + secretKeyRef: + name: {{ .Values.oidc.existingClientSecretName }} + key: {{ .Values.oidc.existingClientSecretKey }} + {{- else }} + value: {{ .Values.oidc.clientSecret }} + {{- end }} + - name: MINIO_IDENTITY_OPENID_CLAIM_NAME + value: {{ .Values.oidc.claimName }} + - name: MINIO_IDENTITY_OPENID_CLAIM_PREFIX + value: {{ .Values.oidc.claimPrefix }} + - name: MINIO_IDENTITY_OPENID_SCOPES + value: {{ .Values.oidc.scopes }} + - name: MINIO_IDENTITY_OPENID_COMMENT + value: {{ .Values.oidc.comment }} + - name: MINIO_IDENTITY_OPENID_REDIRECT_URI + value: {{ .Values.oidc.redirectUri }} + - name: MINIO_IDENTITY_OPENID_DISPLAY_NAME + value: {{ .Values.oidc.displayName }} + {{- end }} + {{- if .Values.etcd.endpoints }} + - name: MINIO_ETCD_ENDPOINTS + value: {{ join "," .Values.etcd.endpoints | quote }} + {{- if .Values.etcd.clientCert }} + - name: MINIO_ETCD_CLIENT_CERT + value: "/tmp/credentials/etcd_client_cert.pem" + {{- end }} + {{- if .Values.etcd.clientCertKey }} + - name: MINIO_ETCD_CLIENT_CERT_KEY + value: "/tmp/credentials/etcd_client_cert_key.pem" + {{- end }} + {{- if .Values.etcd.pathPrefix }} + - name: MINIO_ETCD_PATH_PREFIX + value: {{ .Values.etcd.pathPrefix }} + {{- end }} + {{- if .Values.etcd.corednsPathPrefix }} + - name: MINIO_ETCD_COREDNS_PATH + value: {{ .Values.etcd.corednsPathPrefix }} + {{- end }} + {{- end }} + {{- range $key, $val := .Values.environment }} + - name: {{ $key }} + value: {{ tpl $val $ | quote }} + {{- end }} + resources: {{- toYaml .Values.resources | nindent 12 }} + {{- if and .Values.securityContext.enabled .Values.persistence.enabled }} + {{- with .Values.containerSecurityContext }} + securityContext: {{ toYaml . | nindent 12}} + {{- end }} + {{- end }} + {{- with .Values.extraContainers }} + {{- if eq (typeOf .) "string" }} + {{- tpl . $ | nindent 8 }} + {{- else }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "minio.imagePullSecrets" . | indent 6 }} + {{- with .Values.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + volumes: + - name: export + {{- if .Values.persistence.enabled }} + persistentVolumeClaim: + claimName: {{ .Values.persistence.existingClaim | default (include "minio.fullname" .) }} + {{- else }} + emptyDir: {} + {{- end }} + {{- if .Values.extraSecret }} + - name: extra-secret + secret: + secretName: {{ .Values.extraSecret }} + {{- end }} + - name: minio-user + secret: + secretName: {{ template "minio.secretName" . }} + {{- include "minio.tlsKeysVolume" . | indent 8 }} + {{- if .Values.extraVolumes }} + {{ toYaml .Values.extraVolumes | nindent 8 }} + {{- end }} +{{- end }} diff --git a/helm/minio/templates/ingress.yaml b/helm/minio/templates/ingress.yaml new file mode 100644 index 00000000..1a564c6b --- /dev/null +++ b/helm/minio/templates/ingress.yaml @@ -0,0 +1,55 @@ +{{- if .Values.ingress.enabled -}} +{{- $fullName := include "minio.fullname" . -}} +{{- $servicePort := .Values.service.port -}} +{{- $ingressPath := .Values.ingress.path -}} +apiVersion: {{ template "minio.ingress.apiVersion" . }} +kind: Ingress +metadata: + name: {{ $fullName }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- with .Values.ingress.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} + {{- with .Values.ingress.annotations }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +spec: + {{- if .Values.ingress.ingressClassName }} + ingressClassName: {{ .Values.ingress.ingressClassName }} + {{- end }} + {{- if .Values.ingress.tls }} + tls: + {{- range .Values.ingress.tls }} + - hosts: + {{- range .hosts }} + - {{ . | quote }} + {{- end }} + secretName: {{ .secretName }} + {{- end }} + {{- end }} + rules: + {{- range .Values.ingress.hosts }} + - http: + paths: + - path: {{ $ingressPath }} + {{- if semverCompare ">=1.19-0" $.Capabilities.KubeVersion.GitVersion }} + pathType: Prefix + backend: + service: + name: {{ $fullName }} + port: + number: {{ $servicePort }} + {{- else }} + backend: + serviceName: {{ $fullName }} + servicePort: {{ $servicePort }} + {{- end }} + {{- if . }} + host: {{ tpl . $ | quote }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/minio/templates/networkpolicy.yaml b/helm/minio/templates/networkpolicy.yaml new file mode 100644 index 00000000..bb45a6c6 --- /dev/null +++ b/helm/minio/templates/networkpolicy.yaml @@ -0,0 +1,66 @@ +{{- if and (.Values.networkPolicy.enabled) (eq .Values.networkPolicy.flavor "kubernetes") }} +kind: NetworkPolicy +apiVersion: {{ template "minio.networkPolicy.apiVersion" . }} +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + podSelector: + matchLabels: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} + ingress: + - ports: + - port: {{ .Values.minioAPIPort }} + protocol: TCP + - port: {{ .Values.minioConsolePort }} + protocol: TCP + {{- if not .Values.networkPolicy.allowExternal }} + from: + - podSelector: + matchLabels: + {{ template "minio.name" . }}-client: "true" + {{- end }} + {{- if .Values.networkPolicy.egress.enabled }} + egress: + - ports: + {{ .Values.networkPolicy.egress.ports | toJson }} + {{- with .Values.networkPolicy.egress.to }} + to: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} +--- +kind: NetworkPolicy +apiVersion: {{ template "minio.networkPolicy.apiVersion" . }} +metadata: + name: {{ template "minio.fullname" . }}-post-job + labels: + app: {{ template "minio.name" . }}-post-job + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + podSelector: + matchLabels: + app: {{ template "minio.name" . }}-job + release: {{ .Release.Name }} + egress: + - ports: + - port: {{ .Values.minioAPIPort }} + protocol: TCP + - port: {{ .Values.minioConsolePort }} + protocol: TCP + {{- if .Values.networkPolicy.egress.enabled }} + - ports: + {{ .Values.networkPolicy.egress.ports | toJson }} + {{- with .Values.networkPolicy.egress.to }} + to: + {{- toYaml . | nindent 12 }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/minio/templates/poddisruptionbudget.yaml b/helm/minio/templates/poddisruptionbudget.yaml new file mode 100644 index 00000000..a5f90a08 --- /dev/null +++ b/helm/minio/templates/poddisruptionbudget.yaml @@ -0,0 +1,17 @@ +{{- if .Values.podDisruptionBudget.enabled }} +{{- if .Capabilities.APIVersions.Has "policy/v1beta1/PodDisruptionBudget" }} +apiVersion: policy/v1beta1 +{{- else }} +apiVersion: policy/v1 +{{- end }} +kind: PodDisruptionBudget +metadata: + name: minio + labels: + app: {{ template "minio.name" . }} +spec: + maxUnavailable: {{ .Values.podDisruptionBudget.maxUnavailable }} + selector: + matchLabels: + app: {{ template "minio.name" . }} +{{- end }} diff --git a/helm/minio/templates/post-job.yaml b/helm/minio/templates/post-job.yaml new file mode 100644 index 00000000..955d6558 --- /dev/null +++ b/helm/minio/templates/post-job.yaml @@ -0,0 +1,258 @@ +{{- if or .Values.buckets .Values.users .Values.policies .Values.customCommands .Values.svcaccts }} +apiVersion: batch/v1 +kind: Job +metadata: + name: {{ template "minio.fullname" . }}-post-job + labels: + app: {{ template "minio.name" . }}-post-job + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + annotations: + "helm.sh/hook": post-install,post-upgrade + "helm.sh/hook-delete-policy": hook-succeeded,before-hook-creation + {{- with .Values.postJob.annotations }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + template: + metadata: + labels: + app: {{ template "minio.name" . }}-job + release: {{ .Release.Name }} + {{- if .Values.podLabels }} + {{- toYaml .Values.podLabels | nindent 8 }} + {{- end }} + {{- if .Values.postJob.podAnnotations }} + annotations: {{- toYaml .Values.postJob.podAnnotations | nindent 8 }} + {{- end }} + spec: + restartPolicy: OnFailure + {{- include "minio.imagePullSecrets" . | indent 6 }} + {{- if .Values.nodeSelector }} + nodeSelector: {{- toYaml .Values.postJob.nodeSelector | nindent 8 }} + {{- end }} + {{- with .Values.postJob.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.postJob.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + {{- if .Values.postJob.securityContext.enabled }} + securityContext: {{ omit .Values.postJob.securityContext "enabled" | toYaml | nindent 12 }} + {{- end }} + volumes: + - name: etc-path + emptyDir: {} + - name: tmp + emptyDir: {} + - name: minio-configuration + projected: + sources: + - configMap: + name: {{ template "minio.fullname" . }} + - secret: + name: {{ template "minio.secretName" . }} + {{- range (concat .Values.users (default (list) .Values.svcaccts)) }} + {{- if .existingSecret }} + - secret: + name: {{ tpl .existingSecret $ }} + items: + - key: {{ .existingSecretKey }} + path: secrets/{{ tpl .existingSecret $ }}/{{ tpl .existingSecretKey $ }} + {{- end }} + {{- end }} + {{- range ( default list .Values.svcaccts ) }} + {{- if .existingSecret }} + - secret: + name: {{ tpl .existingSecret $ }} + items: + - key: {{ .existingSecretKey }} + path: secrets-svc/{{ tpl .existingSecret $ }}/{{ tpl .existingSecretKey $ }} + {{- end }} + {{- end }} + {{- if .Values.tls.enabled }} + - name: cert-secret-volume-mc + secret: + secretName: {{ .Values.tls.certSecret }} + items: + - key: {{ .Values.tls.publicCrt }} + path: CAs/public.crt + {{- end }} + {{- if .Values.customCommandJob.extraVolumes }} + {{- toYaml .Values.customCommandJob.extraVolumes | nindent 8 }} + {{- end }} + {{- if .Values.serviceAccount.create }} + serviceAccountName: {{ .Values.serviceAccount.name }} + {{- end }} + {{- if .Values.policies }} + initContainers: + - name: minio-make-policy + image: "{{ .Values.mcImage.repository }}:{{ .Values.mcImage.tag }}" + {{- if .Values.makePolicyJob.securityContext.enabled }} + {{- with .Values.makePolicyJob.containerSecurityContext }} + securityContext: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + imagePullPolicy: {{ .Values.mcImage.pullPolicy }} + {{- if .Values.makePolicyJob.exitCommand }} + command: [ "/bin/sh", "-c" ] + args: [ "/bin/sh /config/add-policy; EV=$?; {{ .Values.makePolicyJob.exitCommand }} && exit $EV" ] + {{- else }} + command: [ "/bin/sh", "/config/add-policy" ] + {{- end }} + env: + - name: MINIO_ENDPOINT + value: {{ template "minio.fullname" . }} + - name: MINIO_PORT + value: {{ .Values.service.port | quote }} + volumeMounts: + - name: etc-path + mountPath: /etc/minio/mc + - name: tmp + mountPath: /tmp + - name: minio-configuration + mountPath: /config + {{- if .Values.tls.enabled }} + - name: cert-secret-volume-mc + mountPath: {{ .Values.configPathmc }}certs + {{- end }} + resources: {{- toYaml .Values.makePolicyJob.resources | nindent 12 }} + {{- end }} + containers: + {{- if .Values.buckets }} + - name: minio-make-bucket + image: "{{ .Values.mcImage.repository }}:{{ .Values.mcImage.tag }}" + {{- if .Values.makeBucketJob.securityContext.enabled }} + {{- with .Values.makeBucketJob.containerSecurityContext }} + securityContext: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + imagePullPolicy: {{ .Values.mcImage.pullPolicy }} + {{- if .Values.makeBucketJob.exitCommand }} + command: [ "/bin/sh", "-c" ] + args: [ "/bin/sh /config/initialize; EV=$?; {{ .Values.makeBucketJob.exitCommand }} && exit $EV" ] + {{- else }} + command: [ "/bin/sh", "/config/initialize" ] + {{- end }} + env: + - name: MINIO_ENDPOINT + value: {{ template "minio.fullname" . }} + - name: MINIO_PORT + value: {{ .Values.service.port | quote }} + volumeMounts: + - name: etc-path + mountPath: /etc/minio/mc + - name: tmp + mountPath: /tmp + - name: minio-configuration + mountPath: /config + {{- if .Values.tls.enabled }} + - name: cert-secret-volume-mc + mountPath: {{ .Values.configPathmc }}certs + {{- end }} + resources: {{- toYaml .Values.makeBucketJob.resources | nindent 12 }} + {{- end }} + {{- if .Values.users }} + - name: minio-make-user + image: "{{ .Values.mcImage.repository }}:{{ .Values.mcImage.tag }}" + {{- if .Values.makeUserJob.securityContext.enabled }} + {{- with .Values.makeUserJob.containerSecurityContext }} + securityContext: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + imagePullPolicy: {{ .Values.mcImage.pullPolicy }} + {{- if .Values.makeUserJob.exitCommand }} + command: [ "/bin/sh", "-c" ] + args: [ "/bin/sh /config/add-user; EV=$?; {{ .Values.makeUserJob.exitCommand }} && exit $EV" ] + {{- else }} + command: [ "/bin/sh", "/config/add-user" ] + {{- end }} + env: + - name: MINIO_ENDPOINT + value: {{ template "minio.fullname" . }} + - name: MINIO_PORT + value: {{ .Values.service.port | quote }} + volumeMounts: + - name: etc-path + mountPath: /etc/minio/mc + - name: tmp + mountPath: /tmp + - name: minio-configuration + mountPath: /config + {{- if .Values.tls.enabled }} + - name: cert-secret-volume-mc + mountPath: {{ .Values.configPathmc }}certs + {{- end }} + resources: {{- toYaml .Values.makeUserJob.resources | nindent 12 }} + {{- end }} + {{- if .Values.customCommands }} + - name: minio-custom-command + image: "{{ .Values.mcImage.repository }}:{{ .Values.mcImage.tag }}" + {{- if .Values.customCommandJob.securityContext.enabled }} + {{- with .Values.customCommandJob.containerSecurityContext }} + securityContext: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + imagePullPolicy: {{ .Values.mcImage.pullPolicy }} + {{- if .Values.customCommandJob.exitCommand }} + command: [ "/bin/sh", "-c" ] + args: [ "/bin/sh /config/custom-command; EV=$?; {{ .Values.customCommandJob.exitCommand }} && exit $EV" ] + {{- else }} + command: [ "/bin/sh", "/config/custom-command" ] + {{- end }} + env: + - name: MINIO_ENDPOINT + value: {{ template "minio.fullname" . }} + - name: MINIO_PORT + value: {{ .Values.service.port | quote }} + volumeMounts: + - name: etc-path + mountPath: /etc/minio/mc + - name: tmp + mountPath: /tmp + - name: minio-configuration + mountPath: /config + {{- if .Values.tls.enabled }} + - name: cert-secret-volume-mc + mountPath: {{ .Values.configPathmc }}certs + {{- end }} + {{- if .Values.customCommandJob.extraVolumeMounts }} + {{- toYaml .Values.customCommandJob.extraVolumeMounts | nindent 12 }} + {{- end }} + resources: {{- toYaml .Values.customCommandJob.resources | nindent 12 }} + {{- end }} + {{- if .Values.svcaccts }} + - name: minio-make-svcacct + image: "{{ .Values.mcImage.repository }}:{{ .Values.mcImage.tag }}" + {{- if .Values.makeServiceAccountJob.securityContext.enabled }} + {{- with .Values.makeServiceAccountJob.containerSecurityContext }} + securityContext: {{ toYaml . | nindent 12 }} + {{- end }} + {{- end }} + imagePullPolicy: {{ .Values.mcImage.pullPolicy }} + {{- if .Values.makeServiceAccountJob.exitCommand }} + command: [ "/bin/sh", "-c" ] + args: ["/bin/sh /config/add-svcacct; EV=$?; {{ .Values.makeServiceAccountJob.exitCommand }} && exit $EV" ] + {{- else }} + command: ["/bin/sh", "/config/add-svcacct"] + {{- end }} + env: + - name: MINIO_ENDPOINT + value: {{ template "minio.fullname" . }} + - name: MINIO_PORT + value: {{ .Values.service.port | quote }} + volumeMounts: + - name: etc-path + mountPath: /etc/minio/mc + - name: tmp + mountPath: /tmp + - name: minio-configuration + mountPath: /config + {{- if .Values.tls.enabled }} + - name: cert-secret-volume-mc + mountPath: {{ .Values.configPathmc }}certs + {{- end }} + resources: {{- toYaml .Values.makeServiceAccountJob.resources | nindent 12 }} + {{- end }} +{{- end }} diff --git a/helm/minio/templates/pvc.yaml b/helm/minio/templates/pvc.yaml new file mode 100644 index 00000000..60f5267b --- /dev/null +++ b/helm/minio/templates/pvc.yaml @@ -0,0 +1,32 @@ +{{- if eq .Values.mode "standalone" }} +{{- if and .Values.persistence.enabled (not .Values.persistence.existingClaim) }} +apiVersion: v1 +kind: PersistentVolumeClaim +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.persistence.annotations }} + annotations: {{- toYaml .Values.persistence.annotations | nindent 4 }} + {{- end }} +spec: + accessModes: + - {{ .Values.persistence.accessMode | quote }} + resources: + requests: + storage: {{ .Values.persistence.size | quote }} + {{- if .Values.persistence.storageClass }} + {{- if (eq "-" .Values.persistence.storageClass) }} + storageClassName: "" + {{- else }} + storageClassName: "{{ .Values.persistence.storageClass }}" + {{- end }} + {{- end }} + {{- if .Values.persistence.volumeName }} + volumeName: "{{ .Values.persistence.volumeName }}" + {{- end }} +{{- end }} +{{- end }} diff --git a/helm/minio/templates/secrets.yaml b/helm/minio/templates/secrets.yaml new file mode 100644 index 00000000..476c3da5 --- /dev/null +++ b/helm/minio/templates/secrets.yaml @@ -0,0 +1,21 @@ +{{- if not .Values.existingSecret }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ template "minio.secretName" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +type: Opaque +data: + rootUser: {{ include "minio.root.username" . | b64enc | quote }} + rootPassword: {{ include "minio.root.password" . | b64enc | quote }} + {{- if .Values.etcd.clientCert }} + etcd_client.crt: {{ .Values.etcd.clientCert | toString | b64enc | quote }} + {{- end }} + {{- if .Values.etcd.clientCertKey }} + etcd_client.key: {{ .Values.etcd.clientCertKey | toString | b64enc | quote }} + {{- end }} +{{- end }} diff --git a/helm/minio/templates/securitycontextconstraints.yaml b/helm/minio/templates/securitycontextconstraints.yaml new file mode 100644 index 00000000..4bac7e37 --- /dev/null +++ b/helm/minio/templates/securitycontextconstraints.yaml @@ -0,0 +1,45 @@ +{{- if and .Values.securityContext.enabled .Values.persistence.enabled (.Capabilities.APIVersions.Has "security.openshift.io/v1") }} +apiVersion: security.openshift.io/v1 +kind: SecurityContextConstraints +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +allowHostDirVolumePlugin: false +allowHostIPC: false +allowHostNetwork: false +allowHostPID: false +allowHostPorts: false +allowPrivilegeEscalation: true +allowPrivilegedContainer: false +allowedCapabilities: [] +readOnlyRootFilesystem: false +defaultAddCapabilities: [] +requiredDropCapabilities: +- KILL +- MKNOD +- SETUID +- SETGID +fsGroup: + type: MustRunAs + ranges: + - max: {{ .Values.securityContext.fsGroup }} + min: {{ .Values.securityContext.fsGroup }} +runAsUser: + type: MustRunAs + uid: {{ .Values.securityContext.runAsUser }} +seLinuxContext: + type: MustRunAs +supplementalGroups: + type: RunAsAny +volumes: +- configMap +- downwardAPI +- emptyDir +- persistentVolumeClaim +- projected +- secret +{{- end }} diff --git a/helm/minio/templates/service.yaml b/helm/minio/templates/service.yaml new file mode 100644 index 00000000..d872cd07 --- /dev/null +++ b/helm/minio/templates/service.yaml @@ -0,0 +1,46 @@ +{{ $scheme := .Values.tls.enabled | ternary "https" "http" }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + monitoring: "true" + {{- if .Values.service.annotations }} + annotations: {{- toYaml .Values.service.annotations | nindent 4 }} + {{- end }} +spec: + type: {{ .Values.service.type }} + {{- if and (eq .Values.service.type "ClusterIP") .Values.service.clusterIP }} + clusterIP: {{ .Values.service.clusterIP }} + {{- end }} + {{- if or (eq .Values.service.type "LoadBalancer") (eq .Values.service.type "NodePort") }} + externalTrafficPolicy: {{ .Values.service.externalTrafficPolicy | quote }} + {{- end }} + {{- if and (eq .Values.service.type "LoadBalancer") .Values.service.loadBalancerSourceRanges }} + loadBalancerSourceRanges: {{ .Values.service.loadBalancerSourceRanges }} + {{ end }} + {{- if and (eq .Values.service.type "LoadBalancer") (not (empty .Values.service.loadBalancerIP)) }} + loadBalancerIP: {{ default "" .Values.service.loadBalancerIP | quote }} + {{- end }} + ports: + - name: {{ $scheme }} + port: {{ .Values.service.port }} + protocol: TCP + {{- if (and (eq .Values.service.type "NodePort") ( .Values.service.nodePort)) }} + nodePort: {{ .Values.service.nodePort }} + {{- else }} + targetPort: {{ .Values.minioAPIPort }} + {{- end }} + {{- if .Values.service.externalIPs }} + externalIPs: + {{- range $i , $ip := .Values.service.externalIPs }} + - {{ $ip }} + {{- end }} + {{- end }} + selector: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} diff --git a/helm/minio/templates/serviceaccount.yaml b/helm/minio/templates/serviceaccount.yaml new file mode 100644 index 00000000..07840153 --- /dev/null +++ b/helm/minio/templates/serviceaccount.yaml @@ -0,0 +1,6 @@ +{{- if .Values.serviceAccount.create }} +apiVersion: v1 +kind: ServiceAccount +metadata: + name: {{ .Values.serviceAccount.name | quote }} +{{- end }} diff --git a/helm/minio/templates/servicemonitor.yaml b/helm/minio/templates/servicemonitor.yaml new file mode 100644 index 00000000..f875a850 --- /dev/null +++ b/helm/minio/templates/servicemonitor.yaml @@ -0,0 +1,112 @@ +{{- if and .Values.metrics.serviceMonitor.enabled .Values.metrics.serviceMonitor.includeNode }} +apiVersion: monitoring.coreos.com/v1 +kind: ServiceMonitor +metadata: + name: {{ template "minio.fullname" . }} + {{- if .Values.metrics.serviceMonitor.namespace }} + namespace: {{ .Values.metrics.serviceMonitor.namespace }} + {{- end }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.metrics.serviceMonitor.additionalLabels }} + {{- toYaml .Values.metrics.serviceMonitor.additionalLabels | nindent 4 }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.annotations }} + annotations: {{- toYaml .Values.metrics.serviceMonitor.annotations | nindent 4 }} + {{- end }} +spec: + endpoints: + {{- if .Values.tls.enabled }} + - port: https + scheme: https + tlsConfig: + ca: + secret: + name: {{ .Values.tls.certSecret }} + key: {{ .Values.tls.publicCrt }} + serverName: {{ template "minio.fullname" . }} + {{- else }} + - port: http + scheme: http + {{- end }} + path: /minio/v2/metrics/node + {{- if .Values.metrics.serviceMonitor.interval }} + interval: {{ .Values.metrics.serviceMonitor.interval }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.relabelConfigs }} + {{- toYaml .Values.metrics.serviceMonitor.relabelConfigs | nindent 6 }} + {{- end }} + {{- if not .Values.metrics.serviceMonitor.public }} + bearerTokenSecret: + name: {{ template "minio.fullname" . }}-prometheus + key: token + {{- end }} + namespaceSelector: + matchNames: + - {{ .Release.Namespace | quote }} + selector: + matchLabels: + app: {{ include "minio.name" . }} + release: {{ .Release.Name }} + monitoring: "true" +{{- end }} +{{- if .Values.metrics.serviceMonitor.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: Probe +metadata: + name: {{ template "minio.fullname" . }}-cluster + {{- if .Values.metrics.serviceMonitor.namespace }} + namespace: {{ .Values.metrics.serviceMonitor.namespace }} + {{- end }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.metrics.serviceMonitor.additionalLabels }} + {{- toYaml .Values.metrics.serviceMonitor.additionalLabels | nindent 4 }} + {{- end }} +spec: + jobName: {{ template "minio.fullname" . }} + {{- if .Values.tls.enabled }} + tlsConfig: + ca: + secret: + name: {{ .Values.tls.certSecret }} + key: {{ .Values.tls.publicCrt }} + serverName: {{ template "minio.fullname" . }} + {{- end }} + prober: + url: {{ template "minio.fullname" . }}.{{ .Release.Namespace }}:{{ .Values.service.port }} + path: /minio/v2/metrics/cluster + {{- if .Values.tls.enabled }} + scheme: https + {{- else }} + scheme: http + {{- end }} + {{- if .Values.metrics.serviceMonitor.relabelConfigsCluster }} + {{- toYaml .Values.metrics.serviceMonitor.relabelConfigsCluster | nindent 2 }} + {{- end }} + targets: + staticConfig: + static: + - {{ template "minio.fullname" . }}.{{ .Release.Namespace }} + {{- if not .Values.metrics.serviceMonitor.public }} + {{- if .Values.metrics.serviceMonitor.interval }} + interval: {{ .Values.metrics.serviceMonitor.interval }} + {{- end }} + {{- if .Values.metrics.serviceMonitor.scrapeTimeout }} + scrapeTimeout: {{ .Values.metrics.serviceMonitor.scrapeTimeout }} + {{- end }} + bearerTokenSecret: + name: {{ template "minio.fullname" . }}-prometheus + key: token + {{- end }} +{{- end }} diff --git a/helm/minio/templates/statefulset.yaml b/helm/minio/templates/statefulset.yaml new file mode 100644 index 00000000..d671eaaf --- /dev/null +++ b/helm/minio/templates/statefulset.yaml @@ -0,0 +1,267 @@ +{{- if eq .Values.mode "distributed" }} +{{ $poolCount := .Values.pools | int }} +{{ $nodeCount := .Values.replicas | int }} +{{ $replicas := mul $poolCount $nodeCount }} +{{ $drivesPerNode := .Values.drivesPerNode | int }} +{{ $scheme := .Values.tls.enabled | ternary "https" "http" }} +{{ $mountPath := .Values.mountPath }} +{{ $bucketRoot := or ($.Values.bucketRoot) ($.Values.mountPath) }} +{{ $subPath := .Values.persistence.subPath }} +{{ $penabled := .Values.persistence.enabled }} +{{ $accessMode := .Values.persistence.accessMode }} +{{ $storageClass := .Values.persistence.storageClass }} +{{ $psize := .Values.persistence.size }} +apiVersion: v1 +kind: Service +metadata: + name: {{ template "minio.fullname" . }}-svc + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} +spec: + publishNotReadyAddresses: true + clusterIP: None + ports: + - name: {{ $scheme }} + port: {{ .Values.service.port }} + protocol: TCP + targetPort: {{ .Values.minioAPIPort }} + selector: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} +--- +apiVersion: {{ template "minio.statefulset.apiVersion" . }} +kind: StatefulSet +metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + chart: {{ template "minio.chart" . }} + release: {{ .Release.Name }} + heritage: {{ .Release.Service }} + {{- if .Values.additionalLabels }} + {{- toYaml .Values.additionalLabels | nindent 4 }} + {{- end }} + {{- if .Values.additionalAnnotations }} + annotations: {{- toYaml .Values.additionalAnnotations | nindent 4 }} + {{- end }} +spec: + updateStrategy: + type: {{ .Values.statefulSetUpdate.updateStrategy }} + podManagementPolicy: "Parallel" + serviceName: {{ template "minio.fullname" . }}-svc + replicas: {{ $replicas }} + selector: + matchLabels: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} + template: + metadata: + name: {{ template "minio.fullname" . }} + labels: + app: {{ template "minio.name" . }} + release: {{ .Release.Name }} + {{- if .Values.podLabels }} + {{- toYaml .Values.podLabels | nindent 8 }} + {{- end }} + annotations: + {{- if not .Values.ignoreChartChecksums }} + checksum/secrets: {{ include (print $.Template.BasePath "/secrets.yaml") . | sha256sum }} + checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- end }} + {{- if .Values.podAnnotations }} + {{- toYaml .Values.podAnnotations | nindent 8 }} + {{- end }} + spec: + {{- if .Values.priorityClassName }} + priorityClassName: "{{ .Values.priorityClassName }}" + {{- end }} + {{- if .Values.runtimeClassName }} + runtimeClassName: "{{ .Values.runtimeClassName }}" + {{- end }} + {{- if and .Values.securityContext.enabled .Values.persistence.enabled }} + securityContext: + {{- omit .Values.securityContext "enabled" | toYaml | nindent 8 }} + {{- end }} + {{- if .Values.serviceAccount.create }} + serviceAccountName: {{ .Values.serviceAccount.name }} + {{- end }} + containers: + - name: {{ .Chart.Name }} + image: {{ .Values.image.repository }}:{{ .Values.image.tag }} + imagePullPolicy: {{ .Values.image.pullPolicy }} + command: [ + "/bin/sh", + "-ce", + "/usr/bin/docker-entrypoint.sh minio server {{- range $i := until $poolCount }}{{ $factor := mul $i $nodeCount }}{{ $endIndex := add $factor $nodeCount }}{{ $beginIndex := mul $i $nodeCount }} {{ $scheme }}://{{ template `minio.fullname` $ }}-{{ `{` }}{{ $beginIndex }}...{{ sub $endIndex 1 }}{{ `}`}}.{{ template `minio.fullname` $ }}-svc.{{ $.Release.Namespace }}.svc{{if (gt $drivesPerNode 1)}}{{ $bucketRoot }}-{{ `{` }}0...{{ sub $drivesPerNode 1 }}{{ `}` }}{{ else }}{{ $bucketRoot }}{{end }}{{- end }} -S {{ .Values.certsPath }} --address :{{ .Values.minioAPIPort }} --console-address :{{ .Values.minioConsolePort }} {{- template `minio.extraArgs` . }}" + ] + volumeMounts: + {{- if $penabled }} + {{- if (gt $drivesPerNode 1) }} + {{- range $i := until $drivesPerNode }} + - name: export-{{ $i }} + mountPath: {{ $mountPath }}-{{ $i }} + {{- if and $penabled $subPath }} + subPath: {{ $subPath }} + {{- end }} + {{- end }} + {{- else }} + - name: export + mountPath: {{ $mountPath }} + {{- if and $penabled $subPath }} + subPath: {{ $subPath }} + {{- end }} + {{- end }} + {{- end }} + {{- if .Values.extraSecret }} + - name: extra-secret + mountPath: "/tmp/minio-config-env" + {{- end }} + {{- include "minio.tlsKeysVolumeMount" . | indent 12 }} + {{- if .Values.extraVolumeMounts }} + {{- toYaml .Values.extraVolumeMounts | nindent 12 }} + {{- end }} + ports: + - name: {{ $scheme }} + containerPort: {{ .Values.minioAPIPort }} + - name: {{ $scheme }}-console + containerPort: {{ .Values.minioConsolePort }} + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: {{ template "minio.secretName" . }} + key: rootUser + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: {{ template "minio.secretName" . }} + key: rootPassword + {{- if .Values.extraSecret }} + - name: MINIO_CONFIG_ENV_FILE + value: "/tmp/minio-config-env/config.env" + {{- end }} + {{- if .Values.metrics.serviceMonitor.public }} + - name: MINIO_PROMETHEUS_AUTH_TYPE + value: "public" + {{- end }} + {{- if .Values.oidc.enabled }} + - name: MINIO_IDENTITY_OPENID_CONFIG_URL + value: {{ .Values.oidc.configUrl }} + - name: MINIO_IDENTITY_OPENID_CLIENT_ID + {{- if and .Values.oidc.existingClientSecretName .Values.oidc.existingClientIdKey }} + valueFrom: + secretKeyRef: + name: {{ .Values.oidc.existingClientSecretName }} + key: {{ .Values.oidc.existingClientIdKey }} + {{- else }} + value: {{ .Values.oidc.clientId }} + {{- end }} + - name: MINIO_IDENTITY_OPENID_CLIENT_SECRET + {{- if and .Values.oidc.existingClientSecretName .Values.oidc.existingClientSecretKey }} + valueFrom: + secretKeyRef: + name: {{ .Values.oidc.existingClientSecretName }} + key: {{ .Values.oidc.existingClientSecretKey }} + {{- else }} + value: {{ .Values.oidc.clientSecret }} + {{- end }} + - name: MINIO_IDENTITY_OPENID_CLAIM_NAME + value: {{ .Values.oidc.claimName }} + - name: MINIO_IDENTITY_OPENID_CLAIM_PREFIX + value: {{ .Values.oidc.claimPrefix }} + - name: MINIO_IDENTITY_OPENID_SCOPES + value: {{ .Values.oidc.scopes }} + - name: MINIO_IDENTITY_OPENID_COMMENT + value: {{ .Values.oidc.comment }} + - name: MINIO_IDENTITY_OPENID_REDIRECT_URI + value: {{ .Values.oidc.redirectUri }} + - name: MINIO_IDENTITY_OPENID_DISPLAY_NAME + value: {{ .Values.oidc.displayName }} + {{- end }} + {{- range $key, $val := .Values.environment }} + - name: {{ $key }} + value: {{ tpl $val $ | quote }} + {{- end }} + resources: {{- toYaml .Values.resources | nindent 12 }} + {{- if and .Values.securityContext.enabled .Values.persistence.enabled }} + {{- with .Values.containerSecurityContext }} + securityContext: {{ toYaml . | nindent 12}} + {{- end }} + {{- end }} + {{- with .Values.extraContainers }} + {{- if eq (typeOf .) "string" }} + {{- tpl . $ | nindent 8 }} + {{- else }} + {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + {{- with .Values.nodeSelector }} + nodeSelector: {{- toYaml . | nindent 8 }} + {{- end }} + {{- include "minio.imagePullSecrets" . | indent 6 }} + {{- with .Values.affinity }} + affinity: {{- toYaml . | nindent 8 }} + {{- end }} + {{- with .Values.tolerations }} + tolerations: {{- toYaml . | nindent 8 }} + {{- end }} + {{- if and (gt $replicas 1) (ge .Capabilities.KubeVersion.Major "1") (ge .Capabilities.KubeVersion.Minor "19") }} + {{- with .Values.topologySpreadConstraints }} + topologySpreadConstraints: {{- toYaml . | nindent 8 }} + {{- end }} + {{- end }} + volumes: + - name: minio-user + secret: + secretName: {{ template "minio.secretName" . }} + {{- if .Values.extraSecret }} + - name: extra-secret + secret: + secretName: {{ .Values.extraSecret }} + {{- end }} + {{- include "minio.tlsKeysVolume" . | indent 8 }} + {{- if .Values.extraVolumes }} + {{- toYaml .Values.extraVolumes | nindent 8 }} + {{- end }} + {{- if .Values.persistence.enabled }} + volumeClaimTemplates: + {{- if gt $drivesPerNode 1 }} + {{- range $diskId := until $drivesPerNode}} + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: export-{{ $diskId }} + {{- if $.Values.persistence.annotations }} + annotations: {{- toYaml $.Values.persistence.annotations | nindent 10 }} + {{- end }} + spec: + accessModes: [ {{ $accessMode | quote }} ] + {{- if $storageClass }} + storageClassName: {{ $storageClass }} + {{- end }} + resources: + requests: + storage: {{ $psize }} + {{- end }} + {{- else }} + - apiVersion: v1 + kind: PersistentVolumeClaim + metadata: + name: export + {{- if $.Values.persistence.annotations }} + annotations: {{- toYaml $.Values.persistence.annotations | nindent 10 }} + {{- end }} + spec: + accessModes: [ {{ $accessMode | quote }} ] + {{- if $storageClass }} + storageClassName: {{ $storageClass }} + {{- end }} + resources: + requests: + storage: {{ $psize }} + {{- end }} + {{- end }} +{{- end }} diff --git a/helm/minio/values.yaml b/helm/minio/values.yaml new file mode 100644 index 00000000..2ea13b10 --- /dev/null +++ b/helm/minio/values.yaml @@ -0,0 +1,627 @@ +## Provide a name in place of minio for `app:` labels +## +nameOverride: "" + +## Provide a name to substitute for the full names of resources +## +fullnameOverride: "" + +## set kubernetes cluster domain where minio is running +## +clusterDomain: cluster.local + +## Set default image, imageTag, and imagePullPolicy. mode is used to indicate the +## +image: + repository: quay.io/minio/minio + tag: RELEASE.2024-12-18T13-15-44Z + pullPolicy: IfNotPresent + +imagePullSecrets: [] +# - name: "image-pull-secret" + +## Set default image, imageTag, and imagePullPolicy for the `mc` (the minio +## client used to create a default bucket). +## +mcImage: + repository: quay.io/minio/mc + tag: RELEASE.2024-11-21T17-21-54Z + pullPolicy: IfNotPresent + +## minio mode, i.e. standalone or distributed +mode: distributed ## other supported values are "standalone" + +## Additional labels to include with deployment or statefulset +additionalLabels: {} + +## Additional annotations to include with deployment or statefulset +additionalAnnotations: {} + +## Typically the deployment/statefulset includes checksums of secrets/config, +## So that when these change on a subsequent helm install, the deployment/statefulset +## is restarted. This can result in unnecessary restarts under GitOps tooling such as +## flux, so set to "true" to disable this behaviour. +ignoreChartChecksums: false + +## Additional arguments to pass to minio binary +extraArgs: [] +# example for enabling FTP: +# - --ftp=\"address=:8021\" +# - --ftp=\"passive-port-range=10000-10010\" + +## Additional volumes to minio container +extraVolumes: [] + +## Additional volumeMounts to minio container +extraVolumeMounts: [] + +## Additional sidecar containers +extraContainers: [] + +## Internal port number for MinIO S3 API container +## Change service.port to change external port number +minioAPIPort: "9000" + +## Internal port number for MinIO Browser Console container +## Change consoleService.port to change external port number +minioConsolePort: "9001" + +## Update strategy for Deployments +deploymentUpdate: + type: RollingUpdate + maxUnavailable: 0 + maxSurge: 100% + +## Update strategy for StatefulSets +statefulSetUpdate: + updateStrategy: RollingUpdate + +## Pod priority settings +## ref: https://kubernetes.io/docs/concepts/configuration/pod-priority-preemption/ +## +priorityClassName: "" + +## Pod runtime class name +## ref https://kubernetes.io/docs/concepts/containers/runtime-class/ +## +runtimeClassName: "" + +## Set default rootUser, rootPassword +## rootUser and rootPassword is generated when not set +## Distributed MinIO ref: https://docs.min.io/community/minio-object-store/operations/install-deploy-manage/deploy-minio-multi-node-multi-drive.html +## +rootUser: "" +rootPassword: "" + +## Use existing Secret that store following variables: +## +## | Chart var | .data. in Secret | +## |:----------------------|:-------------------------| +## | rootUser | rootUser | +## | rootPassword | rootPassword | +## +## All mentioned variables will be ignored in values file. +## .data.rootUser and .data.rootPassword are mandatory, +## others depend on enabled status of corresponding sections. +existingSecret: "" + +## Directory on the MinIO pof +certsPath: "/etc/minio/certs/" +configPathmc: "/etc/minio/mc/" + +## Path where PV would be mounted on the MinIO Pod +mountPath: "/export" +## Override the root directory which the minio server should serve from. +## If left empty, it defaults to the value of {{ .Values.mountPath }} +## If defined, it must be a sub-directory of the path specified in {{ .Values.mountPath }} +## +bucketRoot: "" + +# Number of drives attached to a node +drivesPerNode: 1 +# Number of MinIO containers running +replicas: 16 +# Number of expanded MinIO clusters +pools: 1 + +## TLS Settings for MinIO +tls: + enabled: false + ## Create a secret with private.key and public.crt files and pass that here. Ref: https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret + certSecret: "" + publicCrt: public.crt + privateKey: private.key + +## Trusted Certificates Settings for MinIO. Ref: https://docs.min.io/community/minio-object-store/operations/network-encryption.html#third-party-certificate-authorities +## Bundle multiple trusted certificates into one secret and pass that here. Ref: https://github.com/minio/minio/tree/master/docs/tls/kubernetes#2-create-kubernetes-secret +## When using self-signed certificates, remember to include MinIO's own certificate in the bundle with key public.crt. +## If certSecret is left empty and tls is enabled, this chart installs the public certificate from .Values.tls.certSecret. +trustedCertsSecret: "" + +## Enable persistence using Persistent Volume Claims +## ref: http://kubernetes.io/docs/user-guide/persistent-volumes/ +## +persistence: + enabled: true + annotations: {} + + ## A manually managed Persistent Volume and Claim + ## Requires persistence.enabled: true + ## If defined, PVC must be created manually before volume will be bound + existingClaim: "" + + ## minio data Persistent Volume Storage Class + ## If defined, storageClassName: + ## If set to "-", storageClassName: "", which disables dynamic provisioning + ## If undefined (the default) or set to null, no storageClassName spec is + ## set, choosing the default provisioner. (gp2 on AWS, standard on + ## GKE, AWS & OpenStack) + ## + ## Storage class of PV to bind. By default it looks for standard storage class. + ## If the PV uses a different storage class, specify that here. + storageClass: "" + volumeName: "" + accessMode: ReadWriteOnce + size: 500Gi + + ## If subPath is set mount a sub folder of a volume instead of the root of the volume. + ## This is especially handy for volume plugins that don't natively support sub mounting (like glusterfs). + ## + subPath: "" + +## Expose the MinIO service to be accessed from outside the cluster (LoadBalancer service). +## or access it from within the cluster (ClusterIP service). Set the service type and the port to serve it. +## ref: http://kubernetes.io/docs/user-guide/services/ +## +service: + type: ClusterIP + clusterIP: ~ + port: "9000" + nodePort: 32000 + loadBalancerIP: ~ + externalIPs: [] + annotations: {} + + ## service.loadBalancerSourceRanges Addresses that are allowed when service is LoadBalancer + ## https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## + #loadBalancerSourceRanges: + # - 10.10.10.0/24 + loadBalancerSourceRanges: [] + + ## service.externalTrafficPolicy minio service external traffic policy + ## ref http://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip + ## + externalTrafficPolicy: Cluster + +## Configure Ingress based on the documentation here: https://kubernetes.io/docs/concepts/services-networking/ingress/ +## + +ingress: + enabled: false + ingressClassName: ~ + labels: + {} + # node-role.kubernetes.io/ingress: platform + annotations: + {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + # kubernetes.io/ingress.allow-http: "false" + # kubernetes.io/ingress.global-static-ip-name: "" + # nginx.ingress.kubernetes.io/secure-backends: "true" + # nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + # nginx.ingress.kubernetes.io/whitelist-source-range: 0.0.0.0/0 + path: / + hosts: + - minio-example.local + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +consoleService: + type: ClusterIP + clusterIP: ~ + port: "9001" + nodePort: 32001 + loadBalancerIP: ~ + externalIPs: [] + annotations: {} + ## consoleService.loadBalancerSourceRanges Addresses that are allowed when service is LoadBalancer + ## https://kubernetes.io/docs/tasks/access-application-cluster/configure-cloud-provider-firewall/#restrict-access-for-loadbalancer-service + ## + #loadBalancerSourceRanges: + # - 10.10.10.0/24 + loadBalancerSourceRanges: [] + + ## servconsoleServiceice.externalTrafficPolicy minio service external traffic policy + ## ref http://kubernetes.io/docs/tasks/access-application-cluster/create-external-load-balancer/#preserving-the-client-source-ip + ## + externalTrafficPolicy: Cluster + +consoleIngress: + enabled: false + ingressClassName: ~ + labels: + {} + # node-role.kubernetes.io/ingress: platform + annotations: + {} + # kubernetes.io/ingress.class: nginx + # kubernetes.io/tls-acme: "true" + # kubernetes.io/ingress.allow-http: "false" + # kubernetes.io/ingress.global-static-ip-name: "" + # nginx.ingress.kubernetes.io/secure-backends: "true" + # nginx.ingress.kubernetes.io/backend-protocol: "HTTPS" + # nginx.ingress.kubernetes.io/whitelist-source-range: 0.0.0.0/0 + path: / + hosts: + - console.minio-example.local + tls: [] + # - secretName: chart-example-tls + # hosts: + # - chart-example.local + +## Node labels for pod assignment +## Ref: https://kubernetes.io/docs/user-guide/node-selection/ +## +nodeSelector: {} +tolerations: [] +affinity: {} +topologySpreadConstraints: [] + +## Add stateful containers to have security context, if enabled MinIO will run as this +## user and group NOTE: securityContext is only enabled if persistence.enabled=true +securityContext: + enabled: true + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + fsGroupChangePolicy: "OnRootMismatch" + +containerSecurityContext: + readOnlyRootFilesystem: false + +# Additational pod annotations +podAnnotations: {} + +# Additional pod labels +podLabels: {} + +## Configure resource requests and limits +## ref: http://kubernetes.io/docs/user-guide/compute-resources/ +## +resources: + requests: + memory: 16Gi + +## List of policies to be created after minio install +## +## In addition to default policies [readonly|readwrite|writeonly|consoleAdmin|diagnostics] +## you can define additional policies with custom supported actions and resources +policies: [] +## writeexamplepolicy policy grants creation or deletion of buckets with name +## starting with example. In addition, grants objects write permissions on buckets starting with +## example. +# - name: writeexamplepolicy +# statements: +# - effect: Allow # this is the default +# resources: +# - 'arn:aws:s3:::example*/*' +# actions: +# - "s3:AbortMultipartUpload" +# - "s3:GetObject" +# - "s3:DeleteObject" +# - "s3:PutObject" +# - "s3:ListMultipartUploadParts" +# - resources: +# - 'arn:aws:s3:::example*' +# actions: +# - "s3:CreateBucket" +# - "s3:DeleteBucket" +# - "s3:GetBucketLocation" +# - "s3:ListBucket" +# - "s3:ListBucketMultipartUploads" +## readonlyexamplepolicy policy grants access to buckets with name starting with example. +## In addition, grants objects read permissions on buckets starting with example. +# - name: readonlyexamplepolicy +# statements: +# - resources: +# - 'arn:aws:s3:::example*/*' +# actions: +# - "s3:GetObject" +# - resources: +# - 'arn:aws:s3:::example*' +# actions: +# - "s3:GetBucketLocation" +# - "s3:ListBucket" +# - "s3:ListBucketMultipartUploads" +## conditionsexample policy creates all access to example bucket with aws:username="johndoe" and source ip range 10.0.0.0/8 and 192.168.0.0/24 only +# - name: conditionsexample +# statements: +# - resources: +# - 'arn:aws:s3:::example/*' +# actions: +# - 's3:*' +# conditions: +# - StringEquals: '"aws:username": "johndoe"' +# - IpAddress: | +# "aws:SourceIp": [ +# "10.0.0.0/8", +# "192.168.0.0/24" +# ] +# +## Additional Annotations for the Kubernetes Job makePolicyJob +makePolicyJob: + securityContext: + enabled: false + runAsUser: 1000 + runAsGroup: 1000 + resources: + requests: + memory: 128Mi + # Command to run after the main command on exit + exitCommand: "" + +## List of users to be created after minio install +## +users: + ## Username, password and policy to be assigned to the user + ## Default policies are [readonly|readwrite|writeonly|consoleAdmin|diagnostics] + ## Add new policies as explained here https://docs.min.io/community/minio-object-store/administration/identity-access-management.html#access-management + ## NOTE: this will fail if LDAP is enabled in your MinIO deployment + ## make sure to disable this if you are using LDAP. + - accessKey: console + secretKey: console123 + policy: consoleAdmin + # Or you can refer to specific secret + #- accessKey: externalSecret + # existingSecret: my-secret + # existingSecretKey: password + # policy: readonly + +## Additional Annotations for the Kubernetes Job makeUserJob +makeUserJob: + securityContext: + enabled: false + runAsUser: 1000 + runAsGroup: 1000 + resources: + requests: + memory: 128Mi + # Command to run after the main command on exit + exitCommand: "" + +## List of service accounts to be created after minio install +## +svcaccts: + [] + ## accessKey, secretKey and parent user to be assigned to the service accounts + ## Add new service accounts as explained here https://docs.min.io/community/minio-object-store/administration/identity-access-management/minio-user-management.html#access-keys + # - accessKey: console-svcacct + # secretKey: console123 + # user: console + ## Or you can refer to specific secret + # - accessKey: externalSecret + # existingSecret: my-secret + # existingSecretKey: password + # user: console + ## You also can pass custom policy + # - accessKey: console-svcacct + # secretKey: console123 + # user: console + # policy: + # statements: + # - resources: + # - 'arn:aws:s3:::example*/*' + # actions: + # - "s3:AbortMultipartUpload" + # - "s3:GetObject" + # - "s3:DeleteObject" + # - "s3:PutObject" + # - "s3:ListMultipartUploadParts" + +makeServiceAccountJob: + securityContext: + enabled: false + runAsUser: 1000 + runAsGroup: 1000 + resources: + requests: + memory: 128Mi + # Command to run after the main command on exit + exitCommand: "" + +## List of buckets to be created after minio install +## +buckets: + [] + # # Name of the bucket + # - name: bucket1 + # # Policy to be set on the + # # bucket [none|download|upload|public] + # policy: none + # # Purge if bucket exists already + # purge: false + # # set versioning for + # # bucket [true|false] + # versioning: false # remove this key if you do not want versioning feature + # # set objectlocking for + # # bucket [true|false] NOTE: versioning is enabled by default if you use locking + # objectlocking: false + # - name: bucket2 + # policy: none + # purge: false + # versioning: true + # # set objectlocking for + # # bucket [true|false] NOTE: versioning is enabled by default if you use locking + # objectlocking: false + +## Additional Annotations for the Kubernetes Job makeBucketJob +makeBucketJob: + securityContext: + enabled: false + runAsUser: 1000 + runAsGroup: 1000 + resources: + requests: + memory: 128Mi + # Command to run after the main command on exit + exitCommand: "" + +## List of command to run after minio install +## NOTE: the mc command TARGET is always "myminio" +customCommands: + # - command: "admin policy attach myminio consoleAdmin --group='cn=ops,cn=groups,dc=example,dc=com'" + +## Additional Annotations for the Kubernetes Job customCommandJob +customCommandJob: + securityContext: + enabled: false + runAsUser: 1000 + runAsGroup: 1000 + resources: + requests: + memory: 128Mi + ## Additional volumes to add to the post-job. + extraVolumes: + [] + # - name: extra-policies + # configMap: + # name: my-extra-policies-cm + ## Additional volumeMounts to add to the custom commands container when + ## running the post-job. + extraVolumeMounts: + [] + # - name: extra-policies + # mountPath: /mnt/extras/ + # Command to run after the main command on exit + exitCommand: "" + +## Merge jobs +postJob: + podAnnotations: {} + annotations: {} + securityContext: + enabled: false + runAsUser: 1000 + runAsGroup: 1000 + fsGroup: 1000 + nodeSelector: {} + tolerations: [] + affinity: {} + +## Use this field to add environment variables relevant to MinIO server. These fields will be passed on to MinIO container(s) +## when Chart is deployed +environment: + ## Please refer for comprehensive list https://docs.min.io/community/minio-object-store/reference/minio-server/minio-server.html + ## MINIO_SUBNET_LICENSE: "License key obtained from https://subnet.min.io" + ## MINIO_BROWSER: "off" + +## The name of a secret in the same kubernetes namespace which contain secret values +## This can be useful for LDAP password, etc +## The key in the secret must be 'config.env' +## +extraSecret: ~ + +## OpenID Identity Management +## The following section documents environment variables for enabling external identity management using an OpenID Connect (OIDC)-compatible provider. +## See https://docs.min.io/community/minio-object-store/operations/external-iam/configure-openid-external-identity-management.html for a tutorial on using these variables. +oidc: + enabled: false + configUrl: "https://identity-provider-url/.well-known/openid-configuration" + clientId: "minio" + clientSecret: "" + # Provide existing client secret from the Kubernetes Secret resource, existing secret will have priority over `clientId` and/or `clientSecret`` + existingClientSecretName: "" + existingClientIdKey: "" + existingClientSecretKey: "" + claimName: "policy" + scopes: "openid,profile,email" + redirectUri: "https://console-endpoint-url/oauth_callback" + # Can leave empty + claimPrefix: "" + comment: "" + displayName: "" + +networkPolicy: + enabled: false + # Specifies whether the policies created will be standard Network Policies (flavor: kubernetes) + # or Cilium Network Policies (flavor: cilium) + flavor: kubernetes + # allows external access to the minio api + allowExternal: true + ## @params networkPolicy.egress configuration of the egress traffic + egress: + ## @param networkPolicy.egress.enabled When enabled, an egress network policy will be + ## created allowing minio to connect to external data sources from kubernetes cluster. + ## + enabled: false + ## @param networkPolicy.egress.ports Add individual ports to be allowed by the egress + ## Add ports to the egress by specifying - port: + ## E.X. + ## - port: 80 + ## - port: 443 + ## - port: 53 + ## protocol: UDP + ## + ports: [] + ## @param networkPolicy.egress.to Allow egress traffic to specific destinations + ## Add destinations to the egress by specifying - ipBlock: + ## E.X. + ## to: + ## - namespaceSelector: + ## matchExpressions: + ## - {key: role, operator: In, values: [minio]} + ## + to: [] + # only when using flavor: cilium + egressEntities: + - kube-apiserver + +## PodDisruptionBudget settings +## ref: https://kubernetes.io/docs/concepts/workloads/pods/disruptions/ +## +podDisruptionBudget: + enabled: false + maxUnavailable: 1 + +## Specify the service account to use for the MinIO pods. If 'create' is set to 'false' +## and 'name' is left unspecified, the account 'default' will be used. +serviceAccount: + create: true + ## The name of the service account to use. If 'create' is 'true', a service account with that name + ## will be created. + name: "minio-sa" + +metrics: + serviceMonitor: + enabled: false + # scrape each node/pod individually for additional metrics + includeNode: false + public: true + additionalLabels: {} + annotations: {} + # for node metrics + relabelConfigs: {} + # for cluster metrics + relabelConfigsCluster: + {} + # metricRelabelings: + # - regex: (server|pod) + # action: labeldrop + namespace: ~ + # Scrape interval, for example `interval: 30s` + interval: ~ + # Scrape timeout, for example `scrapeTimeout: 10s` + scrapeTimeout: ~ + +## ETCD settings: https://github.com/minio/minio/blob/master/docs/sts/etcd.md +## Define endpoints to enable this section. +etcd: + endpoints: [] + pathPrefix: "" + corednsPathPrefix: "" + clientCert: "" + clientCertKey: "" diff --git a/index.yaml b/index.yaml new file mode 100644 index 00000000..7d62f602 --- /dev/null +++ b/index.yaml @@ -0,0 +1,1968 @@ +apiVersion: v1 +entries: + minio: + - apiVersion: v1 + appVersion: RELEASE.2024-12-18T13-15-44Z + created: "2025-01-02T21:34:25.234658257-08:00" + description: High Performance Object Storage + digest: 25fa2740480d1ebc9e64340854a6c42d3a7bc39c2a77378da91b21f144faa9af + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-5.4.0.tgz + version: 5.4.0 + - apiVersion: v1 + appVersion: RELEASE.2024-04-18T19-09-19Z + created: "2025-01-02T21:34:25.231025201-08:00" + description: High Performance Object Storage + digest: 5f927286767c285b925a3395e75b4f372367f83d2124395185e21dc7fd4ca177 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-5.3.0.tgz + version: 5.3.0 + - apiVersion: v1 + appVersion: RELEASE.2024-04-18T19-09-19Z + created: "2025-01-02T21:34:25.227480037-08:00" + description: High Performance Object Storage + digest: 8ef4212d7d51be6c8192b3e91138a9ca918ca56142c42500028cfd3b80e0b2dd + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-5.2.0.tgz + version: 5.2.0 + - apiVersion: v1 + appVersion: RELEASE.2024-03-03T17-50-39Z + created: "2025-01-02T21:34:25.221946278-08:00" + description: High Performance Object Storage + digest: 742d658c029616f0a977f255a27e806f2e3ef31f0d30467353a0882b5607001e + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-5.1.0.tgz + version: 5.1.0 + - apiVersion: v1 + appVersion: RELEASE.2024-01-11T07-46-16Z + created: "2025-01-02T21:34:25.188561933-08:00" + description: Multi-Cloud Object Storage + digest: 3a2d8e03ffdd98501026aa7561633c91d9871647f4b01d77b75a2ad9b72ee618 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-5.0.15.tgz + version: 5.0.15 + - apiVersion: v1 + appVersion: RELEASE.2023-09-30T07-02-29Z + created: "2025-01-02T21:34:25.184512596-08:00" + description: Multi-Cloud Object Storage + digest: 6c3656924fbad2cb17f810cd78f352f9b60626aaec64b837c96829095b215ad3 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-5.0.14.tgz + version: 5.0.14 + - apiVersion: v1 + appVersion: RELEASE.2023-07-07T07-13-57Z + created: "2025-01-02T21:34:25.180913342-08:00" + description: Multi-Cloud Object Storage + digest: 3c18f7381efe6d86497f952e6d5f59003ee5a009c54778ddea1ee8d3c7bed9c8 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-5.0.13.tgz + version: 5.0.13 + - apiVersion: v1 + appVersion: RELEASE.2023-07-07T07-13-57Z + created: "2025-01-02T21:34:25.177247018-08:00" + description: Multi-Cloud Object Storage + digest: 5318bc56c73a8f4539c3dd178f4d55c7f41bee4a25d7dc02ac6a5843eeee7976 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-5.0.12.tgz + version: 5.0.12 + - apiVersion: v1 + appVersion: RELEASE.2023-06-19T19-52-50Z + created: "2025-01-02T21:34:25.17337971-08:00" + description: Multi-Cloud Object Storage + digest: cba44c8cddcda1fb5c082dce82004a39f53cc20677ab9698a6998f01efefd8db + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-5.0.11.tgz + version: 5.0.11 + - apiVersion: v1 + appVersion: RELEASE.2023-05-18T00-05-36Z + created: "2025-01-02T21:34:25.169502301-08:00" + description: Multi-Cloud Object Storage + digest: a3d55b12f38a2049ddf3efe35b38b6dc4e59777452b72d18d5a82f3378deb9cd + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-5.0.10.tgz + version: 5.0.10 + - apiVersion: v1 + appVersion: RELEASE.2023-04-28T18-11-17Z + created: "2025-01-02T21:34:25.218260054-08:00" + description: Multi-Cloud Object Storage + digest: cf98985e32675e4ce327304ea9ac61046a788b3d5190d6b501330f7803d41a11 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-5.0.9.tgz + version: 5.0.9 + - apiVersion: v1 + appVersion: RELEASE.2023-04-13T03-08-07Z + created: "2025-01-02T21:34:25.214515045-08:00" + description: Multi-Cloud Object Storage + digest: 034d68f85799f6693836975797f85a91842cf2d003a6c4ff401bd4ea4c946af6 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-5.0.8.tgz + version: 5.0.8 + - apiVersion: v1 + appVersion: RELEASE.2023-02-10T18-48-39Z + created: "2025-01-02T21:34:25.210879405-08:00" + description: Multi-Cloud Object Storage + digest: 3f935a310e1b5b873052629b66005c160356ca7b2bd394cb07b34dbaf9905e3f + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-5.0.7.tgz + version: 5.0.7 + - apiVersion: v1 + appVersion: RELEASE.2023-02-10T18-48-39Z + created: "2025-01-02T21:34:25.207094353-08:00" + description: Multi-Cloud Object Storage + digest: 82ef858ce483c2d736444792986cb36bd0fb4fc90a80b97fe30d7b2f2034d24a + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-5.0.6.tgz + version: 5.0.6 + - apiVersion: v1 + appVersion: RELEASE.2023-01-31T02-24-19Z + created: "2025-01-02T21:34:25.201959046-08:00" + description: Multi-Cloud Object Storage + digest: fefeea10e4e525e45f82fb80a03900d34605ec432dd92f56d94eaf4fb1b98c41 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-5.0.5.tgz + version: 5.0.5 + - apiVersion: v1 + appVersion: RELEASE.2022-12-12T19-27-27Z + created: "2025-01-02T21:34:25.198369173-08:00" + description: Multi-Cloud Object Storage + digest: 6b305783c98b0b97ffab079ff4430094fd0ca6e98e82bb8153cb93033a1bf40f + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-5.0.4.tgz + version: 5.0.4 + - apiVersion: v1 + appVersion: RELEASE.2022-12-12T19-27-27Z + created: "2025-01-02T21:34:25.194953084-08:00" + description: Multi-Cloud Object Storage + digest: bac89157c53b324aece263c294aa49f5c9b64f426b4b06c9bca3d72e77e244f2 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-5.0.3.tgz + version: 5.0.3 + - apiVersion: v1 + appVersion: RELEASE.2022-12-12T19-27-27Z + created: "2025-01-02T21:34:25.191760917-08:00" + description: Multi-Cloud Object Storage + digest: 935ce4f09366231b11d414d626f887fa6fa6024dd30a42e81e810ca1438d5904 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-5.0.2.tgz + version: 5.0.2 + - apiVersion: v1 + appVersion: RELEASE.2022-11-11T03-44-20Z + created: "2025-01-02T21:34:25.164816778-08:00" + description: Multi-Cloud Object Storage + digest: 3e952c5d737980b8ccdfb819021eafb4b4e8da226f764a1dc3de1ba63ceb1ffa + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-5.0.1.tgz + version: 5.0.1 + - apiVersion: v1 + appVersion: RELEASE.2022-10-24T18-35-07Z + created: "2025-01-02T21:34:25.16141762-08:00" + description: Multi-Cloud Object Storage + digest: 6215c800d84fd4c40e4fb4142645fc1c6a039c251776a3cc8c11a24b9e3b59c7 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-5.0.0.tgz + version: 5.0.0 + - apiVersion: v1 + appVersion: RELEASE.2022-10-24T18-35-07Z + created: "2025-01-02T21:34:25.157595167-08:00" + description: Multi-Cloud Object Storage + digest: 2d3d884490ea1127742f938bc9382844bae713caae08b3308f766f3c9000659a + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-4.1.0.tgz + version: 4.1.0 + - apiVersion: v1 + appVersion: RELEASE.2022-09-17T00-09-45Z + created: "2025-01-02T21:34:25.122758935-08:00" + description: Multi-Cloud Object Storage + digest: 6f16f2dbfed91ab81a7fae60b6ea32f554365bd27bf5fda55b64a0fa264f4252 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-4.0.15.tgz + version: 4.0.15 + - apiVersion: v1 + appVersion: RELEASE.2022-09-01T23-53-36Z + created: "2025-01-02T21:34:25.118898654-08:00" + description: Multi-Cloud Object Storage + digest: 35d89d8f49d53ea929466fb88ee26123431326033f1387e6b2d536a629c0a398 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-4.0.14.tgz + version: 4.0.14 + - apiVersion: v1 + appVersion: RELEASE.2022-08-22T23-53-06Z + created: "2025-01-02T21:34:25.115194076-08:00" + description: Multi-Cloud Object Storage + digest: 5b86937ca88d9f6046141fdc2b1cc54760435ed92d289cd0a115fa7148781d4e + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-4.0.13.tgz + version: 4.0.13 + - apiVersion: v1 + appVersion: RELEASE.2022-08-13T21-54-44Z + created: "2025-01-02T21:34:25.111485897-08:00" + description: Multi-Cloud Object Storage + digest: 2d9c227c0f46ea8bdef4d760c212156fd4c6623ddc5406779c569fe925527787 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-4.0.12.tgz + version: 4.0.12 + - apiVersion: v1 + appVersion: RELEASE.2022-08-05T23-27-09Z + created: "2025-01-02T21:34:25.107832294-08:00" + description: Multi-Cloud Object Storage + digest: 6caaffcb636e040cd7e8bc4883a1674a673757f4781c32d53b5ec0f41fea3944 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-4.0.11.tgz + version: 4.0.11 + - apiVersion: v1 + appVersion: RELEASE.2022-08-02T23-59-16Z + created: "2025-01-02T21:34:25.103772055-08:00" + description: Multi-Cloud Object Storage + digest: 841d87788fb094d6a7d8a91e91821fe1e847bc952e054c781fc93742d112e18a + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-4.0.10.tgz + version: 4.0.10 + - apiVersion: v1 + appVersion: RELEASE.2022-08-02T23-59-16Z + created: "2025-01-02T21:34:25.153995865-08:00" + description: Multi-Cloud Object Storage + digest: 6f1a78382df3215deac07495a5e7de7009a1153b4cf6cb565630652a69aec4cf + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-4.0.9.tgz + version: 4.0.9 + - apiVersion: v1 + appVersion: RELEASE.2022-07-29T19-40-48Z + created: "2025-01-02T21:34:25.150678505-08:00" + description: Multi-Cloud Object Storage + digest: d11db37963636922cb778b6bc0ad2ca4724cb391ea7b785995ada52467d7dd83 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-4.0.8.tgz + version: 4.0.8 + - apiVersion: v1 + appVersion: RELEASE.2022-07-26T00-53-03Z + created: "2025-01-02T21:34:25.146767779-08:00" + description: Multi-Cloud Object Storage + digest: ca775e08c84331bb5029d4d29867d30c16e2c62e897788eb432212a756e91e4e + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-4.0.7.tgz + version: 4.0.7 + - apiVersion: v1 + appVersion: RELEASE.2022-05-08T23-50-31Z + created: "2025-01-02T21:34:25.142770749-08:00" + description: Multi-Cloud Object Storage + digest: 06542b8f3d149d5908b15de9a8d6f8cf304af0213830be56dc315785d14f9ccd + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-4.0.6.tgz + version: 4.0.6 + - apiVersion: v1 + appVersion: RELEASE.2022-05-08T23-50-31Z + created: "2025-01-02T21:34:25.139151034-08:00" + description: Multi-Cloud Object Storage + digest: dd2676362f067454a496cdd293609d0c904b08f521625af49f95402a024ba1f5 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-4.0.5.tgz + version: 4.0.5 + - apiVersion: v1 + appVersion: RELEASE.2022-05-08T23-50-31Z + created: "2025-01-02T21:34:25.135573416-08:00" + description: Multi-Cloud Object Storage + digest: bab9ef192d4eda4c572ad0ce0cf551736c847f582d1837d6833ee10543c23167 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-4.0.4.tgz + version: 4.0.4 + - apiVersion: v1 + appVersion: RELEASE.2022-05-08T23-50-31Z + created: "2025-01-02T21:34:25.132238833-08:00" + description: Multi-Cloud Object Storage + digest: c770bb9841c76576e4e8573f78b0ec33e0d729504c9667e67ad62d48df5ed64c + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-4.0.3.tgz + version: 4.0.3 + - apiVersion: v1 + appVersion: RELEASE.2022-05-08T23-50-31Z + created: "2025-01-02T21:34:25.128974045-08:00" + description: Multi-Cloud Object Storage + digest: 95835f4199d963e2a23a2493610b348e6f2ff8b71c1a648c4a3b84af9b7a83eb + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-4.0.2.tgz + version: 4.0.2 + - apiVersion: v1 + appVersion: RELEASE.2022-04-30T22-23-53Z + created: "2025-01-02T21:34:25.099393644-08:00" + description: Multi-Cloud Object Storage + digest: 55a088c403b056e1f055a97426aa11759c3d6cbad38face170fe6cbbec7d568f + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-4.0.1.tgz + version: 4.0.1 + - apiVersion: v1 + appVersion: RELEASE.2022-04-26T01-20-24Z + created: "2025-01-02T21:34:25.095908528-08:00" + description: Multi-Cloud Object Storage + digest: f541237e24336ec3f7f45ae0d523fef694e3a2f9ef648c5b11c15734db6ba2b2 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-4.0.0.tgz + version: 4.0.0 + - apiVersion: v1 + appVersion: RELEASE.2022-04-16T04-26-02Z + created: "2025-01-02T21:34:25.092803423-08:00" + description: Multi-Cloud Object Storage + digest: edc0c3dd6d5246a06b74ba16bb4aff80a6d7225dc9aecf064fd89a8af371b9c1 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.6.6.tgz + version: 3.6.6 + - apiVersion: v1 + appVersion: RELEASE.2022-04-12T06-55-35Z + created: "2025-01-02T21:34:25.089672015-08:00" + description: Multi-Cloud Object Storage + digest: 211e89f6b9eb0b9a3583abaa127be60e1f9717a098e6b2858cb9dc1cc50c1650 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.6.5.tgz + version: 3.6.5 + - apiVersion: v1 + appVersion: RELEASE.2022-04-09T15-09-52Z + created: "2025-01-02T21:34:25.086239968-08:00" + description: Multi-Cloud Object Storage + digest: 534a879d73b370a18b554b93d0930e1c115419619c4ce4ec7dbaae632acacf06 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.6.4.tgz + version: 3.6.4 + - apiVersion: v1 + appVersion: RELEASE.2022-03-24T00-43-44Z + created: "2025-01-02T21:34:25.081664315-08:00" + description: Multi-Cloud Object Storage + digest: 99508b20eb0083a567dcccaf9a6c237e09575ed1d70cd2e8333f89c472d13d75 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.6.3.tgz + version: 3.6.3 + - apiVersion: v1 + appVersion: RELEASE.2022-03-17T06-34-49Z + created: "2025-01-02T21:34:25.078433537-08:00" + description: Multi-Cloud Object Storage + digest: b4cd25611ca322b1d23d23112fdfa6b068fd91eefe0b0663b88ff87ea4282495 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.6.2.tgz + version: 3.6.2 + - apiVersion: v1 + appVersion: RELEASE.2022-03-14T18-25-24Z + created: "2025-01-02T21:34:25.075113944-08:00" + description: Multi-Cloud Object Storage + digest: d75b88162bfe54740a233bcecf87328bba2ae23d170bec3a35c828bc6fdc224c + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.6.1.tgz + version: 3.6.1 + - apiVersion: v1 + appVersion: RELEASE.2022-03-11T23-57-45Z + created: "2025-01-02T21:34:25.07170837-08:00" + description: Multi-Cloud Object Storage + digest: 22e53a1184a21a679bc7d8b94e955777f3506340fc29da5ab0cb6d729bdbde8d + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.6.0.tgz + version: 3.6.0 + - apiVersion: v1 + appVersion: RELEASE.2022-03-03T21-21-16Z + created: "2025-01-02T21:34:25.067175653-08:00" + description: Multi-Cloud Object Storage + digest: 6fda968d3fdfd60470c0055a4e1a3bd8e5aee9ad0af5ba2fb7b7b926fdc9e4a0 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.5.9.tgz + version: 3.5.9 + - apiVersion: v1 + appVersion: RELEASE.2022-02-26T02-54-46Z + created: "2025-01-02T21:34:25.063997563-08:00" + description: Multi-Cloud Object Storage + digest: 8e015369048a3a82bbd53ad36696786f18561c6b25d14eee9e2c93a7336cef46 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.5.8.tgz + version: 3.5.8 + - apiVersion: v1 + appVersion: RELEASE.2022-02-18T01-50-10Z + created: "2025-01-02T21:34:25.058867444-08:00" + description: Multi-Cloud Object Storage + digest: cb3543fe748e5f0d59b3ccf4ab9af8e10b731405ae445d1f5715e30013632373 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.5.7.tgz + version: 3.5.7 + - apiVersion: v1 + appVersion: RELEASE.2022-02-18T01-50-10Z + created: "2025-01-02T21:34:25.055866713-08:00" + description: Multi-Cloud Object Storage + digest: f2e359fa5eefffc59abb3d14a8fa94b11ddeaa99f6cd8dd5f40f4e04121000d6 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.5.6.tgz + version: 3.5.6 + - apiVersion: v1 + appVersion: RELEASE.2022-02-16T00-35-27Z + created: "2025-01-02T21:34:25.052552978-08:00" + description: Multi-Cloud Object Storage + digest: 529d56cca9d83a3d0e5672e63b6e87b5bcbe10a6b45f7a55ba998cceb32f9c81 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.5.5.tgz + version: 3.5.5 + - apiVersion: v1 + appVersion: RELEASE.2022-02-12T00-51-25Z + created: "2025-01-02T21:34:25.049153108-08:00" + description: Multi-Cloud Object Storage + digest: 3d530598f8ece67bec5b7f990d206584893987c713502f9228e4ee24b5535414 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.5.4.tgz + version: 3.5.4 + - apiVersion: v1 + appVersion: RELEASE.2022-02-12T00-51-25Z + created: "2025-01-02T21:34:25.045984459-08:00" + description: Multi-Cloud Object Storage + digest: 53937031348b29615f07fc4869b2d668391d8ba9084630a497abd7a7dea9dfb0 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.5.3.tgz + version: 3.5.3 + - apiVersion: v1 + appVersion: RELEASE.2022-02-07T08-17-33Z + created: "2025-01-02T21:34:25.042945494-08:00" + description: Multi-Cloud Object Storage + digest: 68d643414ff0d565716c5715034fcbf1af262e041915a5c02eb51ec1a65c1ea0 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.5.2.tgz + version: 3.5.2 + - apiVersion: v1 + appVersion: RELEASE.2022-02-01T18-00-14Z + created: "2025-01-02T21:34:25.038683645-08:00" + description: Multi-Cloud Object Storage + digest: a3e855ed0f31233b989fffd775a29d6fbfa0590089010ff16783fd7f142ef6e7 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.5.1.tgz + version: 3.5.1 + - apiVersion: v1 + appVersion: RELEASE.2022-02-01T18-00-14Z + created: "2025-01-02T21:34:25.03587265-08:00" + description: Multi-Cloud Object Storage + digest: b1b0ae3c54b4260a698753e11d7781bb8ddc67b7e3fbf0af82796e4cd4ef92a3 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.5.0.tgz + version: 3.5.0 + - apiVersion: v1 + appVersion: RELEASE.2022-01-28T02-28-16Z + created: "2025-01-02T21:34:25.032826604-08:00" + description: Multi-Cloud Object Storage + digest: fecf25d2d3fb208c6f894fed642a60780a570b7f6d0adddde846af7236dc80aa + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.4.8.tgz + version: 3.4.8 + - apiVersion: v1 + appVersion: RELEASE.2022-01-25T19-56-04Z + created: "2025-01-02T21:34:25.029589236-08:00" + description: Multi-Cloud Object Storage + digest: c78008caa5ce98f64c887630f59d0cbd481cb3f19a7d4e9d3e81bf4e1e45cadc + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.4.7.tgz + version: 3.4.7 + - apiVersion: v1 + appVersion: RELEASE.2022-01-08T03-11-54Z + created: "2025-01-02T21:34:25.026512118-08:00" + description: Multi-Cloud Object Storage + digest: 8f2e2691bf897f74ff094dd370ec56ba9d417e5e8926710c14c2ba346330238d + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.4.6.tgz + version: 3.4.6 + - apiVersion: v1 + appVersion: RELEASE.2022-01-04T07-41-07Z + created: "2025-01-02T21:34:25.023266957-08:00" + description: Multi-Cloud Object Storage + digest: bacd140f0016fab35f516bde787da6449b3a960c071fad9e4b6563118033ac84 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.4.5.tgz + version: 3.4.5 + - apiVersion: v1 + appVersion: RELEASE.2021-12-29T06-49-06Z + created: "2025-01-02T21:34:25.020285989-08:00" + description: Multi-Cloud Object Storage + digest: 48a453ea5ffeef25933904caefd9470bfb26224dfc2d1096bd0031467ba53007 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.4.4.tgz + version: 3.4.4 + - apiVersion: v1 + appVersion: RELEASE.2021-12-20T22-07-16Z + created: "2025-01-02T21:34:25.014477173-08:00" + description: Multi-Cloud Object Storage + digest: 47ef4a930713b98f9438ceca913c6e700f85bb25dba5624b056486254b5f0c60 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.4.3.tgz + version: 3.4.3 + - apiVersion: v1 + appVersion: RELEASE.2021-12-20T22-07-16Z + created: "2025-01-02T21:34:25.011715909-08:00" + description: Multi-Cloud Object Storage + digest: d6763f7e2ea66810bd55eb225579a9c3b968f9ae1256f45fd469362e55d846ff + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.4.2.tgz + version: 3.4.2 + - apiVersion: v1 + appVersion: RELEASE.2021-12-10T23-03-39Z + created: "2025-01-02T21:34:25.009018639-08:00" + description: Multi-Cloud Object Storage + digest: 2fb822c87216ba3fc2ae51a54a0a3e239aa560d86542991504a841cc2a2b9a37 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.4.1.tgz + version: 3.4.1 + - apiVersion: v1 + appVersion: RELEASE.2021-12-18T04-42-33Z + created: "2025-01-02T21:34:25.006295652-08:00" + description: Multi-Cloud Object Storage + digest: fa8ba1aeb1a15316c6be8403416a5e6b5e6139b7166592087e7bddc9e6db5453 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.4.0.tgz + version: 3.4.0 + - apiVersion: v1 + appVersion: RELEASE.2021-12-10T23-03-39Z + created: "2025-01-02T21:34:25.003243793-08:00" + description: Multi-Cloud Object Storage + digest: b9b0af9ca50b8d00868e1f1b989dca275829d9110af6de91bb9b3a398341e894 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.3.4.tgz + version: 3.3.4 + - apiVersion: v1 + appVersion: RELEASE.2021-12-10T23-03-39Z + created: "2025-01-02T21:34:24.999956538-08:00" + description: Multi-Cloud Object Storage + digest: f8b22a5b8fe95a7ddf61b825e17d11c9345fb10e4c126b0d78381608aa300a08 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.3.3.tgz + version: 3.3.3 + - apiVersion: v1 + appVersion: RELEASE.2021-12-10T23-03-39Z + created: "2025-01-02T21:34:24.995166842-08:00" + description: Multi-Cloud Object Storage + digest: c48d474f269427abe5ab446f00687d0625b3d1adfc5c73bdb4b21ca9e42853fb + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.3.2.tgz + version: 3.3.2 + - apiVersion: v1 + appVersion: RELEASE.2021-11-24T23-19-33Z + created: "2025-01-02T21:34:24.992276741-08:00" + description: Multi-Cloud Object Storage + digest: 7c3da39d9b0090cbf5efedf0cc163a1e2df05becc5152c3add8e837384690bc4 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.3.1.tgz + version: 3.3.1 + - apiVersion: v1 + appVersion: RELEASE.2021-11-24T23-19-33Z + created: "2025-01-02T21:34:24.989284049-08:00" + description: Multi-Cloud Object Storage + digest: 50d6590b4cc779c40f81cc13b1586fbe508aa7f3230036c760bfc5f4154fbce4 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.3.0.tgz + version: 3.3.0 + - apiVersion: v1 + appVersion: RELEASE.2021-10-13T00-23-17Z + created: "2025-01-02T21:34:24.986516619-08:00" + description: Multi-Cloud Object Storage + digest: 5b797b7208cd904c11a76cd72938c8652160cb5fcd7f09fa41e4e703e6d64054 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.2.0.tgz + version: 3.2.0 + - apiVersion: v1 + appVersion: RELEASE.2021-10-10T16-53-30Z + created: "2025-01-02T21:34:24.983573512-08:00" + description: Multi-Cloud Object Storage + digest: e084ac4bb095f071e59f8f08bd092e4ab2404c1ddadacfdce7dbe248f1bafff8 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.1.9.tgz + version: 3.1.9 + - apiVersion: v1 + appVersion: RELEASE.2021-10-06T23-36-31Z + created: "2025-01-02T21:34:24.980470597-08:00" + description: Multi-Cloud Object Storage + digest: 2890430a8d9487d1fa5508c26776e4881d0086b2c052aa6bdc65c0e4423b9159 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.1.8.tgz + version: 3.1.8 + - apiVersion: v1 + appVersion: RELEASE.2021-10-02T16-31-05Z + created: "2025-01-02T21:34:24.977221503-08:00" + description: Multi-Cloud Object Storage + digest: 01a92196af6c47e3a01e1c68d7cf693a8bc487cba810c2cecff155071e4d6a11 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.1.7.tgz + version: 3.1.7 + - apiVersion: v1 + appVersion: RELEASE.2021-09-18T18-09-59Z + created: "2025-01-02T21:34:24.972867415-08:00" + description: Multi-Cloud Object Storage + digest: e779d73f80b75f33b9c9d995ab10fa455c9c57ee575ebc54e06725a64cd04310 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.1.6.tgz + version: 3.1.6 + - apiVersion: v1 + appVersion: RELEASE.2021-09-18T18-09-59Z + created: "2025-01-02T21:34:24.969718459-08:00" + description: Multi-Cloud Object Storage + digest: 19de4bbc8a400f0c2a94c5e85fc25c9bfc666e773fb3e368dd621d5a57dd1c2a + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.1.5.tgz + version: 3.1.5 + - apiVersion: v1 + appVersion: RELEASE.2021-09-18T18-09-59Z + created: "2025-01-02T21:34:24.966608057-08:00" + description: Multi-Cloud Object Storage + digest: f789d93a171296dd01af0105a5ce067c663597afbb2432faeda293b752b355c0 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.1.4.tgz + version: 3.1.4 + - apiVersion: v1 + appVersion: RELEASE.2021-09-09T21-37-07Z + created: "2025-01-02T21:34:24.963751369-08:00" + description: Multi-Cloud Object Storage + digest: e2eb34d31560b012ef6581f0ff6004ea4376c968cbe0daed2d8f3a614a892afb + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.1.3.tgz + version: 3.1.3 + - apiVersion: v1 + appVersion: RELEASE.2021-09-09T21-37-07Z + created: "2025-01-02T21:34:24.960755082-08:00" + description: Multi-Cloud Object Storage + digest: 8d7e0cc46b3583abd71b97dc0c071f98321101f90eca17348f1e9e0831be64cd + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.1.2.tgz + version: 3.1.2 + - apiVersion: v1 + appVersion: RELEASE.2021-09-09T21-37-07Z + created: "2025-01-02T21:34:24.957713429-08:00" + description: Multi-Cloud Object Storage + digest: 50dcbf366b1b21f4a6fc429d0b884c0c7ff481d0fb95c5e9b3ae157c348dd124 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.1.1.tgz + version: 3.1.1 + - apiVersion: v1 + appVersion: RELEASE.2021-09-09T21-37-07Z + created: "2025-01-02T21:34:24.954546983-08:00" + description: Multi-Cloud Object Storage + digest: 6c01af55d2e2e5f716eabf6fef3a92a8464d0674529e9bacab292e5478a73b7a + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.1.0.tgz + version: 3.1.0 + - apiVersion: v1 + appVersion: RELEASE.2021-09-03T03-56-13Z + created: "2025-01-02T21:34:24.949999464-08:00" + description: Multi-Cloud Object Storage + digest: 18e10be4d0458bc590ca9abf753227e0c70f60511495387b8d4fb15a4daf932e + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.0.2.tgz + version: 3.0.2 + - apiVersion: v1 + appVersion: RELEASE.2021-08-31T05-46-54Z + created: "2025-01-02T21:34:24.947018538-08:00" + description: Multi-Cloud Object Storage + digest: f5b6e7f6272a9e71aef3b75555f6f756a39eef65cb78873f26451dba79b19906 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.0.1.tgz + version: 3.0.1 + - apiVersion: v1 + appVersion: RELEASE.2021-08-31T05-46-54Z + created: "2025-01-02T21:34:24.943547135-08:00" + description: Multi-Cloud Object Storage + digest: 6d2ee1336c412affaaf209fdb80215be2a6ebb23ab2443adbaffef9e7df13fab + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - minio + - storage + - object-storage + - s3 + - cluster + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-3.0.0.tgz + version: 3.0.0 + - apiVersion: v1 + appVersion: RELEASE.2021-08-31T05-46-54Z + created: "2025-01-02T21:34:24.940463458-08:00" + description: Multi-Cloud Object Storage + digest: 0a004aaf5bb61deed6a5c88256d1695ebe2f9ff1553874a93e4acfd75e8d339b + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - storage + - object-storage + - S3 + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-2.0.1.tgz + version: 2.0.1 + - apiVersion: v1 + appVersion: RELEASE.2021-08-25T00-41-18Z + created: "2025-01-02T21:34:24.937381269-08:00" + description: Multi-Cloud Object Storage + digest: fcd944e837ee481307de6aa3d387ea18c234f995a84c15abb211aab4a4054afc + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - storage + - object-storage + - S3 + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-2.0.0.tgz + version: 2.0.0 + - apiVersion: v1 + appVersion: RELEASE.2021-08-25T00-41-18Z + created: "2025-01-02T21:34:24.934337395-08:00" + description: Multi-Cloud Object Storage + digest: 7b6c033d43a856479eb493ab8ca05b230f77c3e42e209e8f298fac6af1a9796f + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - storage + - object-storage + - S3 + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-1.0.5.tgz + version: 1.0.5 + - apiVersion: v1 + appVersion: RELEASE.2021-08-25T00-41-18Z + created: "2025-01-02T21:34:24.931230726-08:00" + description: Multi-Cloud Object Storage + digest: abd221245ace16c8e0c6c851cf262d1474a5219dcbf25c4b2e7b77142f9c59ed + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - storage + - object-storage + - S3 + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-1.0.4.tgz + version: 1.0.4 + - apiVersion: v1 + appVersion: RELEASE.2021-08-20T18-32-01Z + created: "2025-01-02T21:34:24.926018385-08:00" + description: Multi-Cloud Object Storage + digest: 922a333f5413d1042f7aa81929f43767f6ffca9b260c46713f04ce1dda86d57d + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - storage + - object-storage + - S3 + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-1.0.3.tgz + version: 1.0.3 + - apiVersion: v1 + appVersion: RELEASE.2021-08-20T18-32-01Z + created: "2025-01-02T21:34:24.924448521-08:00" + description: High Performance, Kubernetes Native Object Storage + digest: 10e22773506bbfb1c66442937956534cf4057b94f06a977db78b8cd223588388 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - storage + - object-storage + - S3 + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-1.0.2.tgz + version: 1.0.2 + - apiVersion: v1 + appVersion: RELEASE.2021-08-20T18-32-01Z + created: "2025-01-02T21:34:24.923185443-08:00" + description: High Performance, Kubernetes Native Object Storage + digest: ef86ab6df23d6942705da9ef70991b649638c51bc310587d37a425268ba4a06c + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - storage + - object-storage + - S3 + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-1.0.1.tgz + version: 1.0.1 + - apiVersion: v1 + appVersion: RELEASE.2021-08-17T20-53-08Z + created: "2025-01-02T21:34:24.921774338-08:00" + description: High Performance, Kubernetes Native Object Storage + digest: 1add7608692cbf39aaf9b1252530e566f7b2f306a14e390b0f49b97a20f2b188 + home: https://min.io + icon: https://min.io/resources/img/logo/MINIO_wordmark.png + keywords: + - storage + - object-storage + - S3 + maintainers: + - email: dev@minio.io + name: MinIO, Inc + name: minio + sources: + - https://github.com/minio/minio + urls: + - https://charts.min.io/helm-releases/minio-1.0.0.tgz + version: 1.0.0 +generated: "2025-01-02T21:34:24.920106038-08:00" diff --git a/internal/amztime/iso8601_time.go b/internal/amztime/iso8601_time.go new file mode 100644 index 00000000..d12f9b92 --- /dev/null +++ b/internal/amztime/iso8601_time.go @@ -0,0 +1,59 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package amztime + +import ( + "strings" + "time" +) + +// RFC3339 a subset of the ISO8601 timestamp format. e.g 2014-04-29T18:30:38Z +const ( + iso8601TimeFormat = "2006-01-02T15:04:05.000Z" // Reply date format with millisecond precision. + iso8601TimeFormatLong = "2006-01-02T15:04:05.000000Z" // Reply date format with nanosecond precision. +) + +// ISO8601Format converts time 't' into ISO8601 time format expected in AWS S3 spec. +// +// This function is needed to avoid a Go's float64 precision bug, where Go avoids +// padding the extra '0' before the timezone. +func ISO8601Format(t time.Time) string { + value := t.Format(iso8601TimeFormat) + if len(value) < len(iso8601TimeFormat) { + value = t.Format(iso8601TimeFormat[:len(iso8601TimeFormat)-1]) + // Pad necessary zeroes to full-fill the iso8601TimeFormat + return value + strings.Repeat("0", (len(iso8601TimeFormat)-1)-len(value)) + "Z" + } + return value +} + +// ISO8601Parse parses ISO8601 date string +func ISO8601Parse(iso8601 string) (t time.Time, err error) { + for _, layout := range []string{ + iso8601TimeFormat, + iso8601TimeFormatLong, + time.RFC3339, + } { + t, err = time.Parse(layout, iso8601) + if err == nil { + return t, nil + } + } + + return t, err +} diff --git a/internal/amztime/iso8601_time_test.go b/internal/amztime/iso8601_time_test.go new file mode 100644 index 00000000..34c60204 --- /dev/null +++ b/internal/amztime/iso8601_time_test.go @@ -0,0 +1,57 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package amztime + +import ( + "testing" + "time" +) + +func TestISO8601Format(t *testing.T) { + testCases := []struct { + date time.Time + expectedOutput string + }{ + { + date: time.Date(2009, time.November, 13, 4, 51, 1, 940303531, time.UTC), + expectedOutput: "2009-11-13T04:51:01.940Z", + }, + { + date: time.Date(2009, time.November, 13, 4, 51, 1, 901303531, time.UTC), + expectedOutput: "2009-11-13T04:51:01.901Z", + }, + { + date: time.Date(2009, time.November, 13, 4, 51, 1, 900303531, time.UTC), + expectedOutput: "2009-11-13T04:51:01.900Z", + }, + { + date: time.Date(2009, time.November, 13, 4, 51, 1, 941303531, time.UTC), + expectedOutput: "2009-11-13T04:51:01.941Z", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.expectedOutput, func(t *testing.T) { + gotOutput := ISO8601Format(testCase.date) + t.Log("Go", testCase.date.Format(iso8601TimeFormat)) + if gotOutput != testCase.expectedOutput { + t.Errorf("Expected %s, got %s", testCase.expectedOutput, gotOutput) + } + }) + } +} diff --git a/internal/amztime/parse.go b/internal/amztime/parse.go new file mode 100644 index 00000000..f5bebcf5 --- /dev/null +++ b/internal/amztime/parse.go @@ -0,0 +1,82 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Package amztime implements AWS specific time parsing and deviations +package amztime + +import ( + "errors" + "net/http" + "time" +) + +// Supported amz date formats. +var amzDateFormats = []string{ + // Do not change this order, x-amz-date format is usually in + // iso8601Format rest are meant for relaxed handling of other + // odd SDKs that might be out there. + "20060102T150405Z", + time.RFC1123, + time.RFC1123Z, + // Add new AMZ date formats here. +} + +// ErrMalformedDate always returned for dates that cannot be parsed. +var ErrMalformedDate = errors.New("malformed date") + +// Parse parses date string via supported amz date formats. +func Parse(amzDateStr string) (time.Time, error) { + for _, dateFormat := range amzDateFormats { + amzDate, err := time.Parse(dateFormat, amzDateStr) + if err == nil { + return amzDate, nil + } + } + return time.Time{}, ErrMalformedDate +} + +var httpTimeFormats = []string{ + // Do not change this order, http time format dates + // are usually in http.TimeFormat however there are + // situations where for example aws-sdk-java doesn't + // send the correct format. + http.TimeFormat, + "Mon, 2 Jan 2006 15:04:05 GMT", +} + +// ParseHeader parses http.TimeFormat with an acceptable +// extension for http.TimeFormat - return time might be zero +// if the timeStr is invalid. +func ParseHeader(timeStr string) (time.Time, error) { + for _, dateFormat := range httpTimeFormats { + t, err := time.Parse(dateFormat, timeStr) + if err == nil { + return t, nil + } + } + return time.Time{}, ErrMalformedDate +} + +// ParseReplicationTS parse http.TimeFormat first +// will try time.RFC3339Nano when parse http.TimeFormat failed +func ParseReplicationTS(str string) (time.Time, error) { + tm, err := time.Parse(http.TimeFormat, str) + if tm.IsZero() || err != nil { + tm, err = time.Parse(time.RFC3339Nano, str) + } + return tm, err +} diff --git a/internal/amztime/parse_test.go b/internal/amztime/parse_test.go new file mode 100644 index 00000000..144ef81c --- /dev/null +++ b/internal/amztime/parse_test.go @@ -0,0 +1,57 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Package amztime implements AWS specific time parsing and deviations +package amztime + +import ( + "errors" + "testing" + "time" +) + +func TestParse(t *testing.T) { + type testCase struct { + expectedErr error + expectedTime time.Time + timeStr string + } + testCases := []testCase{ + { + ErrMalformedDate, + time.Time{}, + "Tue Sep 6 07:10:23 PM PDT 2022", + }, + { + nil, + time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC), + "Tue, 10 Nov 2009 23:00:00 UTC", + }, + } + + for _, testCase := range testCases { + t.Run(testCase.timeStr, func(t *testing.T) { + gott, goterr := Parse(testCase.timeStr) + if !errors.Is(goterr, testCase.expectedErr) { + t.Errorf("expected %v, got %v", testCase.expectedErr, goterr) + } + if !gott.Equal(testCase.expectedTime) { + t.Errorf("expected %v, got %v", testCase.expectedTime, gott) + } + }) + } +} diff --git a/internal/arn/arn.go b/internal/arn/arn.go new file mode 100644 index 00000000..4f40748c --- /dev/null +++ b/internal/arn/arn.go @@ -0,0 +1,133 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package arn + +import ( + "errors" + "fmt" + "regexp" + "strings" +) + +// ARN structure: +// +// arn:partition:service:region:account-id:resource-type/resource-id +// +// In this implementation, account-id is empty. +// +// Reference: https://docs.aws.amazon.com/general/latest/gr/aws-arns-and-namespaces.html + +const ( + arnPrefixArn = "arn" + arnPartitionMinio = "minio" + arnServiceIAM = "iam" + arnResourceTypeRole = "role" +) + +// ARN - representation of resources based on AWS ARNs. +type ARN struct { + Partition string + Service string + Region string + ResourceType string + ResourceID string +} + +// Allows english letters, numbers, '.', '-', '_' and '/'. Starts with a +// letter or digit. At least 1 character long. +var validResourceIDRegex = regexp.MustCompile(`[A-Za-z0-9_/\.-]+$`) + +// NewIAMRoleARN - returns an ARN for a role in MinIO. +func NewIAMRoleARN(resourceID, serverRegion string) (ARN, error) { + if !validResourceIDRegex.MatchString(resourceID) { + return ARN{}, fmt.Errorf("invalid resource ID: %s", resourceID) + } + return ARN{ + Partition: arnPartitionMinio, + Service: arnServiceIAM, + Region: serverRegion, + ResourceType: arnResourceTypeRole, + ResourceID: resourceID, + }, nil +} + +// String - returns string representation of the ARN. +func (arn ARN) String() string { + return strings.Join( + []string{ + arnPrefixArn, + arn.Partition, + arn.Service, + arn.Region, + "", // account-id is always empty in this implementation + arn.ResourceType + "/" + arn.ResourceID, + }, + ":", + ) +} + +// Parse - parses an ARN string into a type. +func Parse(arnStr string) (arn ARN, err error) { + ps := strings.Split(arnStr, ":") + if len(ps) != 6 || ps[0] != string(arnPrefixArn) { + err = errors.New("invalid ARN string format") + return arn, err + } + + if ps[1] != string(arnPartitionMinio) { + err = errors.New("invalid ARN - bad partition field") + return arn, err + } + + if ps[2] != string(arnServiceIAM) { + err = errors.New("invalid ARN - bad service field") + return arn, err + } + + // ps[3] is region and is not validated here. If the region is invalid, + // the ARN would not match any configured ARNs in the server. + if ps[4] != "" { + err = errors.New("invalid ARN - unsupported account-id field") + return arn, err + } + + res := strings.SplitN(ps[5], "/", 2) + if len(res) != 2 { + err = errors.New("invalid ARN - resource does not contain a \"/\"") + return arn, err + } + + if res[0] != string(arnResourceTypeRole) { + err = errors.New("invalid ARN: resource type is invalid") + return arn, err + } + + if !validResourceIDRegex.MatchString(res[1]) { + err = fmt.Errorf("invalid resource ID: %s", res[1]) + return arn, err + } + + arn = ARN{ + Partition: arnPartitionMinio, + Service: arnServiceIAM, + Region: ps[3], + ResourceType: arnResourceTypeRole, + ResourceID: res[1], + } + return arn, err +} diff --git a/internal/arn/arn_test.go b/internal/arn/arn_test.go new file mode 100644 index 00000000..7012cfa1 --- /dev/null +++ b/internal/arn/arn_test.go @@ -0,0 +1,236 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package arn + +import ( + "reflect" + "testing" +) + +func TestARN_String(t *testing.T) { + tests := []struct { + arn ARN + want string + }{ + { + arn: ARN{ + Partition: "minio", + Service: "iam", + Region: "us-east-1", + ResourceType: "role", + ResourceID: "my-role", + }, + want: "arn:minio:iam:us-east-1::role/my-role", + }, + { + arn: ARN{ + Partition: "minio", + Service: "", + Region: "us-east-1", + ResourceType: "role", + ResourceID: "my-role", + }, + want: "arn:minio::us-east-1::role/my-role", + }, + } + for _, tt := range tests { + t.Run(tt.want, func(t *testing.T) { + if got := tt.arn.String(); got != tt.want { + t.Errorf("ARN.String() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestNewIAMRoleARN(t *testing.T) { + type args struct { + resourceID string + serverRegion string + } + tests := []struct { + name string + args args + want ARN + wantErr bool + }{ + { + name: "valid resource ID must succeed", + args: args{ + resourceID: "my-role", + serverRegion: "us-east-1", + }, + want: ARN{ + Partition: "minio", + Service: "iam", + Region: "us-east-1", + ResourceType: "role", + ResourceID: "my-role", + }, + wantErr: false, + }, + { + name: "valid resource ID must succeed", + args: args{ + resourceID: "-my-role", + serverRegion: "us-east-1", + }, + want: ARN{ + Partition: "minio", + Service: "iam", + Region: "us-east-1", + ResourceType: "role", + ResourceID: "-my-role", + }, + wantErr: false, + }, + { + name: "empty server region must succeed", + args: args{ + resourceID: "my-role", + serverRegion: "", + }, + want: ARN{ + Partition: "minio", + Service: "iam", + Region: "", + ResourceType: "role", + ResourceID: "my-role", + }, + wantErr: false, + }, + { + name: "empty resource ID must fail", + args: args{ + resourceID: "", + serverRegion: "us-east-1", + }, + want: ARN{}, + wantErr: true, + }, + { + name: "resource ID starting with '=' must fail", + args: args{ + resourceID: "=", + serverRegion: "us-east-1", + }, + want: ARN{}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := NewIAMRoleARN(tt.args.resourceID, tt.args.serverRegion) + if (err != nil) != tt.wantErr { + t.Errorf("NewIAMRoleARN() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("NewIAMRoleARN() got = %v, want %v", got, tt.want) + } + }) + } +} + +func TestParse(t *testing.T) { + type args struct { + arnStr string + } + tests := []struct { + name string + args args + wantArn ARN + wantErr bool + }{ + { + name: "valid ARN must succeed", + args: args{ + arnStr: "arn:minio:iam:us-east-1::role/my-role", + }, + wantArn: ARN{ + Partition: "minio", + Service: "iam", + Region: "us-east-1", + ResourceType: "role", + ResourceID: "my-role", + }, + wantErr: false, + }, + { + name: "valid ARN must succeed", + args: args{ + arnStr: "arn:minio:iam:us-east-1::role/-my-role", + }, + wantArn: ARN{ + Partition: "minio", + Service: "iam", + Region: "us-east-1", + ResourceType: "role", + ResourceID: "-my-role", + }, + wantErr: false, + }, + { + name: "invalid ARN length must fail", + args: args{ + arnStr: "arn:minio:", + }, + wantArn: ARN{}, + wantErr: true, + }, + { + name: "invalid ARN partition must fail", + args: args{ + arnStr: "arn:invalid:iam:us-east-1::role/my-role", + }, + wantArn: ARN{}, + wantErr: true, + }, + { + name: "invalid ARN service must fail", + args: args{ + arnStr: "arn:minio:invalid:us-east-1::role/my-role", + }, + wantArn: ARN{}, + wantErr: true, + }, + { + name: "invalid ARN resource type must fail", + args: args{ + arnStr: "arn:minio:iam:us-east-1::invalid", + }, + wantArn: ARN{}, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotArn, err := Parse(tt.args.arnStr) + if err == nil && tt.wantErr { + t.Errorf("Parse() error = %v, wantErr %v", err, tt.wantErr) + } + if err != nil && !tt.wantErr { + t.Errorf("Parse() error = %v, wantErr %v", err, tt.wantErr) + } + if err == nil { + if !reflect.DeepEqual(gotArn, tt.wantArn) { + t.Errorf("Parse() gotArn = %v, want %v", gotArn, tt.wantArn) + } + } + }) + } +} diff --git a/internal/auth/credentials.go b/internal/auth/credentials.go new file mode 100644 index 00000000..764b7cad --- /dev/null +++ b/internal/auth/credentials.go @@ -0,0 +1,382 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package auth + +import ( + "crypto/rand" + "crypto/subtle" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "strconv" + "strings" + "time" + + jwtgo "github.com/golang-jwt/jwt/v4" + "github.com/minio/minio/internal/jwt" +) + +const ( + // Minimum length for MinIO access key. + accessKeyMinLen = 3 + + // Maximum length for MinIO access key. + // There is no max length enforcement for access keys + accessKeyMaxLen = 20 + + // Minimum length for MinIO secret key for both server + secretKeyMinLen = 8 + + // Maximum secret key length for MinIO, this + // is used when autogenerating new credentials. + // There is no max length enforcement for secret keys + secretKeyMaxLen = 40 + + // Alpha numeric table used for generating access keys. + alphaNumericTable = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ" + + // Total length of the alpha numeric table. + alphaNumericTableLen = byte(len(alphaNumericTable)) + + reservedChars = "=," +) + +// Common errors generated for access and secret key validation. +var ( + ErrInvalidAccessKeyLength = fmt.Errorf("access key length should be between %d and %d", accessKeyMinLen, accessKeyMaxLen) + ErrInvalidSecretKeyLength = fmt.Errorf("secret key length should be between %d and %d", secretKeyMinLen, secretKeyMaxLen) + ErrNoAccessKeyWithSecretKey = fmt.Errorf("access key must be specified if secret key is specified") + ErrNoSecretKeyWithAccessKey = fmt.Errorf("secret key must be specified if access key is specified") + ErrContainsReservedChars = fmt.Errorf("access key contains one of reserved characters '=' or ','") +) + +// AnonymousCredentials simply points to empty credentials +var AnonymousCredentials = Credentials{} + +// ContainsReservedChars - returns whether the input string contains reserved characters. +func ContainsReservedChars(s string) bool { + return strings.ContainsAny(s, reservedChars) +} + +// IsAccessKeyValid - validate access key for right length. +func IsAccessKeyValid(accessKey string) bool { + return len(accessKey) >= accessKeyMinLen +} + +// IsSecretKeyValid - validate secret key for right length. +func IsSecretKeyValid(secretKey string) bool { + return len(secretKey) >= secretKeyMinLen +} + +// Default access and secret keys. +const ( + DefaultAccessKey = "minioadmin" + DefaultSecretKey = "minioadmin" +) + +// Default access credentials +var ( + DefaultCredentials = Credentials{ + AccessKey: DefaultAccessKey, + SecretKey: DefaultSecretKey, + } +) + +// claim key found in credentials which are service accounts +const iamPolicyClaimNameSA = "sa-policy" + +const ( + // AccountOn indicates that credentials are enabled + AccountOn = "on" + // AccountOff indicates that credentials are disabled + AccountOff = "off" +) + +// Credentials holds access and secret keys. +type Credentials struct { + AccessKey string `xml:"AccessKeyId" json:"accessKey,omitempty" yaml:"accessKey"` + SecretKey string `xml:"SecretAccessKey" json:"secretKey,omitempty" yaml:"secretKey"` + SessionToken string `xml:"SessionToken" json:"sessionToken,omitempty" yaml:"sessionToken"` + Expiration time.Time `xml:"Expiration" json:"expiration" yaml:"-"` + Status string `xml:"-" json:"status,omitempty"` + ParentUser string `xml:"-" json:"parentUser,omitempty"` + Groups []string `xml:"-" json:"groups,omitempty"` + Claims map[string]any `xml:"-" json:"claims,omitempty"` + Name string `xml:"-" json:"name,omitempty"` + Description string `xml:"-" json:"description,omitempty"` + + // Deprecated: In favor of Description - when reading credentials from + // storage the value of this field is placed in the Description field above + // if the existing Description from storage is empty. + Comment string `xml:"-" json:"comment,omitempty"` +} + +func (cred Credentials) String() string { + var s strings.Builder + s.WriteString(cred.AccessKey) + s.WriteString(":") + s.WriteString(cred.SecretKey) + if cred.SessionToken != "" { + s.WriteString("\n") + s.WriteString(cred.SessionToken) + } + if !cred.Expiration.IsZero() && !cred.Expiration.Equal(timeSentinel) { + s.WriteString("\n") + s.WriteString(cred.Expiration.String()) + } + return s.String() +} + +// IsExpired - returns whether Credential is expired or not. +func (cred Credentials) IsExpired() bool { + if cred.Expiration.IsZero() || cred.Expiration.Equal(timeSentinel) { + return false + } + + return cred.Expiration.Before(time.Now().UTC()) +} + +// IsTemp - returns whether credential is temporary or not. +func (cred Credentials) IsTemp() bool { + return cred.SessionToken != "" && !cred.Expiration.IsZero() && !cred.Expiration.Equal(timeSentinel) +} + +// IsServiceAccount - returns whether credential is a service account or not +func (cred Credentials) IsServiceAccount() bool { + _, ok := cred.Claims[iamPolicyClaimNameSA] + return cred.ParentUser != "" && ok +} + +// IsImpliedPolicy - returns if the policy is implied via ParentUser or not. +func (cred Credentials) IsImpliedPolicy() bool { + if cred.IsServiceAccount() { + return cred.Claims[iamPolicyClaimNameSA] == "inherited-policy" + } + return false +} + +// IsValid - returns whether credential is valid or not. +func (cred Credentials) IsValid() bool { + // Verify credentials if its enabled or not set. + if cred.Status == AccountOff { + return false + } + return IsAccessKeyValid(cred.AccessKey) && IsSecretKeyValid(cred.SecretKey) && !cred.IsExpired() +} + +// Equal - returns whether two credentials are equal or not. +func (cred Credentials) Equal(ccred Credentials) bool { + if !ccred.IsValid() { + return false + } + return (cred.AccessKey == ccred.AccessKey && subtle.ConstantTimeCompare([]byte(cred.SecretKey), []byte(ccred.SecretKey)) == 1 && + subtle.ConstantTimeCompare([]byte(cred.SessionToken), []byte(ccred.SessionToken)) == 1) +} + +var timeSentinel = time.Unix(0, 0).UTC() + +// ErrInvalidDuration invalid token expiry +var ErrInvalidDuration = errors.New("invalid token expiry") + +// ExpToInt64 - convert input interface value to int64. +func ExpToInt64(expI any) (expAt int64, err error) { + switch exp := expI.(type) { + case string: + expAt, err = strconv.ParseInt(exp, 10, 64) + case float64: + expAt, err = int64(exp), nil + case int64: + expAt, err = exp, nil + case int: + expAt, err = int64(exp), nil + case uint64: + expAt, err = int64(exp), nil + case uint: + expAt, err = int64(exp), nil + case json.Number: + expAt, err = exp.Int64() + case time.Duration: + expAt, err = time.Now().UTC().Add(exp).Unix(), nil + case nil: + expAt, err = 0, nil + default: + expAt, err = 0, ErrInvalidDuration + } + if expAt < 0 { + return 0, ErrInvalidDuration + } + return expAt, err +} + +// GenerateCredentials - creates randomly generated credentials of maximum +// allowed length. +func GenerateCredentials() (accessKey, secretKey string, err error) { + accessKey, err = GenerateAccessKey(accessKeyMaxLen, rand.Reader) + if err != nil { + return "", "", err + } + secretKey, err = GenerateSecretKey(secretKeyMaxLen, rand.Reader) + if err != nil { + return "", "", err + } + return accessKey, secretKey, nil +} + +// GenerateAccessKey returns a new access key generated randomly using +// the given io.Reader. If random is nil, crypto/rand.Reader is used. +// If length <= 0, the access key length is chosen automatically. +// +// GenerateAccessKey returns an error if length is too small for a valid +// access key. +func GenerateAccessKey(length int, random io.Reader) (string, error) { + if random == nil { + random = rand.Reader + } + if length <= 0 { + length = accessKeyMaxLen + } + if length < accessKeyMinLen { + return "", errors.New("auth: access key length is too short") + } + + key := make([]byte, length) + if _, err := io.ReadFull(random, key); err != nil { + return "", err + } + for i := range key { + key[i] = alphaNumericTable[key[i]%alphaNumericTableLen] + } + return string(key), nil +} + +// GenerateSecretKey returns a new secret key generated randomly using +// the given io.Reader. If random is nil, crypto/rand.Reader is used. +// If length <= 0, the secret key length is chosen automatically. +// +// GenerateSecretKey returns an error if length is too small for a valid +// secret key. +func GenerateSecretKey(length int, random io.Reader) (string, error) { + if random == nil { + random = rand.Reader + } + if length <= 0 { + length = secretKeyMaxLen + } + if length < secretKeyMinLen { + return "", errors.New("auth: secret key length is too short") + } + + key := make([]byte, base64.RawStdEncoding.DecodedLen(length)) + if _, err := io.ReadFull(random, key); err != nil { + return "", err + } + + s := base64.RawStdEncoding.EncodeToString(key) + return strings.ReplaceAll(s, "/", "+"), nil +} + +// GetNewCredentialsWithMetadata generates and returns new credential with expiry. +func GetNewCredentialsWithMetadata(m map[string]any, tokenSecret string) (Credentials, error) { + accessKey, secretKey, err := GenerateCredentials() + if err != nil { + return Credentials{}, err + } + return CreateNewCredentialsWithMetadata(accessKey, secretKey, m, tokenSecret) +} + +// CreateNewCredentialsWithMetadata - creates new credentials using the specified access & secret keys +// and generate a session token if a secret token is provided. +func CreateNewCredentialsWithMetadata(accessKey, secretKey string, m map[string]any, tokenSecret string) (cred Credentials, err error) { + if len(accessKey) < accessKeyMinLen || len(accessKey) > accessKeyMaxLen { + return Credentials{}, ErrInvalidAccessKeyLength + } + + if len(secretKey) < secretKeyMinLen || len(secretKey) > secretKeyMaxLen { + return Credentials{}, ErrInvalidSecretKeyLength + } + + cred.AccessKey = accessKey + cred.SecretKey = secretKey + cred.Status = AccountOn + + if tokenSecret == "" { + cred.Expiration = timeSentinel + return cred, nil + } + + expiry, err := ExpToInt64(m["exp"]) + if err != nil { + return cred, err + } + cred.Expiration = time.Unix(expiry, 0).UTC() + + cred.SessionToken, err = JWTSignWithAccessKey(cred.AccessKey, m, tokenSecret) + if err != nil { + return cred, err + } + + return cred, nil +} + +// JWTSignWithAccessKey - generates a session token. +func JWTSignWithAccessKey(accessKey string, m map[string]any, tokenSecret string) (string, error) { + m["accessKey"] = accessKey + jwt := jwtgo.NewWithClaims(jwtgo.SigningMethodHS512, jwtgo.MapClaims(m)) + return jwt.SignedString([]byte(tokenSecret)) +} + +// ExtractClaims extracts JWT claims from a security token using a secret key +func ExtractClaims(token, secretKey string) (*jwt.MapClaims, error) { + if token == "" || secretKey == "" { + return nil, errors.New("invalid argument") + } + + claims := jwt.NewMapClaims() + stsTokenCallback := func(claims *jwt.MapClaims) ([]byte, error) { + return []byte(secretKey), nil + } + + if err := jwt.ParseWithClaims(token, claims, stsTokenCallback); err != nil { + return nil, err + } + + return claims, nil +} + +// GetNewCredentials generates and returns new credential. +func GetNewCredentials() (cred Credentials, err error) { + return GetNewCredentialsWithMetadata(map[string]any{}, "") +} + +// CreateCredentials returns new credential with the given access key and secret key. +// Error is returned if given access key or secret key are invalid length. +func CreateCredentials(accessKey, secretKey string) (cred Credentials, err error) { + if !IsAccessKeyValid(accessKey) { + return cred, ErrInvalidAccessKeyLength + } + if !IsSecretKeyValid(secretKey) { + return cred, ErrInvalidSecretKeyLength + } + cred.AccessKey = accessKey + cred.SecretKey = secretKey + cred.Expiration = timeSentinel + cred.Status = AccountOn + return cred, nil +} diff --git a/internal/auth/credentials_test.go b/internal/auth/credentials_test.go new file mode 100644 index 00000000..9e83f4b5 --- /dev/null +++ b/internal/auth/credentials_test.go @@ -0,0 +1,181 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package auth + +import ( + "encoding/json" + "testing" + "time" +) + +func TestExpToInt64(t *testing.T) { + testCases := []struct { + exp any + expectedFailure bool + }{ + {"", true}, + {"-1", true}, + {"1574812326", false}, + {1574812326, false}, + {int64(1574812326), false}, + {int(1574812326), false}, + {uint(1574812326), false}, + {uint64(1574812326), false}, + {json.Number("1574812326"), false}, + {1574812326.000, false}, + {time.Duration(3) * time.Minute, false}, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + _, err := ExpToInt64(testCase.exp) + if err != nil && !testCase.expectedFailure { + t.Errorf("Expected success but got failure %s", err) + } + if err == nil && testCase.expectedFailure { + t.Error("Expected failure but got success") + } + }) + } +} + +func TestIsAccessKeyValid(t *testing.T) { + testCases := []struct { + accessKey string + expectedResult bool + }{ + {alphaNumericTable[:accessKeyMinLen], true}, + {alphaNumericTable[:accessKeyMinLen+1], true}, + {alphaNumericTable[:accessKeyMinLen-1], false}, + } + + for i, testCase := range testCases { + result := IsAccessKeyValid(testCase.accessKey) + if result != testCase.expectedResult { + t.Fatalf("test %v: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestIsSecretKeyValid(t *testing.T) { + testCases := []struct { + secretKey string + expectedResult bool + }{ + {alphaNumericTable[:secretKeyMinLen], true}, + {alphaNumericTable[:secretKeyMinLen+1], true}, + {alphaNumericTable[:secretKeyMinLen-1], false}, + } + + for i, testCase := range testCases { + result := IsSecretKeyValid(testCase.secretKey) + if result != testCase.expectedResult { + t.Fatalf("test %v: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestGetNewCredentials(t *testing.T) { + cred, err := GetNewCredentials() + if err != nil { + t.Fatalf("Failed to get a new credential") + } + if !cred.IsValid() { + t.Fatalf("Failed to get new valid credential") + } + if len(cred.AccessKey) != accessKeyMaxLen { + t.Fatalf("access key length: expected: %v, got: %v", secretKeyMaxLen, len(cred.AccessKey)) + } + if len(cred.SecretKey) != secretKeyMaxLen { + t.Fatalf("secret key length: expected: %v, got: %v", secretKeyMaxLen, len(cred.SecretKey)) + } +} + +func TestCreateCredentials(t *testing.T) { + testCases := []struct { + accessKey string + secretKey string + valid bool + expectedErr error + }{ + // Valid access and secret keys with minimum length. + {alphaNumericTable[:accessKeyMinLen], alphaNumericTable[:secretKeyMinLen], true, nil}, + // Valid access and/or secret keys are longer than minimum length. + {alphaNumericTable[:accessKeyMinLen+1], alphaNumericTable[:secretKeyMinLen+1], true, nil}, + // Smaller access key. + {alphaNumericTable[:accessKeyMinLen-1], alphaNumericTable[:secretKeyMinLen], false, ErrInvalidAccessKeyLength}, + // Smaller secret key. + {alphaNumericTable[:accessKeyMinLen], alphaNumericTable[:secretKeyMinLen-1], false, ErrInvalidSecretKeyLength}, + } + + for i, testCase := range testCases { + cred, err := CreateCredentials(testCase.accessKey, testCase.secretKey) + + if err != nil { + if testCase.expectedErr == nil { + t.Fatalf("test %v: error: expected = , got = %v", i+1, err) + } + if testCase.expectedErr.Error() != err.Error() { + t.Fatalf("test %v: error: expected = %v, got = %v", i+1, testCase.expectedErr, err) + } + } else { + if testCase.expectedErr != nil { + t.Fatalf("test %v: error: expected = %v, got = ", i+1, testCase.expectedErr) + } + if !cred.IsValid() { + t.Fatalf("test %v: got invalid credentials", i+1) + } + } + } +} + +func TestCredentialsEqual(t *testing.T) { + cred, err := GetNewCredentials() + if err != nil { + t.Fatalf("Failed to get a new credential: %v", err) + } + cred2, err := GetNewCredentials() + if err != nil { + t.Fatalf("Failed to get a new credential: %v", err) + } + testCases := []struct { + cred Credentials + ccred Credentials + expectedResult bool + }{ + // Same Credentialss. + {cred, cred, true}, + // Empty credentials to compare. + {cred, Credentials{}, false}, + // Empty credentials. + {Credentials{}, cred, false}, + // Two different credentialss + {cred, cred2, false}, + // Access key is different in credentials to compare. + {cred, Credentials{AccessKey: "myuser", SecretKey: cred.SecretKey}, false}, + // Secret key is different in credentials to compare. + {cred, Credentials{AccessKey: cred.AccessKey, SecretKey: "mypassword"}, false}, + } + + for i, testCase := range testCases { + result := testCase.cred.Equal(testCase.ccred) + if result != testCase.expectedResult { + t.Fatalf("test %v: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} diff --git a/internal/bpool/bpool.go b/internal/bpool/bpool.go new file mode 100644 index 00000000..fc88c320 --- /dev/null +++ b/internal/bpool/bpool.go @@ -0,0 +1,117 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package bpool + +import ( + "github.com/klauspost/reedsolomon" +) + +// BytePoolCap implements a leaky pool of []byte in the form of a bounded channel. +type BytePoolCap struct { + c chan []byte + w int + wcap int +} + +// NewBytePoolCap creates a new BytePool bounded to the given maxSize, with new +// byte arrays sized based on width. +func NewBytePoolCap(maxSize uint64, width int, capwidth int) (bp *BytePoolCap) { + if capwidth <= 0 { + panic("total buffer capacity must be provided") + } + if capwidth < 64 { + panic("buffer capped with smaller than 64 bytes is not supported") + } + if width > capwidth { + panic("minimum buffer length cannot be > capacity of the buffer") + } + return &BytePoolCap{ + c: make(chan []byte, maxSize), + w: width, + wcap: capwidth, + } +} + +// Populate - populates and pre-warms the byte pool, this function is non-blocking. +func (bp *BytePoolCap) Populate() { + for _, buf := range reedsolomon.AllocAligned(cap(bp.c), bp.wcap) { + bp.Put(buf[:bp.w]) + } +} + +// Get gets a []byte from the BytePool, or creates a new one if none are +// available in the pool. +func (bp *BytePoolCap) Get() (b []byte) { + if bp == nil { + return nil + } + select { + case b = <-bp.c: + // reuse existing buffer + default: + // create new aligned buffer + b = reedsolomon.AllocAligned(1, bp.wcap)[0][:bp.w] + } + return b +} + +// Put returns the given Buffer to the BytePool. +func (bp *BytePoolCap) Put(b []byte) { + if bp == nil { + return + } + + if cap(b) != bp.wcap { + // someone tried to put back buffer which is not part of this buffer pool + // we simply don't put this back into pool, a modified buffer provided + // by this package is no more usable, callers make sure to not modify + // the capacity of the buffer. + return + } + + select { + case bp.c <- b[:bp.w]: + // buffer went back into pool + default: + // buffer didn't go back into pool, just discard + } +} + +// Width returns the width of the byte arrays in this pool. +func (bp *BytePoolCap) Width() (n int) { + if bp == nil { + return 0 + } + return bp.w +} + +// WidthCap returns the cap width of the byte arrays in this pool. +func (bp *BytePoolCap) WidthCap() (n int) { + if bp == nil { + return 0 + } + return bp.wcap +} + +// CurrentSize returns current size of buffer pool +func (bp *BytePoolCap) CurrentSize() int { + if bp == nil { + return 0 + } + return len(bp.c) * bp.w +} diff --git a/internal/bpool/bpool_test.go b/internal/bpool/bpool_test.go new file mode 100644 index 00000000..da673017 --- /dev/null +++ b/internal/bpool/bpool_test.go @@ -0,0 +1,99 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package bpool + +import ( + "testing" +) + +// Tests - bytePool functionality. +func TestBytePool(t *testing.T) { + size := uint64(4) + width := 1024 + capWidth := 2048 + + bp := NewBytePoolCap(size, width, capWidth) + + // Check the width + if bp.Width() != width { + t.Fatalf("bytepool width invalid: got %v want %v", bp.Width(), width) + } + + // Check with width cap + if bp.WidthCap() != capWidth { + t.Fatalf("bytepool capWidth invalid: got %v want %v", bp.WidthCap(), capWidth) + } + + // Check that retrieved buffer are of the expected width + b := bp.Get() + if len(b) != width { + t.Fatalf("bytepool length invalid: got %v want %v", len(b), width) + } + if cap(b) != capWidth { + t.Fatalf("bytepool cap invalid: got %v want %v", cap(b), capWidth) + } + + bp.Put(b) + + // Fill the pool beyond the capped pool size. + for i := uint64(0); i < size*2; i++ { + bp.Put(make([]byte, bp.w, bp.wcap)) + } + + b = bp.Get() + if len(b) != width { + t.Fatalf("bytepool length invalid: got %v want %v", len(b), width) + } + if cap(b) != capWidth { + t.Fatalf("bytepool length invalid: got %v want %v", cap(b), capWidth) + } + + bp.Put(b) + + // Check the size of the pool. + if uint64(len(bp.c)) != size { + t.Fatalf("bytepool size invalid: got %v want %v", len(bp.c), size) + } + + // lets drain the buf channel first before we validate invalid buffers. + for range size { + bp.Get() // discard + } + + // Try putting some invalid buffers into pool + bp.Put(make([]byte, bp.w, bp.wcap-1)) // wrong capacity is rejected (less) + bp.Put(make([]byte, bp.w, bp.wcap+1)) // wrong capacity is rejected (more) + bp.Put(make([]byte, width)) // wrong capacity is rejected (very less) + if len(bp.c) > 0 { + t.Fatal("bytepool should have rejected invalid packets") + } + + // Try putting a short slice into pool + bp.Put(make([]byte, bp.w, bp.wcap)[:2]) + if len(bp.c) != 1 { + t.Fatal("bytepool should have accepted short slice with sufficient capacity") + } + + b = bp.Get() + if len(b) != width { + t.Fatalf("bytepool length invalid: got %v want %v", len(b), width) + } + + // Close the channel. + close(bp.c) +} diff --git a/internal/bpool/pool.go b/internal/bpool/pool.go new file mode 100644 index 00000000..63b56eff --- /dev/null +++ b/internal/bpool/pool.go @@ -0,0 +1,45 @@ +// Copyright (c) 2015-2025 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package bpool + +import "sync" + +// Pool is a single type sync.Pool with a few extra properties: +// If New is not set Get may return the zero value of T. +type Pool[T any] struct { + New func() T + p sync.Pool +} + +// Get will retuen a new T +func (p *Pool[T]) Get() T { + v, ok := p.p.Get().(T) + if ok { + return v + } + if p.New == nil { + var t T + return t + } + return p.New() +} + +// Put a used T. +func (p *Pool[T]) Put(t T) { + p.p.Put(t) +} diff --git a/internal/bucket/bandwidth/measurement.go b/internal/bucket/bandwidth/measurement.go new file mode 100644 index 00000000..069785e4 --- /dev/null +++ b/internal/bucket/bandwidth/measurement.go @@ -0,0 +1,92 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package bandwidth + +import ( + "sync" + "sync/atomic" + "time" +) + +const ( + // betaBucket is the weight used to calculate exponential moving average + betaBucket = 0.1 // Number of averages considered = 1/(1-betaObject) +) + +// bucketMeasurement captures the bandwidth details for one bucket +type bucketMeasurement struct { + lock sync.Mutex + bytesSinceLastWindow uint64 // Total bytes since last window was processed + startTime time.Time // Start time for window + expMovingAvg float64 // Previously calculate sliding window +} + +// newBucketMeasurement creates a new instance of the measurement with the initial start time. +func newBucketMeasurement(initTime time.Time) *bucketMeasurement { + return &bucketMeasurement{ + startTime: initTime, + } +} + +// incrementBytes add bytes reported for a bucket. +func (m *bucketMeasurement) incrementBytes(bytes uint64) { + atomic.AddUint64(&m.bytesSinceLastWindow, bytes) +} + +// updateExponentialMovingAverage processes the measurements captured so far. +func (m *bucketMeasurement) updateExponentialMovingAverage(endTime time.Time) { + // Calculate aggregate avg bandwidth and exp window avg + m.lock.Lock() + defer func() { + m.startTime = endTime + m.lock.Unlock() + }() + + if m.startTime.IsZero() { + return + } + + if endTime.Before(m.startTime) { + return + } + + duration := endTime.Sub(m.startTime) + + bytesSinceLastWindow := atomic.SwapUint64(&m.bytesSinceLastWindow, 0) + + if m.expMovingAvg == 0 { + // Should address initial calculation and should be fine for resuming from 0 + m.expMovingAvg = float64(bytesSinceLastWindow) / duration.Seconds() + return + } + + increment := float64(bytesSinceLastWindow) / duration.Seconds() + m.expMovingAvg = exponentialMovingAverage(betaBucket, m.expMovingAvg, increment) +} + +// exponentialMovingAverage calculates the exponential moving average +func exponentialMovingAverage(beta, previousAvg, incrementAvg float64) float64 { + return (1-beta)*incrementAvg + beta*previousAvg +} + +// getExpMovingAvgBytesPerSecond returns the exponential moving average for the bucket in bytes +func (m *bucketMeasurement) getExpMovingAvgBytesPerSecond() float64 { + m.lock.Lock() + defer m.lock.Unlock() + return m.expMovingAvg +} diff --git a/internal/bucket/bandwidth/monitor.go b/internal/bucket/bandwidth/monitor.go new file mode 100644 index 00000000..b523030b --- /dev/null +++ b/internal/bucket/bandwidth/monitor.go @@ -0,0 +1,215 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package bandwidth + +//go:generate msgp -file=$GOFILE -unexported + +import ( + "context" + "slices" + "sync" + "time" + + "golang.org/x/time/rate" +) + +//msgp:ignore bucketThrottle Monitor + +type bucketThrottle struct { + *rate.Limiter + NodeBandwidthPerSec int64 +} + +// Monitor holds the state of the global bucket monitor +type Monitor struct { + tlock sync.RWMutex // mutex for bucket throttling + mlock sync.RWMutex // mutex for bucket measurement + + bucketsThrottle map[BucketOptions]*bucketThrottle + bucketsMeasurement map[BucketOptions]*bucketMeasurement // Buckets with objects in flight + + bucketMovingAvgTicker *time.Ticker // Ticker for calculating moving averages + ctx context.Context // Context for generate + NodeCount uint64 +} + +// NewMonitor returns a monitor with defaults. +func NewMonitor(ctx context.Context, numNodes uint64) *Monitor { + m := &Monitor{ + bucketsMeasurement: make(map[BucketOptions]*bucketMeasurement), + bucketsThrottle: make(map[BucketOptions]*bucketThrottle), + bucketMovingAvgTicker: time.NewTicker(2 * time.Second), + ctx: ctx, + NodeCount: numNodes, + } + go m.trackEWMA() + return m +} + +func (m *Monitor) updateMeasurement(opts BucketOptions, bytes uint64) { + m.mlock.Lock() + defer m.mlock.Unlock() + + tm, ok := m.bucketsMeasurement[opts] + if !ok { + tm = &bucketMeasurement{} + } + tm.incrementBytes(bytes) + m.bucketsMeasurement[opts] = tm +} + +// SelectionFunction for buckets +type SelectionFunction func(bucket string) bool + +// SelectBuckets will select all the buckets passed in. +func SelectBuckets(buckets ...string) SelectionFunction { + if len(buckets) == 0 { + return func(bucket string) bool { + return true + } + } + return func(bucket string) bool { + return slices.Contains(buckets, bucket) + } +} + +// Details for the measured bandwidth +type Details struct { + LimitInBytesPerSecond int64 `json:"limitInBits"` + CurrentBandwidthInBytesPerSecond float64 `json:"currentBandwidth"` +} + +// BucketBandwidthReport captures the details for all buckets. +type BucketBandwidthReport struct { + BucketStats map[BucketOptions]Details `json:"bucketStats,omitempty"` +} + +// GetReport gets the report for all bucket bandwidth details. +func (m *Monitor) GetReport(selectBucket SelectionFunction) *BucketBandwidthReport { + m.mlock.RLock() + defer m.mlock.RUnlock() + return m.getReport(selectBucket) +} + +func (m *Monitor) getReport(selectBucket SelectionFunction) *BucketBandwidthReport { + report := &BucketBandwidthReport{ + BucketStats: make(map[BucketOptions]Details), + } + for bucketOpts, bucketMeasurement := range m.bucketsMeasurement { + if !selectBucket(bucketOpts.Name) { + continue + } + m.tlock.RLock() + if tgtThrottle, ok := m.bucketsThrottle[bucketOpts]; ok { + currBw := bucketMeasurement.getExpMovingAvgBytesPerSecond() + report.BucketStats[bucketOpts] = Details{ + LimitInBytesPerSecond: tgtThrottle.NodeBandwidthPerSec * int64(m.NodeCount), + CurrentBandwidthInBytesPerSecond: currBw, + } + } + m.tlock.RUnlock() + } + return report +} + +func (m *Monitor) trackEWMA() { + for { + select { + case <-m.bucketMovingAvgTicker.C: + m.updateMovingAvg() + case <-m.ctx.Done(): + return + } + } +} + +func (m *Monitor) updateMovingAvg() { + m.mlock.Lock() + defer m.mlock.Unlock() + for _, bucketMeasurement := range m.bucketsMeasurement { + bucketMeasurement.updateExponentialMovingAverage(time.Now()) + } +} + +func (m *Monitor) init(opts BucketOptions) { + m.mlock.Lock() + defer m.mlock.Unlock() + + _, ok := m.bucketsMeasurement[opts] + if !ok { + m.bucketsMeasurement[opts] = newBucketMeasurement(time.Now()) + } +} + +// DeleteBucket deletes monitoring the 'bucket' +func (m *Monitor) DeleteBucket(bucket string) { + m.tlock.Lock() + for opts := range m.bucketsThrottle { + if opts.Name == bucket { + delete(m.bucketsThrottle, opts) + } + } + m.tlock.Unlock() + + m.mlock.Lock() + for opts := range m.bucketsMeasurement { + if opts.Name == bucket { + delete(m.bucketsMeasurement, opts) + } + } + m.mlock.Unlock() +} + +// DeleteBucketThrottle deletes monitoring for a bucket's target +func (m *Monitor) DeleteBucketThrottle(bucket, arn string) { + m.tlock.Lock() + delete(m.bucketsThrottle, BucketOptions{Name: bucket, ReplicationARN: arn}) + m.tlock.Unlock() + m.mlock.Lock() + delete(m.bucketsMeasurement, BucketOptions{Name: bucket, ReplicationARN: arn}) + m.mlock.Unlock() +} + +// throttle returns currently configured throttle for this bucket +func (m *Monitor) throttle(opts BucketOptions) *bucketThrottle { + m.tlock.RLock() + defer m.tlock.RUnlock() + return m.bucketsThrottle[opts] +} + +// SetBandwidthLimit sets the bandwidth limit for a bucket +func (m *Monitor) SetBandwidthLimit(bucket, arn string, limit int64) { + m.tlock.Lock() + defer m.tlock.Unlock() + limitBytes := limit / int64(m.NodeCount) + throttle, ok := m.bucketsThrottle[BucketOptions{Name: bucket, ReplicationARN: arn}] + if !ok { + throttle = &bucketThrottle{} + } + throttle.NodeBandwidthPerSec = limitBytes + throttle.Limiter = rate.NewLimiter(rate.Limit(float64(limitBytes)), int(limitBytes)) + m.bucketsThrottle[BucketOptions{Name: bucket, ReplicationARN: arn}] = throttle +} + +// IsThrottled returns true if a bucket has bandwidth throttling enabled. +func (m *Monitor) IsThrottled(bucket, arn string) bool { + m.tlock.RLock() + defer m.tlock.RUnlock() + _, ok := m.bucketsThrottle[BucketOptions{Name: bucket, ReplicationARN: arn}] + return ok +} diff --git a/internal/bucket/bandwidth/monitor_gen.go b/internal/bucket/bandwidth/monitor_gen.go new file mode 100644 index 00000000..4492ca2d --- /dev/null +++ b/internal/bucket/bandwidth/monitor_gen.go @@ -0,0 +1,220 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package bandwidth + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *BucketBandwidthReport) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z BucketBandwidthReport) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 0 + _ = z + err = en.Append(0x80) + if err != nil { + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z BucketBandwidthReport) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 0 + _ = z + o = append(o, 0x80) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *BucketBandwidthReport) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z BucketBandwidthReport) Msgsize() (s int) { + s = 1 + return +} + +// DecodeMsg implements msgp.Decodable +func (z *Details) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "LimitInBytesPerSecond": + z.LimitInBytesPerSecond, err = dc.ReadInt64() + if err != nil { + err = msgp.WrapError(err, "LimitInBytesPerSecond") + return + } + case "CurrentBandwidthInBytesPerSecond": + z.CurrentBandwidthInBytesPerSecond, err = dc.ReadFloat64() + if err != nil { + err = msgp.WrapError(err, "CurrentBandwidthInBytesPerSecond") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z Details) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "LimitInBytesPerSecond" + err = en.Append(0x82, 0xb5, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x49, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64) + if err != nil { + return + } + err = en.WriteInt64(z.LimitInBytesPerSecond) + if err != nil { + err = msgp.WrapError(err, "LimitInBytesPerSecond") + return + } + // write "CurrentBandwidthInBytesPerSecond" + err = en.Append(0xd9, 0x20, 0x43, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, 0x74, 0x68, 0x49, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64) + if err != nil { + return + } + err = en.WriteFloat64(z.CurrentBandwidthInBytesPerSecond) + if err != nil { + err = msgp.WrapError(err, "CurrentBandwidthInBytesPerSecond") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z Details) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "LimitInBytesPerSecond" + o = append(o, 0x82, 0xb5, 0x4c, 0x69, 0x6d, 0x69, 0x74, 0x49, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64) + o = msgp.AppendInt64(o, z.LimitInBytesPerSecond) + // string "CurrentBandwidthInBytesPerSecond" + o = append(o, 0xd9, 0x20, 0x43, 0x75, 0x72, 0x72, 0x65, 0x6e, 0x74, 0x42, 0x61, 0x6e, 0x64, 0x77, 0x69, 0x64, 0x74, 0x68, 0x49, 0x6e, 0x42, 0x79, 0x74, 0x65, 0x73, 0x50, 0x65, 0x72, 0x53, 0x65, 0x63, 0x6f, 0x6e, 0x64) + o = msgp.AppendFloat64(o, z.CurrentBandwidthInBytesPerSecond) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *Details) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "LimitInBytesPerSecond": + z.LimitInBytesPerSecond, bts, err = msgp.ReadInt64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "LimitInBytesPerSecond") + return + } + case "CurrentBandwidthInBytesPerSecond": + z.CurrentBandwidthInBytesPerSecond, bts, err = msgp.ReadFloat64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "CurrentBandwidthInBytesPerSecond") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z Details) Msgsize() (s int) { + s = 1 + 22 + msgp.Int64Size + 34 + msgp.Float64Size + return +} diff --git a/internal/bucket/bandwidth/monitor_gen_test.go b/internal/bucket/bandwidth/monitor_gen_test.go new file mode 100644 index 00000000..9f9c1e56 --- /dev/null +++ b/internal/bucket/bandwidth/monitor_gen_test.go @@ -0,0 +1,236 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package bandwidth + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalBucketBandwidthReport(t *testing.T) { + v := BucketBandwidthReport{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgBucketBandwidthReport(b *testing.B) { + v := BucketBandwidthReport{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgBucketBandwidthReport(b *testing.B) { + v := BucketBandwidthReport{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalBucketBandwidthReport(b *testing.B) { + v := BucketBandwidthReport{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeBucketBandwidthReport(t *testing.T) { + v := BucketBandwidthReport{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeBucketBandwidthReport Msgsize() is inaccurate") + } + + vn := BucketBandwidthReport{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeBucketBandwidthReport(b *testing.B) { + v := BucketBandwidthReport{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeBucketBandwidthReport(b *testing.B) { + v := BucketBandwidthReport{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalDetails(t *testing.T) { + v := Details{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgDetails(b *testing.B) { + v := Details{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgDetails(b *testing.B) { + v := Details{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalDetails(b *testing.B) { + v := Details{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeDetails(t *testing.T) { + v := Details{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeDetails Msgsize() is inaccurate") + } + + vn := Details{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeDetails(b *testing.B) { + v := Details{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeDetails(b *testing.B) { + v := Details{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/internal/bucket/bandwidth/monitor_test.go b/internal/bucket/bandwidth/monitor_test.go new file mode 100644 index 00000000..4799ce07 --- /dev/null +++ b/internal/bucket/bandwidth/monitor_test.go @@ -0,0 +1,127 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package bandwidth + +import ( + "reflect" + "testing" + "time" +) + +const ( + oneMiB uint64 = 1024 * 1024 +) + +func TestMonitor_GetReport(t *testing.T) { + type fields struct { + activeBuckets map[BucketOptions]*bucketMeasurement + endTime time.Time + update2 uint64 + endTime2 time.Time + } + start := time.Now() + m0 := newBucketMeasurement(start) + m0.incrementBytes(0) + m1MiBPS := newBucketMeasurement(start) + m1MiBPS.incrementBytes(oneMiB) + + test1Want := make(map[BucketOptions]Details) + test1Want[BucketOptions{Name: "bucket", ReplicationARN: "arn"}] = Details{LimitInBytesPerSecond: 1024 * 1024, CurrentBandwidthInBytesPerSecond: 0} + test1Want2 := make(map[BucketOptions]Details) + test1Want2[BucketOptions{Name: "bucket", ReplicationARN: "arn"}] = Details{ + LimitInBytesPerSecond: 1024 * 1024, + CurrentBandwidthInBytesPerSecond: (1024 * 1024) / start.Add(2*time.Second).Sub(start.Add(1*time.Second)).Seconds(), + } + + test2Want := make(map[BucketOptions]Details) + test2Want[BucketOptions{Name: "bucket", ReplicationARN: "arn"}] = Details{LimitInBytesPerSecond: 1024 * 1024, CurrentBandwidthInBytesPerSecond: float64(oneMiB)} + test2Want2 := make(map[BucketOptions]Details) + test2Want2[BucketOptions{Name: "bucket", ReplicationARN: "arn"}] = Details{ + LimitInBytesPerSecond: 1024 * 1024, + CurrentBandwidthInBytesPerSecond: exponentialMovingAverage(betaBucket, float64(oneMiB), 2*float64(oneMiB)), + } + + test1ActiveBuckets := make(map[BucketOptions]*bucketMeasurement) + test1ActiveBuckets[BucketOptions{Name: "bucket", ReplicationARN: "arn"}] = m0 + test1ActiveBuckets2 := make(map[BucketOptions]*bucketMeasurement) + test1ActiveBuckets2[BucketOptions{Name: "bucket", ReplicationARN: "arn"}] = m1MiBPS + + tests := []struct { + name string + fields fields + want *BucketBandwidthReport + want2 *BucketBandwidthReport + }{ + { + name: "ZeroToOne", + fields: fields{ + activeBuckets: test1ActiveBuckets, + endTime: start.Add(1 * time.Second), + update2: oneMiB, + endTime2: start.Add(2 * time.Second), + }, + want: &BucketBandwidthReport{ + BucketStats: test1Want, + }, + want2: &BucketBandwidthReport{ + BucketStats: test1Want2, + }, + }, + { + name: "OneToTwo", + fields: fields{ + activeBuckets: test1ActiveBuckets2, + endTime: start.Add(1 * time.Second), + update2: 2 * oneMiB, + endTime2: start.Add(2 * time.Second), + }, + want: &BucketBandwidthReport{ + BucketStats: test2Want, + }, + want2: &BucketBandwidthReport{ + BucketStats: test2Want2, + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + thr := bucketThrottle{ + NodeBandwidthPerSec: 1024 * 1024, + } + th := make(map[BucketOptions]*bucketThrottle) + th[BucketOptions{Name: "bucket", ReplicationARN: "arn"}] = &thr + m := &Monitor{ + bucketsMeasurement: tt.fields.activeBuckets, + bucketsThrottle: th, + NodeCount: 1, + } + m.bucketsMeasurement[BucketOptions{Name: "bucket", ReplicationARN: "arn"}].updateExponentialMovingAverage(tt.fields.endTime) + got := m.GetReport(SelectBuckets()) + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("GetReport() = %v, want %v", got, tt.want) + } + m.bucketsMeasurement[BucketOptions{Name: "bucket", ReplicationARN: "arn"}].incrementBytes(tt.fields.update2) + m.bucketsMeasurement[BucketOptions{Name: "bucket", ReplicationARN: "arn"}].updateExponentialMovingAverage(tt.fields.endTime2) + got = m.GetReport(SelectBuckets()) + if !reflect.DeepEqual(got.BucketStats, tt.want2.BucketStats) { + t.Errorf("GetReport() = %v, want %v", got.BucketStats, tt.want2.BucketStats) + } + }) + } +} diff --git a/internal/bucket/bandwidth/reader.go b/internal/bucket/bandwidth/reader.go new file mode 100644 index 00000000..30cb8559 --- /dev/null +++ b/internal/bucket/bandwidth/reader.go @@ -0,0 +1,107 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package bandwidth + +import ( + "context" + "io" + "math" +) + +// MonitoredReader represents a throttled reader subject to bandwidth monitoring +type MonitoredReader struct { + r io.Reader + throttle *bucketThrottle + ctx context.Context // request context + lastErr error // last error reported, if this non-nil all reads will fail. + m *Monitor + opts *MonitorReaderOptions +} + +// BucketOptions represents the bucket and optionally its replication target pair. +type BucketOptions struct { + Name string + ReplicationARN string // This is optional, and not mandatory. +} + +// MonitorReaderOptions provides configurable options for monitor reader implementation. +type MonitorReaderOptions struct { + BucketOptions + HeaderSize int +} + +// Read implements a throttled read +func (r *MonitoredReader) Read(buf []byte) (n int, err error) { + if r.throttle == nil { + return r.r.Read(buf) + } + if r.lastErr != nil { + err = r.lastErr + return n, err + } + b := r.throttle.Burst() // maximum available tokens + need := len(buf) // number of bytes requested by caller + hdr := r.opts.HeaderSize // remaining header bytes + var tokens int // number of tokens to request + + if hdr > 0 { // available tokens go towards header first + if hdr < b { // all of header can be accommodated + r.opts.HeaderSize = 0 + need = int(math.Min(float64(b-hdr), float64(need))) // use remaining tokens towards payload + tokens = need + hdr + } else { // part of header can be accommodated + r.opts.HeaderSize -= b - 1 + need = 1 // to ensure we read at least one byte for every Read + tokens = b + } + } else { // all tokens go towards payload + need = int(math.Min(float64(b), float64(need))) + tokens = need + } + // reduce tokens requested according to availability + av := int(r.throttle.Tokens()) + if av < tokens && av > 0 { + tokens = av + need = int(math.Min(float64(tokens), float64(need))) + } + err = r.throttle.WaitN(r.ctx, tokens) + if err != nil { + return n, err + } + n, err = r.r.Read(buf[:need]) + if err != nil { + r.lastErr = err + return n, err + } + r.m.updateMeasurement(r.opts.BucketOptions, uint64(tokens)) + return n, err +} + +// NewMonitoredReader returns reference to a monitored reader that throttles reads to configured bandwidth for the +// bucket. +func NewMonitoredReader(ctx context.Context, m *Monitor, r io.Reader, opts *MonitorReaderOptions) *MonitoredReader { + reader := MonitoredReader{ + r: r, + throttle: m.throttle(opts.BucketOptions), + m: m, + opts: opts, + ctx: ctx, + } + reader.m.init(opts.BucketOptions) + return &reader +} diff --git a/internal/bucket/encryption/bucket-sse-config.go b/internal/bucket/encryption/bucket-sse-config.go new file mode 100644 index 00000000..ef403a47 --- /dev/null +++ b/internal/bucket/encryption/bucket-sse-config.go @@ -0,0 +1,171 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package sse + +import ( + "encoding/xml" + "errors" + "io" + "net/http" + "strings" + + "github.com/minio/minio/internal/crypto" + xhttp "github.com/minio/minio/internal/http" +) + +const ( + // AES256 is used with SSE-S3 + AES256 Algorithm = "AES256" + // AWSKms is used with SSE-KMS + AWSKms Algorithm = "aws:kms" +) + +// Algorithm - represents valid SSE algorithms supported; currently only AES256 is supported +type Algorithm string + +// UnmarshalXML - Unmarshals XML tag to valid SSE algorithm +func (alg *Algorithm) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + var s string + if err := d.DecodeElement(&s, &start); err != nil { + return err + } + + switch s { + case string(AES256): + *alg = AES256 + case string(AWSKms): + *alg = AWSKms + default: + return errors.New("Unknown SSE algorithm") + } + + return nil +} + +// MarshalXML - Marshals given SSE algorithm to valid XML +func (alg *Algorithm) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + return e.EncodeElement(string(*alg), start) +} + +// EncryptionAction - for ApplyServerSideEncryptionByDefault XML tag +type EncryptionAction struct { + Algorithm Algorithm `xml:"SSEAlgorithm,omitempty"` + MasterKeyID string `xml:"KMSMasterKeyID,omitempty"` +} + +// Rule - for ServerSideEncryptionConfiguration XML tag +type Rule struct { + DefaultEncryptionAction EncryptionAction `xml:"ApplyServerSideEncryptionByDefault"` +} + +const xmlNS = "http://s3.amazonaws.com/doc/2006-03-01/" + +// BucketSSEConfig - represents default bucket encryption configuration +type BucketSSEConfig struct { + XMLNS string `xml:"xmlns,attr,omitempty"` + XMLName xml.Name `xml:"ServerSideEncryptionConfiguration"` + Rules []Rule `xml:"Rule"` +} + +// ParseBucketSSEConfig - Decodes given XML to a valid default bucket encryption config +func ParseBucketSSEConfig(r io.Reader) (*BucketSSEConfig, error) { + var config BucketSSEConfig + err := xml.NewDecoder(r).Decode(&config) + if err != nil { + return nil, err + } + + // Validates server-side encryption config rules + // Only one rule is allowed on AWS S3 + if len(config.Rules) != 1 { + return nil, errors.New("only one server-side encryption rule is allowed at a time") + } + + for _, rule := range config.Rules { + switch rule.DefaultEncryptionAction.Algorithm { + case AES256: + if rule.DefaultEncryptionAction.MasterKeyID != "" { + return nil, errors.New("MasterKeyID is allowed with aws:kms only") + } + case AWSKms: + keyID := rule.DefaultEncryptionAction.MasterKeyID + if keyID == "" { + return nil, errors.New("MasterKeyID is missing with aws:kms") + } + spaces := strings.HasPrefix(keyID, " ") || strings.HasSuffix(keyID, " ") + if spaces { + return nil, errors.New("MasterKeyID contains unsupported characters") + } + } + } + + if config.XMLNS == "" { + config.XMLNS = xmlNS + } + return &config, nil +} + +// ApplyOptions ask for specific features to be enabled, +// when bucketSSEConfig is empty. +type ApplyOptions struct { + AutoEncrypt bool +} + +// Apply applies the SSE bucket configuration on the given HTTP headers and +// sets the specified SSE headers. +// +// Apply does not overwrite any existing SSE headers. Further, it will +// set minimal SSE-KMS headers if autoEncrypt is true and the BucketSSEConfig +// is nil. +func (b *BucketSSEConfig) Apply(headers http.Header, opts ApplyOptions) { + if crypto.Requested(headers) { + return + } + if b == nil { + if opts.AutoEncrypt { + headers.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS) + } + return + } + + switch b.Algo() { + case xhttp.AmzEncryptionAES: + headers.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionAES) + case xhttp.AmzEncryptionKMS: + headers.Set(xhttp.AmzServerSideEncryption, xhttp.AmzEncryptionKMS) + headers.Set(xhttp.AmzServerSideEncryptionKmsID, b.KeyID()) + } +} + +// Algo returns the SSE algorithm specified by the SSE configuration. +func (b *BucketSSEConfig) Algo() Algorithm { + for _, rule := range b.Rules { + return rule.DefaultEncryptionAction.Algorithm + } + return "" +} + +// KeyID returns the KMS key ID specified by the SSE configuration. +// If the SSE configuration does not specify SSE-KMS it returns an +// empty key ID. +func (b *BucketSSEConfig) KeyID() string { + for _, rule := range b.Rules { + return strings.TrimPrefix(rule.DefaultEncryptionAction.MasterKeyID, crypto.ARNPrefix) + } + return "" +} diff --git a/internal/bucket/encryption/bucket-sse-config_test.go b/internal/bucket/encryption/bucket-sse-config_test.go new file mode 100644 index 00000000..5918e22a --- /dev/null +++ b/internal/bucket/encryption/bucket-sse-config_test.go @@ -0,0 +1,153 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package sse + +import ( + "bytes" + "encoding/xml" + "errors" + "testing" +) + +// TestParseBucketSSEConfig performs basic sanity tests on ParseBucketSSEConfig +func TestParseBucketSSEConfig(t *testing.T) { + actualAES256NoNSConfig := &BucketSSEConfig{ + XMLName: xml.Name{ + Local: "ServerSideEncryptionConfiguration", + }, + Rules: []Rule{ + { + DefaultEncryptionAction: EncryptionAction{ + Algorithm: AES256, + }, + }, + }, + } + + actualAES256Config := &BucketSSEConfig{ + XMLNS: xmlNS, + XMLName: xml.Name{ + Local: "ServerSideEncryptionConfiguration", + }, + Rules: []Rule{ + { + DefaultEncryptionAction: EncryptionAction{ + Algorithm: AES256, + }, + }, + }, + } + + actualKMSConfig := &BucketSSEConfig{ + XMLNS: xmlNS, + XMLName: xml.Name{ + Local: "ServerSideEncryptionConfiguration", + }, + Rules: []Rule{ + { + DefaultEncryptionAction: EncryptionAction{ + Algorithm: AWSKms, + MasterKeyID: "arn:aws:kms:my-minio-key", + }, + }, + }, + } + + testCases := []struct { + inputXML string + keyID string + expectedErr error + shouldPass bool + expectedConfig *BucketSSEConfig + }{ + // 1. Valid XML SSE-S3 + { + inputXML: `AES256`, + expectedErr: nil, + shouldPass: true, + expectedConfig: actualAES256Config, + }, + // 2. Valid XML SSE-KMS + { + inputXML: `aws:kmsarn:aws:kms:my-minio-key`, + expectedErr: nil, + shouldPass: true, + expectedConfig: actualKMSConfig, + keyID: "my-minio-key", + }, + // 3. Invalid - more than one rule + { + inputXML: `AES256AES256`, + expectedErr: errors.New("only one server-side encryption rule is allowed at a time"), + shouldPass: false, + }, + // 4. Invalid XML - master key ID present along with AES256 + { + inputXML: `AES256arn:aws:kms:us-east-1:1234/5678example`, + expectedErr: errors.New("MasterKeyID is allowed with aws:kms only"), + shouldPass: false, + }, + // 5. Invalid XML - master key ID not provided when algorithm is set to aws:kms algorithm + { + inputXML: `aws:kms`, + expectedErr: errors.New("MasterKeyID is missing with aws:kms"), + shouldPass: false, + }, + // 6. Invalid Algorithm + { + inputXML: `InvalidAlgorithm`, + expectedErr: errors.New("Unknown SSE algorithm"), + shouldPass: false, + }, + // 7. Valid XML without the namespace set + { + inputXML: `AES256`, + expectedErr: nil, + shouldPass: true, + expectedConfig: actualAES256NoNSConfig, + }, + // 8. Space characters in MasterKeyID + { + inputXML: `aws:kms arn:aws:kms:my-minio-key `, + expectedErr: errors.New("MasterKeyID contains unsupported characters"), + shouldPass: false, + }, + } + + for i, tc := range testCases { + ssec, err := ParseBucketSSEConfig(bytes.NewReader([]byte(tc.inputXML))) + if tc.shouldPass && err != nil { + t.Errorf("Test case %d: Expected to succeed but got %s", i+1, err) + } + + if !tc.shouldPass { + if err == nil || err != nil && err.Error() != tc.expectedErr.Error() { + t.Errorf("Test case %d: Expected %s but got %s", i+1, tc.expectedErr, err) + } + continue + } + + if tc.keyID != "" && tc.keyID != ssec.KeyID() { + t.Errorf("Test case %d: Expected bucket encryption KeyID %s but got %s", i+1, tc.keyID, ssec.KeyID()) + } + + if expectedXML, err := xml.Marshal(tc.expectedConfig); err != nil || !bytes.Equal(expectedXML, []byte(tc.inputXML)) { + t.Errorf("Test case %d: Expected bucket encryption XML %s but got %s", i+1, string(expectedXML), tc.inputXML) + } + } +} diff --git a/internal/bucket/lifecycle/action_string.go b/internal/bucket/lifecycle/action_string.go new file mode 100644 index 00000000..e3f11ee5 --- /dev/null +++ b/internal/bucket/lifecycle/action_string.go @@ -0,0 +1,32 @@ +// Code generated by "stringer -type Action lifecycle.go"; DO NOT EDIT. + +package lifecycle + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[NoneAction-0] + _ = x[DeleteAction-1] + _ = x[DeleteVersionAction-2] + _ = x[TransitionAction-3] + _ = x[TransitionVersionAction-4] + _ = x[DeleteRestoredAction-5] + _ = x[DeleteRestoredVersionAction-6] + _ = x[DeleteAllVersionsAction-7] + _ = x[DelMarkerDeleteAllVersionsAction-8] + _ = x[ActionCount-9] +} + +const _Action_name = "NoneActionDeleteActionDeleteVersionActionTransitionActionTransitionVersionActionDeleteRestoredActionDeleteRestoredVersionActionDeleteAllVersionsActionDelMarkerDeleteAllVersionsActionActionCount" + +var _Action_index = [...]uint8{0, 10, 22, 41, 57, 80, 100, 127, 150, 182, 193} + +func (i Action) String() string { + if i < 0 || i >= Action(len(_Action_index)-1) { + return "Action(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _Action_name[_Action_index[i]:_Action_index[i+1]] +} diff --git a/internal/bucket/lifecycle/and.go b/internal/bucket/lifecycle/and.go new file mode 100644 index 00000000..f46780c3 --- /dev/null +++ b/internal/bucket/lifecycle/and.go @@ -0,0 +1,105 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lifecycle + +import ( + "encoding/xml" +) + +var errDuplicateTagKey = Errorf("Duplicate Tag Keys are not allowed") + +// And - a tag to combine a prefix and multiple tags for lifecycle configuration rule. +type And struct { + XMLName xml.Name `xml:"And"` + ObjectSizeGreaterThan int64 `xml:"ObjectSizeGreaterThan,omitempty"` + ObjectSizeLessThan int64 `xml:"ObjectSizeLessThan,omitempty"` + Prefix Prefix `xml:"Prefix,omitempty"` + Tags []Tag `xml:"Tag,omitempty"` +} + +// isEmpty returns true if Tags field is null +func (a And) isEmpty() bool { + return len(a.Tags) == 0 && !a.Prefix.set && + a.ObjectSizeGreaterThan == 0 && a.ObjectSizeLessThan == 0 +} + +// Validate - validates the And field +func (a And) Validate() error { + // > This is used in a Lifecycle Rule Filter to apply a logical AND to two or more predicates. + // ref: https://docs.aws.amazon.com/AmazonS3/latest/API/API_LifecycleRuleAndOperator.html + // i.e, predCount >= 2 + var predCount int + if a.Prefix.set { + predCount++ + } + predCount += len(a.Tags) + if a.ObjectSizeGreaterThan > 0 { + predCount++ + } + if a.ObjectSizeLessThan > 0 { + predCount++ + } + + if predCount < 2 { + return errXMLNotWellFormed + } + + if a.ContainsDuplicateTag() { + return errDuplicateTagKey + } + for _, t := range a.Tags { + if err := t.Validate(); err != nil { + return err + } + } + + if a.ObjectSizeGreaterThan < 0 || a.ObjectSizeLessThan < 0 { + return errXMLNotWellFormed + } + return nil +} + +// ContainsDuplicateTag - returns true if duplicate keys are present in And +func (a And) ContainsDuplicateTag() bool { + x := make(map[string]struct{}, len(a.Tags)) + + for _, t := range a.Tags { + if _, has := x[t.Key]; has { + return true + } + x[t.Key] = struct{}{} + } + + return false +} + +// BySize returns true when sz satisfies a +// ObjectSizeLessThan/ObjectSizeGreaterthan or a logical AND of these predicates +// Note: And combines size and other predicates like Tags, Prefix, etc. This +// method applies exclusively to size predicates only. +func (a And) BySize(sz int64) bool { + if a.ObjectSizeGreaterThan > 0 && + sz <= a.ObjectSizeGreaterThan { + return false + } + if a.ObjectSizeLessThan > 0 && + sz >= a.ObjectSizeLessThan { + return false + } + return true +} diff --git a/internal/bucket/lifecycle/delmarker-expiration.go b/internal/bucket/lifecycle/delmarker-expiration.go new file mode 100644 index 00000000..db22d291 --- /dev/null +++ b/internal/bucket/lifecycle/delmarker-expiration.go @@ -0,0 +1,74 @@ +// Copyright (c) 2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lifecycle + +import ( + "encoding/xml" + "time" +) + +var errInvalidDaysDelMarkerExpiration = Errorf("Days must be a positive integer with DelMarkerExpiration") + +// DelMarkerExpiration used to xml encode/decode ILM action by the same name +type DelMarkerExpiration struct { + XMLName xml.Name `xml:"DelMarkerExpiration"` + Days int `xml:"Days,omitempty"` +} + +// Empty returns if a DelMarkerExpiration XML element is empty. +// Used to detect if lifecycle.Rule contained a DelMarkerExpiration element. +func (de DelMarkerExpiration) Empty() bool { + return de.Days == 0 +} + +// UnmarshalXML decodes a single XML element into a DelMarkerExpiration value +func (de *DelMarkerExpiration) UnmarshalXML(dec *xml.Decoder, start xml.StartElement) error { + type delMarkerExpiration DelMarkerExpiration + var dexp delMarkerExpiration + err := dec.DecodeElement(&dexp, &start) + if err != nil { + return err + } + + if dexp.Days <= 0 { + return errInvalidDaysDelMarkerExpiration + } + + *de = DelMarkerExpiration(dexp) + return nil +} + +// MarshalXML encodes a DelMarkerExpiration value into an XML element +func (de DelMarkerExpiration) MarshalXML(enc *xml.Encoder, start xml.StartElement) error { + if de.Empty() { + return nil + } + + type delMarkerExpiration DelMarkerExpiration + return enc.EncodeElement(delMarkerExpiration(de), start) +} + +// NextDue returns upcoming DelMarkerExpiration date for obj if +// applicable, returns false otherwise. +func (de DelMarkerExpiration) NextDue(obj ObjectOpts) (time.Time, bool) { + if !obj.IsLatest || !obj.DeleteMarker { + return time.Time{}, false + } + + return ExpectedExpiryTime(obj.ModTime, de.Days), true +} diff --git a/internal/bucket/lifecycle/delmarker-expiration_test.go b/internal/bucket/lifecycle/delmarker-expiration_test.go new file mode 100644 index 00000000..8cba948c --- /dev/null +++ b/internal/bucket/lifecycle/delmarker-expiration_test.go @@ -0,0 +1,63 @@ +// Copyright (c) 2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lifecycle + +import ( + "encoding/xml" + "fmt" + "testing" +) + +func TestDelMarkerExpParseAndValidate(t *testing.T) { + tests := []struct { + xml string + err error + }{ + { + xml: ` 1 `, + err: nil, + }, + { + xml: ` -1 `, + err: errInvalidDaysDelMarkerExpiration, + }, + } + + for i, test := range tests { + t.Run(fmt.Sprintf("TestDelMarker-%d", i), func(t *testing.T) { + var dexp DelMarkerExpiration + var fail bool + err := xml.Unmarshal([]byte(test.xml), &dexp) + if test.err == nil { + if err != nil { + fail = true + } + } else { + if err == nil { + fail = true + } + if test.err.Error() != err.Error() { + fail = true + } + } + if fail { + t.Fatalf("Expected %v but got %v", test.err, err) + } + }) + } +} diff --git a/internal/bucket/lifecycle/error.go b/internal/bucket/lifecycle/error.go new file mode 100644 index 00000000..c3a8572f --- /dev/null +++ b/internal/bucket/lifecycle/error.go @@ -0,0 +1,45 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lifecycle + +import ( + "fmt" +) + +// Error is the generic type for any error happening during tag +// parsing. +type Error struct { + err error +} + +// Errorf - formats according to a format specifier and returns +// the string as a value that satisfies error of type tagging.Error +func Errorf(format string, a ...any) error { + return Error{err: fmt.Errorf(format, a...)} +} + +// Unwrap the internal error. +func (e Error) Unwrap() error { return e.err } + +// Error 'error' compatible method. +func (e Error) Error() string { + if e.err == nil { + return "lifecycle: cause " + } + return e.err.Error() +} diff --git a/internal/bucket/lifecycle/evaluator.go b/internal/bucket/lifecycle/evaluator.go new file mode 100644 index 00000000..ec6f04e6 --- /dev/null +++ b/internal/bucket/lifecycle/evaluator.go @@ -0,0 +1,156 @@ +// Copyright (c) 2015-2025 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lifecycle + +import ( + "fmt" + "time" + + objlock "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/replication" +) + +// Evaluator - evaluates lifecycle policy on objects for the given lifecycle +// configuration, lock retention configuration and replication configuration. +type Evaluator struct { + policy Lifecycle + lockRetention *objlock.Retention + replCfg *replication.Config +} + +// NewEvaluator - creates a new evaluator with the given lifecycle +func NewEvaluator(policy Lifecycle) *Evaluator { + return &Evaluator{ + policy: policy, + } +} + +// WithLockRetention - sets the lock retention configuration for the evaluator +func (e *Evaluator) WithLockRetention(lr *objlock.Retention) *Evaluator { + e.lockRetention = lr + return e +} + +// WithReplicationConfig - sets the replication configuration for the evaluator +func (e *Evaluator) WithReplicationConfig(rcfg *replication.Config) *Evaluator { + e.replCfg = rcfg + return e +} + +// IsPendingReplication checks if the object is pending replication. +func (e *Evaluator) IsPendingReplication(obj ObjectOpts) bool { + if e.replCfg == nil { + return false + } + if e.replCfg.HasActiveRules(obj.Name, true) && !obj.VersionPurgeStatus.Empty() { + return true + } + + return false +} + +// IsObjectLocked checks if it is appropriate to remove an +// object according to locking configuration when this is lifecycle/ bucket quota asking. +// (copied over from enforceRetentionForDeletion) +func (e *Evaluator) IsObjectLocked(obj ObjectOpts) bool { + if e.lockRetention == nil || !e.lockRetention.LockEnabled { + return false + } + + if obj.DeleteMarker { + return false + } + + lhold := objlock.GetObjectLegalHoldMeta(obj.UserDefined) + if lhold.Status.Valid() && lhold.Status == objlock.LegalHoldOn { + return true + } + + ret := objlock.GetObjectRetentionMeta(obj.UserDefined) + if ret.Mode.Valid() && (ret.Mode == objlock.RetCompliance || ret.Mode == objlock.RetGovernance) { + t, err := objlock.UTCNowNTP() + if err != nil { + // it is safe to assume that the object is locked when + // we can't get the current time + return true + } + if ret.RetainUntilDate.After(t) { + return true + } + } + return false +} + +// eval will return a lifecycle event for each object in objs for a given time. +func (e *Evaluator) eval(objs []ObjectOpts, now time.Time) []Event { + events := make([]Event, len(objs)) + var newerNoncurrentVersions int +loop: + for i, obj := range objs { + event := e.policy.eval(obj, now, newerNoncurrentVersions) + switch event.Action { + case DeleteAllVersionsAction, DelMarkerDeleteAllVersionsAction: + // Skip if bucket has object locking enabled; To prevent the + // possibility of violating an object retention on one of the + // noncurrent versions of this object. + if e.lockRetention != nil && e.lockRetention.LockEnabled { + event = Event{} + } else { + // No need to evaluate remaining versions' lifecycle + // events after DeleteAllVersionsAction* + events[i] = event + break loop + } + + case DeleteVersionAction, DeleteRestoredVersionAction: + // Defensive code, should never happen + if obj.VersionID == "" { + event.Action = NoneAction + } + if e.IsObjectLocked(obj) { + event = Event{} + } + + if e.IsPendingReplication(obj) { + event = Event{} + } + } + if !obj.IsLatest { + switch event.Action { + case DeleteVersionAction: + // this noncurrent version will be expired, nothing to add + default: + // this noncurrent version will be spared + newerNoncurrentVersions++ + } + } + events[i] = event + } + return events +} + +// Eval will return a lifecycle event for each object in objs +func (e *Evaluator) Eval(objs []ObjectOpts) ([]Event, error) { + if len(objs) == 0 { + return nil, nil + } + if len(objs) != objs[0].NumVersions { + return nil, fmt.Errorf("number of versions mismatch, expected %d, got %d", objs[0].NumVersions, len(objs)) + } + return e.eval(objs, time.Now().UTC()), nil +} diff --git a/internal/bucket/lifecycle/evaluator_test.go b/internal/bucket/lifecycle/evaluator_test.go new file mode 100644 index 00000000..8225fcd6 --- /dev/null +++ b/internal/bucket/lifecycle/evaluator_test.go @@ -0,0 +1,183 @@ +// Copyright (c) 2015-2025 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lifecycle + +import ( + "fmt" + "testing" + "time" + + "github.com/google/uuid" +) + +func TestNewerNoncurrentVersions(t *testing.T) { + prepLifecycleCfg := func(tagKeys []string, retainVersions []int) Lifecycle { + var lc Lifecycle + for i := range retainVersions { + ruleID := fmt.Sprintf("rule-%d", i) + tag := Tag{ + Key: tagKeys[i], + Value: "minio", + } + lc.Rules = append(lc.Rules, Rule{ + ID: ruleID, + Status: "Enabled", + Filter: Filter{ + Tag: tag, + set: true, + }, + NoncurrentVersionExpiration: NoncurrentVersionExpiration{ + NewerNoncurrentVersions: retainVersions[i], + set: true, + }, + }) + } + return lc + } + + lc := prepLifecycleCfg([]string{"tag3", "tag4", "tag5"}, []int{3, 4, 5}) + evaluator := NewEvaluator(lc) + tagKeys := []string{"tag3", "tag3", "tag3", "tag4", "tag4", "tag5", "tag5"} + verIDs := []string{ + "0NdAikoUVNGEpCUuB9vl.XyoMftMXCSg", "19M6Z405yFZuYygnnU9jKzsOBamTZK_7", "0PmlJdFWi_9d6l_dAkWrrhP.bBgtFk6V", // spellchecker:disable-line + ".MmRalFNNJyOLymgCtQ3.qsdoYpy8qkB", "Bjb4OlMW9Agx.Nrggh15iU6frGu2CLde", "ngBmUd_cVl6ckONI9XsKGpJjzimohrzZ", // spellchecker:disable-line + "T6m1heTHLUtnByW2IOWJ3zM4JP9xXt2O", // spellchecker:disable-line + } + wantEvents := []Event{ + {Action: NoneAction}, + {Action: NoneAction}, + {Action: NoneAction}, + {Action: NoneAction}, + {Action: NoneAction}, + {Action: NoneAction}, + {Action: DeleteVersionAction}, + } + var objs []ObjectOpts + curModTime := time.Date(2025, time.February, 10, 23, 0, 0, 0, time.UTC) + for i := range tagKeys { + obj := ObjectOpts{ + Name: "obj", + VersionID: verIDs[i], + ModTime: curModTime.Add(time.Duration(-i) * time.Second), + UserTags: fmt.Sprintf("%s=minio", tagKeys[i]), + NumVersions: len(verIDs), + } + if i == 0 { + obj.IsLatest = true + } else { + obj.SuccessorModTime = curModTime.Add(time.Duration(-i+1) * time.Second) + } + objs = append(objs, obj) + } + now := time.Date(2025, time.February, 10, 23, 0, 0, 0, time.UTC) + gotEvents := evaluator.eval(objs, now) + for i := range wantEvents { + if gotEvents[i].Action != wantEvents[i].Action { + t.Fatalf("got %v, want %v", gotEvents[i], wantEvents[i]) + } + } + + lc = prepLifecycleCfg([]string{"tag3", "tag4", "tag5"}, []int{1, 2, 3}) + objs = objs[:len(objs)-1] + wantEvents = []Event{ + {Action: NoneAction}, + {Action: NoneAction}, + {Action: DeleteVersionAction}, + {Action: NoneAction}, + {Action: DeleteVersionAction}, + {Action: NoneAction}, + } + evaluator = NewEvaluator(lc) + gotEvents = evaluator.eval(objs, now) + for i := range wantEvents { + if gotEvents[i].Action != wantEvents[i].Action { + t.Fatalf("test-%d: got %v, want %v", i+1, gotEvents[i], wantEvents[i]) + } + } + + lc = Lifecycle{ + Rules: []Rule{ + { + ID: "AllVersionsExpiration", + Status: "Enabled", + Filter: Filter{}, + Expiration: Expiration{ + Days: 1, + DeleteAll: Boolean{ + val: true, + set: true, + }, + set: true, + }, + }, + }, + } + + now = time.Date(2025, time.February, 12, 23, 0, 0, 0, time.UTC) + evaluator = NewEvaluator(lc) + gotEvents = evaluator.eval(objs, now) + wantEvents = []Event{ + {Action: DeleteAllVersionsAction}, + {Action: NoneAction}, + {Action: NoneAction}, + {Action: NoneAction}, + {Action: NoneAction}, + {Action: NoneAction}, + } + for i := range wantEvents { + if gotEvents[i].Action != wantEvents[i].Action { + t.Fatalf("test-%d: got %v, want %v", i+1, gotEvents[i], wantEvents[i]) + } + } + + // Test with zero versions + events, err := evaluator.Eval(nil) + if len(events) != 0 || err != nil { + t.Fatal("expected no events nor error") + } +} + +func TestEmptyEvaluator(t *testing.T) { + var objs []ObjectOpts + curModTime := time.Date(2025, time.February, 10, 23, 0, 0, 0, time.UTC) + for i := range 5 { + obj := ObjectOpts{ + Name: "obj", + VersionID: uuid.New().String(), + ModTime: curModTime.Add(time.Duration(-i) * time.Second), + NumVersions: 5, + } + if i == 0 { + obj.IsLatest = true + } else { + obj.SuccessorModTime = curModTime.Add(time.Duration(-i+1) * time.Second) + } + objs = append(objs, obj) + } + + evaluator := NewEvaluator(Lifecycle{}) + events, err := evaluator.Eval(objs) + if err != nil { + t.Fatal(err) + } + for _, event := range events { + if event.Action != NoneAction { + t.Fatalf("got %v, want %v", event.Action, NoneAction) + } + } +} diff --git a/internal/bucket/lifecycle/expiration.go b/internal/bucket/lifecycle/expiration.go new file mode 100644 index 00000000..8acc2039 --- /dev/null +++ b/internal/bucket/lifecycle/expiration.go @@ -0,0 +1,211 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lifecycle + +import ( + "encoding/xml" + "time" +) + +var ( + errLifecycleInvalidDate = Errorf("Date must be provided in ISO 8601 format") + errLifecycleInvalidDays = Errorf("Days must be positive integer when used with Expiration") + errLifecycleInvalidExpiration = Errorf("Exactly one of Days (positive integer) or Date (positive ISO 8601 format) should be present inside Expiration.") + errLifecycleInvalidDeleteMarker = Errorf("Delete marker cannot be specified with Days or Date in a Lifecycle Expiration Policy") + errLifecycleDateNotMidnight = Errorf("'Date' must be at midnight GMT") + errLifecycleInvalidDeleteAll = Errorf("Days (positive integer) should be present inside Expiration with ExpiredObjectAllVersions.") +) + +// ExpirationDays is a type alias to unmarshal Days in Expiration +type ExpirationDays int + +// UnmarshalXML parses number of days from Expiration and validates if +// greater than zero +func (eDays *ExpirationDays) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error { + var numDays int + err := d.DecodeElement(&numDays, &startElement) + if err != nil { + return err + } + if numDays <= 0 { + return errLifecycleInvalidDays + } + *eDays = ExpirationDays(numDays) + return nil +} + +// MarshalXML encodes number of days to expire if it is non-zero and +// encodes empty string otherwise +func (eDays ExpirationDays) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { + if eDays == 0 { + return nil + } + return e.EncodeElement(int(eDays), startElement) +} + +// ExpirationDate is a embedded type containing time.Time to unmarshal +// Date in Expiration +type ExpirationDate struct { + time.Time +} + +// UnmarshalXML parses date from Expiration and validates date format +func (eDate *ExpirationDate) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error { + var dateStr string + err := d.DecodeElement(&dateStr, &startElement) + if err != nil { + return err + } + // While AWS documentation mentions that the date specified + // must be present in ISO 8601 format, in reality they allow + // users to provide RFC 3339 compliant dates. + expDate, err := time.Parse(time.RFC3339, dateStr) + if err != nil { + return errLifecycleInvalidDate + } + // Allow only date timestamp specifying midnight GMT + hr, m, sec := expDate.Clock() + nsec := expDate.Nanosecond() + loc := expDate.Location() + if hr != 0 || m != 0 || sec != 0 || nsec != 0 || loc.String() != time.UTC.String() { + return errLifecycleDateNotMidnight + } + + *eDate = ExpirationDate{expDate} + return nil +} + +// MarshalXML encodes expiration date if it is non-zero and encodes +// empty string otherwise +func (eDate ExpirationDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { + if eDate.IsZero() { + return nil + } + return e.EncodeElement(eDate.Format(time.RFC3339), startElement) +} + +// ExpireDeleteMarker represents value of ExpiredObjectDeleteMarker field in Expiration XML element. +type ExpireDeleteMarker struct { + Boolean +} + +// Boolean signifies a boolean XML struct with custom marshaling +type Boolean struct { + val bool + set bool + Unused struct{} // Needed for GOB compatibility +} + +// Expiration - expiration actions for a rule in lifecycle configuration. +type Expiration struct { + XMLName xml.Name `xml:"Expiration"` + Days ExpirationDays `xml:"Days,omitempty"` + Date ExpirationDate `xml:"Date,omitempty"` + DeleteMarker ExpireDeleteMarker `xml:"ExpiredObjectDeleteMarker"` + // Indicates whether MinIO will remove all versions. If set to true, all versions will be deleted; + // if set to false the policy takes no action. This action uses the Days/Date to expire objects. + // This check is verified for latest version of the object. + DeleteAll Boolean `xml:"ExpiredObjectAllVersions"` + + set bool +} + +// MarshalXML encodes delete marker boolean into an XML form. +func (b Boolean) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { + if !b.set { + return nil + } + return e.EncodeElement(b.val, startElement) +} + +// UnmarshalXML decodes delete marker boolean from the XML form. +func (b *Boolean) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error { + var exp bool + err := d.DecodeElement(&exp, &startElement) + if err != nil { + return err + } + b.val = exp + b.set = true + return nil +} + +// MarshalXML encodes expiration field into an XML form. +func (e Expiration) MarshalXML(enc *xml.Encoder, startElement xml.StartElement) error { + if !e.set { + return nil + } + type expirationWrapper Expiration + return enc.EncodeElement(expirationWrapper(e), startElement) +} + +// UnmarshalXML decodes expiration field from the XML form. +func (e *Expiration) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error { + type expirationWrapper Expiration + var exp expirationWrapper + err := d.DecodeElement(&exp, &startElement) + if err != nil { + return err + } + *e = Expiration(exp) + e.set = true + return nil +} + +// Validate - validates the "Expiration" element +func (e Expiration) Validate() error { + if !e.set { + return nil + } + + // DeleteMarker cannot be specified if date or dates are specified. + if (!e.IsDaysNull() || !e.IsDateNull()) && e.DeleteMarker.set { + return errLifecycleInvalidDeleteMarker + } + + if !e.DeleteMarker.set && !e.DeleteAll.set && e.IsDaysNull() && e.IsDateNull() { + return errXMLNotWellFormed + } + + // Both expiration days and date are specified + if !e.IsDaysNull() && !e.IsDateNull() { + return errLifecycleInvalidExpiration + } + + // DeleteAll set without expiration days + if e.DeleteAll.set && e.IsDaysNull() { + return errLifecycleInvalidDeleteAll + } + + return nil +} + +// IsDaysNull returns true if days field is null +func (e Expiration) IsDaysNull() bool { + return e.Days == ExpirationDays(0) +} + +// IsDateNull returns true if date field is null +func (e Expiration) IsDateNull() bool { + return e.Date.IsZero() +} + +// IsNull returns true if both date and days fields are null +func (e Expiration) IsNull() bool { + return e.IsDaysNull() && e.IsDateNull() +} diff --git a/internal/bucket/lifecycle/expiration_test.go b/internal/bucket/lifecycle/expiration_test.go new file mode 100644 index 00000000..76cac74d --- /dev/null +++ b/internal/bucket/lifecycle/expiration_test.go @@ -0,0 +1,126 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lifecycle + +import ( + "encoding/xml" + "fmt" + "testing" +) + +// appropriate errors on validation +func TestInvalidExpiration(t *testing.T) { + testCases := []struct { + inputXML string + expectedErr error + }{ + { // Expiration with zero days + inputXML: ` + 0 + `, + expectedErr: errLifecycleInvalidDays, + }, + { // Expiration with invalid date + inputXML: ` + invalid date + `, + expectedErr: errLifecycleInvalidDate, + }, + { // Expiration with both number of days nor a date + inputXML: ` + 2019-04-20T00:01:00Z + `, + expectedErr: errLifecycleDateNotMidnight, + }, + } + + for i, tc := range testCases { + t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) { + var expiration Expiration + err := xml.Unmarshal([]byte(tc.inputXML), &expiration) + if err != tc.expectedErr { + t.Fatalf("%d: Expected %v but got %v", i+1, tc.expectedErr, err) + } + }) + } + + validationTestCases := []struct { + inputXML string + expectedErr error + }{ + { // Expiration with a valid ISO 8601 date + inputXML: ` + 2019-04-20T00:00:00Z + `, + expectedErr: nil, + }, + { // Expiration with a valid number of days + inputXML: ` + 3 + `, + expectedErr: nil, + }, + { // Expiration with neither number of days nor a date + inputXML: ` + `, + expectedErr: errXMLNotWellFormed, + }, + { // Expiration with both number of days and a date + inputXML: ` + 3 + 2019-04-20T00:00:00Z + `, + expectedErr: errLifecycleInvalidExpiration, + }, + { // Expiration with both ExpiredObjectDeleteMarker and days + inputXML: ` + 3 + false + `, + expectedErr: errLifecycleInvalidDeleteMarker, + }, + { // Expiration with a valid number of days and ExpiredObjectAllVersions + inputXML: ` + 3 + true + `, + expectedErr: nil, + }, + { // Expiration with a valid ISO 8601 date and ExpiredObjectAllVersions + inputXML: ` + 2019-04-20T00:00:00Z + true + `, + expectedErr: errLifecycleInvalidDeleteAll, + }, + } + for i, tc := range validationTestCases { + t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) { + var expiration Expiration + err := xml.Unmarshal([]byte(tc.inputXML), &expiration) + if err != nil { + t.Fatalf("%d: %v", i+1, err) + } + + err = expiration.Validate() + if err != tc.expectedErr { + t.Fatalf("%d: got: %v, expected: %v", i+1, err, tc.expectedErr) + } + }) + } +} diff --git a/internal/bucket/lifecycle/filter.go b/internal/bucket/lifecycle/filter.go new file mode 100644 index 00000000..6e605d3e --- /dev/null +++ b/internal/bucket/lifecycle/filter.go @@ -0,0 +1,270 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lifecycle + +import ( + "encoding/xml" + "io" + + "github.com/minio/minio-go/v7/pkg/tags" +) + +var errInvalidFilter = Errorf("Filter must have exactly one of Prefix, Tag, or And specified") + +// Filter - a filter for a lifecycle configuration Rule. +type Filter struct { + XMLName xml.Name `xml:"Filter"` + set bool + + Prefix Prefix + + ObjectSizeGreaterThan int64 `xml:"ObjectSizeGreaterThan,omitempty"` + ObjectSizeLessThan int64 `xml:"ObjectSizeLessThan,omitempty"` + + And And + andSet bool + + Tag Tag + tagSet bool + + // Caching tags, only once + cachedTags map[string]string +} + +// MarshalXML - produces the xml representation of the Filter struct +// only one of Prefix, And and Tag should be present in the output. +func (f Filter) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if !f.set { + return nil + } + + if err := e.EncodeToken(start); err != nil { + return err + } + + switch { + case !f.And.isEmpty(): + if err := e.EncodeElement(f.And, xml.StartElement{Name: xml.Name{Local: "And"}}); err != nil { + return err + } + case !f.Tag.IsEmpty(): + if err := e.EncodeElement(f.Tag, xml.StartElement{Name: xml.Name{Local: "Tag"}}); err != nil { + return err + } + default: + // Always print Prefix field when both And & Tag are empty + if err := e.EncodeElement(f.Prefix, xml.StartElement{Name: xml.Name{Local: "Prefix"}}); err != nil { + return err + } + + if f.ObjectSizeLessThan > 0 { + if err := e.EncodeElement(f.ObjectSizeLessThan, xml.StartElement{Name: xml.Name{Local: "ObjectSizeLessThan"}}); err != nil { + return err + } + } + if f.ObjectSizeGreaterThan > 0 { + if err := e.EncodeElement(f.ObjectSizeGreaterThan, xml.StartElement{Name: xml.Name{Local: "ObjectSizeGreaterThan"}}); err != nil { + return err + } + } + } + + return e.EncodeToken(xml.EndElement{Name: start.Name}) +} + +// UnmarshalXML - decodes XML data. +func (f *Filter) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err error) { + f.set = true + for { + // Read tokens from the XML document in a stream. + t, err := d.Token() + if err != nil { + if err == io.EOF { + break + } + return err + } + + if se, ok := t.(xml.StartElement); ok { + switch se.Name.Local { + case "Prefix": + var p Prefix + if err = d.DecodeElement(&p, &se); err != nil { + return err + } + f.Prefix = p + case "And": + var and And + if err = d.DecodeElement(&and, &se); err != nil { + return err + } + f.And = and + f.andSet = true + case "Tag": + var tag Tag + if err = d.DecodeElement(&tag, &se); err != nil { + return err + } + f.Tag = tag + f.tagSet = true + case "ObjectSizeLessThan": + var sz int64 + if err = d.DecodeElement(&sz, &se); err != nil { + return err + } + f.ObjectSizeLessThan = sz + case "ObjectSizeGreaterThan": + var sz int64 + if err = d.DecodeElement(&sz, &se); err != nil { + return err + } + f.ObjectSizeGreaterThan = sz + default: + return errUnknownXMLTag + } + } + } + return nil +} + +// IsEmpty returns true if Filter is not specified in the XML +func (f Filter) IsEmpty() bool { + return !f.set +} + +// Validate - validates the filter element +func (f Filter) Validate() error { + if f.IsEmpty() { + return errXMLNotWellFormed + } + // A Filter must have exactly one of Prefix, Tag, + // ObjectSize{LessThan,GreaterThan} or And specified. + type predType uint8 + const ( + nonePred predType = iota + prefixPred + andPred + tagPred + sizeLtPred + sizeGtPred + ) + var predCount int + var pType predType + if !f.And.isEmpty() { + pType = andPred + predCount++ + } + if f.Prefix.set { + pType = prefixPred + predCount++ + } + if !f.Tag.IsEmpty() { + pType = tagPred + predCount++ + } + if f.ObjectSizeGreaterThan != 0 { + pType = sizeGtPred + predCount++ + } + if f.ObjectSizeLessThan != 0 { + pType = sizeLtPred + predCount++ + } + // Note: S3 supports empty , so predCount == 0 is + // valid. + if predCount > 1 { + return errInvalidFilter + } + + var err error + switch pType { + case nonePred: + // S3 supports empty + case prefixPred: + case andPred: + err = f.And.Validate() + case tagPred: + err = f.Tag.Validate() + case sizeLtPred: + if f.ObjectSizeLessThan < 0 { + err = errXMLNotWellFormed + } + case sizeGtPred: + if f.ObjectSizeGreaterThan < 0 { + err = errXMLNotWellFormed + } + } + return err +} + +// TestTags tests if the object tags satisfy the Filter tags requirement, +// it returns true if there is no tags in the underlying Filter. +func (f Filter) TestTags(userTags string) bool { + if f.cachedTags == nil { + cache := make(map[string]string) + for _, t := range append(f.And.Tags, f.Tag) { + if !t.IsEmpty() { + cache[t.Key] = t.Value + } + } + f.cachedTags = cache + } + + // This filter does not have any tags, always return true + if len(f.cachedTags) == 0 { + return true + } + + parsedTags, err := tags.ParseObjectTags(userTags) + if err != nil { + return false + } + tagsMap := parsedTags.ToMap() + + // Not enough tags on object to satisfy the rule filter's tags + if len(tagsMap) < len(f.cachedTags) { + return false + } + + var mismatch bool + for k, cv := range f.cachedTags { + v, ok := tagsMap[k] + if !ok || v != cv { + mismatch = true + break + } + } + return !mismatch +} + +// BySize returns true if sz satisfies one of ObjectSizeGreaterThan, +// ObjectSizeLessThan predicates or a combination of them via And. +func (f Filter) BySize(sz int64) bool { + if f.ObjectSizeGreaterThan > 0 && + sz <= f.ObjectSizeGreaterThan { + return false + } + if f.ObjectSizeLessThan > 0 && + sz >= f.ObjectSizeLessThan { + return false + } + if !f.And.isEmpty() { + return f.And.BySize(sz) + } + return true +} diff --git a/internal/bucket/lifecycle/filter_test.go b/internal/bucket/lifecycle/filter_test.go new file mode 100644 index 00000000..31de24a1 --- /dev/null +++ b/internal/bucket/lifecycle/filter_test.go @@ -0,0 +1,343 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lifecycle + +import ( + "encoding/xml" + "fmt" + "testing" + + "github.com/dustin/go-humanize" +) + +// TestUnsupportedFilters checks if parsing Filter xml with +// unsupported elements returns appropriate errors +func TestUnsupportedFilters(t *testing.T) { + testCases := []struct { + inputXML string + expectedErr error + }{ + { // Filter with And tags + inputXML: ` + + key-prefix + + `, + expectedErr: errXMLNotWellFormed, + }, + { // Filter with Tag tags + inputXML: ` + + key1 + value1 + + `, + expectedErr: nil, + }, + { // Filter with Prefix tag + inputXML: ` + key-prefix + `, + expectedErr: nil, + }, + { // Filter without And and multiple Tag tags + inputXML: ` + key-prefix + + key1 + value1 + + + key2 + value2 + + `, + expectedErr: errInvalidFilter, + }, + { // Filter with And, Prefix & multiple Tag tags + inputXML: ` + + key-prefix + + key1 + value1 + + + key2 + value2 + + + `, + expectedErr: nil, + }, + { // Filter with And and multiple Tag tags + inputXML: ` + + + + key1 + value1 + + + key2 + value2 + + + `, + expectedErr: nil, + }, + { // Filter without And and single Tag tag + inputXML: ` + key-prefix + + key1 + value1 + + `, + expectedErr: errInvalidFilter, + }, + } + for i, tc := range testCases { + t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) { + var filter Filter + err := xml.Unmarshal([]byte(tc.inputXML), &filter) + if err != nil { + t.Fatalf("%d: Expected no error but got %v", i+1, err) + } + err = filter.Validate() + if err != tc.expectedErr { + t.Fatalf("%d: Expected %v but got %v", i+1, tc.expectedErr, err) + } + }) + } +} + +func TestObjectSizeFilters(t *testing.T) { + f1 := Filter{ + set: true, + Prefix: Prefix{ + string: "doc/", + set: true, + Unused: struct{}{}, + }, + ObjectSizeGreaterThan: 100 * humanize.MiByte, + ObjectSizeLessThan: 100 * humanize.GiByte, + } + b, err := xml.Marshal(f1) + if err != nil { + t.Fatalf("Failed to marshal %v", f1) + } + var f2 Filter + err = xml.Unmarshal(b, &f2) + if err != nil { + t.Fatalf("Failed to unmarshal %s", string(b)) + } + if f1.ObjectSizeLessThan != f2.ObjectSizeLessThan { + t.Fatalf("Expected %v but got %v", f1.ObjectSizeLessThan, f2.And.ObjectSizeLessThan) + } + if f1.ObjectSizeGreaterThan != f2.ObjectSizeGreaterThan { + t.Fatalf("Expected %v but got %v", f1.ObjectSizeGreaterThan, f2.And.ObjectSizeGreaterThan) + } + + f1 = Filter{ + set: true, + And: And{ + ObjectSizeGreaterThan: 100 * humanize.MiByte, + ObjectSizeLessThan: 1 * humanize.GiByte, + Prefix: Prefix{}, + }, + andSet: true, + } + b, err = xml.Marshal(f1) + if err != nil { + t.Fatalf("Failed to marshal %v", f1) + } + f2 = Filter{} + err = xml.Unmarshal(b, &f2) + if err != nil { + t.Fatalf("Failed to unmarshal %s", string(b)) + } + if f1.And.ObjectSizeLessThan != f2.And.ObjectSizeLessThan { + t.Fatalf("Expected %v but got %v", f1.And.ObjectSizeLessThan, f2.And.ObjectSizeLessThan) + } + if f1.And.ObjectSizeGreaterThan != f2.And.ObjectSizeGreaterThan { + t.Fatalf("Expected %v but got %v", f1.And.ObjectSizeGreaterThan, f2.And.ObjectSizeGreaterThan) + } + + fiGt := Filter{ + ObjectSizeGreaterThan: 1 * humanize.MiByte, + } + fiLt := Filter{ + ObjectSizeLessThan: 100 * humanize.MiByte, + } + fiLtAndGt := Filter{ + And: And{ + ObjectSizeGreaterThan: 1 * humanize.MiByte, + ObjectSizeLessThan: 100 * humanize.MiByte, + }, + } + + tests := []struct { + filter Filter + objSize int64 + want bool + }{ + { + filter: fiLt, + objSize: 101 * humanize.MiByte, + want: false, + }, + { + filter: fiLt, + objSize: 99 * humanize.MiByte, + want: true, + }, + { + filter: fiGt, + objSize: 1*humanize.MiByte - 1, + want: false, + }, + { + filter: fiGt, + objSize: 1*humanize.MiByte + 1, + want: true, + }, + { + filter: fiLtAndGt, + objSize: 1*humanize.MiByte - 1, + want: false, + }, + { + filter: fiLtAndGt, + objSize: 2 * humanize.MiByte, + want: true, + }, + { + filter: fiLtAndGt, + objSize: 100*humanize.MiByte + 1, + want: false, + }, + } + for i, test := range tests { + t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) { + if got := test.filter.BySize(test.objSize); got != test.want { + t.Fatalf("Expected %v but got %v", test.want, got) + } + }) + } +} + +func TestTestTags(t *testing.T) { + noTags := Filter{ + set: true, + And: And{ + Tags: []Tag{}, + }, + andSet: true, + } + + oneTag := Filter{ + set: true, + And: And{ + Tags: []Tag{{Key: "FOO", Value: "1"}}, + }, + andSet: true, + } + + twoTags := Filter{ + set: true, + And: And{ + Tags: []Tag{{Key: "FOO", Value: "1"}, {Key: "BAR", Value: "2"}}, + }, + andSet: true, + } + + tests := []struct { + filter Filter + userTags string + want bool + }{ + { + filter: noTags, + userTags: "", + want: true, + }, + { + filter: noTags, + userTags: "A=3", + want: true, + }, + { + filter: oneTag, + userTags: "A=3", + want: false, + }, + { + filter: oneTag, + userTags: "FOO=1", + want: true, + }, + { + filter: oneTag, + userTags: "A=B&FOO=1", + want: true, + }, + { + filter: twoTags, + userTags: "", + want: false, + }, + { + filter: twoTags, + userTags: "FOO=1", + want: false, + }, + { + filter: twoTags, + userTags: "BAR=2", + want: false, + }, + { + filter: twoTags, + userTags: "FOO=2&BAR=2", + want: false, + }, + { + filter: twoTags, + userTags: "F=1&B=2", + want: false, + }, + { + filter: twoTags, + userTags: "FOO=1&BAR=2", + want: true, + }, + { + filter: twoTags, + userTags: "BAR=2&FOO=1", + want: true, + }, + } + for i, test := range tests { + t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) { + if got := test.filter.TestTags(test.userTags); got != test.want { + t.Errorf("Expected %v but got %v", test.want, got) + } + }) + } +} diff --git a/internal/bucket/lifecycle/lifecycle.go b/internal/bucket/lifecycle/lifecycle.go new file mode 100644 index 00000000..97c4200f --- /dev/null +++ b/internal/bucket/lifecycle/lifecycle.go @@ -0,0 +1,570 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lifecycle + +import ( + "encoding/xml" + "fmt" + "io" + "net/http" + "slices" + "strings" + "time" + + "github.com/google/uuid" + "github.com/minio/minio/internal/bucket/object/lock" + "github.com/minio/minio/internal/bucket/replication" + xhttp "github.com/minio/minio/internal/http" +) + +var ( + errLifecycleTooManyRules = Errorf("Lifecycle configuration allows a maximum of 1000 rules") + errLifecycleNoRule = Errorf("Lifecycle configuration should have at least one rule") + errLifecycleDuplicateID = Errorf("Rule ID must be unique. Found same ID for more than one rule") + errXMLNotWellFormed = Errorf("The XML you provided was not well-formed or did not validate against our published schema") + errLifecycleBucketLocked = Errorf("ExpiredObjectAllVersions element and DelMarkerExpiration action cannot be used on an object locked bucket") +) + +const ( + // TransitionComplete marks completed transition + TransitionComplete = "complete" + // TransitionPending - transition is yet to be attempted + TransitionPending = "pending" +) + +// Action represents a delete action or other transition +// actions that will be implemented later. +type Action int + +//go:generate stringer -type Action $GOFILE + +const ( + // NoneAction means no action required after evaluating lifecycle rules + NoneAction Action = iota + // DeleteAction means the object needs to be removed after evaluating lifecycle rules + DeleteAction + // DeleteVersionAction deletes a particular version + DeleteVersionAction + // TransitionAction transitions a particular object after evaluating lifecycle transition rules + TransitionAction + // TransitionVersionAction transitions a particular object version after evaluating lifecycle transition rules + TransitionVersionAction + // DeleteRestoredAction means the temporarily restored object needs to be removed after evaluating lifecycle rules + DeleteRestoredAction + // DeleteRestoredVersionAction deletes a particular version that was temporarily restored + DeleteRestoredVersionAction + // DeleteAllVersionsAction deletes all versions when an object expires + DeleteAllVersionsAction + // DelMarkerDeleteAllVersionsAction deletes all versions when an object with delete marker as latest version expires + DelMarkerDeleteAllVersionsAction + // ActionCount must be the last action and shouldn't be used as a regular action. + ActionCount +) + +// DeleteRestored - Returns true if action demands delete on restored objects +func (a Action) DeleteRestored() bool { + return a == DeleteRestoredAction || a == DeleteRestoredVersionAction +} + +// DeleteVersioned - Returns true if action demands delete on a versioned object +func (a Action) DeleteVersioned() bool { + return a == DeleteVersionAction || a == DeleteRestoredVersionAction +} + +// DeleteAll - Returns true if the action demands deleting all versions of an object +func (a Action) DeleteAll() bool { + return a == DeleteAllVersionsAction || a == DelMarkerDeleteAllVersionsAction +} + +// Delete - Returns true if action demands delete on all objects (including restored) +func (a Action) Delete() bool { + if a.DeleteRestored() { + return true + } + return a == DeleteVersionAction || a == DeleteAction || a == DeleteAllVersionsAction || a == DelMarkerDeleteAllVersionsAction +} + +// Lifecycle - Configuration for bucket lifecycle. +type Lifecycle struct { + XMLName xml.Name `xml:"LifecycleConfiguration"` + Rules []Rule `xml:"Rule"` + ExpiryUpdatedAt *time.Time `xml:"ExpiryUpdatedAt,omitempty"` +} + +// HasTransition returns 'true' if lifecycle document has Transition enabled. +func (lc Lifecycle) HasTransition() bool { + for _, rule := range lc.Rules { + if rule.Transition.IsEnabled() { + return true + } + } + return false +} + +// HasExpiry returns 'true' if lifecycle document has Expiry enabled. +func (lc Lifecycle) HasExpiry() bool { + for _, rule := range lc.Rules { + if !rule.Expiration.IsNull() || !rule.NoncurrentVersionExpiration.IsNull() { + return true + } + } + return false +} + +// UnmarshalXML - decodes XML data. +func (lc *Lifecycle) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err error) { + switch start.Name.Local { + case "LifecycleConfiguration", "BucketLifecycleConfiguration": + default: + return xml.UnmarshalError(fmt.Sprintf("expected element type / but have <%s>", + start.Name.Local)) + } + for { + // Read tokens from the XML document in a stream. + t, err := d.Token() + if err != nil { + if err == io.EOF { + break + } + return err + } + + if se, ok := t.(xml.StartElement); ok { + switch se.Name.Local { + case "Rule": + var r Rule + if err = d.DecodeElement(&r, &se); err != nil { + return err + } + lc.Rules = append(lc.Rules, r) + case "ExpiryUpdatedAt": + var t time.Time + if err = d.DecodeElement(&t, &start); err != nil { + return err + } + lc.ExpiryUpdatedAt = &t + default: + return xml.UnmarshalError(fmt.Sprintf("expected element type but have <%s>", se.Name.Local)) + } + } + } + return nil +} + +// HasActiveRules - returns whether lc has active rules at any level below or at prefix. +func (lc Lifecycle) HasActiveRules(prefix string) bool { + if len(lc.Rules) == 0 { + return false + } + for _, rule := range lc.Rules { + if rule.Status == Disabled { + continue + } + + if len(prefix) > 0 && len(rule.GetPrefix()) > 0 { + // we can skip this rule if it doesn't match the tested + // prefix. + if !strings.HasPrefix(prefix, rule.GetPrefix()) && !strings.HasPrefix(rule.GetPrefix(), prefix) { + continue + } + } + + if rule.NoncurrentVersionExpiration.NoncurrentDays > 0 { + return true + } + if rule.NoncurrentVersionExpiration.NewerNoncurrentVersions > 0 { + return true + } + if !rule.NoncurrentVersionTransition.IsNull() { + return true + } + if !rule.Expiration.IsDateNull() && rule.Expiration.Date.Before(time.Now().UTC()) { + return true + } + if !rule.Expiration.IsDaysNull() { + return true + } + if rule.Expiration.DeleteMarker.val { + return true + } + if !rule.Transition.IsDateNull() && rule.Transition.Date.Before(time.Now().UTC()) { + return true + } + if !rule.Transition.IsNull() { // this allows for Transition.Days to be zero. + return true + } + } + return false +} + +// ParseLifecycleConfigWithID - parses for a Lifecycle config and assigns +// unique id to rules with empty ID. +func ParseLifecycleConfigWithID(r io.Reader) (*Lifecycle, error) { + var lc Lifecycle + if err := xml.NewDecoder(r).Decode(&lc); err != nil { + return nil, err + } + // assign a unique id for rules with empty ID + for i := range lc.Rules { + if lc.Rules[i].ID == "" { + lc.Rules[i].ID = uuid.New().String() + } + } + return &lc, nil +} + +// ParseLifecycleConfig - parses data in given reader to Lifecycle. +func ParseLifecycleConfig(reader io.Reader) (*Lifecycle, error) { + var lc Lifecycle + if err := xml.NewDecoder(reader).Decode(&lc); err != nil { + return nil, err + } + return &lc, nil +} + +// Validate - validates the lifecycle configuration +func (lc Lifecycle) Validate(lr lock.Retention) error { + // Lifecycle config can't have more than 1000 rules + if len(lc.Rules) > 1000 { + return errLifecycleTooManyRules + } + // Lifecycle config should have at least one rule + if len(lc.Rules) == 0 { + return errLifecycleNoRule + } + + // Validate all the rules in the lifecycle config + for _, r := range lc.Rules { + if err := r.Validate(); err != nil { + return err + } + if lr.LockEnabled && (r.Expiration.DeleteAll.val || !r.DelMarkerExpiration.Empty()) { + return errLifecycleBucketLocked + } + } + // Make sure Rule ID is unique + for i := range lc.Rules { + if i == len(lc.Rules)-1 { + break + } + otherRules := lc.Rules[i+1:] + for _, otherRule := range otherRules { + if lc.Rules[i].ID == otherRule.ID { + return errLifecycleDuplicateID + } + } + } + return nil +} + +// FilterRules returns the rules filtered by the status, prefix and tags +func (lc Lifecycle) FilterRules(obj ObjectOpts) []Rule { + if obj.Name == "" { + return nil + } + var rules []Rule + for _, rule := range lc.Rules { + if rule.Status == Disabled { + continue + } + if !strings.HasPrefix(obj.Name, rule.GetPrefix()) { + continue + } + if !rule.Filter.TestTags(obj.UserTags) { + continue + } + if !obj.DeleteMarker && !rule.Filter.BySize(obj.Size) { + continue + } + rules = append(rules, rule) + } + return rules +} + +// ObjectOpts provides information to deduce the lifecycle actions +// which can be triggered on the resultant object. +type ObjectOpts struct { + Name string + UserTags string + ModTime time.Time + Size int64 + VersionID string + IsLatest bool + DeleteMarker bool + NumVersions int + SuccessorModTime time.Time + TransitionStatus string + RestoreOngoing bool + RestoreExpires time.Time + // to determine if object is locked due to retention + UserDefined map[string]string + VersionPurgeStatus replication.VersionPurgeStatusType + ReplicationStatus replication.StatusType +} + +// ExpiredObjectDeleteMarker returns true if an object version referred to by o +// is the only version remaining and is a delete marker. It returns false +// otherwise. +func (o ObjectOpts) ExpiredObjectDeleteMarker() bool { + return o.DeleteMarker && o.NumVersions == 1 +} + +// Event contains a lifecycle action with associated info +type Event struct { + Action Action + RuleID string + Due time.Time + NoncurrentDays int + NewerNoncurrentVersions int + StorageClass string +} + +// Eval returns the lifecycle event applicable now. +func (lc Lifecycle) Eval(obj ObjectOpts) Event { + return lc.eval(obj, time.Now().UTC(), 0) +} + +// eval returns the lifecycle event applicable at the given now. If now is the +// zero value of time.Time, it returns the upcoming lifecycle event. +func (lc Lifecycle) eval(obj ObjectOpts, now time.Time, remainingVersions int) Event { + var events []Event + if obj.ModTime.IsZero() { + return Event{} + } + + // Handle expiry of restored object; NB Restored Objects have expiry set on + // them as part of RestoreObject API. They aren't governed by lifecycle + // rules. + if !obj.RestoreExpires.IsZero() && now.After(obj.RestoreExpires) { + action := DeleteRestoredAction + if !obj.IsLatest { + action = DeleteRestoredVersionAction + } + + events = append(events, Event{ + Action: action, + Due: now, + }) + } + + for _, rule := range lc.FilterRules(obj) { + if obj.ExpiredObjectDeleteMarker() { + if rule.Expiration.DeleteMarker.val { + // Indicates whether MinIO will remove a delete marker with no noncurrent versions. + // Only latest marker is removed. If set to true, the delete marker will be expired; + // if set to false the policy takes no action. This cannot be specified with Days or + // Date in a Lifecycle Expiration Policy. + events = append(events, Event{ + Action: DeleteVersionAction, + RuleID: rule.ID, + Due: now, + }) + // No other conflicting actions apply to an expired object delete marker + break + } + + if !rule.Expiration.IsDaysNull() { + // Specifying the Days tag will automatically perform ExpiredObjectDeleteMarker cleanup + // once delete markers are old enough to satisfy the age criteria. + // https://docs.aws.amazon.com/AmazonS3/latest/userguide/lifecycle-configuration-examples.html + if expectedExpiry := ExpectedExpiryTime(obj.ModTime, int(rule.Expiration.Days)); now.IsZero() || now.After(expectedExpiry) { + events = append(events, Event{ + Action: DeleteVersionAction, + RuleID: rule.ID, + Due: expectedExpiry, + }) + // No other conflicting actions apply to an expired object delete marker + break + } + } + } + + // DelMarkerExpiration + if obj.IsLatest && obj.DeleteMarker && !rule.DelMarkerExpiration.Empty() { + if due, ok := rule.DelMarkerExpiration.NextDue(obj); ok && (now.IsZero() || now.After(due)) { + events = append(events, Event{ + Action: DelMarkerDeleteAllVersionsAction, + RuleID: rule.ID, + Due: due, + }) + } + // No other conflicting actions in this rule can apply to an object with current version as DEL marker + // Note: There could be other rules with earlier expiration which need to be considered. + // See TestDelMarkerExpiration + continue + } + + // NoncurrentVersionExpiration + if !obj.IsLatest && rule.NoncurrentVersionExpiration.set { + var ( + retainedEnough bool + oldEnough bool + ) + if rule.NoncurrentVersionExpiration.NewerNoncurrentVersions == 0 || remainingVersions >= rule.NoncurrentVersionExpiration.NewerNoncurrentVersions { + retainedEnough = true + } + expectedExpiry := ExpectedExpiryTime(obj.SuccessorModTime, int(rule.NoncurrentVersionExpiration.NoncurrentDays)) + if now.IsZero() || now.After(expectedExpiry) { + oldEnough = true + } + // > For the deletion to occur, both the and the values must be exceeded. + // ref: https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#intro-lifecycle-rules-actions + if retainedEnough && oldEnough { + events = append(events, Event{ + Action: DeleteVersionAction, + RuleID: rule.ID, + Due: expectedExpiry, + }) + } + } + + if !obj.IsLatest && !rule.NoncurrentVersionTransition.IsNull() { + if !obj.DeleteMarker && obj.TransitionStatus != TransitionComplete { + // Non current versions should be transitioned if their age exceeds non current days configuration + // https://docs.aws.amazon.com/AmazonS3/latest/dev/intro-lifecycle-rules.html#intro-lifecycle-rules-actions + if due, ok := rule.NoncurrentVersionTransition.NextDue(obj); ok && (now.IsZero() || now.After(due)) { + events = append(events, Event{ + Action: TransitionVersionAction, + RuleID: rule.ID, + Due: due, + StorageClass: rule.NoncurrentVersionTransition.StorageClass, + }) + } + } + } + + // Remove the object or simply add a delete marker (once) in a versioned bucket + if obj.IsLatest && !obj.DeleteMarker { + switch { + case !rule.Expiration.IsDateNull(): + if now.IsZero() || now.After(rule.Expiration.Date.Time) { + events = append(events, Event{ + Action: DeleteAction, + RuleID: rule.ID, + Due: rule.Expiration.Date.Time, + }) + } + case !rule.Expiration.IsDaysNull(): + if expectedExpiry := ExpectedExpiryTime(obj.ModTime, int(rule.Expiration.Days)); now.IsZero() || now.After(expectedExpiry) { + event := Event{ + Action: DeleteAction, + RuleID: rule.ID, + Due: expectedExpiry, + } + if rule.Expiration.DeleteAll.val { + // Expires all versions of this object once the latest object is old enough. + // This is a MinIO only extension. + event.Action = DeleteAllVersionsAction + } + events = append(events, event) + } + } + + if obj.TransitionStatus != TransitionComplete { + if due, ok := rule.Transition.NextDue(obj); ok && (now.IsZero() || now.After(due)) { + events = append(events, Event{ + Action: TransitionAction, + RuleID: rule.ID, + Due: due, + StorageClass: rule.Transition.StorageClass, + }) + } + } + } + } + + if len(events) > 0 { + slices.SortFunc(events, func(a, b Event) int { + // Prefer Expiration over Transition for both current + // and noncurrent versions when, + // - now is past the expected time to action + // - expected time to action is the same for both actions + if now.After(a.Due) && now.After(b.Due) || a.Due.Equal(b.Due) { + switch a.Action { + case DeleteAllVersionsAction, DelMarkerDeleteAllVersionsAction, + DeleteAction, DeleteVersionAction: + return -1 + } + switch b.Action { + case DeleteAllVersionsAction, DelMarkerDeleteAllVersionsAction, + DeleteAction, DeleteVersionAction: + return 1 + } + return -1 + } + + // Prefer earlier occurring event + if a.Due.Before(b.Due) { + return -1 + } + return 1 + }) + return events[0] + } + + return Event{ + Action: NoneAction, + } +} + +// ExpectedExpiryTime calculates the expiry, transition or restore date/time based on a object modtime. +// The expected transition or restore time is always a midnight time following the object +// modification time plus the number of transition/restore days. +// +// e.g. If the object modtime is `Thu May 21 13:42:50 GMT 2020` and the object should +// transition in 1 day, then the expected transition time is `Fri, 23 May 2020 00:00:00 GMT` +func ExpectedExpiryTime(modTime time.Time, days int) time.Time { + if days == 0 { + return modTime + } + t := modTime.UTC().Add(time.Duration(days+1) * 24 * time.Hour) + return t.Truncate(24 * time.Hour) +} + +// SetPredictionHeaders sets time to expiry and transition headers on w for a +// given obj. +func (lc Lifecycle) SetPredictionHeaders(w http.ResponseWriter, obj ObjectOpts) { + event := lc.eval(obj, time.Time{}, 0) + switch event.Action { + case DeleteAction, DeleteVersionAction, DeleteAllVersionsAction, DelMarkerDeleteAllVersionsAction: + w.Header()[xhttp.AmzExpiration] = []string{ + fmt.Sprintf(`expiry-date="%s", rule-id="%s"`, event.Due.Format(http.TimeFormat), event.RuleID), + } + case TransitionAction, TransitionVersionAction: + w.Header()[xhttp.MinIOTransition] = []string{ + fmt.Sprintf(`transition-date="%s", rule-id="%s"`, event.Due.Format(http.TimeFormat), event.RuleID), + } + } +} + +// NoncurrentVersionsExpirationLimit returns the number of noncurrent versions +// to be retained from the first applicable rule per S3 behavior. +func (lc Lifecycle) NoncurrentVersionsExpirationLimit(obj ObjectOpts) Event { + for _, rule := range lc.FilterRules(obj) { + if rule.NoncurrentVersionExpiration.NewerNoncurrentVersions == 0 { + continue + } + return Event{ + Action: DeleteVersionAction, + RuleID: rule.ID, + NoncurrentDays: int(rule.NoncurrentVersionExpiration.NoncurrentDays), + NewerNoncurrentVersions: rule.NoncurrentVersionExpiration.NewerNoncurrentVersions, + } + } + return Event{} +} diff --git a/internal/bucket/lifecycle/lifecycle_test.go b/internal/bucket/lifecycle/lifecycle_test.go new file mode 100644 index 00000000..2af5556e --- /dev/null +++ b/internal/bucket/lifecycle/lifecycle_test.go @@ -0,0 +1,1520 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lifecycle + +import ( + "bytes" + "encoding/xml" + "fmt" + "net/http" + "net/http/httptest" + "strconv" + "strings" + "testing" + "time" + + "github.com/dustin/go-humanize" + "github.com/google/uuid" + "github.com/minio/minio/internal/bucket/object/lock" + xhttp "github.com/minio/minio/internal/http" +) + +func TestParseAndValidateLifecycleConfig(t *testing.T) { + testCases := []struct { + inputConfig string + expectedParsingErr error + expectedValidationErr error + lr lock.Retention + }{ + { // Valid lifecycle config + inputConfig: ` + + testRule1 + + prefix + + Enabled + 3 + + + testRule2 + + another-prefix + + Enabled + 3 + + `, + expectedParsingErr: nil, + expectedValidationErr: nil, + }, + { // Using ExpiredObjectAllVersions element with an object locked bucket + inputConfig: ` + + ExpiredObjectAllVersions with object locking + + prefix + + Enabled + + 3 + true + + + `, + expectedParsingErr: nil, + expectedValidationErr: errLifecycleBucketLocked, + lr: lock.Retention{ + LockEnabled: true, + }, + }, + { // Using DelMarkerExpiration action with an object locked bucket + inputConfig: ` + + DeleteMarkerExpiration with object locking + + prefix + + Enabled + + 3 + + + `, + expectedParsingErr: nil, + expectedValidationErr: errLifecycleBucketLocked, + lr: lock.Retention{ + LockEnabled: true, + }, + }, + { // lifecycle config with no rules + inputConfig: ` + `, + expectedParsingErr: nil, + expectedValidationErr: errLifecycleNoRule, + }, + { // Valid lifecycle config + inputConfig: ` + + + key1val1key2val2 + + 3 + + `, + expectedParsingErr: errDuplicatedXMLTag, + expectedValidationErr: nil, + }, + { // lifecycle config without prefixes + inputConfig: ` + + 3 + Enabled + + `, + expectedParsingErr: nil, + expectedValidationErr: nil, + }, + { // lifecycle config with rules having overlapping prefix + inputConfig: `rule1Enabled/a/b3rule2Enabled/a/b/ckey1val13 `, + expectedParsingErr: nil, + expectedValidationErr: nil, + }, + { // lifecycle config with rules having duplicate ID + inputConfig: `duplicateIDEnabled/a/b3duplicateIDEnabled/x/zkey1val14`, + expectedParsingErr: nil, + expectedValidationErr: errLifecycleDuplicateID, + }, + // Missing in + { + inputConfig: `sample-rule-2/a/b/cEnabled1`, + expectedParsingErr: nil, + expectedValidationErr: errXMLNotWellFormed, + }, + // Lifecycle with the deprecated Prefix tag + { + inputConfig: `ruleEnabled1`, + expectedParsingErr: nil, + expectedValidationErr: nil, + }, + // Lifecycle with empty Filter tag + { + inputConfig: `ruleEnabled1`, + expectedParsingErr: nil, + expectedValidationErr: nil, + }, + // Lifecycle with zero Transition Days + { + inputConfig: `ruleEnabled0S3TIER-1`, + expectedParsingErr: nil, + expectedValidationErr: nil, + }, + // Lifecycle with max noncurrent versions + { + inputConfig: `ruleEnabled5`, + expectedParsingErr: nil, + expectedValidationErr: nil, + }, + // Lifecycle with delmarker expiration + { + inputConfig: `ruleEnabled5`, + expectedParsingErr: nil, + expectedValidationErr: nil, + }, + // Lifecycle with empty delmarker expiration + { + inputConfig: `ruleEnabled`, + expectedParsingErr: errInvalidDaysDelMarkerExpiration, + expectedValidationErr: nil, + }, + } + + for i, tc := range testCases { + t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) { + lc, err := ParseLifecycleConfig(bytes.NewReader([]byte(tc.inputConfig))) + if err != tc.expectedParsingErr { + t.Fatalf("%d: Expected %v during parsing but got %v", i+1, tc.expectedParsingErr, err) + } + if tc.expectedParsingErr != nil { + // We already expect a parsing error, + // no need to continue this test. + return + } + err = lc.Validate(tc.lr) + if err != tc.expectedValidationErr { + t.Fatalf("%d: Expected %v during validation but got %v", i+1, tc.expectedValidationErr, err) + } + }) + } +} + +// TestMarshalLifecycleConfig checks if lifecycleconfig xml +// marshaling/unmarshaling can handle output from each other +func TestMarshalLifecycleConfig(t *testing.T) { + // Time at midnight UTC + midnightTS := ExpirationDate{time.Date(2019, time.April, 20, 0, 0, 0, 0, time.UTC)} + lc := Lifecycle{ + Rules: []Rule{ + { + Status: "Enabled", + Filter: Filter{Prefix: Prefix{string: "prefix-1", set: true}}, + Expiration: Expiration{Days: ExpirationDays(3)}, + }, + { + Status: "Enabled", + Filter: Filter{Prefix: Prefix{string: "prefix-1", set: true}}, + Expiration: Expiration{Date: midnightTS}, + }, + { + Status: "Enabled", + Filter: Filter{Prefix: Prefix{string: "prefix-1", set: true}}, + Expiration: Expiration{Date: midnightTS}, + NoncurrentVersionTransition: NoncurrentVersionTransition{NoncurrentDays: TransitionDays(2), StorageClass: "TEST"}, + }, + }, + } + b, err := xml.MarshalIndent(&lc, "", "\t") + if err != nil { + t.Fatal(err) + } + var lc1 Lifecycle + err = xml.Unmarshal(b, &lc1) + if err != nil { + t.Fatal(err) + } + + ruleSet := make(map[string]struct{}) + for _, rule := range lc.Rules { + ruleBytes, err := xml.Marshal(rule) + if err != nil { + t.Fatal(err) + } + ruleSet[string(ruleBytes)] = struct{}{} + } + for _, rule := range lc1.Rules { + ruleBytes, err := xml.Marshal(rule) + if err != nil { + t.Fatal(err) + } + if _, ok := ruleSet[string(ruleBytes)]; !ok { + t.Fatalf("Expected %v to be equal to %v, %v missing", lc, lc1, rule) + } + } +} + +func TestExpectedExpiryTime(t *testing.T) { + testCases := []struct { + modTime time.Time + days ExpirationDays + expected time.Time + }{ + { + time.Date(2020, time.March, 15, 10, 10, 10, 0, time.UTC), + 4, + time.Date(2020, time.March, 20, 0, 0, 0, 0, time.UTC), + }, + { + time.Date(2020, time.March, 15, 0, 0, 0, 0, time.UTC), + 1, + time.Date(2020, time.March, 17, 0, 0, 0, 0, time.UTC), + }, + } + + for i, tc := range testCases { + t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) { + got := ExpectedExpiryTime(tc.modTime, int(tc.days)) + if !got.Equal(tc.expected) { + t.Fatalf("Expected %v to be equal to %v", got, tc.expected) + } + }) + } +} + +func TestEval(t *testing.T) { + testCases := []struct { + inputConfig string + objectName string + objectTags string + objectModTime time.Time + isDelMarker bool + hasManyVersions bool + expectedAction Action + isNoncurrent bool + objectSuccessorModTime time.Time + versionID string + }{ + // Empty object name (unexpected case) should always return NoneAction + { + inputConfig: `prefixEnabled5`, + expectedAction: NoneAction, + }, + // Disabled should always return NoneAction + { + inputConfig: `foodir/Disabled5`, + objectName: "foodir/fooobject", + objectModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago + expectedAction: NoneAction, + }, + // No modTime, should be none-action + { + inputConfig: `foodir/Enabled5`, + objectName: "foodir/fooobject", + expectedAction: NoneAction, + }, + // Prefix not matched + { + inputConfig: `foodir/Enabled5`, + objectName: "foxdir/fooobject", + objectModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago + expectedAction: NoneAction, + }, + // Test rule with empty prefix e.g. for whole bucket + { + inputConfig: `Enabled5`, + objectName: "foxdir/fooobject/foo.txt", + objectModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago + expectedAction: DeleteAction, + }, + // Too early to remove (test Days) + { + inputConfig: `foodir/Enabled5`, + objectName: "foxdir/fooobject", + objectModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago + expectedAction: NoneAction, + }, + // Should remove (test Days) + { + inputConfig: `foodir/Enabled5`, + objectName: "foodir/fooobject", + objectModTime: time.Now().UTC().Add(-6 * 24 * time.Hour), // Created 6 days ago + expectedAction: DeleteAction, + }, + // Too early to remove (test Date) + { + inputConfig: `foodir/Enabled` + time.Now().UTC().Truncate(24*time.Hour).Add(24*time.Hour).Format(time.RFC3339) + ``, + objectName: "foodir/fooobject", + objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago + expectedAction: NoneAction, + }, + // Should remove (test Days) + { + inputConfig: `foodir/Enabled` + time.Now().UTC().Truncate(24*time.Hour).Add(-24*time.Hour).Format(time.RFC3339) + ``, + objectName: "foodir/fooobject", + objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago + expectedAction: DeleteAction, + }, + // Should remove (Tags match) + { + inputConfig: `foodir/tag1value1Enabled` + time.Now().UTC().Truncate(24*time.Hour).Add(-24*time.Hour).Format(time.RFC3339) + ``, + objectName: "foodir/fooobject", + objectTags: "tag1=value1&tag2=value2", + objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago + expectedAction: DeleteAction, + }, + // Should remove (Multiple Rules, Tags match) + { + inputConfig: `foodir/tag1value1tag2value2Enabled` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + `abc/tag2valueEnabled` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + ``, + objectName: "foodir/fooobject", + objectTags: "tag1=value1&tag2=value2", + objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago + expectedAction: DeleteAction, + }, + // Should remove (Tags match) + { + inputConfig: `foodir/tag1value1tag2value2Enabled` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + ``, + objectName: "foodir/fooobject", + objectTags: "tag1=value1&tag2=value2", + objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago + expectedAction: DeleteAction, + }, + // Should remove (Tags match with inverted order) + { + inputConfig: `factorytruestoreforeverfalseEnabled` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + ``, + objectName: "fooobject", + objectTags: "storeforever=false&factory=true", + objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago + expectedAction: DeleteAction, + }, + // Should remove (Tags with encoded chars) + { + inputConfig: `factorytruestore foreverfalseEnabled` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + ``, + objectName: "fooobject", + objectTags: "store+forever=false&factory=true", + objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago + expectedAction: DeleteAction, + }, + + // Should not remove (Tags don't match) + { + inputConfig: `foodir/tagvalue1Enabled` + time.Now().UTC().Truncate(24*time.Hour).Add(-24*time.Hour).Format(time.RFC3339) + ``, + objectName: "foodir/fooobject", + objectTags: "tag1=value1", + objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago + expectedAction: NoneAction, + }, + // Should not remove (Tags match, but prefix doesn't match) + { + inputConfig: `foodir/tag1value1Enabled` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + ``, + objectName: "foxdir/fooobject", + objectTags: "tag1=value1", + objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago + expectedAction: NoneAction, + }, + // Should remove - empty prefix, tags match, date expiration kicked in + { + inputConfig: `tag1value1Enabled` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + ``, + objectName: "foxdir/fooobject", + objectTags: "tag1=value1", + objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago + expectedAction: DeleteAction, + }, + // Should remove - empty prefix, tags match, object is expired based on specified Days + { + inputConfig: `tag1value1Enabled1`, + objectName: "foxdir/fooobject", + objectTags: "tag1=value1", + objectModTime: time.Now().UTC().Add(-48 * time.Hour), // Created 2 day ago + expectedAction: DeleteAction, + }, + // Should remove, the second rule has expiration kicked in + { + inputConfig: `Enabled` + time.Now().Truncate(24*time.Hour).UTC().Add(24*time.Hour).Format(time.RFC3339) + `foxdir/Enabled` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + ``, + objectName: "foxdir/fooobject", + objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago + expectedAction: DeleteAction, + }, + // Should accept BucketLifecycleConfiguration root tag + { + inputConfig: `foodir/Enabled` + time.Now().Truncate(24*time.Hour).UTC().Add(-24*time.Hour).Format(time.RFC3339) + ``, + objectName: "foodir/fooobject", + objectModTime: time.Now().UTC().Add(-24 * time.Hour), // Created 1 day ago + expectedAction: DeleteAction, + }, + // Should delete expired delete marker right away + { + inputConfig: `trueEnabled`, + objectName: "foodir/fooobject", + objectModTime: time.Now().UTC().Add(-1 * time.Hour), // Created one hour ago + isDelMarker: true, + expectedAction: DeleteVersionAction, + }, + // Should not expire a delete marker; ExpiredObjectDeleteAllVersions applies only when current version is not a DEL marker. + { + inputConfig: `1trueEnabled`, + objectName: "foodir/fooobject", + objectModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago + isDelMarker: true, + hasManyVersions: true, + expectedAction: NoneAction, + }, + // Should delete all versions of this object since the latest version has past the expiry days criteria + { + inputConfig: `1trueEnabled`, + objectName: "foodir/fooobject", + objectModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago + hasManyVersions: true, + expectedAction: DeleteAllVersionsAction, + }, + // TransitionAction applies since object doesn't meet the age criteria for DeleteAllVersions + { + inputConfig: `30true10WARM-1Enabled`, + objectName: "foodir/fooobject", + objectModTime: time.Now().UTC().Add(-11 * 24 * time.Hour), // Created 11 days ago + hasManyVersions: true, + expectedAction: TransitionAction, + }, + // Should not delete expired marker if its time has not come yet + { + inputConfig: `Enabled1`, + objectName: "foodir/fooobject", + objectModTime: time.Now().UTC().Add(-12 * time.Hour), // Created 12 hours ago + isDelMarker: true, + expectedAction: NoneAction, + }, + // Should delete expired marker since its time has come + { + inputConfig: `Enabled1`, + objectName: "foodir/fooobject", + objectModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago + isDelMarker: true, + expectedAction: DeleteVersionAction, + }, + // Should transition immediately when Transition days is zero + { + inputConfig: `Enabled0S3TIER-1`, + objectName: "foodir/fooobject", + objectModTime: time.Now().Add(-1 * time.Nanosecond).UTC(), // Created now + expectedAction: TransitionAction, + }, + // Should transition immediately when NoncurrentVersion Transition days is zero + { + inputConfig: `Enabled0S3TIER-1`, + objectName: "foodir/fooobject", + objectModTime: time.Now().Add(-1 * time.Nanosecond).UTC(), // Created now + expectedAction: TransitionVersionAction, + isNoncurrent: true, + objectSuccessorModTime: time.Now().Add(-1 * time.Nanosecond).UTC(), + versionID: uuid.New().String(), + }, + // Lifecycle rules with NewerNoncurrentVersions specified must return NoneAction. + { + inputConfig: `foodir/Enabled5`, + objectName: "foodir/fooobject", + versionID: uuid.NewString(), + objectModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago + expectedAction: NoneAction, + }, + // Disabled rules with NewerNoncurrentVersions shouldn't affect outcome. + { + inputConfig: `foodir/Enabled5foodir/Disabled5`, + objectName: "foodir/fooobject", + versionID: uuid.NewString(), + objectModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago + objectSuccessorModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // Created 10 days ago + isNoncurrent: true, + expectedAction: DeleteVersionAction, + }, + { + inputConfig: ` + + Rule 1 + + + Enabled + + 365 + + + + Rule 2 + + logs/ + + Enabled + + STANDARD_IA + 30 + + + `, + objectName: "logs/obj-1", + objectModTime: time.Now().UTC().Add(-31 * 24 * time.Hour), + expectedAction: TransitionAction, + }, + { + inputConfig: ` + + Rule 1 + + logs/ + + Enabled + + 365 + + + + Rule 2 + + logs/ + + Enabled + + STANDARD_IA + 365 + + + `, + objectName: "logs/obj-1", + objectModTime: time.Now().UTC().Add(-366 * 24 * time.Hour), + expectedAction: DeleteAction, + }, + { + inputConfig: ` + + Rule 1 + + + tag1 + value1 + + + Enabled + + GLACIER + 365 + + + + Rule 2 + + + tag2 + value2 + + + Enabled + + 14 + + + `, + objectName: "obj-1", + objectTags: "tag1=value1&tag2=value2", + objectModTime: time.Now().UTC().Add(-15 * 24 * time.Hour), + expectedAction: DeleteAction, + }, + { + inputConfig: ` + + Rule 1 + Enabled + + + WARM-1 + 30 + + + 60 + + + `, + objectName: "obj-1", + objectModTime: time.Now().UTC().Add(-90 * 24 * time.Hour), + expectedAction: DeleteAction, + }, + { + inputConfig: ` + + Rule 2 + + Enabled + + 60 + + + WARM-1 + 30 + + + `, + objectName: "obj-1", + isNoncurrent: true, + objectModTime: time.Now().UTC().Add(-90 * 24 * time.Hour), + objectSuccessorModTime: time.Now().UTC().Add(-90 * 24 * time.Hour), + expectedAction: DeleteVersionAction, + }, + { + // DelMarkerExpiration is preferred since object age is past both transition and expiration days. + inputConfig: ` + + DelMarkerExpiration with Transition + + Enabled + + 60 + + + WARM-1 + 30 + + + `, + objectName: "obj-1", + objectModTime: time.Now().UTC().Add(-90 * 24 * time.Hour), + isDelMarker: true, + expectedAction: DelMarkerDeleteAllVersionsAction, + }, + { + // NoneAction since object doesn't qualify for DelMarkerExpiration yet. + // Note: TransitionAction doesn't apply to DEL marker + inputConfig: ` + + DelMarkerExpiration with Transition + + Enabled + + 60 + + + WARM-1 + 30 + + + `, + objectName: "obj-1", + objectModTime: time.Now().UTC().Add(-50 * 24 * time.Hour), + isDelMarker: true, + expectedAction: NoneAction, + }, + { + inputConfig: ` + + DelMarkerExpiration with non DEL-marker object + + Enabled + + 60 + + + `, + objectName: "obj-1", + objectModTime: time.Now().UTC().Add(-90 * 24 * time.Hour), + expectedAction: NoneAction, + }, + { + inputConfig: ` + + DelMarkerExpiration with noncurrent DEL-marker + + Enabled + + 60 + + + `, + objectName: "obj-1", + objectModTime: time.Now().UTC().Add(-90 * 24 * time.Hour), + objectSuccessorModTime: time.Now().UTC().Add(-60 * 24 * time.Hour), + isDelMarker: true, + isNoncurrent: true, + expectedAction: NoneAction, + }, + } + + for _, tc := range testCases { + t.Run("", func(t *testing.T) { + lc, err := ParseLifecycleConfig(bytes.NewReader([]byte(tc.inputConfig))) + if err != nil { + t.Fatalf("Got unexpected error: %v", err) + } + opts := ObjectOpts{ + Name: tc.objectName, + UserTags: tc.objectTags, + ModTime: tc.objectModTime, + DeleteMarker: tc.isDelMarker, + IsLatest: !tc.isNoncurrent, + SuccessorModTime: tc.objectSuccessorModTime, + VersionID: tc.versionID, + } + opts.NumVersions = 1 + if tc.hasManyVersions { + opts.NumVersions = 2 // at least one noncurrent version + } + if res := lc.Eval(opts); res.Action != tc.expectedAction { + t.Fatalf("Expected action: `%v`, got: `%v`", tc.expectedAction, res.Action) + } + }) + } +} + +func TestHasActiveRules(t *testing.T) { + testCases := []struct { + inputConfig string + prefix string + want bool + }{ + { + inputConfig: `foodir/Enabled5`, + prefix: "foodir/foobject", + want: true, + }, + { // empty prefix + inputConfig: `Enabled5`, + prefix: "foodir/foobject/foo.txt", + want: true, + }, + { + inputConfig: `foodir/Enabled5`, + prefix: "zdir/foobject", + want: false, + }, + { + inputConfig: `foodir/zdir/Enabled5`, + prefix: "foodir/", + want: true, + }, + { + inputConfig: `Disabled5`, + prefix: "foodir/", + want: false, + }, + { + inputConfig: `foodir/Enabled2999-01-01T00:00:00.000Z`, + prefix: "foodir/foobject", + want: false, + }, + { + inputConfig: `EnabledS3TIER-1`, + prefix: "foodir/foobject/foo.txt", + want: true, + }, + { + inputConfig: `EnabledS3TIER-1`, + prefix: "foodir/foobject/foo.txt", + want: true, + }, + { + inputConfig: `Enabledtrue`, + prefix: "", + want: true, + }, + { + inputConfig: `Enabled42true`, + prefix: "", + want: true, + }, + } + + for i, tc := range testCases { + t.Run(fmt.Sprintf("Test_%d", i+1), func(t *testing.T) { + lc, err := ParseLifecycleConfig(bytes.NewReader([]byte(tc.inputConfig))) + if err != nil { + t.Fatalf("Got unexpected error: %v", err) + } + // To ensure input lifecycle configurations are valid + if err := lc.Validate(lock.Retention{}); err != nil { + t.Fatalf("Invalid test case: %d %v", i+1, err) + } + if got := lc.HasActiveRules(tc.prefix); got != tc.want { + t.Fatalf("Expected result: `%v`, got: `%v`", tc.want, got) + } + }) + } +} + +func TestSetPredictionHeaders(t *testing.T) { + lc := Lifecycle{ + Rules: []Rule{ + { + ID: "rule-1", + Status: "Enabled", + Expiration: Expiration{ + Days: ExpirationDays(3), + set: true, + }, + }, + { + ID: "rule-2", + Status: "Enabled", + Transition: Transition{ + Days: TransitionDays(3), + StorageClass: "TIER-1", + set: true, + }, + }, + { + ID: "rule-3", + Status: "Enabled", + NoncurrentVersionTransition: NoncurrentVersionTransition{ + NoncurrentDays: TransitionDays(5), + StorageClass: "TIER-2", + set: true, + }, + }, + }, + } + + // current version + obj1 := ObjectOpts{ + Name: "obj1", + IsLatest: true, + } + // non-current version + obj2 := ObjectOpts{ + Name: "obj2", + } + + tests := []struct { + obj ObjectOpts + expRuleID int + transRuleID int + }{ + { + obj: obj1, + expRuleID: 0, + transRuleID: 1, + }, + { + obj: obj2, + expRuleID: 0, + transRuleID: 2, + }, + } + for i, tc := range tests { + w := httptest.NewRecorder() + lc.SetPredictionHeaders(w, tc.obj) + if expHdrs, ok := w.Header()[xhttp.AmzExpiration]; ok && !strings.Contains(expHdrs[0], lc.Rules[tc.expRuleID].ID) { + t.Fatalf("Test %d: Expected %s header", i+1, xhttp.AmzExpiration) + } + if transHdrs, ok := w.Header()[xhttp.MinIOTransition]; ok { + if !strings.Contains(transHdrs[0], lc.Rules[tc.transRuleID].ID) { + t.Fatalf("Test %d: Expected %s header", i+1, xhttp.MinIOTransition) + } + + if tc.obj.IsLatest { + if expectedDue, _ := lc.Rules[tc.transRuleID].Transition.NextDue(tc.obj); !strings.Contains(transHdrs[0], expectedDue.Format(http.TimeFormat)) { + t.Fatalf("Test %d: Expected transition time %s", i+1, expectedDue) + } + } else { + if expectedDue, _ := lc.Rules[tc.transRuleID].NoncurrentVersionTransition.NextDue(tc.obj); !strings.Contains(transHdrs[0], expectedDue.Format(http.TimeFormat)) { + t.Fatalf("Test %d: Expected transition time %s", i+1, expectedDue) + } + } + } + } +} + +func TestTransitionTier(t *testing.T) { + lc := Lifecycle{ + Rules: []Rule{ + { + ID: "rule-1", + Status: "Enabled", + Transition: Transition{ + Days: TransitionDays(3), + StorageClass: "TIER-1", + }, + }, + { + ID: "rule-2", + Status: "Enabled", + NoncurrentVersionTransition: NoncurrentVersionTransition{ + NoncurrentDays: TransitionDays(3), + StorageClass: "TIER-2", + }, + }, + }, + } + + now := time.Now().UTC() + + obj1 := ObjectOpts{ + Name: "obj1", + IsLatest: true, + ModTime: now, + } + + obj2 := ObjectOpts{ + Name: "obj2", + ModTime: now, + } + + // Go back seven days in the past + now = now.Add(7 * 24 * time.Hour) + + evaluator := NewEvaluator(lc) + evts := evaluator.eval([]ObjectOpts{obj1, obj2}, now) + evt := evts[0] + if evt.Action != TransitionAction { + t.Fatalf("Expected action: %s but got %s", TransitionAction, evt.Action) + } + if evt.StorageClass != "TIER-1" { + t.Fatalf("Expected TIER-1 but got %s", evt.StorageClass) + } + + evt = evts[1] + if evt.Action != TransitionVersionAction { + t.Fatalf("Expected action: %s but got %s", TransitionVersionAction, evt.Action) + } + if evt.StorageClass != "TIER-2" { + t.Fatalf("Expected TIER-2 but got %s", evt.StorageClass) + } +} + +func TestTransitionTierWithPrefixAndTags(t *testing.T) { + lc := Lifecycle{ + Rules: []Rule{ + { + ID: "rule-1", + Status: "Enabled", + Filter: Filter{ + Prefix: Prefix{ + set: true, + string: "abcd/", + }, + }, + Transition: Transition{ + Days: TransitionDays(3), + StorageClass: "TIER-1", + }, + }, + { + ID: "rule-2", + Status: "Enabled", + Filter: Filter{ + tagSet: true, + Tag: Tag{ + Key: "priority", + Value: "low", + }, + }, + Transition: Transition{ + Days: TransitionDays(3), + StorageClass: "TIER-2", + }, + }, + }, + } + + now := time.Now().UTC() + + obj1 := ObjectOpts{ + Name: "obj1", + IsLatest: true, + ModTime: now, + } + + obj2 := ObjectOpts{ + Name: "abcd/obj2", + IsLatest: true, + ModTime: now, + } + + obj3 := ObjectOpts{ + Name: "obj3", + IsLatest: true, + ModTime: now, + UserTags: "priority=low", + } + + // Go back seven days in the past + now = now.Add(7 * 24 * time.Hour) + + evaluator := NewEvaluator(lc) + evts := evaluator.eval([]ObjectOpts{obj1, obj2, obj3}, now) + // Eval object 1 + evt := evts[0] + if evt.Action != NoneAction { + t.Fatalf("Expected action: %s but got %s", NoneAction, evt.Action) + } + + // Eval object 2 + evt = evts[1] + if evt.Action != TransitionAction { + t.Fatalf("Expected action: %s but got %s", TransitionAction, evt.Action) + } + if evt.StorageClass != "TIER-1" { + t.Fatalf("Expected TIER-1 but got %s", evt.StorageClass) + } + + // Eval object 3 + evt = evts[2] + if evt.Action != TransitionAction { + t.Fatalf("Expected action: %s but got %s", TransitionAction, evt.Action) + } + if evt.StorageClass != "TIER-2" { + t.Fatalf("Expected TIER-2 but got %s", evt.StorageClass) + } +} + +func TestNoncurrentVersionsLimit(t *testing.T) { + // test that the lowest max noncurrent versions limit is returned among + // matching rules + var rules []Rule + for i := 1; i <= 10; i++ { + rules = append(rules, Rule{ + ID: strconv.Itoa(i), + Status: "Enabled", + NoncurrentVersionExpiration: NoncurrentVersionExpiration{ + NewerNoncurrentVersions: i, + NoncurrentDays: ExpirationDays(i), + }, + }) + } + lc := Lifecycle{ + Rules: rules, + } + if event := lc.NoncurrentVersionsExpirationLimit(ObjectOpts{Name: "obj"}); event.RuleID != "1" || event.NoncurrentDays != 1 || event.NewerNoncurrentVersions != 1 { + t.Fatalf("Expected (ruleID, days, lim) to be (\"1\", 1, 1) but got (%s, %d, %d)", event.RuleID, event.NoncurrentDays, event.NewerNoncurrentVersions) + } +} + +func TestMaxNoncurrentBackwardCompat(t *testing.T) { + testCases := []struct { + xml string + expected NoncurrentVersionExpiration + }{ + { + xml: `13`, + expected: NoncurrentVersionExpiration{ + XMLName: xml.Name{ + Local: "NoncurrentVersionExpiration", + }, + NoncurrentDays: 1, + NewerNoncurrentVersions: 3, + set: true, + }, + }, + { + xml: `24`, + expected: NoncurrentVersionExpiration{ + XMLName: xml.Name{ + Local: "NoncurrentVersionExpiration", + }, + NoncurrentDays: 2, + NewerNoncurrentVersions: 4, + set: true, + }, + }, + } + for i, tc := range testCases { + var got NoncurrentVersionExpiration + dec := xml.NewDecoder(strings.NewReader(tc.xml)) + if err := dec.Decode(&got); err != nil || got != tc.expected { + if err != nil { + t.Fatalf("%d: Failed to unmarshal xml %v", i+1, err) + } + t.Fatalf("%d: Expected %v but got %v", i+1, tc.expected, got) + } + } +} + +func TestParseLifecycleConfigWithID(t *testing.T) { + r := bytes.NewReader([]byte(` + + rule-1 + + prefix + + Enabled + 3 + + + + another-prefix + + Enabled + 3 + + `)) + lc, err := ParseLifecycleConfigWithID(r) + if err != nil { + t.Fatalf("Expected parsing to succeed but failed with %v", err) + } + for _, rule := range lc.Rules { + if rule.ID == "" { + t.Fatalf("Expected all rules to have a unique id assigned %#v", rule) + } + } +} + +func TestFilterAndSetPredictionHeaders(t *testing.T) { + lc := Lifecycle{ + Rules: []Rule{ + { + ID: "rule-1", + Status: "Enabled", + Filter: Filter{ + set: true, + Prefix: Prefix{ + string: "folder1/folder1/exp_dt=2022-", + set: true, + }, + }, + Expiration: Expiration{ + Days: 1, + set: true, + }, + }, + }, + } + tests := []struct { + opts ObjectOpts + lc Lifecycle + want int + }{ + { + opts: ObjectOpts{ + Name: "folder1/folder1/exp_dt=2022-08-01/obj-1", + ModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), + VersionID: "", + IsLatest: true, + NumVersions: 1, + }, + want: 1, + lc: lc, + }, + { + opts: ObjectOpts{ + Name: "folder1/folder1/exp_dt=9999-01-01/obj-1", + ModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), + VersionID: "", + IsLatest: true, + NumVersions: 1, + }, + want: 0, + lc: lc, + }, + } + for i, tc := range tests { + t.Run(fmt.Sprintf("test-%d", i+1), func(t *testing.T) { + if got := tc.lc.FilterRules(tc.opts); len(got) != tc.want { + t.Fatalf("Expected %d rules to match but got %d", tc.want, len(got)) + } + w := httptest.NewRecorder() + tc.lc.SetPredictionHeaders(w, tc.opts) + expHdr, ok := w.Header()[xhttp.AmzExpiration] + switch { + case ok && tc.want == 0: + t.Fatalf("Expected no rule to match but found x-amz-expiration header set: %v", expHdr) + case !ok && tc.want > 0: + t.Fatal("Expected x-amz-expiration header to be set but not found") + } + }) + } +} + +func TestFilterRules(t *testing.T) { + rules := []Rule{ + { + ID: "rule-1", + Status: "Enabled", + Filter: Filter{ + set: true, + Tag: Tag{ + Key: "key1", + Value: "val1", + }, + }, + Expiration: Expiration{ + set: true, + Days: 1, + }, + }, + { + ID: "rule-with-sz-lt", + Status: "Enabled", + Filter: Filter{ + set: true, + ObjectSizeLessThan: 100 * humanize.MiByte, + }, + Expiration: Expiration{ + set: true, + Days: 1, + }, + }, + { + ID: "rule-with-sz-gt", + Status: "Enabled", + Filter: Filter{ + set: true, + ObjectSizeGreaterThan: 1 * humanize.MiByte, + }, + Expiration: Expiration{ + set: true, + Days: 1, + }, + }, + { + ID: "rule-with-sz-lt-and-tag", + Status: "Enabled", + Filter: Filter{ + set: true, + And: And{ + ObjectSizeLessThan: 100 * humanize.MiByte, + Tags: []Tag{ + { + Key: "key1", + Value: "val1", + }, + }, + }, + }, + Expiration: Expiration{ + set: true, + Days: 1, + }, + }, + { + ID: "rule-with-sz-gt-and-tag", + Status: "Enabled", + Filter: Filter{ + set: true, + And: And{ + ObjectSizeGreaterThan: 1 * humanize.MiByte, + Tags: []Tag{ + { + Key: "key1", + Value: "val1", + }, + }, + }, + }, + Expiration: Expiration{ + set: true, + Days: 1, + }, + }, + { + ID: "rule-with-sz-lt-and-gt", + Status: "Enabled", + Filter: Filter{ + set: true, + And: And{ + ObjectSizeGreaterThan: 101 * humanize.MiByte, + ObjectSizeLessThan: 200 * humanize.MiByte, + }, + }, + Expiration: Expiration{ + set: true, + Days: 1, + }, + }, + } + tests := []struct { + lc Lifecycle + opts ObjectOpts + hasRules bool + }{ + { // Delete marker shouldn't match filter without tags + lc: Lifecycle{ + Rules: []Rule{ + rules[0], + }, + }, + opts: ObjectOpts{ + DeleteMarker: true, + IsLatest: true, + Name: "obj-1", + }, + hasRules: false, + }, + { // PUT version with no matching tags + lc: Lifecycle{ + Rules: []Rule{ + rules[0], + }, + }, + opts: ObjectOpts{ + IsLatest: true, + Name: "obj-1", + Size: 1 * humanize.MiByte, + }, + hasRules: false, + }, + { // PUT version with matching tags + lc: Lifecycle{ + Rules: []Rule{ + rules[0], + }, + }, + opts: ObjectOpts{ + IsLatest: true, + UserTags: "key1=val1", + Name: "obj-1", + Size: 2 * humanize.MiByte, + }, + hasRules: true, + }, + { // PUT version with size based filters + lc: Lifecycle{ + Rules: []Rule{ + rules[1], + rules[2], + rules[3], + rules[4], + rules[5], + }, + }, + opts: ObjectOpts{ + IsLatest: true, + UserTags: "key1=val1", + Name: "obj-1", + Size: 1*humanize.MiByte - 1, + }, + hasRules: true, + }, + { // PUT version with size based filters + lc: Lifecycle{ + Rules: []Rule{ + rules[1], + rules[2], + rules[3], + rules[4], + rules[5], + }, + }, + opts: ObjectOpts{ + IsLatest: true, + Name: "obj-1", + Size: 1*humanize.MiByte + 1, + }, + hasRules: true, + }, + { // DEL version with size based filters + lc: Lifecycle{ + Rules: []Rule{ + rules[1], + rules[2], + rules[3], + rules[4], + rules[5], + }, + }, + opts: ObjectOpts{ + DeleteMarker: true, + IsLatest: true, + Name: "obj-1", + }, + hasRules: true, + }, + } + + for i, tc := range tests { + t.Run(fmt.Sprintf("test-%d", i+1), func(t *testing.T) { + if err := tc.lc.Validate(lock.Retention{}); err != nil { + t.Fatalf("Lifecycle validation failed - %v", err) + } + rules := tc.lc.FilterRules(tc.opts) + if tc.hasRules && len(rules) == 0 { + t.Fatalf("%d: Expected at least one rule to match but none matched", i+1) + } + if !tc.hasRules && len(rules) > 0 { + t.Fatalf("%d: Expected no rules to match but got matches %v", i+1, rules) + } + }) + } +} + +// TestDeleteAllVersions tests ordering among events, especially ones which +// expire all versions like ExpiredObjectDeleteAllVersions and +// DelMarkerExpiration +func TestDeleteAllVersions(t *testing.T) { + // ExpiredObjectDeleteAllVersions + lc := Lifecycle{ + Rules: []Rule{ + { + ID: "ExpiredObjectDeleteAllVersions-20", + Status: "Enabled", + Expiration: Expiration{ + set: true, + DeleteAll: Boolean{val: true, set: true}, + Days: 20, + }, + }, + { + ID: "Transition-10", + Status: "Enabled", + Transition: Transition{ + set: true, + StorageClass: "WARM-1", + Days: 10, + }, + }, + }, + } + opts := ObjectOpts{ + Name: "foo.txt", + ModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // created 10 days ago + Size: 0, + VersionID: uuid.New().String(), + IsLatest: true, + NumVersions: 4, + } + + evaluator := NewEvaluator(lc) + events := evaluator.eval([]ObjectOpts{opts}, time.Time{}) + event := events[0] + if event.Action != TransitionAction { + t.Fatalf("Expected %v action but got %v", TransitionAction, event.Action) + } + // The earlier upcoming lifecycle event must be picked, i.e rule with id "Transition-10" + if exp := ExpectedExpiryTime(opts.ModTime, 10); exp != event.Due { + t.Fatalf("Expected due %v but got %v, ruleID=%v", exp, event.Due, event.RuleID) + } + + // DelMarkerExpiration + lc = Lifecycle{ + Rules: []Rule{ + { + ID: "delmarker-exp-20", + Status: "Enabled", + DelMarkerExpiration: DelMarkerExpiration{ + Days: 20, + }, + }, + { + ID: "delmarker-exp-10", + Status: "Enabled", + DelMarkerExpiration: DelMarkerExpiration{ + Days: 10, + }, + }, + }, + } + opts = ObjectOpts{ + Name: "foo.txt", + ModTime: time.Now().UTC().Add(-10 * 24 * time.Hour), // created 10 days ago + Size: 0, + VersionID: uuid.New().String(), + IsLatest: true, + DeleteMarker: true, + NumVersions: 4, + } + evaluator = NewEvaluator(lc) + events = evaluator.eval([]ObjectOpts{opts}, time.Time{}) + event = events[0] + if event.Action != DelMarkerDeleteAllVersionsAction { + t.Fatalf("Expected %v action but got %v", DelMarkerDeleteAllVersionsAction, event.Action) + } + // The earlier upcoming lifecycle event must be picked, i.e rule with id "delmarker-exp-10" + if exp := ExpectedExpiryTime(opts.ModTime, 10); exp != event.Due { + t.Fatalf("Expected due %v but got %v, ruleID=%v", exp, event.Due, event.RuleID) + } +} diff --git a/internal/bucket/lifecycle/noncurrentversion.go b/internal/bucket/lifecycle/noncurrentversion.go new file mode 100644 index 00000000..23d03c84 --- /dev/null +++ b/internal/bucket/lifecycle/noncurrentversion.go @@ -0,0 +1,156 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lifecycle + +import ( + "encoding/xml" + "time" +) + +// NoncurrentVersionExpiration - an action for lifecycle configuration rule. +type NoncurrentVersionExpiration struct { + XMLName xml.Name `xml:"NoncurrentVersionExpiration"` + NoncurrentDays ExpirationDays `xml:"NoncurrentDays,omitempty"` + NewerNoncurrentVersions int `xml:"NewerNoncurrentVersions,omitempty"` + set bool +} + +// MarshalXML if non-current days not set to non zero value +func (n NoncurrentVersionExpiration) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if n.IsNull() { + return nil + } + type noncurrentVersionExpirationWrapper NoncurrentVersionExpiration + return e.EncodeElement(noncurrentVersionExpirationWrapper(n), start) +} + +// UnmarshalXML decodes NoncurrentVersionExpiration +func (n *NoncurrentVersionExpiration) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error { + // To handle xml with MaxNoncurrentVersions from older MinIO releases. + // note: only one of MaxNoncurrentVersions or NewerNoncurrentVersions would be present. + type noncurrentExpiration struct { + XMLName xml.Name `xml:"NoncurrentVersionExpiration"` + NoncurrentDays ExpirationDays `xml:"NoncurrentDays,omitempty"` + NewerNoncurrentVersions int `xml:"NewerNoncurrentVersions,omitempty"` + MaxNoncurrentVersions int `xml:"MaxNoncurrentVersions,omitempty"` + } + + var val noncurrentExpiration + err := d.DecodeElement(&val, &startElement) + if err != nil { + return err + } + if val.MaxNoncurrentVersions > 0 { + val.NewerNoncurrentVersions = val.MaxNoncurrentVersions + } + *n = NoncurrentVersionExpiration{ + XMLName: val.XMLName, + NoncurrentDays: val.NoncurrentDays, + NewerNoncurrentVersions: val.NewerNoncurrentVersions, + } + n.set = true + return nil +} + +// IsNull returns if both NoncurrentDays and NoncurrentVersions are empty +func (n NoncurrentVersionExpiration) IsNull() bool { + return n.IsDaysNull() && n.NewerNoncurrentVersions == 0 +} + +// IsDaysNull returns true if days field is null +func (n NoncurrentVersionExpiration) IsDaysNull() bool { + return n.NoncurrentDays == ExpirationDays(0) +} + +// Validate returns an error with wrong value +func (n NoncurrentVersionExpiration) Validate() error { + if !n.set { + return nil + } + val := int(n.NoncurrentDays) + switch { + case val == 0 && n.NewerNoncurrentVersions == 0: + // both fields can't be zero + return errXMLNotWellFormed + + case val < 0, n.NewerNoncurrentVersions < 0: + // negative values are not supported + return errXMLNotWellFormed + } + return nil +} + +// NoncurrentVersionTransition - an action for lifecycle configuration rule. +type NoncurrentVersionTransition struct { + NoncurrentDays TransitionDays `xml:"NoncurrentDays"` + StorageClass string `xml:"StorageClass"` + set bool +} + +// MarshalXML is extended to leave out +// tags +func (n NoncurrentVersionTransition) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if n.IsNull() { + return nil + } + type noncurrentVersionTransitionWrapper NoncurrentVersionTransition + return e.EncodeElement(noncurrentVersionTransitionWrapper(n), start) +} + +// UnmarshalXML decodes NoncurrentVersionExpiration +func (n *NoncurrentVersionTransition) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error { + type noncurrentVersionTransitionWrapper NoncurrentVersionTransition + var val noncurrentVersionTransitionWrapper + err := d.DecodeElement(&val, &startElement) + if err != nil { + return err + } + *n = NoncurrentVersionTransition(val) + n.set = true + return nil +} + +// IsNull returns true if NoncurrentTransition doesn't refer to any storage-class. +// Note: It supports immediate transition, i.e zero noncurrent days. +func (n NoncurrentVersionTransition) IsNull() bool { + return n.StorageClass == "" +} + +// Validate returns an error with wrong value +func (n NoncurrentVersionTransition) Validate() error { + if !n.set { + return nil + } + if n.StorageClass == "" { + return errXMLNotWellFormed + } + return nil +} + +// NextDue returns upcoming NoncurrentVersionTransition date for obj if +// applicable, returns false otherwise. +func (n NoncurrentVersionTransition) NextDue(obj ObjectOpts) (time.Time, bool) { + if obj.IsLatest || n.StorageClass == "" { + return time.Time{}, false + } + // Days == 0 indicates immediate tiering, i.e object is eligible for tiering since it became noncurrent. + if n.NoncurrentDays == 0 { + return obj.SuccessorModTime, true + } + return ExpectedExpiryTime(obj.SuccessorModTime, int(n.NoncurrentDays)), true +} diff --git a/internal/bucket/lifecycle/noncurrentversion_test.go b/internal/bucket/lifecycle/noncurrentversion_test.go new file mode 100644 index 00000000..fc6d1b34 --- /dev/null +++ b/internal/bucket/lifecycle/noncurrentversion_test.go @@ -0,0 +1,82 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lifecycle + +import "testing" + +func Test_NoncurrentVersionsExpiration_Validation(t *testing.T) { + testcases := []struct { + n NoncurrentVersionExpiration + err error + }{ + { + n: NoncurrentVersionExpiration{ + NoncurrentDays: 0, + NewerNoncurrentVersions: 0, + set: true, + }, + err: errXMLNotWellFormed, + }, + { + n: NoncurrentVersionExpiration{ + NoncurrentDays: 90, + NewerNoncurrentVersions: 0, + set: true, + }, + err: nil, + }, + { + n: NoncurrentVersionExpiration{ + NoncurrentDays: 90, + NewerNoncurrentVersions: 2, + set: true, + }, + err: nil, + }, + { + n: NoncurrentVersionExpiration{ + NoncurrentDays: -1, + set: true, + }, + err: errXMLNotWellFormed, + }, + { + n: NoncurrentVersionExpiration{ + NoncurrentDays: 90, + NewerNoncurrentVersions: -2, + set: true, + }, + err: errXMLNotWellFormed, + }, + // MinIO extension: supports zero NoncurrentDays when NewerNoncurrentVersions > 0 + { + n: NoncurrentVersionExpiration{ + NoncurrentDays: 0, + NewerNoncurrentVersions: 5, + set: true, + }, + err: nil, + }, + } + + for i, tc := range testcases { + if got := tc.n.Validate(); got != tc.err { + t.Fatalf("%d: expected %v but got %v", i+1, tc.err, got) + } + } +} diff --git a/internal/bucket/lifecycle/prefix.go b/internal/bucket/lifecycle/prefix.go new file mode 100644 index 00000000..a2015669 --- /dev/null +++ b/internal/bucket/lifecycle/prefix.go @@ -0,0 +1,52 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lifecycle + +import ( + "encoding/xml" +) + +// Prefix holds the prefix xml tag in and +type Prefix struct { + string + set bool + Unused struct{} // Needed for GOB compatibility +} + +// UnmarshalXML - decodes XML data. +func (p *Prefix) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err error) { + var s string + if err = d.DecodeElement(&s, &start); err != nil { + return err + } + *p = Prefix{string: s, set: true} + return nil +} + +// MarshalXML - decodes XML data. +func (p Prefix) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { + if !p.set { + return nil + } + return e.EncodeElement(p.string, startElement) +} + +// String returns the prefix string +func (p Prefix) String() string { + return p.string +} diff --git a/internal/bucket/lifecycle/rule.go b/internal/bucket/lifecycle/rule.go new file mode 100644 index 00000000..2fb00606 --- /dev/null +++ b/internal/bucket/lifecycle/rule.go @@ -0,0 +1,194 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lifecycle + +import ( + "bytes" + "encoding/xml" +) + +// Status represents lifecycle configuration status +type Status string + +// Supported status types +const ( + Enabled Status = "Enabled" + Disabled Status = "Disabled" +) + +// Rule - a rule for lifecycle configuration. +type Rule struct { + XMLName xml.Name `xml:"Rule"` + ID string `xml:"ID,omitempty"` + Status Status `xml:"Status"` + Filter Filter `xml:"Filter,omitempty"` + Prefix Prefix `xml:"Prefix,omitempty"` + Expiration Expiration `xml:"Expiration,omitempty"` + Transition Transition `xml:"Transition,omitempty"` + DelMarkerExpiration DelMarkerExpiration `xml:"DelMarkerExpiration,omitempty"` + // FIXME: add a type to catch unsupported AbortIncompleteMultipartUpload AbortIncompleteMultipartUpload `xml:"AbortIncompleteMultipartUpload,omitempty"` + NoncurrentVersionExpiration NoncurrentVersionExpiration `xml:"NoncurrentVersionExpiration,omitempty"` + NoncurrentVersionTransition NoncurrentVersionTransition `xml:"NoncurrentVersionTransition,omitempty"` +} + +var ( + errInvalidRuleID = Errorf("ID length is limited to 255 characters") + errEmptyRuleStatus = Errorf("Status should not be empty") + errInvalidRuleStatus = Errorf("Status must be set to either Enabled or Disabled") + errInvalidRuleDelMarkerExpiration = Errorf("Rule with DelMarkerExpiration cannot have tags based filtering") +) + +// validateID - checks if ID is valid or not. +func (r Rule) validateID() error { + if len(r.ID) > 255 { + return errInvalidRuleID + } + return nil +} + +// validateStatus - checks if status is valid or not. +func (r Rule) validateStatus() error { + // Status can't be empty + if len(r.Status) == 0 { + return errEmptyRuleStatus + } + + // Status must be one of Enabled or Disabled + if r.Status != Enabled && r.Status != Disabled { + return errInvalidRuleStatus + } + return nil +} + +func (r Rule) validateExpiration() error { + return r.Expiration.Validate() +} + +func (r Rule) validateNoncurrentExpiration() error { + return r.NoncurrentVersionExpiration.Validate() +} + +func (r Rule) validatePrefixAndFilter() error { + // In the now deprecated PutBucketLifecycle API, Rule had a mandatory Prefix element and there existed no Filter field. + // See https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycle.html + // In the newer PutBucketLifecycleConfiguration API, Rule has a prefix field that is deprecated, and there exists an optional + // Filter field, and within it, an optional Prefix field. + // See https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutBucketLifecycleConfiguration.html + // A valid rule could be a pre-existing one created using the now deprecated PutBucketLifecycle. + // Or, a valid rule could also be either a pre-existing or a new rule that is created using PutBucketLifecycleConfiguration. + // Prefix validation below may check that either Rule.Prefix or Rule.Filter.Prefix exist but not both. + // Here, we assume the pre-existing rule created using PutBucketLifecycle API is already valid and won't fail the validation if Rule.Prefix is empty. + + if r.Prefix.set && !r.Filter.IsEmpty() && r.Filter.Prefix.set { + return errXMLNotWellFormed + } + + if r.Filter.set { + return r.Filter.Validate() + } + return nil +} + +func (r Rule) validateTransition() error { + return r.Transition.Validate() +} + +func (r Rule) validateNoncurrentTransition() error { + return r.NoncurrentVersionTransition.Validate() +} + +// GetPrefix - a rule can either have prefix under , +// or under . This method returns the prefix from the +// location where it is available. +func (r Rule) GetPrefix() string { + if p := r.Prefix.String(); p != "" { + return p + } + if p := r.Filter.Prefix.String(); p != "" { + return p + } + if p := r.Filter.And.Prefix.String(); p != "" { + return p + } + return "" +} + +// Tags - a rule can either have tag under or under +// . This method returns all the tags from the +// rule in the format tag1=value1&tag2=value2 +func (r Rule) Tags() string { + if !r.Filter.Tag.IsEmpty() { + return r.Filter.Tag.String() + } + if len(r.Filter.And.Tags) != 0 { + var buf bytes.Buffer + for _, t := range r.Filter.And.Tags { + if buf.Len() > 0 { + buf.WriteString("&") + } + buf.WriteString(t.String()) + } + return buf.String() + } + return "" +} + +// Validate - validates the rule element +func (r Rule) Validate() error { + if err := r.validateID(); err != nil { + return err + } + if err := r.validateStatus(); err != nil { + return err + } + if err := r.validateExpiration(); err != nil { + return err + } + if err := r.validateNoncurrentExpiration(); err != nil { + return err + } + if err := r.validatePrefixAndFilter(); err != nil { + return err + } + if err := r.validateTransition(); err != nil { + return err + } + if err := r.validateNoncurrentTransition(); err != nil { + return err + } + if (!r.Filter.Tag.IsEmpty() || len(r.Filter.And.Tags) != 0) && !r.DelMarkerExpiration.Empty() { + return errInvalidRuleDelMarkerExpiration + } + if !r.Expiration.set && !r.Transition.set && !r.NoncurrentVersionExpiration.set && !r.NoncurrentVersionTransition.set && r.DelMarkerExpiration.Empty() { + return errXMLNotWellFormed + } + return nil +} + +// CloneNonTransition - returns a clone of the object containing non transition rules +func (r Rule) CloneNonTransition() Rule { + return Rule{ + XMLName: r.XMLName, + ID: r.ID, + Status: r.Status, + Filter: r.Filter, + Prefix: r.Prefix, + Expiration: r.Expiration, + NoncurrentVersionExpiration: r.NoncurrentVersionExpiration, + } +} diff --git a/internal/bucket/lifecycle/rule_test.go b/internal/bucket/lifecycle/rule_test.go new file mode 100644 index 00000000..f6f13917 --- /dev/null +++ b/internal/bucket/lifecycle/rule_test.go @@ -0,0 +1,148 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lifecycle + +import ( + "encoding/xml" + "fmt" + "testing" +) + +// TestInvalidRules checks if Rule xml with invalid elements returns +// appropriate errors on validation +func TestInvalidRules(t *testing.T) { + invalidTestCases := []struct { + inputXML string + expectedErr error + }{ + { // Rule with ID longer than 255 characters + inputXML: ` + babababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababababab + `, + expectedErr: errInvalidRuleID, + }, + { // Rule with empty ID + inputXML: ` + + + + 365 + + Enabled + `, + expectedErr: nil, + }, + { // Rule with empty status + inputXML: ` + rule with empty status + + `, + expectedErr: errEmptyRuleStatus, + }, + { // Rule with invalid status + inputXML: ` + rule with invalid status + OK + `, + expectedErr: errInvalidRuleStatus, + }, + { // Rule with negative values for ObjectSizeLessThan + inputXML: ` + negative-obj-size-less-than + -1 + + 365 + + Enabled + `, + expectedErr: errXMLNotWellFormed, + }, + { // Rule with negative values for And>ObjectSizeLessThan + inputXML: ` + negative-and-obj-size-less-than + -1 + + 365 + + Enabled + `, + expectedErr: errXMLNotWellFormed, + }, + { // Rule with negative values for ObjectSizeGreaterThan + inputXML: ` + negative-obj-size-greater-than + -1 + + 365 + + Enabled + `, + expectedErr: errXMLNotWellFormed, + }, + { // Rule with negative values for And>ObjectSizeGreaterThan + inputXML: ` + negative-and-obj-size-greater-than + -1 + + 365 + + Enabled + `, + expectedErr: errXMLNotWellFormed, + }, + { + inputXML: ` + Rule with a tag and DelMarkerExpiration + k1v1 + + 365 + + Enabled + `, + expectedErr: errInvalidRuleDelMarkerExpiration, + }, + { + inputXML: ` + Rule with multiple tags and DelMarkerExpiration + + k1v1 + k2v2 + + + 365 + + Enabled + `, + expectedErr: errInvalidRuleDelMarkerExpiration, + }, + } + + for i, tc := range invalidTestCases { + t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) { + var rule Rule + err := xml.Unmarshal([]byte(tc.inputXML), &rule) + if err != nil { + t.Fatal(err) + } + + if err := rule.Validate(); err != tc.expectedErr { + t.Fatalf("%d: Expected %v but got %v", i+1, tc.expectedErr, err) + } + }) + } +} diff --git a/internal/bucket/lifecycle/tag.go b/internal/bucket/lifecycle/tag.go new file mode 100644 index 00000000..bdb3fbe0 --- /dev/null +++ b/internal/bucket/lifecycle/tag.go @@ -0,0 +1,101 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lifecycle + +import ( + "encoding/xml" + "io" + "unicode/utf8" +) + +// Tag - a tag for a lifecycle configuration Rule filter. +type Tag struct { + XMLName xml.Name `xml:"Tag"` + Key string `xml:"Key,omitempty"` + Value string `xml:"Value,omitempty"` +} + +var ( + errInvalidTagKey = Errorf("The TagKey you have provided is invalid") + errInvalidTagValue = Errorf("The TagValue you have provided is invalid") + + errDuplicatedXMLTag = Errorf("duplicated XML Tag") + errUnknownXMLTag = Errorf("unknown XML Tag") +) + +// UnmarshalXML - decodes XML data. +func (tag *Tag) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err error) { + var keyAlreadyParsed, valueAlreadyParsed bool + for { + // Read tokens from the XML document in a stream. + t, err := d.Token() + if err != nil { + if err == io.EOF { + break + } + return err + } + + if se, ok := t.(xml.StartElement); ok { + var s string + if err = d.DecodeElement(&s, &se); err != nil { + return err + } + switch se.Name.Local { + case "Key": + if keyAlreadyParsed { + return errDuplicatedXMLTag + } + tag.Key = s + keyAlreadyParsed = true + case "Value": + if valueAlreadyParsed { + return errDuplicatedXMLTag + } + tag.Value = s + valueAlreadyParsed = true + default: + return errUnknownXMLTag + } + } + } + + return nil +} + +func (tag Tag) String() string { + return tag.Key + "=" + tag.Value +} + +// IsEmpty returns whether this tag is empty or not. +func (tag Tag) IsEmpty() bool { + return tag.Key == "" +} + +// Validate checks this tag. +func (tag Tag) Validate() error { + if len(tag.Key) == 0 || utf8.RuneCountInString(tag.Key) > 128 { + return errInvalidTagKey + } + + if utf8.RuneCountInString(tag.Value) > 256 { + return errInvalidTagValue + } + + return nil +} diff --git a/internal/bucket/lifecycle/transition.go b/internal/bucket/lifecycle/transition.go new file mode 100644 index 00000000..397f4c01 --- /dev/null +++ b/internal/bucket/lifecycle/transition.go @@ -0,0 +1,178 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lifecycle + +import ( + "encoding/xml" + "time" +) + +var ( + errTransitionInvalidDays = Errorf("Days must be 0 or greater when used with Transition") + errTransitionInvalidDate = Errorf("Date must be provided in ISO 8601 format") + errTransitionInvalid = Errorf("Exactly one of Days (0 or greater) or Date (positive ISO 8601 format) should be present in Transition.") + errTransitionDateNotMidnight = Errorf("'Date' must be at midnight GMT") +) + +// TransitionDate is a embedded type containing time.Time to unmarshal +// Date in Transition +type TransitionDate struct { + time.Time +} + +// UnmarshalXML parses date from Transition and validates date format +func (tDate *TransitionDate) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error { + var dateStr string + err := d.DecodeElement(&dateStr, &startElement) + if err != nil { + return err + } + // While AWS documentation mentions that the date specified + // must be present in ISO 8601 format, in reality they allow + // users to provide RFC 3339 compliant dates. + trnDate, err := time.Parse(time.RFC3339, dateStr) + if err != nil { + return errTransitionInvalidDate + } + // Allow only date timestamp specifying midnight GMT + hr, m, sec := trnDate.Clock() + nsec := trnDate.Nanosecond() + loc := trnDate.Location() + if hr != 0 || m != 0 || sec != 0 || nsec != 0 || loc.String() != time.UTC.String() { + return errTransitionDateNotMidnight + } + + *tDate = TransitionDate{trnDate} + return nil +} + +// MarshalXML encodes expiration date if it is non-zero and encodes +// empty string otherwise +func (tDate TransitionDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { + if tDate.IsZero() { + return nil + } + return e.EncodeElement(tDate.Format(time.RFC3339), startElement) +} + +// TransitionDays is a type alias to unmarshal Days in Transition +type TransitionDays int + +// UnmarshalXML parses number of days from Transition and validates if +// >= 0 +func (tDays *TransitionDays) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error { + var days int + err := d.DecodeElement(&days, &startElement) + if err != nil { + return err + } + + if days < 0 { + return errTransitionInvalidDays + } + *tDays = TransitionDays(days) + + return nil +} + +// MarshalXML encodes number of days to expire if it is non-zero and +// encodes empty string otherwise +func (tDays TransitionDays) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { + return e.EncodeElement(int(tDays), startElement) +} + +// Transition - transition actions for a rule in lifecycle configuration. +type Transition struct { + XMLName xml.Name `xml:"Transition"` + Days TransitionDays `xml:"Days,omitempty"` + Date TransitionDate `xml:"Date,omitempty"` + StorageClass string `xml:"StorageClass,omitempty"` + + set bool +} + +// IsEnabled returns if transition is enabled. +func (t Transition) IsEnabled() bool { + return t.set +} + +// MarshalXML encodes transition field into an XML form. +func (t Transition) MarshalXML(enc *xml.Encoder, start xml.StartElement) error { + if !t.set { + return nil + } + type transitionWrapper Transition + return enc.EncodeElement(transitionWrapper(t), start) +} + +// UnmarshalXML decodes transition field from the XML form. +func (t *Transition) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error { + type transitionWrapper Transition + var trw transitionWrapper + err := d.DecodeElement(&trw, &startElement) + if err != nil { + return err + } + *t = Transition(trw) + t.set = true + return nil +} + +// Validate - validates the "Transition" element +func (t Transition) Validate() error { + if !t.set { + return nil + } + + if !t.IsDateNull() && t.Days > 0 { + return errTransitionInvalid + } + + if t.StorageClass == "" { + return errXMLNotWellFormed + } + return nil +} + +// IsDateNull returns true if date field is null +func (t Transition) IsDateNull() bool { + return t.Date.IsZero() +} + +// IsNull returns true if both date and days fields are null +func (t Transition) IsNull() bool { + return t.StorageClass == "" +} + +// NextDue returns upcoming transition date for obj and true if applicable, +// returns false otherwise. +func (t Transition) NextDue(obj ObjectOpts) (time.Time, bool) { + if !obj.IsLatest || t.IsNull() { + return time.Time{}, false + } + + if !t.IsDateNull() { + return t.Date.Time, true + } + + // Days == 0 indicates immediate tiering, i.e object is eligible for tiering since its creation. + if t.Days == 0 { + return obj.ModTime, true + } + return ExpectedExpiryTime(obj.ModTime, int(t.Days)), true +} diff --git a/internal/bucket/lifecycle/transition_test.go b/internal/bucket/lifecycle/transition_test.go new file mode 100644 index 00000000..5a8fba74 --- /dev/null +++ b/internal/bucket/lifecycle/transition_test.go @@ -0,0 +1,93 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lifecycle + +import ( + "encoding/xml" + "testing" +) + +func TestTransitionUnmarshalXML(t *testing.T) { + trTests := []struct { + input string + err error + }{ + { + input: ` + 0 + S3TIER-1 + `, + err: nil, + }, + { + input: ` + 1 + 2021-01-01T00:00:00Z + S3TIER-1 + `, + err: errTransitionInvalid, + }, + { + input: ` + 1 + `, + err: errXMLNotWellFormed, + }, + } + + for i, tc := range trTests { + var tr Transition + err := xml.Unmarshal([]byte(tc.input), &tr) + if err != nil { + t.Fatalf("%d: xml unmarshal failed with %v", i+1, err) + } + if err = tr.Validate(); err != tc.err { + t.Fatalf("%d: Invalid transition %v: err %v", i+1, tr, err) + } + } + + ntrTests := []struct { + input string + err error + }{ + { + input: ` + 0 + S3TIER-1 + `, + err: nil, + }, + { + input: ` + 1 + `, + err: errXMLNotWellFormed, + }, + } + + for i, tc := range ntrTests { + var ntr NoncurrentVersionTransition + err := xml.Unmarshal([]byte(tc.input), &ntr) + if err != nil { + t.Fatalf("%d: xml unmarshal failed with %v", i+1, err) + } + if err = ntr.Validate(); err != tc.err { + t.Fatalf("%d: Invalid noncurrent version transition %v: err %v", i+1, ntr, err) + } + } +} diff --git a/internal/bucket/object/lock/lock.go b/internal/bucket/object/lock/lock.go new file mode 100644 index 00000000..410011b9 --- /dev/null +++ b/internal/bucket/object/lock/lock.go @@ -0,0 +1,622 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lock + +import ( + "bytes" + "context" + "encoding/xml" + "errors" + "fmt" + "io" + "maps" + "net/http" + "net/textproto" + "strings" + "time" + + "github.com/beevik/ntp" + "github.com/minio/minio/internal/amztime" + xhttp "github.com/minio/minio/internal/http" + + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/env" +) + +const ( + logSubsys = "locking" +) + +func lockLogIf(ctx context.Context, err error) { + logger.LogIf(ctx, logSubsys, err) +} + +// Enabled indicates object locking is enabled +const Enabled = "Enabled" + +// RetMode - object retention mode. +type RetMode string + +const ( + // RetGovernance - governance mode. + RetGovernance RetMode = "GOVERNANCE" + + // RetCompliance - compliance mode. + RetCompliance RetMode = "COMPLIANCE" +) + +// Valid - returns if retention mode is valid +func (r RetMode) Valid() bool { + switch r { + case RetGovernance, RetCompliance: + return true + } + return false +} + +func parseRetMode(modeStr string) (mode RetMode) { + switch strings.ToUpper(modeStr) { + case "GOVERNANCE": + mode = RetGovernance + case "COMPLIANCE": + mode = RetCompliance + } + return mode +} + +// LegalHoldStatus - object legal hold status. +type LegalHoldStatus string + +const ( + // LegalHoldOn - legal hold is on. + LegalHoldOn LegalHoldStatus = "ON" + + // LegalHoldOff - legal hold is off. + LegalHoldOff LegalHoldStatus = "OFF" +) + +// Valid - returns true if legal hold status has valid values +func (l LegalHoldStatus) Valid() bool { + switch l { + case LegalHoldOn, LegalHoldOff: + return true + } + return false +} + +func parseLegalHoldStatus(holdStr string) (st LegalHoldStatus) { + switch strings.ToUpper(holdStr) { + case "ON": + st = LegalHoldOn + case "OFF": + st = LegalHoldOff + } + return st +} + +// Bypass retention governance header. +const ( + AmzObjectLockBypassRetGovernance = "X-Amz-Bypass-Governance-Retention" + AmzObjectLockRetainUntilDate = "X-Amz-Object-Lock-Retain-Until-Date" + AmzObjectLockMode = "X-Amz-Object-Lock-Mode" + AmzObjectLockLegalHold = "X-Amz-Object-Lock-Legal-Hold" +) + +var ( + // ErrMalformedBucketObjectConfig -indicates that the bucket object lock config is malformed + ErrMalformedBucketObjectConfig = errors.New("invalid bucket object lock config") + // ErrInvalidRetentionDate - indicates that retention date needs to be in ISO 8601 format + ErrInvalidRetentionDate = errors.New("date must be provided in ISO 8601 format") + // ErrPastObjectLockRetainDate - indicates that retention date must be in the future + ErrPastObjectLockRetainDate = errors.New("the retain until date must be in the future") + // ErrUnknownWORMModeDirective - indicates that the retention mode is invalid + ErrUnknownWORMModeDirective = errors.New("unknown WORM mode directive") + // ErrObjectLockMissingContentMD5 - indicates missing Content-MD5 header for put object requests with locking + ErrObjectLockMissingContentMD5 = errors.New("content-MD5 HTTP header is required for Put Object requests with Object Lock parameters") + // ErrObjectLockInvalidHeaders indicates that object lock headers are missing + ErrObjectLockInvalidHeaders = errors.New("x-amz-object-lock-retain-until-date and x-amz-object-lock-mode must both be supplied") + // ErrMalformedXML - generic error indicating malformed XML + ErrMalformedXML = errors.New("the XML you provided was not well-formed or did not validate against our published schema") +) + +const ( + ntpServerEnv = "MINIO_NTP_SERVER" +) + +var ntpServer = env.Get(ntpServerEnv, "") + +// UTCNowNTP - is similar in functionality to UTCNow() +// but only used when we do not wish to rely on system +// time. +func UTCNowNTP() (time.Time, error) { + // ntp server is disabled + if ntpServer == "" { + return time.Now().UTC(), nil + } + return ntp.Time(ntpServer) +} + +// Retention - bucket level retention configuration. +type Retention struct { + Mode RetMode + Validity time.Duration + LockEnabled bool +} + +// Retain - check whether given date is retainable by validity time. +func (r Retention) Retain(created time.Time) bool { + t, err := UTCNowNTP() + if err != nil { + lockLogIf(context.Background(), err) + // Retain + return true + } + return created.Add(r.Validity).After(t) +} + +// DefaultRetention - default retention configuration. +type DefaultRetention struct { + XMLName xml.Name `xml:"DefaultRetention"` + Mode RetMode `xml:"Mode"` + Days *uint64 `xml:"Days"` + Years *uint64 `xml:"Years"` +} + +// Maximum support retention days and years supported by AWS S3. +const ( + // This tested by using `mc lock` command + maximumRetentionDays = 36500 + maximumRetentionYears = 100 +) + +// UnmarshalXML - decodes XML data. +func (dr *DefaultRetention) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + // Make subtype to avoid recursive UnmarshalXML(). + type defaultRetention DefaultRetention + retention := defaultRetention{} + + if err := d.DecodeElement(&retention, &start); err != nil { + return err + } + + switch retention.Mode { + case RetGovernance, RetCompliance: + default: + return fmt.Errorf("unknown retention mode %v", retention.Mode) + } + + if retention.Days == nil && retention.Years == nil { + return fmt.Errorf("either Days or Years must be specified") + } + + if retention.Days != nil && retention.Years != nil { + return fmt.Errorf("either Days or Years must be specified, not both") + } + + //nolint:gocritic + if retention.Days != nil { + if *retention.Days == 0 { + return fmt.Errorf("Default retention period must be a positive integer value for 'Days'") + } + if *retention.Days > maximumRetentionDays { + return fmt.Errorf("Default retention period too large for 'Days' %d", *retention.Days) + } + } else if *retention.Years == 0 { + return fmt.Errorf("Default retention period must be a positive integer value for 'Years'") + } else if *retention.Years > maximumRetentionYears { + return fmt.Errorf("Default retention period too large for 'Years' %d", *retention.Years) + } + + *dr = DefaultRetention(retention) + + return nil +} + +// Config - object lock configuration specified in +// https://docs.aws.amazon.com/AmazonS3/latest/API/Type_API_ObjectLockConfiguration.html +type Config struct { + XMLNS string `xml:"xmlns,attr,omitempty"` + XMLName xml.Name `xml:"ObjectLockConfiguration"` + ObjectLockEnabled string `xml:"ObjectLockEnabled"` + Rule *struct { + DefaultRetention DefaultRetention `xml:"DefaultRetention"` + } `xml:"Rule,omitempty"` +} + +// String returns the human readable format of object lock configuration, used in audit logs. +func (config Config) String() string { + parts := []string{ + fmt.Sprintf("Enabled: %v", config.Enabled()), + } + if config.Rule != nil { + if config.Rule.DefaultRetention.Mode != "" { + parts = append(parts, fmt.Sprintf("Mode: %s", config.Rule.DefaultRetention.Mode)) + } + if config.Rule.DefaultRetention.Days != nil { + parts = append(parts, fmt.Sprintf("Days: %d", *config.Rule.DefaultRetention.Days)) + } + if config.Rule.DefaultRetention.Years != nil { + parts = append(parts, fmt.Sprintf("Years: %d", *config.Rule.DefaultRetention.Years)) + } + } + return strings.Join(parts, ", ") +} + +// Enabled returns true if config.ObjectLockEnabled is set to Enabled +func (config *Config) Enabled() bool { + return config.ObjectLockEnabled == Enabled +} + +// UnmarshalXML - decodes XML data. +func (config *Config) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + // Make subtype to avoid recursive UnmarshalXML(). + type objectLockConfig Config + parsedConfig := objectLockConfig{} + + if err := d.DecodeElement(&parsedConfig, &start); err != nil { + return err + } + + if parsedConfig.ObjectLockEnabled != Enabled { + return fmt.Errorf("only 'Enabled' value is allowed to ObjectLockEnabled element") + } + + *config = Config(parsedConfig) + return nil +} + +// ToRetention - convert to Retention type. +func (config *Config) ToRetention() Retention { + r := Retention{ + LockEnabled: config.ObjectLockEnabled == Enabled, + } + if config.Rule != nil { + r.Mode = config.Rule.DefaultRetention.Mode + + t, err := UTCNowNTP() + if err != nil { + lockLogIf(context.Background(), err) + // Do not change any configuration + // upon NTP failure. + return r + } + + if config.Rule.DefaultRetention.Days != nil { + r.Validity = t.AddDate(0, 0, int(*config.Rule.DefaultRetention.Days)).Sub(t) + } else { + r.Validity = t.AddDate(int(*config.Rule.DefaultRetention.Years), 0, 0).Sub(t) + } + } + + return r +} + +// Maximum 4KiB size per object lock config. +const maxObjectLockConfigSize = 1 << 12 + +// ParseObjectLockConfig parses ObjectLockConfig from xml +func ParseObjectLockConfig(reader io.Reader) (*Config, error) { + config := Config{} + if err := xml.NewDecoder(io.LimitReader(reader, maxObjectLockConfigSize)).Decode(&config); err != nil { + return nil, err + } + + return &config, nil +} + +// NewObjectLockConfig returns a initialized lock.Config struct +func NewObjectLockConfig() *Config { + return &Config{ + ObjectLockEnabled: Enabled, + } +} + +// RetentionDate is a embedded type containing time.Time to unmarshal +// Date in Retention +type RetentionDate struct { + time.Time +} + +// UnmarshalXML parses date from Retention and validates date format +func (rDate *RetentionDate) UnmarshalXML(d *xml.Decoder, startElement xml.StartElement) error { + var dateStr string + err := d.DecodeElement(&dateStr, &startElement) + if err != nil { + return err + } + // While AWS documentation mentions that the date specified + // must be present in ISO 8601 format, in reality they allow + // users to provide RFC 3339 compliant dates. + retDate, err := amztime.ISO8601Parse(dateStr) + if err != nil { + return ErrInvalidRetentionDate + } + + *rDate = RetentionDate{retDate} + return nil +} + +// MarshalXML encodes expiration date if it is non-zero and encodes +// empty string otherwise +func (rDate RetentionDate) MarshalXML(e *xml.Encoder, startElement xml.StartElement) error { + if rDate.IsZero() { + return nil + } + return e.EncodeElement(amztime.ISO8601Format(rDate.Time), startElement) +} + +// ObjectRetention specified in +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectRetention.html +type ObjectRetention struct { + XMLNS string `xml:"xmlns,attr,omitempty"` + XMLName xml.Name `xml:"Retention"` + Mode RetMode `xml:"Mode,omitempty"` + RetainUntilDate RetentionDate `xml:"RetainUntilDate,omitempty"` +} + +func (o ObjectRetention) String() string { + return fmt.Sprintf("Mode: %s, RetainUntilDate: %s", o.Mode, o.RetainUntilDate.Time) +} + +// Maximum 4KiB size per object retention config. +const maxObjectRetentionSize = 1 << 12 + +// ParseObjectRetention constructs ObjectRetention struct from xml input +func ParseObjectRetention(reader io.Reader) (*ObjectRetention, error) { + ret := ObjectRetention{} + if err := xml.NewDecoder(io.LimitReader(reader, maxObjectRetentionSize)).Decode(&ret); err != nil { + return nil, err + } + if ret.Mode != "" && !ret.Mode.Valid() { + return &ret, ErrUnknownWORMModeDirective + } + + if ret.Mode.Valid() && ret.RetainUntilDate.IsZero() { + return &ret, ErrMalformedXML + } + + if !ret.Mode.Valid() && !ret.RetainUntilDate.IsZero() { + return &ret, ErrMalformedXML + } + + t, err := UTCNowNTP() + if err != nil { + lockLogIf(context.Background(), err) + return &ret, ErrPastObjectLockRetainDate + } + + if !ret.RetainUntilDate.IsZero() && ret.RetainUntilDate.Before(t) { + return &ret, ErrPastObjectLockRetainDate + } + + return &ret, nil +} + +// IsObjectLockRetentionRequested returns true if object lock retention headers are set. +func IsObjectLockRetentionRequested(h http.Header) bool { + if _, ok := h[AmzObjectLockMode]; ok { + return true + } + if _, ok := h[AmzObjectLockRetainUntilDate]; ok { + return true + } + return false +} + +// IsObjectLockLegalHoldRequested returns true if object lock legal hold header is set. +func IsObjectLockLegalHoldRequested(h http.Header) bool { + _, ok := h[AmzObjectLockLegalHold] + return ok +} + +// IsObjectLockGovernanceBypassSet returns true if object lock governance bypass header is set. +func IsObjectLockGovernanceBypassSet(h http.Header) bool { + return strings.EqualFold(h.Get(AmzObjectLockBypassRetGovernance), "true") +} + +// IsObjectLockRequested returns true if legal hold or object lock retention headers are requested. +func IsObjectLockRequested(h http.Header) bool { + return IsObjectLockLegalHoldRequested(h) || IsObjectLockRetentionRequested(h) +} + +// ParseObjectLockRetentionHeaders parses http headers to extract retention mode and retention date +func ParseObjectLockRetentionHeaders(h http.Header) (rmode RetMode, r RetentionDate, err error) { + retMode := h.Get(AmzObjectLockMode) + dateStr := h.Get(AmzObjectLockRetainUntilDate) + if len(retMode) == 0 || len(dateStr) == 0 { + return rmode, r, ErrObjectLockInvalidHeaders + } + + rmode = parseRetMode(retMode) + if !rmode.Valid() { + return rmode, r, ErrUnknownWORMModeDirective + } + + var retDate time.Time + // While AWS documentation mentions that the date specified + // must be present in ISO 8601 format, in reality they allow + // users to provide RFC 3339 compliant dates. + retDate, err = amztime.ISO8601Parse(dateStr) + if err != nil { + return rmode, r, ErrInvalidRetentionDate + } + _, replReq := h[textproto.CanonicalMIMEHeaderKey(xhttp.MinIOSourceReplicationRequest)] + + t, err := UTCNowNTP() + if err != nil { + lockLogIf(context.Background(), err) + return rmode, r, ErrPastObjectLockRetainDate + } + + if retDate.Before(t) && !replReq { + return rmode, r, ErrPastObjectLockRetainDate + } + + return rmode, RetentionDate{retDate}, nil +} + +// GetObjectRetentionMeta constructs ObjectRetention from metadata +func GetObjectRetentionMeta(meta map[string]string) ObjectRetention { + var mode RetMode + var retainTill RetentionDate + + var modeStr, tillStr string + ok := false + + modeStr, ok = meta[strings.ToLower(AmzObjectLockMode)] + if !ok { + modeStr, ok = meta[AmzObjectLockMode] + } + if ok { + mode = parseRetMode(modeStr) + } else { + return ObjectRetention{} + } + + tillStr, ok = meta[strings.ToLower(AmzObjectLockRetainUntilDate)] + if !ok { + tillStr, ok = meta[AmzObjectLockRetainUntilDate] + } + if ok { + if t, e := amztime.ISO8601Parse(tillStr); e == nil { + retainTill = RetentionDate{t.UTC()} + } + } + return ObjectRetention{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/", Mode: mode, RetainUntilDate: retainTill} +} + +// GetObjectLegalHoldMeta constructs ObjectLegalHold from metadata +func GetObjectLegalHoldMeta(meta map[string]string) ObjectLegalHold { + holdStr, ok := meta[strings.ToLower(AmzObjectLockLegalHold)] + if !ok { + holdStr, ok = meta[AmzObjectLockLegalHold] + } + if ok { + return ObjectLegalHold{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/", Status: parseLegalHoldStatus(holdStr)} + } + return ObjectLegalHold{} +} + +// ParseObjectLockLegalHoldHeaders parses request headers to construct ObjectLegalHold +func ParseObjectLockLegalHoldHeaders(h http.Header) (lhold ObjectLegalHold, err error) { + holdStatus, ok := h[AmzObjectLockLegalHold] + if ok { + lh := parseLegalHoldStatus(holdStatus[0]) + if !lh.Valid() { + return lhold, ErrUnknownWORMModeDirective + } + lhold = ObjectLegalHold{XMLNS: "http://s3.amazonaws.com/doc/2006-03-01/", Status: lh} + } + return lhold, nil +} + +// ObjectLegalHold specified in +// https://docs.aws.amazon.com/AmazonS3/latest/API/API_PutObjectLegalHold.html +type ObjectLegalHold struct { + XMLNS string `xml:"xmlns,attr,omitempty"` + XMLName xml.Name `xml:"LegalHold"` + Status LegalHoldStatus `xml:"Status,omitempty"` +} + +// UnmarshalXML - decodes XML data. +func (l *ObjectLegalHold) UnmarshalXML(d *xml.Decoder, start xml.StartElement) (err error) { + switch start.Name.Local { + case "LegalHold", "ObjectLockLegalHold": + default: + return xml.UnmarshalError(fmt.Sprintf("expected element type / but have <%s>", + start.Name.Local)) + } + for { + // Read tokens from the XML document in a stream. + t, err := d.Token() + if err != nil { + if err == io.EOF { + break + } + return err + } + + if se, ok := t.(xml.StartElement); ok { + switch se.Name.Local { + case "Status": + var st LegalHoldStatus + if err = d.DecodeElement(&st, &se); err != nil { + return err + } + l.Status = st + default: + return xml.UnmarshalError(fmt.Sprintf("expected element type but have <%s>", se.Name.Local)) + } + } + } + return nil +} + +// IsEmpty returns true if struct is empty +func (l *ObjectLegalHold) IsEmpty() bool { + return !l.Status.Valid() +} + +// ParseObjectLegalHold decodes the XML into ObjectLegalHold +func ParseObjectLegalHold(reader io.Reader) (hold *ObjectLegalHold, err error) { + buf, err := io.ReadAll(io.LimitReader(reader, maxObjectLockConfigSize)) + if err != nil { + return nil, err + } + + hold = &ObjectLegalHold{} + if err = xml.NewDecoder(bytes.NewReader(buf)).Decode(hold); err != nil { + return nil, err + } + + if !hold.Status.Valid() { + return nil, ErrMalformedXML + } + return hold, err +} + +// FilterObjectLockMetadata filters object lock metadata if s3:GetObjectRetention permission is denied or if isCopy flag set. +func FilterObjectLockMetadata(metadata map[string]string, filterRetention, filterLegalHold bool) map[string]string { + // Copy on write + dst := metadata + var copied bool + delKey := func(key string) { + key = strings.ToLower(key) + if _, ok := metadata[key]; !ok { + return + } + if !copied { + dst = make(map[string]string, len(metadata)) + maps.Copy(dst, metadata) + copied = true + } + delete(dst, key) + } + legalHold := GetObjectLegalHoldMeta(metadata) + if !legalHold.Status.Valid() || filterLegalHold { + delKey(AmzObjectLockLegalHold) + } + + ret := GetObjectRetentionMeta(metadata) + if !ret.Mode.Valid() || filterRetention { + delKey(AmzObjectLockMode) + delKey(AmzObjectLockRetainUntilDate) + return dst + } + return dst +} diff --git a/internal/bucket/object/lock/lock_test.go b/internal/bucket/object/lock/lock_test.go new file mode 100644 index 00000000..be7975e2 --- /dev/null +++ b/internal/bucket/object/lock/lock_test.go @@ -0,0 +1,680 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lock + +import ( + "encoding/xml" + "errors" + "fmt" + "net/http" + "reflect" + "strings" + "testing" + "time" + + xhttp "github.com/minio/minio/internal/http" +) + +func TestParseMode(t *testing.T) { + testCases := []struct { + value string + expectedMode RetMode + }{ + { + value: "governance", + expectedMode: RetGovernance, + }, + { + value: "complIAnce", + expectedMode: RetCompliance, + }, + { + value: "gce", + expectedMode: "", + }, + } + + for _, tc := range testCases { + if parseRetMode(tc.value) != tc.expectedMode { + t.Errorf("Expected Mode %s, got %s", tc.expectedMode, parseRetMode(tc.value)) + } + } +} + +func TestParseLegalHoldStatus(t *testing.T) { + tests := []struct { + value string + expectedStatus LegalHoldStatus + }{ + { + value: "ON", + expectedStatus: LegalHoldOn, + }, + { + value: "Off", + expectedStatus: LegalHoldOff, + }, + { + value: "x", + expectedStatus: "", + }, + } + + for _, tt := range tests { + actualStatus := parseLegalHoldStatus(tt.value) + if actualStatus != tt.expectedStatus { + t.Errorf("Expected legal hold status %s, got %s", tt.expectedStatus, actualStatus) + } + } +} + +// TestUnmarshalDefaultRetention checks if default retention +// marshaling and unmarshalling work as expected +func TestUnmarshalDefaultRetention(t *testing.T) { + days := uint64(4) + years := uint64(1) + zerodays := uint64(0) + invalidDays := uint64(maximumRetentionDays + 1) + tests := []struct { + value DefaultRetention + expectedErr error + expectErr bool + }{ + { + value: DefaultRetention{Mode: "retain"}, + expectedErr: fmt.Errorf("unknown retention mode retain"), + expectErr: true, + }, + { + value: DefaultRetention{Mode: RetGovernance}, + expectedErr: fmt.Errorf("either Days or Years must be specified"), + expectErr: true, + }, + { + value: DefaultRetention{Mode: RetGovernance, Days: &days}, + expectedErr: nil, + expectErr: false, + }, + { + value: DefaultRetention{Mode: RetGovernance, Years: &years}, + expectedErr: nil, + expectErr: false, + }, + { + value: DefaultRetention{Mode: RetGovernance, Days: &days, Years: &years}, + expectedErr: fmt.Errorf("either Days or Years must be specified, not both"), + expectErr: true, + }, + { + value: DefaultRetention{Mode: RetGovernance, Days: &zerodays}, + expectedErr: fmt.Errorf("Default retention period must be a positive integer value for 'Days'"), + expectErr: true, + }, + { + value: DefaultRetention{Mode: RetGovernance, Days: &invalidDays}, + expectedErr: fmt.Errorf("Default retention period too large for 'Days' %d", invalidDays), + expectErr: true, + }, + } + for _, tt := range tests { + d, err := xml.MarshalIndent(&tt.value, "", "\t") + if err != nil { + t.Fatal(err) + } + var dr DefaultRetention + err = xml.Unmarshal(d, &dr) + //nolint:gocritic + if tt.expectedErr == nil { + if err != nil { + t.Fatalf("error: expected = , got = %v", err) + } + } else if err == nil { + t.Fatalf("error: expected = %v, got = ", tt.expectedErr) + } else if tt.expectedErr.Error() != err.Error() { + t.Fatalf("error: expected = %v, got = %v", tt.expectedErr, err) + } + } +} + +func TestParseObjectLockConfig(t *testing.T) { + tests := []struct { + value string + expectedErr error + expectErr bool + }{ + { + value: `yes`, + expectedErr: fmt.Errorf("only 'Enabled' value is allowed to ObjectLockEnabled element"), + expectErr: true, + }, + { + value: `EnabledCOMPLIANCE0`, + expectedErr: fmt.Errorf("Default retention period must be a positive integer value for 'Days'"), + expectErr: true, + }, + { + value: `EnabledCOMPLIANCE30`, + expectedErr: nil, + expectErr: false, + }, + } + for _, tt := range tests { + t.Run("", func(t *testing.T) { + _, err := ParseObjectLockConfig(strings.NewReader(tt.value)) + //nolint:gocritic + if tt.expectedErr == nil { + if err != nil { + t.Fatalf("error: expected = , got = %v", err) + } + } else if err == nil { + t.Fatalf("error: expected = %v, got = ", tt.expectedErr) + } else if tt.expectedErr.Error() != err.Error() { + t.Fatalf("error: expected = %v, got = %v", tt.expectedErr, err) + } + }) + } +} + +func TestParseObjectRetention(t *testing.T) { + tests := []struct { + value string + expectedErr error + expectErr bool + }{ + { + value: `string2020-01-02T15:04:05Z`, + expectedErr: ErrUnknownWORMModeDirective, + expectErr: true, + }, + { + value: `COMPLIANCE2017-01-02T15:04:05Z`, + expectedErr: ErrPastObjectLockRetainDate, + expectErr: true, + }, + { + value: `GOVERNANCE2057-01-02T15:04:05Z`, + expectedErr: nil, + expectErr: false, + }, + { + value: `GOVERNANCE2057-01-02T15:04:05.000Z`, + expectedErr: nil, + expectErr: false, + }, + } + for _, tt := range tests { + t.Run("", func(t *testing.T) { + _, err := ParseObjectRetention(strings.NewReader(tt.value)) + //nolint:gocritic + if tt.expectedErr == nil { + if err != nil { + t.Fatalf("error: expected = , got = %v", err) + } + } else if err == nil { + t.Fatalf("error: expected = %v, got = ", tt.expectedErr) + } else if tt.expectedErr.Error() != err.Error() { + t.Fatalf("error: expected = %v, got = %v", tt.expectedErr, err) + } + }) + } +} + +func TestIsObjectLockRequested(t *testing.T) { + tests := []struct { + header http.Header + expectedVal bool + }{ + { + header: http.Header{ + "Authorization": []string{"AWS4-HMAC-SHA256 "}, + "X-Amz-Content-Sha256": []string{""}, + "Content-Encoding": []string{""}, + }, + expectedVal: false, + }, + { + header: http.Header{ + AmzObjectLockLegalHold: []string{""}, + }, + expectedVal: true, + }, + { + header: http.Header{ + AmzObjectLockRetainUntilDate: []string{""}, + AmzObjectLockMode: []string{""}, + }, + expectedVal: true, + }, + { + header: http.Header{ + AmzObjectLockBypassRetGovernance: []string{""}, + }, + expectedVal: false, + }, + } + for _, tt := range tests { + actualVal := IsObjectLockRequested(tt.header) + if actualVal != tt.expectedVal { + t.Fatalf("error: expected %v, actual %v", tt.expectedVal, actualVal) + } + } +} + +func TestIsObjectLockGovernanceBypassSet(t *testing.T) { + tests := []struct { + header http.Header + expectedVal bool + }{ + { + header: http.Header{ + "Authorization": []string{"AWS4-HMAC-SHA256 "}, + "X-Amz-Content-Sha256": []string{""}, + "Content-Encoding": []string{""}, + }, + expectedVal: false, + }, + { + header: http.Header{ + AmzObjectLockLegalHold: []string{""}, + }, + expectedVal: false, + }, + { + header: http.Header{ + AmzObjectLockRetainUntilDate: []string{""}, + AmzObjectLockMode: []string{""}, + }, + expectedVal: false, + }, + { + header: http.Header{ + AmzObjectLockBypassRetGovernance: []string{""}, + }, + expectedVal: false, + }, + { + header: http.Header{ + AmzObjectLockBypassRetGovernance: []string{"true"}, + }, + expectedVal: true, + }, + } + for _, tt := range tests { + actualVal := IsObjectLockGovernanceBypassSet(tt.header) + if actualVal != tt.expectedVal { + t.Fatalf("error: expected %v, actual %v", tt.expectedVal, actualVal) + } + } +} + +func TestParseObjectLockRetentionHeaders(t *testing.T) { + tests := []struct { + header http.Header + expectedErr error + }{ + { + header: http.Header{ + "Authorization": []string{"AWS4-HMAC-SHA256 "}, + "X-Amz-Content-Sha256": []string{""}, + "Content-Encoding": []string{""}, + }, + expectedErr: ErrObjectLockInvalidHeaders, + }, + { + header: http.Header{ + xhttp.AmzObjectLockMode: []string{"lock"}, + xhttp.AmzObjectLockRetainUntilDate: []string{"2017-01-02"}, + }, + expectedErr: ErrUnknownWORMModeDirective, + }, + { + header: http.Header{ + xhttp.AmzObjectLockMode: []string{"governance"}, + }, + expectedErr: ErrObjectLockInvalidHeaders, + }, + { + header: http.Header{ + xhttp.AmzObjectLockRetainUntilDate: []string{"2017-01-02"}, + xhttp.AmzObjectLockMode: []string{"governance"}, + }, + expectedErr: ErrInvalidRetentionDate, + }, + { + header: http.Header{ + xhttp.AmzObjectLockRetainUntilDate: []string{"2017-01-02T15:04:05Z"}, + xhttp.AmzObjectLockMode: []string{"governance"}, + }, + expectedErr: ErrPastObjectLockRetainDate, + }, + { + header: http.Header{ + xhttp.AmzObjectLockMode: []string{"governance"}, + xhttp.AmzObjectLockRetainUntilDate: []string{"2017-01-02T15:04:05Z"}, + }, + expectedErr: ErrPastObjectLockRetainDate, + }, + { + header: http.Header{ + xhttp.AmzObjectLockMode: []string{"governance"}, + xhttp.AmzObjectLockRetainUntilDate: []string{"2087-01-02T15:04:05Z"}, + }, + expectedErr: nil, + }, + { + header: http.Header{ + xhttp.AmzObjectLockMode: []string{"governance"}, + xhttp.AmzObjectLockRetainUntilDate: []string{"2087-01-02T15:04:05.000Z"}, + }, + expectedErr: nil, + }, + } + + for i, tt := range tests { + _, _, err := ParseObjectLockRetentionHeaders(tt.header) + //nolint:gocritic + if tt.expectedErr == nil { + if err != nil { + t.Fatalf("Case %d error: expected = , got = %v", i, err) + } + } else if err == nil { + t.Fatalf("Case %d error: expected = %v, got = ", i, tt.expectedErr) + } else if tt.expectedErr.Error() != err.Error() { + t.Fatalf("Case %d error: expected = %v, got = %v", i, tt.expectedErr, err) + } + } +} + +func TestGetObjectRetentionMeta(t *testing.T) { + tests := []struct { + metadata map[string]string + expected ObjectRetention + }{ + { + metadata: map[string]string{ + "Authorization": "AWS4-HMAC-SHA256 ", + "X-Amz-Content-Sha256": "", + "Content-Encoding": "", + }, + expected: ObjectRetention{}, + }, + { + metadata: map[string]string{ + "x-amz-object-lock-mode": "governance", + }, + expected: ObjectRetention{Mode: RetGovernance}, + }, + { + metadata: map[string]string{ + "x-amz-object-lock-retain-until-date": "2020-02-01", + }, + expected: ObjectRetention{RetainUntilDate: RetentionDate{time.Date(2020, 2, 1, 12, 0, 0, 0, time.UTC)}}, + }, + } + + for i, tt := range tests { + o := GetObjectRetentionMeta(tt.metadata) + if o.Mode != tt.expected.Mode { + t.Fatalf("Case %d expected %v, got %v", i, tt.expected.Mode, o.Mode) + } + } +} + +func TestGetObjectLegalHoldMeta(t *testing.T) { + tests := []struct { + metadata map[string]string + expected ObjectLegalHold + }{ + { + metadata: map[string]string{ + "x-amz-object-lock-mode": "governance", + }, + expected: ObjectLegalHold{}, + }, + { + metadata: map[string]string{ + "x-amz-object-lock-legal-hold": "on", + }, + expected: ObjectLegalHold{Status: LegalHoldOn}, + }, + { + metadata: map[string]string{ + "x-amz-object-lock-legal-hold": "off", + }, + expected: ObjectLegalHold{Status: LegalHoldOff}, + }, + { + metadata: map[string]string{ + "x-amz-object-lock-legal-hold": "X", + }, + expected: ObjectLegalHold{Status: ""}, + }, + } + + for i, tt := range tests { + o := GetObjectLegalHoldMeta(tt.metadata) + if o.Status != tt.expected.Status { + t.Fatalf("Case %d expected %v, got %v", i, tt.expected.Status, o.Status) + } + } +} + +func TestParseObjectLegalHold(t *testing.T) { + tests := []struct { + value string + expectedErr error + expectErr bool + }{ + { + value: `string`, + expectedErr: ErrMalformedXML, + expectErr: true, + }, + { + value: `ON`, + expectedErr: nil, + expectErr: false, + }, + { + value: `ON`, + expectedErr: nil, + expectErr: false, + }, + // invalid Status key + { + value: `ON`, + expectedErr: errors.New("expected element type but have "), + expectErr: true, + }, + // invalid XML attr + { + value: `ON`, + expectedErr: errors.New("expected element type / but have "), + expectErr: true, + }, + { + value: `On`, + expectedErr: ErrMalformedXML, + expectErr: true, + }, + } + for i, tt := range tests { + _, err := ParseObjectLegalHold(strings.NewReader(tt.value)) + //nolint:gocritic + if tt.expectedErr == nil { + if err != nil { + t.Fatalf("Case %d error: expected = , got = %v", i, err) + } + } else if err == nil { + t.Fatalf("Case %d error: expected = %v, got = ", i, tt.expectedErr) + } else if tt.expectedErr.Error() != err.Error() { + t.Fatalf("Case %d error: expected = %v, got = %v", i, tt.expectedErr, err) + } + } +} + +func TestFilterObjectLockMetadata(t *testing.T) { + tests := []struct { + metadata map[string]string + filterRetention bool + filterLegalHold bool + expected map[string]string + }{ + { + metadata: map[string]string{ + "Authorization": "AWS4-HMAC-SHA256 ", + "X-Amz-Content-Sha256": "", + "Content-Encoding": "", + }, + expected: map[string]string{ + "Authorization": "AWS4-HMAC-SHA256 ", + "X-Amz-Content-Sha256": "", + "Content-Encoding": "", + }, + }, + { + metadata: map[string]string{ + "x-amz-object-lock-mode": "governance", + }, + expected: map[string]string{ + "x-amz-object-lock-mode": "governance", + }, + filterRetention: false, + }, + { + metadata: map[string]string{ + "x-amz-object-lock-mode": "governance", + "x-amz-object-lock-retain-until-date": "2020-02-01", + }, + expected: map[string]string{}, + filterRetention: true, + }, + { + metadata: map[string]string{ + "x-amz-object-lock-legal-hold": "off", + }, + expected: map[string]string{}, + filterLegalHold: true, + }, + { + metadata: map[string]string{ + "x-amz-object-lock-legal-hold": "on", + }, + expected: map[string]string{"x-amz-object-lock-legal-hold": "on"}, + filterLegalHold: false, + }, + { + metadata: map[string]string{ + "x-amz-object-lock-legal-hold": "on", + "x-amz-object-lock-mode": "governance", + "x-amz-object-lock-retain-until-date": "2020-02-01", + }, + expected: map[string]string{}, + filterRetention: true, + filterLegalHold: true, + }, + { + metadata: map[string]string{ + "x-amz-object-lock-legal-hold": "on", + "x-amz-object-lock-mode": "governance", + "x-amz-object-lock-retain-until-date": "2020-02-01", + }, + expected: map[string]string{ + "x-amz-object-lock-legal-hold": "on", + "x-amz-object-lock-mode": "governance", + "x-amz-object-lock-retain-until-date": "2020-02-01", + }, + }, + } + + for i, tt := range tests { + o := FilterObjectLockMetadata(tt.metadata, tt.filterRetention, tt.filterLegalHold) + if !reflect.DeepEqual(o, tt.expected) { + t.Fatalf("Case %d expected %v, got %v", i, tt.metadata, o) + } + } +} + +func TestToString(t *testing.T) { + days := uint64(30) + daysPtr := &days + years := uint64(2) + yearsPtr := &years + + tests := []struct { + name string + c Config + want string + }{ + { + name: "happy case", + c: Config{ + ObjectLockEnabled: "Enabled", + }, + want: "Enabled: true", + }, + { + name: "with default retention days", + c: Config{ + ObjectLockEnabled: "Enabled", + Rule: &struct { + DefaultRetention DefaultRetention `xml:"DefaultRetention"` + }{ + DefaultRetention: DefaultRetention{ + Mode: RetGovernance, + Days: daysPtr, + }, + }, + }, + want: "Enabled: true, Mode: GOVERNANCE, Days: 30", + }, + { + name: "with default retention years", + c: Config{ + ObjectLockEnabled: "Enabled", + Rule: &struct { + DefaultRetention DefaultRetention `xml:"DefaultRetention"` + }{ + DefaultRetention: DefaultRetention{ + Mode: RetCompliance, + Years: yearsPtr, + }, + }, + }, + want: "Enabled: true, Mode: COMPLIANCE, Years: 2", + }, + { + name: "disabled case", + c: Config{ + ObjectLockEnabled: "Disabled", + }, + want: "Enabled: false", + }, + { + name: "empty case", + c: Config{}, + want: "Enabled: false", + }, + } + for _, tt := range tests { + got := tt.c.String() + if got != tt.want { + t.Errorf("test: %s, got: '%v', want: '%v'", tt.name, got, tt.want) + } + } +} diff --git a/internal/bucket/replication/and.go b/internal/bucket/replication/and.go new file mode 100644 index 00000000..9c8de96b --- /dev/null +++ b/internal/bucket/replication/and.go @@ -0,0 +1,63 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package replication + +import ( + "encoding/xml" +) + +// And - a tag to combine a prefix and multiple tags for replication configuration rule. +type And struct { + XMLName xml.Name `xml:"And" json:"And"` + Prefix string `xml:"Prefix,omitempty" json:"Prefix,omitempty"` + Tags []Tag `xml:"Tag,omitempty" json:"Tag,omitempty"` +} + +var errDuplicateTagKey = Errorf("Duplicate Tag Keys are not allowed") + +// isEmpty returns true if Tags field is null +func (a And) isEmpty() bool { + return len(a.Tags) == 0 && a.Prefix == "" +} + +// Validate - validates the And field +func (a And) Validate() error { + if a.ContainsDuplicateTag() { + return errDuplicateTagKey + } + for _, t := range a.Tags { + if err := t.Validate(); err != nil { + return err + } + } + return nil +} + +// ContainsDuplicateTag - returns true if duplicate keys are present in And +func (a And) ContainsDuplicateTag() bool { + x := make(map[string]struct{}, len(a.Tags)) + + for _, t := range a.Tags { + if _, has := x[t.Key]; has { + return true + } + x[t.Key] = struct{}{} + } + + return false +} diff --git a/internal/bucket/replication/datatypes.go b/internal/bucket/replication/datatypes.go new file mode 100644 index 00000000..980f9be5 --- /dev/null +++ b/internal/bucket/replication/datatypes.go @@ -0,0 +1,77 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package replication + +//go:generate msgp -file=$GOFILE + +// StatusType of Replication for x-amz-replication-status header +type StatusType string + +// Type - replication type enum +type Type int + +const ( + // Pending - replication is pending. + Pending StatusType = "PENDING" + + // Completed - replication completed ok. + Completed StatusType = "COMPLETED" + + // CompletedLegacy was called "COMPLETE" incorrectly. + CompletedLegacy StatusType = "COMPLETE" + + // Failed - replication failed. + Failed StatusType = "FAILED" + + // Replica - this is a replica. + Replica StatusType = "REPLICA" +) + +// String returns string representation of status +func (s StatusType) String() string { + return string(s) +} + +// Empty returns true if this status is not set +func (s StatusType) Empty() bool { + return string(s) == "" +} + +// VersionPurgeStatusType represents status of a versioned delete or permanent delete w.r.t bucket replication +type VersionPurgeStatusType string + +const ( + // VersionPurgePending - versioned delete replication is pending. + VersionPurgePending VersionPurgeStatusType = "PENDING" + + // VersionPurgeComplete - versioned delete replication is now complete, erase version on disk. + VersionPurgeComplete VersionPurgeStatusType = "COMPLETE" + + // VersionPurgeFailed - versioned delete replication failed. + VersionPurgeFailed VersionPurgeStatusType = "FAILED" +) + +// Empty returns true if purge status was not set. +func (v VersionPurgeStatusType) Empty() bool { + return string(v) == "" +} + +// Pending returns true if the version is pending purge. +func (v VersionPurgeStatusType) Pending() bool { + return v == VersionPurgePending || v == VersionPurgeFailed +} diff --git a/internal/bucket/replication/datatypes_gen.go b/internal/bucket/replication/datatypes_gen.go new file mode 100644 index 00000000..3a9c20a8 --- /dev/null +++ b/internal/bucket/replication/datatypes_gen.go @@ -0,0 +1,163 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package replication + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *StatusType) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 string + zb0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = StatusType(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z StatusType) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteString(string(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z StatusType) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendString(o, string(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *StatusType) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 string + zb0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = StatusType(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z StatusType) Msgsize() (s int) { + s = msgp.StringPrefixSize + len(string(z)) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *Type) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 int + zb0001, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = Type(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z Type) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteInt(int(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z Type) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendInt(o, int(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *Type) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 int + zb0001, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = Type(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z Type) Msgsize() (s int) { + s = msgp.IntSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *VersionPurgeStatusType) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 string + zb0001, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = VersionPurgeStatusType(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z VersionPurgeStatusType) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteString(string(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z VersionPurgeStatusType) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendString(o, string(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *VersionPurgeStatusType) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 string + zb0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = VersionPurgeStatusType(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z VersionPurgeStatusType) Msgsize() (s int) { + s = msgp.StringPrefixSize + len(string(z)) + return +} diff --git a/internal/bucket/replication/datatypes_gen_test.go b/internal/bucket/replication/datatypes_gen_test.go new file mode 100644 index 00000000..048f6d7b --- /dev/null +++ b/internal/bucket/replication/datatypes_gen_test.go @@ -0,0 +1,3 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package replication diff --git a/internal/bucket/replication/destination.go b/internal/bucket/replication/destination.go new file mode 100644 index 00000000..9f31b323 --- /dev/null +++ b/internal/bucket/replication/destination.go @@ -0,0 +1,136 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package replication + +import ( + "encoding/xml" + "fmt" + "strings" + + "github.com/minio/pkg/v3/wildcard" +) + +// DestinationARNPrefix - destination ARN prefix as per AWS S3 specification. +const DestinationARNPrefix = "arn:aws:s3:::" + +// DestinationARNMinIOPrefix - destination ARN prefix for MinIO. +const DestinationARNMinIOPrefix = "arn:minio:replication:" + +// Destination - destination in ReplicationConfiguration. +type Destination struct { + XMLName xml.Name `xml:"Destination" json:"Destination"` + Bucket string `xml:"Bucket" json:"Bucket"` + StorageClass string `xml:"StorageClass" json:"StorageClass"` + ARN string + // EncryptionConfiguration TODO: not needed for MinIO +} + +func (d Destination) isValidStorageClass() bool { + if d.StorageClass == "" { + return true + } + return d.StorageClass == "STANDARD" || d.StorageClass == "REDUCED_REDUNDANCY" +} + +// IsValid - checks whether Destination is valid or not. +func (d Destination) IsValid() bool { + return d.Bucket != "" || !d.isValidStorageClass() +} + +func (d Destination) String() string { + return d.ARN +} + +// LegacyArn returns true if arn format has prefix "arn:aws:s3:::" which was +// used prior to multi-destination +func (d Destination) LegacyArn() bool { + return strings.HasPrefix(d.ARN, DestinationARNPrefix) +} + +// TargetArn returns true if arn format has prefix "arn:minio:replication:::" +// used for multi-destination targets +func (d Destination) TargetArn() bool { + return strings.HasPrefix(d.ARN, DestinationARNMinIOPrefix) +} + +// MarshalXML - encodes to XML data. +func (d Destination) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if err := e.EncodeToken(start); err != nil { + return err + } + if err := e.EncodeElement(d.String(), xml.StartElement{Name: xml.Name{Local: "Bucket"}}); err != nil { + return err + } + if d.StorageClass != "" { + if err := e.EncodeElement(d.StorageClass, xml.StartElement{Name: xml.Name{Local: "StorageClass"}}); err != nil { + return err + } + } + return e.EncodeToken(xml.EndElement{Name: start.Name}) +} + +// UnmarshalXML - decodes XML data. +func (d *Destination) UnmarshalXML(dec *xml.Decoder, start xml.StartElement) (err error) { + // Make subtype to avoid recursive UnmarshalXML(). + type destination Destination + dest := destination{} + + if err := dec.DecodeElement(&dest, &start); err != nil { + return err + } + parsedDest, err := parseDestination(dest.Bucket) + if err != nil { + return err + } + if dest.StorageClass != "" { + switch dest.StorageClass { + case "STANDARD", "REDUCED_REDUNDANCY": + default: + return fmt.Errorf("unknown storage class %s", dest.StorageClass) + } + } + parsedDest.StorageClass = dest.StorageClass + *d = parsedDest + return nil +} + +// Validate - validates Resource is for given bucket or not. +func (d Destination) Validate(bucketName string) error { + if !d.IsValid() { + return Errorf("invalid destination") + } + + if !wildcard.Match(d.Bucket, bucketName) { + return Errorf("bucket name does not match") + } + return nil +} + +// parseDestination - parses string to Destination. +func parseDestination(s string) (Destination, error) { + if !strings.HasPrefix(s, DestinationARNPrefix) && !strings.HasPrefix(s, DestinationARNMinIOPrefix) { + return Destination{}, Errorf("invalid destination '%s'", s) + } + + bucketName := strings.TrimPrefix(s, DestinationARNPrefix) + + return Destination{ + Bucket: bucketName, + ARN: s, + }, nil +} diff --git a/internal/bucket/replication/error.go b/internal/bucket/replication/error.go new file mode 100644 index 00000000..b653c863 --- /dev/null +++ b/internal/bucket/replication/error.go @@ -0,0 +1,45 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package replication + +import ( + "fmt" +) + +// Error is the generic type for any error happening during tag +// parsing. +type Error struct { + err error +} + +// Errorf - formats according to a format specifier and returns +// the string as a value that satisfies error of type tagging.Error +func Errorf(format string, a ...any) error { + return Error{err: fmt.Errorf(format, a...)} +} + +// Unwrap the internal error. +func (e Error) Unwrap() error { return e.err } + +// Error 'error' compatible method. +func (e Error) Error() string { + if e.err == nil { + return "replication: cause " + } + return e.err.Error() +} diff --git a/internal/bucket/replication/filter.go b/internal/bucket/replication/filter.go new file mode 100644 index 00000000..25cae51e --- /dev/null +++ b/internal/bucket/replication/filter.go @@ -0,0 +1,138 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package replication + +import ( + "encoding/xml" + + "github.com/minio/minio-go/v7/pkg/tags" +) + +var errInvalidFilter = Errorf("Filter must have exactly one of Prefix, Tag, or And specified") + +// Filter - a filter for a replication configuration Rule. +type Filter struct { + XMLName xml.Name `xml:"Filter" json:"Filter"` + Prefix string + And And + Tag Tag + + // Caching tags, only once + cachedTags map[string]string +} + +// IsEmpty returns true if filter is not set +func (f Filter) IsEmpty() bool { + return f.And.isEmpty() && f.Tag.IsEmpty() && f.Prefix == "" +} + +// MarshalXML - produces the xml representation of the Filter struct +// only one of Prefix, And and Tag should be present in the output. +func (f Filter) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if err := e.EncodeToken(start); err != nil { + return err + } + + switch { + case !f.And.isEmpty(): + if err := e.EncodeElement(f.And, xml.StartElement{Name: xml.Name{Local: "And"}}); err != nil { + return err + } + case !f.Tag.IsEmpty(): + if err := e.EncodeElement(f.Tag, xml.StartElement{Name: xml.Name{Local: "Tag"}}); err != nil { + return err + } + default: + // Always print Prefix field when both And & Tag are empty + if err := e.EncodeElement(f.Prefix, xml.StartElement{Name: xml.Name{Local: "Prefix"}}); err != nil { + return err + } + } + + return e.EncodeToken(xml.EndElement{Name: start.Name}) +} + +// Validate - validates the filter element +func (f Filter) Validate() error { + // A Filter must have exactly one of Prefix, Tag, or And specified. + if !f.And.isEmpty() { + if f.Prefix != "" { + return errInvalidFilter + } + if !f.Tag.IsEmpty() { + return errInvalidFilter + } + if err := f.And.Validate(); err != nil { + return err + } + } + if f.Prefix != "" { + if !f.Tag.IsEmpty() { + return errInvalidFilter + } + } + if !f.Tag.IsEmpty() { + if err := f.Tag.Validate(); err != nil { + return err + } + } + return nil +} + +// TestTags tests if the object tags satisfy the Filter tags requirement, +// it returns true if there is no tags in the underlying Filter. +func (f *Filter) TestTags(userTags string) bool { + if f.cachedTags == nil { + cached := make(map[string]string) + for _, t := range append(f.And.Tags, f.Tag) { + if !t.IsEmpty() { + cached[t.Key] = t.Value + } + } + f.cachedTags = cached + } + + // This filter does not have any tags, always return true + if len(f.cachedTags) == 0 { + return true + } + + parsedTags, err := tags.ParseObjectTags(userTags) + if err != nil { + return false + } + + tagsMap := parsedTags.ToMap() + + // This filter has tags configured but this object + // does not have any tag, skip this object + if len(tagsMap) == 0 { + return false + } + + // Both filter and object have tags, find a match, + // skip this object otherwise + for k, cv := range f.cachedTags { + v, ok := tagsMap[k] + if ok && v == cv { + return true + } + } + + return false +} diff --git a/internal/bucket/replication/replication.go b/internal/bucket/replication/replication.go new file mode 100644 index 00000000..409e1c19 --- /dev/null +++ b/internal/bucket/replication/replication.go @@ -0,0 +1,294 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package replication + +import ( + "encoding/xml" + "io" + "sort" + "strconv" + "strings" +) + +var ( + errReplicationTooManyRules = Errorf("Replication configuration allows a maximum of 1000 rules") + errReplicationNoRule = Errorf("Replication configuration should have at least one rule") + errReplicationUniquePriority = Errorf("Replication configuration has duplicate priority") + errRoleArnMissingLegacy = Errorf("Missing required parameter `Role` in ReplicationConfiguration") + errDestinationArnMissing = Errorf("Missing required parameter `Destination` in Replication rule") + errInvalidSourceSelectionCriteria = Errorf("Invalid ReplicaModification status") + errRoleArnPresentForMultipleTargets = Errorf("`Role` should be empty in ReplicationConfiguration for multiple targets") +) + +// Config - replication configuration specified in +// https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html +type Config struct { + XMLName xml.Name `xml:"ReplicationConfiguration" json:"-"` + Rules []Rule `xml:"Rule" json:"Rules"` + // RoleArn is being reused for MinIO replication ARN + RoleArn string `xml:"Role" json:"Role"` +} + +// Maximum 2MiB size per replication config. +const maxReplicationConfigSize = 2 << 20 + +// ParseConfig parses ReplicationConfiguration from xml +func ParseConfig(reader io.Reader) (*Config, error) { + config := Config{} + if err := xml.NewDecoder(io.LimitReader(reader, maxReplicationConfigSize)).Decode(&config); err != nil { + return nil, err + } + // By default, set replica modification to enabled if unset. + for i := range config.Rules { + if len(config.Rules[i].SourceSelectionCriteria.ReplicaModifications.Status) == 0 { + config.Rules[i].SourceSelectionCriteria = SourceSelectionCriteria{ + ReplicaModifications: ReplicaModifications{ + Status: Enabled, + }, + } + } + // Default DeleteReplication to disabled if unset. + if len(config.Rules[i].DeleteReplication.Status) == 0 { + config.Rules[i].DeleteReplication = DeleteReplication{ + Status: Disabled, + } + } + } + return &config, nil +} + +// Validate - validates the replication configuration +func (c Config) Validate(bucket string, sameTarget bool) error { + // replication config can't have more than 1000 rules + if len(c.Rules) > 1000 { + return errReplicationTooManyRules + } + // replication config should have at least one rule + if len(c.Rules) == 0 { + return errReplicationNoRule + } + + // Validate all the rules in the replication config + targetMap := make(map[string]struct{}) + priorityMap := make(map[string]struct{}) + var legacyArn bool + for _, r := range c.Rules { + if _, ok := targetMap[r.Destination.Bucket]; !ok { + targetMap[r.Destination.Bucket] = struct{}{} + } + if err := r.Validate(bucket, sameTarget); err != nil { + return err + } + if _, ok := priorityMap[strconv.Itoa(r.Priority)]; ok { + return errReplicationUniquePriority + } + priorityMap[strconv.Itoa(r.Priority)] = struct{}{} + + if r.Destination.LegacyArn() { + legacyArn = true + } + if c.RoleArn == "" && !r.Destination.TargetArn() { + return errDestinationArnMissing + } + } + // disallow combining old replication configuration which used RoleArn as target ARN with multiple + // destination replication + if c.RoleArn != "" && len(targetMap) > 1 { + return errRoleArnPresentForMultipleTargets + } + // validate RoleArn if destination used legacy ARN format. + if c.RoleArn == "" && legacyArn { + return errRoleArnMissingLegacy + } + return nil +} + +// Types of replication +const ( + UnsetReplicationType Type = 0 + iota + ObjectReplicationType + DeleteReplicationType + MetadataReplicationType + HealReplicationType + ExistingObjectReplicationType + ResyncReplicationType + AllReplicationType +) + +// Valid returns true if replication type is set +func (t Type) Valid() bool { + return t > 0 +} + +// IsDataReplication returns true if content being replicated +func (t Type) IsDataReplication() bool { + switch t { + case ObjectReplicationType, HealReplicationType, ExistingObjectReplicationType: + return true + } + return false +} + +// ObjectOpts provides information to deduce whether replication +// can be triggered on the resultant object. +type ObjectOpts struct { + Name string + UserTags string + VersionID string + DeleteMarker bool + SSEC bool + OpType Type + Replica bool + ExistingObject bool + TargetArn string +} + +// HasExistingObjectReplication returns true if any of the rule returns 'ExistingObjects' replication. +func (c Config) HasExistingObjectReplication(arn string) (hasARN, isEnabled bool) { + for _, rule := range c.Rules { + if rule.Destination.ARN == arn || c.RoleArn == arn { + if !hasARN { + hasARN = true + } + if rule.ExistingObjectReplication.Status == Enabled { + return true, true + } + } + } + return hasARN, false +} + +// FilterActionableRules returns the rules actions that need to be executed +// after evaluating prefix/tag filtering +func (c Config) FilterActionableRules(obj ObjectOpts) []Rule { + if obj.Name == "" && (obj.OpType != ResyncReplicationType && obj.OpType != AllReplicationType) { + return nil + } + var rules []Rule + for _, rule := range c.Rules { + if rule.Status == Disabled { + continue + } + + if obj.TargetArn != "" && rule.Destination.ARN != obj.TargetArn && c.RoleArn != obj.TargetArn { + continue + } + // Ignore other object level and prefix filters for resyncing target/listing bucket targets + if obj.OpType == ResyncReplicationType || obj.OpType == AllReplicationType { + rules = append(rules, rule) + continue + } + if obj.ExistingObject && rule.ExistingObjectReplication.Status == Disabled { + continue + } + if !strings.HasPrefix(obj.Name, rule.Prefix()) { + continue + } + if rule.Filter.TestTags(obj.UserTags) { + rules = append(rules, rule) + } + } + sort.Slice(rules, func(i, j int) bool { + return rules[i].Priority > rules[j].Priority && rules[i].Destination.String() == rules[j].Destination.String() + }) + + return rules +} + +// GetDestination returns destination bucket and storage class. +func (c Config) GetDestination() Destination { + if len(c.Rules) > 0 { + return c.Rules[0].Destination + } + return Destination{} +} + +// Replicate returns true if the object should be replicated. +func (c Config) Replicate(obj ObjectOpts) bool { + for _, rule := range c.FilterActionableRules(obj) { + if rule.Status == Disabled { + continue + } + if obj.ExistingObject && rule.ExistingObjectReplication.Status == Disabled { + return false + } + if obj.OpType == DeleteReplicationType { + switch { + case obj.VersionID != "": + // check MinIO extension for versioned deletes + return rule.DeleteReplication.Status == Enabled + default: + return rule.DeleteMarkerReplication.Status == Enabled + } + } // regular object/metadata replication + return rule.MetadataReplicate(obj) + } + return false +} + +// HasActiveRules - returns whether replication policy has active rules +// Optionally a prefix can be supplied. +// If recursive is specified the function will also return true if any level below the +// prefix has active rules. If no prefix is specified recursive is effectively true. +func (c Config) HasActiveRules(prefix string, recursive bool) bool { + if len(c.Rules) == 0 { + return false + } + for _, rule := range c.Rules { + if rule.Status == Disabled { + continue + } + if len(prefix) > 0 && len(rule.Filter.Prefix) > 0 { + // incoming prefix must be in rule prefix + if !recursive && !strings.HasPrefix(prefix, rule.Filter.Prefix) { + continue + } + // If recursive, we can skip this rule if it doesn't match the tested prefix or level below prefix + // does not match + if recursive && !strings.HasPrefix(rule.Prefix(), prefix) && !strings.HasPrefix(prefix, rule.Prefix()) { + continue + } + } + return true + } + return false +} + +// FilterTargetArns returns a slice of distinct target arns in the config +func (c Config) FilterTargetArns(obj ObjectOpts) []string { + var arns []string + + tgtsMap := make(map[string]struct{}) + rules := c.FilterActionableRules(obj) + for _, rule := range rules { + if rule.Status == Disabled { + continue + } + if c.RoleArn != "" { + arns = append(arns, c.RoleArn) // use legacy RoleArn if present + return arns + } + if _, ok := tgtsMap[rule.Destination.ARN]; !ok { + tgtsMap[rule.Destination.ARN] = struct{}{} + } + } + for k := range tgtsMap { + arns = append(arns, k) + } + return arns +} diff --git a/internal/bucket/replication/replication_test.go b/internal/bucket/replication/replication_test.go new file mode 100644 index 00000000..7732f8ca --- /dev/null +++ b/internal/bucket/replication/replication_test.go @@ -0,0 +1,417 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package replication + +import ( + "bytes" + "fmt" + "testing" +) + +func TestParseAndValidateReplicationConfig(t *testing.T) { + testCases := []struct { + inputConfig string + expectedParsingErr error + expectedValidationErr error + destBucket string + sameTarget bool + }{ + { // 1 Invalid delete marker status in replication config + inputConfig: `arn:aws:iam::AcctID:role/role-nameEnabledstringkey-prefixarn:aws:s3:::destinationbucket`, + destBucket: "destinationbucket", + sameTarget: false, + expectedParsingErr: nil, + expectedValidationErr: errInvalidDeleteMarkerReplicationStatus, + }, + // 2 No delete replication status in replication config + { + inputConfig: `arn:aws:iam::AcctID:role/role-nameEnabledDisabledkey-prefixarn:aws:s3:::destinationbucket`, + destBucket: "destinationbucket", + sameTarget: false, + expectedParsingErr: nil, + expectedValidationErr: nil, + }, + // 3 valid replication config + { + inputConfig: `arn:aws:iam::AcctID:role/role-nameEnabledDisabledDisabledkey-prefixarn:aws:s3:::destinationbucket`, + destBucket: "destinationbucket", + sameTarget: false, + expectedParsingErr: nil, + expectedValidationErr: nil, + }, + // 4 missing role in config and destination ARN is in legacy format + { + inputConfig: `EnabledDisabledDisabledkey-prefixarn:aws:s3:::destinationbucket`, + // destination bucket in config different from bucket specified + destBucket: "destinationbucket", + sameTarget: false, + expectedParsingErr: nil, + expectedValidationErr: errDestinationArnMissing, + }, + // 5 replication destination in different rules not identical + { + inputConfig: `EnabledDisabledDisabledkey-prefixarn:minio:replication:::destinationbucketEnabled3DisabledDisabledkey-prefixarn:minio:replication:::destinationbucket2`, + destBucket: "destinationbucket", + sameTarget: false, + expectedParsingErr: nil, + expectedValidationErr: nil, + }, + // 6 missing rule status in replication config + { + inputConfig: `arn:aws:iam::AcctID:role/role-nameDisabledDisabledkey-prefixarn:aws:s3:::destinationbucket`, + destBucket: "destinationbucket", + sameTarget: false, + expectedParsingErr: nil, + expectedValidationErr: errEmptyRuleStatus, + }, + // 7 invalid rule status in replication config + { + inputConfig: `arn:aws:iam::AcctID:role/role-nameEnssabledDisabledDisabledkey-prefixarn:aws:s3:::destinationbucketEnabledDisabledDisabledkey-prefixarn:aws:s3:::destinationbucket`, + destBucket: "destinationbucket", + sameTarget: false, + expectedParsingErr: nil, + expectedValidationErr: errInvalidRuleStatus, + }, + // 8 invalid rule id exceeds length allowed in replication config + { + inputConfig: `arn:aws:iam::AcctID:role/role-namevsUVERgOc8zZYagLSzSa5lE8qeI6nh1lyLNS4R9W052yfecrhhepGboswSWMMNO8CPcXM4GM3nKyQ72EadlMzzZBFoYWKn7ju5GoE5w9c57a0piHR1vexpdd9FrMquiruvAJ0MTGVupm0EegMVxoIOdjx7VgZhGrmi2XDvpVEFT7WmYMA9fSK297XkTHWyECaNHBySJ1Qp4vwX8tPNauKpfHx4kzUpnKe1PZbptGMWbY5qTcwlNuMhVSmgFffShqEnabledDisabledDisabledkey-prefixarn:aws:s3:::destinationbucket`, + destBucket: "destinationbucket", + sameTarget: false, + expectedParsingErr: nil, + expectedValidationErr: errInvalidRuleID, + }, + // 9 invalid priority status in replication config + { + inputConfig: `arn:aws:iam::AcctID:role/role-nameEnabledDisabledDisabledkey-prefixarn:aws:s3:::destinationbucketEnabledDisabledDisabledkey-prefixarn:aws:s3:::destinationbucket`, + destBucket: "destinationbucket", + sameTarget: false, + expectedParsingErr: nil, + expectedValidationErr: errReplicationUniquePriority, + }, + // 10 no rule in replication config + { + inputConfig: `arn:aws:iam::AcctID:role/role-name`, + destBucket: "destinationbucket", + sameTarget: false, + expectedParsingErr: nil, + expectedValidationErr: errReplicationNoRule, + }, + // 11 no destination in replication config + { + inputConfig: `arn:aws:iam::AcctID:role/role-nameEnabledDisabledDisabledkey-prefix`, + destBucket: "destinationbucket", + sameTarget: false, + expectedParsingErr: Errorf("invalid destination '%v'", ""), + expectedValidationErr: nil, + }, + // 12 destination not matching ARN in replication config + { + inputConfig: `arn:aws:iam::AcctID:role/role-nameEnabledDisabledDisabledkey-prefixdestinationbucket2`, + destBucket: "destinationbucket", + sameTarget: false, + expectedParsingErr: fmt.Errorf("invalid destination '%v'", "destinationbucket2"), + expectedValidationErr: nil, + }, + // 13 missing role in config and destination ARN has target ARN + { + inputConfig: `EnabledDisabledDisabledkey-prefixarn:minio:replication::8320b6d18f9032b4700f1f03b50d8d1853de8f22cab86931ee794e12f190852c:destinationbucket`, + // destination bucket in config different from bucket specified + destBucket: "destinationbucket", + sameTarget: false, + expectedParsingErr: nil, + expectedValidationErr: nil, + }, + // 14 role absent in config and destination ARN has target ARN in invalid format + { + inputConfig: `EnabledDisabledDisabledkey-prefixarn:xx:replication::8320b6d18f9032b4700f1f03b50d8d1853de8f22cab86931ee794e12f190852c:destinationbucket`, + // destination bucket in config different from bucket specified + destBucket: "destinationbucket", + sameTarget: false, + expectedParsingErr: fmt.Errorf("invalid destination '%v'", "arn:xx:replication::8320b6d18f9032b4700f1f03b50d8d1853de8f22cab86931ee794e12f190852c:destinationbucket"), + expectedValidationErr: nil, + }, + } + for i, tc := range testCases { + t.Run(fmt.Sprintf("Test %d", i+1), func(t *testing.T) { + cfg, err := ParseConfig(bytes.NewReader([]byte(tc.inputConfig))) + if err != nil && tc.expectedParsingErr != nil && err.Error() != tc.expectedParsingErr.Error() { + t.Fatalf("%d: Expected '%v' during parsing but got '%v'", i+1, tc.expectedParsingErr, err) + } + if err == nil && tc.expectedParsingErr != nil { + t.Fatalf("%d: Expected '%v' during parsing but got '%v'", i+1, tc.expectedParsingErr, err) + } + if tc.expectedParsingErr != nil { + // We already expect a parsing error, + // no need to continue this test. + return + } + err = cfg.Validate(tc.destBucket, tc.sameTarget) + if err != tc.expectedValidationErr { + t.Fatalf("%d: Expected %v during parsing but got %v", i+1, tc.expectedValidationErr, err) + } + }) + } +} + +func TestReplicate(t *testing.T) { + cfgs := []Config{ + { // Config0 - Replication config has no filters, all replication enabled + Rules: []Rule{ + { + Status: Enabled, + Priority: 3, + DeleteMarkerReplication: DeleteMarkerReplication{Status: Enabled}, + DeleteReplication: DeleteReplication{Status: Enabled}, + Filter: Filter{}, + }, + }, + }, + { // Config1 - Replication config has no filters, delete,delete-marker replication disabled + Rules: []Rule{ + { + Status: Enabled, + Priority: 3, + DeleteMarkerReplication: DeleteMarkerReplication{Status: Disabled}, + DeleteReplication: DeleteReplication{Status: Disabled}, + Filter: Filter{}, + }, + }, + }, + { // Config2 - Replication config has filters and more than 1 matching rule, delete,delete-marker replication disabled + Rules: []Rule{ + { + Status: Enabled, + Priority: 2, + DeleteMarkerReplication: DeleteMarkerReplication{Status: Disabled}, + DeleteReplication: DeleteReplication{Status: Enabled}, + Filter: Filter{Prefix: "xy", And: And{}, Tag: Tag{Key: "k1", Value: "v1"}}, + }, + { + Status: Enabled, + Priority: 1, + DeleteMarkerReplication: DeleteMarkerReplication{Status: Enabled}, + DeleteReplication: DeleteReplication{Status: Disabled}, + Filter: Filter{Prefix: "xyz"}, + }, + }, + }, + { // Config3 - Replication config has filters and no overlapping rules + Rules: []Rule{ + { + Status: Enabled, + Priority: 2, + DeleteMarkerReplication: DeleteMarkerReplication{Status: Disabled}, + DeleteReplication: DeleteReplication{Status: Enabled}, + Filter: Filter{Prefix: "xy", And: And{}, Tag: Tag{Key: "k1", Value: "v1"}}, + }, + { + Status: Enabled, + Priority: 1, + DeleteMarkerReplication: DeleteMarkerReplication{Status: Enabled}, + DeleteReplication: DeleteReplication{Status: Disabled}, + Filter: Filter{Prefix: "abc"}, + }, + }, + }, + { // Config4 - Replication config has filters and SourceSelectionCriteria Disabled + Rules: []Rule{ + { + Status: Enabled, + Priority: 2, + DeleteMarkerReplication: DeleteMarkerReplication{Status: Enabled}, + DeleteReplication: DeleteReplication{Status: Enabled}, + SourceSelectionCriteria: SourceSelectionCriteria{ReplicaModifications: ReplicaModifications{Status: Disabled}}, + }, + }, + }, + } + testCases := []struct { + opts ObjectOpts + c Config + expectedResult bool + }{ + // using config 1 - no filters, all replication enabled + {ObjectOpts{}, cfgs[0], false}, // 1. invalid ObjectOpts missing object name + {ObjectOpts{Name: "c1test"}, cfgs[0], true}, // 2. valid ObjectOpts passing empty Filter + {ObjectOpts{Name: "c1test", VersionID: "vid"}, cfgs[0], true}, // 3. valid ObjectOpts passing empty Filter + + {ObjectOpts{Name: "c1test", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[0], true}, // 4. DeleteMarker version replication valid case - matches DeleteMarkerReplication status + {ObjectOpts{Name: "c1test", VersionID: "vid", OpType: DeleteReplicationType}, cfgs[0], true}, // 5. permanent delete of version, matches DeleteReplication status - valid case + {ObjectOpts{Name: "c1test", VersionID: "vid", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[0], true}, // 6. permanent delete of version, matches DeleteReplication status + {ObjectOpts{Name: "c1test", VersionID: "vid", DeleteMarker: true, SSEC: true, OpType: DeleteReplicationType}, cfgs[0], true}, // 7. permanent delete of version + {ObjectOpts{Name: "c1test", DeleteMarker: true, SSEC: true, OpType: DeleteReplicationType}, cfgs[0], true}, // 8. setting DeleteMarker on SSE-C encrypted object + {ObjectOpts{Name: "c1test", SSEC: true}, cfgs[0], true}, // 9. replication of SSE-C encrypted object + + // using config 2 - no filters, only replication of object, metadata enabled + {ObjectOpts{Name: "c2test"}, cfgs[1], true}, // 10. valid ObjectOpts passing empty Filter + {ObjectOpts{Name: "c2test", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[1], false}, // 11. DeleteMarker version replication not allowed due to DeleteMarkerReplication status + {ObjectOpts{Name: "c2test", VersionID: "vid", OpType: DeleteReplicationType}, cfgs[1], false}, // 12. permanent delete of version, disallowed by DeleteReplication status + {ObjectOpts{Name: "c2test", VersionID: "vid", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[1], false}, // 13. permanent delete of DeleteMarker version, disallowed by DeleteReplication status + {ObjectOpts{Name: "c2test", VersionID: "vid", DeleteMarker: true, SSEC: true, OpType: DeleteReplicationType}, cfgs[1], false}, // 14. permanent delete of version, disqualified by SSE-C & DeleteReplication status + {ObjectOpts{Name: "c2test", DeleteMarker: true, SSEC: true, OpType: DeleteReplicationType}, cfgs[1], false}, // 15. setting DeleteMarker on SSE-C encrypted object, disqualified by SSE-C & DeleteMarkerReplication status + {ObjectOpts{Name: "c2test", SSEC: true}, cfgs[1], true}, // 16. replication of SSE-C encrypted object + // using config 2 - has more than one rule with overlapping prefixes + {ObjectOpts{Name: "xy/c3test", UserTags: "k1=v1"}, cfgs[2], true}, // 17. matches rule 1 for replication of content/metadata + {ObjectOpts{Name: "xyz/c3test", UserTags: "k1=v1"}, cfgs[2], true}, // 18. matches rule 1 for replication of content/metadata + {ObjectOpts{Name: "xyz/c3test", UserTags: "k1=v1", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[2], false}, // 19. matches rule 1 - DeleteMarker replication disallowed by rule + {ObjectOpts{Name: "xyz/c3test", UserTags: "k1=v1", DeleteMarker: true, VersionID: "vid", OpType: DeleteReplicationType}, cfgs[2], true}, // 20. matches rule 1 - DeleteReplication allowed by rule for permanent delete of DeleteMarker + {ObjectOpts{Name: "xyz/c3test", UserTags: "k1=v1", VersionID: "vid", OpType: DeleteReplicationType}, cfgs[2], true}, // 21. matches rule 1 - DeleteReplication allowed by rule for permanent delete of version + {ObjectOpts{Name: "xyz/c3test"}, cfgs[2], true}, // 22. matches rule 2 for replication of content/metadata + {ObjectOpts{Name: "xy/c3test", UserTags: "k1=v2"}, cfgs[2], false}, // 23. does not match rule1 because tag value does not pass filter + {ObjectOpts{Name: "xyz/c3test", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[2], true}, // 24. matches rule 2 - DeleteMarker replication allowed by rule + {ObjectOpts{Name: "xyz/c3test", DeleteMarker: true, VersionID: "vid", OpType: DeleteReplicationType}, cfgs[2], false}, // 25. matches rule 2 - DeleteReplication disallowed by rule for permanent delete of DeleteMarker + {ObjectOpts{Name: "xyz/c3test", VersionID: "vid", OpType: DeleteReplicationType}, cfgs[2], false}, // 26. matches rule 1 - DeleteReplication disallowed by rule for permanent delete of version + {ObjectOpts{Name: "abc/c3test"}, cfgs[2], false}, // 27. matches no rule because object prefix does not match + + // using config 3 - has no overlapping rules + {ObjectOpts{Name: "xy/c4test", UserTags: "k1=v1"}, cfgs[3], true}, // 28. matches rule 1 for replication of content/metadata + {ObjectOpts{Name: "xa/c4test", UserTags: "k1=v1"}, cfgs[3], false}, // 29. no rule match object prefix not in rules + {ObjectOpts{Name: "xyz/c4test", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[3], false}, // 30. rule 1 not matched because of tags filter + {ObjectOpts{Name: "xyz/c4test", UserTags: "k1=v1", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[3], false}, // 31. matches rule 1 - DeleteMarker replication disallowed by rule + {ObjectOpts{Name: "xyz/c4test", UserTags: "k1=v1", DeleteMarker: true, VersionID: "vid", OpType: DeleteReplicationType}, cfgs[3], true}, // 32. matches rule 1 - DeleteReplication allowed by rule for permanent delete of DeleteMarker + {ObjectOpts{Name: "xyz/c4test", UserTags: "k1=v1", VersionID: "vid", OpType: DeleteReplicationType}, cfgs[3], true}, // 33. matches rule 1 - DeleteReplication allowed by rule for permanent delete of version + {ObjectOpts{Name: "abc/c4test"}, cfgs[3], true}, // 34. matches rule 2 for replication of content/metadata + {ObjectOpts{Name: "abc/c4test", UserTags: "k1=v2"}, cfgs[3], true}, // 35. matches rule 2 for replication of content/metadata + {ObjectOpts{Name: "abc/c4test", DeleteMarker: true, OpType: DeleteReplicationType}, cfgs[3], true}, // 36. matches rule 2 - DeleteMarker replication allowed by rule + {ObjectOpts{Name: "abc/c4test", DeleteMarker: true, VersionID: "vid", OpType: DeleteReplicationType}, cfgs[3], false}, // 37. matches rule 2 - DeleteReplication disallowed by rule for permanent delete of DeleteMarker + {ObjectOpts{Name: "abc/c4test", VersionID: "vid", OpType: DeleteReplicationType}, cfgs[3], false}, // 38. matches rule 2 - DeleteReplication disallowed by rule for permanent delete of version + // using config 4 - with replica modification sync disabled. + {ObjectOpts{Name: "xy/c5test", UserTags: "k1=v1", Replica: true}, cfgs[4], false}, // 39. replica syncing disabled, this object is a replica + {ObjectOpts{Name: "xa/c5test", UserTags: "k1=v1", Replica: false}, cfgs[4], true}, // 40. replica syncing disabled, this object is NOT a replica + } + + for _, testCase := range testCases { + t.Run(testCase.opts.Name, func(t *testing.T) { + result := testCase.c.Replicate(testCase.opts) + if result != testCase.expectedResult { + t.Errorf("expected: %v, got: %v", testCase.expectedResult, result) + } + }) + } +} + +func TestHasActiveRules(t *testing.T) { + testCases := []struct { + inputConfig string + prefix string + expectedNonRec bool + expectedRec bool + }{ + // case 1 - only one rule which is in Disabled status + { + inputConfig: `arn:aws:iam::AcctID:role/role-nameDisabledDisabledDisabledkey-prefixarn:aws:s3:::destinationbucket`, + prefix: "miss/prefix", + expectedNonRec: false, + expectedRec: false, + }, + // case 2 - only one rule which matches prefix filter + { + inputConfig: `arn:aws:iam::AcctID:role/role-nameEnabledDisabledDisabledkey/prefixarn:aws:s3:::destinationbucket`, + prefix: "key/prefix1", + expectedNonRec: true, + expectedRec: true, + }, + // case 3 - empty prefix + { + inputConfig: `arn:aws:iam::AcctID:role/role-nameEnabledDisabledDisabledarn:aws:s3:::destinationbucket`, + prefix: "key-prefix", + expectedNonRec: true, + expectedRec: true, + }, + // case 4 - has Filter based on prefix + { + inputConfig: `arn:aws:iam::AcctID:role/role-nameEnabledDisabledDisabledtestdir/dir1/arn:aws:s3:::destinationbucket`, + prefix: "testdir/", + expectedNonRec: false, + expectedRec: true, + }, + // case 5 - has filter with prefix and tags, here we are not matching on tags + { + inputConfig: `arn:aws:iam::AcctID:role/role-nameEnabledDisabledDisabled + key-prefixkey1value1key2value2arn:aws:s3:::destinationbucket`, + prefix: "testdir/", + expectedNonRec: true, + expectedRec: true, + }, + } + + for i, tc := range testCases { + t.Run(fmt.Sprintf("Test_%d", i+1), func(t *testing.T) { + cfg, err := ParseConfig(bytes.NewReader([]byte(tc.inputConfig))) + if err != nil { + t.Fatalf("Got unexpected error: %v", err) + } + if got := cfg.HasActiveRules(tc.prefix, false); got != tc.expectedNonRec { + t.Fatalf("Expected result with recursive set to false: `%v`, got: `%v`", tc.expectedNonRec, got) + } + if got := cfg.HasActiveRules(tc.prefix, true); got != tc.expectedRec { + t.Fatalf("Expected result with recursive set to true: `%v`, got: `%v`", tc.expectedRec, got) + } + }) + } +} + +func TestFilterActionableRules(t *testing.T) { + testCases := []struct { + inputConfig string + prefix string + ExpectedRules []Rule + }{ + // case 1 - only one rule + { + inputConfig: `arn:aws:iam::AcctID:role/role-nameEnabledDisabledDisabledprefix1arn:minio:replication:xxx::destinationbucket`, + prefix: "prefix", + ExpectedRules: []Rule{{Status: Enabled, Priority: 1, DeleteMarkerReplication: DeleteMarkerReplication{Status: Enabled}, DeleteReplication: DeleteReplication{Status: Disabled}, Destination: Destination{Bucket: "destinationbucket", ARN: "arn:minio:replication:xxx::destinationbucket"}}}, + }, + // case 2 - multiple rules for same target, overlapping rules with different priority + { + inputConfig: `arn:aws:iam::AcctID:role/role-nameEnabledDisabledDisabledprefix3arn:minio:replication:xxx::destinationbucketEnabledDisabledDisabledprefix1arn:minio:replication:xxx::destinationbucket`, + prefix: "prefix", + ExpectedRules: []Rule{ + {Status: Enabled, Priority: 3, DeleteMarkerReplication: DeleteMarkerReplication{Status: Enabled}, DeleteReplication: DeleteReplication{Status: Disabled}, Destination: Destination{Bucket: "destinationbucket", ARN: "arn:minio:replication:xxx::destinationbucket"}}, + {Status: Enabled, Priority: 1, DeleteMarkerReplication: DeleteMarkerReplication{Status: Enabled}, DeleteReplication: DeleteReplication{Status: Disabled}, Destination: Destination{Bucket: "destinationbucket", ARN: "arn:minio:replication:xxx::destinationbucket"}}, + }, + }, + // case 3 - multiple rules for different target, overlapping rules on a target + { + inputConfig: `arn:aws:iam::AcctID:role/role-nameEnabledDisabledDisabledprefix2arn:minio:replication:xxx::destinationbucket2EnabledDisabledDisabledprefix4arn:minio:replication:xxx::destinationbucket2EnabledDisabledDisabledprefix3arn:minio:replication:xxx::destinationbucketEnabledDisabledDisabledprefix1arn:minio:replication:xxx::destinationbucket`, + prefix: "prefix", + ExpectedRules: []Rule{ + {Status: Enabled, Priority: 4, DeleteMarkerReplication: DeleteMarkerReplication{Status: Enabled}, DeleteReplication: DeleteReplication{Status: Disabled}, Destination: Destination{Bucket: "destinationbucket2", ARN: "arn:minio:replication:xxx::destinationbucket2"}}, + {Status: Enabled, Priority: 2, DeleteMarkerReplication: DeleteMarkerReplication{Status: Enabled}, DeleteReplication: DeleteReplication{Status: Disabled}, Destination: Destination{Bucket: "destinationbucket2", ARN: "arn:minio:replication:xxx::destinationbucket2"}}, + {Status: Enabled, Priority: 3, DeleteMarkerReplication: DeleteMarkerReplication{Status: Enabled}, DeleteReplication: DeleteReplication{Status: Disabled}, Destination: Destination{Bucket: "destinationbucket", ARN: "arn:minio:replication:xxx::destinationbucket"}}, + {Status: Enabled, Priority: 1, DeleteMarkerReplication: DeleteMarkerReplication{Status: Enabled}, DeleteReplication: DeleteReplication{Status: Disabled}, Destination: Destination{Bucket: "destinationbucket", ARN: "arn:minio:replication:xxx::destinationbucket"}}, + }, + }, + } + for _, tc := range testCases { + cfg, err := ParseConfig(bytes.NewReader([]byte(tc.inputConfig))) + if err != nil { + t.Fatalf("Got unexpected error: %v", err) + } + got := cfg.FilterActionableRules(ObjectOpts{Name: tc.prefix}) + if len(got) != len(tc.ExpectedRules) { + t.Fatalf("Expected matching number of actionable rules: `%v`, got: `%v`", tc.ExpectedRules, got) + } + for i := range got { + if got[i].Destination.ARN != tc.ExpectedRules[i].Destination.ARN || got[i].Priority != tc.ExpectedRules[i].Priority { + t.Fatalf("Expected order of filtered rules to be identical: `%v`, got: `%v`", tc.ExpectedRules, got) + } + } + } +} diff --git a/internal/bucket/replication/rule.go b/internal/bucket/replication/rule.go new file mode 100644 index 00000000..347dfe28 --- /dev/null +++ b/internal/bucket/replication/rule.go @@ -0,0 +1,256 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package replication + +import ( + "bytes" + "encoding/xml" +) + +// Status represents Enabled/Disabled status +type Status string + +// Supported status types +const ( + Enabled Status = "Enabled" + Disabled Status = "Disabled" +) + +// DeleteMarkerReplication - whether delete markers are replicated - https://docs.aws.amazon.com/AmazonS3/latest/dev/replication-add-config.html +type DeleteMarkerReplication struct { + Status Status `xml:"Status"` // should be set to "Disabled" by default +} + +// IsEmpty returns true if DeleteMarkerReplication is not set +func (d DeleteMarkerReplication) IsEmpty() bool { + return len(d.Status) == 0 +} + +// Validate validates whether the status is disabled. +func (d DeleteMarkerReplication) Validate() error { + if d.IsEmpty() { + return errDeleteMarkerReplicationMissing + } + if d.Status != Disabled && d.Status != Enabled { + return errInvalidDeleteMarkerReplicationStatus + } + return nil +} + +// DeleteReplication - whether versioned deletes are replicated - this is a MinIO only +// extension. +type DeleteReplication struct { + Status Status `xml:"Status"` // should be set to "Disabled" by default +} + +// IsEmpty returns true if DeleteReplication is not set +func (d DeleteReplication) IsEmpty() bool { + return len(d.Status) == 0 +} + +// Validate validates whether the status is disabled. +func (d DeleteReplication) Validate() error { + if d.IsEmpty() { + return errDeleteReplicationMissing + } + if d.Status != Disabled && d.Status != Enabled { + return errInvalidDeleteReplicationStatus + } + return nil +} + +// UnmarshalXML - decodes XML data. +func (d *DeleteReplication) UnmarshalXML(dec *xml.Decoder, start xml.StartElement) (err error) { + // Make subtype to avoid recursive UnmarshalXML(). + type deleteReplication DeleteReplication + drep := deleteReplication{} + + if err := dec.DecodeElement(&drep, &start); err != nil { + return err + } + if len(drep.Status) == 0 { + drep.Status = Disabled + } + d.Status = drep.Status + return nil +} + +// ExistingObjectReplication - whether existing object replication is enabled +type ExistingObjectReplication struct { + Status Status `xml:"Status"` // should be set to "Disabled" by default +} + +// IsEmpty returns true if ExistingObjectReplication is not set +func (e ExistingObjectReplication) IsEmpty() bool { + return len(e.Status) == 0 +} + +// Validate validates whether the status is disabled. +func (e ExistingObjectReplication) Validate() error { + if e.IsEmpty() { + return nil + } + if e.Status != Disabled && e.Status != Enabled { + return errInvalidExistingObjectReplicationStatus + } + return nil +} + +// UnmarshalXML - decodes XML data. Default to Disabled unless specified +func (e *ExistingObjectReplication) UnmarshalXML(dec *xml.Decoder, start xml.StartElement) (err error) { + // Make subtype to avoid recursive UnmarshalXML(). + type existingObjectReplication ExistingObjectReplication + erep := existingObjectReplication{} + + if err := dec.DecodeElement(&erep, &start); err != nil { + return err + } + if len(erep.Status) == 0 { + erep.Status = Disabled + } + e.Status = erep.Status + return nil +} + +// Rule - a rule for replication configuration. +type Rule struct { + XMLName xml.Name `xml:"Rule" json:"Rule"` + ID string `xml:"ID,omitempty" json:"ID,omitempty"` + Status Status `xml:"Status" json:"Status"` + Priority int `xml:"Priority" json:"Priority"` + DeleteMarkerReplication DeleteMarkerReplication `xml:"DeleteMarkerReplication" json:"DeleteMarkerReplication"` + // MinIO extension to replicate versioned deletes + DeleteReplication DeleteReplication `xml:"DeleteReplication" json:"DeleteReplication"` + Destination Destination `xml:"Destination" json:"Destination"` + SourceSelectionCriteria SourceSelectionCriteria `xml:"SourceSelectionCriteria" json:"SourceSelectionCriteria"` + Filter Filter `xml:"Filter" json:"Filter"` + ExistingObjectReplication ExistingObjectReplication `xml:"ExistingObjectReplication,omitempty" json:"ExistingObjectReplication"` +} + +var ( + errInvalidRuleID = Errorf("ID must be less than 255 characters") + errEmptyRuleStatus = Errorf("Status should not be empty") + errInvalidRuleStatus = Errorf("Status must be set to either Enabled or Disabled") + errDeleteMarkerReplicationMissing = Errorf("DeleteMarkerReplication must be specified") + errPriorityMissing = Errorf("Priority must be specified") + errInvalidDeleteMarkerReplicationStatus = Errorf("Delete marker replication status is invalid") + errDestinationSourceIdentical = Errorf("Destination bucket cannot be the same as the source bucket.") + errDeleteReplicationMissing = Errorf("Delete replication must be specified") + errInvalidDeleteReplicationStatus = Errorf("Delete replication is either enable|disable") + errInvalidExistingObjectReplicationStatus = Errorf("Existing object replication status is invalid") + errTagsDeleteMarkerReplicationDisallowed = Errorf("Delete marker replication is not supported if any Tag filter is specified") +) + +// validateID - checks if ID is valid or not. +func (r Rule) validateID() error { + // cannot be longer than 255 characters + if len(r.ID) > 255 { + return errInvalidRuleID + } + return nil +} + +// validateStatus - checks if status is valid or not. +func (r Rule) validateStatus() error { + // Status can't be empty + if len(r.Status) == 0 { + return errEmptyRuleStatus + } + + // Status must be one of Enabled or Disabled + if r.Status != Enabled && r.Status != Disabled { + return errInvalidRuleStatus + } + return nil +} + +func (r Rule) validateFilter() error { + return r.Filter.Validate() +} + +// Prefix - a rule can either have prefix under or under +// . This method returns the prefix from the +// location where it is available +func (r Rule) Prefix() string { + if r.Filter.Prefix != "" { + return r.Filter.Prefix + } + return r.Filter.And.Prefix +} + +// Tags - a rule can either have tag under or under +// . This method returns all the tags from the +// rule in the format tag1=value1&tag2=value2 +func (r Rule) Tags() string { + if !r.Filter.Tag.IsEmpty() { + return r.Filter.Tag.String() + } + if len(r.Filter.And.Tags) != 0 { + var buf bytes.Buffer + for _, t := range r.Filter.And.Tags { + if buf.Len() > 0 { + buf.WriteString("&") + } + buf.WriteString(t.String()) + } + return buf.String() + } + return "" +} + +// Validate - validates the rule element +func (r Rule) Validate(bucket string, sameTarget bool) error { + if err := r.validateID(); err != nil { + return err + } + if err := r.validateStatus(); err != nil { + return err + } + if err := r.validateFilter(); err != nil { + return err + } + if err := r.DeleteMarkerReplication.Validate(); err != nil { + return err + } + if err := r.DeleteReplication.Validate(); err != nil { + return err + } + if err := r.SourceSelectionCriteria.Validate(); err != nil { + return err + } + + if r.Priority < 0 { + return errPriorityMissing + } + if r.Destination.Bucket == bucket && sameTarget { + return errDestinationSourceIdentical + } + if !r.Filter.Tag.IsEmpty() && (r.DeleteMarkerReplication.Status == Enabled) { + return errTagsDeleteMarkerReplicationDisallowed + } + return r.ExistingObjectReplication.Validate() +} + +// MetadataReplicate returns true if object is not a replica or in the case of replicas, +// replica modification sync is enabled. +func (r Rule) MetadataReplicate(obj ObjectOpts) bool { + if !obj.Replica { + return true + } + return obj.Replica && r.SourceSelectionCriteria.ReplicaModifications.Status == Enabled +} diff --git a/internal/bucket/replication/rule_test.go b/internal/bucket/replication/rule_test.go new file mode 100644 index 00000000..32722c97 --- /dev/null +++ b/internal/bucket/replication/rule_test.go @@ -0,0 +1,70 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package replication + +import ( + "bytes" + "fmt" + "testing" +) + +func TestMetadataReplicate(t *testing.T) { + testCases := []struct { + inputConfig string + opts ObjectOpts + expectedResult bool + }{ + // case 1 - rule with replica modification enabled; not a replica + { + inputConfig: `arn:aws:iam::AcctID:role/role-nameEnabledDisabledDisabledkey-prefixarn:aws:s3:::destinationbucketEnabled`, + opts: ObjectOpts{Name: "c1test", DeleteMarker: false, OpType: ObjectReplicationType, Replica: false}, // 1. Replica mod sync enabled; not a replica + expectedResult: true, + }, + // case 2 - rule with replica modification disabled; a replica + { + inputConfig: `arn:aws:iam::AcctID:role/role-nameEnabledDisabledDisabledkey-prefixarn:aws:s3:::destinationbucketDisabled`, + opts: ObjectOpts{Name: "c2test", DeleteMarker: false, OpType: ObjectReplicationType, Replica: true}, // 1. Replica mod sync enabled; a replica + expectedResult: false, + }, + // case 3 - rule with replica modification disabled; not a replica + { + inputConfig: `arn:aws:iam::AcctID:role/role-nameEnabledDisabledDisabledkey-prefixarn:aws:s3:::destinationbucketDisabled`, + opts: ObjectOpts{Name: "c2test", DeleteMarker: false, OpType: ObjectReplicationType, Replica: false}, // 1. Replica mod sync disabled; not a replica + expectedResult: true, + }, + + // case 4 - rule with replica modification enabled; a replica + { + inputConfig: `arn:aws:iam::AcctID:role/role-nameEnabledDisabledDisabledkey-prefixarn:aws:s3:::destinationbucketEnabled`, + opts: ObjectOpts{Name: "c2test", DeleteMarker: false, OpType: MetadataReplicationType, Replica: true}, // 1. Replica mod sync enabled; a replica + expectedResult: true, + }, + } + + for i, tc := range testCases { + t.Run(fmt.Sprintf("Test_%d", i+1), func(t *testing.T) { + cfg, err := ParseConfig(bytes.NewReader([]byte(tc.inputConfig))) + if err != nil { + t.Fatalf("Got unexpected error: %v", err) + } + if got := cfg.Rules[0].MetadataReplicate(tc.opts); got != tc.expectedResult { + t.Fatalf("Expected result with recursive set to false: `%v`, got: `%v`", tc.expectedResult, got) + } + }) + } +} diff --git a/internal/bucket/replication/sourceselectioncriteria.go b/internal/bucket/replication/sourceselectioncriteria.go new file mode 100644 index 00000000..19768f52 --- /dev/null +++ b/internal/bucket/replication/sourceselectioncriteria.go @@ -0,0 +1,76 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package replication + +import ( + "encoding/xml" +) + +// ReplicaModifications specifies if replica modification sync is enabled +type ReplicaModifications struct { + Status Status `xml:"Status" json:"Status"` +} + +// SourceSelectionCriteria - specifies additional source selection criteria in ReplicationConfiguration. +type SourceSelectionCriteria struct { + ReplicaModifications ReplicaModifications `xml:"ReplicaModifications" json:"ReplicaModifications"` +} + +// IsValid - checks whether SourceSelectionCriteria is valid or not. +func (s SourceSelectionCriteria) IsValid() bool { + return s.ReplicaModifications.Status == Enabled || s.ReplicaModifications.Status == Disabled +} + +// Validate source selection criteria +func (s SourceSelectionCriteria) Validate() error { + if (s == SourceSelectionCriteria{}) { + return nil + } + if !s.IsValid() { + return errInvalidSourceSelectionCriteria + } + return nil +} + +// UnmarshalXML - decodes XML data. +func (s *SourceSelectionCriteria) UnmarshalXML(dec *xml.Decoder, start xml.StartElement) (err error) { + // Make subtype to avoid recursive UnmarshalXML(). + type sourceSelectionCriteria SourceSelectionCriteria + ssc := sourceSelectionCriteria{} + if err := dec.DecodeElement(&ssc, &start); err != nil { + return err + } + if len(ssc.ReplicaModifications.Status) == 0 { + ssc.ReplicaModifications.Status = Enabled + } + *s = SourceSelectionCriteria(ssc) + return nil +} + +// MarshalXML - encodes to XML data. +func (s SourceSelectionCriteria) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if err := e.EncodeToken(start); err != nil { + return err + } + if s.IsValid() { + if err := e.EncodeElement(s.ReplicaModifications, xml.StartElement{Name: xml.Name{Local: "ReplicaModifications"}}); err != nil { + return err + } + } + return e.EncodeToken(xml.EndElement{Name: start.Name}) +} diff --git a/internal/bucket/replication/tag.go b/internal/bucket/replication/tag.go new file mode 100644 index 00000000..e60989a5 --- /dev/null +++ b/internal/bucket/replication/tag.go @@ -0,0 +1,57 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package replication + +import ( + "encoding/xml" + "unicode/utf8" +) + +// Tag - a tag for a replication configuration Rule filter. +type Tag struct { + XMLName xml.Name `xml:"Tag" json:"Tag"` + Key string `xml:"Key,omitempty" json:"Key,omitempty"` + Value string `xml:"Value,omitempty" json:"Value,omitempty"` +} + +var ( + errInvalidTagKey = Errorf("The TagKey you have provided is invalid") + errInvalidTagValue = Errorf("The TagValue you have provided is invalid") +) + +func (tag Tag) String() string { + return tag.Key + "=" + tag.Value +} + +// IsEmpty returns whether this tag is empty or not. +func (tag Tag) IsEmpty() bool { + return tag.Key == "" +} + +// Validate checks this tag. +func (tag Tag) Validate() error { + if len(tag.Key) == 0 || utf8.RuneCountInString(tag.Key) > 128 { + return errInvalidTagKey + } + + if utf8.RuneCountInString(tag.Value) > 256 { + return errInvalidTagValue + } + + return nil +} diff --git a/internal/bucket/versioning/error.go b/internal/bucket/versioning/error.go new file mode 100644 index 00000000..20bb4caa --- /dev/null +++ b/internal/bucket/versioning/error.go @@ -0,0 +1,45 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package versioning + +import ( + "fmt" +) + +// Error is the generic type for any error happening during tag +// parsing. +type Error struct { + err error +} + +// Errorf - formats according to a format specifier and returns +// the string as a value that satisfies error of type tagging.Error +func Errorf(format string, a ...any) error { + return Error{err: fmt.Errorf(format, a...)} +} + +// Unwrap the internal error. +func (e Error) Unwrap() error { return e.err } + +// Error 'error' compatible method. +func (e Error) Error() string { + if e.err == nil { + return "versioning: cause " + } + return e.err.Error() +} diff --git a/internal/bucket/versioning/versioning.go b/internal/bucket/versioning/versioning.go new file mode 100644 index 00000000..3647f908 --- /dev/null +++ b/internal/bucket/versioning/versioning.go @@ -0,0 +1,167 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package versioning + +import ( + "encoding/xml" + "io" + "strings" + + "github.com/minio/pkg/v3/wildcard" +) + +// State - enabled/disabled/suspended states +// for multifactor and status of versioning. +type State string + +// Various supported states +const ( + Enabled State = "Enabled" + // Disabled State = "Disabled" only used by MFA Delete not supported yet. + Suspended State = "Suspended" +) + +var ( + errExcludedPrefixNotSupported = Errorf("excluded prefixes extension supported only when versioning is enabled") + errTooManyExcludedPrefixes = Errorf("too many excluded prefixes") +) + +// ExcludedPrefix - holds individual prefixes excluded from being versioned. +type ExcludedPrefix struct { + Prefix string +} + +// Versioning - Configuration for bucket versioning. +type Versioning struct { + XMLNS string `xml:"xmlns,attr,omitempty"` + XMLName xml.Name `xml:"VersioningConfiguration"` + // MFADelete State `xml:"MFADelete,omitempty"` // not supported yet. + Status State `xml:"Status,omitempty"` + // MinIO extension - allows selective, prefix-level versioning exclusion. + // Requires versioning to be enabled + ExcludedPrefixes []ExcludedPrefix `xml:",omitempty"` + ExcludeFolders bool `xml:",omitempty"` +} + +// Validate - validates the versioning configuration +func (v Versioning) Validate() error { + // Not supported yet + // switch v.MFADelete { + // case Enabled, Disabled: + // default: + // return Errorf("unsupported MFADelete state %s", v.MFADelete) + // } + switch v.Status { + case Enabled: + const maxExcludedPrefixes = 10 + if len(v.ExcludedPrefixes) > maxExcludedPrefixes { + return errTooManyExcludedPrefixes + } + + case Suspended: + if len(v.ExcludedPrefixes) > 0 { + return errExcludedPrefixNotSupported + } + default: + return Errorf("unsupported Versioning status %s", v.Status) + } + return nil +} + +// Enabled - returns true if versioning is enabled +func (v Versioning) Enabled() bool { + return v.Status == Enabled +} + +// Versioned returns if 'prefix' has versioning enabled or suspended. +func (v Versioning) Versioned(prefix string) bool { + return v.PrefixEnabled(prefix) || v.PrefixSuspended(prefix) +} + +// PrefixEnabled - returns true if versioning is enabled at the bucket and given +// prefix, false otherwise. +func (v Versioning) PrefixEnabled(prefix string) bool { + if v.Status != Enabled { + return false + } + + if prefix == "" { + return true + } + if v.ExcludeFolders && strings.HasSuffix(prefix, "/") { + return false + } + + for _, sprefix := range v.ExcludedPrefixes { + // Note: all excluded prefix patterns end with `/` (See Validate) + sprefix.Prefix += "*" + + if matched := wildcard.MatchSimple(sprefix.Prefix, prefix); matched { + return false + } + } + return true +} + +// Suspended - returns true if versioning is suspended +func (v Versioning) Suspended() bool { + return v.Status == Suspended +} + +// PrefixSuspended - returns true if versioning is suspended at the bucket level +// or suspended on the given prefix. +func (v Versioning) PrefixSuspended(prefix string) bool { + if v.Status == Suspended { + return true + } + if v.Status == Enabled { + if prefix == "" { + return false + } + if v.ExcludeFolders && strings.HasSuffix(prefix, "/") { + return true + } + + for _, sprefix := range v.ExcludedPrefixes { + // Note: all excluded prefix patterns end with `/` (See Validate) + sprefix.Prefix += "*" + if matched := wildcard.MatchSimple(sprefix.Prefix, prefix); matched { + return true + } + } + } + return false +} + +// PrefixesExcluded returns true if v contains one or more excluded object +// prefixes or if ExcludeFolders is true. +func (v Versioning) PrefixesExcluded() bool { + return len(v.ExcludedPrefixes) > 0 || v.ExcludeFolders +} + +// ParseConfig - parses data in given reader to VersioningConfiguration. +func ParseConfig(reader io.Reader) (*Versioning, error) { + var v Versioning + if err := xml.NewDecoder(reader).Decode(&v); err != nil { + return nil, err + } + if err := v.Validate(); err != nil { + return nil, err + } + return &v, nil +} diff --git a/internal/bucket/versioning/versioning_test.go b/internal/bucket/versioning/versioning_test.go new file mode 100644 index 00000000..718da5c0 --- /dev/null +++ b/internal/bucket/versioning/versioning_test.go @@ -0,0 +1,241 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package versioning + +import ( + "encoding/xml" + "strings" + "testing" +) + +func TestParseConfig(t *testing.T) { + testcases := []struct { + input string + err error + excludedPrefixes []string + excludeFolders bool + }{ + { + input: ` + Enabled + `, + err: nil, + }, + { + input: ` + Enabled + + path/to/my/workload/_staging/ + + + path/to/my/workload/_temporary/ + + `, + err: nil, + excludedPrefixes: []string{"path/to/my/workload/_staging/", "path/to/my/workload/_temporary/"}, + }, + { + input: ` + Suspended + + path/to/my/workload/_staging + + `, + err: errExcludedPrefixNotSupported, + }, + { + input: ` + Enabled + + path/to/my/workload/_staging/ab/ + + + path/to/my/workload/_staging/cd/ + + + path/to/my/workload/_staging/ef/ + + + path/to/my/workload/_staging/gh/ + + + path/to/my/workload/_staging/ij/ + + + path/to/my/workload/_staging/kl/ + + + path/to/my/workload/_staging/mn/ + + + path/to/my/workload/_staging/op/ + + + path/to/my/workload/_staging/qr/ + + + path/to/my/workload/_staging/st/ + + + path/to/my/workload/_staging/uv/ + + `, + err: errTooManyExcludedPrefixes, + }, + { + input: ` + Enabled + true + + path/to/my/workload/_staging/ + + + path/to/my/workload/_temporary/ + + `, + err: nil, + excludedPrefixes: []string{"path/to/my/workload/_staging/", "path/to/my/workload/_temporary/"}, + excludeFolders: true, + }, + } + + for i, tc := range testcases { + var v *Versioning + var err error + v, err = ParseConfig(strings.NewReader(tc.input)) + if tc.err != err { + t.Fatalf("Test %d: expected %v but got %v", i+1, tc.err, err) + } + if err != nil { + if tc.err == nil { + t.Fatalf("Test %d: failed due to %v", i+1, err) + } + } else { + if err := v.Validate(); tc.err != err { + t.Fatalf("Test %d: validation failed due to %v", i+1, err) + } + if len(tc.excludedPrefixes) > 0 { + var mismatch bool + if len(v.ExcludedPrefixes) != len(tc.excludedPrefixes) { + t.Fatalf("Test %d: Expected length of excluded prefixes %d but got %d", i+1, len(tc.excludedPrefixes), len(v.ExcludedPrefixes)) + } + var i int + var eprefix string + for i, eprefix = range tc.excludedPrefixes { + if eprefix != v.ExcludedPrefixes[i].Prefix { + mismatch = true + break + } + } + if mismatch { + t.Fatalf("Test %d: Expected excluded prefix %s but got %s", i+1, tc.excludedPrefixes[i], v.ExcludedPrefixes[i].Prefix) + } + } + if tc.excludeFolders != v.ExcludeFolders { + t.Fatalf("Test %d: Expected ExcludeFoldersr=%v but got %v", i+1, tc.excludeFolders, v.ExcludeFolders) + } + } + } +} + +func TestMarshalXML(t *testing.T) { + // Validates if Versioning with no excluded prefixes omits + // ExcludedPrefixes tags + v := Versioning{ + Status: Enabled, + } + buf, err := xml.Marshal(v) + if err != nil { + t.Fatalf("Failed to marshal %v: %v", v, err) + } + + str := string(buf) + if strings.Contains(str, "ExcludedPrefixes") { + t.Fatalf("XML shouldn't contain ExcludedPrefixes tag - %s", str) + } +} + +func TestVersioningZero(t *testing.T) { + var v Versioning + if v.Enabled() { + t.Fatalf("Expected to be disabled but got enabled") + } + if v.Suspended() { + t.Fatalf("Expected to be disabled but got suspended") + } +} + +func TestExcludeFolders(t *testing.T) { + v := Versioning{ + Status: Enabled, + ExcludeFolders: true, + } + testPrefixes := []string{"jobs/output/_temporary/", "jobs/output/", "jobs/"} + for i, prefix := range testPrefixes { + if v.PrefixEnabled(prefix) || !v.PrefixSuspended(prefix) { + t.Fatalf("Test %d: Expected versioning to be excluded for %s", i+1, prefix) + } + } + + // Test applicability for regular objects + if prefix := "prefix-1/obj-1"; !v.PrefixEnabled(prefix) || v.PrefixSuspended(prefix) { + t.Fatalf("Expected versioning to be enabled for %s", prefix) + } + + // Test when ExcludeFolders is disabled + v.ExcludeFolders = false + for i, prefix := range testPrefixes { + if !v.PrefixEnabled(prefix) || v.PrefixSuspended(prefix) { + t.Fatalf("Test %d: Expected versioning to be enabled for %s", i+1, prefix) + } + } +} + +func TestExcludedPrefixesMatch(t *testing.T) { + v := Versioning{ + Status: Enabled, + ExcludedPrefixes: []ExcludedPrefix{{"*/_temporary/"}}, + } + + if err := v.Validate(); err != nil { + t.Fatalf("Invalid test versioning config %v: %v", v, err) + } + tests := []struct { + prefix string + excluded bool + }{ + { + prefix: "app1-jobs/output/_temporary/attempt1/data.csv", + excluded: true, + }, + { + prefix: "app1-jobs/output/final/attempt1/data.csv", + excluded: false, + }, + } + + for i, test := range tests { + if v.PrefixSuspended(test.prefix) != test.excluded { + if test.excluded { + t.Fatalf("Test %d: Expected prefix %s to be excluded from versioning", i+1, test.prefix) + } else { + t.Fatalf("Test %d: Expected prefix %s to have versioning enabled", i+1, test.prefix) + } + } + } +} diff --git a/internal/cachevalue/cache.go b/internal/cachevalue/cache.go new file mode 100644 index 00000000..346e0a5f --- /dev/null +++ b/internal/cachevalue/cache.go @@ -0,0 +1,156 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cachevalue + +import ( + "context" + "sync" + "sync/atomic" + "time" +) + +// Opts contains options for the cache. +type Opts struct { + // When set to true, return the last cached value + // even if updating the value errors out. + // Returns the last good value AND the error. + ReturnLastGood bool + + // If NoWait is set, Get() will return the last good value, + // if TTL has expired but 2x TTL has not yet passed, + // but will fetch a new value in the background. + NoWait bool +} + +// Cache contains a synchronized value that is considered valid +// for a specific amount of time. +// An Update function must be set to provide an updated value when needed. +type Cache[T any] struct { + // updateFn must return an updated value. + // If an error is returned the cached value is not set. + // Only one caller will call this function at any time, others will be blocking. + // The returned value can no longer be modified once returned. + // Should be set before calling Get(). + updateFn func(ctx context.Context) (T, error) + + // ttl for a cached value. + ttl time.Duration + + opts Opts + + // Once can be used to initialize values for lazy initialization. + // Should be set before calling Get(). + Once sync.Once + + // Managed values. + val atomic.Pointer[T] + lastUpdateMs atomic.Int64 + updating sync.Mutex +} + +// New allocates a new cached value instance. Tt must be initialized with +// `.TnitOnce`. +func New[T any]() *Cache[T] { + return &Cache[T]{} +} + +// NewFromFunc allocates a new cached value instance and initializes it with an +// update function, making it ready for use. +func NewFromFunc[T any](ttl time.Duration, opts Opts, update func(ctx context.Context) (T, error)) *Cache[T] { + return &Cache[T]{ + ttl: ttl, + updateFn: update, + opts: opts, + } +} + +// InitOnce initializes the cache with a TTL and an update function. It is +// guaranteed to be called only once. +func (t *Cache[T]) InitOnce(ttl time.Duration, opts Opts, update func(ctx context.Context) (T, error)) { + t.Once.Do(func() { + t.ttl = ttl + t.updateFn = update + t.opts = opts + }) +} + +// GetWithCtx will return a cached value or fetch a new one. +// passes a caller context, if caller context cancels nothing +// is cached. +// If the Update function returns an error the value is forwarded as is and not cached. +func (t *Cache[T]) GetWithCtx(ctx context.Context) (T, error) { + v := t.val.Load() + ttl := t.ttl + vTime := t.lastUpdateMs.Load() + tNow := time.Now().UnixMilli() + if v != nil && tNow-vTime < ttl.Milliseconds() { + return *v, nil + } + + // Fetch new value asynchronously, while we do not return an error + // if v != nil value or + if t.opts.NoWait && v != nil && tNow-vTime < ttl.Milliseconds()*2 { + if t.updating.TryLock() { + go func() { + defer t.updating.Unlock() + t.update(context.Background()) + }() + } + return *v, nil + } + + // Get lock. Either we get it or we wait for it. + t.updating.Lock() + defer t.updating.Unlock() + + if time.Since(time.UnixMilli(t.lastUpdateMs.Load())) < ttl { + // There is a new value, release lock and return it. + if v = t.val.Load(); v != nil { + return *v, nil + } + } + + if err := t.update(ctx); err != nil { + var empty T + return empty, err + } + + return *t.val.Load(), nil +} + +// Get will return a cached value or fetch a new one. +// Tf the Update function returns an error the value is forwarded as is and not cached. +func (t *Cache[T]) Get() (T, error) { + return t.GetWithCtx(context.Background()) +} + +func (t *Cache[T]) update(ctx context.Context) error { + val, err := t.updateFn(ctx) + if err != nil { + if t.opts.ReturnLastGood && t.val.Load() != nil { + // Keep last good value, so update + // does not return an error. + return nil + } + return err + } + + t.val.Store(&val) + t.lastUpdateMs.Store(time.Now().UnixMilli()) + return nil +} diff --git a/internal/cachevalue/cache_test.go b/internal/cachevalue/cache_test.go new file mode 100644 index 00000000..978cc1ca --- /dev/null +++ b/internal/cachevalue/cache_test.go @@ -0,0 +1,127 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package cachevalue + +import ( + "context" + "errors" + "testing" + "time" +) + +func slowCaller(ctx context.Context) error { + sl := time.NewTimer(time.Second) + defer sl.Stop() + + select { + case <-sl.C: + case <-ctx.Done(): + return ctx.Err() + } + + return nil +} + +func TestCacheCtx(t *testing.T) { + cache := New[time.Time]() + t.Parallel() + cache.InitOnce(2*time.Second, Opts{}, + func(ctx context.Context) (time.Time, error) { + return time.Now(), slowCaller(ctx) + }, + ) + + ctx, cancel := context.WithCancel(t.Context()) + cancel() // cancel context to test. + + _, err := cache.GetWithCtx(ctx) + if !errors.Is(err, context.Canceled) { + t.Fatalf("expected context.Canceled err, got %v", err) + } + + ctx, cancel = context.WithCancel(t.Context()) + defer cancel() + + t1, err := cache.GetWithCtx(ctx) + if err != nil { + t.Fatalf("expected nil err, got %v", err) + } + + t2, err := cache.GetWithCtx(ctx) + if err != nil { + t.Fatalf("expected nil err, got %v", err) + } + + if !t1.Equal(t2) { + t.Fatalf("expected time to be equal: %s != %s", t1, t2) + } + + time.Sleep(3 * time.Second) + + t3, err := cache.GetWithCtx(ctx) + if err != nil { + t.Fatalf("expected nil err, got %v", err) + } + + if t1.Equal(t3) { + t.Fatalf("expected time to be un-equal: %s == %s", t1, t3) + } +} + +func TestCache(t *testing.T) { + cache := New[time.Time]() + t.Parallel() + cache.InitOnce(2*time.Second, Opts{}, + func(ctx context.Context) (time.Time, error) { + return time.Now(), nil + }, + ) + + t1, _ := cache.Get() + + t2, _ := cache.Get() + + if !t1.Equal(t2) { + t.Fatalf("expected time to be equal: %s != %s", t1, t2) + } + + time.Sleep(3 * time.Second) + t3, _ := cache.Get() + + if t1.Equal(t3) { + t.Fatalf("expected time to be un-equal: %s == %s", t1, t3) + } +} + +func BenchmarkCache(b *testing.B) { + cache := New[time.Time]() + cache.InitOnce(1*time.Millisecond, Opts{}, + func(ctx context.Context) (time.Time, error) { + return time.Now(), nil + }, + ) + + b.ReportAllocs() + b.ResetTimer() + + b.RunParallel(func(pb *testing.PB) { + for pb.Next() { + cache.Get() + } + }) +} diff --git a/internal/color/color.go b/internal/color/color.go new file mode 100644 index 00000000..e45851e9 --- /dev/null +++ b/internal/color/color.go @@ -0,0 +1,160 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package color + +import ( + "fmt" + + "github.com/fatih/color" +) + +// global colors. +var ( + // Check if we stderr, stdout are dumb terminals, we do not apply + // ansi coloring on dumb terminals. + IsTerminal = func() bool { + return !color.NoColor + } + + Bold = func() func(format string, a ...any) string { + if IsTerminal() { + return color.New(color.Bold).SprintfFunc() + } + return fmt.Sprintf + }() + + RedBold = func() func(a ...any) string { + if IsTerminal() { + return color.New(color.FgRed, color.Bold).SprintFunc() + } + return fmt.Sprint + }() + + RedBoldf = func() func(format string, a ...any) string { + if IsTerminal() { + return color.New(color.FgRed, color.Bold).SprintfFunc() + } + return fmt.Sprintf + }() + + Red = func() func(format string, a ...any) string { + if IsTerminal() { + return color.New(color.FgRed).SprintfFunc() + } + return fmt.Sprintf + }() + + Blue = func() func(format string, a ...any) string { + if IsTerminal() { + return color.New(color.FgBlue).SprintfFunc() + } + return fmt.Sprintf + }() + + Yellow = func() func(format string, a ...any) string { + if IsTerminal() { + return color.New(color.FgYellow).SprintfFunc() + } + return fmt.Sprintf + }() + + Green = func() func(a ...any) string { + if IsTerminal() { + return color.New(color.FgGreen).SprintFunc() + } + return fmt.Sprint + }() + + Greenf = func() func(format string, a ...any) string { + if IsTerminal() { + return color.New(color.FgGreen).SprintfFunc() + } + return fmt.Sprintf + }() + + GreenBold = func() func(a ...any) string { + if IsTerminal() { + return color.New(color.FgGreen, color.Bold).SprintFunc() + } + return fmt.Sprint + }() + + CyanBold = func() func(a ...any) string { + if IsTerminal() { + return color.New(color.FgCyan, color.Bold).SprintFunc() + } + return fmt.Sprint + }() + + YellowBold = func() func(format string, a ...any) string { + if IsTerminal() { + return color.New(color.FgYellow, color.Bold).SprintfFunc() + } + return fmt.Sprintf + }() + + BlueBold = func() func(format string, a ...any) string { + if IsTerminal() { + return color.New(color.FgBlue, color.Bold).SprintfFunc() + } + return fmt.Sprintf + }() + + BgYellow = func() func(format string, a ...any) string { + if IsTerminal() { + return color.New(color.BgYellow).SprintfFunc() + } + return fmt.Sprintf + }() + + Black = func() func(format string, a ...any) string { + if IsTerminal() { + return color.New(color.FgBlack).SprintfFunc() + } + return fmt.Sprintf + }() + + FgRed = func() func(a ...any) string { + if IsTerminal() { + return color.New(color.FgRed).SprintFunc() + } + return fmt.Sprint + }() + + BgRed = func() func(format string, a ...any) string { + if IsTerminal() { + return color.New(color.BgRed).SprintfFunc() + } + return fmt.Sprintf + }() + + FgWhite = func() func(format string, a ...any) string { + if IsTerminal() { + return color.New(color.FgWhite).SprintfFunc() + } + return fmt.Sprintf + }() + + TurnOff = func() { + color.NoColor = true + } + + TurnOn = func() { + color.NoColor = false + } +) diff --git a/internal/config/api/api.go b/internal/config/api/api.go new file mode 100644 index 00000000..f203f7e9 --- /dev/null +++ b/internal/config/api/api.go @@ -0,0 +1,344 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package api + +import ( + "encoding/json" + "errors" + "fmt" + "math" + "slices" + "strconv" + "strings" + "time" + + "github.com/minio/minio/internal/config" + "github.com/minio/pkg/v3/env" +) + +// API sub-system constants +const ( + apiRequestsMax = "requests_max" + apiClusterDeadline = "cluster_deadline" + apiCorsAllowOrigin = "cors_allow_origin" + apiRemoteTransportDeadline = "remote_transport_deadline" + apiListQuorum = "list_quorum" + apiReplicationPriority = "replication_priority" + apiReplicationMaxWorkers = "replication_max_workers" + apiReplicationMaxLWorkers = "replication_max_lrg_workers" + + apiTransitionWorkers = "transition_workers" + apiStaleUploadsCleanupInterval = "stale_uploads_cleanup_interval" + apiStaleUploadsExpiry = "stale_uploads_expiry" + apiDeleteCleanupInterval = "delete_cleanup_interval" + apiDisableODirect = "disable_odirect" + apiODirect = "odirect" + apiGzipObjects = "gzip_objects" + apiRootAccess = "root_access" + apiSyncEvents = "sync_events" + apiObjectMaxVersions = "object_max_versions" + + EnvAPIRequestsMax = "MINIO_API_REQUESTS_MAX" + EnvAPIRequestsDeadline = "MINIO_API_REQUESTS_DEADLINE" + EnvAPIClusterDeadline = "MINIO_API_CLUSTER_DEADLINE" + EnvAPICorsAllowOrigin = "MINIO_API_CORS_ALLOW_ORIGIN" + EnvAPIRemoteTransportDeadline = "MINIO_API_REMOTE_TRANSPORT_DEADLINE" + EnvAPITransitionWorkers = "MINIO_API_TRANSITION_WORKERS" + EnvAPIListQuorum = "MINIO_API_LIST_QUORUM" + EnvAPISecureCiphers = "MINIO_API_SECURE_CIPHERS" // default config.EnableOn + EnvAPIReplicationPriority = "MINIO_API_REPLICATION_PRIORITY" + EnvAPIReplicationMaxWorkers = "MINIO_API_REPLICATION_MAX_WORKERS" + EnvAPIReplicationMaxLWorkers = "MINIO_API_REPLICATION_MAX_LRG_WORKERS" + + EnvAPIStaleUploadsCleanupInterval = "MINIO_API_STALE_UPLOADS_CLEANUP_INTERVAL" + EnvAPIStaleUploadsExpiry = "MINIO_API_STALE_UPLOADS_EXPIRY" + EnvAPIDeleteCleanupInterval = "MINIO_API_DELETE_CLEANUP_INTERVAL" + EnvDeleteCleanupInterval = "MINIO_DELETE_CLEANUP_INTERVAL" + EnvAPIODirect = "MINIO_API_ODIRECT" + EnvAPIDisableODirect = "MINIO_API_DISABLE_ODIRECT" + EnvAPIGzipObjects = "MINIO_API_GZIP_OBJECTS" + EnvAPIRootAccess = "MINIO_API_ROOT_ACCESS" // default config.EnableOn + EnvAPISyncEvents = "MINIO_API_SYNC_EVENTS" // default "off" + EnvAPIObjectMaxVersions = "MINIO_API_OBJECT_MAX_VERSIONS" + EnvAPIObjectMaxVersionsLegacy = "_MINIO_OBJECT_MAX_VERSIONS" +) + +// Deprecated key and ENVs +const ( + apiReadyDeadline = "ready_deadline" + apiRequestsDeadline = "requests_deadline" + apiReplicationWorkers = "replication_workers" + apiReplicationFailedWorkers = "replication_failed_workers" +) + +// DefaultKVS - default storage class config +var ( + DefaultKVS = config.KVS{ + config.KV{ + Key: apiRequestsMax, + Value: "0", + }, + config.KV{ + Key: apiClusterDeadline, + Value: "10s", + }, + config.KV{ + Key: apiCorsAllowOrigin, + Value: "*", + }, + config.KV{ + Key: apiRemoteTransportDeadline, + Value: "2h", + }, + config.KV{ + Key: apiListQuorum, + Value: "strict", + }, + config.KV{ + Key: apiReplicationPriority, + Value: "auto", + }, + config.KV{ + Key: apiReplicationMaxWorkers, + Value: "500", + }, + config.KV{ + Key: apiReplicationMaxLWorkers, + Value: "10", + }, + config.KV{ + Key: apiTransitionWorkers, + Value: "100", + }, + config.KV{ + Key: apiStaleUploadsCleanupInterval, + Value: "6h", + }, + config.KV{ + Key: apiStaleUploadsExpiry, + Value: "24h", + }, + config.KV{ + Key: apiDeleteCleanupInterval, + Value: "5m", + }, + config.KV{ + Key: apiDisableODirect, + Value: "", + HiddenIfEmpty: true, + }, + config.KV{ + Key: apiODirect, + Value: config.EnableOn, + }, + config.KV{ + Key: apiGzipObjects, + Value: config.EnableOff, + }, + config.KV{ + Key: apiRootAccess, + Value: config.EnableOn, + }, + config.KV{ + Key: apiSyncEvents, + Value: config.EnableOff, + }, + config.KV{ + Key: apiObjectMaxVersions, + Value: "9223372036854775807", + }, + } +) + +// Config storage class configuration +type Config struct { + RequestsMax int `json:"requests_max"` + ClusterDeadline time.Duration `json:"cluster_deadline"` + CorsAllowOrigin []string `json:"cors_allow_origin"` + RemoteTransportDeadline time.Duration `json:"remote_transport_deadline"` + ListQuorum string `json:"list_quorum"` + ReplicationPriority string `json:"replication_priority"` + ReplicationMaxWorkers int `json:"replication_max_workers"` + ReplicationMaxLWorkers int `json:"replication_max_lrg_workers"` + TransitionWorkers int `json:"transition_workers"` + StaleUploadsCleanupInterval time.Duration `json:"stale_uploads_cleanup_interval"` + StaleUploadsExpiry time.Duration `json:"stale_uploads_expiry"` + DeleteCleanupInterval time.Duration `json:"delete_cleanup_interval"` + EnableODirect bool `json:"enable_odirect"` + GzipObjects bool `json:"gzip_objects"` + RootAccess bool `json:"root_access"` + SyncEvents bool `json:"sync_events"` + ObjectMaxVersions int64 `json:"object_max_versions"` +} + +// UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON. +func (sCfg *Config) UnmarshalJSON(data []byte) error { + type Alias Config + aux := &struct { + *Alias + }{ + Alias: (*Alias)(sCfg), + } + return json.Unmarshal(data, &aux) +} + +// LookupConfig - lookup api config and override with valid environment settings if any. +func LookupConfig(kvs config.KVS) (cfg Config, err error) { + deprecatedKeys := []string{ + apiReadyDeadline, + apiRequestsDeadline, + "extend_list_cache_life", + apiReplicationWorkers, + apiReplicationFailedWorkers, + "expiry_workers", + } + + disableODirect := env.Get(EnvAPIDisableODirect, kvs.Get(apiDisableODirect)) == config.EnableOn + enableODirect := env.Get(EnvAPIODirect, kvs.Get(apiODirect)) == config.EnableOn + gzipObjects := env.Get(EnvAPIGzipObjects, kvs.Get(apiGzipObjects)) == config.EnableOn + rootAccess := env.Get(EnvAPIRootAccess, kvs.Get(apiRootAccess)) == config.EnableOn + + cfg = Config{ + EnableODirect: enableODirect || !disableODirect, + GzipObjects: gzipObjects, + RootAccess: rootAccess, + } + + var corsAllowOrigin []string + corsList := env.Get(EnvAPICorsAllowOrigin, kvs.Get(apiCorsAllowOrigin)) + if corsList == "" { + corsAllowOrigin = []string{"*"} // defaults to '*' + } else { + corsAllowOrigin = strings.Split(corsList, ",") + if slices.Contains(corsAllowOrigin, "") { + return cfg, errors.New("invalid cors value") + } + } + cfg.CorsAllowOrigin = corsAllowOrigin + + if err = config.CheckValidKeys(config.APISubSys, kvs, DefaultKVS, deprecatedKeys...); err != nil { + return cfg, err + } + + // Check environment variables parameters + requestsMax, err := strconv.Atoi(env.Get(EnvAPIRequestsMax, kvs.GetWithDefault(apiRequestsMax, DefaultKVS))) + if err != nil { + return cfg, err + } + + cfg.RequestsMax = requestsMax + if requestsMax < 0 { + return cfg, errors.New("invalid API max requests value") + } + + clusterDeadline, err := time.ParseDuration(env.Get(EnvAPIClusterDeadline, kvs.GetWithDefault(apiClusterDeadline, DefaultKVS))) + if err != nil { + return cfg, err + } + cfg.ClusterDeadline = clusterDeadline + + remoteTransportDeadline, err := time.ParseDuration(env.Get(EnvAPIRemoteTransportDeadline, kvs.GetWithDefault(apiRemoteTransportDeadline, DefaultKVS))) + if err != nil { + return cfg, err + } + cfg.RemoteTransportDeadline = remoteTransportDeadline + + listQuorum := env.Get(EnvAPIListQuorum, kvs.GetWithDefault(apiListQuorum, DefaultKVS)) + switch listQuorum { + case "strict", "optimal", "reduced", "disk", "auto": + default: + return cfg, fmt.Errorf("invalid value %v for list_quorum: will default to 'strict'", listQuorum) + } + cfg.ListQuorum = listQuorum + + replicationPriority := env.Get(EnvAPIReplicationPriority, kvs.GetWithDefault(apiReplicationPriority, DefaultKVS)) + switch replicationPriority { + case "slow", "fast", "auto": + default: + return cfg, fmt.Errorf("invalid value %v for replication_priority", replicationPriority) + } + cfg.ReplicationPriority = replicationPriority + replicationMaxWorkers, err := strconv.Atoi(env.Get(EnvAPIReplicationMaxWorkers, kvs.GetWithDefault(apiReplicationMaxWorkers, DefaultKVS))) + if err != nil { + return cfg, err + } + if replicationMaxWorkers <= 0 || replicationMaxWorkers > 500 { + return cfg, config.ErrInvalidReplicationWorkersValue(nil).Msg("Number of replication workers should be between 1 and 500") + } + cfg.ReplicationMaxWorkers = replicationMaxWorkers + + replicationMaxLWorkers, err := strconv.Atoi(env.Get(EnvAPIReplicationMaxLWorkers, kvs.GetWithDefault(apiReplicationMaxLWorkers, DefaultKVS))) + if err != nil { + return cfg, err + } + if replicationMaxLWorkers <= 0 || replicationMaxLWorkers > 10 { + return cfg, config.ErrInvalidReplicationWorkersValue(nil).Msg("Number of replication workers for transfers >=128MiB should be between 1 and 10 per node") + } + + cfg.ReplicationMaxLWorkers = replicationMaxLWorkers + + transitionWorkers, err := strconv.Atoi(env.Get(EnvAPITransitionWorkers, kvs.GetWithDefault(apiTransitionWorkers, DefaultKVS))) + if err != nil { + return cfg, err + } + cfg.TransitionWorkers = transitionWorkers + + v := env.Get(EnvAPIDeleteCleanupInterval, kvs.Get(apiDeleteCleanupInterval)) + if v == "" { + v = env.Get(EnvDeleteCleanupInterval, kvs.GetWithDefault(apiDeleteCleanupInterval, DefaultKVS)) + } + + deleteCleanupInterval, err := time.ParseDuration(v) + if err != nil { + return cfg, err + } + cfg.DeleteCleanupInterval = deleteCleanupInterval + + staleUploadsCleanupInterval, err := time.ParseDuration(env.Get(EnvAPIStaleUploadsCleanupInterval, kvs.GetWithDefault(apiStaleUploadsCleanupInterval, DefaultKVS))) + if err != nil { + return cfg, err + } + cfg.StaleUploadsCleanupInterval = staleUploadsCleanupInterval + + staleUploadsExpiry, err := time.ParseDuration(env.Get(EnvAPIStaleUploadsExpiry, kvs.GetWithDefault(apiStaleUploadsExpiry, DefaultKVS))) + if err != nil { + return cfg, err + } + cfg.StaleUploadsExpiry = staleUploadsExpiry + + cfg.SyncEvents = env.Get(EnvAPISyncEvents, kvs.Get(apiSyncEvents)) == config.EnableOn + + maxVerStr := env.Get(EnvAPIObjectMaxVersions, "") + if maxVerStr == "" { + maxVerStr = env.Get(EnvAPIObjectMaxVersionsLegacy, kvs.Get(apiObjectMaxVersions)) + } + if maxVerStr != "" { + maxVersions, err := strconv.ParseInt(maxVerStr, 10, 64) + if err != nil { + return cfg, err + } + if maxVersions <= 0 { + return cfg, fmt.Errorf("invalid object max versions value: %v", maxVersions) + } + cfg.ObjectMaxVersions = maxVersions + } else { + cfg.ObjectMaxVersions = math.MaxInt64 + } + + return cfg, nil +} diff --git a/internal/config/api/help.go b/internal/config/api/help.go new file mode 100644 index 00000000..2c4b8b2b --- /dev/null +++ b/internal/config/api/help.go @@ -0,0 +1,120 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package api + +import "github.com/minio/minio/internal/config" + +var ( + defaultHelpPostfix = func(key string) string { + return config.DefaultHelpPostfix(DefaultKVS, key) + } + + // Help holds configuration keys and their default values for api subsystem. + Help = config.HelpKVS{ + config.HelpKV{ + Key: apiRequestsMax, + Description: `set the maximum number of concurrent requests (default: auto)`, + Optional: true, + Type: "number", + }, + config.HelpKV{ + Key: apiClusterDeadline, + Description: `set the deadline for cluster readiness check` + defaultHelpPostfix(apiClusterDeadline), + Optional: true, + Type: "duration", + }, + config.HelpKV{ + Key: apiCorsAllowOrigin, + Description: `set comma separated list of origins allowed for CORS requests` + defaultHelpPostfix(apiCorsAllowOrigin), + Optional: true, + Type: "csv", + }, + config.HelpKV{ + Key: apiRemoteTransportDeadline, + Description: `set the deadline for API requests on remote transports while proxying between federated instances e.g. "2h"` + defaultHelpPostfix(apiRemoteTransportDeadline), + Optional: true, + Type: "duration", + }, + config.HelpKV{ + Key: apiListQuorum, + Description: `set the acceptable quorum expected for list operations e.g. "optimal", "reduced", "disk", "strict", "auto"` + defaultHelpPostfix(apiListQuorum), + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: apiReplicationPriority, + Description: `set replication priority` + defaultHelpPostfix(apiReplicationPriority), + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: apiReplicationMaxWorkers, + Description: `set the maximum number of replication workers` + defaultHelpPostfix(apiReplicationMaxWorkers), + Optional: true, + Type: "number", + }, + config.HelpKV{ + Key: apiTransitionWorkers, + Description: `set the number of transition workers` + defaultHelpPostfix(apiTransitionWorkers), + Optional: true, + Type: "number", + }, + config.HelpKV{ + Key: apiStaleUploadsExpiry, + Description: `set to expire stale multipart uploads older than this values` + defaultHelpPostfix(apiStaleUploadsExpiry), + Optional: true, + Type: "duration", + }, + config.HelpKV{ + Key: apiStaleUploadsCleanupInterval, + Description: `set to change intervals when stale multipart uploads are expired` + defaultHelpPostfix(apiStaleUploadsCleanupInterval), + Optional: true, + Type: "duration", + }, + config.HelpKV{ + Key: apiDeleteCleanupInterval, + Description: `set to change intervals when deleted objects are permanently deleted from ".trash" folder` + defaultHelpPostfix(apiDeleteCleanupInterval), + Optional: true, + Type: "duration", + }, + config.HelpKV{ + Key: apiODirect, + Description: "set to enable or disable O_DIRECT for writes under special conditions. NOTE: do not disable O_DIRECT without prior testing" + defaultHelpPostfix(apiODirect), + Optional: true, + Type: "boolean", + }, + config.HelpKV{ + Key: apiRootAccess, + Description: "turn 'off' root credential access for all API calls including s3, admin operations" + defaultHelpPostfix(apiRootAccess), + Optional: true, + Type: "boolean", + }, + config.HelpKV{ + Key: apiSyncEvents, + Description: "set to enable synchronous bucket notifications" + defaultHelpPostfix(apiSyncEvents), + Optional: true, + Type: "boolean", + }, + config.HelpKV{ + Key: apiObjectMaxVersions, + Description: "set max allowed number of versions per object" + defaultHelpPostfix(apiObjectMaxVersions), + Optional: true, + Type: "number", + }, + } +) diff --git a/internal/config/batch/batch.go b/internal/config/batch/batch.go new file mode 100644 index 00000000..7404cf86 --- /dev/null +++ b/internal/config/batch/batch.go @@ -0,0 +1,160 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package batch + +import ( + "sync" + "time" + + "github.com/minio/minio/internal/config" + "github.com/minio/pkg/v3/env" +) + +// Batch job environment variables +const ( + ReplicationWorkersWait = "replication_workers_wait" + KeyRotationWorkersWait = "keyrotation_workers_wait" + ExpirationWorkersWait = "expiration_workers_wait" + + EnvReplicationWorkersWait = "MINIO_BATCH_REPLICATION_WORKERS_WAIT" + EnvKeyRotationWorkersWait = "MINIO_BATCH_KEYROTATION_WORKERS_WAIT" + EnvKeyExpirationWorkersWait = "MINIO_BATCH_EXPIRATION_WORKERS_WAIT" +) + +var configMu sync.RWMutex + +// Config represents the batch job settings. +type Config struct { + ReplicationWorkersWait time.Duration `json:"replicationWorkersWait"` + KeyRotationWorkersWait time.Duration `json:"keyRotationWorkersWait"` + ExpirationWorkersWait time.Duration `json:"expirationWorkersWait"` +} + +// ExpirationWait returns the duration for which a batch expiration worker +// would wait before working on next object. +func (opts Config) ExpirationWait() time.Duration { + configMu.RLock() + defer configMu.RUnlock() + + return opts.ExpirationWorkersWait +} + +// ReplicationWait returns the duration for which a batch replication worker +// would wait before working on next object. +func (opts Config) ReplicationWait() time.Duration { + configMu.RLock() + defer configMu.RUnlock() + + return opts.ReplicationWorkersWait +} + +// KeyRotationWait returns the duration for which a batch key-rotation worker +// would wait before working on next object. +func (opts Config) KeyRotationWait() time.Duration { + configMu.RLock() + defer configMu.RUnlock() + + return opts.KeyRotationWorkersWait +} + +// Clone returns a copy of Config value +func (opts Config) Clone() Config { + configMu.RLock() + defer configMu.RUnlock() + + return Config{ + ReplicationWorkersWait: opts.ReplicationWorkersWait, + KeyRotationWorkersWait: opts.KeyRotationWorkersWait, + ExpirationWorkersWait: opts.ExpirationWorkersWait, + } +} + +// Update updates opts with nopts +func (opts *Config) Update(nopts Config) { + configMu.Lock() + defer configMu.Unlock() + + opts.ReplicationWorkersWait = nopts.ReplicationWorkersWait + opts.KeyRotationWorkersWait = nopts.KeyRotationWorkersWait + opts.ExpirationWorkersWait = nopts.ExpirationWorkersWait +} + +// DefaultKVS - default KV config for batch job settings +var DefaultKVS = config.KVS{ + config.KV{ + Key: ReplicationWorkersWait, + Value: "0ms", // No wait by default between each replication attempts. + }, + config.KV{ + Key: KeyRotationWorkersWait, + Value: "0ms", // No wait by default between each key rotation attempts. + }, + config.KV{ + Key: ExpirationWorkersWait, + Value: "0ms", // No wait by default between each expiration attempts. + }, +} + +// LookupConfig - lookup config and override with valid environment settings if any. +func LookupConfig(kvs config.KVS) (cfg Config, err error) { + if err = config.CheckValidKeys(config.BatchSubSys, kvs, DefaultKVS); err != nil { + return cfg, err + } + + cfg.ReplicationWorkersWait = 0 + cfg.KeyRotationWorkersWait = 0 + cfg.ExpirationWorkersWait = 0 + + rduration, err := time.ParseDuration(env.Get(EnvReplicationWorkersWait, kvs.GetWithDefault(ReplicationWorkersWait, DefaultKVS))) + if err != nil { + return cfg, err + } + if rduration < 0 { + return cfg, config.ErrInvalidBatchReplicationWorkersWait(nil) + } + + kduration, err := time.ParseDuration(env.Get(EnvKeyRotationWorkersWait, kvs.GetWithDefault(KeyRotationWorkersWait, DefaultKVS))) + if err != nil { + return cfg, err + } + if kduration < 0 { + return cfg, config.ErrInvalidBatchKeyRotationWorkersWait(nil) + } + + eduration, err := time.ParseDuration(env.Get(EnvKeyExpirationWorkersWait, kvs.GetWithDefault(ExpirationWorkersWait, DefaultKVS))) + if err != nil { + return cfg, err + } + if eduration < 0 { + return cfg, config.ErrInvalidBatchExpirationWorkersWait(nil) + } + + if rduration > 0 { + cfg.ReplicationWorkersWait = rduration + } + + if kduration > 0 { + cfg.KeyRotationWorkersWait = kduration + } + + if eduration > 0 { + cfg.ExpirationWorkersWait = eduration + } + + return cfg, nil +} diff --git a/internal/config/batch/help.go b/internal/config/batch/help.go new file mode 100644 index 00000000..e218edd1 --- /dev/null +++ b/internal/config/batch/help.go @@ -0,0 +1,49 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package batch + +import "github.com/minio/minio/internal/config" + +// Help template for batch feature. +var ( + defaultHelpPostfix = func(key string) string { + return config.DefaultHelpPostfix(DefaultKVS, key) + } + + // Help provides help for config values + Help = config.HelpKVS{ + config.HelpKV{ + Key: ReplicationWorkersWait, + Description: `maximum sleep duration between objects to slow down batch replication operation` + defaultHelpPostfix(ReplicationWorkersWait), + Optional: true, + Type: "duration", + }, + config.HelpKV{ + Key: KeyRotationWorkersWait, + Description: `maximum sleep duration between objects to slow down batch keyrotation operation` + defaultHelpPostfix(KeyRotationWorkersWait), + Optional: true, + Type: "duration", + }, + config.HelpKV{ + Key: ExpirationWorkersWait, + Description: "maximum sleep duration between objects to slow down batch expiration operation" + defaultHelpPostfix(ExpirationWorkersWait), + Optional: true, + Type: "duration", + }, + } +) diff --git a/internal/config/bool-flag.go b/internal/config/bool-flag.go new file mode 100644 index 00000000..950bbe08 --- /dev/null +++ b/internal/config/bool-flag.go @@ -0,0 +1,91 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package config + +import ( + "encoding/json" + "fmt" + "strconv" + "strings" +) + +// BoolFlag - wrapper bool type. +type BoolFlag bool + +// String - returns string of BoolFlag. +func (bf BoolFlag) String() string { + if bf { + return "on" + } + + return "off" +} + +// MarshalJSON - converts BoolFlag into JSON data. +func (bf BoolFlag) MarshalJSON() ([]byte, error) { + return json.Marshal(bf.String()) +} + +// UnmarshalJSON - parses given data into BoolFlag. +func (bf *BoolFlag) UnmarshalJSON(data []byte) (err error) { + var s string + if err = json.Unmarshal(data, &s); err == nil { + b := BoolFlag(true) + if s == "" { + // Empty string is treated as valid. + *bf = b + } else if b, err = ParseBoolFlag(s); err == nil { + *bf = b + } + } + + return err +} + +// FormatBool prints stringified version of boolean. +func FormatBool(b bool) string { + if b { + return "on" + } + return "off" +} + +// ParseBool returns the boolean value represented by the string. +// It accepts 1, t, T, TRUE, true, True, 0, f, F, FALSE, false, False. +// Any other value returns an error. +func ParseBool(str string) (bool, error) { + switch str { + case "1", "t", "T", "true", "TRUE", "True", "on", "ON", "On": + return true, nil + case "0", "f", "F", "false", "FALSE", "False", "off", "OFF", "Off": + return false, nil + } + if strings.EqualFold(str, "enabled") { + return true, nil + } + if strings.EqualFold(str, "disabled") { + return false, nil + } + return false, fmt.Errorf("ParseBool: parsing '%s': %w", str, strconv.ErrSyntax) +} + +// ParseBoolFlag - parses string into BoolFlag. +func ParseBoolFlag(s string) (bf BoolFlag, err error) { + b, err := ParseBool(s) + return BoolFlag(b), err +} diff --git a/internal/config/bool-flag_test.go b/internal/config/bool-flag_test.go new file mode 100644 index 00000000..db99aff9 --- /dev/null +++ b/internal/config/bool-flag_test.go @@ -0,0 +1,129 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package config + +import ( + "testing" +) + +// Test BoolFlag.String() +func TestBoolFlagString(t *testing.T) { + var bf BoolFlag + + testCases := []struct { + flag BoolFlag + expectedResult string + }{ + {bf, "off"}, + {BoolFlag(true), "on"}, + {BoolFlag(false), "off"}, + } + + for _, testCase := range testCases { + str := testCase.flag.String() + if testCase.expectedResult != str { + t.Fatalf("expected: %v, got: %v", testCase.expectedResult, str) + } + } +} + +// Test BoolFlag.MarshalJSON() +func TestBoolFlagMarshalJSON(t *testing.T) { + var bf BoolFlag + + testCases := []struct { + flag BoolFlag + expectedResult string + }{ + {bf, `"off"`}, + {BoolFlag(true), `"on"`}, + {BoolFlag(false), `"off"`}, + } + + for _, testCase := range testCases { + data, _ := testCase.flag.MarshalJSON() + if testCase.expectedResult != string(data) { + t.Fatalf("expected: %v, got: %v", testCase.expectedResult, string(data)) + } + } +} + +// Test BoolFlag.UnmarshalJSON() +func TestBoolFlagUnmarshalJSON(t *testing.T) { + testCases := []struct { + data []byte + expectedResult BoolFlag + expectedErr bool + }{ + {[]byte(`{}`), BoolFlag(false), true}, + {[]byte(`["on"]`), BoolFlag(false), true}, + {[]byte(`"junk"`), BoolFlag(false), true}, + {[]byte(`""`), BoolFlag(true), false}, + {[]byte(`"on"`), BoolFlag(true), false}, + {[]byte(`"off"`), BoolFlag(false), false}, + {[]byte(`"true"`), BoolFlag(true), false}, + {[]byte(`"false"`), BoolFlag(false), false}, + {[]byte(`"ON"`), BoolFlag(true), false}, + {[]byte(`"OFF"`), BoolFlag(false), false}, + } + + for _, testCase := range testCases { + var flag BoolFlag + err := (&flag).UnmarshalJSON(testCase.data) + if !testCase.expectedErr && err != nil { + t.Fatalf("error: expected = , got = %v", err) + } + if testCase.expectedErr && err == nil { + t.Fatalf("error: expected error, got = ") + } + if err == nil && testCase.expectedResult != flag { + t.Fatalf("result: expected: %v, got: %v", testCase.expectedResult, flag) + } + } +} + +// Test ParseBoolFlag() +func TestParseBoolFlag(t *testing.T) { + testCases := []struct { + flagStr string + expectedResult BoolFlag + expectedErr bool + }{ + {"", BoolFlag(false), true}, + {"junk", BoolFlag(false), true}, + {"true", BoolFlag(true), false}, + {"false", BoolFlag(false), false}, + {"ON", BoolFlag(true), false}, + {"OFF", BoolFlag(false), false}, + {"on", BoolFlag(true), false}, + {"off", BoolFlag(false), false}, + } + + for _, testCase := range testCases { + bf, err := ParseBoolFlag(testCase.flagStr) + if !testCase.expectedErr && err != nil { + t.Fatalf("error: expected = , got = %v", err) + } + if testCase.expectedErr && err == nil { + t.Fatalf("error: expected error, got = ") + } + if err == nil && testCase.expectedResult != bf { + t.Fatalf("result: expected: %v, got: %v", testCase.expectedResult, bf) + } + } +} diff --git a/internal/config/browser/browser.go b/internal/config/browser/browser.go new file mode 100644 index 00000000..f5ad11d2 --- /dev/null +++ b/internal/config/browser/browser.go @@ -0,0 +1,174 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package browser + +import ( + "fmt" + "strconv" + "sync" + + "github.com/minio/minio/internal/config" + "github.com/minio/pkg/v3/env" +) + +// Browser sub-system constants +const ( + // browserCSPPolicy setting name for Content-Security-Policy response header value + browserCSPPolicy = "csp_policy" + // browserHSTSSeconds setting name for Strict-Transport-Security response header, amount of seconds for 'max-age' + browserHSTSSeconds = "hsts_seconds" + // browserHSTSIncludeSubdomains setting name for Strict-Transport-Security response header 'includeSubDomains' flag (true or false) + browserHSTSIncludeSubdomains = "hsts_include_subdomains" + // browserHSTSPreload setting name for Strict-Transport-Security response header 'preload' flag (true or false) + browserHSTSPreload = "hsts_preload" + // browserReferrerPolicy setting name for Referrer-Policy response header + browserReferrerPolicy = "referrer_policy" + + EnvBrowserCSPPolicy = "MINIO_BROWSER_CONTENT_SECURITY_POLICY" + EnvBrowserHSTSSeconds = "MINIO_BROWSER_HSTS_SECONDS" + EnvBrowserHSTSIncludeSubdomains = "MINIO_BROWSER_HSTS_INCLUDE_SUB_DOMAINS" + EnvBrowserHSTSPreload = "MINIO_BROWSER_HSTS_PRELOAD" + EnvBrowserReferrerPolicy = "MINIO_BROWSER_REFERRER_POLICY" +) + +// DefaultKVS - default storage class config +var ( + DefaultKVS = config.KVS{ + config.KV{ + Key: browserCSPPolicy, + Value: "default-src 'self' 'unsafe-eval' 'unsafe-inline'; script-src 'self' https://unpkg.com; connect-src 'self' https://unpkg.com;", + }, + config.KV{ + Key: browserHSTSSeconds, + Value: "0", + }, + config.KV{ + Key: browserHSTSIncludeSubdomains, + Value: config.EnableOff, + }, + config.KV{ + Key: browserHSTSPreload, + Value: config.EnableOff, + }, + config.KV{ + Key: browserReferrerPolicy, + Value: "strict-origin-when-cross-origin", + }, + } +) + +// configLock is a global lock for browser config +var configLock sync.RWMutex + +// Config storage class configuration +type Config struct { + CSPPolicy string `json:"csp_policy"` + HSTSSeconds int `json:"hsts_seconds"` + HSTSIncludeSubdomains bool `json:"hsts_include_subdomains"` + HSTSPreload bool `json:"hsts_preload"` + ReferrerPolicy string `json:"referrer_policy"` +} + +// Update Updates browser with new config +func (browseCfg *Config) Update(newCfg Config) { + configLock.Lock() + defer configLock.Unlock() + browseCfg.CSPPolicy = newCfg.CSPPolicy + browseCfg.HSTSSeconds = newCfg.HSTSSeconds + browseCfg.HSTSIncludeSubdomains = newCfg.HSTSIncludeSubdomains + browseCfg.HSTSPreload = newCfg.HSTSPreload + browseCfg.ReferrerPolicy = newCfg.ReferrerPolicy +} + +// LookupConfig - lookup api config and override with valid environment settings if any. +func LookupConfig(kvs config.KVS) (cfg Config, err error) { + cfg = Config{ + CSPPolicy: env.Get(EnvBrowserCSPPolicy, kvs.GetWithDefault(browserCSPPolicy, DefaultKVS)), + HSTSSeconds: 0, + HSTSIncludeSubdomains: true, + HSTSPreload: true, + ReferrerPolicy: "strict-origin-when-cross-origin", + } + + if err = config.CheckValidKeys(config.BrowserSubSys, kvs, DefaultKVS); err != nil { + return cfg, err + } + + hstsIncludeSubdomains := env.Get(EnvBrowserHSTSIncludeSubdomains, kvs.GetWithDefault(browserHSTSIncludeSubdomains, DefaultKVS)) == config.EnableOn + hstsPreload := env.Get(EnvBrowserHSTSPreload, kvs.Get(browserHSTSPreload)) == config.EnableOn + + hstsSeconds, err := strconv.Atoi(env.Get(EnvBrowserHSTSSeconds, kvs.GetWithDefault(browserHSTSSeconds, DefaultKVS))) + if err != nil { + return cfg, err + } + + cfg.HSTSSeconds = hstsSeconds + cfg.HSTSIncludeSubdomains = hstsIncludeSubdomains + cfg.HSTSPreload = hstsPreload + + referrerPolicy := env.Get(EnvBrowserReferrerPolicy, kvs.GetWithDefault(browserReferrerPolicy, DefaultKVS)) + switch referrerPolicy { + case "no-referrer", "no-referrer-when-downgrade", "origin", "origin-when-cross-origin", "same-origin", "strict-origin", "strict-origin-when-cross-origin", "unsafe-url": + cfg.ReferrerPolicy = referrerPolicy + default: + return cfg, fmt.Errorf("invalid value %v for %s", referrerPolicy, browserReferrerPolicy) + } + + return cfg, nil +} + +// GetCSPolicy - Get the Content security Policy +func (browseCfg *Config) GetCSPolicy() string { + configLock.RLock() + defer configLock.RUnlock() + return browseCfg.CSPPolicy +} + +// GetHSTSSeconds - Get the Content security Policy +func (browseCfg *Config) GetHSTSSeconds() int { + configLock.RLock() + defer configLock.RUnlock() + return browseCfg.HSTSSeconds +} + +// IsHSTSIncludeSubdomains - is HSTS 'includeSubdomains' directive enabled +func (browseCfg *Config) IsHSTSIncludeSubdomains() string { + configLock.RLock() + defer configLock.RUnlock() + if browseCfg.HSTSSeconds > 0 && browseCfg.HSTSIncludeSubdomains { + return config.EnableOn + } + return config.EnableOff +} + +// IsHSTSPreload - is HSTS 'preload' directive enabled +func (browseCfg *Config) IsHSTSPreload() string { + configLock.RLock() + defer configLock.RUnlock() + if browseCfg.HSTSSeconds > 0 && browseCfg.HSTSPreload { + return config.EnableOn + } + return config.EnableOff +} + +// GetReferPolicy - Get the ReferPolicy +func (browseCfg *Config) GetReferPolicy() string { + configLock.RLock() + defer configLock.RUnlock() + return browseCfg.ReferrerPolicy +} diff --git a/internal/config/browser/help.go b/internal/config/browser/help.go new file mode 100644 index 00000000..8f8483e2 --- /dev/null +++ b/internal/config/browser/help.go @@ -0,0 +1,60 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// # This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package browser + +import "github.com/minio/minio/internal/config" + +// Help template for browser feature. +var ( + defaultHelpPostfix = func(key string) string { + return config.DefaultHelpPostfix(DefaultKVS, key) + } + + Help = config.HelpKVS{ + config.HelpKV{ + Key: browserCSPPolicy, + Description: `set Content-Security-Policy response header value` + defaultHelpPostfix(browserCSPPolicy), + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: browserHSTSSeconds, + Description: `set Strict-Transport-Security 'max-age' amount of seconds value` + defaultHelpPostfix(browserHSTSSeconds), + Optional: true, + Type: "number", + }, + config.HelpKV{ + Key: browserHSTSIncludeSubdomains, + Description: `turn 'on' to set Strict-Transport-Security 'includeSubDomains' directive` + defaultHelpPostfix(browserHSTSIncludeSubdomains), + Optional: true, + Type: "boolean", + }, + config.HelpKV{ + Key: browserHSTSPreload, + Description: `turn 'on' to set Strict-Transport-Security 'preload' directive` + defaultHelpPostfix(browserHSTSPreload), + Optional: true, + Type: "boolean", + }, + config.HelpKV{ + Key: browserReferrerPolicy, + Description: `set Referrer-Policy response header value` + defaultHelpPostfix(browserReferrerPolicy), + Optional: true, + Type: "string", + }, + } +) diff --git a/internal/config/callhome/callhome.go b/internal/config/callhome/callhome.go new file mode 100644 index 00000000..ef6f8d51 --- /dev/null +++ b/internal/config/callhome/callhome.go @@ -0,0 +1,100 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package callhome + +import ( + "sync" + "time" + + "github.com/minio/minio/internal/config" + "github.com/minio/pkg/v3/env" +) + +// Callhome related keys +const ( + Enable = "enable" + Frequency = "frequency" +) + +// DefaultKVS - default KV config for subnet settings +var DefaultKVS = config.KVS{ + config.KV{ + Key: Enable, + Value: "off", + }, + config.KV{ + Key: Frequency, + Value: "24h", + }, +} + +// callhomeCycleDefault is the default interval between two callhome cycles (24hrs) +const callhomeCycleDefault = 24 * time.Hour + +// Config represents the subnet related configuration +type Config struct { + // Flag indicating whether callhome is enabled. + Enable bool `json:"enable"` + + // The interval between callhome cycles + Frequency time.Duration `json:"frequency"` +} + +var configLock sync.RWMutex + +// Enabled - indicates if callhome is enabled or not +func (c *Config) Enabled() bool { + configLock.RLock() + defer configLock.RUnlock() + + return c.Enable +} + +// FrequencyDur - returns the currently configured callhome frequency +func (c *Config) FrequencyDur() time.Duration { + configLock.RLock() + defer configLock.RUnlock() + + if c.Frequency == 0 { + return callhomeCycleDefault + } + + return c.Frequency +} + +// Update updates new callhome frequency +func (c *Config) Update(ncfg Config) { + configLock.Lock() + defer configLock.Unlock() + + c.Enable = ncfg.Enable + c.Frequency = ncfg.Frequency +} + +// LookupConfig - lookup config and override with valid environment settings if any. +func LookupConfig(kvs config.KVS) (cfg Config, err error) { + if err = config.CheckValidKeys(config.CallhomeSubSys, kvs, DefaultKVS); err != nil { + return cfg, err + } + + cfg.Enable = env.Get(config.EnvMinIOCallhomeEnable, + kvs.GetWithDefault(Enable, DefaultKVS)) == config.EnableOn + cfg.Frequency, err = time.ParseDuration(env.Get(config.EnvMinIOCallhomeFrequency, + kvs.GetWithDefault(Frequency, DefaultKVS))) + return cfg, err +} diff --git a/internal/config/callhome/help.go b/internal/config/callhome/help.go new file mode 100644 index 00000000..8def3fa5 --- /dev/null +++ b/internal/config/callhome/help.go @@ -0,0 +1,42 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package callhome + +import "github.com/minio/minio/internal/config" + +var ( + defaultHelpPostfix = func(key string) string { + return config.DefaultHelpPostfix(DefaultKVS, key) + } + + // HelpCallhome - provides help for callhome config + HelpCallhome = config.HelpKVS{ + config.HelpKV{ + Key: Enable, + Type: "on|off", + Description: "set to enable callhome" + defaultHelpPostfix(Enable), + Optional: true, + }, + config.HelpKV{ + Key: Frequency, + Type: "duration", + Description: "time duration between callhome cycles e.g. 24h" + defaultHelpPostfix(Frequency), + Optional: true, + }, + } +) diff --git a/internal/config/certs.go b/internal/config/certs.go new file mode 100644 index 00000000..e2ba44eb --- /dev/null +++ b/internal/config/certs.go @@ -0,0 +1,114 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package config + +import ( + "bytes" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "errors" + "os" + + "github.com/minio/pkg/v3/env" +) + +// EnvCertPassword is the environment variable which contains the password used +// to decrypt the TLS private key. It must be set if the TLS private key is +// password protected. +const EnvCertPassword = "MINIO_CERT_PASSWD" + +// ParsePublicCertFile - parses public cert into its *x509.Certificate equivalent. +func ParsePublicCertFile(certFile string) (x509Certs []*x509.Certificate, err error) { + // Read certificate file. + var data []byte + if data, err = os.ReadFile(certFile); err != nil { + return nil, err + } + + // Trimming leading and tailing white spaces. + data = bytes.TrimSpace(data) + + // Parse all certs in the chain. + current := data + for len(current) > 0 { + var pemBlock *pem.Block + if pemBlock, current = pem.Decode(current); pemBlock == nil { + return nil, ErrTLSUnexpectedData(nil).Msgf("Could not read PEM block from file %s", certFile) + } + + var x509Cert *x509.Certificate + if x509Cert, err = x509.ParseCertificate(pemBlock.Bytes); err != nil { + return nil, ErrTLSUnexpectedData(nil).Msgf("Failed to parse `%s`: %s", certFile, err.Error()) + } + + x509Certs = append(x509Certs, x509Cert) + } + + if len(x509Certs) == 0 { + return nil, ErrTLSUnexpectedData(nil).Msgf("Empty public certificate file %s", certFile) + } + + return x509Certs, nil +} + +// LoadX509KeyPair - load an X509 key pair (private key , certificate) +// from the provided paths. The private key may be encrypted and is +// decrypted using the ENV_VAR: MINIO_CERT_PASSWD. +func LoadX509KeyPair(certFile, keyFile string) (tls.Certificate, error) { + certPEMBlock, err := os.ReadFile(certFile) + if err != nil { + return tls.Certificate{}, ErrTLSReadError(nil).Msgf("Unable to read the public key: %s", err) + } + keyPEMBlock, err := os.ReadFile(keyFile) + if err != nil { + return tls.Certificate{}, ErrTLSReadError(nil).Msgf("Unable to read the private key: %s", err) + } + key, rest := pem.Decode(keyPEMBlock) + if len(rest) > 0 { + return tls.Certificate{}, ErrTLSUnexpectedData(nil).Msgf("The private key contains additional data") + } + if key == nil { + return tls.Certificate{}, ErrTLSUnexpectedData(nil).Msgf("The private key is not readable") + } + if x509.IsEncryptedPEMBlock(key) { + password := env.Get(EnvCertPassword, "") + if len(password) == 0 { + return tls.Certificate{}, ErrTLSNoPassword(nil) + } + decryptedKey, decErr := x509.DecryptPEMBlock(key, []byte(password)) + if decErr != nil { + return tls.Certificate{}, ErrTLSWrongPassword(decErr) + } + keyPEMBlock = pem.EncodeToMemory(&pem.Block{Type: key.Type, Bytes: decryptedKey}) + } + cert, err := tls.X509KeyPair(certPEMBlock, keyPEMBlock) + if err != nil { + return tls.Certificate{}, ErrTLSUnexpectedData(nil).Msg(err.Error()) + } + return cert, nil +} + +// EnsureCertAndKey checks if both client certificate and key paths are provided +func EnsureCertAndKey(clientCert, clientKey string) error { + if (clientCert != "" && clientKey == "") || + (clientCert == "" && clientKey != "") { + return errors.New("cert and key must be specified as a pair") + } + return nil +} diff --git a/internal/config/certs_test.go b/internal/config/certs_test.go new file mode 100644 index 00000000..d102a249 --- /dev/null +++ b/internal/config/certs_test.go @@ -0,0 +1,454 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package config + +import ( + "os" + "testing" +) + +func createTempFile(t testing.TB, prefix, content string) (tempFile string, err error) { + t.Helper() + var tmpfile *os.File + + if tmpfile, err = os.CreateTemp(t.TempDir(), prefix); err != nil { + return tempFile, err + } + + if _, err = tmpfile.Write([]byte(content)); err != nil { + return tempFile, err + } + + if err = tmpfile.Close(); err != nil { + return tempFile, err + } + + tempFile = tmpfile.Name() + return tempFile, err +} + +func TestParsePublicCertFile(t *testing.T) { + tempFile1, err := createTempFile(t, "public-cert-file", "") + if err != nil { + t.Fatalf("Unable to create temporary file. %v", err) + } + defer os.Remove(tempFile1) + + tempFile2, err := createTempFile(t, "public-cert-file", `-----BEGIN CERTIFICATE----- +MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa +WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN +aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN +AQkBFg50ZXN0c0BtaW5pby5pbzAeFw0xNjEwMTQxMTM0MjJaFw0xNzEwMTQxMTM0 +MjJaMH8xCzAJBgNVBAYTAlpZMQ4wDAYDVQQIEwVNaW5pbzERMA8GA1UEBxMISW50 +ZXJuZXQxDjAMBgNVBA-some-junk-Q4wDAYDVQQLEwVNaW5pbzEOMAwGA1UEAxMF +TWluaW8xHTAbBgkqhkiG9w0BCQEWDnRlc3RzQG1pbmlvLmlvMIGfMA0GCSqGSIb3 +DQEBAQUAA4GNADCBiQKBgQDwNUYB/Sj79WsUE8qnXzzh2glSzWxUE79sCOpQYK83 +HWkrl5WxlG8ZxDR1IQV9Ex/lzigJu8G+KXahon6a+3n5GhNrYRe5kIXHQHz0qvv4 +aMulqlnYpvSfC83aaO9GVBtwXS/O4Nykd7QBg4nZlazVmsGk7POOjhpjGShRsqpU +JwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBALqjOA6bD8BEl7hkQ8XwX/owSAL0URDe +nUfCOsXgIIAqgw4uTCLOfCJVZNKmRT+KguvPAQ6Z80vau2UxPX5Q2Q+OHXDRrEnK +FjqSBgLP06Qw7a++bshlWGTt5bHWOneW3EQikedckVuIKPkOCib9yGi4VmBBjdFE +M9ofSEt/bdRD +-----END CERTIFICATE-----`) + if err != nil { + t.Fatalf("Unable to create temporary file. %v", err) + } + defer os.Remove(tempFile2) + + tempFile3, err := createTempFile(t, "public-cert-file", `-----BEGIN CERTIFICATE----- +MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa +WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN +aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN +AQkBFg50ZXN0c0BtaW5pby5pbzAeFw0xNjEwMTQxMTM0MjJaFw0xNzEwMTQxMTM0 +MjJaMH8xCzAJBgNVBAYTAlpZMQ4wDAYDVQQIEwVNaW5pbzERMA8GA1UEBxMISW50 +ZXJuZXQxDjAMBgNVBAabababababaQ4wDAYDVQQLEwVNaW5pbzEOMAwGA1UEAxMF +TWluaW8xHTAbBgkqhkiG9w0BCQEWDnRlc3RzQG1pbmlvLmlvMIGfMA0GCSqGSIb3 +DQEBAQUAA4GNADCBiQKBgQDwNUYB/Sj79WsUE8qnXzzh2glSzWxUE79sCOpQYK83 +HWkrl5WxlG8ZxDR1IQV9Ex/lzigJu8G+KXahon6a+3n5GhNrYRe5kIXHQHz0qvv4 +aMulqlnYpvSfC83aaO9GVBtwXS/O4Nykd7QBg4nZlazVmsGk7POOjhpjGShRsqpU +JwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBALqjOA6bD8BEl7hkQ8XwX/owSAL0URDe +nUfCOsXgIIAqgw4uTCLOfCJVZNKmRT+KguvPAQ6Z80vau2UxPX5Q2Q+OHXDRrEnK +FjqSBgLP06Qw7a++bshlWGTt5bHWOneW3EQikedckVuIKPkOCib9yGi4VmBBjdFE +M9ofSEt/bdRD +-----END CERTIFICATE-----`) + if err != nil { + t.Fatalf("Unable to create temporary file. %v", err) + } + defer os.Remove(tempFile3) + + tempFile4, err := createTempFile(t, "public-cert-file", `-----BEGIN CERTIFICATE----- +MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa +WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN +aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN +AQkBFg50ZXN0c0BtaW5pby5pbzAeFw0xNjEwMTQxMTM0MjJaFw0xNzEwMTQxMTM0 +MjJaMH8xCzAJBgNVBAYTAlpZMQ4wDAYDVQQIEwVNaW5pbzERMA8GA1UEBxMISW50 +ZXJuZXQxDjAMBgNVBAoTBU1pbmlvMQ4wDAYDVQQLEwVNaW5pbzEOMAwGA1UEAxMF +TWluaW8xHTAbBgkqhkiG9w0BCQEWDnRlc3RzQG1pbmlvLmlvMIGfMA0GCSqGSIb3 +DQEBAQUAA4GNADCBiQKBgQDwNUYB/Sj79WsUE8qnXzzh2glSzWxUE79sCOpQYK83 +HWkrl5WxlG8ZxDR1IQV9Ex/lzigJu8G+KXahon6a+3n5GhNrYRe5kIXHQHz0qvv4 +aMulqlnYpvSfC83aaO9GVBtwXS/O4Nykd7QBg4nZlazVmsGk7POOjhpjGShRsqpU +JwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBALqjOA6bD8BEl7hkQ8XwX/owSAL0URDe +nUfCOsXgIIAqgw4uTCLOfCJVZNKmRT+KguvPAQ6Z80vau2UxPX5Q2Q+OHXDRrEnK +FjqSBgLP06Qw7a++bshlWGTt5bHWOneW3EQikedckVuIKPkOCib9yGi4VmBBjdFE +M9ofSEt/bdRD +-----END CERTIFICATE-----`) + if err != nil { + t.Fatalf("Unable to create temporary file. %v", err) + } + defer os.Remove(tempFile4) + + tempFile5, err := createTempFile(t, "public-cert-file", `-----BEGIN CERTIFICATE----- +MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa +WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN +aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN +AQkBFg50ZXN0c0BtaW5pby5pbzAeFw0xNjEwMTQxMTM0MjJaFw0xNzEwMTQxMTM0 +MjJaMH8xCzAJBgNVBAYTAlpZMQ4wDAYDVQQIEwVNaW5pbzERMA8GA1UEBxMISW50 +ZXJuZXQxDjAMBgNVBAoTBU1pbmlvMQ4wDAYDVQQLEwVNaW5pbzEOMAwGA1UEAxMF +TWluaW8xHTAbBgkqhkiG9w0BCQEWDnRlc3RzQG1pbmlvLmlvMIGfMA0GCSqGSIb3 +DQEBAQUAA4GNADCBiQKBgQDwNUYB/Sj79WsUE8qnXzzh2glSzWxUE79sCOpQYK83 +HWkrl5WxlG8ZxDR1IQV9Ex/lzigJu8G+KXahon6a+3n5GhNrYRe5kIXHQHz0qvv4 +aMulqlnYpvSfC83aaO9GVBtwXS/O4Nykd7QBg4nZlazVmsGk7POOjhpjGShRsqpU +JwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBALqjOA6bD8BEl7hkQ8XwX/owSAL0URDe +nUfCOsXgIIAqgw4uTCLOfCJVZNKmRT+KguvPAQ6Z80vau2UxPX5Q2Q+OHXDRrEnK +FjqSBgLP06Qw7a++bshlWGTt5bHWOneW3EQikedckVuIKPkOCib9yGi4VmBBjdFE +M9ofSEt/bdRD +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +MIICdTCCAd4CCQCO5G/W1xcE9TANBgkqhkiG9w0BAQUFADB/MQswCQYDVQQGEwJa +WTEOMAwGA1UECBMFTWluaW8xETAPBgNVBAcTCEludGVybmV0MQ4wDAYDVQQKEwVN +aW5pbzEOMAwGA1UECxMFTWluaW8xDjAMBgNVBAMTBU1pbmlvMR0wGwYJKoZIhvcN +AQkBFg50ZXN0c0BtaW5pby5pbzAeFw0xNjEwMTQxMTM0MjJaFw0xNzEwMTQxMTM0 +MjJaMH8xCzAJBgNVBAYTAlpZMQ4wDAYDVQQIEwVNaW5pbzERMA8GA1UEBxMISW50 +ZXJuZXQxDjAMBgNVBAoTBU1pbmlvMQ4wDAYDVQQLEwVNaW5pbzEOMAwGA1UEAxMF +TWluaW8xHTAbBgkqhkiG9w0BCQEWDnRlc3RzQG1pbmlvLmlvMIGfMA0GCSqGSIb3 +DQEBAQUAA4GNADCBiQKBgQDwNUYB/Sj79WsUE8qnXzzh2glSzWxUE79sCOpQYK83 +HWkrl5WxlG8ZxDR1IQV9Ex/lzigJu8G+KXahon6a+3n5GhNrYRe5kIXHQHz0qvv4 +aMulqlnYpvSfC83aaO9GVBtwXS/O4Nykd7QBg4nZlazVmsGk7POOjhpjGShRsqpU +JwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBALqjOA6bD8BEl7hkQ8XwX/owSAL0URDe +nUfCOsXgIIAqgw4uTCLOfCJVZNKmRT+KguvPAQ6Z80vau2UxPX5Q2Q+OHXDRrEnK +FjqSBgLP06Qw7a++bshlWGTt5bHWOneW3EQikedckVuIKPkOCib9yGi4VmBBjdFE +M9ofSEt/bdRD +-----END CERTIFICATE-----`) + if err != nil { + t.Fatalf("Unable to create temporary file. %v", err) + } + defer os.Remove(tempFile5) + + testCases := []struct { + certFile string + expectedResultLen int + expectedErr bool + }{ + {"nonexistent-file", 0, true}, + {tempFile1, 0, true}, + {tempFile2, 0, true}, + {tempFile3, 0, true}, + {tempFile4, 1, false}, + {tempFile5, 2, false}, + } + + for _, testCase := range testCases { + certs, err := ParsePublicCertFile(testCase.certFile) + if !testCase.expectedErr && err != nil { + t.Fatalf("error: expected = , got = %v", err) + } + if testCase.expectedErr && err == nil { + t.Fatal("error: expected err, got = ") + } + if len(certs) != testCase.expectedResultLen { + t.Fatalf("certs: expected = %v, got = %v", testCase.expectedResultLen, len(certs)) + } + } +} + +func TestLoadX509KeyPair(t *testing.T) { + t.Cleanup(func() { + os.Unsetenv(EnvCertPassword) + }) + for i, testCase := range loadX509KeyPairTests { + privateKey, err := createTempFile(t, "private.key", testCase.privateKey) + if err != nil { + t.Fatalf("Test %d: failed to create tmp private key file: %v", i, err) + } + certificate, err := createTempFile(t, "public.crt", testCase.certificate) + if err != nil { + os.Remove(privateKey) + t.Fatalf("Test %d: failed to create tmp certificate file: %v", i, err) + } + + if testCase.password != "" { + t.Setenv(EnvCertPassword, testCase.password) + } + _, err = LoadX509KeyPair(certificate, privateKey) + if err != nil && !testCase.shouldFail { + t.Errorf("Test %d: test should succeed but it failed: %v", i, err) + } + if err == nil && testCase.shouldFail { + t.Errorf("Test %d: test should fail but it succeed", i) + } + os.Remove(privateKey) + os.Remove(certificate) + } +} + +var loadX509KeyPairTests = []struct { + password string + privateKey, certificate string + shouldFail bool +}{ + { + password: "", + privateKey: "", + certificate: `-----BEGIN CERTIFICATE----- +MIIDiTCCAnGgAwIBAgIJAK5m5S7EE46kMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV +BAYTAlVTMQ4wDAYDVQQIDAVzdGF0ZTERMA8GA1UEBwwIbG9jYXRpb24xFTATBgNV +BAoMDG9yZ2FuaXphdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIxODE4 +MDUyOFoXDTI3MTIxNjE4MDUyOFowWzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBXN0 +YXRlMREwDwYDVQQHDAhsb2NhdGlvbjEVMBMGA1UECgwMb3JnYW5pemF0aW9uMRIw +EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +AQDPJfYY5Dhsntrqwyu7ZgKM/zrlKEjCwGHhWJBdZdeZCHQlY8ISrtDxxp2XMmI6 +HsszalEhNF9fk3vSXWclTuomG03fgGzP4R6QpcwGUCxhRF1J+0b64Yi8pw2uEGsR +GuMwLhGorcWalNoihgHc0BQ4vO8aaTNTX7iD06olesP6vGNu/S8h0VomE+0v9qYc +VF66Zaiv/6OmxAtDpElJjVd0mY7G85BlDlFrVwzd7zhRiuJZ4iDg749Xt9GuuKla +Dvr14glHhP4dQgUbhluJmIHMdx2ZPjk+5FxaDK6I9IUpxczFDe4agDE6lKzU1eLd +cCXRWFOf6q9lTB1hUZfmWfTxAgMBAAGjUDBOMB0GA1UdDgQWBBTQh7lDTq+8salD +0HBNILochiiNaDAfBgNVHSMEGDAWgBTQh7lDTq+8salD0HBNILochiiNaDAMBgNV +HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAqi9LycxcXKNSDXaPkCKvw7RQy +iMBDGm1kIY++p3tzbUGuaeu85TsswKnqd50AullEU+aQxRRJGfR8eSKzQJMBXLMQ +b4ptYCc5OrZtRHT8NaZ/df2tc6I88kN8dBu6ybcNGsevXA/iNX3kKLW7naxdr5jj +KUudWSuqDCjCmQa5bYb9H6DreLH2lUItSWBa/YmeZ3VSezDCd+XYO53QKwZVj8Jb +bulZmoo7e7HO1qecEzWKL10UYyEbG3UDPtw+NZc142ZYeEhXQ0dsstGAO5hf3hEl +kQyKGUTpDbKLuyYMFsoH73YLjBqNe+UEhPwE+FWpcky1Sp9RTx/oMLpiZaPR +-----END CERTIFICATE-----`, + shouldFail: true, + }, + { + password: "foobar", + privateKey: `-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-128-CBC,CC483BF11678C35F9F02A1AD85DAE285 + +nMDFd+Qxk1f+S7LwMitmMofNXYNbCY4L1QEqPOOx5wnjNF1wSxmEkL7+h8W4Y/vb +AQt/7TCcUSuSqEMl45nUIcCbhBos5wz+ShvFiez3qKwmR5HSURvqyN6PIJeAbU+h +uw/cvAQsCH1Cq+gYkDJqjrizPhGqg7mSkqyeST3PbOl+ZXc0wynIjA34JSwO3c5j +cF7XKHETtNGj1+AiLruX4wYZAJwQnK375fCoNVMO992zC6K83d8kvGMUgmJjkiIj +q3s4ymFGfoo0S/XNDQXgE5A5QjAKRKUyW2i7pHIIhTyOpeJQeFHDi2/zaZRxoCog +lD2/HKLi5xJtRelZaaGyEJ20c05VzaSZ+EtRIN33foNdyQQL6iAUU3hJ6JlcmRIB +bRfX4XPH1w9UfFU5ZKwUciCoDcL65bsyv/y56ItljBp7Ok+UUKl0H4myFNOSfsuU +IIj4neslnAvwQ8SN4XUpug+7pGF+2m/5UDwRzSUN1H2RfgWN95kqR+tYqCq/E+KO +i0svzFrljSHswsFoPBqKngI7hHwc9QTt5q4frXwj9I4F6HHrTKZnC5M4ef26sbJ1 +r7JRmkt0h/GfcS355b0uoBTtF1R8tSJo85Zh47wE+ucdjEvy9/pjnzKqIoJo9bNZ +ri+ue7GhH5EUca1Kd10bH8FqTF+8AHh4yW6xMxSkSgFGp7KtraAVpdp+6kosymqh +dz9VMjA8i28btfkS2isRaCpyumaFYJ3DJMFYhmeyt6gqYovmRLX0qrBf8nrkFTAA +ZmykWsc8ErsCudxlDmKVemuyFL7jtm9IRPq+Jh+IrmixLJFx8PKkNAM6g+A8irx8 +piw+yhRsVy5Jk2QeIqvbpxN6BfCNcix4sWkusiCJrAqQFuSm26Mhh53Ig1DXG4d3 +6QY1T8tW80Q6JHUtDR+iOPqW6EmrNiEopzirvhGv9FicXZ0Lo2yKJueeeihWhFLL +GmlnCjWVMO4hoo8lWCHv95JkPxGMcecCacKKUbHlXzCGyw3+eeTEHMWMEhziLeBy +HZJ1/GReI3Sx7XlUCkG4468Yz3PpmbNIk/U5XKE7TGuxKmfcWQpu022iF/9DrKTz +KVhKimCBXJX345bCFe1rN2z5CV6sv87FkMs5Y+OjPw6qYFZPVKO2TdUUBcpXbQMg +UW+Kuaax9W7214Stlil727MjRCiH1+0yODg4nWj4pTSocA5R3pn5cwqrjMu97OmL +ESx4DHmy4keeSy3+AIAehCZlwgeLb70/xCSRhJMIMS9Q6bz8CPkEWN8bBZt95oeo +37LqZ7lNmq61fs1x1tq0VUnI9HwLFEnsiubp6RG0Yu8l/uImjjjXa/ytW2GXrfUi +zM22dOntu6u23iBxRBJRWdFTVUz7qrdu+PHavr+Y7TbCeiBwiypmz5llf823UIVx +btamI6ziAq2gKZhObIhut7sjaLkAyTLlNVkNN1WNaplAXpW25UFVk93MHbvZ27bx +9iLGs/qB2kDTUjffSQoHTLY1GoLxv83RgVspUGQjslztEEpWfYvGfVLcgYLv933B +aRW9BRoNZ0czKx7Lhuwjreyb5IcWDarhC8q29ZkkWsQQonaPb0kTEFJul80Yqk0k +-----END RSA PRIVATE KEY-----`, + certificate: `-----BEGIN CERTIFICATE----- +MIIDiTCCAnGgAwIBAgIJAK5m5S7EE46kMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV +BAYTAlVTMQ4wDAYDVQQIDAVzdGF0ZTERMA8GA1UEBwwIbG9jYXRpb24xFTATBgNV +BAoMDG9yZ2FuaXphdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIxODE4 +MDUyOFoXDTI3MTIxNjE4MDUyOFowWzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBXN0 +YXRlMREwDwYDVQQHDAhsb2NhdGlvbjEVMBMGA1UECgwMb3JnYW5pemF0aW9uMRIw +EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +AQDPJfYY5Dhsntrqwyu7ZgKM/zrlKEjCwGHhWJBdZdeZCHQlY8ISrtDxxp2XMmI6 +HsszalEhNF9fk3vSXWclTuomG03fgGzP4R6QpcwGUCxhRF1J+0b64Yi8pw2uEGsR +GuMwLhGorcWalNoihgHc0BQ4vO8aaTNTX7iD06olesP6vGNu/S8h0VomE+0v9qYc +VF66Zaiv/6OmxAtDpElJjVd0mY7G85BlDlFrVwzd7zhRiuJZ4iDg749Xt9GuuKla +Dvr14glHhP4dQgUbhluJmIHMdx2ZPjk+5FxaDK6I9IUpxczFDe4agDE6lKzU1eLd +cCXRWFOf6q9lTB1hUZfmWfTxAgMBAAGjUDBOMB0GA1UdDgQWBBTQh7lDTq+8salD +0HBNILochiiNaDAfBgNVHSMEGDAWgBTQh7lDTq+8salD0HBNILochiiNaDAMBgNV +HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAqi9LycxcXKNSDXaPkCKvw7RQy +iMBDGm1kIY++p3tzbUGuaeu85TsswKnqd50AullEU+aQxRRJGfR8eSKzQJMBXLMQ +b4ptYCc5OrZtRHT8NaZ/df2tc6I88kN8dBu6ybcNGsevXA/iNX3kKLW7naxdr5jj +KUudWSuqDCjCmQa5bYb9H6DreLH2lUItSWBa/YmeZ3VSezDCd+XYO53QKwZVj8Jb +bulZmoo7e7HO1qecEzWKL10UYyEbG3UDPtw+NZc142ZYeEhXQ0dsstGAO5hf3hEl +kQyKGUTpDbKLuyYMFsoH73YLjBqNe+UEhPwE+FWpcky1Sp9RTx/oMLpiZaPR +-----END CERTIFICATE-----`, + shouldFail: false, + }, + { + password: "password", + privateKey: `-----BEGIN RSA PRIVATE KEY----- +Proc-Type: 4,ENCRYPTED +DEK-Info: AES-128-CBC,CC483BF11678C35F9F02A1AD85DAE285 + +nMDFd+Qxk1f+S7LwMitmMofNXYNbCY4L1QEqPOOx5wnjNF1wSxmEkL7+h8W4Y/vb +AQt/7TCcUSuSqEMl45nUIcCbhBos5wz+ShvFiez3qKwmR5HSURvqyN6PIJeAbU+h +uw/cvAQsCH1Cq+gYkDJqjrizPhGqg7mSkqyeST3PbOl+ZXc0wynIjA34JSwO3c5j +cF7XKHETtNGj1+AiLruX4wYZAJwQnK375fCoNVMO992zC6K83d8kvGMUgmJjkiIj +q3s4ymFGfoo0S/XNDQXgE5A5QjAKRKUyW2i7pHIIhTyOpeJQeFHDi2/zaZRxoCog +lD2/HKLi5xJtRelZaaGyEJ20c05VzaSZ+EtRIN33foNdyQQL6iAUU3hJ6JlcmRIB +bRfX4XPH1w9UfFU5ZKwUciCoDcL65bsyv/y56ItljBp7Ok+UUKl0H4myFNOSfsuU +IIj4neslnAvwQ8SN4XUpug+7pGF+2m/5UDwRzSUN1H2RfgWN95kqR+tYqCq/E+KO +i0svzFrljSHswsFoPBqKngI7hHwc9QTt5q4frXwj9I4F6HHrTKZnC5M4ef26sbJ1 +r7JRmkt0h/GfcS355b0uoBTtF1R8tSJo85Zh47wE+ucdjEvy9/pjnzKqIoJo9bNZ +ri+ue7GhH5EUca1Kd10bH8FqTF+8AHh4yW6xMxSkSgFGp7KtraAVpdp+6kosymqh +dz9VMjA8i28btfkS2isRaCpyumaFYJ3DJMFYhmeyt6gqYovmRLX0qrBf8nrkFTAA +ZmykWsc8ErsCudxlDmKVemuyFL7jtm9IRPq+Jh+IrmixLJFx8PKkNAM6g+A8irx8 +piw+yhRsVy5Jk2QeIqvbpxN6BfCNcix4sWkusiCJrAqQFuSm26Mhh53Ig1DXG4d3 +6QY1T8tW80Q6JHUtDR+iOPqW6EmrNiEopzirvhGv9FicXZ0Lo2yKJueeeihWhFLL +GmlnCjWVMO4hoo8lWCHv95JkPxGMcecCacKKUbHlXzCGyw3+eeTEHMWMEhziLeBy +HZJ1/GReI3Sx7XlUCkG4468Yz3PpmbNIk/U5XKE7TGuxKmfcWQpu022iF/9DrKTz +KVhKimCBXJX345bCFe1rN2z5CV6sv87FkMs5Y+OjPw6qYFZPVKO2TdUUBcpXbQMg +UW+Kuaax9W7214Stlil727MjRCiH1+0yODg4nWj4pTSocA5R3pn5cwqrjMu97OmL +ESx4DHmy4keeSy3+AIAehCZlwgeLb70/xCSRhJMIMS9Q6bz8CPkEWN8bBZt95oeo +37LqZ7lNmq61fs1x1tq0VUnI9HwLFEnsiubp6RG0Yu8l/uImjjjXa/ytW2GXrfUi +zM22dOntu6u23iBxRBJRWdFTVUz7qrdu+PHavr+Y7TbCeiBwiypmz5llf823UIVx +btamI6ziAq2gKZhObIhut7sjaLkAyTLlNVkNN1WNaplAXpW25UFVk93MHbvZ27bx +9iLGs/qB2kDTUjffSQoHTLY1GoLxv83RgVspUGQjslztEEpWfYvGfVLcgYLv933B +aRW9BRoNZ0czKx7Lhuwjreyb5IcWDarhC8q29ZkkWsQQonaPb0kTEFJul80Yqk0k +-----END RSA PRIVATE KEY-----`, + certificate: `-----BEGIN CERTIFICATE----- +MIIDiTCCAnGgAwIBAgIJAK5m5S7EE46kMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV +BAYTAlVTMQ4wDAYDVQQIDAVzdGF0ZTERMA8GA1UEBwwIbG9jYXRpb24xFTATBgNV +BAoMDG9yZ2FuaXphdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIxODE4 +MDUyOFoXDTI3MTIxNjE4MDUyOFowWzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBXN0 +YXRlMREwDwYDVQQHDAhsb2NhdGlvbjEVMBMGA1UECgwMb3JnYW5pemF0aW9uMRIw +EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +AQDPJfYY5Dhsntrqwyu7ZgKM/zrlKEjCwGHhWJBdZdeZCHQlY8ISrtDxxp2XMmI6 +HsszalEhNF9fk3vSXWclTuomG03fgGzP4R6QpcwGUCxhRF1J+0b64Yi8pw2uEGsR +GuMwLhGorcWalNoihgHc0BQ4vO8aaTNTX7iD06olesP6vGNu/S8h0VomE+0v9qYc +VF66Zaiv/6OmxAtDpElJjVd0mY7G85BlDlFrVwzd7zhRiuJZ4iDg749Xt9GuuKla +Dvr14glHhP4dQgUbhluJmIHMdx2ZPjk+5FxaDK6I9IUpxczFDe4agDE6lKzU1eLd +cCXRWFOf6q9lTB1hUZfmWfTxAgMBAAGjUDBOMB0GA1UdDgQWBBTQh7lDTq+8salD +0HBNILochiiNaDAfBgNVHSMEGDAWgBTQh7lDTq+8salD0HBNILochiiNaDAMBgNV +HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQAqi9LycxcXKNSDXaPkCKvw7RQy +iMBDGm1kIY++p3tzbUGuaeu85TsswKnqd50AullEU+aQxRRJGfR8eSKzQJMBXLMQ +b4ptYCc5OrZtRHT8NaZ/df2tc6I88kN8dBu6ybcNGsevXA/iNX3kKLW7naxdr5jj +KUudWSuqDCjCmQa5bYb9H6DreLH2lUItSWBa/YmeZ3VSezDCd+XYO53QKwZVj8Jb +bulZmoo7e7HO1qecEzWKL10UYyEbG3UDPtw+NZc142ZYeEhXQ0dsstGAO5hf3hEl +kQyKGUTpDbKLuyYMFsoH73YLjBqNe+UEhPwE+FWpcky1Sp9RTx/oMLpiZaPR +-----END CERTIFICATE-----`, + shouldFail: true, + }, + { + password: "", + privateKey: `-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA4K9Qq7vMY2bGkrdFAYpBYNLlCgnnFU+0pi+N+3bjuWmfX/kw +WXBa3SDqKD08PWWzwvBSLPCCUV2IuUd7tBa1pJ2wXkdoDeI5InYHJKrXbSZonni6 +Bex7sgnqV/9o8xFkSOleoQWZgyeKGxtt0J/Z+zhpH+zXahwM4wOL3yzLSQt+NCKM +6N96zXYi16DEa89fYwRxPwE1XTRc7Ddggqx+4iRHvYG0fyTNcPB/+UiFw59EE1Sg +QIyTVntVqpsb6s8XdkFxURoLxefhcMVf2kU0T04OWI3gmeavKfTcj8Z2/bjPSsqP +mgkADv9Ru6VnSK/96TW/NwxWJ32PBz6Sbl9LdwIDAQABAoIBABVh+d5uH/RxyoIZ ++PI9kx1A1NVQvfI0RK/wJKYC2YdCuw0qLOTGIY+b20z7DumU7TenIVrvhKdzrFhd +qjMoWh8RdsByMT/pAKD79JATxi64EgrK2IFJ0TfPY8L+JqHDTPT3aK8QVly5/ZW4 +1YmePOOAqdiE9Lc/diaApuYVYD9SL/X7fYs1ezOB4oGXoz0rthX77zHMxcEurpK3 +VgSnaq7FYTVY7GrFB+ASiAlDIyLwztz08Ijn8aG0QAZ8GFuPGSmPMXWjLwFhRZsa +Gfy5BYiA0bVSnQSPHzAnHu9HyGlsdouVPPvJB3SrvMl+BFhZiUuR8OGSob7z7hfI +hMyHbNECgYEA/gyG7sHAb5mPkhq9JkTv+LrMY5NDZKYcSlbvBlM3kd6Ib3Hxl+6T +FMq2TWIrh2+mT1C14htziHd05dF6St995Tby6CJxTj6a/2Odnfm+JcOou/ula4Sz +92nIGlGPTJXstDbHGnRCpk6AomXK02stydTyrCisOw1H+LyTG6aT0q8CgYEA4mkO +hfLJkgmJzWIhxHR901uWHz/LId0gC6FQCeaqWmRup6Bl97f0U6xokw4tw8DJOncF +yZpYRXUXhdv/FXCjtXvAhKIX5+e+3dlzPHIdekSfcY00ip/ifAS1OyVviJia+cna +eJgq8WLHxJZim9Ah93NlPyiqGPwtasub90qjZbkCgYEA35WK02o1wII3dvCNc7bM +M+3CoAglEdmXoF1uM/TdPUXKcbqoU3ymeXAGjYhOov3CMp/n0z0xqvLnMLPxmx+i +ny6DDYXyjlhO9WFogHYhwP636+mHJl8+PAsfDvqk0VRJZDmpdUDIv7DrSQGpRfRX +8f+2K4oIOlhv9RuRpI4wHwUCgYB8OjaMyn1NEsy4k2qBt4U+jhcdyEv1pbWqi/U1 +qYm5FTgd44VvWVDHBGdQoMv9h28iFCJpzrU2Txv8B4y7v9Ujg+ZLIAFL7j0szt5K +wTZpWvO9Q0Qb98Q2VgL2lADRiyIlglrMJnoRfiisNfOfGKE6e+eGsxI5qUxmN5e5 +JQvoiQKBgQCqgyuUBIu/Qsb3qUED/o0S5wCel43Yh/Rl+mxDinOUvJfKJSW2SyEk ++jDo0xw3Opg6ZC5Lj2V809LA/XteaIuyhRuqOopjhHIvIvrYGe+2O8q9/Mv40BYW +0BhJ/Gdseps0C6Z5mTT5Fee4YVlGZuyuNKmKTd4JmqInfBV3ncMWQg== +-----END RSA PRIVATE KEY-----`, + certificate: `-----BEGIN CERTIFICATE----- +MIIDiTCCAnGgAwIBAgIJAIb84Z5Mh31iMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV +BAYTAlVTMQ4wDAYDVQQIDAVzdGF0ZTERMA8GA1UEBwwIbG9jYXRpb24xFTATBgNV +BAoMDG9yZ2FuaXphdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIxODE4 +NTcyM1oXDTI3MTIxNjE4NTcyM1owWzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBXN0 +YXRlMREwDwYDVQQHDAhsb2NhdGlvbjEVMBMGA1UECgwMb3JnYW5pemF0aW9uMRIw +EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +AQDgr1Cru8xjZsaSt0UBikFg0uUKCecVT7SmL437duO5aZ9f+TBZcFrdIOooPTw9 +ZbPC8FIs8IJRXYi5R3u0FrWknbBeR2gN4jkidgckqtdtJmieeLoF7HuyCepX/2jz +EWRI6V6hBZmDJ4obG23Qn9n7OGkf7NdqHAzjA4vfLMtJC340Iozo33rNdiLXoMRr +z19jBHE/ATVdNFzsN2CCrH7iJEe9gbR/JM1w8H/5SIXDn0QTVKBAjJNWe1Wqmxvq +zxd2QXFRGgvF5+FwxV/aRTRPTg5YjeCZ5q8p9NyPxnb9uM9Kyo+aCQAO/1G7pWdI +r/3pNb83DFYnfY8HPpJuX0t3AgMBAAGjUDBOMB0GA1UdDgQWBBQ2/bSCHscnoV+0 +d+YJxLu4XLSNIDAfBgNVHSMEGDAWgBQ2/bSCHscnoV+0d+YJxLu4XLSNIDAMBgNV +HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQC6p4gPwmkoDtRsP1c8IWgXFka+ +Q59oe79ZK1RqDE6ZZu0rgw07rPzKr4ofW4hTxnx7PUgKOhWLq9VvwEC/9tDbD0Gw +SKknRZZOiEE3qUZbwNtHMd4UBzpzChTRC6RcwC5zT1/WICMUHxa4b8E2umJuf3Qd +5Y23sXEESx5evr49z6DLcVe2i70o2wJeWs2kaXqhCJt0X7z0rnYqjfFdvxd8dyzt +1DXmE45cLadpWHDg26DMsdchamgnqEo79YUxkH6G/Cb8ZX4igQ/CsxCDOKvccjHO +OncDtuIpK8O7OyfHP3+MBpUFG4P6Ctn7RVcZe9fQweTpfAy18G+loVzuUeOD +-----END CERTIFICATE-----`, + shouldFail: false, + }, + { + password: "foobar", + privateKey: `-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEA4K9Qq7vMY2bGkrdFAYpBYNLlCgnnFU+0pi+N+3bjuWmfX/kw +WXBa3SDqKD08PWWzwvBSLPCCUV2IuUd7tBa1pJ2wXkdoDeI5InYHJKrXbSZonni6 +Bex7sgnqV/9o8xFkSOleoQWZgyeKGxtt0J/Z+zhpH+zXahwM4wOL3yzLSQt+NCKM +6N96zXYi16DEa89fYwRxPwE1XTRc7Ddggqx+4iRHvYG0fyTNcPB/+UiFw59EE1Sg +QIyTVntVqpsb6s8XdkFxURoLxefhcMVf2kU0T04OWI3gmeavKfTcj8Z2/bjPSsqP +mgkADv9Ru6VnSK/96TW/NwxWJ32PBz6Sbl9LdwIDAQABAoIBABVh+d5uH/RxyoIZ ++PI9kx1A1NVQvfI0RK/wJKYC2YdCuw0qLOTGIY+b20z7DumU7TenIVrvhKdzrFhd +qjMoWh8RdsByMT/pAKD79JATxi64EgrK2IFJ0TfPY8L+JqHDTPT3aK8QVly5/ZW4 +1YmePOOAqdiE9Lc/diaApuYVYD9SL/X7fYs1ezOB4oGXoz0rthX77zHMxcEurpK3 +VgSnaq7FYTVY7GrFB+ASiAlDIyLwztz08Ijn8aG0QAZ8GFuPGSmPMXWjLwFhRZsa +Gfy5BYiA0bVSnQSPHzAnHu9HyGlsdouVPPvJB3SrvMl+BFhZiUuR8OGSob7z7hfI +hMyHbNECgYEA/gyG7sHAb5mPkhq9JkTv+LrMY5NDZKYcSlbvBlM3kd6Ib3Hxl+6T +FMq2TWIrh2+mT1C14htziHd05dF6St995Tby6CJxTj6a/2Odnfm+JcOou/ula4Sz +92nIGlGPTJXstDbHGnRCpk6AomXK02stydTyrCisOw1H+LyTG6aT0q8CgYEA4mkO +hfLJkgmJzWIhxHR901uWHz/LId0gC6FQCeaqWmRup6Bl97f0U6xokw4tw8DJOncF +yZpYRXUXhdv/FXCjtXvAhKIX5+e+3dlzPHIdekSfcY00ip/ifAS1OyVviJia+cna +eJgq8WLHxJZim9Ah93NlPyiqGPwtasub90qjZbkCgYEA35WK02o1wII3dvCNc7bM +M+3CoAglEdmXoF1uM/TdPUXKcbqoU3ymeXAGjYhOov3CMp/n0z0xqvLnMLPxmx+i +ny6DDYXyjlhO9WFogHYhwP636+mHJl8+PAsfDvqk0VRJZDmpdUDIv7DrSQGpRfRX +8f+2K4oIOlhv9RuRpI4wHwUCgYB8OjaMyn1NEsy4k2qBt4U+jhcdyEv1pbWqi/U1 +qYm5FTgd44VvWVDHBGdQoMv9h28iFCJpzrU2Txv8B4y7v9Ujg+ZLIAFL7j0szt5K +wTZpWvO9Q0Qb98Q2VgL2lADRiyIlglrMJnoRfiisNfOfGKE6e+eGsxI5qUxmN5e5 +JQvoiQKBgQCqgyuUBIu/Qsb3qUED/o0S5wCel43Yh/Rl+mxDinOUvJfKJSW2SyEk ++jDo0xw3Opg6ZC5Lj2V809LA/XteaIuyhRuqOopjhHIvIvrYGe+2O8q9/Mv40BYW +0BhJ/Gdseps0C6Z5mTT5Fee4YVlGZuyuNKmKTd4JmqInfBV3ncMWQg== +-----END RSA PRIVATE KEY-----`, + certificate: `-----BEGIN CERTIFICATE----- +MIIDiTCCAnGgAwIBAgIJAIb84Z5Mh31iMA0GCSqGSIb3DQEBCwUAMFsxCzAJBgNV +BAYTAlVTMQ4wDAYDVQQIDAVzdGF0ZTERMA8GA1UEBwwIbG9jYXRpb24xFTATBgNV +BAoMDG9yZ2FuaXphdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MB4XDTE3MTIxODE4 +NTcyM1oXDTI3MTIxNjE4NTcyM1owWzELMAkGA1UEBhMCVVMxDjAMBgNVBAgMBXN0 +YXRlMREwDwYDVQQHDAhsb2NhdGlvbjEVMBMGA1UECgwMb3JnYW5pemF0aW9uMRIw +EAYDVQQDDAlsb2NhbGhvc3QwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIB +AQDgr1Cru8xjZsaSt0UBikFg0uUKCecVT7SmL437duO5aZ9f+TBZcFrdIOooPTw9 +ZbPC8FIs8IJRXYi5R3u0FrWknbBeR2gN4jkidgckqtdtJmieeLoF7HuyCepX/2jz +EWRI6V6hBZmDJ4obG23Qn9n7OGkf7NdqHAzjA4vfLMtJC340Iozo33rNdiLXoMRr +z19jBHE/ATVdNFzsN2CCrH7iJEe9gbR/JM1w8H/5SIXDn0QTVKBAjJNWe1Wqmxvq +zxd2QXFRGgvF5+FwxV/aRTRPTg5YjeCZ5q8p9NyPxnb9uM9Kyo+aCQAO/1G7pWdI +r/3pNb83DFYnfY8HPpJuX0t3AgMBAAGjUDBOMB0GA1UdDgQWBBQ2/bSCHscnoV+0 +d+YJxLu4XLSNIDAfBgNVHSMEGDAWgBQ2/bSCHscnoV+0d+YJxLu4XLSNIDAMBgNV +HRMEBTADAQH/MA0GCSqGSIb3DQEBCwUAA4IBAQC6p4gPwmkoDtRsP1c8IWgXFka+ +Q59oe79ZK1RqDE6ZZu0rgw07rPzKr4ofW4hTxnx7PUgKOhWLq9VvwEC/9tDbD0Gw +SKknRZZOiEE3qUZbwNtHMd4UBzpzChTRC6RcwC5zT1/WICMUHxa4b8E2umJuf3Qd +5Y23sXEESx5evr49z6DLcVe2i70o2wJeWs2kaXqhCJt0X7z0rnYqjfFdvxd8dyzt +1DXmE45cLadpWHDg26DMsdchamgnqEo79YUxkH6G/Cb8ZX4igQ/CsxCDOKvccjHO +OncDtuIpK8O7OyfHP3+MBpUFG4P6Ctn7RVcZe9fQweTpfAy18G+loVzuUeOD +-----END CERTIFICATE-----`, + shouldFail: false, + }, +} diff --git a/internal/config/certsinfo.go b/internal/config/certsinfo.go new file mode 100644 index 00000000..013bd1d1 --- /dev/null +++ b/internal/config/certsinfo.go @@ -0,0 +1,94 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package config + +import ( + "crypto/x509" + "crypto/x509/pkix" + "fmt" + "net/http" + "strings" + + color "github.com/minio/minio/internal/color" +) + +// Extra ASN1 OIDs that we may need to handle +var ( + oidEmailAddress = []int{1, 2, 840, 113549, 1, 9, 1} +) + +// printName prints the fields of a distinguished name, which include such +// things as its common name and locality. +func printName(names []pkix.AttributeTypeAndValue, buf *strings.Builder) []string { + values := []string{} + for _, name := range names { + oid := name.Type + //nolint:gocritic + if len(oid) == 4 && oid[0] == 2 && oid[1] == 5 && oid[2] == 4 { + switch oid[3] { + case 3: + values = append(values, fmt.Sprintf("CN=%s", name.Value)) + case 6: + values = append(values, fmt.Sprintf("C=%s", name.Value)) + case 8: + values = append(values, fmt.Sprintf("ST=%s", name.Value)) + case 10: + values = append(values, fmt.Sprintf("O=%s", name.Value)) + case 11: + values = append(values, fmt.Sprintf("OU=%s", name.Value)) + default: + values = append(values, fmt.Sprintf("UnknownOID=%s", name.Type.String())) + } + } else if oid.Equal(oidEmailAddress) { + values = append(values, fmt.Sprintf("emailAddress=%s", name.Value)) + } else { + values = append(values, fmt.Sprintf("UnknownOID=%s", name.Type.String())) + } + } + if len(values) > 0 { + buf.WriteString(values[0]) + for i := 1; i < len(values); i++ { + buf.WriteString(", " + values[i]) + } + buf.WriteString("\n") + } + return values +} + +// CertificateText returns a human-readable string representation +// of the certificate cert. The format is similar to the OpenSSL +// way of printing certificates (not identical). +func CertificateText(cert *x509.Certificate) string { + var buf strings.Builder + + buf.WriteString(color.Blue("\nCertificate:\n")) + if cert.SignatureAlgorithm != x509.UnknownSignatureAlgorithm { + buf.WriteString(color.Blue("%4sSignature Algorithm: ", "") + color.Bold(fmt.Sprintf("%s\n", cert.SignatureAlgorithm))) + } + + // Issuer information + buf.WriteString(color.Blue("%4sIssuer: ", "")) + printName(cert.Issuer.Names, &buf) + + // Validity information + buf.WriteString(color.Blue("%4sValidity\n", "")) + buf.WriteString(color.Bold(fmt.Sprintf("%8sNot Before: %s\n", "", cert.NotBefore.Format(http.TimeFormat)))) + buf.WriteString(color.Bold(fmt.Sprintf("%8sNot After : %s\n", "", cert.NotAfter.Format(http.TimeFormat)))) + + return buf.String() +} diff --git a/internal/config/compress/compress.go b/internal/config/compress/compress.go new file mode 100644 index 00000000..dc050e3b --- /dev/null +++ b/internal/config/compress/compress.go @@ -0,0 +1,171 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package compress + +import ( + "fmt" + "strings" + + "github.com/minio/minio/internal/config" + "github.com/minio/pkg/v3/env" +) + +// Config represents the compression settings. +type Config struct { + Enabled bool `json:"enabled"` + AllowEncrypted bool `json:"allow_encryption"` + Extensions []string `json:"extensions"` + MimeTypes []string `json:"mime-types"` +} + +// Compression environment variables +const ( + Extensions = "extensions" + AllowEncrypted = "allow_encryption" + MimeTypes = "mime_types" + + EnvCompressState = "MINIO_COMPRESSION_ENABLE" + EnvCompressAllowEncryption = "MINIO_COMPRESSION_ALLOW_ENCRYPTION" + EnvCompressExtensions = "MINIO_COMPRESSION_EXTENSIONS" + EnvCompressMimeTypes = "MINIO_COMPRESSION_MIME_TYPES" + + // Include-list for compression. + DefaultExtensions = ".txt,.log,.csv,.json,.tar,.xml,.bin" + DefaultMimeTypes = "text/*,application/json,application/xml,binary/octet-stream" +) + +// DefaultKVS - default KV config for compression settings +var ( + DefaultKVS = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOff, + }, + config.KV{ + Key: AllowEncrypted, + Value: config.EnableOff, + }, + config.KV{ + Key: Extensions, + Value: DefaultExtensions, + }, + config.KV{ + Key: MimeTypes, + Value: DefaultMimeTypes, + }, + } +) + +// Parses the given compression exclude list `extensions` or `content-types`. +func parseCompressIncludes(include string) ([]string, error) { + includes := strings.Split(include, config.ValueSeparator) + for _, e := range includes { + if len(e) == 0 { + return nil, config.ErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type cannot be empty") + } + if e == "/" { + return nil, config.ErrInvalidCompressionIncludesValue(nil).Msg("extension/mime-type cannot be '/'") + } + } + return includes, nil +} + +// LookupConfig - lookup compression config. +func LookupConfig(kvs config.KVS) (Config, error) { + var err error + cfg := Config{} + if err = config.CheckValidKeys(config.CompressionSubSys, kvs, DefaultKVS); err != nil { + return cfg, err + } + + compress := env.Get(EnvCompressState, "") + if compress == "" { + compress = env.Get(EnvCompressEnableLegacy, "") + if compress == "" { + compress = env.Get(EnvCompress, kvs.Get(config.Enable)) + } + } + cfg.Enabled, err = config.ParseBool(compress) + if err != nil { + // Parsing failures happen due to empty KVS, ignore it. + if kvs.Empty() { + return cfg, nil + } + return cfg, err + } + if !cfg.Enabled { + return cfg, nil + } + + allowEnc := env.Get(EnvCompressAllowEncryption, "") + if allowEnc == "" { + allowEnc = env.Get(EnvCompressAllowEncryptionLegacy, kvs.Get(AllowEncrypted)) + } + + cfg.AllowEncrypted, err = config.ParseBool(allowEnc) + if err != nil { + return cfg, err + } + + compressExtensions := env.Get(EnvCompressExtensions, kvs.Get(Extensions)) + compressExtensionsLegacy := env.Get(EnvCompressExtensionsLegacy, "") + compressMimeTypes := env.Get(EnvCompressMimeTypes, kvs.Get(MimeTypes)) + compressMimeTypesLegacy1 := env.Get(EnvCompressMimeTypesLegacy1, "") + compressMimeTypesLegacy2 := env.Get(EnvCompressMimeTypesLegacy2, "") + if compressExtensions != "" { + extensions, err := parseCompressIncludes(compressExtensions) + if err != nil { + return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESSION_EXTENSIONS value (`%s`)", err, extensions) + } + cfg.Extensions = extensions + } + + if compressExtensionsLegacy != "" { + extensions, err := parseCompressIncludes(compressExtensions) + if err != nil { + return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_EXTENSIONS value (`%s`)", err, extensions) + } + cfg.Extensions = extensions + } + + if compressMimeTypes != "" { + mimeTypes, err := parseCompressIncludes(compressMimeTypes) + if err != nil { + return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESSION_MIME_TYPES value (`%s`)", err, mimeTypes) + } + cfg.MimeTypes = mimeTypes + } + + if compressMimeTypesLegacy1 != "" { + mimeTypes, err := parseCompressIncludes(compressMimeTypesLegacy1) + if err != nil { + return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIMETYPES value (`%s`)", err, mimeTypes) + } + cfg.MimeTypes = mimeTypes + } + + if compressMimeTypesLegacy2 != "" { + mimeTypes, err := parseCompressIncludes(compressMimeTypesLegacy2) + if err != nil { + return cfg, fmt.Errorf("%s: Invalid MINIO_COMPRESS_MIME_TYPES value (`%s`)", err, mimeTypes) + } + cfg.MimeTypes = mimeTypes + } + + return cfg, nil +} diff --git a/internal/config/compress/compress_test.go b/internal/config/compress/compress_test.go new file mode 100644 index 00000000..12daa9f1 --- /dev/null +++ b/internal/config/compress/compress_test.go @@ -0,0 +1,57 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package compress + +import ( + "reflect" + "testing" +) + +func TestParseCompressIncludes(t *testing.T) { + testCases := []struct { + str string + expectedPatterns []string + success bool + }{ + // invalid input + {",,,", []string{}, false}, + {"", []string{}, false}, + {",", []string{}, false}, + {"/", []string{}, false}, + {"text/*,/", []string{}, false}, + + // valid input + {".txt,.log", []string{".txt", ".log"}, true}, + {"text/*,application/json", []string{"text/*", "application/json"}, true}, + } + + for _, testCase := range testCases { + t.Run(testCase.str, func(t *testing.T) { + gotPatterns, err := parseCompressIncludes(testCase.str) + if !testCase.success && err == nil { + t.Error("expected failure but success instead") + } + if testCase.success && err != nil { + t.Errorf("expected success but failed instead %s", err) + } + if testCase.success && !reflect.DeepEqual(testCase.expectedPatterns, gotPatterns) { + t.Errorf("expected patterns %s but got %s", testCase.expectedPatterns, gotPatterns) + } + }) + } +} diff --git a/internal/config/compress/help.go b/internal/config/compress/help.go new file mode 100644 index 00000000..e7f9a1fc --- /dev/null +++ b/internal/config/compress/help.go @@ -0,0 +1,61 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package compress + +import "github.com/minio/minio/internal/config" + +// Help template for compress feature. +var ( + defaultHelpPostfix = func(key string) string { + return config.DefaultHelpPostfix(DefaultKVS, key) + } + + Help = config.HelpKVS{ + config.HelpKV{ + Key: config.Enable, + Description: "Enable or disable object compression", + Type: "on|off", + Optional: true, + Sensitive: false, + }, + config.HelpKV{ + Key: Extensions, + Description: `comma separated file extensions` + defaultHelpPostfix(Extensions), + Optional: true, + Type: "csv", + }, + config.HelpKV{ + Key: MimeTypes, + Description: `comma separated wildcard mime-types` + defaultHelpPostfix(MimeTypes), + Optional: true, + Type: "csv", + }, + config.HelpKV{ + Key: AllowEncrypted, + Description: `enable 'encryption' along with compression`, + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: config.Comment, + Description: config.DefaultComment, + Optional: true, + Type: "sentence", + }, + } +) diff --git a/internal/config/compress/legacy.go b/internal/config/compress/legacy.go new file mode 100644 index 00000000..034df304 --- /dev/null +++ b/internal/config/compress/legacy.go @@ -0,0 +1,59 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package compress + +import ( + "strings" + + "github.com/minio/minio/internal/config" +) + +// Legacy envs. +const ( + EnvCompress = "MINIO_COMPRESS" + EnvCompressMimeTypesLegacy1 = "MINIO_COMPRESS_MIMETYPES" + + // These envs were wrong but we supported them for a long time + // so keep them here to support existing deployments. + EnvCompressEnableLegacy = "MINIO_COMPRESS_ENABLE" + EnvCompressAllowEncryptionLegacy = "MINIO_COMPRESS_ALLOW_ENCRYPTION" + EnvCompressExtensionsLegacy = "MINIO_COMPRESS_EXTENSIONS" + EnvCompressMimeTypesLegacy2 = "MINIO_COMPRESS_MIME_TYPES" +) + +// SetCompressionConfig - One time migration code needed, for migrating from older config to new for Compression. +func SetCompressionConfig(s config.Config, cfg Config) { + if !cfg.Enabled { + // No need to save disabled settings in new config. + return + } + s[config.CompressionSubSys][config.Default] = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOn, + }, + config.KV{ + Key: Extensions, + Value: strings.Join(cfg.Extensions, config.ValueSeparator), + }, + config.KV{ + Key: MimeTypes, + Value: strings.Join(cfg.MimeTypes, config.ValueSeparator), + }, + } +} diff --git a/internal/config/config.go b/internal/config/config.go new file mode 100644 index 00000000..20a9f302 --- /dev/null +++ b/internal/config/config.go @@ -0,0 +1,1477 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package config + +import ( + "bufio" + "fmt" + "io" + "maps" + "regexp" + "slices" + "sort" + "strings" + "sync" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/auth" + "github.com/minio/pkg/v3/env" +) + +// ErrorConfig holds the config error types +type ErrorConfig interface { + ErrConfigGeneric | ErrConfigNotFound +} + +// ErrConfigGeneric is a generic config type +type ErrConfigGeneric struct { + msg string +} + +func (ge *ErrConfigGeneric) setMsg(msg string) { + ge.msg = msg +} + +func (ge ErrConfigGeneric) Error() string { + return ge.msg +} + +// ErrConfigNotFound is an error to indicate +// that a config parameter is not found +type ErrConfigNotFound struct { + ErrConfigGeneric +} + +// Error creates an error message and wraps +// it with the error type specified in the type parameter +func Error[T ErrorConfig, PT interface { + *T + setMsg(string) +}](format string, vals ...any, +) T { + pt := PT(new(T)) + pt.setMsg(fmt.Sprintf(format, vals...)) + return *pt +} + +// Errorf formats an error and returns it as a generic config error +func Errorf(format string, vals ...any) ErrConfigGeneric { + return Error[ErrConfigGeneric](format, vals...) +} + +// Default keys +const ( + Default = madmin.Default + Enable = madmin.EnableKey + Comment = madmin.CommentKey + + EnvSeparator = "=" + + // Enable values + EnableOn = madmin.EnableOn + EnableOff = madmin.EnableOff + + RegionKey = "region" + NameKey = "name" + RegionName = "name" + AccessKey = "access_key" + SecretKey = "secret_key" + License = "license" // Deprecated Dec 2021 + APIKey = "api_key" + Proxy = "proxy" +) + +// Top level config constants. +const ( + PolicyOPASubSys = madmin.PolicyOPASubSys + PolicyPluginSubSys = madmin.PolicyPluginSubSys + IdentityOpenIDSubSys = madmin.IdentityOpenIDSubSys + IdentityLDAPSubSys = madmin.IdentityLDAPSubSys + IdentityTLSSubSys = madmin.IdentityTLSSubSys + IdentityPluginSubSys = madmin.IdentityPluginSubSys + SiteSubSys = madmin.SiteSubSys + RegionSubSys = madmin.RegionSubSys + EtcdSubSys = madmin.EtcdSubSys + StorageClassSubSys = madmin.StorageClassSubSys + APISubSys = madmin.APISubSys + CompressionSubSys = madmin.CompressionSubSys + LoggerWebhookSubSys = madmin.LoggerWebhookSubSys + AuditWebhookSubSys = madmin.AuditWebhookSubSys + AuditKafkaSubSys = madmin.AuditKafkaSubSys + HealSubSys = madmin.HealSubSys + ScannerSubSys = madmin.ScannerSubSys + CrawlerSubSys = madmin.CrawlerSubSys + SubnetSubSys = madmin.SubnetSubSys + CallhomeSubSys = madmin.CallhomeSubSys + DriveSubSys = madmin.DriveSubSys + BatchSubSys = madmin.BatchSubSys + BrowserSubSys = madmin.BrowserSubSys + ILMSubSys = madmin.ILMSubsys + + // Add new constants here (similar to above) if you add new fields to config. +) + +// Notification config constants. +const ( + NotifyKafkaSubSys = madmin.NotifyKafkaSubSys + NotifyMQTTSubSys = madmin.NotifyMQTTSubSys + NotifyMySQLSubSys = madmin.NotifyMySQLSubSys + NotifyNATSSubSys = madmin.NotifyNATSSubSys + NotifyNSQSubSys = madmin.NotifyNSQSubSys + NotifyESSubSys = madmin.NotifyESSubSys + NotifyAMQPSubSys = madmin.NotifyAMQPSubSys + NotifyPostgresSubSys = madmin.NotifyPostgresSubSys + NotifyRedisSubSys = madmin.NotifyRedisSubSys + NotifyWebhookSubSys = madmin.NotifyWebhookSubSys + + // Add new constants here (similar to above) if you add new fields to config. +) + +// Lambda config constants. +const ( + LambdaWebhookSubSys = madmin.LambdaWebhookSubSys +) + +// NotifySubSystems - all notification sub-systems +var NotifySubSystems = set.CreateStringSet( + NotifyKafkaSubSys, + NotifyMQTTSubSys, + NotifyMySQLSubSys, + NotifyNATSSubSys, + NotifyNSQSubSys, + NotifyESSubSys, + NotifyAMQPSubSys, + NotifyPostgresSubSys, + NotifyRedisSubSys, + NotifyWebhookSubSys, +) + +// LambdaSubSystems - all lambda sub-systems +var LambdaSubSystems = set.CreateStringSet( + LambdaWebhookSubSys, +) + +// LoggerSubSystems - all sub-systems related to logger +var LoggerSubSystems = set.CreateStringSet( + LoggerWebhookSubSys, + AuditWebhookSubSys, + AuditKafkaSubSys, +) + +// SubSystems - all supported sub-systems +var SubSystems = madmin.SubSystems + +// SubSystemsDynamic - all sub-systems that have dynamic config. +var SubSystemsDynamic = set.CreateStringSet( + APISubSys, + CompressionSubSys, + ScannerSubSys, + HealSubSys, + SubnetSubSys, + CallhomeSubSys, + DriveSubSys, + LoggerWebhookSubSys, + AuditWebhookSubSys, + AuditKafkaSubSys, + StorageClassSubSys, + ILMSubSys, + BatchSubSys, + BrowserSubSys, +) + +// SubSystemsSingleTargets - subsystems which only support single target. +var SubSystemsSingleTargets = set.CreateStringSet( + SiteSubSys, + RegionSubSys, + EtcdSubSys, + APISubSys, + StorageClassSubSys, + CompressionSubSys, + PolicyOPASubSys, + PolicyPluginSubSys, + IdentityLDAPSubSys, + IdentityTLSSubSys, + IdentityPluginSubSys, + HealSubSys, + ScannerSubSys, + SubnetSubSys, + CallhomeSubSys, + DriveSubSys, + ILMSubSys, + BatchSubSys, + BrowserSubSys, +) + +// Constant separators +const ( + SubSystemSeparator = madmin.SubSystemSeparator + KvSeparator = madmin.KvSeparator + KvSpaceSeparator = madmin.KvSpaceSeparator + KvComment = madmin.KvComment + KvNewline = madmin.KvNewline + KvDoubleQuote = madmin.KvDoubleQuote + KvSingleQuote = madmin.KvSingleQuote + + // Env prefix used for all envs in MinIO + EnvPrefix = madmin.EnvPrefix + EnvWordDelimiter = madmin.EnvWordDelimiter +) + +// DefaultKVS - default kvs for all sub-systems +var DefaultKVS = map[string]KVS{} + +// RegisterDefaultKVS - this function saves input kvsMap +// globally, this should be called only once preferably +// during `init()`. +func RegisterDefaultKVS(kvsMap map[string]KVS) { + maps.Copy(DefaultKVS, kvsMap) +} + +// HelpSubSysMap - help for all individual KVS for each sub-systems +// also carries a special empty sub-system which dumps +// help for each sub-system key. +var HelpSubSysMap = map[string]HelpKVS{} + +// RegisterHelpSubSys - this function saves +// input help KVS for each sub-system globally, +// this function should be called only once +// preferably in during `init()`. +func RegisterHelpSubSys(helpKVSMap map[string]HelpKVS) { + maps.Copy(HelpSubSysMap, helpKVSMap) +} + +// HelpDeprecatedSubSysMap - help for all deprecated sub-systems, that may be +// removed in the future. +var HelpDeprecatedSubSysMap = map[string]HelpKV{} + +// RegisterHelpDeprecatedSubSys - saves input help KVS for deprecated +// sub-systems globally. Should be called only once at init. +func RegisterHelpDeprecatedSubSys(helpDeprecatedKVMap map[string]HelpKV) { + maps.Copy(HelpDeprecatedSubSysMap, helpDeprecatedKVMap) +} + +// KV - is a shorthand of each key value. +type KV struct { + Key string `json:"key"` + Value string `json:"value"` + + HiddenIfEmpty bool `json:"-"` +} + +func (kv KV) String() string { + var s strings.Builder + s.WriteString(kv.Key) + s.WriteString(KvSeparator) + spc := madmin.HasSpace(kv.Value) + if spc { + s.WriteString(KvDoubleQuote) + } + s.WriteString(kv.Value) + if spc { + s.WriteString(KvDoubleQuote) + } + return s.String() +} + +// KVS - is a shorthand for some wrapper functions +// to operate on list of key values. +type KVS []KV + +// Empty - return if kv is empty +func (kvs KVS) Empty() bool { + return len(kvs) == 0 +} + +// Clone - returns a copy of the KVS +func (kvs KVS) Clone() KVS { + return append(make(KVS, 0, len(kvs)), kvs...) +} + +// GetWithDefault - returns default value if key not set +func (kvs KVS) GetWithDefault(key string, defaultKVS KVS) string { + v := kvs.Get(key) + if len(v) == 0 { + return defaultKVS.Get(key) + } + return v +} + +// Keys returns the list of keys for the current KVS +func (kvs KVS) Keys() []string { + keys := make([]string, len(kvs)) + var foundComment bool + for i := range kvs { + if kvs[i].Key == madmin.CommentKey { + foundComment = true + } + keys[i] = kvs[i].Key + } + // Comment KV not found, add it explicitly. + if !foundComment { + keys = append(keys, madmin.CommentKey) + } + return keys +} + +func (kvs KVS) String() string { + var s strings.Builder + for _, kv := range kvs { + s.WriteString(kv.String()) + s.WriteString(KvSpaceSeparator) + } + return s.String() +} + +// Merge environment values with on disk KVS, environment values overrides +// anything on the disk. +func Merge(cfgKVS map[string]KVS, envname string, defaultKVS KVS) map[string]KVS { + newCfgKVS := make(map[string]KVS) + for _, e := range env.List(envname) { + tgt := strings.TrimPrefix(e, envname+Default) + if tgt == envname { + tgt = Default + } + newCfgKVS[tgt] = defaultKVS + } + maps.Copy(newCfgKVS, cfgKVS) + return newCfgKVS +} + +// Set sets a value, if not sets a default value. +func (kvs *KVS) Set(key, value string) { + for i, kv := range *kvs { + if kv.Key == key { + (*kvs)[i] = KV{ + Key: key, + Value: value, + } + return + } + } + *kvs = append(*kvs, KV{ + Key: key, + Value: value, + }) +} + +// Get - returns the value of a key, if not found returns empty. +func (kvs KVS) Get(key string) string { + v, ok := kvs.Lookup(key) + if ok { + return v + } + return "" +} + +// Delete - deletes the key if present from the KV list. +func (kvs *KVS) Delete(key string) { + for i, kv := range *kvs { + if kv.Key == key { + *kvs = append((*kvs)[:i], (*kvs)[i+1:]...) + return + } + } +} + +// LookupKV returns the KV by its key +func (kvs KVS) LookupKV(key string) (KV, bool) { + for _, kv := range kvs { + if kv.Key == key { + return kv, true + } + } + return KV{}, false +} + +// Lookup - lookup a key in a list of KVS +func (kvs KVS) Lookup(key string) (string, bool) { + for _, kv := range kvs { + if kv.Key == key { + return kv.Value, true + } + } + return "", false +} + +// Config - MinIO server config structure. +type Config map[string]map[string]KVS + +// DelFrom - deletes all keys in the input reader. +func (c Config) DelFrom(r io.Reader) error { + scanner := bufio.NewScanner(r) + for scanner.Scan() { + // Skip any empty lines, or comment like characters + text := scanner.Text() + if text == "" || strings.HasPrefix(text, KvComment) { + continue + } + if err := c.DelKVS(text); err != nil { + return err + } + } + return scanner.Err() +} + +// ContextKeyString is type(string) for contextKey +type ContextKeyString string + +// ContextKeyForTargetFromConfig - key for context for target from config +const ContextKeyForTargetFromConfig = ContextKeyString("ContextKeyForTargetFromConfig") + +// ParseConfigTargetID - read all targetIDs from reader +func ParseConfigTargetID(r io.Reader) (ids map[string]bool, err error) { + ids = make(map[string]bool) + scanner := bufio.NewScanner(r) + for scanner.Scan() { + // Skip any empty lines, or comment like characters + text := scanner.Text() + if text == "" || strings.HasPrefix(text, KvComment) { + continue + } + _, _, tgt, err := GetSubSys(text) + if err != nil { + return nil, err + } + ids[tgt] = true + } + if err := scanner.Err(); err != nil { + return nil, err + } + return ids, err +} + +// ReadConfig - read content from input and write into c. +// Returns whether all parameters were dynamic. +func (c Config) ReadConfig(r io.Reader) (dynOnly bool, err error) { + var n int + scanner := bufio.NewScanner(r) + dynOnly = true + for scanner.Scan() { + // Skip any empty lines, or comment like characters + text := scanner.Text() + if text == "" || strings.HasPrefix(text, KvComment) { + continue + } + dynamic, err := c.SetKVS(text, DefaultKVS) + if err != nil { + return false, err + } + dynOnly = dynOnly && dynamic + n += len(text) + } + if err := scanner.Err(); err != nil { + return false, err + } + return dynOnly, nil +} + +// RedactSensitiveInfo - removes sensitive information +// like urls and credentials from the configuration +func (c Config) RedactSensitiveInfo() Config { + nc := c.Clone() + + for configName, configVals := range nc { + for _, helpKV := range HelpSubSysMap[configName] { + if helpKV.Sensitive { + for name, kvs := range configVals { + for i := range kvs { + if kvs[i].Key == helpKV.Key && len(kvs[i].Value) > 0 { + kvs[i].Value = "*redacted*" + } + } + configVals[name] = kvs + } + } + } + } + + return nc +} + +// Default KV configs for worm and region +var ( + DefaultCredentialKVS = KVS{ + KV{ + Key: AccessKey, + Value: auth.DefaultAccessKey, + }, + KV{ + Key: SecretKey, + Value: auth.DefaultSecretKey, + }, + } + + DefaultSiteKVS = KVS{ + KV{ + Key: NameKey, + Value: "", + }, + KV{ + Key: RegionKey, + Value: "", + }, + } + + DefaultRegionKVS = KVS{ + KV{ + Key: RegionName, + Value: "", + }, + } +) + +var siteLK sync.RWMutex + +// Site - holds site info - name and region. +type Site struct { + name string + region string +} + +// Update safe update the new site name and region +func (s *Site) Update(n Site) { + siteLK.Lock() + s.name = n.name + s.region = n.region + siteLK.Unlock() +} + +// Name returns currently configured site name +func (s *Site) Name() string { + siteLK.RLock() + defer siteLK.RUnlock() + + return s.name +} + +// Region returns currently configured site region +func (s *Site) Region() string { + siteLK.RLock() + defer siteLK.RUnlock() + + return s.region +} + +var validRegionRegex = regexp.MustCompile("^[a-zA-Z][a-zA-Z0-9-_-]+$") + +// validSiteNameRegex - allows lowercase letters, digits and '-', starts with +// letter. At least 2 characters long. +var validSiteNameRegex = regexp.MustCompile("^[a-z][a-z0-9-]+$") + +// LookupSite - get site related configuration. Loads configuration from legacy +// region sub-system as well. +func LookupSite(siteKV KVS, regionKV KVS) (s Site, err error) { + if err = CheckValidKeys(SiteSubSys, siteKV, DefaultSiteKVS); err != nil { + return s, err + } + region := env.Get(EnvRegion, "") + if region == "" { + env.Get(EnvRegionName, "") + } + if region == "" { + region = env.Get(EnvSiteRegion, siteKV.Get(RegionKey)) + } + if region == "" { + // No region config found in the site-subsystem. So lookup the legacy + // region sub-system. + if err = CheckValidKeys(RegionSubSys, regionKV, DefaultRegionKVS); err != nil { + // An invalid key was found in the region sub-system. + // Since the region sub-system cannot be (re)set as it + // is legacy, we return an error to tell the user to + // reset the region via the new command. + err = Errorf("could not load region from legacy configuration as it was invalid - use 'mc admin config set myminio site region=myregion name=myname' to set a region and name (%v)", err) + return s, err + } + + region = regionKV.Get(RegionName) + } + if region != "" { + if !validRegionRegex.MatchString(region) { + err = Errorf( + "region '%s' is invalid, expected simple characters such as [us-east-1, myregion...]", + region) + return s, err + } + s.region = region + } + + name := env.Get(EnvSiteName, siteKV.Get(NameKey)) + if name != "" { + if !validSiteNameRegex.MatchString(name) { + err = Errorf( + "site name '%s' is invalid, expected simple characters such as [cal-rack0, myname...]", + name) + return s, err + } + s.name = name + } + return s, err +} + +// CheckValidKeys - checks if inputs KVS has the necessary keys, +// returns error if it find extra or superfluous keys. +func CheckValidKeys(subSys string, kv KVS, validKVS KVS, deprecatedKeys ...string) error { + nkv := KVS{} + for _, kv := range kv { + // Comment is a valid key, its also fully optional + // ignore it since it is a valid key for all + // sub-systems. + if kv.Key == Comment { + continue + } + var skip bool + if slices.Contains(deprecatedKeys, kv.Key) { + skip = true + } + if skip { + continue + } + if _, ok := validKVS.Lookup(kv.Key); !ok { + nkv = append(nkv, kv) + } + } + if len(nkv) > 0 { + return Errorf( + "found invalid keys (%s) for '%s' sub-system, use 'mc admin config reset myminio %s' to fix invalid keys", nkv.String(), subSys, subSys) + } + return nil +} + +// LookupWorm - check if worm is enabled +func LookupWorm() (bool, error) { + return ParseBool(env.Get(EnvWorm, EnableOff)) +} + +// Carries all the renamed sub-systems from their +// previously known names +var renamedSubsys = map[string]string{ + CrawlerSubSys: ScannerSubSys, + // Add future sub-system renames +} + +const ( // deprecated keys + apiReplicationWorkers = "replication_workers" + apiReplicationFailedWorkers = "replication_failed_workers" +) + +// map of subsystem to deleted keys +var deletedSubSysKeys = map[string][]string{ + APISubSys: {apiReplicationWorkers, apiReplicationFailedWorkers}, + // Add future sub-system deleted keys +} + +// Merge - merges a new config with all the +// missing values for default configs, +// returns a config. +func (c Config) Merge() Config { + cp := New() + for subSys, tgtKV := range c { + for tgt := range tgtKV { + ckvs := c[subSys][tgt] + for _, kv := range cp[subSys][Default] { + _, ok := c[subSys][tgt].Lookup(kv.Key) + if !ok { + ckvs.Set(kv.Key, kv.Value) + } + } + if _, ok := cp[subSys]; !ok { + rnSubSys, ok := renamedSubsys[subSys] + if !ok { + // A config subsystem was removed or server was downgraded. + continue + } + // Copy over settings from previous sub-system + // to newly renamed sub-system + for _, kv := range cp[rnSubSys][Default] { + _, ok := c[subSys][tgt].Lookup(kv.Key) + if !ok { + ckvs.Set(kv.Key, kv.Value) + } + } + subSys = rnSubSys + } + // Delete deprecated keys for subsystem if any + if keys, ok := deletedSubSysKeys[subSys]; ok { + for _, key := range keys { + ckvs.Delete(key) + } + } + cp[subSys][tgt] = ckvs + } + } + + return cp +} + +// New - initialize a new server config. +func New() Config { + srvCfg := make(Config) + for _, k := range SubSystems.ToSlice() { + srvCfg[k] = map[string]KVS{} + srvCfg[k][Default] = DefaultKVS[k] + } + return srvCfg +} + +// Target signifies an individual target +type Target struct { + SubSystem string + KVS KVS +} + +// Targets sub-system targets +type Targets []Target + +// GetKVS - get kvs from specific subsystem. +func (c Config) GetKVS(s string, defaultKVS map[string]KVS) (Targets, error) { + if len(s) == 0 { + return nil, Errorf("input cannot be empty") + } + inputs := strings.Fields(s) + if len(inputs) > 1 { + return nil, Errorf("invalid number of arguments %s", s) + } + subSystemValue := strings.SplitN(inputs[0], SubSystemSeparator, 2) + if len(subSystemValue) == 0 { + return nil, Errorf("invalid number of arguments %s", s) + } + found := SubSystems.Contains(subSystemValue[0]) + if !found { + // Check for sub-prefix only if the input value is only a + // single value, this rejects invalid inputs if any. + found = !SubSystems.FuncMatch(strings.HasPrefix, subSystemValue[0]).IsEmpty() && len(subSystemValue) == 1 + } + if !found { + return nil, Errorf("unknown sub-system %s", s) + } + + targets := Targets{} + subSysPrefix := subSystemValue[0] + if len(subSystemValue) == 2 { + if len(subSystemValue[1]) == 0 { + return nil, Errorf("sub-system target '%s' cannot be empty", s) + } + kvs, ok := c[subSysPrefix][subSystemValue[1]] + if !ok { + return nil, Errorf("sub-system target '%s' doesn't exist", s) + } + for _, kv := range defaultKVS[subSysPrefix] { + _, ok = kvs.Lookup(kv.Key) + if !ok { + kvs.Set(kv.Key, kv.Value) + } + } + targets = append(targets, Target{ + SubSystem: inputs[0], + KVS: kvs, + }) + } else { + // Use help for sub-system to preserve the order. Add deprecated + // keys at the end (in some order). + kvsOrder := append([]HelpKV{}, HelpSubSysMap[""]...) + for _, v := range HelpDeprecatedSubSysMap { + kvsOrder = append(kvsOrder, v) + } + + for _, hkv := range kvsOrder { + if !strings.HasPrefix(hkv.Key, subSysPrefix) { + continue + } + if c[hkv.Key][Default].Empty() { + targets = append(targets, Target{ + SubSystem: hkv.Key, + KVS: defaultKVS[hkv.Key], + }) + } + for k, kvs := range c[hkv.Key] { + for _, dkv := range defaultKVS[hkv.Key] { + _, ok := kvs.Lookup(dkv.Key) + if !ok { + kvs.Set(dkv.Key, dkv.Value) + } + } + if k != Default { + targets = append(targets, Target{ + SubSystem: hkv.Key + SubSystemSeparator + k, + KVS: kvs, + }) + } else { + targets = append(targets, Target{ + SubSystem: hkv.Key, + KVS: kvs, + }) + } + } + } + } + return targets, nil +} + +// DelKVS - delete a specific key. +func (c Config) DelKVS(s string) error { + subSys, inputs, tgt, err := GetSubSys(s) + if err != nil { + if !SubSystems.Contains(subSys) && len(inputs) == 1 { + // Unknown sub-system found try to remove it anyways. + delete(c, subSys) + return nil + } + return err + } + + ck, ok := c[subSys][tgt] + if !ok { + return Error[ErrConfigNotFound]("sub-system %s:%s already deleted or does not exist", subSys, tgt) + } + + if len(inputs) == 2 { + currKVS := ck.Clone() + defKVS := DefaultKVS[subSys] + for delKey := range strings.FieldsSeq(inputs[1]) { + _, ok := currKVS.Lookup(delKey) + if !ok { + return Error[ErrConfigNotFound]("key %s doesn't exist", delKey) + } + defVal, isDef := defKVS.Lookup(delKey) + if isDef { + currKVS.Set(delKey, defVal) + } else { + currKVS.Delete(delKey) + } + } + c[subSys][tgt] = currKVS + } else { + delete(c[subSys], tgt) + } + return nil +} + +// Clone - clones a config map entirely. +func (c Config) Clone() Config { + cp := New() + for subSys, tgtKV := range c { + cp[subSys] = make(map[string]KVS) + for tgt, kv := range tgtKV { + cp[subSys][tgt] = append(cp[subSys][tgt], kv...) + } + } + return cp +} + +// GetSubSys - extracts subssystem info from given config string +func GetSubSys(s string) (subSys string, inputs []string, tgt string, e error) { + tgt = Default + if len(s) == 0 { + return subSys, inputs, tgt, Errorf("input arguments cannot be empty") + } + inputs = strings.SplitN(s, KvSpaceSeparator, 2) + + subSystemValue := strings.SplitN(inputs[0], SubSystemSeparator, 2) + subSys = subSystemValue[0] + if !SubSystems.Contains(subSys) { + return subSys, inputs, tgt, Errorf("unknown sub-system %s", s) + } + + if SubSystemsSingleTargets.Contains(subSystemValue[0]) && len(subSystemValue) == 2 { + return subSys, inputs, tgt, Errorf("sub-system '%s' only supports single target", subSystemValue[0]) + } + + if len(subSystemValue) == 2 { + tgt = subSystemValue[1] + } + + return subSys, inputs, tgt, e +} + +// kvFields - converts an input string of form "k1=v1 k2=v2" into +// fields of ["k1=v1", "k2=v2"], the tokenization of each `k=v` +// happens with the right number of input keys, if keys +// input is empty returned value is empty slice as well. +func kvFields(input string, keys []string) []string { + valueIndexes := make([]int, 0, len(keys)) + for _, key := range keys { + i := strings.Index(input, key+KvSeparator) + if i == -1 { + continue + } + valueIndexes = append(valueIndexes, i) + } + + sort.Ints(valueIndexes) + fields := make([]string, len(valueIndexes)) + for i := range valueIndexes { + j := i + 1 + if j < len(valueIndexes) { + fields[i] = strings.TrimSpace(input[valueIndexes[i]:valueIndexes[j]]) + } else { + fields[i] = strings.TrimSpace(input[valueIndexes[i]:]) + } + } + return fields +} + +// SetKVS - set specific key values per sub-system. +func (c Config) SetKVS(s string, defaultKVS map[string]KVS) (dynamic bool, err error) { + subSys, inputs, tgt, err := GetSubSys(s) + if err != nil { + return false, err + } + if len(inputs) < 2 { + return false, Errorf("sub-system '%s' must have key", subSys) + } + + dynamic = SubSystemsDynamic.Contains(subSys) + + fields := kvFields(inputs[1], defaultKVS[subSys].Keys()) + if len(fields) == 0 { + return false, Errorf("sub-system '%s' cannot have empty keys", subSys) + } + + kvs := KVS{} + var prevK string + for _, v := range fields { + kv := strings.SplitN(v, KvSeparator, 2) + if len(kv) == 0 { + continue + } + if len(kv) == 1 && prevK != "" { + value := strings.Join([]string{ + kvs.Get(prevK), + madmin.SanitizeValue(kv[0]), + }, KvSpaceSeparator) + kvs.Set(prevK, value) + continue + } + if len(kv) == 2 { + prevK = kv[0] + kvs.Set(prevK, madmin.SanitizeValue(kv[1])) + continue + } + return false, Errorf("key '%s', cannot have empty value", kv[0]) + } + + _, ok := kvs.Lookup(Enable) + // Check if state is required + _, enableRequired := defaultKVS[subSys].Lookup(Enable) + if !ok && enableRequired { + // implicit state "on" if not specified. + kvs.Set(Enable, EnableOn) + } + + var currKVS KVS + ck, ok := c[subSys][tgt] + if !ok { + currKVS = defaultKVS[subSys].Clone() + } else { + currKVS = ck.Clone() + for _, kv := range defaultKVS[subSys] { + if _, ok = currKVS.Lookup(kv.Key); !ok { + currKVS.Set(kv.Key, kv.Value) + } + } + } + + for _, kv := range kvs { + if kv.Key == Comment { + // Skip comment and add it later. + continue + } + currKVS.Set(kv.Key, kv.Value) + } + + v, ok := kvs.Lookup(Comment) + if ok { + currKVS.Set(Comment, v) + } + + hkvs := HelpSubSysMap[subSys] + for _, hkv := range hkvs { + var enabled bool + if enableRequired { + enabled = currKVS.Get(Enable) == EnableOn + } else { + // when enable arg is not required + // then it is implicit on for the sub-system. + enabled = true + } + v, _ := currKVS.Lookup(hkv.Key) + if v == "" && !hkv.Optional && enabled { + // Return error only if the + // key is enabled, for state=off + // let it be empty. + return false, Errorf( + "'%s' is not optional for '%s' sub-system, please check '%s' documentation", + hkv.Key, subSys, subSys) + } + } + c[subSys][tgt] = currKVS + return dynamic, nil +} + +// CheckValidKeys - checks if the config parameters for the given subsystem and +// target are valid. It checks both the configuration store as well as +// environment variables. +func (c Config) CheckValidKeys(subSys string, deprecatedKeys []string) error { + defKVS, ok := DefaultKVS[subSys] + if !ok { + return Errorf("Subsystem %s does not exist", subSys) + } + + // Make a list of valid keys for the subsystem including the `comment` + // key. + validKeys := make([]string, 0, len(defKVS)+1) + for _, param := range defKVS { + validKeys = append(validKeys, param.Key) + } + validKeys = append(validKeys, Comment) + + subSysEnvVars := env.List(fmt.Sprintf("%s%s", EnvPrefix, strings.ToUpper(subSys))) + + // Set of env vars for the sub-system to validate. + candidates := set.CreateStringSet(subSysEnvVars...) + + // Remove all default target env vars from the candidates set (as they + // are valid). + for _, param := range validKeys { + paramEnvName := getEnvVarName(subSys, Default, param) + candidates.Remove(paramEnvName) + } + + isSingleTarget := SubSystemsSingleTargets.Contains(subSys) + if isSingleTarget && len(candidates) > 0 { + return Errorf("The following environment variables are unknown: %s", + strings.Join(candidates.ToSlice(), ", ")) + } + + if !isSingleTarget { + // Validate other env vars for all targets. + envVars := candidates.ToSlice() + for _, envVar := range envVars { + for _, param := range validKeys { + pEnvName := getEnvVarName(subSys, Default, param) + Default + if len(envVar) > len(pEnvName) && strings.HasPrefix(envVar, pEnvName) { + // This envVar is valid - it has a + // non-empty target. + candidates.Remove(envVar) + } + } + } + + // Whatever remains are invalid env vars - return an error. + if len(candidates) > 0 { + return Errorf("The following environment variables are unknown: %s", + strings.Join(candidates.ToSlice(), ", ")) + } + } + + validKeysSet := set.CreateStringSet(validKeys...) + validKeysSet = validKeysSet.Difference(set.CreateStringSet(deprecatedKeys...)) + kvsMap := c[subSys] + for tgt, kvs := range kvsMap { + invalidKV := KVS{} + for _, kv := range kvs { + if !validKeysSet.Contains(kv.Key) { + invalidKV = append(invalidKV, kv) + } + } + if len(invalidKV) > 0 { + return Errorf( + "found invalid keys (%s) for '%s:%s' sub-system, use 'mc admin config reset myminio %s:%s' to fix invalid keys", + invalidKV.String(), subSys, tgt, subSys, tgt) + } + } + return nil +} + +// GetAvailableTargets - returns a list of targets configured for the given +// subsystem (whether they are enabled or not). A target could be configured via +// environment variables or via the configuration store. The default target is +// `_` and is always returned. The result is sorted so that the default target +// is the first one and the remaining entries are sorted in ascending order. +func (c Config) GetAvailableTargets(subSys string) ([]string, error) { + if SubSystemsSingleTargets.Contains(subSys) { + return []string{Default}, nil + } + + defKVS, ok := DefaultKVS[subSys] + if !ok { + return nil, Errorf("Subsystem %s does not exist", subSys) + } + + kvsMap := c[subSys] + seen := set.NewStringSet() + + // Add all targets that are configured in the config store. + for k := range kvsMap { + seen.Add(k) + } + + // env:prefix + filterMap := map[string]string{} + // Add targets that are configured via environment variables. + for _, param := range defKVS { + envVarPrefix := getEnvVarName(subSys, Default, param.Key) + Default + envsWithPrefix := env.List(envVarPrefix) + for _, k := range envsWithPrefix { + tgtName := strings.TrimPrefix(k, envVarPrefix) + if tgtName != "" { + if v, ok := filterMap[k]; ok { + if strings.HasPrefix(envVarPrefix, v) { + filterMap[k] = envVarPrefix + } + } else { + filterMap[k] = envVarPrefix + } + } + } + } + + for k, v := range filterMap { + seen.Add(strings.TrimPrefix(k, v)) + } + + seen.Remove(Default) + targets := seen.ToSlice() + sort.Strings(targets) + targets = append([]string{Default}, targets...) + + return targets, nil +} + +func getEnvVarName(subSys, target, param string) string { + if target == Default { + return fmt.Sprintf("%s%s%s%s", EnvPrefix, strings.ToUpper(subSys), Default, strings.ToUpper(param)) + } + + return fmt.Sprintf("%s%s%s%s%s%s", EnvPrefix, strings.ToUpper(subSys), Default, strings.ToUpper(param), + Default, target) +} + +var resolvableSubsystems = set.CreateStringSet(IdentityOpenIDSubSys, IdentityLDAPSubSys, PolicyPluginSubSys) + +// ValueSource represents the source of a config parameter value. +type ValueSource uint8 + +// Constants for ValueSource +const ( + ValueSourceAbsent ValueSource = iota // this is an error case + ValueSourceDef + ValueSourceCfg + ValueSourceEnv +) + +// ResolveConfigParam returns the effective value of a configuration parameter, +// within a subsystem and subsystem target. The effective value is, in order of +// decreasing precedence: +// +// 1. the value of the corresponding environment variable if set, +// 2. the value of the parameter in the config store if set, +// 3. the default value, +// +// This function only works for a subset of sub-systems, others return +// `ValueSourceAbsent`. FIXME: some parameters have custom environment +// variables for which support needs to be added. +// +// When redactSecrets is true, the returned value is empty if the configuration +// parameter is a secret, and the returned isRedacted flag is set. +func (c Config) ResolveConfigParam(subSys, target, cfgParam string, redactSecrets bool, +) (value string, cs ValueSource, isRedacted bool) { + // cs = ValueSourceAbsent initially as it is iota by default. + + // Initially only support OpenID + if !resolvableSubsystems.Contains(subSys) { + return value, cs, isRedacted + } + + // Check if config param requested is valid. + defKVS, ok := DefaultKVS[subSys] + if !ok { + return value, cs, isRedacted + } + + defValue, isFound := defKVS.Lookup(cfgParam) + // Comments usually are absent from `defKVS`, so we handle it specially. + if !isFound && cfgParam == Comment { + defValue, isFound = "", true + } + if !isFound { + return value, cs, isRedacted + } + + if target == "" { + target = Default + } + + if redactSecrets { + // If the configuration parameter is a secret, make sure to redact it when + // we return. + helpKV, _ := HelpSubSysMap[subSys].Lookup(cfgParam) + if helpKV.Secret { + defer func() { + value = "" + isRedacted = true + }() + } + } + + envVar := getEnvVarName(subSys, target, cfgParam) + + // Lookup Env var. + value = env.Get(envVar, "") + if value != "" { + cs = ValueSourceEnv + return value, cs, isRedacted + } + + // Lookup config store. + if subSysStore, ok := c[subSys]; ok { + if kvs, ok2 := subSysStore[target]; ok2 { + var ok3 bool + value, ok3 = kvs.Lookup(cfgParam) + if ok3 { + cs = ValueSourceCfg + return value, cs, isRedacted + } + } + } + + // Return the default value. + value = defValue + cs = ValueSourceDef + return value, cs, isRedacted +} + +// KVSrc represents a configuration parameter key and value along with the +// source of the value. +type KVSrc struct { + Key string + Value string + Src ValueSource +} + +// GetResolvedConfigParams returns all applicable config parameters with their +// value sources. +func (c Config) GetResolvedConfigParams(subSys, target string, redactSecrets bool) ([]KVSrc, error) { + if !resolvableSubsystems.Contains(subSys) { + return nil, Errorf("unsupported subsystem: %s", subSys) + } + + // Check if config param requested is valid. + defKVS, ok := DefaultKVS[subSys] + if !ok { + return nil, Errorf("unknown subsystem: %s", subSys) + } + + r := make([]KVSrc, 0, len(defKVS)+1) + for _, kv := range defKVS { + v, vs, isRedacted := c.ResolveConfigParam(subSys, target, kv.Key, redactSecrets) + + // Fix `vs` when default. + if v == kv.Value { + vs = ValueSourceDef + } + + if redactSecrets && isRedacted { + // Skip adding redacted secrets to the output. + continue + } + + r = append(r, KVSrc{ + Key: kv.Key, + Value: v, + Src: vs, + }) + } + + // Add the comment key as well if non-empty (and comments are never + // redacted). + v, vs, _ := c.ResolveConfigParam(subSys, target, Comment, redactSecrets) + if vs != ValueSourceDef { + r = append(r, KVSrc{ + Key: Comment, + Value: v, + Src: vs, + }) + } + + return r, nil +} + +// getTargetKVS returns configuration KVs for the given subsystem and target. It +// does not return any secrets in the configuration values when `redactSecrets` +// is set. +func (c Config) getTargetKVS(subSys, target string, redactSecrets bool) KVS { + store, ok := c[subSys] + if !ok { + return nil + } + + // Lookup will succeed, because this function only works with valid subSys + // values. + resultKVS := make([]KV, 0, len(store[target])) + hkvs := HelpSubSysMap[subSys] + for _, kv := range store[target] { + hkv, _ := hkvs.Lookup(kv.Key) + if hkv.Secret && redactSecrets && kv.Value != "" { + // Skip returning secrets. + continue + // clonedKV := kv + // clonedKV.Value = redactedSecret + // resultKVS = append(resultKVS, clonedKV) + } + resultKVS = append(resultKVS, kv) + } + + return resultKVS +} + +// getTargetEnvs returns configured environment variable settings for the given +// subsystem and target. +func (c Config) getTargetEnvs(subSys, target string, defKVS KVS, redactSecrets bool) map[string]EnvPair { + hkvs := HelpSubSysMap[subSys] + envMap := make(map[string]EnvPair) + + // Add all env vars that are set. + for _, kv := range defKVS { + envName := getEnvVarName(subSys, target, kv.Key) + envPair := EnvPair{ + Name: envName, + Value: env.Get(envName, ""), + } + if envPair.Value != "" { + hkv, _ := hkvs.Lookup(kv.Key) + if hkv.Secret && redactSecrets { + // Skip adding any secret to the returned value. + continue + // envPair.Value = redactedSecret + } + envMap[kv.Key] = envPair + } + } + return envMap +} + +// EnvPair represents an environment variable and its value. +type EnvPair struct { + Name, Value string +} + +// SubsysInfo holds config info for a subsystem target. +type SubsysInfo struct { + SubSys, Target string + Defaults KVS + Config KVS + + // map of config parameter name to EnvPair. + EnvMap map[string]EnvPair +} + +// GetSubsysInfo returns `SubsysInfo`s for all targets for the subsystem, when +// target is empty. Otherwise returns `SubsysInfo` for the desired target only. +// To request the default target only, target must be set to `Default`. +func (c Config) GetSubsysInfo(subSys, target string, redactSecrets bool) ([]SubsysInfo, error) { + // Check if config param requested is valid. + defKVS1, ok := DefaultKVS[subSys] + if !ok { + return nil, Errorf("unknown subsystem: %s", subSys) + } + + targets, err := c.GetAvailableTargets(subSys) + if err != nil { + return nil, err + } + + if target != "" { + found := slices.Contains(targets, target) + if !found { + return nil, Errorf("there is no target `%s` for subsystem `%s`", target, subSys) + } + targets = []string{target} + } + + // The `Comment` configuration variable is optional but is available to be + // set for all sub-systems. It is not present in the `DefaultKVS` map's + // values. To enable fetching a configured comment value from the + // environment we add it to the list of default keys for the subsystem. + defKVS := make([]KV, len(defKVS1), len(defKVS1)+1) + copy(defKVS, defKVS1) + defKVS = append(defKVS, KV{Key: Comment}) + + r := make([]SubsysInfo, 0, len(targets)) + for _, target := range targets { + r = append(r, SubsysInfo{ + SubSys: subSys, + Target: target, + Defaults: defKVS, + Config: c.getTargetKVS(subSys, target, redactSecrets), + EnvMap: c.getTargetEnvs(subSys, target, defKVS, redactSecrets), + }) + } + + return r, nil +} + +// AddEnvString adds env vars to the given string builder. +func (cs *SubsysInfo) AddEnvString(b *strings.Builder) { + for _, v := range cs.Defaults { + if ep, ok := cs.EnvMap[v.Key]; ok { + b.WriteString(KvComment) + b.WriteString(KvSpaceSeparator) + b.WriteString(ep.Name) + b.WriteString(EnvSeparator) + b.WriteString(ep.Value) + b.WriteString(KvNewline) + } + } +} + +// WriteTo writes the string representation of the configuration to the given +// builder. When off is true, adds a comment character before the config system +// output. It also ignores values when empty and deprecated. +func (cs *SubsysInfo) WriteTo(b *strings.Builder, off bool) { + cs.AddEnvString(b) + if off { + b.WriteString(KvComment) + b.WriteString(KvSpaceSeparator) + } + b.WriteString(cs.SubSys) + if cs.Target != Default { + b.WriteString(SubSystemSeparator) + b.WriteString(cs.Target) + } + b.WriteString(KvSpaceSeparator) + for _, kv := range cs.Config { + dkv, ok := cs.Defaults.LookupKV(kv.Key) + if !ok { + continue + } + // Ignore empty and deprecated values + if dkv.HiddenIfEmpty && kv.Value == "" { + continue + } + // Do not need to print if state is on + if kv.Key == Enable && kv.Value == EnableOn { + continue + } + b.WriteString(kv.String()) + b.WriteString(KvSpaceSeparator) + } + + b.WriteString(KvNewline) +} diff --git a/internal/config/config_test.go b/internal/config/config_test.go new file mode 100644 index 00000000..e55d446b --- /dev/null +++ b/internal/config/config_test.go @@ -0,0 +1,130 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package config + +import ( + "testing" +) + +func TestKVFields(t *testing.T) { + tests := []struct { + input string + keys []string + expectedFields map[string]struct{} + }{ + // No keys present + { + input: "", + keys: []string{"comment"}, + expectedFields: map[string]struct{}{}, + }, + // No keys requested for tokenizing + { + input: `comment="Hi this is my comment ="`, + keys: []string{}, + expectedFields: map[string]struct{}{}, + }, + // Single key requested and present + { + input: `comment="Hi this is my comment ="`, + keys: []string{"comment"}, + expectedFields: map[string]struct{}{`comment="Hi this is my comment ="`: {}}, + }, + // Keys and input order of k=v is same. + { + input: `connection_string="host=localhost port=2832" comment="really long comment"`, + keys: []string{"connection_string", "comment"}, + expectedFields: map[string]struct{}{ + `connection_string="host=localhost port=2832"`: {}, + `comment="really long comment"`: {}, + }, + }, + // Keys with spaces in between + { + input: `enable=on format=namespace connection_string=" host=localhost port=5432 dbname = cesnietor sslmode=disable" table=holicrayoli`, + keys: []string{"enable", "connection_string", "comment", "format", "table"}, + expectedFields: map[string]struct{}{ + `enable=on`: {}, + `format=namespace`: {}, + `connection_string=" host=localhost port=5432 dbname = cesnietor sslmode=disable"`: {}, + `table=holicrayoli`: {}, + }, + }, + // One of the keys is not present and order of input has changed. + { + input: `comment="really long comment" connection_string="host=localhost port=2832"`, + keys: []string{"connection_string", "comment", "format"}, + expectedFields: map[string]struct{}{ + `connection_string="host=localhost port=2832"`: {}, + `comment="really long comment"`: {}, + }, + }, + // Incorrect delimiter, expected fields should be empty. + { + input: `comment:"really long comment" connection_string:"host=localhost port=2832"`, + keys: []string{"connection_string", "comment"}, + expectedFields: map[string]struct{}{}, + }, + // Incorrect type of input v/s required keys. + { + input: `comme="really long comment" connection_str="host=localhost port=2832"`, + keys: []string{"connection_string", "comment"}, + expectedFields: map[string]struct{}{}, + }, + } + for _, test := range tests { + t.Run("", func(t *testing.T) { + gotFields := kvFields(test.input, test.keys) + if len(gotFields) != len(test.expectedFields) { + t.Errorf("Expected keys %d, found %d", len(test.expectedFields), len(gotFields)) + } + found := true + for _, field := range gotFields { + _, ok := test.expectedFields[field] + found = found && ok + } + if !found { + t.Errorf("Expected %s, got %s", test.expectedFields, gotFields) + } + }) + } +} + +func TestValidRegion(t *testing.T) { + tests := []struct { + name string + success bool + }{ + {name: "us-east-1", success: true}, + {name: "us_east", success: true}, + {name: "helloWorld", success: true}, + {name: "-fdslka", success: false}, + {name: "^00[", success: false}, + {name: "my region", success: false}, + {name: "%%$#!", success: false}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + ok := validRegionRegex.MatchString(test.name) + if test.success != ok { + t.Errorf("Expected %t, got %t", test.success, ok) + } + }) + } +} diff --git a/internal/config/constants.go b/internal/config/constants.go new file mode 100644 index 00000000..bd00ae34 --- /dev/null +++ b/internal/config/constants.go @@ -0,0 +1,95 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package config + +// Config value separator +const ( + ValueSeparator = "," +) + +// Top level common ENVs +const ( + EnvAccessKey = "MINIO_ACCESS_KEY" + EnvSecretKey = "MINIO_SECRET_KEY" + EnvRootUser = "MINIO_ROOT_USER" + EnvRootPassword = "MINIO_ROOT_PASSWORD" + + // Legacy files + EnvAccessKeyFile = "MINIO_ACCESS_KEY_FILE" + EnvSecretKeyFile = "MINIO_SECRET_KEY_FILE" + + // Current files + EnvRootUserFile = "MINIO_ROOT_USER_FILE" + EnvRootPasswordFile = "MINIO_ROOT_PASSWORD_FILE" + + // Set all config environment variables from 'config.env' + // if necessary. Overrides all previous settings and also + // overrides all environment values passed from + // 'podman run -e ENV=value' + EnvConfigEnvFile = "MINIO_CONFIG_ENV_FILE" + + EnvBrowser = "MINIO_BROWSER" + EnvDomain = "MINIO_DOMAIN" + EnvPublicIPs = "MINIO_PUBLIC_IPS" + EnvFSOSync = "MINIO_FS_OSYNC" + EnvArgs = "MINIO_ARGS" + EnvVolumes = "MINIO_VOLUMES" + EnvDNSWebhook = "MINIO_DNS_WEBHOOK_ENDPOINT" + + EnvSiteName = "MINIO_SITE_NAME" + EnvSiteRegion = "MINIO_SITE_REGION" + + EnvMinIOSubnetLicense = "MINIO_SUBNET_LICENSE" // Deprecated Dec 2021 + EnvMinIOSubnetAPIKey = "MINIO_SUBNET_API_KEY" + EnvMinIOSubnetProxy = "MINIO_SUBNET_PROXY" + + EnvMinIOCallhomeEnable = "MINIO_CALLHOME_ENABLE" + EnvMinIOCallhomeFrequency = "MINIO_CALLHOME_FREQUENCY" + + EnvMinIOServerURL = "MINIO_SERVER_URL" + EnvBrowserRedirect = "MINIO_BROWSER_REDIRECT" // On by default + EnvBrowserRedirectURL = "MINIO_BROWSER_REDIRECT_URL" + EnvRootDriveThresholdSize = "MINIO_ROOTDRIVE_THRESHOLD_SIZE" + EnvRootDiskThresholdSize = "MINIO_ROOTDISK_THRESHOLD_SIZE" // Deprecated Sep 2023 + EnvBrowserLoginAnimation = "MINIO_BROWSER_LOGIN_ANIMATION" + EnvBrowserSessionDuration = "MINIO_BROWSER_SESSION_DURATION" // Deprecated after November 2023 + EnvMinioStsDuration = "MINIO_STS_DURATION" + EnvMinIOLogQueryURL = "MINIO_LOG_QUERY_URL" + EnvMinIOLogQueryAuthToken = "MINIO_LOG_QUERY_AUTH_TOKEN" + EnvMinIOPrometheusURL = "MINIO_PROMETHEUS_URL" + EnvMinIOPrometheusJobID = "MINIO_PROMETHEUS_JOB_ID" + EnvMinIOPrometheusExtraLabels = "MINIO_PROMETHEUS_EXTRA_LABELS" + EnvMinIOPrometheusAuthToken = "MINIO_PROMETHEUS_AUTH_TOKEN" + EnvConsoleDebugLogLevel = "MINIO_CONSOLE_DEBUG_LOGLEVEL" + + EnvUpdate = "MINIO_UPDATE" + + EnvEndpoints = "MINIO_ENDPOINTS" // legacy + EnvWorm = "MINIO_WORM" // legacy + EnvRegion = "MINIO_REGION" // legacy + EnvRegionName = "MINIO_REGION_NAME" // legacy + +) + +// Expiration Token durations +// These values are used to validate the expiration time range from +// either the exp claim or MINI_STS_DURATION value +const ( + MinExpiration = 900 + MaxExpiration = 31536000 +) diff --git a/internal/config/crypto.go b/internal/config/crypto.go new file mode 100644 index 00000000..757b1db4 --- /dev/null +++ b/internal/config/crypto.go @@ -0,0 +1,172 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package config + +import ( + "bytes" + "context" + "crypto/rand" + "encoding/binary" + "errors" + "fmt" + "io" + + jsoniter "github.com/json-iterator/go" + "github.com/minio/minio/internal/kms" + "github.com/secure-io/sio-go" + "github.com/secure-io/sio-go/sioutil" +) + +// EncryptBytes encrypts the plaintext with a key managed by KMS. +// The context is bound to the returned ciphertext. +// +// The same context must be provided when decrypting the +// ciphertext. +func EncryptBytes(k *kms.KMS, plaintext []byte, context kms.Context) ([]byte, error) { + ciphertext, err := Encrypt(k, bytes.NewReader(plaintext), context) + if err != nil { + return nil, err + } + return io.ReadAll(ciphertext) +} + +// DecryptBytes decrypts the ciphertext using a key managed by the KMS. +// The same context that have been used during encryption must be +// provided. +func DecryptBytes(k *kms.KMS, ciphertext []byte, context kms.Context) ([]byte, error) { + plaintext, err := Decrypt(k, bytes.NewReader(ciphertext), context) + if err != nil { + return nil, err + } + return io.ReadAll(plaintext) +} + +// Encrypt encrypts the plaintext with a key managed by KMS. +// The context is bound to the returned ciphertext. +// +// The same context must be provided when decrypting the +// ciphertext. +func Encrypt(k *kms.KMS, plaintext io.Reader, ctx kms.Context) (io.Reader, error) { + algorithm := sio.AES_256_GCM + if !sioutil.NativeAES() { + algorithm = sio.ChaCha20Poly1305 + } + + key, err := k.GenerateKey(context.Background(), &kms.GenerateKeyRequest{AssociatedData: ctx}) + if err != nil { + return nil, err + } + stream, err := algorithm.Stream(key.Plaintext) + if err != nil { + return nil, err + } + nonce := make([]byte, stream.NonceSize()) + if _, err := rand.Read(nonce); err != nil { + return nil, err + } + + const ( + MaxMetadataSize = 1 << 20 // max. size of the metadata + Version = 1 + ) + var ( + header [5]byte + buffer bytes.Buffer + ) + json := jsoniter.ConfigCompatibleWithStandardLibrary + metadata, err := json.Marshal(encryptedObject{ + KeyID: key.KeyID, + KMSKey: key.Ciphertext, + Algorithm: algorithm, + Nonce: nonce, + }) + if err != nil { + return nil, err + } + if len(metadata) > MaxMetadataSize { + return nil, errors.New("config: encryption metadata is too large") + } + header[0] = Version + binary.LittleEndian.PutUint32(header[1:], uint32(len(metadata))) + buffer.Write(header[:]) + buffer.Write(metadata) + + return io.MultiReader( + &buffer, + stream.EncryptReader(plaintext, nonce, nil), + ), nil +} + +// Decrypt decrypts the ciphertext using a key managed by the KMS. +// The same context that have been used during encryption must be +// provided. +func Decrypt(k *kms.KMS, ciphertext io.Reader, associatedData kms.Context) (io.Reader, error) { + const ( + MaxMetadataSize = 1 << 20 // max. size of the metadata + Version = 1 + ) + + var header [5]byte + if _, err := io.ReadFull(ciphertext, header[:]); err != nil { + return nil, err + } + if header[0] != Version { + return nil, fmt.Errorf("config: unknown ciphertext version %d", header[0]) + } + size := binary.LittleEndian.Uint32(header[1:]) + if size > MaxMetadataSize { + return nil, errors.New("config: encryption metadata is too large") + } + + var ( + metadataBuffer = make([]byte, size) + metadata encryptedObject + ) + if _, err := io.ReadFull(ciphertext, metadataBuffer); err != nil { + return nil, err + } + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err := json.Unmarshal(metadataBuffer, &metadata); err != nil { + return nil, err + } + + key, err := k.Decrypt(context.TODO(), &kms.DecryptRequest{ + Name: metadata.KeyID, + Ciphertext: metadata.KMSKey, + AssociatedData: associatedData, + }) + if err != nil { + return nil, err + } + stream, err := metadata.Algorithm.Stream(key) + if err != nil { + return nil, err + } + if stream.NonceSize() != len(metadata.Nonce) { + return nil, sio.NotAuthentic + } + return stream.DecryptReader(ciphertext, metadata.Nonce, nil), nil +} + +type encryptedObject struct { + KeyID string `json:"keyid"` + KMSKey []byte `json:"kmskey"` + + Algorithm sio.Algorithm `json:"algorithm"` + Nonce []byte `json:"nonce"` +} diff --git a/internal/config/crypto_test.go b/internal/config/crypto_test.go new file mode 100644 index 00000000..224d230b --- /dev/null +++ b/internal/config/crypto_test.go @@ -0,0 +1,118 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package config + +import ( + "bytes" + "encoding/hex" + "io" + "testing" + + "github.com/minio/minio/internal/kms" +) + +var encryptDecryptTests = []struct { + Data []byte + Context kms.Context +}{ + { + Data: nil, + Context: nil, + }, + { + Data: []byte{1}, + Context: nil, + }, + { + Data: []byte{1}, + Context: kms.Context{"key": "value"}, + }, + { + Data: make([]byte, 1<<20), + Context: kms.Context{"key": "value", "a": "b"}, + }, +} + +func TestEncryptDecrypt(t *testing.T) { + key, err := hex.DecodeString("ddedadb867afa3f73bd33c25499a723ed7f9f51172ee7b1b679e08dc795debcc") + if err != nil { + t.Fatalf("Failed to decode master key: %v", err) + } + KMS, err := kms.NewBuiltin("my-key", key) + if err != nil { + t.Fatalf("Failed to create KMS: %v", err) + } + + for i, test := range encryptDecryptTests { + ciphertext, err := Encrypt(KMS, bytes.NewReader(test.Data), test.Context) + if err != nil { + t.Fatalf("Test %d: failed to encrypt stream: %v", i, err) + } + data, err := io.ReadAll(ciphertext) + if err != nil { + t.Fatalf("Test %d: failed to encrypt stream: %v", i, err) + } + + plaintext, err := Decrypt(KMS, bytes.NewReader(data), test.Context) + if err != nil { + t.Fatalf("Test %d: failed to decrypt stream: %v", i, err) + } + data, err = io.ReadAll(plaintext) + if err != nil { + t.Fatalf("Test %d: failed to decrypt stream: %v", i, err) + } + + if !bytes.Equal(data, test.Data) { + t.Fatalf("Test %d: decrypted data does not match original data", i) + } + } +} + +func BenchmarkEncrypt(b *testing.B) { + key, err := hex.DecodeString("ddedadb867afa3f73bd33c25499a723ed7f9f51172ee7b1b679e08dc795debcc") + if err != nil { + b.Fatalf("Failed to decode master key: %v", err) + } + KMS, err := kms.NewBuiltin("my-key", key) + if err != nil { + b.Fatalf("Failed to create KMS: %v", err) + } + + benchmarkEncrypt := func(size int, b *testing.B) { + var ( + data = make([]byte, size) + plaintext = bytes.NewReader(data) + context = kms.Context{"key": "value"} + ) + b.SetBytes(int64(size)) + for b.Loop() { + ciphertext, err := Encrypt(KMS, plaintext, context) + if err != nil { + b.Fatal(err) + } + if _, err = io.Copy(io.Discard, ciphertext); err != nil { + b.Fatal(err) + } + plaintext.Reset(data) + } + } + b.Run("1KB", func(b *testing.B) { benchmarkEncrypt(1*1024, b) }) + b.Run("512KB", func(b *testing.B) { benchmarkEncrypt(512*1024, b) }) + b.Run("1MB", func(b *testing.B) { benchmarkEncrypt(1024*1024, b) }) + b.Run("10MB", func(b *testing.B) { benchmarkEncrypt(10*1024*1024, b) }) +} diff --git a/internal/config/dns/dns_path.go b/internal/config/dns/dns_path.go new file mode 100644 index 00000000..9e9a7208 --- /dev/null +++ b/internal/config/dns/dns_path.go @@ -0,0 +1,57 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dns + +import ( + "path" + "strings" + + "github.com/miekg/dns" +) + +// msgPath converts a domainname to an etcd path. If s looks like service.staging.skydns.local., +// the resulting key will be /skydns/local/skydns/staging/service . +func msgPath(s, prefix string) string { + l := dns.SplitDomainName(s) + for i, j := 0, len(l)-1; i < j; i, j = i+1, j-1 { + l[i], l[j] = l[j], l[i] + } + return path.Join(append([]string{etcdPathSeparator + prefix + etcdPathSeparator}, l...)...) +} + +// dnsJoin joins labels to form a fully qualified domain name. If the last label is +// the root label it is ignored. Not other syntax checks are performed. +func dnsJoin(labels ...string) string { + if len(labels) == 0 { + return "" + } + ll := len(labels) + if labels[ll-1] == "." { + return strings.Join(labels[:ll-1], ".") + "." + } + return dns.Fqdn(strings.Join(labels, ".")) +} + +// msgUnPath converts a etcd path to domainName. +func msgUnPath(s string) string { + ks := strings.Split(strings.Trim(s, etcdPathSeparator), etcdPathSeparator) + for i, j := 0, len(ks)-1; i < j; i, j = i+1, j-1 { + ks[i], ks[j] = ks[j], ks[i] + } + return strings.Join(ks, ".") +} diff --git a/internal/config/dns/etcd_dns.go b/internal/config/dns/etcd_dns.go new file mode 100644 index 00000000..120eab5e --- /dev/null +++ b/internal/config/dns/etcd_dns.go @@ -0,0 +1,291 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dns + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "sort" + "strings" + "time" + + "github.com/minio/minio-go/v7/pkg/set" + clientv3 "go.etcd.io/etcd/client/v3" +) + +// ErrNoEntriesFound - Indicates no entries were found for the given key (directory) +var ErrNoEntriesFound = errors.New("No entries found for this key") + +// ErrDomainMissing - Indicates domain is missing +var ErrDomainMissing = errors.New("domain is missing") + +const etcdPathSeparator = "/" + +// create a new coredns service record for the bucket. +func newCoreDNSMsg(ip string, port string, ttl uint32, t time.Time) ([]byte, error) { + return json.Marshal(&SrvRecord{ + Host: ip, + Port: json.Number(port), + TTL: ttl, + CreationDate: t, + }) +} + +// Close closes the internal etcd client and cannot be used further +func (c *CoreDNS) Close() error { + c.etcdClient.Close() + return nil +} + +// List - Retrieves list of DNS entries for the domain. +func (c *CoreDNS) List() (map[string][]SrvRecord, error) { + srvRecords := map[string][]SrvRecord{} + for _, domainName := range c.domainNames { + key := msgPath(fmt.Sprintf("%s.", domainName), c.prefixPath) + records, err := c.list(key+etcdPathSeparator, true) + if err != nil { + return srvRecords, err + } + for _, record := range records { + if record.Key == "" { + continue + } + srvRecords[record.Key] = append(srvRecords[record.Key], record) + } + } + return srvRecords, nil +} + +// Get - Retrieves DNS records for a bucket. +func (c *CoreDNS) Get(bucket string) ([]SrvRecord, error) { + var srvRecords []SrvRecord + for _, domainName := range c.domainNames { + key := msgPath(fmt.Sprintf("%s.%s.", bucket, domainName), c.prefixPath) + records, err := c.list(key, false) + if err != nil { + return nil, err + } + // Make sure we have record.Key is empty + // this can only happen when record.Key + // has bucket entry with exact prefix + // match any record.Key which do not + // match the prefixes we skip them. + for _, record := range records { + if record.Key != "" { + continue + } + srvRecords = append(srvRecords, record) + } + } + if len(srvRecords) == 0 { + return nil, ErrNoEntriesFound + } + return srvRecords, nil +} + +// Retrieves list of entries under the key passed. +// Note that this method fetches entries upto only two levels deep. +func (c *CoreDNS) list(key string, domain bool) ([]SrvRecord, error) { + ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout) + r, err := c.etcdClient.Get(ctx, key, clientv3.WithPrefix()) + defer cancel() + if err != nil { + return nil, err + } + + if r.Count == 0 { + key = strings.TrimSuffix(key, etcdPathSeparator) + r, err = c.etcdClient.Get(ctx, key) + if err != nil { + return nil, err + } + // only if we are looking at `domain` as true + // we should return error here. + if domain && r.Count == 0 { + return nil, ErrDomainMissing + } + } + + var srvRecords []SrvRecord + for _, n := range r.Kvs { + var srvRecord SrvRecord + if err = json.Unmarshal(n.Value, &srvRecord); err != nil { + return nil, err + } + srvRecord.Key = strings.TrimPrefix(string(n.Key), key) + srvRecord.Key = strings.TrimSuffix(srvRecord.Key, srvRecord.Host) + + // Skip non-bucket entry like for a key + // /skydns/net/miniocloud/10.0.0.1 that may exist as + // dns entry for the server (rather than the bucket + // itself). + if srvRecord.Key == "" { + continue + } + + srvRecord.Key = msgUnPath(srvRecord.Key) + srvRecords = append(srvRecords, srvRecord) + } + sort.Slice(srvRecords, func(i int, j int) bool { + return srvRecords[i].Key < srvRecords[j].Key + }) + return srvRecords, nil +} + +// Put - Adds DNS entries into etcd endpoint in CoreDNS etcd message format. +func (c *CoreDNS) Put(bucket string) error { + c.Delete(bucket) // delete any existing entries. + + t := time.Now().UTC() + for ip := range c.domainIPs { + bucketMsg, err := newCoreDNSMsg(ip, c.domainPort, defaultTTL, t) + if err != nil { + return err + } + for _, domainName := range c.domainNames { + key := msgPath(fmt.Sprintf("%s.%s", bucket, domainName), c.prefixPath) + key = key + etcdPathSeparator + ip + ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout) + _, err = c.etcdClient.Put(ctx, key, string(bucketMsg)) + cancel() + if err != nil { + ctx, cancel = context.WithTimeout(context.Background(), defaultContextTimeout) + c.etcdClient.Delete(ctx, key) + cancel() + return err + } + } + } + return nil +} + +// Delete - Removes DNS entries added in Put(). +func (c *CoreDNS) Delete(bucket string) error { + for _, domainName := range c.domainNames { + key := msgPath(fmt.Sprintf("%s.%s.", bucket, domainName), c.prefixPath) + ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout) + _, err := c.etcdClient.Delete(ctx, key+etcdPathSeparator, clientv3.WithPrefix()) + cancel() + if err != nil { + return err + } + } + return nil +} + +// DeleteRecord - Removes a specific DNS entry +func (c *CoreDNS) DeleteRecord(record SrvRecord) error { + for _, domainName := range c.domainNames { + key := msgPath(fmt.Sprintf("%s.%s.", record.Key, domainName), c.prefixPath) + + ctx, cancel := context.WithTimeout(context.Background(), defaultContextTimeout) + _, err := c.etcdClient.Delete(ctx, key+etcdPathSeparator+record.Host) + cancel() + if err != nil { + return err + } + } + return nil +} + +// String stringer name for this implementation of dns.Store +func (c *CoreDNS) String() string { + return "etcdDNS" +} + +// CoreDNS - represents dns config for coredns server. +type CoreDNS struct { + domainNames []string + domainIPs set.StringSet + domainPort string + prefixPath string + etcdClient *clientv3.Client +} + +// EtcdOption - functional options pattern style +type EtcdOption func(*CoreDNS) + +// DomainNames set a list of domain names used by this CoreDNS +// client setting, note this will fail if set to empty when +// constructor initializes. +func DomainNames(domainNames []string) EtcdOption { + return func(args *CoreDNS) { + args.domainNames = domainNames + } +} + +// DomainIPs set a list of custom domain IPs, note this will +// fail if set to empty when constructor initializes. +func DomainIPs(domainIPs set.StringSet) EtcdOption { + return func(args *CoreDNS) { + args.domainIPs = domainIPs + } +} + +// DomainPort - is a string version of server port +func DomainPort(domainPort string) EtcdOption { + return func(args *CoreDNS) { + args.domainPort = domainPort + } +} + +// CoreDNSPath - custom prefix on etcd to populate DNS +// service records, optional and can be empty. +// if empty then c.prefixPath is used i.e "/skydns" +func CoreDNSPath(prefix string) EtcdOption { + return func(args *CoreDNS) { + args.prefixPath = prefix + } +} + +// NewCoreDNS - initialize a new coreDNS set/unset values. +func NewCoreDNS(cfg clientv3.Config, setters ...EtcdOption) (Store, error) { + etcdClient, err := clientv3.New(cfg) + if err != nil { + return nil, err + } + + args := &CoreDNS{ + etcdClient: etcdClient, + } + + for _, setter := range setters { + setter(args) + } + + if len(args.domainNames) == 0 || args.domainIPs.IsEmpty() { + return nil, errors.New("invalid argument") + } + + // strip ports off of domainIPs + domainIPsWithoutPorts := args.domainIPs.ApplyFunc(func(ip string) string { + host, _, err := net.SplitHostPort(ip) + if err != nil { + if strings.Contains(err.Error(), "missing port in address") { + host = ip + } + } + return host + }) + args.domainIPs = domainIPsWithoutPorts + + return args, nil +} diff --git a/internal/config/dns/etcd_dns_test.go b/internal/config/dns/etcd_dns_test.go new file mode 100644 index 00000000..4be271bf --- /dev/null +++ b/internal/config/dns/etcd_dns_test.go @@ -0,0 +1,74 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dns + +import "testing" + +func TestDNSJoin(t *testing.T) { + tests := []struct { + in []string + out string + }{ + {[]string{"bla", "bliep", "example", "org"}, "bla.bliep.example.org."}, + {[]string{"example", "."}, "example."}, + {[]string{"example", "org."}, "example.org."}, // technically we should not be called like this. + {[]string{"."}, "."}, + } + + for i, tc := range tests { + if x := dnsJoin(tc.in...); x != tc.out { + t.Errorf("Test %d, expected %s, got %s", i, tc.out, x) + } + } +} + +func TestPath(t *testing.T) { + for _, path := range []string{"mydns", "skydns"} { + result := msgPath("service.staging.skydns.local.", path) + if result != etcdPathSeparator+path+"/local/skydns/staging/service" { + t.Errorf("Failure to get domain's path with prefix: %s", result) + } + } +} + +func TestUnPath(t *testing.T) { + result1 := msgUnPath("/skydns/local/cluster/staging/service/") + if result1 != "service.staging.cluster.local.skydns" { + t.Errorf("Failure to get domain from etcd key (with a trailing '/'), expect: 'service.staging.cluster.local.', actually get: '%s'", result1) + } + + result2 := msgUnPath("/skydns/local/cluster/staging/service") + if result2 != "service.staging.cluster.local.skydns" { + t.Errorf("Failure to get domain from etcd key (without trailing '/'), expect: 'service.staging.cluster.local.' actually get: '%s'", result2) + } + + result3 := msgUnPath("/singleleveldomain/") + if result3 != "singleleveldomain" { + t.Errorf("Failure to get domain from etcd key (with leading and trailing '/'), expect: 'singleleveldomain.' actually get: '%s'", result3) + } + + result4 := msgUnPath("/singleleveldomain") + if result4 != "singleleveldomain" { + t.Errorf("Failure to get domain from etcd key (without trailing '/'), expect: 'singleleveldomain.' actually get: '%s'", result4) + } + + result5 := msgUnPath("singleleveldomain") + if result5 != "singleleveldomain" { + t.Errorf("Failure to get domain from etcd key (without leading and trailing '/'), expect: 'singleleveldomain.' actually get: '%s'", result5) + } +} diff --git a/internal/config/dns/operator_dns.go b/internal/config/dns/operator_dns.go new file mode 100644 index 00000000..e703103c --- /dev/null +++ b/internal/config/dns/operator_dns.go @@ -0,0 +1,240 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dns + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "io" + "net" + "net/http" + "net/url" + "strconv" + "strings" + "time" + + "github.com/golang-jwt/jwt/v4" + "github.com/minio/minio/internal/config" + xhttp "github.com/minio/minio/internal/http" +) + +var ( + defaultOperatorContextTimeout = 10 * time.Second + // ErrNotImplemented - Indicates the functionality which is not implemented + ErrNotImplemented = errors.New("The method is not implemented") +) + +func (c *OperatorDNS) addAuthHeader(r *http.Request) error { + if c.username == "" || c.password == "" { + return nil + } + + claims := &jwt.StandardClaims{ + ExpiresAt: int64(15 * time.Minute), + Issuer: c.username, + Subject: config.EnvDNSWebhook, + } + + token := jwt.NewWithClaims(jwt.SigningMethodHS512, claims) + ss, err := token.SignedString([]byte(c.password)) + if err != nil { + return err + } + + r.Header.Set("Authorization", "Bearer "+ss) + return nil +} + +func (c *OperatorDNS) endpoint(bucket string, del bool) (string, error) { + u, err := url.Parse(c.Endpoint) + if err != nil { + return "", err + } + q := u.Query() + q.Add("bucket", bucket) + q.Add("delete", strconv.FormatBool(del)) + u.RawQuery = q.Encode() + return u.String(), nil +} + +// Put - Adds DNS entries into operator webhook server +func (c *OperatorDNS) Put(bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), defaultOperatorContextTimeout) + defer cancel() + e, err := c.endpoint(bucket, false) + if err != nil { + return newError(bucket, err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, e, nil) + if err != nil { + return newError(bucket, err) + } + if err = c.addAuthHeader(req); err != nil { + return newError(bucket, err) + } + + resp, err := c.httpClient.Do(req) + if err != nil { + if derr := c.Delete(bucket); derr != nil { + return newError(bucket, derr) + } + return err + } + defer xhttp.DrainBody(resp.Body) + + if resp.StatusCode != http.StatusOK { + var errorStringBuilder strings.Builder + io.Copy(&errorStringBuilder, io.LimitReader(resp.Body, resp.ContentLength)) + errorString := errorStringBuilder.String() + if resp.StatusCode == http.StatusConflict { + return ErrBucketConflict(Error{bucket, errors.New(errorString)}) + } + return newError(bucket, fmt.Errorf("service create for bucket %s, failed with status %s, error %s", bucket, resp.Status, errorString)) + } + return nil +} + +func newError(bucket string, err error) error { + e := Error{bucket, err} + if strings.Contains(err.Error(), "invalid bucket name") { + return ErrInvalidBucketName(e) + } + return e +} + +// Delete - Removes DNS entries added in Put(). +func (c *OperatorDNS) Delete(bucket string) error { + ctx, cancel := context.WithTimeout(context.Background(), defaultOperatorContextTimeout) + defer cancel() + e, err := c.endpoint(bucket, true) + if err != nil { + return err + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, e, nil) + if err != nil { + return err + } + if err = c.addAuthHeader(req); err != nil { + return err + } + resp, err := c.httpClient.Do(req) + if err != nil { + return err + } + xhttp.DrainBody(resp.Body) + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("request to delete the service for bucket %s, failed with status %s", bucket, resp.Status) + } + return nil +} + +// DeleteRecord - Removes a specific DNS entry +// No Op for Operator because operator deals on with bucket entries +func (c *OperatorDNS) DeleteRecord(record SrvRecord) error { + return ErrNotImplemented +} + +// Close closes the internal http client +func (c *OperatorDNS) Close() error { + return nil +} + +// List - Retrieves list of DNS entries for the domain. +// This is a No Op for Operator because, there is no intent to enforce global +// namespace at MinIO level with this DNS entry. The global namespace in +// enforced by the Kubernetes Operator +func (c *OperatorDNS) List() (srvRecords map[string][]SrvRecord, err error) { + return nil, ErrNotImplemented +} + +// Get - Retrieves DNS records for a bucket. +// This is a No Op for Operator because, there is no intent to enforce global +// namespace at MinIO level with this DNS entry. The global namespace in +// enforced by the Kubernetes Operator +func (c *OperatorDNS) Get(bucket string) (srvRecords []SrvRecord, err error) { + return nil, ErrNotImplemented +} + +// String stringer name for this implementation of dns.Store +func (c *OperatorDNS) String() string { + return "webhookDNS" +} + +// OperatorDNS - represents dns config for MinIO k8s operator. +type OperatorDNS struct { + httpClient *http.Client + Endpoint string + rootCAs *x509.CertPool + username string + password string +} + +// OperatorOption - functional options pattern style for OperatorDNS +type OperatorOption func(*OperatorDNS) + +// Authentication - custom username and password for authenticating at the endpoint +func Authentication(username, password string) OperatorOption { + return func(args *OperatorDNS) { + args.username = username + args.password = password + } +} + +// RootCAs - add custom trust certs pool +func RootCAs(certPool *x509.CertPool) OperatorOption { + return func(args *OperatorDNS) { + args.rootCAs = certPool + } +} + +// NewOperatorDNS - initialize a new K8S Operator DNS set/unset values. +func NewOperatorDNS(endpoint string, setters ...OperatorOption) (Store, error) { + if endpoint == "" { + return nil, errors.New("invalid argument") + } + + args := &OperatorDNS{ + Endpoint: endpoint, + } + for _, setter := range setters { + setter(args) + } + args.httpClient = &http.Client{ + Transport: &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: (&net.Dialer{ + Timeout: 3 * time.Second, + KeepAlive: 5 * time.Second, + }).DialContext, + ResponseHeaderTimeout: 3 * time.Second, + TLSHandshakeTimeout: 3 * time.Second, + ExpectContinueTimeout: 3 * time.Second, + TLSClientConfig: &tls.Config{ + RootCAs: args.rootCAs, + }, + // Go net/http automatically unzip if content-type is + // gzip disable this feature, as we are always interested + // in raw stream. + DisableCompression: true, + }, + } + return args, nil +} diff --git a/internal/config/dns/store.go b/internal/config/dns/store.go new file mode 100644 index 00000000..831b4539 --- /dev/null +++ b/internal/config/dns/store.go @@ -0,0 +1,53 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dns + +// Error - DNS related errors error. +type Error struct { + Bucket string + Err error +} + +// ErrInvalidBucketName for buckets with invalid name +type ErrInvalidBucketName Error + +func (e ErrInvalidBucketName) Error() string { + return e.Bucket + " invalid bucket name error: " + e.Err.Error() +} + +func (e Error) Error() string { + return "dns related error: " + e.Err.Error() +} + +// ErrBucketConflict for buckets that already exist +type ErrBucketConflict Error + +func (e ErrBucketConflict) Error() string { + return e.Bucket + " bucket conflict error: " + e.Err.Error() +} + +// Store dns record store +type Store interface { + Put(bucket string) error + Get(bucket string) ([]SrvRecord, error) + Delete(bucket string) error + List() (map[string][]SrvRecord, error) + DeleteRecord(record SrvRecord) error + Close() error + String() string +} diff --git a/internal/config/dns/types.go b/internal/config/dns/types.go new file mode 100644 index 00000000..44d37854 --- /dev/null +++ b/internal/config/dns/types.go @@ -0,0 +1,57 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dns + +import ( + "encoding/json" + "time" +) + +const ( + defaultTTL = 30 + defaultContextTimeout = 5 * time.Minute +) + +// SrvRecord - represents a DNS service record +type SrvRecord struct { + Host string `json:"host,omitempty"` + Port json.Number `json:"port,omitempty"` + Priority int `json:"priority,omitempty"` + Weight int `json:"weight,omitempty"` + Text string `json:"text,omitempty"` + Mail bool `json:"mail,omitempty"` // Be an MX record. Priority becomes Preference. + TTL uint32 `json:"ttl,omitempty"` + + // Holds info about when the entry was created first. + CreationDate time.Time `json:"creationDate"` + + // When a SRV record with a "Host: IP-address" is added, we synthesize + // a srv.Target domain name. Normally we convert the full Key where + // the record lives to a DNS name and use this as the srv.Target. When + // TargetStrip > 0 we strip the left most TargetStrip labels from the + // DNS name. + TargetStrip int `json:"targetstrip,omitempty"` + + // Group is used to group (or *not* to group) different services + // together. Services with an identical Group are returned in + // the same answer. + Group string `json:"group,omitempty"` + + // Key carries the original key used during Put(). + Key string `json:"-"` +} diff --git a/internal/config/drive/drive.go b/internal/config/drive/drive.go new file mode 100644 index 00000000..862c62ab --- /dev/null +++ b/internal/config/drive/drive.go @@ -0,0 +1,110 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package drive + +import ( + "sync" + "time" + + "github.com/minio/minio/internal/config" + "github.com/minio/pkg/v3/env" +) + +// Drive specific timeout environment variables +const ( + EnvMaxDriveTimeout = "MINIO_DRIVE_MAX_TIMEOUT" + EnvMaxDriveTimeoutLegacy = "_MINIO_DRIVE_MAX_TIMEOUT" + EnvMaxDiskTimeoutLegacy = "_MINIO_DISK_MAX_TIMEOUT" +) + +// DefaultKVS - default KVS for drive +var DefaultKVS = config.KVS{ + config.KV{ + Key: MaxTimeout, + Value: "30s", + }, +} + +var configLk sync.RWMutex + +// Config represents the subnet related configuration +type Config struct { + // MaxTimeout - maximum timeout for a drive operation + MaxTimeout time.Duration `json:"maxTimeout"` +} + +// Update - updates the config with latest values +func (c *Config) Update(updated Config) error { + configLk.Lock() + defer configLk.Unlock() + c.MaxTimeout = getMaxTimeout(updated.MaxTimeout) + return nil +} + +// GetMaxTimeout - returns the per call drive operation timeout +func (c *Config) GetMaxTimeout() time.Duration { + return c.GetOPTimeout() +} + +// GetOPTimeout - returns the per call drive operation timeout +func (c *Config) GetOPTimeout() time.Duration { + configLk.RLock() + defer configLk.RUnlock() + + return getMaxTimeout(c.MaxTimeout) +} + +// LookupConfig - lookup config and override with valid environment settings if any. +func LookupConfig(kvs config.KVS) (cfg Config, err error) { + cfg = Config{ + MaxTimeout: 30 * time.Second, + } + if err = config.CheckValidKeys(config.DriveSubSys, kvs, DefaultKVS); err != nil { + return cfg, err + } + + // if not set. Get default value from environment + d := env.Get(EnvMaxDriveTimeout, env.Get(EnvMaxDriveTimeoutLegacy, env.Get(EnvMaxDiskTimeoutLegacy, kvs.GetWithDefault(MaxTimeout, DefaultKVS)))) + if d == "" { + cfg.MaxTimeout = 30 * time.Second + } else { + dur, _ := time.ParseDuration(d) + if dur < time.Second { + cfg.MaxTimeout = 30 * time.Second + } else { + cfg.MaxTimeout = getMaxTimeout(dur) + } + } + return cfg, err +} + +func getMaxTimeout(t time.Duration) time.Duration { + if t > time.Second { + return t + } + // get default value + d := env.Get(EnvMaxDriveTimeoutLegacy, env.Get(EnvMaxDiskTimeoutLegacy, "")) + if d == "" { + return 30 * time.Second + } + dur, _ := time.ParseDuration(d) + if dur < time.Second { + return 30 * time.Second + } + return dur +} diff --git a/internal/config/drive/help.go b/internal/config/drive/help.go new file mode 100644 index 00000000..5964dcce --- /dev/null +++ b/internal/config/drive/help.go @@ -0,0 +1,35 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package drive + +import "github.com/minio/minio/internal/config" + +var ( + // MaxTimeout is the max timeout for drive + MaxTimeout = "max_timeout" + + // HelpDrive is help for drive + HelpDrive = config.HelpKVS{ + config.HelpKV{ + Key: MaxTimeout, + Type: "string", + Description: "set per call max_timeout for the drive, defaults to 30 seconds", + Optional: true, + }, + } +) diff --git a/internal/config/errors-utils.go b/internal/config/errors-utils.go new file mode 100644 index 00000000..3d75ab73 --- /dev/null +++ b/internal/config/errors-utils.go @@ -0,0 +1,161 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package config + +import ( + "errors" + "fmt" + "net" + "syscall" + + "github.com/minio/minio/internal/color" +) + +// Err is a structure which contains all information +// to print a fatal error message in json or pretty mode +// Err implements error so we can use it anywhere +type Err struct { + msg string + detail string + action string + hint string +} + +// Clone returns a new Err struct with the same information +func (u Err) Clone() Err { + return Err{ + msg: u.msg, + detail: u.detail, + action: u.action, + hint: u.hint, + } +} + +// Error returns the error message +func (u Err) Error() string { + if u.detail == "" { + if u.msg != "" { + return u.msg + } + return "" + } + return u.detail +} + +// Msg - Replace the current error's message +func (u Err) Msg(m string) Err { + e := u.Clone() + e.msg = m + return e +} + +// Msgf - Replace the current error's message +func (u Err) Msgf(m string, args ...any) Err { + e := u.Clone() + if len(args) == 0 { + e.msg = m + } else { + e.msg = fmt.Sprintf(m, args...) + } + return e +} + +// Hint - Replace the current error's message +func (u Err) Hint(m string, args ...any) Err { + e := u.Clone() + e.hint = fmt.Sprintf(m, args...) + return e +} + +// ErrFn function wrapper +type ErrFn func(err error) Err + +// Create a UI error generator, this is needed to simplify +// the update of the detailed error message in several places +// in MinIO code +func newErrFn(msg, action, hint string) ErrFn { + return func(err error) Err { + u := Err{ + msg: msg, + action: action, + hint: hint, + } + if err != nil { + u.detail = err.Error() + } + return u + } +} + +// ErrorToErr inspects the passed error and transforms it +// to the appropriate UI error. +func ErrorToErr(err error) Err { + if err == nil { + return Err{} + } + + // If this is already a Err, do nothing + if e, ok := err.(Err); ok { + return e + } + + // Show a generic message for known golang errors + if errors.Is(err, syscall.EADDRINUSE) { + return ErrPortAlreadyInUse(err).Msg("Specified port is already in use") + } else if errors.Is(err, syscall.EACCES) || errors.Is(err, syscall.EPERM) { + if netErr, ok := err.(*net.OpError); ok { + return ErrPortAccess(netErr).Msg("Insufficient permissions to use specified port") + } + } + + // Failed to identify what type of error this, return a simple UI error + return Err{msg: err.Error()} +} + +// FmtError converts a fatal error message to a more clear error +// using some colors +func FmtError(introMsg string, err error, jsonFlag bool) string { + renderedTxt := "" + uiErr := ErrorToErr(err) + // JSON print + if jsonFlag { + // Message text in json should be simple + if uiErr.detail != "" { + return uiErr.msg + ": " + uiErr.detail + } + return uiErr.msg + } + // Pretty print error message + introMsg += ": " + if uiErr.msg != "" { + introMsg += color.Bold(uiErr.msg) + } else { + introMsg += color.Bold(err.Error()) + } + renderedTxt += color.Red(introMsg) + "\n" + // Add action message + if uiErr.action != "" { + renderedTxt += "> " + color.BgYellow(color.Black(uiErr.action)) + "\n" + } + // Add hint + if uiErr.hint != "" { + renderedTxt += color.Bold("HINT:") + "\n" + renderedTxt += " " + uiErr.hint + } + return renderedTxt +} diff --git a/internal/config/errors.go b/internal/config/errors.go new file mode 100644 index 00000000..3bcb79ce --- /dev/null +++ b/internal/config/errors.go @@ -0,0 +1,248 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package config + +// UI errors +var ( + ErrInvalidXLValue = newErrFn( + "Invalid drive path", + "Please provide a fresh drive for single drive MinIO setup", + "MinIO only supports fresh drive paths", + ) + + ErrInvalidBrowserValue = newErrFn( + "Invalid console value", + "Please check the passed value", + "Environment can only accept `on` and `off` values. To disable Console access, set this value to `off`", + ) + + ErrInvalidFSOSyncValue = newErrFn( + "Invalid O_SYNC value", + "Please check the passed value", + "Can only accept `on` and `off` values. To enable O_SYNC for fs backend, set this value to `on`", + ) + + ErrOverlappingDomainValue = newErrFn( + "Overlapping domain values", + "Please check the passed value", + "MINIO_DOMAIN only accepts non-overlapping domain values", + ) + + ErrInvalidDomainValue = newErrFn( + "Invalid domain value", + "Please check the passed value", + "Domain can only accept DNS compatible values", + ) + + ErrInvalidErasureSetSize = newErrFn( + "Invalid erasure set size", + "Please check the passed value", + "Erasure set can only accept any of [2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16] values", + ) + + ErrInvalidWormValue = newErrFn( + "Invalid WORM value", + "Please check the passed value", + "WORM can only accept `on` and `off` values. To enable WORM, set this value to `on`", + ) + + ErrInvalidConfigDecryptionKey = newErrFn( + "Incorrect encryption key to decrypt internal data", + "Please set the correct default KMS key value or the correct root credentials for older MinIO versions.", + `Revert MINIO_KMS_KES_KEY_NAME or MINIO_ROOT_USER/MINIO_ROOT_PASSWORD (for older MinIO versions) to be able to decrypt the internal data again.`, + ) + + ErrInvalidCredentials = newErrFn( + "Invalid credentials", + "Please provide correct credentials", + `Access key length should be at least 3, and secret key length at least 8 characters`, + ) + + ErrInvalidRootUserCredentials = newErrFn( + "Invalid credentials", + "Please provide correct credentials", + EnvRootUser+` length should be at least 3, and `+EnvRootPassword+` length at least 8 characters`, + ) + + ErrMissingEnvCredentialRootUser = newErrFn( + "Missing credential environment variable, \""+EnvRootUser+"\"", + "Environment variable \""+EnvRootUser+"\" is missing", + `Root user name (access key) and root password (secret key) are expected to be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD respectively`, + ) + + ErrMissingEnvCredentialRootPassword = newErrFn( + "Missing credential environment variable, \""+EnvRootPassword+"\"", + "Environment variable \""+EnvRootPassword+"\" is missing", + `Root user name (access key) and root password (secret key) are expected to be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD respectively`, + ) + + ErrMissingEnvCredentialAccessKey = newErrFn( + "Missing credential environment variable, \""+EnvAccessKey+"\"", + "Environment variables \""+EnvAccessKey+"\" and \""+EnvSecretKey+"\" are deprecated", + `Root user name (access key) and root password (secret key) are expected to be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD respectively`, + ) + + ErrMissingEnvCredentialSecretKey = newErrFn( + "Missing credential environment variable, \""+EnvSecretKey+"\"", + "Environment variables \""+EnvSecretKey+"\" and \""+EnvAccessKey+"\" are deprecated", + `Root user name (access key) and root password (secret key) are expected to be specified via environment variables MINIO_ROOT_USER and MINIO_ROOT_PASSWORD respectively`, + ) + + ErrInvalidErasureEndpoints = newErrFn( + "Invalid endpoint(s) in erasure mode", + "Please provide correct combination of local/remote paths", + "For more information, please refer to https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html", + ) + + ErrInvalidNumberOfErasureEndpoints = newErrFn( + "Invalid total number of endpoints for erasure mode", + "Please provide number of endpoints greater or equal to 2", + "For more information, please refer to https://docs.min.io/community/minio-object-store/operations/concepts/erasure-coding.html", + ) + + ErrStorageClassValue = newErrFn( + "Invalid storage class value", + "Please check the value", + `MINIO_STORAGE_CLASS_STANDARD: Format "EC:" (e.g. "EC:3"). This sets the number of parity drives for MinIO server in Standard mode. Objects are stored in Standard mode, if storage class is not defined in Put request +MINIO_STORAGE_CLASS_RRS: Format "EC:" (e.g. "EC:3"). This sets the number of parity drives for MinIO server in Reduced Redundancy mode. Objects are stored in Reduced Redundancy mode, if Put request specifies RRS storage class +Refer to the link https://github.com/minio/minio/tree/master/docs/erasure/storage-class for more information`, + ) + + ErrUnexpectedBackendVersion = newErrFn( + "Backend version seems to be too recent", + "Please update to the latest MinIO version", + "", + ) + + ErrInvalidAddressFlag = newErrFn( + "--address input is invalid", + "Please check --address parameter", + `--address binds to a specific ADDRESS:PORT, ADDRESS can be an IPv4/IPv6 address or hostname (default port is ':9000') + Examples: --address ':443' + --address '172.16.34.31:9000' + --address '[fe80::da00:a6c8:e3ae:ddd7]:9000'`, + ) + + ErrInvalidEndpoint = newErrFn( + "Invalid endpoint for single drive mode", + "Please check the endpoint", + `Single-Node modes requires absolute path without hostnames: +Examples: + $ minio server /data/minio/ #Single Node Single Drive + $ minio server /data-{1...4}/minio # Single Node Multi Drive`, + ) + + ErrUnsupportedBackend = newErrFn( + "Unable to write to the backend", + "Please ensure your drive supports O_DIRECT", + "", + ) + + ErrUnableToWriteInBackend = newErrFn( + "Unable to write to the backend", + "Please ensure MinIO binary has write permissions for the backend", + `Verify if MinIO binary is running as the same user who has write permissions for the backend`, + ) + + ErrPortAlreadyInUse = newErrFn( + "Port is already in use", + "Please ensure no other program uses the same address/port", + "", + ) + + ErrPortAccess = newErrFn( + "Unable to use specified port", + "Please ensure MinIO binary has 'cap_net_bind_service=+ep' permissions", + `Use 'sudo setcap cap_net_bind_service=+ep /path/to/minio' to provide sufficient permissions`, + ) + + ErrTLSReadError = newErrFn( + "Cannot read the TLS certificate", + "Please check if the certificate has the proper owner and read permissions", + "", + ) + + ErrTLSUnexpectedData = newErrFn( + "Invalid TLS certificate", + "Please check your certificate", + "", + ) + + ErrTLSNoPassword = newErrFn( + "Missing TLS password", + "Please set the password to environment variable `MINIO_CERT_PASSWD` so that the private key can be decrypted", + "", + ) + + ErrNoCertsAndHTTPSEndpoints = newErrFn( + "HTTPS specified in endpoints, but no TLS certificate is found on the local machine", + "Please add TLS certificate or use HTTP endpoints only", + "Refer to https://docs.min.io/community/minio-object-store/operations/network-encryption.html for information about how to load a TLS certificate in your server", + ) + + ErrCertsAndHTTPEndpoints = newErrFn( + "HTTP specified in endpoints, but the server in the local machine is configured with a TLS certificate", + "Please remove the certificate in the configuration directory or switch to HTTPS", + "", + ) + + ErrTLSWrongPassword = newErrFn( + "Unable to decrypt the private key using the provided password", + "Please set the correct password in environment variable `MINIO_CERT_PASSWD`", + "", + ) + + ErrUnexpectedError = newErrFn( + "Unexpected error", + "Please contact MinIO at https://slack.min.io", + "", + ) + + ErrInvalidCompressionIncludesValue = newErrFn( + "Invalid compression include value", + "Please check the passed value", + "Compress extensions/mime-types are delimited by `,`. For eg, MINIO_COMPRESS_MIME_TYPES=\"A,B,C\"", + ) + + ErrInvalidReplicationWorkersValue = newErrFn( + "Invalid value for replication workers", + "", + "MINIO_API_REPLICATION_WORKERS: should be > 0", + ) + + ErrInvalidTransitionWorkersValue = newErrFn( + "Invalid value for transition workers", + "", + "MINIO_API_TRANSITION_WORKERS: should be >= GOMAXPROCS/2", + ) + ErrInvalidBatchKeyRotationWorkersWait = newErrFn( + "Invalid value for batch key rotation workers wait", + "Please input a non-negative duration", + "keyrotation_workers_wait should be > 0ms", + ) + ErrInvalidBatchReplicationWorkersWait = newErrFn( + "Invalid value for batch replication workers wait", + "Please input a non-negative duration", + "replication_workers_wait should be > 0ms", + ) + ErrInvalidBatchExpirationWorkersWait = newErrFn( + "Invalid value for batch expiration workers wait", + "Please input a non-negative duration", + "expiration_workers_wait should be > 0ms", + ) +) diff --git a/internal/config/etcd/etcd.go b/internal/config/etcd/etcd.go new file mode 100644 index 00000000..87e18012 --- /dev/null +++ b/internal/config/etcd/etcd.go @@ -0,0 +1,183 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package etcd + +import ( + "crypto/tls" + "crypto/x509" + "strings" + "time" + + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/crypto" + "github.com/minio/pkg/v3/env" + xnet "github.com/minio/pkg/v3/net" + clientv3 "go.etcd.io/etcd/client/v3" + "go.etcd.io/etcd/client/v3/namespace" + "go.uber.org/zap" +) + +const ( + // Default values used while communicating with etcd. + defaultDialTimeout = 5 * time.Second + defaultDialKeepAlive = 30 * time.Second +) + +// etcd environment values +const ( + Endpoints = "endpoints" + PathPrefix = "path_prefix" + CoreDNSPath = "coredns_path" + ClientCert = "client_cert" + ClientCertKey = "client_cert_key" + + EnvEtcdEndpoints = "MINIO_ETCD_ENDPOINTS" + EnvEtcdPathPrefix = "MINIO_ETCD_PATH_PREFIX" + EnvEtcdCoreDNSPath = "MINIO_ETCD_COREDNS_PATH" + EnvEtcdClientCert = "MINIO_ETCD_CLIENT_CERT" + EnvEtcdClientCertKey = "MINIO_ETCD_CLIENT_CERT_KEY" +) + +// DefaultKVS - default KV settings for etcd. +var ( + DefaultKVS = config.KVS{ + config.KV{ + Key: Endpoints, + Value: "", + }, + config.KV{ + Key: PathPrefix, + Value: "", + }, + config.KV{ + Key: CoreDNSPath, + Value: "/skydns", + }, + config.KV{ + Key: ClientCert, + Value: "", + }, + config.KV{ + Key: ClientCertKey, + Value: "", + }, + } +) + +// Config - server etcd config. +type Config struct { + Enabled bool `json:"enabled"` + PathPrefix string `json:"pathPrefix"` + CoreDNSPath string `json:"coreDNSPath"` + clientv3.Config +} + +// New - initialize new etcd client. +func New(cfg Config) (*clientv3.Client, error) { + if !cfg.Enabled { + return nil, nil + } + cli, err := clientv3.New(cfg.Config) + if err != nil { + return nil, err + } + cli.KV = namespace.NewKV(cli.KV, cfg.PathPrefix) + cli.Watcher = namespace.NewWatcher(cli.Watcher, cfg.PathPrefix) + cli.Lease = namespace.NewLease(cli.Lease, cfg.PathPrefix) + return cli, nil +} + +func parseEndpoints(endpoints string) ([]string, bool, error) { + etcdEndpoints := strings.Split(endpoints, config.ValueSeparator) + + var etcdSecure bool + for _, endpoint := range etcdEndpoints { + u, err := xnet.ParseHTTPURL(endpoint) + if err != nil { + return nil, false, err + } + if etcdSecure && u.Scheme == "http" { + return nil, false, config.Errorf("all endpoints should be https or http: %s", endpoint) + } + // If one of the endpoint is https, we will use https directly. + etcdSecure = etcdSecure || u.Scheme == "https" + } + + return etcdEndpoints, etcdSecure, nil +} + +// Enabled returns if etcd is enabled. +func Enabled(kvs config.KVS) bool { + endpoints := kvs.Get(Endpoints) + return endpoints != "" +} + +// LookupConfig - Initialize new etcd config. +func LookupConfig(kvs config.KVS, rootCAs *x509.CertPool) (Config, error) { + cfg := Config{} + if err := config.CheckValidKeys(config.EtcdSubSys, kvs, DefaultKVS); err != nil { + return cfg, err + } + + endpoints := env.Get(EnvEtcdEndpoints, kvs.Get(Endpoints)) + if endpoints == "" { + return cfg, nil + } + + etcdEndpoints, etcdSecure, err := parseEndpoints(endpoints) + if err != nil { + return cfg, err + } + + cfg.Enabled = true + cfg.DialTimeout = defaultDialTimeout + cfg.DialKeepAliveTime = defaultDialKeepAlive + // Disable etcd client SDK logging, etcd client + // incorrectly starts logging in unexpected data + // format. + cfg.LogConfig = &zap.Config{ + Level: zap.NewAtomicLevelAt(zap.FatalLevel), + Encoding: "console", + } + cfg.Endpoints = etcdEndpoints + cfg.CoreDNSPath = env.Get(EnvEtcdCoreDNSPath, kvs.Get(CoreDNSPath)) + // Default path prefix for all keys on etcd, other than CoreDNSPath. + cfg.PathPrefix = env.Get(EnvEtcdPathPrefix, kvs.Get(PathPrefix)) + if etcdSecure { + cfg.TLS = &tls.Config{ + RootCAs: rootCAs, + PreferServerCipherSuites: true, + MinVersion: tls.VersionTLS12, + NextProtos: []string{"http/1.1", "h2"}, + ClientSessionCache: tls.NewLRUClientSessionCache(64), + CipherSuites: crypto.TLSCiphersBackwardCompatible(), + CurvePreferences: crypto.TLSCurveIDs(), + } + // This is only to support client side certificate authentication + // https://coreos.com/etcd/docs/latest/op-guide/security.html + etcdClientCertFile := env.Get(EnvEtcdClientCert, kvs.Get(ClientCert)) + etcdClientCertKey := env.Get(EnvEtcdClientCertKey, kvs.Get(ClientCertKey)) + if etcdClientCertFile != "" && etcdClientCertKey != "" { + cfg.TLS.GetClientCertificate = func(unused *tls.CertificateRequestInfo) (*tls.Certificate, error) { + cert, err := tls.LoadX509KeyPair(etcdClientCertFile, etcdClientCertKey) + return &cert, err + } + } + } + return cfg, nil +} diff --git a/internal/config/etcd/etcd_test.go b/internal/config/etcd/etcd_test.go new file mode 100644 index 00000000..e7aad0f7 --- /dev/null +++ b/internal/config/etcd/etcd_test.go @@ -0,0 +1,70 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package etcd + +import ( + "reflect" + "testing" +) + +// TestParseEndpoints - tests parseEndpoints function with valid and invalid inputs. +func TestParseEndpoints(t *testing.T) { + testCases := []struct { + s string + endpoints []string + secure bool + success bool + }{ + // Invalid inputs + {"https://localhost:2379,http://localhost:2380", nil, false, false}, + {",,,", nil, false, false}, + {"", nil, false, false}, + {"ftp://localhost:2379", nil, false, false}, + {"http://localhost:2379000", nil, false, false}, + + // Valid inputs + { + "https://localhost:2379,https://localhost:2380", + []string{ + "https://localhost:2379", "https://localhost:2380", + }, + true, true, + }, + {"http://localhost:2379", []string{"http://localhost:2379"}, false, true}, + } + + for _, testCase := range testCases { + t.Run(testCase.s, func(t *testing.T) { + endpoints, secure, err := parseEndpoints(testCase.s) + if err != nil && testCase.success { + t.Errorf("expected to succeed but failed with %s", err) + } + if !testCase.success && err == nil { + t.Error("expected failure but succeeded instead") + } + if testCase.success { + if !reflect.DeepEqual(endpoints, testCase.endpoints) { + t.Errorf("expected %s, got %s", testCase.endpoints, endpoints) + } + if secure != testCase.secure { + t.Errorf("expected %t, got %t", testCase.secure, secure) + } + } + }) + } +} diff --git a/internal/config/etcd/help.go b/internal/config/etcd/help.go new file mode 100644 index 00000000..df883f3c --- /dev/null +++ b/internal/config/etcd/help.go @@ -0,0 +1,68 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package etcd + +import "github.com/minio/minio/internal/config" + +// etcd config documented in default config +var ( + defaultHelpPostfix = func(key string) string { + return config.DefaultHelpPostfix(DefaultKVS, key) + } + + Help = config.HelpKVS{ + config.HelpKV{ + Key: Endpoints, + Description: `comma separated list of etcd endpoints` + defaultHelpPostfix(Endpoints), + Type: "csv", + Sensitive: true, + }, + config.HelpKV{ + Key: PathPrefix, + Description: `namespace prefix to isolate tenants` + defaultHelpPostfix(PathPrefix), + Optional: true, + Type: "path", + }, + config.HelpKV{ + Key: CoreDNSPath, + Description: `shared bucket DNS records` + defaultHelpPostfix(CoreDNSPath), + Optional: true, + Type: "path", + }, + config.HelpKV{ + Key: ClientCert, + Description: `client cert for mTLS authentication` + defaultHelpPostfix(ClientCert), + Optional: true, + Type: "path", + Sensitive: true, + }, + config.HelpKV{ + Key: ClientCertKey, + Description: `client cert key for mTLS authentication` + defaultHelpPostfix(ClientCertKey), + Optional: true, + Type: "path", + Sensitive: true, + }, + config.HelpKV{ + Key: config.Comment, + Description: config.DefaultComment, + Optional: true, + Type: "sentence", + }, + } +) diff --git a/internal/config/heal/heal.go b/internal/config/heal/heal.go new file mode 100644 index 00000000..edca0eec --- /dev/null +++ b/internal/config/heal/heal.go @@ -0,0 +1,188 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package heal + +import ( + "errors" + "fmt" + "strconv" + "strings" + "sync" + "time" + + "github.com/minio/minio/internal/config" + "github.com/minio/pkg/v3/env" +) + +// Compression environment variables +const ( + Bitrot = "bitrotscan" + Sleep = "max_sleep" + IOCount = "max_io" + DriveWorkers = "drive_workers" + + EnvBitrot = "MINIO_HEAL_BITROTSCAN" + EnvSleep = "MINIO_HEAL_MAX_SLEEP" + EnvIOCount = "MINIO_HEAL_MAX_IO" + EnvDriveWorkers = "MINIO_HEAL_DRIVE_WORKERS" +) + +var configMutex sync.RWMutex + +// Config represents the heal settings. +type Config struct { + // Bitrot will perform bitrot scan on local disk when checking objects. + Bitrot string `json:"bitrotscan"` + + // maximum sleep duration between objects to slow down heal operation. + Sleep time.Duration `json:"sleep"` + IOCount int `json:"iocount"` + + DriveWorkers int `json:"drive_workers"` + + // Cached value from Bitrot field + cache struct { + // -1: bitrot enabled, 0: bitrot disabled, > 0: bitrot cycle + bitrotCycle time.Duration + } +} + +// BitrotScanCycle returns the configured cycle for the scanner healing +// - '-1' for not enabled +// - '0' for continuous bitrot scanning +// - '> 0' interval duration between cycles +func (opts Config) BitrotScanCycle() (d time.Duration) { + configMutex.RLock() + defer configMutex.RUnlock() + return opts.cache.bitrotCycle +} + +// Clone safely the heal configuration +func (opts Config) Clone() (int, time.Duration, string) { + configMutex.RLock() + defer configMutex.RUnlock() + return opts.IOCount, opts.Sleep, opts.Bitrot +} + +// GetWorkers returns the number of workers, -1 is none configured +func (opts Config) GetWorkers() int { + configMutex.RLock() + defer configMutex.RUnlock() + return opts.DriveWorkers +} + +// Update updates opts with nopts +func (opts *Config) Update(nopts Config) { + configMutex.Lock() + defer configMutex.Unlock() + + opts.Bitrot = nopts.Bitrot + opts.IOCount = nopts.IOCount + opts.Sleep = nopts.Sleep + opts.DriveWorkers = nopts.DriveWorkers + + opts.cache.bitrotCycle, _ = parseBitrotConfig(nopts.Bitrot) +} + +// DefaultKVS - default KV config for heal settings +var DefaultKVS = config.KVS{ + config.KV{ + Key: Bitrot, + Value: config.EnableOff, + }, + config.KV{ + Key: Sleep, + Value: "250ms", + }, + config.KV{ + Key: IOCount, + Value: "100", + }, + config.KV{ + Key: DriveWorkers, + Value: "", + }, +} + +const minimumBitrotCycleInMonths = 1 + +func parseBitrotConfig(s string) (time.Duration, error) { + // Try to parse as a boolean + enabled, err := config.ParseBool(s) + if err == nil { + switch enabled { + case true: + return 0, nil + case false: + return -1, nil + } + } + + // Try to parse as a number of months + if !strings.HasSuffix(s, "m") { + return -1, errors.New("unknown format") + } + + months, err := strconv.Atoi(strings.TrimSuffix(s, "m")) + if err != nil { + return -1, err + } + + if months < minimumBitrotCycleInMonths { + return -1, fmt.Errorf("minimum bitrot cycle is %d month(s)", minimumBitrotCycleInMonths) + } + + return time.Duration(months) * 30 * 24 * time.Hour, nil +} + +// LookupConfig - lookup config and override with valid environment settings if any. +func LookupConfig(kvs config.KVS) (cfg Config, err error) { + if err = config.CheckValidKeys(config.HealSubSys, kvs, DefaultKVS); err != nil { + return cfg, err + } + + bitrot := env.Get(EnvBitrot, kvs.GetWithDefault(Bitrot, DefaultKVS)) + if _, err = parseBitrotConfig(bitrot); err != nil { + return cfg, fmt.Errorf("'heal:bitrotscan' value invalid: %w", err) + } + + cfg.Bitrot = bitrot + + cfg.Sleep, err = time.ParseDuration(env.Get(EnvSleep, kvs.GetWithDefault(Sleep, DefaultKVS))) + if err != nil { + return cfg, fmt.Errorf("'heal:max_sleep' value invalid: %w", err) + } + cfg.IOCount, err = strconv.Atoi(env.Get(EnvIOCount, kvs.GetWithDefault(IOCount, DefaultKVS))) + if err != nil { + return cfg, fmt.Errorf("'heal:max_io' value invalid: %w", err) + } + if ws := env.Get(EnvDriveWorkers, kvs.GetWithDefault(DriveWorkers, DefaultKVS)); ws != "" { + w, err := strconv.Atoi(ws) + if err != nil { + return cfg, fmt.Errorf("'heal:drive_workers' value invalid: %w", err) + } + if w < 1 { + return cfg, fmt.Errorf("'heal:drive_workers' value invalid: zero or negative integer unsupported") + } + cfg.DriveWorkers = w + } else { + cfg.DriveWorkers = -1 + } + + return cfg, nil +} diff --git a/internal/config/heal/help.go b/internal/config/heal/help.go new file mode 100644 index 00000000..5c743e31 --- /dev/null +++ b/internal/config/heal/help.go @@ -0,0 +1,55 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package heal + +import "github.com/minio/minio/internal/config" + +// Help template for caching feature. +var ( + defaultHelpPostfix = func(key string) string { + return config.DefaultHelpPostfix(DefaultKVS, key) + } + + // Help provides help for config values + Help = config.HelpKVS{ + config.HelpKV{ + Key: Bitrot, + Description: `perform bitrot scan on drives when checking objects during scanner` + defaultHelpPostfix(Bitrot), + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: Sleep, + Description: `maximum sleep duration between objects to slow down heal operation` + defaultHelpPostfix(Sleep), + Optional: true, + Type: "duration", + }, + config.HelpKV{ + Key: IOCount, + Description: `maximum IO requests allowed between objects to slow down heal operation` + defaultHelpPostfix(IOCount), + Optional: true, + Type: "int", + }, + config.HelpKV{ + Key: DriveWorkers, + Description: `the number of workers per drive to heal a new disk replacement` + defaultHelpPostfix(DriveWorkers), + Optional: true, + Type: "int", + }, + } +) diff --git a/internal/config/help.go b/internal/config/help.go new file mode 100644 index 00000000..9f0d265c --- /dev/null +++ b/internal/config/help.go @@ -0,0 +1,102 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package config + +// HelpKV - implements help messages for keys +// with value as description of the keys. +type HelpKV struct { + Key string `json:"key"` + Type string `json:"type"` + Description string `json:"description"` + Optional bool `json:"optional"` + + // Indicates if the value contains sensitive info that shouldn't be exposed + // in certain apis (such as Health Diagnostics/Callhome) + Sensitive bool `json:"-"` + + // Indicates if the value is a secret such as a password that shouldn't be + // exposed by the server + Secret bool `json:"-"` + + // Indicates if sub-sys supports multiple targets. + MultipleTargets bool `json:"multipleTargets"` +} + +// HelpKVS - implement order of keys help messages. +type HelpKVS []HelpKV + +// Lookup - lookup a key from help kvs. +func (hkvs HelpKVS) Lookup(key string) (HelpKV, bool) { + for _, hkv := range hkvs { + if hkv.Key == key { + return hkv, true + } + } + return HelpKV{}, false +} + +// DefaultComment used across all sub-systems. +const DefaultComment = "optionally add a comment to this setting" + +// Region help is documented in default config +var ( + SiteHelp = HelpKVS{ + HelpKV{ + Key: NameKey, + Type: "string", + Description: `name for the site e.g. "cal-rack0"`, + Optional: true, + }, + HelpKV{ + Key: RegionKey, + Type: "string", + Description: `name of the location of the server e.g. "us-west-1"`, + Optional: true, + }, + HelpKV{ + Key: Comment, + Type: "sentence", + Description: DefaultComment, + Optional: true, + }, + } + + RegionHelp = HelpKVS{ + HelpKV{ + Key: RegionName, + Type: "string", + Description: `[DEPRECATED] name of the location of the server e.g. "us-west-rack2"`, + Optional: true, + }, + HelpKV{ + Key: Comment, + Type: "sentence", + Description: DefaultComment, + Optional: true, + }, + } +) + +// DefaultHelpPostfix - Helper function to add (default: $value) messages in config help +func DefaultHelpPostfix(subsystem KVS, key string) string { + val, found := subsystem.Lookup(key) + if !found || val == "" { + return "" + } + return " (default: '" + val + "')" +} diff --git a/internal/config/identity/ldap/config.go b/internal/config/identity/ldap/config.go new file mode 100644 index 00000000..79bcf86a --- /dev/null +++ b/internal/config/identity/ldap/config.go @@ -0,0 +1,331 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ldap + +import ( + "crypto/tls" + "crypto/x509" + "errors" + "net" + "sort" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/crypto" + "github.com/minio/pkg/v3/ldap" +) + +const ( + defaultLDAPExpiry = time.Hour * 1 + + minLDAPExpiry time.Duration = 15 * time.Minute + maxLDAPExpiry time.Duration = 365 * 24 * time.Hour +) + +// Config contains AD/LDAP server connectivity information. +type Config struct { + LDAP ldap.Config + + stsExpiryDuration time.Duration // contains converted value +} + +// Enabled returns if LDAP is enabled. +func (l *Config) Enabled() bool { + return l.LDAP.Enabled +} + +// Clone returns a cloned copy of LDAP config. +func (l *Config) Clone() Config { + if l == nil { + return Config{} + } + cfg := Config{ + LDAP: l.LDAP.Clone(), + stsExpiryDuration: l.stsExpiryDuration, + } + return cfg +} + +// LDAP keys and envs. +const ( + ServerAddr = "server_addr" + SRVRecordName = "srv_record_name" + LookupBindDN = "lookup_bind_dn" + LookupBindPassword = "lookup_bind_password" + UserDNSearchBaseDN = "user_dn_search_base_dn" + UserDNSearchFilter = "user_dn_search_filter" + UserDNAttributes = "user_dn_attributes" + GroupSearchFilter = "group_search_filter" + GroupSearchBaseDN = "group_search_base_dn" + TLSSkipVerify = "tls_skip_verify" + ServerInsecure = "server_insecure" + ServerStartTLS = "server_starttls" + + EnvServerAddr = "MINIO_IDENTITY_LDAP_SERVER_ADDR" + EnvSRVRecordName = "MINIO_IDENTITY_LDAP_SRV_RECORD_NAME" + EnvTLSSkipVerify = "MINIO_IDENTITY_LDAP_TLS_SKIP_VERIFY" + EnvServerInsecure = "MINIO_IDENTITY_LDAP_SERVER_INSECURE" + EnvServerStartTLS = "MINIO_IDENTITY_LDAP_SERVER_STARTTLS" + EnvUsernameFormat = "MINIO_IDENTITY_LDAP_USERNAME_FORMAT" + EnvUserDNSearchBaseDN = "MINIO_IDENTITY_LDAP_USER_DN_SEARCH_BASE_DN" + EnvUserDNSearchFilter = "MINIO_IDENTITY_LDAP_USER_DN_SEARCH_FILTER" + EnvUserDNAttributes = "MINIO_IDENTITY_LDAP_USER_DN_ATTRIBUTES" + EnvGroupSearchFilter = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_FILTER" + EnvGroupSearchBaseDN = "MINIO_IDENTITY_LDAP_GROUP_SEARCH_BASE_DN" + EnvLookupBindDN = "MINIO_IDENTITY_LDAP_LOOKUP_BIND_DN" + EnvLookupBindPassword = "MINIO_IDENTITY_LDAP_LOOKUP_BIND_PASSWORD" +) + +var removedKeys = []string{ + "sts_expiry", + "username_format", + "username_search_filter", + "username_search_base_dn", + "group_name_attribute", +} + +// DefaultKVS - default config for LDAP config +var ( + DefaultKVS = config.KVS{ + config.KV{ + Key: config.Enable, + Value: "", + }, + config.KV{ + Key: ServerAddr, + Value: "", + }, + config.KV{ + Key: SRVRecordName, + Value: "", + }, + config.KV{ + Key: UserDNSearchBaseDN, + Value: "", + }, + config.KV{ + Key: UserDNSearchFilter, + Value: "", + }, + config.KV{ + Key: UserDNAttributes, + Value: "", + }, + config.KV{ + Key: GroupSearchFilter, + Value: "", + }, + config.KV{ + Key: GroupSearchBaseDN, + Value: "", + }, + config.KV{ + Key: TLSSkipVerify, + Value: config.EnableOff, + }, + config.KV{ + Key: ServerInsecure, + Value: config.EnableOff, + }, + config.KV{ + Key: ServerStartTLS, + Value: config.EnableOff, + }, + config.KV{ + Key: LookupBindDN, + Value: "", + }, + config.KV{ + Key: LookupBindPassword, + Value: "", + }, + } +) + +// Enabled returns if LDAP config is enabled. +func Enabled(kvs config.KVS) bool { + return kvs.Get(ServerAddr) != "" +} + +// Lookup - initializes LDAP config, overrides config, if any ENV values are set. +func Lookup(s config.Config, rootCAs *x509.CertPool) (l Config, err error) { + l = Config{} + + // Purge all removed keys first + kvs := s[config.IdentityLDAPSubSys][config.Default] + if len(kvs) > 0 { + for _, k := range removedKeys { + kvs.Delete(k) + } + s[config.IdentityLDAPSubSys][config.Default] = kvs + } + + if err := s.CheckValidKeys(config.IdentityLDAPSubSys, removedKeys); err != nil { + return l, err + } + + getCfgVal := func(cfgParam string) string { + // As parameters are already validated, we skip checking + // if the config param was found. + val, _, _ := s.ResolveConfigParam(config.IdentityLDAPSubSys, config.Default, cfgParam, false) + return val + } + + ldapServer := getCfgVal(ServerAddr) + if ldapServer == "" { + return l, nil + } + + // Set ServerName in TLS config for proper certificate validation + host, _, err := net.SplitHostPort(ldapServer) + if err != nil { + host = ldapServer + } + + l.LDAP = ldap.Config{ + ServerAddr: ldapServer, + SRVRecordName: getCfgVal(SRVRecordName), + TLS: &tls.Config{ + ServerName: host, + MinVersion: tls.VersionTLS12, + NextProtos: []string{"h2", "http/1.1"}, + ClientSessionCache: tls.NewLRUClientSessionCache(100), + CipherSuites: crypto.TLSCiphersBackwardCompatible(), // Contains RSA key exchange + RootCAs: rootCAs, + }, + } + + // Parse explicitly set enable=on/off flag. + isEnableFlagExplicitlySet := false + if v := getCfgVal(config.Enable); v != "" { + isEnableFlagExplicitlySet = true + l.LDAP.Enabled, err = config.ParseBool(v) + if err != nil { + return l, err + } + } + + l.stsExpiryDuration = defaultLDAPExpiry + + // LDAP connection configuration + if v := getCfgVal(ServerInsecure); v != "" { + l.LDAP.ServerInsecure, err = config.ParseBool(v) + if err != nil { + return l, err + } + } + if v := getCfgVal(ServerStartTLS); v != "" { + l.LDAP.ServerStartTLS, err = config.ParseBool(v) + if err != nil { + return l, err + } + } + if v := getCfgVal(TLSSkipVerify); v != "" { + l.LDAP.TLS.InsecureSkipVerify, err = config.ParseBool(v) + if err != nil { + return l, err + } + } + + // Lookup bind user configuration + l.LDAP.LookupBindDN = getCfgVal(LookupBindDN) + l.LDAP.LookupBindPassword = getCfgVal(LookupBindPassword) + + // User DN search configuration + l.LDAP.UserDNSearchFilter = getCfgVal(UserDNSearchFilter) + l.LDAP.UserDNSearchBaseDistName = getCfgVal(UserDNSearchBaseDN) + l.LDAP.UserDNAttributes = getCfgVal(UserDNAttributes) + + // Group search params configuration + l.LDAP.GroupSearchFilter = getCfgVal(GroupSearchFilter) + l.LDAP.GroupSearchBaseDistName = getCfgVal(GroupSearchBaseDN) + + // If enable flag was not explicitly set, we treat it as implicitly set at + // this point as necessary configuration is available. + if !isEnableFlagExplicitlySet && !l.LDAP.Enabled { + l.LDAP.Enabled = true + } + // Validate and test configuration. + valResult := l.LDAP.Validate() + if !valResult.IsOk() { + // Set to false if configuration fails to validate. + l.LDAP.Enabled = false + return l, valResult + } + + return l, nil +} + +// GetConfigList - returns a list of LDAP configurations. +func (l *Config) GetConfigList(s config.Config) ([]madmin.IDPListItem, error) { + ldapConfigs, err := s.GetAvailableTargets(config.IdentityLDAPSubSys) + if err != nil { + return nil, err + } + + // For now, ldapConfigs will only have a single entry for the default + // configuration. + + var res []madmin.IDPListItem + for _, cfg := range ldapConfigs { + res = append(res, madmin.IDPListItem{ + Type: "ldap", + Name: cfg, + Enabled: l.Enabled(), + }) + } + + return res, nil +} + +// ErrProviderConfigNotFound - represents a non-existing provider error. +var ErrProviderConfigNotFound = errors.New("provider configuration not found") + +// GetConfigInfo - returns config details for an LDAP configuration. +func (l *Config) GetConfigInfo(s config.Config, cfgName string) ([]madmin.IDPCfgInfo, error) { + // For now only a single LDAP config is supported. + if cfgName != madmin.Default { + return nil, ErrProviderConfigNotFound + } + kvsrcs, err := s.GetResolvedConfigParams(config.IdentityLDAPSubSys, cfgName, true) + if err != nil { + return nil, err + } + + res := make([]madmin.IDPCfgInfo, 0, len(kvsrcs)) + for _, kvsrc := range kvsrcs { + // skip default values. + if kvsrc.Src == config.ValueSourceDef { + continue + } + res = append(res, madmin.IDPCfgInfo{ + Key: kvsrc.Key, + Value: kvsrc.Value, + IsCfg: true, + IsEnv: kvsrc.Src == config.ValueSourceEnv, + }) + } + + // sort the structs by the key + sort.Slice(res, func(i, j int) bool { + return res[i].Key < res[j].Key + }) + + return res, nil +} diff --git a/internal/config/identity/ldap/help.go b/internal/config/identity/ldap/help.go new file mode 100644 index 00000000..300039ba --- /dev/null +++ b/internal/config/identity/ldap/help.go @@ -0,0 +1,112 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ldap + +import "github.com/minio/minio/internal/config" + +// Help template for LDAP identity feature. +var ( + defaultHelpPostfix = func(key string) string { + return config.DefaultHelpPostfix(DefaultKVS, key) + } + + Help = config.HelpKVS{ + config.HelpKV{ + Key: ServerAddr, + Description: `AD/LDAP server address e.g. "myldap.com" or "myldapserver.com:636"` + defaultHelpPostfix(ServerAddr), + Type: "address", + Sensitive: true, + }, + config.HelpKV{ + Key: SRVRecordName, + Description: `DNS SRV record name for LDAP service, if given, must be one of "ldap", "ldaps" or "on"` + defaultHelpPostfix(SRVRecordName), + Optional: true, + Type: "string", + Sensitive: false, + }, + config.HelpKV{ + Key: LookupBindDN, + Description: `DN for LDAP read-only service account used to perform DN and group lookups` + defaultHelpPostfix(LookupBindDN), + Optional: true, + Type: "string", + Sensitive: true, + }, + config.HelpKV{ + Key: LookupBindPassword, + Description: `Password for LDAP read-only service account used to perform DN and group lookups` + defaultHelpPostfix(LookupBindPassword), + Optional: true, + Type: "string", + Sensitive: true, + Secret: true, + }, + config.HelpKV{ + Key: UserDNSearchBaseDN, + Description: `";" separated list of user search base DNs e.g. "dc=myldapserver,dc=com"` + defaultHelpPostfix(UserDNSearchBaseDN), + Optional: true, + Type: "list", + }, + config.HelpKV{ + Key: UserDNSearchFilter, + Description: `Search filter to lookup user DN` + defaultHelpPostfix(UserDNSearchFilter), + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: UserDNAttributes, + Description: `"," separated list of user DN attributes e.g. "uid,cn,mail,sshPublicKey"` + defaultHelpPostfix(UserDNAttributes), + Optional: true, + Type: "list", + }, + config.HelpKV{ + Key: GroupSearchFilter, + Description: `search filter for groups e.g. "(&(objectclass=groupOfNames)(memberUid=%s))"` + defaultHelpPostfix(GroupSearchFilter), + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: GroupSearchBaseDN, + Description: `";" separated list of group search base DNs e.g. "dc=myldapserver,dc=com"` + defaultHelpPostfix(GroupSearchBaseDN), + Optional: true, + Type: "list", + }, + config.HelpKV{ + Key: TLSSkipVerify, + Description: `trust server TLS without verification` + defaultHelpPostfix(TLSSkipVerify), + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: ServerInsecure, + Description: `allow plain text connection to AD/LDAP server` + defaultHelpPostfix(ServerInsecure), + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: ServerStartTLS, + Description: `use StartTLS connection to AD/LDAP server` + defaultHelpPostfix(ServerStartTLS), + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: config.Comment, + Description: config.DefaultComment, + Optional: true, + Type: "sentence", + }, + } +) diff --git a/internal/config/identity/ldap/ldap.go b/internal/config/identity/ldap/ldap.go new file mode 100644 index 00000000..1c1c704c --- /dev/null +++ b/internal/config/identity/ldap/ldap.go @@ -0,0 +1,422 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ldap + +import ( + "errors" + "fmt" + "strconv" + "strings" + "time" + + ldap "github.com/go-ldap/ldap/v3" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/auth" + xldap "github.com/minio/pkg/v3/ldap" +) + +// LookupUserDN searches for the full DN and groups of a given short/login +// username. +func (l *Config) LookupUserDN(username string) (*xldap.DNSearchResult, []string, error) { + conn, err := l.LDAP.Connect() + if err != nil { + return nil, nil, err + } + defer conn.Close() + + // Bind to the lookup user account + if err = l.LDAP.LookupBind(conn); err != nil { + return nil, nil, err + } + + // Lookup user DN + lookupRes, err := l.LDAP.LookupUsername(conn, username) + if err != nil { + errRet := fmt.Errorf("Unable to find user DN: %w", err) + return nil, nil, errRet + } + + groups, err := l.LDAP.SearchForUserGroups(conn, username, lookupRes.ActualDN) + if err != nil { + return nil, nil, err + } + + return lookupRes, groups, nil +} + +// GetValidatedDNForUsername checks if the given username exists in the LDAP directory. +// The given username could be just the short "login" username or the full DN. +// +// When the username/DN is found, the full DN returned by the **server** is +// returned, otherwise the returned string is empty. The value returned here is +// the value sent by the LDAP server and is used in minio as the server performs +// LDAP specific normalization (including Unicode normalization). +// +// If the user is not found, err = nil, otherwise, err != nil. +func (l *Config) GetValidatedDNForUsername(username string) (*xldap.DNSearchResult, error) { + conn, err := l.LDAP.Connect() + if err != nil { + return nil, err + } + defer conn.Close() + + // Bind to the lookup user account + if err = l.LDAP.LookupBind(conn); err != nil { + return nil, err + } + + // Check if the passed in username is a valid DN. + if !l.ParsesAsDN(username) { + // We consider it as a login username and attempt to check it exists in + // the directory. + bindDN, err := l.LDAP.LookupUsername(conn, username) + if err != nil { + if strings.Contains(err.Error(), "User DN not found for") { + return nil, nil + } + return nil, fmt.Errorf("Unable to find user DN: %w", err) + } + return bindDN, nil + } + + // Since the username parses as a valid DN, check that it exists and is + // under a configured base DN in the LDAP directory. + validDN, isUnderBaseDN, err := l.GetValidatedUserDN(conn, username) + if err == nil && !isUnderBaseDN { + // Not under any configured base DN, so treat as not found. + return nil, nil + } + return validDN, err +} + +// GetValidatedUserDN validates the given user DN. Will error out if conn is nil. The returned +// boolean is true iff the user DN is found under one of the LDAP user base DNs. +func (l *Config) GetValidatedUserDN(conn *ldap.Conn, userDN string) (*xldap.DNSearchResult, bool, error) { + return l.GetValidatedDNUnderBaseDN(conn, userDN, + l.LDAP.GetUserDNSearchBaseDistNames(), l.LDAP.GetUserDNAttributesList()) +} + +// GetValidatedGroupDN validates the given group DN. If conn is nil, creates a +// connection. The returned boolean is true iff the group DN is found under one +// of the configured LDAP base DNs. +func (l *Config) GetValidatedGroupDN(conn *ldap.Conn, groupDN string) (*xldap.DNSearchResult, bool, error) { + if conn == nil { + var err error + conn, err = l.LDAP.Connect() + if err != nil { + return nil, false, err + } + defer conn.Close() + + // Bind to the lookup user account + if err = l.LDAP.LookupBind(conn); err != nil { + return nil, false, err + } + } + + return l.GetValidatedDNUnderBaseDN(conn, groupDN, + l.LDAP.GetGroupSearchBaseDistNames(), nil) +} + +// GetValidatedDNUnderBaseDN checks if the given DN exists in the LDAP +// directory. +// +// The `NormDN` value returned here in the search result may not be equal to the +// input DN, as LDAP equality is not a simple Golang string equality. However, +// we assume the value returned by the LDAP server is canonical. Additionally, +// the attribute type names in the DN are lower-cased. +// +// Return values: +// +// If the DN is found, the normalized (string) value and any requested +// attributes are returned and error is nil. +// +// If the DN is not found, a nil result and error are returned. +// +// The returned boolean is true iff the DN is found under one of the LDAP +// subtrees listed in `baseDNList`. +func (l *Config) GetValidatedDNUnderBaseDN(conn *ldap.Conn, dn string, baseDNList []xldap.BaseDNInfo, attrs []string) (*xldap.DNSearchResult, bool, error) { + if len(baseDNList) == 0 { + return nil, false, errors.New("no Base DNs given") + } + + // Check that DN exists in the LDAP directory. + searchRes, err := xldap.LookupDN(conn, dn, attrs) + if err != nil { + return nil, false, fmt.Errorf("Error looking up DN %s: %w", dn, err) + } + if searchRes == nil { + return nil, false, nil + } + + // This will not return an error as the argument is validated to be a DN. + pdn, _ := ldap.ParseDN(searchRes.NormDN) + + // Check that the DN is under a configured base DN in the LDAP + // directory. + for _, baseDN := range baseDNList { + if baseDN.Parsed.AncestorOf(pdn) { + return searchRes, true, nil + } + } + + // Not under any configured base DN so return false. + return searchRes, false, nil +} + +// GetValidatedDNWithGroups - Gets validated DN from given DN or short username +// and returns the DN and the groups the user is a member of. +// +// If username is required in group search but a DN is passed, no groups are +// returned. +func (l *Config) GetValidatedDNWithGroups(username string) (*xldap.DNSearchResult, []string, error) { + conn, err := l.LDAP.Connect() + if err != nil { + return nil, nil, err + } + defer conn.Close() + + // Bind to the lookup user account + if err = l.LDAP.LookupBind(conn); err != nil { + return nil, nil, err + } + + var lookupRes *xldap.DNSearchResult + shortUsername := "" + // Check if the passed in username is a valid DN. + if !l.ParsesAsDN(username) { + // We consider it as a login username and attempt to check it exists in + // the directory. + lookupRes, err = l.LDAP.LookupUsername(conn, username) + if err != nil { + if strings.Contains(err.Error(), "User DN not found for") { + return nil, nil, nil + } + return nil, nil, fmt.Errorf("Unable to find user DN: %w", err) + } + shortUsername = username + } else { + // Since the username parses as a valid DN, check that it exists and is + // under a configured base DN in the LDAP directory. + var isUnderBaseDN bool + lookupRes, isUnderBaseDN, err = l.GetValidatedUserDN(conn, username) + if err == nil && !isUnderBaseDN { + return nil, nil, fmt.Errorf("Unable to find user DN: %w", err) + } + } + + groups, err := l.LDAP.SearchForUserGroups(conn, shortUsername, lookupRes.ActualDN) + if err != nil { + return nil, nil, err + } + return lookupRes, groups, nil +} + +// Bind - binds to ldap, searches LDAP and returns the distinguished name of the +// user and the list of groups. +func (l *Config) Bind(username, password string) (*xldap.DNSearchResult, []string, error) { + conn, err := l.LDAP.Connect() + if err != nil { + return nil, nil, err + } + defer conn.Close() + + // Bind to the lookup user account + if err = l.LDAP.LookupBind(conn); err != nil { + return nil, nil, err + } + + // Lookup user DN + lookupResult, err := l.LDAP.LookupUsername(conn, username) + if err != nil { + errRet := fmt.Errorf("Unable to find user DN: %w", err) + return nil, nil, errRet + } + + // Authenticate the user credentials. + err = conn.Bind(lookupResult.ActualDN, password) + if err != nil { + errRet := fmt.Errorf("LDAP auth failed for DN %s: %w", lookupResult.ActualDN, err) + return nil, nil, errRet + } + + // Bind to the lookup user account again to perform group search. + if err = l.LDAP.LookupBind(conn); err != nil { + return nil, nil, err + } + + // User groups lookup. + groups, err := l.LDAP.SearchForUserGroups(conn, username, lookupResult.ActualDN) + if err != nil { + return nil, nil, err + } + + return lookupResult, groups, nil +} + +// GetExpiryDuration - return parsed expiry duration. +func (l Config) GetExpiryDuration(dsecs string) (time.Duration, error) { + if dsecs == "" { + return l.stsExpiryDuration, nil + } + + d, err := strconv.Atoi(dsecs) + if err != nil { + return 0, auth.ErrInvalidDuration + } + + dur := time.Duration(d) * time.Second + + if dur < minLDAPExpiry || dur > maxLDAPExpiry { + return 0, auth.ErrInvalidDuration + } + return dur, nil +} + +// ParsesAsDN determines if the given string could be a valid DN based on +// parsing alone. +func (l Config) ParsesAsDN(dn string) bool { + _, err := ldap.ParseDN(dn) + return err == nil +} + +// IsLDAPUserDN determines if the given string could be a user DN from LDAP. +func (l Config) IsLDAPUserDN(user string) bool { + udn, err := ldap.ParseDN(user) + if err != nil { + return false + } + for _, baseDN := range l.LDAP.GetUserDNSearchBaseDistNames() { + if baseDN.Parsed.AncestorOf(udn) { + return true + } + } + return false +} + +// IsLDAPGroupDN determines if the given string could be a group DN from LDAP. +func (l Config) IsLDAPGroupDN(group string) bool { + gdn, err := ldap.ParseDN(group) + if err != nil { + return false + } + for _, baseDN := range l.LDAP.GetGroupSearchBaseDistNames() { + if baseDN.Parsed.AncestorOf(gdn) { + return true + } + } + return false +} + +// GetNonEligibleUserDistNames - find user accounts (DNs) that are no longer +// present in the LDAP server or do not meet filter criteria anymore +func (l *Config) GetNonEligibleUserDistNames(userDistNames []string) ([]string, error) { + conn, err := l.LDAP.Connect() + if err != nil { + return nil, err + } + defer conn.Close() + + // Bind to the lookup user account + if err = l.LDAP.LookupBind(conn); err != nil { + return nil, err + } + + // Evaluate the filter again with generic wildcard instead of specific values + filter := strings.ReplaceAll(l.LDAP.UserDNSearchFilter, "%s", "*") + + nonExistentUsers := []string{} + for _, dn := range userDistNames { + searchRequest := ldap.NewSearchRequest( + dn, + ldap.ScopeBaseObject, ldap.NeverDerefAliases, 0, 0, false, + filter, + []string{}, // only need DN, so pass no attributes here + nil, + ) + + searchResult, err := conn.Search(searchRequest) + if err != nil { + // Object does not exist error? + if ldap.IsErrorWithCode(err, 32) { + ndn, err := ldap.ParseDN(dn) + if err != nil { + return nil, err + } + nonExistentUsers = append(nonExistentUsers, ndn.String()) + continue + } + return nil, err + } + if len(searchResult.Entries) == 0 { + // DN was not found - this means this user account is + // expired. + ndn, err := ldap.ParseDN(dn) + if err != nil { + return nil, err + } + nonExistentUsers = append(nonExistentUsers, ndn.String()) + } + } + return nonExistentUsers, nil +} + +// LookupGroupMemberships - for each DN finds the set of LDAP groups they are a +// member of. +func (l *Config) LookupGroupMemberships(userDistNames []string, userDNToUsernameMap map[string]string) (map[string]set.StringSet, error) { + conn, err := l.LDAP.Connect() + if err != nil { + return nil, err + } + defer conn.Close() + + // Bind to the lookup user account + if err = l.LDAP.LookupBind(conn); err != nil { + return nil, err + } + + res := make(map[string]set.StringSet, len(userDistNames)) + for _, userDistName := range userDistNames { + username := userDNToUsernameMap[userDistName] + groups, err := l.LDAP.SearchForUserGroups(conn, username, userDistName) + if err != nil { + return nil, err + } + res[userDistName] = set.CreateStringSet(groups...) + } + + return res, nil +} + +// QuickNormalizeDN - normalizes the given DN without checking if it is valid or +// exists in the LDAP directory. Returns input if error +func (l Config) QuickNormalizeDN(dn string) string { + if normDN, err := xldap.NormalizeDN(dn); err == nil { + return normDN + } + return dn +} + +// DecodeDN - denormalizes the given DN by unescaping any escaped characters. +// Returns input if error +func (l Config) DecodeDN(dn string) string { + if decodedDN, err := xldap.DecodeDN(dn); err == nil { + return decodedDN + } + return dn +} diff --git a/internal/config/identity/ldap/legacy.go b/internal/config/identity/ldap/legacy.go new file mode 100644 index 00000000..13fe4f9c --- /dev/null +++ b/internal/config/identity/ldap/legacy.go @@ -0,0 +1,67 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ldap + +import ( + "github.com/minio/minio/internal/config" +) + +// LegacyConfig contains AD/LDAP server connectivity information from old config +// V33. +type LegacyConfig struct { + Enabled bool `json:"enabled"` + + // E.g. "ldap.minio.io:636" + ServerAddr string `json:"serverAddr"` + + // User DN search parameters + UserDNSearchBaseDistName string `json:"userDNSearchBaseDN"` + UserDNSearchBaseDistNames []string `json:"-"` // Generated field + UserDNSearchFilter string `json:"userDNSearchFilter"` + + // Group search parameters + GroupSearchBaseDistName string `json:"groupSearchBaseDN"` + GroupSearchBaseDistNames []string `json:"-"` // Generated field + GroupSearchFilter string `json:"groupSearchFilter"` + + // Lookup bind LDAP service account + LookupBindDN string `json:"lookupBindDN"` + LookupBindPassword string `json:"lookupBindPassword"` +} + +// SetIdentityLDAP - One time migration code needed, for migrating from older config to new for LDAPConfig. +func SetIdentityLDAP(s config.Config, ldapArgs LegacyConfig) { + if !ldapArgs.Enabled { + // ldap not enabled no need to preserve it in new settings. + return + } + s[config.IdentityLDAPSubSys][config.Default] = config.KVS{ + config.KV{ + Key: ServerAddr, + Value: ldapArgs.ServerAddr, + }, + config.KV{ + Key: GroupSearchFilter, + Value: ldapArgs.GroupSearchFilter, + }, + config.KV{ + Key: GroupSearchBaseDN, + Value: ldapArgs.GroupSearchBaseDistName, + }, + } +} diff --git a/internal/config/identity/openid/ecdsa-sha3_contrib.go b/internal/config/identity/openid/ecdsa-sha3_contrib.go new file mode 100644 index 00000000..11d7acb7 --- /dev/null +++ b/internal/config/identity/openid/ecdsa-sha3_contrib.go @@ -0,0 +1,50 @@ +// MinIO Object Storage (c) 2021 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openid + +import ( + "crypto" + + "github.com/golang-jwt/jwt/v4" + + // Needed for SHA3 to work - See: https://golang.org/src/crypto/crypto.go?s=1034:1288 + _ "golang.org/x/crypto/sha3" +) + +// Specific instances for EC256 and company +var ( + SigningMethodES3256 *jwt.SigningMethodECDSA + SigningMethodES3384 *jwt.SigningMethodECDSA + SigningMethodES3512 *jwt.SigningMethodECDSA +) + +func init() { + // ES256 + SigningMethodES3256 = &jwt.SigningMethodECDSA{Name: "ES3256", Hash: crypto.SHA3_256, KeySize: 32, CurveBits: 256} + jwt.RegisterSigningMethod(SigningMethodES3256.Alg(), func() jwt.SigningMethod { + return SigningMethodES3256 + }) + + // ES384 + SigningMethodES3384 = &jwt.SigningMethodECDSA{Name: "ES3384", Hash: crypto.SHA3_384, KeySize: 48, CurveBits: 384} + jwt.RegisterSigningMethod(SigningMethodES3384.Alg(), func() jwt.SigningMethod { + return SigningMethodES3384 + }) + + // ES512 + SigningMethodES3512 = &jwt.SigningMethodECDSA{Name: "ES3512", Hash: crypto.SHA3_512, KeySize: 66, CurveBits: 521} + jwt.RegisterSigningMethod(SigningMethodES3512.Alg(), func() jwt.SigningMethod { + return SigningMethodES3512 + }) +} diff --git a/internal/config/identity/openid/help.go b/internal/config/identity/openid/help.go new file mode 100644 index 00000000..1469034c --- /dev/null +++ b/internal/config/identity/openid/help.go @@ -0,0 +1,119 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package openid + +import "github.com/minio/minio/internal/config" + +// Help template for OpenID identity feature. +var ( + defaultHelpPostfix = func(key string) string { + return config.DefaultHelpPostfix(DefaultKVS, key) + } + + Help = config.HelpKVS{ + config.HelpKV{ + Key: DisplayName, + Description: "Friendly display name for this Provider/App" + defaultHelpPostfix(DisplayName), + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: ConfigURL, + Description: `openid discovery document e.g. "https://accounts.google.com/.well-known/openid-configuration"` + defaultHelpPostfix(ConfigURL), + Type: "url", + }, + config.HelpKV{ + Key: ClientID, + Description: `unique public identifier for apps e.g. "292085223830.apps.googleusercontent.com"` + defaultHelpPostfix(ClientID), + Type: "string", + }, + config.HelpKV{ + Key: ClientSecret, + Description: `secret for the unique public identifier for apps` + defaultHelpPostfix(ClientSecret), + Sensitive: true, + Type: "string", + Secret: true, + }, + config.HelpKV{ + Key: RolePolicy, + Description: `Set the IAM access policies applicable to this client application and IDP e.g. "app-bucket-write,app-bucket-list"` + defaultHelpPostfix(RolePolicy), + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: ClaimName, + Description: `JWT canned policy claim name` + defaultHelpPostfix(ClaimName), + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: Scopes, + Description: `Comma separated list of OpenID scopes for server, defaults to advertised scopes from discovery document e.g. "email,admin"` + defaultHelpPostfix(Scopes), + Optional: true, + Type: "csv", + }, + config.HelpKV{ + Key: Vendor, + Description: `Specify vendor type for vendor specific behavior to checking validity of temporary credentials and service accounts on MinIO` + defaultHelpPostfix(Vendor), + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: ClaimUserinfo, + Description: `Enable fetching claims from UserInfo Endpoint for authenticated user` + defaultHelpPostfix(ClaimUserinfo), + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: KeyCloakRealm, + Description: `Specify Keycloak 'realm' name, only honored if vendor was set to 'keycloak' as value, if no realm is specified 'master' is default` + defaultHelpPostfix(KeyCloakRealm), + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: KeyCloakAdminURL, + Description: `Specify Keycloak 'admin' REST API endpoint e.g. http://localhost:8080/auth/admin/` + defaultHelpPostfix(KeyCloakAdminURL), + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: RedirectURIDynamic, + Description: `Enable 'Host' header based dynamic redirect URI` + defaultHelpPostfix(RedirectURIDynamic), + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: ClaimPrefix, + Description: `[DEPRECATED use 'claim_name'] JWT claim namespace prefix e.g. "customer1/"` + defaultHelpPostfix(ClaimPrefix), + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: RedirectURI, + Description: `[DEPRECATED use env 'MINIO_BROWSER_REDIRECT_URL'] Configure custom redirect_uri for OpenID login flow callback` + defaultHelpPostfix(RedirectURI), + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: config.Comment, + Description: config.DefaultComment, + Optional: true, + Type: "sentence", + }, + } +) diff --git a/internal/config/identity/openid/jwks.go b/internal/config/identity/openid/jwks.go new file mode 100644 index 00000000..e1c00532 --- /dev/null +++ b/internal/config/identity/openid/jwks.go @@ -0,0 +1,130 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package openid + +import ( + "crypto" + "crypto/ecdsa" + "crypto/ed25519" + "crypto/elliptic" + "crypto/rsa" + "encoding/base64" + "errors" + "fmt" + "math/big" +) + +// JWKS - https://tools.ietf.org/html/rfc7517 +type JWKS struct { + Keys []*JWKS `json:"keys,omitempty"` + + Kty string `json:"kty"` + Use string `json:"use,omitempty"` + Kid string `json:"kid,omitempty"` + Alg string `json:"alg,omitempty"` + + Crv string `json:"crv,omitempty"` + X string `json:"x,omitempty"` + Y string `json:"y,omitempty"` + D string `json:"d,omitempty"` + N string `json:"n,omitempty"` + E string `json:"e,omitempty"` + K string `json:"k,omitempty"` +} + +var ( + errMalformedJWKRSAKey = errors.New("malformed JWK RSA key") + errMalformedJWKECKey = errors.New("malformed JWK EC key") +) + +// DecodePublicKey - decodes JSON Web Key (JWK) as public key +func (key *JWKS) DecodePublicKey() (crypto.PublicKey, error) { + switch key.Kty { + case "RSA": + if key.N == "" || key.E == "" { + return nil, errMalformedJWKRSAKey + } + + // decode exponent + ebuf, err := base64.RawURLEncoding.DecodeString(key.E) + if err != nil { + return nil, errMalformedJWKRSAKey + } + + nbuf, err := base64.RawURLEncoding.DecodeString(key.N) + if err != nil { + return nil, errMalformedJWKRSAKey + } + + var n, e big.Int + n.SetBytes(nbuf) + e.SetBytes(ebuf) + + return &rsa.PublicKey{ + E: int(e.Int64()), + N: &n, + }, nil + case "EC": + if key.Crv == "" || key.X == "" || key.Y == "" { + return nil, errMalformedJWKECKey + } + + var curve elliptic.Curve + switch key.Crv { + case "P-224": + curve = elliptic.P224() + case "P-256": + curve = elliptic.P256() + case "P-384": + curve = elliptic.P384() + case "P-521": + curve = elliptic.P521() + default: + return nil, fmt.Errorf("Unknown curve type: %s", key.Crv) + } + + xbuf, err := base64.RawURLEncoding.DecodeString(key.X) + if err != nil { + return nil, errMalformedJWKECKey + } + + ybuf, err := base64.RawURLEncoding.DecodeString(key.Y) + if err != nil { + return nil, errMalformedJWKECKey + } + + var x, y big.Int + x.SetBytes(xbuf) + y.SetBytes(ybuf) + + return &ecdsa.PublicKey{ + Curve: curve, + X: &x, + Y: &y, + }, nil + default: + if key.Alg == "EdDSA" && key.Crv == "Ed25519" && key.X != "" { + pb, err := base64.RawURLEncoding.DecodeString(key.X) + if err != nil { + return nil, errMalformedJWKECKey + } + return ed25519.PublicKey(pb), nil + } + return nil, fmt.Errorf("Unknown JWK key type %s", key.Kty) + } +} diff --git a/internal/config/identity/openid/jwks_test.go b/internal/config/identity/openid/jwks_test.go new file mode 100644 index 00000000..e85d7fa3 --- /dev/null +++ b/internal/config/identity/openid/jwks_test.go @@ -0,0 +1,134 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package openid + +import ( + "bytes" + "crypto" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rsa" + "encoding/json" + "testing" +) + +func TestAzurePublicKey(t *testing.T) { + const jsonkey = `{"keys":[{"kty":"RSA","use":"sig","kid":"SsZsBNhZcF3Q9S4trpQBTByNRRI","x5t":"SsZsBNhZcF3Q9S4trpQBTByNRRI","n":"uHPewhg4WC3eLVPkEFlj7RDtaKYWXCI5G-LPVzsMKOuIu7qQQbeytIA6P6HT9_iIRt8zNQvuw4P9vbNjgUCpI6vfZGsjk3XuCVoB_bAIhvuBcQh9ePH2yEwS5reR-NrG1PsqzobnZZuigKCoDmuOb_UDx1DiVyNCbMBlEG7UzTQwLf5NP6HaRHx027URJeZvPAWY7zjHlSOuKoS_d1yUveaBFIgZqPWLCg44ck4gvik45HsNVWT9zYfT74dvUSSrMSR-SHFT7Hy1XjbVXpHJHNNAXpPoGoWXTuc0BxMsB4cqjfJqoftFGOG4x32vEzakArLPxAKwGvkvu0jToAyvSQ","e":"AQAB","x5c":"MIIDBTCCAe2gAwIBAgIQWHw7h/Ysh6hPcXpnrJ0N8DANBgkqhkiG9w0BAQsFADAtMSswKQYDVQQDEyJhY2NvdW50cy5hY2Nlc3Njb250cm9sLndpbmRvd3MubmV0MB4XDTIwMDQyNzAwMDAwMFoXDTI1MDQyNzAwMDAwMFowLTErMCkGA1UEAxMiYWNjb3VudHMuYWNjZXNzY29udHJvbC53aW5kb3dzLm5ldDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBALhz3sIYOFgt3i1T5BBZY+0Q7WimFlwiORviz1c7DCjriLu6kEG3srSAOj+h0/f4iEbfMzUL7sOD/b2zY4FAqSOr32RrI5N17glaAf2wCIb7gXEIfXjx9shMEua3kfjaxtT7Ks6G52WbooCgqA5rjm/1A8dQ4lcjQmzAZRBu1M00MC3+TT+h2kR8dNu1ESXmbzwFmO84x5UjriqEv3dclL3mgRSIGaj1iwoOOHJOIL4pOOR7DVVk/c2H0++Hb1EkqzEkfkhxU+x8tV421V6RyRzTQF6T6BqFl07nNAcTLAeHKo3yaqH7RRjhuMd9rxM2pAKyz8QCsBr5L7tI06AMr0kCAwEAAaMhMB8wHQYDVR0OBBYEFOI7M+DDFMlP7Ac3aomPnWo1QL1SMA0GCSqGSIb3DQEBCwUAA4IBAQBv+8rBiDY8sZDBoUDYwFQM74QjqCmgNQfv5B0Vjwg20HinERjQeH24uAWzyhWN9++FmeY4zcRXDY5UNmB0nJz7UGlprA9s7voQ0Lkyiud0DO072RPBg38LmmrqoBsLb3MB9MZ2CGBaHftUHfpdTvrgmXSP0IJn7mCUq27g+hFk7n/MLbN1k8JswEODIgdMRvGqN+mnrPKkviWmcVAZccsWfcmS1pKwXqICTKzd6WmVdz+cL7ZSd9I2X0pY4oRwauoE2bS95vrXljCYgLArI3XB2QcnglDDBRYu3Z3aIJb26PTIyhkVKT7xaXhXl4OgrbmQon9/O61G2dzpjzzBPqNP","issuer":"https://login.microsoftonline.com/906aefe9-76a7-4f65-b82d-5ec20775d5aa/v2.0"},{"kty":"RSA","use":"sig","kid":"huN95IvPfehq34GzBDZ1GXGirnM","x5t":"huN95IvPfehq34GzBDZ1GXGirnM","n":"6lldKm5Rc_vMKa1RM_TtUv3tmtj52wLRrJqu13yGM3_h0dwru2ZP53y65wDfz6_tLCjoYuRCuVsjoW37-0zXUORJvZ0L90CAX-58lW7NcE4bAzA1pXv7oR9kQw0X8dp0atU4HnHeaTU8LZxcjJO79_H9cxgwa-clKfGxllcos8TsuurM8xi2dx5VqwzqNMB2s62l3MTN7AzctHUiQCiX2iJArGjAhs-mxS1wmyMIyOSipdodhjQWRAcseW-aFVyRTFVi8okl2cT1HJjPXdx0b1WqYSOzeRdrrLUcA0oR2Tzp7xzOYJZSGNnNLQqa9f6h6h52XbX0iAgxKgEDlRpbJw","e":"AQAB","x5c":["MIIDBTCCAe2gAwIBAgIQPCxFbySVSLZOggeWRzBWOjANBgkqhkiG9w0BAQsFADAtMSswKQYDVQQDEyJhY2NvdW50cy5hY2Nlc3Njb250cm9sLndpbmRvd3MubmV0MB4XDTIwMDYwNzAwMDAwMFoXDTI1MDYwNzAwMDAwMFowLTErMCkGA1UEAxMiYWNjb3VudHMuYWNjZXNzY29udHJvbC53aW5kb3dzLm5ldDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOpZXSpuUXP7zCmtUTP07VL97ZrY+dsC0ayartd8hjN/4dHcK7tmT+d8uucA38+v7Swo6GLkQrlbI6Ft+/tM11DkSb2dC/dAgF/ufJVuzXBOGwMwNaV7+6EfZEMNF/HadGrVOB5x3mk1PC2cXIyTu/fx/XMYMGvnJSnxsZZXKLPE7LrqzPMYtnceVasM6jTAdrOtpdzEzewM3LR1IkAol9oiQKxowIbPpsUtcJsjCMjkoqXaHYY0FkQHLHlvmhVckUxVYvKJJdnE9RyYz13cdG9VqmEjs3kXa6y1HANKEdk86e8czmCWUhjZzS0KmvX+oeoedl219IgIMSoBA5UaWycCAwEAAaMhMB8wHQYDVR0OBBYEFFXP0ODFhjf3RS6oRijM5Tb+yB8CMA0GCSqGSIb3DQEBCwUAA4IBAQB9GtVikLTbJWIu5x9YCUTTKzNhi44XXogP/v8VylRSUHI5YTMdnWwvDIt/Y1sjNonmSy9PrioEjcIiI1U8nicveafMwIq5VLn+gEY2lg6KDJAzgAvA88CXqwfHHvtmYBovN7goolp8TY/kddMTf6TpNzN3lCTM2MK4Ye5xLLVGdp4bqWCOJ/qjwDxpTRSydYIkLUDwqNjv+sYfOElJpYAB4rTL/aw3ChJ1iaA4MtXEt6OjbUtbOa21lShfLzvNRbYK3+ukbrhmRl9lemJEeUls51vPuIe+jg+Ssp43aw7PQjxt4/MpfNMS2BfZ5F8GVSVG7qNb352cLLeJg5rc398Z"],"issuer":"https://login.microsoftonline.com/906aefe9-76a7-4f65-b82d-5ec20775d5aa/v2.0"},{"kty":"RSA","use":"sig","kid":"M6pX7RHoraLsprfJeRCjSxuURhc","x5t":"M6pX7RHoraLsprfJeRCjSxuURhc","n":"xHScZMPo8FifoDcrgncWQ7mGJtiKhrsho0-uFPXg-OdnRKYudTD7-Bq1MDjcqWRf3IfDVjFJixQS61M7wm9wALDj--lLuJJ9jDUAWTA3xWvQLbiBM-gqU0sj4mc2lWm6nPfqlyYeWtQcSC0sYkLlayNgX4noKDaXivhVOp7bwGXq77MRzeL4-9qrRYKjuzHfZL7kNBCsqO185P0NI2Jtmw-EsqYsrCaHsfNRGRrTvUHUq3hWa859kK_5uNd7TeY2ZEwKVD8ezCmSfR59ZzyxTtuPpkCSHS9OtUvS3mqTYit73qcvprjl3R8hpjXLb8oftfpWr3hFRdpxrwuoQEO4QQ","e":"AQAB","x5c":["MIIC8TCCAdmgAwIBAgIQfEWlTVc1uINEc9RBi6qHMjANBgkqhkiG9w0BAQsFADAjMSEwHwYDVQQDExhsb2dpbi5taWNyb3NvZnRvbmxpbmUudXMwHhcNMTgxMDE0MDAwMDAwWhcNMjAxMDE0MDAwMDAwWjAjMSEwHwYDVQQDExhsb2dpbi5taWNyb3NvZnRvbmxpbmUudXMwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQDEdJxkw+jwWJ+gNyuCdxZDuYYm2IqGuyGjT64U9eD452dEpi51MPv4GrUwONypZF/ch8NWMUmLFBLrUzvCb3AAsOP76Uu4kn2MNQBZMDfFa9AtuIEz6CpTSyPiZzaVabqc9+qXJh5a1BxILSxiQuVrI2BfiegoNpeK+FU6ntvAZervsxHN4vj72qtFgqO7Md9kvuQ0EKyo7Xzk/Q0jYm2bD4SypiysJoex81EZGtO9QdSreFZrzn2Qr/m413tN5jZkTApUPx7MKZJ9Hn1nPLFO24+mQJIdL061S9LeapNiK3vepy+muOXdHyGmNctvyh+1+laveEVF2nGvC6hAQ7hBAgMBAAGjITAfMB0GA1UdDgQWBBQ5TKadw06O0cvXrQbXW0Nb3M3h/DANBgkqhkiG9w0BAQsFAAOCAQEAI48JaFtwOFcYS/3pfS5+7cINrafXAKTL+/+he4q+RMx4TCu/L1dl9zS5W1BeJNO2GUznfI+b5KndrxdlB6qJIDf6TRHh6EqfA18oJP5NOiKhU4pgkF2UMUw4kjxaZ5fQrSoD9omjfHAFNjradnHA7GOAoF4iotvXDWDBWx9K4XNZHWvD11Td66zTg5IaEQDIZ+f8WS6nn/98nAVMDtR9zW7Te5h9kGJGfe6WiHVaGRPpBvqC4iypGHjbRwANwofZvmp5wP08hY1CsnKY5tfP+E2k/iAQgKKa6QoxXToYvP7rsSkglak8N5g/+FJGnq4wP6cOzgZpjdPMwaVt5432GA=="],"issuer":"https://login.microsoftonline.com/906aefe9-76a7-4f65-b82d-5ec20775d5aa/v2.0"}]}` + + var jk JWKS + if err := json.Unmarshal([]byte(jsonkey), &jk); err != nil { + t.Fatal("Unmarshal: ", err) + } else if len(jk.Keys) != 3 { + t.Fatalf("Expected 3 keys, got %d", len(jk.Keys)) + } + + var kids []string + for ii, jks := range jk.Keys { + _, err := jks.DecodePublicKey() + if err != nil { + t.Fatalf("Failed to decode key %d: %v", ii, err) + } + kids = append(kids, jks.Kid) + } + if len(kids) != 3 { + t.Fatalf("Failed to find the expected number of kids: 3, got %d", len(kids)) + } +} + +// A.1 - Example public keys +func TestPublicKey(t *testing.T) { + const jsonkey = `{"keys": + [ + {"kty":"EC", + "crv":"P-256", + "x":"MKBCTNIcKUSDii11ySs3526iDZ8AiTo7Tu6KPAqv7D4", + "y":"4Etl6SRW2YiLUrN5vfvVHuhp7x8PxltmWWlbbM4IFyM", + "use":"enc", + "kid":"1"}, + + {"kty":"RSA", + "n": "0vx7agoebGcQSuuPiLJXZptN9nndrQmbXEps2aiAFbWhM78LhWx4cbbfAAtVT86zwu1RK7aPFFxuhDR1L6tSoc_BJECPebWKRXjBZCiFV4n3oknjhMstn64tZ_2W-5JsGY4Hc5n9yBXArwl93lqt7_RN5w6Cf0h4QyQ5v-65YGjQR0_FDW2QvzqY368QQMicAtaSqzs8KJZgnYb9c7d0zgdAZHzu6qMQvRL5hajrn1n91CbOpbISD08qNLyrdkt-bFTWhAI4vMQFh6WeZu0fM4lFd2NcRwr3XPksINHaQ-G_xBniIqbw0Ls1jF44-csFCur-kEgU8awapJzKnqDKgw", + "e":"AQAB", + "alg":"RS256", + "kid":"2011-04-29"} + ] + }` + + var jk JWKS + if err := json.Unmarshal([]byte(jsonkey), &jk); err != nil { + t.Fatal("Unmarshal: ", err) + } else if len(jk.Keys) != 2 { + t.Fatalf("Expected 2 keys, got %d", len(jk.Keys)) + } + + keys := make([]crypto.PublicKey, len(jk.Keys)) + for ii, jks := range jk.Keys { + var err error + keys[ii], err = jks.DecodePublicKey() + if err != nil { + t.Fatalf("Failed to decode key %d: %v", ii, err) + } + } + + //nolint:gocritic + if key0, ok := keys[0].(*ecdsa.PublicKey); !ok { + t.Fatalf("Expected ECDSA key[0], got %T", keys[0]) + } else if key1, ok := keys[1].(*rsa.PublicKey); !ok { + t.Fatalf("Expected RSA key[1], got %T", keys[1]) + } else if key0.Curve != elliptic.P256() { + t.Fatal("Key[0] is not using P-256 curve") + } else if !bytes.Equal(key0.X.Bytes(), []byte{ + 0x30, 0xa0, 0x42, 0x4c, 0xd2, + 0x1c, 0x29, 0x44, 0x83, 0x8a, 0x2d, 0x75, 0xc9, 0x2b, 0x37, 0xe7, 0x6e, 0xa2, + 0xd, 0x9f, 0x0, 0x89, 0x3a, 0x3b, 0x4e, 0xee, 0x8a, 0x3c, 0xa, 0xaf, 0xec, 0x3e, + }) { + t.Fatalf("Bad key[0].X, got %v", key0.X.Bytes()) + } else if !bytes.Equal(key0.Y.Bytes(), []byte{ + 0xe0, 0x4b, 0x65, 0xe9, 0x24, + 0x56, 0xd9, 0x88, 0x8b, 0x52, 0xb3, 0x79, 0xbd, 0xfb, 0xd5, 0x1e, 0xe8, + 0x69, 0xef, 0x1f, 0xf, 0xc6, 0x5b, 0x66, 0x59, 0x69, 0x5b, 0x6c, 0xce, + 0x8, 0x17, 0x23, + }) { + t.Fatalf("Bad key[0].Y, got %v", key0.Y.Bytes()) + } else if key1.E != 0x10001 { + t.Fatalf("Bad key[1].E: %d", key1.E) + } else if !bytes.Equal(key1.N.Bytes(), []byte{ + 0xd2, 0xfc, 0x7b, 0x6a, 0xa, 0x1e, + 0x6c, 0x67, 0x10, 0x4a, 0xeb, 0x8f, 0x88, 0xb2, 0x57, 0x66, 0x9b, 0x4d, 0xf6, + 0x79, 0xdd, 0xad, 0x9, 0x9b, 0x5c, 0x4a, 0x6c, 0xd9, 0xa8, 0x80, 0x15, 0xb5, + 0xa1, 0x33, 0xbf, 0xb, 0x85, 0x6c, 0x78, 0x71, 0xb6, 0xdf, 0x0, 0xb, 0x55, + 0x4f, 0xce, 0xb3, 0xc2, 0xed, 0x51, 0x2b, 0xb6, 0x8f, 0x14, 0x5c, 0x6e, 0x84, + 0x34, 0x75, 0x2f, 0xab, 0x52, 0xa1, 0xcf, 0xc1, 0x24, 0x40, 0x8f, 0x79, 0xb5, + 0x8a, 0x45, 0x78, 0xc1, 0x64, 0x28, 0x85, 0x57, 0x89, 0xf7, 0xa2, 0x49, 0xe3, + 0x84, 0xcb, 0x2d, 0x9f, 0xae, 0x2d, 0x67, 0xfd, 0x96, 0xfb, 0x92, 0x6c, 0x19, + 0x8e, 0x7, 0x73, 0x99, 0xfd, 0xc8, 0x15, 0xc0, 0xaf, 0x9, 0x7d, 0xde, 0x5a, + 0xad, 0xef, 0xf4, 0x4d, 0xe7, 0xe, 0x82, 0x7f, 0x48, 0x78, 0x43, 0x24, 0x39, + 0xbf, 0xee, 0xb9, 0x60, 0x68, 0xd0, 0x47, 0x4f, 0xc5, 0xd, 0x6d, 0x90, 0xbf, + 0x3a, 0x98, 0xdf, 0xaf, 0x10, 0x40, 0xc8, 0x9c, 0x2, 0xd6, 0x92, 0xab, 0x3b, + 0x3c, 0x28, 0x96, 0x60, 0x9d, 0x86, 0xfd, 0x73, 0xb7, 0x74, 0xce, 0x7, 0x40, + 0x64, 0x7c, 0xee, 0xea, 0xa3, 0x10, 0xbd, 0x12, 0xf9, 0x85, 0xa8, 0xeb, 0x9f, + 0x59, 0xfd, 0xd4, 0x26, 0xce, 0xa5, 0xb2, 0x12, 0xf, 0x4f, 0x2a, 0x34, 0xbc, + 0xab, 0x76, 0x4b, 0x7e, 0x6c, 0x54, 0xd6, 0x84, 0x2, 0x38, 0xbc, 0xc4, 0x5, 0x87, + 0xa5, 0x9e, 0x66, 0xed, 0x1f, 0x33, 0x89, 0x45, 0x77, 0x63, 0x5c, 0x47, 0xa, + 0xf7, 0x5c, 0xf9, 0x2c, 0x20, 0xd1, 0xda, 0x43, 0xe1, 0xbf, 0xc4, 0x19, 0xe2, + 0x22, 0xa6, 0xf0, 0xd0, 0xbb, 0x35, 0x8c, 0x5e, 0x38, 0xf9, 0xcb, 0x5, 0xa, 0xea, + 0xfe, 0x90, 0x48, 0x14, 0xf1, 0xac, 0x1a, 0xa4, 0x9c, 0xca, 0x9e, 0xa0, 0xca, 0x83, + }) { + t.Fatalf("Bad key[1].N, got %v", key1.N.Bytes()) + } +} diff --git a/internal/config/identity/openid/jwt.go b/internal/config/identity/openid/jwt.go new file mode 100644 index 00000000..2a422010 --- /dev/null +++ b/internal/config/identity/openid/jwt.go @@ -0,0 +1,286 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package openid + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "sync" + "time" + + jwtgo "github.com/golang-jwt/jwt/v4" + "github.com/minio/minio/internal/arn" + "github.com/minio/minio/internal/auth" + xnet "github.com/minio/pkg/v3/net" + "github.com/minio/pkg/v3/policy" +) + +type publicKeys struct { + *sync.RWMutex + + // map of kid to public key + pkMap map[string]any +} + +func (pk *publicKeys) parseAndAdd(b io.Reader) error { + var jwk JWKS + err := json.NewDecoder(b).Decode(&jwk) + if err != nil { + return err + } + + for _, key := range jwk.Keys { + pkey, err := key.DecodePublicKey() + if err != nil { + return err + } + pk.add(key.Kid, pkey) + } + + return nil +} + +func (pk *publicKeys) add(keyID string, key any) { + pk.Lock() + defer pk.Unlock() + + pk.pkMap[keyID] = key +} + +func (pk *publicKeys) get(kid string) any { + pk.RLock() + defer pk.RUnlock() + return pk.pkMap[kid] +} + +// PopulatePublicKey - populates a new publickey from the JWKS URL. +func (r *Config) PopulatePublicKey(arn arn.ARN) error { + pCfg := r.arnProviderCfgsMap[arn] + if pCfg.JWKS.URL == nil || pCfg.JWKS.URL.String() == "" { + return nil + } + + // Add client secret for the client ID for HMAC based signature. + r.pubKeys.add(pCfg.ClientID, []byte(pCfg.ClientSecret)) + + client := &http.Client{ + Transport: r.transport, + } + + resp, err := client.Get(pCfg.JWKS.URL.String()) + if err != nil { + return err + } + defer r.closeRespFn(resp.Body) + if resp.StatusCode != http.StatusOK { + return errors.New(resp.Status) + } + + return r.pubKeys.parseAndAdd(resp.Body) +} + +// ErrTokenExpired - error token expired +var ( + ErrTokenExpired = errors.New("token expired") +) + +func updateClaimsExpiry(dsecs string, claims map[string]any) error { + expStr := claims["exp"] + if expStr == "" { + return ErrTokenExpired + } + + // No custom duration requested, the claims can be used as is. + if dsecs == "" { + return nil + } + + if _, err := auth.ExpToInt64(expStr); err != nil { + return err + } + + defaultExpiryDuration, err := GetDefaultExpiration(dsecs) + if err != nil { + return err + } + + claims["exp"] = time.Now().UTC().Add(defaultExpiryDuration).Unix() // update with new expiry. + return nil +} + +const ( + audClaim = "aud" + azpClaim = "azp" +) + +// Validate - validates the id_token. +func (r *Config) Validate(ctx context.Context, arn arn.ARN, token, accessToken, dsecs string, claims map[string]any) error { + jp := new(jwtgo.Parser) + jp.ValidMethods = []string{ + "RS256", "RS384", "RS512", + "ES256", "ES384", "ES512", + "HS256", "HS384", "HS512", + "RS3256", "RS3384", "RS3512", + "ES3256", "ES3384", "ES3512", + } + + keyFuncCallback := func(jwtToken *jwtgo.Token) (any, error) { + kid, ok := jwtToken.Header["kid"].(string) + if !ok { + return nil, fmt.Errorf("Invalid kid value %v", jwtToken.Header["kid"]) + } + pubkey := r.pubKeys.get(kid) + if pubkey == nil { + return nil, fmt.Errorf("No public key found for kid %s", kid) + } + return pubkey, nil + } + + pCfg, ok := r.arnProviderCfgsMap[arn] + if !ok { + return fmt.Errorf("Role %s does not exist", arn) + } + + mclaims := jwtgo.MapClaims(claims) + jwtToken, err := jp.ParseWithClaims(token, &mclaims, keyFuncCallback) + if err != nil { + // Re-populate the public key in-case the JWKS + // pubkeys are refreshed + if err = r.PopulatePublicKey(arn); err != nil { + return err + } + jwtToken, err = jwtgo.ParseWithClaims(token, &mclaims, keyFuncCallback) + if err != nil { + return err + } + } + + if !jwtToken.Valid { + return ErrTokenExpired + } + + if err = updateClaimsExpiry(dsecs, mclaims); err != nil { + return err + } + + if err = r.updateUserinfoClaims(ctx, arn, accessToken, mclaims); err != nil { + return err + } + + // Validate that matching clientID appears in the aud or azp claims. + + // REQUIRED. Audience(s) that this ID Token is intended for. + // It MUST contain the OAuth 2.0 client_id of the Relying Party + // as an audience value. It MAY also contain identifiers for + // other audiences. In the general case, the aud value is an + // array of case sensitive strings. In the common special case + // when there is one audience, the aud value MAY be a single + // case sensitive + audValues, ok := policy.GetValuesFromClaims(mclaims, audClaim) + if !ok { + return errors.New("STS JWT Token has `aud` claim invalid, `aud` must match configured OpenID Client ID") + } + if !audValues.Contains(pCfg.ClientID) { + // if audience claims is missing, look for "azp" claims. + // OPTIONAL. Authorized party - the party to which the ID + // Token was issued. If present, it MUST contain the OAuth + // 2.0 Client ID of this party. This Claim is only needed + // when the ID Token has a single audience value and that + // audience is different than the authorized party. It MAY + // be included even when the authorized party is the same + // as the sole audience. The azp value is a case sensitive + // string containing a StringOrURI value + azpValues, ok := policy.GetValuesFromClaims(mclaims, azpClaim) + if !ok { + return errors.New("STS JWT Token has `azp` claim invalid, `azp` must match configured OpenID Client ID") + } + if !azpValues.Contains(pCfg.ClientID) { + return errors.New("STS JWT Token has `azp` claim invalid, `azp` must match configured OpenID Client ID") + } + } + + return nil +} + +func (r *Config) updateUserinfoClaims(ctx context.Context, arn arn.ARN, accessToken string, claims map[string]any) error { + pCfg, ok := r.arnProviderCfgsMap[arn] + // If claim user info is enabled, get claims from userInfo + // and overwrite them with the claims from JWT. + if ok && pCfg.ClaimUserinfo { + if accessToken == "" { + return errors.New("access_token is mandatory if user_info claim is enabled") + } + uclaims, err := pCfg.UserInfo(ctx, accessToken, r.transport) + if err != nil { + return err + } + for k, v := range uclaims { + if _, ok := claims[k]; !ok { // only add to claims not update it. + claims[k] = v + } + } + } + return nil +} + +// DiscoveryDoc - parses the output from openid-configuration +// for example https://accounts.google.com/.well-known/openid-configuration +type DiscoveryDoc struct { + Issuer string `json:"issuer,omitempty"` + AuthEndpoint string `json:"authorization_endpoint,omitempty"` + TokenEndpoint string `json:"token_endpoint,omitempty"` + EndSessionEndpoint string `json:"end_session_endpoint,omitempty"` + UserInfoEndpoint string `json:"userinfo_endpoint,omitempty"` + RevocationEndpoint string `json:"revocation_endpoint,omitempty"` + JwksURI string `json:"jwks_uri,omitempty"` + ResponseTypesSupported []string `json:"response_types_supported,omitempty"` + SubjectTypesSupported []string `json:"subject_types_supported,omitempty"` + IDTokenSigningAlgValuesSupported []string `json:"id_token_signing_alg_values_supported,omitempty"` + ScopesSupported []string `json:"scopes_supported,omitempty"` + TokenEndpointAuthMethods []string `json:"token_endpoint_auth_methods_supported,omitempty"` + ClaimsSupported []string `json:"claims_supported,omitempty"` + CodeChallengeMethodsSupported []string `json:"code_challenge_methods_supported,omitempty"` +} + +func parseDiscoveryDoc(u *xnet.URL, transport http.RoundTripper, closeRespFn func(io.ReadCloser)) (DiscoveryDoc, error) { + d := DiscoveryDoc{} + req, err := http.NewRequest(http.MethodGet, u.String(), nil) + if err != nil { + return d, err + } + clnt := http.Client{ + Transport: transport, + } + resp, err := clnt.Do(req) + if err != nil { + return d, err + } + defer closeRespFn(resp.Body) + if resp.StatusCode != http.StatusOK { + return d, fmt.Errorf("unexpected error returned by %s : status(%s)", u, resp.Status) + } + dec := json.NewDecoder(resp.Body) + if err = dec.Decode(&d); err != nil { + return d, err + } + return d, nil +} diff --git a/internal/config/identity/openid/jwt_test.go b/internal/config/identity/openid/jwt_test.go new file mode 100644 index 00000000..cb54faff --- /dev/null +++ b/internal/config/identity/openid/jwt_test.go @@ -0,0 +1,300 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package openid + +import ( + "bytes" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "sync" + "testing" + "time" + + jwtgo "github.com/golang-jwt/jwt/v4" + "github.com/minio/minio/internal/arn" + "github.com/minio/minio/internal/config" + jwtm "github.com/minio/minio/internal/jwt" + xnet "github.com/minio/pkg/v3/net" +) + +func TestUpdateClaimsExpiry(t *testing.T) { + testCases := []struct { + exp any + dsecs string + expectedFailure bool + }{ + {"", "", true}, + {"-1", "0", true}, + {"-1", "900", true}, + {"1574812326", "900", false}, + {1574812326, "900", false}, + {int64(1574812326), "900", false}, + {int(1574812326), "900", false}, + {uint(1574812326), "900", false}, + {uint64(1574812326), "900", false}, + {json.Number("1574812326"), "900", false}, + {1574812326.000, "900", false}, + {time.Duration(3) * time.Minute, "900", false}, + } + + for _, testCase := range testCases { + t.Run("", func(t *testing.T) { + claims := map[string]any{} + claims["exp"] = testCase.exp + err := updateClaimsExpiry(testCase.dsecs, claims) + if err != nil && !testCase.expectedFailure { + t.Errorf("Expected success, got failure %s", err) + } + if err == nil && testCase.expectedFailure { + t.Error("Expected failure, got success") + } + }) + } +} + +func initJWKSServer() *httptest.Server { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + const jsonkey = `{"keys": + [ + {"kty":"RSA", + "n": "0vx7agoebGcQSuuPiLJXZptN9nndrQmbXEps2aiAFbWhM78LhWx4cbbfAAtVT86zwu1RK7aPFFxuhDR1L6tSoc_BJECPebWKRXjBZCiFV4n3oknjhMstn64tZ_2W-5JsGY4Hc5n9yBXArwl93lqt7_RN5w6Cf0h4QyQ5v-65YGjQR0_FDW2QvzqY368QQMicAtaSqzs8KJZgnYb9c7d0zgdAZHzu6qMQvRL5hajrn1n91CbOpbISD08qNLyrdkt-bFTWhAI4vMQFh6WeZu0fM4lFd2NcRwr3XPksINHaQ-G_xBniIqbw0Ls1jF44-csFCur-kEgU8awapJzKnqDKgw", + "e":"AQAB", + "alg":"RS256", + "kid":"2011-04-29"} + ] + }` + w.Write([]byte(jsonkey)) + })) + return server +} + +func TestJWTHMACType(t *testing.T) { + server := initJWKSServer() + defer server.Close() + + jwt := &jwtgo.Token{ + Method: jwtgo.SigningMethodHS256, + Claims: jwtgo.StandardClaims{ + ExpiresAt: 253428928061, + Audience: "76b95ae5-33ef-4283-97b7-d2a85dc2d8f4", + }, + Header: map[string]any{ + "typ": "JWT", + "alg": jwtgo.SigningMethodHS256.Alg(), + "kid": "76b95ae5-33ef-4283-97b7-d2a85dc2d8f4", + }, + } + + token, err := jwt.SignedString([]byte("WNGvKVyyNmXq0TraSvjaDN9CtpFgx35IXtGEffMCPR0")) + if err != nil { + t.Fatal(err) + } + fmt.Println(token) + + u1, err := xnet.ParseHTTPURL(server.URL) + if err != nil { + t.Fatal(err) + } + + pubKeys := publicKeys{ + RWMutex: &sync.RWMutex{}, + pkMap: map[string]any{}, + } + pubKeys.add("76b95ae5-33ef-4283-97b7-d2a85dc2d8f4", []byte("WNGvKVyyNmXq0TraSvjaDN9CtpFgx35IXtGEffMCPR0")) + + if len(pubKeys.pkMap) != 1 { + t.Fatalf("Expected 1 keys, got %d", len(pubKeys.pkMap)) + } + + provider := providerCfg{ + ClientID: "76b95ae5-33ef-4283-97b7-d2a85dc2d8f4", + ClientSecret: "WNGvKVyyNmXq0TraSvjaDN9CtpFgx35IXtGEffMCPR0", + } + provider.JWKS.URL = u1 + cfg := Config{ + Enabled: true, + pubKeys: pubKeys, + arnProviderCfgsMap: map[arn.ARN]*providerCfg{ + DummyRoleARN: &provider, + }, + ProviderCfgs: map[string]*providerCfg{ + "1": &provider, + }, + closeRespFn: func(rc io.ReadCloser) { + rc.Close() + }, + } + + var claims jwtgo.MapClaims + if err = cfg.Validate(t.Context(), DummyRoleARN, token, "", "", claims); err != nil { + t.Fatal(err) + } +} + +func TestJWT(t *testing.T) { + const jsonkey = `{"keys": + [ + {"kty":"RSA", + "n": "0vx7agoebGcQSuuPiLJXZptN9nndrQmbXEps2aiAFbWhM78LhWx4cbbfAAtVT86zwu1RK7aPFFxuhDR1L6tSoc_BJECPebWKRXjBZCiFV4n3oknjhMstn64tZ_2W-5JsGY4Hc5n9yBXArwl93lqt7_RN5w6Cf0h4QyQ5v-65YGjQR0_FDW2QvzqY368QQMicAtaSqzs8KJZgnYb9c7d0zgdAZHzu6qMQvRL5hajrn1n91CbOpbISD08qNLyrdkt-bFTWhAI4vMQFh6WeZu0fM4lFd2NcRwr3XPksINHaQ-G_xBniIqbw0Ls1jF44-csFCur-kEgU8awapJzKnqDKgw", + "e":"AQAB", + "alg":"RS256", + "kid":"2011-04-29"} + ] + }` + + pubKeys := publicKeys{ + RWMutex: &sync.RWMutex{}, + pkMap: map[string]any{}, + } + err := pubKeys.parseAndAdd(bytes.NewBuffer([]byte(jsonkey))) + if err != nil { + t.Fatal("Error loading pubkeys:", err) + } + if len(pubKeys.pkMap) != 1 { + t.Fatalf("Expected 1 keys, got %d", len(pubKeys.pkMap)) + } + + u1, err := xnet.ParseHTTPURL("http://127.0.0.1:8443") + if err != nil { + t.Fatal(err) + } + + provider := providerCfg{} + provider.JWKS.URL = u1 + cfg := Config{ + Enabled: true, + pubKeys: pubKeys, + arnProviderCfgsMap: map[arn.ARN]*providerCfg{ + DummyRoleARN: &provider, + }, + ProviderCfgs: map[string]*providerCfg{ + "1": &provider, + }, + } + + u, err := url.Parse("http://127.0.0.1:8443/?Token=invalid") + if err != nil { + t.Fatal(err) + } + + var claims jwtgo.MapClaims + if err = cfg.Validate(t.Context(), DummyRoleARN, u.Query().Get("Token"), "", "", claims); err == nil { + t.Fatal(err) + } +} + +func TestDefaultExpiryDuration(t *testing.T) { + testCases := []struct { + reqURL string + duration time.Duration + expectErr bool + }{ + { + reqURL: "http://127.0.0.1:8443/?Token=xxxxx", + duration: time.Duration(60) * time.Minute, + }, + { + reqURL: "http://127.0.0.1:8443/?DurationSeconds=9s", + expectErr: true, + }, + { + reqURL: "http://127.0.0.1:8443/?DurationSeconds=31536001", + expectErr: true, + }, + { + reqURL: "http://127.0.0.1:8443/?DurationSeconds=800", + expectErr: true, + }, + { + reqURL: "http://127.0.0.1:8443/?DurationSeconds=901", + duration: time.Duration(901) * time.Second, + }, + } + + for i, testCase := range testCases { + u, err := url.Parse(testCase.reqURL) + if err != nil { + t.Fatal(err) + } + d, err := GetDefaultExpiration(u.Query().Get("DurationSeconds")) + gotErr := (err != nil) + if testCase.expectErr != gotErr { + t.Errorf("Test %d: Expected %v, got %v with error %s", i+1, testCase.expectErr, gotErr, err) + } + if d != testCase.duration { + t.Errorf("Test %d: Expected duration %d, got %d", i+1, testCase.duration, d) + } + } +} + +func TestExpCorrect(t *testing.T) { + signKey, _ := base64.StdEncoding.DecodeString("NTNv7j0TuYARvmNMmWXo6fKvM4o6nv/aUi9ryX38ZH+L1bkrnD1ObOQ8JAUmHCBq7Iy7otZcyAagBLHVKvvYaIpmMuxmARQ97jUVG16Jkpkp1wXOPsrF9zwew6TpczyHkHgX5EuLg2MeBuiT/qJACs1J0apruOOJCg/gOtkjB4c=") + + claimsMap := jwtm.NewMapClaims() + claimsMap.SetExpiry(time.Now().Add(time.Minute)) + claimsMap.SetAccessKey("test-access") + if err := updateClaimsExpiry("3600", claimsMap.MapClaims); err != nil { + t.Error(err) + } + // Build simple token with updated expiration claim + token := jwtgo.NewWithClaims(jwtgo.SigningMethodHS256, claimsMap) + tokenString, err := token.SignedString(signKey) + if err != nil { + t.Error(err) + } + + // Parse token to be sure it is valid + err = jwtm.ParseWithClaims(tokenString, claimsMap, func(*jwtm.MapClaims) ([]byte, error) { + return signKey, nil + }) + if err != nil { + t.Error(err) + } +} + +func TestKeycloakProviderInitialization(t *testing.T) { + testConfig := providerCfg{ + DiscoveryDoc: DiscoveryDoc{ + TokenEndpoint: "http://keycloak.test/token/endpoint", + }, + } + testKvs := config.KVS{} + testKvs.Set(Vendor, "keycloak") + testKvs.Set(KeyCloakRealm, "TestRealm") + testKvs.Set(KeyCloakAdminURL, "http://keycloak.test/auth/admin") + cfgGet := func(param string) string { + return testKvs.Get(param) + } + + if testConfig.provider != nil { + t.Errorf("Empty config cannot have any provider!") + } + + if err := testConfig.initializeProvider(cfgGet, http.DefaultTransport); err != nil { + t.Error(err) + } + + if testConfig.provider == nil { + t.Errorf("keycloak provider must be initialized!") + } +} diff --git a/internal/config/identity/openid/openid.go b/internal/config/identity/openid/openid.go new file mode 100644 index 00000000..003ede92 --- /dev/null +++ b/internal/config/identity/openid/openid.go @@ -0,0 +1,652 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package openid + +import ( + "crypto/sha1" + "encoding/base64" + "errors" + "io" + "maps" + "net/http" + "slices" + "sort" + "strconv" + "strings" + "sync" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio-go/v7/pkg/set" + "github.com/minio/minio/internal/arn" + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/config/identity/openid/provider" + "github.com/minio/minio/internal/hash/sha256" + "github.com/minio/pkg/v3/env" + xnet "github.com/minio/pkg/v3/net" + "github.com/minio/pkg/v3/policy" +) + +// OpenID keys and envs. +const ( + ClientID = "client_id" + ClientSecret = "client_secret" + ConfigURL = "config_url" + ClaimName = "claim_name" + ClaimUserinfo = "claim_userinfo" + RolePolicy = "role_policy" + DisplayName = "display_name" + UserReadableClaim = "user_readable_claim" + UserIDClaim = "user_id_claim" + + Scopes = "scopes" + RedirectURI = "redirect_uri" + RedirectURIDynamic = "redirect_uri_dynamic" + Vendor = "vendor" + + // Vendor specific ENV only enabled if the Vendor matches == "vendor" + KeyCloakRealm = "keycloak_realm" + KeyCloakAdminURL = "keycloak_admin_url" + + // Removed params + JwksURL = "jwks_url" + ClaimPrefix = "claim_prefix" +) + +// DefaultKVS - default config for OpenID config +var ( + DefaultKVS = config.KVS{ + config.KV{ + Key: config.Enable, + Value: "", + }, + config.KV{ + Key: DisplayName, + Value: "", + }, + config.KV{ + Key: ConfigURL, + Value: "", + }, + config.KV{ + Key: ClientID, + Value: "", + }, + config.KV{ + Key: ClientSecret, + Value: "", + }, + config.KV{ + Key: ClaimName, + Value: policy.PolicyName, + }, + config.KV{ + Key: ClaimUserinfo, + Value: "", + }, + config.KV{ + Key: RolePolicy, + Value: "", + }, + config.KV{ + Key: ClaimPrefix, + Value: "", + HiddenIfEmpty: true, + }, + config.KV{ + Key: RedirectURI, + Value: "", + HiddenIfEmpty: true, + }, + config.KV{ + Key: RedirectURIDynamic, + Value: "off", + }, + config.KV{ + Key: Scopes, + Value: "", + }, + config.KV{ + Key: Vendor, + Value: "", + }, + config.KV{ + Key: KeyCloakRealm, + Value: "", + }, + config.KV{ + Key: KeyCloakAdminURL, + Value: "", + }, + config.KV{ + Key: UserReadableClaim, + Value: "", + }, + config.KV{ + Key: UserIDClaim, + Value: "", + }, + } +) + +var errSingleProvider = config.Errorf("Only one OpenID provider can be configured if not using role policy mapping") + +// DummyRoleARN is used to indicate that the user associated with it was +// authenticated via policy-claim based OpenID provider. +var DummyRoleARN = func() arn.ARN { + v, err := arn.NewIAMRoleARN("dummy-internal", "") + if err != nil { + panic("should not happen!") + } + return v +}() + +// Config - OpenID Config +type Config struct { + Enabled bool + + // map of roleARN to providerCfg's + arnProviderCfgsMap map[arn.ARN]*providerCfg + + // map of config names to providerCfg's + ProviderCfgs map[string]*providerCfg + + pubKeys publicKeys + roleArnPolicyMap map[arn.ARN]string + + transport http.RoundTripper + closeRespFn func(io.ReadCloser) +} + +// Clone returns a cloned copy of OpenID config. +func (r *Config) Clone() Config { + if r == nil { + return Config{} + } + cfg := Config{ + Enabled: r.Enabled, + arnProviderCfgsMap: make(map[arn.ARN]*providerCfg, len(r.arnProviderCfgsMap)), + ProviderCfgs: make(map[string]*providerCfg, len(r.ProviderCfgs)), + pubKeys: r.pubKeys, + roleArnPolicyMap: make(map[arn.ARN]string, len(r.roleArnPolicyMap)), + transport: r.transport, + closeRespFn: r.closeRespFn, + } + maps.Copy(cfg.arnProviderCfgsMap, r.arnProviderCfgsMap) + maps.Copy(cfg.ProviderCfgs, r.ProviderCfgs) + maps.Copy(cfg.roleArnPolicyMap, r.roleArnPolicyMap) + return cfg +} + +// LookupConfig lookup jwks from config, override with any ENVs. +func LookupConfig(s config.Config, transport http.RoundTripper, closeRespFn func(io.ReadCloser), serverRegion string) (c Config, err error) { + openIDClientTransport := http.DefaultTransport + if transport != nil { + openIDClientTransport = transport + } + c = Config{ + Enabled: false, + arnProviderCfgsMap: map[arn.ARN]*providerCfg{}, + ProviderCfgs: map[string]*providerCfg{}, + pubKeys: publicKeys{ + RWMutex: &sync.RWMutex{}, + pkMap: map[string]any{}, + }, + roleArnPolicyMap: map[arn.ARN]string{}, + transport: openIDClientTransport, + closeRespFn: closeRespFn, + } + + seenClientIDs := set.NewStringSet() + + deprecatedKeys := []string{JwksURL} + + // remove this since we have removed support for this already. + for k := range s[config.IdentityOpenIDSubSys] { + for _, dk := range deprecatedKeys { + kvs := s[config.IdentityOpenIDSubSys][k] + kvs.Delete(dk) + s[config.IdentityOpenIDSubSys][k] = kvs + } + } + + if err := s.CheckValidKeys(config.IdentityOpenIDSubSys, deprecatedKeys); err != nil { + return c, err + } + + openIDTargets, err := s.GetAvailableTargets(config.IdentityOpenIDSubSys) + if err != nil { + return c, err + } + + for _, cfgName := range openIDTargets { + getCfgVal := func(cfgParam string) string { + // As parameters are already validated, we skip checking + // if the config param was found. + val, _, _ := s.ResolveConfigParam(config.IdentityOpenIDSubSys, cfgName, cfgParam, false) + return val + } + + // In the past, when only one openID provider was allowed, there + // was no `enable` parameter - the configuration is turned off + // by clearing the values. With multiple providers, we support + // individually enabling/disabling provider configurations. If + // the enable parameter's value is non-empty, we use that + // setting, otherwise we treat it as enabled if some important + // parameters are non-empty. + var ( + cfgEnableVal = getCfgVal(config.Enable) + isExplicitlyEnabled = cfgEnableVal != "" + ) + + var enabled bool + if isExplicitlyEnabled { + enabled, err = config.ParseBool(cfgEnableVal) + if err != nil { + return c, err + } + // No need to continue loading if the config is not enabled. + if !enabled { + continue + } + } + + p := newProviderCfgFromConfig(getCfgVal) + configURL := getCfgVal(ConfigURL) + + if !isExplicitlyEnabled { + enabled = true + if p.ClientID == "" && p.ClientSecret == "" && configURL == "" { + enabled = false + } + } + + // No need to continue loading if the config is not enabled. + if !enabled { + continue + } + + // Validate that client ID has not been duplicately specified. + if seenClientIDs.Contains(p.ClientID) { + return c, config.Errorf("Client ID %s is present with multiple OpenID configurations", p.ClientID) + } + seenClientIDs.Add(p.ClientID) + + p.URL, err = xnet.ParseHTTPURL(configURL) + if err != nil { + return c, err + } + configURLDomain := p.URL.Hostname() + p.DiscoveryDoc, err = parseDiscoveryDoc(p.URL, transport, closeRespFn) + if err != nil { + return c, err + } + + if p.ClaimUserinfo && configURL == "" { + return c, errors.New("please specify config_url to enable fetching claims from UserInfo endpoint") + } + + if scopeList := getCfgVal(Scopes); scopeList != "" { + var scopes []string + for scope := range strings.SplitSeq(scopeList, ",") { + scope = strings.TrimSpace(scope) + if scope == "" { + return c, config.Errorf("empty scope value is not allowed '%s', please refer to our documentation", scopeList) + } + scopes = append(scopes, scope) + } + // Replace the discovery document scopes by client customized scopes. + p.DiscoveryDoc.ScopesSupported = scopes + } + + // Check if claim name is the non-default value and role policy is set. + if p.ClaimName != policy.PolicyName && p.RolePolicy != "" { + // In the unlikely event that the user specifies + // `policy.PolicyName` as the claim name explicitly and sets + // a role policy, this check is thwarted, but we will be using + // the role policy anyway. + return c, config.Errorf("Role Policy (=`%s`) and Claim Name (=`%s`) cannot both be set", p.RolePolicy, p.ClaimName) + } + + jwksURL := p.DiscoveryDoc.JwksURI + if jwksURL == "" { + return c, config.Errorf("no JWKS URI found in your provider's discovery doc (config_url=%s)", configURL) + } + + p.JWKS.URL, err = xnet.ParseHTTPURL(jwksURL) + if err != nil { + return c, err + } + + if p.RolePolicy != "" { + // RolePolicy is validated by IAM System during its + // initialization. + + // Generate role ARN as combination of provider domain and + // prefix of client ID. + domain := configURLDomain + if domain == "" { + // Attempt to parse the JWKs URI. + domain = p.JWKS.URL.Hostname() + if domain == "" { + return c, config.Errorf("unable to parse a domain from the OpenID config") + } + } + if p.ClientID == "" { + return c, config.Errorf("client ID must not be empty") + } + + // We set the resource ID of the role arn as a hash of client + // ID, so we can get a short roleARN that stays the same on + // restart. + var resourceID string + { + h := sha1.New() + h.Write([]byte(p.ClientID)) + bs := h.Sum(nil) + resourceID = base64.RawURLEncoding.EncodeToString(bs) + } + p.roleArn, err = arn.NewIAMRoleARN(resourceID, serverRegion) + if err != nil { + return c, config.Errorf("unable to generate ARN from the OpenID config: %v", err) + } + + c.roleArnPolicyMap[p.roleArn] = p.RolePolicy + } else if p.ClaimName == "" { + return c, config.Errorf("A role policy or claim name must be specified") + } + + if err = p.initializeProvider(getCfgVal, c.transport); err != nil { + return c, err + } + + arnKey := p.roleArn + if p.RolePolicy == "" { + arnKey = DummyRoleARN + // Ensure that at most one JWT policy claim based provider may be + // defined. + if _, ok := c.arnProviderCfgsMap[DummyRoleARN]; ok { + return c, errSingleProvider + } + } + + c.arnProviderCfgsMap[arnKey] = &p + c.ProviderCfgs[cfgName] = &p + + if err = c.PopulatePublicKey(arnKey); err != nil { + return c, err + } + } + + c.Enabled = true + + return c, nil +} + +// ErrProviderConfigNotFound - represents a non-existing provider error. +var ErrProviderConfigNotFound = errors.New("provider configuration not found") + +// GetConfigInfo - returns configuration and related info for the given IDP +// provider. +func (r *Config) GetConfigInfo(s config.Config, cfgName string) ([]madmin.IDPCfgInfo, error) { + openIDConfigs, err := s.GetAvailableTargets(config.IdentityOpenIDSubSys) + if err != nil { + return nil, err + } + + present := slices.Contains(openIDConfigs, cfgName) + + if !present { + return nil, ErrProviderConfigNotFound + } + + kvsrcs, err := s.GetResolvedConfigParams(config.IdentityOpenIDSubSys, cfgName, true) + if err != nil { + return nil, err + } + + res := make([]madmin.IDPCfgInfo, 0, len(kvsrcs)+1) + for _, kvsrc := range kvsrcs { + // skip returning default config values. + if kvsrc.Src == config.ValueSourceDef { + if kvsrc.Key != madmin.EnableKey { + continue + } + // for EnableKey we set an explicit on/off from live configuration + // if it is present. + if _, ok := r.ProviderCfgs[cfgName]; !ok { + // No live config is present + continue + } + if r.Enabled { + kvsrc.Value = "on" + } else { + kvsrc.Value = "off" + } + } + res = append(res, madmin.IDPCfgInfo{ + Key: kvsrc.Key, + Value: kvsrc.Value, + IsCfg: true, + IsEnv: kvsrc.Src == config.ValueSourceEnv, + }) + } + + if provCfg, exists := r.ProviderCfgs[cfgName]; exists && provCfg.RolePolicy != "" { + // Append roleARN + res = append(res, madmin.IDPCfgInfo{ + Key: "roleARN", + Value: provCfg.roleArn.String(), + IsCfg: false, + }) + } + + // sort the structs by the key + sort.Slice(res, func(i, j int) bool { + return res[i].Key < res[j].Key + }) + + return res, nil +} + +// GetConfigList - list openID configurations +func (r *Config) GetConfigList(s config.Config) ([]madmin.IDPListItem, error) { + openIDConfigs, err := s.GetAvailableTargets(config.IdentityOpenIDSubSys) + if err != nil { + return nil, err + } + + var res []madmin.IDPListItem + for _, cfg := range openIDConfigs { + pcfg, ok := r.ProviderCfgs[cfg] + if !ok { + res = append(res, madmin.IDPListItem{ + Type: "openid", + Name: cfg, + Enabled: false, + }) + } else { + var roleARN string + if pcfg.RolePolicy != "" { + roleARN = pcfg.roleArn.String() + } + res = append(res, madmin.IDPListItem{ + Type: "openid", + Name: cfg, + Enabled: r.Enabled, + RoleARN: roleARN, + }) + } + } + + return res, nil +} + +// Enabled returns if configURL is enabled. +func Enabled(kvs config.KVS) bool { + return kvs.Get(ConfigURL) != "" +} + +// GetSettings - fetches OIDC settings for site-replication related validation. +// NOTE that region must be populated by caller as this package does not know. +func (r *Config) GetSettings() madmin.OpenIDSettings { + res := madmin.OpenIDSettings{} + if !r.Enabled { + return res + } + h := sha256.New() + hashedSecret := "" + for arn, provCfg := range r.arnProviderCfgsMap { + h.Write([]byte(provCfg.ClientSecret)) + hashedSecret = base64.RawURLEncoding.EncodeToString(h.Sum(nil)) + h.Reset() + if arn != DummyRoleARN { + if res.Roles == nil { + res.Roles = make(map[string]madmin.OpenIDProviderSettings) + } + res.Roles[arn.String()] = madmin.OpenIDProviderSettings{ + ClaimUserinfoEnabled: provCfg.ClaimUserinfo, + RolePolicy: provCfg.RolePolicy, + ClientID: provCfg.ClientID, + HashedClientSecret: hashedSecret, + } + } else { + res.ClaimProvider = madmin.OpenIDProviderSettings{ + ClaimUserinfoEnabled: provCfg.ClaimUserinfo, + RolePolicy: provCfg.RolePolicy, + ClientID: provCfg.ClientID, + HashedClientSecret: hashedSecret, + } + } + } + + return res +} + +// GetIAMPolicyClaimName - returns the policy claim name for the (at most one) +// provider configured without a role policy. +func (r *Config) GetIAMPolicyClaimName() string { + pCfg, ok := r.arnProviderCfgsMap[DummyRoleARN] + if !ok { + return "" + } + return pCfg.ClaimPrefix + pCfg.ClaimName +} + +// LookupUser lookup userid for the provider +func (r Config) LookupUser(roleArn, userid string) (provider.User, error) { + // Can safely ignore error here as empty or invalid ARNs will not be + // mapped. + arnVal, _ := arn.Parse(roleArn) + pCfg, ok := r.arnProviderCfgsMap[arnVal] + if ok { + user, err := pCfg.provider.LookupUser(userid) + if err != nil && err != provider.ErrAccessTokenExpired { + return user, err + } + if err == provider.ErrAccessTokenExpired { + if err = pCfg.provider.LoginWithClientID(pCfg.ClientID, pCfg.ClientSecret); err != nil { + return user, err + } + user, err = pCfg.provider.LookupUser(userid) + } + return user, err + } + // Without any specific logic for a provider, all accounts + // are always enabled. + return provider.User{ID: userid, Enabled: true}, nil +} + +// ProviderEnabled returns true if any vendor specific provider is enabled. +func (r Config) ProviderEnabled() bool { + if !r.Enabled { + return false + } + for _, v := range r.arnProviderCfgsMap { + if v.provider != nil { + return true + } + } + return false +} + +// GetRoleInfo - returns ARN to policies map if a role policy based openID +// provider is configured. Otherwise returns nil. +func (r Config) GetRoleInfo() map[arn.ARN]string { + for _, p := range r.arnProviderCfgsMap { + if p.RolePolicy != "" { + return r.roleArnPolicyMap + } + } + return nil +} + +// GetDefaultExpiration - returns the expiration seconds expected. +func GetDefaultExpiration(dsecs string) (time.Duration, error) { + timeout := env.Get(config.EnvMinioStsDuration, "") + defaultExpiryDuration, err := time.ParseDuration(timeout) + if err != nil { + defaultExpiryDuration = time.Hour + } + if timeout == "" && dsecs != "" { + expirySecs, err := strconv.ParseInt(dsecs, 10, 64) + if err != nil { + return 0, auth.ErrInvalidDuration + } + + // The duration, in seconds, of the role session. + // The value can range from 900 seconds (15 minutes) + // up to 365 days. + if expirySecs < config.MinExpiration || expirySecs > config.MaxExpiration { + return 0, auth.ErrInvalidDuration + } + + defaultExpiryDuration = time.Duration(expirySecs) * time.Second + } else if timeout == "" && dsecs == "" { + return time.Hour, nil + } + + if defaultExpiryDuration.Seconds() < config.MinExpiration || defaultExpiryDuration.Seconds() > config.MaxExpiration { + return 0, auth.ErrInvalidDuration + } + + return defaultExpiryDuration, nil +} + +// GetUserReadableClaim returns the human readable claim name for the given +// configuration name. +func (r Config) GetUserReadableClaim(cfgName string) string { + pCfg, ok := r.ProviderCfgs[cfgName] + if ok { + return pCfg.UserReadableClaim + } + return "" +} + +// GetUserIDClaim returns the user ID claim for the given configuration name, or "sub" if not set. +func (r Config) GetUserIDClaim(cfgName string) string { + pCfg, ok := r.ProviderCfgs[cfgName] + if ok { + if pCfg.UserIDClaim != "" { + return pCfg.UserIDClaim + } + return "sub" + } + return "" // an incorrect config should be handled outside this function +} diff --git a/internal/config/identity/openid/provider/keycloak.go b/internal/config/identity/openid/provider/keycloak.go new file mode 100644 index 00000000..3e9648d6 --- /dev/null +++ b/internal/config/identity/openid/provider/keycloak.go @@ -0,0 +1,186 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package provider + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + "net/url" + "path" + "strings" + "sync" +) + +// Token - parses the output from IDP id_token. +type Token struct { + AccessToken string `json:"access_token"` + Expiry int `json:"expires_in"` +} + +// KeycloakProvider implements Provider interface for KeyCloak Identity Provider. +type KeycloakProvider struct { + sync.Mutex + + oeConfig DiscoveryDoc + client http.Client + adminURL string + realm string + + // internal value refreshed + accessToken Token +} + +// LoginWithUser authenticates username/password, not needed for Keycloak +func (k *KeycloakProvider) LoginWithUser(username, password string) error { + return ErrNotImplemented +} + +// LoginWithClientID is implemented by Keycloak service account support +func (k *KeycloakProvider) LoginWithClientID(clientID, clientSecret string) error { + values := url.Values{} + values.Set("client_id", clientID) + values.Set("client_secret", clientSecret) + values.Set("grant_type", "client_credentials") + + req, err := http.NewRequest(http.MethodPost, k.oeConfig.TokenEndpoint, strings.NewReader(values.Encode())) + if err != nil { + return err + } + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + + resp, err := k.client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + var accessToken Token + if err = json.NewDecoder(resp.Body).Decode(&accessToken); err != nil { + return err + } + + k.Lock() + k.accessToken = accessToken + k.Unlock() + return nil +} + +// LookupUser lookup user by their userid. +func (k *KeycloakProvider) LookupUser(userid string) (User, error) { + req, err := http.NewRequest(http.MethodGet, k.adminURL, nil) + if err != nil { + return User{}, err + } + req.URL.Path = path.Join(req.URL.Path, "realms", k.realm, "users", userid) + + k.Lock() + accessToken := k.accessToken + k.Unlock() + if accessToken.AccessToken == "" { + return User{}, ErrAccessTokenExpired + } + req.Header.Set("Authorization", "Bearer "+accessToken.AccessToken) + resp, err := k.client.Do(req) + if err != nil { + return User{}, err + } + defer resp.Body.Close() + switch resp.StatusCode { + case http.StatusOK, http.StatusPartialContent: + var u User + if err = json.NewDecoder(resp.Body).Decode(&u); err != nil { + return User{}, err + } + return u, nil + case http.StatusNotFound: + return User{ + ID: userid, + Enabled: false, + }, nil + case http.StatusUnauthorized: + return User{}, ErrAccessTokenExpired + } + return User{}, fmt.Errorf("Unable to lookup - keycloak user lookup returned %v", resp.Status) +} + +// Option is a function type that accepts a pointer Target +type Option func(*KeycloakProvider) + +// WithTransport provide custom transport +func WithTransport(transport http.RoundTripper) Option { + return func(p *KeycloakProvider) { + p.client = http.Client{ + Transport: transport, + } + } +} + +// WithOpenIDConfig provide OpenID Endpoint configuration discovery document +func WithOpenIDConfig(oeConfig DiscoveryDoc) Option { + return func(p *KeycloakProvider) { + p.oeConfig = oeConfig + } +} + +// WithAdminURL provide admin URL configuration for Keycloak +func WithAdminURL(url string) Option { + return func(p *KeycloakProvider) { + p.adminURL = url + } +} + +// WithRealm provide realm configuration for Keycloak +func WithRealm(realm string) Option { + return func(p *KeycloakProvider) { + p.realm = realm + } +} + +// KeyCloak initializes a new keycloak provider +func KeyCloak(opts ...Option) (Provider, error) { + p := &KeycloakProvider{} + + for _, opt := range opts { + opt(p) + } + + if p.adminURL == "" { + return nil, errors.New("Admin URL cannot be empty") + } + + _, err := url.Parse(p.adminURL) + if err != nil { + return nil, fmt.Errorf("Unable to parse the adminURL %s: %w", p.adminURL, err) + } + + if p.client.Transport == nil { + p.client.Transport = http.DefaultTransport + } + + if p.oeConfig.TokenEndpoint == "" { + return nil, errors.New("missing OpenID token endpoint") + } + + if p.realm == "" { + p.realm = "master" // default realm + } + + return p, nil +} diff --git a/internal/config/identity/openid/provider/provider.go b/internal/config/identity/openid/provider/provider.go new file mode 100644 index 00000000..8b514e98 --- /dev/null +++ b/internal/config/identity/openid/provider/provider.go @@ -0,0 +1,62 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package provider + +import "errors" + +// DiscoveryDoc - parses the output from openid-configuration +// for example https://accounts.google.com/.well-known/openid-configuration +// +//nolint:unused +type DiscoveryDoc struct { + Issuer string `json:"issuer,omitempty"` + AuthEndpoint string `json:"authorization_endpoint,omitempty"` + TokenEndpoint string `json:"token_endpoint,omitempty"` + EndSessionEndpoint string `json:"end_session_endpoint,omitempty"` + UserInfoEndpoint string `json:"userinfo_endpoint,omitempty"` + RevocationEndpoint string `json:"revocation_endpoint,omitempty"` + JwksURI string `json:"jwks_uri,omitempty"` + ResponseTypesSupported []string `json:"response_types_supported,omitempty"` + SubjectTypesSupported []string `json:"subject_types_supported,omitempty"` + IDTokenSigningAlgValuesSupported []string `json:"id_token_signing_alg_values_supported,omitempty"` + ScopesSupported []string `json:"scopes_supported,omitempty"` + TokenEndpointAuthMethods []string `json:"token_endpoint_auth_methods_supported,omitempty"` + ClaimsSupported []string `json:"claims_supported,omitempty"` + CodeChallengeMethodsSupported []string `json:"code_challenge_methods_supported,omitempty"` +} + +// User represents information about user. +type User struct { + Name string `json:"username"` + ID string `json:"id"` + Enabled bool `json:"enabled"` +} + +// Standard errors. +var ( + ErrNotImplemented = errors.New("function not implemented") + ErrAccessTokenExpired = errors.New("access_token expired or unauthorized") +) + +// Provider implements identity provider specific admin operations, such as +// looking up users, fetching additional attributes etc. +type Provider interface { + LoginWithUser(username, password string) error + LoginWithClientID(clientID, clientSecret string) error + LookupUser(userid string) (User, error) +} diff --git a/internal/config/identity/openid/providercfg.go b/internal/config/identity/openid/providercfg.go new file mode 100644 index 00000000..1ccc230c --- /dev/null +++ b/internal/config/identity/openid/providercfg.go @@ -0,0 +1,161 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package openid + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/http" + + "github.com/minio/minio/internal/arn" + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/config/identity/openid/provider" + xhttp "github.com/minio/minio/internal/http" + xnet "github.com/minio/pkg/v3/net" +) + +type providerCfg struct { + // Used for user interface like console + DisplayName string + + JWKS struct { + URL *xnet.URL + } + URL *xnet.URL + ClaimPrefix string + ClaimName string + ClaimUserinfo bool + RedirectURI string + RedirectURIDynamic bool + DiscoveryDoc DiscoveryDoc + ClientID string + ClientSecret string + RolePolicy string + UserReadableClaim string + UserIDClaim string + + roleArn arn.ARN + provider provider.Provider +} + +func newProviderCfgFromConfig(getCfgVal func(cfgName string) string) providerCfg { + return providerCfg{ + DisplayName: getCfgVal(DisplayName), + ClaimName: getCfgVal(ClaimName), + ClaimUserinfo: getCfgVal(ClaimUserinfo) == config.EnableOn, + ClaimPrefix: getCfgVal(ClaimPrefix), + RedirectURI: getCfgVal(RedirectURI), + RedirectURIDynamic: getCfgVal(RedirectURIDynamic) == config.EnableOn, + ClientID: getCfgVal(ClientID), + ClientSecret: getCfgVal(ClientSecret), + RolePolicy: getCfgVal(RolePolicy), + UserReadableClaim: getCfgVal(UserReadableClaim), + UserIDClaim: getCfgVal(UserIDClaim), + } +} + +const ( + keyCloakVendor = "keycloak" +) + +// initializeProvider initializes if any additional vendor specific information +// was provided, initialization will return an error initial login fails. +func (p *providerCfg) initializeProvider(cfgGet func(string) string, transport http.RoundTripper) error { + vendor := cfgGet(Vendor) + if vendor == "" { + return nil + } + var err error + switch vendor { + case keyCloakVendor: + adminURL := cfgGet(KeyCloakAdminURL) + realm := cfgGet(KeyCloakRealm) + p.provider, err = provider.KeyCloak( + provider.WithAdminURL(adminURL), + provider.WithOpenIDConfig(provider.DiscoveryDoc(p.DiscoveryDoc)), + provider.WithTransport(transport), + provider.WithRealm(realm), + ) + return err + default: + return fmt.Errorf("Unsupported vendor %s", keyCloakVendor) + } +} + +// GetRoleArn returns the role ARN. +func (p *providerCfg) GetRoleArn() string { + if p.RolePolicy == "" { + return "" + } + return p.roleArn.String() +} + +// UserInfo returns claims for authenticated user from userInfo endpoint. +// +// Some OIDC implementations such as GitLab do not support +// claims as part of the normal oauth2 flow, instead rely +// on service providers making calls to IDP to fetch additional +// claims available from the UserInfo endpoint +func (p *providerCfg) UserInfo(ctx context.Context, accessToken string, transport http.RoundTripper) (map[string]any, error) { + if p.JWKS.URL == nil || p.JWKS.URL.String() == "" { + return nil, errors.New("openid not configured") + } + + req, err := http.NewRequestWithContext(ctx, http.MethodPost, p.DiscoveryDoc.UserInfoEndpoint, nil) + if err != nil { + return nil, err + } + + req.Header.Set("Content-Type", "application/x-www-form-urlencoded") + if accessToken != "" { + req.Header.Set("Authorization", "Bearer "+accessToken) + } + + client := &http.Client{ + Transport: transport, + } + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + + defer xhttp.DrainBody(resp.Body) + if resp.StatusCode != http.StatusOK { + // uncomment this for debugging when needed. + // reqBytes, _ := httputil.DumpRequest(req, false) + // fmt.Println(string(reqBytes)) + // respBytes, _ := httputil.DumpResponse(resp, true) + // fmt.Println(string(respBytes)) + return nil, errors.New(resp.Status) + } + + claims := map[string]any{} + if err = json.NewDecoder(resp.Body).Decode(&claims); err != nil { + // uncomment this for debugging when needed. + // reqBytes, _ := httputil.DumpRequest(req, false) + // fmt.Println(string(reqBytes)) + // respBytes, _ := httputil.DumpResponse(resp, true) + // fmt.Println(string(respBytes)) + return nil, err + } + + return claims, nil +} diff --git a/internal/config/identity/openid/rsa-sha3_contrib.go b/internal/config/identity/openid/rsa-sha3_contrib.go new file mode 100644 index 00000000..82607473 --- /dev/null +++ b/internal/config/identity/openid/rsa-sha3_contrib.go @@ -0,0 +1,51 @@ +// MinIO Object Storage (c) 2021 MinIO, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package openid + +import ( + "crypto" + + "github.com/golang-jwt/jwt/v4" + + // Needed for SHA3 to work - See: https://golang.org/src/crypto/crypto.go?s=1034:1288 + _ "golang.org/x/crypto/sha3" +) + +// Specific instances for RS256 and company +var ( + SigningMethodRS3256 *jwt.SigningMethodRSA + SigningMethodRS3384 *jwt.SigningMethodRSA + SigningMethodRS3512 *jwt.SigningMethodRSA +) + +func init() { + // RS3256 + SigningMethodRS3256 = &jwt.SigningMethodRSA{Name: "RS3256", Hash: crypto.SHA3_256} + jwt.RegisterSigningMethod(SigningMethodRS3256.Alg(), func() jwt.SigningMethod { + return SigningMethodRS3256 + }) + + // RS3384 + SigningMethodRS3384 = &jwt.SigningMethodRSA{Name: "RS3384", Hash: crypto.SHA3_384} + jwt.RegisterSigningMethod(SigningMethodRS3384.Alg(), func() jwt.SigningMethod { + return SigningMethodRS3384 + }) + + // RS3512 + SigningMethodRS3512 = &jwt.SigningMethodRSA{Name: "RS3512", Hash: crypto.SHA3_512} + jwt.RegisterSigningMethod(SigningMethodRS3512.Alg(), func() jwt.SigningMethod { + return SigningMethodRS3512 + }) +} diff --git a/internal/config/identity/plugin/config.go b/internal/config/identity/plugin/config.go new file mode 100644 index 00000000..3b0ac830 --- /dev/null +++ b/internal/config/identity/plugin/config.go @@ -0,0 +1,514 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package plugin + +import ( + "bytes" + "context" + "crypto/sha1" + "encoding/base64" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "regexp" + "sync" + "time" + + "github.com/minio/minio/internal/arn" + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/env" + xnet "github.com/minio/pkg/v3/net" +) + +func authNLogIf(ctx context.Context, err error) { + logger.LogIf(ctx, "authN", err) +} + +// Authentication Plugin config and env variables +const ( + URL = "url" + AuthToken = "auth_token" + RolePolicy = "role_policy" + RoleID = "role_id" + + EnvIdentityPluginURL = "MINIO_IDENTITY_PLUGIN_URL" + EnvIdentityPluginAuthToken = "MINIO_IDENTITY_PLUGIN_AUTH_TOKEN" + EnvIdentityPluginRolePolicy = "MINIO_IDENTITY_PLUGIN_ROLE_POLICY" + EnvIdentityPluginRoleID = "MINIO_IDENTITY_PLUGIN_ROLE_ID" +) + +var ( + // DefaultKVS - default config for AuthN plugin config + DefaultKVS = config.KVS{ + config.KV{ + Key: URL, + Value: "", + }, + config.KV{ + Key: AuthToken, + Value: "", + }, + config.KV{ + Key: RolePolicy, + Value: "", + }, + config.KV{ + Key: RoleID, + Value: "", + }, + } + + defaultHelpPostfix = func(key string) string { + return config.DefaultHelpPostfix(DefaultKVS, key) + } + + // Help for Identity Plugin + Help = config.HelpKVS{ + config.HelpKV{ + Key: URL, + Description: `plugin hook endpoint (HTTP(S)) e.g. "http://localhost:8181/path/to/endpoint"` + defaultHelpPostfix(URL), + Type: "url", + }, + config.HelpKV{ + Key: AuthToken, + Description: "authorization token for plugin hook endpoint" + defaultHelpPostfix(AuthToken), + Optional: true, + Type: "string", + Sensitive: true, + Secret: true, + }, + config.HelpKV{ + Key: RolePolicy, + Description: "policies to apply for plugin authorized users" + defaultHelpPostfix(RolePolicy), + Type: "string", + }, + config.HelpKV{ + Key: RoleID, + Description: "unique ID to generate the ARN" + defaultHelpPostfix(RoleID), + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: config.Comment, + Description: config.DefaultComment, + Optional: true, + Type: "sentence", + }, + } +) + +// Allows only Base64 URL encoding characters. +var validRoleIDRegex = regexp.MustCompile(`^[A-Za-z0-9_-]+$`) + +// Args for authentication plugin. +type Args struct { + URL *xnet.URL + AuthToken string + Transport http.RoundTripper + CloseRespFn func(r io.ReadCloser) + + RolePolicy string + RoleARN arn.ARN +} + +// Validate - validate configuration params. +func (a *Args) Validate() error { + req, err := http.NewRequest(http.MethodPost, a.URL.String(), bytes.NewReader([]byte(""))) + if err != nil { + return err + } + + req.Header.Set("Content-Type", "application/json") + if a.AuthToken != "" { + req.Header.Set("Authorization", a.AuthToken) + } + + client := &http.Client{Transport: a.Transport} + resp, err := client.Do(req) + if err != nil { + return err + } + defer a.CloseRespFn(resp.Body) + + return nil +} + +type serviceRTTMinuteStats struct { + statsTime time.Time + rttMsSum, maxRttMs float64 + successRequestCount int64 + failedRequestCount int64 +} + +type metrics struct { + sync.Mutex + LastCheckSuccess time.Time + LastCheckFailure time.Time + lastFullMinute serviceRTTMinuteStats + currentMinute serviceRTTMinuteStats +} + +func (h *metrics) setConnSuccess(reqStartTime time.Time) { + h.Lock() + defer h.Unlock() + h.LastCheckSuccess = reqStartTime +} + +func (h *metrics) setConnFailure(reqStartTime time.Time) { + h.Lock() + defer h.Unlock() + h.LastCheckFailure = reqStartTime +} + +func (h *metrics) updateLastFullMinute(currReqMinute time.Time) { + // Assumes the caller has h.Lock()'ed + h.lastFullMinute = h.currentMinute + h.currentMinute = serviceRTTMinuteStats{ + statsTime: currReqMinute, + } +} + +func (h *metrics) accumRequestRTT(reqStartTime time.Time, rttMs float64, isSuccess bool) { + h.Lock() + defer h.Unlock() + + // Update connectivity times + if isSuccess { + if reqStartTime.After(h.LastCheckSuccess) { + h.LastCheckSuccess = reqStartTime + } + } else { + if reqStartTime.After(h.LastCheckFailure) { + h.LastCheckFailure = reqStartTime + } + } + + // Round the request time *down* to whole minute. + reqTimeMinute := reqStartTime.Truncate(time.Minute) + if reqTimeMinute.After(h.currentMinute.statsTime) { + // Drop the last full minute now, since we got a request for a time we + // are not yet tracking. + h.updateLastFullMinute(reqTimeMinute) + } + var entry *serviceRTTMinuteStats + switch { + case reqTimeMinute.Equal(h.currentMinute.statsTime): + entry = &h.currentMinute + case reqTimeMinute.Equal(h.lastFullMinute.statsTime): + entry = &h.lastFullMinute + default: + // This request is too old, it should never happen, ignore it as we + // cannot return an error. + return + } + + // Update stats + if isSuccess { + if entry.maxRttMs < rttMs { + entry.maxRttMs = rttMs + } + entry.rttMsSum += rttMs + entry.successRequestCount++ + } else { + entry.failedRequestCount++ + } +} + +// AuthNPlugin - implements pluggable authentication via webhook. +type AuthNPlugin struct { + args Args + client *http.Client + shutdownCtx context.Context + serviceMetrics *metrics +} + +// Enabled returns if AuthNPlugin is enabled. +func Enabled(kvs config.KVS) bool { + return kvs.Get(URL) != "" +} + +// LookupConfig lookup AuthNPlugin from config, override with any ENVs. +func LookupConfig(kv config.KVS, transport *http.Transport, closeRespFn func(io.ReadCloser), serverRegion string) (Args, error) { + args := Args{} + + if err := config.CheckValidKeys(config.IdentityPluginSubSys, kv, DefaultKVS); err != nil { + return args, err + } + + pluginURL := env.Get(EnvIdentityPluginURL, kv.Get(URL)) + if pluginURL == "" { + return args, nil + } + + authToken := env.Get(EnvIdentityPluginAuthToken, kv.Get(AuthToken)) + + u, err := xnet.ParseHTTPURL(pluginURL) + if err != nil { + return args, err + } + + rolePolicy := env.Get(EnvIdentityPluginRolePolicy, kv.Get(RolePolicy)) + if rolePolicy == "" { + return args, config.Errorf("A role policy must be specified for Identity Management Plugin") + } + + resourceID := "idmp-" + roleID := env.Get(EnvIdentityPluginRoleID, kv.Get(RoleID)) + if roleID == "" { + // We use a hash of the plugin URL so that the ARN remains + // constant across restarts. + h := sha1.New() + h.Write([]byte(pluginURL)) + bs := h.Sum(nil) + resourceID += base64.RawURLEncoding.EncodeToString(bs) + } else { + // Check that the roleID is restricted to URL safe characters + // (base64 URL encoding chars). + if !validRoleIDRegex.MatchString(roleID) { + return args, config.Errorf("Role ID must match the regexp `^[a-zA-Z0-9_-]+$`") + } + + // Use the user provided ID here. + resourceID += roleID + } + + roleArn, err := arn.NewIAMRoleARN(resourceID, serverRegion) + if err != nil { + return args, config.Errorf("unable to generate ARN from the plugin config: %v", err) + } + + args = Args{ + URL: u, + AuthToken: authToken, + Transport: transport, + CloseRespFn: closeRespFn, + RolePolicy: rolePolicy, + RoleARN: roleArn, + } + if err = args.Validate(); err != nil { + return args, err + } + return args, nil +} + +// New - initializes Authorization Management Plugin. +func New(shutdownCtx context.Context, args Args) *AuthNPlugin { + if args.URL == nil || args.URL.Scheme == "" && args.AuthToken == "" { + return nil + } + plugin := AuthNPlugin{ + args: args, + client: &http.Client{Transport: args.Transport}, + shutdownCtx: shutdownCtx, + serviceMetrics: &metrics{ + Mutex: sync.Mutex{}, + LastCheckSuccess: time.Unix(0, 0), + LastCheckFailure: time.Unix(0, 0), + lastFullMinute: serviceRTTMinuteStats{}, + currentMinute: serviceRTTMinuteStats{}, + }, + } + go plugin.doPeriodicHealthCheck() + return &plugin +} + +// AuthNSuccessResponse - represents the response from the authentication plugin +// service. +type AuthNSuccessResponse struct { + User string `json:"user"` + MaxValiditySeconds int `json:"maxValiditySeconds"` + Claims map[string]any `json:"claims"` +} + +// AuthNErrorResponse - represents an error response from the authN plugin. +type AuthNErrorResponse struct { + Reason string `json:"reason"` +} + +// AuthNResponse - represents a result of the authentication operation. +type AuthNResponse struct { + Success *AuthNSuccessResponse + Failure *AuthNErrorResponse +} + +const ( + minValidityDurationSeconds int = 900 + maxValidityDurationSeconds int = 365 * 24 * 3600 +) + +// Authenticate authenticates the token with the external hook endpoint and +// returns a parent user, max expiry duration for the authentication and a set +// of claims. +func (o *AuthNPlugin) Authenticate(roleArn arn.ARN, token string) (AuthNResponse, error) { + if o == nil { + return AuthNResponse{}, nil + } + + if roleArn != o.args.RoleARN { + return AuthNResponse{}, fmt.Errorf("Invalid role ARN value: %s", roleArn.String()) + } + + u := url.URL(*o.args.URL) + q := u.Query() + q.Set("token", token) + u.RawQuery = q.Encode() + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodPost, u.String(), nil) + if err != nil { + return AuthNResponse{}, err + } + + if o.args.AuthToken != "" { + req.Header.Set("Authorization", o.args.AuthToken) + } + + reqStartTime := time.Now() + resp, err := o.client.Do(req) + if err != nil { + o.serviceMetrics.accumRequestRTT(reqStartTime, 0, false) + return AuthNResponse{}, err + } + defer o.args.CloseRespFn(resp.Body) + reqDurNanos := time.Since(reqStartTime).Nanoseconds() + o.serviceMetrics.accumRequestRTT(reqStartTime, float64(reqDurNanos)/1e6, true) + + switch resp.StatusCode { + case 200: + var result AuthNSuccessResponse + if err = json.NewDecoder(resp.Body).Decode(&result); err != nil { + return AuthNResponse{}, err + } + + if result.MaxValiditySeconds < minValidityDurationSeconds || result.MaxValiditySeconds > maxValidityDurationSeconds { + return AuthNResponse{}, fmt.Errorf("Plugin returned an invalid validity duration (%d) - should be between %d and %d", + result.MaxValiditySeconds, minValidityDurationSeconds, maxValidityDurationSeconds) + } + + return AuthNResponse{ + Success: &result, + }, nil + + case 403: + var result AuthNErrorResponse + if err = json.NewDecoder(resp.Body).Decode(&result); err != nil { + return AuthNResponse{}, err + } + return AuthNResponse{ + Failure: &result, + }, nil + + default: + return AuthNResponse{}, fmt.Errorf("Invalid status code %d from auth plugin", resp.StatusCode) + } +} + +// GetRoleInfo - returns ARN to policies map. +func (o *AuthNPlugin) GetRoleInfo() map[arn.ARN]string { + return map[arn.ARN]string{ + o.args.RoleARN: o.args.RolePolicy, + } +} + +// checkConnectivity returns true if we are able to connect to the plugin +// service. +func (o *AuthNPlugin) checkConnectivity(ctx context.Context) bool { + ctx, cancel := context.WithTimeout(ctx, healthCheckTimeout) + defer cancel() + u := url.URL(*o.args.URL) + + req, err := http.NewRequestWithContext(ctx, http.MethodHead, u.String(), nil) + if err != nil { + authNLogIf(ctx, err) + return false + } + + if o.args.AuthToken != "" { + req.Header.Set("Authorization", o.args.AuthToken) + } + + resp, err := o.client.Do(req) + if err != nil { + return false + } + defer o.args.CloseRespFn(resp.Body) + return true +} + +var ( + healthCheckInterval = 1 * time.Minute + healthCheckTimeout = 5 * time.Second +) + +func (o *AuthNPlugin) doPeriodicHealthCheck() { + ticker := time.NewTicker(healthCheckInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + now := time.Now() + isConnected := o.checkConnectivity(o.shutdownCtx) + if isConnected { + o.serviceMetrics.setConnSuccess(now) + } else { + o.serviceMetrics.setConnFailure(now) + } + case <-o.shutdownCtx.Done(): + return + } + } +} + +// Metrics contains metrics about the authentication plugin service. +type Metrics struct { + LastReachableSecs, LastUnreachableSecs float64 + + // Last whole minute stats + TotalRequests, FailedRequests int64 + AvgSuccRTTMs float64 + MaxSuccRTTMs float64 +} + +// Metrics reports metrics related to plugin service reachability and stats for the last whole minute +func (o *AuthNPlugin) Metrics() Metrics { + if o == nil { + // Return empty metrics when not configured. + return Metrics{} + } + o.serviceMetrics.Lock() + defer o.serviceMetrics.Unlock() + l := &o.serviceMetrics.lastFullMinute + var avg float64 + if l.successRequestCount > 0 { + avg = l.rttMsSum / float64(l.successRequestCount) + } + now := time.Now().UTC() + return Metrics{ + LastReachableSecs: now.Sub(o.serviceMetrics.LastCheckSuccess).Seconds(), + LastUnreachableSecs: now.Sub(o.serviceMetrics.LastCheckFailure).Seconds(), + TotalRequests: l.failedRequestCount + l.successRequestCount, + FailedRequests: l.failedRequestCount, + AvgSuccRTTMs: avg, + MaxSuccRTTMs: l.maxRttMs, + } +} diff --git a/internal/config/identity/tls/config.go b/internal/config/identity/tls/config.go new file mode 100644 index 00000000..b002aab7 --- /dev/null +++ b/internal/config/identity/tls/config.go @@ -0,0 +1,126 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package tls + +import ( + "strconv" + "time" + + "github.com/minio/minio/internal/auth" + "github.com/minio/minio/internal/config" + "github.com/minio/pkg/v3/env" +) + +const ( + // EnvIdentityTLSEnabled is an environment variable that controls whether the X.509 + // TLS STS API is enabled. By default, if not set, it is enabled. + EnvIdentityTLSEnabled = "MINIO_IDENTITY_TLS_ENABLE" + + // EnvIdentityTLSSkipVerify is an environment variable that controls whether + // MinIO verifies the client certificate present by the client + // when requesting temp. credentials. + // By default, MinIO always verify the client certificate. + // + // The client certificate verification should only be skipped + // when debugging or testing a setup since it allows arbitrary + // clients to obtain temp. credentials with arbitrary policy + // permissions - including admin permissions. + EnvIdentityTLSSkipVerify = "MINIO_IDENTITY_TLS_SKIP_VERIFY" +) + +// Config contains the STS TLS configuration for generating temp. +// credentials and mapping client certificates to S3 policies. +type Config struct { + Enabled bool `json:"enabled"` + + // InsecureSkipVerify, if set to true, disables the client + // certificate verification. It should only be set for + // debugging or testing purposes. + InsecureSkipVerify bool `json:"skip_verify"` +} + +const ( + defaultExpiry time.Duration = 1 * time.Hour + minExpiry time.Duration = 15 * time.Minute + maxExpiry time.Duration = 365 * 24 * time.Hour +) + +// GetExpiryDuration - return parsed expiry duration. +func (l Config) GetExpiryDuration(dsecs string) (time.Duration, error) { + if dsecs == "" { + return defaultExpiry, nil + } + + d, err := strconv.Atoi(dsecs) + if err != nil { + return 0, auth.ErrInvalidDuration + } + + dur := time.Duration(d) * time.Second + + if dur < minExpiry || dur > maxExpiry { + return 0, auth.ErrInvalidDuration + } + return dur, nil +} + +// Lookup returns a new Config by merging the given K/V config +// system with environment variables. +func Lookup(kvs config.KVS) (Config, error) { + if err := config.CheckValidKeys(config.IdentityTLSSubSys, kvs, DefaultKVS); err != nil { + return Config{}, err + } + cfg := Config{} + var err error + v := env.Get(EnvIdentityTLSEnabled, "") + if v == "" { + return cfg, nil + } + cfg.Enabled, err = config.ParseBool(v) + if err != nil { + return Config{}, err + } + cfg.InsecureSkipVerify, err = config.ParseBool(env.Get(EnvIdentityTLSSkipVerify, kvs.Get(skipVerify))) + if err != nil { + return Config{}, err + } + return cfg, nil +} + +const ( + skipVerify = "skip_verify" +) + +// DefaultKVS is the default K/V config system for +// the STS TLS API. +var DefaultKVS = config.KVS{ + config.KV{ + Key: skipVerify, + Value: "off", + }, +} + +// Help is the help and description for the STS API K/V configuration. +var Help = config.HelpKVS{ + config.HelpKV{ + Key: skipVerify, + Description: `trust client certificates without verification (default: 'off')`, + Optional: true, + Type: "on|off", + }, +} diff --git a/internal/config/ilm/help.go b/internal/config/ilm/help.go new file mode 100644 index 00000000..9cbf30b1 --- /dev/null +++ b/internal/config/ilm/help.go @@ -0,0 +1,52 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ilm + +import "github.com/minio/minio/internal/config" + +const ( + transitionWorkers = "transition_workers" + expirationWorkers = "expiration_workers" + // EnvILMTransitionWorkers env variable to configure number of transition workers + EnvILMTransitionWorkers = "MINIO_ILM_TRANSITION_WORKERS" + // EnvILMExpirationWorkers env variable to configure number of expiration workers + EnvILMExpirationWorkers = "MINIO_ILM_EXPIRATION_WORKERS" +) + +var ( + defaultHelpPostfix = func(key string) string { + return config.DefaultHelpPostfix(DefaultKVS, key) + } + + // Help holds configuration keys and their default values for the ILM + // subsystem + Help = config.HelpKVS{ + config.HelpKV{ + Key: transitionWorkers, + Type: "number", + Description: `set the number of transition workers` + defaultHelpPostfix(transitionWorkers), + Optional: true, + }, + config.HelpKV{ + Key: expirationWorkers, + Type: "number", + Description: `set the number of expiration workers` + defaultHelpPostfix(expirationWorkers), + Optional: true, + }, + } +) diff --git a/internal/config/ilm/ilm.go b/internal/config/ilm/ilm.go new file mode 100644 index 00000000..3ecf68fa --- /dev/null +++ b/internal/config/ilm/ilm.go @@ -0,0 +1,69 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ilm + +import ( + "strconv" + + "github.com/minio/minio/internal/config" + "github.com/minio/pkg/v3/env" +) + +// DefaultKVS default configuration values for ILM subsystem +var DefaultKVS = config.KVS{ + config.KV{ + Key: transitionWorkers, + Value: "100", + }, + config.KV{ + Key: expirationWorkers, + Value: "100", + }, +} + +// Config represents the different configuration values for ILM subsystem +type Config struct { + TransitionWorkers int + ExpirationWorkers int +} + +// LookupConfig - lookup ilm config and override with valid environment settings if any. +func LookupConfig(kvs config.KVS) (cfg Config, err error) { + cfg = Config{ + TransitionWorkers: 100, + ExpirationWorkers: 100, + } + + if err = config.CheckValidKeys(config.ILMSubSys, kvs, DefaultKVS); err != nil { + return cfg, err + } + + tw, err := strconv.Atoi(env.Get(EnvILMTransitionWorkers, kvs.GetWithDefault(transitionWorkers, DefaultKVS))) + if err != nil { + return cfg, err + } + + ew, err := strconv.Atoi(env.Get(EnvILMExpirationWorkers, kvs.GetWithDefault(expirationWorkers, DefaultKVS))) + if err != nil { + return cfg, err + } + + cfg.TransitionWorkers = tw + cfg.ExpirationWorkers = ew + return cfg, nil +} diff --git a/internal/config/lambda/config.go b/internal/config/lambda/config.go new file mode 100644 index 00000000..0e85e99b --- /dev/null +++ b/internal/config/lambda/config.go @@ -0,0 +1,40 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lambda + +import "github.com/minio/minio/internal/event/target" + +// Config - lambda target configuration structure, holds +// information about various lambda targets. +type Config struct { + Webhook map[string]target.WebhookArgs `json:"webhook"` +} + +const ( + defaultTarget = "1" +) + +// NewConfig - initialize lambda config. +func NewConfig() Config { + // Make sure to initialize lambda targets + cfg := Config{ + Webhook: make(map[string]target.WebhookArgs), + } + cfg.Webhook[defaultTarget] = target.WebhookArgs{} + return cfg +} diff --git a/internal/config/lambda/event/arn.go b/internal/config/lambda/event/arn.go new file mode 100644 index 00000000..65a9644d --- /dev/null +++ b/internal/config/lambda/event/arn.go @@ -0,0 +1,62 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "strings" +) + +// ARN - SQS resource name representation. +type ARN struct { + TargetID + region string +} + +// String - returns string representation. +func (arn ARN) String() string { + if arn.ID == "" && arn.Name == "" && arn.region == "" { + return "" + } + + return "arn:minio:s3-object-lambda:" + arn.region + ":" + arn.TargetID.String() +} + +// ParseARN - parses string to ARN. +func ParseARN(s string) (*ARN, error) { + // ARN must be in the format of arn:minio:s3-object-lambda::: + if !strings.HasPrefix(s, "arn:minio:s3-object-lambda:") { + return nil, &ErrInvalidARN{s} + } + + tokens := strings.Split(s, ":") + if len(tokens) != 6 { + return nil, &ErrInvalidARN{s} + } + + if tokens[4] == "" || tokens[5] == "" { + return nil, &ErrInvalidARN{s} + } + + return &ARN{ + region: tokens[3], + TargetID: TargetID{ + ID: tokens[4], + Name: tokens[5], + }, + }, nil +} diff --git a/internal/config/lambda/event/arn_test.go b/internal/config/lambda/event/arn_test.go new file mode 100644 index 00000000..68c24569 --- /dev/null +++ b/internal/config/lambda/event/arn_test.go @@ -0,0 +1,72 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "testing" +) + +func TestARNString(t *testing.T) { + testCases := []struct { + arn ARN + expectedResult string + }{ + {ARN{}, ""}, + {ARN{TargetID{"1", "webhook"}, ""}, "arn:minio:s3-object-lambda::1:webhook"}, + {ARN{TargetID{"1", "webhook"}, "us-east-1"}, "arn:minio:s3-object-lambda:us-east-1:1:webhook"}, + } + + for i, testCase := range testCases { + result := testCase.arn.String() + + if result != testCase.expectedResult { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestParseARN(t *testing.T) { + testCases := []struct { + s string + expectedARN *ARN + expectErr bool + }{ + {"", nil, true}, + {"arn:minio:s3-object-lambda:::", nil, true}, + {"arn:minio:s3-object-lambda::1:webhook:remote", nil, true}, + {"arn:aws:s3-object-lambda::1:webhook", nil, true}, + {"arn:minio:sns::1:webhook", nil, true}, + {"arn:minio:s3-object-lambda::1:webhook", &ARN{TargetID{"1", "webhook"}, ""}, false}, + {"arn:minio:s3-object-lambda:us-east-1:1:webhook", &ARN{TargetID{"1", "webhook"}, "us-east-1"}, false}, + } + + for i, testCase := range testCases { + arn, err := ParseARN(testCase.s) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + + if !testCase.expectErr { + if *arn != *testCase.expectedARN { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedARN, arn) + } + } + } +} diff --git a/internal/config/lambda/event/errors.go b/internal/config/lambda/event/errors.go new file mode 100644 index 00000000..8d5d74c9 --- /dev/null +++ b/internal/config/lambda/event/errors.go @@ -0,0 +1,49 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "fmt" +) + +// ErrUnknownRegion - unknown region error. +type ErrUnknownRegion struct { + Region string +} + +func (err ErrUnknownRegion) Error() string { + return fmt.Sprintf("unknown region '%v'", err.Region) +} + +// ErrARNNotFound - ARN not found error. +type ErrARNNotFound struct { + ARN ARN +} + +func (err ErrARNNotFound) Error() string { + return fmt.Sprintf("ARN '%v' not found", err.ARN) +} + +// ErrInvalidARN - invalid ARN error. +type ErrInvalidARN struct { + ARN string +} + +func (err ErrInvalidARN) Error() string { + return fmt.Sprintf("invalid ARN '%v'", err.ARN) +} diff --git a/internal/config/lambda/event/event.go b/internal/config/lambda/event/event.go new file mode 100644 index 00000000..d3d8cb7b --- /dev/null +++ b/internal/config/lambda/event/event.go @@ -0,0 +1,80 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import "net/http" + +// Identity represents access key who caused the event. +type Identity struct { + Type string `json:"type"` + PrincipalID string `json:"principalId"` + AccessKeyID string `json:"accessKeyId"` +} + +// UserRequest user request headers +type UserRequest struct { + URL string `json:"url"` + Headers http.Header `json:"headers"` +} + +// GetObjectContext provides the necessary details to perform +// download of the object, and return back the processed response +// to the server. +type GetObjectContext struct { + OutputRoute string `json:"outputRoute"` + OutputToken string `json:"outputToken"` + InputS3URL string `json:"inputS3Url"` +} + +// Event represents lambda function event, this is undocumented in AWS S3. This +// structure bases itself on this structure but there is no binding. +// +// { +// "xAmzRequestId": "a2871150-1df5-4dc9-ad9f-3da283ca1bf3", +// "getObjectContext": { +// "outputRoute": "...", +// "outputToken": "...", +// "inputS3Url": "" +// }, +// "configuration": { // not useful in MinIO +// "accessPointArn": "...", +// "supportingAccessPointArn": "...", +// "payload": "" +// }, +// "userRequest": { +// "url": "...", +// "headers": { +// "Host": "...", +// "X-Amz-Content-SHA256": "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855" +// } +// }, +// "userIdentity": { +// "type": "IAMUser", +// "principalId": "AIDAJF5MO57RFXQCE5ZNC", +// "arn": "...", +// "accountId": "...", +// "accessKeyId": "AKIA3WNQJCXE2DYPAU7R" +// }, +// "protocolVersion": "1.00" +// } +type Event struct { + ProtocolVersion string `json:"protocolVersion"` + GetObjectContext *GetObjectContext `json:"getObjectContext"` + UserIdentity Identity `json:"userIdentity"` + UserRequest UserRequest `json:"userRequest"` +} diff --git a/internal/config/lambda/event/targetid.go b/internal/config/lambda/event/targetid.go new file mode 100644 index 00000000..8cce5d4b --- /dev/null +++ b/internal/config/lambda/event/targetid.go @@ -0,0 +1,74 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "encoding/json" + "fmt" + "strings" +) + +// TargetID - holds identification and name strings of notification target. +type TargetID struct { + ID string + Name string +} + +// String - returns string representation. +func (tid TargetID) String() string { + return tid.ID + ":" + tid.Name +} + +// ToARN - converts to ARN. +func (tid TargetID) ToARN(region string) ARN { + return ARN{TargetID: tid, region: region} +} + +// MarshalJSON - encodes to JSON data. +func (tid TargetID) MarshalJSON() ([]byte, error) { + return json.Marshal(tid.String()) +} + +// UnmarshalJSON - decodes JSON data. +func (tid *TargetID) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + + targetID, err := parseTargetID(s) + if err != nil { + return err + } + + *tid = *targetID + return nil +} + +// parseTargetID - parses string to TargetID. +func parseTargetID(s string) (*TargetID, error) { + tokens := strings.Split(s, ":") + if len(tokens) != 2 { + return nil, fmt.Errorf("invalid TargetID format '%v'", s) + } + + return &TargetID{ + ID: tokens[0], + Name: tokens[1], + }, nil +} diff --git a/internal/config/lambda/event/targetid_test.go b/internal/config/lambda/event/targetid_test.go new file mode 100644 index 00000000..ffee1106 --- /dev/null +++ b/internal/config/lambda/event/targetid_test.go @@ -0,0 +1,118 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "reflect" + "testing" +) + +func TestTargetDString(t *testing.T) { + testCases := []struct { + tid TargetID + expectedResult string + }{ + {TargetID{}, ":"}, + {TargetID{"1", "webhook"}, "1:webhook"}, + {TargetID{"httpclient+2e33cdee-fbec-4bdd-917e-7d8e3c5a2531", "localhost:55638"}, "httpclient+2e33cdee-fbec-4bdd-917e-7d8e3c5a2531:localhost:55638"}, + } + + for i, testCase := range testCases { + result := testCase.tid.String() + + if result != testCase.expectedResult { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestTargetDToARN(t *testing.T) { + tid := TargetID{"1", "webhook"} + testCases := []struct { + tid TargetID + region string + expectedARN ARN + }{ + {tid, "", ARN{TargetID: tid, region: ""}}, + {tid, "us-east-1", ARN{TargetID: tid, region: "us-east-1"}}, + } + + for i, testCase := range testCases { + arn := testCase.tid.ToARN(testCase.region) + + if arn != testCase.expectedARN { + t.Fatalf("test %v: ARN: expected: %v, got: %v", i+1, testCase.expectedARN, arn) + } + } +} + +func TestTargetDMarshalJSON(t *testing.T) { + testCases := []struct { + tid TargetID + expectedData []byte + expectErr bool + }{ + {TargetID{}, []byte(`":"`), false}, + {TargetID{"1", "webhook"}, []byte(`"1:webhook"`), false}, + {TargetID{"httpclient+2e33cdee-fbec-4bdd-917e-7d8e3c5a2531", "localhost:55638"}, []byte(`"httpclient+2e33cdee-fbec-4bdd-917e-7d8e3c5a2531:localhost:55638"`), false}, + } + + for i, testCase := range testCases { + data, err := testCase.tid.MarshalJSON() + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + + if !testCase.expectErr { + if !reflect.DeepEqual(data, testCase.expectedData) { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, string(testCase.expectedData), string(data)) + } + } + } +} + +func TestTargetDUnmarshalJSON(t *testing.T) { + testCases := []struct { + data []byte + expectedTargetID *TargetID + expectErr bool + }{ + {[]byte(`""`), nil, true}, + {[]byte(`"httpclient+2e33cdee-fbec-4bdd-917e-7d8e3c5a2531:localhost:55638"`), nil, true}, + {[]byte(`":"`), &TargetID{}, false}, + {[]byte(`"1:webhook"`), &TargetID{"1", "webhook"}, false}, + } + + for i, testCase := range testCases { + targetID := &TargetID{} + err := targetID.UnmarshalJSON(testCase.data) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + + if !testCase.expectErr { + if *targetID != *testCase.expectedTargetID { + t.Fatalf("test %v: TargetID: expected: %v, got: %v", i+1, testCase.expectedTargetID, targetID) + } + } + } +} diff --git a/internal/config/lambda/event/targetidset.go b/internal/config/lambda/event/targetidset.go new file mode 100644 index 00000000..eea48102 --- /dev/null +++ b/internal/config/lambda/event/targetidset.go @@ -0,0 +1,72 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import "maps" + +// TargetIDSet - Set representation of TargetIDs. +type TargetIDSet map[TargetID]struct{} + +// IsEmpty returns true if the set is empty. +func (set TargetIDSet) IsEmpty() bool { + return len(set) != 0 +} + +// Clone - returns copy of this set. +func (set TargetIDSet) Clone() TargetIDSet { + setCopy := NewTargetIDSet() + maps.Copy(setCopy, set) + return setCopy +} + +// add - adds TargetID to the set. +func (set TargetIDSet) add(targetID TargetID) { + set[targetID] = struct{}{} +} + +// Union - returns union with given set as new set. +func (set TargetIDSet) Union(sset TargetIDSet) TargetIDSet { + nset := set.Clone() + + for k := range sset { + nset.add(k) + } + + return nset +} + +// Difference - returns difference with given set as new set. +func (set TargetIDSet) Difference(sset TargetIDSet) TargetIDSet { + nset := NewTargetIDSet() + for k := range set { + if _, ok := sset[k]; !ok { + nset.add(k) + } + } + + return nset +} + +// NewTargetIDSet - creates new TargetID set with given TargetIDs. +func NewTargetIDSet(targetIDs ...TargetID) TargetIDSet { + set := make(TargetIDSet) + for _, targetID := range targetIDs { + set.add(targetID) + } + return set +} diff --git a/internal/config/lambda/event/targetidset_test.go b/internal/config/lambda/event/targetidset_test.go new file mode 100644 index 00000000..d798992e --- /dev/null +++ b/internal/config/lambda/event/targetidset_test.go @@ -0,0 +1,110 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "reflect" + "testing" +) + +func TestTargetIDSetClone(t *testing.T) { + testCases := []struct { + set TargetIDSet + targetIDToAdd TargetID + }{ + {NewTargetIDSet(), TargetID{"1", "webhook"}}, + {NewTargetIDSet(TargetID{"1", "webhook"}), TargetID{"2", "webhook"}}, + {NewTargetIDSet(TargetID{"1", "webhook"}, TargetID{"2", "amqp"}), TargetID{"2", "webhook"}}, + } + + for i, testCase := range testCases { + result := testCase.set.Clone() + + if !reflect.DeepEqual(result, testCase.set) { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.set, result) + } + + result.add(testCase.targetIDToAdd) + if reflect.DeepEqual(result, testCase.set) { + t.Fatalf("test %v: result: expected: not equal, got: equal", i+1) + } + } +} + +func TestTargetIDSetUnion(t *testing.T) { + testCases := []struct { + set TargetIDSet + setToAdd TargetIDSet + expectedResult TargetIDSet + }{ + {NewTargetIDSet(), NewTargetIDSet(), NewTargetIDSet()}, + {NewTargetIDSet(), NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"1", "webhook"})}, + {NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(), NewTargetIDSet(TargetID{"1", "webhook"})}, + {NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"2", "amqp"}), NewTargetIDSet(TargetID{"1", "webhook"}, TargetID{"2", "amqp"})}, + {NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"1", "webhook"})}, + } + + for i, testCase := range testCases { + result := testCase.set.Union(testCase.setToAdd) + + if !reflect.DeepEqual(testCase.expectedResult, result) { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestTargetIDSetDifference(t *testing.T) { + testCases := []struct { + set TargetIDSet + setToRemove TargetIDSet + expectedResult TargetIDSet + }{ + {NewTargetIDSet(), NewTargetIDSet(), NewTargetIDSet()}, + {NewTargetIDSet(), NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet()}, + {NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(), NewTargetIDSet(TargetID{"1", "webhook"})}, + {NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"2", "amqp"}), NewTargetIDSet(TargetID{"1", "webhook"})}, + {NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet()}, + } + + for i, testCase := range testCases { + result := testCase.set.Difference(testCase.setToRemove) + + if !reflect.DeepEqual(testCase.expectedResult, result) { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestNewTargetIDSet(t *testing.T) { + testCases := []struct { + targetIDs []TargetID + expectedResult TargetIDSet + }{ + {[]TargetID{}, NewTargetIDSet()}, + {[]TargetID{{"1", "webhook"}}, NewTargetIDSet(TargetID{"1", "webhook"})}, + {[]TargetID{{"1", "webhook"}, {"2", "amqp"}}, NewTargetIDSet(TargetID{"1", "webhook"}, TargetID{"2", "amqp"})}, + } + + for i, testCase := range testCases { + result := NewTargetIDSet(testCase.targetIDs...) + + if !reflect.DeepEqual(testCase.expectedResult, result) { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} diff --git a/internal/config/lambda/event/targetlist.go b/internal/config/lambda/event/targetlist.go new file mode 100644 index 00000000..34350318 --- /dev/null +++ b/internal/config/lambda/event/targetlist.go @@ -0,0 +1,188 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "fmt" + "maps" + "net/http" + "strings" + "sync" +) + +// Target - lambda target interface +type Target interface { + ID() TargetID + IsActive() (bool, error) + Send(Event) (*http.Response, error) + Stat() TargetStat + Close() error +} + +// TargetStats is a collection of stats for multiple targets. +type TargetStats struct { + TargetStats map[string]TargetStat +} + +// TargetStat is the stats of a single target. +type TargetStat struct { + ID TargetID + ActiveRequests int64 + TotalRequests int64 + FailedRequests int64 +} + +// TargetList - holds list of targets indexed by target ID. +type TargetList struct { + sync.RWMutex + targets map[TargetID]Target +} + +// Add - adds unique target to target list. +func (list *TargetList) Add(targets ...Target) error { + list.Lock() + defer list.Unlock() + + for _, target := range targets { + if _, ok := list.targets[target.ID()]; ok { + return fmt.Errorf("target %v already exists", target.ID()) + } + list.targets[target.ID()] = target + } + + return nil +} + +// Lookup - checks whether target by target ID exists is valid or not. +func (list *TargetList) Lookup(arnStr string) (Target, error) { + list.RLock() + defer list.RUnlock() + + arn, err := ParseARN(arnStr) + if err != nil { + return nil, err + } + + id, found := list.targets[arn.TargetID] + if !found { + return nil, &ErrARNNotFound{} + } + return id, nil +} + +// TargetIDResult returns result of Remove/Send operation, sets err if +// any for the associated TargetID +type TargetIDResult struct { + // ID where the remove or send were initiated. + ID TargetID + // Stores any error while removing a target or while sending an event. + Err error +} + +// Remove - closes and removes targets by given target IDs. +func (list *TargetList) Remove(targetIDSet TargetIDSet) { + list.Lock() + defer list.Unlock() + + for id := range targetIDSet { + target, ok := list.targets[id] + if ok { + target.Close() + delete(list.targets, id) + } + } +} + +// Targets - list all targets +func (list *TargetList) Targets() []Target { + if list == nil { + return []Target{} + } + + list.RLock() + defer list.RUnlock() + + targets := make([]Target, 0, len(list.targets)) + for _, tgt := range list.targets { + targets = append(targets, tgt) + } + + return targets +} + +// Empty returns true if targetList is empty. +func (list *TargetList) Empty() bool { + list.RLock() + defer list.RUnlock() + + return len(list.targets) == 0 +} + +// List - returns available target IDs. +func (list *TargetList) List(region string) []ARN { + list.RLock() + defer list.RUnlock() + + keys := make([]ARN, 0, len(list.targets)) + for k := range list.targets { + keys = append(keys, k.ToARN(region)) + } + + return keys +} + +// TargetMap - returns available targets. +func (list *TargetList) TargetMap() map[TargetID]Target { + list.RLock() + defer list.RUnlock() + + ntargets := make(map[TargetID]Target, len(list.targets)) + maps.Copy(ntargets, list.targets) + return ntargets +} + +// Send - sends events to targets identified by target IDs. +func (list *TargetList) Send(event Event, id TargetID) (*http.Response, error) { + list.RLock() + target, ok := list.targets[id] + list.RUnlock() + if ok { + return target.Send(event) + } + return nil, ErrARNNotFound{} +} + +// Stats returns stats for targets. +func (list *TargetList) Stats() TargetStats { + t := TargetStats{} + if list == nil { + return t + } + list.RLock() + defer list.RUnlock() + t.TargetStats = make(map[string]TargetStat, len(list.targets)) + for id, target := range list.targets { + t.TargetStats[strings.ReplaceAll(id.String(), ":", "_")] = target.Stat() + } + return t +} + +// NewTargetList - creates TargetList. +func NewTargetList() *TargetList { + return &TargetList{targets: make(map[TargetID]Target)} +} diff --git a/internal/config/lambda/help.go b/internal/config/lambda/help.go new file mode 100644 index 00000000..2f728ad3 --- /dev/null +++ b/internal/config/lambda/help.go @@ -0,0 +1,63 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lambda + +import ( + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/event/target" +) + +// Help template inputs for all lambda targets +var ( + HelpWebhook = config.HelpKVS{ + config.HelpKV{ + Key: target.WebhookEndpoint, + Description: "webhook server endpoint e.g. http://localhost:8080/minio/lambda", + Type: "url", + Sensitive: true, + }, + config.HelpKV{ + Key: target.WebhookAuthToken, + Description: "opaque string or JWT authorization token", + Optional: true, + Type: "string", + Sensitive: true, + Secret: true, + }, + config.HelpKV{ + Key: config.Comment, + Description: config.DefaultComment, + Optional: true, + Type: "sentence", + }, + config.HelpKV{ + Key: target.WebhookClientCert, + Description: "client cert for Webhook mTLS auth", + Optional: true, + Type: "string", + Sensitive: true, + }, + config.HelpKV{ + Key: target.WebhookClientKey, + Description: "client cert key for Webhook mTLS auth", + Optional: true, + Type: "string", + Sensitive: true, + }, + } +) diff --git a/internal/config/lambda/parse.go b/internal/config/lambda/parse.go new file mode 100644 index 00000000..eac6a5de --- /dev/null +++ b/internal/config/lambda/parse.go @@ -0,0 +1,219 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package lambda + +import ( + "context" + "errors" + "fmt" + "net/http" + + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/config/lambda/event" + "github.com/minio/minio/internal/config/lambda/target" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/env" + xnet "github.com/minio/pkg/v3/net" +) + +const ( + logSubsys = "notify" +) + +func logOnceIf(ctx context.Context, err error, id string, errKind ...any) { + logger.LogOnceIf(ctx, logSubsys, err, id, errKind...) +} + +// ErrTargetsOffline - Indicates single/multiple target failures. +var ErrTargetsOffline = errors.New("one or more targets are offline. Please use `mc admin info --json` to check the offline targets") + +// TestSubSysLambdaTargets - tests notification targets of given subsystem +func TestSubSysLambdaTargets(ctx context.Context, cfg config.Config, subSys string, transport *http.Transport) error { + if err := checkValidLambdaKeysForSubSys(subSys, cfg[subSys]); err != nil { + return err + } + + targetList, err := fetchSubSysTargets(ctx, cfg, subSys, transport) + if err != nil { + return err + } + + for _, target := range targetList { + defer target.Close() + } + + for _, target := range targetList { + yes, err := target.IsActive() + if err == nil && !yes { + err = ErrTargetsOffline + } + if err != nil { + return fmt.Errorf("error (%s): %w", target.ID(), err) + } + } + + return nil +} + +func fetchSubSysTargets(ctx context.Context, cfg config.Config, subSys string, transport *http.Transport) (targets []event.Target, err error) { + if err := checkValidLambdaKeysForSubSys(subSys, cfg[subSys]); err != nil { + return nil, err + } + + if subSys == config.LambdaWebhookSubSys { + webhookTargets, err := GetLambdaWebhook(cfg[config.LambdaWebhookSubSys], transport) + if err != nil { + return nil, err + } + for id, args := range webhookTargets { + if !args.Enable { + continue + } + t, err := target.NewWebhookTarget(ctx, id, args, logOnceIf, transport) + if err != nil { + return nil, err + } + targets = append(targets, t) + } + } + return targets, nil +} + +// FetchEnabledTargets - Returns a set of configured TargetList +func FetchEnabledTargets(ctx context.Context, cfg config.Config, transport *http.Transport) (*event.TargetList, error) { + targetList := event.NewTargetList() + for _, subSys := range config.LambdaSubSystems.ToSlice() { + targets, err := fetchSubSysTargets(ctx, cfg, subSys, transport) + if err != nil { + return nil, err + } + for _, t := range targets { + if err = targetList.Add(t); err != nil { + return nil, err + } + } + } + return targetList, nil +} + +// DefaultLambdaKVS - default notification list of kvs. +var ( + DefaultLambdaKVS = map[string]config.KVS{ + config.LambdaWebhookSubSys: DefaultWebhookKVS, + } +) + +// DefaultWebhookKVS - default KV for webhook config +var ( + DefaultWebhookKVS = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOff, + }, + config.KV{ + Key: target.WebhookEndpoint, + Value: "", + }, + config.KV{ + Key: target.WebhookAuthToken, + Value: "", + }, + config.KV{ + Key: target.WebhookClientCert, + Value: "", + }, + config.KV{ + Key: target.WebhookClientKey, + Value: "", + }, + } +) + +func checkValidLambdaKeysForSubSys(subSys string, tgt map[string]config.KVS) error { + validKVS, ok := DefaultLambdaKVS[subSys] + if !ok { + return nil + } + for tname, kv := range tgt { + subSysTarget := subSys + if tname != config.Default { + subSysTarget = subSys + config.SubSystemSeparator + tname + } + if v, ok := kv.Lookup(config.Enable); ok && v == config.EnableOn { + if err := config.CheckValidKeys(subSysTarget, kv, validKVS); err != nil { + return err + } + } + } + return nil +} + +// GetLambdaWebhook - returns a map of registered notification 'webhook' targets +func GetLambdaWebhook(webhookKVS map[string]config.KVS, transport *http.Transport) ( + map[string]target.WebhookArgs, error, +) { + webhookTargets := make(map[string]target.WebhookArgs) + for k, kv := range config.Merge(webhookKVS, target.EnvWebhookEnable, DefaultWebhookKVS) { + enableEnv := target.EnvWebhookEnable + if k != config.Default { + enableEnv = enableEnv + config.Default + k + } + enabled, err := config.ParseBool(env.Get(enableEnv, kv.Get(config.Enable))) + if err != nil { + return nil, err + } + if !enabled { + continue + } + urlEnv := target.EnvWebhookEndpoint + if k != config.Default { + urlEnv = urlEnv + config.Default + k + } + url, err := xnet.ParseHTTPURL(env.Get(urlEnv, kv.Get(target.WebhookEndpoint))) + if err != nil { + return nil, err + } + authEnv := target.EnvWebhookAuthToken + if k != config.Default { + authEnv = authEnv + config.Default + k + } + clientCertEnv := target.EnvWebhookClientCert + if k != config.Default { + clientCertEnv = clientCertEnv + config.Default + k + } + + clientKeyEnv := target.EnvWebhookClientKey + if k != config.Default { + clientKeyEnv = clientKeyEnv + config.Default + k + } + + webhookArgs := target.WebhookArgs{ + Enable: enabled, + Endpoint: *url, + Transport: transport, + AuthToken: env.Get(authEnv, kv.Get(target.WebhookAuthToken)), + ClientCert: env.Get(clientCertEnv, kv.Get(target.WebhookClientCert)), + ClientKey: env.Get(clientKeyEnv, kv.Get(target.WebhookClientKey)), + } + if err = webhookArgs.Validate(); err != nil { + return nil, err + } + webhookTargets[k] = webhookArgs + } + return webhookTargets, nil +} diff --git a/internal/config/lambda/target/lazyinit.go b/internal/config/lambda/target/lazyinit.go new file mode 100644 index 00000000..c7abf38d --- /dev/null +++ b/internal/config/lambda/target/lazyinit.go @@ -0,0 +1,51 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package target + +import ( + "sync" + "sync/atomic" +) + +// Inspired from Golang sync.Once but it is only marked +// initialized when the provided function returns nil. + +type lazyInit struct { + done uint32 + m sync.Mutex +} + +func (l *lazyInit) Do(f func() error) error { + if atomic.LoadUint32(&l.done) == 0 { + return l.doSlow(f) + } + return nil +} + +func (l *lazyInit) doSlow(f func() error) error { + l.m.Lock() + defer l.m.Unlock() + if atomic.LoadUint32(&l.done) == 0 { + if err := f(); err != nil { + return err + } + // Mark as done only when f() is successful + atomic.StoreUint32(&l.done, 1) + } + return nil +} diff --git a/internal/config/lambda/target/webhook.go b/internal/config/lambda/target/webhook.go new file mode 100644 index 00000000..20149f02 --- /dev/null +++ b/internal/config/lambda/target/webhook.go @@ -0,0 +1,246 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package target + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "errors" + "net/http" + "strings" + "sync/atomic" + "syscall" + "time" + + "github.com/minio/minio/internal/config/lambda/event" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/certs" + xnet "github.com/minio/pkg/v3/net" +) + +// Webhook constants +const ( + WebhookEndpoint = "endpoint" + WebhookAuthToken = "auth_token" + WebhookClientCert = "client_cert" + WebhookClientKey = "client_key" + + EnvWebhookEnable = "MINIO_LAMBDA_WEBHOOK_ENABLE" + EnvWebhookEndpoint = "MINIO_LAMBDA_WEBHOOK_ENDPOINT" + EnvWebhookAuthToken = "MINIO_LAMBDA_WEBHOOK_AUTH_TOKEN" + EnvWebhookClientCert = "MINIO_LAMBDA_WEBHOOK_CLIENT_CERT" + EnvWebhookClientKey = "MINIO_LAMBDA_WEBHOOK_CLIENT_KEY" +) + +// WebhookArgs - Webhook target arguments. +type WebhookArgs struct { + Enable bool `json:"enable"` + Endpoint xnet.URL `json:"endpoint"` + AuthToken string `json:"authToken"` + Transport *http.Transport `json:"-"` + ClientCert string `json:"clientCert"` + ClientKey string `json:"clientKey"` +} + +// Validate WebhookArgs fields +func (w WebhookArgs) Validate() error { + if !w.Enable { + return nil + } + if w.Endpoint.IsEmpty() { + return errors.New("endpoint empty") + } + if w.ClientCert != "" && w.ClientKey == "" || w.ClientCert == "" && w.ClientKey != "" { + return errors.New("cert and key must be specified as a pair") + } + return nil +} + +// WebhookTarget - Webhook target. +type WebhookTarget struct { + activeRequests int64 + totalRequests int64 + failedRequests int64 + + lazyInit lazyInit + + id event.TargetID + args WebhookArgs + transport *http.Transport + httpClient *http.Client + loggerOnce logger.LogOnce + cancel context.CancelFunc + cancelCh <-chan struct{} +} + +// ID - returns target ID. +func (target *WebhookTarget) ID() event.TargetID { + return target.id +} + +// IsActive - Return true if target is up and active +func (target *WebhookTarget) IsActive() (bool, error) { + if err := target.init(); err != nil { + return false, err + } + return target.isActive() +} + +// errNotConnected - indicates that the target connection is not active. +var errNotConnected = errors.New("not connected to target server/service") + +func (target *WebhookTarget) isActive() (bool, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + req, err := http.NewRequestWithContext(ctx, http.MethodHead, target.args.Endpoint.String(), nil) + if err != nil { + if xnet.IsNetworkOrHostDown(err, false) { + return false, errNotConnected + } + return false, err + } + tokens := strings.Fields(target.args.AuthToken) + switch len(tokens) { + case 2: + req.Header.Set("Authorization", target.args.AuthToken) + case 1: + req.Header.Set("Authorization", "Bearer "+target.args.AuthToken) + } + + resp, err := target.httpClient.Do(req) + if err != nil { + if xnet.IsNetworkOrHostDown(err, true) { + return false, errNotConnected + } + return false, err + } + xhttp.DrainBody(resp.Body) + // No network failure i.e response from the target means its up + return true, nil +} + +// Stat - returns lambda webhook target statistics such as +// current calls in progress, successfully completed functions +// failed functions. +func (target *WebhookTarget) Stat() event.TargetStat { + return event.TargetStat{ + ID: target.id, + ActiveRequests: atomic.LoadInt64(&target.activeRequests), + TotalRequests: atomic.LoadInt64(&target.totalRequests), + FailedRequests: atomic.LoadInt64(&target.failedRequests), + } +} + +// Send - sends an event to the webhook. +func (target *WebhookTarget) Send(eventData event.Event) (resp *http.Response, err error) { + atomic.AddInt64(&target.activeRequests, 1) + defer atomic.AddInt64(&target.activeRequests, -1) + + atomic.AddInt64(&target.totalRequests, 1) + defer func() { + if err != nil { + atomic.AddInt64(&target.failedRequests, 1) + } + }() + + if err = target.init(); err != nil { + return nil, err + } + + data, err := json.Marshal(eventData) + if err != nil { + return nil, err + } + + req, err := http.NewRequest(http.MethodPost, target.args.Endpoint.String(), bytes.NewReader(data)) + if err != nil { + return nil, err + } + + // Verify if the authToken already contains + // like format, if this is + // already present we can blindly use the + // authToken as is instead of adding 'Bearer' + tokens := strings.Fields(target.args.AuthToken) + switch len(tokens) { + case 2: + req.Header.Set("Authorization", target.args.AuthToken) + case 1: + req.Header.Set("Authorization", "Bearer "+target.args.AuthToken) + } + + req.Header.Set("Content-Type", "application/json") + + return target.httpClient.Do(req) +} + +// Close the target. Will cancel all active requests. +func (target *WebhookTarget) Close() error { + target.cancel() + return nil +} + +func (target *WebhookTarget) init() error { + return target.lazyInit.Do(target.initWebhook) +} + +// Only called from init() +func (target *WebhookTarget) initWebhook() error { + args := target.args + transport := target.transport + + if args.ClientCert != "" && args.ClientKey != "" { + manager, err := certs.NewManager(context.Background(), args.ClientCert, args.ClientKey, tls.LoadX509KeyPair) + if err != nil { + return err + } + manager.ReloadOnSignal(syscall.SIGHUP) // allow reloads upon SIGHUP + transport.TLSClientConfig.GetClientCertificate = manager.GetClientCertificate + } + target.httpClient = &http.Client{Transport: transport} + + yes, err := target.isActive() + if err != nil { + return err + } + if !yes { + return errNotConnected + } + + return nil +} + +// NewWebhookTarget - creates new Webhook target. +func NewWebhookTarget(ctx context.Context, id string, args WebhookArgs, loggerOnce logger.LogOnce, transport *http.Transport) (*WebhookTarget, error) { + ctx, cancel := context.WithCancel(ctx) + + target := &WebhookTarget{ + id: event.TargetID{ID: id, Name: "webhook"}, + args: args, + loggerOnce: loggerOnce, + transport: transport, + cancel: cancel, + cancelCh: ctx.Done(), + } + + return target, nil +} diff --git a/internal/config/legacy.go b/internal/config/legacy.go new file mode 100644 index 00000000..e76bdefe --- /dev/null +++ b/internal/config/legacy.go @@ -0,0 +1,33 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package config + +// One time migration code section + +// SetRegion - One time migration code needed, for migrating from older config to new for server Region. +func SetRegion(c Config, name string) { + if name == "" { + return + } + c[RegionSubSys][Default] = KVS{ + KV{ + Key: RegionName, + Value: name, + }, + } +} diff --git a/internal/config/notify/config.go b/internal/config/notify/config.go new file mode 100644 index 00000000..cfb852c9 --- /dev/null +++ b/internal/config/notify/config.go @@ -0,0 +1,69 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package notify + +import ( + "github.com/minio/minio/internal/event/target" +) + +// Config - notification target configuration structure, holds +// information about various notification targets. +type Config struct { + AMQP map[string]target.AMQPArgs `json:"amqp"` + Elasticsearch map[string]target.ElasticsearchArgs `json:"elasticsearch"` + Kafka map[string]target.KafkaArgs `json:"kafka"` + MQTT map[string]target.MQTTArgs `json:"mqtt"` + MySQL map[string]target.MySQLArgs `json:"mysql"` + NATS map[string]target.NATSArgs `json:"nats"` + NSQ map[string]target.NSQArgs `json:"nsq"` + PostgreSQL map[string]target.PostgreSQLArgs `json:"postgresql"` + Redis map[string]target.RedisArgs `json:"redis"` + Webhook map[string]target.WebhookArgs `json:"webhook"` +} + +const ( + defaultTarget = "1" +) + +// NewConfig - initialize notification config. +func NewConfig() Config { + // Make sure to initialize notification targets + cfg := Config{ + NSQ: make(map[string]target.NSQArgs), + AMQP: make(map[string]target.AMQPArgs), + MQTT: make(map[string]target.MQTTArgs), + NATS: make(map[string]target.NATSArgs), + Redis: make(map[string]target.RedisArgs), + MySQL: make(map[string]target.MySQLArgs), + Kafka: make(map[string]target.KafkaArgs), + Webhook: make(map[string]target.WebhookArgs), + PostgreSQL: make(map[string]target.PostgreSQLArgs), + Elasticsearch: make(map[string]target.ElasticsearchArgs), + } + cfg.NSQ[defaultTarget] = target.NSQArgs{} + cfg.AMQP[defaultTarget] = target.AMQPArgs{} + cfg.MQTT[defaultTarget] = target.MQTTArgs{} + cfg.NATS[defaultTarget] = target.NATSArgs{} + cfg.Redis[defaultTarget] = target.RedisArgs{} + cfg.MySQL[defaultTarget] = target.MySQLArgs{} + cfg.Kafka[defaultTarget] = target.KafkaArgs{} + cfg.Webhook[defaultTarget] = target.WebhookArgs{} + cfg.PostgreSQL[defaultTarget] = target.PostgreSQLArgs{} + cfg.Elasticsearch[defaultTarget] = target.ElasticsearchArgs{} + return cfg +} diff --git a/internal/config/notify/help.go b/internal/config/notify/help.go new file mode 100644 index 00000000..343f46c7 --- /dev/null +++ b/internal/config/notify/help.go @@ -0,0 +1,714 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package notify + +import ( + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/event/target" +) + +const ( + formatComment = `'namespace' reflects current bucket/object list and 'access' reflects a journal of object operations, defaults to 'namespace'` + queueDirComment = `staging dir for undelivered messages e.g. '/home/events'` + queueLimitComment = `maximum limit for undelivered messages, defaults to '100000'` +) + +// Help template inputs for all notification targets +var ( + HelpWebhook = config.HelpKVS{ + config.HelpKV{ + Key: target.WebhookEndpoint, + Description: "webhook server endpoint e.g. http://localhost:8080/minio/events", + Type: "url", + Sensitive: true, + }, + config.HelpKV{ + Key: target.WebhookAuthToken, + Description: "opaque string or JWT authorization token", + Optional: true, + Type: "string", + Sensitive: true, + Secret: true, + }, + config.HelpKV{ + Key: target.WebhookQueueDir, + Description: queueDirComment, + Optional: true, + Type: "path", + }, + config.HelpKV{ + Key: target.WebhookQueueLimit, + Description: queueLimitComment, + Optional: true, + Type: "number", + }, + config.HelpKV{ + Key: config.Comment, + Description: config.DefaultComment, + Optional: true, + Type: "sentence", + }, + config.HelpKV{ + Key: target.WebhookClientCert, + Description: "client cert for Webhook mTLS auth", + Optional: true, + Type: "string", + Sensitive: true, + }, + config.HelpKV{ + Key: target.WebhookClientKey, + Description: "client cert key for Webhook mTLS auth", + Optional: true, + Type: "string", + Sensitive: true, + }, + } + + HelpAMQP = config.HelpKVS{ + config.HelpKV{ + Key: target.AmqpURL, + Description: "AMQP server endpoint e.g. `amqp://myuser:mypassword@localhost:5672`", + Type: "url", + Sensitive: true, + }, + config.HelpKV{ + Key: target.AmqpExchange, + Description: "name of the AMQP exchange", + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: target.AmqpExchangeType, + Description: "AMQP exchange type", + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: target.AmqpRoutingKey, + Description: "routing key for publishing", + Optional: true, + Type: "string", + Sensitive: true, + }, + config.HelpKV{ + Key: target.AmqpMandatory, + Description: "quietly ignore undelivered messages when set to 'off', default is 'on'", + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: target.AmqpDurable, + Description: "persist queue across broker restarts when set to 'on', default is 'off'", + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: target.AmqpNoWait, + Description: "non-blocking message delivery when set to 'on', default is 'off'", + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: target.AmqpInternal, + Description: "set to 'on' for exchange to be not used directly by publishers, but only when bound to other exchanges", + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: target.AmqpAutoDeleted, + Description: "auto delete queue when set to 'on', when there are no consumers", + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: target.AmqpDeliveryMode, + Description: "set to '1' for non-persistent or '2' for persistent queue", + Optional: true, + Type: "number", + }, + config.HelpKV{ + Key: target.AmqpPublisherConfirms, + Description: "enable consumer acknowledgement and publisher confirms, use this along with queue_dir for guaranteed delivery of all events", + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: target.AmqpQueueDir, + Description: queueDirComment, + Optional: true, + Type: "path", + }, + config.HelpKV{ + Key: target.AmqpQueueLimit, + Description: queueLimitComment, + Optional: true, + Type: "number", + }, + config.HelpKV{ + Key: config.Comment, + Description: config.DefaultComment, + Optional: true, + Type: "sentence", + }, + } + + HelpKafka = config.HelpKVS{ + config.HelpKV{ + Key: target.KafkaBrokers, + Description: "comma separated list of Kafka broker addresses", + Type: "csv", + }, + config.HelpKV{ + Key: target.KafkaTopic, + Description: "Kafka topic used for bucket notifications", + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: target.KafkaSASLUsername, + Description: "username for SASL/PLAIN or SASL/SCRAM authentication", + Optional: true, + Type: "string", + Sensitive: true, + }, + config.HelpKV{ + Key: target.KafkaSASLPassword, + Description: "password for SASL/PLAIN or SASL/SCRAM authentication", + Optional: true, + Type: "string", + Sensitive: true, + Secret: true, + }, + config.HelpKV{ + Key: target.KafkaSASLMechanism, + Description: "sasl authentication mechanism, default 'plain'", + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: target.KafkaTLSClientAuth, + Description: "clientAuth determines the Kafka server's policy for TLS client auth", + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: target.KafkaSASL, + Description: "set to 'on' to enable SASL authentication", + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: target.KafkaTLS, + Description: "set to 'on' to enable TLS", + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: target.KafkaTLSSkipVerify, + Description: `trust server TLS without verification, defaults to "on" (verify)`, + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: target.KafkaClientTLSCert, + Description: "path to client certificate for mTLS auth", + Optional: true, + Type: "path", + Sensitive: true, + }, + config.HelpKV{ + Key: target.KafkaClientTLSKey, + Description: "path to client key for mTLS auth", + Optional: true, + Type: "path", + Sensitive: true, + }, + config.HelpKV{ + Key: target.KafkaQueueDir, + Description: queueDirComment, + Optional: true, + Type: "path", + }, + config.HelpKV{ + Key: target.KafkaQueueLimit, + Description: queueLimitComment, + Optional: true, + Type: "number", + }, + config.HelpKV{ + Key: target.KafkaVersion, + Description: "specify the version of the Kafka cluster", + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: config.Comment, + Description: config.DefaultComment, + Optional: true, + Type: "sentence", + }, + config.HelpKV{ + Key: target.KafkaCompressionCodec, + Description: "specify compression_codec of the Kafka cluster", + Optional: true, + Type: "none|snappy|gzip|lz4|zstd", + }, + config.HelpKV{ + Key: target.KafkaCompressionLevel, + Description: "specify compression level of the Kafka cluster", + Optional: true, + Type: "number", + }, + config.HelpKV{ + Key: target.KafkaBatchSize, + Description: "batch size of the events; used only when queue_dir is set", + Optional: true, + Type: "number", + }, + config.HelpKV{ + Key: target.KafkaBatchCommitTimeout, + Description: "commit timeout set for the batch; used only when batch_size > 1", + Optional: true, + Type: "duration", + }, + } + + HelpMQTT = config.HelpKVS{ + config.HelpKV{ + Key: target.MqttBroker, + Description: "MQTT server endpoint e.g. `tcp://localhost:1883`", + Type: "uri", + Sensitive: true, + }, + config.HelpKV{ + Key: target.MqttTopic, + Description: "name of the MQTT topic to publish", + Type: "string", + }, + config.HelpKV{ + Key: target.MqttUsername, + Description: "MQTT username", + Optional: true, + Type: "string", + Sensitive: true, + }, + config.HelpKV{ + Key: target.MqttPassword, + Description: "MQTT password", + Optional: true, + Type: "string", + Sensitive: true, + Secret: true, + }, + config.HelpKV{ + Key: target.MqttQoS, + Description: "set the quality of service priority, defaults to '0'", + Optional: true, + Type: "number", + }, + config.HelpKV{ + Key: target.MqttKeepAliveInterval, + Description: "keep-alive interval for MQTT connections in s,m,h,d", + Optional: true, + Type: "duration", + }, + config.HelpKV{ + Key: target.MqttReconnectInterval, + Description: "reconnect interval for MQTT connections in s,m,h,d", + Optional: true, + Type: "duration", + }, + config.HelpKV{ + Key: target.MqttQueueDir, + Description: queueDirComment, + Optional: true, + Type: "path", + }, + config.HelpKV{ + Key: target.MqttQueueLimit, + Description: queueLimitComment, + Optional: true, + Type: "number", + }, + config.HelpKV{ + Key: config.Comment, + Description: config.DefaultComment, + Optional: true, + Type: "sentence", + }, + } + + HelpPostgres = config.HelpKVS{ + config.HelpKV{ + Key: target.PostgresConnectionString, + Description: `Postgres server connection-string e.g. "host=localhost port=5432 dbname=minio_events user=postgres password=password sslmode=disable"`, + Type: "string", + Sensitive: true, + }, + config.HelpKV{ + Key: target.PostgresTable, + Description: "DB table name to store/update events, table is auto-created", + Type: "string", + }, + config.HelpKV{ + Key: target.PostgresFormat, + Description: formatComment, + Type: "namespace*|access", + }, + config.HelpKV{ + Key: target.PostgresQueueDir, + Description: queueDirComment, + Optional: true, + Type: "path", + }, + config.HelpKV{ + Key: target.PostgresQueueLimit, + Description: queueLimitComment, + Optional: true, + Type: "number", + }, + config.HelpKV{ + Key: config.Comment, + Description: config.DefaultComment, + Optional: true, + Type: "sentence", + }, + config.HelpKV{ + Key: target.PostgresMaxOpenConnections, + Description: "To set the maximum number of open connections to the database. The value is set to `2` by default.", + Optional: true, + Type: "number", + }, + } + + HelpMySQL = config.HelpKVS{ + config.HelpKV{ + Key: target.MySQLDSNString, + Description: `MySQL data-source-name connection string e.g. ":@tcp(:)/"`, + Optional: true, + Type: "string", + Sensitive: true, + }, + config.HelpKV{ + Key: target.MySQLTable, + Description: "DB table name to store/update events, table is auto-created", + Type: "string", + }, + config.HelpKV{ + Key: target.MySQLFormat, + Description: formatComment, + Type: "namespace*|access", + }, + config.HelpKV{ + Key: target.MySQLQueueDir, + Description: queueDirComment, + Optional: true, + Type: "path", + }, + config.HelpKV{ + Key: target.MySQLQueueLimit, + Description: queueLimitComment, + Optional: true, + Type: "number", + }, + config.HelpKV{ + Key: config.Comment, + Description: config.DefaultComment, + Optional: true, + Type: "sentence", + }, + config.HelpKV{ + Key: target.MySQLMaxOpenConnections, + Description: "To set the maximum number of open connections to the database. The value is set to `2` by default.", + Optional: true, + Type: "number", + }, + } + + HelpNATS = config.HelpKVS{ + config.HelpKV{ + Key: target.NATSAddress, + Description: "NATS server address e.g. '0.0.0.0:4222'", + Type: "address", + Sensitive: true, + }, + config.HelpKV{ + Key: target.NATSSubject, + Description: "NATS subscription subject", + Type: "string", + }, + config.HelpKV{ + Key: target.NATSUsername, + Description: "NATS username", + Optional: true, + Type: "string", + Sensitive: true, + }, + config.HelpKV{ + Key: target.NATSPassword, + Description: "NATS password", + Optional: true, + Type: "string", + Sensitive: true, + Secret: true, + }, + config.HelpKV{ + Key: target.NATSToken, + Description: "NATS token", + Optional: true, + Type: "string", + Sensitive: true, + Secret: true, + }, + config.HelpKV{ + Key: target.NATSTLS, + Description: "set to 'on' to enable TLS", + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: target.NATSTLSSkipVerify, + Description: `trust server TLS without verification, defaults to "on" (verify)`, + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: target.NATSPingInterval, + Description: "client ping commands interval in s,m,h,d. Disabled by default", + Optional: true, + Type: "duration", + }, + config.HelpKV{ + Key: target.NATSCertAuthority, + Description: "path to certificate chain of the target NATS server", + Optional: true, + Type: "string", + Sensitive: true, + }, + config.HelpKV{ + Key: target.NATSClientCert, + Description: "client cert for NATS mTLS auth", + Optional: true, + Type: "string", + Sensitive: true, + }, + config.HelpKV{ + Key: target.NATSClientKey, + Description: "client cert key for NATS mTLS auth", + Optional: true, + Type: "string", + Sensitive: true, + }, + config.HelpKV{ + Key: target.NATSJetStream, + Description: "enable JetStream support", + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: target.NATSQueueDir, + Description: queueDirComment, + Optional: true, + Type: "path", + }, + config.HelpKV{ + Key: target.NATSQueueLimit, + Description: queueLimitComment, + Optional: true, + Type: "number", + }, + config.HelpKV{ + Key: target.NATSStreaming, + Description: "[DEPRECATED] set to 'on', to use streaming NATS server", + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: target.NATSStreamingAsync, + Description: "[DEPRECATED] set to 'on', to enable asynchronous publish", + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: target.NATSStreamingMaxPubAcksInFlight, + Description: "[DEPRECATED] number of messages to publish without waiting for ACKs", + Optional: true, + Type: "number", + }, + config.HelpKV{ + Key: target.NATSStreamingClusterID, + Description: "[DEPRECATED] unique ID for NATS streaming cluster", + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: config.Comment, + Description: config.DefaultComment, + Optional: true, + Type: "sentence", + }, + } + + HelpNSQ = config.HelpKVS{ + config.HelpKV{ + Key: target.NSQAddress, + Description: "NSQ server address e.g. '127.0.0.1:4150'", + Type: "address", + Sensitive: true, + }, + config.HelpKV{ + Key: target.NSQTopic, + Description: "NSQ topic", + Type: "string", + }, + config.HelpKV{ + Key: target.NSQTLS, + Description: "set to 'on' to enable TLS", + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: target.NSQTLSSkipVerify, + Description: `trust server TLS without verification, defaults to "on" (verify)`, + Optional: true, + Type: "on|off", + }, + config.HelpKV{ + Key: target.NSQQueueDir, + Description: queueDirComment, + Optional: true, + Type: "path", + }, + config.HelpKV{ + Key: target.NSQQueueLimit, + Description: queueLimitComment, + Optional: true, + Type: "number", + }, + config.HelpKV{ + Key: config.Comment, + Description: config.DefaultComment, + Optional: true, + Type: "sentence", + }, + } + + HelpES = config.HelpKVS{ + config.HelpKV{ + Key: target.ElasticURL, + Description: "Elasticsearch server's address, with optional authentication info", + Type: "url", + Sensitive: true, + }, + config.HelpKV{ + Key: target.ElasticIndex, + Description: `Elasticsearch index to store/update events, index is auto-created`, + Type: "string", + }, + config.HelpKV{ + Key: target.ElasticFormat, + Description: formatComment, + Type: "namespace*|access", + }, + config.HelpKV{ + Key: target.ElasticQueueDir, + Description: queueDirComment, + Optional: true, + Type: "path", + }, + config.HelpKV{ + Key: target.ElasticQueueLimit, + Description: queueLimitComment, + Optional: true, + Type: "number", + }, + config.HelpKV{ + Key: target.ElasticUsername, + Description: "username for Elasticsearch basic-auth", + Optional: true, + Type: "string", + Sensitive: true, + }, + config.HelpKV{ + Key: target.ElasticPassword, + Description: "password for Elasticsearch basic-auth", + Optional: true, + Type: "string", + Sensitive: true, + Secret: true, + }, + config.HelpKV{ + Key: config.Comment, + Description: config.DefaultComment, + Optional: true, + Type: "sentence", + }, + } + + HelpRedis = config.HelpKVS{ + config.HelpKV{ + Key: target.RedisAddress, + Description: "Redis server's address. For example: `localhost:6379`", + Type: "address", + Sensitive: true, + }, + config.HelpKV{ + Key: target.RedisKey, + Description: "Redis key to store/update events, key is auto-created", + Type: "string", + Sensitive: true, + }, + config.HelpKV{ + Key: target.RedisFormat, + Description: formatComment, + Type: "namespace*|access", + }, + config.HelpKV{ + Key: target.RedisPassword, + Description: "Redis server password", + Optional: true, + Type: "string", + Sensitive: true, + Secret: true, + }, + config.HelpKV{ + Key: target.RedisUser, + Description: "Redis server user for the auth", + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: target.RedisQueueDir, + Description: queueDirComment, + Optional: true, + Type: "path", + }, + config.HelpKV{ + Key: target.RedisQueueLimit, + Description: queueLimitComment, + Optional: true, + Type: "number", + }, + config.HelpKV{ + Key: config.Comment, + Description: config.DefaultComment, + Optional: true, + Type: "sentence", + }, + } +) diff --git a/internal/config/notify/legacy.go b/internal/config/notify/legacy.go new file mode 100644 index 00000000..c72aff12 --- /dev/null +++ b/internal/config/notify/legacy.go @@ -0,0 +1,649 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package notify + +import ( + "fmt" + "strconv" + "strings" + + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/event/target" +) + +// SetNotifyKafka - helper for config migration from older config. +func SetNotifyKafka(s config.Config, name string, cfg target.KafkaArgs) error { + if !cfg.Enable { + return nil + } + + if err := cfg.Validate(); err != nil { + return err + } + + s[config.NotifyKafkaSubSys][name] = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOn, + }, + config.KV{ + Key: target.KafkaBrokers, + Value: func() string { + var brokers []string + for _, broker := range cfg.Brokers { + brokers = append(brokers, broker.String()) + } + return strings.Join(brokers, config.ValueSeparator) + }(), + }, + config.KV{ + Key: target.KafkaTopic, + Value: cfg.Topic, + }, + config.KV{ + Key: target.KafkaQueueDir, + Value: cfg.QueueDir, + }, + config.KV{ + Key: target.KafkaClientTLSCert, + Value: cfg.TLS.ClientTLSCert, + }, + config.KV{ + Key: target.KafkaClientTLSKey, + Value: cfg.TLS.ClientTLSKey, + }, + config.KV{ + Key: target.KafkaQueueLimit, + Value: strconv.Itoa(int(cfg.QueueLimit)), + }, + config.KV{ + Key: target.KafkaTLS, + Value: config.FormatBool(cfg.TLS.Enable), + }, + config.KV{ + Key: target.KafkaTLSSkipVerify, + Value: config.FormatBool(cfg.TLS.SkipVerify), + }, + config.KV{ + Key: target.KafkaTLSClientAuth, + Value: strconv.Itoa(int(cfg.TLS.ClientAuth)), + }, + config.KV{ + Key: target.KafkaSASL, + Value: config.FormatBool(cfg.SASL.Enable), + }, + config.KV{ + Key: target.KafkaSASLUsername, + Value: cfg.SASL.User, + }, + config.KV{ + Key: target.KafkaSASLPassword, + Value: cfg.SASL.Password, + }, + config.KV{ + Key: target.KafkaCompressionCodec, + Value: cfg.Producer.Compression, + }, + config.KV{ + Key: target.KafkaCompressionLevel, + Value: strconv.Itoa(cfg.Producer.CompressionLevel), + }, + } + return nil +} + +// SetNotifyAMQP - helper for config migration from older config. +func SetNotifyAMQP(s config.Config, amqpName string, cfg target.AMQPArgs) error { + if !cfg.Enable { + return nil + } + + if err := cfg.Validate(); err != nil { + return err + } + + s[config.NotifyAMQPSubSys][amqpName] = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOn, + }, + config.KV{ + Key: target.AmqpURL, + Value: cfg.URL.String(), + }, + config.KV{ + Key: target.AmqpExchange, + Value: cfg.Exchange, + }, + config.KV{ + Key: target.AmqpRoutingKey, + Value: cfg.RoutingKey, + }, + config.KV{ + Key: target.AmqpExchangeType, + Value: cfg.ExchangeType, + }, + config.KV{ + Key: target.AmqpDeliveryMode, + Value: strconv.Itoa(int(cfg.DeliveryMode)), + }, + config.KV{ + Key: target.AmqpMandatory, + Value: config.FormatBool(cfg.Mandatory), + }, + config.KV{ + Key: target.AmqpInternal, + Value: config.FormatBool(cfg.Immediate), + }, + config.KV{ + Key: target.AmqpDurable, + Value: config.FormatBool(cfg.Durable), + }, + config.KV{ + Key: target.AmqpNoWait, + Value: config.FormatBool(cfg.NoWait), + }, + config.KV{ + Key: target.AmqpAutoDeleted, + Value: config.FormatBool(cfg.AutoDeleted), + }, + config.KV{ + Key: target.AmqpQueueDir, + Value: cfg.QueueDir, + }, + config.KV{ + Key: target.AmqpQueueLimit, + Value: strconv.Itoa(int(cfg.QueueLimit)), + }, + } + + return nil +} + +// SetNotifyES - helper for config migration from older config. +func SetNotifyES(s config.Config, esName string, cfg target.ElasticsearchArgs) error { + if !cfg.Enable { + return nil + } + + if err := cfg.Validate(); err != nil { + return err + } + + s[config.NotifyESSubSys][esName] = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOn, + }, + config.KV{ + Key: target.ElasticFormat, + Value: cfg.Format, + }, + config.KV{ + Key: target.ElasticURL, + Value: cfg.URL.String(), + }, + config.KV{ + Key: target.ElasticIndex, + Value: cfg.Index, + }, + config.KV{ + Key: target.ElasticQueueDir, + Value: cfg.QueueDir, + }, + config.KV{ + Key: target.ElasticQueueLimit, + Value: strconv.Itoa(int(cfg.QueueLimit)), + }, + config.KV{ + Key: target.ElasticUsername, + Value: cfg.Username, + }, + config.KV{ + Key: target.ElasticPassword, + Value: cfg.Password, + }, + } + + return nil +} + +// SetNotifyRedis - helper for config migration from older config. +func SetNotifyRedis(s config.Config, redisName string, cfg target.RedisArgs) error { + if !cfg.Enable { + return nil + } + + if err := cfg.Validate(); err != nil { + return err + } + + s[config.NotifyRedisSubSys][redisName] = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOn, + }, + config.KV{ + Key: target.RedisFormat, + Value: cfg.Format, + }, + config.KV{ + Key: target.RedisAddress, + Value: cfg.Addr.String(), + }, + config.KV{ + Key: target.RedisPassword, + Value: cfg.Password, + }, + config.KV{ + Key: target.RedisUser, + Value: cfg.User, + }, + config.KV{ + Key: target.RedisKey, + Value: cfg.Key, + }, + config.KV{ + Key: target.RedisQueueDir, + Value: cfg.QueueDir, + }, + config.KV{ + Key: target.RedisQueueLimit, + Value: strconv.Itoa(int(cfg.QueueLimit)), + }, + } + + return nil +} + +// SetNotifyWebhook - helper for config migration from older config. +func SetNotifyWebhook(s config.Config, whName string, cfg target.WebhookArgs) error { + if !cfg.Enable { + return nil + } + + if err := cfg.Validate(); err != nil { + return err + } + + s[config.NotifyWebhookSubSys][whName] = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOn, + }, + config.KV{ + Key: target.WebhookEndpoint, + Value: cfg.Endpoint.String(), + }, + config.KV{ + Key: target.WebhookAuthToken, + Value: cfg.AuthToken, + }, + config.KV{ + Key: target.WebhookQueueDir, + Value: cfg.QueueDir, + }, + config.KV{ + Key: target.WebhookQueueLimit, + Value: strconv.Itoa(int(cfg.QueueLimit)), + }, + config.KV{ + Key: target.WebhookClientCert, + Value: cfg.ClientCert, + }, + config.KV{ + Key: target.WebhookClientKey, + Value: cfg.ClientKey, + }, + } + + return nil +} + +// SetNotifyPostgres - helper for config migration from older config. +func SetNotifyPostgres(s config.Config, psqName string, cfg target.PostgreSQLArgs) error { + if !cfg.Enable { + return nil + } + + if err := cfg.Validate(); err != nil { + return err + } + + s[config.NotifyPostgresSubSys][psqName] = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOn, + }, + config.KV{ + Key: target.PostgresFormat, + Value: cfg.Format, + }, + config.KV{ + Key: target.PostgresConnectionString, + Value: cfg.ConnectionString, + }, + config.KV{ + Key: target.PostgresTable, + Value: cfg.Table, + }, + config.KV{ + Key: target.PostgresHost, + Value: cfg.Host.String(), + }, + config.KV{ + Key: target.PostgresPort, + Value: cfg.Port, + }, + config.KV{ + Key: target.PostgresUsername, + Value: cfg.Username, + }, + config.KV{ + Key: target.PostgresPassword, + Value: cfg.Password, + }, + config.KV{ + Key: target.PostgresDatabase, + Value: cfg.Database, + }, + config.KV{ + Key: target.PostgresQueueDir, + Value: cfg.QueueDir, + }, + config.KV{ + Key: target.PostgresQueueLimit, + Value: strconv.Itoa(int(cfg.QueueLimit)), + }, + config.KV{ + Key: target.PostgresMaxOpenConnections, + Value: strconv.Itoa(cfg.MaxOpenConnections), + }, + } + + return nil +} + +// SetNotifyNSQ - helper for config migration from older config. +func SetNotifyNSQ(s config.Config, nsqName string, cfg target.NSQArgs) error { + if !cfg.Enable { + return nil + } + + if err := cfg.Validate(); err != nil { + return err + } + + s[config.NotifyNSQSubSys][nsqName] = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOn, + }, + config.KV{ + Key: target.NSQAddress, + Value: cfg.NSQDAddress.String(), + }, + config.KV{ + Key: target.NSQTopic, + Value: cfg.Topic, + }, + config.KV{ + Key: target.NSQTLS, + Value: config.FormatBool(cfg.TLS.Enable), + }, + config.KV{ + Key: target.NSQTLSSkipVerify, + Value: config.FormatBool(cfg.TLS.SkipVerify), + }, + config.KV{ + Key: target.NSQQueueDir, + Value: cfg.QueueDir, + }, + config.KV{ + Key: target.NSQQueueLimit, + Value: strconv.Itoa(int(cfg.QueueLimit)), + }, + } + + return nil +} + +// SetNotifyNATS - helper for config migration from older config. +func SetNotifyNATS(s config.Config, natsName string, cfg target.NATSArgs) error { + if !cfg.Enable { + return nil + } + + if err := cfg.Validate(); err != nil { + return err + } + + s[config.NotifyNATSSubSys][natsName] = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOn, + }, + config.KV{ + Key: target.NATSAddress, + Value: cfg.Address.String(), + }, + config.KV{ + Key: target.NATSSubject, + Value: cfg.Subject, + }, + config.KV{ + Key: target.NATSUsername, + Value: cfg.Username, + }, + config.KV{ + Key: target.NATSUserCredentials, + Value: cfg.UserCredentials, + }, + config.KV{ + Key: target.NATSPassword, + Value: cfg.Password, + }, + config.KV{ + Key: target.NATSToken, + Value: cfg.Token, + }, + config.KV{ + Key: target.NATSNKeySeed, + Value: cfg.NKeySeed, + }, + config.KV{ + Key: target.NATSCertAuthority, + Value: cfg.CertAuthority, + }, + config.KV{ + Key: target.NATSClientCert, + Value: cfg.ClientCert, + }, + config.KV{ + Key: target.NATSClientKey, + Value: cfg.ClientKey, + }, + config.KV{ + Key: target.NATSTLS, + Value: config.FormatBool(cfg.Secure), + }, + config.KV{ + Key: target.NATSTLSSkipVerify, + Value: config.FormatBool(cfg.Secure), + }, + config.KV{ + Key: target.NATSTLSHandshakeFirst, + Value: config.FormatBool(cfg.TLSHandshakeFirst), + }, + config.KV{ + Key: target.NATSPingInterval, + Value: strconv.FormatInt(cfg.PingInterval, 10), + }, + config.KV{ + Key: target.NATSQueueDir, + Value: cfg.QueueDir, + }, + config.KV{ + Key: target.NATSQueueLimit, + Value: strconv.Itoa(int(cfg.QueueLimit)), + }, + config.KV{ + Key: target.NATSStreaming, + Value: func() string { + if cfg.Streaming.Enable { + return config.EnableOn + } + return config.EnableOff + }(), + }, + config.KV{ + Key: target.NATSStreamingClusterID, + Value: cfg.Streaming.ClusterID, + }, + config.KV{ + Key: target.NATSStreamingAsync, + Value: config.FormatBool(cfg.Streaming.Async), + }, + config.KV{ + Key: target.NATSStreamingMaxPubAcksInFlight, + Value: strconv.Itoa(cfg.Streaming.MaxPubAcksInflight), + }, + } + + return nil +} + +// SetNotifyMySQL - helper for config migration from older config. +func SetNotifyMySQL(s config.Config, sqlName string, cfg target.MySQLArgs) error { + if !cfg.Enable { + return nil + } + + if err := cfg.Validate(); err != nil { + return err + } + + s[config.NotifyMySQLSubSys][sqlName] = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOn, + }, + config.KV{ + Key: target.MySQLFormat, + Value: cfg.Format, + }, + config.KV{ + Key: target.MySQLDSNString, + Value: cfg.DSN, + }, + config.KV{ + Key: target.MySQLTable, + Value: cfg.Table, + }, + config.KV{ + Key: target.MySQLHost, + Value: cfg.Host.String(), + }, + config.KV{ + Key: target.MySQLPort, + Value: cfg.Port, + }, + config.KV{ + Key: target.MySQLUsername, + Value: cfg.User, + }, + config.KV{ + Key: target.MySQLPassword, + Value: cfg.Password, + }, + config.KV{ + Key: target.MySQLDatabase, + Value: cfg.Database, + }, + config.KV{ + Key: target.MySQLQueueDir, + Value: cfg.QueueDir, + }, + config.KV{ + Key: target.MySQLQueueLimit, + Value: strconv.Itoa(int(cfg.QueueLimit)), + }, + config.KV{ + Key: target.MySQLMaxOpenConnections, + Value: strconv.Itoa(cfg.MaxOpenConnections), + }, + } + + return nil +} + +// SetNotifyMQTT - helper for config migration from older config. +func SetNotifyMQTT(s config.Config, mqttName string, cfg target.MQTTArgs) error { + if !cfg.Enable { + return nil + } + + if err := cfg.Validate(); err != nil { + return err + } + + s[config.NotifyMQTTSubSys][mqttName] = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOn, + }, + config.KV{ + Key: target.MqttBroker, + Value: cfg.Broker.String(), + }, + config.KV{ + Key: target.MqttTopic, + Value: cfg.Topic, + }, + config.KV{ + Key: target.MqttQoS, + Value: fmt.Sprintf("%d", cfg.QoS), + }, + config.KV{ + Key: target.MqttUsername, + Value: cfg.User, + }, + config.KV{ + Key: target.MqttPassword, + Value: cfg.Password, + }, + config.KV{ + Key: target.MqttReconnectInterval, + Value: cfg.MaxReconnectInterval.String(), + }, + config.KV{ + Key: target.MqttKeepAliveInterval, + Value: cfg.KeepAlive.String(), + }, + config.KV{ + Key: target.MqttQueueDir, + Value: cfg.QueueDir, + }, + config.KV{ + Key: target.MqttQueueLimit, + Value: strconv.Itoa(int(cfg.QueueLimit)), + }, + } + + return nil +} diff --git a/internal/config/notify/parse.go b/internal/config/notify/parse.go new file mode 100644 index 00000000..b479d0d4 --- /dev/null +++ b/internal/config/notify/parse.go @@ -0,0 +1,1796 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package notify + +import ( + "context" + "crypto/tls" + "crypto/x509" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + "time" + + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/event/target" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/env" + xnet "github.com/minio/pkg/v3/net" + "github.com/rabbitmq/amqp091-go" +) + +const ( + formatNamespace = "namespace" +) + +const ( + logSubsys = "notify" +) + +func logOnceIf(ctx context.Context, err error, id string, errKind ...any) { + logger.LogOnceIf(ctx, logSubsys, err, id, errKind...) +} + +// ErrTargetsOffline - Indicates single/multiple target failures. +var ErrTargetsOffline = errors.New("one or more targets are offline. Please use `mc admin info --json` to check the offline targets") + +// TestSubSysNotificationTargets - tests notification targets of given subsystem +func TestSubSysNotificationTargets(ctx context.Context, cfg config.Config, subSys string, transport *http.Transport) error { + if err := checkValidNotificationKeysForSubSys(subSys, cfg[subSys]); err != nil { + return err + } + + targetList, err := fetchSubSysTargets(ctx, cfg, subSys, transport) + if err != nil { + return err + } + + for _, target := range targetList { + defer target.Close() + } + + tgts, ok := ctx.Value(config.ContextKeyForTargetFromConfig).(map[string]bool) + if !ok { + tgts = make(map[string]bool) + } + for _, target := range targetList { + if tgts[target.ID().ID] { + // When target set should be online + yes, err := target.IsActive() + if err == nil && !yes { + err = ErrTargetsOffline + } + if err != nil { + return fmt.Errorf("error (%s): %w", target.ID(), err) + } + } else { + // Just for call init. + // Ignore target is online or offline + _, _ = target.IsActive() + } + } + + return nil +} + +func fetchSubSysTargets(ctx context.Context, cfg config.Config, subSys string, transport *http.Transport) (targets []event.Target, err error) { + if err := checkValidNotificationKeysForSubSys(subSys, cfg[subSys]); err != nil { + return nil, err + } + + switch subSys { + case config.NotifyAMQPSubSys: + amqpTargets, err := GetNotifyAMQP(cfg[config.NotifyAMQPSubSys]) + if err != nil { + return nil, err + } + for id, args := range amqpTargets { + if !args.Enable { + continue + } + t, err := target.NewAMQPTarget(id, args, logOnceIf) + if err != nil { + return nil, err + } + targets = append(targets, t) + } + case config.NotifyESSubSys: + esTargets, err := GetNotifyES(cfg[config.NotifyESSubSys], transport) + if err != nil { + return nil, err + } + for id, args := range esTargets { + if !args.Enable { + continue + } + t, err := target.NewElasticsearchTarget(id, args, logOnceIf) + if err != nil { + return nil, err + } + targets = append(targets, t) + } + case config.NotifyKafkaSubSys: + kafkaTargets, err := GetNotifyKafka(cfg[config.NotifyKafkaSubSys]) + if err != nil { + return nil, err + } + for id, args := range kafkaTargets { + if !args.Enable { + continue + } + args.TLS.RootCAs = transport.TLSClientConfig.RootCAs + t, err := target.NewKafkaTarget(id, args, logOnceIf) + if err != nil { + return nil, err + } + targets = append(targets, t) + } + + case config.NotifyMQTTSubSys: + mqttTargets, err := GetNotifyMQTT(cfg[config.NotifyMQTTSubSys], transport.TLSClientConfig.RootCAs) + if err != nil { + return nil, err + } + for id, args := range mqttTargets { + if !args.Enable { + continue + } + args.RootCAs = transport.TLSClientConfig.RootCAs + t, err := target.NewMQTTTarget(id, args, logOnceIf) + if err != nil { + return nil, err + } + targets = append(targets, t) + } + case config.NotifyMySQLSubSys: + mysqlTargets, err := GetNotifyMySQL(cfg[config.NotifyMySQLSubSys]) + if err != nil { + return nil, err + } + for id, args := range mysqlTargets { + if !args.Enable { + continue + } + t, err := target.NewMySQLTarget(id, args, logOnceIf) + if err != nil { + return nil, err + } + targets = append(targets, t) + } + case config.NotifyNATSSubSys: + natsTargets, err := GetNotifyNATS(cfg[config.NotifyNATSSubSys], transport.TLSClientConfig.RootCAs) + if err != nil { + return nil, err + } + for id, args := range natsTargets { + if !args.Enable { + continue + } + t, err := target.NewNATSTarget(id, args, logOnceIf) + if err != nil { + return nil, err + } + targets = append(targets, t) + } + case config.NotifyNSQSubSys: + nsqTargets, err := GetNotifyNSQ(cfg[config.NotifyNSQSubSys]) + if err != nil { + return nil, err + } + for id, args := range nsqTargets { + if !args.Enable { + continue + } + t, err := target.NewNSQTarget(id, args, logOnceIf) + if err != nil { + return nil, err + } + targets = append(targets, t) + } + case config.NotifyPostgresSubSys: + postgresTargets, err := GetNotifyPostgres(cfg[config.NotifyPostgresSubSys]) + if err != nil { + return nil, err + } + for id, args := range postgresTargets { + if !args.Enable { + continue + } + t, err := target.NewPostgreSQLTarget(id, args, logOnceIf) + if err != nil { + return nil, err + } + targets = append(targets, t) + } + case config.NotifyRedisSubSys: + redisTargets, err := GetNotifyRedis(cfg[config.NotifyRedisSubSys]) + if err != nil { + return nil, err + } + for id, args := range redisTargets { + if !args.Enable { + continue + } + t, err := target.NewRedisTarget(id, args, logOnceIf) + if err != nil { + return nil, err + } + targets = append(targets, t) + } + case config.NotifyWebhookSubSys: + webhookTargets, err := GetNotifyWebhook(cfg[config.NotifyWebhookSubSys], transport) + if err != nil { + return nil, err + } + for id, args := range webhookTargets { + if !args.Enable { + continue + } + t, err := target.NewWebhookTarget(ctx, id, args, logOnceIf, transport) + if err != nil { + return nil, err + } + targets = append(targets, t) + } + } + return targets, nil +} + +// FetchEnabledTargets - Returns a set of configured TargetList +func FetchEnabledTargets(ctx context.Context, cfg config.Config, transport *http.Transport) (_ *event.TargetList, err error) { + targetList := event.NewTargetList(ctx) + for _, subSys := range config.NotifySubSystems.ToSlice() { + targets, err := fetchSubSysTargets(ctx, cfg, subSys, transport) + if err != nil { + return nil, err + } + for _, t := range targets { + if err = targetList.Add(t); err != nil { + return nil, err + } + } + } + return targetList, nil +} + +// DefaultNotificationKVS - default notification list of kvs. +var ( + DefaultNotificationKVS = map[string]config.KVS{ + config.NotifyAMQPSubSys: DefaultAMQPKVS, + config.NotifyKafkaSubSys: DefaultKafkaKVS, + config.NotifyMQTTSubSys: DefaultMQTTKVS, + config.NotifyMySQLSubSys: DefaultMySQLKVS, + config.NotifyNATSSubSys: DefaultNATSKVS, + config.NotifyNSQSubSys: DefaultNSQKVS, + config.NotifyPostgresSubSys: DefaultPostgresKVS, + config.NotifyRedisSubSys: DefaultRedisKVS, + config.NotifyWebhookSubSys: DefaultWebhookKVS, + config.NotifyESSubSys: DefaultESKVS, + } +) + +func checkValidNotificationKeysForSubSys(subSys string, tgt map[string]config.KVS) error { + validKVS, ok := DefaultNotificationKVS[subSys] + if !ok { + return nil + } + for tname, kv := range tgt { + subSysTarget := subSys + if tname != config.Default { + subSysTarget = subSys + config.SubSystemSeparator + tname + } + if v, ok := kv.Lookup(config.Enable); ok && v == config.EnableOn { + if err := config.CheckValidKeys(subSysTarget, kv, validKVS); err != nil { + return err + } + } + } + return nil +} + +// DefaultKafkaKVS - default KV for kafka target +var ( + DefaultKafkaKVS = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOff, + }, + config.KV{ + Key: target.KafkaTopic, + Value: "", + }, + config.KV{ + Key: target.KafkaBrokers, + Value: "", + }, + config.KV{ + Key: target.KafkaSASLUsername, + Value: "", + }, + config.KV{ + Key: target.KafkaSASLPassword, + Value: "", + }, + config.KV{ + Key: target.KafkaSASLMechanism, + Value: "plain", + }, + config.KV{ + Key: target.KafkaClientTLSCert, + Value: "", + }, + config.KV{ + Key: target.KafkaClientTLSKey, + Value: "", + }, + config.KV{ + Key: target.KafkaTLSClientAuth, + Value: "0", + }, + config.KV{ + Key: target.KafkaSASL, + Value: config.EnableOff, + }, + config.KV{ + Key: target.KafkaTLS, + Value: config.EnableOff, + }, + config.KV{ + Key: target.KafkaTLSSkipVerify, + Value: config.EnableOff, + }, + config.KV{ + Key: target.KafkaQueueLimit, + Value: "0", + }, + config.KV{ + Key: target.KafkaQueueDir, + Value: "", + }, + config.KV{ + Key: target.KafkaVersion, + Value: "", + }, + config.KV{ + Key: target.KafkaBatchSize, + Value: "0", + }, + config.KV{ + Key: target.KafkaBatchCommitTimeout, + Value: "0s", + }, + config.KV{ + Key: target.KafkaCompressionCodec, + Value: "", + }, + config.KV{ + Key: target.KafkaCompressionLevel, + Value: "", + }, + } +) + +// GetNotifyKafka - returns a map of registered notification 'kafka' targets +func GetNotifyKafka(kafkaKVS map[string]config.KVS) (map[string]target.KafkaArgs, error) { + kafkaTargets := make(map[string]target.KafkaArgs) + for k, kv := range config.Merge(kafkaKVS, target.EnvKafkaEnable, DefaultKafkaKVS) { + enableEnv := target.EnvKafkaEnable + if k != config.Default { + enableEnv = enableEnv + config.Default + k + } + enabled, err := config.ParseBool(env.Get(enableEnv, kv.Get(config.Enable))) + if err != nil { + return nil, err + } + if !enabled { + continue + } + var brokers []xnet.Host + brokersEnv := target.EnvKafkaBrokers + if k != config.Default { + brokersEnv = brokersEnv + config.Default + k + } + kafkaBrokers := env.Get(brokersEnv, kv.Get(target.KafkaBrokers)) + if len(kafkaBrokers) == 0 { + return nil, config.Errorf("kafka 'brokers' cannot be empty") + } + for s := range strings.SplitSeq(kafkaBrokers, config.ValueSeparator) { + var host *xnet.Host + host, err = xnet.ParseHost(s) + if err != nil { + break + } + brokers = append(brokers, *host) + } + if err != nil { + return nil, err + } + + queueLimitEnv := target.EnvKafkaQueueLimit + if k != config.Default { + queueLimitEnv = queueLimitEnv + config.Default + k + } + queueLimit, err := strconv.ParseUint(env.Get(queueLimitEnv, kv.Get(target.KafkaQueueLimit)), 10, 64) + if err != nil { + return nil, err + } + + clientAuthEnv := target.EnvKafkaTLSClientAuth + if k != config.Default { + clientAuthEnv = clientAuthEnv + config.Default + k + } + clientAuth, err := strconv.Atoi(env.Get(clientAuthEnv, kv.Get(target.KafkaTLSClientAuth))) + if err != nil { + return nil, err + } + + topicEnv := target.EnvKafkaTopic + if k != config.Default { + topicEnv = topicEnv + config.Default + k + } + + queueDirEnv := target.EnvKafkaQueueDir + if k != config.Default { + queueDirEnv = queueDirEnv + config.Default + k + } + + versionEnv := target.EnvKafkaVersion + if k != config.Default { + versionEnv = versionEnv + config.Default + k + } + + batchSizeEnv := target.EnvKafkaBatchSize + if k != config.Default { + batchSizeEnv = batchSizeEnv + config.Default + k + } + batchSize, err := strconv.ParseUint(env.Get(batchSizeEnv, kv.Get(target.KafkaBatchSize)), 10, 32) + if err != nil { + return nil, err + } + + batchCommitTimeoutEnv := target.EnvKafkaBatchCommitTimeout + if k != config.Default { + batchCommitTimeoutEnv = batchCommitTimeoutEnv + config.Default + k + } + batchCommitTimeout, err := time.ParseDuration(env.Get(batchCommitTimeoutEnv, kv.Get(target.KafkaBatchCommitTimeout))) + if err != nil { + return nil, err + } + kafkaArgs := target.KafkaArgs{ + Enable: enabled, + Brokers: brokers, + Topic: env.Get(topicEnv, kv.Get(target.KafkaTopic)), + QueueDir: env.Get(queueDirEnv, kv.Get(target.KafkaQueueDir)), + QueueLimit: queueLimit, + Version: env.Get(versionEnv, kv.Get(target.KafkaVersion)), + BatchSize: uint32(batchSize), + BatchCommitTimeout: batchCommitTimeout, + } + + tlsEnableEnv := target.EnvKafkaTLS + if k != config.Default { + tlsEnableEnv = tlsEnableEnv + config.Default + k + } + tlsSkipVerifyEnv := target.EnvKafkaTLSSkipVerify + if k != config.Default { + tlsSkipVerifyEnv = tlsSkipVerifyEnv + config.Default + k + } + + tlsClientTLSCertEnv := target.EnvKafkaClientTLSCert + if k != config.Default { + tlsClientTLSCertEnv = tlsClientTLSCertEnv + config.Default + k + } + + tlsClientTLSKeyEnv := target.EnvKafkaClientTLSKey + if k != config.Default { + tlsClientTLSKeyEnv = tlsClientTLSKeyEnv + config.Default + k + } + + kafkaArgs.TLS.Enable = env.Get(tlsEnableEnv, kv.Get(target.KafkaTLS)) == config.EnableOn + kafkaArgs.TLS.SkipVerify = env.Get(tlsSkipVerifyEnv, kv.Get(target.KafkaTLSSkipVerify)) == config.EnableOn + kafkaArgs.TLS.ClientAuth = tls.ClientAuthType(clientAuth) + + kafkaArgs.TLS.ClientTLSCert = env.Get(tlsClientTLSCertEnv, kv.Get(target.KafkaClientTLSCert)) + kafkaArgs.TLS.ClientTLSKey = env.Get(tlsClientTLSKeyEnv, kv.Get(target.KafkaClientTLSKey)) + + compressionCodecEnv := target.EnvKafkaProducerCompressionCodec + if k != config.Default { + compressionCodecEnv = compressionCodecEnv + config.Default + k + } + kafkaArgs.Producer.Compression = env.Get(compressionCodecEnv, kv.Get(target.KafkaCompressionCodec)) + + compressionLevelEnv := target.EnvKafkaProducerCompressionLevel + if k != config.Default { + compressionLevelEnv = compressionLevelEnv + config.Default + k + } + compressionLevel, _ := strconv.Atoi(env.Get(compressionLevelEnv, kv.Get(target.KafkaCompressionLevel))) + kafkaArgs.Producer.CompressionLevel = compressionLevel + + saslEnableEnv := target.EnvKafkaSASLEnable + if k != config.Default { + saslEnableEnv = saslEnableEnv + config.Default + k + } + saslUsernameEnv := target.EnvKafkaSASLUsername + if k != config.Default { + saslUsernameEnv = saslUsernameEnv + config.Default + k + } + saslPasswordEnv := target.EnvKafkaSASLPassword + if k != config.Default { + saslPasswordEnv = saslPasswordEnv + config.Default + k + } + saslMechanismEnv := target.EnvKafkaSASLMechanism + if k != config.Default { + saslMechanismEnv = saslMechanismEnv + config.Default + k + } + kafkaArgs.SASL.Enable = env.Get(saslEnableEnv, kv.Get(target.KafkaSASL)) == config.EnableOn + kafkaArgs.SASL.User = env.Get(saslUsernameEnv, kv.Get(target.KafkaSASLUsername)) + kafkaArgs.SASL.Password = env.Get(saslPasswordEnv, kv.Get(target.KafkaSASLPassword)) + kafkaArgs.SASL.Mechanism = env.Get(saslMechanismEnv, kv.Get(target.KafkaSASLMechanism)) + + if err = kafkaArgs.Validate(); err != nil { + return nil, err + } + + kafkaTargets[k] = kafkaArgs + } + + return kafkaTargets, nil +} + +// DefaultMQTTKVS - default MQTT config +var ( + DefaultMQTTKVS = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOff, + }, + config.KV{ + Key: target.MqttBroker, + Value: "", + }, + config.KV{ + Key: target.MqttTopic, + Value: "", + }, + config.KV{ + Key: target.MqttPassword, + Value: "", + }, + config.KV{ + Key: target.MqttUsername, + Value: "", + }, + config.KV{ + Key: target.MqttQoS, + Value: "0", + }, + config.KV{ + Key: target.MqttKeepAliveInterval, + Value: "0s", + }, + config.KV{ + Key: target.MqttReconnectInterval, + Value: "0s", + }, + config.KV{ + Key: target.MqttQueueDir, + Value: "", + }, + config.KV{ + Key: target.MqttQueueLimit, + Value: "0", + }, + } +) + +// GetNotifyMQTT - returns a map of registered notification 'mqtt' targets +func GetNotifyMQTT(mqttKVS map[string]config.KVS, rootCAs *x509.CertPool) (map[string]target.MQTTArgs, error) { + mqttTargets := make(map[string]target.MQTTArgs) + for k, kv := range config.Merge(mqttKVS, target.EnvMQTTEnable, DefaultMQTTKVS) { + enableEnv := target.EnvMQTTEnable + if k != config.Default { + enableEnv = enableEnv + config.Default + k + } + + enabled, err := config.ParseBool(env.Get(enableEnv, kv.Get(config.Enable))) + if err != nil { + return nil, err + } + if !enabled { + continue + } + + brokerEnv := target.EnvMQTTBroker + if k != config.Default { + brokerEnv = brokerEnv + config.Default + k + } + + brokerURL, err := xnet.ParseURL(env.Get(brokerEnv, kv.Get(target.MqttBroker))) + if err != nil { + return nil, err + } + + reconnectIntervalEnv := target.EnvMQTTReconnectInterval + if k != config.Default { + reconnectIntervalEnv = reconnectIntervalEnv + config.Default + k + } + reconnectInterval, err := time.ParseDuration(env.Get(reconnectIntervalEnv, + kv.Get(target.MqttReconnectInterval))) + if err != nil { + return nil, err + } + + keepAliveIntervalEnv := target.EnvMQTTKeepAliveInterval + if k != config.Default { + keepAliveIntervalEnv = keepAliveIntervalEnv + config.Default + k + } + keepAliveInterval, err := time.ParseDuration(env.Get(keepAliveIntervalEnv, + kv.Get(target.MqttKeepAliveInterval))) + if err != nil { + return nil, err + } + + queueLimitEnv := target.EnvMQTTQueueLimit + if k != config.Default { + queueLimitEnv = queueLimitEnv + config.Default + k + } + queueLimit, err := strconv.ParseUint(env.Get(queueLimitEnv, kv.Get(target.MqttQueueLimit)), 10, 64) + if err != nil { + return nil, err + } + + qosEnv := target.EnvMQTTQoS + if k != config.Default { + qosEnv = qosEnv + config.Default + k + } + + // Parse uint8 value + qos, err := strconv.ParseUint(env.Get(qosEnv, kv.Get(target.MqttQoS)), 10, 8) + if err != nil { + return nil, err + } + + topicEnv := target.EnvMQTTTopic + if k != config.Default { + topicEnv = topicEnv + config.Default + k + } + + usernameEnv := target.EnvMQTTUsername + if k != config.Default { + usernameEnv = usernameEnv + config.Default + k + } + + passwordEnv := target.EnvMQTTPassword + if k != config.Default { + passwordEnv = passwordEnv + config.Default + k + } + + queueDirEnv := target.EnvMQTTQueueDir + if k != config.Default { + queueDirEnv = queueDirEnv + config.Default + k + } + + mqttArgs := target.MQTTArgs{ + Enable: enabled, + Broker: *brokerURL, + Topic: env.Get(topicEnv, kv.Get(target.MqttTopic)), + QoS: byte(qos), + User: env.Get(usernameEnv, kv.Get(target.MqttUsername)), + Password: env.Get(passwordEnv, kv.Get(target.MqttPassword)), + MaxReconnectInterval: reconnectInterval, + KeepAlive: keepAliveInterval, + RootCAs: rootCAs, + QueueDir: env.Get(queueDirEnv, kv.Get(target.MqttQueueDir)), + QueueLimit: queueLimit, + } + + if err = mqttArgs.Validate(); err != nil { + return nil, err + } + mqttTargets[k] = mqttArgs + } + return mqttTargets, nil +} + +// DefaultMySQLKVS - default KV for MySQL +var ( + DefaultMySQLKVS = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOff, + }, + config.KV{ + Key: target.MySQLFormat, + Value: formatNamespace, + }, + config.KV{ + Key: target.MySQLDSNString, + Value: "", + }, + config.KV{ + Key: target.MySQLTable, + Value: "", + }, + config.KV{ + Key: target.MySQLQueueDir, + Value: "", + }, + config.KV{ + Key: target.MySQLQueueLimit, + Value: "0", + }, + config.KV{ + Key: target.MySQLMaxOpenConnections, + Value: "2", + }, + } +) + +// GetNotifyMySQL - returns a map of registered notification 'mysql' targets +func GetNotifyMySQL(mysqlKVS map[string]config.KVS) (map[string]target.MySQLArgs, error) { + mysqlTargets := make(map[string]target.MySQLArgs) + for k, kv := range config.Merge(mysqlKVS, target.EnvMySQLEnable, DefaultMySQLKVS) { + enableEnv := target.EnvMySQLEnable + if k != config.Default { + enableEnv = enableEnv + config.Default + k + } + + enabled, err := config.ParseBool(env.Get(enableEnv, kv.Get(config.Enable))) + if err != nil { + return nil, err + } + if !enabled { + continue + } + + queueLimitEnv := target.EnvMySQLQueueLimit + if k != config.Default { + queueLimitEnv = queueLimitEnv + config.Default + k + } + queueLimit, err := strconv.ParseUint(env.Get(queueLimitEnv, kv.Get(target.MySQLQueueLimit)), 10, 64) + if err != nil { + return nil, err + } + + formatEnv := target.EnvMySQLFormat + if k != config.Default { + formatEnv = formatEnv + config.Default + k + } + + dsnStringEnv := target.EnvMySQLDSNString + if k != config.Default { + dsnStringEnv = dsnStringEnv + config.Default + k + } + + tableEnv := target.EnvMySQLTable + if k != config.Default { + tableEnv = tableEnv + config.Default + k + } + + queueDirEnv := target.EnvMySQLQueueDir + if k != config.Default { + queueDirEnv = queueDirEnv + config.Default + k + } + + maxOpenConnectionsEnv := target.EnvMySQLMaxOpenConnections + if k != config.Default { + maxOpenConnectionsEnv = maxOpenConnectionsEnv + config.Default + k + } + + maxOpenConnections, cErr := strconv.Atoi(env.Get(maxOpenConnectionsEnv, kv.Get(target.MySQLMaxOpenConnections))) + if cErr != nil { + return nil, cErr + } + + mysqlArgs := target.MySQLArgs{ + Enable: enabled, + Format: env.Get(formatEnv, kv.Get(target.MySQLFormat)), + DSN: env.Get(dsnStringEnv, kv.Get(target.MySQLDSNString)), + Table: env.Get(tableEnv, kv.Get(target.MySQLTable)), + QueueDir: env.Get(queueDirEnv, kv.Get(target.MySQLQueueDir)), + QueueLimit: queueLimit, + MaxOpenConnections: maxOpenConnections, + } + if err = mysqlArgs.Validate(); err != nil { + return nil, err + } + mysqlTargets[k] = mysqlArgs + } + return mysqlTargets, nil +} + +// DefaultNATSKVS - NATS KV for nats config. +var ( + DefaultNATSKVS = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOff, + }, + config.KV{ + Key: target.NATSAddress, + Value: "", + }, + config.KV{ + Key: target.NATSSubject, + Value: "", + }, + config.KV{ + Key: target.NATSUsername, + Value: "", + }, + config.KV{ + Key: target.NATSPassword, + Value: "", + }, + config.KV{ + Key: target.NATSToken, + Value: "", + }, + config.KV{ + Key: target.NATSTLS, + Value: config.EnableOff, + }, + config.KV{ + Key: target.NATSTLSSkipVerify, + Value: config.EnableOff, + }, + config.KV{ + Key: target.NATSCertAuthority, + Value: "", + }, + config.KV{ + Key: target.NATSClientCert, + Value: "", + }, + config.KV{ + Key: target.NATSClientKey, + Value: "", + }, + config.KV{ + Key: target.NATSPingInterval, + Value: "0", + }, + config.KV{ + Key: target.NATSJetStream, + Value: config.EnableOff, + }, + config.KV{ + Key: target.NATSStreaming, + Value: config.EnableOff, + HiddenIfEmpty: true, + }, + config.KV{ + Key: target.NATSStreamingAsync, + Value: config.EnableOff, + HiddenIfEmpty: true, + }, + config.KV{ + Key: target.NATSStreamingMaxPubAcksInFlight, + Value: "0", + HiddenIfEmpty: true, + }, + config.KV{ + Key: target.NATSStreamingClusterID, + Value: "", + HiddenIfEmpty: true, + }, + config.KV{ + Key: target.NATSQueueDir, + Value: "", + }, + config.KV{ + Key: target.NATSQueueLimit, + Value: "0", + }, + } +) + +// GetNotifyNATS - returns a map of registered notification 'nats' targets +func GetNotifyNATS(natsKVS map[string]config.KVS, rootCAs *x509.CertPool) (map[string]target.NATSArgs, error) { + natsTargets := make(map[string]target.NATSArgs) + for k, kv := range config.Merge(natsKVS, target.EnvNATSEnable, DefaultNATSKVS) { + enableEnv := target.EnvNATSEnable + if k != config.Default { + enableEnv = enableEnv + config.Default + k + } + + enabled, err := config.ParseBool(env.Get(enableEnv, kv.Get(config.Enable))) + if err != nil { + return nil, err + } + if !enabled { + continue + } + + addressEnv := target.EnvNATSAddress + if k != config.Default { + addressEnv = addressEnv + config.Default + k + } + + address, err := xnet.ParseHost(env.Get(addressEnv, kv.Get(target.NATSAddress))) + if err != nil { + return nil, err + } + + pingIntervalEnv := target.EnvNATSPingInterval + if k != config.Default { + pingIntervalEnv = pingIntervalEnv + config.Default + k + } + + pingInterval, err := strconv.ParseInt(env.Get(pingIntervalEnv, kv.Get(target.NATSPingInterval)), 10, 64) + if err != nil { + return nil, err + } + + queueLimitEnv := target.EnvNATSQueueLimit + if k != config.Default { + queueLimitEnv = queueLimitEnv + config.Default + k + } + + queueLimit, err := strconv.ParseUint(env.Get(queueLimitEnv, kv.Get(target.NATSQueueLimit)), 10, 64) + if err != nil { + return nil, err + } + + tlsEnv := target.EnvNATSTLS + if k != config.Default { + tlsEnv = tlsEnv + config.Default + k + } + + tlsSkipVerifyEnv := target.EnvNATSTLSSkipVerify + if k != config.Default { + tlsSkipVerifyEnv = tlsSkipVerifyEnv + config.Default + k + } + + tlsHandshakeFirstEnv := target.EnvNatsTLSHandshakeFirst + if k != config.Default { + tlsHandshakeFirstEnv = tlsHandshakeFirstEnv + config.Default + k + } + + subjectEnv := target.EnvNATSSubject + if k != config.Default { + subjectEnv = subjectEnv + config.Default + k + } + + usernameEnv := target.EnvNATSUsername + if k != config.Default { + usernameEnv = usernameEnv + config.Default + k + } + + userCredentialsEnv := target.NATSUserCredentials + if k != config.Default { + userCredentialsEnv = userCredentialsEnv + config.Default + k + } + + passwordEnv := target.EnvNATSPassword + if k != config.Default { + passwordEnv = passwordEnv + config.Default + k + } + + tokenEnv := target.EnvNATSToken + if k != config.Default { + tokenEnv = tokenEnv + config.Default + k + } + + nKeySeedEnv := target.EnvNATSNKeySeed + if k != config.Default { + nKeySeedEnv = nKeySeedEnv + config.Default + k + } + + queueDirEnv := target.EnvNATSQueueDir + if k != config.Default { + queueDirEnv = queueDirEnv + config.Default + k + } + + certAuthorityEnv := target.EnvNATSCertAuthority + if k != config.Default { + certAuthorityEnv = certAuthorityEnv + config.Default + k + } + + clientCertEnv := target.EnvNATSClientCert + if k != config.Default { + clientCertEnv = clientCertEnv + config.Default + k + } + + clientKeyEnv := target.EnvNATSClientKey + if k != config.Default { + clientKeyEnv = clientKeyEnv + config.Default + k + } + + jetStreamEnableEnv := target.EnvNATSJetStream + if k != config.Default { + jetStreamEnableEnv = jetStreamEnableEnv + config.Default + k + } + + natsArgs := target.NATSArgs{ + Enable: true, + Address: *address, + Subject: env.Get(subjectEnv, kv.Get(target.NATSSubject)), + Username: env.Get(usernameEnv, kv.Get(target.NATSUsername)), + UserCredentials: env.Get(userCredentialsEnv, kv.Get(target.NATSUserCredentials)), + Password: env.Get(passwordEnv, kv.Get(target.NATSPassword)), + CertAuthority: env.Get(certAuthorityEnv, kv.Get(target.NATSCertAuthority)), + ClientCert: env.Get(clientCertEnv, kv.Get(target.NATSClientCert)), + ClientKey: env.Get(clientKeyEnv, kv.Get(target.NATSClientKey)), + Token: env.Get(tokenEnv, kv.Get(target.NATSToken)), + NKeySeed: env.Get(nKeySeedEnv, kv.Get(target.NATSNKeySeed)), + TLS: env.Get(tlsEnv, kv.Get(target.NATSTLS)) == config.EnableOn, + TLSSkipVerify: env.Get(tlsSkipVerifyEnv, kv.Get(target.NATSTLSSkipVerify)) == config.EnableOn, + TLSHandshakeFirst: env.Get(tlsHandshakeFirstEnv, kv.Get(target.NATSTLSHandshakeFirst)) == config.EnableOn, + PingInterval: pingInterval, + QueueDir: env.Get(queueDirEnv, kv.Get(target.NATSQueueDir)), + QueueLimit: queueLimit, + RootCAs: rootCAs, + } + natsArgs.JetStream.Enable = env.Get(jetStreamEnableEnv, kv.Get(target.NATSJetStream)) == config.EnableOn + + streamingEnableEnv := target.EnvNATSStreaming + if k != config.Default { + streamingEnableEnv = streamingEnableEnv + config.Default + k + } + + streamingEnabled := env.Get(streamingEnableEnv, kv.Get(target.NATSStreaming)) == config.EnableOn + if streamingEnabled { + asyncEnv := target.EnvNATSStreamingAsync + if k != config.Default { + asyncEnv = asyncEnv + config.Default + k + } + maxPubAcksInflightEnv := target.EnvNATSStreamingMaxPubAcksInFlight + if k != config.Default { + maxPubAcksInflightEnv = maxPubAcksInflightEnv + config.Default + k + } + maxPubAcksInflight, err := strconv.Atoi(env.Get(maxPubAcksInflightEnv, + kv.Get(target.NATSStreamingMaxPubAcksInFlight))) + if err != nil { + return nil, err + } + clusterIDEnv := target.EnvNATSStreamingClusterID + if k != config.Default { + clusterIDEnv = clusterIDEnv + config.Default + k + } + natsArgs.Streaming.Enable = streamingEnabled + natsArgs.Streaming.ClusterID = env.Get(clusterIDEnv, kv.Get(target.NATSStreamingClusterID)) + natsArgs.Streaming.Async = env.Get(asyncEnv, kv.Get(target.NATSStreamingAsync)) == config.EnableOn + natsArgs.Streaming.MaxPubAcksInflight = maxPubAcksInflight + } + + if err = natsArgs.Validate(); err != nil { + return nil, err + } + + natsTargets[k] = natsArgs + } + return natsTargets, nil +} + +// DefaultNSQKVS - NSQ KV for config +var ( + DefaultNSQKVS = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOff, + }, + config.KV{ + Key: target.NSQAddress, + Value: "", + }, + config.KV{ + Key: target.NSQTopic, + Value: "", + }, + config.KV{ + Key: target.NSQTLS, + Value: config.EnableOff, + }, + config.KV{ + Key: target.NSQTLSSkipVerify, + Value: config.EnableOff, + }, + config.KV{ + Key: target.NSQQueueDir, + Value: "", + }, + config.KV{ + Key: target.NSQQueueLimit, + Value: "0", + }, + } +) + +// GetNotifyNSQ - returns a map of registered notification 'nsq' targets +func GetNotifyNSQ(nsqKVS map[string]config.KVS) (map[string]target.NSQArgs, error) { + nsqTargets := make(map[string]target.NSQArgs) + for k, kv := range config.Merge(nsqKVS, target.EnvNSQEnable, DefaultNSQKVS) { + enableEnv := target.EnvNSQEnable + if k != config.Default { + enableEnv = enableEnv + config.Default + k + } + + enabled, err := config.ParseBool(env.Get(enableEnv, kv.Get(config.Enable))) + if err != nil { + return nil, err + } + if !enabled { + continue + } + + addressEnv := target.EnvNSQAddress + if k != config.Default { + addressEnv = addressEnv + config.Default + k + } + nsqdAddress, err := xnet.ParseHost(env.Get(addressEnv, kv.Get(target.NSQAddress))) + if err != nil { + return nil, err + } + tlsEnableEnv := target.EnvNSQTLS + if k != config.Default { + tlsEnableEnv = tlsEnableEnv + config.Default + k + } + tlsSkipVerifyEnv := target.EnvNSQTLSSkipVerify + if k != config.Default { + tlsSkipVerifyEnv = tlsSkipVerifyEnv + config.Default + k + } + + queueLimitEnv := target.EnvNSQQueueLimit + if k != config.Default { + queueLimitEnv = queueLimitEnv + config.Default + k + } + queueLimit, err := strconv.ParseUint(env.Get(queueLimitEnv, kv.Get(target.NSQQueueLimit)), 10, 64) + if err != nil { + return nil, err + } + + topicEnv := target.EnvNSQTopic + if k != config.Default { + topicEnv = topicEnv + config.Default + k + } + queueDirEnv := target.EnvNSQQueueDir + if k != config.Default { + queueDirEnv = queueDirEnv + config.Default + k + } + + nsqArgs := target.NSQArgs{ + Enable: enabled, + NSQDAddress: *nsqdAddress, + Topic: env.Get(topicEnv, kv.Get(target.NSQTopic)), + QueueDir: env.Get(queueDirEnv, kv.Get(target.NSQQueueDir)), + QueueLimit: queueLimit, + } + nsqArgs.TLS.Enable = env.Get(tlsEnableEnv, kv.Get(target.NSQTLS)) == config.EnableOn + nsqArgs.TLS.SkipVerify = env.Get(tlsSkipVerifyEnv, kv.Get(target.NSQTLSSkipVerify)) == config.EnableOn + + if err = nsqArgs.Validate(); err != nil { + return nil, err + } + + nsqTargets[k] = nsqArgs + } + return nsqTargets, nil +} + +// DefaultPostgresKVS - default Postgres KV for server config. +var ( + DefaultPostgresKVS = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOff, + }, + config.KV{ + Key: target.PostgresFormat, + Value: formatNamespace, + }, + config.KV{ + Key: target.PostgresConnectionString, + Value: "", + }, + config.KV{ + Key: target.PostgresTable, + Value: "", + }, + config.KV{ + Key: target.PostgresQueueDir, + Value: "", + }, + config.KV{ + Key: target.PostgresQueueLimit, + Value: "0", + }, + config.KV{ + Key: target.PostgresMaxOpenConnections, + Value: "2", + }, + } +) + +// GetNotifyPostgres - returns a map of registered notification 'postgres' targets +func GetNotifyPostgres(postgresKVS map[string]config.KVS) (map[string]target.PostgreSQLArgs, error) { + psqlTargets := make(map[string]target.PostgreSQLArgs) + for k, kv := range config.Merge(postgresKVS, target.EnvPostgresEnable, DefaultPostgresKVS) { + enableEnv := target.EnvPostgresEnable + if k != config.Default { + enableEnv = enableEnv + config.Default + k + } + + enabled, err := config.ParseBool(env.Get(enableEnv, kv.Get(config.Enable))) + if err != nil { + return nil, err + } + if !enabled { + continue + } + + queueLimitEnv := target.EnvPostgresQueueLimit + if k != config.Default { + queueLimitEnv = queueLimitEnv + config.Default + k + } + + queueLimit, err := strconv.Atoi(env.Get(queueLimitEnv, kv.Get(target.PostgresQueueLimit))) + if err != nil { + return nil, err + } + + formatEnv := target.EnvPostgresFormat + if k != config.Default { + formatEnv = formatEnv + config.Default + k + } + + connectionStringEnv := target.EnvPostgresConnectionString + if k != config.Default { + connectionStringEnv = connectionStringEnv + config.Default + k + } + + tableEnv := target.EnvPostgresTable + if k != config.Default { + tableEnv = tableEnv + config.Default + k + } + + queueDirEnv := target.EnvPostgresQueueDir + if k != config.Default { + queueDirEnv = queueDirEnv + config.Default + k + } + + maxOpenConnectionsEnv := target.EnvPostgresMaxOpenConnections + if k != config.Default { + maxOpenConnectionsEnv = maxOpenConnectionsEnv + config.Default + k + } + + maxOpenConnections, cErr := strconv.Atoi(env.Get(maxOpenConnectionsEnv, kv.Get(target.PostgresMaxOpenConnections))) + if cErr != nil { + return nil, cErr + } + + psqlArgs := target.PostgreSQLArgs{ + Enable: enabled, + Format: env.Get(formatEnv, kv.Get(target.PostgresFormat)), + ConnectionString: env.Get(connectionStringEnv, kv.Get(target.PostgresConnectionString)), + Table: env.Get(tableEnv, kv.Get(target.PostgresTable)), + QueueDir: env.Get(queueDirEnv, kv.Get(target.PostgresQueueDir)), + QueueLimit: uint64(queueLimit), + MaxOpenConnections: maxOpenConnections, + } + if err = psqlArgs.Validate(); err != nil { + return nil, err + } + psqlTargets[k] = psqlArgs + } + + return psqlTargets, nil +} + +// DefaultRedisKVS - default KV for redis config +var ( + DefaultRedisKVS = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOff, + }, + config.KV{ + Key: target.RedisFormat, + Value: formatNamespace, + }, + config.KV{ + Key: target.RedisAddress, + Value: "", + }, + config.KV{ + Key: target.RedisKey, + Value: "", + }, + config.KV{ + Key: target.RedisPassword, + Value: "", + }, + config.KV{ + Key: target.RedisUser, + Value: "", + }, + config.KV{ + Key: target.RedisQueueDir, + Value: "", + }, + config.KV{ + Key: target.RedisQueueLimit, + Value: "0", + }, + } +) + +// GetNotifyRedis - returns a map of registered notification 'redis' targets +func GetNotifyRedis(redisKVS map[string]config.KVS) (map[string]target.RedisArgs, error) { + redisTargets := make(map[string]target.RedisArgs) + for k, kv := range config.Merge(redisKVS, target.EnvRedisEnable, DefaultRedisKVS) { + enableEnv := target.EnvRedisEnable + if k != config.Default { + enableEnv = enableEnv + config.Default + k + } + + enabled, err := config.ParseBool(env.Get(enableEnv, kv.Get(config.Enable))) + if err != nil { + return nil, err + } + if !enabled { + continue + } + + addressEnv := target.EnvRedisAddress + if k != config.Default { + addressEnv = addressEnv + config.Default + k + } + addr, err := xnet.ParseHost(env.Get(addressEnv, kv.Get(target.RedisAddress))) + if err != nil { + return nil, err + } + queueLimitEnv := target.EnvRedisQueueLimit + if k != config.Default { + queueLimitEnv = queueLimitEnv + config.Default + k + } + queueLimit, err := strconv.Atoi(env.Get(queueLimitEnv, kv.Get(target.RedisQueueLimit))) + if err != nil { + return nil, err + } + formatEnv := target.EnvRedisFormat + if k != config.Default { + formatEnv = formatEnv + config.Default + k + } + passwordEnv := target.EnvRedisPassword + if k != config.Default { + passwordEnv = passwordEnv + config.Default + k + } + userEnv := target.EnvRedisUser + if k != config.Default { + userEnv = userEnv + config.Default + k + } + keyEnv := target.EnvRedisKey + if k != config.Default { + keyEnv = keyEnv + config.Default + k + } + queueDirEnv := target.EnvRedisQueueDir + if k != config.Default { + queueDirEnv = queueDirEnv + config.Default + k + } + redisArgs := target.RedisArgs{ + Enable: enabled, + Format: env.Get(formatEnv, kv.Get(target.RedisFormat)), + Addr: *addr, + Password: env.Get(passwordEnv, kv.Get(target.RedisPassword)), + User: env.Get(userEnv, kv.Get(target.RedisUser)), + Key: env.Get(keyEnv, kv.Get(target.RedisKey)), + QueueDir: env.Get(queueDirEnv, kv.Get(target.RedisQueueDir)), + QueueLimit: uint64(queueLimit), + } + if err = redisArgs.Validate(); err != nil { + return nil, err + } + redisTargets[k] = redisArgs + } + return redisTargets, nil +} + +// DefaultWebhookKVS - default KV for webhook config +var ( + DefaultWebhookKVS = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOff, + }, + config.KV{ + Key: target.WebhookEndpoint, + Value: "", + }, + config.KV{ + Key: target.WebhookAuthToken, + Value: "", + }, + config.KV{ + Key: target.WebhookQueueLimit, + Value: "0", + }, + config.KV{ + Key: target.WebhookQueueDir, + Value: "", + }, + config.KV{ + Key: target.WebhookClientCert, + Value: "", + }, + config.KV{ + Key: target.WebhookClientKey, + Value: "", + }, + } +) + +// GetNotifyWebhook - returns a map of registered notification 'webhook' targets +func GetNotifyWebhook(webhookKVS map[string]config.KVS, transport *http.Transport) ( + map[string]target.WebhookArgs, error, +) { + webhookTargets := make(map[string]target.WebhookArgs) + for k, kv := range config.Merge(webhookKVS, target.EnvWebhookEnable, DefaultWebhookKVS) { + enableEnv := target.EnvWebhookEnable + if k != config.Default { + enableEnv = enableEnv + config.Default + k + } + enabled, err := config.ParseBool(env.Get(enableEnv, kv.Get(config.Enable))) + if err != nil { + return nil, err + } + if !enabled { + continue + } + urlEnv := target.EnvWebhookEndpoint + if k != config.Default { + urlEnv = urlEnv + config.Default + k + } + url, err := xnet.ParseHTTPURL(env.Get(urlEnv, kv.Get(target.WebhookEndpoint))) + if err != nil { + return nil, err + } + queueLimitEnv := target.EnvWebhookQueueLimit + if k != config.Default { + queueLimitEnv = queueLimitEnv + config.Default + k + } + queueLimit, err := strconv.Atoi(env.Get(queueLimitEnv, kv.Get(target.WebhookQueueLimit))) + if err != nil { + return nil, err + } + queueDirEnv := target.EnvWebhookQueueDir + if k != config.Default { + queueDirEnv = queueDirEnv + config.Default + k + } + authEnv := target.EnvWebhookAuthToken + if k != config.Default { + authEnv = authEnv + config.Default + k + } + clientCertEnv := target.EnvWebhookClientCert + if k != config.Default { + clientCertEnv = clientCertEnv + config.Default + k + } + + clientKeyEnv := target.EnvWebhookClientKey + if k != config.Default { + clientKeyEnv = clientKeyEnv + config.Default + k + } + + webhookArgs := target.WebhookArgs{ + Enable: enabled, + Endpoint: *url, + Transport: transport, + AuthToken: env.Get(authEnv, kv.Get(target.WebhookAuthToken)), + QueueDir: env.Get(queueDirEnv, kv.Get(target.WebhookQueueDir)), + QueueLimit: uint64(queueLimit), + ClientCert: env.Get(clientCertEnv, kv.Get(target.WebhookClientCert)), + ClientKey: env.Get(clientKeyEnv, kv.Get(target.WebhookClientKey)), + } + if err = webhookArgs.Validate(); err != nil { + return nil, err + } + webhookTargets[k] = webhookArgs + } + return webhookTargets, nil +} + +// DefaultESKVS - default KV config for Elasticsearch target +var ( + DefaultESKVS = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOff, + }, + config.KV{ + Key: target.ElasticURL, + Value: "", + }, + config.KV{ + Key: target.ElasticFormat, + Value: formatNamespace, + }, + config.KV{ + Key: target.ElasticIndex, + Value: "", + }, + config.KV{ + Key: target.ElasticQueueDir, + Value: "", + }, + config.KV{ + Key: target.ElasticQueueLimit, + Value: "0", + }, + config.KV{ + Key: target.ElasticUsername, + Value: "", + }, + config.KV{ + Key: target.ElasticPassword, + Value: "", + }, + } +) + +// GetNotifyES - returns a map of registered notification 'elasticsearch' targets +func GetNotifyES(esKVS map[string]config.KVS, transport *http.Transport) (map[string]target.ElasticsearchArgs, error) { + esTargets := make(map[string]target.ElasticsearchArgs) + for k, kv := range config.Merge(esKVS, target.EnvElasticEnable, DefaultESKVS) { + enableEnv := target.EnvElasticEnable + if k != config.Default { + enableEnv = enableEnv + config.Default + k + } + enabled, err := config.ParseBool(env.Get(enableEnv, kv.Get(config.Enable))) + if err != nil { + return nil, err + } + if !enabled { + continue + } + + urlEnv := target.EnvElasticURL + if k != config.Default { + urlEnv = urlEnv + config.Default + k + } + + url, err := xnet.ParseHTTPURL(env.Get(urlEnv, kv.Get(target.ElasticURL))) + if err != nil { + return nil, err + } + + queueLimitEnv := target.EnvElasticQueueLimit + if k != config.Default { + queueLimitEnv = queueLimitEnv + config.Default + k + } + + queueLimit, err := strconv.Atoi(env.Get(queueLimitEnv, kv.Get(target.ElasticQueueLimit))) + if err != nil { + return nil, err + } + + formatEnv := target.EnvElasticFormat + if k != config.Default { + formatEnv = formatEnv + config.Default + k + } + + indexEnv := target.EnvElasticIndex + if k != config.Default { + indexEnv = indexEnv + config.Default + k + } + + queueDirEnv := target.EnvElasticQueueDir + if k != config.Default { + queueDirEnv = queueDirEnv + config.Default + k + } + + usernameEnv := target.EnvElasticUsername + if k != config.Default { + usernameEnv = usernameEnv + config.Default + k + } + + passwordEnv := target.EnvElasticPassword + if k != config.Default { + passwordEnv = passwordEnv + config.Default + k + } + + esArgs := target.ElasticsearchArgs{ + Enable: enabled, + Format: env.Get(formatEnv, kv.Get(target.ElasticFormat)), + URL: *url, + Index: env.Get(indexEnv, kv.Get(target.ElasticIndex)), + QueueDir: env.Get(queueDirEnv, kv.Get(target.ElasticQueueDir)), + QueueLimit: uint64(queueLimit), + Transport: transport, + Username: env.Get(usernameEnv, kv.Get(target.ElasticUsername)), + Password: env.Get(passwordEnv, kv.Get(target.ElasticPassword)), + } + if err = esArgs.Validate(); err != nil { + return nil, err + } + esTargets[k] = esArgs + } + return esTargets, nil +} + +// DefaultAMQPKVS - default KV for AMQP config +var ( + DefaultAMQPKVS = config.KVS{ + config.KV{ + Key: config.Enable, + Value: config.EnableOff, + }, + config.KV{ + Key: target.AmqpURL, + Value: "", + }, + config.KV{ + Key: target.AmqpExchange, + Value: "", + }, + config.KV{ + Key: target.AmqpExchangeType, + Value: "", + }, + config.KV{ + Key: target.AmqpRoutingKey, + Value: "", + }, + config.KV{ + Key: target.AmqpMandatory, + Value: config.EnableOff, + }, + config.KV{ + Key: target.AmqpDurable, + Value: config.EnableOff, + }, + config.KV{ + Key: target.AmqpNoWait, + Value: config.EnableOff, + }, + config.KV{ + Key: target.AmqpInternal, + Value: config.EnableOff, + }, + config.KV{ + Key: target.AmqpAutoDeleted, + Value: config.EnableOff, + }, + config.KV{ + Key: target.AmqpDeliveryMode, + Value: "0", + }, + config.KV{ + Key: target.AmqpPublisherConfirms, + Value: config.EnableOff, + }, + config.KV{ + Key: target.AmqpQueueLimit, + Value: "0", + }, + config.KV{ + Key: target.AmqpQueueDir, + Value: "", + }, + } +) + +// GetNotifyAMQP - returns a map of registered notification 'amqp' targets +func GetNotifyAMQP(amqpKVS map[string]config.KVS) (map[string]target.AMQPArgs, error) { + amqpTargets := make(map[string]target.AMQPArgs) + for k, kv := range config.Merge(amqpKVS, target.EnvAMQPEnable, DefaultAMQPKVS) { + enableEnv := target.EnvAMQPEnable + if k != config.Default { + enableEnv = enableEnv + config.Default + k + } + enabled, err := config.ParseBool(env.Get(enableEnv, kv.Get(config.Enable))) + if err != nil { + return nil, err + } + if !enabled { + continue + } + urlEnv := target.EnvAMQPURL + if k != config.Default { + urlEnv = urlEnv + config.Default + k + } + url, err := amqp091.ParseURI(env.Get(urlEnv, kv.Get(target.AmqpURL))) + if err != nil { + return nil, err + } + deliveryModeEnv := target.EnvAMQPDeliveryMode + if k != config.Default { + deliveryModeEnv = deliveryModeEnv + config.Default + k + } + deliveryMode, err := strconv.Atoi(env.Get(deliveryModeEnv, kv.Get(target.AmqpDeliveryMode))) + if err != nil { + return nil, err + } + exchangeEnv := target.EnvAMQPExchange + if k != config.Default { + exchangeEnv = exchangeEnv + config.Default + k + } + routingKeyEnv := target.EnvAMQPRoutingKey + if k != config.Default { + routingKeyEnv = routingKeyEnv + config.Default + k + } + exchangeTypeEnv := target.EnvAMQPExchangeType + if k != config.Default { + exchangeTypeEnv = exchangeTypeEnv + config.Default + k + } + mandatoryEnv := target.EnvAMQPMandatory + if k != config.Default { + mandatoryEnv = mandatoryEnv + config.Default + k + } + immediateEnv := target.EnvAMQPImmediate + if k != config.Default { + immediateEnv = immediateEnv + config.Default + k + } + durableEnv := target.EnvAMQPDurable + if k != config.Default { + durableEnv = durableEnv + config.Default + k + } + internalEnv := target.EnvAMQPInternal + if k != config.Default { + internalEnv = internalEnv + config.Default + k + } + noWaitEnv := target.EnvAMQPNoWait + if k != config.Default { + noWaitEnv = noWaitEnv + config.Default + k + } + autoDeletedEnv := target.EnvAMQPAutoDeleted + if k != config.Default { + autoDeletedEnv = autoDeletedEnv + config.Default + k + } + publisherConfirmsEnv := target.EnvAMQPPublisherConfirms + if k != config.Default { + publisherConfirmsEnv = publisherConfirmsEnv + config.Default + k + } + queueDirEnv := target.EnvAMQPQueueDir + if k != config.Default { + queueDirEnv = queueDirEnv + config.Default + k + } + queueLimitEnv := target.EnvAMQPQueueLimit + if k != config.Default { + queueLimitEnv = queueLimitEnv + config.Default + k + } + queueLimit, err := strconv.ParseUint(env.Get(queueLimitEnv, kv.Get(target.AmqpQueueLimit)), 10, 64) + if err != nil { + return nil, err + } + amqpArgs := target.AMQPArgs{ + Enable: enabled, + URL: url, + Exchange: env.Get(exchangeEnv, kv.Get(target.AmqpExchange)), + RoutingKey: env.Get(routingKeyEnv, kv.Get(target.AmqpRoutingKey)), + ExchangeType: env.Get(exchangeTypeEnv, kv.Get(target.AmqpExchangeType)), + DeliveryMode: uint8(deliveryMode), + Mandatory: env.Get(mandatoryEnv, kv.Get(target.AmqpMandatory)) == config.EnableOn, + Immediate: env.Get(immediateEnv, kv.Get(target.AmqpImmediate)) == config.EnableOn, + Durable: env.Get(durableEnv, kv.Get(target.AmqpDurable)) == config.EnableOn, + Internal: env.Get(internalEnv, kv.Get(target.AmqpInternal)) == config.EnableOn, + NoWait: env.Get(noWaitEnv, kv.Get(target.AmqpNoWait)) == config.EnableOn, + AutoDeleted: env.Get(autoDeletedEnv, kv.Get(target.AmqpAutoDeleted)) == config.EnableOn, + PublisherConfirms: env.Get(publisherConfirmsEnv, kv.Get(target.AmqpPublisherConfirms)) == config.EnableOn, + QueueDir: env.Get(queueDirEnv, kv.Get(target.AmqpQueueDir)), + QueueLimit: queueLimit, + } + if err = amqpArgs.Validate(); err != nil { + return nil, err + } + amqpTargets[k] = amqpArgs + } + return amqpTargets, nil +} diff --git a/internal/config/policy/opa/config.go b/internal/config/policy/opa/config.go new file mode 100644 index 00000000..47185fb8 --- /dev/null +++ b/internal/config/policy/opa/config.go @@ -0,0 +1,230 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package opa + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + + "github.com/minio/minio/internal/config" + "github.com/minio/pkg/v3/env" + xnet "github.com/minio/pkg/v3/net" + "github.com/minio/pkg/v3/policy" +) + +// Env IAM OPA URL +const ( + URL = "url" + AuthToken = "auth_token" + + EnvPolicyOpaURL = "MINIO_POLICY_OPA_URL" + EnvPolicyOpaAuthToken = "MINIO_POLICY_OPA_AUTH_TOKEN" +) + +// DefaultKVS - default config for OPA config +var ( + DefaultKVS = config.KVS{ + config.KV{ + Key: URL, + Value: "", + HiddenIfEmpty: true, + }, + config.KV{ + Key: AuthToken, + Value: "", + HiddenIfEmpty: true, + }, + } +) + +// Args opa general purpose policy engine configuration. +type Args struct { + URL *xnet.URL `json:"url"` + AuthToken string `json:"authToken"` + Transport http.RoundTripper `json:"-"` + CloseRespFn func(r io.ReadCloser) `json:"-"` +} + +// Validate - validate opa configuration params. +func (a *Args) Validate() error { + req, err := http.NewRequest(http.MethodPost, a.URL.String(), bytes.NewReader([]byte(""))) + if err != nil { + return err + } + + req.Header.Set("Content-Type", "application/json") + if a.AuthToken != "" { + req.Header.Set("Authorization", a.AuthToken) + } + + client := &http.Client{Transport: a.Transport} + resp, err := client.Do(req) + if err != nil { + return err + } + defer a.CloseRespFn(resp.Body) + + return nil +} + +// UnmarshalJSON - decodes JSON data. +func (a *Args) UnmarshalJSON(data []byte) error { + // subtype to avoid recursive call to UnmarshalJSON() + type subArgs Args + var so subArgs + + if err := json.Unmarshal(data, &so); err != nil { + return err + } + + oa := Args(so) + if oa.URL == nil || oa.URL.String() == "" { + *a = oa + return nil + } + + *a = oa + return nil +} + +// Opa - implements opa policy agent calls. +type Opa struct { + args Args + client *http.Client +} + +// Enabled returns if opa is enabled. +func Enabled(kvs config.KVS) bool { + return kvs.Get(URL) != "" +} + +// LookupConfig lookup Opa from config, override with any ENVs. +func LookupConfig(kv config.KVS, transport *http.Transport, closeRespFn func(io.ReadCloser)) (Args, error) { + args := Args{} + + if err := config.CheckValidKeys(config.PolicyOPASubSys, kv, DefaultKVS); err != nil { + return args, err + } + + opaURL := env.Get(EnvIamOpaURL, "") + if opaURL == "" { + opaURL = env.Get(EnvPolicyOpaURL, kv.Get(URL)) + if opaURL == "" { + return args, nil + } + } + authToken := env.Get(EnvIamOpaAuthToken, "") + if authToken == "" { + authToken = env.Get(EnvPolicyOpaAuthToken, kv.Get(AuthToken)) + } + + u, err := xnet.ParseHTTPURL(opaURL) + if err != nil { + return args, err + } + args = Args{ + URL: u, + AuthToken: authToken, + Transport: transport, + CloseRespFn: closeRespFn, + } + if err = args.Validate(); err != nil { + return args, err + } + return args, nil +} + +// New - initializes opa policy engine connector. +func New(args Args) *Opa { + // No opa args. + if args.URL == nil || args.URL.Scheme == "" && args.AuthToken == "" { + return nil + } + return &Opa{ + args: args, + client: &http.Client{Transport: args.Transport}, + } +} + +// IsAllowed - checks given policy args is allowed to continue the REST API. +func (o *Opa) IsAllowed(args policy.Args) (bool, error) { + if o == nil { + return false, nil + } + + // OPA input + body := make(map[string]any) + body["input"] = args + + inputBytes, err := json.Marshal(body) + if err != nil { + return false, err + } + + req, err := http.NewRequest(http.MethodPost, o.args.URL.String(), bytes.NewReader(inputBytes)) + if err != nil { + return false, err + } + + req.Header.Set("Content-Type", "application/json") + if o.args.AuthToken != "" { + req.Header.Set("Authorization", o.args.AuthToken) + } + + resp, err := o.client.Do(req) + if err != nil { + return false, err + } + defer o.args.CloseRespFn(resp.Body) + + // Read the body to be saved later. + opaRespBytes, err := io.ReadAll(resp.Body) + if err != nil { + return false, err + } + + // Handle large OPA responses when OPA URL is of + // form http://localhost:8181/v1/data/httpapi/authz + type opaResultAllow struct { + Result struct { + Allow bool `json:"allow"` + } `json:"result"` + } + + // Handle simpler OPA responses when OPA URL is of + // form http://localhost:8181/v1/data/httpapi/authz/allow + type opaResult struct { + Result bool `json:"result"` + } + + respBody := bytes.NewReader(opaRespBytes) + + var result opaResult + if err = json.NewDecoder(respBody).Decode(&result); err != nil { + respBody.Seek(0, 0) + var resultAllow opaResultAllow + if err = json.NewDecoder(respBody).Decode(&resultAllow); err != nil { + return false, err + } + return resultAllow.Result.Allow, nil + } + + return result.Result, nil +} diff --git a/internal/config/policy/opa/help.go b/internal/config/policy/opa/help.go new file mode 100644 index 00000000..c0825b65 --- /dev/null +++ b/internal/config/policy/opa/help.go @@ -0,0 +1,50 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package opa + +import "github.com/minio/minio/internal/config" + +// Help template for OPA policy feature. +var ( + defaultHelpPostfix = func(key string) string { + return config.DefaultHelpPostfix(DefaultKVS, key) + } + + Help = config.HelpKVS{ + config.HelpKV{ + Key: URL, + Description: `[DEPRECATED] OPA HTTP(s) endpoint e.g. "http://localhost:8181/v1/data/httpapi/authz/allow"` + defaultHelpPostfix(URL), + Type: "url", + Sensitive: true, + }, + config.HelpKV{ + Key: AuthToken, + Description: "[DEPRECATED] authorization token for OPA endpoint" + defaultHelpPostfix(AuthToken), + Optional: true, + Type: "string", + Sensitive: true, + Secret: true, + }, + config.HelpKV{ + Key: config.Comment, + Description: config.DefaultComment, + Optional: true, + Type: "sentence", + }, + } +) diff --git a/internal/config/policy/opa/legacy.go b/internal/config/policy/opa/legacy.go new file mode 100644 index 00000000..b0055e53 --- /dev/null +++ b/internal/config/policy/opa/legacy.go @@ -0,0 +1,46 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package opa + +import ( + "github.com/minio/minio/internal/config" +) + +// Legacy OPA envs +const ( + EnvIamOpaURL = "MINIO_IAM_OPA_URL" + EnvIamOpaAuthToken = "MINIO_IAM_OPA_AUTHTOKEN" +) + +// SetPolicyOPAConfig - One time migration code needed, for migrating from older config to new for PolicyOPAConfig. +func SetPolicyOPAConfig(s config.Config, opaArgs Args) { + if opaArgs.URL == nil || opaArgs.URL.String() == "" { + // Do not enable if opaArgs was empty. + return + } + s[config.PolicyOPASubSys][config.Default] = config.KVS{ + config.KV{ + Key: URL, + Value: opaArgs.URL.String(), + }, + config.KV{ + Key: AuthToken, + Value: opaArgs.AuthToken, + }, + } +} diff --git a/internal/config/policy/plugin/config.go b/internal/config/policy/plugin/config.go new file mode 100644 index 00000000..6e651c5a --- /dev/null +++ b/internal/config/policy/plugin/config.go @@ -0,0 +1,245 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package plugin + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "time" + + "github.com/minio/minio/internal/config" + xhttp "github.com/minio/minio/internal/http" + xnet "github.com/minio/pkg/v3/net" + "github.com/minio/pkg/v3/policy" +) + +// Authorization Plugin config and env variables +const ( + URL = "url" + AuthToken = "auth_token" + EnableHTTP2 = "enable_http2" + + EnvPolicyPluginURL = "MINIO_POLICY_PLUGIN_URL" + EnvPolicyPluginAuthToken = "MINIO_POLICY_PLUGIN_AUTH_TOKEN" + EnvPolicyPluginEnableHTTP2 = "MINIO_POLICY_PLUGIN_ENABLE_HTTP2" +) + +// DefaultKVS - default config for Authz plugin config +var ( + DefaultKVS = config.KVS{ + config.KV{ + Key: URL, + Value: "", + }, + config.KV{ + Key: AuthToken, + Value: "", + }, + config.KV{ + Key: EnableHTTP2, + Value: "off", + }, + } +) + +// Args for general purpose policy engine configuration. +type Args struct { + URL *xnet.URL `json:"url"` + AuthToken string `json:"authToken"` + Transport http.RoundTripper `json:"-"` + CloseRespFn func(r io.ReadCloser) `json:"-"` +} + +// Validate - validate opa configuration params. +func (a *Args) Validate() error { + req, err := http.NewRequest(http.MethodPost, a.URL.String(), bytes.NewReader([]byte(""))) + if err != nil { + return err + } + + req.Header.Set("Content-Type", "application/json") + if a.AuthToken != "" { + req.Header.Set("Authorization", a.AuthToken) + } + + client := &http.Client{Transport: a.Transport} + resp, err := client.Do(req) + if err != nil { + return err + } + defer a.CloseRespFn(resp.Body) + + return nil +} + +// UnmarshalJSON - decodes JSON data. +func (a *Args) UnmarshalJSON(data []byte) error { + // subtype to avoid recursive call to UnmarshalJSON() + type subArgs Args + var so subArgs + + if err := json.Unmarshal(data, &so); err != nil { + return err + } + + oa := Args(so) + if oa.URL == nil || oa.URL.String() == "" { + *a = oa + return nil + } + + *a = oa + return nil +} + +// AuthZPlugin - implements opa policy agent calls. +type AuthZPlugin struct { + args Args + client *http.Client +} + +// Enabled returns if AuthZPlugin is enabled. +func Enabled(kvs config.KVS) bool { + return kvs.Get(URL) != "" +} + +// LookupConfig lookup AuthZPlugin from config, override with any ENVs. +func LookupConfig(s config.Config, httpSettings xhttp.ConnSettings, closeRespFn func(io.ReadCloser)) (Args, error) { + args := Args{} + + if err := s.CheckValidKeys(config.PolicyPluginSubSys, nil); err != nil { + return args, err + } + + getCfg := func(cfgParam string) string { + // As parameters are already validated, we skip checking + // if the config param was found. + val, _, _ := s.ResolveConfigParam(config.PolicyPluginSubSys, config.Default, cfgParam, false) + return val + } + + pluginURL := getCfg(URL) + if pluginURL == "" { + return args, nil + } + + u, err := xnet.ParseHTTPURL(pluginURL) + if err != nil { + return args, err + } + + enableHTTP2 := false + if v := getCfg(EnableHTTP2); v != "" { + enableHTTP2, err = config.ParseBool(v) + if err != nil { + return args, err + } + } + httpSettings.EnableHTTP2 = enableHTTP2 + transport := httpSettings.NewHTTPTransportWithTimeout(time.Minute) + + args = Args{ + URL: u, + AuthToken: getCfg(AuthToken), + Transport: transport, + CloseRespFn: closeRespFn, + } + if err = args.Validate(); err != nil { + return args, err + } + return args, nil +} + +// New - initializes Authorization Management Plugin. +func New(args Args) *AuthZPlugin { + if args.URL == nil || args.URL.Scheme == "" && args.AuthToken == "" { + return nil + } + return &AuthZPlugin{ + args: args, + client: &http.Client{Transport: args.Transport}, + } +} + +// IsAllowed - checks given policy args is allowed to continue the REST API. +func (o *AuthZPlugin) IsAllowed(args policy.Args) (bool, error) { + if o == nil { + return false, nil + } + + // Access Management Plugin Input + body := make(map[string]any) + body["input"] = args + + inputBytes, err := json.Marshal(body) + if err != nil { + return false, err + } + + req, err := http.NewRequest(http.MethodPost, o.args.URL.String(), bytes.NewReader(inputBytes)) + if err != nil { + return false, err + } + + req.Header.Set("Content-Type", "application/json") + if o.args.AuthToken != "" { + req.Header.Set("Authorization", o.args.AuthToken) + } + + resp, err := o.client.Do(req) + if err != nil { + return false, err + } + defer o.args.CloseRespFn(resp.Body) + + // Read the body to be saved later. + opaRespBytes, err := io.ReadAll(resp.Body) + if err != nil { + return false, err + } + + // Handle large OPA responses when OPA URL is of + // form http://localhost:8181/v1/data/httpapi/authz + type opaResultAllow struct { + Result struct { + Allow bool `json:"allow"` + } `json:"result"` + } + + // Handle simpler OPA responses when OPA URL is of + // form http://localhost:8181/v1/data/httpapi/authz/allow + type opaResult struct { + Result bool `json:"result"` + } + + respBody := bytes.NewReader(opaRespBytes) + + var result opaResult + if err = json.NewDecoder(respBody).Decode(&result); err != nil { + respBody.Seek(0, 0) + var resultAllow opaResultAllow + if err = json.NewDecoder(respBody).Decode(&resultAllow); err != nil { + return false, err + } + return resultAllow.Result.Allow, nil + } + + return result.Result, nil +} diff --git a/internal/config/policy/plugin/help.go b/internal/config/policy/plugin/help.go new file mode 100644 index 00000000..182de61e --- /dev/null +++ b/internal/config/policy/plugin/help.go @@ -0,0 +1,56 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package plugin + +import "github.com/minio/minio/internal/config" + +// Help template for Access Management Plugin policy feature. +var ( + defaultHelpPostfix = func(key string) string { + return config.DefaultHelpPostfix(DefaultKVS, key) + } + + Help = config.HelpKVS{ + config.HelpKV{ + Key: URL, + Description: `plugin hook endpoint (HTTP(S)) e.g. "http://localhost:8181/v1/data/httpapi/authz/allow"` + defaultHelpPostfix(URL), + Type: "url", + Sensitive: true, + }, + config.HelpKV{ + Key: AuthToken, + Description: "authorization header for plugin hook endpoint" + defaultHelpPostfix(AuthToken), + Optional: true, + Type: "string", + Sensitive: true, + Secret: true, + }, + config.HelpKV{ + Key: EnableHTTP2, + Description: "Enable experimental HTTP2 support to connect to plugin service" + defaultHelpPostfix(EnableHTTP2), + Optional: true, + Type: "bool", + }, + config.HelpKV{ + Key: config.Comment, + Description: config.DefaultComment, + Optional: true, + Type: "sentence", + }, + } +) diff --git a/internal/config/scanner/help.go b/internal/config/scanner/help.go new file mode 100644 index 00000000..363ccaed --- /dev/null +++ b/internal/config/scanner/help.go @@ -0,0 +1,48 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package scanner + +import "github.com/minio/minio/internal/config" + +var ( + defaultHelpPostfix = func(key string) string { + return config.DefaultHelpPostfix(DefaultKVS, key) + } + + // Help provides help for config values + Help = config.HelpKVS{ + config.HelpKV{ + Key: Speed, + Description: `customize scanner speed (default|slowest|slow|fast|fastest)` + defaultHelpPostfix(Speed), + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: ExcessVersions, + Description: `alert per object beyond this many versions` + defaultHelpPostfix(ExcessVersions), + Optional: true, + Type: "int", + }, + config.HelpKV{ + Key: ExcessFolders, + Description: `alert beyond this many sub-folders per folder in an erasure set` + defaultHelpPostfix(ExcessFolders), + Optional: true, + Type: "int", + }, + } +) diff --git a/internal/config/scanner/scanner.go b/internal/config/scanner/scanner.go new file mode 100644 index 00000000..7d171413 --- /dev/null +++ b/internal/config/scanner/scanner.go @@ -0,0 +1,202 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package scanner + +import ( + "fmt" + "strconv" + "time" + + "github.com/minio/minio/internal/config" + "github.com/minio/pkg/v3/env" +) + +// Compression environment variables +const ( + Speed = "speed" + EnvSpeed = "MINIO_SCANNER_SPEED" + + IdleSpeed = "idle_speed" + EnvIdleSpeed = "MINIO_SCANNER_IDLE_SPEED" + + ExcessVersions = "alert_excess_versions" + EnvExcessVersions = "MINIO_SCANNER_ALERT_EXCESS_VERSIONS" + + ExcessFolders = "alert_excess_folders" + EnvExcessFolders = "MINIO_SCANNER_ALERT_EXCESS_FOLDERS" + + // All below are deprecated in October 2022 and + // replaced them with a single speed parameter + Delay = "delay" + MaxWait = "max_wait" + Cycle = "cycle" + EnvDelay = "MINIO_SCANNER_DELAY" + EnvCycle = "MINIO_SCANNER_CYCLE" + EnvDelayLegacy = "MINIO_CRAWLER_DELAY" + EnvMaxWait = "MINIO_SCANNER_MAX_WAIT" + EnvMaxWaitLegacy = "MINIO_CRAWLER_MAX_WAIT" +) + +// Config represents the heal settings. +type Config struct { + // Delay is the sleep multiplier. + Delay float64 `json:"delay"` + + // Sleep always or based on incoming S3 requests. + IdleMode int32 // 0 => on, 1 => off + + // Alert upon this many excess object versions + ExcessVersions int64 // 100 + + // Alert upon this many excess sub-folders per folder in an erasure set. + ExcessFolders int64 // 50000 + + // MaxWait is maximum wait time between operations + MaxWait time.Duration + // Cycle is the time.Duration between each scanner cycles + Cycle time.Duration +} + +// DefaultKVS - default KV config for heal settings +var DefaultKVS = config.KVS{ + config.KV{ + Key: Speed, + Value: "default", + }, + config.KV{ + Key: IdleSpeed, + Value: "", + HiddenIfEmpty: true, + }, + config.KV{ + Key: ExcessVersions, + Value: "100", + }, + config.KV{ + Key: ExcessFolders, + Value: "50000", + }, + + // Deprecated Oct 2022 + config.KV{ + Key: Delay, + Value: "", + HiddenIfEmpty: true, + }, + // Deprecated Oct 2022 + config.KV{ + Key: MaxWait, + Value: "", + HiddenIfEmpty: true, + }, + // Deprecated Oct 2022 + config.KV{ + Key: Cycle, + Value: "", + HiddenIfEmpty: true, + }, +} + +// LookupConfig - lookup config and override with valid environment settings if any. +func LookupConfig(kvs config.KVS) (cfg Config, err error) { + cfg = Config{ + ExcessVersions: 100, + ExcessFolders: 50000, + IdleMode: 0, // Default is on + } + + if err = config.CheckValidKeys(config.ScannerSubSys, kvs, DefaultKVS); err != nil { + return cfg, err + } + + excessVersions, err := strconv.ParseInt(env.Get(EnvExcessVersions, kvs.GetWithDefault(ExcessVersions, DefaultKVS)), 10, 64) + if err != nil { + return cfg, err + } + cfg.ExcessVersions = excessVersions + + excessFolders, err := strconv.ParseInt(env.Get(EnvExcessFolders, kvs.GetWithDefault(ExcessFolders, DefaultKVS)), 10, 64) + if err != nil { + return cfg, err + } + cfg.ExcessFolders = excessFolders + + switch idleSpeed := env.Get(EnvIdleSpeed, kvs.GetWithDefault(IdleSpeed, DefaultKVS)); idleSpeed { + case "", config.EnableOn: + cfg.IdleMode = 0 + case config.EnableOff: + cfg.IdleMode = 1 + default: + return cfg, fmt.Errorf("unknown value: '%s'", idleSpeed) + } + + // Stick to loading deprecated config/env if they are already set, and the Speed value + // has not been changed from its "default" value, if it has been changed honor new settings. + if kvs.GetWithDefault(Speed, DefaultKVS) == "default" { + if kvs.Get(Delay) != "" && kvs.Get(MaxWait) != "" { + if err = lookupDeprecatedScannerConfig(kvs, &cfg); err != nil { + return cfg, err + } + } + } + + switch speed := env.Get(EnvSpeed, kvs.GetWithDefault(Speed, DefaultKVS)); speed { + case "fastest": + cfg.Delay, cfg.MaxWait, cfg.Cycle = 0, 0, time.Second + case "fast": + cfg.Delay, cfg.MaxWait, cfg.Cycle = 1, 100*time.Millisecond, time.Minute + case "default": + cfg.Delay, cfg.MaxWait, cfg.Cycle = 2, time.Second, time.Minute + case "slow": + cfg.Delay, cfg.MaxWait, cfg.Cycle = 10, 15*time.Second, time.Minute + case "slowest": + cfg.Delay, cfg.MaxWait, cfg.Cycle = 100, 15*time.Second, 30*time.Minute + default: + return cfg, fmt.Errorf("unknown '%s' value", speed) + } + + return cfg, nil +} + +func lookupDeprecatedScannerConfig(kvs config.KVS, cfg *Config) (err error) { + delay := env.Get(EnvDelayLegacy, "") + if delay == "" { + delay = env.Get(EnvDelay, kvs.GetWithDefault(Delay, DefaultKVS)) + } + cfg.Delay, err = strconv.ParseFloat(delay, 64) + if err != nil { + return err + } + maxWait := env.Get(EnvMaxWaitLegacy, "") + if maxWait == "" { + maxWait = env.Get(EnvMaxWait, kvs.GetWithDefault(MaxWait, DefaultKVS)) + } + cfg.MaxWait, err = time.ParseDuration(maxWait) + if err != nil { + return err + } + cycle := env.Get(EnvCycle, kvs.GetWithDefault(Cycle, DefaultKVS)) + if cycle == "" { + cycle = "1m" + } + cfg.Cycle, err = time.ParseDuration(cycle) + if err != nil { + return err + } + return nil +} diff --git a/internal/config/server.go b/internal/config/server.go new file mode 100644 index 00000000..e9f750f4 --- /dev/null +++ b/internal/config/server.go @@ -0,0 +1,62 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package config + +// Opts holds MinIO configuration options +type Opts struct { + FTP struct { + Address string `yaml:"address"` + PassivePortRange string `yaml:"passive-port-range"` + } `yaml:"ftp"` + SFTP struct { + Address string `yaml:"address"` + SSHPrivateKey string `yaml:"ssh-private-key"` + } `yaml:"sftp"` +} + +// ServerConfigVersion struct is used to extract the version +type ServerConfigVersion struct { + Version string `yaml:"version"` +} + +// ServerConfigCommon struct for server config common options +type ServerConfigCommon struct { + RootUser string `yaml:"rootUser"` + RootPwd string `yaml:"rootPassword"` + Addr string `yaml:"address"` + ConsoleAddr string `yaml:"console-address"` + CertsDir string `yaml:"certs-dir"` + Options Opts `yaml:"options"` +} + +// ServerConfigV1 represents a MinIO configuration file v1 +type ServerConfigV1 struct { + ServerConfigVersion + ServerConfigCommon + Pools [][]string `yaml:"pools"` +} + +// ServerConfig represents a MinIO configuration file +type ServerConfig struct { + ServerConfigVersion + ServerConfigCommon + Pools []struct { + Args []string `yaml:"args"` + SetDriveCount uint64 `yaml:"set-drive-count"` + } `yaml:"pools"` +} diff --git a/internal/config/storageclass/help.go b/internal/config/storageclass/help.go new file mode 100644 index 00000000..0b57085e --- /dev/null +++ b/internal/config/storageclass/help.go @@ -0,0 +1,54 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package storageclass + +import "github.com/minio/minio/internal/config" + +// Help template for storageclass feature. +var ( + defaultHelpPostfix = func(key string) string { + return config.DefaultHelpPostfix(DefaultKVS, key) + } + + Help = config.HelpKVS{ + config.HelpKV{ + Key: ClassStandard, + Description: `set the parity count for default standard storage class` + defaultHelpPostfix(ClassStandard), + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: ClassRRS, + Description: `set the parity count for reduced redundancy storage class` + defaultHelpPostfix(ClassRRS), + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: Optimize, + Description: `optimize parity calculation for standard storage class, set 'capacity' for capacity optimized (no additional parity)` + defaultHelpPostfix(Optimize), + Optional: true, + Type: "string", + }, + config.HelpKV{ + Key: config.Comment, + Description: config.DefaultComment, + Optional: true, + Type: "sentence", + }, + } +) diff --git a/internal/config/storageclass/legacy.go b/internal/config/storageclass/legacy.go new file mode 100644 index 00000000..447e9762 --- /dev/null +++ b/internal/config/storageclass/legacy.go @@ -0,0 +1,40 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package storageclass + +import ( + "github.com/minio/minio/internal/config" +) + +// SetStorageClass - One time migration code needed, for migrating from older config to new for StorageClass. +func SetStorageClass(s config.Config, cfg Config) { + if len(cfg.Standard.String()) == 0 && len(cfg.RRS.String()) == 0 { + // Do not enable storage-class if no settings found. + return + } + s[config.StorageClassSubSys][config.Default] = config.KVS{ + config.KV{ + Key: ClassStandard, + Value: cfg.Standard.String(), + }, + config.KV{ + Key: ClassRRS, + Value: cfg.RRS.String(), + }, + } +} diff --git a/internal/config/storageclass/storage-class.go b/internal/config/storageclass/storage-class.go new file mode 100644 index 00000000..18121141 --- /dev/null +++ b/internal/config/storageclass/storage-class.go @@ -0,0 +1,435 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package storageclass + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + "strings" + "sync" + + "github.com/dustin/go-humanize" + "github.com/minio/minio/internal/config" + "github.com/minio/minio/internal/logger" + "github.com/minio/pkg/v3/env" +) + +// Standard constants for all storage class +const ( + // Reduced redundancy storage class + RRS = "REDUCED_REDUNDANCY" + // Standard storage class + STANDARD = "STANDARD" +) + +// Standard constants for config info storage class +const ( + ClassStandard = "standard" + ClassRRS = "rrs" + Optimize = "optimize" + InlineBlock = "inline_block" + + // Reduced redundancy storage class environment variable + RRSEnv = "MINIO_STORAGE_CLASS_RRS" + // Standard storage class environment variable + StandardEnv = "MINIO_STORAGE_CLASS_STANDARD" + // Optimize storage class environment variable + OptimizeEnv = "MINIO_STORAGE_CLASS_OPTIMIZE" + // Inline block indicates the size of the shard + // that is considered for inlining, remember this + // shard value is the value per drive shard it + // will vary based on the parity that is configured + // for the STANDARD storage_class. + // inlining means data and metadata are written + // together in a single file i.e xl.meta + InlineBlockEnv = "MINIO_STORAGE_CLASS_INLINE_BLOCK" + + // Supported storage class scheme is EC + schemePrefix = "EC" + + // Min parity drives + minParityDrives = 0 + + // Default RRS parity is always minimum parity. + defaultRRSParity = 1 +) + +// DefaultKVS - default storage class config +var ( + DefaultKVS = config.KVS{ + config.KV{ + Key: ClassStandard, + Value: "", + }, + config.KV{ + Key: ClassRRS, + Value: "EC:1", + }, + config.KV{ + Key: Optimize, + Value: "availability", + }, + config.KV{ + Key: InlineBlock, + Value: "", + HiddenIfEmpty: true, + }, + } +) + +// StorageClass - holds storage class information +type StorageClass struct { + Parity int +} + +// ConfigLock is a global lock for storage-class config +var ConfigLock sync.RWMutex + +// Config storage class configuration +type Config struct { + Standard StorageClass `json:"standard"` + RRS StorageClass `json:"rrs"` + Optimize string `json:"optimize"` + inlineBlock int64 + + initialized bool +} + +// UnmarshalJSON - Validate SS and RRS parity when unmarshalling JSON. +func (sCfg *Config) UnmarshalJSON(data []byte) error { + type Alias Config + aux := &struct { + *Alias + }{ + Alias: (*Alias)(sCfg), + } + return json.Unmarshal(data, &aux) +} + +// IsValid - returns true if input string is a valid +// storage class kind supported. +func IsValid(sc string) bool { + return sc == RRS || sc == STANDARD +} + +// UnmarshalText unmarshals storage class from its textual form into +// storageClass structure. +func (sc *StorageClass) UnmarshalText(b []byte) error { + scStr := string(b) + if scStr == "" { + return nil + } + s, err := parseStorageClass(scStr) + if err != nil { + return err + } + sc.Parity = s.Parity + return nil +} + +// MarshalText - marshals storage class string. +func (sc *StorageClass) MarshalText() ([]byte, error) { + if sc.Parity != 0 { + return fmt.Appendf(nil, "%s:%d", schemePrefix, sc.Parity), nil + } + return []byte{}, nil +} + +func (sc *StorageClass) String() string { + if sc.Parity != 0 { + return fmt.Sprintf("%s:%d", schemePrefix, sc.Parity) + } + return "" +} + +// Parses given storageClassEnv and returns a storageClass structure. +// Supported Storage Class format is "Scheme:Number of parity drives". +// Currently only supported scheme is "EC". +func parseStorageClass(storageClassEnv string) (sc StorageClass, err error) { + s := strings.Split(storageClassEnv, ":") + + // only two elements allowed in the string - "scheme" and "number of parity drives" + if len(s) > 2 { + return StorageClass{}, config.ErrStorageClassValue(nil).Msg("Too many sections in " + storageClassEnv) + } else if len(s) < 2 { + return StorageClass{}, config.ErrStorageClassValue(nil).Msg("Too few sections in " + storageClassEnv) + } + + // only allowed scheme is "EC" + if s[0] != schemePrefix { + return StorageClass{}, config.ErrStorageClassValue(nil).Msg("Unsupported scheme " + s[0] + ". Supported scheme is EC") + } + + // Number of parity drives should be integer + parityDrives, err := strconv.Atoi(s[1]) + if err != nil { + return StorageClass{}, config.ErrStorageClassValue(err) + } + if parityDrives < 0 { + return StorageClass{}, config.ErrStorageClassValue(nil).Msg("Unsupported parity value " + s[1] + " provided") + } + return StorageClass{ + Parity: parityDrives, + }, nil +} + +// ValidateParity validate standard storage class parity. +func ValidateParity(ssParity, setDriveCount int) error { + // SS parity drives should be greater than or equal to minParityDrives. + // Parity below minParityDrives is not supported. + if ssParity > 0 && ssParity < minParityDrives { + return fmt.Errorf("parity %d should be greater than or equal to %d", + ssParity, minParityDrives) + } + + if ssParity > setDriveCount/2 { + return fmt.Errorf("parity %d should be less than or equal to %d", ssParity, setDriveCount/2) + } + + return nil +} + +// Validates the parity drives. +func validateParity(ssParity, rrsParity, setDriveCount int) (err error) { + // SS parity drives should be greater than or equal to minParityDrives. + // Parity below minParityDrives is not supported. + if ssParity > 0 && ssParity < minParityDrives { + return fmt.Errorf("Standard storage class parity %d should be greater than or equal to %d", + ssParity, minParityDrives) + } + + // RRS parity drives should be greater than or equal to minParityDrives. + // Parity below minParityDrives is not supported. + if rrsParity > 0 && rrsParity < minParityDrives { + return fmt.Errorf("Reduced redundancy storage class parity %d should be greater than or equal to %d", rrsParity, minParityDrives) + } + + if setDriveCount > 2 { + if ssParity > setDriveCount/2 { + return fmt.Errorf("Standard storage class parity %d should be less than or equal to %d", ssParity, setDriveCount/2) + } + + if rrsParity > setDriveCount/2 { + return fmt.Errorf("Reduced redundancy storage class parity %d should be less than or equal to %d", rrsParity, setDriveCount/2) + } + } + + if ssParity > 0 && rrsParity > 0 { + if ssParity < rrsParity { + return fmt.Errorf("Standard storage class parity drives %d should be greater than or equal to Reduced redundancy storage class parity drives %d", ssParity, rrsParity) + } + } + return nil +} + +// GetParityForSC - Returns the data and parity drive count based on storage class +// If storage class is set using the env vars MINIO_STORAGE_CLASS_RRS and +// MINIO_STORAGE_CLASS_STANDARD or server config fields corresponding values are +// returned. +// +// -- if input storage class is empty then standard is assumed +// +// -- if input is RRS but RRS is not configured/initialized '-1' parity +// +// for RRS is assumed, the caller is expected to choose the right parity +// at that point. +// +// -- if input is STANDARD but STANDARD is not configured/initialized '-1' parity +// +// is returned, the caller is expected to choose the right parity +// at that point. +func (sCfg *Config) GetParityForSC(sc string) (parity int) { + ConfigLock.RLock() + defer ConfigLock.RUnlock() + switch strings.TrimSpace(sc) { + case RRS: + if !sCfg.initialized { + return -1 + } + return sCfg.RRS.Parity + default: + if !sCfg.initialized { + return -1 + } + return sCfg.Standard.Parity + } +} + +// ShouldInline returns true if the shardSize is worthy of inline +// if versioned is true then we chosen 1/8th inline block size +// to satisfy the same constraints. +func (sCfg *Config) ShouldInline(shardSize int64, versioned bool) bool { + if shardSize < 0 { + return false + } + + ConfigLock.RLock() + inlineBlock := int64(128 * humanize.KiByte) + if sCfg.initialized { + inlineBlock = sCfg.inlineBlock + } + ConfigLock.RUnlock() + + if versioned { + return shardSize <= inlineBlock/8 + } + return shardSize <= inlineBlock +} + +// InlineBlock indicates the size of the block which will be used to inline +// an erasure shard and written along with xl.meta on the drive, on a versioned +// bucket this value is automatically chosen to 1/8th of the this value, make +// sure to put this into consideration when choosing this value. +func (sCfg *Config) InlineBlock() int64 { + ConfigLock.RLock() + defer ConfigLock.RUnlock() + if !sCfg.initialized { + return 128 * humanize.KiByte + } + return sCfg.inlineBlock +} + +// CapacityOptimized - returns true if the storage-class is capacity optimized +// meaning we will not use additional parities when drives are offline. +// +// Default is "availability" optimized, unless this is configured. +func (sCfg *Config) CapacityOptimized() bool { + ConfigLock.RLock() + defer ConfigLock.RUnlock() + if !sCfg.initialized { + return false + } + return sCfg.Optimize == "capacity" +} + +// AvailabilityOptimized - returns true if the storage-class is availability +// optimized, meaning we will use additional parities when drives are offline +// to retain parity SLA. +// +// Default is "availability" optimized. +func (sCfg *Config) AvailabilityOptimized() bool { + ConfigLock.RLock() + defer ConfigLock.RUnlock() + if !sCfg.initialized { + return true + } + return sCfg.Optimize == "availability" || sCfg.Optimize == "" +} + +// Update update storage-class with new config +func (sCfg *Config) Update(newCfg Config) { + ConfigLock.Lock() + defer ConfigLock.Unlock() + sCfg.RRS = newCfg.RRS + sCfg.Standard = newCfg.Standard + sCfg.Optimize = newCfg.Optimize + sCfg.inlineBlock = newCfg.inlineBlock + sCfg.initialized = true +} + +// Enabled returns if storageClass is enabled is enabled. +func Enabled(kvs config.KVS) bool { + ssc := kvs.Get(ClassStandard) + rrsc := kvs.Get(ClassRRS) + return ssc != "" || rrsc != "" +} + +// DefaultParityBlocks returns default parity blocks for 'drive' count +func DefaultParityBlocks(drive int) int { + switch drive { + case 1: + return 0 + case 3, 2: + return 1 + case 4, 5: + return 2 + case 6, 7: + return 3 + default: + return 4 + } +} + +// LookupConfig - lookup storage class config and override with valid environment settings if any. +func LookupConfig(kvs config.KVS, setDriveCount int) (cfg Config, err error) { + cfg = Config{} + + deprecatedKeys := []string{ + "dma", + } + + if err = config.CheckValidKeys(config.StorageClassSubSys, kvs, DefaultKVS, deprecatedKeys...); err != nil { + return Config{}, err + } + + ssc := env.Get(StandardEnv, kvs.Get(ClassStandard)) + rrsc := env.Get(RRSEnv, kvs.Get(ClassRRS)) + // Check for environment variables and parse into storageClass struct + if ssc != "" { + cfg.Standard, err = parseStorageClass(ssc) + if err != nil { + return Config{}, err + } + } else { + cfg.Standard.Parity = DefaultParityBlocks(setDriveCount) + } + + if rrsc != "" { + cfg.RRS, err = parseStorageClass(rrsc) + if err != nil { + return Config{}, err + } + } else { + cfg.RRS.Parity = defaultRRSParity + if setDriveCount == 1 { + cfg.RRS.Parity = 0 + } + } + + // Validation is done after parsing both the storage classes. This is needed because we need one + // storage class value to deduce the correct value of the other storage class. + if err = validateParity(cfg.Standard.Parity, cfg.RRS.Parity, setDriveCount); err != nil { + return Config{}, err + } + + cfg.Optimize = env.Get(OptimizeEnv, kvs.Get(Optimize)) + + inlineBlockStr := env.Get(InlineBlockEnv, kvs.Get(InlineBlock)) + if inlineBlockStr != "" { + inlineBlock, err := humanize.ParseBytes(inlineBlockStr) + if err != nil { + return cfg, err + } + if inlineBlock > 128*humanize.KiByte { + configLogOnceIf(context.Background(), fmt.Errorf("inline block value bigger than recommended max of 128KiB -> %s, performance may degrade for PUT please benchmark the changes", inlineBlockStr), inlineBlockStr) + } + cfg.inlineBlock = int64(inlineBlock) + } else { + cfg.inlineBlock = 128 * humanize.KiByte + } + + cfg.initialized = true + + return cfg, nil +} + +func configLogOnceIf(ctx context.Context, err error, id string, errKind ...any) { + logger.LogOnceIf(ctx, "config", err, id, errKind...) +} diff --git a/internal/config/storageclass/storage-class_test.go b/internal/config/storageclass/storage-class_test.go new file mode 100644 index 00000000..be225581 --- /dev/null +++ b/internal/config/storageclass/storage-class_test.go @@ -0,0 +1,190 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package storageclass + +import ( + "errors" + "reflect" + "testing" +) + +func TestParseStorageClass(t *testing.T) { + tests := []struct { + storageClassEnv string + wantSc StorageClass + expectedError error + }{ + { + "EC:3", + StorageClass{ + Parity: 3, + }, + nil, + }, + { + "EC:4", + StorageClass{ + Parity: 4, + }, + nil, + }, + { + "AB:4", + StorageClass{ + Parity: 4, + }, + errors.New("Unsupported scheme AB. Supported scheme is EC"), + }, + { + "EC:4:5", + StorageClass{ + Parity: 4, + }, + errors.New("Too many sections in EC:4:5"), + }, + { + "EC:A", + StorageClass{ + Parity: 4, + }, + errors.New(`strconv.Atoi: parsing "A": invalid syntax`), + }, + { + "AB", + StorageClass{ + Parity: 4, + }, + errors.New("Too few sections in AB"), + }, + } + for i, tt := range tests { + gotSc, err := parseStorageClass(tt.storageClassEnv) + if err != nil && tt.expectedError == nil { + t.Errorf("Test %d, Expected %s, got %s", i+1, tt.expectedError, err) + return + } + if err == nil && tt.expectedError != nil { + t.Errorf("Test %d, Expected %s, got %s", i+1, tt.expectedError, err) + return + } + if tt.expectedError == nil && !reflect.DeepEqual(gotSc, tt.wantSc) { + t.Errorf("Test %d, Expected %v, got %v", i+1, tt.wantSc, gotSc) + return + } + if tt.expectedError != nil && err.Error() != tt.expectedError.Error() { + t.Errorf("Test %d, Expected `%v`, got `%v`", i+1, tt.expectedError, err) + } + } +} + +func TestValidateParity(t *testing.T) { + tests := []struct { + rrsParity int + ssParity int + success bool + setDriveCount int + }{ + {2, 4, true, 16}, + {3, 3, true, 16}, + {0, 0, true, 16}, + {1, 4, true, 16}, + {0, 4, true, 16}, + {7, 6, false, 16}, + {9, 0, false, 16}, + {9, 9, false, 16}, + {2, 9, false, 16}, + {9, 2, false, 16}, + } + for i, tt := range tests { + err := validateParity(tt.ssParity, tt.rrsParity, tt.setDriveCount) + if err != nil && tt.success { + t.Errorf("Test %d, Expected success, got %s", i+1, err) + } + if err == nil && !tt.success { + t.Errorf("Test %d, Expected failure, got success", i+1) + } + } +} + +func TestParityCount(t *testing.T) { + tests := []struct { + sc string + drivesCount int + expectedData int + expectedParity int + }{ + {RRS, 16, 14, 2}, + {STANDARD, 16, 8, 8}, + {"", 16, 8, 8}, + {RRS, 16, 9, 7}, + {STANDARD, 16, 10, 6}, + {"", 16, 9, 7}, + } + for i, tt := range tests { + scfg := Config{ + Standard: StorageClass{ + Parity: 8, + }, + RRS: StorageClass{ + Parity: 2, + }, + initialized: true, + } + // Set env var for test case 4 + if i+1 == 4 { + scfg.RRS.Parity = 7 + } + // Set env var for test case 5 + if i+1 == 5 { + scfg.Standard.Parity = 6 + } + // Set env var for test case 6 + if i+1 == 6 { + scfg.Standard.Parity = 7 + } + parity := scfg.GetParityForSC(tt.sc) + if (tt.drivesCount - parity) != tt.expectedData { + t.Errorf("Test %d, Expected data drives %d, got %d", i+1, tt.expectedData, tt.drivesCount-parity) + continue + } + if parity != tt.expectedParity { + t.Errorf("Test %d, Expected parity drives %d, got %d", i+1, tt.expectedParity, parity) + } + } +} + +// Test IsValid method with valid and invalid inputs +func TestIsValidStorageClassKind(t *testing.T) { + tests := []struct { + sc string + want bool + }{ + {"STANDARD", true}, + {"REDUCED_REDUNDANCY", true}, + {"", false}, + {"INVALID", false}, + {"123", false}, + {"MINIO_STORAGE_CLASS_RRS", false}, + {"MINIO_STORAGE_CLASS_STANDARD", false}, + } + for i, tt := range tests { + if got := IsValid(tt.sc); got != tt.want { + t.Errorf("Test %d, Expected Storage Class to be %t, got %t", i+1, tt.want, got) + } + } +} diff --git a/internal/config/subnet/config.go b/internal/config/subnet/config.go new file mode 100644 index 00000000..9e2420a6 --- /dev/null +++ b/internal/config/subnet/config.go @@ -0,0 +1,154 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package subnet + +import ( + "net/http" + "net/url" + "os" + "strings" + "sync" + + "github.com/minio/minio/internal/config" + "github.com/minio/pkg/v3/env" + xnet "github.com/minio/pkg/v3/net" +) + +const ( + baseURL = "https://subnet.min.io" + baseURLDev = "http://localhost:9000" +) + +// DefaultKVS - default KV config for subnet settings +var DefaultKVS = config.KVS{ + config.KV{ + Key: config.License, // Deprecated Dec 2021 + Value: "", + }, + config.KV{ + Key: config.APIKey, + Value: "", + }, + config.KV{ + Key: config.Proxy, + Value: "", + }, +} + +// Config represents the subnet related configuration +type Config struct { + // The subnet license token - Deprecated Dec 2021 + License string `json:"license"` + + // The subnet api key + APIKey string `json:"apiKey"` + + // The HTTP(S) proxy URL to use for connecting to SUBNET + Proxy string `json:"proxy"` + + // Transport configured with proxy_url if set optionally. + transport http.RoundTripper + + // The subnet base URL + BaseURL string +} + +var configLock sync.RWMutex + +// Registered indicates if cluster is registered or not +func (c *Config) Registered() bool { + configLock.RLock() + defer configLock.RUnlock() + + return len(c.APIKey) > 0 +} + +// ApplyEnv - applies the current subnet config to Console UI specific environment variables. +func (c *Config) ApplyEnv() { + configLock.RLock() + defer configLock.RUnlock() + + if c.License != "" { + os.Setenv("CONSOLE_SUBNET_LICENSE", c.License) + } + if c.APIKey != "" { + os.Setenv("CONSOLE_SUBNET_API_KEY", c.APIKey) + } + if c.Proxy != "" { + os.Setenv("CONSOLE_SUBNET_PROXY", c.Proxy) + } + os.Setenv("CONSOLE_SUBNET_URL", c.BaseURL) +} + +// Update - in-place update with new license and registration information. +func (c *Config) Update(ncfg Config, isDevEnv bool) { + configLock.Lock() + defer configLock.Unlock() + + c.License = ncfg.License + c.APIKey = ncfg.APIKey + c.Proxy = ncfg.Proxy + c.transport = ncfg.transport + c.BaseURL = baseURL + + if isDevEnv { + c.BaseURL = os.Getenv("_MINIO_SUBNET_URL") + if c.BaseURL == "" { + c.BaseURL = baseURLDev + } + } +} + +// LookupConfig - lookup config and override with valid environment settings if any. +func LookupConfig(kvs config.KVS, transport http.RoundTripper) (cfg Config, err error) { + if err = config.CheckValidKeys(config.SubnetSubSys, kvs, DefaultKVS); err != nil { + return cfg, err + } + + var proxyURL *xnet.URL + proxy := env.Get(config.EnvMinIOSubnetProxy, kvs.Get(config.Proxy)) + if len(proxy) > 0 { + proxyURL, err = xnet.ParseHTTPURL(proxy) + if err != nil { + return cfg, err + } + } + + cfg.License = strings.TrimSpace(env.Get(config.EnvMinIOSubnetLicense, kvs.Get(config.License))) + cfg.APIKey = strings.TrimSpace(env.Get(config.EnvMinIOSubnetAPIKey, kvs.Get(config.APIKey))) + cfg.Proxy = proxy + + if transport == nil { + // when transport is nil, it means we are just validating the + // inputs not performing any network calls. + return cfg, nil + } + + // Make sure to clone the transport before editing the ProxyURL + if proxyURL != nil { + if tr, ok := transport.(*http.Transport); ok { + ctransport := tr.Clone() + ctransport.Proxy = http.ProxyURL((*url.URL)(proxyURL)) + cfg.transport = ctransport + } + } else { + cfg.transport = transport + } + + return cfg, nil +} diff --git a/internal/config/subnet/help.go b/internal/config/subnet/help.go new file mode 100644 index 00000000..da4451d5 --- /dev/null +++ b/internal/config/subnet/help.go @@ -0,0 +1,51 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package subnet + +import "github.com/minio/minio/internal/config" + +var ( + defaultHelpPostfix = func(key string) string { + return config.DefaultHelpPostfix(DefaultKVS, key) + } + + // HelpSubnet - provides help for subnet api key config + HelpSubnet = config.HelpKVS{ + config.HelpKV{ + Key: config.License, + Type: "string", + Description: "Enterprise license for the cluster" + defaultHelpPostfix(config.License), + Optional: true, + Sensitive: true, + }, + config.HelpKV{ + Key: config.APIKey, + Type: "string", + Description: "Enterprise license API key for the cluster" + defaultHelpPostfix(config.APIKey), + Optional: true, + Sensitive: true, + }, + config.HelpKV{ + Key: config.Proxy, + Type: "string", + Description: "HTTP(s) proxy URL to use for connecting to SUBNET" + defaultHelpPostfix(config.Proxy), + Optional: true, + Sensitive: true, + }, + } +) diff --git a/internal/config/subnet/subnet.go b/internal/config/subnet/subnet.go new file mode 100644 index 00000000..c4ba0198 --- /dev/null +++ b/internal/config/subnet/subnet.go @@ -0,0 +1,114 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package subnet + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "io" + "mime/multipart" + "net/http" + "time" + + xhttp "github.com/minio/minio/internal/http" +) + +const ( + respBodyLimit = 1 << 20 // 1 MiB + + // LoggerWebhookName - subnet logger webhook target + LoggerWebhookName = "subnet" +) + +// Upload given file content (payload) to specified URL +func (c Config) Upload(reqURL string, filename string, payload []byte) (string, error) { + if !c.Registered() { + return "", errors.New("Deployment is not registered with SUBNET. Please register the deployment via 'mc license register ALIAS'") + } + + var body bytes.Buffer + writer := multipart.NewWriter(&body) + part, e := writer.CreateFormFile("file", filename) + if e != nil { + return "", e + } + + if _, e = part.Write(payload); e != nil { + return "", e + } + writer.Close() + + r, e := http.NewRequest(http.MethodPost, reqURL, &body) + if e != nil { + return "", e + } + r.Header.Add("Content-Type", writer.FormDataContentType()) + + return c.submitPost(r) +} + +func (c Config) submitPost(r *http.Request) (string, error) { + configLock.RLock() + r.Header.Set(xhttp.SubnetAPIKey, c.APIKey) + configLock.RUnlock() + r.Header.Set(xhttp.MinioDeploymentID, xhttp.GlobalDeploymentID) + + client := &http.Client{ + Timeout: 10 * time.Second, + Transport: c.transport, + } + + resp, err := client.Do(r) + if err != nil { + return "", err + } + defer xhttp.DrainBody(resp.Body) + + respBytes, err := io.ReadAll(io.LimitReader(resp.Body, respBodyLimit)) + if err != nil { + return "", err + } + respStr := string(respBytes) + + if resp.StatusCode == http.StatusOK { + return respStr, nil + } + + return respStr, fmt.Errorf("SUBNET request failed with code %d and error: %s", resp.StatusCode, respStr) +} + +// Post submit 'payload' to specified URL +func (c Config) Post(reqURL string, payload any) (string, error) { + if !c.Registered() { + return "", errors.New("Deployment is not registered with SUBNET. Please register the deployment via 'mc license register ALIAS'") + } + body, err := json.Marshal(payload) + if err != nil { + return "", err + } + r, err := http.NewRequest(http.MethodPost, reqURL, bytes.NewReader(body)) + if err != nil { + return "", err + } + + r.Header.Set("Content-Type", "application/json") + + return c.submitPost(r) +} diff --git a/internal/crypto/auto-encryption.go b/internal/crypto/auto-encryption.go new file mode 100644 index 00000000..f2cdcc5c --- /dev/null +++ b/internal/crypto/auto-encryption.go @@ -0,0 +1,40 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package crypto + +import ( + "github.com/minio/minio/internal/config" + "github.com/minio/pkg/v3/env" +) + +const ( + // EnvKMSAutoEncryption is the environment variable used to en/disable + // SSE-S3 auto-encryption. SSE-S3 auto-encryption, if enabled, + // requires a valid KMS configuration and turns any non-SSE-C + // request into an SSE-S3 request. + // If present EnvAutoEncryption must be either "on" or "off". + EnvKMSAutoEncryption = "MINIO_KMS_AUTO_ENCRYPTION" +) + +// LookupAutoEncryption returns true if and only if +// the MINIO_KMS_AUTO_ENCRYPTION env. variable is +// set to "on". +func LookupAutoEncryption() bool { + auto, _ := config.ParseBool(env.Get(EnvKMSAutoEncryption, config.EnableOff)) + return auto +} diff --git a/internal/crypto/crypto.go b/internal/crypto/crypto.go new file mode 100644 index 00000000..f4a9b943 --- /dev/null +++ b/internal/crypto/crypto.go @@ -0,0 +1,78 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package crypto + +import ( + "crypto/tls" + + "github.com/minio/sio" +) + +// DARECiphers returns a list of supported cipher suites +// for the DARE object encryption. +func DARECiphers() []byte { return []byte{sio.AES_256_GCM, sio.CHACHA20_POLY1305} } + +// TLSCiphers returns a list of supported TLS transport +// cipher suite IDs. +func TLSCiphers() []uint16 { + return []uint16{ + tls.TLS_CHACHA20_POLY1305_SHA256, // TLS 1.3 + tls.TLS_AES_128_GCM_SHA256, + tls.TLS_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305, // TLS 1.2 + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + } +} + +// TLSCiphersBackwardCompatible returns a list of supported +// TLS transport cipher suite IDs. +// +// In contrast to TLSCiphers, the list contains additional +// ciphers for backward compatibility. In particular, AES-CBC +// and non-ECDHE ciphers. +func TLSCiphersBackwardCompatible() []uint16 { + return []uint16{ + tls.TLS_CHACHA20_POLY1305_SHA256, // TLS 1.3 + tls.TLS_AES_128_GCM_SHA256, + tls.TLS_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256, // TLS 1.2 ECDHE GCM / POLY1305 + tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, + tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, + tls.TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA, // TLS 1.2 ECDHE CBC + tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, + tls.TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA, + tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, + tls.TLS_RSA_WITH_AES_128_GCM_SHA256, // TLS 1.2 non-ECDHE + tls.TLS_RSA_WITH_AES_256_GCM_SHA384, + tls.TLS_RSA_WITH_AES_128_CBC_SHA, + tls.TLS_RSA_WITH_AES_256_CBC_SHA, + } +} + +// TLSCurveIDs returns a list of supported elliptic curve IDs +// in preference order. +func TLSCurveIDs() []tls.CurveID { + return []tls.CurveID{tls.X25519MLKEM768, tls.CurveP256, tls.X25519, tls.CurveP384, tls.CurveP521} +} diff --git a/internal/crypto/doc.go b/internal/crypto/doc.go new file mode 100644 index 00000000..eac2e4ac --- /dev/null +++ b/internal/crypto/doc.go @@ -0,0 +1,117 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Package crypto implements AWS S3 related cryptographic building blocks +// for implementing Server-Side-Encryption (SSE-S3) and Server-Side-Encryption +// with customer provided keys (SSE-C). +// +// All objects are encrypted with an unique and randomly generated 'ObjectKey'. +// The ObjectKey itself is never stored in plaintext. Instead it is only stored +// in a sealed from. The sealed 'ObjectKey' is created by encrypting the 'ObjectKey' +// with an unique key-encryption-key. Given the correct key-encryption-key the +// sealed 'ObjectKey' can be unsealed and the object can be decrypted. +// +// ## SSE-C +// +// SSE-C computes the key-encryption-key from the client-provided key, an +// initialization vector (IV) and the bucket/object path. +// +// 1. Encrypt: +// Input: ClientKey, bucket, object, metadata, object_data +// - IV := Random({0,1}²⁵⁶) +// - ObjectKey := SHA256(ClientKey || Random({0,1}²⁵⁶)) +// - KeyEncKey := HMAC-SHA256(ClientKey, IV || 'SSE-C' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object) +// - SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey) +// - enc_object_data := DAREv2_Enc(ObjectKey, object_data) +// - metadata <- IV +// - metadata <- SealedKey +// Output: enc_object_data, metadata +// +// 2. Decrypt: +// Input: ClientKey, bucket, object, metadata, enc_object_data +// - IV <- metadata +// - SealedKey <- metadata +// - KeyEncKey := HMAC-SHA256(ClientKey, IV || 'SSE-C' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object) +// - ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey) +// - object_data := DAREv2_Dec(ObjectKey, enc_object_data) +// Output: object_data +// +// ## SSE-S3 +// +// SSE-S3 can use either a master key or a KMS as root-of-trust. +// The en/decryption slightly depens upon which root-of-trust is used. +// +// ### SSE-S3 and single master key +// +// The master key is used to derive unique object- and key-encryption-keys. +// SSE-S3 with a single master key works as SSE-C where the master key is +// used as the client-provided key. +// +// 1. Encrypt: +// Input: MasterKey, bucket, object, metadata, object_data +// - IV := Random({0,1}²⁵⁶) +// - ObjectKey := SHA256(MasterKey || Random({0,1}²⁵⁶)) +// - KeyEncKey := HMAC-SHA256(MasterKey, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object) +// - SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey) +// - enc_object_data := DAREv2_Enc(ObjectKey, object_data) +// - metadata <- IV +// - metadata <- SealedKey +// Output: enc_object_data, metadata +// +// 2. Decrypt: +// Input: MasterKey, bucket, object, metadata, enc_object_data +// - IV <- metadata +// - SealedKey <- metadata +// - KeyEncKey := HMAC-SHA256(MasterKey, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object) +// - ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey) +// - object_data := DAREv2_Dec(ObjectKey, enc_object_data) +// Output: object_data +// +// ### SSE-S3 and KMS +// +// SSE-S3 requires that the KMS provides two functions: +// +// 1. Generate(KeyID) -> (Key, EncKey) +// +// 2. Unseal(KeyID, EncKey) -> Key +// +// 1. Encrypt: +// Input: KeyID, bucket, object, metadata, object_data +// - Key, EncKey := Generate(KeyID) +// - IV := Random({0,1}²⁵⁶) +// - ObjectKey := SHA256(Key, Random({0,1}²⁵⁶)) +// - KeyEncKey := HMAC-SHA256(Key, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object) +// - SealedKey := DAREv2_Enc(KeyEncKey, ObjectKey) +// - enc_object_data := DAREv2_Enc(ObjectKey, object_data) +// - metadata <- IV +// - metadata <- KeyID +// - metadata <- EncKey +// - metadata <- SealedKey +// Output: enc_object_data, metadata +// +// 2. Decrypt: +// Input: bucket, object, metadata, enc_object_data +// - KeyID <- metadata +// - EncKey <- metadata +// - IV <- metadata +// - SealedKey <- metadata +// - Key := Unseal(KeyID, EncKey) +// - KeyEncKey := HMAC-SHA256(Key, IV || 'SSE-S3' || 'DAREv2-HMAC-SHA256' || bucket || '/' || object) +// - ObjectKey := DAREv2_Dec(KeyEncKey, SealedKey) +// - object_data := DAREv2_Dec(ObjectKey, enc_object_data) +// Output: object_data +package crypto diff --git a/internal/crypto/error.go b/internal/crypto/error.go new file mode 100644 index 00000000..4d21b936 --- /dev/null +++ b/internal/crypto/error.go @@ -0,0 +1,102 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package crypto + +import ( + "errors" + "fmt" +) + +// Error is the generic type for any error happening during decrypting +// an object. It indicates that the object itself or its metadata was +// modified accidentally or maliciously. +type Error struct { + msg string + cause error +} + +// Errorf - formats according to a format specifier and returns +// the string as a value that satisfies error of type crypto.Error +func Errorf(format string, a ...any) error { + e := fmt.Errorf(format, a...) + ee := Error{} + ee.msg = e.Error() + ee.cause = errors.Unwrap(e) + return ee +} + +// Unwrap the internal error. +func (e Error) Unwrap() error { return e.cause } + +// Error 'error' compatible method. +func (e Error) Error() string { + if e.msg == "" { + return "crypto: cause " + } + return e.msg +} + +var ( + // ErrInvalidEncryptionMethod indicates that the specified SSE encryption method + // is not supported. + ErrInvalidEncryptionMethod = Errorf("The encryption method is not supported") + + // ErrInvalidCustomerAlgorithm indicates that the specified SSE-C algorithm + // is not supported. + ErrInvalidCustomerAlgorithm = Errorf("The SSE-C algorithm is not supported") + + // ErrMissingCustomerKey indicates that the HTTP headers contains no SSE-C client key. + ErrMissingCustomerKey = Errorf("The SSE-C request is missing the customer key") + + // ErrMissingCustomerKeyMD5 indicates that the HTTP headers contains no SSE-C client key + // MD5 checksum. + ErrMissingCustomerKeyMD5 = Errorf("The SSE-C request is missing the customer key MD5") + + // ErrInvalidCustomerKey indicates that the SSE-C client key is not valid - e.g. not a + // base64-encoded string or not 256 bits long. + ErrInvalidCustomerKey = Errorf("The SSE-C client key is invalid") + + // ErrSecretKeyMismatch indicates that the provided secret key (SSE-C client key / SSE-S3 KMS key) + // does not match the secret key used during encrypting the object. + ErrSecretKeyMismatch = Errorf("The secret key does not match the secret key used during upload") + + // ErrCustomerKeyMD5Mismatch indicates that the SSE-C key MD5 does not match the + // computed MD5 sum. This means that the client provided either the wrong key for + // a certain MD5 checksum or the wrong MD5 for a certain key. + ErrCustomerKeyMD5Mismatch = Errorf("The provided SSE-C key MD5 does not match the computed MD5 of the SSE-C key") + // ErrIncompatibleEncryptionMethod indicates that both SSE-C headers and SSE-S3 headers were specified, and are incompatible + // The client needs to remove the SSE-S3 header or the SSE-C headers + ErrIncompatibleEncryptionMethod = Errorf("Server side encryption specified with both SSE-C and SSE-S3 headers") + // ErrIncompatibleEncryptionWithCompression indicates that both data compression and SSE-C not allowed at the same time + ErrIncompatibleEncryptionWithCompression = Errorf("Server side encryption specified with SSE-C with compression not allowed") + + // ErrInvalidEncryptionKeyID returns error when KMS key id contains invalid characters + ErrInvalidEncryptionKeyID = Errorf("KMS KeyID contains unsupported characters") +) + +var ( + errMissingInternalIV = Errorf("The object metadata is missing the internal encryption IV") + errMissingInternalSealAlgorithm = Errorf("The object metadata is missing the internal seal algorithm") + + errInvalidInternalIV = Errorf("The internal encryption IV is malformed") + errInvalidInternalSealAlgorithm = Errorf("The internal seal algorithm is invalid and not supported") +) + +// errOutOfEntropy indicates that the a source of randomness (PRNG) wasn't able +// to produce enough random data. This is fatal error and should cause a panic. +var errOutOfEntropy = Errorf("Unable to read enough randomness from the system") diff --git a/internal/crypto/header.go b/internal/crypto/header.go new file mode 100644 index 00000000..213f7888 --- /dev/null +++ b/internal/crypto/header.go @@ -0,0 +1,84 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package crypto + +import ( + "bytes" + "crypto/md5" + "encoding/base64" + "net/http" + + xhttp "github.com/minio/minio/internal/http" +) + +// RemoveSensitiveHeaders removes confidential encryption +// information - e.g. the SSE-C key - from the HTTP headers. +// It has the same semantics as RemoveSensitiveEntries. +func RemoveSensitiveHeaders(h http.Header) { + h.Del(xhttp.AmzServerSideEncryptionCustomerKey) + h.Del(xhttp.AmzServerSideEncryptionCopyCustomerKey) + h.Del(xhttp.AmzMetaUnencryptedContentLength) + h.Del(xhttp.AmzMetaUnencryptedContentMD5) +} + +// SSECopy represents AWS SSE-C for copy requests. It provides +// functionality to handle SSE-C copy requests. +var SSECopy = ssecCopy{} + +type ssecCopy struct{} + +// IsRequested returns true if the HTTP headers contains +// at least one SSE-C copy header. Regular SSE-C headers +// are ignored. +func (ssecCopy) IsRequested(h http.Header) bool { + if _, ok := h[xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm]; ok { + return true + } + if _, ok := h[xhttp.AmzServerSideEncryptionCopyCustomerKey]; ok { + return true + } + if _, ok := h[xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5]; ok { + return true + } + return false +} + +// ParseHTTP parses the SSE-C copy headers and returns the SSE-C client key +// on success. Regular SSE-C headers are ignored. +func (ssecCopy) ParseHTTP(h http.Header) (key [32]byte, err error) { + if h.Get(xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm) != xhttp.AmzEncryptionAES { + return key, ErrInvalidCustomerAlgorithm + } + if h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKey) == "" { + return key, ErrMissingCustomerKey + } + if h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5) == "" { + return key, ErrMissingCustomerKeyMD5 + } + + clientKey, err := base64.StdEncoding.DecodeString(h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKey)) + if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long + return key, ErrInvalidCustomerKey + } + keyMD5, err := base64.StdEncoding.DecodeString(h.Get(xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5)) + if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) { + return key, ErrCustomerKeyMD5Mismatch + } + copy(key[:], clientKey) + return key, nil +} diff --git a/internal/crypto/header_test.go b/internal/crypto/header_test.go new file mode 100644 index 00000000..424d889f --- /dev/null +++ b/internal/crypto/header_test.go @@ -0,0 +1,538 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package crypto + +import ( + "encoding/base64" + "net/http" + "sort" + "testing" + + xhttp "github.com/minio/minio/internal/http" +) + +func TestIsRequested(t *testing.T) { + for i, test := range kmsIsRequestedTests { + _, got := IsRequested(test.Header) + if Requested(test.Header) != got { + // Test if result matches. + t.Errorf("Requested mismatch, want %v, got %v", Requested(test.Header), got) + } + got = got && S3KMS.IsRequested(test.Header) + if got != test.Expected { + t.Errorf("SSE-KMS: Test %d: Wanted %v but got %v", i, test.Expected, got) + } + } + for i, test := range s3IsRequestedTests { + _, got := IsRequested(test.Header) + if Requested(test.Header) != got { + // Test if result matches. + t.Errorf("Requested mismatch, want %v, got %v", Requested(test.Header), got) + } + got = got && S3.IsRequested(test.Header) + if got != test.Expected { + t.Errorf("SSE-S3: Test %d: Wanted %v but got %v", i, test.Expected, got) + } + } + for i, test := range ssecIsRequestedTests { + _, got := IsRequested(test.Header) + if Requested(test.Header) != got { + // Test if result matches. + t.Errorf("Requested mismatch, want %v, got %v", Requested(test.Header), got) + } + got = got && SSEC.IsRequested(test.Header) + if got != test.Expected { + t.Errorf("SSE-C: Test %d: Wanted %v but got %v", i, test.Expected, got) + } + } +} + +var kmsIsRequestedTests = []struct { + Header http.Header + Expected bool +}{ + {Header: http.Header{}, Expected: false}, // 0 + {Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"aws:kms"}}, Expected: true}, // 1 + {Header: http.Header{"X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{"0839-9047947-844842874-481"}}, Expected: true}, // 2 + {Header: http.Header{"X-Amz-Server-Side-Encryption-Context": []string{"7PpPLAK26ONlVUGOWlusfg=="}}, Expected: true}, // 3 + { + Header: http.Header{ + "X-Amz-Server-Side-Encryption": []string{""}, + "X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{""}, + "X-Amz-Server-Side-Encryption-Context": []string{""}, + }, + Expected: true, + }, // 4 + { + Header: http.Header{ + "X-Amz-Server-Side-Encryption": []string{"AES256"}, + "X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{""}, + }, + Expected: true, + }, // 5 + {Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, Expected: false}, // 6 +} + +func TestKMSIsRequested(t *testing.T) { + for i, test := range kmsIsRequestedTests { + if got := S3KMS.IsRequested(test.Header); got != test.Expected { + t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got) + } + } +} + +var kmsParseHTTPTests = []struct { + Header http.Header + ShouldFail bool +}{ + {Header: http.Header{}, ShouldFail: true}, // 0 + {Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"aws:kms"}}, ShouldFail: false}, // 1 + {Header: http.Header{ + "X-Amz-Server-Side-Encryption": []string{"aws:kms"}, + "X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{"s3-007-293847485-724784"}, + }, ShouldFail: false}, // 2 + {Header: http.Header{ + "X-Amz-Server-Side-Encryption": []string{"aws:kms"}, + "X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{"s3-007-293847485-724784"}, + "X-Amz-Server-Side-Encryption-Context": []string{base64.StdEncoding.EncodeToString([]byte("{}"))}, + }, ShouldFail: false}, // 3 + {Header: http.Header{ + "X-Amz-Server-Side-Encryption": []string{"aws:kms"}, + "X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{"s3-007-293847485-724784"}, + "X-Amz-Server-Side-Encryption-Context": []string{base64.StdEncoding.EncodeToString([]byte(`{"bucket": "some-bucket"}`))}, + }, ShouldFail: false}, // 4 + {Header: http.Header{ + "X-Amz-Server-Side-Encryption": []string{"aws:kms"}, + "X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{"s3-007-293847485-724784"}, + "X-Amz-Server-Side-Encryption-Context": []string{base64.StdEncoding.EncodeToString([]byte(`{"bucket": "some-bucket"}`))}, + }, ShouldFail: false}, // 5 + {Header: http.Header{ + "X-Amz-Server-Side-Encryption": []string{"AES256"}, + "X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{"s3-007-293847485-724784"}, + "X-Amz-Server-Side-Encryption-Context": []string{base64.StdEncoding.EncodeToString([]byte(`{"bucket": "some-bucket"}`))}, + }, ShouldFail: true}, // 6 + {Header: http.Header{ + "X-Amz-Server-Side-Encryption": []string{"aws:kms"}, + "X-Amz-Server-Side-Encryption-Aws-Kms-Key-Id": []string{"s3-007-293847485-724784"}, + "X-Amz-Server-Side-Encryption-Context": []string{base64.StdEncoding.EncodeToString([]byte(`{"bucket": "some-bucket"`))}, // invalid JSON + }, ShouldFail: true}, // 7 + +} + +func TestKMSParseHTTP(t *testing.T) { + for i, test := range kmsParseHTTPTests { + _, _, err := S3KMS.ParseHTTP(test.Header) + if err == nil && test.ShouldFail { + t.Errorf("Test %d: should fail but succeeded", i) + } + if err != nil && !test.ShouldFail { + t.Errorf("Test %d: should pass but failed with: %v", i, err) + } + } +} + +var s3IsRequestedTests = []struct { + Header http.Header + Expected bool +}{ + {Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, Expected: true}, // 0 + {Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, Expected: true}, // 1 + {Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, Expected: true}, // 2 + {Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, Expected: false}, // 3 + {Header: http.Header{"X-Amz-Server-Side-Encryption": []string{xhttp.AmzEncryptionKMS}}, Expected: false}, // 4 +} + +func TestS3IsRequested(t *testing.T) { + for i, test := range s3IsRequestedTests { + if got := S3.IsRequested(test.Header); got != test.Expected { + t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got) + } + } +} + +var s3ParseTests = []struct { + Header http.Header + ExpectedErr error +}{ + {Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES256"}}, ExpectedErr: nil}, // 0 + {Header: http.Header{"X-Amz-Server-Side-Encryption": []string{"AES-256"}}, ExpectedErr: ErrInvalidEncryptionMethod}, // 1 + {Header: http.Header{"X-Amz-Server-Side-Encryption": []string{""}}, ExpectedErr: ErrInvalidEncryptionMethod}, // 2 + {Header: http.Header{"X-Amz-Server-Side-Encryptio": []string{"AES256"}}, ExpectedErr: ErrInvalidEncryptionMethod}, // 3 +} + +func TestS3Parse(t *testing.T) { + for i, test := range s3ParseTests { + if err := S3.ParseHTTP(test.Header); err != test.ExpectedErr { + t.Errorf("Test %d: Wanted '%v' but got '%v'", i, test.ExpectedErr, err) + } + } +} + +var ssecIsRequestedTests = []struct { + Header http.Header + Expected bool +}{ + {Header: http.Header{}, Expected: false}, // 0 + {Header: http.Header{"X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}}, Expected: true}, // 1 + {Header: http.Header{"X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}}, Expected: true}, // 2 + {Header: http.Header{"X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}}, Expected: true}, // 3 + { + Header: http.Header{ + "X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{""}, + "X-Amz-Server-Side-Encryption-Customer-Key": []string{""}, + "X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{""}, + }, + Expected: true, + }, // 4 + { + Header: http.Header{ + "X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + "X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + Expected: true, + }, // 5 + { + Header: http.Header{ + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + Expected: false, + }, // 6 +} + +func TestSSECIsRequested(t *testing.T) { + for i, test := range ssecIsRequestedTests { + if got := SSEC.IsRequested(test.Header); got != test.Expected { + t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got) + } + } +} + +var ssecCopyIsRequestedTests = []struct { + Header http.Header + Expected bool +}{ + {Header: http.Header{}, Expected: false}, // 0 + {Header: http.Header{"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}}, Expected: true}, // 1 + {Header: http.Header{"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}}, Expected: true}, // 2 + {Header: http.Header{"X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}}, Expected: true}, // 3 + { + Header: http.Header{ + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{""}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{""}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{""}, + }, + Expected: true, + }, // 4 + { + Header: http.Header{ + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + Expected: true, + }, // 5 + { + Header: http.Header{ + "X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + "X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + Expected: false, + }, // 6 +} + +func TestSSECopyIsRequested(t *testing.T) { + for i, test := range ssecCopyIsRequestedTests { + if got := SSECopy.IsRequested(test.Header); got != test.Expected { + t.Errorf("Test %d: Wanted %v but got %v", i, test.Expected, got) + } + } +} + +var ssecParseTests = []struct { + Header http.Header + ExpectedErr error +}{ + { + Header: http.Header{ + "X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + "X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + ExpectedErr: nil, // 0 + }, + { + Header: http.Header{ + "X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES-256"}, // invalid algorithm + "X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + "X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + ExpectedErr: ErrInvalidCustomerAlgorithm, // 1 + }, + { + Header: http.Header{ + "X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Server-Side-Encryption-Customer-Key": []string{""}, // no client key + "X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + ExpectedErr: ErrMissingCustomerKey, // 2 + }, + { + Header: http.Header{ + "X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRr.ZXltdXN0cHJvdmlkZWQ="}, // invalid key + "X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + ExpectedErr: ErrInvalidCustomerKey, // 3 + }, + { + Header: http.Header{ + "X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + "X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{""}, // no key MD5 + }, + ExpectedErr: ErrMissingCustomerKeyMD5, // 4 + }, + { + Header: http.Header{ + "X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Server-Side-Encryption-Customer-Key": []string{"DzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, // wrong client key + "X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + ExpectedErr: ErrCustomerKeyMD5Mismatch, // 5 + }, + { + Header: http.Header{ + "X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + "X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{".7PpPLAK26ONlVUGOWlusfg=="}, // wrong key MD5 + }, + ExpectedErr: ErrCustomerKeyMD5Mismatch, // 6 + }, +} + +func TestSSECParse(t *testing.T) { + var zeroKey [32]byte + for i, test := range ssecParseTests { + key, err := SSEC.ParseHTTP(test.Header) + if err != test.ExpectedErr { + t.Errorf("Test %d: want error '%v' but got '%v'", i, test.ExpectedErr, err) + } + + if err != nil && key != zeroKey { + t.Errorf("Test %d: parsing failed and client key is not zero key", i) + } + if err == nil && key == zeroKey { + t.Errorf("Test %d: parsed client key is zero key", i) + } + } +} + +var ssecCopyParseTests = []struct { + Header http.Header + ExpectedErr error +}{ + { + Header: http.Header{ + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + ExpectedErr: nil, // 0 + }, + { + Header: http.Header{ + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES-256"}, // invalid algorithm + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + ExpectedErr: ErrInvalidCustomerAlgorithm, // 1 + }, + { + Header: http.Header{ + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{""}, // no client key + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + ExpectedErr: ErrMissingCustomerKey, // 2 + }, + { + Header: http.Header{ + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRr.ZXltdXN0cHJvdmlkZWQ="}, // invalid key + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + ExpectedErr: ErrInvalidCustomerKey, // 3 + }, + { + Header: http.Header{ + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{""}, // no key MD5 + }, + ExpectedErr: ErrMissingCustomerKeyMD5, // 4 + }, + { + Header: http.Header{ + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"DzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, // wrong client key + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + ExpectedErr: ErrCustomerKeyMD5Mismatch, // 5 + }, + { + Header: http.Header{ + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{".7PpPLAK26ONlVUGOWlusfg=="}, // wrong key MD5 + }, + ExpectedErr: ErrCustomerKeyMD5Mismatch, // 6 + }, +} + +func TestSSECopyParse(t *testing.T) { + var zeroKey [32]byte + for i, test := range ssecCopyParseTests { + key, err := SSECopy.ParseHTTP(test.Header) + if err != test.ExpectedErr { + t.Errorf("Test %d: want error '%v' but got '%v'", i, test.ExpectedErr, err) + } + + if err != nil && key != zeroKey { + t.Errorf("Test %d: parsing failed and client key is not zero key", i) + } + if err == nil && key == zeroKey { + t.Errorf("Test %d: parsed client key is zero key", i) + } + if _, ok := test.Header[xhttp.AmzServerSideEncryptionCustomerKey]; ok { + t.Errorf("Test %d: client key is not removed from HTTP headers after parsing", i) + } + } +} + +var removeSensitiveHeadersTests = []struct { + Header, ExpectedHeader http.Header +}{ + { + Header: http.Header{ + xhttp.AmzServerSideEncryptionCustomerKey: []string{""}, + xhttp.AmzServerSideEncryptionCopyCustomerKey: []string{""}, + }, + ExpectedHeader: http.Header{}, + }, + { // Standard SSE-C request headers + Header: http.Header{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES}, + xhttp.AmzServerSideEncryptionCustomerKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + ExpectedHeader: http.Header{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES}, + xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + }, + { // Standard SSE-C + SSE-C-copy request headers + Header: http.Header{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES}, + xhttp.AmzServerSideEncryptionCustomerKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}, + xhttp.AmzServerSideEncryptionCopyCustomerKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + ExpectedHeader: http.Header{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES}, + xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}, + xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + }, + { // Standard SSE-C + metadata request headers + Header: http.Header{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES}, + xhttp.AmzServerSideEncryptionCustomerKey: []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}, + "X-Amz-Meta-Test-1": []string{"Test-1"}, + }, + ExpectedHeader: http.Header{ + xhttp.AmzServerSideEncryptionCustomerAlgorithm: []string{xhttp.AmzEncryptionAES}, + xhttp.AmzServerSideEncryptionCustomerKeyMD5: []string{"7PpPLAK26ONlVUGOWlusfg=="}, + "X-Amz-Meta-Test-1": []string{"Test-1"}, + }, + }, + { // https://github.com/google/security-research/security/advisories/GHSA-76wf-9vgp-pj7w + Header: http.Header{ + "X-Amz-Meta-X-Amz-Unencrypted-Content-Md5": []string{"value"}, + "X-Amz-Meta-X-Amz-Unencrypted-Content-Length": []string{"value"}, + "X-Amz-Meta-Test-1": []string{"Test-1"}, + }, + ExpectedHeader: http.Header{ + "X-Amz-Meta-Test-1": []string{"Test-1"}, + }, + }, +} + +func TestRemoveSensitiveHeaders(t *testing.T) { + isEqual := func(x, y http.Header) bool { + if len(x) != len(y) { + return false + } + for k, v := range x { + u, ok := y[k] + if !ok || len(v) != len(u) { + return false + } + sort.Strings(v) + sort.Strings(u) + for j := range v { + if v[j] != u[j] { + return false + } + } + } + return true + } + areKeysEqual := func(h http.Header, metadata map[string]string) bool { + if len(h) != len(metadata) { + return false + } + for k := range h { + if _, ok := metadata[k]; !ok { + return false + } + } + return true + } + + for i, test := range removeSensitiveHeadersTests { + metadata := make(map[string]string, len(test.Header)) + for k := range test.Header { + metadata[k] = "" // set metadata key - we don't care about the value + } + + RemoveSensitiveHeaders(test.Header) + if !isEqual(test.ExpectedHeader, test.Header) { + t.Errorf("Test %d: filtered headers do not match expected headers - got: %v , want: %v", i, test.Header, test.ExpectedHeader) + } + RemoveSensitiveEntries(metadata) + if !areKeysEqual(test.ExpectedHeader, metadata) { + t.Errorf("Test %d: filtered headers do not match expected headers - got: %v , want: %v", i, test.Header, test.ExpectedHeader) + } + } +} diff --git a/internal/crypto/key.go b/internal/crypto/key.go new file mode 100644 index 00000000..7c59ecd3 --- /dev/null +++ b/internal/crypto/key.go @@ -0,0 +1,178 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package crypto + +import ( + "bytes" + "context" + "crypto/hmac" + "crypto/rand" + "encoding/binary" + "errors" + "io" + "path" + + "github.com/minio/minio/internal/hash/sha256" + "github.com/minio/minio/internal/logger" + "github.com/minio/sio" +) + +// ObjectKey is a 256 bit secret key used to encrypt the object. +// It must never be stored in plaintext. +type ObjectKey [32]byte + +// GenerateKey generates a unique ObjectKey from a 256 bit external key +// and a source of randomness. If random is nil the default PRNG of the +// system (crypto/rand) is used. +func GenerateKey(extKey []byte, random io.Reader) (key ObjectKey) { + if random == nil { + random = rand.Reader + } + if len(extKey) != 32 { // safety check + logger.CriticalIf(context.Background(), errors.New("crypto: invalid key length")) + } + var nonce [32]byte + if _, err := io.ReadFull(random, nonce[:]); err != nil { + logger.CriticalIf(context.Background(), errOutOfEntropy) + } + + const Context = "object-encryption-key generation" + mac := hmac.New(sha256.New, extKey) + mac.Write([]byte(Context)) + mac.Write(nonce[:]) + mac.Sum(key[:0]) + return key +} + +// GenerateIV generates a new random 256 bit IV from the provided source +// of randomness. If random is nil the default PRNG of the system +// (crypto/rand) is used. +func GenerateIV(random io.Reader) (iv [32]byte) { + if random == nil { + random = rand.Reader + } + if _, err := io.ReadFull(random, iv[:]); err != nil { + logger.CriticalIf(context.Background(), errOutOfEntropy) + } + return iv +} + +// SealedKey represents a sealed object key. It can be stored +// at an untrusted location. +type SealedKey struct { + Key [64]byte // The encrypted and authenticated object-key. + IV [32]byte // The random IV used to encrypt the object-key. + Algorithm string // The sealing algorithm used to encrypt the object key. +} + +// Seal encrypts the ObjectKey using the 256 bit external key and IV. The sealed +// key is also cryptographically bound to the object's path (bucket/object) and the +// domain (SSE-C or SSE-S3). +func (key ObjectKey) Seal(extKey []byte, iv [32]byte, domain, bucket, object string) SealedKey { + if len(extKey) != 32 { + logger.CriticalIf(context.Background(), errors.New("crypto: invalid key length")) + } + var ( + sealingKey [32]byte + encryptedKey bytes.Buffer + ) + mac := hmac.New(sha256.New, extKey) + mac.Write(iv[:]) + mac.Write([]byte(domain)) + mac.Write([]byte(SealAlgorithm)) + mac.Write([]byte(path.Join(bucket, object))) // use path.Join for canonical 'bucket/object' + mac.Sum(sealingKey[:0]) + if n, err := sio.Encrypt(&encryptedKey, bytes.NewReader(key[:]), sio.Config{Key: sealingKey[:]}); n != 64 || err != nil { + logger.CriticalIf(context.Background(), errors.New("Unable to generate sealed key")) + } + sealedKey := SealedKey{ + IV: iv, + Algorithm: SealAlgorithm, + } + copy(sealedKey.Key[:], encryptedKey.Bytes()) + return sealedKey +} + +// Unseal decrypts a sealed key using the 256 bit external key. Since the sealed key +// may be cryptographically bound to the object's path the same bucket/object as during sealing +// must be provided. On success the ObjectKey contains the decrypted sealed key. +func (key *ObjectKey) Unseal(extKey []byte, sealedKey SealedKey, domain, bucket, object string) error { + var unsealConfig sio.Config + switch sealedKey.Algorithm { + default: + return Errorf("The sealing algorithm '%s' is not supported", sealedKey.Algorithm) + case SealAlgorithm: + mac := hmac.New(sha256.New, extKey) + mac.Write(sealedKey.IV[:]) + mac.Write([]byte(domain)) + mac.Write([]byte(SealAlgorithm)) + mac.Write([]byte(path.Join(bucket, object))) // use path.Join for canonical 'bucket/object' + unsealConfig = sio.Config{MinVersion: sio.Version20, Key: mac.Sum(nil)} + case InsecureSealAlgorithm: + sha := sha256.New() + sha.Write(extKey) + sha.Write(sealedKey.IV[:]) + unsealConfig = sio.Config{MinVersion: sio.Version10, Key: sha.Sum(nil)} + } + + if out, err := sio.DecryptBuffer(key[:0], sealedKey.Key[:], unsealConfig); len(out) != 32 || err != nil { + return ErrSecretKeyMismatch + } + return nil +} + +// DerivePartKey derives an unique 256 bit key from an ObjectKey and the part index. +func (key ObjectKey) DerivePartKey(id uint32) (partKey [32]byte) { + var bin [4]byte + binary.LittleEndian.PutUint32(bin[:], id) + + mac := hmac.New(sha256.New, key[:]) + mac.Write(bin[:]) + mac.Sum(partKey[:0]) + return partKey +} + +// SealETag seals the etag using the object key. +// It does not encrypt empty ETags because such ETags indicate +// that the S3 client hasn't sent an ETag = MD5(object) and +// the backend can pick an ETag value. +func (key ObjectKey) SealETag(etag []byte) []byte { + if len(etag) == 0 { // don't encrypt empty ETag - only if client sent ETag = MD5(object) + return etag + } + var buffer bytes.Buffer + mac := hmac.New(sha256.New, key[:]) + mac.Write([]byte("SSE-etag")) + if _, err := sio.Encrypt(&buffer, bytes.NewReader(etag), sio.Config{Key: mac.Sum(nil)}); err != nil { + logger.CriticalIf(context.Background(), errors.New("Unable to encrypt ETag using object key")) + } + return buffer.Bytes() +} + +// UnsealETag unseals the etag using the provided object key. +// It does not try to decrypt the ETag if len(etag) == 16 +// because such ETags indicate that the S3 client hasn't sent +// an ETag = MD5(object) and the backend has picked an ETag value. +func (key ObjectKey) UnsealETag(etag []byte) ([]byte, error) { + if !IsETagSealed(etag) { + return etag, nil + } + mac := hmac.New(sha256.New, key[:]) + mac.Write([]byte("SSE-etag")) + return sio.DecryptBuffer(make([]byte, 0, len(etag)), etag, sio.Config{Key: mac.Sum(nil)}) +} diff --git a/internal/crypto/key_test.go b/internal/crypto/key_test.go new file mode 100644 index 00000000..bf15fd88 --- /dev/null +++ b/internal/crypto/key_test.go @@ -0,0 +1,199 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package crypto + +import ( + "bytes" + "crypto/rand" + "encoding/hex" + "io" + "testing" + + "github.com/minio/minio/internal/logger" +) + +var shortRandom = func(limit int64) io.Reader { return io.LimitReader(rand.Reader, limit) } + +func recoverTest(i int, shouldPass bool, t *testing.T) { + if err := recover(); err == nil && !shouldPass { + t.Errorf("Test %d should fail but passed successfully", i) + } else if err != nil && shouldPass { + t.Errorf("Test %d should pass but failed: %v", i, err) + } +} + +var generateKeyTests = []struct { + ExtKey [32]byte + Random io.Reader + ShouldPass bool +}{ + {ExtKey: [32]byte{}, Random: nil, ShouldPass: true}, // 0 + {ExtKey: [32]byte{}, Random: rand.Reader, ShouldPass: true}, // 1 + {ExtKey: [32]byte{}, Random: shortRandom(32), ShouldPass: true}, // 2 + {ExtKey: [32]byte{}, Random: shortRandom(31), ShouldPass: false}, // 3 +} + +func TestGenerateKey(t *testing.T) { + defer func(l bool) { logger.DisableLog = l }(logger.DisableLog) + logger.DisableLog = true + + for i, test := range generateKeyTests { + i, test := i, test + func() { + defer recoverTest(i, test.ShouldPass, t) + key := GenerateKey(test.ExtKey[:], test.Random) + if [32]byte(key) == [32]byte{} { + t.Errorf("Test %d: generated key is zero key", i) // check that we generate random and unique key + } + }() + } +} + +var generateIVTests = []struct { + Random io.Reader + ShouldPass bool +}{ + {Random: nil, ShouldPass: true}, // 0 + {Random: rand.Reader, ShouldPass: true}, // 1 + {Random: shortRandom(32), ShouldPass: true}, // 2 + {Random: shortRandom(31), ShouldPass: false}, // 3 +} + +func TestGenerateIV(t *testing.T) { + defer func(l bool) { logger.DisableLog = l }(logger.DisableLog) + logger.DisableLog = true + + for i, test := range generateIVTests { + i, test := i, test + func() { + defer recoverTest(i, test.ShouldPass, t) + iv := GenerateIV(test.Random) + if iv == [32]byte{} { + t.Errorf("Test %d: generated IV is zero IV", i) // check that we generate random and unique IV + } + }() + } +} + +var sealUnsealKeyTests = []struct { + SealExtKey, SealIV [32]byte + SealDomain, SealBucket, SealObject string + + UnsealExtKey [32]byte + UnsealDomain, UnsealBucket, UnsealObject string + + ShouldPass bool +}{ + { + SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-C", SealBucket: "bucket", SealObject: "object", + UnsealExtKey: [32]byte{}, UnsealDomain: "SSE-C", UnsealBucket: "bucket", UnsealObject: "object", + ShouldPass: true, + }, // 0 + { + SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-C", SealBucket: "bucket", SealObject: "object", + UnsealExtKey: [32]byte{1}, UnsealDomain: "SSE-C", UnsealBucket: "bucket", UnsealObject: "object", // different ext-key + ShouldPass: false, + }, // 1 + { + SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-S3", SealBucket: "bucket", SealObject: "object", + UnsealExtKey: [32]byte{}, UnsealDomain: "SSE-C", UnsealBucket: "bucket", UnsealObject: "object", // different domain + ShouldPass: false, + }, // 2 + { + SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-C", SealBucket: "bucket", SealObject: "object", + UnsealExtKey: [32]byte{}, UnsealDomain: "SSE-C", UnsealBucket: "Bucket", UnsealObject: "object", // different bucket + ShouldPass: false, + }, // 3 + { + SealExtKey: [32]byte{}, SealIV: [32]byte{}, SealDomain: "SSE-C", SealBucket: "bucket", SealObject: "object", + UnsealExtKey: [32]byte{}, UnsealDomain: "SSE-C", UnsealBucket: "bucket", UnsealObject: "Object", // different object + ShouldPass: false, + }, // 4 +} + +func TestSealUnsealKey(t *testing.T) { + for i, test := range sealUnsealKeyTests { + key := GenerateKey(test.SealExtKey[:], rand.Reader) + sealedKey := key.Seal(test.SealExtKey[:], test.SealIV, test.SealDomain, test.SealBucket, test.SealObject) + if err := key.Unseal(test.UnsealExtKey[:], sealedKey, test.UnsealDomain, test.UnsealBucket, test.UnsealObject); err == nil && !test.ShouldPass { + t.Errorf("Test %d should fail but passed successfully", i) + } else if err != nil && test.ShouldPass { + t.Errorf("Test %d should pass put failed: %v", i, err) + } + } + + // Test legacy InsecureSealAlgorithm + var extKey, iv [32]byte + key := GenerateKey(extKey[:], rand.Reader) + sealedKey := key.Seal(extKey[:], iv, "SSE-S3", "bucket", "object") + sealedKey.Algorithm = InsecureSealAlgorithm + if err := key.Unseal(extKey[:], sealedKey, "SSE-S3", "bucket", "object"); err == nil { + t.Errorf("'%s' test succeeded but it should fail because the legacy algorithm was used", sealedKey.Algorithm) + } +} + +var derivePartKeyTest = []struct { + PartID uint32 + PartKey string +}{ + {PartID: 0, PartKey: "aa7855e13839dd767cd5da7c1ff5036540c9264b7a803029315e55375287b4af"}, + {PartID: 1, PartKey: "a3e7181c6eed030fd52f79537c56c4d07da92e56d374ff1dd2043350785b37d8"}, + {PartID: 10000, PartKey: "f86e65c396ed52d204ee44bd1a0bbd86eb8b01b7354e67a3b3ae0e34dd5bd115"}, +} + +func TestDerivePartKey(t *testing.T) { + var key ObjectKey + for i, test := range derivePartKeyTest { + expectedPartKey, err := hex.DecodeString(test.PartKey) + if err != nil { + t.Fatalf("Test %d failed to decode expected part-key: %v", i, err) + } + partKey := key.DerivePartKey(test.PartID) + if !bytes.Equal(partKey[:], expectedPartKey) { + t.Errorf("Test %d derives wrong part-key: got '%s' want: '%s'", i, hex.EncodeToString(partKey[:]), test.PartKey) + } + } +} + +var sealUnsealETagTests = []string{ + "", + "90682b8e8cc7609c", + "90682b8e8cc7609c4671e1d64c73fc30", + "90682b8e8cc7609c4671e1d64c73fc307fb3104f", +} + +func TestSealETag(t *testing.T) { + var key ObjectKey + for i := range key { + key[i] = byte(i) + } + for i, etag := range sealUnsealETagTests { + tag, err := hex.DecodeString(etag) + if err != nil { + t.Errorf("Test %d: failed to decode etag: %s", i, err) + } + sealedETag := key.SealETag(tag) + unsealedETag, err := key.UnsealETag(sealedETag) + if err != nil { + t.Errorf("Test %d: failed to decrypt etag: %s", i, err) + } + if !bytes.Equal(unsealedETag, tag) { + t.Errorf("Test %d: unsealed etag does not match: got %s - want %s", i, hex.EncodeToString(unsealedETag), etag) + } + } +} diff --git a/internal/crypto/metadata.go b/internal/crypto/metadata.go new file mode 100644 index 00000000..43e8cfe4 --- /dev/null +++ b/internal/crypto/metadata.go @@ -0,0 +1,174 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package crypto + +import ( + xhttp "github.com/minio/minio/internal/http" +) + +const ( + // MetaMultipart indicates that the object has been uploaded + // in multiple parts - via the S3 multipart API. + MetaMultipart = "X-Minio-Internal-Encrypted-Multipart" + + // MetaIV is the random initialization vector (IV) used for + // the MinIO-internal key derivation. + MetaIV = "X-Minio-Internal-Server-Side-Encryption-Iv" + + // MetaAlgorithm is the algorithm used to derive internal keys + // and encrypt the objects. + MetaAlgorithm = "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm" + + // MetaSealedKeySSEC is the sealed object encryption key in case of SSE-C. + MetaSealedKeySSEC = "X-Minio-Internal-Server-Side-Encryption-Sealed-Key" + // MetaSealedKeyS3 is the sealed object encryption key in case of SSE-S3 + MetaSealedKeyS3 = "X-Minio-Internal-Server-Side-Encryption-S3-Sealed-Key" + // MetaSealedKeyKMS is the sealed object encryption key in case of SSE-KMS + MetaSealedKeyKMS = "X-Minio-Internal-Server-Side-Encryption-Kms-Sealed-Key" + + // MetaKeyID is the KMS master key ID used to generate/encrypt the data + // encryption key (DEK). + MetaKeyID = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Key-Id" + // MetaDataEncryptionKey is the sealed data encryption key (DEK) received from + // the KMS. + MetaDataEncryptionKey = "X-Minio-Internal-Server-Side-Encryption-S3-Kms-Sealed-Key" + + // MetaSsecCRC is the encrypted checksum of the SSE-C encrypted object. + MetaSsecCRC = "X-Minio-Replication-Ssec-Crc" + + // MetaContext is the KMS context provided by a client when encrypting an + // object with SSE-KMS. A client may not send a context in which case the + // MetaContext will not be present. + // MetaContext only contains the bucket/object name if the client explicitly + // added it. However, when decrypting an object the bucket/object name must + // be part of the object. Therefore, the bucket/object name must be added + // to the context, if not present, whenever a decryption is performed. + MetaContext = "X-Minio-Internal-Server-Side-Encryption-Context" + + // ARNPrefix prefix for "arn:aws:kms" + ARNPrefix = "arn:aws:kms:" +) + +// IsMultiPart returns true if the object metadata indicates +// that it was uploaded using some form of server-side-encryption +// and the S3 multipart API. +func IsMultiPart(metadata map[string]string) bool { + if _, ok := metadata[MetaMultipart]; ok { + return true + } + return false +} + +// RemoveSensitiveEntries removes confidential encryption +// information - e.g. the SSE-C key - from the metadata map. +// It has the same semantics as RemoveSensitiveHeaders. +func RemoveSensitiveEntries(metadata map[string]string) { // The functions is tested in TestRemoveSensitiveHeaders for compatibility reasons + delete(metadata, xhttp.AmzServerSideEncryptionCustomerKey) + delete(metadata, xhttp.AmzServerSideEncryptionCopyCustomerKey) + delete(metadata, xhttp.AmzMetaUnencryptedContentLength) + delete(metadata, xhttp.AmzMetaUnencryptedContentMD5) +} + +// RemoveSSEHeaders removes all crypto-specific SSE +// header entries from the metadata map. +func RemoveSSEHeaders(metadata map[string]string) { + delete(metadata, xhttp.AmzServerSideEncryption) + delete(metadata, xhttp.AmzServerSideEncryptionKmsID) + delete(metadata, xhttp.AmzServerSideEncryptionKmsContext) + delete(metadata, xhttp.AmzServerSideEncryptionCustomerAlgorithm) + delete(metadata, xhttp.AmzServerSideEncryptionCustomerKey) + delete(metadata, xhttp.AmzServerSideEncryptionCustomerKeyMD5) + delete(metadata, xhttp.AmzServerSideEncryptionCopyCustomerAlgorithm) + delete(metadata, xhttp.AmzServerSideEncryptionCopyCustomerKey) + delete(metadata, xhttp.AmzServerSideEncryptionCopyCustomerKeyMD5) +} + +// RemoveInternalEntries removes all crypto-specific internal +// metadata entries from the metadata map. +func RemoveInternalEntries(metadata map[string]string) { + delete(metadata, MetaMultipart) + delete(metadata, MetaAlgorithm) + delete(metadata, MetaIV) + delete(metadata, MetaSealedKeySSEC) + delete(metadata, MetaSealedKeyS3) + delete(metadata, MetaSealedKeyKMS) + delete(metadata, MetaKeyID) + delete(metadata, MetaDataEncryptionKey) + delete(metadata, MetaSsecCRC) +} + +// IsSourceEncrypted returns true if the source is encrypted +func IsSourceEncrypted(metadata map[string]string) bool { + if _, ok := metadata[xhttp.AmzServerSideEncryptionCustomerAlgorithm]; ok { + return true + } + if _, ok := metadata[xhttp.AmzServerSideEncryption]; ok { + return true + } + return false +} + +// IsEncrypted returns true if the object metadata indicates +// that it was uploaded using some form of server-side-encryption. +// +// IsEncrypted only checks whether the metadata contains at least +// one entry indicating SSE-C or SSE-S3. +func IsEncrypted(metadata map[string]string) (Type, bool) { + if S3KMS.IsEncrypted(metadata) { + return S3KMS, true + } + if S3.IsEncrypted(metadata) { + return S3, true + } + if SSEC.IsEncrypted(metadata) { + return SSEC, true + } + if IsMultiPart(metadata) { + return nil, true + } + if _, ok := metadata[MetaIV]; ok { + return nil, true + } + if _, ok := metadata[MetaAlgorithm]; ok { + return nil, true + } + if _, ok := metadata[MetaKeyID]; ok { + return nil, true + } + if _, ok := metadata[MetaDataEncryptionKey]; ok { + return nil, true + } + if _, ok := metadata[MetaContext]; ok { + return nil, true + } + return nil, false +} + +// CreateMultipartMetadata adds the multipart flag entry to metadata +// and returns modified metadata. It allocates a new metadata map if +// metadata is nil. +func CreateMultipartMetadata(metadata map[string]string) map[string]string { + if metadata == nil { + return map[string]string{MetaMultipart: ""} + } + metadata[MetaMultipart] = "" + return metadata +} + +// IsETagSealed returns true if the etag seems to be encrypted. +func IsETagSealed(etag []byte) bool { return len(etag) > 16 } diff --git a/internal/crypto/metadata_test.go b/internal/crypto/metadata_test.go new file mode 100644 index 00000000..612cf19c --- /dev/null +++ b/internal/crypto/metadata_test.go @@ -0,0 +1,459 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package crypto + +import ( + "bytes" + "encoding/base64" + "encoding/hex" + "testing" + + "github.com/minio/minio/internal/logger" +) + +var isMultipartTests = []struct { + Metadata map[string]string + Multipart bool +}{ + {Multipart: true, Metadata: map[string]string{MetaMultipart: ""}}, // 0 + {Multipart: true, Metadata: map[string]string{"X-Minio-Internal-Encrypted-Multipart": ""}}, // 1 + {Multipart: true, Metadata: map[string]string{MetaMultipart: "some-value"}}, // 2 + {Multipart: false, Metadata: map[string]string{"": ""}}, // 3 + {Multipart: false, Metadata: map[string]string{"X-Minio-Internal-EncryptedMultipart": ""}}, // 4 +} + +func TestIsMultipart(t *testing.T) { + for i, test := range isMultipartTests { + if isMultipart := IsMultiPart(test.Metadata); isMultipart != test.Multipart { + t.Errorf("Test %d: got '%v' - want '%v'", i, isMultipart, test.Multipart) + } + } +} + +var isEncryptedTests = []struct { + Metadata map[string]string + Encrypted bool +}{ + {Encrypted: true, Metadata: map[string]string{MetaMultipart: ""}}, // 0 + {Encrypted: true, Metadata: map[string]string{MetaIV: ""}}, // 1 + {Encrypted: true, Metadata: map[string]string{MetaAlgorithm: ""}}, // 2 + {Encrypted: true, Metadata: map[string]string{MetaSealedKeySSEC: ""}}, // 3 + {Encrypted: true, Metadata: map[string]string{MetaSealedKeyS3: ""}}, // 4 + {Encrypted: true, Metadata: map[string]string{MetaKeyID: ""}}, // 5 + {Encrypted: true, Metadata: map[string]string{MetaDataEncryptionKey: ""}}, // 6 + {Encrypted: false, Metadata: map[string]string{"": ""}}, // 7 + {Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8 +} + +func TestIsEncrypted(t *testing.T) { + for i, test := range isEncryptedTests { + if _, isEncrypted := IsEncrypted(test.Metadata); isEncrypted != test.Encrypted { + t.Errorf("Test %d: got '%v' - want '%v'", i, isEncrypted, test.Encrypted) + } + } +} + +var s3IsEncryptedTests = []struct { + Metadata map[string]string + Encrypted bool +}{ + {Encrypted: false, Metadata: map[string]string{MetaMultipart: ""}}, // 0 + {Encrypted: false, Metadata: map[string]string{MetaIV: ""}}, // 1 + {Encrypted: false, Metadata: map[string]string{MetaAlgorithm: ""}}, // 2 + {Encrypted: false, Metadata: map[string]string{MetaSealedKeySSEC: ""}}, // 3 + {Encrypted: true, Metadata: map[string]string{MetaSealedKeyS3: ""}}, // 4 + {Encrypted: false, Metadata: map[string]string{MetaKeyID: ""}}, // 5 + {Encrypted: false, Metadata: map[string]string{MetaDataEncryptionKey: ""}}, // 6 + {Encrypted: false, Metadata: map[string]string{"": ""}}, // 7 + {Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8 +} + +func TestS3IsEncrypted(t *testing.T) { + for i, test := range s3IsEncryptedTests { + if isEncrypted := S3.IsEncrypted(test.Metadata); isEncrypted != test.Encrypted { + t.Errorf("Test %d: got '%v' - want '%v'", i, isEncrypted, test.Encrypted) + } + } +} + +var ssecIsEncryptedTests = []struct { + Metadata map[string]string + Encrypted bool +}{ + {Encrypted: false, Metadata: map[string]string{MetaMultipart: ""}}, // 0 + {Encrypted: false, Metadata: map[string]string{MetaIV: ""}}, // 1 + {Encrypted: false, Metadata: map[string]string{MetaAlgorithm: ""}}, // 2 + {Encrypted: true, Metadata: map[string]string{MetaSealedKeySSEC: ""}}, // 3 + {Encrypted: false, Metadata: map[string]string{MetaSealedKeyS3: ""}}, // 4 + {Encrypted: false, Metadata: map[string]string{MetaKeyID: ""}}, // 5 + {Encrypted: false, Metadata: map[string]string{MetaDataEncryptionKey: ""}}, // 6 + {Encrypted: false, Metadata: map[string]string{"": ""}}, // 7 + {Encrypted: false, Metadata: map[string]string{"X-Minio-Internal-Server-Side-Encryption": ""}}, // 8 +} + +func TestSSECIsEncrypted(t *testing.T) { + for i, test := range ssecIsEncryptedTests { + if isEncrypted := SSEC.IsEncrypted(test.Metadata); isEncrypted != test.Encrypted { + t.Errorf("Test %d: got '%v' - want '%v'", i, isEncrypted, test.Encrypted) + } + } +} + +var s3ParseMetadataTests = []struct { + Metadata map[string]string + ExpectedErr error + + DataKey []byte + KeyID string + SealedKey SealedKey +}{ + {ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{}}, // 0 + { + ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{MetaIV: ""}, + DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{}, + }, // 1 + { + ExpectedErr: Errorf("The object metadata is missing the internal sealed key for SSE-S3"), + Metadata: map[string]string{MetaIV: "", MetaAlgorithm: ""}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{}, + }, // 2 + { + ExpectedErr: Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3"), + Metadata: map[string]string{MetaIV: "", MetaAlgorithm: "", MetaSealedKeyS3: "", MetaDataEncryptionKey: "IAAF0b=="}, DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{}, + }, // 3 + { + ExpectedErr: Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3"), + Metadata: map[string]string{MetaIV: "", MetaAlgorithm: "", MetaSealedKeyS3: "", MetaKeyID: ""}, + DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{}, + }, // 4 + { + ExpectedErr: errInvalidInternalIV, + Metadata: map[string]string{MetaIV: "", MetaAlgorithm: "", MetaSealedKeyS3: "", MetaKeyID: "", MetaDataEncryptionKey: ""}, + DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{}, + }, // 5 + { + ExpectedErr: errInvalidInternalSealAlgorithm, + Metadata: map[string]string{ + MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: "", MetaSealedKeyS3: "", MetaKeyID: "", MetaDataEncryptionKey: "", + }, + DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{}, + }, // 6 + { + ExpectedErr: Errorf("The internal sealed key for SSE-S3 is invalid"), + Metadata: map[string]string{ + MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm, MetaSealedKeyS3: "", + MetaKeyID: "", MetaDataEncryptionKey: "", + }, + DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{}, + }, // 7 + { + ExpectedErr: Errorf("The internal sealed KMS data key for SSE-S3 is invalid"), + Metadata: map[string]string{ + MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm, + MetaSealedKeyS3: base64.StdEncoding.EncodeToString(make([]byte, 64)), MetaKeyID: "key-1", + MetaDataEncryptionKey: ".MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ=", // invalid base64 + }, + DataKey: []byte{}, KeyID: "key-1", SealedKey: SealedKey{}, + }, // 8 + { + ExpectedErr: nil, + Metadata: map[string]string{ + MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm, + MetaSealedKeyS3: base64.StdEncoding.EncodeToString(make([]byte, 64)), MetaKeyID: "", MetaDataEncryptionKey: "", + }, + DataKey: []byte{}, KeyID: "", SealedKey: SealedKey{Algorithm: SealAlgorithm}, + }, // 9 + { + ExpectedErr: nil, + Metadata: map[string]string{ + MetaIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), MetaAlgorithm: SealAlgorithm, + MetaSealedKeyS3: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)), MetaKeyID: "key-1", + MetaDataEncryptionKey: base64.StdEncoding.EncodeToString(make([]byte, 48)), + }, + DataKey: make([]byte, 48), KeyID: "key-1", SealedKey: SealedKey{Algorithm: SealAlgorithm, Key: [64]byte{1}, IV: [32]byte{1}}, + }, // 10 +} + +func TestS3ParseMetadata(t *testing.T) { + for i, test := range s3ParseMetadataTests { + keyID, dataKey, sealedKey, err := S3.ParseMetadata(test.Metadata) + if err != nil && test.ExpectedErr == nil { + t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr) + } + if err == nil && test.ExpectedErr != nil { + t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr) + } + if err != nil && test.ExpectedErr != nil { + if err.Error() != test.ExpectedErr.Error() { + t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr) + } + } + if !bytes.Equal(dataKey, test.DataKey) { + t.Errorf("Test %d: got data key '%v' - want data key '%v'", i, dataKey, test.DataKey) + } + if keyID != test.KeyID { + t.Errorf("Test %d: got key-ID '%v' - want key-ID '%v'", i, keyID, test.KeyID) + } + if sealedKey.Algorithm != test.SealedKey.Algorithm { + t.Errorf("Test %d: got sealed key algorithm '%v' - want sealed key algorithm '%v'", i, sealedKey.Algorithm, test.SealedKey.Algorithm) + } + if !bytes.Equal(sealedKey.Key[:], test.SealedKey.Key[:]) { + t.Errorf("Test %d: got sealed key '%v' - want sealed key '%v'", i, sealedKey.Key, test.SealedKey.Key) + } + if !bytes.Equal(sealedKey.IV[:], test.SealedKey.IV[:]) { + t.Errorf("Test %d: got sealed key IV '%v' - want sealed key IV '%v'", i, sealedKey.IV, test.SealedKey.IV) + } + } +} + +var ssecParseMetadataTests = []struct { + Metadata map[string]string + ExpectedErr error + + SealedKey SealedKey +}{ + {ExpectedErr: errMissingInternalIV, Metadata: map[string]string{}, SealedKey: SealedKey{}}, // 0 + {ExpectedErr: errMissingInternalSealAlgorithm, Metadata: map[string]string{MetaIV: ""}, SealedKey: SealedKey{}}, // 1 + { + ExpectedErr: Errorf("The object metadata is missing the internal sealed key for SSE-C"), + Metadata: map[string]string{MetaIV: "", MetaAlgorithm: ""}, SealedKey: SealedKey{}, + }, // 2 + { + ExpectedErr: errInvalidInternalIV, + Metadata: map[string]string{MetaIV: "", MetaAlgorithm: "", MetaSealedKeySSEC: ""}, SealedKey: SealedKey{}, + }, // 3 + { + ExpectedErr: errInvalidInternalSealAlgorithm, + Metadata: map[string]string{ + MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: "", MetaSealedKeySSEC: "", + }, + SealedKey: SealedKey{}, + }, // 4 + { + ExpectedErr: Errorf("The internal sealed key for SSE-C is invalid"), + Metadata: map[string]string{ + MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm, MetaSealedKeySSEC: "", + }, + SealedKey: SealedKey{}, + }, // 5 + { + ExpectedErr: nil, + Metadata: map[string]string{ + MetaIV: base64.StdEncoding.EncodeToString(make([]byte, 32)), MetaAlgorithm: SealAlgorithm, + MetaSealedKeySSEC: base64.StdEncoding.EncodeToString(make([]byte, 64)), + }, + SealedKey: SealedKey{Algorithm: SealAlgorithm}, + }, // 6 + { + ExpectedErr: nil, + Metadata: map[string]string{ + MetaIV: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 31)...)), MetaAlgorithm: InsecureSealAlgorithm, + MetaSealedKeySSEC: base64.StdEncoding.EncodeToString(append([]byte{1}, make([]byte, 63)...)), + }, + SealedKey: SealedKey{Algorithm: InsecureSealAlgorithm, Key: [64]byte{1}, IV: [32]byte{1}}, + }, // 7 +} + +func TestCreateMultipartMetadata(t *testing.T) { + metadata := CreateMultipartMetadata(nil) + if v, ok := metadata[MetaMultipart]; !ok || v != "" { + t.Errorf("Metadata is missing the correct value for '%s': got '%s' - want '%s'", MetaMultipart, v, "") + } +} + +func TestSSECParseMetadata(t *testing.T) { + for i, test := range ssecParseMetadataTests { + sealedKey, err := SSEC.ParseMetadata(test.Metadata) + if err != nil && test.ExpectedErr == nil { + t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr) + } + if err == nil && test.ExpectedErr != nil { + t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr) + } + if err != nil && test.ExpectedErr != nil { + if err.Error() != test.ExpectedErr.Error() { + t.Errorf("Test %d: got error '%v' - want error '%v'", i, err, test.ExpectedErr) + } + } + if sealedKey.Algorithm != test.SealedKey.Algorithm { + t.Errorf("Test %d: got sealed key algorithm '%v' - want sealed key algorithm '%v'", i, sealedKey.Algorithm, test.SealedKey.Algorithm) + } + if !bytes.Equal(sealedKey.Key[:], test.SealedKey.Key[:]) { + t.Errorf("Test %d: got sealed key '%v' - want sealed key '%v'", i, sealedKey.Key, test.SealedKey.Key) + } + if !bytes.Equal(sealedKey.IV[:], test.SealedKey.IV[:]) { + t.Errorf("Test %d: got sealed key IV '%v' - want sealed key IV '%v'", i, sealedKey.IV, test.SealedKey.IV) + } + } +} + +var s3CreateMetadataTests = []struct { + KeyID string + SealedDataKey []byte + SealedKey SealedKey +}{ + {KeyID: "", SealedDataKey: nil, SealedKey: SealedKey{Algorithm: SealAlgorithm}}, + {KeyID: "my-minio-key", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}}, + {KeyID: "cafebabe", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}}, + {KeyID: "deadbeef", SealedDataKey: make([]byte, 32), SealedKey: SealedKey{IV: [32]byte{0xf7}, Key: [64]byte{0xea}, Algorithm: SealAlgorithm}}, +} + +func TestS3CreateMetadata(t *testing.T) { + defer func(l bool) { logger.DisableLog = l }(logger.DisableLog) + logger.DisableLog = true + for i, test := range s3CreateMetadataTests { + metadata := S3.CreateMetadata(nil, test.KeyID, test.SealedDataKey, test.SealedKey) + keyID, kmsKey, sealedKey, err := S3.ParseMetadata(metadata) + if err != nil { + t.Errorf("Test %d: failed to parse metadata: %v", i, err) + continue + } + if keyID != test.KeyID { + t.Errorf("Test %d: Key-ID mismatch: got '%s' - want '%s'", i, keyID, test.KeyID) + } + if !bytes.Equal(kmsKey, test.SealedDataKey) { + t.Errorf("Test %d: sealed KMS data mismatch: got '%v' - want '%v'", i, kmsKey, test.SealedDataKey) + } + if sealedKey.Algorithm != test.SealedKey.Algorithm { + t.Errorf("Test %d: seal algorithm mismatch: got '%s' - want '%s'", i, sealedKey.Algorithm, test.SealedKey.Algorithm) + } + if !bytes.Equal(sealedKey.IV[:], test.SealedKey.IV[:]) { + t.Errorf("Test %d: IV mismatch: got '%v' - want '%v'", i, sealedKey.IV, test.SealedKey.IV) + } + if !bytes.Equal(sealedKey.Key[:], test.SealedKey.Key[:]) { + t.Errorf("Test %d: sealed key mismatch: got '%v' - want '%v'", i, sealedKey.Key, test.SealedKey.Key) + } + } + + defer func() { + if err := recover(); err == nil || err != logger.ErrCritical { + t.Errorf("Expected '%s' panic for invalid seal algorithm but got '%s'", logger.ErrCritical, err) + } + }() + _ = S3.CreateMetadata(nil, "", []byte{}, SealedKey{Algorithm: InsecureSealAlgorithm}) +} + +var ssecCreateMetadataTests = []struct { + KeyID string + SealedDataKey []byte + SealedKey SealedKey +}{ + {KeyID: "", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}}, + {KeyID: "cafebabe", SealedDataKey: make([]byte, 48), SealedKey: SealedKey{Algorithm: SealAlgorithm}}, + {KeyID: "deadbeef", SealedDataKey: make([]byte, 32), SealedKey: SealedKey{IV: [32]byte{0xf7}, Key: [64]byte{0xea}, Algorithm: SealAlgorithm}}, +} + +func TestSSECCreateMetadata(t *testing.T) { + defer func(l bool) { logger.DisableLog = l }(logger.DisableLog) + logger.DisableLog = true + for i, test := range ssecCreateMetadataTests { + metadata := SSEC.CreateMetadata(nil, test.SealedKey) + sealedKey, err := SSEC.ParseMetadata(metadata) + if err != nil { + t.Errorf("Test %d: failed to parse metadata: %v", i, err) + continue + } + if sealedKey.Algorithm != test.SealedKey.Algorithm { + t.Errorf("Test %d: seal algorithm mismatch: got '%s' - want '%s'", i, sealedKey.Algorithm, test.SealedKey.Algorithm) + } + if !bytes.Equal(sealedKey.IV[:], test.SealedKey.IV[:]) { + t.Errorf("Test %d: IV mismatch: got '%v' - want '%v'", i, sealedKey.IV, test.SealedKey.IV) + } + if !bytes.Equal(sealedKey.Key[:], test.SealedKey.Key[:]) { + t.Errorf("Test %d: sealed key mismatch: got '%v' - want '%v'", i, sealedKey.Key, test.SealedKey.Key) + } + } + + defer func() { + if err := recover(); err == nil || err != logger.ErrCritical { + t.Errorf("Expected '%s' panic for invalid seal algorithm but got '%s'", logger.ErrCritical, err) + } + }() + _ = SSEC.CreateMetadata(nil, SealedKey{Algorithm: InsecureSealAlgorithm}) +} + +var isETagSealedTests = []struct { + ETag string + IsSealed bool +}{ + {ETag: "", IsSealed: false}, // 0 + {ETag: "90682b8e8cc7609c4671e1d64c73fc30", IsSealed: false}, // 1 + {ETag: "f201040c9dc593e39ea004dc1323699bcd", IsSealed: true}, // 2 not valid ciphertext but looks like sealed ETag + {ETag: "20000f00fba2ee2ae4845f725964eeb9e092edfabc7ab9f9239e8344341f769a51ce99b4801b0699b92b16a72fa94972", IsSealed: true}, // 3 +} + +func TestIsETagSealed(t *testing.T) { + for i, test := range isETagSealedTests { + etag, err := hex.DecodeString(test.ETag) + if err != nil { + t.Errorf("Test %d: failed to decode etag: %s", i, err) + } + if sealed := IsETagSealed(etag); sealed != test.IsSealed { + t.Errorf("Test %d: got %v - want %v", i, sealed, test.IsSealed) + } + } +} + +var removeInternalEntriesTests = []struct { + Metadata, Expected map[string]string +}{ + { // 0 + Metadata: map[string]string{ + MetaMultipart: "", + MetaIV: "", + MetaAlgorithm: "", + MetaSealedKeySSEC: "", + MetaSealedKeyS3: "", + MetaKeyID: "", + MetaDataEncryptionKey: "", + }, + Expected: map[string]string{}, + }, + { // 1 + Metadata: map[string]string{ + MetaMultipart: "", + MetaIV: "", + "X-Amz-Meta-A": "X", + "X-Minio-Internal-B": "Y", + }, + Expected: map[string]string{ + "X-Amz-Meta-A": "X", + "X-Minio-Internal-B": "Y", + }, + }, +} + +func TestRemoveInternalEntries(t *testing.T) { + isEqual := func(x, y map[string]string) bool { + if len(x) != len(y) { + return false + } + for k, v := range x { + if u, ok := y[k]; !ok || v != u { + return false + } + } + return true + } + + for i, test := range removeInternalEntriesTests { + RemoveInternalEntries(test.Metadata) + if !isEqual(test.Metadata, test.Expected) { + t.Errorf("Test %d: got %v - want %v", i, test.Metadata, test.Expected) + } + } +} diff --git a/internal/crypto/sse-c.go b/internal/crypto/sse-c.go new file mode 100644 index 00000000..2b96fc02 --- /dev/null +++ b/internal/crypto/sse-c.go @@ -0,0 +1,158 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package crypto + +import ( + "bytes" + "context" + "crypto/md5" + "encoding/base64" + "net/http" + + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" +) + +type ssec struct{} + +var ( + // SSEC represents AWS SSE-C. It provides functionality to handle + // SSE-C requests. + SSEC = ssec{} + + _ Type = SSEC +) + +// String returns the SSE domain as string. For SSE-C the +// domain is "SSE-C". +func (ssec) String() string { return "SSE-C" } + +// IsRequested returns true if the HTTP headers contains +// at least one SSE-C header. SSE-C copy headers are ignored. +func (ssec) IsRequested(h http.Header) bool { + if _, ok := h[xhttp.AmzServerSideEncryptionCustomerAlgorithm]; ok { + return true + } + if _, ok := h[xhttp.AmzServerSideEncryptionCustomerKey]; ok { + return true + } + if _, ok := h[xhttp.AmzServerSideEncryptionCustomerKeyMD5]; ok { + return true + } + return false +} + +// IsEncrypted returns true if the metadata contains an SSE-C +// entry indicating that the object has been encrypted using +// SSE-C. +func (ssec) IsEncrypted(metadata map[string]string) bool { + if _, ok := metadata[MetaSealedKeySSEC]; ok { + return true + } + return false +} + +// ParseHTTP parses the SSE-C headers and returns the SSE-C client key +// on success. SSE-C copy headers are ignored. +func (ssec) ParseHTTP(h http.Header) (key [32]byte, err error) { + if h.Get(xhttp.AmzServerSideEncryptionCustomerAlgorithm) != xhttp.AmzEncryptionAES { + return key, ErrInvalidCustomerAlgorithm + } + if h.Get(xhttp.AmzServerSideEncryptionCustomerKey) == "" { + return key, ErrMissingCustomerKey + } + if h.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5) == "" { + return key, ErrMissingCustomerKeyMD5 + } + + clientKey, err := base64.StdEncoding.DecodeString(h.Get(xhttp.AmzServerSideEncryptionCustomerKey)) + if err != nil || len(clientKey) != 32 { // The client key must be 256 bits long + return key, ErrInvalidCustomerKey + } + keyMD5, err := base64.StdEncoding.DecodeString(h.Get(xhttp.AmzServerSideEncryptionCustomerKeyMD5)) + if md5Sum := md5.Sum(clientKey); err != nil || !bytes.Equal(md5Sum[:], keyMD5) { + return key, ErrCustomerKeyMD5Mismatch + } + copy(key[:], clientKey) + return key, nil +} + +// UnsealObjectKey extracts and decrypts the sealed object key +// from the metadata using the SSE-C client key of the HTTP headers +// and returns the decrypted object key. +func (s3 ssec) UnsealObjectKey(h http.Header, metadata map[string]string, bucket, object string) (key ObjectKey, err error) { + clientKey, err := s3.ParseHTTP(h) + if err != nil { + return key, err + } + return unsealObjectKey(clientKey[:], metadata, bucket, object) +} + +// CreateMetadata encodes the sealed key into the metadata +// and returns the modified metadata. It allocates a new +// metadata map if metadata is nil. +func (ssec) CreateMetadata(metadata map[string]string, sealedKey SealedKey) map[string]string { + if sealedKey.Algorithm != SealAlgorithm { + logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-C", sealedKey.Algorithm)) + } + + if metadata == nil { + metadata = make(map[string]string, 3) + } + metadata[MetaAlgorithm] = SealAlgorithm + metadata[MetaIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:]) + metadata[MetaSealedKeySSEC] = base64.StdEncoding.EncodeToString(sealedKey.Key[:]) + return metadata +} + +// ParseMetadata extracts all SSE-C related values from the object metadata +// and checks whether they are well-formed. It returns the sealed object key +// on success. +func (ssec) ParseMetadata(metadata map[string]string) (sealedKey SealedKey, err error) { + // Extract all required values from object metadata + b64IV, ok := metadata[MetaIV] + if !ok { + return sealedKey, errMissingInternalIV + } + algorithm, ok := metadata[MetaAlgorithm] + if !ok { + return sealedKey, errMissingInternalSealAlgorithm + } + b64SealedKey, ok := metadata[MetaSealedKeySSEC] + if !ok { + return sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-C") + } + + // Check whether all extracted values are well-formed + iv, err := base64.StdEncoding.DecodeString(b64IV) + if err != nil || len(iv) != 32 { + return sealedKey, errInvalidInternalIV + } + if algorithm != SealAlgorithm && algorithm != InsecureSealAlgorithm { + return sealedKey, errInvalidInternalSealAlgorithm + } + encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey) + if err != nil || len(encryptedKey) != 64 { + return sealedKey, Errorf("The internal sealed key for SSE-C is invalid") + } + + sealedKey.Algorithm = algorithm + copy(sealedKey.IV[:], iv) + copy(sealedKey.Key[:], encryptedKey) + return sealedKey, nil +} diff --git a/internal/crypto/sse-kms.go b/internal/crypto/sse-kms.go new file mode 100644 index 00000000..dd0aa46a --- /dev/null +++ b/internal/crypto/sse-kms.go @@ -0,0 +1,241 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package crypto + +import ( + "context" + "encoding/base64" + "errors" + "net/http" + "path" + "strings" + + jsoniter "github.com/json-iterator/go" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/kms" + "github.com/minio/minio/internal/logger" +) + +type ssekms struct{} + +var ( + // S3KMS represents AWS SSE-KMS. It provides functionality to + // handle SSE-KMS requests. + S3KMS = ssekms{} + + _ Type = S3KMS +) + +// String returns the SSE domain as string. For SSE-KMS the +// domain is "SSE-KMS". +func (ssekms) String() string { return "SSE-KMS" } + +// IsRequested returns true if the HTTP headers contains +// at least one SSE-KMS header. +func (ssekms) IsRequested(h http.Header) bool { + if _, ok := h[xhttp.AmzServerSideEncryptionKmsID]; ok { + return true + } + if _, ok := h[xhttp.AmzServerSideEncryptionKmsContext]; ok { + return true + } + if _, ok := h[xhttp.AmzServerSideEncryption]; ok { + // Return only true if the SSE header is specified and does not contain the SSE-S3 value + return strings.ToUpper(h.Get(xhttp.AmzServerSideEncryption)) != xhttp.AmzEncryptionAES + } + return false +} + +// ParseHTTP parses the SSE-KMS headers and returns the SSE-KMS key ID +// and the KMS context on success. +func (ssekms) ParseHTTP(h http.Header) (string, kms.Context, error) { + if h == nil { + return "", nil, ErrInvalidEncryptionMethod + } + + algorithm := h.Get(xhttp.AmzServerSideEncryption) + if algorithm != xhttp.AmzEncryptionKMS { + return "", nil, ErrInvalidEncryptionMethod + } + + var ctx kms.Context + if context, ok := h[xhttp.AmzServerSideEncryptionKmsContext]; ok { + b, err := base64.StdEncoding.DecodeString(context[0]) + if err != nil { + return "", nil, err + } + + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err := json.Unmarshal(b, &ctx); err != nil { + return "", nil, err + } + } + + keyID := h.Get(xhttp.AmzServerSideEncryptionKmsID) + spaces := strings.HasPrefix(keyID, " ") || strings.HasSuffix(keyID, " ") + if spaces { + return "", nil, ErrInvalidEncryptionKeyID + } + return strings.TrimPrefix(keyID, ARNPrefix), ctx, nil +} + +// IsEncrypted returns true if the object metadata indicates +// that the object was uploaded using SSE-KMS. +func (ssekms) IsEncrypted(metadata map[string]string) bool { + if _, ok := metadata[MetaSealedKeyKMS]; ok { + return true + } + return false +} + +// UnsealObjectKey extracts and decrypts the sealed object key +// from the metadata using KMS and returns the decrypted object +// key. +func (s3 ssekms) UnsealObjectKey(k *kms.KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) { + if k == nil { + return key, Errorf("KMS not configured") + } + + keyID, kmsKey, sealedKey, ctx, err := s3.ParseMetadata(metadata) + if err != nil { + return key, err + } + if ctx == nil { + ctx = kms.Context{bucket: path.Join(bucket, object)} + } else if _, ok := ctx[bucket]; !ok { + ctx[bucket] = path.Join(bucket, object) + } + unsealKey, err := k.Decrypt(context.TODO(), &kms.DecryptRequest{ + Name: keyID, + Ciphertext: kmsKey, + AssociatedData: ctx, + }) + if err != nil { + return key, err + } + err = key.Unseal(unsealKey, sealedKey, s3.String(), bucket, object) + return key, err +} + +// CreateMetadata encodes the sealed object key into the metadata and returns +// the modified metadata. If the keyID and the kmsKey is not empty it encodes +// both into the metadata as well. It allocates a new metadata map if metadata +// is nil. +func (ssekms) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey, ctx kms.Context) map[string]string { + if sealedKey.Algorithm != SealAlgorithm { + logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm)) + } + + // There are two possibilities: + // - We use a KMS -> There must be non-empty key ID and a KMS data key. + // - We use a K/V -> There must be no key ID and no KMS data key. + // Otherwise, the caller has passed an invalid argument combination. + if keyID == "" && len(kmsKey) != 0 { + logger.CriticalIf(context.Background(), errors.New("The key ID must not be empty if a KMS data key is present")) + } + if keyID != "" && len(kmsKey) == 0 { + logger.CriticalIf(context.Background(), errors.New("The KMS data key must not be empty if a key ID is present")) + } + + if metadata == nil { + metadata = make(map[string]string, 5) + } + + metadata[MetaAlgorithm] = sealedKey.Algorithm + metadata[MetaIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:]) + metadata[MetaSealedKeyKMS] = base64.StdEncoding.EncodeToString(sealedKey.Key[:]) + if len(ctx) > 0 { + b, _ := ctx.MarshalText() + metadata[MetaContext] = base64.StdEncoding.EncodeToString(b) + } + if len(kmsKey) > 0 && keyID != "" { // We use a KMS -> Store key ID and sealed KMS data key. + metadata[MetaKeyID] = keyID + metadata[MetaDataEncryptionKey] = base64.StdEncoding.EncodeToString(kmsKey) + } + return metadata +} + +// ParseMetadata extracts all SSE-KMS related values from the object metadata +// and checks whether they are well-formed. It returns the sealed object key +// on success. If the metadata contains both, a KMS master key ID and a sealed +// KMS data key it returns both. If the metadata does not contain neither a +// KMS master key ID nor a sealed KMS data key it returns an empty keyID and +// KMS data key. Otherwise, it returns an error. +func (ssekms) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, ctx kms.Context, err error) { + // Extract all required values from object metadata + b64IV, ok := metadata[MetaIV] + if !ok { + return keyID, kmsKey, sealedKey, ctx, errMissingInternalIV + } + algorithm, ok := metadata[MetaAlgorithm] + if !ok { + return keyID, kmsKey, sealedKey, ctx, errMissingInternalSealAlgorithm + } + b64SealedKey, ok := metadata[MetaSealedKeyKMS] + if !ok { + return keyID, kmsKey, sealedKey, ctx, Errorf("The object metadata is missing the internal sealed key for SSE-S3") + } + + // There are two possibilities: + // - We use a KMS -> There must be a key ID and a KMS data key. + // - We use a K/V -> There must be no key ID and no KMS data key. + // Otherwise, the metadata is corrupted. + keyID, idPresent := metadata[MetaKeyID] + b64KMSSealedKey, kmsKeyPresent := metadata[MetaDataEncryptionKey] + if !idPresent && kmsKeyPresent { + return keyID, kmsKey, sealedKey, ctx, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3") + } + if idPresent && !kmsKeyPresent { + return keyID, kmsKey, sealedKey, ctx, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3") + } + + // Check whether all extracted values are well-formed + iv, err := base64.StdEncoding.DecodeString(b64IV) + if err != nil || len(iv) != 32 { + return keyID, kmsKey, sealedKey, ctx, errInvalidInternalIV + } + if algorithm != SealAlgorithm { + return keyID, kmsKey, sealedKey, ctx, errInvalidInternalSealAlgorithm + } + encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey) + if err != nil || len(encryptedKey) != 64 { + return keyID, kmsKey, sealedKey, ctx, Errorf("The internal sealed key for SSE-KMS is invalid") + } + if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key. + kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey) + if err != nil { + return keyID, kmsKey, sealedKey, ctx, Errorf("The internal sealed KMS data key for SSE-KMS is invalid") + } + } + b64Ctx, ok := metadata[MetaContext] + if ok { + b, err := base64.StdEncoding.DecodeString(b64Ctx) + if err != nil { + return keyID, kmsKey, sealedKey, ctx, Errorf("The internal KMS context is not base64-encoded") + } + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err = json.Unmarshal(b, &ctx); err != nil { + return keyID, kmsKey, sealedKey, ctx, Errorf("The internal sealed KMS context is invalid %w", err) + } + } + + sealedKey.Algorithm = algorithm + copy(sealedKey.IV[:], iv) + copy(sealedKey.Key[:], encryptedKey) + return keyID, kmsKey, sealedKey, ctx, nil +} diff --git a/internal/crypto/sse-s3.go b/internal/crypto/sse-s3.go new file mode 100644 index 00000000..ce34d5a4 --- /dev/null +++ b/internal/crypto/sse-s3.go @@ -0,0 +1,209 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package crypto + +import ( + "context" + "encoding/base64" + "errors" + "net/http" + "path" + "strings" + + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/kms" + "github.com/minio/minio/internal/logger" +) + +type sses3 struct{} + +var ( + // S3 represents AWS SSE-S3. It provides functionality to handle + // SSE-S3 requests. + S3 = sses3{} + + _ Type = S3 +) + +// String returns the SSE domain as string. For SSE-S3 the +// domain is "SSE-S3". +func (sses3) String() string { return "SSE-S3" } + +func (sses3) IsRequested(h http.Header) bool { + _, ok := h[xhttp.AmzServerSideEncryption] + // Return only true if the SSE header is specified and does not contain the SSE-KMS value + return ok && !strings.EqualFold(h.Get(xhttp.AmzServerSideEncryption), xhttp.AmzEncryptionKMS) +} + +// ParseHTTP parses the SSE-S3 related HTTP headers and checks +// whether they contain valid values. +func (sses3) ParseHTTP(h http.Header) error { + if h.Get(xhttp.AmzServerSideEncryption) != xhttp.AmzEncryptionAES { + return ErrInvalidEncryptionMethod + } + return nil +} + +// IsEncrypted returns true if the object metadata indicates +// that the object was uploaded using SSE-S3. +func (sses3) IsEncrypted(metadata map[string]string) bool { + if _, ok := metadata[MetaSealedKeyS3]; ok { + return true + } + return false +} + +// UnsealObjectKey extracts and decrypts the sealed object key +// from the metadata using KMS and returns the decrypted object +// key. +func (s3 sses3) UnsealObjectKey(k *kms.KMS, metadata map[string]string, bucket, object string) (key ObjectKey, err error) { + if k == nil { + return key, Errorf("KMS not configured") + } + keyID, kmsKey, sealedKey, err := s3.ParseMetadata(metadata) + if err != nil { + return key, err + } + unsealKey, err := k.Decrypt(context.TODO(), &kms.DecryptRequest{ + Name: keyID, + Ciphertext: kmsKey, + AssociatedData: kms.Context{bucket: path.Join(bucket, object)}, + }) + if err != nil { + return key, err + } + err = key.Unseal(unsealKey, sealedKey, s3.String(), bucket, object) + return key, err +} + +// UnsealObjectsKeys extracts and decrypts all sealed object keys +// from the metadata using the KMS and returns the decrypted object +// keys. +// +// The metadata, buckets and objects slices must have the same length. +func (s3 sses3) UnsealObjectKeys(ctx context.Context, k *kms.KMS, metadata []map[string]string, buckets, objects []string) ([]ObjectKey, error) { + if k == nil { + return nil, Errorf("KMS not configured") + } + + if len(metadata) != len(buckets) || len(metadata) != len(objects) { + return nil, Errorf("invalid metadata/object count: %d != %d != %d", len(metadata), len(buckets), len(objects)) + } + keys := make([]ObjectKey, 0, len(metadata)) + for i := range metadata { + key, err := s3.UnsealObjectKey(k, metadata[i], buckets[i], objects[i]) + if err != nil { + return nil, err + } + keys = append(keys, key) + } + return keys, nil +} + +// CreateMetadata encodes the sealed object key into the metadata and returns +// the modified metadata. If the keyID and the kmsKey is not empty it encodes +// both into the metadata as well. It allocates a new metadata map if metadata +// is nil. +func (sses3) CreateMetadata(metadata map[string]string, keyID string, kmsKey []byte, sealedKey SealedKey) map[string]string { + if sealedKey.Algorithm != SealAlgorithm { + logger.CriticalIf(context.Background(), Errorf("The seal algorithm '%s' is invalid for SSE-S3", sealedKey.Algorithm)) + } + + // There are two possibilities: + // - We use a KMS -> There must be non-empty key ID and a KMS data key. + // - We use a K/V -> There must be no key ID and no KMS data key. + // Otherwise, the caller has passed an invalid argument combination. + if keyID == "" && len(kmsKey) != 0 { + logger.CriticalIf(context.Background(), errors.New("The key ID must not be empty if a KMS data key is present")) + } + if keyID != "" && len(kmsKey) == 0 { + logger.CriticalIf(context.Background(), errors.New("The KMS data key must not be empty if a key ID is present")) + } + + if metadata == nil { + metadata = make(map[string]string, 5) + } + + metadata[MetaAlgorithm] = sealedKey.Algorithm + metadata[MetaIV] = base64.StdEncoding.EncodeToString(sealedKey.IV[:]) + metadata[MetaSealedKeyS3] = base64.StdEncoding.EncodeToString(sealedKey.Key[:]) + if len(kmsKey) > 0 && keyID != "" { // We use a KMS -> Store key ID and sealed KMS data key. + metadata[MetaKeyID] = keyID + metadata[MetaDataEncryptionKey] = base64.StdEncoding.EncodeToString(kmsKey) + } + return metadata +} + +// ParseMetadata extracts all SSE-S3 related values from the object metadata +// and checks whether they are well-formed. It returns the sealed object key +// on success. If the metadata contains both, a KMS master key ID and a sealed +// KMS data key it returns both. If the metadata does not contain neither a +// KMS master key ID nor a sealed KMS data key it returns an empty keyID and +// KMS data key. Otherwise, it returns an error. +func (sses3) ParseMetadata(metadata map[string]string) (keyID string, kmsKey []byte, sealedKey SealedKey, err error) { + // Extract all required values from object metadata + b64IV, ok := metadata[MetaIV] + if !ok { + return keyID, kmsKey, sealedKey, errMissingInternalIV + } + algorithm, ok := metadata[MetaAlgorithm] + if !ok { + return keyID, kmsKey, sealedKey, errMissingInternalSealAlgorithm + } + b64SealedKey, ok := metadata[MetaSealedKeyS3] + if !ok { + return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed key for SSE-S3") + } + + // There are two possibilities: + // - We use a KMS -> There must be a key ID and a KMS data key. + // - We use a K/V -> There must be no key ID and no KMS data key. + // Otherwise, the metadata is corrupted. + keyID, idPresent := metadata[MetaKeyID] + b64KMSSealedKey, kmsKeyPresent := metadata[MetaDataEncryptionKey] + if !idPresent && kmsKeyPresent { + return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal KMS key-ID for SSE-S3") + } + if idPresent && !kmsKeyPresent { + return keyID, kmsKey, sealedKey, Errorf("The object metadata is missing the internal sealed KMS data key for SSE-S3") + } + + // Check whether all extracted values are well-formed + iv, err := base64.StdEncoding.DecodeString(b64IV) + if err != nil || len(iv) != 32 { + return keyID, kmsKey, sealedKey, errInvalidInternalIV + } + if algorithm != SealAlgorithm { + return keyID, kmsKey, sealedKey, errInvalidInternalSealAlgorithm + } + encryptedKey, err := base64.StdEncoding.DecodeString(b64SealedKey) + if err != nil || len(encryptedKey) != 64 { + return keyID, kmsKey, sealedKey, Errorf("The internal sealed key for SSE-S3 is invalid") + } + if idPresent && kmsKeyPresent { // We are using a KMS -> parse the sealed KMS data key. + kmsKey, err = base64.StdEncoding.DecodeString(b64KMSSealedKey) + if err != nil { + return keyID, kmsKey, sealedKey, Errorf("The internal sealed KMS data key for SSE-S3 is invalid") + } + } + + sealedKey.Algorithm = algorithm + copy(sealedKey.IV[:], iv) + copy(sealedKey.Key[:], encryptedKey) + return keyID, kmsKey, sealedKey, nil +} diff --git a/internal/crypto/sse.go b/internal/crypto/sse.go new file mode 100644 index 00000000..e0daf750 --- /dev/null +++ b/internal/crypto/sse.go @@ -0,0 +1,130 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package crypto + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + + "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/logger" + "github.com/minio/sio" +) + +const ( + // SealAlgorithm is the encryption/sealing algorithm used to derive & seal + // the key-encryption-key and to en/decrypt the object data. + SealAlgorithm = "DAREv2-HMAC-SHA256" + + // InsecureSealAlgorithm is the legacy encryption/sealing algorithm used + // to derive & seal the key-encryption-key and to en/decrypt the object data. + // This algorithm should not be used for new objects because its key derivation + // is not optimal. See: https://github.com/minio/minio/pull/6121 + InsecureSealAlgorithm = "DARE-SHA256" +) + +// Type represents an AWS SSE type: +// - SSE-C +// - SSE-S3 +// - SSE-KMS +type Type interface { + fmt.Stringer + + IsRequested(http.Header) bool + IsEncrypted(map[string]string) bool +} + +// IsRequested returns true and the SSE Type if the HTTP headers +// indicate that some form server-side encryption is requested. +// +// If no SSE headers are present then IsRequested returns false +// and no Type. +func IsRequested(h http.Header) (Type, bool) { + switch { + case S3.IsRequested(h): + return S3, true + case S3KMS.IsRequested(h): + return S3KMS, true + case SSEC.IsRequested(h): + return SSEC, true + default: + return nil, false + } +} + +// Requested returns whether any type of encryption is requested. +func Requested(h http.Header) bool { + return S3.IsRequested(h) || S3KMS.IsRequested(h) || SSEC.IsRequested(h) +} + +// UnsealObjectKey extracts and decrypts the sealed object key +// from the metadata using the SSE-Copy client key of the HTTP headers +// and returns the decrypted object key. +func (sse ssecCopy) UnsealObjectKey(h http.Header, metadata map[string]string, bucket, object string) (key ObjectKey, err error) { + clientKey, err := sse.ParseHTTP(h) + if err != nil { + return key, err + } + return unsealObjectKey(clientKey[:], metadata, bucket, object) +} + +// unsealObjectKey decrypts and returns the sealed object key +// from the metadata using the SSE-C client key. +func unsealObjectKey(clientKey []byte, metadata map[string]string, bucket, object string) (key ObjectKey, err error) { + sealedKey, err := SSEC.ParseMetadata(metadata) + if err != nil { + return key, err + } + err = key.Unseal(clientKey, sealedKey, SSEC.String(), bucket, object) + return key, err +} + +// EncryptSinglePart encrypts an io.Reader which must be the +// body of a single-part PUT request. +func EncryptSinglePart(r io.Reader, key ObjectKey) io.Reader { + r, err := sio.EncryptReader(r, sio.Config{MinVersion: sio.Version20, Key: key[:]}) + if err != nil { + logger.CriticalIf(context.Background(), errors.New("Unable to encrypt io.Reader using object key")) + } + return r +} + +// EncryptMultiPart encrypts an io.Reader which must be the body of +// multi-part PUT request. It derives an unique encryption key from +// the partID and the object key. +func EncryptMultiPart(r io.Reader, partID int, key ObjectKey) io.Reader { + partKey := key.DerivePartKey(uint32(partID)) + return EncryptSinglePart(r, ObjectKey(partKey)) +} + +// DecryptSinglePart decrypts an io.Writer which must an object +// uploaded with the single-part PUT API. The offset and length +// specify the requested range. +func DecryptSinglePart(w io.Writer, offset, length int64, key ObjectKey) io.WriteCloser { + const PayloadSize = 1 << 16 // DARE 2.0 + w = ioutil.LimitedWriter(w, offset%PayloadSize, length) + + decWriter, err := sio.DecryptWriter(w, sio.Config{Key: key[:]}) + if err != nil { + logger.CriticalIf(context.Background(), errors.New("Unable to decrypt io.Writer using object key")) + } + return decWriter +} diff --git a/internal/crypto/sse_test.go b/internal/crypto/sse_test.go new file mode 100644 index 00000000..0ed6ac05 --- /dev/null +++ b/internal/crypto/sse_test.go @@ -0,0 +1,187 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package crypto + +import ( + "net/http" + "testing" +) + +func TestS3String(t *testing.T) { + const Domain = "SSE-S3" + if domain := S3.String(); domain != Domain { + t.Errorf("S3's string method returns wrong domain: got '%s' - want '%s'", domain, Domain) + } +} + +func TestSSECString(t *testing.T) { + const Domain = "SSE-C" + if domain := SSEC.String(); domain != Domain { + t.Errorf("SSEC's string method returns wrong domain: got '%s' - want '%s'", domain, Domain) + } +} + +var ssecUnsealObjectKeyTests = []struct { + Headers http.Header + Bucket, Object string + Metadata map[string]string + + ExpectedErr error +}{ + { // 0 - Valid HTTP headers and valid metadata entries for bucket/object + Headers: http.Header{ + "X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + "X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + Bucket: "bucket", + Object: "object", + Metadata: map[string]string{ + "X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==", + "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256", + "X-Minio-Internal-Server-Side-Encryption-Iv": "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=", + }, + ExpectedErr: nil, + }, + { // 1 - Valid HTTP headers but invalid metadata entries for bucket/object2 + Headers: http.Header{ + "X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + "X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + Bucket: "bucket", + Object: "object2", + Metadata: map[string]string{ + "X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==", + "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256", + "X-Minio-Internal-Server-Side-Encryption-Iv": "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=", + }, + ExpectedErr: ErrSecretKeyMismatch, + }, + { // 2 - Valid HTTP headers but invalid metadata entries for bucket/object + Headers: http.Header{ + "X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + "X-Amz-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + Bucket: "bucket", + Object: "object", + Metadata: map[string]string{ + "X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==", + "X-Minio-Internal-Server-Side-Encryption-Iv": "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=", + }, + ExpectedErr: errMissingInternalSealAlgorithm, + }, + { // 3 - Invalid HTTP headers for valid metadata entries for bucket/object + Headers: http.Header{ + "X-Amz-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + }, + Bucket: "bucket", + Object: "object", + Metadata: map[string]string{ + "X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==", + "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256", + "X-Minio-Internal-Server-Side-Encryption-Iv": "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=", + }, + ExpectedErr: ErrMissingCustomerKeyMD5, + }, +} + +func TestSSECUnsealObjectKey(t *testing.T) { + for i, test := range ssecUnsealObjectKeyTests { + if _, err := SSEC.UnsealObjectKey(test.Headers, test.Metadata, test.Bucket, test.Object); err != test.ExpectedErr { + t.Errorf("Test %d: got: %v - want: %v", i, err, test.ExpectedErr) + } + } +} + +var sseCopyUnsealObjectKeyTests = []struct { + Headers http.Header + Bucket, Object string + Metadata map[string]string + + ExpectedErr error +}{ + { // 0 - Valid HTTP headers and valid metadata entries for bucket/object + Headers: http.Header{ + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + Bucket: "bucket", + Object: "object", + Metadata: map[string]string{ + "X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==", + "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256", + "X-Minio-Internal-Server-Side-Encryption-Iv": "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=", + }, + ExpectedErr: nil, + }, + { // 1 - Valid HTTP headers but invalid metadata entries for bucket/object2 + Headers: http.Header{ + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + Bucket: "bucket", + Object: "object2", + Metadata: map[string]string{ + "X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==", + "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256", + "X-Minio-Internal-Server-Side-Encryption-Iv": "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=", + }, + ExpectedErr: ErrSecretKeyMismatch, + }, + { // 2 - Valid HTTP headers but invalid metadata entries for bucket/object + Headers: http.Header{ + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5": []string{"7PpPLAK26ONlVUGOWlusfg=="}, + }, + Bucket: "bucket", + Object: "object", + Metadata: map[string]string{ + "X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==", + "X-Minio-Internal-Server-Side-Encryption-Iv": "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=", + }, + ExpectedErr: errMissingInternalSealAlgorithm, + }, + { // 3 - Invalid HTTP headers for valid metadata entries for bucket/object + Headers: http.Header{ + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm": []string{"AES256"}, + "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key": []string{"MzJieXRlc2xvbmdzZWNyZXRrZXltdXN0cHJvdmlkZWQ="}, + }, + Bucket: "bucket", + Object: "object", + Metadata: map[string]string{ + "X-Minio-Internal-Server-Side-Encryption-Sealed-Key": "IAAfAMBdYor5tf/UlVaQvwYlw5yKbPBeQqfygqsfHqhu1wHD9KDAP4bw38AhL12prFTS23JbbR9Re5Qv26ZnlQ==", + "X-Minio-Internal-Server-Side-Encryption-Seal-Algorithm": "DAREv2-HMAC-SHA256", + "X-Minio-Internal-Server-Side-Encryption-Iv": "coVfGS3I/CTrqexX5vUN+PQPoP9aUFiPYYrSzqTWfBA=", + }, + ExpectedErr: ErrMissingCustomerKeyMD5, + }, +} + +func TestSSECopyUnsealObjectKey(t *testing.T) { + for i, test := range sseCopyUnsealObjectKeyTests { + if _, err := SSECopy.UnsealObjectKey(test.Headers, test.Metadata, test.Bucket, test.Object); err != test.ExpectedErr { + t.Errorf("Test %d: got: %v - want: %v", i, err, test.ExpectedErr) + } + } +} diff --git a/internal/deadlineconn/deadlineconn.go b/internal/deadlineconn/deadlineconn.go new file mode 100644 index 00000000..95bb43ef --- /dev/null +++ b/internal/deadlineconn/deadlineconn.go @@ -0,0 +1,176 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Package deadlineconn implements net.Conn wrapper with configured deadlines. +package deadlineconn + +import ( + "context" + "net" + "sync" + "sync/atomic" + "time" +) + +// updateInterval is the minimum time between deadline updates. +const updateInterval = 250 * time.Millisecond + +// DeadlineConn - is a generic stream-oriented network connection supporting buffered reader and read/write timeout. +type DeadlineConn struct { + net.Conn + readDeadline time.Duration // sets the read deadline on a connection. + readSetAt time.Time + writeDeadline time.Duration // sets the write deadline on a connection. + writeSetAt time.Time + abortReads, abortWrites atomic.Bool // A deadline was set to indicate caller wanted the conn to time out. + infReads, infWrites atomic.Bool + mu sync.Mutex +} + +// Unwrap will unwrap the connection and remove the deadline if applied. +// If not a *DeadlineConn, the unmodified net.Conn is returned. +func Unwrap(c net.Conn) net.Conn { + if dc, ok := c.(*DeadlineConn); ok { + return dc.Conn + } + return c +} + +// Sets read deadline +func (c *DeadlineConn) setReadDeadline() { + // Do not set a Read deadline, if upstream wants to cancel all reads. + if c.readDeadline <= 0 || c.abortReads.Load() || c.infReads.Load() { + return + } + + c.mu.Lock() + defer c.mu.Unlock() + if c.abortReads.Load() { + return + } + + now := time.Now() + if now.Sub(c.readSetAt) > updateInterval { + c.Conn.SetReadDeadline(now.Add(c.readDeadline + updateInterval)) + c.readSetAt = now + } +} + +func (c *DeadlineConn) setWriteDeadline() { + // Do not set a Write deadline, if upstream wants to cancel all reads. + if c.writeDeadline <= 0 || c.abortWrites.Load() || c.infWrites.Load() { + return + } + + c.mu.Lock() + defer c.mu.Unlock() + if c.abortWrites.Load() { + return + } + now := time.Now() + if now.Sub(c.writeSetAt) > updateInterval { + c.Conn.SetWriteDeadline(now.Add(c.writeDeadline + updateInterval)) + c.writeSetAt = now + } +} + +// Read - reads data from the connection using wrapped buffered reader. +func (c *DeadlineConn) Read(b []byte) (n int, err error) { + if c.abortReads.Load() { + return 0, context.DeadlineExceeded + } + c.setReadDeadline() + n, err = c.Conn.Read(b) + return n, err +} + +// Write - writes data to the connection. +func (c *DeadlineConn) Write(b []byte) (n int, err error) { + if c.abortWrites.Load() { + return 0, context.DeadlineExceeded + } + c.setWriteDeadline() + n, err = c.Conn.Write(b) + return n, err +} + +// SetDeadline will set the deadline for reads and writes. +// A zero value for t means I/O operations will not time out. +func (c *DeadlineConn) SetDeadline(t time.Time) error { + c.mu.Lock() + defer c.mu.Unlock() + + c.readSetAt = time.Time{} + c.writeSetAt = time.Time{} + c.abortReads.Store(!t.IsZero() && time.Until(t) < 0) + c.abortWrites.Store(!t.IsZero() && time.Until(t) < 0) + c.infReads.Store(t.IsZero()) + c.infWrites.Store(t.IsZero()) + return c.Conn.SetDeadline(t) +} + +// SetReadDeadline sets the deadline for future Read calls +// and any currently-blocked Read call. +// A zero value for t means Read will not time out. +func (c *DeadlineConn) SetReadDeadline(t time.Time) error { + c.mu.Lock() + defer c.mu.Unlock() + c.abortReads.Store(!t.IsZero() && time.Until(t) < 0) + c.infReads.Store(t.IsZero()) + c.readSetAt = time.Time{} + return c.Conn.SetReadDeadline(t) +} + +// SetWriteDeadline sets the deadline for future Write calls +// and any currently-blocked Write call. +// Even if write times out, it may return n > 0, indicating that +// some of the data was successfully written. +// A zero value for t means Write will not time out. +func (c *DeadlineConn) SetWriteDeadline(t time.Time) error { + c.mu.Lock() + defer c.mu.Unlock() + c.abortWrites.Store(!t.IsZero() && time.Until(t) < 0) + c.infWrites.Store(t.IsZero()) + c.writeSetAt = time.Time{} + return c.Conn.SetWriteDeadline(t) +} + +// Close wraps conn.Close and stops sending deadline updates. +func (c *DeadlineConn) Close() error { + c.abortReads.Store(true) + c.abortWrites.Store(true) + return c.Conn.Close() +} + +// WithReadDeadline sets a new read side net.Conn deadline. +func (c *DeadlineConn) WithReadDeadline(d time.Duration) *DeadlineConn { + c.readDeadline = d + return c +} + +// WithWriteDeadline sets a new write side net.Conn deadline. +func (c *DeadlineConn) WithWriteDeadline(d time.Duration) *DeadlineConn { + c.writeDeadline = d + return c +} + +// New - creates a new connection object wrapping net.Conn with deadlines. +func New(c net.Conn) *DeadlineConn { + return &DeadlineConn{ + Conn: c, + } +} diff --git a/internal/deadlineconn/deadlineconn_test.go b/internal/deadlineconn/deadlineconn_test.go new file mode 100644 index 00000000..6921e47b --- /dev/null +++ b/internal/deadlineconn/deadlineconn_test.go @@ -0,0 +1,192 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package deadlineconn + +import ( + "bufio" + "fmt" + "io" + "net" + "sync" + "testing" + "time" +) + +// Test deadlineconn handles read timeout properly by reading two messages beyond deadline. +func TestBuffConnReadTimeout(t *testing.T) { + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("unable to create listener. %v", err) + } + defer l.Close() + serverAddr := l.Addr().String() + + tcpListener, ok := l.(*net.TCPListener) + if !ok { + t.Fatalf("failed to assert to net.TCPListener") + } + + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + + tcpConn, terr := tcpListener.AcceptTCP() + if terr != nil { + t.Errorf("failed to accept new connection. %v", terr) + return + } + deadlineconn := New(tcpConn) + deadlineconn.WithReadDeadline(time.Second) + deadlineconn.WithWriteDeadline(time.Second) + defer deadlineconn.Close() + + // Read a line + b := make([]byte, 12) + _, terr = deadlineconn.Read(b) + if terr != nil { + t.Errorf("failed to read from client. %v", terr) + return + } + received := string(b) + if received != "message one\n" { + t.Errorf(`server: expected: "message one\n", got: %v`, received) + return + } + + // Wait for more than read timeout to simulate processing. + time.Sleep(3 * time.Second) + + _, terr = deadlineconn.Read(b) + if terr != nil { + t.Errorf("failed to read from client. %v", terr) + return + } + received = string(b) + if received != "message two\n" { + t.Errorf(`server: expected: "message two\n", got: %v`, received) + return + } + + // Send a response. + _, terr = io.WriteString(deadlineconn, "messages received\n") + if terr != nil { + t.Errorf("failed to write to client. %v", terr) + return + } + }() + + c, err := net.Dial("tcp", serverAddr) + if err != nil { + t.Fatalf("unable to connect to server. %v", err) + } + defer c.Close() + + _, err = io.WriteString(c, "message one\n") + if err != nil { + t.Fatalf("failed to write to server. %v", err) + } + _, err = io.WriteString(c, "message two\n") + if err != nil { + t.Fatalf("failed to write to server. %v", err) + } + + received, err := bufio.NewReader(c).ReadString('\n') + if err != nil { + t.Fatalf("failed to read from server. %v", err) + } + if received != "messages received\n" { + t.Fatalf(`client: expected: "messages received\n", got: %v`, received) + } + + wg.Wait() +} + +// Test deadlineconn handles read timeout properly by reading two messages beyond deadline. +func TestBuffConnReadCheckTimeout(t *testing.T) { + l, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatalf("unable to create listener. %v", err) + } + defer l.Close() + serverAddr := l.Addr().String() + + tcpListener, ok := l.(*net.TCPListener) + if !ok { + t.Fatalf("failed to assert to net.TCPListener") + } + var cerr error + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + + tcpConn, terr := tcpListener.AcceptTCP() + if terr != nil { + cerr = fmt.Errorf("failed to accept new connection. %v", terr) + return + } + deadlineconn := New(tcpConn) + deadlineconn.WithReadDeadline(time.Second) + deadlineconn.WithWriteDeadline(time.Second) + defer deadlineconn.Close() + + // Read a line + b := make([]byte, 12) + _, terr = deadlineconn.Read(b) + if terr != nil { + cerr = fmt.Errorf("failed to read from client. %v", terr) + return + } + received := string(b) + if received != "message one\n" { + cerr = fmt.Errorf(`server: expected: "message one\n", got: %v`, received) + return + } + + // Set a deadline in the past to indicate we want the next read to fail. + // Ensure we don't override it on read. + deadlineconn.SetReadDeadline(time.Unix(1, 0)) + + // Be sure to exceed update interval + time.Sleep(updateInterval * 2) + + _, terr = deadlineconn.Read(b) + if terr == nil { + cerr = fmt.Errorf("could read from client, expected error, got %v", terr) + return + } + }() + + c, err := net.Dial("tcp", serverAddr) + if err != nil { + t.Fatalf("unable to connect to server. %v", err) + } + defer c.Close() + + _, err = io.WriteString(c, "message one\n") + if err != nil { + t.Fatalf("failed to write to server. %v", err) + } + _, _ = io.WriteString(c, "message two\n") + + wg.Wait() + if cerr != nil { + t.Fatal(cerr) + } +} diff --git a/internal/disk/directio_darwin.go b/internal/disk/directio_darwin.go new file mode 100644 index 00000000..af09799d --- /dev/null +++ b/internal/disk/directio_darwin.go @@ -0,0 +1,47 @@ +//go:build darwin + +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +import ( + "os" + + "github.com/ncw/directio" + "golang.org/x/sys/unix" +) + +// ODirectPlatform indicates if the platform supports O_DIRECT +const ODirectPlatform = true + +// OpenFileDirectIO - bypass kernel cache. +func OpenFileDirectIO(filePath string, flag int, perm os.FileMode) (*os.File, error) { + return directio.OpenFile(filePath, flag, perm) +} + +// DisableDirectIO - disables directio mode. +func DisableDirectIO(f *os.File) error { + fd := f.Fd() + _, err := unix.FcntlInt(fd, unix.F_NOCACHE, 0) + return err +} + +// AlignedBlock - pass through to directio implementation. +func AlignedBlock(blockSize int) []byte { + return directio.AlignedBlock(blockSize) +} diff --git a/internal/disk/directio_unix.go b/internal/disk/directio_unix.go new file mode 100644 index 00000000..883df95d --- /dev/null +++ b/internal/disk/directio_unix.go @@ -0,0 +1,54 @@ +//go:build !windows && !darwin && !openbsd && !plan9 +// +build !windows,!darwin,!openbsd,!plan9 + +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +import ( + "os" + "syscall" + + "github.com/ncw/directio" + "golang.org/x/sys/unix" +) + +// ODirectPlatform indicates if the platform supports O_DIRECT +const ODirectPlatform = true + +// OpenFileDirectIO - bypass kernel cache. +func OpenFileDirectIO(filePath string, flag int, perm os.FileMode) (*os.File, error) { + return directio.OpenFile(filePath, flag, perm) +} + +// DisableDirectIO - disables directio mode. +func DisableDirectIO(f *os.File) error { + fd := f.Fd() + flag, err := unix.FcntlInt(fd, unix.F_GETFL, 0) + if err != nil { + return err + } + flag &= ^(syscall.O_DIRECT) + _, err = unix.FcntlInt(fd, unix.F_SETFL, flag) + return err +} + +// AlignedBlock - pass through to directio implementation. +func AlignedBlock(blockSize int) []byte { + return directio.AlignedBlock(blockSize) +} diff --git a/internal/disk/directio_unsupported.go b/internal/disk/directio_unsupported.go new file mode 100644 index 00000000..6c579aa4 --- /dev/null +++ b/internal/disk/directio_unsupported.go @@ -0,0 +1,68 @@ +//go:build windows || openbsd || plan9 + +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +import ( + "os" +) + +// ODirectPlatform indicates if the platform supports O_DIRECT +const ODirectPlatform = false + +// OpenBSD, Windows, and illumos do not support O_DIRECT. +// On Windows there is no documentation on disabling O_DIRECT. +// For these systems we do not attempt to build the 'directio' dependency since +// the O_DIRECT symbol may not be exposed resulting in a failed build. +// +// +// On illumos an explicit O_DIRECT flag is not necessary for two primary +// reasons. Note that ZFS is effectively the default filesystem on illumos +// systems. +// +// One benefit of using DirectIO on Linux is that the page cache will not be +// polluted with single-access data. The ZFS read cache (ARC) is scan-resistant +// so there is no risk of polluting the entire cache with data accessed once. +// Another goal of DirectIO is to minimize the mutation of data by the kernel +// before issuing IO to underlying devices. ZFS users often enable features like +// compression and checksumming which currently necessitates mutating data in +// the kernel. +// +// DirectIO semantics for a filesystem like ZFS would be quite different than +// the semantics on filesystems like XFS, and these semantics are not +// implemented at this time. +// For more information on why typical DirectIO semantics do not apply to ZFS +// see this ZFS-on-Linux commit message: +// https://github.com/openzfs/zfs/commit/a584ef26053065f486d46a7335bea222cb03eeea + +// OpenFileDirectIO wrapper around os.OpenFile nothing special +func OpenFileDirectIO(filePath string, flag int, perm os.FileMode) (*os.File, error) { + return os.OpenFile(filePath, flag, perm) +} + +// DisableDirectIO is a no-op +func DisableDirectIO(f *os.File) error { + return nil +} + +// AlignedBlock simply returns an unaligned buffer +// for systems that do not support DirectIO. +func AlignedBlock(blockSize int) []byte { + return make([]byte, blockSize) +} diff --git a/internal/disk/disk.go b/internal/disk/disk.go new file mode 100644 index 00000000..a0b51f12 --- /dev/null +++ b/internal/disk/disk.go @@ -0,0 +1,62 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +// Info stat fs struct is container which holds following values +// Total - total size of the volume / disk +// Free - free size of the volume / disk +// Files - total inodes available +// Ffree - free inodes available +// FSType - file system type +// Major - major dev id +// Minor - minor dev id +// Devname - device name +type Info struct { + Total uint64 + Free uint64 + Used uint64 + Files uint64 + Ffree uint64 + FSType string + Major uint32 + Minor uint32 + Name string + Rotational *bool + NRRequests uint64 +} + +// IOStats contains stats of a single drive +type IOStats struct { + ReadIOs uint64 + ReadMerges uint64 + ReadSectors uint64 + ReadTicks uint64 + WriteIOs uint64 + WriteMerges uint64 + WriteSectors uint64 + WriteTicks uint64 + CurrentIOs uint64 + TotalTicks uint64 + ReqTicks uint64 + DiscardIOs uint64 + DiscardMerges uint64 + DiscardSectors uint64 + DiscardTicks uint64 + FlushIOs uint64 + FlushTicks uint64 +} diff --git a/internal/disk/disk_test.go b/internal/disk/disk_test.go new file mode 100644 index 00000000..d8e77438 --- /dev/null +++ b/internal/disk/disk_test.go @@ -0,0 +1,38 @@ +//go:build !netbsd && !solaris +// +build !netbsd,!solaris + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk_test + +import ( + "testing" + + "github.com/minio/minio/internal/disk" +) + +func TestFree(t *testing.T) { + di, err := disk.GetInfo(t.TempDir(), true) + if err != nil { + t.Fatal(err) + } + + if di.FSType == "UNKNOWN" { + t.Error("Unexpected FSType", di.FSType) + } +} diff --git a/internal/disk/disk_unix.go b/internal/disk/disk_unix.go new file mode 100644 index 00000000..a7cfabf5 --- /dev/null +++ b/internal/disk/disk_unix.go @@ -0,0 +1,41 @@ +//go:build !windows +// +build !windows + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +import ( + "syscall" +) + +// SameDisk reports whether di1 and di2 describe the same disk. +func SameDisk(disk1, disk2 string) (bool, error) { + st1 := syscall.Stat_t{} + st2 := syscall.Stat_t{} + + if err := syscall.Stat(disk1, &st1); err != nil { + return false, err + } + + if err := syscall.Stat(disk2, &st2); err != nil { + return false, err + } + + return st1.Dev == st2.Dev, nil +} diff --git a/internal/disk/disk_windows.go b/internal/disk/disk_windows.go new file mode 100644 index 00000000..0412ab9b --- /dev/null +++ b/internal/disk/disk_windows.go @@ -0,0 +1,26 @@ +//go:build windows +// +build windows + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +// SameDisk reports whether di1 and di2 describe the same disk. +func SameDisk(disk1, disk2 string) (bool, error) { + return false, nil +} diff --git a/internal/disk/fdatasync_linux.go b/internal/disk/fdatasync_linux.go new file mode 100644 index 00000000..4d398677 --- /dev/null +++ b/internal/disk/fdatasync_linux.go @@ -0,0 +1,47 @@ +//go:build linux +// +build linux + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +import ( + "os" + "syscall" + + "golang.org/x/sys/unix" +) + +// Fdatasync - fdatasync() is similar to fsync(), but does not flush modified metadata +// unless that metadata is needed in order to allow a subsequent data retrieval +// to be correctly handled. For example, changes to st_atime or st_mtime +// (respectively, time of last access and time of last modification; see inode(7)) +// do not require flushing because they are not necessary for a subsequent data +// read to be handled correctly. On the other hand, a change to the file size +// (st_size, as made by say ftruncate(2)), would require a metadata flush. +// +// The aim of fdatasync() is to reduce disk activity for applications that +// do not require all metadata to be synchronized with the disk. +func Fdatasync(f *os.File) error { + return syscall.Fdatasync(int(f.Fd())) +} + +// FadviseDontNeed invalidates page-cache +func FadviseDontNeed(f *os.File) error { + return unix.Fadvise(int(f.Fd()), 0, 0, unix.FADV_DONTNEED) +} diff --git a/internal/disk/fdatasync_unix.go b/internal/disk/fdatasync_unix.go new file mode 100644 index 00000000..10c224cd --- /dev/null +++ b/internal/disk/fdatasync_unix.go @@ -0,0 +1,36 @@ +//go:build freebsd || netbsd || openbsd || darwin +// +build freebsd netbsd openbsd darwin + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +import ( + "os" + "syscall" +) + +// Fdatasync is fsync on freebsd/darwin +func Fdatasync(f *os.File) error { + return syscall.Fsync(int(f.Fd())) +} + +// FadviseDontNeed is a no-op +func FadviseDontNeed(f *os.File) error { + return nil +} diff --git a/internal/disk/fdatasync_unsupported.go b/internal/disk/fdatasync_unsupported.go new file mode 100644 index 00000000..a3beedb5 --- /dev/null +++ b/internal/disk/fdatasync_unsupported.go @@ -0,0 +1,35 @@ +//go:build !linux && !netbsd && !freebsd && !darwin && !openbsd +// +build !linux,!netbsd,!freebsd,!darwin,!openbsd + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +import ( + "os" +) + +// Fdatasync is a no-op +func Fdatasync(f *os.File) error { + return nil +} + +// FadviseDontNeed is a no-op +func FadviseDontNeed(f *os.File) error { + return nil +} diff --git a/internal/disk/root_disk.go b/internal/disk/root_disk.go new file mode 100644 index 00000000..d29b4c75 --- /dev/null +++ b/internal/disk/root_disk.go @@ -0,0 +1,29 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +import "runtime" + +// IsRootDisk returns if diskPath belongs to root-disk, i.e the disk mounted at "/" +func IsRootDisk(diskPath string, rootDisk string) (bool, error) { + if runtime.GOOS == "windows" { + // On windows this function is not implemented. + return false, nil + } + return SameDisk(diskPath, rootDisk) +} diff --git a/internal/disk/stat_bsd.go b/internal/disk/stat_bsd.go new file mode 100644 index 00000000..b0d8e741 --- /dev/null +++ b/internal/disk/stat_bsd.go @@ -0,0 +1,54 @@ +//go:build darwin || dragonfly +// +build darwin dragonfly + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +import ( + "errors" + "fmt" + "syscall" +) + +// GetInfo returns total and free bytes available in a directory, e.g. `/`. +func GetInfo(path string, _ bool) (info Info, err error) { + s := syscall.Statfs_t{} + err = syscall.Statfs(path, &s) + if err != nil { + return Info{}, err + } + reservedBlocks := s.Bfree - s.Bavail + info = Info{ + Total: uint64(s.Bsize) * (s.Blocks - reservedBlocks), + Free: uint64(s.Bsize) * s.Bavail, + Files: s.Files, + Ffree: s.Ffree, + FSType: getFSType(s.Fstypename[:]), + } + if info.Free > info.Total { + return info, fmt.Errorf("detected free space (%d) > total drive space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path) + } + info.Used = info.Total - info.Free + return info, nil +} + +// GetDriveStats returns IO stats of the drive by its major:minor +func GetDriveStats(major, minor uint32) (iostats IOStats, err error) { + return IOStats{}, errors.New("operation unsupported") +} diff --git a/internal/disk/stat_freebsd.go b/internal/disk/stat_freebsd.go new file mode 100644 index 00000000..f76c9cf2 --- /dev/null +++ b/internal/disk/stat_freebsd.go @@ -0,0 +1,54 @@ +//go:build freebsd +// +build freebsd + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +import ( + "errors" + "fmt" + "syscall" +) + +// GetInfo returns total and free bytes available in a directory, e.g. `/`. +func GetInfo(path string, _ bool) (info Info, err error) { + s := syscall.Statfs_t{} + err = syscall.Statfs(path, &s) + if err != nil { + return Info{}, err + } + reservedBlocks := s.Bfree - uint64(s.Bavail) + info = Info{ + Total: uint64(s.Bsize) * (s.Blocks - reservedBlocks), + Free: uint64(s.Bsize) * uint64(s.Bavail), + Files: s.Files, + Ffree: uint64(s.Ffree), + FSType: getFSType(s.Fstypename[:]), + } + if info.Free > info.Total { + return info, fmt.Errorf("detected free space (%d) > total drive space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path) + } + info.Used = info.Total - info.Free + return info, nil +} + +// GetDriveStats returns IO stats of the drive by its major:minor +func GetDriveStats(major, minor uint32) (iostats IOStats, err error) { + return IOStats{}, errors.New("operation unsupported") +} diff --git a/internal/disk/stat_linux.go b/internal/disk/stat_linux.go new file mode 100644 index 00000000..390cdced --- /dev/null +++ b/internal/disk/stat_linux.go @@ -0,0 +1,173 @@ +//go:build linux && !s390x && !arm && !386 +// +build linux,!s390x,!arm,!386 + +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +import ( + "bufio" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + + "github.com/prometheus/procfs/blockdevice" + "golang.org/x/sys/unix" +) + +// GetInfo returns total and free bytes available in a directory, e.g. `/`. +func GetInfo(path string, firstTime bool) (info Info, err error) { + s := syscall.Statfs_t{} + err = syscall.Statfs(path, &s) + if err != nil { + return Info{}, err + } + reservedBlocks := s.Bfree - s.Bavail + info = Info{ + Total: uint64(s.Frsize) * (s.Blocks - reservedBlocks), + Free: uint64(s.Frsize) * s.Bavail, + Files: s.Files, + Ffree: s.Ffree, + //nolint:unconvert + FSType: getFSType(int64(s.Type)), + } + + st := syscall.Stat_t{} + err = syscall.Stat(path, &st) + if err != nil { + return Info{}, err + } + //nolint:unconvert + devID := uint64(st.Dev) // Needed to support multiple GOARCHs + info.Major = unix.Major(devID) + info.Minor = unix.Minor(devID) + + // Check for overflows. + // https://github.com/minio/minio/issues/8035 + // XFS can show wrong values at times error out + // in such scenarios. + if info.Free > info.Total { + return info, fmt.Errorf("detected free space (%d) > total drive space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path) + } + info.Used = info.Total - info.Free + + if firstTime { + bfs, err := blockdevice.NewDefaultFS() + if err == nil { + devName := "" + diskstats, _ := bfs.ProcDiskstats() + for _, dstat := range diskstats { + // ignore all loop devices + if strings.HasPrefix(dstat.DeviceName, "loop") { + continue + } + if dstat.MajorNumber == info.Major && dstat.MinorNumber == info.Minor { + devName = dstat.DeviceName + break + } + } + if devName != "" { + info.Name = devName + qst, err := bfs.SysBlockDeviceQueueStats(devName) + if err != nil { // Mostly not found error + // Check if there is a parent device: + // e.g. if the mount is based on /dev/nvme0n1p1, let's calculate the + // real device name (nvme0n1) to get its sysfs information + parentDevPath, e := os.Readlink("/sys/class/block/" + devName) + if e == nil { + parentDev := filepath.Base(filepath.Dir(parentDevPath)) + qst, err = bfs.SysBlockDeviceQueueStats(parentDev) + } + } + if err == nil { + info.NRRequests = qst.NRRequests + rot := qst.Rotational == 1 // Rotational is '1' if the device is HDD + info.Rotational = &rot + } + } + } + } + + return info, nil +} + +// GetDriveStats returns IO stats of the drive by its major:minor +func GetDriveStats(major, minor uint32) (iostats IOStats, err error) { + return readDriveStats(fmt.Sprintf("/sys/dev/block/%v:%v/stat", major, minor)) +} + +func readDriveStats(statsFile string) (iostats IOStats, err error) { + stats, err := readStat(statsFile) + if err != nil { + return IOStats{}, err + } + if len(stats) < 11 { + return IOStats{}, fmt.Errorf("found invalid format while reading %v", statsFile) + } + // refer https://www.kernel.org/doc/Documentation/block/stat.txt + iostats = IOStats{ + ReadIOs: stats[0], + ReadMerges: stats[1], + ReadSectors: stats[2], + ReadTicks: stats[3], + WriteIOs: stats[4], + WriteMerges: stats[5], + WriteSectors: stats[6], + WriteTicks: stats[7], + CurrentIOs: stats[8], + TotalTicks: stats[9], + ReqTicks: stats[10], + } + // as per the doc, only 11 fields are guaranteed + // only set if available + if len(stats) > 14 { + iostats.DiscardIOs = stats[11] + iostats.DiscardMerges = stats[12] + iostats.DiscardSectors = stats[13] + iostats.DiscardTicks = stats[14] + } + return iostats, err +} + +func readStat(fileName string) (stats []uint64, err error) { + file, err := os.Open(fileName) + if err != nil { + return nil, err + } + defer file.Close() + + s, err := bufio.NewReader(file).ReadString('\n') + if err != nil && !errors.Is(err, io.EOF) { + return nil, err + } + statLine := strings.TrimSpace(s) + for _, token := range strings.Fields(statLine) { + ui64, err := strconv.ParseUint(token, 10, 64) + if err != nil { + return nil, err + } + stats = append(stats, ui64) + } + + return stats, nil +} diff --git a/internal/disk/stat_linux_32bit.go b/internal/disk/stat_linux_32bit.go new file mode 100644 index 00000000..ee438408 --- /dev/null +++ b/internal/disk/stat_linux_32bit.go @@ -0,0 +1,89 @@ +//go:build (linux && arm) || (linux && 386) +// +build linux,arm linux,386 + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +import ( + "errors" + "fmt" + "strconv" + "syscall" +) + +// fsType2StringMap - list of filesystems supported on linux +var fsType2StringMap = map[string]string{ + "1021994": "TMPFS", + "137d": "EXT", + "4244": "HFS", + "4d44": "MSDOS", + "52654973": "REISERFS", + "5346544e": "NTFS", + "58465342": "XFS", + "61756673": "AUFS", + "6969": "NFS", + "ef51": "EXT2OLD", + "ef53": "EXT4", + "f15f": "ecryptfs", + "794c7630": "overlayfs", + "2fc12fc1": "zfs", + "ff534d42": "cifs", + "53464846": "wslfs", +} + +// getFSType returns the filesystem type of the underlying mounted filesystem +func getFSType(ftype int32) string { + fsTypeHex := strconv.FormatInt(int64(ftype), 16) + fsTypeString, ok := fsType2StringMap[fsTypeHex] + if !ok { + return "UNKNOWN" + } + return fsTypeString +} + +// GetInfo returns total and free bytes available in a directory, e.g. `/`. +func GetInfo(path string, _ bool) (info Info, err error) { + s := syscall.Statfs_t{} + err = syscall.Statfs(path, &s) + if err != nil { + return Info{}, err + } + reservedBlocks := s.Bfree - s.Bavail + info = Info{ + Total: uint64(s.Frsize) * (s.Blocks - reservedBlocks), + Free: uint64(s.Frsize) * s.Bavail, + Files: s.Files, + Ffree: s.Ffree, + FSType: getFSType(s.Type), + } + // Check for overflows. + // https://github.com/minio/minio/issues/8035 + // XFS can show wrong values at times error out + // in such scenarios. + if info.Free > info.Total { + return info, fmt.Errorf("detected free space (%d) > total drive space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path) + } + info.Used = info.Total - info.Free + return info, nil +} + +// GetDriveStats returns IO stats of the drive by its major:minor +func GetDriveStats(major, minor uint32) (iostats IOStats, err error) { + return IOStats{}, errors.New("operation unsupported") +} diff --git a/internal/disk/stat_linux_s390x.go b/internal/disk/stat_linux_s390x.go new file mode 100644 index 00000000..5ee6eabd --- /dev/null +++ b/internal/disk/stat_linux_s390x.go @@ -0,0 +1,89 @@ +//go:build linux && s390x +// +build linux,s390x + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +import ( + "errors" + "fmt" + "strconv" + "syscall" +) + +// fsType2StringMap - list of filesystems supported on linux +var fsType2StringMap = map[string]string{ + "1021994": "TMPFS", + "137d": "EXT", + "4244": "HFS", + "4d44": "MSDOS", + "52654973": "REISERFS", + "5346544e": "NTFS", + "58465342": "XFS", + "61756673": "AUFS", + "6969": "NFS", + "ef51": "EXT2OLD", + "ef53": "EXT4", + "f15f": "ecryptfs", + "794c7630": "overlayfs", + "2fc12fc1": "zfs", + "ff534d42": "cifs", + "53464846": "wslfs", +} + +// getFSType returns the filesystem type of the underlying mounted filesystem +func getFSType(ftype uint32) string { + fsTypeHex := strconv.FormatUint(uint64(ftype), 16) + fsTypeString, ok := fsType2StringMap[fsTypeHex] + if !ok { + return "UNKNOWN" + } + return fsTypeString +} + +// GetInfo returns total and free bytes available in a directory, e.g. `/`. +func GetInfo(path string, _ bool) (info Info, err error) { + s := syscall.Statfs_t{} + err = syscall.Statfs(path, &s) + if err != nil { + return Info{}, err + } + reservedBlocks := s.Bfree - s.Bavail + info = Info{ + Total: uint64(s.Frsize) * (s.Blocks - reservedBlocks), + Free: uint64(s.Frsize) * s.Bavail, + Files: s.Files, + Ffree: s.Ffree, + FSType: getFSType(s.Type), + } + // Check for overflows. + // https://github.com/minio/minio/issues/8035 + // XFS can show wrong values at times error out + // in such scenarios. + if info.Free > info.Total { + return info, fmt.Errorf("detected free space (%d) > total drive space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path) + } + info.Used = info.Total - info.Free + return info, nil +} + +// GetDriveStats returns IO stats of the drive by its major:minor +func GetDriveStats(major, minor uint32) (iostats IOStats, err error) { + return IOStats{}, errors.New("operation unsupported") +} diff --git a/internal/disk/stat_netbsd.go b/internal/disk/stat_netbsd.go new file mode 100644 index 00000000..ba368bd4 --- /dev/null +++ b/internal/disk/stat_netbsd.go @@ -0,0 +1,54 @@ +//go:build netbsd +// +build netbsd + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +import ( + "errors" + "fmt" + + "golang.org/x/sys/unix" +) + +// GetInfo returns total and free bytes available in a directory, e.g. `/`. +func GetInfo(path string, _ bool) (info Info, err error) { + s := unix.Statvfs_t{} + if err = unix.Statvfs(path, &s); err != nil { + return Info{}, err + } + reservedBlocks := uint64(s.Bfree) - uint64(s.Bavail) + info = Info{ + Total: uint64(s.Frsize) * (uint64(s.Blocks) - reservedBlocks), + Free: uint64(s.Frsize) * uint64(s.Bavail), + Files: uint64(s.Files), + Ffree: uint64(s.Ffree), + FSType: string(s.Fstypename[:]), + } + if info.Free > info.Total { + return info, fmt.Errorf("detected free space (%d) > total drive space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path) + } + info.Used = info.Total - info.Free + return info, nil +} + +// GetDriveStats returns IO stats of the drive by its major:minor +func GetDriveStats(major, minor uint32) (iostats IOStats, err error) { + return IOStats{}, errors.New("operation unsupported") +} diff --git a/internal/disk/stat_openbsd.go b/internal/disk/stat_openbsd.go new file mode 100644 index 00000000..f1ec7dd6 --- /dev/null +++ b/internal/disk/stat_openbsd.go @@ -0,0 +1,54 @@ +//go:build openbsd +// +build openbsd + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +import ( + "errors" + "fmt" + "syscall" +) + +// GetInfo returns total and free bytes available in a directory, e.g. `/`. +func GetInfo(path string, _ bool) (info Info, err error) { + s := syscall.Statfs_t{} + err = syscall.Statfs(path, &s) + if err != nil { + return Info{}, err + } + reservedBlocks := uint64(s.F_bfree) - uint64(s.F_bavail) + info = Info{ + Total: uint64(s.F_bsize) * (uint64(s.F_blocks) - reservedBlocks), + Free: uint64(s.F_bsize) * uint64(s.F_bavail), + Files: uint64(s.F_files), + Ffree: uint64(s.F_ffree), + FSType: getFSType(s.F_fstypename[:]), + } + if info.Free > info.Total { + return info, fmt.Errorf("detected free space (%d) > total drive space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path) + } + info.Used = info.Total - info.Free + return info, nil +} + +// GetDriveStats returns IO stats of the drive by its major:minor +func GetDriveStats(major, minor uint32) (iostats IOStats, err error) { + return IOStats{}, errors.New("operation unsupported") +} diff --git a/internal/disk/stat_solaris.go b/internal/disk/stat_solaris.go new file mode 100644 index 00000000..0409c0ec --- /dev/null +++ b/internal/disk/stat_solaris.go @@ -0,0 +1,54 @@ +//go:build solaris +// +build solaris + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +import ( + "errors" + "fmt" + + "golang.org/x/sys/unix" +) + +// GetInfo returns total and free bytes available in a directory, e.g. `/`. +func GetInfo(path string, _ bool) (info Info, err error) { + s := unix.Statvfs_t{} + if err = unix.Statvfs(path, &s); err != nil { + return Info{}, err + } + reservedBlocks := uint64(s.Bfree) - uint64(s.Bavail) + info = Info{ + Total: uint64(s.Frsize) * (uint64(s.Blocks) - reservedBlocks), + Free: uint64(s.Frsize) * uint64(s.Bavail), + Files: uint64(s.Files), + Ffree: uint64(s.Ffree), + FSType: getFSType(s.Fstr[:]), + } + if info.Free > info.Total { + return info, fmt.Errorf("detected free space (%d) > total drive space (%d), fs corruption at (%s). please run 'fsck'", info.Free, info.Total, path) + } + info.Used = info.Total - info.Free + return info, nil +} + +// GetDriveStats returns IO stats of the drive by its major:minor +func GetDriveStats(major, minor uint32) (iostats IOStats, err error) { + return IOStats{}, errors.New("operation unsupported") +} diff --git a/internal/disk/stat_test.go b/internal/disk/stat_test.go new file mode 100644 index 00000000..73ca017f --- /dev/null +++ b/internal/disk/stat_test.go @@ -0,0 +1,126 @@ +//go:build linux && !s390x && !arm && !386 +// +build linux,!s390x,!arm,!386 + +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +import ( + "os" + "reflect" + "testing" +) + +func TestReadDriveStats(t *testing.T) { + testCases := []struct { + stat string + expectedIOStats IOStats + expectErr bool + }{ + { + stat: "1432553 420084 66247626 2398227 7077314 8720147 157049224 7469810 0 7580552 9869354 46037 0 41695120 1315 0 0", + expectedIOStats: IOStats{ + ReadIOs: 1432553, + ReadMerges: 420084, + ReadSectors: 66247626, + ReadTicks: 2398227, + WriteIOs: 7077314, + WriteMerges: 8720147, + WriteSectors: 157049224, + WriteTicks: 7469810, + CurrentIOs: 0, + TotalTicks: 7580552, + ReqTicks: 9869354, + DiscardIOs: 46037, + DiscardMerges: 0, + DiscardSectors: 41695120, + DiscardTicks: 1315, + FlushIOs: 0, + FlushTicks: 0, + }, + expectErr: false, + }, + { + stat: "1432553 420084 66247626 2398227 7077314 8720147 157049224 7469810 0 7580552 9869354 46037 0 41695120 1315", + expectedIOStats: IOStats{ + ReadIOs: 1432553, + ReadMerges: 420084, + ReadSectors: 66247626, + ReadTicks: 2398227, + WriteIOs: 7077314, + WriteMerges: 8720147, + WriteSectors: 157049224, + WriteTicks: 7469810, + CurrentIOs: 0, + TotalTicks: 7580552, + ReqTicks: 9869354, + DiscardIOs: 46037, + DiscardMerges: 0, + DiscardSectors: 41695120, + DiscardTicks: 1315, + }, + expectErr: false, + }, + { + stat: "1432553 420084 66247626 2398227 7077314 8720147 157049224 7469810 0 7580552 9869354", + expectedIOStats: IOStats{ + ReadIOs: 1432553, + ReadMerges: 420084, + ReadSectors: 66247626, + ReadTicks: 2398227, + WriteIOs: 7077314, + WriteMerges: 8720147, + WriteSectors: 157049224, + WriteTicks: 7469810, + CurrentIOs: 0, + TotalTicks: 7580552, + ReqTicks: 9869354, + }, + expectErr: false, + }, + { + stat: "1432553 420084 66247626 2398227", + expectedIOStats: IOStats{}, + expectErr: true, + }, + } + + for _, testCase := range testCases { + testCase := testCase + t.Run("", func(t *testing.T) { + tmpfile, err := os.CreateTemp(t.TempDir(), "testfile") + if err != nil { + t.Error(err) + } + tmpfile.WriteString(testCase.stat) + tmpfile.Sync() + tmpfile.Close() + + iostats, err := readDriveStats(tmpfile.Name()) + if err != nil && !testCase.expectErr { + t.Fatalf("unexpected err; %v", err) + } + if testCase.expectErr && err == nil { + t.Fatal("expected to fail but err is nil") + } + if !reflect.DeepEqual(iostats, testCase.expectedIOStats) { + t.Fatalf("expected iostats: %v but got %v", testCase.expectedIOStats, iostats) + } + }) + } +} diff --git a/internal/disk/stat_windows.go b/internal/disk/stat_windows.go new file mode 100644 index 00000000..7037932b --- /dev/null +++ b/internal/disk/stat_windows.go @@ -0,0 +1,114 @@ +//go:build windows +// +build windows + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +import ( + "errors" + "fmt" + "os" + "syscall" + "unsafe" + + "golang.org/x/sys/windows" +) + +var ( + kernel32 = windows.NewLazySystemDLL("kernel32.dll") + + // GetDiskFreeSpaceEx - https://msdn.microsoft.com/en-us/library/windows/desktop/aa364937(v=vs.85).aspx + // Retrieves information about the amount of space that is available on a disk volume, + // which is the total amount of space, the total amount of free space, and the total + // amount of free space available to the user that is associated with the calling thread. + GetDiskFreeSpaceEx = kernel32.NewProc("GetDiskFreeSpaceExW") + + // GetDiskFreeSpace - https://msdn.microsoft.com/en-us/library/windows/desktop/aa364935(v=vs.85).aspx + // Retrieves information about the specified disk, including the amount of free space on the disk. + GetDiskFreeSpace = kernel32.NewProc("GetDiskFreeSpaceW") +) + +// GetInfo returns total and free bytes available in a directory, e.g. `C:\`. +// It returns free space available to the user (including quota limitations) +// +// https://msdn.microsoft.com/en-us/library/windows/desktop/aa364937(v=vs.85).aspx +func GetInfo(path string, _ bool) (info Info, err error) { + // Stat to know if the path exists. + if _, err = os.Stat(path); err != nil { + return Info{}, err + } + + lpFreeBytesAvailable := int64(0) + lpTotalNumberOfBytes := int64(0) + lpTotalNumberOfFreeBytes := int64(0) + + // Extract values safely + // BOOL WINAPI GetDiskFreeSpaceEx( + // _In_opt_ LPCTSTR lpDirectoryName, + // _Out_opt_ PULARGE_INTEGER lpFreeBytesAvailable, + // _Out_opt_ PULARGE_INTEGER lpTotalNumberOfBytes, + // _Out_opt_ PULARGE_INTEGER lpTotalNumberOfFreeBytes + // ); + _, _, _ = GetDiskFreeSpaceEx.Call(uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))), + uintptr(unsafe.Pointer(&lpFreeBytesAvailable)), + uintptr(unsafe.Pointer(&lpTotalNumberOfBytes)), + uintptr(unsafe.Pointer(&lpTotalNumberOfFreeBytes))) + + if uint64(lpTotalNumberOfFreeBytes) > uint64(lpTotalNumberOfBytes) { + return info, fmt.Errorf("detected free space (%d) > total drive space (%d), fs corruption at (%s). please run 'fsck'", + uint64(lpTotalNumberOfFreeBytes), uint64(lpTotalNumberOfBytes), path) + } + + info = Info{ + Total: uint64(lpTotalNumberOfBytes), + Free: uint64(lpTotalNumberOfFreeBytes), + Used: uint64(lpTotalNumberOfBytes) - uint64(lpTotalNumberOfFreeBytes), + FSType: getFSType(path), + } + + // Return values of GetDiskFreeSpace() + lpSectorsPerCluster := uint32(0) + lpBytesPerSector := uint32(0) + lpNumberOfFreeClusters := uint32(0) + lpTotalNumberOfClusters := uint32(0) + + // Extract values safely + // BOOL WINAPI GetDiskFreeSpace( + // _In_ LPCTSTR lpRootPathName, + // _Out_ LPDWORD lpSectorsPerCluster, + // _Out_ LPDWORD lpBytesPerSector, + // _Out_ LPDWORD lpNumberOfFreeClusters, + // _Out_ LPDWORD lpTotalNumberOfClusters + // ); + _, _, _ = GetDiskFreeSpace.Call(uintptr(unsafe.Pointer(syscall.StringToUTF16Ptr(path))), + uintptr(unsafe.Pointer(&lpSectorsPerCluster)), + uintptr(unsafe.Pointer(&lpBytesPerSector)), + uintptr(unsafe.Pointer(&lpNumberOfFreeClusters)), + uintptr(unsafe.Pointer(&lpTotalNumberOfClusters))) + + info.Files = uint64(lpTotalNumberOfClusters) + info.Ffree = uint64(lpNumberOfFreeClusters) + + return info, nil +} + +// GetDriveStats returns IO stats of the drive by its major:minor +func GetDriveStats(major, minor uint32) (iostats IOStats, err error) { + return IOStats{}, errors.New("operation unsupported") +} diff --git a/internal/disk/type_bsd.go b/internal/disk/type_bsd.go new file mode 100644 index 00000000..5212864e --- /dev/null +++ b/internal/disk/type_bsd.go @@ -0,0 +1,30 @@ +//go:build darwin || freebsd || dragonfly || openbsd || solaris +// +build darwin freebsd dragonfly openbsd solaris + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +// getFSType returns the filesystem type of the underlying mounted filesystem +func getFSType(fstype []int8) string { + b := make([]byte, len(fstype)) + for i, v := range fstype { + b[i] = byte(v) + } + return string(b) +} diff --git a/internal/disk/type_linux.go b/internal/disk/type_linux.go new file mode 100644 index 00000000..37788d2b --- /dev/null +++ b/internal/disk/type_linux.go @@ -0,0 +1,53 @@ +//go:build linux && !s390x && !arm && !386 +// +build linux,!s390x,!arm,!386 + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +import "strconv" + +// fsType2StringMap - list of filesystems supported on linux +var fsType2StringMap = map[string]string{ + "1021994": "TMPFS", + "137d": "EXT", + "4244": "HFS", + "4d44": "MSDOS", + "52654973": "REISERFS", + "5346544e": "NTFS", + "58465342": "XFS", + "61756673": "AUFS", + "6969": "NFS", + "ef51": "EXT2OLD", + "ef53": "EXT4", + "f15f": "ecryptfs", + "794c7630": "overlayfs", + "2fc12fc1": "zfs", + "ff534d42": "cifs", + "53464846": "wslfs", +} + +// getFSType returns the filesystem type of the underlying mounted filesystem +func getFSType(ftype int64) string { + fsTypeHex := strconv.FormatInt(ftype, 16) + fsTypeString, ok := fsType2StringMap[fsTypeHex] + if !ok { + return "UNKNOWN" + } + return fsTypeString +} diff --git a/internal/disk/type_windows.go b/internal/disk/type_windows.go new file mode 100644 index 00000000..579f5e81 --- /dev/null +++ b/internal/disk/type_windows.go @@ -0,0 +1,62 @@ +//go:build windows +// +build windows + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package disk + +import ( + "path/filepath" + "syscall" + "unsafe" +) + +// GetVolumeInformation provides windows drive volume information. +var GetVolumeInformation = kernel32.NewProc("GetVolumeInformationW") + +// getFSType returns the filesystem type of the underlying mounted filesystem +func getFSType(path string) string { + volumeNameSize, nFileSystemNameSize := uint32(260), uint32(260) + var lpVolumeSerialNumber uint32 + var lpFileSystemFlags, lpMaximumComponentLength uint32 + var lpFileSystemNameBuffer, volumeName [260]uint16 + ps := syscall.StringToUTF16Ptr(filepath.VolumeName(path)) + + // Extract values safely + // BOOL WINAPI GetVolumeInformation( + // _In_opt_ LPCTSTR lpRootPathName, + // _Out_opt_ LPTSTR lpVolumeNameBuffer, + // _In_ DWORD nVolumeNameSize, + // _Out_opt_ LPDWORD lpVolumeSerialNumber, + // _Out_opt_ LPDWORD lpMaximumComponentLength, + // _Out_opt_ LPDWORD lpFileSystemFlags, + // _Out_opt_ LPTSTR lpFileSystemNameBuffer, + // _In_ DWORD nFileSystemNameSize + // ); + + _, _, _ = GetVolumeInformation.Call(uintptr(unsafe.Pointer(ps)), + uintptr(unsafe.Pointer(&volumeName)), + uintptr(volumeNameSize), + uintptr(unsafe.Pointer(&lpVolumeSerialNumber)), + uintptr(unsafe.Pointer(&lpMaximumComponentLength)), + uintptr(unsafe.Pointer(&lpFileSystemFlags)), + uintptr(unsafe.Pointer(&lpFileSystemNameBuffer)), + uintptr(nFileSystemNameSize)) + + return syscall.UTF16ToString(lpFileSystemNameBuffer[:]) +} diff --git a/internal/dsync/.gitignore b/internal/dsync/.gitignore new file mode 100644 index 00000000..c17ac1ee --- /dev/null +++ b/internal/dsync/.gitignore @@ -0,0 +1,3 @@ +dsync.test +coverage.txt +*.out \ No newline at end of file diff --git a/internal/dsync/drwmutex.go b/internal/dsync/drwmutex.go new file mode 100644 index 00000000..7d6506ea --- /dev/null +++ b/internal/dsync/drwmutex.go @@ -0,0 +1,735 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dsync + +import ( + "context" + "errors" + "math/rand" + "slices" + "sort" + "strconv" + "sync" + "time" + + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/mcontext" + "github.com/minio/pkg/v3/console" + "github.com/minio/pkg/v3/env" +) + +// Indicator if logging is enabled. +var dsyncLog bool + +// Retry unit interval +var lockRetryMinInterval time.Duration + +var lockRetryBackOff func(*rand.Rand, uint) time.Duration + +func init() { + // Check for MINIO_DSYNC_TRACE env variable, if set logging will be enabled for failed REST operations. + dsyncLog = env.Get("_MINIO_DSYNC_TRACE", "0") == "1" + + lockRetryMinInterval = 250 * time.Millisecond + if lri := env.Get("_MINIO_LOCK_RETRY_INTERVAL", ""); lri != "" { + v, err := strconv.Atoi(lri) + if err != nil { + panic(err) + } + lockRetryMinInterval = time.Duration(v) * time.Millisecond + } + + lockRetryBackOff = backoffWait( + lockRetryMinInterval, + 100*time.Millisecond, + 5*time.Second, + ) +} + +func log(format string, data ...any) { + if dsyncLog { + console.Printf(format, data...) + } +} + +const ( + // dRWMutexAcquireTimeout - default tolerance limit to wait for lock acquisition before. + drwMutexAcquireTimeout = 1 * time.Second // 1 second. + + // dRWMutexRefreshTimeout - default timeout for the refresh call + drwMutexRefreshCallTimeout = 5 * time.Second + + // dRWMutexUnlockTimeout - default timeout for the unlock call + drwMutexUnlockCallTimeout = 30 * time.Second + + // dRWMutexForceUnlockTimeout - default timeout for the unlock call + drwMutexForceUnlockCallTimeout = 30 * time.Second + + // dRWMutexRefreshInterval - default the interval between two refresh calls + drwMutexRefreshInterval = 10 * time.Second + + drwMutexInfinite = 1<<63 - 1 +) + +// Timeouts are timeouts for specific operations. +type Timeouts struct { + // Acquire - tolerance limit to wait for lock acquisition before. + Acquire time.Duration + + // RefreshCall - timeout for the refresh call + RefreshCall time.Duration + + // UnlockCall - timeout for the unlock call + UnlockCall time.Duration + + // ForceUnlockCall - timeout for the force unlock call + ForceUnlockCall time.Duration +} + +// DefaultTimeouts contains default timeouts. +var DefaultTimeouts = Timeouts{ + Acquire: drwMutexAcquireTimeout, + RefreshCall: drwMutexRefreshCallTimeout, + UnlockCall: drwMutexUnlockCallTimeout, + ForceUnlockCall: drwMutexForceUnlockCallTimeout, +} + +// A DRWMutex is a distributed mutual exclusion lock. +type DRWMutex struct { + Names []string + writeLocks []string // Array of nodes that granted a write lock + readLocks []string // Array of array of nodes that granted reader locks + rng *rand.Rand + m sync.Mutex // Mutex to prevent multiple simultaneous locks from this node + clnt *Dsync + cancelRefresh context.CancelFunc + refreshInterval time.Duration + lockRetryMinInterval time.Duration +} + +// Granted - represents a structure of a granted lock. +type Granted struct { + index int + lockUID string // Locked if set with UID string, unlocked if empty +} + +func (g *Granted) isLocked() bool { + return isLocked(g.lockUID) +} + +func isLocked(uid string) bool { + return len(uid) > 0 +} + +// NewDRWMutex - initializes a new dsync RW mutex. +func NewDRWMutex(clnt *Dsync, names ...string) *DRWMutex { + restClnts, _ := clnt.GetLockers() + sort.Strings(names) + return &DRWMutex{ + writeLocks: make([]string, len(restClnts)), + readLocks: make([]string, len(restClnts)), + Names: names, + clnt: clnt, + rng: rand.New(&lockedRandSource{src: rand.NewSource(time.Now().UTC().UnixNano())}), + refreshInterval: drwMutexRefreshInterval, + lockRetryMinInterval: lockRetryMinInterval, + } +} + +// Lock holds a write lock on dm. +// +// If the lock is already in use, the calling go routine +// blocks until the mutex is available. +func (dm *DRWMutex) Lock(id, source string) { + isReadLock := false + dm.lockBlocking(context.Background(), nil, id, source, isReadLock, Options{ + Timeout: drwMutexInfinite, + }) +} + +// Options lock options. +type Options struct { + Timeout time.Duration + RetryInterval time.Duration +} + +// GetLock tries to get a write lock on dm before the timeout elapses. +// +// If the lock is already in use, the calling go routine +// blocks until either the mutex becomes available and return success or +// more time has passed than the timeout value and return false. +func (dm *DRWMutex) GetLock(ctx context.Context, cancel context.CancelFunc, id, source string, opts Options) (locked bool) { + isReadLock := false + return dm.lockBlocking(ctx, cancel, id, source, isReadLock, opts) +} + +// RLock holds a read lock on dm. +// +// If one or more read locks are already in use, it will grant another lock. +// Otherwise the calling go routine blocks until the mutex is available. +func (dm *DRWMutex) RLock(id, source string) { + isReadLock := true + dm.lockBlocking(context.Background(), nil, id, source, isReadLock, Options{ + Timeout: drwMutexInfinite, + }) +} + +// GetRLock tries to get a read lock on dm before the timeout elapses. +// +// If one or more read locks are already in use, it will grant another lock. +// Otherwise the calling go routine blocks until either the mutex becomes +// available and return success or more time has passed than the timeout +// value and return false. +func (dm *DRWMutex) GetRLock(ctx context.Context, cancel context.CancelFunc, id, source string, opts Options) (locked bool) { + isReadLock := true + return dm.lockBlocking(ctx, cancel, id, source, isReadLock, opts) +} + +// lockBlocking will try to acquire either a read or a write lock +// +// The function will loop using a built-in timing randomized back-off +// algorithm until either the lock is acquired successfully or more +// time has elapsed than the timeout value. +func (dm *DRWMutex) lockBlocking(ctx context.Context, lockLossCallback func(), id, source string, isReadLock bool, opts Options) (locked bool) { + restClnts, _ := dm.clnt.GetLockers() + + // Create lock array to capture the successful lockers + locks := make([]string, len(restClnts)) + + // Add total timeout + ctx, cancel := context.WithTimeout(ctx, opts.Timeout) + defer cancel() + + // Tolerance is not set, defaults to half of the locker clients. + tolerance := len(restClnts) / 2 + + // Quorum is effectively = total clients subtracted with tolerance limit + quorum := len(restClnts) - tolerance + if !isReadLock { + // In situations for write locks, as a special case + // to avoid split brains we make sure to acquire + // quorum + 1 when tolerance is exactly half of the + // total locker clients. + if quorum == tolerance { + quorum++ + } + } + + log("lockBlocking %s/%s for %#v: lockType readLock(%t), additional opts: %#v, quorum: %d, tolerance: %d, lockClients: %d\n", id, source, dm.Names, isReadLock, opts, quorum, tolerance, len(restClnts)) + + tolerance = len(restClnts) - quorum + attempt := uint(0) + + for { + select { + case <-ctx.Done(): + return false + default: + // Try to acquire the lock. + if locked = lock(ctx, dm.clnt, &locks, id, source, isReadLock, tolerance, quorum, dm.Names...); locked { + dm.m.Lock() + + // If success, copy array to object + if isReadLock { + copy(dm.readLocks, locks) + } else { + copy(dm.writeLocks, locks) + } + + dm.m.Unlock() + log("lockBlocking %s/%s for %#v: granted\n", id, source, dm.Names) + + // Refresh lock continuously and cancel if there is no quorum in the lock anymore + dm.startContinuousLockRefresh(lockLossCallback, id, source, quorum) + + return locked + } + + switch { + case opts.RetryInterval < 0: + return false + case opts.RetryInterval > 0: + time.Sleep(opts.RetryInterval) + default: + attempt++ + time.Sleep(lockRetryBackOff(dm.rng, attempt)) + } + } + } +} + +func (dm *DRWMutex) startContinuousLockRefresh(lockLossCallback func(), id, source string, quorum int) { + ctx, cancel := context.WithCancel(context.Background()) + + dm.m.Lock() + dm.cancelRefresh = cancel + dm.m.Unlock() + + go func() { + defer cancel() + + refreshTimer := time.NewTimer(dm.refreshInterval) + defer refreshTimer.Stop() + + for { + select { + case <-ctx.Done(): + return + case <-refreshTimer.C: + noQuorum, err := refreshLock(ctx, dm.clnt, id, source, quorum) + if err == nil && noQuorum { + // Clean the lock locally and in remote nodes + forceUnlock(ctx, dm.clnt, id) + // Execute the caller lock loss callback + if lockLossCallback != nil { + lockLossCallback() + } + return + } + + refreshTimer.Reset(dm.refreshInterval) + } + } + }() +} + +func forceUnlock(ctx context.Context, ds *Dsync, id string) { + ctx, cancel := context.WithTimeout(ctx, ds.Timeouts.ForceUnlockCall) + defer cancel() + + restClnts, _ := ds.GetLockers() + + args := LockArgs{ + UID: id, + } + + var wg sync.WaitGroup + for index, c := range restClnts { + wg.Add(1) + // Send refresh request to all nodes + go func(index int, c NetLocker) { + defer wg.Done() + c.ForceUnlock(ctx, args) + }(index, c) + } + wg.Wait() +} + +type refreshResult struct { + offline bool + refreshed bool +} + +// Refresh the given lock in all nodes, return true to indicate if a lock +// does not exist in enough quorum nodes. +func refreshLock(ctx context.Context, ds *Dsync, id, source string, quorum int) (bool, error) { + restClnts, _ := ds.GetLockers() + + // Create buffered channel of size equal to total number of nodes. + ch := make(chan refreshResult, len(restClnts)) + var wg sync.WaitGroup + + args := LockArgs{ + UID: id, + } + + for index, c := range restClnts { + wg.Add(1) + // Send refresh request to all nodes + go func(index int, c NetLocker) { + defer wg.Done() + + if c == nil { + ch <- refreshResult{offline: true} + return + } + + ctx, cancel := context.WithTimeout(ctx, ds.Timeouts.RefreshCall) + defer cancel() + + refreshed, err := c.Refresh(ctx, args) + if err != nil { + ch <- refreshResult{offline: true} + log("dsync: Unable to call Refresh failed with %s for %#v at %s\n", err, args, c) + } else { + ch <- refreshResult{refreshed: refreshed} + log("dsync: Refresh returned false for %#v at %s\n", args, c) + } + }(index, c) + } + + // Wait until we have either + // + // a) received all refresh responses + // b) received too many refreshed for quorum to be still possible + // c) timed out + // + lockNotFound, lockRefreshed := 0, 0 + done := false + + for range len(restClnts) { + select { + case refreshResult := <-ch: + if refreshResult.offline { + continue + } + if refreshResult.refreshed { + lockRefreshed++ + } else { + lockNotFound++ + } + if lockRefreshed >= quorum || lockNotFound > len(restClnts)-quorum { + done = true + } + case <-ctx.Done(): + // Refreshing is canceled + return false, ctx.Err() + } + if done { + break + } + } + + // We may have some unused results in ch, release them async. + go func() { + wg.Wait() + xioutil.SafeClose(ch) + for range ch { + } + }() + + noQuorum := lockNotFound > len(restClnts)-quorum + return noQuorum, nil +} + +// lock tries to acquire the distributed lock, returning true or false. +func lock(ctx context.Context, ds *Dsync, locks *[]string, id, source string, isReadLock bool, tolerance, quorum int, names ...string) bool { + for i := range *locks { + (*locks)[i] = "" + } + + restClnts, owner := ds.GetLockers() + + // Create buffered channel of size equal to total number of nodes. + ch := make(chan Granted, len(restClnts)) + var wg sync.WaitGroup + + args := LockArgs{ + Owner: owner, + UID: id, + Resources: names, + Source: source, + Quorum: &quorum, + } + + // Combined timeout for the lock attempt. + ctx, cancel := context.WithTimeout(ctx, ds.Timeouts.Acquire) + defer cancel() + + // Special context for NetLockers - do not use timeouts. + // Also, pass the trace context info if found for debugging + netLockCtx := context.Background() + + tc, ok := ctx.Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt) + if ok { + netLockCtx = context.WithValue(netLockCtx, mcontext.ContextTraceKey, tc) + } + + for index, c := range restClnts { + wg.Add(1) + // broadcast lock request to all nodes + go func(index int, isReadLock bool, c NetLocker) { + defer wg.Done() + + g := Granted{index: index} + if c == nil { + log("dsync: nil locker\n") + ch <- g + return + } + + var locked bool + var err error + if isReadLock { + if locked, err = c.RLock(netLockCtx, args); err != nil { + log("dsync: Unable to call RLock failed with %s for %#v at %s\n", err, args, c) + } + } else { + if locked, err = c.Lock(netLockCtx, args); err != nil { + log("dsync: Unable to call Lock failed with %s for %#v at %s\n", err, args, c) + } + } + if locked { + g.lockUID = args.UID + } + ch <- g + }(index, isReadLock, c) + } + + // Wait until we have either + // + // a) received all lock responses + // b) received too many 'non-'locks for quorum to be still possible + // c) timed out + // + i, locksFailed := 0, 0 + done := false + + for ; i < len(restClnts); i++ { // Loop until we acquired all locks + select { + case grant := <-ch: + if grant.isLocked() { + // Mark that this node has acquired the lock + (*locks)[grant.index] = grant.lockUID + } else { + locksFailed++ + if locksFailed > tolerance { + // We know that we are not going to get the lock anymore, + // so exit out and release any locks that did get acquired + done = true + } + } + case <-ctx.Done(): + // Capture timedout locks as failed or took too long + locksFailed++ + if locksFailed > tolerance { + // We know that we are not going to get the lock anymore, + // so exit out and release any locks that did get acquired + done = true + } + } + + if done { + break + } + } + + quorumLocked := checkQuorumLocked(locks, quorum) && locksFailed <= tolerance + if !quorumLocked { + log("dsync: Unable to acquire lock in quorum %#v\n", args) + // Release all acquired locks without quorum. + if !releaseAll(ctx, ds, tolerance, owner, locks, isReadLock, restClnts, names...) { + log("Unable to release acquired locks, these locks will expire automatically %#v\n", args) + } + } + + // We may have some unused results in ch, release them async. + go func() { + wg.Wait() + xioutil.SafeClose(ch) + for grantToBeReleased := range ch { + if grantToBeReleased.isLocked() { + // release abandoned lock + log("Releasing abandoned lock\n") + sendRelease(ctx, ds, restClnts[grantToBeReleased.index], + owner, grantToBeReleased.lockUID, isReadLock, names...) + } + } + }() + + return quorumLocked +} + +// checkFailedUnlocks determines whether we have sufficiently unlocked all +// resources to ensure no deadlocks for future callers +func checkFailedUnlocks(locks []string, tolerance int) bool { + unlocksFailed := 0 + for lockID := range locks { + if isLocked(locks[lockID]) { + unlocksFailed++ + } + } + + // Unlock failures are higher than tolerance limit + // for this instance of unlocker, we should let the + // caller know that lock is not successfully released + // yet. + if len(locks)-tolerance == tolerance { + // In case of split brain scenarios where + // tolerance is exactly half of the len(*locks) + // then we need to make sure we have unlocked + // upto tolerance+1 - especially for RUnlock + // to ensure that we don't end up with active + // read locks on the resource after unlocking + // only half of the lockers. + return unlocksFailed >= tolerance + } + return unlocksFailed > tolerance +} + +// checkQuorumLocked determines whether we have locked the required quorum of underlying locks or not +func checkQuorumLocked(locks *[]string, quorum int) bool { + count := 0 + for _, uid := range *locks { + if isLocked(uid) { + count++ + } + } + + return count >= quorum +} + +// releaseAll releases all locks that are marked as locked +func releaseAll(ctx context.Context, ds *Dsync, tolerance int, owner string, locks *[]string, isReadLock bool, restClnts []NetLocker, names ...string) bool { + var wg sync.WaitGroup + for lockID := range restClnts { + wg.Add(1) + go func(lockID int) { + defer wg.Done() + if sendRelease(ctx, ds, restClnts[lockID], owner, (*locks)[lockID], isReadLock, names...) { + (*locks)[lockID] = "" + } + }(lockID) + } + wg.Wait() + + // Return true if releaseAll was successful, otherwise we return 'false' + // to indicate we haven't sufficiently unlocked lockers to avoid deadlocks. + // + // Caller may use this as an indication to call again. + return !checkFailedUnlocks(*locks, tolerance) +} + +// Unlock unlocks the write lock. +// +// It is a run-time error if dm is not locked on entry to Unlock. +func (dm *DRWMutex) Unlock(ctx context.Context) { + dm.m.Lock() + dm.cancelRefresh() + dm.m.Unlock() + + restClnts, owner := dm.clnt.GetLockers() + // create temp array on stack + locks := make([]string, len(restClnts)) + + { + dm.m.Lock() + defer dm.m.Unlock() + + // Check if minimally a single bool is set in the writeLocks array + lockFound := slices.ContainsFunc(dm.writeLocks, isLocked) + if !lockFound { + panic("Trying to Unlock() while no Lock() is active") + } + + // Copy write locks to stack array + copy(locks, dm.writeLocks) + } + + // Tolerance is not set, defaults to half of the locker clients. + tolerance := len(restClnts) / 2 + + isReadLock := false + started := time.Now() + // Do async unlocking. + // This means unlock will no longer block on the network or missing quorum. + go func() { + ctx, done := context.WithTimeout(ctx, drwMutexUnlockCallTimeout) + defer done() + for !releaseAll(ctx, dm.clnt, tolerance, owner, &locks, isReadLock, restClnts, dm.Names...) { + time.Sleep(time.Duration(dm.rng.Float64() * float64(dm.lockRetryMinInterval))) + if time.Since(started) > dm.clnt.Timeouts.UnlockCall { + return + } + } + }() +} + +// RUnlock releases a read lock held on dm. +// +// It is a run-time error if dm is not locked on entry to RUnlock. +func (dm *DRWMutex) RUnlock(ctx context.Context) { + dm.m.Lock() + dm.cancelRefresh() + dm.m.Unlock() + + restClnts, owner := dm.clnt.GetLockers() + // create temp array on stack + locks := make([]string, len(restClnts)) + + { + dm.m.Lock() + defer dm.m.Unlock() + + // Check if minimally a single bool is set in the writeLocks array + lockFound := slices.ContainsFunc(dm.readLocks, isLocked) + if !lockFound { + panic("Trying to RUnlock() while no RLock() is active") + } + + // Copy write locks to stack array + copy(locks, dm.readLocks) + } + + // Tolerance is not set, defaults to half of the locker clients. + tolerance := len(restClnts) / 2 + isReadLock := true + started := time.Now() + // Do async unlocking. + // This means unlock will no longer block on the network or missing quorum. + go func() { + for !releaseAll(ctx, dm.clnt, tolerance, owner, &locks, isReadLock, restClnts, dm.Names...) { + time.Sleep(time.Duration(dm.rng.Float64() * float64(dm.lockRetryMinInterval))) + // If we have been waiting for more than the force unlock timeout, return + // Remotes will have canceled due to the missing refreshes anyway. + if time.Since(started) > dm.clnt.Timeouts.UnlockCall { + return + } + } + }() +} + +// sendRelease sends a release message to a node that previously granted a lock +func sendRelease(ctx context.Context, ds *Dsync, c NetLocker, owner string, uid string, isReadLock bool, names ...string) bool { + if c == nil { + log("Unable to call RUnlock failed with %s\n", errors.New("netLocker is offline")) + return false + } + + if len(uid) == 0 { + return false + } + + args := LockArgs{ + Owner: owner, + UID: uid, + Resources: names, + } + + netLockCtx, cancel := context.WithTimeout(context.Background(), ds.Timeouts.UnlockCall) + defer cancel() + + tc, ok := ctx.Value(mcontext.ContextTraceKey).(*mcontext.TraceCtxt) + if ok { + netLockCtx = context.WithValue(netLockCtx, mcontext.ContextTraceKey, tc) + } + + if isReadLock { + if _, err := c.RUnlock(netLockCtx, args); err != nil { + log("dsync: Unable to call RUnlock failed with %s for %#v at %s\n", err, args, c) + return false + } + } else { + if _, err := c.Unlock(netLockCtx, args); err != nil { + log("dsync: Unable to call Unlock failed with %s for %#v at %s\n", err, args, c) + return false + } + } + + return true +} diff --git a/internal/dsync/drwmutex_test.go b/internal/dsync/drwmutex_test.go new file mode 100644 index 00000000..78d37b4e --- /dev/null +++ b/internal/dsync/drwmutex_test.go @@ -0,0 +1,355 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dsync + +import ( + "context" + "fmt" + "runtime" + "sync/atomic" + "testing" + "time" +) + +const ( + id = "1234-5678" + source = "main.go" +) + +func testSimpleWriteLock(t *testing.T, duration time.Duration) (locked bool) { + drwm1 := NewDRWMutex(ds, "simplelock") + ctx1, cancel1 := context.WithCancel(t.Context()) + if !drwm1.GetRLock(ctx1, cancel1, id, source, Options{Timeout: time.Second}) { + panic("Failed to acquire read lock") + } + // fmt.Println("1st read lock acquired, waiting...") + + drwm2 := NewDRWMutex(ds, "simplelock") + ctx2, cancel2 := context.WithCancel(t.Context()) + if !drwm2.GetRLock(ctx2, cancel2, id, source, Options{Timeout: time.Second}) { + panic("Failed to acquire read lock") + } + // fmt.Println("2nd read lock acquired, waiting...") + + go func() { + time.Sleep(2 * testDrwMutexAcquireTimeout) + drwm1.RUnlock(t.Context()) + // fmt.Println("1st read lock released, waiting...") + }() + + go func() { + time.Sleep(3 * testDrwMutexAcquireTimeout) + drwm2.RUnlock(t.Context()) + // fmt.Println("2nd read lock released, waiting...") + }() + + drwm3 := NewDRWMutex(ds, "simplelock") + // fmt.Println("Trying to acquire write lock, waiting...") + ctx3, cancel3 := context.WithCancel(t.Context()) + locked = drwm3.GetLock(ctx3, cancel3, id, source, Options{Timeout: duration}) + if locked { + // fmt.Println("Write lock acquired, waiting...") + time.Sleep(testDrwMutexAcquireTimeout) + + drwm3.Unlock(t.Context()) + } + // fmt.Println("Write lock failed due to timeout") + return locked +} + +func TestSimpleWriteLockAcquired(t *testing.T) { + locked := testSimpleWriteLock(t, 10*testDrwMutexAcquireTimeout) + + expected := true + if locked != expected { + t.Errorf("TestSimpleWriteLockAcquired(): \nexpected %#v\ngot %#v", expected, locked) + } +} + +func TestSimpleWriteLockTimedOut(t *testing.T) { + locked := testSimpleWriteLock(t, testDrwMutexAcquireTimeout) + + expected := false + if locked != expected { + t.Errorf("TestSimpleWriteLockTimedOut(): \nexpected %#v\ngot %#v", expected, locked) + } +} + +func testDualWriteLock(t *testing.T, duration time.Duration) (locked bool) { + drwm1 := NewDRWMutex(ds, "duallock") + + // fmt.Println("Getting initial write lock") + ctx1, cancel1 := context.WithCancel(t.Context()) + if !drwm1.GetLock(ctx1, cancel1, id, source, Options{Timeout: time.Second}) { + panic("Failed to acquire initial write lock") + } + + go func() { + time.Sleep(3 * testDrwMutexAcquireTimeout) + drwm1.Unlock(t.Context()) + // fmt.Println("Initial write lock released, waiting...") + }() + + // fmt.Println("Trying to acquire 2nd write lock, waiting...") + drwm2 := NewDRWMutex(ds, "duallock") + ctx2, cancel2 := context.WithCancel(t.Context()) + locked = drwm2.GetLock(ctx2, cancel2, id, source, Options{Timeout: duration}) + if locked { + // fmt.Println("2nd write lock acquired, waiting...") + time.Sleep(testDrwMutexAcquireTimeout) + + drwm2.Unlock(t.Context()) + } + // fmt.Println("2nd write lock failed due to timeout") + return locked +} + +func TestDualWriteLockAcquired(t *testing.T) { + locked := testDualWriteLock(t, 10*testDrwMutexAcquireTimeout) + + expected := true + if locked != expected { + t.Errorf("TestDualWriteLockAcquired(): \nexpected %#v\ngot %#v", expected, locked) + } +} + +func TestDualWriteLockTimedOut(t *testing.T) { + locked := testDualWriteLock(t, testDrwMutexAcquireTimeout) + + expected := false + if locked != expected { + t.Errorf("TestDualWriteLockTimedOut(): \nexpected %#v\ngot %#v", expected, locked) + } +} + +// Test cases below are copied 1 to 1 from sync/rwmutex_test.go (adapted to use DRWMutex) + +// Borrowed from rwmutex_test.go +func parallelReader(ctx context.Context, m *DRWMutex, clocked, cunlock, cdone chan bool) { + if m.GetRLock(ctx, nil, id, source, Options{Timeout: time.Second}) { + clocked <- true + <-cunlock + m.RUnlock(context.Background()) + cdone <- true + } +} + +// Borrowed from rwmutex_test.go +func doTestParallelReaders(numReaders, gomaxprocs int) { + runtime.GOMAXPROCS(gomaxprocs) + m := NewDRWMutex(ds, "test-parallel") + + clocked := make(chan bool) + cunlock := make(chan bool) + cdone := make(chan bool) + for range numReaders { + go parallelReader(context.Background(), m, clocked, cunlock, cdone) + } + // Wait for all parallel RLock()s to succeed. + for range numReaders { + <-clocked + } + for range numReaders { + cunlock <- true + } + // Wait for the goroutines to finish. + for range numReaders { + <-cdone + } +} + +// Borrowed from rwmutex_test.go +func TestParallelReaders(t *testing.T) { + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(-1)) + doTestParallelReaders(1, 4) + doTestParallelReaders(3, 4) + doTestParallelReaders(4, 2) +} + +// Borrowed from rwmutex_test.go +func reader(resource string, numIterations int, activity *int32, cdone chan bool) { + rwm := NewDRWMutex(ds, resource) + for range numIterations { + if rwm.GetRLock(context.Background(), nil, id, source, Options{Timeout: time.Second}) { + n := atomic.AddInt32(activity, 1) + if n < 1 || n >= 10000 { + panic(fmt.Sprintf("wlock(%d)\n", n)) + } + for range 100 { + } + atomic.AddInt32(activity, -1) + rwm.RUnlock(context.Background()) + } + } + cdone <- true +} + +// Borrowed from rwmutex_test.go +func writer(resource string, numIterations int, activity *int32, cdone chan bool) { + rwm := NewDRWMutex(ds, resource) + for range numIterations { + if rwm.GetLock(context.Background(), nil, id, source, Options{Timeout: time.Second}) { + n := atomic.AddInt32(activity, 10000) + if n != 10000 { + panic(fmt.Sprintf("wlock(%d)\n", n)) + } + for range 100 { + } + atomic.AddInt32(activity, -10000) + rwm.Unlock(context.Background()) + } + } + cdone <- true +} + +// Borrowed from rwmutex_test.go +func hammerRWMutex(t *testing.T, gomaxprocs, numReaders, numIterations int) { + t.Run(fmt.Sprintf("%d-%d-%d", gomaxprocs, numReaders, numIterations), func(t *testing.T) { + resource := "test" + runtime.GOMAXPROCS(gomaxprocs) + // Number of active readers + 10000 * number of active writers. + var activity int32 + cdone := make(chan bool) + go writer(resource, numIterations, &activity, cdone) + var i int + for i = 0; i < numReaders/2; i++ { + go reader(resource, numIterations, &activity, cdone) + } + go writer(resource, numIterations, &activity, cdone) + for ; i < numReaders; i++ { + go reader(resource, numIterations, &activity, cdone) + } + // Wait for the 2 writers and all readers to finish. + for i := 0; i < 2+numReaders; i++ { + <-cdone + } + }) +} + +// Borrowed from rwmutex_test.go +func TestRWMutex(t *testing.T) { + defer runtime.GOMAXPROCS(runtime.GOMAXPROCS(-1)) + n := 100 + if testing.Short() { + n = 5 + } + hammerRWMutex(t, 1, 1, n) + hammerRWMutex(t, 1, 3, n) + hammerRWMutex(t, 1, 10, n) + hammerRWMutex(t, 4, 1, n) + hammerRWMutex(t, 4, 3, n) + hammerRWMutex(t, 4, 10, n) + hammerRWMutex(t, 10, 1, n) + hammerRWMutex(t, 10, 3, n) + hammerRWMutex(t, 10, 10, n) + hammerRWMutex(t, 10, 5, n) +} + +// Borrowed from rwmutex_test.go +func TestUnlockPanic(t *testing.T) { + defer func() { + if recover() == nil { + t.Fatalf("unlock of unlocked RWMutex did not panic") + } + }() + mu := NewDRWMutex(ds, "test") + mu.Unlock(t.Context()) +} + +// Borrowed from rwmutex_test.go +func TestUnlockPanic2(t *testing.T) { + mu := NewDRWMutex(ds, "test-unlock-panic-2") + defer func() { + if recover() == nil { + t.Fatalf("unlock of unlocked RWMutex did not panic") + } + mu.RUnlock(t.Context()) // Unlock, so -test.count > 1 works + }() + mu.RLock(id, source) + mu.Unlock(t.Context()) +} + +// Borrowed from rwmutex_test.go +func TestRUnlockPanic(t *testing.T) { + defer func() { + if recover() == nil { + t.Fatalf("read unlock of unlocked RWMutex did not panic") + } + }() + mu := NewDRWMutex(ds, "test") + mu.RUnlock(t.Context()) +} + +// Borrowed from rwmutex_test.go +func TestRUnlockPanic2(t *testing.T) { + mu := NewDRWMutex(ds, "test-runlock-panic-2") + defer func() { + if recover() == nil { + t.Fatalf("read unlock of unlocked RWMutex did not panic") + } + mu.Unlock(t.Context()) // Unlock, so -test.count > 1 works + }() + mu.Lock(id, source) + mu.RUnlock(t.Context()) +} + +// Borrowed from rwmutex_test.go +func benchmarkRWMutex(b *testing.B, localWork, writeRatio int) { + b.ResetTimer() + b.ReportAllocs() + + b.RunParallel(func(pb *testing.PB) { + foo := 0 + for pb.Next() { + rwm := NewDRWMutex(ds, "test") + foo++ + if foo%writeRatio == 0 { + rwm.Lock(id, source) + rwm.Unlock(b.Context()) + } else { + rwm.RLock(id, source) + for i := 0; i != localWork; i++ { + foo *= 2 + foo /= 2 + } + rwm.RUnlock(b.Context()) + } + } + _ = foo + }) +} + +// Borrowed from rwmutex_test.go +func BenchmarkRWMutexWrite100(b *testing.B) { + benchmarkRWMutex(b, 0, 100) +} + +// Borrowed from rwmutex_test.go +func BenchmarkRWMutexWrite10(b *testing.B) { + benchmarkRWMutex(b, 0, 10) +} + +// Borrowed from rwmutex_test.go +func BenchmarkRWMutexWorkWrite100(b *testing.B) { + benchmarkRWMutex(b, 100, 100) +} + +// Borrowed from rwmutex_test.go +func BenchmarkRWMutexWorkWrite10(b *testing.B) { + benchmarkRWMutex(b, 100, 10) +} diff --git a/internal/dsync/dsync-client_test.go b/internal/dsync/dsync-client_test.go new file mode 100644 index 00000000..8172505e --- /dev/null +++ b/internal/dsync/dsync-client_test.go @@ -0,0 +1,149 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dsync + +import ( + "bytes" + "context" + "errors" + "net/http" + "net/url" + "time" + + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/rest" +) + +// ReconnectRESTClient is a wrapper type for rest.Client which provides reconnect on first failure. +type ReconnectRESTClient struct { + u *url.URL + rest *rest.Client +} + +// newClient constructs a ReconnectRESTClient object with addr and endpoint initialized. +// It _doesn't_ connect to the remote endpoint. See Call method to see when the +// connect happens. +func newClient(endpoint string) NetLocker { + u, err := url.Parse(endpoint) + if err != nil { + panic(err) + } + + tr := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + MaxIdleConnsPerHost: 1024, + WriteBufferSize: 32 << 10, // 32KiB moving up from 4KiB default + ReadBufferSize: 32 << 10, // 32KiB moving up from 4KiB default + IdleConnTimeout: 15 * time.Second, + ResponseHeaderTimeout: 15 * time.Minute, // Set conservative timeouts for MinIO internode. + TLSHandshakeTimeout: 15 * time.Second, + ExpectContinueTimeout: 15 * time.Second, + // Go net/http automatically unzip if content-type is + // gzip disable this feature, as we are always interested + // in raw stream. + DisableCompression: true, + } + + return &ReconnectRESTClient{ + u: u, + rest: rest.NewClient(u, tr, nil), + } +} + +// Close closes the underlying socket file descriptor. +func (restClient *ReconnectRESTClient) IsOnline() bool { + // If rest client has not connected yet there is nothing to close. + return restClient.rest != nil +} + +func (restClient *ReconnectRESTClient) IsLocal() bool { + return false +} + +// Close closes the underlying socket file descriptor. +func (restClient *ReconnectRESTClient) Close() error { + return nil +} + +var ( + errLockConflict = errors.New("lock conflict") + errLockNotFound = errors.New("lock not found") +) + +func toLockError(err error) error { + if err == nil { + return nil + } + + switch err.Error() { + case errLockConflict.Error(): + return errLockConflict + case errLockNotFound.Error(): + return errLockNotFound + } + return err +} + +// Call makes a REST call to the remote endpoint using the msgp codec +func (restClient *ReconnectRESTClient) Call(method string, args LockArgs) (status bool, err error) { + buf, err := args.MarshalMsg(nil) + if err != nil { + return false, err + } + body := bytes.NewReader(buf) + respBody, err := restClient.rest.Call(context.Background(), method, + url.Values{}, body, body.Size()) + defer xhttp.DrainBody(respBody) + + switch toLockError(err) { + case nil: + return true, nil + case errLockConflict, errLockNotFound: + return false, nil + default: + return false, err + } +} + +func (restClient *ReconnectRESTClient) RLock(ctx context.Context, args LockArgs) (status bool, err error) { + return restClient.Call("/v1/rlock", args) +} + +func (restClient *ReconnectRESTClient) Lock(ctx context.Context, args LockArgs) (status bool, err error) { + return restClient.Call("/v1/lock", args) +} + +func (restClient *ReconnectRESTClient) RUnlock(ctx context.Context, args LockArgs) (status bool, err error) { + return restClient.Call("/v1/runlock", args) +} + +func (restClient *ReconnectRESTClient) Unlock(ctx context.Context, args LockArgs) (status bool, err error) { + return restClient.Call("/v1/unlock", args) +} + +func (restClient *ReconnectRESTClient) Refresh(ctx context.Context, args LockArgs) (refreshed bool, err error) { + return restClient.Call("/v1/refresh", args) +} + +func (restClient *ReconnectRESTClient) ForceUnlock(ctx context.Context, args LockArgs) (reply bool, err error) { + return restClient.Call("/v1/force-unlock", args) +} + +func (restClient *ReconnectRESTClient) String() string { + return restClient.u.String() +} diff --git a/internal/dsync/dsync-server_test.go b/internal/dsync/dsync-server_test.go new file mode 100644 index 00000000..bdf7ac7b --- /dev/null +++ b/internal/dsync/dsync-server_test.go @@ -0,0 +1,304 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dsync + +import ( + "fmt" + "io" + "net/http" + "net/http/httptest" + "sync" + "sync/atomic" + "time" + + "github.com/minio/mux" +) + +const numberOfNodes = 5 + +var ( + ds *Dsync + nodes = make([]*httptest.Server, numberOfNodes) // list of node IP addrs or hostname with ports. + lockServers = make([]*lockServer, numberOfNodes) +) + +func getLockArgs(r *http.Request) (args LockArgs, err error) { + buf, err := io.ReadAll(r.Body) + if err != nil { + return args, err + } + _, err = args.UnmarshalMsg(buf) + return args, err +} + +type lockServerHandler struct { + lsrv *lockServer +} + +func (lh *lockServerHandler) writeErrorResponse(w http.ResponseWriter, err error) { + w.WriteHeader(http.StatusForbidden) + w.Write([]byte(err.Error())) +} + +func (lh *lockServerHandler) ForceUnlockHandler(w http.ResponseWriter, r *http.Request) { + args, err := getLockArgs(r) + if err != nil { + lh.writeErrorResponse(w, err) + return + } + + if _, err = lh.lsrv.ForceUnlock(&args); err != nil { + lh.writeErrorResponse(w, err) + return + } +} + +func (lh *lockServerHandler) RefreshHandler(w http.ResponseWriter, r *http.Request) { + args, err := getLockArgs(r) + if err != nil { + lh.writeErrorResponse(w, err) + return + } + + reply, err := lh.lsrv.Refresh(&args) + if err != nil { + lh.writeErrorResponse(w, err) + return + } + + if !reply { + lh.writeErrorResponse(w, errLockNotFound) + return + } +} + +func (lh *lockServerHandler) LockHandler(w http.ResponseWriter, r *http.Request) { + args, err := getLockArgs(r) + if err != nil { + lh.writeErrorResponse(w, err) + return + } + reply, err := lh.lsrv.Lock(&args) + if err == nil && !reply { + err = errLockConflict + } + if err != nil { + lh.writeErrorResponse(w, err) + return + } +} + +func (lh *lockServerHandler) UnlockHandler(w http.ResponseWriter, r *http.Request) { + args, err := getLockArgs(r) + if err != nil { + lh.writeErrorResponse(w, err) + return + } + _, err = lh.lsrv.Unlock(&args) + if err != nil { + lh.writeErrorResponse(w, err) + return + } +} + +func (lh *lockServerHandler) RUnlockHandler(w http.ResponseWriter, r *http.Request) { + args, err := getLockArgs(r) + if err != nil { + lh.writeErrorResponse(w, err) + return + } + _, err = lh.lsrv.RUnlock(&args) + if err != nil { + lh.writeErrorResponse(w, err) + return + } +} + +func (lh *lockServerHandler) HealthHandler(w http.ResponseWriter, r *http.Request) {} + +func (lh *lockServerHandler) RLockHandler(w http.ResponseWriter, r *http.Request) { + args, err := getLockArgs(r) + if err != nil { + lh.writeErrorResponse(w, err) + return + } + + reply, err := lh.lsrv.RLock(&args) + if err == nil && !reply { + err = errLockConflict + } + if err != nil { + lh.writeErrorResponse(w, err) + return + } +} + +func stopLockServers() { + for i := range numberOfNodes { + nodes[i].Close() + } +} + +func startLockServers() { + for i := range numberOfNodes { + lsrv := &lockServer{ + mutex: sync.Mutex{}, + lockMap: make(map[string]int64), + } + lockServer := lockServerHandler{ + lsrv: lsrv, + } + lockServers[i] = lsrv + + router := mux.NewRouter().SkipClean(true) + subrouter := router.PathPrefix("/").Subrouter() + subrouter.Methods(http.MethodPost).Path("/v1/health").HandlerFunc(lockServer.HealthHandler) + subrouter.Methods(http.MethodPost).Path("/v1/refresh").HandlerFunc(lockServer.RefreshHandler) + subrouter.Methods(http.MethodPost).Path("/v1/lock").HandlerFunc(lockServer.LockHandler) + subrouter.Methods(http.MethodPost).Path("/v1/rlock").HandlerFunc(lockServer.RLockHandler) + subrouter.Methods(http.MethodPost).Path("/v1/unlock").HandlerFunc(lockServer.UnlockHandler) + subrouter.Methods(http.MethodPost).Path("/v1/runlock").HandlerFunc(lockServer.RUnlockHandler) + subrouter.Methods(http.MethodPost).Path("/v1/force-unlock").HandlerFunc(lockServer.ForceUnlockHandler) + + nodes[i] = httptest.NewServer(router) + } +} + +const WriteLock = -1 + +type lockServer struct { + mutex sync.Mutex + // Map of locks, with negative value indicating (exclusive) write lock + // and positive values indicating number of read locks + lockMap map[string]int64 + + // Refresh returns lock not found if set to true + lockNotFound bool + + // Set to true if you want peers servers to do not respond + responseDelay int64 +} + +func (l *lockServer) setRefreshReply(refreshed bool) { + l.mutex.Lock() + defer l.mutex.Unlock() + l.lockNotFound = !refreshed +} + +func (l *lockServer) setResponseDelay(responseDelay time.Duration) { + atomic.StoreInt64(&l.responseDelay, int64(responseDelay)) +} + +func (l *lockServer) Lock(args *LockArgs) (reply bool, err error) { + if d := atomic.LoadInt64(&l.responseDelay); d != 0 { + time.Sleep(time.Duration(d)) + } + + l.mutex.Lock() + defer l.mutex.Unlock() + if _, reply = l.lockMap[args.Resources[0]]; !reply { + l.lockMap[args.Resources[0]] = WriteLock // No locks held on the given name, so claim write lock + } + reply = !reply // Negate *reply to return true when lock is granted or false otherwise + return reply, nil +} + +func (l *lockServer) Unlock(args *LockArgs) (reply bool, err error) { + if d := atomic.LoadInt64(&l.responseDelay); d != 0 { + time.Sleep(time.Duration(d)) + } + + l.mutex.Lock() + defer l.mutex.Unlock() + var locksHeld int64 + if locksHeld, reply = l.lockMap[args.Resources[0]]; !reply { // No lock is held on the given name + return false, fmt.Errorf("Unlock attempted on an unlocked entity: %s", args.Resources[0]) + } + if reply = locksHeld == WriteLock; !reply { // Unless it is a write lock + return false, fmt.Errorf("Unlock attempted on a read locked entity: %s (%d read locks active)", args.Resources[0], locksHeld) + } + delete(l.lockMap, args.Resources[0]) // Remove the write lock + return true, nil +} + +const ReadLock = 1 + +func (l *lockServer) RLock(args *LockArgs) (reply bool, err error) { + if d := atomic.LoadInt64(&l.responseDelay); d != 0 { + time.Sleep(time.Duration(d)) + } + + l.mutex.Lock() + defer l.mutex.Unlock() + var locksHeld int64 + if locksHeld, reply = l.lockMap[args.Resources[0]]; !reply { + l.lockMap[args.Resources[0]] = ReadLock // No locks held on the given name, so claim (first) read lock + reply = true + } else if reply = locksHeld != WriteLock; reply { // Unless there is a write lock + l.lockMap[args.Resources[0]] = locksHeld + ReadLock // Grant another read lock + } + return reply, nil +} + +func (l *lockServer) RUnlock(args *LockArgs) (reply bool, err error) { + if d := atomic.LoadInt64(&l.responseDelay); d != 0 { + time.Sleep(time.Duration(d)) + } + + l.mutex.Lock() + defer l.mutex.Unlock() + var locksHeld int64 + if locksHeld, reply = l.lockMap[args.Resources[0]]; !reply { // No lock is held on the given name + return false, fmt.Errorf("RUnlock attempted on an unlocked entity: %s", args.Resources[0]) + } + if reply = locksHeld != WriteLock; !reply { // A write-lock is held, cannot release a read lock + return false, fmt.Errorf("RUnlock attempted on a write locked entity: %s", args.Resources[0]) + } + if locksHeld > ReadLock { + l.lockMap[args.Resources[0]] = locksHeld - ReadLock // Remove one of the read locks held + } else { + delete(l.lockMap, args.Resources[0]) // Remove the (last) read lock + } + return reply, nil +} + +func (l *lockServer) Refresh(args *LockArgs) (reply bool, err error) { + if d := atomic.LoadInt64(&l.responseDelay); d != 0 { + time.Sleep(time.Duration(d)) + } + + l.mutex.Lock() + defer l.mutex.Unlock() + reply = !l.lockNotFound + return reply, nil +} + +func (l *lockServer) ForceUnlock(args *LockArgs) (reply bool, err error) { + if d := atomic.LoadInt64(&l.responseDelay); d != 0 { + time.Sleep(time.Duration(d)) + } + + l.mutex.Lock() + defer l.mutex.Unlock() + if len(args.UID) != 0 { + return false, fmt.Errorf("ForceUnlock called with non-empty UID: %s", args.UID) + } + delete(l.lockMap, args.Resources[0]) // Remove the lock (irrespective of write or read lock) + reply = true + return reply, nil +} diff --git a/internal/dsync/dsync.go b/internal/dsync/dsync.go new file mode 100644 index 00000000..02ec60aa --- /dev/null +++ b/internal/dsync/dsync.go @@ -0,0 +1,28 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dsync + +// Dsync represents dsync client object which is initialized with +// authenticated clients, used to initiate lock REST calls. +type Dsync struct { + // List of rest client objects, one per lock server. + GetLockers func() ([]NetLocker, string) + + // Timeouts to apply. + Timeouts Timeouts +} diff --git a/internal/dsync/dsync_test.go b/internal/dsync/dsync_test.go new file mode 100644 index 00000000..18b6c0ba --- /dev/null +++ b/internal/dsync/dsync_test.go @@ -0,0 +1,451 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dsync + +import ( + "context" + "math/rand" + "os" + "sync" + "testing" + "time" + + "github.com/google/uuid" +) + +const ( + testDrwMutexAcquireTimeout = 250 * time.Millisecond + testDrwMutexRefreshCallTimeout = 250 * time.Millisecond + testDrwMutexUnlockCallTimeout = 250 * time.Millisecond + testDrwMutexForceUnlockCallTimeout = 250 * time.Millisecond + testDrwMutexRefreshInterval = 100 * time.Millisecond +) + +// TestMain initializes the testing framework +func TestMain(m *testing.M) { + startLockServers() + + // Initialize locker clients for dsync. + var clnts []NetLocker + for i := range nodes { + clnts = append(clnts, newClient(nodes[i].URL)) + } + + ds = &Dsync{ + GetLockers: func() ([]NetLocker, string) { return clnts, uuid.New().String() }, + Timeouts: Timeouts{ + Acquire: testDrwMutexAcquireTimeout, + RefreshCall: testDrwMutexRefreshCallTimeout, + UnlockCall: testDrwMutexUnlockCallTimeout, + ForceUnlockCall: testDrwMutexForceUnlockCallTimeout, + }, + } + + code := m.Run() + stopLockServers() + os.Exit(code) +} + +func TestSimpleLock(t *testing.T) { + dm := NewDRWMutex(ds, "test") + + dm.Lock(id, source) + + // fmt.Println("Lock acquired, waiting...") + time.Sleep(testDrwMutexRefreshCallTimeout) + + dm.Unlock(t.Context()) +} + +func TestSimpleLockUnlockMultipleTimes(t *testing.T) { + dm := NewDRWMutex(ds, "test") + + dm.Lock(id, source) + time.Sleep(time.Duration(10+(rand.Float32()*50)) * time.Millisecond) + dm.Unlock(t.Context()) + + dm.Lock(id, source) + time.Sleep(time.Duration(10+(rand.Float32()*50)) * time.Millisecond) + dm.Unlock(t.Context()) + + dm.Lock(id, source) + time.Sleep(time.Duration(10+(rand.Float32()*50)) * time.Millisecond) + dm.Unlock(t.Context()) + + dm.Lock(id, source) + time.Sleep(time.Duration(10+(rand.Float32()*50)) * time.Millisecond) + dm.Unlock(t.Context()) + + dm.Lock(id, source) + time.Sleep(time.Duration(10+(rand.Float32()*50)) * time.Millisecond) + dm.Unlock(t.Context()) +} + +// Test two locks for same resource, one succeeds, one fails (after timeout) +func TestTwoSimultaneousLocksForSameResource(t *testing.T) { + dm1st := NewDRWMutex(ds, "aap") + dm2nd := NewDRWMutex(ds, "aap") + + dm1st.Lock(id, source) + + // Release lock after 10 seconds + go func() { + time.Sleep(5 * testDrwMutexAcquireTimeout) + // fmt.Println("Unlocking dm1") + + dm1st.Unlock(t.Context()) + }() + + dm2nd.Lock(id, source) + + // fmt.Printf("2nd lock obtained after 1st lock is released\n") + time.Sleep(testDrwMutexRefreshCallTimeout * 2) + + dm2nd.Unlock(t.Context()) +} + +// Test three locks for same resource, one succeeds, one fails (after timeout) +func TestThreeSimultaneousLocksForSameResource(t *testing.T) { + dm1st := NewDRWMutex(ds, "aap") + dm2nd := NewDRWMutex(ds, "aap") + dm3rd := NewDRWMutex(ds, "aap") + + dm1st.Lock(id, source) + started := time.Now() + var expect time.Duration + // Release lock after 10 seconds + go func() { + // TOTAL + time.Sleep(2 * testDrwMutexAcquireTimeout) + // fmt.Println("Unlocking dm1") + + dm1st.Unlock(t.Context()) + }() + expect += 2 * testDrwMutexAcquireTimeout + + var wg sync.WaitGroup + wg.Add(2) + + go func() { + defer wg.Done() + + dm2nd.Lock(id, source) + + // Release lock after 10 seconds + go func() { + time.Sleep(2 * testDrwMutexAcquireTimeout) + // fmt.Println("Unlocking dm2") + + dm2nd.Unlock(t.Context()) + }() + + dm3rd.Lock(id, source) + + // fmt.Printf("3rd lock obtained after 1st & 2nd locks are released\n") + time.Sleep(testDrwMutexRefreshCallTimeout) + + dm3rd.Unlock(t.Context()) + }() + expect += 2*testDrwMutexAcquireTimeout + testDrwMutexRefreshCallTimeout + + go func() { + defer wg.Done() + + dm3rd.Lock(id, source) + + // Release lock after 10 seconds + go func() { + time.Sleep(2 * testDrwMutexAcquireTimeout) + // fmt.Println("Unlocking dm3") + + dm3rd.Unlock(t.Context()) + }() + + dm2nd.Lock(id, source) + + // fmt.Printf("2nd lock obtained after 1st & 3rd locks are released\n") + time.Sleep(testDrwMutexRefreshCallTimeout) + + dm2nd.Unlock(t.Context()) + }() + expect += 2*testDrwMutexAcquireTimeout + testDrwMutexRefreshCallTimeout + + wg.Wait() + // We expect at least 3 x 2 x testDrwMutexAcquireTimeout to have passed + elapsed := time.Since(started) + if elapsed < expect { + t.Errorf("expected at least %v time have passed, however %v passed", expect, elapsed) + } + t.Logf("expected at least %v time have passed, %v passed", expect, elapsed) +} + +// Test two locks for different resources, both succeed +func TestTwoSimultaneousLocksForDifferentResources(t *testing.T) { + dm1 := NewDRWMutex(ds, "aap") + dm2 := NewDRWMutex(ds, "noot") + + dm1.Lock(id, source) + dm2.Lock(id, source) + dm1.Unlock(t.Context()) + dm2.Unlock(t.Context()) +} + +// Test refreshing lock - refresh should always return true +func TestSuccessfulLockRefresh(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + dm := NewDRWMutex(ds, "aap") + dm.refreshInterval = testDrwMutexRefreshInterval + + ctx, cancel := context.WithCancel(t.Context()) + + if !dm.GetLock(ctx, cancel, id, source, Options{Timeout: 5 * time.Minute}) { + t.Fatal("GetLock() should be successful") + } + + // Make it run twice. + timer := time.NewTimer(testDrwMutexRefreshInterval * 2) + + select { + case <-ctx.Done(): + t.Fatal("Lock context canceled which is not expected") + case <-timer.C: + } + + // Should be safe operation in all cases + dm.Unlock(t.Context()) +} + +// Test canceling context while quorum servers report lock not found +func TestFailedRefreshLock(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + // Simulate Refresh response to return no locking found + for i := range lockServers[:3] { + lockServers[i].setRefreshReply(false) + defer lockServers[i].setRefreshReply(true) + } + + dm := NewDRWMutex(ds, "aap") + dm.refreshInterval = 500 * time.Millisecond + var wg sync.WaitGroup + wg.Add(1) + + ctx, cl := context.WithCancel(t.Context()) + cancel := func() { + cl() + wg.Done() + } + + if !dm.GetLock(ctx, cancel, id, source, Options{Timeout: 5 * time.Minute}) { + t.Fatal("GetLock() should be successful") + } + + // Wait until context is canceled + wg.Wait() + if ctx.Err() == nil { + t.Fatal("Unexpected error", ctx.Err()) + } + + // Should be safe operation in all cases + dm.Unlock(t.Context()) +} + +// Test Unlock should not timeout +func TestUnlockShouldNotTimeout(t *testing.T) { + if testing.Short() { + t.Skip("skipping test in short mode.") + } + + dm := NewDRWMutex(ds, "aap") + dm.refreshInterval = testDrwMutexUnlockCallTimeout + if !dm.GetLock(t.Context(), nil, id, source, Options{Timeout: 5 * time.Minute}) { + t.Fatal("GetLock() should be successful") + } + + // Add delay to lock server responses to ensure that lock does not timeout + for i := range lockServers { + lockServers[i].setResponseDelay(5 * testDrwMutexUnlockCallTimeout) + defer lockServers[i].setResponseDelay(0) + } + + unlockReturned := make(chan struct{}, 1) + go func() { + ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond) + defer cancel() + dm.Unlock(ctx) + // Unlock is not blocking. Try to get a new lock. + dm.GetLock(ctx, nil, id, source, Options{Timeout: 5 * time.Minute}) + unlockReturned <- struct{}{} + }() + + timer := time.NewTimer(2 * testDrwMutexUnlockCallTimeout) + defer timer.Stop() + + select { + case <-unlockReturned: + t.Fatal("Unlock timed out, which should not happen") + case <-timer.C: + } +} + +// Borrowed from mutex_test.go +func HammerMutex(m *DRWMutex, loops int, cdone chan bool) { + for range loops { + m.Lock(id, source) + m.Unlock(context.Background()) + } + cdone <- true +} + +// Borrowed from mutex_test.go +func TestMutex(t *testing.T) { + loops := 200 + if testing.Short() { + loops = 5 + } + c := make(chan bool) + m := NewDRWMutex(ds, "test") + for range 10 { + go HammerMutex(m, loops, c) + } + for range 10 { + <-c + } +} + +func BenchmarkMutexUncontended(b *testing.B) { + b.ResetTimer() + b.ReportAllocs() + + type PaddedMutex struct { + *DRWMutex + } + b.RunParallel(func(pb *testing.PB) { + mu := PaddedMutex{NewDRWMutex(ds, "")} + for pb.Next() { + mu.Lock(id, source) + mu.Unlock(b.Context()) + } + }) +} + +func benchmarkMutex(b *testing.B, slack, work bool) { + b.ResetTimer() + b.ReportAllocs() + + mu := NewDRWMutex(ds, "") + if slack { + b.SetParallelism(10) + } + b.RunParallel(func(pb *testing.PB) { + foo := 0 + for pb.Next() { + mu.Lock(id, source) + mu.Unlock(b.Context()) + if work { + for range 100 { + foo *= 2 + foo /= 2 + } + } + } + _ = foo + }) +} + +func BenchmarkMutex(b *testing.B) { + benchmarkMutex(b, false, false) +} + +func BenchmarkMutexSlack(b *testing.B) { + benchmarkMutex(b, true, false) +} + +func BenchmarkMutexWork(b *testing.B) { + benchmarkMutex(b, false, true) +} + +func BenchmarkMutexWorkSlack(b *testing.B) { + benchmarkMutex(b, true, true) +} + +func BenchmarkMutexNoSpin(b *testing.B) { + b.ResetTimer() + b.ReportAllocs() + + // This benchmark models a situation where spinning in the mutex should be + // non-profitable and allows to confirm that spinning does not do harm. + // To achieve this we create excess of goroutines most of which do local work. + // These goroutines yield during local work, so that switching from + // a blocked goroutine to other goroutines is profitable. + // As a matter of fact, this benchmark still triggers some spinning in the mutex. + m := NewDRWMutex(ds, "") + var acc0, acc1 uint64 + b.SetParallelism(4) + b.RunParallel(func(pb *testing.PB) { + c := make(chan bool) + var data [4 << 10]uint64 + for i := 0; pb.Next(); i++ { + if i%4 == 0 { + m.Lock(id, source) + acc0 -= 100 + acc1 += 100 + m.Unlock(b.Context()) + } else { + for i := 0; i < len(data); i += 4 { + data[i]++ + } + // Elaborate way to say runtime.Gosched + // that does not put the goroutine onto global runq. + go func() { + c <- true + }() + <-c + } + } + }) +} + +func BenchmarkMutexSpin(b *testing.B) { + b.ResetTimer() + b.ReportAllocs() + + // This benchmark models a situation where spinning in the mutex should be + // profitable. To achieve this we create a goroutine per-proc. + // These goroutines access considerable amount of local data so that + // unnecessary rescheduling is penalized by cache misses. + m := NewDRWMutex(ds, "") + var acc0, acc1 uint64 + b.RunParallel(func(pb *testing.PB) { + var data [16 << 10]uint64 + for i := 0; pb.Next(); i++ { + m.Lock(id, source) + acc0 -= 100 + acc1 += 100 + m.Unlock(b.Context()) + for i := 0; i < len(data); i += 4 { + data[i]++ + } + } + }) +} diff --git a/internal/dsync/lock-args.go b/internal/dsync/lock-args.go new file mode 100644 index 00000000..6fba08f9 --- /dev/null +++ b/internal/dsync/lock-args.go @@ -0,0 +1,58 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dsync + +//go:generate msgp -file $GOFILE + +// LockArgs is minimal required values for any dsync compatible lock operation. +type LockArgs struct { + // Unique ID of lock/unlock request. + UID string + + // Resources contains single or multiple entries to be locked/unlocked. + Resources []string + + // Owner represents unique ID for this instance, an owner who originally requested + // the locked resource, useful primarily in figuring out stale locks. + Owner string + + // Source contains the line number, function and file name of the code + // on the client node that requested the lock. + Source string `msgp:"omitempty"` + + // Quorum represents the expected quorum for this lock type. + Quorum *int `msgp:"omitempty"` +} + +// ResponseCode is the response code for a locking request. +type ResponseCode uint8 + +// Response codes for a locking request. +const ( + RespOK ResponseCode = iota + RespLockConflict + RespLockNotInitialized + RespLockNotFound + RespErr +) + +// LockResp is a locking request response. +type LockResp struct { + Code ResponseCode + Err string +} diff --git a/internal/dsync/lock-args_gen.go b/internal/dsync/lock-args_gen.go new file mode 100644 index 00000000..98a5fe3e --- /dev/null +++ b/internal/dsync/lock-args_gen.go @@ -0,0 +1,477 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package dsync + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *LockArgs) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "UID": + z.UID, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "UID") + return + } + case "Resources": + var zb0002 uint32 + zb0002, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err, "Resources") + return + } + if cap(z.Resources) >= int(zb0002) { + z.Resources = (z.Resources)[:zb0002] + } else { + z.Resources = make([]string, zb0002) + } + for za0001 := range z.Resources { + z.Resources[za0001], err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Resources", za0001) + return + } + } + case "Owner": + z.Owner, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Owner") + return + } + case "Source": + z.Source, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Source") + return + } + case "Quorum": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "Quorum") + return + } + z.Quorum = nil + } else { + if z.Quorum == nil { + z.Quorum = new(int) + } + *z.Quorum, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Quorum") + return + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *LockArgs) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 5 + // write "UID" + err = en.Append(0x85, 0xa3, 0x55, 0x49, 0x44) + if err != nil { + return + } + err = en.WriteString(z.UID) + if err != nil { + err = msgp.WrapError(err, "UID") + return + } + // write "Resources" + err = en.Append(0xa9, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73) + if err != nil { + return + } + err = en.WriteArrayHeader(uint32(len(z.Resources))) + if err != nil { + err = msgp.WrapError(err, "Resources") + return + } + for za0001 := range z.Resources { + err = en.WriteString(z.Resources[za0001]) + if err != nil { + err = msgp.WrapError(err, "Resources", za0001) + return + } + } + // write "Owner" + err = en.Append(0xa5, 0x4f, 0x77, 0x6e, 0x65, 0x72) + if err != nil { + return + } + err = en.WriteString(z.Owner) + if err != nil { + err = msgp.WrapError(err, "Owner") + return + } + // write "Source" + err = en.Append(0xa6, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65) + if err != nil { + return + } + err = en.WriteString(z.Source) + if err != nil { + err = msgp.WrapError(err, "Source") + return + } + // write "Quorum" + err = en.Append(0xa6, 0x51, 0x75, 0x6f, 0x72, 0x75, 0x6d) + if err != nil { + return + } + if z.Quorum == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = en.WriteInt(*z.Quorum) + if err != nil { + err = msgp.WrapError(err, "Quorum") + return + } + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *LockArgs) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 5 + // string "UID" + o = append(o, 0x85, 0xa3, 0x55, 0x49, 0x44) + o = msgp.AppendString(o, z.UID) + // string "Resources" + o = append(o, 0xa9, 0x52, 0x65, 0x73, 0x6f, 0x75, 0x72, 0x63, 0x65, 0x73) + o = msgp.AppendArrayHeader(o, uint32(len(z.Resources))) + for za0001 := range z.Resources { + o = msgp.AppendString(o, z.Resources[za0001]) + } + // string "Owner" + o = append(o, 0xa5, 0x4f, 0x77, 0x6e, 0x65, 0x72) + o = msgp.AppendString(o, z.Owner) + // string "Source" + o = append(o, 0xa6, 0x53, 0x6f, 0x75, 0x72, 0x63, 0x65) + o = msgp.AppendString(o, z.Source) + // string "Quorum" + o = append(o, 0xa6, 0x51, 0x75, 0x6f, 0x72, 0x75, 0x6d) + if z.Quorum == nil { + o = msgp.AppendNil(o) + } else { + o = msgp.AppendInt(o, *z.Quorum) + } + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *LockArgs) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "UID": + z.UID, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "UID") + return + } + case "Resources": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Resources") + return + } + if cap(z.Resources) >= int(zb0002) { + z.Resources = (z.Resources)[:zb0002] + } else { + z.Resources = make([]string, zb0002) + } + for za0001 := range z.Resources { + z.Resources[za0001], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Resources", za0001) + return + } + } + case "Owner": + z.Owner, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Owner") + return + } + case "Source": + z.Source, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Source") + return + } + case "Quorum": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.Quorum = nil + } else { + if z.Quorum == nil { + z.Quorum = new(int) + } + *z.Quorum, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Quorum") + return + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *LockArgs) Msgsize() (s int) { + s = 1 + 4 + msgp.StringPrefixSize + len(z.UID) + 10 + msgp.ArrayHeaderSize + for za0001 := range z.Resources { + s += msgp.StringPrefixSize + len(z.Resources[za0001]) + } + s += 6 + msgp.StringPrefixSize + len(z.Owner) + 7 + msgp.StringPrefixSize + len(z.Source) + 7 + if z.Quorum == nil { + s += msgp.NilSize + } else { + s += msgp.IntSize + } + return +} + +// DecodeMsg implements msgp.Decodable +func (z *LockResp) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Code": + { + var zb0002 uint8 + zb0002, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err, "Code") + return + } + z.Code = ResponseCode(zb0002) + } + case "Err": + z.Err, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Err") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z LockResp) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "Code" + err = en.Append(0x82, 0xa4, 0x43, 0x6f, 0x64, 0x65) + if err != nil { + return + } + err = en.WriteUint8(uint8(z.Code)) + if err != nil { + err = msgp.WrapError(err, "Code") + return + } + // write "Err" + err = en.Append(0xa3, 0x45, 0x72, 0x72) + if err != nil { + return + } + err = en.WriteString(z.Err) + if err != nil { + err = msgp.WrapError(err, "Err") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z LockResp) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "Code" + o = append(o, 0x82, 0xa4, 0x43, 0x6f, 0x64, 0x65) + o = msgp.AppendUint8(o, uint8(z.Code)) + // string "Err" + o = append(o, 0xa3, 0x45, 0x72, 0x72) + o = msgp.AppendString(o, z.Err) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *LockResp) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Code": + { + var zb0002 uint8 + zb0002, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Code") + return + } + z.Code = ResponseCode(zb0002) + } + case "Err": + z.Err, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Err") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z LockResp) Msgsize() (s int) { + s = 1 + 5 + msgp.Uint8Size + 4 + msgp.StringPrefixSize + len(z.Err) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *ResponseCode) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 uint8 + zb0001, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = ResponseCode(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z ResponseCode) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteUint8(uint8(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z ResponseCode) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendUint8(o, uint8(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *ResponseCode) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 uint8 + zb0001, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = ResponseCode(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z ResponseCode) Msgsize() (s int) { + s = msgp.Uint8Size + return +} diff --git a/internal/dsync/lock-args_gen_test.go b/internal/dsync/lock-args_gen_test.go new file mode 100644 index 00000000..c5de2cc0 --- /dev/null +++ b/internal/dsync/lock-args_gen_test.go @@ -0,0 +1,236 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package dsync + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalLockArgs(t *testing.T) { + v := LockArgs{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgLockArgs(b *testing.B) { + v := LockArgs{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgLockArgs(b *testing.B) { + v := LockArgs{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalLockArgs(b *testing.B) { + v := LockArgs{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeLockArgs(t *testing.T) { + v := LockArgs{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeLockArgs Msgsize() is inaccurate") + } + + vn := LockArgs{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeLockArgs(b *testing.B) { + v := LockArgs{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeLockArgs(b *testing.B) { + v := LockArgs{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalLockResp(t *testing.T) { + v := LockResp{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgLockResp(b *testing.B) { + v := LockResp{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgLockResp(b *testing.B) { + v := LockResp{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalLockResp(b *testing.B) { + v := LockResp{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeLockResp(t *testing.T) { + v := LockResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeLockResp Msgsize() is inaccurate") + } + + vn := LockResp{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeLockResp(b *testing.B) { + v := LockResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeLockResp(b *testing.B) { + v := LockResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/internal/dsync/locked_rand.go b/internal/dsync/locked_rand.go new file mode 100644 index 00000000..680b7384 --- /dev/null +++ b/internal/dsync/locked_rand.go @@ -0,0 +1,45 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dsync + +import ( + "math/rand" + "sync" +) + +// lockedRandSource provides protected rand source, implements rand.Source interface. +type lockedRandSource struct { + lk sync.Mutex + src rand.Source +} + +// Int63 returns a non-negative pseudo-random 63-bit integer as an int64. +func (r *lockedRandSource) Int63() (n int64) { + r.lk.Lock() + n = r.src.Int63() + r.lk.Unlock() + return n +} + +// Seed uses the provided seed value to initialize the generator to a +// deterministic state. +func (r *lockedRandSource) Seed(seed int64) { + r.lk.Lock() + r.src.Seed(seed) + r.lk.Unlock() +} diff --git a/internal/dsync/locker.go b/internal/dsync/locker.go new file mode 100644 index 00000000..5b457868 --- /dev/null +++ b/internal/dsync/locker.go @@ -0,0 +1,67 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dsync + +import "context" + +// NetLocker is dsync compatible locker interface. +type NetLocker interface { + // Do read lock for given LockArgs. It should return + // * a boolean to indicate success/failure of the operation + // * an error on failure of lock request operation. + RLock(ctx context.Context, args LockArgs) (bool, error) + + // Do write lock for given LockArgs. It should return + // * a boolean to indicate success/failure of the operation + // * an error on failure of lock request operation. + Lock(ctx context.Context, args LockArgs) (bool, error) + + // Do read unlock for given LockArgs. It should return + // * a boolean to indicate success/failure of the operation + // * an error on failure of unlock request operation. + // Canceling the context will abort the remote call. + // In that case, the resource may or may not be unlocked. + RUnlock(ctx context.Context, args LockArgs) (bool, error) + + // Do write unlock for given LockArgs. It should return + // * a boolean to indicate success/failure of the operation + // * an error on failure of unlock request operation. + // Canceling the context will abort the remote call. + // In that case, the resource may or may not be unlocked. + Unlock(ctx context.Context, args LockArgs) (bool, error) + + // Refresh the given lock to prevent it from becoming stale + Refresh(ctx context.Context, args LockArgs) (bool, error) + + // Unlock (read/write) forcefully for given LockArgs. It should return + // * a boolean to indicate success/failure of the operation + // * an error on failure of unlock request operation. + ForceUnlock(ctx context.Context, args LockArgs) (bool, error) + + // Returns underlying endpoint of this lock client instance. + String() string + + // Close closes any underlying connection to the service endpoint + Close() error + + // Is the underlying connection online? (is always true for any local lockers) + IsOnline() bool + + // Is the underlying locker local to this server? + IsLocal() bool +} diff --git a/internal/dsync/utils.go b/internal/dsync/utils.go new file mode 100644 index 00000000..6a6d2914 --- /dev/null +++ b/internal/dsync/utils.go @@ -0,0 +1,39 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package dsync + +import ( + "math/rand" + "time" +) + +func backoffWait(minSleep, unit, maxSleep time.Duration) func(*rand.Rand, uint) time.Duration { + if unit > time.Hour { + // Protect against integer overflow + panic("unit cannot exceed one hour") + } + return func(r *rand.Rand, attempt uint) time.Duration { + sleep := minSleep + sleep += unit * time.Duration(attempt) + if sleep > maxSleep { + sleep = maxSleep + } + sleep -= time.Duration(r.Float64() * float64(sleep)) + return sleep + } +} diff --git a/internal/etag/etag.go b/internal/etag/etag.go new file mode 100644 index 00000000..78d0e5d4 --- /dev/null +++ b/internal/etag/etag.go @@ -0,0 +1,424 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Package etag provides an implementation of S3 ETags. +// +// Each S3 object has an associated ETag that can be +// used to e.g. quickly compare objects or check whether +// the content of an object has changed. +// +// In general, an S3 ETag is an MD5 checksum of the object +// content. However, there are many exceptions to this rule. +// +// # Single-part Upload +// +// In case of a basic single-part PUT operation - without server +// side encryption or object compression - the ETag of an object +// is its content MD5. +// +// # Multi-part Upload +// +// The ETag of an object does not correspond to its content MD5 +// when the object is uploaded in multiple parts via the S3 +// multipart API. Instead, S3 first computes a MD5 of each part: +// +// e1 := MD5(part-1) +// e2 := MD5(part-2) +// ... +// eN := MD5(part-N) +// +// Then, the ETag of the object is computed as MD5 of all individual +// part checksums. S3 also encodes the number of parts into the ETag +// by appending a - at the end: +// +// ETag := MD5(e1 || e2 || e3 ... || eN) || -N +// +// For example: ceb8853ddc5086cc4ab9e149f8f09c88-5 +// +// However, this scheme is only used for multipart objects that are +// not encrypted. +// +// # Server-side Encryption +// +// S3 specifies three types of server-side-encryption - SSE-C, SSE-S3 +// and SSE-KMS - with different semantics w.r.t. ETags. +// In case of SSE-S3, the ETag of an object is computed the same as +// for single resp. multipart plaintext objects. In particular, +// the ETag of a singlepart SSE-S3 object is its content MD5. +// +// In case of SSE-C and SSE-KMS, the ETag of an object is computed +// differently. For singlepart uploads the ETag is not the content +// MD5 of the object. For multipart uploads the ETag is also not +// the MD5 of the individual part checksums but it still contains +// the number of parts as suffix. +// +// Instead, the ETag is kind of unpredictable for S3 clients when +// an object is encrypted using SSE-C or SSE-KMS. Maybe AWS S3 +// computes the ETag as MD5 of the encrypted content but there is +// no way to verify this assumption since the encryption happens +// inside AWS S3. +// Therefore, S3 clients must not make any assumption about ETags +// in case of SSE-C or SSE-KMS except that the ETag is well-formed. +// +// To put all of this into a simple rule: +// +// SSE-S3 : ETag == MD5 +// SSE-C : ETag != MD5 +// SSE-KMS: ETag != MD5 +// +// # Encrypted ETags +// +// An S3 implementation has to remember the content MD5 of objects +// in case of SSE-S3. However, storing the ETag of an encrypted +// object in plaintext may reveal some information about the object. +// For example, two objects with the same ETag are identical with +// a very high probability. +// +// Therefore, an S3 implementation may encrypt an ETag before storing +// it. In this case, the stored ETag may not be a well-formed S3 ETag. +// For example, it can be larger due to a checksum added by authenticated +// encryption schemes. Such an ETag must be decrypted before sent to an +// S3 client. +// +// # S3 Clients +// +// There are many different S3 client implementations. Most of them +// access the ETag by looking for the HTTP response header key "Etag". +// However, some of them assume that the header key has to be "ETag" +// (case-sensitive) and will fail otherwise. +// Further, some clients require that the ETag value is a double-quoted +// string. Therefore, this package provides dedicated functions for +// adding and extracting the ETag to/from HTTP headers. +package etag + +import ( + "bytes" + "crypto/hmac" + "crypto/md5" + "encoding/base64" + "encoding/hex" + "errors" + "fmt" + "net/http" + "strconv" + "strings" + + "github.com/minio/minio/internal/hash/sha256" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/sio" +) + +// ETag is a single S3 ETag. +// +// An S3 ETag sometimes corresponds to the MD5 of +// the S3 object content. However, when an object +// is encrypted, compressed or uploaded using +// the S3 multipart API then its ETag is not +// necessarily the MD5 of the object content. +// +// For a more detailed description of S3 ETags +// take a look at the package documentation. +type ETag []byte + +// String returns the string representation of the ETag. +// +// The returned string is a hex representation of the +// binary ETag with an optional '-' suffix. +func (e ETag) String() string { + if e.IsMultipart() { + return hex.EncodeToString(e[:16]) + string(e[16:]) + } + return hex.EncodeToString(e) +} + +// IsEncrypted reports whether the ETag is encrypted. +func (e ETag) IsEncrypted() bool { + // An encrypted ETag must be at least 32 bytes long. + // It contains the encrypted ETag value + an authentication + // code generated by the AEAD cipher. + // + // Here is an incorrect implementation of IsEncrypted: + // + // return len(e) > 16 && !bytes.ContainsRune(e, '-') + // + // An encrypted ETag may contain some random bytes - e.g. + // and nonce value. This nonce value may contain a '-' + // just by its nature of being randomly generated. + // The above implementation would incorrectly consider + // such an ETag (with a nonce value containing a '-') + // as non-encrypted. + + return len(e) >= 32 // We consider all ETags longer than 32 bytes as encrypted +} + +// IsMultipart reports whether the ETag belongs to an +// object that has been uploaded using the S3 multipart +// API. +// An S3 multipart ETag has a - suffix. +func (e ETag) IsMultipart() bool { + return len(e) > 16 && !e.IsEncrypted() && bytes.ContainsRune(e, '-') +} + +// Parts returns the number of object parts that are +// referenced by this ETag. It returns 1 if the object +// has been uploaded using the S3 singlepart API. +// +// Parts may panic if the ETag is an invalid multipart +// ETag. +func (e ETag) Parts() int { + if !e.IsMultipart() { + return 1 + } + + n := bytes.IndexRune(e, '-') + parts, err := strconv.Atoi(string(e[n+1:])) + if err != nil { + panic(err) // malformed ETag + } + return parts +} + +// Format returns an ETag that is formatted as specified +// by AWS S3. +// +// An AWS S3 ETag is 16 bytes long and, in case of a multipart +// upload, has a `-N` suffix encoding the number of object parts. +// An ETag is not AWS S3 compatible when encrypted. When sending +// an ETag back to an S3 client it has to be formatted to be +// AWS S3 compatible. +// +// Therefore, Format returns the last 16 bytes of an encrypted +// ETag. +// +// In general, a caller has to distinguish the following cases: +// - The object is a multipart object. In this case, +// Format returns the ETag unmodified. +// - The object is a SSE-KMS or SSE-C encrypted single- +// part object. In this case, Format returns the last +// 16 bytes of the encrypted ETag which will be a random +// value. +// - The object is a SSE-S3 encrypted single-part object. +// In this case, the caller has to decrypt the ETag first +// before calling Format. +// S3 clients expect that the ETag of an SSE-S3 encrypted +// single-part object is equal to the object's content MD5. +// Formatting the SSE-S3 ETag before decryption will result +// in a random-looking ETag which an S3 client will not accept. +// +// Hence, a caller has to check: +// +// if method == SSE-S3 { +// ETag, err := Decrypt(key, ETag) +// if err != nil { +// } +// } +// ETag = ETag.Format() +func (e ETag) Format() ETag { + if !e.IsEncrypted() { + return e + } + return e[len(e)-16:] +} + +var _ Tagger = ETag{} // compiler check + +// ETag returns the ETag itself. +// +// By providing this method ETag implements +// the Tagger interface. +func (e ETag) ETag() ETag { return e } + +// FromContentMD5 decodes and returns the Content-MD5 +// as ETag, if set. If no Content-MD5 header is set +// it returns an empty ETag and no error. +func FromContentMD5(h http.Header) (ETag, error) { + v, ok := h["Content-Md5"] + if !ok { + return nil, nil + } + if v[0] == "" { + return nil, errors.New("etag: content-md5 is set but contains no value") + } + b, err := base64.StdEncoding.Strict().DecodeString(v[0]) + if err != nil { + return nil, err + } + if len(b) != md5.Size { + return nil, errors.New("etag: invalid content-md5") + } + return ETag(b), nil +} + +// ContentMD5Requested - for http.request.header is not request Content-Md5 +func ContentMD5Requested(h http.Header) bool { + _, ok := h[xhttp.ContentMD5] + return ok +} + +// Multipart computes an S3 multipart ETag given a list of +// S3 singlepart ETags. It returns nil if the list of +// ETags is empty. +// +// Any encrypted or multipart ETag will be ignored and not +// used to compute the returned ETag. +func Multipart(etags ...ETag) ETag { + if len(etags) == 0 { + return nil + } + + var n int64 + h := md5.New() + for _, etag := range etags { + if !etag.IsMultipart() && !etag.IsEncrypted() { + h.Write(etag) + n++ + } + } + etag := append(h.Sum(nil), '-') + return strconv.AppendInt(etag, n, 10) +} + +// Set adds the ETag to the HTTP headers. It overwrites any +// existing ETag entry. +// +// Due to legacy S3 clients, that make incorrect assumptions +// about HTTP headers, Set should be used instead of +// http.Header.Set(...). Otherwise, some S3 clients will not +// able to extract the ETag. +func Set(etag ETag, h http.Header) { + // Some (broken) S3 clients expect the ETag header to + // literally "ETag" - not "Etag". Further, some clients + // expect an ETag in double quotes. Therefore, we set the + // ETag directly as map entry instead of using http.Header.Set + h["ETag"] = []string{`"` + etag.String() + `"`} +} + +// Get extracts and parses an ETag from the given HTTP headers. +// It returns an error when the HTTP headers do not contain +// an ETag entry or when the ETag is malformed. +// +// Get only accepts AWS S3 compatible ETags - i.e. no +// encrypted ETags - and therefore is stricter than Parse. +func Get(h http.Header) (ETag, error) { + const strict = true + if v := h.Get("Etag"); v != "" { + return parse(v, strict) + } + v, ok := h["ETag"] + if !ok || len(v) == 0 { + return nil, errors.New("etag: HTTP header does not contain an ETag") + } + return parse(v[0], strict) +} + +// Equal returns true if and only if the two ETags are +// identical. +func Equal(a, b ETag) bool { return bytes.Equal(a, b) } + +// Decrypt decrypts the ETag with the given key. +// +// If the ETag is not encrypted, Decrypt returns +// the ETag unmodified. +func Decrypt(key []byte, etag ETag) (ETag, error) { + const HMACContext = "SSE-etag" + + if !etag.IsEncrypted() { + return etag, nil + } + mac := hmac.New(sha256.New, key) + mac.Write([]byte(HMACContext)) + decryptionKey := mac.Sum(nil) + + plaintext := make([]byte, 0, 16) + etag, err := sio.DecryptBuffer(plaintext, etag, sio.Config{ + Key: decryptionKey, + }) + if err != nil { + return nil, err + } + return etag, nil +} + +// Parse parses s as an S3 ETag, returning the result. +// The string can be an encrypted, singlepart +// or multipart S3 ETag. It returns an error if s is +// not a valid textual representation of an ETag. +func Parse(s string) (ETag, error) { + const strict = false + return parse(s, strict) +} + +// parse parse s as an S3 ETag, returning the result. +// It operates in one of two modes: +// - strict +// - non-strict +// +// In strict mode, parse only accepts ETags that +// are AWS S3 compatible. In particular, an AWS +// S3 ETag always consists of a 128 bit checksum +// value and an optional - suffix. +// Therefore, s must have the following form in +// strict mode: <32-hex-characters>[-] +// +// In non-strict mode, parse also accepts ETags +// that are not AWS S3 compatible - e.g. encrypted +// ETags. +func parse(s string, strict bool) (ETag, error) { + // An S3 ETag may be a double-quoted string. + // Therefore, we remove double quotes at the + // start and end, if any. + if strings.HasPrefix(s, `"`) && strings.HasSuffix(s, `"`) { + s = s[1 : len(s)-1] + } + + // An S3 ETag may be a multipart ETag that + // contains a '-' followed by a number. + // If the ETag does not a '-' is either + // a singlepart or encrypted ETag. + n := strings.IndexRune(s, '-') + if n == -1 { + etag, err := hex.DecodeString(s) + if err != nil { + return nil, err + } + if strict && len(etag) != 16 { // AWS S3 ETags are always 128 bit long + return nil, fmt.Errorf("etag: invalid length %d", len(etag)) + } + return ETag(etag), nil + } + + prefix, suffix := s[:n], s[n:] + if len(prefix) != 32 { + return nil, fmt.Errorf("etag: invalid prefix length %d", len(prefix)) + } + if len(suffix) <= 1 { + return nil, errors.New("etag: suffix is not a part number") + } + + etag, err := hex.DecodeString(prefix) + if err != nil { + return nil, err + } + partNumber, err := strconv.Atoi(suffix[1:]) // suffix[0] == '-' Therefore, we start parsing at suffix[1] + if err != nil { + return nil, err + } + if strict && (partNumber == 0 || partNumber > 10000) { + return nil, fmt.Errorf("etag: invalid part number %d", partNumber) + } + return ETag(append(etag, suffix...)), nil +} diff --git a/internal/etag/etag_test.go b/internal/etag/etag_test.go new file mode 100644 index 00000000..4d18d4f4 --- /dev/null +++ b/internal/etag/etag_test.go @@ -0,0 +1,316 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package etag + +import ( + "io" + "net/http" + "strings" + "testing" +) + +var _ Tagger = Wrap(nil, nil).(Tagger) // runtime check that wrapReader implements Tagger + +var parseTests = []struct { + String string + ETag ETag + ShouldFail bool +}{ + {String: "3b83ef96387f1465", ETag: ETag{59, 131, 239, 150, 56, 127, 20, 101}}, // 0 + {String: "3b83ef96387f14655fc854ddc3c6bd57", ETag: ETag{59, 131, 239, 150, 56, 127, 20, 101, 95, 200, 84, 221, 195, 198, 189, 87}}, // 1 + {String: `"3b83ef96387f14655fc854ddc3c6bd57"`, ETag: ETag{59, 131, 239, 150, 56, 127, 20, 101, 95, 200, 84, 221, 195, 198, 189, 87}}, // 2 + {String: "ceb8853ddc5086cc4ab9e149f8f09c88-1", ETag: ETag{206, 184, 133, 61, 220, 80, 134, 204, 74, 185, 225, 73, 248, 240, 156, 136, 45, 49}}, // 3 + {String: `"ceb8853ddc5086cc4ab9e149f8f09c88-2"`, ETag: ETag{206, 184, 133, 61, 220, 80, 134, 204, 74, 185, 225, 73, 248, 240, 156, 136, 45, 50}}, // 4 + { // 5 + String: "90402c78d2dccddee1e9e86222ce2c6361675f3529d26000ae2e900ff216b3cb59e130e092d8a2981e776f4d0bd60941", + ETag: ETag{144, 64, 44, 120, 210, 220, 205, 222, 225, 233, 232, 98, 34, 206, 44, 99, 97, 103, 95, 53, 41, 210, 96, 0, 174, 46, 144, 15, 242, 22, 179, 203, 89, 225, 48, 224, 146, 216, 162, 152, 30, 119, 111, 77, 11, 214, 9, 65}, + }, + + {String: `"3b83ef96387f14655fc854ddc3c6bd57`, ShouldFail: true}, // 6 + {String: "ceb8853ddc5086cc4ab9e149f8f09c88-", ShouldFail: true}, // 7 + {String: "ceb8853ddc5086cc4ab9e149f8f09c88-2a", ShouldFail: true}, // 8 + {String: "ceb8853ddc5086cc4ab9e149f8f09c88-2-1", ShouldFail: true}, // 9 + {String: "90402c78d2dccddee1e9e86222ce2c-1", ShouldFail: true}, // 10 + {String: "90402c78d2dccddee1e9e86222ce2c6361675f3529d26000ae2e900ff216b3cb59e130e092d8a2981e776f4d0bd60941-1", ShouldFail: true}, // 11 +} + +func TestParse(t *testing.T) { + for i, test := range parseTests { + etag, err := Parse(test.String) + if err == nil && test.ShouldFail { + t.Fatalf("Test %d: parse should have failed but succeeded", i) + } + if err != nil && !test.ShouldFail { + t.Fatalf("Test %d: failed to parse ETag %q: %v", i, test.String, err) + } + if !Equal(etag, test.ETag) { + t.Log([]byte(etag)) + t.Fatalf("Test %d: ETags don't match", i) + } + } +} + +var stringTests = []struct { + ETag ETag + String string +}{ + {ETag: ETag{59, 131, 239, 150, 56, 127, 20, 101}, String: "3b83ef96387f1465"}, // 0 + {ETag: ETag{59, 131, 239, 150, 56, 127, 20, 101, 95, 200, 84, 221, 195, 198, 189, 87}, String: "3b83ef96387f14655fc854ddc3c6bd57"}, // 1 + {ETag: ETag{206, 184, 133, 61, 220, 80, 134, 204, 74, 185, 225, 73, 248, 240, 156, 136, 45, 49}, String: "ceb8853ddc5086cc4ab9e149f8f09c88-1"}, // 2 + {ETag: ETag{206, 184, 133, 61, 220, 80, 134, 204, 74, 185, 225, 73, 248, 240, 156, 136, 45, 50}, String: "ceb8853ddc5086cc4ab9e149f8f09c88-2"}, // 3 + { // 4 + ETag: ETag{144, 64, 44, 120, 210, 220, 205, 222, 225, 233, 232, 98, 34, 206, 44, 99, 97, 103, 95, 53, 41, 210, 96, 0, 174, 46, 144, 15, 242, 22, 179, 203, 89, 225, 48, 224, 146, 216, 162, 152, 30, 119, 111, 77, 11, 214, 9, 65}, + String: "90402c78d2dccddee1e9e86222ce2c6361675f3529d26000ae2e900ff216b3cb59e130e092d8a2981e776f4d0bd60941", + }, + { // 5 + ETag: ETag{32, 0, 15, 0, 219, 45, 144, 167, 180, 7, 130, 212, 207, 242, 180, 26, 119, 153, 252, 30, 126, 173, 37, 151, 45, 182, 81, 80, 17, 141, 251, 226, 186, 118, 163, 192, 2, 218, 40, 248, 92, 132, 12, 210, 0, 26, 40, 169}, + String: "20000f00db2d90a7b40782d4cff2b41a7799fc1e7ead25972db65150118dfbe2ba76a3c002da28f85c840cd2001a28a9", + }, +} + +func TestString(t *testing.T) { + for i, test := range stringTests { + s := test.ETag.String() + if s != test.String { + t.Fatalf("Test %d: got %s - want %s", i, s, test.String) + } + } +} + +var equalTests = []struct { + A string + B string + Equal bool +}{ + {A: "3b83ef96387f14655fc854ddc3c6bd57", B: "3b83ef96387f14655fc854ddc3c6bd57", Equal: true}, // 0 + {A: "3b83ef96387f14655fc854ddc3c6bd57", B: `"3b83ef96387f14655fc854ddc3c6bd57"`, Equal: true}, // 1 + + {A: "3b83ef96387f14655fc854ddc3c6bd57", B: "3b83ef96387f14655fc854ddc3c6bd57-2", Equal: false}, // 2 + {A: "3b83ef96387f14655fc854ddc3c6bd57", B: "ceb8853ddc5086cc4ab9e149f8f09c88", Equal: false}, // 3 +} + +func TestEqual(t *testing.T) { + for i, test := range equalTests { + A, err := Parse(test.A) + if err != nil { + t.Fatalf("Test %d: %v", i, err) + } + B, err := Parse(test.B) + if err != nil { + t.Fatalf("Test %d: %v", i, err) + } + if equal := Equal(A, B); equal != test.Equal { + t.Fatalf("Test %d: got %v - want %v", i, equal, test.Equal) + } + } +} + +var readerTests = []struct { // Reference values computed by: echo | md5sum + Content string + ETag ETag +}{ + { + Content: "", ETag: ETag{212, 29, 140, 217, 143, 0, 178, 4, 233, 128, 9, 152, 236, 248, 66, 126}, + }, + { + Content: " ", ETag: ETag{114, 21, 238, 156, 125, 157, 194, 41, 210, 146, 26, 64, 232, 153, 236, 95}, + }, + { + Content: "Hello World", ETag: ETag{177, 10, 141, 177, 100, 224, 117, 65, 5, 183, 169, 155, 231, 46, 63, 229}, + }, +} + +func TestReader(t *testing.T) { + for i, test := range readerTests { + reader := NewReader(t.Context(), strings.NewReader(test.Content), test.ETag, nil) + if _, err := io.Copy(io.Discard, reader); err != nil { + t.Fatalf("Test %d: read failed: %v", i, err) + } + if ETag := reader.ETag(); !Equal(ETag, test.ETag) { + t.Fatalf("Test %d: ETag mismatch: got %q - want %q", i, ETag, test.ETag) + } + } +} + +var multipartTests = []struct { // Test cases have been generated using AWS S3 + ETags []ETag + Multipart ETag +}{ + { + ETags: []ETag{}, + Multipart: ETag{}, + }, + { + ETags: []ETag{must("b10a8db164e0754105b7a99be72e3fe5")}, + Multipart: must("7b976cc68452e003eec7cb0eb631a19a-1"), + }, + { + ETags: []ETag{must("5f363e0e58a95f06cbe9bbc662c5dfb6"), must("5f363e0e58a95f06cbe9bbc662c5dfb6")}, + Multipart: must("a7d414b9133d6483d9a1c4e04e856e3b-2"), + }, + { + ETags: []ETag{must("5f363e0e58a95f06cbe9bbc662c5dfb6"), must("a096eb5968d607c2975fb2c4af9ab225"), must("b10a8db164e0754105b7a99be72e3fe5")}, + Multipart: must("9a0d1febd9265f59f368ceb652770bc2-3"), + }, + { // Check that multipart ETags are ignored + ETags: []ETag{must("5f363e0e58a95f06cbe9bbc662c5dfb6"), must("5f363e0e58a95f06cbe9bbc662c5dfb6"), must("ceb8853ddc5086cc4ab9e149f8f09c88-1")}, + Multipart: must("a7d414b9133d6483d9a1c4e04e856e3b-2"), + }, + { // Check that encrypted ETags are ignored + ETags: []ETag{ + must("90402c78d2dccddee1e9e86222ce2c6361675f3529d26000ae2e900ff216b3cb59e130e092d8a2981e776f4d0bd60941"), + must("5f363e0e58a95f06cbe9bbc662c5dfb6"), must("5f363e0e58a95f06cbe9bbc662c5dfb6"), + }, + Multipart: must("a7d414b9133d6483d9a1c4e04e856e3b-2"), + }, +} + +func TestMultipart(t *testing.T) { + for i, test := range multipartTests { + if multipart := Multipart(test.ETags...); !Equal(multipart, test.Multipart) { + t.Fatalf("Test %d: got %q - want %q", i, multipart, test.Multipart) + } + } +} + +var isEncryptedTests = []struct { + ETag string + IsEncrypted bool +}{ + {ETag: "20000f00db2d90a7b40782d4cff2b41a7799fc1e7ead25972db65150118dfbe2ba76a3c002da28f85c840cd2001a28a9", IsEncrypted: true}, // 0 + + {ETag: "3b83ef96387f14655fc854ddc3c6bd57"}, // 1 + {ETag: "7b976cc68452e003eec7cb0eb631a19a-1"}, // 2 + {ETag: "a7d414b9133d6483d9a1c4e04e856e3b-2"}, // 3 + {ETag: "7b976cc68452e003eec7cb0eb631a19a-10000"}, // 4 +} + +func TestIsEncrypted(t *testing.T) { + for i, test := range isEncryptedTests { + tag, err := Parse(test.ETag) + if err != nil { + t.Fatalf("Test %d: failed to parse ETag: %v", i, err) + } + if isEncrypted := tag.IsEncrypted(); isEncrypted != test.IsEncrypted { + t.Fatalf("Test %d: got '%v' - want '%v'", i, isEncrypted, test.IsEncrypted) + } + } +} + +var formatTests = []struct { + ETag string + AWSETag string +}{ + {ETag: "3b83ef96387f14655fc854ddc3c6bd57", AWSETag: "3b83ef96387f14655fc854ddc3c6bd57"}, // 0 + {ETag: "7b976cc68452e003eec7cb0eb631a19a-1", AWSETag: "7b976cc68452e003eec7cb0eb631a19a-1"}, // 1 + {ETag: "a7d414b9133d6483d9a1c4e04e856e3b-2", AWSETag: "a7d414b9133d6483d9a1c4e04e856e3b-2"}, // 2 + {ETag: "7b976cc68452e003eec7cb0eb631a19a-10000", AWSETag: "7b976cc68452e003eec7cb0eb631a19a-10000"}, // 3 + {ETag: "20000f00db2d90a7b40782d4cff2b41a7799fc1e7ead25972db65150118dfbe2ba76a3c002da28f85c840cd2001a28a9", AWSETag: "ba76a3c002da28f85c840cd2001a28a9"}, // 4 +} + +func TestFormat(t *testing.T) { + for i, test := range formatTests { + tag, err := Parse(test.ETag) + if err != nil { + t.Fatalf("Test %d: failed to parse ETag: %v", i, err) + } + if s := tag.Format().String(); s != test.AWSETag { + t.Fatalf("Test %d: got '%v' - want '%v'", i, s, test.AWSETag) + } + } +} + +var fromContentMD5Tests = []struct { + Header http.Header + ETag ETag + ShouldFail bool +}{ + {Header: http.Header{}, ETag: nil}, // 0 + {Header: http.Header{"Content-Md5": []string{"1B2M2Y8AsgTpgAmY7PhCfg=="}}, ETag: must("d41d8cd98f00b204e9800998ecf8427e")}, // 1 + {Header: http.Header{"Content-Md5": []string{"sQqNsWTgdUEFt6mb5y4/5Q=="}}, ETag: must("b10a8db164e0754105b7a99be72e3fe5")}, // 2 + {Header: http.Header{"Content-MD5": []string{"1B2M2Y8AsgTpgAmY7PhCfg=="}}, ETag: nil}, // 3 (Content-MD5 vs Content-Md5) + {Header: http.Header{"Content-Md5": []string{"sQqNsWTgdUEFt6mb5y4/5Q==", "1B2M2Y8AsgTpgAmY7PhCfg=="}}, ETag: must("b10a8db164e0754105b7a99be72e3fe5")}, // 4 + + {Header: http.Header{"Content-Md5": []string{""}}, ShouldFail: true}, // 5 (empty value) + {Header: http.Header{"Content-Md5": []string{"", "sQqNsWTgdUEFt6mb5y4/5Q=="}}, ShouldFail: true}, // 6 (empty value) + {Header: http.Header{"Content-Md5": []string{"d41d8cd98f00b204e9800998ecf8427e"}}, ShouldFail: true}, // 7 (content-md5 is invalid b64 / of invalid length) +} + +func TestFromContentMD5(t *testing.T) { + for i, test := range fromContentMD5Tests { + ETag, err := FromContentMD5(test.Header) + if err != nil && !test.ShouldFail { + t.Fatalf("Test %d: failed to convert Content-MD5 to ETag: %v", i, err) + } + if err == nil && test.ShouldFail { + t.Fatalf("Test %d: should have failed but succeeded", i) + } + if err == nil { + if !Equal(ETag, test.ETag) { + t.Fatalf("Test %d: got %q - want %q", i, ETag, test.ETag) + } + } + } +} + +var decryptTests = []struct { + Key []byte + ETag ETag + Plaintext ETag +}{ + { // 0 + Key: make([]byte, 32), + ETag: must("3b83ef96387f14655fc854ddc3c6bd57"), + Plaintext: must("3b83ef96387f14655fc854ddc3c6bd57"), + }, + { // 1 + Key: make([]byte, 32), + ETag: must("7b976cc68452e003eec7cb0eb631a19a-1"), + Plaintext: must("7b976cc68452e003eec7cb0eb631a19a-1"), + }, + { // 2 + Key: make([]byte, 32), + ETag: must("7b976cc68452e003eec7cb0eb631a19a-10000"), + Plaintext: must("7b976cc68452e003eec7cb0eb631a19a-10000"), + }, + { // 3 + Key: make([]byte, 32), + ETag: must("20000f00f2cc184414bc982927ec56abb7e18426faa205558982e9a8125c1370a9cf5754406e428b3343f21ee1125965"), + Plaintext: must("6d6cdccb9a7498c871bde8eab2f49141"), + }, +} + +func TestDecrypt(t *testing.T) { + for i, test := range decryptTests { + etag, err := Decrypt(test.Key, test.ETag) + if err != nil { + t.Fatalf("Test %d: failed to decrypt ETag: %v", i, err) + } + if !Equal(etag, test.Plaintext) { + t.Fatalf("Test %d: got '%v' - want '%v'", i, etag, test.Plaintext) + } + } +} + +func must(s string) ETag { + t, err := Parse(s) + if err != nil { + panic(err) + } + return t +} diff --git a/internal/etag/reader.go b/internal/etag/reader.go new file mode 100644 index 00000000..56da3da9 --- /dev/null +++ b/internal/etag/reader.go @@ -0,0 +1,198 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package etag + +import ( + "context" + "crypto/md5" + "fmt" + "hash" + "io" +) + +// Tagger is the interface that wraps the basic ETag method. +type Tagger interface { + ETag() ETag +} + +type wrapReader struct { + io.Reader + Tagger +} + +var _ Tagger = wrapReader{} // compiler check + +// ETag returns the ETag of the underlying Tagger. +func (r wrapReader) ETag() ETag { + if r.Tagger == nil { + return nil + } + return r.Tagger.ETag() +} + +// Wrap returns an io.Reader that reads from the wrapped +// io.Reader and implements the Tagger interaface. +// +// If content implements Tagger then the returned Reader +// returns ETag of the content. Otherwise, it returns +// nil as ETag. +// +// Wrap provides an adapter for io.Reader implementations +// that don't implement the Tagger interface. +// It is mainly used to provide a high-level io.Reader +// access to the ETag computed by a low-level io.Reader: +// +// content := etag.NewReader(r.Body, nil) +// +// compressedContent := Compress(content) +// encryptedContent := Encrypt(compressedContent) +// +// // Now, we need an io.Reader that can access +// // the ETag computed over the content. +// reader := etag.Wrap(encryptedContent, content) +func Wrap(wrapped, content io.Reader) io.Reader { + if t, ok := content.(Tagger); ok { + return wrapReader{ + Reader: wrapped, + Tagger: t, + } + } + return wrapReader{ + Reader: wrapped, + } +} + +// A Reader wraps an io.Reader and computes the +// MD5 checksum of the read content as ETag. +// +// Optionally, a Reader can also verify that +// the computed ETag matches an expected value. +// Therefore, it compares both ETags once the +// underlying io.Reader returns io.EOF. +// If the computed ETag does not match the +// expected ETag then Read returns a VerifyError. +// +// Reader implements the Tagger interface. +type Reader struct { + src io.Reader + + md5 hash.Hash + checksum ETag + + readN int64 +} + +// NewReader returns a new Reader that computes the +// MD5 checksum of the content read from r as ETag. +// +// If the provided etag is not nil the returned +// Reader compares the etag with the computed +// MD5 sum once the r returns io.EOF. +func NewReader(ctx context.Context, r io.Reader, etag ETag, forceMD5 []byte) *Reader { + if er, ok := r.(*Reader); ok { + if er.readN == 0 && Equal(etag, er.checksum) { + return er + } + } + if len(forceMD5) != 0 { + return &Reader{ + src: r, + md5: NewUUIDHash(forceMD5), + checksum: etag, + } + } + return &Reader{ + src: r, + md5: md5.New(), + checksum: etag, + } +} + +// Read reads up to len(p) bytes from the underlying +// io.Reader as specified by the io.Reader interface. +func (r *Reader) Read(p []byte) (int, error) { + n, err := r.src.Read(p) + r.readN += int64(n) + r.md5.Write(p[:n]) + + if err == io.EOF && len(r.checksum) != 0 { + if etag := r.ETag(); !Equal(etag, r.checksum) { + return n, VerifyError{ + Expected: r.checksum, + Computed: etag, + } + } + } + return n, err +} + +// ETag returns the ETag of all the content read +// so far. Reading more content changes the MD5 +// checksum. Therefore, calling ETag multiple +// times may return different results. +func (r *Reader) ETag() ETag { + sum := r.md5.Sum(nil) + return ETag(sum) +} + +// VerifyError is an error signaling that a +// computed ETag does not match an expected +// ETag. +type VerifyError struct { + Expected ETag + Computed ETag +} + +func (v VerifyError) Error() string { + return fmt.Sprintf("etag: expected ETag %q does not match computed ETag %q", v.Expected, v.Computed) +} + +// UUIDHash - use uuid to make md5sum +type UUIDHash struct { + uuid []byte +} + +// Write - implement hash.Hash Write +func (u UUIDHash) Write(p []byte) (n int, err error) { + return len(p), nil +} + +// Sum - implement md5.Sum +func (u UUIDHash) Sum(b []byte) []byte { + return u.uuid +} + +// Reset - implement hash.Hash Reset +func (u UUIDHash) Reset() {} + +// Size - implement hash.Hash Size +func (u UUIDHash) Size() int { + return len(u.uuid) +} + +// BlockSize - implement hash.Hash BlockSize +func (u UUIDHash) BlockSize() int { + return md5.BlockSize +} + +var _ hash.Hash = &UUIDHash{} + +// NewUUIDHash - new UUIDHash +func NewUUIDHash(uuid []byte) *UUIDHash { + return &UUIDHash{uuid: uuid} +} diff --git a/internal/event/arn.go b/internal/event/arn.go new file mode 100644 index 00000000..6c263560 --- /dev/null +++ b/internal/event/arn.go @@ -0,0 +1,84 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "encoding/xml" + "strings" +) + +// ARN - SQS resource name representation. +type ARN struct { + TargetID + region string +} + +// String - returns string representation. +func (arn ARN) String() string { + if arn.ID == "" && arn.Name == "" && arn.region == "" { + return "" + } + + return "arn:minio:sqs:" + arn.region + ":" + arn.TargetID.String() +} + +// MarshalXML - encodes to XML data. +func (arn ARN) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + return e.EncodeElement(arn.String(), start) +} + +// UnmarshalXML - decodes XML data. +func (arn *ARN) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + var s string + if err := d.DecodeElement(&s, &start); err != nil { + return err + } + + parsedARN, err := parseARN(s) + if err != nil { + return err + } + + *arn = *parsedARN + return nil +} + +// parseARN - parses string to ARN. +func parseARN(s string) (*ARN, error) { + // ARN must be in the format of arn:minio:sqs::: + if !strings.HasPrefix(s, "arn:minio:sqs:") { + return nil, &ErrInvalidARN{s} + } + + tokens := strings.Split(s, ":") + if len(tokens) != 6 { + return nil, &ErrInvalidARN{s} + } + + if tokens[4] == "" || tokens[5] == "" { + return nil, &ErrInvalidARN{s} + } + + return &ARN{ + region: tokens[3], + TargetID: TargetID{ + ID: tokens[4], + Name: tokens[5], + }, + }, nil +} diff --git a/internal/event/arn_test.go b/internal/event/arn_test.go new file mode 100644 index 00000000..f9281fad --- /dev/null +++ b/internal/event/arn_test.go @@ -0,0 +1,130 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "encoding/xml" + "reflect" + "testing" +) + +func TestARNString(t *testing.T) { + testCases := []struct { + arn ARN + expectedResult string + }{ + {ARN{}, ""}, + {ARN{TargetID{"1", "webhook"}, ""}, "arn:minio:sqs::1:webhook"}, + {ARN{TargetID{"1", "webhook"}, "us-east-1"}, "arn:minio:sqs:us-east-1:1:webhook"}, + } + + for i, testCase := range testCases { + result := testCase.arn.String() + + if result != testCase.expectedResult { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestARNMarshalXML(t *testing.T) { + testCases := []struct { + arn ARN + expectedData []byte + expectErr bool + }{ + {ARN{}, []byte(""), false}, + {ARN{TargetID{"1", "webhook"}, ""}, []byte("arn:minio:sqs::1:webhook"), false}, + {ARN{TargetID{"1", "webhook"}, "us-east-1"}, []byte("arn:minio:sqs:us-east-1:1:webhook"), false}, + } + + for i, testCase := range testCases { + data, err := xml.Marshal(testCase.arn) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + + if !testCase.expectErr { + if !reflect.DeepEqual(data, testCase.expectedData) { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, string(testCase.expectedData), string(data)) + } + } + } +} + +func TestARNUnmarshalXML(t *testing.T) { + testCases := []struct { + data []byte + expectedARN *ARN + expectErr bool + }{ + {[]byte(""), nil, true}, + {[]byte("arn:minio:sqs:::"), nil, true}, + {[]byte("arn:minio:sqs::1:webhook"), &ARN{TargetID{"1", "webhook"}, ""}, false}, + {[]byte("arn:minio:sqs:us-east-1:1:webhook"), &ARN{TargetID{"1", "webhook"}, "us-east-1"}, false}, + } + + for i, testCase := range testCases { + arn := &ARN{} + err := xml.Unmarshal(testCase.data, &arn) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + + if !testCase.expectErr { + if *arn != *testCase.expectedARN { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedARN, arn) + } + } + } +} + +func TestParseARN(t *testing.T) { + testCases := []struct { + s string + expectedARN *ARN + expectErr bool + }{ + {"", nil, true}, + {"arn:minio:sqs:::", nil, true}, + {"arn:minio:sqs::1:webhook:remote", nil, true}, + {"arn:aws:sqs::1:webhook", nil, true}, + {"arn:minio:sns::1:webhook", nil, true}, + {"arn:minio:sqs::1:webhook", &ARN{TargetID{"1", "webhook"}, ""}, false}, + {"arn:minio:sqs:us-east-1:1:webhook", &ARN{TargetID{"1", "webhook"}, "us-east-1"}, false}, + } + + for i, testCase := range testCases { + arn, err := parseARN(testCase.s) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + + if !testCase.expectErr { + if *arn != *testCase.expectedARN { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedARN, arn) + } + } + } +} diff --git a/internal/event/config.go b/internal/event/config.go new file mode 100644 index 00000000..de21764b --- /dev/null +++ b/internal/event/config.go @@ -0,0 +1,326 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "encoding/xml" + "errors" + "io" + "reflect" + "strings" + "unicode/utf8" + + "github.com/minio/minio-go/v7/pkg/set" +) + +// ValidateFilterRuleValue - checks if given value is filter rule value or not. +func ValidateFilterRuleValue(value string) error { + for segment := range strings.SplitSeq(value, "/") { + if segment == "." || segment == ".." { + return &ErrInvalidFilterValue{value} + } + } + + if len(value) <= 1024 && utf8.ValidString(value) && !strings.Contains(value, `\`) { + return nil + } + + return &ErrInvalidFilterValue{value} +} + +// FilterRule - represents elements inside ... +type FilterRule struct { + Name string `xml:"Name"` + Value string `xml:"Value"` +} + +func (filter FilterRule) isEmpty() bool { + return filter.Name == "" && filter.Value == "" +} + +// MarshalXML implements a custom marshaller to support `omitempty` feature. +func (filter FilterRule) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if filter.isEmpty() { + return nil + } + type filterRuleWrapper FilterRule + return e.EncodeElement(filterRuleWrapper(filter), start) +} + +// UnmarshalXML - decodes XML data. +func (filter *FilterRule) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + // Make subtype to avoid recursive UnmarshalXML(). + type filterRule FilterRule + rule := filterRule{} + if err := d.DecodeElement(&rule, &start); err != nil { + return err + } + + if rule.Name != "prefix" && rule.Name != "suffix" { + return &ErrInvalidFilterName{rule.Name} + } + + if err := ValidateFilterRuleValue(filter.Value); err != nil { + return err + } + + *filter = FilterRule(rule) + + return nil +} + +// FilterRuleList - represents multiple ... +type FilterRuleList struct { + Rules []FilterRule `xml:"FilterRule,omitempty"` +} + +// UnmarshalXML - decodes XML data. +func (ruleList *FilterRuleList) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + // Make subtype to avoid recursive UnmarshalXML(). + type filterRuleList FilterRuleList + rules := filterRuleList{} + if err := d.DecodeElement(&rules, &start); err != nil { + return err + } + + // FilterRuleList must have only one prefix and/or suffix. + nameSet := set.NewStringSet() + for _, rule := range rules.Rules { + if nameSet.Contains(rule.Name) { + if rule.Name == "prefix" { + return &ErrFilterNamePrefix{} + } + + return &ErrFilterNameSuffix{} + } + + nameSet.Add(rule.Name) + } + + *ruleList = FilterRuleList(rules) + return nil +} + +func (ruleList FilterRuleList) isEmpty() bool { + return len(ruleList.Rules) == 0 +} + +// Pattern - returns pattern using prefix and suffix values. +func (ruleList FilterRuleList) Pattern() string { + var prefix string + var suffix string + + for _, rule := range ruleList.Rules { + switch rule.Name { + case "prefix": + prefix = rule.Value + case "suffix": + suffix = rule.Value + } + } + + return NewPattern(prefix, suffix) +} + +// S3Key - represents elements inside ... +type S3Key struct { + RuleList FilterRuleList `xml:"S3Key,omitempty" json:"S3Key"` +} + +// MarshalXML implements a custom marshaller to support `omitempty` feature. +func (s3Key S3Key) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + if s3Key.RuleList.isEmpty() { + return nil + } + type s3KeyWrapper S3Key + return e.EncodeElement(s3KeyWrapper(s3Key), start) +} + +// common - represents common elements inside , +// and +type common struct { + ID string `xml:"Id" json:"Id"` + Filter S3Key `xml:"Filter" json:"Filter"` + Events []Name `xml:"Event" json:"Event"` +} + +// Queue - represents elements inside +type Queue struct { + common + ARN ARN `xml:"Queue"` +} + +// UnmarshalXML - decodes XML data. +func (q *Queue) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + // Make subtype to avoid recursive UnmarshalXML(). + type queue Queue + parsedQueue := queue{} + if err := d.DecodeElement(&parsedQueue, &start); err != nil { + return err + } + + if len(parsedQueue.Events) == 0 { + return errors.New("missing event name(s)") + } + + eventStringSet := set.NewStringSet() + for _, eventName := range parsedQueue.Events { + if eventStringSet.Contains(eventName.String()) { + return &ErrDuplicateEventName{eventName} + } + + eventStringSet.Add(eventName.String()) + } + + *q = Queue(parsedQueue) + + return nil +} + +// Validate - checks whether queue has valid values or not. +func (q Queue) Validate(region string, targetList *TargetList) error { + if q.ARN.region == "" { + if !targetList.Exists(q.ARN.TargetID) { + return &ErrARNNotFound{q.ARN} + } + return nil + } + + if region != "" && q.ARN.region != region { + return &ErrUnknownRegion{q.ARN.region} + } + + if !targetList.Exists(q.ARN.TargetID) { + return &ErrARNNotFound{q.ARN} + } + + return nil +} + +// SetRegion - sets region value to queue's ARN. +func (q *Queue) SetRegion(region string) { + q.ARN.region = region +} + +// ToRulesMap - converts Queue to RulesMap +func (q Queue) ToRulesMap() RulesMap { + pattern := q.Filter.RuleList.Pattern() + return NewRulesMap(q.Events, pattern, q.ARN.TargetID) +} + +// Unused. Available for completion. +type lambda struct { + ARN string `xml:"CloudFunction"` +} + +// Unused. Available for completion. +type topic struct { + ARN string `xml:"Topic" json:"Topic"` +} + +// Config - notification configuration described in +// http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html +type Config struct { + XMLNS string `xml:"xmlns,attr,omitempty"` + XMLName xml.Name `xml:"NotificationConfiguration"` + QueueList []Queue `xml:"QueueConfiguration,omitempty"` + LambdaList []lambda `xml:"CloudFunctionConfiguration,omitempty"` + TopicList []topic `xml:"TopicConfiguration,omitempty"` +} + +// UnmarshalXML - decodes XML data. +func (conf *Config) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + // Make subtype to avoid recursive UnmarshalXML(). + type config Config + parsedConfig := config{} + if err := d.DecodeElement(&parsedConfig, &start); err != nil { + return err + } + + // Empty queue list means user wants to delete the notification configuration. + if len(parsedConfig.QueueList) > 0 { + for i, q1 := range parsedConfig.QueueList[:len(parsedConfig.QueueList)-1] { + for _, q2 := range parsedConfig.QueueList[i+1:] { + // Removes the region from ARN if server region is not set + if q2.ARN.region != "" && q1.ARN.region == "" { + q2.ARN.region = "" + } + if reflect.DeepEqual(q1, q2) { + return &ErrDuplicateQueueConfiguration{q1} + } + } + } + } + + if len(parsedConfig.LambdaList) > 0 || len(parsedConfig.TopicList) > 0 { + return &ErrUnsupportedConfiguration{} + } + + *conf = Config(parsedConfig) + + return nil +} + +// Validate - checks whether config has valid values or not. +func (conf Config) Validate(region string, targetList *TargetList) error { + for _, queue := range conf.QueueList { + if err := queue.Validate(region, targetList); err != nil { + return err + } + } + + return nil +} + +// SetRegion - sets region to all queue configuration. +func (conf *Config) SetRegion(region string) { + for i := range conf.QueueList { + conf.QueueList[i].SetRegion(region) + } +} + +// ToRulesMap - converts all queue configuration to RulesMap. +func (conf *Config) ToRulesMap() RulesMap { + rulesMap := make(RulesMap) + + for _, queue := range conf.QueueList { + rulesMap.Add(queue.ToRulesMap()) + } + + return rulesMap +} + +// ParseConfig - parses data in reader to notification configuration. +func ParseConfig(reader io.Reader, region string, targetList *TargetList) (*Config, error) { + var config Config + + if err := xml.NewDecoder(reader).Decode(&config); err != nil { + return nil, err + } + + if err := config.Validate(region, targetList); err != nil { + return nil, err + } + + config.SetRegion(region) + // If xml namespace is empty, set a default value before returning. + if config.XMLNS == "" { + config.XMLNS = "http://s3.amazonaws.com/doc/2006-03-01/" + } + return &config, nil +} diff --git a/internal/event/config_test.go b/internal/event/config_test.go new file mode 100644 index 00000000..5190de38 --- /dev/null +++ b/internal/event/config_test.go @@ -0,0 +1,961 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "encoding/xml" + "reflect" + "strings" + "testing" +) + +func TestValidateFilterRuleValue(t *testing.T) { + testCases := []struct { + value string + expectErr bool + }{ + {"foo/.", true}, + {"../foo", true}, + {`foo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/bazfoo/bar/baz`, true}, + {string([]byte{0xff, 0xfe, 0xfd}), true}, + {`foo\bar`, true}, + {"Hello/世界", false}, + } + + for i, testCase := range testCases { + err := ValidateFilterRuleValue(testCase.value) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + } +} + +func TestFilterRuleUnmarshalXML(t *testing.T) { + testCases := []struct { + data []byte + expectedResult *FilterRule + expectErr bool + }{ + {[]byte(``), nil, true}, + {[]byte(``), nil, true}, + {[]byte(``), nil, true}, + {[]byte(``), nil, true}, + {[]byte(`PrefixHello/世界`), nil, true}, + {[]byte(`endsfoo/bar`), nil, true}, + {[]byte(`prefixHello/世界`), &FilterRule{"prefix", "Hello/世界"}, false}, + {[]byte(`suffixfoo/bar`), &FilterRule{"suffix", "foo/bar"}, false}, + } + + for i, testCase := range testCases { + result := &FilterRule{} + err := xml.Unmarshal(testCase.data, result) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + + if !testCase.expectErr { + if !reflect.DeepEqual(result, testCase.expectedResult) { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } + } +} + +func TestFilterRuleListUnmarshalXML(t *testing.T) { + testCases := []struct { + data []byte + expectedResult *FilterRuleList + expectErr bool + }{ + {[]byte(`suffixHello/世界suffixfoo/bar`), nil, true}, + {[]byte(`prefixHello/世界prefixfoo/bar`), nil, true}, + {[]byte(`prefixHello/世界`), &FilterRuleList{[]FilterRule{{"prefix", "Hello/世界"}}}, false}, + {[]byte(`suffixfoo/bar`), &FilterRuleList{[]FilterRule{{"suffix", "foo/bar"}}}, false}, + {[]byte(`prefixHello/世界suffixfoo/bar`), &FilterRuleList{[]FilterRule{{"prefix", "Hello/世界"}, {"suffix", "foo/bar"}}}, false}, + } + + for i, testCase := range testCases { + result := &FilterRuleList{} + err := xml.Unmarshal(testCase.data, result) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + + if !testCase.expectErr { + if !reflect.DeepEqual(result, testCase.expectedResult) { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } + } +} + +func TestFilterRuleListPattern(t *testing.T) { + testCases := []struct { + filterRuleList FilterRuleList + expectedResult string + }{ + {FilterRuleList{}, ""}, + {FilterRuleList{[]FilterRule{{"prefix", "Hello/世界"}}}, "Hello/世界*"}, + {FilterRuleList{[]FilterRule{{"suffix", "foo/bar"}}}, "*foo/bar"}, + {FilterRuleList{[]FilterRule{{"prefix", "Hello/世界"}, {"suffix", "foo/bar"}}}, "Hello/世界*foo/bar"}, + } + + for i, testCase := range testCases { + result := testCase.filterRuleList.Pattern() + + if result != testCase.expectedResult { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestQueueUnmarshalXML(t *testing.T) { + dataCase1 := []byte(` + + 1 + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectAccessed:* + s3:ObjectCreated:* + s3:ObjectRemoved:* +`) + + dataCase2 := []byte(` + + 1 + + + + prefix + images/ + + + suffix + jpg + + + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectCreated:Put +`) + + dataCase3 := []byte(` + + 1 + + + + prefix + images/ + + + suffix + jpg + + + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectCreated:Put + s3:ObjectCreated:Put +`) + + testCases := []struct { + data []byte + expectErr bool + }{ + {dataCase1, false}, + {dataCase2, false}, + {dataCase3, true}, + } + + for i, testCase := range testCases { + err := xml.Unmarshal(testCase.data, &Queue{}) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + } +} + +func TestQueueValidate(t *testing.T) { + data := []byte(` + + 1 + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectAccessed:* + s3:ObjectCreated:* + s3:ObjectRemoved:* +`) + queue1 := &Queue{} + if err := xml.Unmarshal(data, queue1); err != nil { + panic(err) + } + + data = []byte(` + + 1 + + + + prefix + images/ + + + suffix + jpg + + + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectCreated:Put +`) + queue2 := &Queue{} + if err := xml.Unmarshal(data, queue2); err != nil { + panic(err) + } + + data = []byte(` + + 1 + + arn:minio:sqs:eu-west-2:1:webhook + s3:ObjectAccessed:* + s3:ObjectCreated:* + s3:ObjectRemoved:* +`) + queue3 := &Queue{} + if err := xml.Unmarshal(data, queue3); err != nil { + panic(err) + } + + targetList1 := NewTargetList(t.Context()) + + targetList2 := NewTargetList(t.Context()) + if err := targetList2.Add(&ExampleTarget{TargetID{"1", "webhook"}, false, false}); err != nil { + panic(err) + } + + testCases := []struct { + queue *Queue + region string + targetList *TargetList + expectErr bool + }{ + {queue1, "eu-west-1", nil, true}, + {queue2, "us-east-1", targetList1, true}, + {queue3, "", targetList2, false}, + {queue2, "us-east-1", targetList2, false}, + } + + for i, testCase := range testCases { + err := testCase.queue.Validate(testCase.region, testCase.targetList) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + } +} + +func TestQueueSetRegion(t *testing.T) { + data := []byte(` + + 1 + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectAccessed:* + s3:ObjectCreated:* + s3:ObjectRemoved:* +`) + queue1 := &Queue{} + if err := xml.Unmarshal(data, queue1); err != nil { + panic(err) + } + + data = []byte(` + + 1 + + + + prefix + images/ + + + suffix + jpg + + + + arn:minio:sqs::1:webhook + s3:ObjectCreated:Put +`) + queue2 := &Queue{} + if err := xml.Unmarshal(data, queue2); err != nil { + panic(err) + } + + testCases := []struct { + queue *Queue + region string + expectedResult ARN + }{ + {queue1, "eu-west-1", ARN{TargetID{"1", "webhook"}, "eu-west-1"}}, + {queue1, "", ARN{TargetID{"1", "webhook"}, ""}}, + {queue2, "us-east-1", ARN{TargetID{"1", "webhook"}, "us-east-1"}}, + {queue2, "", ARN{TargetID{"1", "webhook"}, ""}}, + } + + for i, testCase := range testCases { + testCase.queue.SetRegion(testCase.region) + result := testCase.queue.ARN + + if !reflect.DeepEqual(result, testCase.expectedResult) { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestQueueToRulesMap(t *testing.T) { + data := []byte(` + + 1 + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectAccessed:* + s3:ObjectCreated:* + s3:ObjectRemoved:* +`) + queueCase1 := &Queue{} + if err := xml.Unmarshal(data, queueCase1); err != nil { + panic(err) + } + + data = []byte(` + + 1 + + + + prefix + images/ + + + suffix + jpg + + + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectCreated:Put +`) + queueCase2 := &Queue{} + if err := xml.Unmarshal(data, queueCase2); err != nil { + panic(err) + } + + rulesMapCase1 := NewRulesMap([]Name{ObjectAccessedAll, ObjectCreatedAll, ObjectRemovedAll}, "*", TargetID{"1", "webhook"}) + rulesMapCase2 := NewRulesMap([]Name{ObjectCreatedPut}, "images/*jpg", TargetID{"1", "webhook"}) + + testCases := []struct { + queue *Queue + expectedResult RulesMap + }{ + {queueCase1, rulesMapCase1}, + {queueCase2, rulesMapCase2}, + } + + for i, testCase := range testCases { + result := testCase.queue.ToRulesMap() + + if !reflect.DeepEqual(result, testCase.expectedResult) { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestConfigUnmarshalXML(t *testing.T) { + dataCase1 := []byte(` + + + 1 + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectAccessed:* + s3:ObjectCreated:* + s3:ObjectRemoved:* + + +`) + + dataCase2 := []byte(` + + + 1 + + + + prefix + images/ + + + suffix + jpg + + + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectCreated:Put + + + `) + + dataCase3 := []byte(` + + + 1 + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectAccessed:* + s3:ObjectCreated:* + s3:ObjectRemoved:* + + + 2 + + + + prefix + images/ + + + suffix + jpg + + + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectCreated:Put + + + `) + + dataCase4 := []byte(` + + + 1 + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectAccessed:* + s3:ObjectCreated:* + s3:ObjectRemoved:* + + + 1 + + + + suffix + .jpg + + + + arn:aws:lambda:us-west-2:444455556666:cloud-function-A + s3:ObjectCreated:Put + + + arn:aws:sns:us-west-2:444455556666:sns-notification-one + s3:ObjectCreated:* + + + `) + + dataCase5 := []byte(``) + + testCases := []struct { + data []byte + expectErr bool + }{ + {dataCase1, false}, + {dataCase2, false}, + {dataCase3, false}, + {dataCase4, true}, + // make sure we don't fail when queue is empty. + {dataCase5, false}, + } + + for i, testCase := range testCases { + err := xml.Unmarshal(testCase.data, &Config{}) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + } +} + +func TestConfigValidate(t *testing.T) { + data := []byte(` + + + 1 + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectAccessed:* + s3:ObjectCreated:* + s3:ObjectRemoved:* + + +`) + config1 := &Config{} + if err := xml.Unmarshal(data, config1); err != nil { + panic(err) + } + + data = []byte(` + + + 1 + + + + prefix + images/ + + + suffix + jpg + + + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectCreated:Put + + +`) + config2 := &Config{} + if err := xml.Unmarshal(data, config2); err != nil { + panic(err) + } + + data = []byte(` + + + 1 + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectAccessed:* + s3:ObjectCreated:* + s3:ObjectRemoved:* + + + 2 + + + + prefix + images/ + + + suffix + jpg + + + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectCreated:Put + + +`) + config3 := &Config{} + if err := xml.Unmarshal(data, config3); err != nil { + panic(err) + } + + targetList1 := NewTargetList(t.Context()) + + targetList2 := NewTargetList(t.Context()) + if err := targetList2.Add(&ExampleTarget{TargetID{"1", "webhook"}, false, false}); err != nil { + panic(err) + } + + testCases := []struct { + config *Config + region string + targetList *TargetList + expectErr bool + }{ + {config1, "eu-west-1", nil, true}, + {config2, "us-east-1", targetList1, true}, + {config3, "", targetList2, false}, + {config2, "us-east-1", targetList2, false}, + } + + for i, testCase := range testCases { + err := testCase.config.Validate(testCase.region, testCase.targetList) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + } +} + +func TestConfigSetRegion(t *testing.T) { + data := []byte(` + + + 1 + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectAccessed:* + s3:ObjectCreated:* + s3:ObjectRemoved:* + + +`) + config1 := &Config{} + if err := xml.Unmarshal(data, config1); err != nil { + panic(err) + } + + data = []byte(` + + + 1 + + + + prefix + images/ + + + suffix + jpg + + + + arn:minio:sqs::1:webhook + s3:ObjectCreated:Put + + +`) + config2 := &Config{} + if err := xml.Unmarshal(data, config2); err != nil { + panic(err) + } + + data = []byte(` + + + 1 + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectAccessed:* + s3:ObjectCreated:* + s3:ObjectRemoved:* + + + 2 + + + + prefix + images/ + + + suffix + jpg + + + + arn:minio:sqs:us-east-1:2:amqp + s3:ObjectCreated:Put + + +`) + config3 := &Config{} + if err := xml.Unmarshal(data, config3); err != nil { + panic(err) + } + + testCases := []struct { + config *Config + region string + expectedResult []ARN + }{ + {config1, "eu-west-1", []ARN{{TargetID{"1", "webhook"}, "eu-west-1"}}}, + {config1, "", []ARN{{TargetID{"1", "webhook"}, ""}}}, + {config2, "us-east-1", []ARN{{TargetID{"1", "webhook"}, "us-east-1"}}}, + {config2, "", []ARN{{TargetID{"1", "webhook"}, ""}}}, + {config3, "us-east-1", []ARN{{TargetID{"1", "webhook"}, "us-east-1"}, {TargetID{"2", "amqp"}, "us-east-1"}}}, + {config3, "", []ARN{{TargetID{"1", "webhook"}, ""}, {TargetID{"2", "amqp"}, ""}}}, + } + + for i, testCase := range testCases { + testCase.config.SetRegion(testCase.region) + result := []ARN{} + for _, queue := range testCase.config.QueueList { + result = append(result, queue.ARN) + } + + if !reflect.DeepEqual(result, testCase.expectedResult) { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestConfigToRulesMap(t *testing.T) { + data := []byte(` + + + 1 + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectAccessed:* + s3:ObjectCreated:* + s3:ObjectRemoved:* + + +`) + config1 := &Config{} + if err := xml.Unmarshal(data, config1); err != nil { + panic(err) + } + + data = []byte(` + + + 1 + + + + prefix + images/ + + + suffix + jpg + + + + arn:minio:sqs::1:webhook + s3:ObjectCreated:Put + + +`) + config2 := &Config{} + if err := xml.Unmarshal(data, config2); err != nil { + panic(err) + } + + data = []byte(` + + + 1 + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectAccessed:* + s3:ObjectCreated:* + s3:ObjectRemoved:* + + + 2 + + + + prefix + images/ + + + suffix + jpg + + + + arn:minio:sqs:us-east-1:2:amqp + s3:ObjectCreated:Put + + +`) + config3 := &Config{} + if err := xml.Unmarshal(data, config3); err != nil { + panic(err) + } + + rulesMapCase1 := NewRulesMap([]Name{ObjectAccessedAll, ObjectCreatedAll, ObjectRemovedAll}, "*", TargetID{"1", "webhook"}) + + rulesMapCase2 := NewRulesMap([]Name{ObjectCreatedPut}, "images/*jpg", TargetID{"1", "webhook"}) + + rulesMapCase3 := NewRulesMap([]Name{ObjectAccessedAll, ObjectCreatedAll, ObjectRemovedAll}, "*", TargetID{"1", "webhook"}) + rulesMapCase3.add([]Name{ObjectCreatedPut}, "images/*jpg", TargetID{"2", "amqp"}) + + testCases := []struct { + config *Config + expectedResult RulesMap + }{ + {config1, rulesMapCase1}, + {config2, rulesMapCase2}, + {config3, rulesMapCase3}, + } + + for i, testCase := range testCases { + result := testCase.config.ToRulesMap() + + if !reflect.DeepEqual(result, testCase.expectedResult) { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestParseConfig(t *testing.T) { + reader1 := strings.NewReader(` + + + 1 + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectAccessed:* + s3:ObjectCreated:* + s3:ObjectRemoved:* + + +`) + + reader2 := strings.NewReader(` + + + 1 + + + + prefix + images/ + + + suffix + jpg + + + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectCreated:Put + + +`) + + reader3 := strings.NewReader(` + + + 1 + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectAccessed:* + s3:ObjectCreated:* + s3:ObjectRemoved:* + + + 2 + + + + prefix + images/ + + + suffix + jpg + + + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectCreated:Put + + +`) + + reader4 := strings.NewReader(` + + + 1 + + arn:minio:sqs:us-east-1:1:webhook + s3:ObjectAccessed:* + s3:ObjectCreated:* + s3:ObjectRemoved:* + + + 1 + + + + suffix + .jpg + + + + arn:aws:lambda:us-west-2:444455556666:cloud-function-A + s3:ObjectCreated:Put + + + arn:aws:sns:us-west-2:444455556666:sns-notification-one + s3:ObjectCreated:* + + +`) + + targetList1 := NewTargetList(t.Context()) + + targetList2 := NewTargetList(t.Context()) + if err := targetList2.Add(&ExampleTarget{TargetID{"1", "webhook"}, false, false}); err != nil { + panic(err) + } + + testCases := []struct { + reader *strings.Reader + region string + targetList *TargetList + expectErr bool + }{ + {reader1, "eu-west-1", nil, true}, + {reader2, "us-east-1", targetList1, true}, + {reader4, "us-east-1", targetList1, true}, + {reader3, "", targetList2, false}, + {reader2, "us-east-1", targetList2, false}, + } + + for i, testCase := range testCases { + if _, err := testCase.reader.Seek(0, 0); err != nil { + panic(err) + } + _, err := ParseConfig(testCase.reader, testCase.region, testCase.targetList) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + } +} diff --git a/internal/event/errors.go b/internal/event/errors.go new file mode 100644 index 00000000..f1b54206 --- /dev/null +++ b/internal/event/errors.go @@ -0,0 +1,153 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "encoding/xml" + "fmt" +) + +// IsEventError - checks whether given error is event error or not. +func IsEventError(err error) bool { + switch err.(type) { + case ErrInvalidFilterName, *ErrInvalidFilterName: + return true + case ErrFilterNamePrefix, *ErrFilterNamePrefix: + return true + case ErrFilterNameSuffix, *ErrFilterNameSuffix: + return true + case ErrInvalidFilterValue, *ErrInvalidFilterValue: + return true + case ErrDuplicateEventName, *ErrDuplicateEventName: + return true + case ErrUnsupportedConfiguration, *ErrUnsupportedConfiguration: + return true + case ErrDuplicateQueueConfiguration, *ErrDuplicateQueueConfiguration: + return true + case ErrUnknownRegion, *ErrUnknownRegion: + return true + case ErrARNNotFound, *ErrARNNotFound: + return true + case ErrInvalidARN, *ErrInvalidARN: + return true + case ErrInvalidEventName, *ErrInvalidEventName: + return true + } + + return false +} + +// ErrInvalidFilterName - invalid filter name error. +type ErrInvalidFilterName struct { + FilterName string +} + +func (err ErrInvalidFilterName) Error() string { + return fmt.Sprintf("invalid filter name '%v'", err.FilterName) +} + +// ErrFilterNamePrefix - more than one prefix usage error. +type ErrFilterNamePrefix struct{} + +func (err ErrFilterNamePrefix) Error() string { + return "more than one prefix in filter rule" +} + +// ErrFilterNameSuffix - more than one suffix usage error. +type ErrFilterNameSuffix struct{} + +func (err ErrFilterNameSuffix) Error() string { + return "more than one suffix in filter rule" +} + +// ErrInvalidFilterValue - invalid filter value error. +type ErrInvalidFilterValue struct { + FilterValue string +} + +func (err ErrInvalidFilterValue) Error() string { + return fmt.Sprintf("invalid filter value '%v'", err.FilterValue) +} + +// ErrDuplicateEventName - duplicate event name error. +type ErrDuplicateEventName struct { + EventName Name +} + +func (err ErrDuplicateEventName) Error() string { + return fmt.Sprintf("duplicate event name '%v' found", err.EventName) +} + +// ErrUnsupportedConfiguration - unsupported configuration error. +type ErrUnsupportedConfiguration struct{} + +func (err ErrUnsupportedConfiguration) Error() string { + return "topic or cloud function configuration is not supported" +} + +// ErrDuplicateQueueConfiguration - duplicate queue configuration error. +type ErrDuplicateQueueConfiguration struct { + Queue Queue +} + +func (err ErrDuplicateQueueConfiguration) Error() string { + var message string + if data, xerr := xml.Marshal(err.Queue); xerr != nil { + message = fmt.Sprintf("%+v", err.Queue) + } else { + message = string(data) + } + + return fmt.Sprintf("duplicate queue configuration %v", message) +} + +// ErrUnknownRegion - unknown region error. +type ErrUnknownRegion struct { + Region string +} + +func (err ErrUnknownRegion) Error() string { + return fmt.Sprintf("unknown region '%v'", err.Region) +} + +// ErrARNNotFound - ARN not found error. +type ErrARNNotFound struct { + ARN ARN +} + +func (err ErrARNNotFound) Error() string { + return fmt.Sprintf("ARN '%v' not found", err.ARN) +} + +// ErrInvalidARN - invalid ARN error. +type ErrInvalidARN struct { + ARN string +} + +func (err ErrInvalidARN) Error() string { + return fmt.Sprintf("invalid ARN '%v'", err.ARN) +} + +// ErrInvalidEventName - invalid event name error. +type ErrInvalidEventName struct { + Name string +} + +func (err ErrInvalidEventName) Error() string { + return fmt.Sprintf("invalid event name '%v'", err.Name) +} diff --git a/internal/event/event.go b/internal/event/event.go new file mode 100644 index 00000000..cbffe363 --- /dev/null +++ b/internal/event/event.go @@ -0,0 +1,102 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "github.com/minio/madmin-go/v3" +) + +const ( + // NamespaceFormat - namespace log format used in some event targets. + NamespaceFormat = "namespace" + + // AccessFormat - access log format used in some event targets. + AccessFormat = "access" + + // AMZTimeFormat - event time format. + AMZTimeFormat = "2006-01-02T15:04:05.000Z" + + // StoreExtension - file extension of an event file in store + StoreExtension = ".event" +) + +// Identity represents access key who caused the event. +type Identity struct { + PrincipalID string `json:"principalId"` +} + +// Bucket represents bucket metadata of the event. +type Bucket struct { + Name string `json:"name"` + OwnerIdentity Identity `json:"ownerIdentity"` + ARN string `json:"arn"` +} + +// Object represents object metadata of the event. +type Object struct { + Key string `json:"key"` + Size int64 `json:"size,omitempty"` + ETag string `json:"eTag,omitempty"` + ContentType string `json:"contentType,omitempty"` + UserMetadata map[string]string `json:"userMetadata,omitempty"` + VersionID string `json:"versionId,omitempty"` + Sequencer string `json:"sequencer"` +} + +// Metadata represents event metadata. +type Metadata struct { + SchemaVersion string `json:"s3SchemaVersion"` + ConfigurationID string `json:"configurationId"` + Bucket Bucket `json:"bucket"` + Object Object `json:"object"` +} + +// Source represents client information who triggered the event. +type Source struct { + Host string `json:"host"` + Port string `json:"port"` + UserAgent string `json:"userAgent"` +} + +// Event represents event notification information defined in +// http://docs.aws.amazon.com/AmazonS3/latest/dev/notification-content-structure.html. +type Event struct { + EventVersion string `json:"eventVersion"` + EventSource string `json:"eventSource"` + AwsRegion string `json:"awsRegion"` + EventTime string `json:"eventTime"` + EventName Name `json:"eventName"` + UserIdentity Identity `json:"userIdentity"` + RequestParameters map[string]string `json:"requestParameters"` + ResponseElements map[string]string `json:"responseElements"` + S3 Metadata `json:"s3"` + Source Source `json:"source"` + Type madmin.TraceType `json:"-"` +} + +// Mask returns the type as mask. +func (e Event) Mask() uint64 { + return e.EventName.Mask() +} + +// Log represents event information for some event targets. +type Log struct { + EventName Name + Key string + Records []Event +} diff --git a/internal/event/name.go b/internal/event/name.go new file mode 100644 index 00000000..0dee9cc3 --- /dev/null +++ b/internal/event/name.go @@ -0,0 +1,364 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "encoding/json" + "encoding/xml" +) + +// Name - event type enum. +// Refer http://docs.aws.amazon.com/AmazonS3/latest/dev/NotificationHowTo.html#notification-how-to-event-types-and-destinations +// for most basic values we have since extend this and its not really much applicable other than a reference point. +// "s3:Replication:OperationCompletedReplication" is a MinIO extension. +type Name int + +// Values of event Name +const ( + // Single event types (does not require expansion) + + ObjectAccessedGet Name = 1 + iota + ObjectAccessedGetRetention + ObjectAccessedGetLegalHold + ObjectAccessedHead + ObjectAccessedAttributes + ObjectCreatedCompleteMultipartUpload + ObjectCreatedCopy + ObjectCreatedPost + ObjectCreatedPut + ObjectCreatedPutRetention + ObjectCreatedPutLegalHold + ObjectCreatedPutTagging + ObjectCreatedDeleteTagging + ObjectRemovedDelete + ObjectRemovedDeleteMarkerCreated + ObjectRemovedDeleteAllVersions + ObjectRemovedNoOP + BucketCreated + BucketRemoved + ObjectReplicationFailed + ObjectReplicationComplete + ObjectReplicationMissedThreshold + ObjectReplicationReplicatedAfterThreshold + ObjectReplicationNotTracked + ObjectRestorePost + ObjectRestoreCompleted + ObjectTransitionFailed + ObjectTransitionComplete + ObjectManyVersions + ObjectLargeVersions + PrefixManyFolders + ILMDelMarkerExpirationDelete + + objectSingleTypesEnd + // Start Compound types that require expansion: + + ObjectAccessedAll + ObjectCreatedAll + ObjectRemovedAll + ObjectReplicationAll + ObjectRestoreAll + ObjectTransitionAll + ObjectScannerAll + Everything +) + +// The number of single names should not exceed 64. +// This will break masking. Use bit 63 as extension. +var _ = uint64(1 << objectSingleTypesEnd) + +// Expand - returns expanded values of abbreviated event type. +func (name Name) Expand() []Name { + switch name { + case ObjectAccessedAll: + return []Name{ + ObjectAccessedGet, ObjectAccessedHead, + ObjectAccessedGetRetention, ObjectAccessedGetLegalHold, ObjectAccessedAttributes, + } + case ObjectCreatedAll: + return []Name{ + ObjectCreatedCompleteMultipartUpload, ObjectCreatedCopy, + ObjectCreatedPost, ObjectCreatedPut, + ObjectCreatedPutRetention, ObjectCreatedPutLegalHold, + ObjectCreatedPutTagging, ObjectCreatedDeleteTagging, + } + case ObjectRemovedAll: + return []Name{ + ObjectRemovedDelete, + ObjectRemovedDeleteMarkerCreated, + ObjectRemovedNoOP, + ObjectRemovedDeleteAllVersions, + } + case ObjectReplicationAll: + return []Name{ + ObjectReplicationFailed, + ObjectReplicationComplete, + ObjectReplicationNotTracked, + ObjectReplicationMissedThreshold, + ObjectReplicationReplicatedAfterThreshold, + } + case ObjectRestoreAll: + return []Name{ + ObjectRestorePost, + ObjectRestoreCompleted, + } + case ObjectTransitionAll: + return []Name{ + ObjectTransitionFailed, + ObjectTransitionComplete, + } + case ObjectScannerAll: + return []Name{ + ObjectManyVersions, + ObjectLargeVersions, + PrefixManyFolders, + } + case Everything: + res := make([]Name, objectSingleTypesEnd-1) + for i := range res { + res[i] = Name(i + 1) + } + return res + default: + return []Name{name} + } +} + +// Mask returns the type as mask. +// Compound "All" types are expanded. +func (name Name) Mask() uint64 { + if name < objectSingleTypesEnd { + return 1 << (name - 1) + } + var mask uint64 + for _, n := range name.Expand() { + mask |= 1 << (n - 1) + } + return mask +} + +// String - returns string representation of event type. +func (name Name) String() string { + switch name { + case BucketCreated: + return "s3:BucketCreated:*" + case BucketRemoved: + return "s3:BucketRemoved:*" + case ObjectAccessedAll: + return "s3:ObjectAccessed:*" + case ObjectAccessedGet: + return "s3:ObjectAccessed:Get" + case ObjectAccessedGetRetention: + return "s3:ObjectAccessed:GetRetention" + case ObjectAccessedGetLegalHold: + return "s3:ObjectAccessed:GetLegalHold" + case ObjectAccessedHead: + return "s3:ObjectAccessed:Head" + case ObjectAccessedAttributes: + return "s3:ObjectAccessed:Attributes" + case ObjectCreatedAll: + return "s3:ObjectCreated:*" + case ObjectCreatedCompleteMultipartUpload: + return "s3:ObjectCreated:CompleteMultipartUpload" + case ObjectCreatedCopy: + return "s3:ObjectCreated:Copy" + case ObjectCreatedPost: + return "s3:ObjectCreated:Post" + case ObjectCreatedPut: + return "s3:ObjectCreated:Put" + case ObjectCreatedPutTagging: + return "s3:ObjectCreated:PutTagging" + case ObjectCreatedDeleteTagging: + return "s3:ObjectCreated:DeleteTagging" + case ObjectCreatedPutRetention: + return "s3:ObjectCreated:PutRetention" + case ObjectCreatedPutLegalHold: + return "s3:ObjectCreated:PutLegalHold" + case ObjectRemovedAll: + return "s3:ObjectRemoved:*" + case ObjectRemovedDelete: + return "s3:ObjectRemoved:Delete" + case ObjectRemovedDeleteMarkerCreated: + return "s3:ObjectRemoved:DeleteMarkerCreated" + case ObjectRemovedNoOP: + return "s3:ObjectRemoved:NoOP" + case ObjectRemovedDeleteAllVersions: + return "s3:ObjectRemoved:DeleteAllVersions" + case ILMDelMarkerExpirationDelete: + return "s3:LifecycleDelMarkerExpiration:Delete" + case ObjectReplicationAll: + return "s3:Replication:*" + case ObjectReplicationFailed: + return "s3:Replication:OperationFailedReplication" + case ObjectReplicationComplete: + return "s3:Replication:OperationCompletedReplication" + case ObjectReplicationNotTracked: + return "s3:Replication:OperationNotTracked" + case ObjectReplicationMissedThreshold: + return "s3:Replication:OperationMissedThreshold" + case ObjectReplicationReplicatedAfterThreshold: + return "s3:Replication:OperationReplicatedAfterThreshold" + case ObjectRestoreAll: + return "s3:ObjectRestore:*" + case ObjectRestorePost: + return "s3:ObjectRestore:Post" + case ObjectRestoreCompleted: + return "s3:ObjectRestore:Completed" + case ObjectTransitionAll: + return "s3:ObjectTransition:*" + case ObjectTransitionFailed: + return "s3:ObjectTransition:Failed" + case ObjectTransitionComplete: + return "s3:ObjectTransition:Complete" + case ObjectManyVersions: + return "s3:Scanner:ManyVersions" + case ObjectLargeVersions: + return "s3:Scanner:LargeVersions" + + case PrefixManyFolders: + return "s3:Scanner:BigPrefix" + } + + return "" +} + +// MarshalXML - encodes to XML data. +func (name Name) MarshalXML(e *xml.Encoder, start xml.StartElement) error { + return e.EncodeElement(name.String(), start) +} + +// UnmarshalXML - decodes XML data. +func (name *Name) UnmarshalXML(d *xml.Decoder, start xml.StartElement) error { + var s string + if err := d.DecodeElement(&s, &start); err != nil { + return err + } + + eventName, err := ParseName(s) + if err != nil { + return err + } + + *name = eventName + return nil +} + +// MarshalJSON - encodes to JSON data. +func (name Name) MarshalJSON() ([]byte, error) { + return json.Marshal(name.String()) +} + +// UnmarshalJSON - decodes JSON data. +func (name *Name) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + + eventName, err := ParseName(s) + if err != nil { + return err + } + + *name = eventName + return nil +} + +// ParseName - parses string to Name. +func ParseName(s string) (Name, error) { + switch s { + case "s3:BucketCreated:*": + return BucketCreated, nil + case "s3:BucketRemoved:*": + return BucketRemoved, nil + case "s3:ObjectAccessed:*": + return ObjectAccessedAll, nil + case "s3:ObjectAccessed:Get": + return ObjectAccessedGet, nil + case "s3:ObjectAccessed:GetRetention": + return ObjectAccessedGetRetention, nil + case "s3:ObjectAccessed:GetLegalHold": + return ObjectAccessedGetLegalHold, nil + case "s3:ObjectAccessed:Head": + return ObjectAccessedHead, nil + case "s3:ObjectAccessed:Attributes": + return ObjectAccessedAttributes, nil + case "s3:ObjectCreated:*": + return ObjectCreatedAll, nil + case "s3:ObjectCreated:CompleteMultipartUpload": + return ObjectCreatedCompleteMultipartUpload, nil + case "s3:ObjectCreated:Copy": + return ObjectCreatedCopy, nil + case "s3:ObjectCreated:Post": + return ObjectCreatedPost, nil + case "s3:ObjectCreated:Put": + return ObjectCreatedPut, nil + case "s3:ObjectCreated:PutRetention": + return ObjectCreatedPutRetention, nil + case "s3:ObjectCreated:PutLegalHold": + return ObjectCreatedPutLegalHold, nil + case "s3:ObjectCreated:PutTagging": + return ObjectCreatedPutTagging, nil + case "s3:ObjectCreated:DeleteTagging": + return ObjectCreatedDeleteTagging, nil + case "s3:ObjectRemoved:*": + return ObjectRemovedAll, nil + case "s3:ObjectRemoved:Delete": + return ObjectRemovedDelete, nil + case "s3:ObjectRemoved:DeleteMarkerCreated": + return ObjectRemovedDeleteMarkerCreated, nil + case "s3:ObjectRemoved:NoOP": + return ObjectRemovedNoOP, nil + case "s3:ObjectRemoved:DeleteAllVersions": + return ObjectRemovedDeleteAllVersions, nil + case "s3:LifecycleDelMarkerExpiration:Delete": + return ILMDelMarkerExpirationDelete, nil + case "s3:Replication:*": + return ObjectReplicationAll, nil + case "s3:Replication:OperationFailedReplication": + return ObjectReplicationFailed, nil + case "s3:Replication:OperationCompletedReplication": + return ObjectReplicationComplete, nil + case "s3:Replication:OperationMissedThreshold": + return ObjectReplicationMissedThreshold, nil + case "s3:Replication:OperationReplicatedAfterThreshold": + return ObjectReplicationReplicatedAfterThreshold, nil + case "s3:Replication:OperationNotTracked": + return ObjectReplicationNotTracked, nil + case "s3:ObjectRestore:*": + return ObjectRestoreAll, nil + case "s3:ObjectRestore:Post": + return ObjectRestorePost, nil + case "s3:ObjectRestore:Completed": + return ObjectRestoreCompleted, nil + case "s3:ObjectTransition:Failed": + return ObjectTransitionFailed, nil + case "s3:ObjectTransition:Complete": + return ObjectTransitionComplete, nil + case "s3:ObjectTransition:*": + return ObjectTransitionAll, nil + case "s3:Scanner:ManyVersions": + return ObjectManyVersions, nil + case "s3:Scanner:LargeVersions": + return ObjectLargeVersions, nil + case "s3:Scanner:BigPrefix": + return PrefixManyFolders, nil + default: + return 0, &ErrInvalidEventName{s} + } +} diff --git a/internal/event/name_test.go b/internal/event/name_test.go new file mode 100644 index 00000000..7bafa2ee --- /dev/null +++ b/internal/event/name_test.go @@ -0,0 +1,242 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "encoding/json" + "encoding/xml" + "reflect" + "testing" +) + +func TestNameExpand(t *testing.T) { + testCases := []struct { + name Name + expectedResult []Name + }{ + {BucketCreated, []Name{BucketCreated}}, + {BucketRemoved, []Name{BucketRemoved}}, + {ObjectAccessedAll, []Name{ObjectAccessedGet, ObjectAccessedHead, ObjectAccessedGetRetention, ObjectAccessedGetLegalHold, ObjectAccessedAttributes}}, + {ObjectCreatedAll, []Name{ + ObjectCreatedCompleteMultipartUpload, ObjectCreatedCopy, ObjectCreatedPost, ObjectCreatedPut, + ObjectCreatedPutRetention, ObjectCreatedPutLegalHold, ObjectCreatedPutTagging, ObjectCreatedDeleteTagging, + }}, + {ObjectRemovedAll, []Name{ObjectRemovedDelete, ObjectRemovedDeleteMarkerCreated, ObjectRemovedNoOP, ObjectRemovedDeleteAllVersions}}, + {ObjectAccessedHead, []Name{ObjectAccessedHead}}, + } + + for i, testCase := range testCases { + result := testCase.name.Expand() + + if !reflect.DeepEqual(result, testCase.expectedResult) { + t.Errorf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestNameString(t *testing.T) { + var blankName Name + + testCases := []struct { + name Name + expectedResult string + }{ + {BucketCreated, "s3:BucketCreated:*"}, + {BucketRemoved, "s3:BucketRemoved:*"}, + {ObjectAccessedAll, "s3:ObjectAccessed:*"}, + {ObjectAccessedGet, "s3:ObjectAccessed:Get"}, + {ObjectAccessedHead, "s3:ObjectAccessed:Head"}, + {ObjectCreatedAll, "s3:ObjectCreated:*"}, + {ObjectCreatedCompleteMultipartUpload, "s3:ObjectCreated:CompleteMultipartUpload"}, + {ObjectCreatedCopy, "s3:ObjectCreated:Copy"}, + {ObjectCreatedPost, "s3:ObjectCreated:Post"}, + {ObjectCreatedPut, "s3:ObjectCreated:Put"}, + {ObjectRemovedAll, "s3:ObjectRemoved:*"}, + {ObjectRemovedDelete, "s3:ObjectRemoved:Delete"}, + {ObjectRemovedDeleteAllVersions, "s3:ObjectRemoved:DeleteAllVersions"}, + {ILMDelMarkerExpirationDelete, "s3:LifecycleDelMarkerExpiration:Delete"}, + {ObjectRemovedNoOP, "s3:ObjectRemoved:NoOP"}, + {ObjectCreatedPutRetention, "s3:ObjectCreated:PutRetention"}, + {ObjectCreatedPutLegalHold, "s3:ObjectCreated:PutLegalHold"}, + {ObjectAccessedGetRetention, "s3:ObjectAccessed:GetRetention"}, + {ObjectAccessedGetLegalHold, "s3:ObjectAccessed:GetLegalHold"}, + + {blankName, ""}, + } + + for i, testCase := range testCases { + result := testCase.name.String() + + if result != testCase.expectedResult { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestNameMarshalXML(t *testing.T) { + var blankName Name + + testCases := []struct { + name Name + expectedData []byte + expectErr bool + }{ + {ObjectAccessedAll, []byte("s3:ObjectAccessed:*"), false}, + {ObjectRemovedDelete, []byte("s3:ObjectRemoved:Delete"), false}, + {ObjectRemovedNoOP, []byte("s3:ObjectRemoved:NoOP"), false}, + {blankName, []byte(""), false}, + } + + for i, testCase := range testCases { + data, err := xml.Marshal(testCase.name) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + + if !testCase.expectErr { + if !reflect.DeepEqual(data, testCase.expectedData) { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, string(testCase.expectedData), string(data)) + } + } + } +} + +func TestNameUnmarshalXML(t *testing.T) { + var blankName Name + + testCases := []struct { + data []byte + expectedName Name + expectErr bool + }{ + {[]byte("s3:ObjectAccessed:*"), ObjectAccessedAll, false}, + {[]byte("s3:ObjectRemoved:Delete"), ObjectRemovedDelete, false}, + {[]byte("s3:ObjectRemoved:NoOP"), ObjectRemovedNoOP, false}, + {[]byte(""), blankName, true}, + } + + for i, testCase := range testCases { + var name Name + err := xml.Unmarshal(testCase.data, &name) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + + if !testCase.expectErr { + if !reflect.DeepEqual(name, testCase.expectedName) { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedName, name) + } + } + } +} + +func TestNameMarshalJSON(t *testing.T) { + var blankName Name + + testCases := []struct { + name Name + expectedData []byte + expectErr bool + }{ + {ObjectAccessedAll, []byte(`"s3:ObjectAccessed:*"`), false}, + {ObjectRemovedDelete, []byte(`"s3:ObjectRemoved:Delete"`), false}, + {ObjectRemovedNoOP, []byte(`"s3:ObjectRemoved:NoOP"`), false}, + {blankName, []byte(`""`), false}, + } + + for i, testCase := range testCases { + data, err := json.Marshal(testCase.name) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + + if !testCase.expectErr { + if !reflect.DeepEqual(data, testCase.expectedData) { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, string(testCase.expectedData), string(data)) + } + } + } +} + +func TestNameUnmarshalJSON(t *testing.T) { + var blankName Name + + testCases := []struct { + data []byte + expectedName Name + expectErr bool + }{ + {[]byte(`"s3:ObjectAccessed:*"`), ObjectAccessedAll, false}, + {[]byte(`"s3:ObjectRemoved:Delete"`), ObjectRemovedDelete, false}, + {[]byte(`"s3:ObjectRemoved:NoOP"`), ObjectRemovedNoOP, false}, + {[]byte(`""`), blankName, true}, + } + + for i, testCase := range testCases { + var name Name + err := json.Unmarshal(testCase.data, &name) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + + if !testCase.expectErr { + if !reflect.DeepEqual(name, testCase.expectedName) { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedName, name) + } + } + } +} + +func TestParseName(t *testing.T) { + var blankName Name + + testCases := []struct { + s string + expectedName Name + expectErr bool + }{ + {"s3:ObjectAccessed:*", ObjectAccessedAll, false}, + {"s3:ObjectRemoved:Delete", ObjectRemovedDelete, false}, + {"s3:ObjectRemoved:NoOP", ObjectRemovedNoOP, false}, + {"s3:LifecycleDelMarkerExpiration:Delete", ILMDelMarkerExpirationDelete, false}, + {"", blankName, true}, + } + + for i, testCase := range testCases { + name, err := ParseName(testCase.s) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + + if !testCase.expectErr { + if !reflect.DeepEqual(name, testCase.expectedName) { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedName, name) + } + } + } +} diff --git a/internal/event/rules.go b/internal/event/rules.go new file mode 100644 index 00000000..0218aabc --- /dev/null +++ b/internal/event/rules.go @@ -0,0 +1,113 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "strings" + + "github.com/minio/pkg/v3/wildcard" +) + +// NewPattern - create new pattern for prefix/suffix. +func NewPattern(prefix, suffix string) (pattern string) { + if prefix != "" { + if !strings.HasSuffix(prefix, "*") { + prefix += "*" + } + + pattern = prefix + } + + if suffix != "" { + if !strings.HasPrefix(suffix, "*") { + suffix = "*" + suffix + } + + pattern += suffix + } + + pattern = strings.ReplaceAll(pattern, "**", "*") + + return pattern +} + +// Rules - event rules +type Rules map[string]TargetIDSet + +// Add - adds pattern and target ID. +func (rules Rules) Add(pattern string, targetID TargetID) { + rules[pattern] = NewTargetIDSet(targetID).Union(rules[pattern]) +} + +// MatchSimple - returns true one of the matching object name in rules. +func (rules Rules) MatchSimple(objectName string) bool { + for pattern := range rules { + if wildcard.MatchSimple(pattern, objectName) { + return true + } + } + return false +} + +// Match - returns TargetIDSet matching object name in rules. +func (rules Rules) Match(objectName string) TargetIDSet { + targetIDs := NewTargetIDSet() + + for pattern, targetIDSet := range rules { + if wildcard.MatchSimple(pattern, objectName) { + targetIDs = targetIDs.Union(targetIDSet) + } + } + + return targetIDs +} + +// Clone - returns copy of this rules. +func (rules Rules) Clone() Rules { + rulesCopy := make(Rules) + + for pattern, targetIDSet := range rules { + rulesCopy[pattern] = targetIDSet.Clone() + } + + return rulesCopy +} + +// Union - returns union with given rules as new rules. +func (rules Rules) Union(rules2 Rules) Rules { + nrules := rules.Clone() + + for pattern, targetIDSet := range rules2 { + nrules[pattern] = nrules[pattern].Union(targetIDSet) + } + + return nrules +} + +// Difference - returns difference with given rules as new rules. +func (rules Rules) Difference(rules2 Rules) Rules { + nrules := make(Rules) + + for pattern, targetIDSet := range rules { + if nv := targetIDSet.Difference(rules2[pattern]); len(nv) > 0 { + nrules[pattern] = nv + } + } + + return nrules +} diff --git a/internal/event/rules_test.go b/internal/event/rules_test.go new file mode 100644 index 00000000..128fcccc --- /dev/null +++ b/internal/event/rules_test.go @@ -0,0 +1,276 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "reflect" + "testing" +) + +func TestNewPattern(t *testing.T) { + testCases := []struct { + prefix string + suffix string + expectedResult string + }{ + {"", "", ""}, + {"*", "", "*"}, + {"", "*", "*"}, + {"images/", "", "images/*"}, + {"images/*", "", "images/*"}, + {"", "jpg", "*jpg"}, + {"", "*jpg", "*jpg"}, + {"images/", "jpg", "images/*jpg"}, + {"images/*", "jpg", "images/*jpg"}, + {"images/", "*jpg", "images/*jpg"}, + {"images/*", "*jpg", "images/*jpg"}, + {"201*/images/", "jpg", "201*/images/*jpg"}, + } + + for i, testCase := range testCases { + result := NewPattern(testCase.prefix, testCase.suffix) + + if result != testCase.expectedResult { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestRulesAdd(t *testing.T) { + rulesCase1 := make(Rules) + + rulesCase2 := make(Rules) + rulesCase2.Add(NewPattern("2010*", ""), TargetID{"1", "webhook"}) + + rulesCase3 := make(Rules) + rulesCase3.Add(NewPattern("2010*", ""), TargetID{"1", "webhook"}) + + rulesCase4 := make(Rules) + rulesCase4.Add(NewPattern("", "*.jpg"), TargetID{"1", "webhook"}) + + rulesCase5 := make(Rules) + + rulesCase6 := make(Rules) + rulesCase6.Add(NewPattern("", "*.jpg"), TargetID{"1", "webhook"}) + + rulesCase7 := make(Rules) + rulesCase7.Add(NewPattern("", "*.jpg"), TargetID{"1", "webhook"}) + + rulesCase8 := make(Rules) + rulesCase8.Add(NewPattern("2010*", ""), TargetID{"1", "webhook"}) + + testCases := []struct { + rules Rules + pattern string + targetID TargetID + expectedResult int + }{ + {rulesCase1, NewPattern("*", ""), TargetID{"1", "webhook"}, 1}, + {rulesCase2, NewPattern("*", ""), TargetID{"2", "amqp"}, 2}, + {rulesCase3, NewPattern("2010*", ""), TargetID{"1", "webhook"}, 1}, + {rulesCase4, NewPattern("*", ""), TargetID{"1", "webhook"}, 2}, + {rulesCase5, NewPattern("", "*.jpg"), TargetID{"1", "webhook"}, 1}, + {rulesCase6, NewPattern("", "*"), TargetID{"2", "amqp"}, 2}, + {rulesCase7, NewPattern("", "*.jpg"), TargetID{"1", "webhook"}, 1}, + {rulesCase8, NewPattern("", "*.jpg"), TargetID{"1", "webhook"}, 2}, + } + + for i, testCase := range testCases { + testCase.rules.Add(testCase.pattern, testCase.targetID) + result := len(testCase.rules) + + if result != testCase.expectedResult { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestRulesMatch(t *testing.T) { + rulesCase1 := make(Rules) + + rulesCase2 := make(Rules) + rulesCase2.Add(NewPattern("*", "*"), TargetID{"1", "webhook"}) + + rulesCase3 := make(Rules) + rulesCase3.Add(NewPattern("2010*", ""), TargetID{"1", "webhook"}) + rulesCase3.Add(NewPattern("", "*.png"), TargetID{"2", "amqp"}) + + rulesCase4 := make(Rules) + rulesCase4.Add(NewPattern("2010*", ""), TargetID{"1", "webhook"}) + + testCases := []struct { + rules Rules + objectName string + expectedResult TargetIDSet + }{ + {rulesCase1, "photos.jpg", NewTargetIDSet()}, + {rulesCase2, "photos.jpg", NewTargetIDSet(TargetID{"1", "webhook"})}, + {rulesCase3, "2010/photos.jpg", NewTargetIDSet(TargetID{"1", "webhook"})}, + {rulesCase4, "2000/photos.jpg", NewTargetIDSet()}, + } + + for i, testCase := range testCases { + result := testCase.rules.Match(testCase.objectName) + + if !reflect.DeepEqual(testCase.expectedResult, result) { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestRulesClone(t *testing.T) { + rulesCase1 := make(Rules) + + rulesCase2 := make(Rules) + rulesCase2.Add(NewPattern("2010*", ""), TargetID{"1", "webhook"}) + + rulesCase3 := make(Rules) + rulesCase3.Add(NewPattern("", "*.jpg"), TargetID{"1", "webhook"}) + + testCases := []struct { + rules Rules + prefix string + targetID TargetID + }{ + {rulesCase1, "2010*", TargetID{"1", "webhook"}}, + {rulesCase2, "2000*", TargetID{"2", "amqp"}}, + {rulesCase3, "2010*", TargetID{"1", "webhook"}}, + } + + for i, testCase := range testCases { + result := testCase.rules.Clone() + + if !reflect.DeepEqual(result, testCase.rules) { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.rules, result) + } + + result.Add(NewPattern(testCase.prefix, ""), testCase.targetID) + if reflect.DeepEqual(result, testCase.rules) { + t.Fatalf("test %v: result: expected: not equal, got: equal", i+1) + } + } +} + +func TestRulesUnion(t *testing.T) { + rulesCase1 := make(Rules) + rules2Case1 := make(Rules) + expectedResultCase1 := make(Rules) + + rulesCase2 := make(Rules) + rules2Case2 := make(Rules) + rules2Case2.Add(NewPattern("*", ""), TargetID{"1", "webhook"}) + expectedResultCase2 := make(Rules) + expectedResultCase2.Add(NewPattern("*", ""), TargetID{"1", "webhook"}) + + rulesCase3 := make(Rules) + rulesCase3.Add(NewPattern("", "*"), TargetID{"1", "webhook"}) + rules2Case3 := make(Rules) + expectedResultCase3 := make(Rules) + expectedResultCase3.Add(NewPattern("", "*"), TargetID{"1", "webhook"}) + + rulesCase4 := make(Rules) + rulesCase4.Add(NewPattern("2010*", ""), TargetID{"1", "webhook"}) + rules2Case4 := make(Rules) + rules2Case4.Add(NewPattern("2010*", ""), TargetID{"1", "webhook"}) + expectedResultCase4 := make(Rules) + expectedResultCase4.Add(NewPattern("2010*", ""), TargetID{"1", "webhook"}) + + rulesCase5 := make(Rules) + rulesCase5.Add(NewPattern("2010*", ""), TargetID{"1", "webhook"}) + rulesCase5.Add(NewPattern("", "*.png"), TargetID{"2", "amqp"}) + rules2Case5 := make(Rules) + rules2Case5.Add(NewPattern("*", ""), TargetID{"1", "webhook"}) + expectedResultCase5 := make(Rules) + expectedResultCase5.Add(NewPattern("2010*", ""), TargetID{"1", "webhook"}) + expectedResultCase5.Add(NewPattern("", "*.png"), TargetID{"2", "amqp"}) + expectedResultCase5.Add(NewPattern("*", ""), TargetID{"1", "webhook"}) + + testCases := []struct { + rules Rules + rules2 Rules + expectedResult Rules + }{ + {rulesCase1, rules2Case1, expectedResultCase1}, + {rulesCase2, rules2Case2, expectedResultCase2}, + {rulesCase3, rules2Case3, expectedResultCase3}, + {rulesCase4, rules2Case4, expectedResultCase4}, + {rulesCase5, rules2Case5, expectedResultCase5}, + } + + for i, testCase := range testCases { + result := testCase.rules.Union(testCase.rules2) + + if !reflect.DeepEqual(testCase.expectedResult, result) { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestRulesDifference(t *testing.T) { + rulesCase1 := make(Rules) + rules2Case1 := make(Rules) + expectedResultCase1 := make(Rules) + + rulesCase2 := make(Rules) + rules2Case2 := make(Rules) + rules2Case2.Add(NewPattern("*", "*"), TargetID{"1", "webhook"}) + expectedResultCase2 := make(Rules) + + rulesCase3 := make(Rules) + rulesCase3.Add(NewPattern("*", "*"), TargetID{"1", "webhook"}) + rules2Case3 := make(Rules) + expectedResultCase3 := make(Rules) + expectedResultCase3.Add(NewPattern("*", "*"), TargetID{"1", "webhook"}) + + rulesCase4 := make(Rules) + rulesCase4.Add(NewPattern("*", "*"), TargetID{"1", "webhook"}) + rules2Case4 := make(Rules) + rules2Case4.Add(NewPattern("2010*", ""), TargetID{"1", "webhook"}) + rules2Case4.Add(NewPattern("", "*.png"), TargetID{"2", "amqp"}) + expectedResultCase4 := make(Rules) + expectedResultCase4.Add(NewPattern("*", "*"), TargetID{"1", "webhook"}) + + rulesCase5 := make(Rules) + rulesCase5.Add(NewPattern("*", ""), TargetID{"1", "webhook"}) + rulesCase5.Add(NewPattern("", "*"), TargetID{"2", "amqp"}) + rules2Case5 := make(Rules) + rules2Case5.Add(NewPattern("2010*", ""), TargetID{"1", "webhook"}) + rules2Case5.Add(NewPattern("", "*"), TargetID{"2", "amqp"}) + expectedResultCase5 := make(Rules) + expectedResultCase5.Add(NewPattern("*", ""), TargetID{"1", "webhook"}) + + testCases := []struct { + rules Rules + rules2 Rules + expectedResult Rules + }{ + {rulesCase1, rules2Case1, expectedResultCase1}, + {rulesCase2, rules2Case2, expectedResultCase2}, + {rulesCase3, rules2Case3, expectedResultCase3}, + {rulesCase4, rules2Case4, expectedResultCase4}, + {rulesCase5, rules2Case5, expectedResultCase5}, + } + + for i, testCase := range testCases { + result := testCase.rules.Difference(testCase.rules2) + + if !reflect.DeepEqual(testCase.expectedResult, result) { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} diff --git a/internal/event/rulesmap.go b/internal/event/rulesmap.go new file mode 100644 index 00000000..835d25be --- /dev/null +++ b/internal/event/rulesmap.go @@ -0,0 +1,84 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +// RulesMap - map of rules for every event name. +type RulesMap map[Name]Rules + +// add - adds event names, prefixes, suffixes and target ID to rules map. +func (rulesMap RulesMap) add(eventNames []Name, pattern string, targetID TargetID) { + rules := make(Rules) + rules.Add(pattern, targetID) + + for _, eventName := range eventNames { + for _, name := range eventName.Expand() { + rulesMap[name] = rulesMap[name].Union(rules) + } + } +} + +// Clone - returns copy of this rules map. +func (rulesMap RulesMap) Clone() RulesMap { + rulesMapCopy := make(RulesMap) + + for eventName, rules := range rulesMap { + rulesMapCopy[eventName] = rules.Clone() + } + + return rulesMapCopy +} + +// Add - adds given rules map. +func (rulesMap RulesMap) Add(rulesMap2 RulesMap) { + for eventName, rules := range rulesMap2 { + rulesMap[eventName] = rules.Union(rulesMap[eventName]) + } +} + +// Remove - removes given rules map. +func (rulesMap RulesMap) Remove(rulesMap2 RulesMap) { + for eventName, rules := range rulesMap { + if nr := rules.Difference(rulesMap2[eventName]); len(nr) != 0 { + rulesMap[eventName] = nr + } else { + delete(rulesMap, eventName) + } + } +} + +// MatchSimple - returns true if matching object name and event name in rules map. +func (rulesMap RulesMap) MatchSimple(eventName Name, objectName string) bool { + return rulesMap[eventName].MatchSimple(objectName) +} + +// Match - returns TargetIDSet matching object name and event name in rules map. +func (rulesMap RulesMap) Match(eventName Name, objectName string) TargetIDSet { + return rulesMap[eventName].Match(objectName) +} + +// NewRulesMap - creates new rules map with given values. +func NewRulesMap(eventNames []Name, pattern string, targetID TargetID) RulesMap { + // If pattern is empty, add '*' wildcard to match all. + if pattern == "" { + pattern = "*" + } + + rulesMap := make(RulesMap) + rulesMap.add(eventNames, pattern, targetID) + return rulesMap +} diff --git a/internal/event/rulesmap_test.go b/internal/event/rulesmap_test.go new file mode 100644 index 00000000..e7fb89bd --- /dev/null +++ b/internal/event/rulesmap_test.go @@ -0,0 +1,187 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "reflect" + "testing" +) + +func TestRulesMapClone(t *testing.T) { + rulesMapCase1 := make(RulesMap) + rulesMapToAddCase1 := NewRulesMap([]Name{ObjectCreatedAll}, "*", TargetID{"1", "webhook"}) + + rulesMapCase2 := NewRulesMap([]Name{ObjectCreatedAll}, "*", TargetID{"1", "webhook"}) + rulesMapToAddCase2 := NewRulesMap([]Name{ObjectCreatedAll}, "2010*.jpg", TargetID{"1", "webhook"}) + + rulesMapCase3 := NewRulesMap([]Name{ObjectCreatedAll}, "2010*.jpg", TargetID{"1", "webhook"}) + rulesMapToAddCase3 := NewRulesMap([]Name{ObjectCreatedAll}, "*", TargetID{"1", "webhook"}) + + testCases := []struct { + rulesMap RulesMap + rulesMapToAdd RulesMap + }{ + {rulesMapCase1, rulesMapToAddCase1}, + {rulesMapCase2, rulesMapToAddCase2}, + {rulesMapCase3, rulesMapToAddCase3}, + } + + for i, testCase := range testCases { + result := testCase.rulesMap.Clone() + + if !reflect.DeepEqual(result, testCase.rulesMap) { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.rulesMap, result) + } + + result.Add(testCase.rulesMapToAdd) + if reflect.DeepEqual(result, testCase.rulesMap) { + t.Fatalf("test %v: result: expected: not equal, got: equal", i+1) + } + } +} + +func TestRulesMapAdd(t *testing.T) { + rulesMapCase1 := make(RulesMap) + rulesMapToAddCase1 := make(RulesMap) + expectedResultCase1 := make(RulesMap) + + rulesMapCase2 := make(RulesMap) + rulesMapToAddCase2 := NewRulesMap([]Name{ObjectCreatedAll}, "*", TargetID{"1", "webhook"}) + expectedResultCase2 := NewRulesMap([]Name{ObjectCreatedAll}, "*", TargetID{"1", "webhook"}) + + rulesMapCase3 := NewRulesMap([]Name{ObjectCreatedAll}, "*", TargetID{"1", "webhook"}) + rulesMapToAddCase3 := NewRulesMap([]Name{ObjectCreatedAll}, "2010*.jpg", TargetID{"1", "webhook"}) + expectedResultCase3 := NewRulesMap([]Name{ObjectCreatedAll}, "2010*.jpg", TargetID{"1", "webhook"}) + expectedResultCase3.add([]Name{ObjectCreatedAll}, "*", TargetID{"1", "webhook"}) + + testCases := []struct { + rulesMap RulesMap + rulesMapToAdd RulesMap + expectedResult RulesMap + }{ + {rulesMapCase1, rulesMapToAddCase1, expectedResultCase1}, + {rulesMapCase2, rulesMapToAddCase2, expectedResultCase2}, + {rulesMapCase3, rulesMapToAddCase3, expectedResultCase3}, + } + + for i, testCase := range testCases { + testCase.rulesMap.Add(testCase.rulesMapToAdd) + + if !reflect.DeepEqual(testCase.rulesMap, testCase.expectedResult) { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, testCase.rulesMap) + } + } +} + +func TestRulesMapRemove(t *testing.T) { + rulesMapCase1 := make(RulesMap) + rulesMapToAddCase1 := make(RulesMap) + expectedResultCase1 := make(RulesMap) + + rulesMapCase2 := NewRulesMap([]Name{ObjectCreatedAll}, "*", TargetID{"1", "webhook"}) + rulesMapToAddCase2 := NewRulesMap([]Name{ObjectCreatedAll}, "*", TargetID{"1", "webhook"}) + expectedResultCase2 := make(RulesMap) + + rulesMapCase3 := NewRulesMap([]Name{ObjectCreatedAll}, "2010*.jpg", TargetID{"1", "webhook"}) + rulesMapCase3.add([]Name{ObjectCreatedAll}, "*", TargetID{"1", "webhook"}) + rulesMapToAddCase3 := NewRulesMap([]Name{ObjectCreatedAll}, "2010*.jpg", TargetID{"1", "webhook"}) + expectedResultCase3 := NewRulesMap([]Name{ObjectCreatedAll}, "*", TargetID{"1", "webhook"}) + + testCases := []struct { + rulesMap RulesMap + rulesMapToAdd RulesMap + expectedResult RulesMap + }{ + {rulesMapCase1, rulesMapToAddCase1, expectedResultCase1}, + {rulesMapCase2, rulesMapToAddCase2, expectedResultCase2}, + {rulesMapCase3, rulesMapToAddCase3, expectedResultCase3}, + } + + for i, testCase := range testCases { + testCase.rulesMap.Remove(testCase.rulesMapToAdd) + + if !reflect.DeepEqual(testCase.rulesMap, testCase.expectedResult) { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, testCase.rulesMap) + } + } +} + +func TestRulesMapMatch(t *testing.T) { + rulesMapCase1 := make(RulesMap) + + rulesMapCase2 := NewRulesMap([]Name{ObjectCreatedAll}, "*", TargetID{"1", "webhook"}) + + rulesMapCase3 := NewRulesMap([]Name{ObjectCreatedAll}, "2010*.jpg", TargetID{"1", "webhook"}) + + rulesMapCase4 := NewRulesMap([]Name{ObjectCreatedAll}, "2010*.jpg", TargetID{"1", "webhook"}) + rulesMapCase4.add([]Name{ObjectCreatedAll}, "*", TargetID{"2", "amqp"}) + + testCases := []struct { + rulesMap RulesMap + eventName Name + objectName string + expectedResult TargetIDSet + }{ + {rulesMapCase1, ObjectCreatedPut, "2010/photo.jpg", NewTargetIDSet()}, + {rulesMapCase2, ObjectCreatedPut, "2010/photo.jpg", NewTargetIDSet(TargetID{"1", "webhook"})}, + {rulesMapCase3, ObjectCreatedPut, "2000/photo.png", NewTargetIDSet()}, + {rulesMapCase4, ObjectCreatedPut, "2000/photo.png", NewTargetIDSet(TargetID{"2", "amqp"})}, + } + + for i, testCase := range testCases { + result := testCase.rulesMap.Match(testCase.eventName, testCase.objectName) + + if !reflect.DeepEqual(result, testCase.expectedResult) { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestNewRulesMap(t *testing.T) { + rulesMapCase1 := make(RulesMap) + rulesMapCase1.add([]Name{ObjectAccessedGet, ObjectAccessedHead, ObjectAccessedGetRetention, ObjectAccessedGetLegalHold, ObjectAccessedAttributes}, + "*", TargetID{"1", "webhook"}) + + rulesMapCase2 := make(RulesMap) + rulesMapCase2.add([]Name{ + ObjectAccessedGet, ObjectAccessedHead, ObjectAccessedAttributes, + ObjectCreatedPut, ObjectAccessedGetRetention, ObjectAccessedGetLegalHold, + }, "*", TargetID{"1", "webhook"}) + + rulesMapCase3 := make(RulesMap) + rulesMapCase3.add([]Name{ObjectRemovedDelete}, "2010*.jpg", TargetID{"1", "webhook"}) + + testCases := []struct { + eventNames []Name + pattern string + targetID TargetID + expectedResult RulesMap + }{ + {[]Name{ObjectAccessedAll}, "", TargetID{"1", "webhook"}, rulesMapCase1}, + {[]Name{ObjectAccessedAll, ObjectCreatedPut}, "", TargetID{"1", "webhook"}, rulesMapCase2}, + {[]Name{ObjectRemovedDelete}, "2010*.jpg", TargetID{"1", "webhook"}, rulesMapCase3}, + } + + for i, testCase := range testCases { + result := NewRulesMap(testCase.eventNames, testCase.pattern, testCase.targetID) + + if !reflect.DeepEqual(result, testCase.expectedResult) { + t.Errorf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} diff --git a/internal/event/target/amqp.go b/internal/event/target/amqp.go new file mode 100644 index 00000000..8b001f5d --- /dev/null +++ b/internal/event/target/amqp.go @@ -0,0 +1,376 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package target + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net" + "net/url" + "os" + "path/filepath" + "sync" + + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/once" + "github.com/minio/minio/internal/store" + xnet "github.com/minio/pkg/v3/net" + "github.com/rabbitmq/amqp091-go" +) + +// AMQPArgs - AMQP target arguments. +type AMQPArgs struct { + Enable bool `json:"enable"` + URL amqp091.URI `json:"url"` + Exchange string `json:"exchange"` + RoutingKey string `json:"routingKey"` + ExchangeType string `json:"exchangeType"` + DeliveryMode uint8 `json:"deliveryMode"` + Mandatory bool `json:"mandatory"` + Immediate bool `json:"immediate"` + Durable bool `json:"durable"` + Internal bool `json:"internal"` + NoWait bool `json:"noWait"` + AutoDeleted bool `json:"autoDeleted"` + PublisherConfirms bool `json:"publisherConfirms"` + QueueDir string `json:"queueDir"` + QueueLimit uint64 `json:"queueLimit"` +} + +// AMQP input constants. +// +// ST1003 We cannot change these exported names. +// +//nolint:staticcheck +const ( + AmqpQueueDir = "queue_dir" + AmqpQueueLimit = "queue_limit" + + AmqpURL = "url" + AmqpExchange = "exchange" + AmqpRoutingKey = "routing_key" + AmqpExchangeType = "exchange_type" + AmqpDeliveryMode = "delivery_mode" + AmqpMandatory = "mandatory" + AmqpImmediate = "immediate" + AmqpDurable = "durable" + AmqpInternal = "internal" + AmqpNoWait = "no_wait" + AmqpAutoDeleted = "auto_deleted" + AmqpArguments = "arguments" + AmqpPublisherConfirms = "publisher_confirms" + + EnvAMQPEnable = "MINIO_NOTIFY_AMQP_ENABLE" + EnvAMQPURL = "MINIO_NOTIFY_AMQP_URL" + EnvAMQPExchange = "MINIO_NOTIFY_AMQP_EXCHANGE" + EnvAMQPRoutingKey = "MINIO_NOTIFY_AMQP_ROUTING_KEY" + EnvAMQPExchangeType = "MINIO_NOTIFY_AMQP_EXCHANGE_TYPE" + EnvAMQPDeliveryMode = "MINIO_NOTIFY_AMQP_DELIVERY_MODE" + EnvAMQPMandatory = "MINIO_NOTIFY_AMQP_MANDATORY" + EnvAMQPImmediate = "MINIO_NOTIFY_AMQP_IMMEDIATE" + EnvAMQPDurable = "MINIO_NOTIFY_AMQP_DURABLE" + EnvAMQPInternal = "MINIO_NOTIFY_AMQP_INTERNAL" + EnvAMQPNoWait = "MINIO_NOTIFY_AMQP_NO_WAIT" + EnvAMQPAutoDeleted = "MINIO_NOTIFY_AMQP_AUTO_DELETED" + EnvAMQPArguments = "MINIO_NOTIFY_AMQP_ARGUMENTS" + EnvAMQPPublisherConfirms = "MINIO_NOTIFY_AMQP_PUBLISHING_CONFIRMS" + EnvAMQPQueueDir = "MINIO_NOTIFY_AMQP_QUEUE_DIR" + EnvAMQPQueueLimit = "MINIO_NOTIFY_AMQP_QUEUE_LIMIT" +) + +// Validate AMQP arguments +func (a *AMQPArgs) Validate() error { + if !a.Enable { + return nil + } + if _, err := amqp091.ParseURI(a.URL.String()); err != nil { + return err + } + if a.QueueDir != "" { + if !filepath.IsAbs(a.QueueDir) { + return errors.New("queueDir path should be absolute") + } + } + + return nil +} + +// AMQPTarget - AMQP target +type AMQPTarget struct { + initOnce once.Init + + id event.TargetID + args AMQPArgs + conn *amqp091.Connection + connMutex sync.Mutex + store store.Store[event.Event] + loggerOnce logger.LogOnce + + quitCh chan struct{} +} + +// ID - returns TargetID. +func (target *AMQPTarget) ID() event.TargetID { + return target.id +} + +// Name - returns the Name of the target. +func (target *AMQPTarget) Name() string { + return target.ID().String() +} + +// Store returns any underlying store if set. +func (target *AMQPTarget) Store() event.TargetStore { + return target.store +} + +// IsActive - Return true if target is up and active +func (target *AMQPTarget) IsActive() (bool, error) { + if err := target.init(); err != nil { + return false, err + } + + return target.isActive() +} + +func (target *AMQPTarget) isActive() (bool, error) { + ch, _, err := target.channel() + if err != nil { + return false, err + } + defer func() { + ch.Close() + }() + return true, nil +} + +func (target *AMQPTarget) channel() (*amqp091.Channel, chan amqp091.Confirmation, error) { + var err error + var conn *amqp091.Connection + var ch *amqp091.Channel + + isAMQPClosedErr := func(err error) bool { + if err == amqp091.ErrClosed { + return true + } + + if nerr, ok := err.(*net.OpError); ok { + return (nerr.Err.Error() == "use of closed network connection") + } + + return false + } + + target.connMutex.Lock() + defer target.connMutex.Unlock() + + if target.conn != nil { + ch, err = target.conn.Channel() + if err == nil { + if target.args.PublisherConfirms { + confirms := ch.NotifyPublish(make(chan amqp091.Confirmation, 1)) + if err := ch.Confirm(false); err != nil { + ch.Close() + return nil, nil, err + } + return ch, confirms, nil + } + return ch, nil, nil + } + + if !isAMQPClosedErr(err) { + return nil, nil, err + } + + // close when we know this is a network error. + target.conn.Close() + } + + conn, err = amqp091.Dial(target.args.URL.String()) + if err != nil { + if xnet.IsConnRefusedErr(err) { + return nil, nil, store.ErrNotConnected + } + return nil, nil, err + } + + ch, err = conn.Channel() + if err != nil { + return nil, nil, err + } + + target.conn = conn + + if target.args.PublisherConfirms { + confirms := ch.NotifyPublish(make(chan amqp091.Confirmation, 1)) + if err := ch.Confirm(false); err != nil { + ch.Close() + return nil, nil, err + } + return ch, confirms, nil + } + + return ch, nil, nil +} + +// send - sends an event to the AMQP091. +func (target *AMQPTarget) send(eventData event.Event, ch *amqp091.Channel, confirms chan amqp091.Confirmation) error { + objectName, err := url.QueryUnescape(eventData.S3.Object.Key) + if err != nil { + return err + } + key := eventData.S3.Bucket.Name + "/" + objectName + + data, err := json.Marshal(event.Log{EventName: eventData.EventName, Key: key, Records: []event.Event{eventData}}) + if err != nil { + return err + } + + headers := make(amqp091.Table) + // Add more information here as required, but be aware to not overload headers + headers["minio-bucket"] = eventData.S3.Bucket.Name + headers["minio-event"] = eventData.EventName.String() + + if err = ch.ExchangeDeclare(target.args.Exchange, target.args.ExchangeType, target.args.Durable, + target.args.AutoDeleted, target.args.Internal, target.args.NoWait, nil); err != nil { + return err + } + + if err = ch.Publish(target.args.Exchange, target.args.RoutingKey, target.args.Mandatory, + target.args.Immediate, amqp091.Publishing{ + Headers: headers, + ContentType: "application/json", + DeliveryMode: target.args.DeliveryMode, + Body: data, + }); err != nil { + return err + } + + // check for publisher confirms only if its enabled + if target.args.PublisherConfirms { + confirmed := <-confirms + if !confirmed.Ack { + return fmt.Errorf("failed delivery of delivery tag: %d", confirmed.DeliveryTag) + } + } + + return nil +} + +// Save - saves the events to the store which will be replayed when the amqp connection is active. +func (target *AMQPTarget) Save(eventData event.Event) error { + if target.store != nil { + _, err := target.store.Put(eventData) + return err + } + if err := target.init(); err != nil { + return err + } + ch, confirms, err := target.channel() + if err != nil { + return err + } + defer ch.Close() + + return target.send(eventData, ch, confirms) +} + +// SendFromStore - reads an event from store and sends it to AMQP091. +func (target *AMQPTarget) SendFromStore(key store.Key) error { + if err := target.init(); err != nil { + return err + } + + ch, confirms, err := target.channel() + if err != nil { + return err + } + defer ch.Close() + + eventData, eErr := target.store.Get(key) + if eErr != nil { + // The last event key in a successful batch will be sent in the channel atmost once by the replayEvents() + // Such events will not exist and wouldve been already been sent successfully. + if os.IsNotExist(eErr) { + return nil + } + return eErr + } + + if err := target.send(eventData, ch, confirms); err != nil { + return err + } + + // Delete the event from store. + return target.store.Del(key) +} + +// Close - does nothing and available for interface compatibility. +func (target *AMQPTarget) Close() error { + close(target.quitCh) + if target.conn != nil { + return target.conn.Close() + } + return nil +} + +func (target *AMQPTarget) init() error { + return target.initOnce.Do(target.initAMQP) +} + +func (target *AMQPTarget) initAMQP() error { + conn, err := amqp091.Dial(target.args.URL.String()) + if err != nil { + if xnet.IsConnRefusedErr(err) || xnet.IsConnResetErr(err) { + target.loggerOnce(context.Background(), err, target.ID().String()) + } + return err + } + target.conn = conn + + return nil +} + +// NewAMQPTarget - creates new AMQP target. +func NewAMQPTarget(id string, args AMQPArgs, loggerOnce logger.LogOnce) (*AMQPTarget, error) { + var queueStore store.Store[event.Event] + if args.QueueDir != "" { + queueDir := filepath.Join(args.QueueDir, storePrefix+"-amqp-"+id) + queueStore = store.NewQueueStore[event.Event](queueDir, args.QueueLimit, event.StoreExtension) + if err := queueStore.Open(); err != nil { + return nil, fmt.Errorf("unable to initialize the queue store of AMQP `%s`: %w", id, err) + } + } + + target := &AMQPTarget{ + id: event.TargetID{ID: id, Name: "amqp"}, + args: args, + loggerOnce: loggerOnce, + store: queueStore, + quitCh: make(chan struct{}), + } + + if target.store != nil { + store.StreamItems(target.store, target, target.quitCh, target.loggerOnce) + } + + return target, nil +} diff --git a/internal/event/target/elasticsearch.go b/internal/event/target/elasticsearch.go new file mode 100644 index 00000000..9cdd861b --- /dev/null +++ b/internal/event/target/elasticsearch.go @@ -0,0 +1,584 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package target + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/json" + "fmt" + "net/http" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + elasticsearch7 "github.com/elastic/go-elasticsearch/v7" + "github.com/minio/highwayhash" + "github.com/minio/minio/internal/event" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/once" + "github.com/minio/minio/internal/store" + xnet "github.com/minio/pkg/v3/net" + "github.com/pkg/errors" +) + +// Elastic constants +const ( + ElasticFormat = "format" + ElasticURL = "url" + ElasticIndex = "index" + ElasticQueueDir = "queue_dir" + ElasticQueueLimit = "queue_limit" + ElasticUsername = "username" + ElasticPassword = "password" + + EnvElasticEnable = "MINIO_NOTIFY_ELASTICSEARCH_ENABLE" + EnvElasticFormat = "MINIO_NOTIFY_ELASTICSEARCH_FORMAT" + EnvElasticURL = "MINIO_NOTIFY_ELASTICSEARCH_URL" + EnvElasticIndex = "MINIO_NOTIFY_ELASTICSEARCH_INDEX" + EnvElasticQueueDir = "MINIO_NOTIFY_ELASTICSEARCH_QUEUE_DIR" + EnvElasticQueueLimit = "MINIO_NOTIFY_ELASTICSEARCH_QUEUE_LIMIT" + EnvElasticUsername = "MINIO_NOTIFY_ELASTICSEARCH_USERNAME" + EnvElasticPassword = "MINIO_NOTIFY_ELASTICSEARCH_PASSWORD" +) + +// ESSupportStatus is a typed string representing the support status for +// Elasticsearch +type ESSupportStatus string + +const ( + // ESSUnknown is default value + ESSUnknown ESSupportStatus = "ESSUnknown" + // ESSDeprecated -> support will be removed in future + ESSDeprecated ESSupportStatus = "ESSDeprecated" + // ESSUnsupported -> we won't work with this ES server + ESSUnsupported ESSupportStatus = "ESSUnsupported" + // ESSSupported -> all good! + ESSSupported ESSupportStatus = "ESSSupported" +) + +func getESVersionSupportStatus(version string) (res ESSupportStatus, err error) { + parts := strings.Split(version, ".") + if len(parts) < 1 { + err = fmt.Errorf("bad ES version string: %s", version) + return res, err + } + + majorVersion, err := strconv.Atoi(parts[0]) + if err != nil { + err = fmt.Errorf("bad ES version string: %s", version) + return res, err + } + + switch { + case majorVersion <= 6: + res = ESSUnsupported + default: + res = ESSSupported + } + return res, err +} + +// magic HH-256 key as HH-256 hash of the first 100 decimals of π as utf-8 string with a zero key. +var magicHighwayHash256Key = []byte("\x4b\xe7\x34\xfa\x8e\x23\x8a\xcd\x26\x3e\x83\xe6\xbb\x96\x85\x52\x04\x0f\x93\x5d\xa3\x9f\x44\x14\x97\xe0\x9d\x13\x22\xde\x36\xa0") + +// Interface for elasticsearch client objects +type esClient interface { + isAtleastV7() bool + createIndex(ElasticsearchArgs) error + ping(context.Context, ElasticsearchArgs) (bool, error) + stop() + entryExists(context.Context, string, string) (bool, error) + removeEntry(context.Context, string, string) error + updateEntry(context.Context, string, string, event.Event) error + addEntry(context.Context, string, event.Event) error +} + +// ElasticsearchArgs - Elasticsearch target arguments. +type ElasticsearchArgs struct { + Enable bool `json:"enable"` + Format string `json:"format"` + URL xnet.URL `json:"url"` + Index string `json:"index"` + QueueDir string `json:"queueDir"` + QueueLimit uint64 `json:"queueLimit"` + Transport *http.Transport `json:"-"` + Username string `json:"username"` + Password string `json:"password"` +} + +// Validate ElasticsearchArgs fields +func (a ElasticsearchArgs) Validate() error { + if !a.Enable { + return nil + } + if a.URL.IsEmpty() { + return errors.New("empty URL") + } + if a.Format != "" { + f := strings.ToLower(a.Format) + if f != event.NamespaceFormat && f != event.AccessFormat { + return errors.New("format value unrecognized") + } + } + if a.Index == "" { + return errors.New("empty index value") + } + + if (a.Username == "" && a.Password != "") || (a.Username != "" && a.Password == "") { + return errors.New("username and password should be set in pairs") + } + + return nil +} + +// ElasticsearchTarget - Elasticsearch target. +type ElasticsearchTarget struct { + initOnce once.Init + + id event.TargetID + args ElasticsearchArgs + client esClient + store store.Store[event.Event] + loggerOnce logger.LogOnce + quitCh chan struct{} +} + +// ID - returns target ID. +func (target *ElasticsearchTarget) ID() event.TargetID { + return target.id +} + +// Name - returns the Name of the target. +func (target *ElasticsearchTarget) Name() string { + return target.ID().String() +} + +// Store returns any underlying store if set. +func (target *ElasticsearchTarget) Store() event.TargetStore { + return target.store +} + +// IsActive - Return true if target is up and active +func (target *ElasticsearchTarget) IsActive() (bool, error) { + if err := target.init(); err != nil { + return false, err + } + return target.isActive() +} + +func (target *ElasticsearchTarget) isActive() (bool, error) { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + err := target.checkAndInitClient(ctx) + if err != nil { + return false, err + } + + return target.client.ping(ctx, target.args) +} + +// Save - saves the events to the store if queuestore is configured, which will be replayed when the elasticsearch connection is active. +func (target *ElasticsearchTarget) Save(eventData event.Event) error { + if target.store != nil { + _, err := target.store.Put(eventData) + return err + } + if err := target.init(); err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + err := target.checkAndInitClient(ctx) + if err != nil { + return err + } + + err = target.send(eventData) + if xnet.IsNetworkOrHostDown(err, false) { + return store.ErrNotConnected + } + return err +} + +// send - sends the event to the target. +func (target *ElasticsearchTarget) send(eventData event.Event) error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + if target.args.Format == event.NamespaceFormat { + objectName, err := url.QueryUnescape(eventData.S3.Object.Key) + if err != nil { + return err + } + + // Calculate a hash of the key for the id of the ES document. + // Id's are limited to 512 bytes in V7+, so we need to do this. + var keyHash string + { + key := eventData.S3.Bucket.Name + "/" + objectName + if target.client.isAtleastV7() { + hh, _ := highwayhash.New(magicHighwayHash256Key) // New will never return error since key is 256 bit + hh.Write([]byte(key)) + hashBytes := hh.Sum(nil) + keyHash = base64.URLEncoding.EncodeToString(hashBytes) + } else { + keyHash = key + } + } + + if eventData.EventName == event.ObjectRemovedDelete { + err = target.client.removeEntry(ctx, target.args.Index, keyHash) + } else { + err = target.client.updateEntry(ctx, target.args.Index, keyHash, eventData) + } + return err + } + + if target.args.Format == event.AccessFormat { + return target.client.addEntry(ctx, target.args.Index, eventData) + } + + return nil +} + +// SendFromStore - reads an event from store and sends it to Elasticsearch. +func (target *ElasticsearchTarget) SendFromStore(key store.Key) error { + if err := target.init(); err != nil { + return err + } + + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + err := target.checkAndInitClient(ctx) + if err != nil { + return err + } + + eventData, eErr := target.store.Get(key) + if eErr != nil { + // The last event key in a successful batch will be sent in the channel atmost once by the replayEvents() + // Such events will not exist and wouldve been already been sent successfully. + if os.IsNotExist(eErr) { + return nil + } + return eErr + } + + if err := target.send(eventData); err != nil { + if xnet.IsNetworkOrHostDown(err, false) { + return store.ErrNotConnected + } + return err + } + + // Delete the event from store. + return target.store.Del(key) +} + +// Close - does nothing and available for interface compatibility. +func (target *ElasticsearchTarget) Close() error { + close(target.quitCh) + if target.client != nil { + // Stops the background processes that the client is running. + target.client.stop() + } + return nil +} + +func (target *ElasticsearchTarget) checkAndInitClient(ctx context.Context) error { + if target.client != nil { + return nil + } + + clientV7, err := newClientV7(target.args) + if err != nil { + return err + } + + // Check es version to confirm if it is supported. + serverSupportStatus, version, err := clientV7.getServerSupportStatus(ctx) + if err != nil { + return err + } + + switch serverSupportStatus { + case ESSUnknown: + return errors.New("unable to determine support status of ES (should not happen)") + + case ESSDeprecated: + return errors.New("there is no currently deprecated version of ES in MinIO") + + case ESSSupported: + target.client = clientV7 + + default: + // ESSUnsupported case + return fmt.Errorf("Elasticsearch version '%s' is not supported! Please use at least version 7.x.", version) + } + + target.client.createIndex(target.args) + return nil +} + +func (target *ElasticsearchTarget) init() error { + return target.initOnce.Do(target.initElasticsearch) +} + +func (target *ElasticsearchTarget) initElasticsearch() error { + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + defer cancel() + + err := target.checkAndInitClient(ctx) + if err != nil { + if err != store.ErrNotConnected { + target.loggerOnce(context.Background(), err, target.ID().String()) + } + return err + } + + return nil +} + +// NewElasticsearchTarget - creates new Elasticsearch target. +func NewElasticsearchTarget(id string, args ElasticsearchArgs, loggerOnce logger.LogOnce) (*ElasticsearchTarget, error) { + var queueStore store.Store[event.Event] + if args.QueueDir != "" { + queueDir := filepath.Join(args.QueueDir, storePrefix+"-elasticsearch-"+id) + queueStore = store.NewQueueStore[event.Event](queueDir, args.QueueLimit, event.StoreExtension) + if err := queueStore.Open(); err != nil { + return nil, fmt.Errorf("unable to initialize the queue store of Elasticsearch `%s`: %w", id, err) + } + } + + target := &ElasticsearchTarget{ + id: event.TargetID{ID: id, Name: "elasticsearch"}, + args: args, + store: queueStore, + loggerOnce: loggerOnce, + quitCh: make(chan struct{}), + } + + if target.store != nil { + store.StreamItems(target.store, target, target.quitCh, target.loggerOnce) + } + + return target, nil +} + +// ES Client definitions and methods + +type esClientV7 struct { + *elasticsearch7.Client +} + +func newClientV7(args ElasticsearchArgs) (*esClientV7, error) { + // Client options + elasticConfig := elasticsearch7.Config{ + Addresses: []string{args.URL.String()}, + Transport: args.Transport, + MaxRetries: 10, + } + // Set basic auth + if args.Username != "" && args.Password != "" { + elasticConfig.Username = args.Username + elasticConfig.Password = args.Password + } + // Create a client + client, err := elasticsearch7.NewClient(elasticConfig) + if err != nil { + return nil, err + } + clientV7 := &esClientV7{client} + return clientV7, nil +} + +func (c *esClientV7) getServerSupportStatus(ctx context.Context) (ESSupportStatus, string, error) { + resp, err := c.Info( + c.Info.WithContext(ctx), + ) + if err != nil { + return ESSUnknown, "", store.ErrNotConnected + } + + defer resp.Body.Close() + + m := make(map[string]any) + err = json.NewDecoder(resp.Body).Decode(&m) + if err != nil { + return ESSUnknown, "", fmt.Errorf("unable to get ES Server version - json parse error: %v", err) + } + + if v, ok := m["version"].(map[string]any); ok { + if ver, ok := v["number"].(string); ok { + status, err := getESVersionSupportStatus(ver) + return status, ver, err + } + } + return ESSUnknown, "", fmt.Errorf("Unable to get ES Server Version - got INFO response: %v", m) +} + +func (c *esClientV7) isAtleastV7() bool { + return true +} + +// createIndex - creates the index if it does not exist. +func (c *esClientV7) createIndex(args ElasticsearchArgs) error { + res, err := c.Indices.ResolveIndex([]string{args.Index}) + if err != nil { + return err + } + defer res.Body.Close() + + var v map[string]any + found := false + if err := json.NewDecoder(res.Body).Decode(&v); err != nil { + return fmt.Errorf("Error parsing response body: %v", err) + } + + indices, ok := v["indices"].([]any) + if ok { + for _, index := range indices { + if name, ok := index.(map[string]any); ok && name["name"] == args.Index { + found = true + break + } + } + } + + if !found { + resp, err := c.Indices.Create(args.Index) + if err != nil { + return err + } + defer xhttp.DrainBody(resp.Body) + if resp.IsError() { + return fmt.Errorf("Create index err: %v", res) + } + return nil + } + return nil +} + +func (c *esClientV7) ping(ctx context.Context, _ ElasticsearchArgs) (bool, error) { + resp, err := c.Ping( + c.Ping.WithContext(ctx), + ) + if err != nil { + return false, store.ErrNotConnected + } + xhttp.DrainBody(resp.Body) + return !resp.IsError(), nil +} + +func (c *esClientV7) entryExists(ctx context.Context, index string, key string) (bool, error) { + res, err := c.Exists( + index, + key, + c.Exists.WithContext(ctx), + ) + if err != nil { + return false, err + } + xhttp.DrainBody(res.Body) + return !res.IsError(), nil +} + +func (c *esClientV7) removeEntry(ctx context.Context, index string, key string) error { + exists, err := c.entryExists(ctx, index, key) + if err == nil && exists { + res, err := c.Delete( + index, + key, + c.Delete.WithContext(ctx), + ) + if err != nil { + return err + } + defer xhttp.DrainBody(res.Body) + if res.IsError() { + return fmt.Errorf("Delete err: %s", res.String()) + } + return nil + } + return err +} + +func (c *esClientV7) updateEntry(ctx context.Context, index string, key string, eventData event.Event) error { + doc := map[string]any{ + "Records": []event.Event{eventData}, + } + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + err := enc.Encode(doc) + if err != nil { + return err + } + res, err := c.Index( + index, + &buf, + c.Index.WithDocumentID(key), + c.Index.WithContext(ctx), + ) + if err != nil { + return err + } + defer xhttp.DrainBody(res.Body) + if res.IsError() { + return fmt.Errorf("Update err: %s", res.String()) + } + + return nil +} + +func (c *esClientV7) addEntry(ctx context.Context, index string, eventData event.Event) error { + doc := map[string]any{ + "Records": []event.Event{eventData}, + } + var buf bytes.Buffer + enc := json.NewEncoder(&buf) + err := enc.Encode(doc) + if err != nil { + return err + } + res, err := c.Index( + index, + &buf, + c.Index.WithContext(ctx), + ) + if err != nil { + return err + } + defer xhttp.DrainBody(res.Body) + if res.IsError() { + return fmt.Errorf("Add err: %s", res.String()) + } + return nil +} + +func (c *esClientV7) stop() { +} diff --git a/internal/event/target/kafka.go b/internal/event/target/kafka.go new file mode 100644 index 00000000..d6af69b8 --- /dev/null +++ b/internal/event/target/kafka.go @@ -0,0 +1,465 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package target + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "log" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/once" + "github.com/minio/minio/internal/store" + xnet "github.com/minio/pkg/v3/net" + + "github.com/IBM/sarama" + saramatls "github.com/IBM/sarama/tools/tls" +) + +// Kafka input constants +const ( + KafkaBrokers = "brokers" + KafkaTopic = "topic" + KafkaQueueDir = "queue_dir" + KafkaQueueLimit = "queue_limit" + KafkaTLS = "tls" + KafkaTLSSkipVerify = "tls_skip_verify" + KafkaTLSClientAuth = "tls_client_auth" + KafkaSASL = "sasl" + KafkaSASLUsername = "sasl_username" + KafkaSASLPassword = "sasl_password" + KafkaSASLMechanism = "sasl_mechanism" + KafkaClientTLSCert = "client_tls_cert" + KafkaClientTLSKey = "client_tls_key" + KafkaVersion = "version" + KafkaBatchSize = "batch_size" + KafkaBatchCommitTimeout = "batch_commit_timeout" + KafkaCompressionCodec = "compression_codec" + KafkaCompressionLevel = "compression_level" + + EnvKafkaEnable = "MINIO_NOTIFY_KAFKA_ENABLE" + EnvKafkaBrokers = "MINIO_NOTIFY_KAFKA_BROKERS" + EnvKafkaTopic = "MINIO_NOTIFY_KAFKA_TOPIC" + EnvKafkaQueueDir = "MINIO_NOTIFY_KAFKA_QUEUE_DIR" + EnvKafkaQueueLimit = "MINIO_NOTIFY_KAFKA_QUEUE_LIMIT" + EnvKafkaTLS = "MINIO_NOTIFY_KAFKA_TLS" + EnvKafkaTLSSkipVerify = "MINIO_NOTIFY_KAFKA_TLS_SKIP_VERIFY" + EnvKafkaTLSClientAuth = "MINIO_NOTIFY_KAFKA_TLS_CLIENT_AUTH" + EnvKafkaSASLEnable = "MINIO_NOTIFY_KAFKA_SASL" + EnvKafkaSASLUsername = "MINIO_NOTIFY_KAFKA_SASL_USERNAME" + EnvKafkaSASLPassword = "MINIO_NOTIFY_KAFKA_SASL_PASSWORD" + EnvKafkaSASLMechanism = "MINIO_NOTIFY_KAFKA_SASL_MECHANISM" + EnvKafkaClientTLSCert = "MINIO_NOTIFY_KAFKA_CLIENT_TLS_CERT" + EnvKafkaClientTLSKey = "MINIO_NOTIFY_KAFKA_CLIENT_TLS_KEY" + EnvKafkaVersion = "MINIO_NOTIFY_KAFKA_VERSION" + EnvKafkaBatchSize = "MINIO_NOTIFY_KAFKA_BATCH_SIZE" + EnvKafkaBatchCommitTimeout = "MINIO_NOTIFY_KAFKA_BATCH_COMMIT_TIMEOUT" + EnvKafkaProducerCompressionCodec = "MINIO_NOTIFY_KAFKA_PRODUCER_COMPRESSION_CODEC" + EnvKafkaProducerCompressionLevel = "MINIO_NOTIFY_KAFKA_PRODUCER_COMPRESSION_LEVEL" +) + +var codecs = map[string]sarama.CompressionCodec{ + "none": sarama.CompressionNone, + "gzip": sarama.CompressionGZIP, + "snappy": sarama.CompressionSnappy, + "lz4": sarama.CompressionLZ4, + "zstd": sarama.CompressionZSTD, +} + +// KafkaArgs - Kafka target arguments. +type KafkaArgs struct { + Enable bool `json:"enable"` + Brokers []xnet.Host `json:"brokers"` + Topic string `json:"topic"` + QueueDir string `json:"queueDir"` + QueueLimit uint64 `json:"queueLimit"` + Version string `json:"version"` + BatchSize uint32 `json:"batchSize"` + BatchCommitTimeout time.Duration `json:"batchCommitTimeout"` + TLS struct { + Enable bool `json:"enable"` + RootCAs *x509.CertPool `json:"-"` + SkipVerify bool `json:"skipVerify"` + ClientAuth tls.ClientAuthType `json:"clientAuth"` + ClientTLSCert string `json:"clientTLSCert"` + ClientTLSKey string `json:"clientTLSKey"` + } `json:"tls"` + SASL struct { + Enable bool `json:"enable"` + User string `json:"username"` + Password string `json:"password"` + Mechanism string `json:"mechanism"` + } `json:"sasl"` + Producer struct { + Compression string `json:"compression"` + CompressionLevel int `json:"compressionLevel"` + } `json:"producer"` +} + +// Validate KafkaArgs fields +func (k KafkaArgs) Validate() error { + if !k.Enable { + return nil + } + if len(k.Brokers) == 0 { + return errors.New("no broker address found") + } + for _, b := range k.Brokers { + if _, err := xnet.ParseHost(b.String()); err != nil { + return err + } + } + if k.QueueDir != "" { + if !filepath.IsAbs(k.QueueDir) { + return errors.New("queueDir path should be absolute") + } + } + if k.Version != "" { + if _, err := sarama.ParseKafkaVersion(k.Version); err != nil { + return err + } + } + if k.BatchSize > 1 { + if k.QueueDir == "" { + return errors.New("batch should be enabled only if queue dir is enabled") + } + } + if k.BatchCommitTimeout > 0 { + if k.QueueDir == "" || k.BatchSize <= 1 { + return errors.New("batch commit timeout should be set only if queue dir is enabled and batch size > 1") + } + } + return nil +} + +// KafkaTarget - Kafka target. +type KafkaTarget struct { + initOnce once.Init + + id event.TargetID + args KafkaArgs + client sarama.Client + producer sarama.SyncProducer + config *sarama.Config + store store.Store[event.Event] + batch *store.Batch[event.Event] + loggerOnce logger.LogOnce + quitCh chan struct{} +} + +// ID - returns target ID. +func (target *KafkaTarget) ID() event.TargetID { + return target.id +} + +// Name - returns the Name of the target. +func (target *KafkaTarget) Name() string { + return target.ID().String() +} + +// Store returns any underlying store if set. +func (target *KafkaTarget) Store() event.TargetStore { + return target.store +} + +// IsActive - Return true if target is up and active +func (target *KafkaTarget) IsActive() (bool, error) { + if err := target.init(); err != nil { + return false, err + } + return target.isActive() +} + +func (target *KafkaTarget) isActive() (bool, error) { + // Refer https://github.com/IBM/sarama/issues/1341 + brokers := target.client.Brokers() + if len(brokers) == 0 { + return false, store.ErrNotConnected + } + return true, nil +} + +// Save - saves the events to the store which will be replayed when the Kafka connection is active. +func (target *KafkaTarget) Save(eventData event.Event) error { + if target.store != nil { + if target.batch != nil { + return target.batch.Add(eventData) + } + _, err := target.store.Put(eventData) + return err + } + if err := target.init(); err != nil { + return err + } + return target.send(eventData) +} + +// send - sends an event to the kafka. +func (target *KafkaTarget) send(eventData event.Event) error { + if target.producer == nil { + return store.ErrNotConnected + } + msg, err := target.toProducerMessage(eventData) + if err != nil { + return err + } + _, _, err = target.producer.SendMessage(msg) + return err +} + +// sendMultiple sends multiple messages to the kafka. +func (target *KafkaTarget) sendMultiple(events []event.Event) error { + if target.producer == nil { + return store.ErrNotConnected + } + var msgs []*sarama.ProducerMessage + for _, event := range events { + msg, err := target.toProducerMessage(event) + if err != nil { + return err + } + msgs = append(msgs, msg) + } + return target.producer.SendMessages(msgs) +} + +// SendFromStore - reads an event from store and sends it to Kafka. +func (target *KafkaTarget) SendFromStore(key store.Key) (err error) { + if err = target.init(); err != nil { + return err + } + switch { + case key.ItemCount == 1: + var event event.Event + event, err = target.store.Get(key) + if err != nil { + // The last event key in a successful batch will be sent in the channel atmost once by the replayEvents() + // Such events will not exist and wouldve been already been sent successfully. + if os.IsNotExist(err) { + return nil + } + return err + } + err = target.send(event) + case key.ItemCount > 1: + var events []event.Event + events, err = target.store.GetMultiple(key) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return err + } + err = target.sendMultiple(events) + } + if err != nil { + if isKafkaConnErr(err) { + return store.ErrNotConnected + } + return err + } + // Delete the event from store. + return target.store.Del(key) +} + +func (target *KafkaTarget) toProducerMessage(eventData event.Event) (*sarama.ProducerMessage, error) { + objectName, err := url.QueryUnescape(eventData.S3.Object.Key) + if err != nil { + return nil, err + } + + key := eventData.S3.Bucket.Name + "/" + objectName + data, err := json.Marshal(event.Log{EventName: eventData.EventName, Key: key, Records: []event.Event{eventData}}) + if err != nil { + return nil, err + } + + return &sarama.ProducerMessage{ + Topic: target.args.Topic, + Key: sarama.StringEncoder(key), + Value: sarama.ByteEncoder(data), + }, nil +} + +// Close - closes underneath kafka connection. +func (target *KafkaTarget) Close() error { + close(target.quitCh) + + if target.batch != nil { + target.batch.Close() + } + + if target.producer != nil { + if target.store != nil { + // It is safe to abort the current transaction if + // queue_dir is configured + target.producer.AbortTxn() + } else { + target.producer.CommitTxn() + } + target.producer.Close() + return target.client.Close() + } + + return nil +} + +func (target *KafkaTarget) init() error { + return target.initOnce.Do(target.initKafka) +} + +func (target *KafkaTarget) initKafka() error { + if os.Getenv("_MINIO_KAFKA_DEBUG") != "" { + sarama.DebugLogger = log.Default() + } + + args := target.args + + config := sarama.NewConfig() + if args.Version != "" { + kafkaVersion, err := sarama.ParseKafkaVersion(args.Version) + if err != nil { + target.loggerOnce(context.Background(), err, target.ID().String()) + return err + } + config.Version = kafkaVersion + } + + config.Net.KeepAlive = 60 * time.Second + config.Net.SASL.User = args.SASL.User + config.Net.SASL.Password = args.SASL.Password + initScramClient(args, config) // initializes configured scram client. + config.Net.SASL.Enable = args.SASL.Enable + + tlsConfig, err := saramatls.NewConfig(args.TLS.ClientTLSCert, args.TLS.ClientTLSKey) + if err != nil { + target.loggerOnce(context.Background(), err, target.ID().String()) + return err + } + + config.Net.TLS.Enable = args.TLS.Enable + config.Net.TLS.Config = tlsConfig + config.Net.TLS.Config.InsecureSkipVerify = args.TLS.SkipVerify + config.Net.TLS.Config.ClientAuth = args.TLS.ClientAuth + config.Net.TLS.Config.RootCAs = args.TLS.RootCAs + + // These settings are needed to ensure that kafka client doesn't hang on brokers + // refer https://github.com/IBM/sarama/issues/765#issuecomment-254333355 + config.Producer.Retry.Max = 2 + config.Producer.Retry.Backoff = (1 * time.Second) + config.Producer.Return.Successes = true + config.Producer.Return.Errors = true + config.Producer.RequiredAcks = 1 + config.Producer.Timeout = (5 * time.Second) + // Set Producer Compression + cc, ok := codecs[strings.ToLower(args.Producer.Compression)] + if ok { + config.Producer.Compression = cc + config.Producer.CompressionLevel = args.Producer.CompressionLevel + } + + config.Net.ReadTimeout = (5 * time.Second) + config.Net.DialTimeout = (5 * time.Second) + config.Net.WriteTimeout = (5 * time.Second) + config.Metadata.Retry.Max = 1 + config.Metadata.Retry.Backoff = (1 * time.Second) + config.Metadata.RefreshFrequency = (15 * time.Minute) + + target.config = config + + brokers := []string{} + for _, broker := range args.Brokers { + brokers = append(brokers, broker.String()) + } + + client, err := sarama.NewClient(brokers, config) + if err != nil { + if !errors.Is(err, sarama.ErrOutOfBrokers) { + target.loggerOnce(context.Background(), err, target.ID().String()) + } + return err + } + + producer, err := sarama.NewSyncProducerFromClient(client) + if err != nil { + if !errors.Is(err, sarama.ErrOutOfBrokers) { + target.loggerOnce(context.Background(), err, target.ID().String()) + } + return err + } + target.client = client + target.producer = producer + + yes, err := target.isActive() + if err != nil { + return err + } + if !yes { + return store.ErrNotConnected + } + + return nil +} + +// NewKafkaTarget - creates new Kafka target with auth credentials. +func NewKafkaTarget(id string, args KafkaArgs, loggerOnce logger.LogOnce) (*KafkaTarget, error) { + var queueStore store.Store[event.Event] + if args.QueueDir != "" { + queueDir := filepath.Join(args.QueueDir, storePrefix+"-kafka-"+id) + queueStore = store.NewQueueStore[event.Event](queueDir, args.QueueLimit, event.StoreExtension) + if err := queueStore.Open(); err != nil { + return nil, fmt.Errorf("unable to initialize the queue store of Kafka `%s`: %w", id, err) + } + } + + target := &KafkaTarget{ + id: event.TargetID{ID: id, Name: "kafka"}, + args: args, + store: queueStore, + loggerOnce: loggerOnce, + quitCh: make(chan struct{}), + } + if target.store != nil { + if args.BatchSize > 1 { + target.batch = store.NewBatch[event.Event](store.BatchConfig[event.Event]{ + Limit: args.BatchSize, + Log: loggerOnce, + Store: queueStore, + CommitTimeout: args.BatchCommitTimeout, + }) + } + store.StreamItems(target.store, target, target.quitCh, target.loggerOnce) + } + + return target, nil +} + +func isKafkaConnErr(err error) bool { + // Sarama opens the circuit breaker after 3 consecutive connection failures. + return err == sarama.ErrLeaderNotAvailable || err.Error() == "circuit breaker is open" +} diff --git a/internal/event/target/kafka_scram_client_contrib.go b/internal/event/target/kafka_scram_client_contrib.go new file mode 100644 index 00000000..a6027208 --- /dev/null +++ b/internal/event/target/kafka_scram_client_contrib.go @@ -0,0 +1,86 @@ +/* + * MinIO Object Storage (c) 2021-2023 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package target + +import ( + "crypto/sha512" + "strings" + + "github.com/IBM/sarama" + "github.com/xdg/scram" + + "github.com/minio/minio/internal/hash/sha256" +) + +func initScramClient(args KafkaArgs, config *sarama.Config) { + switch strings.ToLower(args.SASL.Mechanism) { + case "sha512": + config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: KafkaSHA512} } + config.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypeSCRAMSHA512) + case "sha256": + config.Net.SASL.SCRAMClientGeneratorFunc = func() sarama.SCRAMClient { return &XDGSCRAMClient{HashGeneratorFcn: KafkaSHA256} } + config.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypeSCRAMSHA256) + default: + // default to PLAIN + config.Net.SASL.Mechanism = sarama.SASLMechanism(sarama.SASLTypePlaintext) + } +} + +// KafkaSHA256 is a function that returns a crypto/sha256 hasher and should be used +// to create Client objects configured for SHA-256 hashing. +var KafkaSHA256 scram.HashGeneratorFcn = sha256.New + +// KafkaSHA512 is a function that returns a crypto/sha512 hasher and should be used +// to create Client objects configured for SHA-512 hashing. +var KafkaSHA512 scram.HashGeneratorFcn = sha512.New + +// XDGSCRAMClient implements the client-side of an authentication +// conversation with a server. A new conversation must be created for +// each authentication attempt. +type XDGSCRAMClient struct { + *scram.Client + *scram.ClientConversation + scram.HashGeneratorFcn +} + +// Begin constructs a SCRAM client component based on a given hash.Hash +// factory receiver. This constructor will normalize the username, password +// and authzID via the SASLprep algorithm, as recommended by RFC-5802. If +// SASLprep fails, the method returns an error. +func (x *XDGSCRAMClient) Begin(userName, password, authzID string) (err error) { + x.Client, err = x.NewClient(userName, password, authzID) + if err != nil { + return err + } + x.ClientConversation = x.NewConversation() + return nil +} + +// Step takes a string provided from a server (or just an empty string for the +// very first conversation step) and attempts to move the authentication +// conversation forward. It returns a string to be sent to the server or an +// error if the server message is invalid. Calling Step after a conversation +// completes is also an error. +func (x *XDGSCRAMClient) Step(challenge string) (response string, err error) { + response, err = x.ClientConversation.Step(challenge) + return response, err +} + +// Done returns true if the conversation is completed or has errored. +func (x *XDGSCRAMClient) Done() bool { + return x.ClientConversation.Done() +} diff --git a/internal/event/target/mqtt.go b/internal/event/target/mqtt.go new file mode 100644 index 00000000..8f568cd3 --- /dev/null +++ b/internal/event/target/mqtt.go @@ -0,0 +1,308 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package target + +import ( + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "time" + + mqtt "github.com/eclipse/paho.mqtt.golang" + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/once" + "github.com/minio/minio/internal/store" + xnet "github.com/minio/pkg/v3/net" +) + +const ( + reconnectInterval = 5 * time.Second + storePrefix = "minio" +) + +// MQTT input constants +const ( + MqttBroker = "broker" + MqttTopic = "topic" + MqttQoS = "qos" + MqttUsername = "username" + MqttPassword = "password" + MqttReconnectInterval = "reconnect_interval" + MqttKeepAliveInterval = "keep_alive_interval" + MqttQueueDir = "queue_dir" + MqttQueueLimit = "queue_limit" + + EnvMQTTEnable = "MINIO_NOTIFY_MQTT_ENABLE" + EnvMQTTBroker = "MINIO_NOTIFY_MQTT_BROKER" + EnvMQTTTopic = "MINIO_NOTIFY_MQTT_TOPIC" + EnvMQTTQoS = "MINIO_NOTIFY_MQTT_QOS" + EnvMQTTUsername = "MINIO_NOTIFY_MQTT_USERNAME" + EnvMQTTPassword = "MINIO_NOTIFY_MQTT_PASSWORD" + EnvMQTTReconnectInterval = "MINIO_NOTIFY_MQTT_RECONNECT_INTERVAL" + EnvMQTTKeepAliveInterval = "MINIO_NOTIFY_MQTT_KEEP_ALIVE_INTERVAL" + EnvMQTTQueueDir = "MINIO_NOTIFY_MQTT_QUEUE_DIR" + EnvMQTTQueueLimit = "MINIO_NOTIFY_MQTT_QUEUE_LIMIT" +) + +// MQTTArgs - MQTT target arguments. +type MQTTArgs struct { + Enable bool `json:"enable"` + Broker xnet.URL `json:"broker"` + Topic string `json:"topic"` + QoS byte `json:"qos"` + User string `json:"username"` + Password string `json:"password"` + MaxReconnectInterval time.Duration `json:"reconnectInterval"` + KeepAlive time.Duration `json:"keepAliveInterval"` + RootCAs *x509.CertPool `json:"-"` + QueueDir string `json:"queueDir"` + QueueLimit uint64 `json:"queueLimit"` +} + +// Validate MQTTArgs fields +func (m MQTTArgs) Validate() error { + if !m.Enable { + return nil + } + u, err := xnet.ParseURL(m.Broker.String()) + if err != nil { + return err + } + switch u.Scheme { + case "ws", "wss", "tcp", "ssl", "tls", "tcps": + default: + return errors.New("unknown protocol in broker address") + } + if m.QueueDir != "" { + if !filepath.IsAbs(m.QueueDir) { + return errors.New("queueDir path should be absolute") + } + if m.QoS == 0 { + return errors.New("qos should be set to 1 or 2 if queueDir is set") + } + } + + return nil +} + +// MQTTTarget - MQTT target. +type MQTTTarget struct { + initOnce once.Init + + id event.TargetID + args MQTTArgs + client mqtt.Client + store store.Store[event.Event] + quitCh chan struct{} + loggerOnce logger.LogOnce +} + +// ID - returns target ID. +func (target *MQTTTarget) ID() event.TargetID { + return target.id +} + +// Name - returns the Name of the target. +func (target *MQTTTarget) Name() string { + return target.ID().String() +} + +// Store returns any underlying store if set. +func (target *MQTTTarget) Store() event.TargetStore { + return target.store +} + +// IsActive - Return true if target is up and active +func (target *MQTTTarget) IsActive() (bool, error) { + if err := target.init(); err != nil { + return false, err + } + return target.isActive() +} + +func (target *MQTTTarget) isActive() (bool, error) { + if !target.client.IsConnectionOpen() { + return false, store.ErrNotConnected + } + return true, nil +} + +// send - sends an event to the mqtt. +func (target *MQTTTarget) send(eventData event.Event) error { + objectName, err := url.QueryUnescape(eventData.S3.Object.Key) + if err != nil { + return err + } + key := eventData.S3.Bucket.Name + "/" + objectName + + data, err := json.Marshal(event.Log{EventName: eventData.EventName, Key: key, Records: []event.Event{eventData}}) + if err != nil { + return err + } + + token := target.client.Publish(target.args.Topic, target.args.QoS, false, string(data)) + if !token.WaitTimeout(reconnectInterval) { + return store.ErrNotConnected + } + return token.Error() +} + +// SendFromStore - reads an event from store and sends it to MQTT. +func (target *MQTTTarget) SendFromStore(key store.Key) error { + if err := target.init(); err != nil { + return err + } + + // Do not send if the connection is not active. + _, err := target.isActive() + if err != nil { + return err + } + + eventData, err := target.store.Get(key) + if err != nil { + // The last event key in a successful batch will be sent in the channel atmost once by the replayEvents() + // Such events will not exist and wouldve been already been sent successfully. + if os.IsNotExist(err) { + return nil + } + return err + } + + if err = target.send(eventData); err != nil { + return err + } + + // Delete the event from store. + return target.store.Del(key) +} + +// Save - saves the events to the store if queuestore is configured, which will +// be replayed when the mqtt connection is active. +func (target *MQTTTarget) Save(eventData event.Event) error { + if target.store != nil { + _, err := target.store.Put(eventData) + return err + } + if err := target.init(); err != nil { + return err + } + + // Do not send if the connection is not active. + _, err := target.isActive() + if err != nil { + return err + } + + return target.send(eventData) +} + +// Close - does nothing and available for interface compatibility. +func (target *MQTTTarget) Close() error { + if target.client != nil { + target.client.Disconnect(100) + } + close(target.quitCh) + return nil +} + +func (target *MQTTTarget) init() error { + return target.initOnce.Do(target.initMQTT) +} + +func (target *MQTTTarget) initMQTT() error { + args := target.args + + // Using hex here, to make sure we avoid 23 + // character limit on client_id according to + // MQTT spec. + clientID := fmt.Sprintf("%x", time.Now().UnixNano()) + + options := mqtt.NewClientOptions(). + SetClientID(clientID). + SetCleanSession(true). + SetUsername(args.User). + SetPassword(args.Password). + SetMaxReconnectInterval(args.MaxReconnectInterval). + SetKeepAlive(args.KeepAlive). + SetTLSConfig(&tls.Config{RootCAs: args.RootCAs}). + AddBroker(args.Broker.String()) + + target.client = mqtt.NewClient(options) + + token := target.client.Connect() + ok := token.WaitTimeout(reconnectInterval) + if !ok { + return store.ErrNotConnected + } + if token.Error() != nil { + return token.Error() + } + + yes, err := target.isActive() + if err != nil { + return err + } + if !yes { + return store.ErrNotConnected + } + + return nil +} + +// NewMQTTTarget - creates new MQTT target. +func NewMQTTTarget(id string, args MQTTArgs, loggerOnce logger.LogOnce) (*MQTTTarget, error) { + if args.MaxReconnectInterval == 0 { + // Default interval + // https://github.com/eclipse/paho.mqtt.golang/blob/master/options.go#L115 + args.MaxReconnectInterval = 10 * time.Minute + } + + if args.KeepAlive == 0 { + args.KeepAlive = 10 * time.Second + } + + var queueStore store.Store[event.Event] + if args.QueueDir != "" { + queueDir := filepath.Join(args.QueueDir, storePrefix+"-mqtt-"+id) + queueStore = store.NewQueueStore[event.Event](queueDir, args.QueueLimit, event.StoreExtension) + if err := queueStore.Open(); err != nil { + return nil, fmt.Errorf("unable to initialize the queue store of MQTT `%s`: %w", id, err) + } + } + + target := &MQTTTarget{ + id: event.TargetID{ID: id, Name: "mqtt"}, + args: args, + store: queueStore, + quitCh: make(chan struct{}), + loggerOnce: loggerOnce, + } + + if target.store != nil { + store.StreamItems(target.store, target, target.quitCh, target.loggerOnce) + } + + return target, nil +} diff --git a/internal/event/target/mysql.go b/internal/event/target/mysql.go new file mode 100644 index 00000000..0f311232 --- /dev/null +++ b/internal/event/target/mysql.go @@ -0,0 +1,444 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package target + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "strconv" + "strings" + "time" + + "github.com/go-sql-driver/mysql" + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/once" + "github.com/minio/minio/internal/store" + xnet "github.com/minio/pkg/v3/net" +) + +const ( + mysqlTableExists = `SELECT 1 FROM %s;` + // Some MySQL has a 3072 byte limit on key sizes. + mysqlCreateNamespaceTable = `CREATE TABLE %s ( + key_name VARCHAR(3072) NOT NULL, + key_hash CHAR(64) GENERATED ALWAYS AS (SHA2(key_name, 256)) STORED NOT NULL PRIMARY KEY, + value JSON) + CHARACTER SET = utf8mb4 COLLATE = utf8mb4_bin ROW_FORMAT = Dynamic;` + mysqlCreateAccessTable = `CREATE TABLE %s (event_time DATETIME NOT NULL, event_data JSON) + ROW_FORMAT = Dynamic;` + + mysqlUpdateRow = `INSERT INTO %s (key_name, value) VALUES (?, ?) ON DUPLICATE KEY UPDATE value=VALUES(value);` + mysqlDeleteRow = `DELETE FROM %s WHERE key_hash = SHA2(?, 256);` + mysqlInsertRow = `INSERT INTO %s (event_time, event_data) VALUES (?, ?);` +) + +// MySQL related constants +const ( + MySQLFormat = "format" + MySQLDSNString = "dsn_string" + MySQLTable = "table" + MySQLHost = "host" + MySQLPort = "port" + MySQLUsername = "username" + MySQLPassword = "password" + MySQLDatabase = "database" + MySQLQueueLimit = "queue_limit" + MySQLQueueDir = "queue_dir" + MySQLMaxOpenConnections = "max_open_connections" + + EnvMySQLEnable = "MINIO_NOTIFY_MYSQL_ENABLE" + EnvMySQLFormat = "MINIO_NOTIFY_MYSQL_FORMAT" + EnvMySQLDSNString = "MINIO_NOTIFY_MYSQL_DSN_STRING" + EnvMySQLTable = "MINIO_NOTIFY_MYSQL_TABLE" + EnvMySQLHost = "MINIO_NOTIFY_MYSQL_HOST" + EnvMySQLPort = "MINIO_NOTIFY_MYSQL_PORT" + EnvMySQLUsername = "MINIO_NOTIFY_MYSQL_USERNAME" + EnvMySQLPassword = "MINIO_NOTIFY_MYSQL_PASSWORD" + EnvMySQLDatabase = "MINIO_NOTIFY_MYSQL_DATABASE" + EnvMySQLQueueLimit = "MINIO_NOTIFY_MYSQL_QUEUE_LIMIT" + EnvMySQLQueueDir = "MINIO_NOTIFY_MYSQL_QUEUE_DIR" + EnvMySQLMaxOpenConnections = "MINIO_NOTIFY_MYSQL_MAX_OPEN_CONNECTIONS" +) + +// MySQLArgs - MySQL target arguments. +type MySQLArgs struct { + Enable bool `json:"enable"` + Format string `json:"format"` + DSN string `json:"dsnString"` + Table string `json:"table"` + Host xnet.URL `json:"host"` + Port string `json:"port"` + User string `json:"user"` + Password string `json:"password"` + Database string `json:"database"` + QueueDir string `json:"queueDir"` + QueueLimit uint64 `json:"queueLimit"` + MaxOpenConnections int `json:"maxOpenConnections"` +} + +// Validate MySQLArgs fields +func (m MySQLArgs) Validate() error { + if !m.Enable { + return nil + } + + if m.Format != "" { + f := strings.ToLower(m.Format) + if f != event.NamespaceFormat && f != event.AccessFormat { + return fmt.Errorf("unrecognized format") + } + } + + if m.Table == "" { + return fmt.Errorf("table unspecified") + } + + if m.DSN != "" { + if _, err := mysql.ParseDSN(m.DSN); err != nil { + return err + } + } else { + // Some fields need to be specified when DSN is unspecified + if m.Port == "" { + return fmt.Errorf("unspecified port") + } + if _, err := strconv.Atoi(m.Port); err != nil { + return fmt.Errorf("invalid port") + } + if m.Database == "" { + return fmt.Errorf("database unspecified") + } + } + + if m.QueueDir != "" { + if !filepath.IsAbs(m.QueueDir) { + return errors.New("queueDir path should be absolute") + } + } + + if m.MaxOpenConnections < 0 { + return errors.New("maxOpenConnections cannot be less than zero") + } + + return nil +} + +// MySQLTarget - MySQL target. +type MySQLTarget struct { + initOnce once.Init + + id event.TargetID + args MySQLArgs + updateStmt *sql.Stmt + deleteStmt *sql.Stmt + insertStmt *sql.Stmt + db *sql.DB + store store.Store[event.Event] + firstPing bool + loggerOnce logger.LogOnce + + quitCh chan struct{} +} + +// ID - returns target ID. +func (target *MySQLTarget) ID() event.TargetID { + return target.id +} + +// Name - returns the Name of the target. +func (target *MySQLTarget) Name() string { + return target.ID().String() +} + +// Store returns any underlying store if set. +func (target *MySQLTarget) Store() event.TargetStore { + return target.store +} + +// IsActive - Return true if target is up and active +func (target *MySQLTarget) IsActive() (bool, error) { + if err := target.init(); err != nil { + return false, err + } + return target.isActive() +} + +func (target *MySQLTarget) isActive() (bool, error) { + if err := target.db.Ping(); err != nil { + if IsConnErr(err) { + return false, store.ErrNotConnected + } + return false, err + } + return true, nil +} + +// Save - saves the events to the store which will be replayed when the SQL connection is active. +func (target *MySQLTarget) Save(eventData event.Event) error { + if target.store != nil { + _, err := target.store.Put(eventData) + return err + } + if err := target.init(); err != nil { + return err + } + + _, err := target.isActive() + if err != nil { + return err + } + return target.send(eventData) +} + +// send - sends an event to the mysql. +func (target *MySQLTarget) send(eventData event.Event) error { + if target.args.Format == event.NamespaceFormat { + objectName, err := url.QueryUnescape(eventData.S3.Object.Key) + if err != nil { + return err + } + key := eventData.S3.Bucket.Name + "/" + objectName + + if eventData.EventName == event.ObjectRemovedDelete { + _, err = target.deleteStmt.Exec(key) + } else { + var data []byte + if data, err = json.Marshal(struct{ Records []event.Event }{[]event.Event{eventData}}); err != nil { + return err + } + + _, err = target.updateStmt.Exec(key, data) + } + + return err + } + + if target.args.Format == event.AccessFormat { + eventTime, err := time.Parse(event.AMZTimeFormat, eventData.EventTime) + if err != nil { + return err + } + + data, err := json.Marshal(struct{ Records []event.Event }{[]event.Event{eventData}}) + if err != nil { + return err + } + + _, err = target.insertStmt.Exec(eventTime, data) + + return err + } + + return nil +} + +// SendFromStore - reads an event from store and sends it to MySQL. +func (target *MySQLTarget) SendFromStore(key store.Key) error { + if err := target.init(); err != nil { + return err + } + + _, err := target.isActive() + if err != nil { + return err + } + + if !target.firstPing { + if err := target.executeStmts(); err != nil { + if IsConnErr(err) { + return store.ErrNotConnected + } + return err + } + } + + eventData, eErr := target.store.Get(key) + if eErr != nil { + // The last event key in a successful batch will be sent in the channel atmost once by the replayEvents() + // Such events will not exist and wouldve been already been sent successfully. + if os.IsNotExist(eErr) { + return nil + } + return eErr + } + + if err := target.send(eventData); err != nil { + if IsConnErr(err) { + return store.ErrNotConnected + } + return err + } + + // Delete the event from store. + return target.store.Del(key) +} + +// Close - closes underneath connections to MySQL database. +func (target *MySQLTarget) Close() error { + close(target.quitCh) + if target.updateStmt != nil { + // FIXME: log returned error. ignore time being. + _ = target.updateStmt.Close() + } + + if target.deleteStmt != nil { + // FIXME: log returned error. ignore time being. + _ = target.deleteStmt.Close() + } + + if target.insertStmt != nil { + // FIXME: log returned error. ignore time being. + _ = target.insertStmt.Close() + } + + if target.db != nil { + return target.db.Close() + } + + return nil +} + +// Executes the table creation statements. +func (target *MySQLTarget) executeStmts() error { + _, err := target.db.Exec(fmt.Sprintf(mysqlTableExists, target.args.Table)) + if err != nil { + createStmt := mysqlCreateNamespaceTable + if target.args.Format == event.AccessFormat { + createStmt = mysqlCreateAccessTable + } + + if _, dbErr := target.db.Exec(fmt.Sprintf(createStmt, target.args.Table)); dbErr != nil { + return dbErr + } + } + + switch target.args.Format { + case event.NamespaceFormat: + // insert or update statement + if target.updateStmt, err = target.db.Prepare(fmt.Sprintf(mysqlUpdateRow, target.args.Table)); err != nil { + return err + } + // delete statement + if target.deleteStmt, err = target.db.Prepare(fmt.Sprintf(mysqlDeleteRow, target.args.Table)); err != nil { + return err + } + case event.AccessFormat: + // insert statement + if target.insertStmt, err = target.db.Prepare(fmt.Sprintf(mysqlInsertRow, target.args.Table)); err != nil { + return err + } + } + + return nil +} + +func (target *MySQLTarget) init() error { + return target.initOnce.Do(target.initMySQL) +} + +func (target *MySQLTarget) initMySQL() error { + args := target.args + + db, err := sql.Open("mysql", args.DSN) + if err != nil { + target.loggerOnce(context.Background(), err, target.ID().String()) + return err + } + target.db = db + + if args.MaxOpenConnections > 0 { + // Set the maximum connections limit + target.db.SetMaxOpenConns(args.MaxOpenConnections) + } + + err = target.db.Ping() + if err != nil { + if !xnet.IsConnRefusedErr(err) && !xnet.IsConnResetErr(err) { + target.loggerOnce(context.Background(), err, target.ID().String()) + } + } else { + if err = target.executeStmts(); err != nil { + target.loggerOnce(context.Background(), err, target.ID().String()) + } else { + target.firstPing = true + } + } + + if err != nil { + target.db.Close() + return err + } + + yes, err := target.isActive() + if err != nil { + return err + } + if !yes { + return store.ErrNotConnected + } + + return nil +} + +// NewMySQLTarget - creates new MySQL target. +func NewMySQLTarget(id string, args MySQLArgs, loggerOnce logger.LogOnce) (*MySQLTarget, error) { + var queueStore store.Store[event.Event] + if args.QueueDir != "" { + queueDir := filepath.Join(args.QueueDir, storePrefix+"-mysql-"+id) + queueStore = store.NewQueueStore[event.Event](queueDir, args.QueueLimit, event.StoreExtension) + if err := queueStore.Open(); err != nil { + return nil, fmt.Errorf("unable to initialize the queue store of MySQL `%s`: %w", id, err) + } + } + + if args.DSN == "" { + config := mysql.Config{ + User: args.User, + Passwd: args.Password, + Net: "tcp", + Addr: args.Host.String() + ":" + args.Port, + DBName: args.Database, + AllowNativePasswords: true, + CheckConnLiveness: true, + } + + args.DSN = config.FormatDSN() + } + + target := &MySQLTarget{ + id: event.TargetID{ID: id, Name: "mysql"}, + args: args, + firstPing: false, + store: queueStore, + loggerOnce: loggerOnce, + quitCh: make(chan struct{}), + } + + if target.store != nil { + store.StreamItems(target.store, target, target.quitCh, target.loggerOnce) + } + + return target, nil +} diff --git a/internal/event/target/mysql_test.go b/internal/event/target/mysql_test.go new file mode 100644 index 00000000..d8799dc7 --- /dev/null +++ b/internal/event/target/mysql_test.go @@ -0,0 +1,36 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package target + +import ( + "database/sql" + "slices" + "testing" +) + +// TestPostgreSQLRegistration checks if sql driver +// is registered and fails otherwise. +func TestMySQLRegistration(t *testing.T) { + var found bool + if slices.Contains(sql.Drivers(), "mysql") { + found = true + } + if !found { + t.Fatal("mysql driver not registered") + } +} diff --git a/internal/event/target/nats.go b/internal/event/target/nats.go new file mode 100644 index 00000000..c96833bc --- /dev/null +++ b/internal/event/target/nats.go @@ -0,0 +1,480 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package target + +import ( + "context" + "crypto/tls" + "crypto/x509" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + + "github.com/google/uuid" + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/once" + "github.com/minio/minio/internal/store" + xnet "github.com/minio/pkg/v3/net" + "github.com/nats-io/nats.go" + "github.com/nats-io/stan.go" +) + +// NATS related constants +const ( + NATSAddress = "address" + NATSSubject = "subject" + NATSUsername = "username" + NATSPassword = "password" + NATSToken = "token" + NATSNKeySeed = "nkey_seed" + NATSTLS = "tls" + NATSTLSSkipVerify = "tls_skip_verify" + NATSTLSHandshakeFirst = "tls_handshake_first" + NATSPingInterval = "ping_interval" + NATSQueueDir = "queue_dir" + NATSQueueLimit = "queue_limit" + NATSCertAuthority = "cert_authority" + NATSClientCert = "client_cert" + NATSClientKey = "client_key" + + // Streaming constants - deprecated + NATSStreaming = "streaming" + NATSStreamingClusterID = "streaming_cluster_id" + NATSStreamingAsync = "streaming_async" + NATSStreamingMaxPubAcksInFlight = "streaming_max_pub_acks_in_flight" + + // JetStream constants + NATSJetStream = "jetstream" + + EnvNATSEnable = "MINIO_NOTIFY_NATS_ENABLE" + EnvNATSAddress = "MINIO_NOTIFY_NATS_ADDRESS" + EnvNATSSubject = "MINIO_NOTIFY_NATS_SUBJECT" + EnvNATSUsername = "MINIO_NOTIFY_NATS_USERNAME" + NATSUserCredentials = "MINIO_NOTIFY_NATS_USER_CREDENTIALS" + EnvNATSPassword = "MINIO_NOTIFY_NATS_PASSWORD" + EnvNATSToken = "MINIO_NOTIFY_NATS_TOKEN" + EnvNATSNKeySeed = "MINIO_NOTIFY_NATS_NKEY_SEED" + EnvNATSTLS = "MINIO_NOTIFY_NATS_TLS" + EnvNATSTLSSkipVerify = "MINIO_NOTIFY_NATS_TLS_SKIP_VERIFY" + EnvNatsTLSHandshakeFirst = "MINIO_NOTIFY_NATS_TLS_HANDSHAKE_FIRST" + EnvNATSPingInterval = "MINIO_NOTIFY_NATS_PING_INTERVAL" + EnvNATSQueueDir = "MINIO_NOTIFY_NATS_QUEUE_DIR" + EnvNATSQueueLimit = "MINIO_NOTIFY_NATS_QUEUE_LIMIT" + EnvNATSCertAuthority = "MINIO_NOTIFY_NATS_CERT_AUTHORITY" + EnvNATSClientCert = "MINIO_NOTIFY_NATS_CLIENT_CERT" + EnvNATSClientKey = "MINIO_NOTIFY_NATS_CLIENT_KEY" + + // Streaming constants - deprecated + EnvNATSStreaming = "MINIO_NOTIFY_NATS_STREAMING" + EnvNATSStreamingClusterID = "MINIO_NOTIFY_NATS_STREAMING_CLUSTER_ID" + EnvNATSStreamingAsync = "MINIO_NOTIFY_NATS_STREAMING_ASYNC" + EnvNATSStreamingMaxPubAcksInFlight = "MINIO_NOTIFY_NATS_STREAMING_MAX_PUB_ACKS_IN_FLIGHT" + + // Jetstream constants + EnvNATSJetStream = "MINIO_NOTIFY_NATS_JETSTREAM" +) + +// NATSArgs - NATS target arguments. +type NATSArgs struct { + Enable bool `json:"enable"` + Address xnet.Host `json:"address"` + Subject string `json:"subject"` + Username string `json:"username"` + UserCredentials string `json:"userCredentials"` + Password string `json:"password"` + Token string `json:"token"` + NKeySeed string `json:"nKeySeed"` + TLS bool `json:"tls"` + TLSSkipVerify bool `json:"tlsSkipVerify"` + TLSHandshakeFirst bool `json:"tlsHandshakeFirst"` + Secure bool `json:"secure"` + CertAuthority string `json:"certAuthority"` + ClientCert string `json:"clientCert"` + ClientKey string `json:"clientKey"` + PingInterval int64 `json:"pingInterval"` + QueueDir string `json:"queueDir"` + QueueLimit uint64 `json:"queueLimit"` + JetStream struct { + Enable bool `json:"enable"` + } `json:"jetStream"` + Streaming struct { + Enable bool `json:"enable"` + ClusterID string `json:"clusterID"` + Async bool `json:"async"` + MaxPubAcksInflight int `json:"maxPubAcksInflight"` + } `json:"streaming"` + + RootCAs *x509.CertPool `json:"-"` +} + +// Validate NATSArgs fields +func (n NATSArgs) Validate() error { + if !n.Enable { + return nil + } + + if n.Address.IsEmpty() { + return errors.New("empty address") + } + + if n.Subject == "" { + return errors.New("empty subject") + } + + if n.ClientCert != "" && n.ClientKey == "" || n.ClientCert == "" && n.ClientKey != "" { + return errors.New("cert and key must be specified as a pair") + } + + if n.Username != "" && n.Password == "" || n.Username == "" && n.Password != "" { + return errors.New("username and password must be specified as a pair") + } + + if n.Streaming.Enable { + if n.Streaming.ClusterID == "" { + return errors.New("empty cluster id") + } + } + + if n.JetStream.Enable { + if n.Subject == "" { + return errors.New("empty subject") + } + } + + if n.QueueDir != "" { + if !filepath.IsAbs(n.QueueDir) { + return errors.New("queueDir path should be absolute") + } + } + + return nil +} + +// To obtain a nats connection from args. +func (n NATSArgs) connectNats() (*nats.Conn, error) { + connOpts := []nats.Option{nats.Name("Minio Notification"), nats.MaxReconnects(-1)} + if n.Username != "" && n.Password != "" { + connOpts = append(connOpts, nats.UserInfo(n.Username, n.Password)) + } + if n.UserCredentials != "" { + connOpts = append(connOpts, nats.UserCredentials(n.UserCredentials)) + } + if n.Token != "" { + connOpts = append(connOpts, nats.Token(n.Token)) + } + if n.NKeySeed != "" { + nkeyOpt, err := nats.NkeyOptionFromSeed(n.NKeySeed) + if err != nil { + return nil, err + } + connOpts = append(connOpts, nkeyOpt) + } + if n.Secure || n.TLS && n.TLSSkipVerify { + connOpts = append(connOpts, nats.Secure(nil)) + } else if n.TLS { + connOpts = append(connOpts, nats.Secure(&tls.Config{RootCAs: n.RootCAs})) + } + if n.TLSHandshakeFirst { + connOpts = append(connOpts, nats.TLSHandshakeFirst()) + } + if n.CertAuthority != "" { + connOpts = append(connOpts, nats.RootCAs(n.CertAuthority)) + } + if n.ClientCert != "" && n.ClientKey != "" { + connOpts = append(connOpts, nats.ClientCert(n.ClientCert, n.ClientKey)) + } + return nats.Connect(n.Address.String(), connOpts...) +} + +// To obtain a streaming connection from args. +func (n NATSArgs) connectStan() (stan.Conn, error) { + scheme := "nats" + if n.Secure { + scheme = "tls" + } + + var addressURL string + //nolint:gocritic + if n.Username != "" && n.Password != "" { + addressURL = scheme + "://" + n.Username + ":" + n.Password + "@" + n.Address.String() + } else if n.Token != "" { + addressURL = scheme + "://" + n.Token + "@" + n.Address.String() + } else { + addressURL = scheme + "://" + n.Address.String() + } + + u, err := uuid.NewRandom() + if err != nil { + return nil, err + } + clientID := u.String() + + connOpts := []stan.Option{stan.NatsURL(addressURL)} + if n.Streaming.MaxPubAcksInflight > 0 { + connOpts = append(connOpts, stan.MaxPubAcksInflight(n.Streaming.MaxPubAcksInflight)) + } + if n.UserCredentials != "" { + connOpts = append(connOpts, stan.NatsOptions(nats.UserCredentials(n.UserCredentials))) + } + + return stan.Connect(n.Streaming.ClusterID, clientID, connOpts...) +} + +// NATSTarget - NATS target. +type NATSTarget struct { + initOnce once.Init + + id event.TargetID + args NATSArgs + natsConn *nats.Conn + stanConn stan.Conn + jstream nats.JetStream + store store.Store[event.Event] + loggerOnce logger.LogOnce + quitCh chan struct{} +} + +// ID - returns target ID. +func (target *NATSTarget) ID() event.TargetID { + return target.id +} + +// Name - returns the Name of the target. +func (target *NATSTarget) Name() string { + return target.ID().String() +} + +// Store returns any underlying store if set. +func (target *NATSTarget) Store() event.TargetStore { + return target.store +} + +// IsActive - Return true if target is up and active +func (target *NATSTarget) IsActive() (bool, error) { + if err := target.init(); err != nil { + return false, err + } + return target.isActive() +} + +func (target *NATSTarget) isActive() (bool, error) { + var connErr error + if target.args.Streaming.Enable { + if target.stanConn == nil || target.stanConn.NatsConn() == nil { + target.stanConn, connErr = target.args.connectStan() + } else if !target.stanConn.NatsConn().IsConnected() { + return false, store.ErrNotConnected + } + } else { + if target.natsConn == nil { + target.natsConn, connErr = target.args.connectNats() + } else if !target.natsConn.IsConnected() { + return false, store.ErrNotConnected + } + } + + if connErr != nil { + if connErr.Error() == nats.ErrNoServers.Error() { + return false, store.ErrNotConnected + } + return false, connErr + } + + if target.natsConn != nil && target.args.JetStream.Enable { + target.jstream, connErr = target.natsConn.JetStream() + if connErr != nil { + if connErr.Error() == nats.ErrNoServers.Error() { + return false, store.ErrNotConnected + } + return false, connErr + } + } + + return true, nil +} + +// Save - saves the events to the store which will be replayed when the Nats connection is active. +func (target *NATSTarget) Save(eventData event.Event) error { + if target.store != nil { + _, err := target.store.Put(eventData) + return err + } + + if err := target.init(); err != nil { + return err + } + + _, err := target.isActive() + if err != nil { + return err + } + return target.send(eventData) +} + +// send - sends an event to the Nats. +func (target *NATSTarget) send(eventData event.Event) error { + objectName, err := url.QueryUnescape(eventData.S3.Object.Key) + if err != nil { + return err + } + key := eventData.S3.Bucket.Name + "/" + objectName + + data, err := json.Marshal(event.Log{EventName: eventData.EventName, Key: key, Records: []event.Event{eventData}}) + if err != nil { + return err + } + + if target.stanConn != nil { + if target.args.Streaming.Async { + _, err = target.stanConn.PublishAsync(target.args.Subject, data, nil) + } else { + err = target.stanConn.Publish(target.args.Subject, data) + } + } else { + if target.jstream != nil { + _, err = target.jstream.Publish(target.args.Subject, data) + } else { + err = target.natsConn.Publish(target.args.Subject, data) + } + } + return err +} + +// SendFromStore - reads an event from store and sends it to Nats. +func (target *NATSTarget) SendFromStore(key store.Key) error { + if err := target.init(); err != nil { + return err + } + + _, err := target.isActive() + if err != nil { + return err + } + + eventData, eErr := target.store.Get(key) + if eErr != nil { + // The last event key in a successful batch will be sent in the channel atmost once by the replayEvents() + // Such events will not exist and wouldve been already been sent successfully. + if os.IsNotExist(eErr) { + return nil + } + return eErr + } + + if err := target.send(eventData); err != nil { + return err + } + + return target.store.Del(key) +} + +// Close - closes underneath connections to NATS server. +func (target *NATSTarget) Close() (err error) { + close(target.quitCh) + if target.stanConn != nil { + // closing the streaming connection does not close the provided NATS connection. + if target.stanConn.NatsConn() != nil { + target.stanConn.NatsConn().Close() + } + return target.stanConn.Close() + } + + if target.natsConn != nil { + target.natsConn.Close() + } + + return nil +} + +func (target *NATSTarget) init() error { + return target.initOnce.Do(target.initNATS) +} + +func (target *NATSTarget) initNATS() error { + args := target.args + + var err error + if args.Streaming.Enable { + target.loggerOnce(context.Background(), errors.New("NATS Streaming is deprecated please migrate to JetStream"), target.ID().String()) + var stanConn stan.Conn + stanConn, err = args.connectStan() + target.stanConn = stanConn + } else { + var natsConn *nats.Conn + natsConn, err = args.connectNats() + target.natsConn = natsConn + } + if err != nil { + if err.Error() != nats.ErrNoServers.Error() { + target.loggerOnce(context.Background(), err, target.ID().String()) + } + return err + } + + if target.natsConn != nil && args.JetStream.Enable { + var jstream nats.JetStream + jstream, err = target.natsConn.JetStream() + if err != nil { + if err.Error() != nats.ErrNoServers.Error() { + target.loggerOnce(context.Background(), err, target.ID().String()) + } + return err + } + target.jstream = jstream + } + + yes, err := target.isActive() + if err != nil { + return err + } + if !yes { + return store.ErrNotConnected + } + + return nil +} + +// NewNATSTarget - creates new NATS target. +func NewNATSTarget(id string, args NATSArgs, loggerOnce logger.LogOnce) (*NATSTarget, error) { + var queueStore store.Store[event.Event] + if args.QueueDir != "" { + queueDir := filepath.Join(args.QueueDir, storePrefix+"-nats-"+id) + queueStore = store.NewQueueStore[event.Event](queueDir, args.QueueLimit, event.StoreExtension) + if err := queueStore.Open(); err != nil { + return nil, fmt.Errorf("unable to initialize the queue store of NATS `%s`: %w", id, err) + } + } + + target := &NATSTarget{ + id: event.TargetID{ID: id, Name: "nats"}, + args: args, + loggerOnce: loggerOnce, + store: queueStore, + quitCh: make(chan struct{}), + } + + if target.store != nil { + store.StreamItems(target.store, target, target.quitCh, target.loggerOnce) + } + + return target, nil +} diff --git a/internal/event/target/nats_contrib_test.go b/internal/event/target/nats_contrib_test.go new file mode 100644 index 00000000..42a5f060 --- /dev/null +++ b/internal/event/target/nats_contrib_test.go @@ -0,0 +1,131 @@ +/* + * MinIO Object Storage (c) 2021-2023 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package target + +import ( + "testing" + + "github.com/nats-io/nats-server/v2/server" + + xnet "github.com/minio/pkg/v3/net" + natsserver "github.com/nats-io/nats-server/v2/test" +) + +func TestNatsConnPlain(t *testing.T) { + opts := natsserver.DefaultTestOptions + opts.Port = 14222 + s := natsserver.RunServer(&opts) + defer s.Shutdown() + + clientConfig := &NATSArgs{ + Enable: true, + Address: xnet.Host{ + Name: "localhost", + Port: (xnet.Port(opts.Port)), + IsPortSet: true, + }, + Subject: "test", + } + con, err := clientConfig.connectNats() + if err != nil { + t.Errorf("Could not connect to nats: %v", err) + } + defer con.Close() +} + +func TestNatsConnUserPass(t *testing.T) { + opts := natsserver.DefaultTestOptions + opts.Port = 14223 + opts.Username = "testminio" + opts.Password = "miniotest" + s := natsserver.RunServer(&opts) + defer s.Shutdown() + + clientConfig := &NATSArgs{ + Enable: true, + Address: xnet.Host{ + Name: "localhost", + Port: (xnet.Port(opts.Port)), + IsPortSet: true, + }, + Subject: "test", + Username: opts.Username, + Password: opts.Password, + } + + con, err := clientConfig.connectNats() + if err != nil { + t.Errorf("Could not connect to nats: %v", err) + } + defer con.Close() +} + +func TestNatsConnToken(t *testing.T) { + opts := natsserver.DefaultTestOptions + opts.Port = 14223 + opts.Authorization = "s3cr3t" + s := natsserver.RunServer(&opts) + defer s.Shutdown() + + clientConfig := &NATSArgs{ + Enable: true, + Address: xnet.Host{ + Name: "localhost", + Port: (xnet.Port(opts.Port)), + IsPortSet: true, + }, + Subject: "test", + Token: opts.Authorization, + } + + con, err := clientConfig.connectNats() + if err != nil { + t.Errorf("Could not connect to nats: %v", err) + } + defer con.Close() +} + +func TestNatsConnNKeySeed(t *testing.T) { + opts := natsserver.DefaultTestOptions + opts.Port = 14223 + opts.Nkeys = []*server.NkeyUser{ + { + // Not a real NKey + // Taken from https://docs.nats.io/running-a-nats-service/configuration/securing_nats/auth_intro/nkey_auth + Nkey: "UDXU4RCSJNZOIQHZNWXHXORDPRTGNJAHAHFRGZNEEJCPQTT2M7NLCNF4", + }, + } + s := natsserver.RunServer(&opts) + defer s.Shutdown() + + clientConfig := &NATSArgs{ + Enable: true, + Address: xnet.Host{ + Name: "localhost", + Port: (xnet.Port(opts.Port)), + IsPortSet: true, + }, + Subject: "test", + NKeySeed: "testdata/contrib/test.nkey", + } + + con, err := clientConfig.connectNats() + if err != nil { + t.Errorf("Could not connect to nats: %v", err) + } + defer con.Close() +} diff --git a/internal/event/target/nats_tls_contrib_test.go b/internal/event/target/nats_tls_contrib_test.go new file mode 100644 index 00000000..30cf5b46 --- /dev/null +++ b/internal/event/target/nats_tls_contrib_test.go @@ -0,0 +1,98 @@ +/* + * MinIO Object Storage (c) 2021-2023 MinIO, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package target + +import ( + "path" + "path/filepath" + "testing" + + xnet "github.com/minio/pkg/v3/net" + natsserver "github.com/nats-io/nats-server/v2/test" +) + +func TestNatsConnTLSCustomCA(t *testing.T) { + s, opts := natsserver.RunServerWithConfig(filepath.Join("testdata", "contrib", "nats_tls.conf")) + defer s.Shutdown() + + clientConfig := &NATSArgs{ + Enable: true, + Address: xnet.Host{ + Name: "localhost", + Port: (xnet.Port(opts.Port)), + IsPortSet: true, + }, + Subject: "test", + Secure: true, + CertAuthority: path.Join("testdata", "contrib", "certs", "root_ca_cert.pem"), + } + + con, err := clientConfig.connectNats() + if err != nil { + t.Errorf("Could not connect to nats: %v", err) + } + defer con.Close() +} + +func TestNatsConnTLSCustomCAHandshakeFirst(t *testing.T) { + s, opts := natsserver.RunServerWithConfig(filepath.Join("testdata", "contrib", "nats_tls_handshake_first.conf")) + defer s.Shutdown() + + clientConfig := &NATSArgs{ + Enable: true, + Address: xnet.Host{ + Name: "localhost", + Port: (xnet.Port(opts.Port)), + IsPortSet: true, + }, + Subject: "test", + Secure: true, + CertAuthority: path.Join("testdata", "contrib", "certs", "root_ca_cert.pem"), + TLSHandshakeFirst: true, + } + + con, err := clientConfig.connectNats() + if err != nil { + t.Errorf("Could not connect to nats: %v", err) + } + defer con.Close() +} + +func TestNatsConnTLSClientAuthorization(t *testing.T) { + s, opts := natsserver.RunServerWithConfig(filepath.Join("testdata", "contrib", "nats_tls_client_cert.conf")) + defer s.Shutdown() + + clientConfig := &NATSArgs{ + Enable: true, + Address: xnet.Host{ + Name: "localhost", + Port: (xnet.Port(opts.Port)), + IsPortSet: true, + }, + Subject: "test", + Secure: true, + CertAuthority: path.Join("testdata", "contrib", "certs", "root_ca_cert.pem"), + ClientCert: path.Join("testdata", "contrib", "certs", "nats_client_cert.pem"), + ClientKey: path.Join("testdata", "contrib", "certs", "nats_client_key.pem"), + } + + con, err := clientConfig.connectNats() + if err != nil { + t.Errorf("Could not connect to nats: %v", err) + } + defer con.Close() +} diff --git a/internal/event/target/nsq.go b/internal/event/target/nsq.go new file mode 100644 index 00000000..ec04d993 --- /dev/null +++ b/internal/event/target/nsq.go @@ -0,0 +1,288 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package target + +import ( + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + + "github.com/nsqio/go-nsq" + + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/once" + "github.com/minio/minio/internal/store" + xnet "github.com/minio/pkg/v3/net" +) + +// NSQ constants +const ( + NSQAddress = "nsqd_address" + NSQTopic = "topic" + NSQTLS = "tls" + NSQTLSSkipVerify = "tls_skip_verify" + NSQQueueDir = "queue_dir" + NSQQueueLimit = "queue_limit" + + EnvNSQEnable = "MINIO_NOTIFY_NSQ_ENABLE" + EnvNSQAddress = "MINIO_NOTIFY_NSQ_NSQD_ADDRESS" + EnvNSQTopic = "MINIO_NOTIFY_NSQ_TOPIC" + EnvNSQTLS = "MINIO_NOTIFY_NSQ_TLS" + EnvNSQTLSSkipVerify = "MINIO_NOTIFY_NSQ_TLS_SKIP_VERIFY" + EnvNSQQueueDir = "MINIO_NOTIFY_NSQ_QUEUE_DIR" + EnvNSQQueueLimit = "MINIO_NOTIFY_NSQ_QUEUE_LIMIT" +) + +// NSQArgs - NSQ target arguments. +type NSQArgs struct { + Enable bool `json:"enable"` + NSQDAddress xnet.Host `json:"nsqdAddress"` + Topic string `json:"topic"` + TLS struct { + Enable bool `json:"enable"` + SkipVerify bool `json:"skipVerify"` + } `json:"tls"` + QueueDir string `json:"queueDir"` + QueueLimit uint64 `json:"queueLimit"` +} + +// Validate NSQArgs fields +func (n NSQArgs) Validate() error { + if !n.Enable { + return nil + } + + if n.NSQDAddress.IsEmpty() { + return errors.New("empty nsqdAddress") + } + + if n.Topic == "" { + return errors.New("empty topic") + } + if n.QueueDir != "" { + if !filepath.IsAbs(n.QueueDir) { + return errors.New("queueDir path should be absolute") + } + } + + return nil +} + +// NSQTarget - NSQ target. +type NSQTarget struct { + initOnce once.Init + + id event.TargetID + args NSQArgs + producer *nsq.Producer + store store.Store[event.Event] + config *nsq.Config + loggerOnce logger.LogOnce + quitCh chan struct{} +} + +// ID - returns target ID. +func (target *NSQTarget) ID() event.TargetID { + return target.id +} + +// Name - returns the Name of the target. +func (target *NSQTarget) Name() string { + return target.ID().String() +} + +// Store returns any underlying store if set. +func (target *NSQTarget) Store() event.TargetStore { + return target.store +} + +// IsActive - Return true if target is up and active +func (target *NSQTarget) IsActive() (bool, error) { + if err := target.init(); err != nil { + return false, err + } + return target.isActive() +} + +func (target *NSQTarget) isActive() (bool, error) { + if target.producer == nil { + producer, err := nsq.NewProducer(target.args.NSQDAddress.String(), target.config) + if err != nil { + return false, err + } + target.producer = producer + } + + if err := target.producer.Ping(); err != nil { + // To treat "connection refused" errors as errNotConnected. + if xnet.IsConnRefusedErr(err) { + return false, store.ErrNotConnected + } + return false, err + } + return true, nil +} + +// Save - saves the events to the store which will be replayed when the nsq connection is active. +func (target *NSQTarget) Save(eventData event.Event) error { + if target.store != nil { + _, err := target.store.Put(eventData) + return err + } + + if err := target.init(); err != nil { + return err + } + + _, err := target.isActive() + if err != nil { + return err + } + return target.send(eventData) +} + +// send - sends an event to the NSQ. +func (target *NSQTarget) send(eventData event.Event) error { + objectName, err := url.QueryUnescape(eventData.S3.Object.Key) + if err != nil { + return err + } + key := eventData.S3.Bucket.Name + "/" + objectName + + data, err := json.Marshal(event.Log{EventName: eventData.EventName, Key: key, Records: []event.Event{eventData}}) + if err != nil { + return err + } + + return target.producer.Publish(target.args.Topic, data) +} + +// SendFromStore - reads an event from store and sends it to NSQ. +func (target *NSQTarget) SendFromStore(key store.Key) error { + if err := target.init(); err != nil { + return err + } + + _, err := target.isActive() + if err != nil { + return err + } + + eventData, eErr := target.store.Get(key) + if eErr != nil { + // The last event key in a successful batch will be sent in the channel atmost once by the replayEvents() + // Such events will not exist and wouldve been already been sent successfully. + if os.IsNotExist(eErr) { + return nil + } + return eErr + } + + if err := target.send(eventData); err != nil { + return err + } + + // Delete the event from store. + return target.store.Del(key) +} + +// Close - closes underneath connections to NSQD server. +func (target *NSQTarget) Close() (err error) { + close(target.quitCh) + if target.producer != nil { + // this blocks until complete: + target.producer.Stop() + } + return nil +} + +func (target *NSQTarget) init() error { + return target.initOnce.Do(target.initNSQ) +} + +func (target *NSQTarget) initNSQ() error { + args := target.args + + config := nsq.NewConfig() + if args.TLS.Enable { + config.TlsV1 = true + config.TlsConfig = &tls.Config{ + InsecureSkipVerify: args.TLS.SkipVerify, + } + } + target.config = config + + producer, err := nsq.NewProducer(args.NSQDAddress.String(), config) + if err != nil { + target.loggerOnce(context.Background(), err, target.ID().String()) + return err + } + target.producer = producer + + err = target.producer.Ping() + if err != nil { + // To treat "connection refused" errors as errNotConnected. + if !xnet.IsConnRefusedErr(err) && !xnet.IsConnResetErr(err) { + target.loggerOnce(context.Background(), err, target.ID().String()) + } + target.producer.Stop() + return err + } + + yes, err := target.isActive() + if err != nil { + return err + } + if !yes { + return store.ErrNotConnected + } + + return nil +} + +// NewNSQTarget - creates new NSQ target. +func NewNSQTarget(id string, args NSQArgs, loggerOnce logger.LogOnce) (*NSQTarget, error) { + var queueStore store.Store[event.Event] + if args.QueueDir != "" { + queueDir := filepath.Join(args.QueueDir, storePrefix+"-nsq-"+id) + queueStore = store.NewQueueStore[event.Event](queueDir, args.QueueLimit, event.StoreExtension) + if err := queueStore.Open(); err != nil { + return nil, fmt.Errorf("unable to initialize the queue store of NSQ `%s`: %w", id, err) + } + } + + target := &NSQTarget{ + id: event.TargetID{ID: id, Name: "nsq"}, + args: args, + loggerOnce: loggerOnce, + store: queueStore, + quitCh: make(chan struct{}), + } + + if target.store != nil { + store.StreamItems(target.store, target, target.quitCh, target.loggerOnce) + } + + return target, nil +} diff --git a/internal/event/target/nsq_test.go b/internal/event/target/nsq_test.go new file mode 100644 index 00000000..32926ab5 --- /dev/null +++ b/internal/event/target/nsq_test.go @@ -0,0 +1,98 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package target + +import ( + "testing" + + xnet "github.com/minio/pkg/v3/net" +) + +func TestNSQArgs_Validate(t *testing.T) { + type fields struct { + Enable bool + NSQDAddress xnet.Host + Topic string + TLS struct { + Enable bool + SkipVerify bool + } + } + tests := []struct { + name string + fields fields + wantErr bool + }{ + { + name: "test1_missing_topic", + fields: fields{ + Enable: true, + NSQDAddress: xnet.Host{ + Name: "127.0.0.1", + Port: 4150, + IsPortSet: true, + }, + Topic: "", + }, + wantErr: true, + }, + { + name: "test2_disabled", + fields: fields{ + Enable: false, + NSQDAddress: xnet.Host{}, + Topic: "topic", + }, + wantErr: false, + }, + { + name: "test3_OK", + fields: fields{ + Enable: true, + NSQDAddress: xnet.Host{ + Name: "127.0.0.1", + Port: 4150, + IsPortSet: true, + }, + Topic: "topic", + }, + wantErr: false, + }, + { + name: "test4_emptynsqdaddr", + fields: fields{ + Enable: true, + NSQDAddress: xnet.Host{}, + Topic: "topic", + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + n := NSQArgs{ + Enable: tt.fields.Enable, + NSQDAddress: tt.fields.NSQDAddress, + Topic: tt.fields.Topic, + } + if err := n.Validate(); (err != nil) != tt.wantErr { + t.Errorf("NSQArgs.Validate() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} diff --git a/internal/event/target/postgresql.go b/internal/event/target/postgresql.go new file mode 100644 index 00000000..9bd9a886 --- /dev/null +++ b/internal/event/target/postgresql.go @@ -0,0 +1,493 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package target + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + "unicode" + + _ "github.com/lib/pq" // Register postgres driver + + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/once" + "github.com/minio/minio/internal/store" + xnet "github.com/minio/pkg/v3/net" +) + +const ( + psqlTableExists = `SELECT 1 FROM %s;` + psqlCreateNamespaceTable = `CREATE TABLE %s (key VARCHAR PRIMARY KEY, value JSONB);` + psqlCreateAccessTable = `CREATE TABLE %s (event_time TIMESTAMP WITH TIME ZONE NOT NULL, event_data JSONB);` + + psqlUpdateRow = `INSERT INTO %s (key, value) VALUES ($1, $2) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value;` + psqlDeleteRow = `DELETE FROM %s WHERE key = $1;` + psqlInsertRow = `INSERT INTO %s (event_time, event_data) VALUES ($1, $2);` +) + +// Postgres constants +const ( + PostgresFormat = "format" + PostgresConnectionString = "connection_string" + PostgresTable = "table" + PostgresHost = "host" + PostgresPort = "port" + PostgresUsername = "username" + PostgresPassword = "password" + PostgresDatabase = "database" + PostgresQueueDir = "queue_dir" + PostgresQueueLimit = "queue_limit" + PostgresMaxOpenConnections = "max_open_connections" + + EnvPostgresEnable = "MINIO_NOTIFY_POSTGRES_ENABLE" + EnvPostgresFormat = "MINIO_NOTIFY_POSTGRES_FORMAT" + EnvPostgresConnectionString = "MINIO_NOTIFY_POSTGRES_CONNECTION_STRING" + EnvPostgresTable = "MINIO_NOTIFY_POSTGRES_TABLE" + EnvPostgresHost = "MINIO_NOTIFY_POSTGRES_HOST" + EnvPostgresPort = "MINIO_NOTIFY_POSTGRES_PORT" + EnvPostgresUsername = "MINIO_NOTIFY_POSTGRES_USERNAME" + EnvPostgresPassword = "MINIO_NOTIFY_POSTGRES_PASSWORD" + EnvPostgresDatabase = "MINIO_NOTIFY_POSTGRES_DATABASE" + EnvPostgresQueueDir = "MINIO_NOTIFY_POSTGRES_QUEUE_DIR" + EnvPostgresQueueLimit = "MINIO_NOTIFY_POSTGRES_QUEUE_LIMIT" + EnvPostgresMaxOpenConnections = "MINIO_NOTIFY_POSTGRES_MAX_OPEN_CONNECTIONS" +) + +// PostgreSQLArgs - PostgreSQL target arguments. +type PostgreSQLArgs struct { + Enable bool `json:"enable"` + Format string `json:"format"` + ConnectionString string `json:"connectionString"` + Table string `json:"table"` + Host xnet.Host `json:"host"` // default: localhost + Port string `json:"port"` // default: 5432 + Username string `json:"username"` // default: user running minio + Password string `json:"password"` // default: no password + Database string `json:"database"` // default: same as user + QueueDir string `json:"queueDir"` + QueueLimit uint64 `json:"queueLimit"` + MaxOpenConnections int `json:"maxOpenConnections"` +} + +// Validate PostgreSQLArgs fields +func (p PostgreSQLArgs) Validate() error { + if !p.Enable { + return nil + } + if p.Table == "" { + return fmt.Errorf("empty table name") + } + if err := validatePsqlTableName(p.Table); err != nil { + return err + } + + if p.Format != "" { + f := strings.ToLower(p.Format) + if f != event.NamespaceFormat && f != event.AccessFormat { + return fmt.Errorf("unrecognized format value") + } + } + + if p.ConnectionString != "" { + // No pq API doesn't help to validate connection string + // prior connection, so no validation for now. + } else { + // Some fields need to be specified when ConnectionString is unspecified + if p.Port == "" { + return fmt.Errorf("unspecified port") + } + if _, err := strconv.Atoi(p.Port); err != nil { + return fmt.Errorf("invalid port") + } + if p.Database == "" { + return fmt.Errorf("database unspecified") + } + } + + if p.QueueDir != "" { + if !filepath.IsAbs(p.QueueDir) { + return errors.New("queueDir path should be absolute") + } + } + + if p.MaxOpenConnections < 0 { + return errors.New("maxOpenConnections cannot be less than zero") + } + + return nil +} + +// PostgreSQLTarget - PostgreSQL target. +type PostgreSQLTarget struct { + initOnce once.Init + + id event.TargetID + args PostgreSQLArgs + updateStmt *sql.Stmt + deleteStmt *sql.Stmt + insertStmt *sql.Stmt + db *sql.DB + store store.Store[event.Event] + firstPing bool + connString string + loggerOnce logger.LogOnce + quitCh chan struct{} +} + +// ID - returns target ID. +func (target *PostgreSQLTarget) ID() event.TargetID { + return target.id +} + +// Name - returns the Name of the target. +func (target *PostgreSQLTarget) Name() string { + return target.ID().String() +} + +// Store returns any underlying store if set. +func (target *PostgreSQLTarget) Store() event.TargetStore { + return target.store +} + +// IsActive - Return true if target is up and active +func (target *PostgreSQLTarget) IsActive() (bool, error) { + if err := target.init(); err != nil { + return false, err + } + return target.isActive() +} + +func (target *PostgreSQLTarget) isActive() (bool, error) { + if err := target.db.Ping(); err != nil { + if IsConnErr(err) { + return false, store.ErrNotConnected + } + return false, err + } + return true, nil +} + +// Save - saves the events to the store if questore is configured, which will be replayed when the PostgreSQL connection is active. +func (target *PostgreSQLTarget) Save(eventData event.Event) error { + if target.store != nil { + _, err := target.store.Put(eventData) + return err + } + + if err := target.init(); err != nil { + return err + } + + _, err := target.isActive() + if err != nil { + return err + } + return target.send(eventData) +} + +// IsConnErr - To detect a connection error. +func IsConnErr(err error) bool { + return xnet.IsConnRefusedErr(err) || err.Error() == "sql: database is closed" || err.Error() == "sql: statement is closed" || err.Error() == "invalid connection" +} + +// send - sends an event to the PostgreSQL. +func (target *PostgreSQLTarget) send(eventData event.Event) error { + if target.args.Format == event.NamespaceFormat { + objectName, err := url.QueryUnescape(eventData.S3.Object.Key) + if err != nil { + return err + } + key := eventData.S3.Bucket.Name + "/" + objectName + + if eventData.EventName == event.ObjectRemovedDelete { + _, err = target.deleteStmt.Exec(key) + } else { + var data []byte + if data, err = json.Marshal(struct{ Records []event.Event }{[]event.Event{eventData}}); err != nil { + return err + } + + _, err = target.updateStmt.Exec(key, data) + } + return err + } + + if target.args.Format == event.AccessFormat { + eventTime, err := time.Parse(event.AMZTimeFormat, eventData.EventTime) + if err != nil { + return err + } + + data, err := json.Marshal(struct{ Records []event.Event }{[]event.Event{eventData}}) + if err != nil { + return err + } + + if _, err = target.insertStmt.Exec(eventTime, data); err != nil { + return err + } + } + + return nil +} + +// SendFromStore - reads an event from store and sends it to PostgreSQL. +func (target *PostgreSQLTarget) SendFromStore(key store.Key) error { + if err := target.init(); err != nil { + return err + } + + _, err := target.isActive() + if err != nil { + return err + } + if !target.firstPing { + if err := target.executeStmts(); err != nil { + if IsConnErr(err) { + return store.ErrNotConnected + } + return err + } + } + + eventData, eErr := target.store.Get(key) + if eErr != nil { + // The last event key in a successful batch will be sent in the channel atmost once by the replayEvents() + // Such events will not exist and wouldve been already been sent successfully. + if os.IsNotExist(eErr) { + return nil + } + return eErr + } + + if err := target.send(eventData); err != nil { + if IsConnErr(err) { + return store.ErrNotConnected + } + return err + } + + // Delete the event from store. + return target.store.Del(key) +} + +// Close - closes underneath connections to PostgreSQL database. +func (target *PostgreSQLTarget) Close() error { + close(target.quitCh) + if target.updateStmt != nil { + // FIXME: log returned error. ignore time being. + _ = target.updateStmt.Close() + } + + if target.deleteStmt != nil { + // FIXME: log returned error. ignore time being. + _ = target.deleteStmt.Close() + } + + if target.insertStmt != nil { + // FIXME: log returned error. ignore time being. + _ = target.insertStmt.Close() + } + + if target.db != nil { + target.db.Close() + } + + return nil +} + +// Executes the table creation statements. +func (target *PostgreSQLTarget) executeStmts() error { + _, err := target.db.Exec(fmt.Sprintf(psqlTableExists, target.args.Table)) + if err != nil { + createStmt := psqlCreateNamespaceTable + if target.args.Format == event.AccessFormat { + createStmt = psqlCreateAccessTable + } + + if _, dbErr := target.db.Exec(fmt.Sprintf(createStmt, target.args.Table)); dbErr != nil { + return dbErr + } + } + + switch target.args.Format { + case event.NamespaceFormat: + // insert or update statement + if target.updateStmt, err = target.db.Prepare(fmt.Sprintf(psqlUpdateRow, target.args.Table)); err != nil { + return err + } + // delete statement + if target.deleteStmt, err = target.db.Prepare(fmt.Sprintf(psqlDeleteRow, target.args.Table)); err != nil { + return err + } + case event.AccessFormat: + // insert statement + if target.insertStmt, err = target.db.Prepare(fmt.Sprintf(psqlInsertRow, target.args.Table)); err != nil { + return err + } + } + + return nil +} + +func (target *PostgreSQLTarget) init() error { + return target.initOnce.Do(target.initPostgreSQL) +} + +func (target *PostgreSQLTarget) initPostgreSQL() error { + args := target.args + + db, err := sql.Open("postgres", target.connString) + if err != nil { + return err + } + target.db = db + + if args.MaxOpenConnections > 0 { + // Set the maximum connections limit + target.db.SetMaxOpenConns(args.MaxOpenConnections) + } + + err = target.db.Ping() + if err != nil { + if !xnet.IsConnRefusedErr(err) && !xnet.IsConnResetErr(err) { + target.loggerOnce(context.Background(), err, target.ID().String()) + } + } else { + if err = target.executeStmts(); err != nil { + target.loggerOnce(context.Background(), err, target.ID().String()) + } else { + target.firstPing = true + } + } + + if err != nil { + target.db.Close() + return err + } + + yes, err := target.isActive() + if err != nil { + return err + } + if !yes { + return store.ErrNotConnected + } + + return nil +} + +// NewPostgreSQLTarget - creates new PostgreSQL target. +func NewPostgreSQLTarget(id string, args PostgreSQLArgs, loggerOnce logger.LogOnce) (*PostgreSQLTarget, error) { + params := []string{args.ConnectionString} + if args.ConnectionString == "" { + params = []string{} + if !args.Host.IsEmpty() { + params = append(params, "host="+args.Host.String()) + } + if args.Port != "" { + params = append(params, "port="+args.Port) + } + if args.Username != "" { + params = append(params, "username="+args.Username) + } + if args.Password != "" { + params = append(params, "password="+args.Password) + } + if args.Database != "" { + params = append(params, "dbname="+args.Database) + } + } + connStr := strings.Join(params, " ") + + var queueStore store.Store[event.Event] + if args.QueueDir != "" { + queueDir := filepath.Join(args.QueueDir, storePrefix+"-postgresql-"+id) + queueStore = store.NewQueueStore[event.Event](queueDir, args.QueueLimit, event.StoreExtension) + if err := queueStore.Open(); err != nil { + return nil, fmt.Errorf("unable to initialize the queue store of PostgreSQL `%s`: %w", id, err) + } + } + + target := &PostgreSQLTarget{ + id: event.TargetID{ID: id, Name: "postgresql"}, + args: args, + firstPing: false, + store: queueStore, + connString: connStr, + loggerOnce: loggerOnce, + quitCh: make(chan struct{}), + } + + if target.store != nil { + store.StreamItems(target.store, target, target.quitCh, target.loggerOnce) + } + + return target, nil +} + +var errInvalidPsqlTablename = errors.New("invalid PostgreSQL table") + +func validatePsqlTableName(name string) error { + // check for quoted string (string may not contain a quote) + if match, err := regexp.MatchString("^\"[^\"]+\"$", name); err != nil { + return err + } else if match { + return nil + } + + // normalize the name to letters, digits, _ or $ + valid := true + cleaned := strings.Map(func(r rune) rune { + switch { + case unicode.IsLetter(r): + return 'a' + case unicode.IsDigit(r): + return '0' + case r == '_', r == '$': + return r + default: + valid = false + return -1 + } + }, name) + + if valid { + // check for simple name or quoted name + // - letter/underscore followed by one or more letter/digit/underscore + // - any text between quotes (text cannot contain a quote itself) + if match, err := regexp.MatchString("^[a_][a0_$]*$", cleaned); err != nil { + return err + } else if match { + return nil + } + } + + return errInvalidPsqlTablename +} diff --git a/internal/event/target/postgresql_test.go b/internal/event/target/postgresql_test.go new file mode 100644 index 00000000..9b5130e2 --- /dev/null +++ b/internal/event/target/postgresql_test.go @@ -0,0 +1,52 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package target + +import ( + "database/sql" + "slices" + "testing" +) + +// TestPostgreSQLRegistration checks if postgres driver +// is registered and fails otherwise. +func TestPostgreSQLRegistration(t *testing.T) { + var found bool + if slices.Contains(sql.Drivers(), "postgres") { + found = true + } + if !found { + t.Fatal("postgres driver not registered") + } +} + +func TestPsqlTableNameValidation(t *testing.T) { + validTables := []string{"táblë", "table", "TableName", "\"Table name\"", "\"✅✅\"", "table$one", "\"táblë\""} + invalidTables := []string{"table name", "table \"name\"", "✅✅", "$table$"} + + for _, name := range validTables { + if err := validatePsqlTableName(name); err != nil { + t.Errorf("Should be valid: %s - %s", name, err) + } + } + for _, name := range invalidTables { + if err := validatePsqlTableName(name); err != errInvalidPsqlTablename { + t.Errorf("Should be invalid: %s - %s", name, err) + } + } +} diff --git a/internal/event/target/redis.go b/internal/event/target/redis.go new file mode 100644 index 00000000..8f7d4272 --- /dev/null +++ b/internal/event/target/redis.go @@ -0,0 +1,382 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package target + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + "time" + + "github.com/gomodule/redigo/redis" + "github.com/minio/minio/internal/event" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/once" + "github.com/minio/minio/internal/store" + xnet "github.com/minio/pkg/v3/net" +) + +// Redis constants +const ( + RedisFormat = "format" + RedisAddress = "address" + RedisPassword = "password" + RedisUser = "user" + RedisKey = "key" + RedisQueueDir = "queue_dir" + RedisQueueLimit = "queue_limit" + + EnvRedisEnable = "MINIO_NOTIFY_REDIS_ENABLE" + EnvRedisFormat = "MINIO_NOTIFY_REDIS_FORMAT" + EnvRedisAddress = "MINIO_NOTIFY_REDIS_ADDRESS" + EnvRedisPassword = "MINIO_NOTIFY_REDIS_PASSWORD" + EnvRedisUser = "MINIO_NOTIFY_REDIS_USER" + EnvRedisKey = "MINIO_NOTIFY_REDIS_KEY" + EnvRedisQueueDir = "MINIO_NOTIFY_REDIS_QUEUE_DIR" + EnvRedisQueueLimit = "MINIO_NOTIFY_REDIS_QUEUE_LIMIT" +) + +// RedisArgs - Redis target arguments. +type RedisArgs struct { + Enable bool `json:"enable"` + Format string `json:"format"` + Addr xnet.Host `json:"address"` + Password string `json:"password"` + User string `json:"user"` + Key string `json:"key"` + QueueDir string `json:"queueDir"` + QueueLimit uint64 `json:"queueLimit"` +} + +// RedisAccessEvent holds event log data and timestamp +type RedisAccessEvent struct { + Event []event.Event + EventTime string +} + +// Validate RedisArgs fields +func (r RedisArgs) Validate() error { + if !r.Enable { + return nil + } + + if r.Format != "" { + f := strings.ToLower(r.Format) + if f != event.NamespaceFormat && f != event.AccessFormat { + return fmt.Errorf("unrecognized format") + } + } + + if r.Key == "" { + return fmt.Errorf("empty key") + } + + if r.QueueDir != "" { + if !filepath.IsAbs(r.QueueDir) { + return errors.New("queueDir path should be absolute") + } + } + + return nil +} + +func (r RedisArgs) validateFormat(c redis.Conn) error { + typeAvailable, err := redis.String(c.Do("TYPE", r.Key)) + if err != nil { + return err + } + + if typeAvailable != "none" { + expectedType := "hash" + if r.Format == event.AccessFormat { + expectedType = "list" + } + + if typeAvailable != expectedType { + return fmt.Errorf("expected type %v does not match with available type %v", expectedType, typeAvailable) + } + } + + return nil +} + +// RedisTarget - Redis target. +type RedisTarget struct { + initOnce once.Init + + id event.TargetID + args RedisArgs + pool *redis.Pool + store store.Store[event.Event] + firstPing bool + loggerOnce logger.LogOnce + quitCh chan struct{} +} + +// ID - returns target ID. +func (target *RedisTarget) ID() event.TargetID { + return target.id +} + +// Name - returns the Name of the target. +func (target *RedisTarget) Name() string { + return target.ID().String() +} + +// Store returns any underlying store if set. +func (target *RedisTarget) Store() event.TargetStore { + return target.store +} + +// IsActive - Return true if target is up and active +func (target *RedisTarget) IsActive() (bool, error) { + if err := target.init(); err != nil { + return false, err + } + return target.isActive() +} + +func (target *RedisTarget) isActive() (bool, error) { + conn := target.pool.Get() + defer conn.Close() + + _, pingErr := conn.Do("PING") + if pingErr != nil { + if xnet.IsConnRefusedErr(pingErr) { + return false, store.ErrNotConnected + } + return false, pingErr + } + return true, nil +} + +// Save - saves the events to the store if questore is configured, which will be replayed when the redis connection is active. +func (target *RedisTarget) Save(eventData event.Event) error { + if target.store != nil { + _, err := target.store.Put(eventData) + return err + } + if err := target.init(); err != nil { + return err + } + _, err := target.isActive() + if err != nil { + return err + } + return target.send(eventData) +} + +// send - sends an event to the redis. +func (target *RedisTarget) send(eventData event.Event) error { + conn := target.pool.Get() + defer conn.Close() + + if target.args.Format == event.NamespaceFormat { + objectName, err := url.QueryUnescape(eventData.S3.Object.Key) + if err != nil { + return err + } + key := eventData.S3.Bucket.Name + "/" + objectName + + if eventData.EventName == event.ObjectRemovedDelete { + _, err = conn.Do("HDEL", target.args.Key, key) + } else { + var data []byte + if data, err = json.Marshal(struct{ Records []event.Event }{[]event.Event{eventData}}); err != nil { + return err + } + + _, err = conn.Do("HSET", target.args.Key, key, data) + } + if err != nil { + return err + } + } + + if target.args.Format == event.AccessFormat { + data, err := json.Marshal([]RedisAccessEvent{{Event: []event.Event{eventData}, EventTime: eventData.EventTime}}) + if err != nil { + return err + } + if _, err := conn.Do("RPUSH", target.args.Key, data); err != nil { + return err + } + } + + return nil +} + +// SendFromStore - reads an event from store and sends it to redis. +func (target *RedisTarget) SendFromStore(key store.Key) error { + if err := target.init(); err != nil { + return err + } + + conn := target.pool.Get() + defer conn.Close() + + _, pingErr := conn.Do("PING") + if pingErr != nil { + if xnet.IsConnRefusedErr(pingErr) { + return store.ErrNotConnected + } + return pingErr + } + + if !target.firstPing { + if err := target.args.validateFormat(conn); err != nil { + if xnet.IsConnRefusedErr(err) { + return store.ErrNotConnected + } + return err + } + target.firstPing = true + } + + eventData, eErr := target.store.Get(key) + if eErr != nil { + // The last event key in a successful batch will be sent in the channel atmost once by the replayEvents() + // Such events will not exist and would've been already been sent successfully. + if os.IsNotExist(eErr) { + return nil + } + return eErr + } + + if err := target.send(eventData); err != nil { + if xnet.IsConnRefusedErr(err) { + return store.ErrNotConnected + } + return err + } + + // Delete the event from store. + return target.store.Del(key) +} + +// Close - releases the resources used by the pool. +func (target *RedisTarget) Close() error { + close(target.quitCh) + if target.pool != nil { + return target.pool.Close() + } + return nil +} + +func (target *RedisTarget) init() error { + return target.initOnce.Do(target.initRedis) +} + +func (target *RedisTarget) initRedis() error { + conn := target.pool.Get() + defer conn.Close() + + _, pingErr := conn.Do("PING") + if pingErr != nil { + if !xnet.IsConnRefusedErr(pingErr) && !xnet.IsConnResetErr(pingErr) { + target.loggerOnce(context.Background(), pingErr, target.ID().String()) + } + return pingErr + } + + if err := target.args.validateFormat(conn); err != nil { + target.loggerOnce(context.Background(), err, target.ID().String()) + return err + } + + target.firstPing = true + + yes, err := target.isActive() + if err != nil { + return err + } + if !yes { + return store.ErrNotConnected + } + + return nil +} + +// NewRedisTarget - creates new Redis target. +func NewRedisTarget(id string, args RedisArgs, loggerOnce logger.LogOnce) (*RedisTarget, error) { + var queueStore store.Store[event.Event] + if args.QueueDir != "" { + queueDir := filepath.Join(args.QueueDir, storePrefix+"-redis-"+id) + queueStore = store.NewQueueStore[event.Event](queueDir, args.QueueLimit, event.StoreExtension) + if err := queueStore.Open(); err != nil { + return nil, fmt.Errorf("unable to initialize the queue store of Redis `%s`: %w", id, err) + } + } + + pool := &redis.Pool{ + MaxIdle: 3, + IdleTimeout: 2 * 60 * time.Second, + Dial: func() (redis.Conn, error) { + conn, err := redis.Dial("tcp", args.Addr.String()) + if err != nil { + return nil, err + } + + if args.Password != "" { + if args.User != "" { + if _, err = conn.Do("AUTH", args.User, args.Password); err != nil { + conn.Close() + return nil, err + } + } else { + if _, err = conn.Do("AUTH", args.Password); err != nil { + conn.Close() + return nil, err + } + } + } + + // Must be done after AUTH + if _, err = conn.Do("CLIENT", "SETNAME", "MinIO"); err != nil { + conn.Close() + return nil, err + } + + return conn, nil + }, + TestOnBorrow: func(c redis.Conn, t time.Time) error { + _, err := c.Do("PING") + return err + }, + } + + target := &RedisTarget{ + id: event.TargetID{ID: id, Name: "redis"}, + args: args, + pool: pool, + store: queueStore, + loggerOnce: loggerOnce, + quitCh: make(chan struct{}), + } + + if target.store != nil { + store.StreamItems(target.store, target, target.quitCh, target.loggerOnce) + } + + return target, nil +} diff --git a/internal/event/target/testdata/contrib/certs/nats_client_cert.pem b/internal/event/target/testdata/contrib/certs/nats_client_cert.pem new file mode 100644 index 00000000..fd898272 --- /dev/null +++ b/internal/event/target/testdata/contrib/certs/nats_client_cert.pem @@ -0,0 +1,13 @@ +-----BEGIN CERTIFICATE----- +MIICCjCCAbGgAwIBAgIUKLFyLD0Ze9gR3A2aBxgEiT6MgZUwCgYIKoZIzj0EAwIw +GDEWMBQGA1UEAwwNTWluaW8gUm9vdCBDQTAeFw0yMDA5MTQxMzI0MzNaFw0zMDA5 +MTIxMzI0MzNaMEIxCzAJBgNVBAYTAkNBMQ4wDAYDVQQKDAVNaW5JTzEPMA0GA1UE +CwwGQ2xpZW50MRIwEAYDVQQDDAlsb2NhbGhvc3QwWTATBgcqhkjOPQIBBggqhkjO +PQMBBwNCAARAhYrQXYbzeKyVSw8nf57gBphwFP1o5S7CjxoGKCfghzdhExKiEmbi +sK+FSS2YtltU7cM7L7AduLIbuEnGHHYQo4GuMIGrMAkGA1UdEwQCMAAwUwYDVR0j +BEwwSoAUWN6Fr30E5vvvNOBkuGGkqGzA3SihHKQaMBgxFjAUBgNVBAMMDU1pbmlv +IFJvb3QgQ0GCFHiTsAON45VvwFb0MxHEdLPeWi95MA4GA1UdDwEB/wQEAwIFoDAd +BgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwGgYDVR0RBBMwEYcEfwAAAYIJ +bG9jYWxob3N0MAoGCCqGSM49BAMCA0cAMEQCIC7MHOEf0C/zqw/ZOaCffeJIMeFm +iT8ugBfhFbgGkd5YAiBz9FEfV4JMZQ4N29WLmvxxDSxkL8g5e3fnIK8Aa4excw== +-----END CERTIFICATE----- diff --git a/internal/event/target/testdata/contrib/certs/nats_client_key.pem b/internal/event/target/testdata/contrib/certs/nats_client_key.pem new file mode 100644 index 00000000..e3200637 --- /dev/null +++ b/internal/event/target/testdata/contrib/certs/nats_client_key.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIBluB2BuspJcz1e58rnXpQEx48/ZwNmygNw06NbdTZDroAoGCCqGSM49 +AwEHoUQDQgAEQIWK0F2G83islUsPJ3+e4AaYcBT9aOUuwo8aBign4Ic3YRMSohJm +4rCvhUktmLZbVO3DOy+wHbiyG7hJxhx2EA== +-----END EC PRIVATE KEY----- diff --git a/internal/event/target/testdata/contrib/certs/nats_server_cert.pem b/internal/event/target/testdata/contrib/certs/nats_server_cert.pem new file mode 100644 index 00000000..9ddc5959 --- /dev/null +++ b/internal/event/target/testdata/contrib/certs/nats_server_cert.pem @@ -0,0 +1,12 @@ +-----BEGIN CERTIFICATE----- +MIIByTCCAW+gAwIBAgIUdAg80BTm1El7s5ZZezgjsls9BwkwCgYIKoZIzj0EAwIw +GDEWMBQGA1UEAwwNTWluaW8gUm9vdCBDQTAeFw0yMDA5MTQxMjQzMjNaFw0zMDA5 +MTIxMjQzMjNaMAAwWTATBgcqhkjOPQIBBggqhkjOPQMBBwNCAASolKUI7FVSA2Ts ++GSW/DHDKNczDNjfccI2GLETso6ie8buveOODj1JIL9ff5pRDN+U6QvwwlDmXEqh +1a6XBI4Ho4GuMIGrMAkGA1UdEwQCMAAwUwYDVR0jBEwwSoAUWN6Fr30E5vvvNOBk +uGGkqGzA3SihHKQaMBgxFjAUBgNVBAMMDU1pbmlvIFJvb3QgQ0GCFHiTsAON45Vv +wFb0MxHEdLPeWi95MA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcD +AQYIKwYBBQUHAwIwGgYDVR0RBBMwEYcEfwAAAYIJbG9jYWxob3N0MAoGCCqGSM49 +BAMCA0gAMEUCIB7WXnQAkmjw2QE6A3uOscOIctJnlVNREfm4V9CrF6UGAiEA734B +vKlhMk8H459BRoIp8GpOuUWqLqocSmMM1febvcg= +-----END CERTIFICATE----- diff --git a/internal/event/target/testdata/contrib/certs/nats_server_key.pem b/internal/event/target/testdata/contrib/certs/nats_server_key.pem new file mode 100644 index 00000000..156f69c1 --- /dev/null +++ b/internal/event/target/testdata/contrib/certs/nats_server_key.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEILFuMS2xvsc/CsuqtSv3S2iSCcc28rZsg1wpR2kirXFloAoGCCqGSM49 +AwEHoUQDQgAEqJSlCOxVUgNk7PhklvwxwyjXMwzY33HCNhixE7KOonvG7r3jjg49 +SSC/X3+aUQzflOkL8MJQ5lxKodWulwSOBw== +-----END EC PRIVATE KEY----- diff --git a/internal/event/target/testdata/contrib/certs/root_ca_cert.pem b/internal/event/target/testdata/contrib/certs/root_ca_cert.pem new file mode 100644 index 00000000..12c578b9 --- /dev/null +++ b/internal/event/target/testdata/contrib/certs/root_ca_cert.pem @@ -0,0 +1,11 @@ +-----BEGIN CERTIFICATE----- +MIIBlTCCATygAwIBAgIUeJOwA43jlW/AVvQzEcR0s95aL3kwCgYIKoZIzj0EAwIw +GDEWMBQGA1UEAwwNTWluaW8gUm9vdCBDQTAeFw0yMDA5MTQxMjMwMDJaFw0zMDA5 +MTIxMjMwMDJaMBgxFjAUBgNVBAMMDU1pbmlvIFJvb3QgQ0EwWTATBgcqhkjOPQIB +BggqhkjOPQMBBwNCAARK9fVNGHc1h5B5fpOMyEdyhh18xNNcNUGQ5iGLO97Z0KtK +5vRlDeeE1I0SaJgqppm9OEHw32JU0HMi4FBZi2Rso2QwYjAdBgNVHQ4EFgQUWN6F +r30E5vvvNOBkuGGkqGzA3SgwHwYDVR0jBBgwFoAUWN6Fr30E5vvvNOBkuGGkqGzA +3SgwDwYDVR0TAQH/BAUwAwEB/zAPBgNVHREECDAGhwR/AAABMAoGCCqGSM49BAMC +A0cAMEQCIDPOiks2Vs3RmuJZl5HHjuqaFSOAp1g7pZpMb3Qrh9YDAiAtjO2xOpkS +WynK8P7EfyQP/IUa7GxJIoHk6/H/TCsYvQ== +-----END CERTIFICATE----- diff --git a/internal/event/target/testdata/contrib/certs/root_ca_key.pem b/internal/event/target/testdata/contrib/certs/root_ca_key.pem new file mode 100644 index 00000000..aea77090 --- /dev/null +++ b/internal/event/target/testdata/contrib/certs/root_ca_key.pem @@ -0,0 +1,5 @@ +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIB8tAGuc9FP4XbYqMP67TKgjL7OTrACGgEmTf+zMvYRhoAoGCCqGSM49 +AwEHoUQDQgAESvX1TRh3NYeQeX6TjMhHcoYdfMTTXDVBkOYhizve2dCrSub0ZQ3n +hNSNEmiYKqaZvThB8N9iVNBzIuBQWYtkbA== +-----END EC PRIVATE KEY----- diff --git a/internal/event/target/testdata/contrib/nats_tls.conf b/internal/event/target/testdata/contrib/nats_tls.conf new file mode 100644 index 00000000..d7c90b68 --- /dev/null +++ b/internal/event/target/testdata/contrib/nats_tls.conf @@ -0,0 +1,7 @@ +port: 14225 +net: localhost + +tls { + cert_file: "./testdata/contrib/certs/nats_server_cert.pem" + key_file: "./testdata/contrib/certs/nats_server_key.pem" +} diff --git a/internal/event/target/testdata/contrib/nats_tls_client_cert.conf b/internal/event/target/testdata/contrib/nats_tls_client_cert.conf new file mode 100644 index 00000000..c112581e --- /dev/null +++ b/internal/event/target/testdata/contrib/nats_tls_client_cert.conf @@ -0,0 +1,18 @@ +port: 14226 +net: localhost + +tls { + cert_file: "./testdata/contrib/certs/nats_server_cert.pem" + key_file: "./testdata/contrib/certs/nats_server_key.pem" + ca_file: "./testdata/contrib/certs/root_ca_cert.pem" + verify_and_map: true +} +authorization { + ADMIN = { + publish = ">" + subscribe = ">" + } + users = [ + {user: "CN=localhost,OU=Client,O=MinIO,C=CA", permissions: $ADMIN} + ] +} diff --git a/internal/event/target/testdata/contrib/nats_tls_handshake_first.conf b/internal/event/target/testdata/contrib/nats_tls_handshake_first.conf new file mode 100644 index 00000000..069eac48 --- /dev/null +++ b/internal/event/target/testdata/contrib/nats_tls_handshake_first.conf @@ -0,0 +1,8 @@ +port: 14227 +net: localhost + +tls { + cert_file: "./testdata/contrib/certs/nats_server_cert.pem" + key_file: "./testdata/contrib/certs/nats_server_key.pem" + handshake_first: true +} diff --git a/internal/event/target/testdata/contrib/test.nkey b/internal/event/target/testdata/contrib/test.nkey new file mode 100644 index 00000000..e75f2719 --- /dev/null +++ b/internal/event/target/testdata/contrib/test.nkey @@ -0,0 +1 @@ +SUACSSL3UAHUDXKFSNVUZRF5UHPMWZ6BFDTJ7M6USDXIEDNPPQYYYCU3VY \ No newline at end of file diff --git a/internal/event/target/webhook.go b/internal/event/target/webhook.go new file mode 100644 index 00000000..e5dc4f69 --- /dev/null +++ b/internal/event/target/webhook.go @@ -0,0 +1,316 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package target + +import ( + "bytes" + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "strings" + "syscall" + "time" + + "github.com/minio/minio/internal/event" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/once" + "github.com/minio/minio/internal/store" + "github.com/minio/pkg/v3/certs" + xnet "github.com/minio/pkg/v3/net" +) + +// Webhook constants +const ( + WebhookEndpoint = "endpoint" + WebhookAuthToken = "auth_token" + WebhookQueueDir = "queue_dir" + WebhookQueueLimit = "queue_limit" + WebhookClientCert = "client_cert" + WebhookClientKey = "client_key" + + EnvWebhookEnable = "MINIO_NOTIFY_WEBHOOK_ENABLE" + EnvWebhookEndpoint = "MINIO_NOTIFY_WEBHOOK_ENDPOINT" + EnvWebhookAuthToken = "MINIO_NOTIFY_WEBHOOK_AUTH_TOKEN" + EnvWebhookQueueDir = "MINIO_NOTIFY_WEBHOOK_QUEUE_DIR" + EnvWebhookQueueLimit = "MINIO_NOTIFY_WEBHOOK_QUEUE_LIMIT" + EnvWebhookClientCert = "MINIO_NOTIFY_WEBHOOK_CLIENT_CERT" + EnvWebhookClientKey = "MINIO_NOTIFY_WEBHOOK_CLIENT_KEY" +) + +// WebhookArgs - Webhook target arguments. +type WebhookArgs struct { + Enable bool `json:"enable"` + Endpoint xnet.URL `json:"endpoint"` + AuthToken string `json:"authToken"` + Transport *http.Transport `json:"-"` + QueueDir string `json:"queueDir"` + QueueLimit uint64 `json:"queueLimit"` + ClientCert string `json:"clientCert"` + ClientKey string `json:"clientKey"` +} + +// Validate WebhookArgs fields +func (w WebhookArgs) Validate() error { + if !w.Enable { + return nil + } + if w.Endpoint.IsEmpty() { + return errors.New("endpoint empty") + } + if w.QueueDir != "" { + if !filepath.IsAbs(w.QueueDir) { + return errors.New("queueDir path should be absolute") + } + } + if w.ClientCert != "" && w.ClientKey == "" || w.ClientCert == "" && w.ClientKey != "" { + return errors.New("cert and key must be specified as a pair") + } + return nil +} + +// WebhookTarget - Webhook target. +type WebhookTarget struct { + initOnce once.Init + + id event.TargetID + args WebhookArgs + transport *http.Transport + httpClient *http.Client + store store.Store[event.Event] + loggerOnce logger.LogOnce + cancel context.CancelFunc + cancelCh <-chan struct{} + + addr string // full address ip/dns with a port number, e.g. x.x.x.x:8080 +} + +// ID - returns target ID. +func (target *WebhookTarget) ID() event.TargetID { + return target.id +} + +// Name - returns the Name of the target. +func (target *WebhookTarget) Name() string { + return target.ID().String() +} + +// IsActive - Return true if target is up and active +func (target *WebhookTarget) IsActive() (bool, error) { + if err := target.init(); err != nil { + return false, err + } + return target.isActive() +} + +// Store returns any underlying store if set. +func (target *WebhookTarget) Store() event.TargetStore { + return target.store +} + +func (target *WebhookTarget) isActive() (bool, error) { + conn, err := net.DialTimeout("tcp", target.addr, 5*time.Second) + if err != nil { + if xnet.IsNetworkOrHostDown(err, false) { + return false, store.ErrNotConnected + } + return false, err + } + defer conn.Close() + return true, nil +} + +// Save - saves the events to the store if queuestore is configured, +// which will be replayed when the webhook connection is active. +func (target *WebhookTarget) Save(eventData event.Event) error { + if target.store != nil { + _, err := target.store.Put(eventData) + return err + } + if err := target.init(); err != nil { + return err + } + err := target.send(eventData) + if err != nil { + if xnet.IsNetworkOrHostDown(err, false) { + return store.ErrNotConnected + } + } + return err +} + +// send - sends an event to the webhook. +func (target *WebhookTarget) send(eventData event.Event) error { + objectName, err := url.QueryUnescape(eventData.S3.Object.Key) + if err != nil { + return err + } + key := eventData.S3.Bucket.Name + "/" + objectName + + data, err := json.Marshal(event.Log{EventName: eventData.EventName, Key: key, Records: []event.Event{eventData}}) + if err != nil { + return err + } + + req, err := http.NewRequest(http.MethodPost, target.args.Endpoint.String(), bytes.NewReader(data)) + if err != nil { + return err + } + + // Verify if the authToken already contains + // like format, if this is + // already present we can blindly use the + // authToken as is instead of adding 'Bearer' + tokens := strings.Fields(target.args.AuthToken) + switch len(tokens) { + case 2: + req.Header.Set("Authorization", target.args.AuthToken) + case 1: + req.Header.Set("Authorization", "Bearer "+target.args.AuthToken) + } + + req.Header.Set("Content-Type", "application/json") + + resp, err := target.httpClient.Do(req) + if err != nil { + return err + } + xhttp.DrainBody(resp.Body) + + if resp.StatusCode >= 200 && resp.StatusCode <= 299 { + // accepted HTTP status codes. + return nil + } else if resp.StatusCode == http.StatusForbidden { + return fmt.Errorf("%s returned '%s', please check if your auth token is correctly set", target.args.Endpoint, resp.Status) + } + return fmt.Errorf("%s returned '%s', please check your endpoint configuration", target.args.Endpoint, resp.Status) +} + +// SendFromStore - reads an event from store and sends it to webhook. +func (target *WebhookTarget) SendFromStore(key store.Key) error { + if err := target.init(); err != nil { + return err + } + + eventData, eErr := target.store.Get(key) + if eErr != nil { + // The last event key in a successful batch will be sent in the channel atmost once by the replayEvents() + // Such events will not exist and would've been already been sent successfully. + if os.IsNotExist(eErr) { + return nil + } + return eErr + } + + if err := target.send(eventData); err != nil { + if xnet.IsNetworkOrHostDown(err, false) { + return store.ErrNotConnected + } + return err + } + + // Delete the event from store. + return target.store.Del(key) +} + +// Close - does nothing and available for interface compatibility. +func (target *WebhookTarget) Close() error { + target.cancel() + return nil +} + +func (target *WebhookTarget) init() error { + return target.initOnce.Do(target.initWebhook) +} + +// Only called from init() +func (target *WebhookTarget) initWebhook() error { + args := target.args + transport := target.transport + + if args.ClientCert != "" && args.ClientKey != "" { + manager, err := certs.NewManager(context.Background(), args.ClientCert, args.ClientKey, tls.LoadX509KeyPair) + if err != nil { + return err + } + manager.ReloadOnSignal(syscall.SIGHUP) // allow reloads upon SIGHUP + transport.TLSClientConfig.GetClientCertificate = manager.GetClientCertificate + } + target.httpClient = &http.Client{Transport: transport} + + yes, err := target.isActive() + if err != nil { + return err + } + if !yes { + return store.ErrNotConnected + } + + return nil +} + +// NewWebhookTarget - creates new Webhook target. +func NewWebhookTarget(ctx context.Context, id string, args WebhookArgs, loggerOnce logger.LogOnce, transport *http.Transport) (*WebhookTarget, error) { + ctx, cancel := context.WithCancel(ctx) + + var queueStore store.Store[event.Event] + if args.QueueDir != "" { + queueDir := filepath.Join(args.QueueDir, storePrefix+"-webhook-"+id) + queueStore = store.NewQueueStore[event.Event](queueDir, args.QueueLimit, event.StoreExtension) + if err := queueStore.Open(); err != nil { + cancel() + return nil, fmt.Errorf("unable to initialize the queue store of Webhook `%s`: %w", id, err) + } + } + + target := &WebhookTarget{ + id: event.TargetID{ID: id, Name: "webhook"}, + args: args, + loggerOnce: loggerOnce, + transport: transport, + store: queueStore, + cancel: cancel, + cancelCh: ctx.Done(), + } + + // Calculate the webhook addr with the port number format + target.addr = args.Endpoint.Host + if _, _, err := net.SplitHostPort(args.Endpoint.Host); err != nil && strings.Contains(err.Error(), "missing port in address") { + switch strings.ToLower(args.Endpoint.Scheme) { + case "http": + target.addr += ":80" + case "https": + target.addr += ":443" + default: + return nil, errors.New("unsupported scheme") + } + } + + if target.store != nil { + store.StreamItems(target.store, target, target.cancelCh, target.loggerOnce) + } + + return target, nil +} diff --git a/internal/event/targetid.go b/internal/event/targetid.go new file mode 100644 index 00000000..7d8664fe --- /dev/null +++ b/internal/event/targetid.go @@ -0,0 +1,74 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "encoding/json" + "fmt" + "strings" +) + +// TargetID - holds identification and name strings of notification target. +type TargetID struct { + ID string + Name string +} + +// String - returns string representation. +func (tid TargetID) String() string { + return tid.ID + ":" + tid.Name +} + +// ToARN - converts to ARN. +func (tid TargetID) ToARN(region string) ARN { + return ARN{TargetID: tid, region: region} +} + +// MarshalJSON - encodes to JSON data. +func (tid TargetID) MarshalJSON() ([]byte, error) { + return json.Marshal(tid.String()) +} + +// UnmarshalJSON - decodes JSON data. +func (tid *TargetID) UnmarshalJSON(data []byte) error { + var s string + if err := json.Unmarshal(data, &s); err != nil { + return err + } + + targetID, err := parseTargetID(s) + if err != nil { + return err + } + + *tid = *targetID + return nil +} + +// parseTargetID - parses string to TargetID. +func parseTargetID(s string) (*TargetID, error) { + tokens := strings.Split(s, ":") + if len(tokens) != 2 { + return nil, fmt.Errorf("invalid TargetID format '%v'", s) + } + + return &TargetID{ + ID: tokens[0], + Name: tokens[1], + }, nil +} diff --git a/internal/event/targetid_test.go b/internal/event/targetid_test.go new file mode 100644 index 00000000..83d8c67d --- /dev/null +++ b/internal/event/targetid_test.go @@ -0,0 +1,118 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "reflect" + "testing" +) + +func TestTargetDString(t *testing.T) { + testCases := []struct { + tid TargetID + expectedResult string + }{ + {TargetID{}, ":"}, + {TargetID{"1", "webhook"}, "1:webhook"}, + {TargetID{"httpclient+2e33cdee-fbec-4bdd-917e-7d8e3c5a2531", "localhost:55638"}, "httpclient+2e33cdee-fbec-4bdd-917e-7d8e3c5a2531:localhost:55638"}, + } + + for i, testCase := range testCases { + result := testCase.tid.String() + + if result != testCase.expectedResult { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestTargetDToARN(t *testing.T) { + tid := TargetID{"1", "webhook"} + testCases := []struct { + tid TargetID + region string + expectedARN ARN + }{ + {tid, "", ARN{TargetID: tid, region: ""}}, + {tid, "us-east-1", ARN{TargetID: tid, region: "us-east-1"}}, + } + + for i, testCase := range testCases { + arn := testCase.tid.ToARN(testCase.region) + + if arn != testCase.expectedARN { + t.Fatalf("test %v: ARN: expected: %v, got: %v", i+1, testCase.expectedARN, arn) + } + } +} + +func TestTargetDMarshalJSON(t *testing.T) { + testCases := []struct { + tid TargetID + expectedData []byte + expectErr bool + }{ + {TargetID{}, []byte(`":"`), false}, + {TargetID{"1", "webhook"}, []byte(`"1:webhook"`), false}, + {TargetID{"httpclient+2e33cdee-fbec-4bdd-917e-7d8e3c5a2531", "localhost:55638"}, []byte(`"httpclient+2e33cdee-fbec-4bdd-917e-7d8e3c5a2531:localhost:55638"`), false}, + } + + for i, testCase := range testCases { + data, err := testCase.tid.MarshalJSON() + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + + if !testCase.expectErr { + if !reflect.DeepEqual(data, testCase.expectedData) { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, string(testCase.expectedData), string(data)) + } + } + } +} + +func TestTargetDUnmarshalJSON(t *testing.T) { + testCases := []struct { + data []byte + expectedTargetID *TargetID + expectErr bool + }{ + {[]byte(`""`), nil, true}, + {[]byte(`"httpclient+2e33cdee-fbec-4bdd-917e-7d8e3c5a2531:localhost:55638"`), nil, true}, + {[]byte(`":"`), &TargetID{}, false}, + {[]byte(`"1:webhook"`), &TargetID{"1", "webhook"}, false}, + } + + for i, testCase := range testCases { + targetID := &TargetID{} + err := targetID.UnmarshalJSON(testCase.data) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + + if !testCase.expectErr { + if *targetID != *testCase.expectedTargetID { + t.Fatalf("test %v: TargetID: expected: %v, got: %v", i+1, testCase.expectedTargetID, targetID) + } + } + } +} diff --git a/internal/event/targetidset.go b/internal/event/targetidset.go new file mode 100644 index 00000000..88fa9177 --- /dev/null +++ b/internal/event/targetidset.go @@ -0,0 +1,67 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import "maps" + +// TargetIDSet - Set representation of TargetIDs. +type TargetIDSet map[TargetID]struct{} + +// Clone - returns copy of this set. +func (set TargetIDSet) Clone() TargetIDSet { + setCopy := NewTargetIDSet() + maps.Copy(setCopy, set) + return setCopy +} + +// add - adds TargetID to the set. +func (set TargetIDSet) add(targetID TargetID) { + set[targetID] = struct{}{} +} + +// Union - returns union with given set as new set. +func (set TargetIDSet) Union(sset TargetIDSet) TargetIDSet { + nset := set.Clone() + + for k := range sset { + nset.add(k) + } + + return nset +} + +// Difference - returns difference with given set as new set. +func (set TargetIDSet) Difference(sset TargetIDSet) TargetIDSet { + nset := NewTargetIDSet() + for k := range set { + if _, ok := sset[k]; !ok { + nset.add(k) + } + } + + return nset +} + +// NewTargetIDSet - creates new TargetID set with given TargetIDs. +func NewTargetIDSet(targetIDs ...TargetID) TargetIDSet { + set := make(TargetIDSet) + for _, targetID := range targetIDs { + set.add(targetID) + } + return set +} diff --git a/internal/event/targetidset_test.go b/internal/event/targetidset_test.go new file mode 100644 index 00000000..6d9feafc --- /dev/null +++ b/internal/event/targetidset_test.go @@ -0,0 +1,110 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "reflect" + "testing" +) + +func TestTargetIDSetClone(t *testing.T) { + testCases := []struct { + set TargetIDSet + targetIDToAdd TargetID + }{ + {NewTargetIDSet(), TargetID{"1", "webhook"}}, + {NewTargetIDSet(TargetID{"1", "webhook"}), TargetID{"2", "webhook"}}, + {NewTargetIDSet(TargetID{"1", "webhook"}, TargetID{"2", "amqp"}), TargetID{"2", "webhook"}}, + } + + for i, testCase := range testCases { + result := testCase.set.Clone() + + if !reflect.DeepEqual(result, testCase.set) { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.set, result) + } + + result.add(testCase.targetIDToAdd) + if reflect.DeepEqual(result, testCase.set) { + t.Fatalf("test %v: result: expected: not equal, got: equal", i+1) + } + } +} + +func TestTargetIDSetUnion(t *testing.T) { + testCases := []struct { + set TargetIDSet + setToAdd TargetIDSet + expectedResult TargetIDSet + }{ + {NewTargetIDSet(), NewTargetIDSet(), NewTargetIDSet()}, + {NewTargetIDSet(), NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"1", "webhook"})}, + {NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(), NewTargetIDSet(TargetID{"1", "webhook"})}, + {NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"2", "amqp"}), NewTargetIDSet(TargetID{"1", "webhook"}, TargetID{"2", "amqp"})}, + {NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"1", "webhook"})}, + } + + for i, testCase := range testCases { + result := testCase.set.Union(testCase.setToAdd) + + if !reflect.DeepEqual(testCase.expectedResult, result) { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestTargetIDSetDifference(t *testing.T) { + testCases := []struct { + set TargetIDSet + setToRemove TargetIDSet + expectedResult TargetIDSet + }{ + {NewTargetIDSet(), NewTargetIDSet(), NewTargetIDSet()}, + {NewTargetIDSet(), NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet()}, + {NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(), NewTargetIDSet(TargetID{"1", "webhook"})}, + {NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"2", "amqp"}), NewTargetIDSet(TargetID{"1", "webhook"})}, + {NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet(TargetID{"1", "webhook"}), NewTargetIDSet()}, + } + + for i, testCase := range testCases { + result := testCase.set.Difference(testCase.setToRemove) + + if !reflect.DeepEqual(testCase.expectedResult, result) { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestNewTargetIDSet(t *testing.T) { + testCases := []struct { + targetIDs []TargetID + expectedResult TargetIDSet + }{ + {[]TargetID{}, NewTargetIDSet()}, + {[]TargetID{{"1", "webhook"}}, NewTargetIDSet(TargetID{"1", "webhook"})}, + {[]TargetID{{"1", "webhook"}, {"2", "amqp"}}, NewTargetIDSet(TargetID{"1", "webhook"}, TargetID{"2", "amqp"})}, + } + + for i, testCase := range testCases { + result := NewTargetIDSet(testCase.targetIDs...) + + if !reflect.DeepEqual(testCase.expectedResult, result) { + t.Fatalf("test %v: result: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} diff --git a/internal/event/targetlist.go b/internal/event/targetlist.go new file mode 100644 index 00000000..3aeee5d2 --- /dev/null +++ b/internal/event/targetlist.go @@ -0,0 +1,396 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "context" + "fmt" + "maps" + "runtime" + "sync" + "sync/atomic" + + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/store" + "github.com/minio/pkg/v3/workers" +) + +const ( + logSubsys = "notify" + + // The maximum allowed number of concurrent Send() calls to all configured notifications targets + maxConcurrentAsyncSend = 50000 +) + +// Target - event target interface +type Target interface { + ID() TargetID + IsActive() (bool, error) + Save(Event) error + SendFromStore(store.Key) error + Close() error + Store() TargetStore +} + +// TargetStore is a shallow version of a target.Store +type TargetStore interface { + Len() int +} + +// Stats is a collection of stats for multiple targets. +type Stats struct { + TotalEvents int64 // Deprecated + EventsSkipped int64 + CurrentQueuedCalls int64 // Deprecated + EventsErrorsTotal int64 // Deprecated + CurrentSendCalls int64 // Deprecated + + TargetStats map[TargetID]TargetStat +} + +// TargetStat is the stats of a single target. +type TargetStat struct { + CurrentSendCalls int64 // CurrentSendCalls is the number of concurrent async Send calls to all targets + CurrentQueue int // Populated if target has a store. + TotalEvents int64 + FailedEvents int64 // Number of failed events per target +} + +// TargetList - holds list of targets indexed by target ID. +type TargetList struct { + // The number of concurrent async Send calls to all targets + currentSendCalls atomic.Int64 + totalEvents atomic.Int64 + eventsSkipped atomic.Int64 + eventsErrorsTotal atomic.Int64 + + sync.RWMutex + targets map[TargetID]Target + queue chan asyncEvent + ctx context.Context + + statLock sync.RWMutex + targetStats map[TargetID]targetStat +} + +type targetStat struct { + // The number of concurrent async Send calls per targets + currentSendCalls int64 + // The number of total events per target + totalEvents int64 + // The number of failed events per target + failedEvents int64 +} + +func (list *TargetList) getStatsByTargetID(id TargetID) (stat targetStat) { + list.statLock.RLock() + defer list.statLock.RUnlock() + + return list.targetStats[id] +} + +func (list *TargetList) incCurrentSendCalls(id TargetID) { + list.statLock.Lock() + defer list.statLock.Unlock() + + stats, ok := list.targetStats[id] + if !ok { + stats = targetStat{} + } + + stats.currentSendCalls++ + list.targetStats[id] = stats +} + +func (list *TargetList) decCurrentSendCalls(id TargetID) { + list.statLock.Lock() + defer list.statLock.Unlock() + + stats, ok := list.targetStats[id] + if !ok { + // should not happen + return + } + + stats.currentSendCalls-- + list.targetStats[id] = stats +} + +func (list *TargetList) incFailedEvents(id TargetID) { + list.statLock.Lock() + defer list.statLock.Unlock() + + stats, ok := list.targetStats[id] + if !ok { + stats = targetStat{} + } + + stats.failedEvents++ + list.targetStats[id] = stats +} + +func (list *TargetList) incTotalEvents(id TargetID) { + list.statLock.Lock() + defer list.statLock.Unlock() + + stats, ok := list.targetStats[id] + if !ok { + stats = targetStat{} + } + + stats.totalEvents++ + list.targetStats[id] = stats +} + +type asyncEvent struct { + ev Event + targetSet TargetIDSet +} + +// Add - adds unique target to target list. +func (list *TargetList) Add(targets ...Target) error { + list.Lock() + defer list.Unlock() + + for _, target := range targets { + if _, ok := list.targets[target.ID()]; ok { + return fmt.Errorf("target %v already exists", target.ID()) + } + list.targets[target.ID()] = target + } + + return nil +} + +// Exists - checks whether target by target ID exists or not. +func (list *TargetList) Exists(id TargetID) bool { + list.RLock() + defer list.RUnlock() + + _, found := list.targets[id] + return found +} + +// TargetIDResult returns result of Remove/Send operation, sets err if +// any for the associated TargetID +type TargetIDResult struct { + // ID where the remove or send were initiated. + ID TargetID + // Stores any error while removing a target or while sending an event. + Err error +} + +// Remove - closes and removes targets by given target IDs. +func (list *TargetList) Remove(targetIDSet TargetIDSet) { + list.Lock() + defer list.Unlock() + + for id := range targetIDSet { + target, ok := list.targets[id] + if ok { + target.Close() + delete(list.targets, id) + } + } +} + +// Targets - list all targets +func (list *TargetList) Targets() []Target { + if list == nil { + return []Target{} + } + + list.RLock() + defer list.RUnlock() + + targets := []Target{} + for _, tgt := range list.targets { + targets = append(targets, tgt) + } + + return targets +} + +// List - returns available target IDs. +func (list *TargetList) List() []TargetID { + list.RLock() + defer list.RUnlock() + + keys := []TargetID{} + for k := range list.targets { + keys = append(keys, k) + } + + return keys +} + +func (list *TargetList) get(id TargetID) (Target, bool) { + list.RLock() + defer list.RUnlock() + + target, ok := list.targets[id] + return target, ok +} + +// TargetMap - returns available targets. +func (list *TargetList) TargetMap() map[TargetID]Target { + list.RLock() + defer list.RUnlock() + + ntargets := make(map[TargetID]Target, len(list.targets)) + maps.Copy(ntargets, list.targets) + return ntargets +} + +// Send - sends events to targets identified by target IDs. +func (list *TargetList) Send(event Event, targetIDset TargetIDSet, sync bool) { + if sync { + list.sendSync(event, targetIDset) + } else { + list.sendAsync(event, targetIDset) + } +} + +func (list *TargetList) sendSync(event Event, targetIDset TargetIDSet) { + var wg sync.WaitGroup + for id := range targetIDset { + target, ok := list.get(id) + if !ok { + continue + } + wg.Add(1) + go func(id TargetID, target Target) { + list.currentSendCalls.Add(1) + list.incCurrentSendCalls(id) + list.incTotalEvents(id) + defer list.decCurrentSendCalls(id) + defer list.currentSendCalls.Add(-1) + defer wg.Done() + + if err := target.Save(event); err != nil { + list.eventsErrorsTotal.Add(1) + list.incFailedEvents(id) + reqInfo := &logger.ReqInfo{} + reqInfo.AppendTags("targetID", id.String()) + logger.LogOnceIf(logger.SetReqInfo(context.Background(), reqInfo), logSubsys, err, id.String()) + } + }(id, target) + } + wg.Wait() + list.totalEvents.Add(1) +} + +func (list *TargetList) sendAsync(event Event, targetIDset TargetIDSet) { + select { + case list.queue <- asyncEvent{ + ev: event, + targetSet: targetIDset.Clone(), + }: + case <-list.ctx.Done(): + list.eventsSkipped.Add(int64(len(list.queue))) + return + default: + list.eventsSkipped.Add(1) + err := fmt.Errorf("concurrent target notifications exceeded %d, configured notification target is too slow to accept events for the incoming request rate", maxConcurrentAsyncSend) + for id := range targetIDset { + reqInfo := &logger.ReqInfo{} + reqInfo.AppendTags("targetID", id.String()) + logger.LogOnceIf(logger.SetReqInfo(context.Background(), reqInfo), logSubsys, err, id.String()) + } + return + } +} + +// Stats returns stats for targets. +func (list *TargetList) Stats() Stats { + t := Stats{} + if list == nil { + return t + } + t.CurrentSendCalls = list.currentSendCalls.Load() + t.EventsSkipped = list.eventsSkipped.Load() + t.TotalEvents = list.totalEvents.Load() + t.CurrentQueuedCalls = int64(len(list.queue)) + t.EventsErrorsTotal = list.eventsErrorsTotal.Load() + + list.RLock() + defer list.RUnlock() + t.TargetStats = make(map[TargetID]TargetStat, len(list.targets)) + for id, target := range list.targets { + var currentQueue int + if st := target.Store(); st != nil { + currentQueue = st.Len() + } + stats := list.getStatsByTargetID(id) + t.TargetStats[id] = TargetStat{ + CurrentSendCalls: stats.currentSendCalls, + CurrentQueue: currentQueue, + FailedEvents: stats.failedEvents, + TotalEvents: stats.totalEvents, + } + } + + return t +} + +func (list *TargetList) startSendWorkers(workerCount int) { + if workerCount == 0 { + workerCount = runtime.GOMAXPROCS(0) + } + wk, err := workers.New(workerCount) + if err != nil { + panic(err) + } + for range workerCount { + wk.Take() + go func() { + defer wk.Give() + + for { + select { + case av := <-list.queue: + list.sendSync(av.ev, av.targetSet) + case <-list.ctx.Done(): + return + } + } + }() + } + wk.Wait() +} + +var startOnce sync.Once + +// Init initialize target send workers. +func (list *TargetList) Init(workers int) *TargetList { + startOnce.Do(func() { + go list.startSendWorkers(workers) + }) + return list +} + +// NewTargetList - creates TargetList. +func NewTargetList(ctx context.Context) *TargetList { + list := &TargetList{ + targets: make(map[TargetID]Target), + queue: make(chan asyncEvent, maxConcurrentAsyncSend), + targetStats: make(map[TargetID]targetStat), + ctx: ctx, + } + return list +} diff --git a/internal/event/targetlist_test.go b/internal/event/targetlist_test.go new file mode 100644 index 00000000..f6aed658 --- /dev/null +++ b/internal/event/targetlist_test.go @@ -0,0 +1,225 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package event + +import ( + "crypto/rand" + "errors" + "reflect" + "testing" + "time" + + "github.com/minio/minio/internal/store" +) + +type ExampleTarget struct { + id TargetID + sendErr bool + closeErr bool +} + +func (target ExampleTarget) ID() TargetID { + return target.id +} + +// Save - Sends event directly without persisting. +func (target ExampleTarget) Save(eventData Event) error { + return target.send(eventData) +} + +// Store - Returns a nil store. +func (target ExampleTarget) Store() TargetStore { + return nil +} + +func (target ExampleTarget) send(eventData Event) error { + b := make([]byte, 1) + if _, err := rand.Read(b); err != nil { + panic(err) + } + + time.Sleep(time.Duration(b[0]) * time.Millisecond) + + if target.sendErr { + return errors.New("send error") + } + + return nil +} + +// SendFromStore - interface compatible method does no-op. +func (target *ExampleTarget) SendFromStore(_ store.Key) error { + return nil +} + +func (target ExampleTarget) Close() error { + if target.closeErr { + return errors.New("close error") + } + + return nil +} + +func (target ExampleTarget) IsActive() (bool, error) { + return false, errors.New("not connected to target server/service") +} + +// FlushQueueStore - No-Op. Added for interface compatibility +func (target ExampleTarget) FlushQueueStore() error { + return nil +} + +func TestTargetListAdd(t *testing.T) { + targetListCase1 := NewTargetList(t.Context()) + + targetListCase2 := NewTargetList(t.Context()) + if err := targetListCase2.Add(&ExampleTarget{TargetID{"2", "testcase"}, false, false}); err != nil { + panic(err) + } + + targetListCase3 := NewTargetList(t.Context()) + if err := targetListCase3.Add(&ExampleTarget{TargetID{"3", "testcase"}, false, false}); err != nil { + panic(err) + } + + testCases := []struct { + targetList *TargetList + target Target + expectedResult []TargetID + expectErr bool + }{ + {targetListCase1, &ExampleTarget{TargetID{"1", "webhook"}, false, false}, []TargetID{{"1", "webhook"}}, false}, + {targetListCase2, &ExampleTarget{TargetID{"1", "webhook"}, false, false}, []TargetID{{"2", "testcase"}, {"1", "webhook"}}, false}, + {targetListCase3, &ExampleTarget{TargetID{"3", "testcase"}, false, false}, nil, true}, + } + + for i, testCase := range testCases { + err := testCase.targetList.Add(testCase.target) + expectErr := (err != nil) + + if expectErr != testCase.expectErr { + t.Fatalf("test %v: error: expected: %v, got: %v", i+1, testCase.expectErr, expectErr) + } + + if !testCase.expectErr { + result := testCase.targetList.List() + + if len(result) != len(testCase.expectedResult) { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + + for _, targetID1 := range result { + var found bool + for _, targetID2 := range testCase.expectedResult { + if reflect.DeepEqual(targetID1, targetID2) { + found = true + break + } + } + if !found { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } + } + } +} + +func TestTargetListExists(t *testing.T) { + targetListCase1 := NewTargetList(t.Context()) + + targetListCase2 := NewTargetList(t.Context()) + if err := targetListCase2.Add(&ExampleTarget{TargetID{"2", "testcase"}, false, false}); err != nil { + panic(err) + } + + targetListCase3 := NewTargetList(t.Context()) + if err := targetListCase3.Add(&ExampleTarget{TargetID{"3", "testcase"}, false, false}); err != nil { + panic(err) + } + + testCases := []struct { + targetList *TargetList + targetID TargetID + expectedResult bool + }{ + {targetListCase1, TargetID{"1", "webhook"}, false}, + {targetListCase2, TargetID{"1", "webhook"}, false}, + {targetListCase3, TargetID{"3", "testcase"}, true}, + } + + for i, testCase := range testCases { + result := testCase.targetList.Exists(testCase.targetID) + + if result != testCase.expectedResult { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } +} + +func TestTargetListList(t *testing.T) { + targetListCase1 := NewTargetList(t.Context()) + + targetListCase2 := NewTargetList(t.Context()) + if err := targetListCase2.Add(&ExampleTarget{TargetID{"2", "testcase"}, false, false}); err != nil { + panic(err) + } + + targetListCase3 := NewTargetList(t.Context()) + if err := targetListCase3.Add(&ExampleTarget{TargetID{"3", "testcase"}, false, false}); err != nil { + panic(err) + } + if err := targetListCase3.Add(&ExampleTarget{TargetID{"1", "webhook"}, false, false}); err != nil { + panic(err) + } + + testCases := []struct { + targetList *TargetList + expectedResult []TargetID + }{ + {targetListCase1, []TargetID{}}, + {targetListCase2, []TargetID{{"2", "testcase"}}}, + {targetListCase3, []TargetID{{"3", "testcase"}, {"1", "webhook"}}}, + } + + for i, testCase := range testCases { + result := testCase.targetList.List() + + if len(result) != len(testCase.expectedResult) { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + + for _, targetID1 := range result { + var found bool + for _, targetID2 := range testCase.expectedResult { + if reflect.DeepEqual(targetID1, targetID2) { + found = true + break + } + } + if !found { + t.Fatalf("test %v: data: expected: %v, got: %v", i+1, testCase.expectedResult, result) + } + } + } +} + +func TestNewTargetList(t *testing.T) { + if result := NewTargetList(t.Context()); result == nil { + t.Fatalf("test: result: expected: , got: ") + } +} diff --git a/internal/grid/README.md b/internal/grid/README.md new file mode 100644 index 00000000..d098b24f --- /dev/null +++ b/internal/grid/README.md @@ -0,0 +1,251 @@ +# MinIO Grid + +The MinIO Grid is a package that provides two-way communication between servers. +It uses a single two-way connection to send and receive messages between servers. + +It includes built in muxing of concurrent requests as well as congestion handling for streams. + +Requests can be "Single Payload" or "Streamed". + +Use the MinIO Grid for: + +* Small, frequent requests with low latency requirements. +* Long-running requests with small/medium payloads. + +Do *not* use the MinIO Grid for: + +* Large payloads. + +Only a single connection is ever made between two servers. +Likely this means that this connection will not be able to saturate network bandwidth. +Therefore, using this for large payloads will likely be slower than using a separate connection, +and other connections will be blocked while the large payload is being sent. + +## Handlers & Routes + +Handlers have a predefined Handler ID. +In addition, there can be several *static* subroutes used to differentiate between different handlers of the same ID. +A subroute on a client must match a subroute on the server. So routes cannot be used for dynamic routing, unlike HTTP. + +Handlers should remain backwards compatible. If a breaking API change is required, a new handler ID should be created. + +## Setup & Configuration + +A **Manager** is used to manage all incoming and outgoing connections to a server. + +On startup all remote servers must be specified. +From that individual connections will be spawned to each remote server, +or incoming requests will be hooked up to the appropriate connection. + +To get a connection to a specific server, use `Manager.Connection(host)` to get a connection to the specified host. +From this connection individual requests can be made. + +Each handler, with optional subroutes can be registered with the manager using +`Manager.RegisterXHandler(handlerID, handler, subroutes...)`. + +A `Handler()` function provides an HTTP handler, which should be hooked up to the appropriate route on the server. + +On startup, the manager will start connecting to remotes and also starts listening for incoming connections. +Until a connection is established, all outgoing requests will return `ErrDisconnected`. + +# Usage + +## Single Payload Requests + +Single payload requests are requests and responses that are sent in a single message. +In essence, they are `[]byte` -> `[]byte, error` functions. + +It is not possible to return *both* an error and a response. + +Handlers are registered on the manager using `(*Manager).RegisterSingleHandler(id HandlerID, h SingleHandlerFn, subroute ...string)`. + +The server handler function has this signature: `type SingleHandlerFn func(payload []byte) ([]byte, *RemoteErr)`. + +Sample handler: +```go + handler := func(payload []byte) ([]byte, *grid.RemoteErr) { + // Do something with payload + return []byte("response"), nil + } + + err := manager.RegisterSingleHandler(grid.HandlerDiskInfo, handler) +``` + +Sample call: +```go + // Get a connection to the remote host + conn := manager.Connection(host) + + payload := []byte("request") + response, err := conn.SingleRequest(ctx, grid.HandlerDiskInfo, payload) +``` + +If the error type is `*RemoteErr`, then the error was returned by the remote server. Otherwise it is a local error. + +Context timeouts are propagated, and a default timeout of 1 minute is added if none is specified. + +There is no cancellation propagation for single payload requests. +When the context is canceled, the request will return at once with an appropriate error. +However, the remote call will not see the cancellation - as can be seen from the 'missing' context on the handler. +The result will be discarded. + +### Typed handlers + +Typed handlers are handlers that have a specific type for the request and response payloads. +These must provide `msgp` serialization and deserialization. + +In the examples we use a `MSS` type, which is a `map[string]string` that is `msgp` serializable. + +```go + handler := func(request *grid.MSS) (*grid.MSS, *grid.RemoteErr) { + fmt.Println("Got request with field", request["myfield"]) + // Do something with payload + return NewMSSWith(map[string]string{"result": "ok"}), nil + } + + // Create a typed handler. + // Due to current generics limitations, a constructor of the empty type must be provided. + instance := grid.NewSingleHandler[*grid.MSS, *grid.MSS](h, grid.NewMSS, grid.NewMSS) + + // Register the handler on the manager + instance.Register(manager, handler) + + // The typed instance is also used for calls + conn := manager.Connection("host") + resp, err := instance.Call(ctx, conn, grid.NewMSSWith(map[string]string{"myfield": "myvalue"})) + if err == nil { + fmt.Println("Got response with field", resp["result"]) + } +``` + +The wrapper will handle all serialization and de-serialization of the request and response, +and furthermore provides reuse of the structs used for the request and response. + +Note that Responses sent for serialization are automatically reused for similar requests. +If the response contains shared data it will cause issues, since each unique response is reused. +To disable this behavior, use `(SingleHandler).WithSharedResponse()` to disable it. + +## Streaming Requests + +Streams consists of an initial request with payload and allows for full two-way communication between the client and server. + +The handler function has this signature. + +Sample handler: +```go + handler := func(ctx context.Context, payload []byte, in <-chan []byte, out chan<- []byte) *RemoteErr { + fmt.Println("Got request with initial payload", p, "from", GetCaller(ctx context.Context)) + fmt.Println("Subroute:", GetSubroute(ctx)) + for { + select { + case <-ctx.Done(): + return nil + case req, ok := <-in: + if !ok { + break + } + // Do something with payload + out <- []byte("response") + + // Return the request for reuse + grid.PutByteBuffer(req) + } + } + // out is closed by the caller and should never be closed by the handler. + return nil + } + + err := manager.RegisterStreamingHandler(grid.HandlerDiskInfo, StreamHandler{ + Handle: handler, + Subroute: "asubroute", + OutCapacity: 1, + InCapacity: 1, + }) +``` + +Sample call: +```go + // Get a connection to the remote host + conn := manager.Connection(host).Subroute("asubroute") + + payload := []byte("request") + stream, err := conn.NewStream(ctx, grid.HandlerDiskInfo, payload) + if err != nil { + return err + } + // Read results from the stream + err = stream.Results(func(result []byte) error { + fmt.Println("Got result", string(result)) + + // Return the response for reuse + grid.PutByteBuffer(result) + return nil + }) +``` + +Context cancellation and timeouts are propagated to the handler. +The client does not wait for the remote handler to finish before returning. +Returning any error will also cancel the stream remotely. + +CAREFUL: When utilizing two-way communication, it is important to ensure that the remote handler is not blocked on a send. +If the remote handler is blocked on a send, and the client is trying to send without the remote receiving, +the operation would become deadlocked if the channels are full. + +### Typed handlers + +Typed handlers are handlers that have a specific type for the request and response payloads. + +```go + // Create a typed handler. + handler := func(ctx context.Context, p *Payload, in <-chan *Req, out chan<- *Resp) *RemoteErr { + fmt.Println("Got request with initial payload", p, "from", GetCaller(ctx context.Context)) + fmt.Println("Subroute:", GetSubroute(ctx)) + for { + select { + case <-ctx.Done(): + return nil + case req, ok := <-in: + if !ok { + break + } + fmt.Println("Got request", in) + // Do something with payload + out <- Resp{"response"} + } + // out is closed by the caller and should never be closed by the handler. + return nil + } + + // Create a typed handler. + // Due to current generics limitations, a constructor of the empty type must be provided. + instance := grid.NewStream[*Payload, *Req, *Resp](h, newPayload, newReq, newResp) + + // Tweakable options + instance.WithPayload = true // default true when newPayload != nil + instance.OutCapacity = 1 // default + instance.InCapacity = 1 // default true when newReq != nil + + // Register the handler on the manager + instance.Register(manager, handler, "asubroute") + + // The typed instance is also used for calls + conn := manager.Connection("host").Subroute("asubroute") + stream, err := instance.Call(ctx, conn, &Payload{"request payload"}) + if err != nil { ... } + + // Read results from the stream + err = stream.Results(func(resp *Resp) error { + fmt.Println("Got result", resp) + // Return the response for reuse + instance.PutResponse(resp) + return nil + }) +``` + +There are handlers for requests with: + * No input stream: `RegisterNoInput`. + * No initial payload: `RegisterNoPayload`. + +Note that Responses sent for serialization are automatically reused for similar requests. +If the response contains shared data it will cause issues, since each unique response is reused. +To disable this behavior, use `(StreamTypeHandler).WithSharedResponse()` to disable it. diff --git a/internal/grid/benchmark_test.go b/internal/grid/benchmark_test.go new file mode 100644 index 00000000..9bd17d1a --- /dev/null +++ b/internal/grid/benchmark_test.go @@ -0,0 +1,567 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package grid + +import ( + "context" + "fmt" + "math/rand" + "runtime" + "strconv" + "sync/atomic" + "testing" + "time" + + "github.com/minio/minio/internal/logger/target/testlogger" +) + +func BenchmarkRequests(b *testing.B) { + for n := 2; n <= 32; n *= 2 { + b.Run("servers="+strconv.Itoa(n), func(b *testing.B) { + benchmarkGridRequests(b, n) + }) + } +} + +func benchmarkGridRequests(b *testing.B, n int) { + defer testlogger.T.SetErrorTB(b)() + errFatal := func(err error) { + b.Helper() + if err != nil { + b.Fatal(err) + } + } + rpc := NewSingleHandler[*testRequest, *testResponse](handlerTest2, newTestRequest, newTestResponse) + grid, err := SetupTestGrid(n) + errFatal(err) + b.Cleanup(grid.Cleanup) + // Create n managers. + for _, remote := range grid.Managers { + // Register a single handler which echos the payload. + errFatal(remote.RegisterSingleHandler(handlerTest, func(payload []byte) ([]byte, *RemoteErr) { + defer PutByteBuffer(payload) + return append(GetByteBuffer()[:0], payload...), nil + })) + errFatal(rpc.Register(remote, func(req *testRequest) (resp *testResponse, err *RemoteErr) { + return &testResponse{ + OrgNum: req.Num, + OrgString: req.String, + Embedded: *req, + }, nil + })) + errFatal(err) + } + const payloadSize = 512 + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + payload := make([]byte, payloadSize) + _, err = rng.Read(payload) + errFatal(err) + + // Wait for all to connect + // Parallel writes per server. + b.Run("bytes", func(b *testing.B) { + for par := 1; par <= 32; par *= 2 { + b.Run("par="+strconv.Itoa(par*runtime.GOMAXPROCS(0)), func(b *testing.B) { + defer timeout(60 * time.Second)() + ctx, cancel := context.WithTimeout(b.Context(), 30*time.Second) + defer cancel() + b.ReportAllocs() + b.SetBytes(int64(len(payload) * 2)) + b.ResetTimer() + t := time.Now() + var ops int64 + var lat int64 + b.SetParallelism(par) + b.RunParallel(func(pb *testing.PB) { + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + n := 0 + var latency int64 + managers := grid.Managers + hosts := grid.Hosts + for pb.Next() { + // Pick a random manager. + src, dst := rng.Intn(len(managers)), rng.Intn(len(managers)) + if src == dst { + dst = (dst + 1) % len(managers) + } + local := managers[src] + conn := local.Connection(hosts[dst]) + if conn == nil { + b.Fatal("No connection") + } + // Send the payload. + t := time.Now() + resp, err := conn.Request(ctx, handlerTest, payload) + latency += time.Since(t).Nanoseconds() + if err != nil { + if debugReqs { + fmt.Println(err.Error()) + } + b.Fatal(err.Error()) + } + PutByteBuffer(resp) + n++ + } + atomic.AddInt64(&ops, int64(n)) + atomic.AddInt64(&lat, latency) + }) + spent := time.Since(t) + if spent > 0 && n > 0 { + // Since we are benchmarking n parallel servers we need to multiply by n. + // This will give an estimate of the total ops/s. + latency := float64(atomic.LoadInt64(&lat)) / float64(time.Millisecond) + b.ReportMetric(float64(n)*float64(ops)/spent.Seconds(), "vops/s") + b.ReportMetric(latency/float64(ops), "ms/op") + } + }) + } + }) + b.Run("rpc", func(b *testing.B) { + for par := 1; par <= 32; par *= 2 { + b.Run("par="+strconv.Itoa(par*runtime.GOMAXPROCS(0)), func(b *testing.B) { + defer timeout(60 * time.Second)() + ctx, cancel := context.WithTimeout(b.Context(), 30*time.Second) + defer cancel() + b.ReportAllocs() + b.ResetTimer() + t := time.Now() + var ops int64 + var lat int64 + b.SetParallelism(par) + b.RunParallel(func(pb *testing.PB) { + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + n := 0 + var latency int64 + managers := grid.Managers + hosts := grid.Hosts + req := testRequest{ + Num: rng.Int(), + String: "hello", + } + for pb.Next() { + // Pick a random manager. + src, dst := rng.Intn(len(managers)), rng.Intn(len(managers)) + if src == dst { + dst = (dst + 1) % len(managers) + } + local := managers[src] + conn := local.Connection(hosts[dst]) + if conn == nil { + b.Fatal("No connection") + } + // Send the payload. + t := time.Now() + resp, err := rpc.Call(ctx, conn, &req) + latency += time.Since(t).Nanoseconds() + if err != nil { + if debugReqs { + fmt.Println(err.Error()) + } + b.Fatal(err.Error()) + } + rpc.PutResponse(resp) + n++ + } + atomic.AddInt64(&ops, int64(n)) + atomic.AddInt64(&lat, latency) + }) + spent := time.Since(t) + if spent > 0 && n > 0 { + // Since we are benchmarking n parallel servers we need to multiply by n. + // This will give an estimate of the total ops/s. + latency := float64(atomic.LoadInt64(&lat)) / float64(time.Millisecond) + b.ReportMetric(float64(n)*float64(ops)/spent.Seconds(), "vops/s") + b.ReportMetric(latency/float64(ops), "ms/op") + } + }) + } + }) +} + +func BenchmarkStream(b *testing.B) { + tests := []struct { + name string + fn func(b *testing.B, n int) + }{ + {name: "request", fn: benchmarkGridStreamReqOnly}, + {name: "responses", fn: benchmarkGridStreamRespOnly}, + {name: "twoway", fn: benchmarkGridStreamTwoway}, + } + for _, test := range tests { + b.Run(test.name, func(b *testing.B) { + for n := 2; n <= 32; n *= 2 { + b.Run("servers="+strconv.Itoa(n), func(b *testing.B) { + test.fn(b, n) + }) + } + }) + } +} + +func benchmarkGridStreamRespOnly(b *testing.B, n int) { + defer testlogger.T.SetErrorTB(b)() + errFatal := func(err error) { + b.Helper() + if err != nil { + b.Fatal(err) + } + } + grid, err := SetupTestGrid(n) + errFatal(err) + b.Cleanup(grid.Cleanup) + const responses = 10 + // Create n managers. + for _, remote := range grid.Managers { + // Register a single handler which echos the payload. + errFatal(remote.RegisterStreamingHandler(handlerTest, StreamHandler{ + // Send 10x response. + Handle: func(ctx context.Context, payload []byte, _ <-chan []byte, out chan<- []byte) *RemoteErr { + for i := range responses { + toSend := GetByteBuffer()[:0] + toSend = append(toSend, byte(i)) + toSend = append(toSend, payload...) + select { + case <-ctx.Done(): + return nil + case out <- toSend: + } + } + return nil + }, + + Subroute: "some-subroute", + OutCapacity: 1, // Only one message buffered. + InCapacity: 0, + })) + errFatal(err) + } + const payloadSize = 512 + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + payload := make([]byte, payloadSize) + _, err = rng.Read(payload) + errFatal(err) + + // Wait for all to connect + // Parallel writes per server. + for par := 1; par <= 32; par *= 2 { + b.Run("par="+strconv.Itoa(par*runtime.GOMAXPROCS(0)), func(b *testing.B) { + defer timeout(30 * time.Second)() + b.ReportAllocs() + b.SetBytes(int64(len(payload) * (responses + 1))) + b.ResetTimer() + t := time.Now() + var ops int64 + var lat int64 + b.SetParallelism(par) + b.RunParallel(func(pb *testing.PB) { + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + n := 0 + var latency int64 + managers := grid.Managers + hosts := grid.Hosts + for pb.Next() { + // Pick a random manager. + src, dst := rng.Intn(len(managers)), rng.Intn(len(managers)) + if src == dst { + dst = (dst + 1) % len(managers) + } + local := managers[src] + conn := local.Connection(hosts[dst]).Subroute("some-subroute") + if conn == nil { + b.Fatal("No connection") + } + ctx, cancel := context.WithTimeout(b.Context(), 30*time.Second) + // Send the payload. + t := time.Now() + st, err := conn.NewStream(ctx, handlerTest, payload) + if err != nil { + if debugReqs { + fmt.Println(err.Error()) + } + b.Fatal(err.Error()) + } + got := 0 + err = st.Results(func(b []byte) error { + got++ + PutByteBuffer(b) + return nil + }) + if err != nil { + if debugReqs { + fmt.Println(err.Error()) + } + b.Fatal(err.Error()) + } + latency += time.Since(t).Nanoseconds() + cancel() + n += got + } + atomic.AddInt64(&ops, int64(n)) + atomic.AddInt64(&lat, latency) + }) + spent := time.Since(t) + if spent > 0 && n > 0 { + // Since we are benchmarking n parallel servers we need to multiply by n. + // This will give an estimate of the total ops/s. + latency := float64(atomic.LoadInt64(&lat)) / float64(time.Millisecond) + b.ReportMetric(float64(n)*float64(ops)/spent.Seconds(), "vops/s") + b.ReportMetric(latency/float64(ops), "ms/op") + } + }) + } +} + +func benchmarkGridStreamReqOnly(b *testing.B, n int) { + defer testlogger.T.SetErrorTB(b)() + errFatal := func(err error) { + b.Helper() + if err != nil { + b.Fatal(err) + } + } + grid, err := SetupTestGrid(n) + errFatal(err) + b.Cleanup(grid.Cleanup) + const requests = 10 + // Create n managers. + for _, remote := range grid.Managers { + // Register a single handler which echos the payload. + errFatal(remote.RegisterStreamingHandler(handlerTest, StreamHandler{ + // Send 10x requests. + Handle: func(ctx context.Context, payload []byte, in <-chan []byte, out chan<- []byte) *RemoteErr { + got := 0 + for b := range in { + PutByteBuffer(b) + got++ + } + if got != requests { + return NewRemoteErrf("wrong number of requests. want %d, got %d", requests, got) + } + return nil + }, + + Subroute: "some-subroute", + OutCapacity: 1, + InCapacity: 1, // Only one message buffered. + })) + errFatal(err) + } + const payloadSize = 512 + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + payload := make([]byte, payloadSize) + _, err = rng.Read(payload) + errFatal(err) + + // Wait for all to connect + // Parallel writes per server. + for par := 1; par <= 32; par *= 2 { + b.Run("par="+strconv.Itoa(par*runtime.GOMAXPROCS(0)), func(b *testing.B) { + defer timeout(30 * time.Second)() + b.ReportAllocs() + b.SetBytes(int64(len(payload) * (requests + 1))) + b.ResetTimer() + t := time.Now() + var ops int64 + var lat int64 + b.SetParallelism(par) + b.RunParallel(func(pb *testing.PB) { + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + n := 0 + var latency int64 + managers := grid.Managers + hosts := grid.Hosts + for pb.Next() { + // Pick a random manager. + src, dst := rng.Intn(len(managers)), rng.Intn(len(managers)) + if src == dst { + dst = (dst + 1) % len(managers) + } + local := managers[src] + conn := local.Connection(hosts[dst]).Subroute("some-subroute") + if conn == nil { + b.Fatal("No connection") + } + ctx, cancel := context.WithTimeout(b.Context(), 30*time.Second) + // Send the payload. + t := time.Now() + st, err := conn.NewStream(ctx, handlerTest, payload) + if err != nil { + if debugReqs { + fmt.Println(err.Error()) + } + b.Fatal(err.Error()) + } + got := 0 + for range requests { + got++ + st.Requests <- append(GetByteBuffer()[:0], payload...) + } + close(st.Requests) + err = st.Results(func(b []byte) error { + return nil + }) + if err != nil { + if debugReqs { + fmt.Println(err.Error()) + } + b.Fatal(err.Error()) + } + latency += time.Since(t).Nanoseconds() + cancel() + n += got + } + atomic.AddInt64(&ops, int64(n)) + atomic.AddInt64(&lat, latency) + }) + spent := time.Since(t) + if spent > 0 && n > 0 { + // Since we are benchmarking n parallel servers we need to multiply by n. + // This will give an estimate of the total ops/s. + latency := float64(atomic.LoadInt64(&lat)) / float64(time.Millisecond) + b.ReportMetric(float64(n)*float64(ops)/spent.Seconds(), "vops/s") + b.ReportMetric(latency/float64(ops), "ms/op") + } + }) + } +} + +func benchmarkGridStreamTwoway(b *testing.B, n int) { + defer testlogger.T.SetErrorTB(b)() + + errFatal := func(err error) { + b.Helper() + if err != nil { + b.Fatal(err) + } + } + grid, err := SetupTestGrid(n) + errFatal(err) + b.Cleanup(grid.Cleanup) + const messages = 10 + // Create n managers. + const payloadSize = 512 + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + payload := make([]byte, payloadSize) + _, err = rng.Read(payload) + errFatal(err) + + for _, remote := range grid.Managers { + // Register a single handler which echos the payload. + errFatal(remote.RegisterStreamingHandler(handlerTest, StreamHandler{ + // Send 10x requests. + Handle: func(ctx context.Context, payload []byte, in <-chan []byte, out chan<- []byte) *RemoteErr { + got := 0 + for b := range in { + out <- b + got++ + } + if got != messages { + return NewRemoteErrf("wrong number of requests. want %d, got %d", messages, got) + } + return nil + }, + + Subroute: "some-subroute", + OutCapacity: 1, + InCapacity: 1, // Only one message buffered. + })) + errFatal(err) + } + + // Wait for all to connect + // Parallel writes per server. + for par := 1; par <= 32; par *= 2 { + b.Run("par="+strconv.Itoa(par*runtime.GOMAXPROCS(0)), func(b *testing.B) { + defer timeout(30 * time.Second)() + b.ReportAllocs() + b.SetBytes(int64(len(payload) * (2*messages + 1))) + b.ResetTimer() + t := time.Now() + var ops int64 + var lat int64 + b.SetParallelism(par) + b.RunParallel(func(pb *testing.PB) { + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + n := 0 + var latency int64 + managers := grid.Managers + hosts := grid.Hosts + for pb.Next() { + // Pick a random manager. + src, dst := rng.Intn(len(managers)), rng.Intn(len(managers)) + if src == dst { + dst = (dst + 1) % len(managers) + } + local := managers[src] + conn := local.Connection(hosts[dst]).Subroute("some-subroute") + if conn == nil { + b.Fatal("No connection") + } + ctx, cancel := context.WithTimeout(b.Context(), 30*time.Second) + // Send the payload. + t := time.Now() + st, err := conn.NewStream(ctx, handlerTest, payload) + if err != nil { + if debugReqs { + fmt.Println(err.Error()) + } + b.Fatal(err.Error()) + } + got := 0 + sent := 0 + go func() { + for range messages { + st.Requests <- append(GetByteBuffer()[:0], payload...) + if sent++; sent == messages { + close(st.Requests) + return + } + } + }() + err = st.Results(func(b []byte) error { + got++ + PutByteBuffer(b) + return nil + }) + if err != nil { + if debugReqs { + fmt.Println(err.Error()) + } + b.Fatal(err.Error()) + } + if got != messages { + b.Fatalf("wrong number of responses. want %d, got %d", messages, got) + } + latency += time.Since(t).Nanoseconds() + cancel() + n += got + } + atomic.AddInt64(&ops, int64(n*2)) + atomic.AddInt64(&lat, latency) + }) + spent := time.Since(t) + if spent > 0 && n > 0 { + // Since we are benchmarking n parallel servers we need to multiply by n. + // This will give an estimate of the total ops/s. + latency := float64(atomic.LoadInt64(&lat)) / float64(time.Millisecond) + b.ReportMetric(float64(n)*float64(ops)/spent.Seconds(), "vops/s") + b.ReportMetric(latency/float64(ops), "ms/op") + } + }) + } +} diff --git a/internal/grid/connection.go b/internal/grid/connection.go new file mode 100644 index 00000000..576f4229 --- /dev/null +++ b/internal/grid/connection.go @@ -0,0 +1,1851 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package grid + +import ( + "bytes" + "context" + "encoding/binary" + "errors" + "fmt" + "io" + "math" + "math/rand" + "net" + "runtime" + "runtime/debug" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/gobwas/ws" + "github.com/gobwas/ws/wsutil" + "github.com/google/uuid" + "github.com/minio/madmin-go/v3" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/minio/minio/internal/logger" + "github.com/minio/minio/internal/pubsub" + xnet "github.com/minio/pkg/v3/net" + "github.com/puzpuzpuz/xsync/v3" + "github.com/tinylib/msgp/msgp" + "github.com/zeebo/xxh3" +) + +func gridLogIf(ctx context.Context, err error, errKind ...any) { + logger.LogIf(ctx, "grid", err, errKind...) +} + +func gridLogIfNot(ctx context.Context, err error, ignored ...error) { + logger.LogIfNot(ctx, "grid", err, ignored...) +} + +func gridLogOnceIf(ctx context.Context, err error, id string, errKind ...any) { + logger.LogOnceIf(ctx, "grid", err, id, errKind...) +} + +// A Connection is a remote connection. +// There is no distinction externally whether the connection was initiated from +// this server or from the remote. +type Connection struct { + // NextID is the next ID that can be used (atomic). + NextID uint64 + + // LastPong is last pong time (atomic) + // Only valid when StateConnected. + LastPong int64 + + // State of the connection (atomic) + state State + + // Non-atomic + Remote string + Local string + + // ID of this connection instance. + id uuid.UUID + + // Remote uuid, if we have been connected. + remoteID *uuid.UUID + reconnectMu sync.Mutex + + // Context for the server. + ctx context.Context + + // Active mux connections. + outgoing *xsync.MapOf[uint64, *muxClient] + + // Incoming streams + inStream *xsync.MapOf[uint64, *muxServer] + + // outQueue is the output queue + outQueue chan []byte + + // Client or serverside. + side ws.State + + // Dialer for outgoing connections. + dial ConnDialer + authFn AuthFn + + handleMsgWg sync.WaitGroup + + // connChange will be signaled whenever State has been updated, or at regular intervals. + // Holding the lock allows safe reads of State, and guarantees that changes will be detected. + connChange *sync.Cond + handlers *handlers + + remote *RemoteClient + clientPingInterval time.Duration + connPingInterval time.Duration + blockConnect chan struct{} + + incomingBytes func(n int64) // Record incoming bytes. + outgoingBytes func(n int64) // Record outgoing bytes. + trace *tracer // tracer for this connection. + baseFlags Flags + outBytes atomic.Int64 + inBytes atomic.Int64 + inMessages atomic.Int64 + outMessages atomic.Int64 + reconnects atomic.Int64 + lastConnect atomic.Pointer[time.Time] + lastPingDur atomic.Int64 + + // For testing only + debugInConn net.Conn + debugOutConn net.Conn + blockMessages atomic.Pointer[<-chan struct{}] + addDeadline time.Duration + connMu sync.Mutex +} + +// Subroute is a connection subroute that can be used to route to a specific handler with the same handler ID. +type Subroute struct { + *Connection + trace *tracer + route string + subID subHandlerID +} + +// String returns a string representation of the connection. +func (c *Connection) String() string { + return fmt.Sprintf("%s->%s", c.Local, c.Remote) +} + +// StringReverse returns a string representation of the reverse connection. +func (c *Connection) StringReverse() string { + return fmt.Sprintf("%s->%s", c.Remote, c.Local) +} + +// State is a connection state. +type State uint32 + +// MANUAL go:generate stringer -type=State -output=state_string.go -trimprefix=State $GOFILE + +const ( + // StateUnconnected is the initial state of a connection. + // When the first message is sent it will attempt to connect. + StateUnconnected = iota + + // StateConnecting is the state from StateUnconnected while the connection is attempted to be established. + // After this connection will be StateConnected or StateConnectionError. + StateConnecting + + // StateConnected is the state when the connection has been established and is considered stable. + // If the connection is lost, state will switch to StateConnecting. + StateConnected + + // StateConnectionError is the state once a connection attempt has been made, and it failed. + // The connection will remain in this stat until the connection has been successfully re-established. + StateConnectionError + + // StateShutdown is the state when the server has been shut down. + // This will not be used under normal operation. + StateShutdown + + // MaxDeadline is the maximum deadline allowed, + // Approx 49 days. + MaxDeadline = time.Duration(math.MaxUint32) * time.Millisecond +) + +// ContextDialer is a dialer that can be used to dial a remote. +type ContextDialer func(ctx context.Context, network, address string) (net.Conn, error) + +// DialContext implements the Dialer interface. +func (c ContextDialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { + return c(ctx, network, address) +} + +const ( + defaultOutQueue = 65535 // kind of close to max open fds per user + readBufferSize = 32 << 10 // 32 KiB is the most optimal on Linux + writeBufferSize = 32 << 10 // 32 KiB is the most optimal on Linux + defaultDialTimeout = 2 * time.Second + connPingInterval = 10 * time.Second + connWriteTimeout = 3 * time.Second +) + +type connectionParams struct { + ctx context.Context + id uuid.UUID + local, remote string + handlers *handlers + incomingBytes func(n int64) // Record incoming bytes. + outgoingBytes func(n int64) // Record outgoing bytes. + publisher *pubsub.PubSub[madmin.TraceInfo, madmin.TraceType] + dialer ConnDialer + authFn AuthFn + + blockConnect chan struct{} +} + +// newConnection will create an unconnected connection to a remote. +func newConnection(o connectionParams) *Connection { + c := &Connection{ + state: StateUnconnected, + Remote: o.remote, + Local: o.local, + id: o.id, + ctx: o.ctx, + outgoing: xsync.NewMapOfPresized[uint64, *muxClient](1000), + inStream: xsync.NewMapOfPresized[uint64, *muxServer](1000), + outQueue: make(chan []byte, defaultOutQueue), + side: ws.StateServerSide, + connChange: &sync.Cond{L: &sync.Mutex{}}, + handlers: o.handlers, + remote: &RemoteClient{Name: o.remote}, + clientPingInterval: clientPingInterval, + connPingInterval: connPingInterval, + dial: o.dialer, + authFn: o.authFn, + } + if debugPrint { + // Random Mux ID + c.NextID = rand.Uint64() + } + + // Record per connection stats. + c.outgoingBytes = func(n int64) { + if o.outgoingBytes != nil { + o.outgoingBytes(n) + } + c.outBytes.Add(n) + } + c.incomingBytes = func(n int64) { + if o.incomingBytes != nil { + o.incomingBytes(n) + } + c.inBytes.Add(n) + } + if !strings.HasPrefix(o.remote, "https://") && !strings.HasPrefix(o.remote, "wss://") { + c.baseFlags |= FlagCRCxxh3 + } + if !strings.HasPrefix(o.local, "https://") && !strings.HasPrefix(o.local, "wss://") { + c.baseFlags |= FlagCRCxxh3 + } + if o.publisher != nil { + c.traceRequests(o.publisher) + } + if o.local == o.remote { + panic("equal hosts") + } + if c.shouldConnect() { + c.side = ws.StateClientSide + + go func() { + if o.blockConnect != nil { + <-o.blockConnect + } + c.connect() + }() + } + if debugPrint { + fmt.Println(c.Local, "->", c.Remote, "Should local connect:", c.shouldConnect(), "side:", c.side) + } + if debugReqs { + fmt.Println("Created connection", c.String()) + } + return c +} + +// Subroute returns a static subroute for the connection. +func (c *Connection) Subroute(s string) *Subroute { + if c == nil { + return nil + } + return &Subroute{ + Connection: c, + route: s, + subID: makeSubHandlerID(0, s), + trace: c.trace.subroute(s), + } +} + +// Subroute adds a subroute to the subroute. +// The subroutes are combined with '/'. +func (c *Subroute) Subroute(s string) *Subroute { + route := strings.Join([]string{c.route, s}, "/") + return &Subroute{ + Connection: c.Connection, + route: route, + subID: makeSubHandlerID(0, route), + trace: c.trace.subroute(route), + } +} + +// newMuxClient returns a mux client for manual use. +func (c *Connection) newMuxClient(ctx context.Context) (*muxClient, error) { + client := newMuxClient(ctx, atomic.AddUint64(&c.NextID, 1), c) + if dl, ok := ctx.Deadline(); ok { + client.deadline = getDeadline(time.Until(dl)) + if client.deadline == 0 { + client.cancelFn(context.DeadlineExceeded) + return nil, context.DeadlineExceeded + } + } + for { + // Handle the extremely unlikely scenario that we wrapped. + if _, loaded := c.outgoing.LoadOrStore(client.MuxID, client); client.MuxID != 0 && !loaded { + if debugReqs { + _, found := c.outgoing.Load(client.MuxID) + fmt.Println(client.MuxID, c.String(), "Connection.newMuxClient: RELOADED MUX. loaded:", loaded, "found:", found) + } + return client, nil + } + client.MuxID = atomic.AddUint64(&c.NextID, 1) + } +} + +// newMuxClient returns a mux client for manual use. +func (c *Subroute) newMuxClient(ctx context.Context) (*muxClient, error) { + cl, err := c.Connection.newMuxClient(ctx) + if err != nil { + return nil, err + } + cl.subroute = &c.subID + return cl, nil +} + +// Request allows to do a single remote request. +// 'req' will not be used after the call and caller can reuse. +// If no deadline is set on ctx, a 1-minute deadline will be added. +func (c *Connection) Request(ctx context.Context, h HandlerID, req []byte) ([]byte, error) { + if !h.valid() { + return nil, ErrUnknownHandler + } + if c.State() != StateConnected { + return nil, ErrDisconnected + } + // Create mux client and call. + client, err := c.newMuxClient(ctx) + if err != nil { + return nil, err + } + defer func() { + if debugReqs { + _, ok := c.outgoing.Load(client.MuxID) + fmt.Println(client.MuxID, c.String(), "Connection.Request: DELETING MUX. Exists:", ok) + } + client.cancelFn(context.Canceled) + c.outgoing.Delete(client.MuxID) + }() + return client.traceRoundtrip(ctx, c.trace, h, req) +} + +// Request allows to do a single remote request. +// 'req' will not be used after the call and caller can reuse. +// If no deadline is set on ctx, a 1-minute deadline will be added. +func (c *Subroute) Request(ctx context.Context, h HandlerID, req []byte) ([]byte, error) { + if !h.valid() { + return nil, ErrUnknownHandler + } + if c.State() != StateConnected { + return nil, ErrDisconnected + } + // Create mux client and call. + client, err := c.newMuxClient(ctx) + if err != nil { + return nil, err + } + client.subroute = &c.subID + defer func() { + if debugReqs { + fmt.Println(client.MuxID, c.String(), "Subroute.Request: DELETING MUX") + } + client.cancelFn(context.Canceled) + c.outgoing.Delete(client.MuxID) + }() + return client.traceRoundtrip(ctx, c.trace, h, req) +} + +// NewStream creates a new stream. +// Initial payload can be reused by the caller. +func (c *Connection) NewStream(ctx context.Context, h HandlerID, payload []byte) (st *Stream, err error) { + if !h.valid() { + return nil, ErrUnknownHandler + } + if c.State() != StateConnected { + return nil, ErrDisconnected + } + handler := c.handlers.streams[h] + if handler == nil { + return nil, ErrUnknownHandler + } + + var requests chan []byte + var responses chan Response + if handler.InCapacity > 0 { + requests = make(chan []byte, handler.InCapacity) + } + if handler.OutCapacity > 0 { + responses = make(chan Response, handler.OutCapacity) + } else { + responses = make(chan Response, 1) + } + + cl, err := c.newMuxClient(ctx) + if err != nil { + return nil, err + } + + return cl.RequestStream(h, payload, requests, responses) +} + +// NewStream creates a new stream. +// Initial payload can be reused by the caller. +func (c *Subroute) NewStream(ctx context.Context, h HandlerID, payload []byte) (st *Stream, err error) { + if !h.valid() { + return nil, ErrUnknownHandler + } + if c.State() != StateConnected { + return nil, ErrDisconnected + } + handler := c.handlers.subStreams[makeZeroSubHandlerID(h)] + if handler == nil { + if debugPrint { + fmt.Println("want", makeZeroSubHandlerID(h), c.route, "got", c.handlers.subStreams) + } + return nil, ErrUnknownHandler + } + + var requests chan []byte + var responses chan Response + if handler.InCapacity > 0 { + requests = make(chan []byte, handler.InCapacity) + } + if handler.OutCapacity > 0 { + responses = make(chan Response, handler.OutCapacity) + } else { + responses = make(chan Response, 1) + } + + cl, err := c.newMuxClient(ctx) + if err != nil { + return nil, err + } + cl.subroute = &c.subID + + return cl.RequestStream(h, payload, requests, responses) +} + +// WaitForConnect will block until a connection has been established or +// the context is canceled, in which case the context error is returned. +func (c *Connection) WaitForConnect(ctx context.Context) error { + if debugPrint { + fmt.Println(c.Local, "->", c.Remote, "WaitForConnect") + defer fmt.Println(c.Local, "->", c.Remote, "WaitForConnect done") + } + c.connChange.L.Lock() + if atomic.LoadUint32((*uint32)(&c.state)) == StateConnected { + c.connChange.L.Unlock() + // Happy path. + return nil + } + ctx, cancel := context.WithCancel(ctx) + defer cancel() + changed := make(chan State, 1) + go func() { + defer xioutil.SafeClose(changed) + for { + c.connChange.Wait() + newState := c.State() + select { + case changed <- newState: + if newState == StateConnected || newState == StateShutdown { + c.connChange.L.Unlock() + return + } + case <-ctx.Done(): + c.connChange.L.Unlock() + return + } + } + }() + + for { + select { + case <-ctx.Done(): + return context.Cause(ctx) + case newState := <-changed: + if newState == StateConnected { + return nil + } + } + } +} + +/* +var ErrDone = errors.New("done for now") + +var ErrRemoteRestart = errors.New("remote restarted") + + +// Stateless connects to the remote handler and return all packets sent back. +// If the remote is restarted will return ErrRemoteRestart. +// If nil will be returned remote call sent EOF or ErrDone is returned by the callback. +// If ErrDone is returned on cb nil will be returned. +func (c *Connection) Stateless(ctx context.Context, h HandlerID, req []byte, cb func([]byte) error) error { + client, err := c.newMuxClient(ctx) + if err != nil { + return err + } + defer c.outgoing.Delete(client.MuxID) + resp := make(chan Response, 10) + client.RequestStateless(h, req, resp) + + for r := range resp { + if r.Err != nil { + return r.Err + } + if len(r.Msg) > 0 { + err := cb(r.Msg) + if err != nil { + if errors.Is(err, ErrDone) { + break + } + return err + } + } + } + return nil +} +*/ + +// shouldConnect returns a deterministic bool whether the local should initiate the connection. +// It should be 50% chance of any host initiating the connection. +func (c *Connection) shouldConnect() bool { + // The remote should have the opposite result. + h0 := xxh3.HashString(c.Local + c.Remote) + h1 := xxh3.HashString(c.Remote + c.Local) + if h0 == h1 { + return c.Local < c.Remote + } + return h0 < h1 +} + +func (c *Connection) send(ctx context.Context, msg []byte) error { + select { + case <-ctx.Done(): + // Returning error here is too noisy. + return nil + case c.outQueue <- msg: + return nil + } +} + +// queueMsg queues a message, with an optional payload. +// sender should not reference msg.Payload +func (c *Connection) queueMsg(msg message, payload sender) error { + // Add baseflags. + msg.Flags.Set(c.baseFlags) + // This cannot encode subroute. + msg.Flags.Clear(FlagSubroute) + if payload != nil { + if sz := payload.Msgsize(); cap(msg.Payload) < sz { + PutByteBuffer(msg.Payload) + msg.Payload = GetByteBufferCap(sz) + } + var err error + msg.Payload, err = payload.MarshalMsg(msg.Payload[:0]) + msg.Op = payload.Op() + if err != nil { + return err + } + } + defer PutByteBuffer(msg.Payload) + dst := GetByteBufferCap(msg.Msgsize()) + dst, err := msg.MarshalMsg(dst) + if err != nil { + return err + } + if msg.Flags&FlagCRCxxh3 != 0 { + h := xxh3.Hash(dst) + dst = binary.LittleEndian.AppendUint32(dst, uint32(h)) + } + return c.send(c.ctx, dst) +} + +// sendMsg will send +func (c *Connection) sendMsg(conn net.Conn, msg message, payload msgp.MarshalSizer) error { + if payload != nil { + if sz := payload.Msgsize(); cap(msg.Payload) < sz { + PutByteBuffer(msg.Payload) + msg.Payload = GetByteBufferCap(sz)[:0] + } + var err error + msg.Payload, err = payload.MarshalMsg(msg.Payload) + if err != nil { + return err + } + defer PutByteBuffer(msg.Payload) + } + dst := GetByteBufferCap(msg.Msgsize()) + dst, err := msg.MarshalMsg(dst) + if err != nil { + return err + } + if msg.Flags&FlagCRCxxh3 != 0 { + h := xxh3.Hash(dst) + dst = binary.LittleEndian.AppendUint32(dst, uint32(h)) + } + if debugPrint { + fmt.Println(c.Local, "sendMsg: Sending", msg.Op, "as", len(dst), "bytes") + } + if c.outgoingBytes != nil { + c.outgoingBytes(int64(len(dst))) + } + err = conn.SetWriteDeadline(time.Now().Add(connWriteTimeout)) + if err != nil { + return err + } + return wsutil.WriteMessage(conn, c.side, ws.OpBinary, dst) +} + +func (c *Connection) connect() { + c.updateState(StateConnecting) + rng := rand.New(rand.NewSource(time.Now().UnixNano())) + // Runs until the server is shut down. + for { + if c.State() == StateShutdown { + return + } + dialStarted := time.Now() + if debugPrint { + fmt.Println(c.Local, "Connecting to ", c.Remote) + } + conn, err := c.dial(c.ctx, c.Remote) + c.connMu.Lock() + c.debugOutConn = conn + c.connMu.Unlock() + retry := func(err error) { + if debugPrint { + fmt.Printf("%v Connecting to %v: %v. Retrying.\n", c.Local, c.Remote, err) + } + sleep := defaultDialTimeout + time.Duration(rng.Int63n(int64(defaultDialTimeout))) + next := dialStarted.Add(sleep / 2) + sleep = max(time.Until(next).Round(time.Millisecond), 0) + gotState := c.State() + if gotState == StateShutdown { + return + } + if gotState != StateConnecting { + // Don't print error on first attempt, and after that only once per hour. + gridLogOnceIf(c.ctx, fmt.Errorf("grid: %s re-connecting to %s: %w (%T) Sleeping %v (%v)", c.Local, c.Remote, err, err, sleep, gotState), c.Remote) + } + c.updateState(StateConnectionError) + time.Sleep(sleep) + } + if err != nil { + retry(err) + continue + } + // Send connect message. + m := message{ + Op: OpConnect, + } + req := connectReq{ + Host: c.Local, + ID: c.id, + Time: time.Now(), + } + req.addToken(c.authFn) + err = c.sendMsg(conn, m, &req) + if err != nil { + retry(err) + continue + } + // Wait for response + var r connectResp + err = c.receive(conn, &r) + if err != nil { + if debugPrint { + fmt.Println(c.Local, "receive err:", err, "side:", c.side) + } + retry(err) + continue + } + if debugPrint { + fmt.Println(c.Local, "Got connectResp:", r) + } + if !r.Accepted { + retry(fmt.Errorf("connection rejected: %s", r.RejectedReason)) + continue + } + t := time.Now().UTC() + c.lastConnect.Store(&t) + c.reconnectMu.Lock() + remoteUUID := uuid.UUID(r.ID) + if c.remoteID != nil { + c.reconnected() + } + c.remoteID = &remoteUUID + if debugPrint { + fmt.Println(c.Local, "Connected Waiting for Messages") + } + // Handle messages... + c.handleMessages(c.ctx, conn) + // Reconnect unless we are shutting down (debug only). + if c.State() == StateShutdown { + conn.Close() + return + } + if debugPrint { + fmt.Println(c.Local, "Disconnected. Attempting to reconnect.") + } + } +} + +func (c *Connection) disconnected() { + c.outgoing.Range(func(key uint64, client *muxClient) bool { + if !client.stateless { + client.cancelFn(ErrDisconnected) + } + return true + }) + if debugReqs { + fmt.Println(c.String(), "Disconnected. Clearing outgoing.") + } + c.outgoing.Clear() + c.inStream.Range(func(key uint64, client *muxServer) bool { + client.cancel() + return true + }) + c.inStream.Clear() +} + +func (c *Connection) receive(conn net.Conn, r receiver) error { + b, op, err := wsutil.ReadData(conn, c.side) + if err != nil { + return err + } + if op != ws.OpBinary { + return fmt.Errorf("unexpected connect response type %v", op) + } + if c.incomingBytes != nil { + c.incomingBytes(int64(len(b))) + } + + var m message + _, _, err = m.parse(b) + if err != nil { + return err + } + if m.Op != r.Op() { + return fmt.Errorf("unexpected response OP, want %v, got %v", r.Op(), m.Op) + } + _, err = r.UnmarshalMsg(m.Payload) + return err +} + +func (c *Connection) handleIncoming(ctx context.Context, conn net.Conn, req connectReq) error { + c.connMu.Lock() + c.debugInConn = conn + c.connMu.Unlock() + if c.blockConnect != nil { + // Block until we are allowed to connect. + <-c.blockConnect + } + if req.Host != c.Remote { + err := fmt.Errorf("expected remote '%s', got '%s'", c.Remote, req.Host) + if debugPrint { + fmt.Println(err) + } + return err + } + if c.shouldConnect() { + if debugPrint { + fmt.Println("expected to be client side, not server side") + } + return errors.New("grid: expected to be client side, not server side") + } + msg := message{ + Op: OpConnectResponse, + } + + resp := connectResp{ + ID: c.id, + Accepted: true, + } + err := c.sendMsg(conn, msg, &resp) + if debugPrint { + fmt.Printf("grid: Queued Response %+v Side: %v\n", resp, c.side) + } + if err != nil { + return err + } + t := time.Now().UTC() + c.lastConnect.Store(&t) + // Signal that we are reconnected, update state and handle messages. + // Prevent other connections from connecting while we process. + c.reconnectMu.Lock() + if c.remoteID != nil { + c.reconnected() + } + rid := uuid.UUID(req.ID) + c.remoteID = &rid + + // Handle incoming messages until disconnect. + c.handleMessages(ctx, conn) + return nil +} + +// reconnected signals the connection has been reconnected. +// It will close all active requests and streams. +// caller *must* hold reconnectMu. +func (c *Connection) reconnected() { + c.updateState(StateConnectionError) + c.reconnects.Add(1) + + // Drain the outQueue, so any blocked messages can be sent. + // We keep the queue, but start draining it, if it gets full. + stopDraining := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + defer func() { + close(stopDraining) + wg.Wait() + }() + go func() { + defer wg.Done() + for { + select { + case <-stopDraining: + return + default: + if cap(c.outQueue)-len(c.outQueue) > 100 { + // Queue is not full, wait a bit. + time.Sleep(1 * time.Millisecond) + continue + } + select { + case v := <-c.outQueue: + PutByteBuffer(v) + case <-stopDraining: + return + } + } + } + }() + // Close all active requests. + if debugReqs { + fmt.Println(c.String(), "Reconnected. Clearing outgoing.") + } + c.outgoing.Range(func(key uint64, client *muxClient) bool { + client.close() + return true + }) + c.inStream.Range(func(key uint64, value *muxServer) bool { + value.close() + return true + }) + + c.inStream.Clear() + c.outgoing.Clear() + + // Wait for existing to exit + c.handleMsgWg.Wait() +} + +func (c *Connection) updateState(s State) { + c.connChange.L.Lock() + defer c.connChange.L.Unlock() + + // We may have reads that aren't locked, so update atomically. + gotState := atomic.LoadUint32((*uint32)(&c.state)) + if gotState == StateShutdown || State(gotState) == s { + return + } + if s == StateConnected { + atomic.StoreInt64(&c.LastPong, time.Now().UnixNano()) + } + atomic.StoreUint32((*uint32)(&c.state), uint32(s)) + if debugPrint { + fmt.Println(c.Local, "updateState:", gotState, "->", s) + } + c.connChange.Broadcast() +} + +// monitorState will monitor the state of the connection and close the net.Conn if it changes. +func (c *Connection) monitorState(conn net.Conn, cancel context.CancelCauseFunc) { + c.connChange.L.Lock() + defer c.connChange.L.Unlock() + for { + newState := c.State() + if newState != StateConnected { + conn.Close() + cancel(ErrDisconnected) + return + } + // Unlock and wait for state change. + c.connChange.Wait() + } +} + +// handleMessages will handle incoming messages on conn. +// caller *must* hold reconnectMu. +func (c *Connection) handleMessages(ctx context.Context, conn net.Conn) { + c.updateState(StateConnected) + ctx, cancel := context.WithCancelCause(ctx) + defer cancel(ErrDisconnected) + + // This will ensure that is something asks to disconnect and we are blocked on reads/writes + // the connection will be closed and readers/writers will unblock. + go c.monitorState(conn, cancel) + + c.handleMsgWg.Add(2) + c.reconnectMu.Unlock() + + // Start reader and writer + go c.readStream(ctx, conn, cancel) + c.writeStream(ctx, conn, cancel) +} + +// readStream handles the read side of the connection. +// It will read messages and send them to c.handleMsg. +// If an error occurs the cancel function will be called and conn be closed. +// The function will block until the connection is closed or an error occurs. +func (c *Connection) readStream(ctx context.Context, conn net.Conn, cancel context.CancelCauseFunc) { + defer func() { + if rec := recover(); rec != nil { + gridLogIf(ctx, fmt.Errorf("handleMessages: panic recovered: %v", rec)) + debug.PrintStack() + } + cancel(ErrDisconnected) + c.connChange.L.Lock() + if atomic.CompareAndSwapUint32((*uint32)(&c.state), StateConnected, StateConnectionError) { + c.connChange.Broadcast() + } + c.connChange.L.Unlock() + conn.Close() + c.handleMsgWg.Done() + }() + + controlHandler := wsutil.ControlFrameHandler(conn, c.side) + wsReader := wsutil.Reader{ + Source: conn, + State: c.side, + CheckUTF8: true, + SkipHeaderCheck: false, + OnIntermediate: controlHandler, + } + readDataInto := func(dst []byte, s ws.State, want ws.OpCode) ([]byte, error) { + dst = dst[:0] + for { + hdr, err := wsReader.NextFrame() + if err != nil { + return nil, err + } + if hdr.OpCode.IsControl() { + if err := controlHandler(hdr, &wsReader); err != nil { + return nil, err + } + continue + } + if hdr.OpCode&want == 0 { + if err := wsReader.Discard(); err != nil { + return nil, err + } + continue + } + if int64(cap(dst)) < hdr.Length+1 { + dst = make([]byte, 0, hdr.Length+hdr.Length>>3) + } + if !hdr.Fin { + hdr.Length = -1 + } + return readAllInto(dst[:0], &wsReader, hdr.Length) + } + } + + // Keep reusing the same buffer. + var msg []byte + for atomic.LoadUint32((*uint32)(&c.state)) == StateConnected { + if cap(msg) > readBufferSize*4 { + // Don't keep too much memory around. + msg = nil + } + + var err error + msg, err = readDataInto(msg, c.side, ws.OpBinary) + if err != nil { + if !xnet.IsNetworkOrHostDown(err, true) { + gridLogIfNot(ctx, fmt.Errorf("ws read: %w", err), net.ErrClosed, io.EOF) + } + return + } + block := c.blockMessages.Load() + if block != nil && *block != nil { + <-*block + } + + if c.incomingBytes != nil { + c.incomingBytes(int64(len(msg))) + } + + // Parse the received message + var m message + subID, remain, err := m.parse(msg) + if err != nil { + if !xnet.IsNetworkOrHostDown(err, true) { + gridLogIf(ctx, fmt.Errorf("ws parse package: %w", err)) + } + return + } + if debugPrint { + fmt.Printf("%s Got msg: %v\n", c.Local, m) + } + if m.Op != OpMerged { + c.inMessages.Add(1) + c.handleMsg(ctx, m, subID) + continue + } + // Handle merged messages. + messages := int(m.Seq) + c.inMessages.Add(int64(messages)) + for range messages { + if atomic.LoadUint32((*uint32)(&c.state)) != StateConnected { + return + } + var next []byte + next, remain, err = msgp.ReadBytesZC(remain) + if err != nil { + if !xnet.IsNetworkOrHostDown(err, true) { + gridLogIf(ctx, fmt.Errorf("ws read merged: %w", err)) + } + return + } + + m.Payload = nil + subID, _, err = m.parse(next) + if err != nil { + if !xnet.IsNetworkOrHostDown(err, true) { + gridLogIf(ctx, fmt.Errorf("ws parse merged: %w", err)) + } + return + } + c.handleMsg(ctx, m, subID) + } + } +} + +// writeStream handles the read side of the connection. +// It will grab messages from c.outQueue and write them to the connection. +// If an error occurs the cancel function will be called and conn be closed. +// The function will block until the connection is closed or an error occurs. +func (c *Connection) writeStream(ctx context.Context, conn net.Conn, cancel context.CancelCauseFunc) { + defer func() { + if rec := recover(); rec != nil { + gridLogIf(ctx, fmt.Errorf("handleMessages: panic recovered: %v", rec)) + debug.PrintStack() + } + if debugPrint { + fmt.Println("handleMessages: write goroutine exited") + } + cancel(ErrDisconnected) + c.connChange.L.Lock() + if atomic.CompareAndSwapUint32((*uint32)(&c.state), StateConnected, StateConnectionError) { + c.connChange.Broadcast() + } + c.disconnected() + c.connChange.L.Unlock() + + conn.Close() + c.handleMsgWg.Done() + }() + + c.connMu.Lock() + connPingInterval := c.connPingInterval + c.connMu.Unlock() + ping := time.NewTicker(connPingInterval) + pingFrame := message{ + Op: OpPing, + DeadlineMS: uint32(connPingInterval.Milliseconds()), + Payload: make([]byte, pingMsg{}.Msgsize()), + } + + defer ping.Stop() + queue := make([][]byte, 0, maxMergeMessages) + var queueSize int + var buf bytes.Buffer + var wsw wsWriter + var lastSetDeadline time.Time + + // Helper to write everything in buf. + // Return false if an error occurred and the connection is unusable. + // Buffer will be reset empty when returning successfully. + writeBuffer := func() (ok bool) { + now := time.Now() + // Only set write deadline once every second + if now.Sub(lastSetDeadline) > time.Second { + err := conn.SetWriteDeadline(now.Add(connWriteTimeout + time.Second)) + if err != nil { + gridLogIf(ctx, fmt.Errorf("conn.SetWriteDeadline: %w", err)) + return false + } + lastSetDeadline = now + } + + _, err := buf.WriteTo(conn) + if err != nil { + if !xnet.IsNetworkOrHostDown(err, true) { + gridLogIf(ctx, fmt.Errorf("ws write: %w", err)) + } + return false + } + if buf.Cap() > writeBufferSize*8 { + // Reset buffer if it gets too big, so we don't keep it around. + buf = bytes.Buffer{} + } + buf.Reset() + return true + } + + // Merge buffer to keep between calls + merged := make([]byte, 0, writeBufferSize) + for { + var toSend []byte + select { + case <-ctx.Done(): + return + case <-ping.C: + if c.State() != StateConnected { + continue + } + lastPong := atomic.LoadInt64(&c.LastPong) + if lastPong > 0 { + lastPongTime := time.Unix(0, lastPong) + if d := time.Since(lastPongTime); d > connPingInterval*2 { + gridLogIf(ctx, fmt.Errorf("host %s last pong too old (%v); disconnecting", c.Remote, d.Round(time.Millisecond))) + return + } + } + ping := pingMsg{ + T: time.Now(), + } + var err error + if pingFrame.Payload, err = ping.MarshalMsg(pingFrame.Payload[:0]); err != nil { + gridLogIf(ctx, err) // Fake it... Though this should never fail. + atomic.StoreInt64(&c.LastPong, time.Now().UnixNano()) + continue + } + toSend, err = pingFrame.MarshalMsg(GetByteBuffer()[:0]) + if err != nil { + gridLogIf(ctx, err) // Fake it... Though this should never fail. + atomic.StoreInt64(&c.LastPong, time.Now().UnixNano()) + continue + } + case toSend = <-c.outQueue: + if len(toSend) == 0 { + continue + } + } + if len(queue) < maxMergeMessages && queueSize+len(toSend) < writeBufferSize-1024 { + if len(c.outQueue) == 0 { + // Yield to allow more messages to fill. + runtime.Gosched() + } + if len(c.outQueue) > 0 { + queue = append(queue, toSend) + queueSize += len(toSend) + continue + } + } + c.outMessages.Add(int64(len(queue) + 1)) + if c.outgoingBytes != nil { + c.outgoingBytes(int64(len(toSend) + queueSize)) + } + + c.connChange.L.Lock() + for { + state := c.State() + if state == StateConnected { + break + } + if debugPrint { + fmt.Println(c.Local, "Waiting for connection ->", c.Remote, "state: ", state) + } + if state == StateShutdown || state == StateConnectionError { + c.connChange.L.Unlock() + return + } + c.connChange.Wait() + select { + case <-ctx.Done(): + c.connChange.L.Unlock() + return + default: + } + } + c.connChange.L.Unlock() + if len(queue) == 0 { + // Send single message without merging. + err := wsw.writeMessage(&buf, c.side, ws.OpBinary, toSend) + if err != nil { + if !xnet.IsNetworkOrHostDown(err, true) { + gridLogIf(ctx, fmt.Errorf("ws writeMessage: %w", err)) + } + return + } + PutByteBuffer(toSend) + + if !writeBuffer() { + return + } + continue + } + + // Merge entries and send + queue = append(queue, toSend) + if debugPrint { + fmt.Println("Merging", len(queue), "messages") + } + + merged = merged[:0] + m := message{Op: OpMerged, Seq: uint32(len(queue))} + var err error + merged, err = m.MarshalMsg(merged) + if err != nil { + gridLogIf(ctx, fmt.Errorf("msg.MarshalMsg: %w", err)) + return + } + // Append as byte slices. + for _, q := range queue { + merged = msgp.AppendBytes(merged, q) + PutByteBuffer(q) + } + queue = queue[:0] + queueSize = 0 + + // Combine writes. + // Consider avoiding buffer copy. + err = wsw.writeMessage(&buf, c.side, ws.OpBinary, merged) + if err != nil { + if !xnet.IsNetworkOrHostDown(err, true) { + gridLogIf(ctx, fmt.Errorf("ws writeMessage: %w", err)) + } + return + } + if cap(merged) > writeBufferSize*8 { + // If we had to send an excessively large package, reset size. + merged = make([]byte, 0, writeBufferSize) + } + if !writeBuffer() { + return + } + } +} + +func (c *Connection) handleMsg(ctx context.Context, m message, subID *subHandlerID) { + switch m.Op { + case OpMuxServerMsg: + c.handleMuxServerMsg(ctx, m) + case OpResponse: + c.handleResponse(m) + case OpMuxClientMsg: + c.handleMuxClientMsg(ctx, m) + case OpUnblockSrvMux: + c.handleUnblockSrvMux(m) + case OpUnblockClMux: + c.handleUnblockClMux(m) + case OpDisconnectServerMux: + c.handleDisconnectServerMux(m) + case OpDisconnectClientMux: + c.handleDisconnectClientMux(m) + case OpPing: + c.handlePing(ctx, m) + case OpPong: + c.handlePong(ctx, m) + case OpRequest: + c.handleRequest(ctx, m, subID) + case OpAckMux: + c.handleAckMux(ctx, m) + case OpConnectMux: + c.handleConnectMux(ctx, m, subID) + case OpMuxConnectError: + c.handleConnectMuxError(ctx, m) + default: + gridLogIf(ctx, fmt.Errorf("unknown message type: %v", m.Op)) + } +} + +func (c *Connection) handleConnectMux(ctx context.Context, m message, subID *subHandlerID) { + // Stateless stream: + if m.Flags&FlagStateless != 0 { + // Reject for now, so we can safely add it later. + if true { + gridLogIf(ctx, c.queueMsg(m, muxConnectError{Error: "Stateless streams not supported"})) + return + } + + var handler *StatelessHandler + if subID == nil { + handler = c.handlers.stateless[m.Handler] + } else { + handler = c.handlers.subStateless[*subID] + } + if handler == nil { + msg := fmt.Sprintf("Invalid Handler for type: %v", m.Handler) + if subID != nil { + msg = fmt.Sprintf("Invalid Handler for type: %v", *subID) + } + gridLogIf(ctx, c.queueMsg(m, muxConnectError{Error: msg})) + return + } + _, _ = c.inStream.LoadOrCompute(m.MuxID, func() *muxServer { + return newMuxStateless(ctx, m, c, *handler) + }) + } else { + // Stream: + var handler *StreamHandler + if subID == nil { + if !m.Handler.valid() { + gridLogIf(ctx, c.queueMsg(m, muxConnectError{Error: "Invalid Handler"})) + return + } + handler = c.handlers.streams[m.Handler] + } else { + handler = c.handlers.subStreams[*subID] + } + if handler == nil { + msg := fmt.Sprintf("Invalid Handler for type: %v", m.Handler) + if subID != nil { + msg = fmt.Sprintf("Invalid Handler for type: %v", *subID) + } + gridLogIf(ctx, c.queueMsg(m, muxConnectError{Error: msg})) + return + } + + // Start a new server handler if none exists. + _, _ = c.inStream.LoadOrCompute(m.MuxID, func() *muxServer { + return newMuxStream(ctx, m, c, *handler) + }) + } +} + +// handleConnectMuxError when mux connect was rejected. +func (c *Connection) handleConnectMuxError(ctx context.Context, m message) { + if v, ok := c.outgoing.Load(m.MuxID); ok { + var cErr muxConnectError + _, err := cErr.UnmarshalMsg(m.Payload) + gridLogIf(ctx, err) + v.error(RemoteErr(cErr.Error)) + return + } + PutByteBuffer(m.Payload) +} + +func (c *Connection) handleAckMux(ctx context.Context, m message) { + PutByteBuffer(m.Payload) + v, ok := c.outgoing.Load(m.MuxID) + if !ok { + if m.Flags&FlagEOF == 0 { + gridLogIf(ctx, c.queueMsg(message{Op: OpDisconnectClientMux, MuxID: m.MuxID}, nil)) + } + return + } + if debugPrint { + fmt.Println(c.Local, "Mux", m.MuxID, "Acknowledged") + } + v.ack(m.Seq) +} + +func (c *Connection) handleRequest(ctx context.Context, m message, subID *subHandlerID) { + if !m.Handler.valid() { + gridLogIf(ctx, c.queueMsg(m, muxConnectError{Error: "Invalid Handler"})) + return + } + if debugReqs { + fmt.Println(m.MuxID, c.StringReverse(), "INCOMING") + } + // Singleshot message + var handler SingleHandlerFn + if subID == nil { + handler = c.handlers.single[m.Handler] + } else { + handler = c.handlers.subSingle[*subID] + } + if handler == nil { + msg := fmt.Sprintf("Invalid Handler for type: %v", m.Handler) + if subID != nil { + msg = fmt.Sprintf("Invalid Handler for type: %v", *subID) + } + gridLogIf(ctx, c.queueMsg(m, muxConnectError{Error: msg})) + return + } + + // TODO: This causes allocations, but escape analysis doesn't really show the cause. + // If another faithful engineer wants to take a stab, feel free. + go func(m message) { + var start time.Time + if m.DeadlineMS > 0 { + start = time.Now() + } + var b []byte + var err *RemoteErr + func() { + defer func() { + if rec := recover(); rec != nil { + err = NewRemoteErrString(fmt.Sprintf("handleMessages: panic recovered: %v", rec)) + debug.PrintStack() + gridLogIf(ctx, err) + } + }() + b, err = handler(m.Payload) + if debugPrint { + fmt.Println(c.Local, "Handler returned payload:", bytesOrLength(b), "err:", err) + } + }() + + if m.DeadlineMS > 0 && time.Since(start).Milliseconds()+c.addDeadline.Milliseconds() > int64(m.DeadlineMS) { + if debugReqs { + fmt.Println(m.MuxID, c.StringReverse(), "DEADLINE EXCEEDED") + } + // No need to return result + PutByteBuffer(b) + return + } + if debugReqs { + fmt.Println(m.MuxID, c.StringReverse(), "RESPONDING") + } + m = message{ + MuxID: m.MuxID, + Seq: m.Seq, + Op: OpResponse, + Flags: FlagEOF, + } + if err != nil { + m.Flags |= FlagPayloadIsErr + m.Payload = []byte(*err) + } else { + m.Payload = b + m.setZeroPayloadFlag() + } + gridLogIf(ctx, c.queueMsg(m, nil)) + }(m) +} + +func (c *Connection) handlePong(ctx context.Context, m message) { + var pong pongMsg + _, err := pong.UnmarshalMsg(m.Payload) + PutByteBuffer(m.Payload) + m.Payload = nil + + if m.MuxID == 0 { + atomic.StoreInt64(&c.LastPong, time.Now().UnixNano()) + c.lastPingDur.Store(int64(time.Since(pong.T))) + return + } + gridLogIf(ctx, err) + if m.MuxID == 0 { + atomic.StoreInt64(&c.LastPong, time.Now().UnixNano()) + return + } + if v, ok := c.outgoing.Load(m.MuxID); ok { + v.pong(pong) + } else { + // We don't care if the client was removed in the meantime, + // but we send a disconnect message to the server just in case. + gridLogIf(ctx, c.queueMsg(message{Op: OpDisconnectClientMux, MuxID: m.MuxID}, nil)) + } +} + +func (c *Connection) handlePing(ctx context.Context, m message) { + var ping pingMsg + if len(m.Payload) > 0 { + _, err := ping.UnmarshalMsg(m.Payload) + if err != nil { + gridLogIf(ctx, err) + } + } + // c.queueMsg will reuse m.Payload + + if m.MuxID == 0 { + gridLogIf(ctx, c.queueMsg(m, &pongMsg{T: ping.T})) + return + } + // Single calls do not support pinging. + if v, ok := c.inStream.Load(m.MuxID); ok { + pong := v.ping(m.Seq) + pong.T = ping.T + gridLogIf(ctx, c.queueMsg(m, &pong)) + } else { + pong := pongMsg{NotFound: true, T: ping.T} + gridLogIf(ctx, c.queueMsg(m, &pong)) + } +} + +func (c *Connection) handleDisconnectClientMux(m message) { + if v, ok := c.outgoing.Load(m.MuxID); ok { + if m.Flags&FlagPayloadIsErr != 0 { + v.error(RemoteErr(m.Payload)) + } else { + v.error(ErrDisconnected) + } + return + } + PutByteBuffer(m.Payload) +} + +func (c *Connection) handleDisconnectServerMux(m message) { + if debugPrint { + fmt.Println(c.Local, "Disconnect server mux:", m.MuxID) + } + PutByteBuffer(m.Payload) + m.Payload = nil + if v, ok := c.inStream.Load(m.MuxID); ok { + v.close() + } +} + +func (c *Connection) handleUnblockClMux(m message) { + PutByteBuffer(m.Payload) + m.Payload = nil + v, ok := c.outgoing.Load(m.MuxID) + if !ok { + if debugPrint { + fmt.Println(c.Local, "Unblock: Unknown Mux:", m.MuxID) + } + // We can expect to receive unblocks for closed muxes + return + } + v.unblockSend(m.Seq) +} + +func (c *Connection) handleUnblockSrvMux(m message) { + if m.Payload != nil { + PutByteBuffer(m.Payload) + } + m.Payload = nil + if v, ok := c.inStream.Load(m.MuxID); ok { + v.unblockSend(m.Seq) + return + } + // We can expect to receive unblocks for closed muxes + if debugPrint { + fmt.Println(c.Local, "Unblock: Unknown Mux:", m.MuxID) + } +} + +func (c *Connection) handleMuxClientMsg(ctx context.Context, m message) { + v, ok := c.inStream.Load(m.MuxID) + if !ok { + if debugPrint { + fmt.Println(c.Local, "OpMuxClientMsg: Unknown Mux:", m.MuxID) + } + gridLogIf(ctx, c.queueMsg(message{Op: OpDisconnectClientMux, MuxID: m.MuxID}, nil)) + PutByteBuffer(m.Payload) + return + } + v.message(m) +} + +func (c *Connection) handleResponse(m message) { + if debugPrint { + fmt.Printf("%s Got mux response: %v\n", c.Local, m) + } + v, ok := c.outgoing.Load(m.MuxID) + if !ok { + if debugReqs { + fmt.Println(m.MuxID, c.String(), "Got response for unknown mux") + } + PutByteBuffer(m.Payload) + return + } + if m.Flags&FlagPayloadIsErr != 0 { + v.response(m.Seq, Response{ + Msg: nil, + Err: RemoteErr(m.Payload), + }) + PutByteBuffer(m.Payload) + } else { + v.response(m.Seq, Response{ + Msg: m.Payload, + Err: nil, + }) + } + v.close() + if debugReqs { + fmt.Println(m.MuxID, c.String(), "handleResponse: closing mux") + } +} + +func (c *Connection) handleMuxServerMsg(ctx context.Context, m message) { + if debugPrint { + fmt.Printf("%s Got mux msg: %v\n", c.Local, m) + } + v, ok := c.outgoing.Load(m.MuxID) + if !ok { + if m.Flags&FlagEOF == 0 { + gridLogIf(ctx, c.queueMsg(message{Op: OpDisconnectClientMux, MuxID: m.MuxID}, nil)) + } + PutByteBuffer(m.Payload) + return + } + if m.Flags&FlagPayloadIsErr != 0 { + v.response(m.Seq, Response{ + Msg: nil, + Err: RemoteErr(m.Payload), + }) + if v.cancelFn != nil { + v.cancelFn(RemoteErr(m.Payload)) + } + PutByteBuffer(m.Payload) + v.close() + c.outgoing.Delete(m.MuxID) + return + } + // Return payload. + if m.Payload != nil { + v.response(m.Seq, Response{ + Msg: m.Payload, + Err: nil, + }) + } + // Close when EOF. + if m.Flags&FlagEOF != 0 { + // We must obtain the lock before closing + // Otherwise others may pick up the error before close is called. + v.respMu.Lock() + v.closeLocked() + v.respMu.Unlock() + if debugReqs { + fmt.Println(m.MuxID, c.String(), "handleMuxServerMsg: DELETING MUX") + } + c.outgoing.Delete(m.MuxID) + } +} + +func (c *Connection) deleteMux(incoming bool, muxID uint64) { + if incoming { + if debugPrint { + fmt.Println("deleteMux: disconnect incoming mux", muxID) + } + v, loaded := c.inStream.LoadAndDelete(muxID) + if loaded && v != nil { + gridLogIf(c.ctx, c.queueMsg(message{Op: OpDisconnectClientMux, MuxID: muxID}, nil)) + v.close() + } + } else { + if debugPrint { + fmt.Println("deleteMux: disconnect outgoing mux", muxID) + } + v, loaded := c.outgoing.LoadAndDelete(muxID) + if loaded && v != nil { + if debugReqs { + fmt.Println(muxID, c.String(), "deleteMux: DELETING MUX") + } + v.close() + gridLogIf(c.ctx, c.queueMsg(message{Op: OpDisconnectServerMux, MuxID: muxID}, nil)) + } + } +} + +// State returns the current connection status. +func (c *Connection) State() State { + return State(atomic.LoadUint32((*uint32)(&c.state))) +} + +// Stats returns the current connection stats. +func (c *Connection) Stats() madmin.RPCMetrics { + conn := 0 + if c.State() == StateConnected { + conn++ + } + var lastConn time.Time + if t := c.lastConnect.Load(); t != nil { + lastConn = *t + } + pingMS := float64(c.lastPingDur.Load()) / float64(time.Millisecond) + m := madmin.RPCMetrics{ + CollectedAt: time.Now(), + Connected: conn, + Disconnected: 1 - conn, + IncomingStreams: c.inStream.Size(), + OutgoingStreams: c.outgoing.Size(), + IncomingBytes: c.inBytes.Load(), + OutgoingBytes: c.outBytes.Load(), + IncomingMessages: c.inMessages.Load(), + OutgoingMessages: c.outMessages.Load(), + OutQueue: len(c.outQueue), + LastPongTime: time.Unix(0, c.LastPong).UTC(), + LastConnectTime: lastConn, + ReconnectCount: int(c.reconnects.Load()), + LastPingMS: pingMS, + MaxPingDurMS: pingMS, + } + m.ByDestination = map[string]madmin.RPCMetrics{ + c.Remote: m, + } + return m +} + +func (c *Connection) debugMsg(d debugMsg, args ...any) { + if debugPrint { + fmt.Println("debug: sending message", d, args) + } + + switch d { + case debugShutdown: + c.updateState(StateShutdown) + case debugKillInbound: + c.connMu.Lock() + defer c.connMu.Unlock() + if c.debugInConn != nil { + if debugPrint { + fmt.Println("debug: closing inbound connection") + } + c.debugInConn.Close() + } + case debugKillOutbound: + c.connMu.Lock() + defer c.connMu.Unlock() + if c.debugInConn != nil { + if debugPrint { + fmt.Println("debug: closing outgoing connection") + } + c.debugInConn.Close() + } + case debugWaitForExit: + c.reconnectMu.Lock() + c.handleMsgWg.Wait() + c.reconnectMu.Unlock() + case debugSetConnPingDuration: + c.connMu.Lock() + defer c.connMu.Unlock() + c.connPingInterval, _ = args[0].(time.Duration) + if c.connPingInterval < time.Second { + panic("CONN ping interval too low") + } + case debugSetClientPingDuration: + c.connMu.Lock() + defer c.connMu.Unlock() + c.clientPingInterval, _ = args[0].(time.Duration) + case debugAddToDeadline: + c.addDeadline, _ = args[0].(time.Duration) + case debugIsOutgoingClosed: + // params: muxID uint64, isClosed func(bool) + muxID, _ := args[0].(uint64) + resp, _ := args[1].(func(b bool)) + mid, ok := c.outgoing.Load(muxID) + if !ok || mid == nil { + resp(true) + return + } + mid.respMu.Lock() + resp(mid.closed) + mid.respMu.Unlock() + case debugBlockInboundMessages: + c.connMu.Lock() + a, _ := args[0].(chan struct{}) + block := (<-chan struct{})(a) + c.blockMessages.Store(&block) + c.connMu.Unlock() + } +} + +// wsWriter writes websocket messages. +type wsWriter struct { + tmp [ws.MaxHeaderSize]byte +} + +// writeMessage writes a message to w without allocations. +func (ww *wsWriter) writeMessage(w io.Writer, s ws.State, op ws.OpCode, p []byte) error { + const fin = true + var frame ws.Frame + if s.ClientSide() { + // We do not need to copy the payload, since we own it. + payload := p + + frame = ws.NewFrame(op, fin, payload) + frame = ws.MaskFrameInPlace(frame) + } else { + frame = ws.NewFrame(op, fin, p) + } + + return ww.writeFrame(w, frame) +} + +// writeFrame writes frame binary representation into w. +func (ww *wsWriter) writeFrame(w io.Writer, f ws.Frame) error { + const ( + bit0 = 0x80 + len7 = int64(125) + len16 = int64(^(uint16(0))) + len64 = int64(^(uint64(0)) >> 1) + ) + + bts := ww.tmp[:] + if f.Header.Fin { + bts[0] |= bit0 + } + bts[0] |= f.Header.Rsv << 4 + bts[0] |= byte(f.Header.OpCode) + + var n int + switch { + case f.Header.Length <= len7: + bts[1] = byte(f.Header.Length) + n = 2 + + case f.Header.Length <= len16: + bts[1] = 126 + binary.BigEndian.PutUint16(bts[2:4], uint16(f.Header.Length)) + n = 4 + + case f.Header.Length <= len64: + bts[1] = 127 + binary.BigEndian.PutUint64(bts[2:10], uint64(f.Header.Length)) + n = 10 + + default: + return ws.ErrHeaderLengthUnexpected + } + + if f.Header.Masked { + bts[1] |= bit0 + n += copy(bts[n:], f.Header.Mask[:]) + } + + if _, err := w.Write(bts[:n]); err != nil { + return err + } + + _, err := w.Write(f.Payload) + return err +} diff --git a/internal/grid/connection_test.go b/internal/grid/connection_test.go new file mode 100644 index 00000000..b81e4860 --- /dev/null +++ b/internal/grid/connection_test.go @@ -0,0 +1,209 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package grid + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/minio/minio/internal/logger/target/testlogger" +) + +func TestDisconnect(t *testing.T) { + defer testlogger.T.SetLogTB(t)() + defer timeout(10 * time.Second)() + hosts, listeners, _ := getHosts(2) + dialer := &net.Dialer{ + Timeout: 1 * time.Second, + } + errFatal := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + wrapServer := func(handler http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + t.Logf("Got a %s request for: %v", r.Method, r.URL) + handler.ServeHTTP(w, r) + }) + } + connReady := make(chan struct{}) + // We fake a local and remote server. + localHost := hosts[0] + remoteHost := hosts[1] + local, err := NewManager(t.Context(), ManagerOptions{ + Dialer: ConnectWS(dialer.DialContext, + dummyNewToken, + nil), + Local: localHost, + Hosts: hosts, + AuthFn: dummyNewToken, + AuthToken: dummyTokenValidate, + BlockConnect: connReady, + }) + errFatal(err) + + // 1: Echo + errFatal(local.RegisterSingleHandler(handlerTest, func(payload []byte) ([]byte, *RemoteErr) { + t.Log("1: server payload: ", len(payload), "bytes.") + return append([]byte{}, payload...), nil + })) + // 2: Return as error + errFatal(local.RegisterSingleHandler(handlerTest2, func(payload []byte) ([]byte, *RemoteErr) { + t.Log("2: server payload: ", len(payload), "bytes.") + err := RemoteErr(payload) + return nil, &err + })) + + remote, err := NewManager(t.Context(), ManagerOptions{ + Dialer: ConnectWS(dialer.DialContext, + dummyNewToken, + nil), + Local: remoteHost, + Hosts: hosts, + AuthFn: dummyNewToken, + AuthToken: dummyTokenValidate, + BlockConnect: connReady, + }) + errFatal(err) + + localServer := startServer(t, listeners[0], wrapServer(local.Handler(dummyRequestValidate))) + remoteServer := startServer(t, listeners[1], wrapServer(remote.Handler(dummyRequestValidate))) + close(connReady) + + defer func() { + local.debugMsg(debugShutdown) + remote.debugMsg(debugShutdown) + remoteServer.Close() + localServer.Close() + remote.debugMsg(debugWaitForExit) + local.debugMsg(debugWaitForExit) + }() + + cleanReqs := make(chan struct{}) + gotCall := make(chan struct{}) + defer close(cleanReqs) + // 1: Block forever + h1 := func(payload []byte) ([]byte, *RemoteErr) { + gotCall <- struct{}{} + <-cleanReqs + return nil, nil + } + // 2: Also block, but with streaming. + h2 := StreamHandler{ + Handle: func(ctx context.Context, payload []byte, request <-chan []byte, resp chan<- []byte) *RemoteErr { + gotCall <- struct{}{} + select { + case <-ctx.Done(): + gotCall <- struct{}{} + case <-cleanReqs: + panic("should not be called") + } + return nil + }, + OutCapacity: 1, + InCapacity: 1, + } + errFatal(remote.RegisterSingleHandler(handlerTest, h1)) + errFatal(remote.RegisterStreamingHandler(handlerTest2, h2)) + errFatal(local.RegisterSingleHandler(handlerTest, h1)) + errFatal(local.RegisterStreamingHandler(handlerTest2, h2)) + + // local to remote + remoteConn := local.Connection(remoteHost) + errFatal(remoteConn.WaitForConnect(t.Context())) + const testPayload = "Hello Grid World!" + + gotResp := make(chan struct{}) + go func() { + start := time.Now() + t.Log("Roundtrip: sending request") + resp, err := remoteConn.Request(t.Context(), handlerTest, []byte(testPayload)) + t.Log("Roundtrip:", time.Since(start), resp, err) + gotResp <- struct{}{} + }() + <-gotCall + remote.debugMsg(debugKillInbound) + local.debugMsg(debugKillInbound) + <-gotResp + + // Must reconnect + errFatal(remoteConn.WaitForConnect(t.Context())) + + stream, err := remoteConn.NewStream(t.Context(), handlerTest2, []byte(testPayload)) + errFatal(err) + go func() { + for resp := range stream.responses { + t.Log("Resp:", resp, err) + } + gotResp <- struct{}{} + }() + + <-gotCall + remote.debugMsg(debugKillOutbound) + local.debugMsg(debugKillOutbound) + errFatal(remoteConn.WaitForConnect(t.Context())) + + <-gotResp + // Killing should cancel the context on the request. + <-gotCall +} + +func TestShouldConnect(t *testing.T) { + var c Connection + var cReverse Connection + hosts := []string{"a", "b", "c", "d", "e", "f", "g", "h", "i", "j", "k", "l", "m", "n", "o", "p", "q", "r", "s", "t", "u", "v", "x", "y", "z", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9"} + for x := range hosts { + should := 0 + for y := range hosts { + if x == y { + continue + } + c.Local = hosts[x] + c.Remote = hosts[y] + cReverse.Local = hosts[y] + cReverse.Remote = hosts[x] + if c.shouldConnect() == cReverse.shouldConnect() { + t.Errorf("shouldConnect(%q, %q) != shouldConnect(%q, %q)", hosts[x], hosts[y], hosts[y], hosts[x]) + } + if c.shouldConnect() { + should++ + } + } + if should < 10 { + t.Errorf("host %q only connects to %d hosts", hosts[x], should) + } + t.Logf("host %q should connect to %d hosts", hosts[x], should) + } +} + +func startServer(t testing.TB, listener net.Listener, handler http.Handler) (server *httptest.Server) { + t.Helper() + server = httptest.NewUnstartedServer(handler) + server.Config.Addr = listener.Addr().String() + server.Listener = listener + server.Start() + // t.Cleanup(server.Close) + t.Log("Started server on", server.Config.Addr, "URL:", server.URL) + return server +} diff --git a/internal/grid/debug.go b/internal/grid/debug.go new file mode 100644 index 00000000..f80559f3 --- /dev/null +++ b/internal/grid/debug.go @@ -0,0 +1,181 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package grid + +import ( + "context" + "fmt" + "net" + "net/http" + "net/http/httptest" + "sync" + "time" + + "github.com/minio/mux" +) + +//go:generate stringer -type=debugMsg $GOFILE + +// debugMsg is a debug message for testing purposes. +// may only be used for tests. +type debugMsg int + +const ( + debugPrint = false + debugReqs = false +) + +const ( + debugShutdown debugMsg = iota + debugKillInbound + debugKillOutbound + debugWaitForExit + debugSetConnPingDuration + debugSetClientPingDuration + debugAddToDeadline + debugIsOutgoingClosed + debugBlockInboundMessages +) + +// TestGrid contains a grid of servers for testing purposes. +type TestGrid struct { + Servers []*httptest.Server + Listeners []net.Listener + Managers []*Manager + Mux []*mux.Router + Hosts []string + cleanupOnce sync.Once + cancel context.CancelFunc +} + +// SetupTestGrid creates a new grid for testing purposes. +// Select the number of hosts to create. +// Call (TestGrid).Cleanup() when done. +func SetupTestGrid(n int) (*TestGrid, error) { + hosts, listeners, err := getHosts(n) + if err != nil { + return nil, err + } + dialer := &net.Dialer{ + Timeout: 5 * time.Second, + } + var res TestGrid + res.Hosts = hosts + ready := make(chan struct{}) + ctx, cancel := context.WithCancel(context.Background()) + res.cancel = cancel + for i, host := range hosts { + manager, err := NewManager(ctx, ManagerOptions{ + Dialer: ConnectWS(dialer.DialContext, + dummyNewToken, + nil), + Local: host, + Hosts: hosts, + AuthFn: dummyNewToken, + AuthToken: dummyTokenValidate, + BlockConnect: ready, + RoutePath: RoutePath, + }) + if err != nil { + return nil, err + } + m := mux.NewRouter() + m.Handle(RoutePath, manager.Handler(dummyRequestValidate)) + res.Managers = append(res.Managers, manager) + res.Servers = append(res.Servers, startHTTPServer(listeners[i], m)) + res.Listeners = append(res.Listeners, listeners[i]) + res.Mux = append(res.Mux, m) + } + close(ready) + for _, m := range res.Managers { + for _, remote := range m.Targets() { + if err := m.Connection(remote).WaitForConnect(ctx); err != nil { + return nil, err + } + } + } + return &res, nil +} + +// Cleanup will clean up the test grid. +func (t *TestGrid) Cleanup() { + t.cancel() + t.cleanupOnce.Do(func() { + for _, manager := range t.Managers { + manager.debugMsg(debugShutdown) + } + for _, server := range t.Servers { + server.Close() + } + for _, listener := range t.Listeners { + listener.Close() + } + }) +} + +// WaitAllConnect will wait for all connections to be established. +func (t *TestGrid) WaitAllConnect(ctx context.Context) { + for _, manager := range t.Managers { + for _, remote := range manager.Targets() { + if manager.HostName() == remote { + continue + } + if err := manager.Connection(remote).WaitForConnect(ctx); err != nil { + panic(err) + } + } + } +} + +func getHosts(n int) (hosts []string, listeners []net.Listener, err error) { + for range n { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + if l, err = net.Listen("tcp6", "[::1]:0"); err != nil { + return nil, nil, fmt.Errorf("httptest: failed to listen on a port: %v", err) + } + } + addr := l.Addr() + hosts = append(hosts, "http://"+addr.String()) + listeners = append(listeners, l) + } + return hosts, listeners, err +} + +func startHTTPServer(listener net.Listener, handler http.Handler) (server *httptest.Server) { + server = httptest.NewUnstartedServer(handler) + server.Config.Addr = listener.Addr().String() + server.Listener = listener + server.Start() + return server +} + +func dummyRequestValidate(r *http.Request) error { + return nil +} + +func dummyTokenValidate(token string) error { + if token == "debug" { + return nil + } + return fmt.Errorf("invalid token. want empty, got %s", token) +} + +func dummyNewToken() string { + return "debug" +} diff --git a/internal/grid/debugmsg_string.go b/internal/grid/debugmsg_string.go new file mode 100644 index 00000000..52c92cb4 --- /dev/null +++ b/internal/grid/debugmsg_string.go @@ -0,0 +1,31 @@ +// Code generated by "stringer -type=debugMsg debug.go"; DO NOT EDIT. + +package grid + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[debugShutdown-0] + _ = x[debugKillInbound-1] + _ = x[debugKillOutbound-2] + _ = x[debugWaitForExit-3] + _ = x[debugSetConnPingDuration-4] + _ = x[debugSetClientPingDuration-5] + _ = x[debugAddToDeadline-6] + _ = x[debugIsOutgoingClosed-7] + _ = x[debugBlockInboundMessages-8] +} + +const _debugMsg_name = "debugShutdowndebugKillInbounddebugKillOutbounddebugWaitForExitdebugSetConnPingDurationdebugSetClientPingDurationdebugAddToDeadlinedebugIsOutgoingCloseddebugBlockInboundMessages" + +var _debugMsg_index = [...]uint8{0, 13, 29, 46, 62, 86, 112, 130, 151, 176} + +func (i debugMsg) String() string { + if i < 0 || i >= debugMsg(len(_debugMsg_index)-1) { + return "debugMsg(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _debugMsg_name[_debugMsg_index[i]:_debugMsg_index[i+1]] +} diff --git a/internal/grid/errors.go b/internal/grid/errors.go new file mode 100644 index 00000000..dbf62f7e --- /dev/null +++ b/internal/grid/errors.go @@ -0,0 +1,43 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package grid + +import ( + "errors" + "fmt" +) + +var ( + // ErrUnknownHandler is returned when an unknown handler is requested. + ErrUnknownHandler = errors.New("unknown mux handler") + + // ErrHandlerAlreadyExists is returned when a handler is already registered. + ErrHandlerAlreadyExists = errors.New("mux handler already exists") + + // ErrIncorrectSequence is returned when an out-of-sequence item is received. + ErrIncorrectSequence = errors.New("out-of-sequence item received") +) + +// ErrResponse is a remote error response. +type ErrResponse struct { + msg string +} + +func (e ErrResponse) Error() string { + return fmt.Sprintf("remote: %s", e.msg) +} diff --git a/internal/grid/grid.go b/internal/grid/grid.go new file mode 100644 index 00000000..42872b06 --- /dev/null +++ b/internal/grid/grid.go @@ -0,0 +1,243 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Package grid provides single-connection two-way grid communication. +package grid + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "net" + "net/http" + "strconv" + "strings" + "time" + + "github.com/gobwas/ws" + "github.com/gobwas/ws/wsutil" + "github.com/minio/minio/internal/bpool" +) + +// ErrDisconnected is returned when the connection to the remote has been lost during the call. +var ErrDisconnected = RemoteErr("remote disconnected") + +const ( + // minBufferSize is the minimum buffer size. + // Buffers below this is not reused. + minBufferSize = 1 << 10 + + // defaultBufferSize is the default buffer allocation size. + defaultBufferSize = 4 << 10 + + // maxBufferSize is the maximum buffer size. + // Buffers larger than this is not reused. + maxBufferSize = 96 << 10 + + // This is the assumed size of bigger buffers and allocation size. + biggerBufMin = 32 << 10 + + // This is the maximum size of bigger buffers. + biggerBufMax = maxBufferSize + + // If there is a queue, merge up to this many messages. + maxMergeMessages = 50 + + // clientPingInterval will ping the remote handler every 15 seconds. + // Clients disconnect when we exceed 2 intervals. + clientPingInterval = 15 * time.Second + + // Deadline for single (non-streaming) requests to complete. + // Used if no deadline is provided on context. + defaultSingleRequestTimeout = time.Minute +) + +var internalByteBuffer = bpool.Pool[*[]byte]{ + New: func() *[]byte { + m := make([]byte, 0, defaultBufferSize) + return &m + }, +} + +var internal32KByteBuffer = bpool.Pool[*[]byte]{ + New: func() *[]byte { + m := make([]byte, 0, biggerBufMin) + return &m + }, +} + +// GetByteBuffer can be replaced with a function that returns a small +// byte buffer. +// When replacing PutByteBuffer should also be replaced +// There is no minimum size. +var GetByteBuffer = func() []byte { + b := *internalByteBuffer.Get() + return b[:0] +} + +// GetByteBufferCap returns a length 0 byte buffer with at least the given capacity. +func GetByteBufferCap(wantSz int) []byte { + if wantSz < defaultBufferSize { + b := GetByteBuffer()[:0] + if cap(b) >= wantSz { + return b + } + PutByteBuffer(b) + } + if wantSz <= maxBufferSize { + b := *internal32KByteBuffer.Get() + if cap(b) >= wantSz { + return b[:0] + } + internal32KByteBuffer.Put(&b) + } + return make([]byte, 0, wantSz) +} + +// PutByteBuffer is for returning byte buffers. +var PutByteBuffer = func(b []byte) { + if cap(b) >= biggerBufMin && cap(b) < biggerBufMax { + internal32KByteBuffer.Put(&b) + return + } + if cap(b) >= minBufferSize && cap(b) < biggerBufMin { + internalByteBuffer.Put(&b) + return + } +} + +// readAllInto reads from r and appends to b until an error or EOF and returns the data it read. +// A successful call returns err == nil, not err == EOF. Because readAllInto is +// defined to read from src until EOF, it does not treat an EOF from Read +// as an error to be reported. +func readAllInto(b []byte, r *wsutil.Reader, want int64) ([]byte, error) { + read := int64(0) + for { + if len(b) == cap(b) { + // Add more capacity (let append pick how much). + b = append(b, 0)[:len(b)] + } + n, err := r.Read(b[len(b):cap(b)]) + b = b[:len(b)+n] + if err != nil { + if errors.Is(err, io.EOF) { + if want >= 0 && read+int64(n) != want { + return nil, io.ErrUnexpectedEOF + } + err = nil + } + return b, err + } + read += int64(n) + if want >= 0 && read == want { + // No need to read more... + return b, nil + } + } +} + +// getDeadline will truncate the deadline so it is at least 1ms and at most MaxDeadline. +func getDeadline(d time.Duration) time.Duration { + if d < time.Millisecond { + return 0 + } + if d > MaxDeadline { + return MaxDeadline + } + return d +} + +type writerWrapper struct { + ch chan<- []byte + ctx context.Context +} + +func (w *writerWrapper) Write(p []byte) (n int, err error) { + buf := GetByteBufferCap(len(p)) + buf = buf[:len(p)] + copy(buf, p) + select { + case w.ch <- buf: + return len(p), nil + case <-w.ctx.Done(): + return 0, context.Cause(w.ctx) + } +} + +// WriterToChannel will return an io.Writer that writes to the given channel. +// The context both allows returning errors on writes and to ensure that +// this isn't abandoned if the channel is no longer being read from. +func WriterToChannel(ctx context.Context, ch chan<- []byte) io.Writer { + return &writerWrapper{ch: ch, ctx: ctx} +} + +// bytesOrLength returns small (<=100b) byte slices as string, otherwise length. +func bytesOrLength(b []byte) string { + if len(b) > 100 { + return fmt.Sprintf("%d bytes", len(b)) + } + return fmt.Sprint(string(b)) +} + +// ConnDialer is a function that dials a connection to the given address. +// There should be no retries in this function, +// and should have a timeout of something like 2 seconds. +// The returned net.Conn should also have quick disconnect on errors. +// The net.Conn must support all features as described by the net.Conn interface. +type ConnDialer func(ctx context.Context, address string) (net.Conn, error) + +// ConnectWSWithRoutePath is like ConnectWS but with a custom grid route path. +func ConnectWSWithRoutePath(dial ContextDialer, auth AuthFn, tls *tls.Config, routePath string) func(ctx context.Context, remote string) (net.Conn, error) { + return func(ctx context.Context, remote string) (net.Conn, error) { + toDial := strings.Replace(remote, "http://", "ws://", 1) + toDial = strings.Replace(toDial, "https://", "wss://", 1) + toDial += routePath + + dialer := ws.DefaultDialer + dialer.ReadBufferSize = readBufferSize + dialer.WriteBufferSize = writeBufferSize + dialer.Timeout = defaultDialTimeout + if dial != nil { + dialer.NetDial = dial + } + header := make(http.Header, 2) + header.Set("Authorization", "Bearer "+auth()) + header.Set("X-Minio-Time", strconv.FormatInt(time.Now().UnixNano(), 10)) + + if len(header) > 0 { + dialer.Header = ws.HandshakeHeaderHTTP(header) + } + dialer.TLSConfig = tls + + conn, br, _, err := dialer.Dial(ctx, toDial) + if br != nil { + ws.PutReader(br) + } + return conn, err + } +} + +// ConnectWS returns a function that dials a websocket connection to the given address. +// Route and auth are added to the connection. +func ConnectWS(dial ContextDialer, auth AuthFn, tls *tls.Config) func(ctx context.Context, remote string) (net.Conn, error) { + return ConnectWSWithRoutePath(dial, auth, tls, RoutePath) +} + +// ValidateTokenFn must validate the token and return an error if it is invalid. +type ValidateTokenFn func(token string) error diff --git a/internal/grid/grid_test.go b/internal/grid/grid_test.go new file mode 100644 index 00000000..3d1156e6 --- /dev/null +++ b/internal/grid/grid_test.go @@ -0,0 +1,1328 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package grid + +import ( + "bytes" + "context" + "errors" + "fmt" + "maps" + "os" + "runtime" + "strconv" + "strings" + "sync" + "testing" + "time" + + "github.com/minio/minio/internal/logger/target/testlogger" +) + +func TestSingleRoundtrip(t *testing.T) { + defer testlogger.T.SetLogTB(t)() + errFatal := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + grid, err := SetupTestGrid(2) + errFatal(err) + remoteHost := grid.Hosts[1] + local := grid.Managers[0] + + // 1: Echo + errFatal(local.RegisterSingleHandler(handlerTest, func(payload []byte) ([]byte, *RemoteErr) { + t.Log("1: server payload: ", len(payload), "bytes.") + return append([]byte{}, payload...), nil + })) + // 2: Return as error + errFatal(local.RegisterSingleHandler(handlerTest2, func(payload []byte) ([]byte, *RemoteErr) { + t.Log("2: server payload: ", len(payload), "bytes.") + err := RemoteErr(payload) + return nil, &err + })) + + remote := grid.Managers[1] + + // 1: Echo + errFatal(remote.RegisterSingleHandler(handlerTest, func(payload []byte) ([]byte, *RemoteErr) { + t.Log("1: server payload: ", len(payload), "bytes.") + return append([]byte{}, payload...), nil + })) + // 2: Return as error + errFatal(remote.RegisterSingleHandler(handlerTest2, func(payload []byte) ([]byte, *RemoteErr) { + t.Log("2: server payload: ", len(payload), "bytes.") + err := RemoteErr(payload) + return nil, &err + })) + + // local to remote + remoteConn := local.Connection(remoteHost) + remoteConn.WaitForConnect(t.Context()) + defer testlogger.T.SetErrorTB(t)() + + t.Run("localToRemote", func(t *testing.T) { + const testPayload = "Hello Grid World!" + + start := time.Now() + resp, err := remoteConn.Request(t.Context(), handlerTest, []byte(testPayload)) + errFatal(err) + if string(resp) != testPayload { + t.Errorf("want %q, got %q", testPayload, string(resp)) + } + t.Log("Roundtrip:", time.Since(start)) + }) + + t.Run("localToRemoteErr", func(t *testing.T) { + const testPayload = "Hello Grid World!" + start := time.Now() + resp, err := remoteConn.Request(t.Context(), handlerTest2, []byte(testPayload)) + t.Log("Roundtrip:", time.Since(start)) + if len(resp) != 0 { + t.Errorf("want nil, got %q", string(resp)) + } + if err != RemoteErr(testPayload) { + t.Errorf("want error %v(%T), got %v(%T)", RemoteErr(testPayload), RemoteErr(testPayload), err, err) + } + t.Log("Roundtrip:", time.Since(start)) + }) + + t.Run("localToRemoteHuge", func(t *testing.T) { + testPayload := bytes.Repeat([]byte("?"), 1<<20) + + start := time.Now() + resp, err := remoteConn.Request(t.Context(), handlerTest, testPayload) + errFatal(err) + if string(resp) != string(testPayload) { + t.Errorf("want %q, got %q", testPayload, string(resp)) + } + t.Log("Roundtrip:", time.Since(start)) + }) + + t.Run("localToRemoteErrHuge", func(t *testing.T) { + testPayload := bytes.Repeat([]byte("!"), 1<<10) + + start := time.Now() + resp, err := remoteConn.Request(t.Context(), handlerTest2, testPayload) + if len(resp) != 0 { + t.Errorf("want nil, got %q", string(resp)) + } + if err != RemoteErr(testPayload) { + t.Errorf("want error %v(%T), got %v(%T)", RemoteErr(testPayload), RemoteErr(testPayload), err, err) + } + t.Log("Roundtrip:", time.Since(start)) + }) +} + +func TestSingleRoundtripNotReady(t *testing.T) { + defer testlogger.T.SetLogTB(t)() + errFatal := func(t testing.TB, err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + grid, err := SetupTestGrid(2) + errFatal(t, err) + remoteHost := grid.Hosts[1] + local := grid.Managers[0] + + // 1: Echo + errFatal(t, local.RegisterSingleHandler(handlerTest, func(payload []byte) ([]byte, *RemoteErr) { + t.Log("1: server payload: ", len(payload), "bytes.") + return append([]byte{}, payload...), nil + })) + // 2: Return as error + errFatal(t, local.RegisterSingleHandler(handlerTest2, func(payload []byte) ([]byte, *RemoteErr) { + t.Log("2: server payload: ", len(payload), "bytes.") + err := RemoteErr(payload) + return nil, &err + })) + + // Do not register remote handlers + + // local to remote + remoteConn := local.Connection(remoteHost) + remoteConn.WaitForConnect(t.Context()) + defer testlogger.T.SetErrorTB(t)() + + t.Run("localToRemote", func(t *testing.T) { + const testPayload = "Hello Grid World!" + // Single requests should have remote errors. + _, err := remoteConn.Request(t.Context(), handlerTest, []byte(testPayload)) + if _, ok := err.(*RemoteErr); !ok { + t.Fatalf("Unexpected error: %v, %T", err, err) + } + // Streams should not be able to set up until registered. + // Thus, the error is a local error. + _, err = remoteConn.NewStream(t.Context(), handlerTest, []byte(testPayload)) + if !errors.Is(err, ErrUnknownHandler) { + t.Fatalf("Unexpected error: %v, %T", err, err) + } + }) +} + +func TestSingleRoundtripGenerics(t *testing.T) { + defer testlogger.T.SetLogTB(t)() + errFatal := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + grid, err := SetupTestGrid(2) + errFatal(err) + remoteHost := grid.Hosts[1] + local := grid.Managers[0] + remote := grid.Managers[1] + + // 1: Echo + h1 := NewSingleHandler[*testRequest, *testResponse](handlerTest, func() *testRequest { + return &testRequest{} + }, func() *testResponse { + return &testResponse{} + }) + // Handles incoming requests, returns a response + handler1 := func(req *testRequest) (resp *testResponse, err *RemoteErr) { + resp = h1.NewResponse() + *resp = testResponse{ + OrgNum: req.Num, + OrgString: req.String, + Embedded: *req, + } + return resp, nil + } + // Return error + h2 := NewSingleHandler[*testRequest, *testResponse](handlerTest2, newTestRequest, newTestResponse) + handler2 := func(req *testRequest) (resp *testResponse, err *RemoteErr) { + r := RemoteErr(req.String) + return nil, &r + } + errFatal(h1.Register(local, handler1)) + errFatal(h2.Register(local, handler2)) + + errFatal(h1.Register(remote, handler1)) + errFatal(h2.Register(remote, handler2)) + + // local to remote connection + remoteConn := local.Connection(remoteHost) + const testPayload = "Hello Grid World!" + + start := time.Now() + req := testRequest{Num: 1, String: testPayload} + resp, err := h1.Call(t.Context(), remoteConn, &req) + errFatal(err) + if resp.OrgString != testPayload { + t.Errorf("want %q, got %q", testPayload, resp.OrgString) + } + t.Log("Roundtrip:", time.Since(start)) + h1.PutResponse(resp) + + start = time.Now() + resp, err = h2.Call(t.Context(), remoteConn, &testRequest{Num: 1, String: testPayload}) + t.Log("Roundtrip:", time.Since(start)) + if err != RemoteErr(testPayload) { + t.Errorf("want error %v(%T), got %v(%T)", RemoteErr(testPayload), RemoteErr(testPayload), err, err) + } + if resp != nil { + t.Errorf("want nil, got %q", resp) + } + h2.PutResponse(resp) + t.Log("Roundtrip:", time.Since(start)) +} + +func TestSingleRoundtripGenericsRecycle(t *testing.T) { + defer testlogger.T.SetLogTB(t)() + errFatal := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + grid, err := SetupTestGrid(2) + errFatal(err) + remoteHost := grid.Hosts[1] + local := grid.Managers[0] + remote := grid.Managers[1] + + // 1: Echo + h1 := NewSingleHandler[*MSS, *MSS](handlerTest, NewMSS, NewMSS) + // Handles incoming requests, returns a response + handler1 := func(req *MSS) (resp *MSS, err *RemoteErr) { + resp = h1.NewResponse() + maps.Copy((*resp), *req) + return resp, nil + } + // Return error + h2 := NewSingleHandler[*MSS, *MSS](handlerTest2, NewMSS, NewMSS) + handler2 := func(req *MSS) (resp *MSS, err *RemoteErr) { + defer req.Recycle() + r := RemoteErr(req.Get("err")) + return nil, &r + } + errFatal(h1.Register(local, handler1)) + errFatal(h2.Register(local, handler2)) + + errFatal(h1.Register(remote, handler1)) + errFatal(h2.Register(remote, handler2)) + + // local to remote connection + remoteConn := local.Connection(remoteHost) + const testPayload = "Hello Grid World!" + + start := time.Now() + req := NewMSSWith(map[string]string{"test": testPayload}) + resp, err := h1.Call(t.Context(), remoteConn, req) + errFatal(err) + if resp.Get("test") != testPayload { + t.Errorf("want %q, got %q", testPayload, resp.Get("test")) + } + t.Log("Roundtrip:", time.Since(start)) + h1.PutResponse(resp) + + start = time.Now() + resp, err = h2.Call(t.Context(), remoteConn, NewMSSWith(map[string]string{"err": testPayload})) + t.Log("Roundtrip:", time.Since(start)) + if err != RemoteErr(testPayload) { + t.Errorf("want error %v(%T), got %v(%T)", RemoteErr(testPayload), RemoteErr(testPayload), err, err) + } + if resp != nil { + t.Errorf("want nil, got %q", resp) + } + t.Log("Roundtrip:", time.Since(start)) + h2.PutResponse(resp) +} + +func TestStreamSuite(t *testing.T) { + defer testlogger.T.SetErrorTB(t)() + errFatal := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + grid, err := SetupTestGrid(2) + errFatal(err) + t.Cleanup(grid.Cleanup) + + local := grid.Managers[0] + localHost := grid.Hosts[0] + remote := grid.Managers[1] + remoteHost := grid.Hosts[1] + + connLocalToRemote := local.Connection(remoteHost) + connRemoteLocal := remote.Connection(localHost) + + t.Run("testStreamRoundtrip", func(t *testing.T) { + defer timeout(5 * time.Second)() + testStreamRoundtrip(t, local, remote) + assertNoActive(t, connRemoteLocal) + assertNoActive(t, connLocalToRemote) + }) + t.Run("testStreamCancel", func(t *testing.T) { + defer timeout(5 * time.Second)() + testStreamCancel(t, local, remote) + assertNoActive(t, connRemoteLocal) + assertNoActive(t, connLocalToRemote) + }) + t.Run("testStreamDeadline", func(t *testing.T) { + defer timeout(5 * time.Second)() + testStreamDeadline(t, local, remote) + assertNoActive(t, connRemoteLocal) + assertNoActive(t, connLocalToRemote) + }) + t.Run("testServerOutCongestion", func(t *testing.T) { + defer timeout(1 * time.Minute)() + testServerOutCongestion(t, local, remote) + assertNoActive(t, connRemoteLocal) + assertNoActive(t, connLocalToRemote) + }) + t.Run("testServerInCongestion", func(t *testing.T) { + defer timeout(1 * time.Minute)() + testServerInCongestion(t, local, remote) + assertNoActive(t, connRemoteLocal) + assertNoActive(t, connLocalToRemote) + }) + t.Run("testGenericsStreamRoundtrip", func(t *testing.T) { + defer timeout(1 * time.Minute)() + testGenericsStreamRoundtrip(t, local, remote) + assertNoActive(t, connRemoteLocal) + assertNoActive(t, connLocalToRemote) + }) + t.Run("testGenericsStreamRoundtripSubroute", func(t *testing.T) { + defer timeout(1 * time.Minute)() + testGenericsStreamRoundtripSubroute(t, local, remote) + assertNoActive(t, connRemoteLocal) + assertNoActive(t, connLocalToRemote) + }) + t.Run("testServerStreamResponseBlocked", func(t *testing.T) { + defer timeout(1 * time.Minute)() + testServerStreamResponseBlocked(t, local, remote) + assertNoActive(t, connRemoteLocal) + assertNoActive(t, connLocalToRemote) + }) + t.Run("testServerStreamOnewayNoPing", func(t *testing.T) { + defer timeout(1 * time.Minute)() + testServerStreamNoPing(t, local, remote, 0) + assertNoActive(t, connRemoteLocal) + assertNoActive(t, connLocalToRemote) + }) + t.Run("testServerStreamTwowayNoPing", func(t *testing.T) { + defer timeout(1 * time.Minute)() + testServerStreamNoPing(t, local, remote, 1) + assertNoActive(t, connRemoteLocal) + assertNoActive(t, connLocalToRemote) + }) + t.Run("testServerStreamTwowayPing", func(t *testing.T) { + defer timeout(1 * time.Minute)() + testServerStreamPingRunning(t, local, remote, 1, false, false) + assertNoActive(t, connRemoteLocal) + assertNoActive(t, connLocalToRemote) + }) + t.Run("testServerStreamTwowayPingReq", func(t *testing.T) { + defer timeout(1 * time.Minute)() + testServerStreamPingRunning(t, local, remote, 1, false, true) + assertNoActive(t, connRemoteLocal) + assertNoActive(t, connLocalToRemote) + }) + t.Run("testServerStreamTwowayPingResp", func(t *testing.T) { + defer timeout(1 * time.Minute)() + testServerStreamPingRunning(t, local, remote, 1, true, false) + assertNoActive(t, connRemoteLocal) + assertNoActive(t, connLocalToRemote) + }) + t.Run("testServerStreamTwowayPingReqResp", func(t *testing.T) { + defer timeout(1 * time.Minute)() + testServerStreamPingRunning(t, local, remote, 1, true, true) + assertNoActive(t, connRemoteLocal) + assertNoActive(t, connLocalToRemote) + }) + t.Run("testServerStreamOnewayPing", func(t *testing.T) { + defer timeout(1 * time.Minute)() + testServerStreamPingRunning(t, local, remote, 0, false, true) + assertNoActive(t, connRemoteLocal) + assertNoActive(t, connLocalToRemote) + }) + t.Run("testServerStreamOnewayPingUnblocked", func(t *testing.T) { + defer timeout(1 * time.Minute)() + testServerStreamPingRunning(t, local, remote, 0, false, false) + assertNoActive(t, connRemoteLocal) + assertNoActive(t, connLocalToRemote) + }) +} + +func testStreamRoundtrip(t *testing.T, local, remote *Manager) { + defer testlogger.T.SetErrorTB(t)() + defer timeout(5 * time.Second)() + errFatal := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + + // We fake a local and remote server. + remoteHost := remote.HostName() + + // 1: Echo + register := func(manager *Manager) { + errFatal(manager.RegisterStreamingHandler(handlerTest, StreamHandler{ + Handle: func(ctx context.Context, payload []byte, request <-chan []byte, resp chan<- []byte) *RemoteErr { + for in := range request { + b := append([]byte{}, payload...) + b = append(b, in...) + resp <- b + } + t.Log(GetCaller(ctx).Name, "Handler done") + return nil + }, + OutCapacity: 1, + InCapacity: 1, + })) + // 2: Return as error + errFatal(manager.RegisterStreamingHandler(handlerTest2, StreamHandler{ + Handle: func(ctx context.Context, payload []byte, request <-chan []byte, resp chan<- []byte) *RemoteErr { + for in := range request { + t.Log("2: Got err request", string(in)) + err := RemoteErr(append(payload, in...)) + return &err + } + return nil + }, + OutCapacity: 1, + InCapacity: 1, + })) + } + register(local) + register(remote) + + // local to remote + remoteConn := local.Connection(remoteHost) + const testPayload = "Hello Grid World!" + + start := time.Now() + stream, err := remoteConn.NewStream(t.Context(), handlerTest, []byte(testPayload)) + errFatal(err) + var n int + stream.Requests <- []byte(strconv.Itoa(n)) + for resp := range stream.responses { + errFatal(resp.Err) + t.Logf("got resp: %+v", string(resp.Msg)) + if string(resp.Msg) != testPayload+strconv.Itoa(n) { + t.Errorf("want %q, got %q", testPayload+strconv.Itoa(n), string(resp.Msg)) + } + if n == 10 { + close(stream.Requests) + continue + } + n++ + t.Log("sending new client request") + stream.Requests <- []byte(strconv.Itoa(n)) + } + t.Log("EOF. 10 Roundtrips:", time.Since(start)) +} + +func testStreamCancel(t *testing.T, local, remote *Manager) { + defer testlogger.T.SetErrorTB(t)() + errFatal := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + + // We fake a local and remote server. + remoteHost := remote.HostName() + + // 1: Echo + serverCanceled := make(chan struct{}) + register := func(manager *Manager) { + errFatal(manager.RegisterStreamingHandler(handlerTest, StreamHandler{ + Handle: func(ctx context.Context, payload []byte, request <-chan []byte, resp chan<- []byte) *RemoteErr { + <-ctx.Done() + serverCanceled <- struct{}{} + fmt.Println(GetCaller(ctx).Name, "Server Context canceled") + return nil + }, + OutCapacity: 1, + InCapacity: 0, + })) + errFatal(manager.RegisterStreamingHandler(handlerTest2, StreamHandler{ + Handle: func(ctx context.Context, payload []byte, request <-chan []byte, resp chan<- []byte) *RemoteErr { + <-ctx.Done() + serverCanceled <- struct{}{} + fmt.Println(GetCaller(ctx).Name, "Server Context canceled") + return nil + }, + OutCapacity: 1, + InCapacity: 1, + })) + } + register(local) + register(remote) + + // local to remote + testHandler := func(t *testing.T, handler HandlerID, sendReq bool) { + remoteConn := local.Connection(remoteHost) + const testPayload = "Hello Grid World!" + + ctx, cancel := context.WithCancel(t.Context()) + st, err := remoteConn.NewStream(ctx, handler, []byte(testPayload)) + errFatal(err) + clientCanceled := make(chan time.Time, 1) + err = nil + go func(t *testing.T) { + for resp := range st.responses { + t.Log("got resp:", string(resp.Msg), "err:", resp.Err) + if err != nil { + t.Log("ERROR: got second error:", resp.Err, "first:", err) + continue + } + err = resp.Err + } + t.Log("Client Context canceled. err state:", err) + clientCanceled <- time.Now() + }(t) + start := time.Now() + if st.Requests != nil { + defer close(st.Requests) + } + // Fill up queue. + for sendReq { + select { + case st.Requests <- []byte("Hello"): + time.Sleep(10 * time.Millisecond) + default: + sendReq = false + } + } + cancel() + <-serverCanceled + t.Log("server cancel time:", time.Since(start)) + clientEnd := <-clientCanceled + if !errors.Is(err, context.Canceled) { + t.Error("expected context.Canceled, got", err) + } + t.Log("client after", time.Since(clientEnd)) + } + // local to remote, unbuffered + t.Run("unbuffered", func(t *testing.T) { + testHandler(t, handlerTest, false) + }) + t.Run("buffered", func(t *testing.T) { + testHandler(t, handlerTest2, false) + }) + t.Run("buffered", func(t *testing.T) { + testHandler(t, handlerTest2, true) + }) +} + +// testStreamDeadline will test if server +func testStreamDeadline(t *testing.T, local, remote *Manager) { + defer testlogger.T.SetErrorTB(t)() + errFatal := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + + const wantDL = 50 * time.Millisecond + // We fake a local and remote server. + remoteHost := remote.HostName() + + // 1: Echo + serverCanceled := make(chan time.Duration, 1) + register := func(manager *Manager) { + errFatal(manager.RegisterStreamingHandler(handlerTest, StreamHandler{ + Handle: func(ctx context.Context, payload []byte, request <-chan []byte, resp chan<- []byte) *RemoteErr { + started := time.Now() + dl, _ := ctx.Deadline() + if testing.Verbose() { + fmt.Println(GetCaller(ctx).Name, "Server deadline:", time.Until(dl)) + } + <-ctx.Done() + serverCanceled <- time.Since(started) + if testing.Verbose() { + fmt.Println(GetCaller(ctx).Name, "Server Context canceled with", ctx.Err(), "after", time.Since(started)) + } + return nil + }, + OutCapacity: 1, + InCapacity: 0, + })) + errFatal(manager.RegisterStreamingHandler(handlerTest2, StreamHandler{ + Handle: func(ctx context.Context, payload []byte, request <-chan []byte, resp chan<- []byte) *RemoteErr { + started := time.Now() + dl, _ := ctx.Deadline() + if testing.Verbose() { + fmt.Println(GetCaller(ctx).Name, "Server deadline:", time.Until(dl)) + } + <-ctx.Done() + serverCanceled <- time.Since(started) + if testing.Verbose() { + fmt.Println(GetCaller(ctx).Name, "Server Context canceled with", ctx.Err(), "after", time.Since(started)) + } + return nil + }, + OutCapacity: 1, + InCapacity: 1, + })) + } + register(local) + register(remote) + // Double remote DL + local.debugMsg(debugAddToDeadline, wantDL) + defer local.debugMsg(debugAddToDeadline, time.Duration(0)) + remote.debugMsg(debugAddToDeadline, wantDL) + defer remote.debugMsg(debugAddToDeadline, time.Duration(0)) + + testHandler := func(t *testing.T, handler HandlerID) { + remoteConn := local.Connection(remoteHost) + const testPayload = "Hello Grid World!" + + ctx, cancel := context.WithTimeout(t.Context(), wantDL) + defer cancel() + st, err := remoteConn.NewStream(ctx, handler, []byte(testPayload)) + errFatal(err) + clientCanceled := make(chan time.Duration, 1) + go func() { + started := time.Now() + for resp := range st.responses { + err = resp.Err + } + clientCanceled <- time.Since(started) + }() + serverEnd := <-serverCanceled + clientEnd := <-clientCanceled + t.Log("server cancel time:", serverEnd) + t.Log("client cancel time:", clientEnd) + if !errors.Is(err, context.DeadlineExceeded) { + t.Error("expected context.DeadlineExceeded, got", err) + } + } + // local to remote, unbuffered + t.Run("unbuffered", func(t *testing.T) { + testHandler(t, handlerTest) + }) + + t.Run("buffered", func(t *testing.T) { + testHandler(t, handlerTest2) + }) +} + +func testServerOutCongestion(t *testing.T, local, remote *Manager) { + defer testlogger.T.SetErrorTB(t)() + errFatal := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + + // We fake a local and remote server. + remoteHost := remote.HostName() + + // 1: Echo + serverSent := make(chan struct{}) + register := func(manager *Manager) { + errFatal(manager.RegisterStreamingHandler(handlerTest, StreamHandler{ + Handle: func(ctx context.Context, payload []byte, request <-chan []byte, resp chan<- []byte) *RemoteErr { + // Send many responses. + // Test that this doesn't block. + for i := range byte(100) { + select { + case resp <- []byte{i}: + // ok + case <-ctx.Done(): + return NewRemoteErr(ctx.Err()) + } + if i == 0 { + close(serverSent) + } + } + return nil + }, + OutCapacity: 1, + InCapacity: 0, + })) + errFatal(manager.RegisterSingleHandler(handlerTest2, func(payload []byte) ([]byte, *RemoteErr) { + // Simple roundtrip + return append([]byte{}, payload...), nil + })) + } + register(local) + register(remote) + + remoteConn := local.Connection(remoteHost) + const testPayload = "Hello Grid World!" + + ctx, cancel := context.WithTimeout(t.Context(), time.Minute) + defer cancel() + st, err := remoteConn.NewStream(ctx, handlerTest, []byte(testPayload)) + errFatal(err) + + // Wait for the server to send the first response. + <-serverSent + + // Now do 100 other requests to ensure that the server doesn't block. + for range 100 { + _, err := remoteConn.Request(ctx, handlerTest2, []byte(testPayload)) + errFatal(err) + } + // Drain responses + got := 0 + for resp := range st.responses { + // t.Log("got response", resp) + errFatal(resp.Err) + if resp.Msg[0] != byte(got) { + t.Error("expected response", got, "got", resp.Msg[0]) + } + got++ + } + if got != 100 { + t.Error("expected 100 responses, got", got) + } +} + +func testServerInCongestion(t *testing.T, local, remote *Manager) { + defer testlogger.T.SetErrorTB(t)() + errFatal := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + + // We fake a local and remote server. + remoteHost := remote.HostName() + + // 1: Echo + processHandler := make(chan struct{}) + register := func(manager *Manager) { + errFatal(manager.RegisterStreamingHandler(handlerTest, StreamHandler{ + Handle: func(ctx context.Context, payload []byte, request <-chan []byte, resp chan<- []byte) *RemoteErr { + // Block incoming requests. + var n byte + <-processHandler + for { + select { + case in, ok := <-request: + if !ok { + return nil + } + if in[0] != n { + return NewRemoteErrString(fmt.Sprintf("expected incoming %d, got %d", n, in[0])) + } + n++ + resp <- append([]byte{}, in...) + case <-ctx.Done(): + return NewRemoteErr(ctx.Err()) + } + } + }, + OutCapacity: 5, + InCapacity: 5, + })) + errFatal(manager.RegisterSingleHandler(handlerTest2, func(payload []byte) ([]byte, *RemoteErr) { + // Simple roundtrip + return append([]byte{}, payload...), nil + })) + } + register(local) + register(remote) + + remoteConn := local.Connection(remoteHost) + const testPayload = "Hello Grid World!" + + ctx, cancel := context.WithTimeout(t.Context(), time.Minute) + defer cancel() + st, err := remoteConn.NewStream(ctx, handlerTest, []byte(testPayload)) + errFatal(err) + + // Start sending requests. + go func() { + for i := range byte(100) { + st.Requests <- []byte{i} + } + close(st.Requests) + }() + // Now do 100 other requests to ensure that the server doesn't block. + for range 100 { + _, err := remoteConn.Request(ctx, handlerTest2, []byte(testPayload)) + errFatal(err) + } + // Start processing requests. + close(processHandler) + + // Drain responses + got := 0 + for resp := range st.responses { + // t.Log("got response", resp) + errFatal(resp.Err) + if resp.Msg[0] != byte(got) { + t.Error("expected response", got, "got", resp.Msg[0]) + } + got++ + } + if got != 100 { + t.Error("expected 100 responses, got", got) + } +} + +func testGenericsStreamRoundtrip(t *testing.T, local, remote *Manager) { + defer testlogger.T.SetErrorTB(t)() + defer timeout(5 * time.Second)() + errFatal := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + + // We fake a local and remote server. + remoteHost := remote.HostName() + handler := NewStream[*testRequest, *testRequest, *testResponse](handlerTest, newTestRequest, newTestRequest, newTestResponse) + handler.InCapacity = 1 + handler.OutCapacity = 1 + const payloads = 10 + + // 1: Echo + register := func(manager *Manager) { + errFatal(handler.Register(manager, func(ctx context.Context, pp *testRequest, in <-chan *testRequest, out chan<- *testResponse) *RemoteErr { + n := 0 + for i := range in { + if n > payloads { + panic("too many requests") + } + + // t.Log("Got request:", *i) + out <- &testResponse{ + OrgNum: i.Num + pp.Num, + OrgString: pp.String + i.String, + Embedded: *i, + } + n++ + } + return nil + })) + } + register(local) + register(remote) + + // local to remote + remoteConn := local.Connection(remoteHost) + const testPayload = "Hello Grid World!" + + start := time.Now() + stream, err := handler.Call(t.Context(), remoteConn, &testRequest{Num: 1, String: testPayload}) + errFatal(err) + go func() { + defer close(stream.Requests) + for i := range payloads { + // t.Log("sending new client request") + stream.Requests <- &testRequest{Num: i, String: testPayload} + } + }() + var n int + err = stream.Results(func(resp *testResponse) error { + const wantString = testPayload + testPayload + if resp.OrgString != testPayload+testPayload { + t.Errorf("want %q, got %q", wantString, resp.OrgString) + } + if resp.OrgNum != n+1 { + t.Errorf("want %d, got %d", n+1, resp.OrgNum) + } + handler.PutResponse(resp) + n++ + return nil + }) + errFatal(err) + t.Log("EOF.", payloads, " Roundtrips:", time.Since(start)) +} + +func testGenericsStreamRoundtripSubroute(t *testing.T, local, remote *Manager) { + defer testlogger.T.SetErrorTB(t)() + defer timeout(5 * time.Second)() + errFatal := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + + // We fake a local and remote server. + remoteHost := remote.HostName() + handler := NewStream[*testRequest, *testRequest, *testResponse](handlerTest, newTestRequest, newTestRequest, newTestResponse) + handler.InCapacity = 1 + handler.OutCapacity = 1 + const payloads = 10 + + // 1: Echo + register := func(manager *Manager) { + errFatal(handler.Register(manager, func(ctx context.Context, pp *testRequest, in <-chan *testRequest, out chan<- *testResponse) *RemoteErr { + sub := GetSubroute(ctx) + if sub != "subroute/1" { + t.Fatal("expected subroute/1, got", sub) + } + n := 0 + for i := range in { + if n > payloads { + panic("too many requests") + } + + // t.Log("Got request:", *i) + out <- &testResponse{ + OrgNum: i.Num + pp.Num, + OrgString: pp.String + i.String, + Embedded: *i, + } + n++ + } + return nil + }, "subroute", "1")) + } + register(local) + register(remote) + + // local to remote + remoteConn := local.Connection(remoteHost) + const testPayload = "Hello Grid World!" + // Add subroute + remoteSub := remoteConn.Subroute(strings.Join([]string{"subroute", "1"}, "/")) + + start := time.Now() + stream, err := handler.Call(t.Context(), remoteSub, &testRequest{Num: 1, String: testPayload}) + errFatal(err) + go func() { + defer close(stream.Requests) + for i := range payloads { + // t.Log("sending new client request") + stream.Requests <- &testRequest{Num: i, String: testPayload} + } + }() + var n int + err = stream.Results(func(resp *testResponse) error { + // t.Logf("got resp: %+v", *resp.Msg) + const wantString = testPayload + testPayload + if resp.OrgString != testPayload+testPayload { + t.Errorf("want %q, got %q", wantString, resp.OrgString) + } + if resp.OrgNum != n+1 { + t.Errorf("want %d, got %d", n+1, resp.OrgNum) + } + handler.PutResponse(resp) + n++ + return nil + }) + + errFatal(err) + t.Log("EOF.", payloads, " Roundtrips:", time.Since(start)) +} + +// testServerStreamResponseBlocked will test if server can handle a blocked response stream +func testServerStreamResponseBlocked(t *testing.T, local, remote *Manager) { + defer testlogger.T.SetErrorTB(t)() + errFatal := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + + // We fake a local and remote server. + remoteHost := remote.HostName() + + // 1: Echo + serverSent := make(chan struct{}) + serverCanceled := make(chan struct{}) + register := func(manager *Manager) { + errFatal(manager.RegisterStreamingHandler(handlerTest, StreamHandler{ + Handle: func(ctx context.Context, payload []byte, _ <-chan []byte, resp chan<- []byte) *RemoteErr { + // Send many responses. + // Test that this doesn't block. + for i := range byte(100) { + select { + case resp <- []byte{i}: + // ok + case <-ctx.Done(): + close(serverCanceled) + return NewRemoteErr(ctx.Err()) + } + if i == 1 { + close(serverSent) + } + } + return nil + }, + OutCapacity: 1, + InCapacity: 0, + })) + } + register(local) + register(remote) + + remoteConn := local.Connection(remoteHost) + const testPayload = "Hello Grid World!" + + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + + st, err := remoteConn.NewStream(ctx, handlerTest, []byte(testPayload)) + errFatal(err) + + // Wait for the server to send the first response. + <-serverSent + + // Read back from the stream and block. + nowBlocking := make(chan struct{}) + stopBlocking := make(chan struct{}) + defer close(stopBlocking) + go func() { + st.Results(func(b []byte) error { + close(nowBlocking) + // Block until test is done. + <-stopBlocking + return nil + }) + }() + + <-nowBlocking + // Wait for the receiver channel to fill. + for len(st.responses) != cap(st.responses) { + time.Sleep(time.Millisecond) + } + cancel() + <-serverCanceled + local.debugMsg(debugIsOutgoingClosed, st.muxID, func(closed bool) { + if !closed { + t.Error("expected outgoing closed") + } else { + t.Log("outgoing was closed") + } + }) + + // Drain responses and check if error propagated. + err = st.Results(func(b []byte) error { + return nil + }) + if !errors.Is(err, context.Canceled) { + t.Error("expected context.Canceled, got", err) + } +} + +// testServerStreamNoPing will test if server and client handle no pings. +func testServerStreamNoPing(t *testing.T, local, remote *Manager, inCap int) { + defer testlogger.T.SetErrorTB(t)() + errFatal := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + + // We fake a local and remote server. + remoteHost := remote.HostName() + + // 1: Echo + reqStarted := make(chan struct{}) + serverCanceled := make(chan struct{}) + register := func(manager *Manager) { + errFatal(manager.RegisterStreamingHandler(handlerTest, StreamHandler{ + Handle: func(ctx context.Context, payload []byte, _ <-chan []byte, resp chan<- []byte) *RemoteErr { + close(reqStarted) + // Just wait for it to cancel. + <-ctx.Done() + close(serverCanceled) + return NewRemoteErr(ctx.Err()) + }, + OutCapacity: 1, + InCapacity: inCap, + })) + } + register(local) + register(remote) + + remoteConn := local.Connection(remoteHost) + const testPayload = "Hello Grid World!" + remoteConn.debugMsg(debugSetClientPingDuration, 100*time.Millisecond) + defer remoteConn.debugMsg(debugSetClientPingDuration, clientPingInterval) + + ctx, cancel := context.WithTimeout(t.Context(), time.Minute) + defer cancel() + st, err := remoteConn.NewStream(ctx, handlerTest, []byte(testPayload)) + errFatal(err) + + // Wait for the server start the request. + <-reqStarted + + // Stop processing requests + nowBlocking := make(chan struct{}) + remoteConn.debugMsg(debugBlockInboundMessages, nowBlocking) + + // Check that local returned. + err = st.Results(func(b []byte) error { + return nil + }) + if err == nil { + t.Fatal("expected error, got nil") + } + t.Logf("response: %v", err) + + // Check that remote is canceled. + <-serverCanceled + close(nowBlocking) +} + +// testServerStreamPingRunning will test if server and client handle ping even when blocked. +func testServerStreamPingRunning(t *testing.T, local, remote *Manager, inCap int, blockResp, blockReq bool) { + defer testlogger.T.SetErrorTB(t)() + errFatal := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + + // We fake a local and remote server. + remoteHost := remote.HostName() + + // 1: Echo + reqStarted := make(chan struct{}) + serverCanceled := make(chan struct{}) + register := func(manager *Manager) { + errFatal(manager.RegisterStreamingHandler(handlerTest, StreamHandler{ + Handle: func(ctx context.Context, payload []byte, req <-chan []byte, resp chan<- []byte) *RemoteErr { + close(reqStarted) + // Just wait for it to cancel. + for blockResp { + select { + case <-ctx.Done(): + close(serverCanceled) + return NewRemoteErr(ctx.Err()) + case resp <- []byte{1}: + time.Sleep(10 * time.Millisecond) + } + } + // Just wait for it to cancel. + <-ctx.Done() + close(serverCanceled) + return NewRemoteErr(ctx.Err()) + }, + OutCapacity: 1, + InCapacity: inCap, + })) + } + register(local) + register(remote) + + remoteConn := local.Connection(remoteHost) + const testPayload = "Hello Grid World!" + remoteConn.debugMsg(debugSetClientPingDuration, 100*time.Millisecond) + defer remoteConn.debugMsg(debugSetClientPingDuration, clientPingInterval) + + ctx, cancel := context.WithTimeout(t.Context(), time.Minute) + defer cancel() + st, err := remoteConn.NewStream(ctx, handlerTest, []byte(testPayload)) + errFatal(err) + + // Wait for the server start the request. + <-reqStarted + + // Block until we have exceeded the deadline several times over. + nowBlocking := make(chan struct{}) + var mu sync.Mutex + time.AfterFunc(time.Second, func() { + mu.Lock() + cancel() + close(nowBlocking) + mu.Unlock() + }) + if inCap > 0 { + go func() { + defer close(st.Requests) + if !blockReq { + <-nowBlocking + return + } + for { + select { + case <-nowBlocking: + return + case <-st.Done(): + case st.Requests <- []byte{1}: + time.Sleep(10 * time.Millisecond) + } + } + }() + } + // Check that local returned. + err = st.Results(func(b []byte) error { + <-st.Done() + return ctx.Err() + }) + mu.Lock() + select { + case <-nowBlocking: + default: + t.Fatal("expected to be blocked. got err", err) + } + if err == nil { + t.Fatal("expected error, got nil") + } + t.Logf("response: %v", err) + // Check that remote is canceled. + <-serverCanceled +} + +func timeout(after time.Duration) (cancel func()) { + c := time.After(after) + cc := make(chan struct{}) + go func() { + select { + case <-cc: + return + case <-c: + buf := make([]byte, 1<<20) + stacklen := runtime.Stack(buf, true) + fmt.Printf("=== Timeout, assuming deadlock ===\n*** goroutine dump...\n%s\n*** end\n", string(buf[:stacklen])) + os.Exit(2) + } + }() + return func() { + close(cc) + } +} + +func assertNoActive(t *testing.T, c *Connection) { + t.Helper() + // Tiny bit racy for tests, but we try to play nice. + for i := 10; i >= 0; i-- { + runtime.Gosched() + stats := c.Stats() + if stats.IncomingStreams != 0 { + if i > 0 { + time.Sleep(100 * time.Millisecond) + continue + } + var found []uint64 + c.inStream.Range(func(key uint64, value *muxServer) bool { + found = append(found, key) + return true + }) + t.Errorf("expected no active streams, got %d incoming: %v", stats.IncomingStreams, found) + } + if stats.OutgoingStreams != 0 { + if i > 0 { + time.Sleep(100 * time.Millisecond) + continue + } + var found []uint64 + c.outgoing.Range(func(key uint64, value *muxClient) bool { + found = append(found, key) + return true + }) + t.Errorf("expected no active streams, got %d outgoing: %v", stats.OutgoingStreams, found) + } + return + } +} + +// Inserted manually. +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[StateUnconnected-0] + _ = x[StateConnecting-1] + _ = x[StateConnected-2] + _ = x[StateConnectionError-3] + _ = x[StateShutdown-4] +} + +const stateName = "UnconnectedConnectingConnectedConnectionErrorShutdown" + +var stateIndex = [...]uint8{0, 11, 21, 30, 45, 53} + +func (i State) String() string { + if i >= State(len(stateIndex)-1) { + return "State(" + strconv.FormatInt(int64(i), 10) + ")" + } + return stateName[stateIndex[i]:stateIndex[i+1]] +} diff --git a/internal/grid/grid_types_msgp_test.go b/internal/grid/grid_types_msgp_test.go new file mode 100644 index 00000000..6ac8a6dd --- /dev/null +++ b/internal/grid/grid_types_msgp_test.go @@ -0,0 +1,368 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package grid + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *testRequest) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Num": + z.Num, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Num") + return + } + case "String": + z.String, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "String") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z testRequest) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 2 + // write "Num" + err = en.Append(0x82, 0xa3, 0x4e, 0x75, 0x6d) + if err != nil { + return + } + err = en.WriteInt(z.Num) + if err != nil { + err = msgp.WrapError(err, "Num") + return + } + // write "String" + err = en.Append(0xa6, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67) + if err != nil { + return + } + err = en.WriteString(z.String) + if err != nil { + err = msgp.WrapError(err, "String") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z testRequest) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 2 + // string "Num" + o = append(o, 0x82, 0xa3, 0x4e, 0x75, 0x6d) + o = msgp.AppendInt(o, z.Num) + // string "String" + o = append(o, 0xa6, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67) + o = msgp.AppendString(o, z.String) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *testRequest) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Num": + z.Num, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Num") + return + } + case "String": + z.String, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "String") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z testRequest) Msgsize() (s int) { + s = 1 + 4 + msgp.IntSize + 7 + msgp.StringPrefixSize + len(z.String) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *testResponse) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "OrgNum": + z.OrgNum, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "OrgNum") + return + } + case "OrgString": + z.OrgString, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "OrgString") + return + } + case "Embedded": + var zb0002 uint32 + zb0002, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err, "Embedded") + return + } + for zb0002 > 0 { + zb0002-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err, "Embedded") + return + } + switch msgp.UnsafeString(field) { + case "Num": + z.Embedded.Num, err = dc.ReadInt() + if err != nil { + err = msgp.WrapError(err, "Embedded", "Num") + return + } + case "String": + z.Embedded.String, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Embedded", "String") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err, "Embedded") + return + } + } + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *testResponse) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "OrgNum" + err = en.Append(0x83, 0xa6, 0x4f, 0x72, 0x67, 0x4e, 0x75, 0x6d) + if err != nil { + return + } + err = en.WriteInt(z.OrgNum) + if err != nil { + err = msgp.WrapError(err, "OrgNum") + return + } + // write "OrgString" + err = en.Append(0xa9, 0x4f, 0x72, 0x67, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67) + if err != nil { + return + } + err = en.WriteString(z.OrgString) + if err != nil { + err = msgp.WrapError(err, "OrgString") + return + } + // write "Embedded" + err = en.Append(0xa8, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x65, 0x64) + if err != nil { + return + } + // map header, size 2 + // write "Num" + err = en.Append(0x82, 0xa3, 0x4e, 0x75, 0x6d) + if err != nil { + return + } + err = en.WriteInt(z.Embedded.Num) + if err != nil { + err = msgp.WrapError(err, "Embedded", "Num") + return + } + // write "String" + err = en.Append(0xa6, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67) + if err != nil { + return + } + err = en.WriteString(z.Embedded.String) + if err != nil { + err = msgp.WrapError(err, "Embedded", "String") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *testResponse) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "OrgNum" + o = append(o, 0x83, 0xa6, 0x4f, 0x72, 0x67, 0x4e, 0x75, 0x6d) + o = msgp.AppendInt(o, z.OrgNum) + // string "OrgString" + o = append(o, 0xa9, 0x4f, 0x72, 0x67, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67) + o = msgp.AppendString(o, z.OrgString) + // string "Embedded" + o = append(o, 0xa8, 0x45, 0x6d, 0x62, 0x65, 0x64, 0x64, 0x65, 0x64) + // map header, size 2 + // string "Num" + o = append(o, 0x82, 0xa3, 0x4e, 0x75, 0x6d) + o = msgp.AppendInt(o, z.Embedded.Num) + // string "String" + o = append(o, 0xa6, 0x53, 0x74, 0x72, 0x69, 0x6e, 0x67) + o = msgp.AppendString(o, z.Embedded.String) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *testResponse) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "OrgNum": + z.OrgNum, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "OrgNum") + return + } + case "OrgString": + z.OrgString, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "OrgString") + return + } + case "Embedded": + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Embedded") + return + } + for zb0002 > 0 { + zb0002-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err, "Embedded") + return + } + switch msgp.UnsafeString(field) { + case "Num": + z.Embedded.Num, bts, err = msgp.ReadIntBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Embedded", "Num") + return + } + case "String": + z.Embedded.String, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Embedded", "String") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err, "Embedded") + return + } + } + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *testResponse) Msgsize() (s int) { + s = 1 + 7 + msgp.IntSize + 10 + msgp.StringPrefixSize + len(z.OrgString) + 9 + 1 + 4 + msgp.IntSize + 7 + msgp.StringPrefixSize + len(z.Embedded.String) + return +} diff --git a/internal/grid/grid_types_test.go b/internal/grid/grid_types_test.go new file mode 100644 index 00000000..1c3b3bf7 --- /dev/null +++ b/internal/grid/grid_types_test.go @@ -0,0 +1,39 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package grid + +//go:generate msgp -unexported -file=$GOFILE -tests=false -o=grid_types_msgp_test.go + +type testRequest struct { + Num int + String string +} + +type testResponse struct { + OrgNum int + OrgString string + Embedded testRequest +} + +func newTestRequest() *testRequest { + return &testRequest{} +} + +func newTestResponse() *testResponse { + return &testResponse{} +} diff --git a/internal/grid/handlers.go b/internal/grid/handlers.go new file mode 100644 index 00000000..1d20fa9b --- /dev/null +++ b/internal/grid/handlers.go @@ -0,0 +1,907 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package grid + +import ( + "context" + "encoding/hex" + "errors" + "fmt" + "strings" + + "github.com/minio/minio/internal/bpool" + "github.com/minio/minio/internal/hash/sha256" + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/tinylib/msgp/msgp" +) + +//go:generate stringer -type=HandlerID -output=handlers_string.go -trimprefix=Handler msg.go $GOFILE + +// HandlerID is a handler identifier. +// It is used to determine request routing on the server. +// Handlers can be registered with a static subroute. +// Do NOT remove or change the order of existing handlers. +const ( + // handlerInvalid is reserved to check for uninitialized values. + handlerInvalid HandlerID = iota + HandlerLockLock + HandlerLockRLock + HandlerLockUnlock + HandlerLockRUnlock + HandlerLockRefresh + HandlerLockForceUnlock + HandlerWalkDir + HandlerStatVol + HandlerDiskInfo + HandlerNSScanner + HandlerReadXL + HandlerReadVersion + HandlerDeleteFile + HandlerDeleteVersion + HandlerUpdateMetadata + HandlerWriteMetadata + HandlerCheckParts + HandlerRenameData + HandlerRenameFile + HandlerReadAll + HandlerServerVerify + HandlerTrace + HandlerListen + HandlerDeleteBucketMetadata + HandlerLoadBucketMetadata + HandlerReloadSiteReplicationConfig + HandlerReloadPoolMeta + HandlerStopRebalance + HandlerLoadRebalanceMeta + HandlerLoadTransitionTierConfig + HandlerDeletePolicy + HandlerLoadPolicy + HandlerLoadPolicyMapping + HandlerDeleteServiceAccount + HandlerLoadServiceAccount + HandlerDeleteUser + HandlerLoadUser + HandlerLoadGroup + HandlerHealBucket + HandlerMakeBucket + HandlerHeadBucket + HandlerDeleteBucket + HandlerGetMetrics + HandlerGetResourceMetrics + HandlerGetMemInfo + HandlerGetProcInfo + HandlerGetOSInfo + HandlerGetPartitions + HandlerGetNetInfo + HandlerGetCPUs + HandlerServerInfo + HandlerGetSysConfig + HandlerGetSysServices + HandlerGetSysErrors + HandlerGetAllBucketStats + HandlerGetBucketStats + HandlerGetSRMetrics + HandlerGetPeerMetrics + HandlerGetMetacacheListing + HandlerUpdateMetacacheListing + HandlerGetPeerBucketMetrics + HandlerStorageInfo + HandlerConsoleLog + HandlerListDir + HandlerGetLocks + HandlerBackgroundHealStatus + HandlerGetLastDayTierStats + HandlerSignalService + HandlerGetBandwidth + HandlerWriteAll + HandlerListBuckets + HandlerRenameDataInline + HandlerRenameData2 + HandlerCheckParts2 + HandlerRenamePart + HandlerClearUploadID + HandlerCheckParts3 + + // Add more above here ^^^ + // If all handlers are used, the type of Handler can be changed. + // Handlers have no versioning, so non-compatible handler changes must result in new IDs. + handlerTest + handlerTest2 + handlerLast +) + +// handlerPrefixes are prefixes for handler IDs used for tracing. +// If a handler is not listed here, it will be traced with "grid" prefix. +var handlerPrefixes = [handlerLast]string{ + HandlerLockLock: lockPrefix, + HandlerLockRLock: lockPrefix, + HandlerLockUnlock: lockPrefix, + HandlerLockRUnlock: lockPrefix, + HandlerLockRefresh: lockPrefix, + HandlerLockForceUnlock: lockPrefix, + HandlerWalkDir: storagePrefix, + HandlerStatVol: storagePrefix, + HandlerDiskInfo: storagePrefix, + HandlerNSScanner: storagePrefix, + HandlerReadXL: storagePrefix, + HandlerReadVersion: storagePrefix, + HandlerDeleteFile: storagePrefix, + HandlerDeleteVersion: storagePrefix, + HandlerUpdateMetadata: storagePrefix, + HandlerWriteMetadata: storagePrefix, + HandlerCheckParts: storagePrefix, + HandlerRenameData: storagePrefix, + HandlerRenameFile: storagePrefix, + HandlerReadAll: storagePrefix, + HandlerWriteAll: storagePrefix, + HandlerServerVerify: bootstrapPrefix, + HandlerTrace: peerPrefix, + HandlerListen: peerPrefix, + HandlerDeleteBucketMetadata: peerPrefix, + HandlerLoadBucketMetadata: peerPrefix, + HandlerReloadSiteReplicationConfig: peerPrefix, + HandlerReloadPoolMeta: peerPrefix, + HandlerStopRebalance: peerPrefix, + HandlerLoadRebalanceMeta: peerPrefix, + HandlerLoadTransitionTierConfig: peerPrefix, + HandlerDeletePolicy: peerPrefix, + HandlerLoadPolicy: peerPrefix, + HandlerLoadPolicyMapping: peerPrefix, + HandlerDeleteServiceAccount: peerPrefix, + HandlerLoadServiceAccount: peerPrefix, + HandlerDeleteUser: peerPrefix, + HandlerLoadUser: peerPrefix, + HandlerLoadGroup: peerPrefix, + HandlerMakeBucket: peerPrefixS3, + HandlerHeadBucket: peerPrefixS3, + HandlerDeleteBucket: peerPrefixS3, + HandlerHealBucket: healPrefix, + HandlerGetMetrics: peerPrefix, + HandlerGetResourceMetrics: peerPrefix, + HandlerGetMemInfo: peerPrefix, + HandlerGetProcInfo: peerPrefix, + HandlerGetOSInfo: peerPrefix, + HandlerGetPartitions: peerPrefix, + HandlerGetNetInfo: peerPrefix, + HandlerGetCPUs: peerPrefix, + HandlerServerInfo: peerPrefix, + HandlerGetSysConfig: peerPrefix, + HandlerGetSysServices: peerPrefix, + HandlerGetSysErrors: peerPrefix, + HandlerGetAllBucketStats: peerPrefix, + HandlerGetBucketStats: peerPrefix, + HandlerGetSRMetrics: peerPrefix, + HandlerGetPeerMetrics: peerPrefix, + HandlerGetMetacacheListing: peerPrefix, + HandlerUpdateMetacacheListing: peerPrefix, + HandlerGetPeerBucketMetrics: peerPrefix, + HandlerStorageInfo: peerPrefix, + HandlerConsoleLog: peerPrefix, + HandlerListDir: storagePrefix, + HandlerListBuckets: peerPrefixS3, + HandlerRenameDataInline: storagePrefix, + HandlerRenameData2: storagePrefix, + HandlerCheckParts2: storagePrefix, + HandlerCheckParts3: storagePrefix, + HandlerRenamePart: storagePrefix, + HandlerClearUploadID: peerPrefix, +} + +const ( + lockPrefix = "lockR" + storagePrefix = "storageR" + bootstrapPrefix = "bootstrap" + peerPrefix = "peer" + peerPrefixS3 = "peerS3" + healPrefix = "heal" +) + +func init() { + // Static check if we exceed 255 handler ids. + // Extend the type to uint16 when hit. + if uint32(handlerLast) > 255 { + panic(fmt.Sprintf("out of handler IDs. %d > %d", handlerLast, 255)) + } +} + +func (h HandlerID) valid() bool { + return h != handlerInvalid && h < handlerLast +} + +func (h HandlerID) isTestHandler() bool { + return h >= handlerTest && h <= handlerTest2 +} + +// RemoteErr is a remote error type. +// Any error seen on a remote will be returned like this. +type RemoteErr string + +// NewRemoteErr creates a new remote error. +// The error type is not preserved. +func NewRemoteErr(err error) *RemoteErr { + if err == nil { + return nil + } + r := RemoteErr(err.Error()) + return &r +} + +// NewRemoteErrf creates a new remote error from a format string. +func NewRemoteErrf(format string, a ...any) *RemoteErr { + r := RemoteErr(fmt.Sprintf(format, a...)) + return &r +} + +// NewNPErr is a helper to no payload and optional remote error. +// The error type is not preserved. +func NewNPErr(err error) (NoPayload, *RemoteErr) { + if err == nil { + return NoPayload{}, nil + } + r := RemoteErr(err.Error()) + return NoPayload{}, &r +} + +// NewRemoteErrString creates a new remote error from a string. +func NewRemoteErrString(msg string) *RemoteErr { + r := RemoteErr(msg) + return &r +} + +func (r RemoteErr) Error() string { + return string(r) +} + +// Is returns if the string representation matches. +func (r *RemoteErr) Is(other error) bool { + if r == nil || other == nil { + return r == other + } + var o RemoteErr + if errors.As(other, &o) { + return r == &o + } + return false +} + +// IsRemoteErr returns the value if the error is a RemoteErr. +func IsRemoteErr(err error) *RemoteErr { + var r RemoteErr + if errors.As(err, &r) { + return &r + } + return nil +} + +type ( + // SingleHandlerFn is handlers for one to one requests. + // A non-nil error value will be returned as RemoteErr(msg) to client. + // No client information or cancellation (deadline) is available. + // Include this in payload if needed. + // Payload should be recycled with PutByteBuffer if not needed after the call. + SingleHandlerFn func(payload []byte) ([]byte, *RemoteErr) + + // StatelessHandlerFn must handle incoming stateless request. + // A non-nil error value will be returned as RemoteErr(msg) to client. + StatelessHandlerFn func(ctx context.Context, payload []byte, resp chan<- []byte) *RemoteErr + + // StatelessHandler is handlers for one to many requests, + // where responses may be dropped. + // Stateless requests provide no incoming stream and there is no flow control + // on outgoing messages. + StatelessHandler struct { + Handle StatelessHandlerFn + // OutCapacity is the output capacity on the caller. + // If <= 0 capacity will be 1. + OutCapacity int + } + + // StreamHandlerFn must process a request with an optional initial payload. + // It must keep consuming from 'in' until it returns. + // 'in' and 'out' are independent. + // The handler should never close out. + // Buffers received from 'in' can be recycled with PutByteBuffer. + // Buffers sent on out can not be referenced once sent. + StreamHandlerFn func(ctx context.Context, payload []byte, in <-chan []byte, out chan<- []byte) *RemoteErr + + // StreamHandler handles fully bidirectional streams, + // There is flow control in both directions. + StreamHandler struct { + // Handle an incoming request. Initial payload is sent. + // Additional input packets (if any) are streamed to request. + // Upstream will block when request channel is full. + // Response packets can be sent at any time. + // Any non-nil error sent as response means no more responses are sent. + Handle StreamHandlerFn + + // Subroute for handler. + // Subroute must be static and clients should specify a matching subroute. + // Should not be set unless there are different handlers for the same HandlerID. + Subroute string + + // OutCapacity is the output capacity. If <= 0 capacity will be 1. + OutCapacity int + + // InCapacity is the output capacity. + // If == 0 no input is expected + InCapacity int + } +) + +type subHandlerID [32]byte + +func makeSubHandlerID(id HandlerID, subRoute string) subHandlerID { + b := subHandlerID(sha256.Sum256([]byte(subRoute))) + b[0] = byte(id) + b[1] = 0 // Reserved + return b +} + +func (s subHandlerID) withHandler(id HandlerID) subHandlerID { + s[0] = byte(id) + s[1] = 0 // Reserved + return s +} + +func (s *subHandlerID) String() string { + if s == nil { + return "" + } + return hex.EncodeToString(s[:]) +} + +func makeZeroSubHandlerID(id HandlerID) subHandlerID { + return subHandlerID{byte(id)} +} + +type handlers struct { + single [handlerLast]SingleHandlerFn + stateless [handlerLast]*StatelessHandler + streams [handlerLast]*StreamHandler + + subSingle map[subHandlerID]SingleHandlerFn + subStateless map[subHandlerID]*StatelessHandler + subStreams map[subHandlerID]*StreamHandler +} + +func (h *handlers) init() { + h.subSingle = make(map[subHandlerID]SingleHandlerFn) + h.subStateless = make(map[subHandlerID]*StatelessHandler) + h.subStreams = make(map[subHandlerID]*StreamHandler) +} + +func (h *handlers) hasAny(id HandlerID) bool { + if !id.valid() { + return false + } + return h.single[id] != nil || h.stateless[id] != nil || h.streams[id] != nil +} + +func (h *handlers) hasSubhandler(id subHandlerID) bool { + return h.subSingle[id] != nil || h.subStateless[id] != nil || h.subStreams[id] != nil +} + +// RoundTripper provides an interface for type roundtrip serialization. +type RoundTripper interface { + msgp.Unmarshaler + msgp.Marshaler + msgp.Sizer + + comparable +} + +// SingleHandler is a type safe handler for single roundtrip requests. +type SingleHandler[Req, Resp RoundTripper] struct { + id HandlerID + sharedResp bool + callReuseReq bool + ignoreNilConn bool + + newReq func() Req + newResp func() Resp + + recycleReq func(Req) + recycleResp func(Resp) +} + +func recycleFunc[RT RoundTripper](newRT func() RT) (newFn func() RT, recycle func(r RT)) { + rAny := any(newRT()) + var rZero RT + if _, ok := rAny.(Recycler); ok { + return newRT, func(r RT) { + if r != rZero { + if rc, ok := any(r).(Recycler); ok { + rc.Recycle() + } + } + } + } + pool := bpool.Pool[RT]{ + New: func() RT { + return newRT() + }, + } + return pool.Get, + func(r RT) { + if r != rZero { + //nolint:staticcheck // SA6002 IT IS A GENERIC VALUE! + pool.Put(r) + } + } +} + +// NewSingleHandler creates a typed handler that can provide Marshal/Unmarshal. +// Use Register to register a server handler. +// Use Call to initiate a clientside call. +func NewSingleHandler[Req, Resp RoundTripper](h HandlerID, newReq func() Req, newResp func() Resp) *SingleHandler[Req, Resp] { + s := SingleHandler[Req, Resp]{id: h} + s.newReq, s.recycleReq = recycleFunc[Req](newReq) + s.newResp, s.recycleResp = recycleFunc[Resp](newResp) + if _, ok := any(newReq()).(Recycler); ok { + s.callReuseReq = true + } + return &s +} + +// PutResponse will accept a response for reuse. +// This can be used by a caller to recycle a response after receiving it from a Call. +func (h *SingleHandler[Req, Resp]) PutResponse(r Resp) { + h.recycleResp(r) +} + +// AllowCallRequestPool indicates it is safe to reuse the request +// on the client side, meaning the request is recycled/pooled when a request is sent. +// CAREFUL: This should only be used when there are no pointers, slices that aren't freshly constructed. +func (h *SingleHandler[Req, Resp]) AllowCallRequestPool(b bool) *SingleHandler[Req, Resp] { + h.callReuseReq = b + return h +} + +// IgnoreNilConn will ignore nil connections when calling. +// This will make Call return nil instead of ErrDisconnected when the connection is nil. +// This may only be set ONCE before use. +func (h *SingleHandler[Req, Resp]) IgnoreNilConn() *SingleHandler[Req, Resp] { + if h.ignoreNilConn { + gridLogOnceIf(context.Background(), fmt.Errorf("%s: IgnoreNilConn called twice", h.id.String()), h.id.String()+"IgnoreNilConn") + } + h.ignoreNilConn = true + return h +} + +// WithSharedResponse indicates it is unsafe to reuse the response +// when it has been returned on a handler. +// This will disable automatic response recycling/pooling. +// Typically this is used when the response sharing part of its data structure. +func (h *SingleHandler[Req, Resp]) WithSharedResponse() *SingleHandler[Req, Resp] { + h.sharedResp = true + return h +} + +// NewResponse creates a new response. +// Handlers can use this to create a reusable response. +// The response may be reused, so caller should clear any fields. +func (h *SingleHandler[Req, Resp]) NewResponse() Resp { + return h.newResp() +} + +// NewRequest creates a new request. +// Handlers can use this to create a reusable request. +// The request may be reused, so caller should clear any fields. +func (h *SingleHandler[Req, Resp]) NewRequest() Req { + return h.newReq() +} + +// Register a handler for a Req -> Resp roundtrip. +// Requests are automatically recycled. +func (h *SingleHandler[Req, Resp]) Register(m *Manager, handle func(req Req) (resp Resp, err *RemoteErr), subroute ...string) error { + if h.newReq == nil { + return errors.New("newReq nil in NewSingleHandler") + } + if h.newResp == nil { + return errors.New("newResp nil in NewSingleHandler") + } + return m.RegisterSingleHandler(h.id, func(payload []byte) ([]byte, *RemoteErr) { + req := h.NewRequest() + _, err := req.UnmarshalMsg(payload) + if err != nil { + PutByteBuffer(payload) + r := RemoteErr(err.Error()) + return nil, &r + } + resp, rerr := handle(req) + h.recycleReq(req) + + if rerr != nil { + PutByteBuffer(payload) + return nil, rerr + } + payload, err = resp.MarshalMsg(payload[:0]) + if !h.sharedResp { + h.PutResponse(resp) + } + if err != nil { + PutByteBuffer(payload) + r := RemoteErr(err.Error()) + return nil, &r + } + return payload, nil + }, subroute...) +} + +// Requester is able to send requests to a remote. +type Requester interface { + Request(ctx context.Context, h HandlerID, req []byte) ([]byte, error) +} + +// Call the remote with the request and return the response. +// The response should be returned with PutResponse when no error. +// If no deadline is set, a 1-minute deadline is added. +func (h *SingleHandler[Req, Resp]) Call(ctx context.Context, c Requester, req Req) (resp Resp, err error) { + if c == nil { + if h.ignoreNilConn { + return resp, nil + } + return resp, ErrDisconnected + } + payload, err := req.MarshalMsg(GetByteBufferCap(req.Msgsize())) + if err != nil { + return resp, err + } + switch any(req).(type) { + case *MSS, *URLValues: + ctx = context.WithValue(ctx, TraceParamsKey{}, req) + case *NoPayload, *Bytes: + // do not need to trace nopayload and bytes payload + default: + ctx = context.WithValue(ctx, TraceParamsKey{}, fmt.Sprintf("type=%T", req)) + } + if h.callReuseReq { + defer h.recycleReq(req) + } + res, err := c.Request(ctx, h.id, payload) + PutByteBuffer(payload) + if err != nil { + return resp, err + } + defer PutByteBuffer(res) + r := h.NewResponse() + _, err = r.UnmarshalMsg(res) + if err != nil { + h.PutResponse(r) + return resp, err + } + return r, err +} + +// RemoteClient contains information about the caller. +type RemoteClient struct { + Name string +} + +type ( + ctxCallerKey = struct{} + ctxSubrouteKey = struct{} +) + +// GetCaller returns caller information from contexts provided to handlers. +func GetCaller(ctx context.Context) *RemoteClient { + val, _ := ctx.Value(ctxCallerKey{}).(*RemoteClient) + return val +} + +// GetSubroute returns caller information from contexts provided to handlers. +func GetSubroute(ctx context.Context) string { + //nolint:staticcheck // SA1029 Staticcheck is drunk. + val, _ := ctx.Value(ctxSubrouteKey{}).(string) + return val +} + +func setCaller(ctx context.Context, cl *RemoteClient) context.Context { + //nolint:staticcheck // SA1029 Staticcheck is drunk. + return context.WithValue(ctx, ctxCallerKey{}, cl) +} + +func setSubroute(ctx context.Context, s string) context.Context { + //nolint:staticcheck // SA1029 Staticcheck is drunk. + return context.WithValue(ctx, ctxSubrouteKey{}, s) +} + +// StreamTypeHandler is a type safe handler for streaming requests. +type StreamTypeHandler[Payload, Req, Resp RoundTripper] struct { + WithPayload bool + + // Override the default capacities (1) + OutCapacity int + + // Set to 0 if no input is expected. + // Will be 0 if newReq is nil. + InCapacity int + + reqPool bpool.Pool[Req] + respPool bpool.Pool[Resp] + id HandlerID + newPayload func() Payload + nilReq Req + nilResp Resp + sharedResponse bool +} + +// NewStream creates a typed handler that can provide Marshal/Unmarshal. +// Use Register to register a server handler. +// Use Call to initiate a clientside call. +// newPayload can be nil. In that case payloads will always be nil. +// newReq can be nil. In that case no input stream is expected and the handler will be called with nil 'in' channel. +func NewStream[Payload, Req, Resp RoundTripper](h HandlerID, newPayload func() Payload, newReq func() Req, newResp func() Resp) *StreamTypeHandler[Payload, Req, Resp] { + if newResp == nil { + panic("newResp missing in NewStream") + } + + s := newStreamHandler[Payload, Req, Resp](h) + if newReq != nil { + s.reqPool.New = func() Req { + return newReq() + } + } else { + s.InCapacity = 0 + } + s.respPool.New = func() Resp { + return newResp() + } + s.newPayload = newPayload + s.WithPayload = newPayload != nil + return s +} + +// WithSharedResponse indicates it is unsafe to reuse the response. +// Typically this is used when the response sharing part of its data structure. +func (h *StreamTypeHandler[Payload, Req, Resp]) WithSharedResponse() *StreamTypeHandler[Payload, Req, Resp] { + h.sharedResponse = true + return h +} + +// NewPayload creates a new payload. +func (h *StreamTypeHandler[Payload, Req, Resp]) NewPayload() Payload { + return h.newPayload() +} + +// NewRequest creates a new request. +// The struct may be reused, so caller should clear any fields. +func (h *StreamTypeHandler[Payload, Req, Resp]) NewRequest() Req { + return h.reqPool.Get() +} + +// PutRequest will accept a request for reuse. +// These should be returned by the handler. +func (h *StreamTypeHandler[Payload, Req, Resp]) PutRequest(r Req) { + if r != h.nilReq { + //nolint:staticcheck // SA6002 IT IS A GENERIC VALUE! (and always a pointer) + h.reqPool.Put(r) + } +} + +// PutResponse will accept a response for reuse. +// These should be returned by the caller. +func (h *StreamTypeHandler[Payload, Req, Resp]) PutResponse(r Resp) { + if r != h.nilResp { + //nolint:staticcheck // SA6002 IT IS A GENERIC VALUE! (and always a pointer) + h.respPool.Put(r) + } +} + +// NewResponse creates a new response. +// Handlers can use this to create a reusable response. +func (h *StreamTypeHandler[Payload, Req, Resp]) NewResponse() Resp { + return h.respPool.Get() +} + +func newStreamHandler[Payload, Req, Resp RoundTripper](h HandlerID) *StreamTypeHandler[Payload, Req, Resp] { + return &StreamTypeHandler[Payload, Req, Resp]{id: h, InCapacity: 1, OutCapacity: 1} +} + +// Register a handler for two-way streaming with payload, input stream and output stream. +// An optional subroute can be given. Multiple entries are joined with '/'. +func (h *StreamTypeHandler[Payload, Req, Resp]) Register(m *Manager, handle func(ctx context.Context, p Payload, in <-chan Req, out chan<- Resp) *RemoteErr, subroute ...string) error { + return h.register(m, handle, subroute...) +} + +// WithOutCapacity adjusts the output capacity from the handler perspective. +// This must be done prior to registering the handler. +func (h *StreamTypeHandler[Payload, Req, Resp]) WithOutCapacity(out int) *StreamTypeHandler[Payload, Req, Resp] { + h.OutCapacity = out + return h +} + +// WithInCapacity adjusts the input capacity from the handler perspective. +// This must be done prior to registering the handler. +func (h *StreamTypeHandler[Payload, Req, Resp]) WithInCapacity(in int) *StreamTypeHandler[Payload, Req, Resp] { + h.InCapacity = in + return h +} + +// RegisterNoInput a handler for one-way streaming with payload and output stream. +// An optional subroute can be given. Multiple entries are joined with '/'. +func (h *StreamTypeHandler[Payload, Req, Resp]) RegisterNoInput(m *Manager, handle func(ctx context.Context, p Payload, out chan<- Resp) *RemoteErr, subroute ...string) error { + h.InCapacity = 0 + return h.register(m, func(ctx context.Context, p Payload, in <-chan Req, out chan<- Resp) *RemoteErr { + return handle(ctx, p, out) + }, subroute...) +} + +// RegisterNoPayload a handler for one-way streaming with payload and output stream. +// An optional subroute can be given. Multiple entries are joined with '/'. +func (h *StreamTypeHandler[Payload, Req, Resp]) RegisterNoPayload(m *Manager, handle func(ctx context.Context, in <-chan Req, out chan<- Resp) *RemoteErr, subroute ...string) error { + h.WithPayload = false + return h.register(m, func(ctx context.Context, p Payload, in <-chan Req, out chan<- Resp) *RemoteErr { + return handle(ctx, in, out) + }, subroute...) +} + +// Register a handler for two-way streaming with optional payload and input stream. +func (h *StreamTypeHandler[Payload, Req, Resp]) register(m *Manager, handle func(ctx context.Context, p Payload, in <-chan Req, out chan<- Resp) *RemoteErr, subroute ...string) error { + return m.RegisterStreamingHandler(h.id, StreamHandler{ + Handle: func(ctx context.Context, payload []byte, in <-chan []byte, out chan<- []byte) *RemoteErr { + var plT Payload + if h.WithPayload { + plT = h.NewPayload() + _, err := plT.UnmarshalMsg(payload) + PutByteBuffer(payload) + if err != nil { + r := RemoteErr(err.Error()) + return &r + } + } + + var inT chan Req + if h.InCapacity > 0 { + // Don't add extra buffering + inT = make(chan Req) + go func() { + defer xioutil.SafeClose(inT) + for { + select { + case <-ctx.Done(): + return + case v, ok := <-in: + if !ok { + return + } + input := h.NewRequest() + _, err := input.UnmarshalMsg(v) + if err != nil { + gridLogOnceIf(ctx, err, err.Error()) + } + PutByteBuffer(v) + // Send input + select { + case <-ctx.Done(): + return + case inT <- input: + } + } + } + }() + } + outT := make(chan Resp) + outDone := make(chan struct{}) + go func() { + defer xioutil.SafeClose(outDone) + dropOutput := false + for v := range outT { + if dropOutput { + continue + } + dst, err := v.MarshalMsg(GetByteBufferCap(v.Msgsize())) + if err != nil { + gridLogOnceIf(ctx, err, err.Error()) + } + if !h.sharedResponse { + h.PutResponse(v) + } + select { + case <-ctx.Done(): + dropOutput = true + case out <- dst: + } + } + }() + rErr := handle(ctx, plT, inT, outT) + xioutil.SafeClose(outT) + <-outDone + return rErr + }, OutCapacity: h.OutCapacity, InCapacity: h.InCapacity, Subroute: strings.Join(subroute, "/"), + }) +} + +// TypedStream is a stream with specific types. +type TypedStream[Req, Resp RoundTripper] struct { + // responses from the remote server. + // Channel will be closed after error or when remote closes. + // responses *must* be read to either an error is returned or the channel is closed. + responses *Stream + newResp func() Resp + + // Requests sent to the server. + // If the handler is defined with 0 incoming capacity this will be nil. + // Channel *must* be closed to signal the end of the stream. + // If the request context is canceled, the stream will no longer process requests. + Requests chan<- Req +} + +// Results returns the results from the remote server one by one. +// If any error is returned by the callback, the stream will be canceled. +// If the context is canceled, the stream will be canceled. +func (s *TypedStream[Req, Resp]) Results(next func(resp Resp) error) (err error) { + return s.responses.Results(func(b []byte) error { + resp := s.newResp() + _, err := resp.UnmarshalMsg(b) + if err != nil { + return err + } + return next(resp) + }) +} + +// Streamer creates a stream. +type Streamer interface { + NewStream(ctx context.Context, h HandlerID, payload []byte) (st *Stream, err error) +} + +// Call the remove with the request and +func (h *StreamTypeHandler[Payload, Req, Resp]) Call(ctx context.Context, c Streamer, payload Payload) (st *TypedStream[Req, Resp], err error) { + if c == nil { + return nil, ErrDisconnected + } + var payloadB []byte + if h.WithPayload { + var err error + payloadB, err = payload.MarshalMsg(GetByteBufferCap(payload.Msgsize())) + if err != nil { + return nil, err + } + } + stream, err := c.NewStream(ctx, h.id, payloadB) + PutByteBuffer(payloadB) + if err != nil { + return nil, err + } + + // respT := make(chan TypedResponse[Resp]) + var reqT chan Req + if h.InCapacity > 0 { + reqT = make(chan Req) + // Request handler + if stream.Requests == nil { + return nil, fmt.Errorf("internal error: stream request channel nil") + } + go func() { + defer xioutil.SafeClose(stream.Requests) + for req := range reqT { + b, err := req.MarshalMsg(GetByteBufferCap(req.Msgsize())) + if err != nil { + gridLogOnceIf(ctx, err, err.Error()) + } + h.PutRequest(req) + stream.Requests <- b + } + }() + } else if stream.Requests != nil { + xioutil.SafeClose(stream.Requests) + } + + return &TypedStream[Req, Resp]{responses: stream, newResp: h.NewResponse, Requests: reqT}, nil +} diff --git a/internal/grid/handlers_string.go b/internal/grid/handlers_string.go new file mode 100644 index 00000000..454c90b8 --- /dev/null +++ b/internal/grid/handlers_string.go @@ -0,0 +1,103 @@ +// Code generated by "stringer -type=HandlerID -output=handlers_string.go -trimprefix=Handler msg.go handlers.go"; DO NOT EDIT. + +package grid + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[handlerInvalid-0] + _ = x[HandlerLockLock-1] + _ = x[HandlerLockRLock-2] + _ = x[HandlerLockUnlock-3] + _ = x[HandlerLockRUnlock-4] + _ = x[HandlerLockRefresh-5] + _ = x[HandlerLockForceUnlock-6] + _ = x[HandlerWalkDir-7] + _ = x[HandlerStatVol-8] + _ = x[HandlerDiskInfo-9] + _ = x[HandlerNSScanner-10] + _ = x[HandlerReadXL-11] + _ = x[HandlerReadVersion-12] + _ = x[HandlerDeleteFile-13] + _ = x[HandlerDeleteVersion-14] + _ = x[HandlerUpdateMetadata-15] + _ = x[HandlerWriteMetadata-16] + _ = x[HandlerCheckParts-17] + _ = x[HandlerRenameData-18] + _ = x[HandlerRenameFile-19] + _ = x[HandlerReadAll-20] + _ = x[HandlerServerVerify-21] + _ = x[HandlerTrace-22] + _ = x[HandlerListen-23] + _ = x[HandlerDeleteBucketMetadata-24] + _ = x[HandlerLoadBucketMetadata-25] + _ = x[HandlerReloadSiteReplicationConfig-26] + _ = x[HandlerReloadPoolMeta-27] + _ = x[HandlerStopRebalance-28] + _ = x[HandlerLoadRebalanceMeta-29] + _ = x[HandlerLoadTransitionTierConfig-30] + _ = x[HandlerDeletePolicy-31] + _ = x[HandlerLoadPolicy-32] + _ = x[HandlerLoadPolicyMapping-33] + _ = x[HandlerDeleteServiceAccount-34] + _ = x[HandlerLoadServiceAccount-35] + _ = x[HandlerDeleteUser-36] + _ = x[HandlerLoadUser-37] + _ = x[HandlerLoadGroup-38] + _ = x[HandlerHealBucket-39] + _ = x[HandlerMakeBucket-40] + _ = x[HandlerHeadBucket-41] + _ = x[HandlerDeleteBucket-42] + _ = x[HandlerGetMetrics-43] + _ = x[HandlerGetResourceMetrics-44] + _ = x[HandlerGetMemInfo-45] + _ = x[HandlerGetProcInfo-46] + _ = x[HandlerGetOSInfo-47] + _ = x[HandlerGetPartitions-48] + _ = x[HandlerGetNetInfo-49] + _ = x[HandlerGetCPUs-50] + _ = x[HandlerServerInfo-51] + _ = x[HandlerGetSysConfig-52] + _ = x[HandlerGetSysServices-53] + _ = x[HandlerGetSysErrors-54] + _ = x[HandlerGetAllBucketStats-55] + _ = x[HandlerGetBucketStats-56] + _ = x[HandlerGetSRMetrics-57] + _ = x[HandlerGetPeerMetrics-58] + _ = x[HandlerGetMetacacheListing-59] + _ = x[HandlerUpdateMetacacheListing-60] + _ = x[HandlerGetPeerBucketMetrics-61] + _ = x[HandlerStorageInfo-62] + _ = x[HandlerConsoleLog-63] + _ = x[HandlerListDir-64] + _ = x[HandlerGetLocks-65] + _ = x[HandlerBackgroundHealStatus-66] + _ = x[HandlerGetLastDayTierStats-67] + _ = x[HandlerSignalService-68] + _ = x[HandlerGetBandwidth-69] + _ = x[HandlerWriteAll-70] + _ = x[HandlerListBuckets-71] + _ = x[HandlerRenameDataInline-72] + _ = x[HandlerRenameData2-73] + _ = x[HandlerCheckParts2-74] + _ = x[HandlerRenamePart-75] + _ = x[HandlerClearUploadID-76] + _ = x[HandlerCheckParts3-77] + _ = x[handlerTest-78] + _ = x[handlerTest2-79] + _ = x[handlerLast-80] +} + +const _HandlerID_name = "handlerInvalidLockLockLockRLockLockUnlockLockRUnlockLockRefreshLockForceUnlockWalkDirStatVolDiskInfoNSScannerReadXLReadVersionDeleteFileDeleteVersionUpdateMetadataWriteMetadataCheckPartsRenameDataRenameFileReadAllServerVerifyTraceListenDeleteBucketMetadataLoadBucketMetadataReloadSiteReplicationConfigReloadPoolMetaStopRebalanceLoadRebalanceMetaLoadTransitionTierConfigDeletePolicyLoadPolicyLoadPolicyMappingDeleteServiceAccountLoadServiceAccountDeleteUserLoadUserLoadGroupHealBucketMakeBucketHeadBucketDeleteBucketGetMetricsGetResourceMetricsGetMemInfoGetProcInfoGetOSInfoGetPartitionsGetNetInfoGetCPUsServerInfoGetSysConfigGetSysServicesGetSysErrorsGetAllBucketStatsGetBucketStatsGetSRMetricsGetPeerMetricsGetMetacacheListingUpdateMetacacheListingGetPeerBucketMetricsStorageInfoConsoleLogListDirGetLocksBackgroundHealStatusGetLastDayTierStatsSignalServiceGetBandwidthWriteAllListBucketsRenameDataInlineRenameData2CheckParts2RenamePartClearUploadIDCheckParts3handlerTesthandlerTest2handlerLast" + +var _HandlerID_index = [...]uint16{0, 14, 22, 31, 41, 52, 63, 78, 85, 92, 100, 109, 115, 126, 136, 149, 163, 176, 186, 196, 206, 213, 225, 230, 236, 256, 274, 301, 315, 328, 345, 369, 381, 391, 408, 428, 446, 456, 464, 473, 483, 493, 503, 515, 525, 543, 553, 564, 573, 586, 596, 603, 613, 625, 639, 651, 668, 682, 694, 708, 727, 749, 769, 780, 790, 797, 805, 825, 844, 857, 869, 877, 888, 904, 915, 926, 936, 949, 960, 971, 983, 994} + +func (i HandlerID) String() string { + if i >= HandlerID(len(_HandlerID_index)-1) { + return "HandlerID(" + strconv.FormatInt(int64(i), 10) + ")" + } + return _HandlerID_name[_HandlerID_index[i]:_HandlerID_index[i+1]] +} diff --git a/internal/grid/manager.go b/internal/grid/manager.go new file mode 100644 index 00000000..3e4829e3 --- /dev/null +++ b/internal/grid/manager.go @@ -0,0 +1,385 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package grid + +import ( + "context" + "errors" + "fmt" + "io" + "net" + "net/http" + "runtime/debug" + "strings" + "time" + + "github.com/gobwas/ws" + "github.com/gobwas/ws/wsutil" + "github.com/google/uuid" + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/deadlineconn" + "github.com/minio/minio/internal/pubsub" + "github.com/minio/mux" +) + +const ( + // apiVersion is a major version of the entire api. + // Bumping this should only be done when overall, + // incompatible changes are made, not when adding a new handler + // or changing an existing handler. + apiVersion = "v1" + + // RoutePath is the remote path to connect to. + RoutePath = "/minio/grid/" + apiVersion + + // RouteLockPath is the remote lock path to connect to. + RouteLockPath = "/minio/grid/lock/" + apiVersion +) + +// Manager will contain all the connections to the grid. +// It also handles incoming requests and routes them to the appropriate connection. +type Manager struct { + // ID is an instance ID, that will change whenever the server restarts. + // This allows remotes to keep track of whether state is preserved. + ID uuid.UUID + + // Immutable after creation, so no locks. + targets map[string]*Connection + + // serverside handlers. + handlers handlers + + // local host name. + local string + + // authToken is a function that will validate a token. + authToken ValidateTokenFn + + // routePath indicates the dial route path + routePath string +} + +// ManagerOptions are options for creating a new grid manager. +type ManagerOptions struct { + Local string // Local host name. + Hosts []string // All hosts, including local in the grid. + Incoming func(n int64) // Record incoming bytes. + Outgoing func(n int64) // Record outgoing bytes. + BlockConnect chan struct{} // If set, incoming and outgoing connections will be blocked until closed. + RoutePath string + TraceTo *pubsub.PubSub[madmin.TraceInfo, madmin.TraceType] + Dialer ConnDialer + // Sign a token for the given audience. + AuthFn AuthFn + // Callbacks to validate incoming connections. + AuthToken ValidateTokenFn +} + +// NewManager creates a new grid manager +func NewManager(ctx context.Context, o ManagerOptions) (*Manager, error) { + found := false + if o.AuthToken == nil { + return nil, fmt.Errorf("grid: AuthToken not set") + } + if o.Dialer == nil { + return nil, fmt.Errorf("grid: Dialer not set") + } + if o.AuthFn == nil { + return nil, fmt.Errorf("grid: AuthFn not set") + } + m := &Manager{ + ID: uuid.New(), + targets: make(map[string]*Connection, len(o.Hosts)), + local: o.Local, + authToken: o.AuthToken, + routePath: o.RoutePath, + } + m.handlers.init() + if ctx == nil { + ctx = context.Background() + } + + for _, host := range o.Hosts { + if host == o.Local { + if found { + return nil, fmt.Errorf("grid: local host found multiple times") + } + found = true + // No connection to local. + continue + } + m.targets[host] = newConnection(connectionParams{ + ctx: ctx, + id: m.ID, + local: o.Local, + remote: host, + handlers: &m.handlers, + blockConnect: o.BlockConnect, + publisher: o.TraceTo, + incomingBytes: o.Incoming, + outgoingBytes: o.Outgoing, + dialer: o.Dialer, + authFn: o.AuthFn, + }) + } + if !found { + return nil, fmt.Errorf("grid: local host (%s) not found in cluster setup", o.Local) + } + + return m, nil +} + +// AddToMux will add the grid manager to the given mux. +func (m *Manager) AddToMux(router *mux.Router, authReq func(r *http.Request) error) { + router.Handle(m.routePath, m.Handler(authReq)) +} + +// Handler returns a handler that can be used to serve grid requests. +// This should be connected on RoutePath to the main server. +func (m *Manager) Handler(authReq func(r *http.Request) error) http.HandlerFunc { + return func(w http.ResponseWriter, req *http.Request) { + defer func() { + if debugPrint { + fmt.Printf("grid: Handler returning from: %v %v\n", req.Method, req.URL) + } + if r := recover(); r != nil { + debug.PrintStack() + err := fmt.Errorf("grid: panic: %v\n", r) + gridLogIf(context.Background(), err, err.Error()) + w.WriteHeader(http.StatusInternalServerError) + } + }() + if debugPrint { + fmt.Printf("grid: Got a %s request for: %v\n", req.Method, req.URL) + } + ctx := req.Context() + if err := authReq(req); err != nil { + gridLogOnceIf(ctx, fmt.Errorf("auth %s: %w", req.RemoteAddr, err), req.RemoteAddr) + w.WriteHeader(http.StatusForbidden) + return + } + conn, _, _, err := ws.UpgradeHTTP(req, w) + if err != nil { + if debugPrint { + fmt.Printf("grid: Unable to upgrade: %v. http.ResponseWriter is type %T\n", err, w) + } + w.WriteHeader(http.StatusUpgradeRequired) + return + } + m.IncomingConn(ctx, conn) + } +} + +// IncomingConn will handle an incoming connection. +// This should be called with the incoming connection after accept. +// Auth is handled internally, as well as disconnecting any connections from the same host. +func (m *Manager) IncomingConn(ctx context.Context, conn net.Conn) { + // We manage our own deadlines. + conn = deadlineconn.Unwrap(conn) + remoteAddr := conn.RemoteAddr().String() + // will write an OpConnectResponse message to the remote and log it once locally. + defer conn.Close() + writeErr := func(err error) { + if err == nil { + return + } + if errors.Is(err, io.EOF) { + return + } + gridLogOnceIf(ctx, err, remoteAddr) + resp := connectResp{ + ID: m.ID, + Accepted: false, + RejectedReason: err.Error(), + } + if b, err := resp.MarshalMsg(nil); err == nil { + msg := message{ + Op: OpConnectResponse, + Payload: b, + } + if b, err := msg.MarshalMsg(nil); err == nil { + wsutil.WriteMessage(conn, ws.StateServerSide, ws.OpBinary, b) + } + } + } + defer conn.Close() + if debugPrint { + fmt.Printf("grid: Upgraded request: %v\n", remoteAddr) + } + + msg, _, err := wsutil.ReadClientData(conn) + if err != nil { + writeErr(fmt.Errorf("reading connect: %w", err)) + return + } + if debugPrint { + fmt.Printf("%s handler: Got message, length %v\n", m.local, len(msg)) + } + + var message message + _, _, err = message.parse(msg) + if err != nil { + writeErr(fmt.Errorf("error parsing grid connect: %w", err)) + return + } + if message.Op != OpConnect { + writeErr(fmt.Errorf("unexpected connect op: %v", message.Op)) + return + } + var cReq connectReq + _, err = cReq.UnmarshalMsg(message.Payload) + if err != nil { + writeErr(fmt.Errorf("error parsing connectReq: %w", err)) + return + } + remote := m.targets[cReq.Host] + if remote == nil { + writeErr(fmt.Errorf("unknown incoming host: %v", cReq.Host)) + return + } + if time.Since(cReq.Time).Abs() > 5*time.Minute { + writeErr(fmt.Errorf("time difference too large between servers: %v", time.Since(cReq.Time).Abs())) + return + } + if err := m.authToken(cReq.Token); err != nil { + writeErr(fmt.Errorf("auth token: %w", err)) + return + } + + if debugPrint { + fmt.Printf("handler: Got Connect Req %+v\n", cReq) + } + writeErr(remote.handleIncoming(ctx, conn, cReq)) +} + +// AuthFn should provide an authentication string for the given aud. +type AuthFn func() string + +// ValidateAuthFn should check authentication for the given aud. +type ValidateAuthFn func(auth string) string + +// Connection will return the connection for the specified host. +// If the host does not exist nil will be returned. +func (m *Manager) Connection(host string) *Connection { + return m.targets[host] +} + +// RegisterSingleHandler will register a stateless handler that serves +// []byte -> ([]byte, error) requests. +// subroutes are joined with "/" to a single subroute. +func (m *Manager) RegisterSingleHandler(id HandlerID, h SingleHandlerFn, subroute ...string) error { + if !id.valid() { + return ErrUnknownHandler + } + s := strings.Join(subroute, "/") + if debugPrint { + fmt.Println("RegisterSingleHandler: ", id.String(), "subroute:", s) + } + + if len(subroute) == 0 { + if m.handlers.hasAny(id) && !id.isTestHandler() { + return fmt.Errorf("handler %v: %w", id.String(), ErrHandlerAlreadyExists) + } + + m.handlers.single[id] = h + return nil + } + subID := makeSubHandlerID(id, s) + if m.handlers.hasSubhandler(subID) && !id.isTestHandler() { + return fmt.Errorf("handler %v, subroute:%v: %w", id.String(), s, ErrHandlerAlreadyExists) + } + m.handlers.subSingle[subID] = h + // Copy so clients can also pick it up for other subpaths. + m.handlers.subSingle[makeZeroSubHandlerID(id)] = h + return nil +} + +/* +// RegisterStateless will register a stateless handler that serves +// []byte -> stream of ([]byte, error) requests. +func (m *Manager) RegisterStateless(id HandlerID, h StatelessHandler) error { + if !id.valid() { + return ErrUnknownHandler + } + if m.handlers.hasAny(id) && !id.isTestHandler() { + return ErrHandlerAlreadyExists + } + + m.handlers.stateless[id] = &h + return nil +} +*/ + +// RegisterStreamingHandler will register a stateless handler that serves +// two-way streaming requests. +func (m *Manager) RegisterStreamingHandler(id HandlerID, h StreamHandler) error { + if !id.valid() { + return ErrUnknownHandler + } + if debugPrint { + fmt.Println("RegisterStreamingHandler: subroute:", h.Subroute) + } + if h.Subroute == "" { + if m.handlers.hasAny(id) && !id.isTestHandler() { + return ErrHandlerAlreadyExists + } + m.handlers.streams[id] = &h + return nil + } + subID := makeSubHandlerID(id, h.Subroute) + if m.handlers.hasSubhandler(subID) && !id.isTestHandler() { + return ErrHandlerAlreadyExists + } + m.handlers.subStreams[subID] = &h + // Copy so clients can also pick it up for other subpaths. + m.handlers.subStreams[makeZeroSubHandlerID(id)] = &h + return nil +} + +// HostName returns the name of the local host. +func (m *Manager) HostName() string { + return m.local +} + +// Targets returns the names of all remote targets. +func (m *Manager) Targets() []string { + var res []string + for k := range m.targets { + res = append(res, k) + } + return res +} + +// debugMsg should *only* be used by tests. +// +//lint:ignore U1000 This is used by tests. +func (m *Manager) debugMsg(d debugMsg, args ...any) { + for _, c := range m.targets { + c.debugMsg(d, args...) + } +} + +// ConnStats returns the connection statistics for all connections. +func (m *Manager) ConnStats() madmin.RPCMetrics { + var res madmin.RPCMetrics + for _, c := range m.targets { + t := c.Stats() + res.Merge(&t) + } + return res +} diff --git a/internal/grid/msg.go b/internal/grid/msg.go new file mode 100644 index 00000000..b72520f0 --- /dev/null +++ b/internal/grid/msg.go @@ -0,0 +1,308 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package grid + +import ( + "encoding/binary" + "fmt" + "strings" + "time" + + "github.com/tinylib/msgp/msgp" + "github.com/zeebo/xxh3" +) + +// Op is operation type. +// +//go:generate msgp -unexported -file=$GOFILE +//go:generate stringer -type=Op -output=msg_string.go -trimprefix=Op $GOFILE + +// Op is operation type messages. +type Op uint8 + +// HandlerID is the ID for the handler of a specific type. +type HandlerID uint8 + +const ( + // OpConnect is a connect request. + OpConnect Op = iota + 1 + + // OpConnectResponse is a response to a connect request. + OpConnectResponse + + // OpPing is a ping request. + // If a mux id is specified that mux is pinged. + // Clients send ping requests. + OpPing + + // OpPong is a OpPing response returned by the server. + OpPong + + // OpConnectMux will connect a new mux with optional payload. + OpConnectMux + + // OpMuxConnectError is an error while connecting a mux. + OpMuxConnectError + + // OpDisconnectClientMux instructs a client to disconnect a mux + OpDisconnectClientMux + + // OpDisconnectServerMux instructs a server to disconnect (cancel) a server mux + OpDisconnectServerMux + + // OpMuxClientMsg contains a message to a client Mux + OpMuxClientMsg + + // OpMuxServerMsg contains a message to a server Mux + OpMuxServerMsg + + // OpUnblockSrvMux contains a message that a server mux is unblocked with one. + // Only Stateful streams has flow control. + OpUnblockSrvMux + + // OpUnblockClMux contains a message that a client mux is unblocked with one. + // Only Stateful streams has flow control. + OpUnblockClMux + + // OpAckMux acknowledges a mux was created. + OpAckMux + + // OpRequest is a single request + response. + // MuxID is returned in response. + OpRequest + + // OpResponse is a response to a single request. + // FlagPayloadIsErr is used to signify that the payload is a string error converted to byte slice. + // When a response is received, the mux is already removed from the remote. + OpResponse + + // OpDisconnect instructs that remote wants to disconnect + OpDisconnect + + // OpMerged is several operations merged into one. + OpMerged +) + +const ( + // FlagCRCxxh3 indicates that, the lower 32 bits of xxhash3 of the serialized + // message will be sent after the serialized message as little endian. + FlagCRCxxh3 Flags = 1 << iota + + // FlagEOF the stream (either direction) is at EOF. + FlagEOF + + // FlagStateless indicates the message is stateless. + // This will retain clients across reconnections or + // if sequence numbers are unexpected. + FlagStateless + + // FlagPayloadIsErr can be used by individual ops to signify that + // The payload is a string error converted to byte slice. + FlagPayloadIsErr + + // FlagPayloadIsZero means that payload is 0-length slice and not nil. + FlagPayloadIsZero + + // FlagSubroute indicates that the message has subroute. + // Subroute will be 32 bytes long and added before any CRC. + FlagSubroute +) + +// This struct cannot be changed and retain backwards compatibility. +// If changed, endpoint version must be bumped. +// +//msgp:tuple message +type message struct { + MuxID uint64 // Mux to receive message if any. + Seq uint32 // Sequence number. + DeadlineMS uint32 // If non-zero, milliseconds until deadline (max 1193h2m47.295s, ~49 days) + Handler HandlerID // ID of handler if invoking a remote handler. + Op Op // Operation. Other fields change based on this value. + Flags Flags // Optional flags. + Payload []byte // Optional payload. +} + +// Flags is a set of flags set on a message. +type Flags uint8 + +func (m message) String() string { + var res []string + if m.MuxID != 0 { + res = append(res, fmt.Sprintf("MuxID: %v", m.MuxID)) + } + if m.Seq != 0 { + res = append(res, fmt.Sprintf("Seq: %v", m.Seq)) + } + if m.DeadlineMS != 0 { + res = append(res, fmt.Sprintf("Deadline: %vms", m.DeadlineMS)) + } + if m.Handler != handlerInvalid { + res = append(res, fmt.Sprintf("Handler: %v", m.Handler)) + } + if m.Op != 0 { + res = append(res, fmt.Sprintf("Op: %v", m.Op)) + } + res = append(res, fmt.Sprintf("Flags: %s", m.Flags.String())) + if len(m.Payload) != 0 { + res = append(res, fmt.Sprintf("Payload: %v", bytesOrLength(m.Payload))) + } + return "{" + strings.Join(res, ", ") + "}" +} + +func (f Flags) String() string { + var res []string + if f&FlagCRCxxh3 != 0 { + res = append(res, "CRC") + } + if f&FlagEOF != 0 { + res = append(res, "EOF") + } + if f&FlagStateless != 0 { + res = append(res, "SL") + } + if f&FlagPayloadIsErr != 0 { + res = append(res, "ERR") + } + if f&FlagPayloadIsZero != 0 { + res = append(res, "ZERO") + } + if f&FlagSubroute != 0 { + res = append(res, "SUB") + } + return "[" + strings.Join(res, ",") + "]" +} + +// Set one or more flags on f. +func (f *Flags) Set(flags Flags) { + *f |= flags +} + +// Clear one or more flags on f. +func (f *Flags) Clear(flags Flags) { + *f &^= flags +} + +// parse an incoming message. +func (m *message) parse(b []byte) (*subHandlerID, []byte, error) { + var sub *subHandlerID + if m.Payload == nil { + m.Payload = GetByteBuffer()[:0] + } + h, err := m.UnmarshalMsg(b) + if err != nil { + return nil, nil, fmt.Errorf("read write: %v", err) + } + if len(m.Payload) == 0 && m.Flags&FlagPayloadIsZero == 0 { + PutByteBuffer(m.Payload) + m.Payload = nil + } + if m.Flags&FlagCRCxxh3 != 0 { + const hashLen = 4 + if len(h) < hashLen { + return nil, nil, fmt.Errorf("want crc len 4, got %v", len(h)) + } + got := uint32(xxh3.Hash(b[:len(b)-hashLen])) + want := binary.LittleEndian.Uint32(h[len(h)-hashLen:]) + if got != want { + return nil, nil, fmt.Errorf("crc mismatch: 0x%08x (given) != 0x%08x (bytes)", want, got) + } + h = h[:len(h)-hashLen] + } + // Extract subroute if any. + if m.Flags&FlagSubroute != 0 { + if len(h) < 32 { + return nil, nil, fmt.Errorf("want subroute len 32, got %v", len(h)) + } + subID := (*[32]byte)(h[len(h)-32:]) + sub = (*subHandlerID)(subID) + // Add if more modifications to h is needed + h = h[:len(h)-32] + } + return sub, h, nil +} + +// setZeroPayloadFlag will clear or set the FlagPayloadIsZero if +// m.Payload is length 0, but not nil. +func (m *message) setZeroPayloadFlag() { + m.Flags &^= FlagPayloadIsZero + if len(m.Payload) == 0 && m.Payload != nil { + m.Flags |= FlagPayloadIsZero + } +} + +type receiver interface { + msgp.Unmarshaler + Op() Op +} + +type sender interface { + msgp.MarshalSizer + Op() Op +} + +type connectReq struct { + ID [16]byte + Host string + Time time.Time + Token string +} + +// addToken will add the token to the connect request. +func (c *connectReq) addToken(fn AuthFn) { + c.Token = fn() +} + +func (connectReq) Op() Op { + return OpConnect +} + +type connectResp struct { + ID [16]byte + Accepted bool + RejectedReason string +} + +func (connectResp) Op() Op { + return OpConnectResponse +} + +type muxConnectError struct { + Error string +} + +func (muxConnectError) Op() Op { + return OpMuxConnectError +} + +type pongMsg struct { + NotFound bool `msg:"nf"` + Err *string `msg:"e,allownil"` + T time.Time `msg:"t"` +} + +func (pongMsg) Op() Op { + return OpPong +} + +type pingMsg struct { + T time.Time `msg:"t"` +} + +func (pingMsg) Op() Op { + return OpPing +} diff --git a/internal/grid/msg_gen.go b/internal/grid/msg_gen.go new file mode 100644 index 00000000..aff7a2bf --- /dev/null +++ b/internal/grid/msg_gen.go @@ -0,0 +1,1084 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package grid + +import ( + "github.com/tinylib/msgp/msgp" +) + +// DecodeMsg implements msgp.Decodable +func (z *Flags) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 uint8 + zb0001, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = Flags(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z Flags) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteUint8(uint8(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z Flags) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendUint8(o, uint8(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *Flags) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 uint8 + zb0001, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = Flags(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z Flags) Msgsize() (s int) { + s = msgp.Uint8Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *HandlerID) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 uint8 + zb0001, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = HandlerID(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z HandlerID) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteUint8(uint8(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z HandlerID) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendUint8(o, uint8(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *HandlerID) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 uint8 + zb0001, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = HandlerID(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z HandlerID) Msgsize() (s int) { + s = msgp.Uint8Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *Op) DecodeMsg(dc *msgp.Reader) (err error) { + { + var zb0001 uint8 + zb0001, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = Op(zb0001) + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z Op) EncodeMsg(en *msgp.Writer) (err error) { + err = en.WriteUint8(uint8(z)) + if err != nil { + err = msgp.WrapError(err) + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z Op) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + o = msgp.AppendUint8(o, uint8(z)) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *Op) UnmarshalMsg(bts []byte) (o []byte, err error) { + { + var zb0001 uint8 + zb0001, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + (*z) = Op(zb0001) + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z Op) Msgsize() (s int) { + s = msgp.Uint8Size + return +} + +// DecodeMsg implements msgp.Decodable +func (z *connectReq) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ID": + err = dc.ReadExactBytes((z.ID)[:]) + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + case "Host": + z.Host, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Host") + return + } + case "Time": + z.Time, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "Time") + return + } + case "Token": + z.Token, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Token") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *connectReq) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 4 + // write "ID" + err = en.Append(0x84, 0xa2, 0x49, 0x44) + if err != nil { + return + } + err = en.WriteBytes((z.ID)[:]) + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + // write "Host" + err = en.Append(0xa4, 0x48, 0x6f, 0x73, 0x74) + if err != nil { + return + } + err = en.WriteString(z.Host) + if err != nil { + err = msgp.WrapError(err, "Host") + return + } + // write "Time" + err = en.Append(0xa4, 0x54, 0x69, 0x6d, 0x65) + if err != nil { + return + } + err = en.WriteTime(z.Time) + if err != nil { + err = msgp.WrapError(err, "Time") + return + } + // write "Token" + err = en.Append(0xa5, 0x54, 0x6f, 0x6b, 0x65, 0x6e) + if err != nil { + return + } + err = en.WriteString(z.Token) + if err != nil { + err = msgp.WrapError(err, "Token") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *connectReq) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 4 + // string "ID" + o = append(o, 0x84, 0xa2, 0x49, 0x44) + o = msgp.AppendBytes(o, (z.ID)[:]) + // string "Host" + o = append(o, 0xa4, 0x48, 0x6f, 0x73, 0x74) + o = msgp.AppendString(o, z.Host) + // string "Time" + o = append(o, 0xa4, 0x54, 0x69, 0x6d, 0x65) + o = msgp.AppendTime(o, z.Time) + // string "Token" + o = append(o, 0xa5, 0x54, 0x6f, 0x6b, 0x65, 0x6e) + o = msgp.AppendString(o, z.Token) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *connectReq) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ID": + bts, err = msgp.ReadExactBytes(bts, (z.ID)[:]) + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + case "Host": + z.Host, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Host") + return + } + case "Time": + z.Time, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Time") + return + } + case "Token": + z.Token, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Token") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *connectReq) Msgsize() (s int) { + s = 1 + 3 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + 5 + msgp.StringPrefixSize + len(z.Host) + 5 + msgp.TimeSize + 6 + msgp.StringPrefixSize + len(z.Token) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *connectResp) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ID": + err = dc.ReadExactBytes((z.ID)[:]) + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + case "Accepted": + z.Accepted, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "Accepted") + return + } + case "RejectedReason": + z.RejectedReason, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "RejectedReason") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *connectResp) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "ID" + err = en.Append(0x83, 0xa2, 0x49, 0x44) + if err != nil { + return + } + err = en.WriteBytes((z.ID)[:]) + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + // write "Accepted" + err = en.Append(0xa8, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64) + if err != nil { + return + } + err = en.WriteBool(z.Accepted) + if err != nil { + err = msgp.WrapError(err, "Accepted") + return + } + // write "RejectedReason" + err = en.Append(0xae, 0x52, 0x65, 0x6a, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e) + if err != nil { + return + } + err = en.WriteString(z.RejectedReason) + if err != nil { + err = msgp.WrapError(err, "RejectedReason") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *connectResp) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "ID" + o = append(o, 0x83, 0xa2, 0x49, 0x44) + o = msgp.AppendBytes(o, (z.ID)[:]) + // string "Accepted" + o = append(o, 0xa8, 0x41, 0x63, 0x63, 0x65, 0x70, 0x74, 0x65, 0x64) + o = msgp.AppendBool(o, z.Accepted) + // string "RejectedReason" + o = append(o, 0xae, 0x52, 0x65, 0x6a, 0x65, 0x63, 0x74, 0x65, 0x64, 0x52, 0x65, 0x61, 0x73, 0x6f, 0x6e) + o = msgp.AppendString(o, z.RejectedReason) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *connectResp) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "ID": + bts, err = msgp.ReadExactBytes(bts, (z.ID)[:]) + if err != nil { + err = msgp.WrapError(err, "ID") + return + } + case "Accepted": + z.Accepted, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Accepted") + return + } + case "RejectedReason": + z.RejectedReason, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "RejectedReason") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *connectResp) Msgsize() (s int) { + s = 1 + 3 + msgp.ArrayHeaderSize + (16 * (msgp.ByteSize)) + 9 + msgp.BoolSize + 15 + msgp.StringPrefixSize + len(z.RejectedReason) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *message) DecodeMsg(dc *msgp.Reader) (err error) { + var zb0001 uint32 + zb0001, err = dc.ReadArrayHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 7 { + err = msgp.ArrayError{Wanted: 7, Got: zb0001} + return + } + z.MuxID, err = dc.ReadUint64() + if err != nil { + err = msgp.WrapError(err, "MuxID") + return + } + z.Seq, err = dc.ReadUint32() + if err != nil { + err = msgp.WrapError(err, "Seq") + return + } + z.DeadlineMS, err = dc.ReadUint32() + if err != nil { + err = msgp.WrapError(err, "DeadlineMS") + return + } + { + var zb0002 uint8 + zb0002, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err, "Handler") + return + } + z.Handler = HandlerID(zb0002) + } + { + var zb0003 uint8 + zb0003, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err, "Op") + return + } + z.Op = Op(zb0003) + } + { + var zb0004 uint8 + zb0004, err = dc.ReadUint8() + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + z.Flags = Flags(zb0004) + } + z.Payload, err = dc.ReadBytes(z.Payload) + if err != nil { + err = msgp.WrapError(err, "Payload") + return + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *message) EncodeMsg(en *msgp.Writer) (err error) { + // array header, size 7 + err = en.Append(0x97) + if err != nil { + return + } + err = en.WriteUint64(z.MuxID) + if err != nil { + err = msgp.WrapError(err, "MuxID") + return + } + err = en.WriteUint32(z.Seq) + if err != nil { + err = msgp.WrapError(err, "Seq") + return + } + err = en.WriteUint32(z.DeadlineMS) + if err != nil { + err = msgp.WrapError(err, "DeadlineMS") + return + } + err = en.WriteUint8(uint8(z.Handler)) + if err != nil { + err = msgp.WrapError(err, "Handler") + return + } + err = en.WriteUint8(uint8(z.Op)) + if err != nil { + err = msgp.WrapError(err, "Op") + return + } + err = en.WriteUint8(uint8(z.Flags)) + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + err = en.WriteBytes(z.Payload) + if err != nil { + err = msgp.WrapError(err, "Payload") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *message) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // array header, size 7 + o = append(o, 0x97) + o = msgp.AppendUint64(o, z.MuxID) + o = msgp.AppendUint32(o, z.Seq) + o = msgp.AppendUint32(o, z.DeadlineMS) + o = msgp.AppendUint8(o, uint8(z.Handler)) + o = msgp.AppendUint8(o, uint8(z.Op)) + o = msgp.AppendUint8(o, uint8(z.Flags)) + o = msgp.AppendBytes(o, z.Payload) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *message) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0001 uint32 + zb0001, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + if zb0001 != 7 { + err = msgp.ArrayError{Wanted: 7, Got: zb0001} + return + } + z.MuxID, bts, err = msgp.ReadUint64Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "MuxID") + return + } + z.Seq, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Seq") + return + } + z.DeadlineMS, bts, err = msgp.ReadUint32Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "DeadlineMS") + return + } + { + var zb0002 uint8 + zb0002, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Handler") + return + } + z.Handler = HandlerID(zb0002) + } + { + var zb0003 uint8 + zb0003, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Op") + return + } + z.Op = Op(zb0003) + } + { + var zb0004 uint8 + zb0004, bts, err = msgp.ReadUint8Bytes(bts) + if err != nil { + err = msgp.WrapError(err, "Flags") + return + } + z.Flags = Flags(zb0004) + } + z.Payload, bts, err = msgp.ReadBytesBytes(bts, z.Payload) + if err != nil { + err = msgp.WrapError(err, "Payload") + return + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *message) Msgsize() (s int) { + s = 1 + msgp.Uint64Size + msgp.Uint32Size + msgp.Uint32Size + msgp.Uint8Size + msgp.Uint8Size + msgp.Uint8Size + msgp.BytesPrefixSize + len(z.Payload) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *muxConnectError) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Error": + z.Error, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Error") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z muxConnectError) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "Error" + err = en.Append(0x81, 0xa5, 0x45, 0x72, 0x72, 0x6f, 0x72) + if err != nil { + return + } + err = en.WriteString(z.Error) + if err != nil { + err = msgp.WrapError(err, "Error") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z muxConnectError) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "Error" + o = append(o, 0x81, 0xa5, 0x45, 0x72, 0x72, 0x6f, 0x72) + o = msgp.AppendString(o, z.Error) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *muxConnectError) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "Error": + z.Error, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Error") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z muxConnectError) Msgsize() (s int) { + s = 1 + 6 + msgp.StringPrefixSize + len(z.Error) + return +} + +// DecodeMsg implements msgp.Decodable +func (z *pingMsg) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "t": + z.T, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "T") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z pingMsg) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 1 + // write "t" + err = en.Append(0x81, 0xa1, 0x74) + if err != nil { + return + } + err = en.WriteTime(z.T) + if err != nil { + err = msgp.WrapError(err, "T") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z pingMsg) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 1 + // string "t" + o = append(o, 0x81, 0xa1, 0x74) + o = msgp.AppendTime(o, z.T) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *pingMsg) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "t": + z.T, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "T") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z pingMsg) Msgsize() (s int) { + s = 1 + 2 + msgp.TimeSize + return +} + +// DecodeMsg implements msgp.Decodable +func (z *pongMsg) DecodeMsg(dc *msgp.Reader) (err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, err = dc.ReadMapHeader() + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, err = dc.ReadMapKeyPtr() + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "nf": + z.NotFound, err = dc.ReadBool() + if err != nil { + err = msgp.WrapError(err, "NotFound") + return + } + case "e": + if dc.IsNil() { + err = dc.ReadNil() + if err != nil { + err = msgp.WrapError(err, "Err") + return + } + z.Err = nil + } else { + if z.Err == nil { + z.Err = new(string) + } + *z.Err, err = dc.ReadString() + if err != nil { + err = msgp.WrapError(err, "Err") + return + } + } + case "t": + z.T, err = dc.ReadTime() + if err != nil { + err = msgp.WrapError(err, "T") + return + } + default: + err = dc.Skip() + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + return +} + +// EncodeMsg implements msgp.Encodable +func (z *pongMsg) EncodeMsg(en *msgp.Writer) (err error) { + // map header, size 3 + // write "nf" + err = en.Append(0x83, 0xa2, 0x6e, 0x66) + if err != nil { + return + } + err = en.WriteBool(z.NotFound) + if err != nil { + err = msgp.WrapError(err, "NotFound") + return + } + // write "e" + err = en.Append(0xa1, 0x65) + if err != nil { + return + } + if z.Err == nil { + err = en.WriteNil() + if err != nil { + return + } + } else { + err = en.WriteString(*z.Err) + if err != nil { + err = msgp.WrapError(err, "Err") + return + } + } + // write "t" + err = en.Append(0xa1, 0x74) + if err != nil { + return + } + err = en.WriteTime(z.T) + if err != nil { + err = msgp.WrapError(err, "T") + return + } + return +} + +// MarshalMsg implements msgp.Marshaler +func (z *pongMsg) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, z.Msgsize()) + // map header, size 3 + // string "nf" + o = append(o, 0x83, 0xa2, 0x6e, 0x66) + o = msgp.AppendBool(o, z.NotFound) + // string "e" + o = append(o, 0xa1, 0x65) + if z.Err == nil { + o = msgp.AppendNil(o) + } else { + o = msgp.AppendString(o, *z.Err) + } + // string "t" + o = append(o, 0xa1, 0x74) + o = msgp.AppendTime(o, z.T) + return +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (z *pongMsg) UnmarshalMsg(bts []byte) (o []byte, err error) { + var field []byte + _ = field + var zb0001 uint32 + zb0001, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + for zb0001 > 0 { + zb0001-- + field, bts, err = msgp.ReadMapKeyZC(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + switch msgp.UnsafeString(field) { + case "nf": + z.NotFound, bts, err = msgp.ReadBoolBytes(bts) + if err != nil { + err = msgp.WrapError(err, "NotFound") + return + } + case "e": + if msgp.IsNil(bts) { + bts, err = msgp.ReadNilBytes(bts) + if err != nil { + return + } + z.Err = nil + } else { + if z.Err == nil { + z.Err = new(string) + } + *z.Err, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Err") + return + } + } + case "t": + z.T, bts, err = msgp.ReadTimeBytes(bts) + if err != nil { + err = msgp.WrapError(err, "T") + return + } + default: + bts, err = msgp.Skip(bts) + if err != nil { + err = msgp.WrapError(err) + return + } + } + } + o = bts + return +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (z *pongMsg) Msgsize() (s int) { + s = 1 + 3 + msgp.BoolSize + 2 + if z.Err == nil { + s += msgp.NilSize + } else { + s += msgp.StringPrefixSize + len(*z.Err) + } + s += 2 + msgp.TimeSize + return +} diff --git a/internal/grid/msg_gen_test.go b/internal/grid/msg_gen_test.go new file mode 100644 index 00000000..7e5574af --- /dev/null +++ b/internal/grid/msg_gen_test.go @@ -0,0 +1,688 @@ +// Code generated by github.com/tinylib/msgp DO NOT EDIT. + +package grid + +import ( + "bytes" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalconnectReq(t *testing.T) { + v := connectReq{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgconnectReq(b *testing.B) { + v := connectReq{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgconnectReq(b *testing.B) { + v := connectReq{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalconnectReq(b *testing.B) { + v := connectReq{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeconnectReq(t *testing.T) { + v := connectReq{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeconnectReq Msgsize() is inaccurate") + } + + vn := connectReq{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeconnectReq(b *testing.B) { + v := connectReq{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeconnectReq(b *testing.B) { + v := connectReq{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalconnectResp(t *testing.T) { + v := connectResp{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgconnectResp(b *testing.B) { + v := connectResp{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgconnectResp(b *testing.B) { + v := connectResp{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalconnectResp(b *testing.B) { + v := connectResp{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodeconnectResp(t *testing.T) { + v := connectResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodeconnectResp Msgsize() is inaccurate") + } + + vn := connectResp{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodeconnectResp(b *testing.B) { + v := connectResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodeconnectResp(b *testing.B) { + v := connectResp{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalmessage(t *testing.T) { + v := message{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgmessage(b *testing.B) { + v := message{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgmessage(b *testing.B) { + v := message{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalmessage(b *testing.B) { + v := message{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodemessage(t *testing.T) { + v := message{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodemessage Msgsize() is inaccurate") + } + + vn := message{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodemessage(b *testing.B) { + v := message{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodemessage(b *testing.B) { + v := message{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalmuxConnectError(t *testing.T) { + v := muxConnectError{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgmuxConnectError(b *testing.B) { + v := muxConnectError{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgmuxConnectError(b *testing.B) { + v := muxConnectError{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalmuxConnectError(b *testing.B) { + v := muxConnectError{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodemuxConnectError(t *testing.T) { + v := muxConnectError{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodemuxConnectError Msgsize() is inaccurate") + } + + vn := muxConnectError{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodemuxConnectError(b *testing.B) { + v := muxConnectError{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodemuxConnectError(b *testing.B) { + v := muxConnectError{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalpingMsg(t *testing.T) { + v := pingMsg{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgpingMsg(b *testing.B) { + v := pingMsg{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgpingMsg(b *testing.B) { + v := pingMsg{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalpingMsg(b *testing.B) { + v := pingMsg{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodepingMsg(t *testing.T) { + v := pingMsg{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodepingMsg Msgsize() is inaccurate") + } + + vn := pingMsg{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodepingMsg(b *testing.B) { + v := pingMsg{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodepingMsg(b *testing.B) { + v := pingMsg{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalpongMsg(t *testing.T) { + v := pongMsg{} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + left, err := v.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } +} + +func BenchmarkMarshalMsgpongMsg(b *testing.B) { + v := pongMsg{} + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgpongMsg(b *testing.B) { + v := pongMsg{} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalpongMsg(b *testing.B) { + v := pongMsg{} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestEncodeDecodepongMsg(t *testing.T) { + v := pongMsg{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + + m := v.Msgsize() + if buf.Len() > m { + t.Log("WARNING: TestEncodeDecodepongMsg Msgsize() is inaccurate") + } + + vn := pongMsg{} + err := msgp.Decode(&buf, &vn) + if err != nil { + t.Error(err) + } + + buf.Reset() + msgp.Encode(&buf, &v) + err = msgp.NewReader(&buf).Skip() + if err != nil { + t.Error(err) + } +} + +func BenchmarkEncodepongMsg(b *testing.B) { + v := pongMsg{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + en := msgp.NewWriter(msgp.Nowhere) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + v.EncodeMsg(en) + } + en.Flush() +} + +func BenchmarkDecodepongMsg(b *testing.B) { + v := pongMsg{} + var buf bytes.Buffer + msgp.Encode(&buf, &v) + b.SetBytes(int64(buf.Len())) + rd := msgp.NewEndlessReader(buf.Bytes(), b) + dc := msgp.NewReader(rd) + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + err := v.DecodeMsg(dc) + if err != nil { + b.Fatal(err) + } + } +} diff --git a/internal/grid/msg_string.go b/internal/grid/msg_string.go new file mode 100644 index 00000000..f89986b3 --- /dev/null +++ b/internal/grid/msg_string.go @@ -0,0 +1,40 @@ +// Code generated by "stringer -type=Op -output=msg_string.go -trimprefix=Op msg.go"; DO NOT EDIT. + +package grid + +import "strconv" + +func _() { + // An "invalid array index" compiler error signifies that the constant values have changed. + // Re-run the stringer command to generate them again. + var x [1]struct{} + _ = x[OpConnect-1] + _ = x[OpConnectResponse-2] + _ = x[OpPing-3] + _ = x[OpPong-4] + _ = x[OpConnectMux-5] + _ = x[OpMuxConnectError-6] + _ = x[OpDisconnectClientMux-7] + _ = x[OpDisconnectServerMux-8] + _ = x[OpMuxClientMsg-9] + _ = x[OpMuxServerMsg-10] + _ = x[OpUnblockSrvMux-11] + _ = x[OpUnblockClMux-12] + _ = x[OpAckMux-13] + _ = x[OpRequest-14] + _ = x[OpResponse-15] + _ = x[OpDisconnect-16] + _ = x[OpMerged-17] +} + +const _Op_name = "ConnectConnectResponsePingPongConnectMuxMuxConnectErrorDisconnectClientMuxDisconnectServerMuxMuxClientMsgMuxServerMsgUnblockSrvMuxUnblockClMuxAckMuxRequestResponseDisconnectMerged" + +var _Op_index = [...]uint8{0, 7, 22, 26, 30, 40, 55, 74, 93, 105, 117, 130, 142, 148, 155, 163, 173, 179} + +func (i Op) String() string { + i -= 1 + if i >= Op(len(_Op_index)-1) { + return "Op(" + strconv.FormatInt(int64(i+1), 10) + ")" + } + return _Op_name[_Op_index[i]:_Op_index[i+1]] +} diff --git a/internal/grid/muxclient.go b/internal/grid/muxclient.go new file mode 100644 index 00000000..769f0854 --- /dev/null +++ b/internal/grid/muxclient.go @@ -0,0 +1,662 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package grid + +import ( + "context" + "encoding/binary" + "errors" + "fmt" + "sync" + "sync/atomic" + "time" + + xioutil "github.com/minio/minio/internal/ioutil" + "github.com/zeebo/xxh3" +) + +// muxClient is a stateful connection to a remote. +type muxClient struct { + MuxID uint64 + SendSeq, RecvSeq uint32 + LastPong int64 + BaseFlags Flags + ctx context.Context + cancelFn context.CancelCauseFunc + parent *Connection + respWait chan<- Response + respMu sync.Mutex + singleResp bool + closed bool + stateless bool + acked bool + init bool + deadline time.Duration + outBlock chan struct{} + subroute *subHandlerID + respErr atomic.Pointer[error] + clientPingInterval time.Duration +} + +// Response is a response from the server. +type Response struct { + Msg []byte + Err error +} + +func newMuxClient(ctx context.Context, muxID uint64, parent *Connection) *muxClient { + ctx, cancelFn := context.WithCancelCause(ctx) + return &muxClient{ + MuxID: muxID, + ctx: ctx, + cancelFn: cancelFn, + parent: parent, + LastPong: time.Now().UnixNano(), + BaseFlags: parent.baseFlags, + clientPingInterval: parent.clientPingInterval, + } +} + +// roundtrip performs a roundtrip, returning the first response. +// This cannot be used concurrently. +func (m *muxClient) roundtrip(h HandlerID, req []byte) ([]byte, error) { + if m.init { + return nil, errors.New("mux client already used") + } + m.init = true + m.singleResp = true + msg := message{ + Op: OpRequest, + MuxID: m.MuxID, + Handler: h, + Flags: m.BaseFlags | FlagEOF, + Payload: req, + DeadlineMS: uint32(m.deadline.Milliseconds()), + } + if m.subroute != nil { + msg.Flags |= FlagSubroute + } + ch := make(chan Response, 1) + m.respMu.Lock() + if m.closed { + m.respMu.Unlock() + return nil, ErrDisconnected + } + m.respWait = ch + m.respMu.Unlock() + ctx := m.ctx + + // Add deadline if none. + if msg.DeadlineMS == 0 { + msg.DeadlineMS = uint32(defaultSingleRequestTimeout / time.Millisecond) + var cancel context.CancelFunc + ctx, cancel = context.WithTimeout(ctx, defaultSingleRequestTimeout) + defer cancel() + } + // Send request + if err := m.send(msg); err != nil { + return nil, err + } + if debugReqs { + fmt.Println(m.MuxID, m.parent.String(), "SEND") + } + // Wait for response or context. + select { + case v, ok := <-ch: + if !ok { + return nil, ErrDisconnected + } + if debugReqs && v.Err != nil { + v.Err = fmt.Errorf("%d %s RESP ERR: %w", m.MuxID, m.parent.String(), v.Err) + } + return v.Msg, v.Err + case <-ctx.Done(): + if debugReqs { + return nil, fmt.Errorf("%d %s ERR: %w", m.MuxID, m.parent.String(), context.Cause(ctx)) + } + return nil, context.Cause(ctx) + } +} + +// send the message. msg.Seq and msg.MuxID will be set +func (m *muxClient) send(msg message) error { + m.respMu.Lock() + defer m.respMu.Unlock() + if m.closed { + return errors.New("mux client closed") + } + return m.sendLocked(msg) +} + +// sendLocked the message. msg.Seq and msg.MuxID will be set. +// m.respMu must be held. +func (m *muxClient) sendLocked(msg message) error { + dst := GetByteBufferCap(msg.Msgsize()) + msg.Seq = m.SendSeq + msg.MuxID = m.MuxID + msg.Flags |= m.BaseFlags + if debugPrint { + fmt.Println("Client sending", &msg, "to", m.parent.Remote) + } + m.SendSeq++ + + dst, err := msg.MarshalMsg(dst) + if err != nil { + return err + } + if msg.Flags&FlagSubroute != 0 { + if m.subroute == nil { + return fmt.Errorf("internal error: subroute not defined on client") + } + hid := m.subroute.withHandler(msg.Handler) + before := len(dst) + dst = append(dst, hid[:]...) + if debugPrint { + fmt.Println("Added subroute", hid.String(), "to message", msg, "len", len(dst)-before) + } + } + if msg.Flags&FlagCRCxxh3 != 0 { + h := xxh3.Hash(dst) + dst = binary.LittleEndian.AppendUint32(dst, uint32(h)) + } + return m.parent.send(m.ctx, dst) +} + +// RequestStateless will send a single payload request and stream back results. +// req may not be read/written to after calling. +// TODO: Not implemented +func (m *muxClient) RequestStateless(h HandlerID, req []byte, out chan<- Response) { + if m.init { + out <- Response{Err: errors.New("mux client already used")} + } + m.init = true + + // Try to grab an initial block. + m.singleResp = false + msg := message{ + Op: OpConnectMux, + Handler: h, + Flags: FlagEOF, + Payload: req, + DeadlineMS: uint32(m.deadline.Milliseconds()), + } + msg.setZeroPayloadFlag() + if m.subroute != nil { + msg.Flags |= FlagSubroute + } + + // Send... + err := m.send(msg) + if err != nil { + out <- Response{Err: err} + return + } + + // Route directly to output. + m.respWait = out +} + +// RequestStream will send a single payload request and stream back results. +// 'requests' can be nil, in which case only req is sent as input. +// It will however take less resources. +func (m *muxClient) RequestStream(h HandlerID, payload []byte, requests chan []byte, responses chan Response) (*Stream, error) { + if m.init { + return nil, errors.New("mux client already used") + } + if responses == nil { + return nil, errors.New("RequestStream: responses channel is nil") + } + m.init = true + m.respMu.Lock() + if m.closed { + m.respMu.Unlock() + return nil, ErrDisconnected + } + m.respWait = responses // Route directly to output. + m.respMu.Unlock() + + // Try to grab an initial block. + m.singleResp = false + m.RecvSeq = m.SendSeq // Sync + if cap(requests) > 0 { + m.outBlock = make(chan struct{}, cap(requests)) + } + msg := message{ + Op: OpConnectMux, + Handler: h, + Payload: payload, + DeadlineMS: uint32(m.deadline.Milliseconds()), + } + msg.setZeroPayloadFlag() + if requests == nil { + msg.Flags |= FlagEOF + } + if m.subroute != nil { + msg.Flags |= FlagSubroute + } + + // Send... + err := m.send(msg) + if err != nil { + return nil, err + } + if debugPrint { + fmt.Println("Connecting Mux", m.MuxID, ",to", m.parent.Remote) + } + + // Space for one message and an error. + responseCh := make(chan Response, 1) + + // Spawn simple disconnect + if requests == nil { + go m.handleOneWayStream(responseCh, responses) + return &Stream{responses: responseCh, Requests: nil, ctx: m.ctx, cancel: m.cancelFn, muxID: m.MuxID}, nil + } + + // Deliver responses and send unblocks back to the server. + go m.handleTwowayResponses(responseCh, responses) + go m.handleTwowayRequests(responses, requests) + + return &Stream{responses: responseCh, Requests: requests, ctx: m.ctx, cancel: m.cancelFn, muxID: m.MuxID}, nil +} + +func (m *muxClient) addErrorNonBlockingClose(respHandler chan<- Response, err error) { + m.respMu.Lock() + defer m.respMu.Unlock() + if !m.closed { + m.respErr.Store(&err) + // Do not block. + select { + case respHandler <- Response{Err: err}: + xioutil.SafeClose(respHandler) + default: + go func() { + respHandler <- Response{Err: err} + xioutil.SafeClose(respHandler) + }() + } + gridLogIf(m.ctx, m.sendLocked(message{Op: OpDisconnectServerMux, MuxID: m.MuxID})) + m.closed = true + } +} + +// respHandler +func (m *muxClient) handleOneWayStream(respHandler chan<- Response, respServer <-chan Response) { + if debugPrint { + start := time.Now() + defer func() { + fmt.Println("Mux", m.MuxID, "Request took", time.Since(start).Round(time.Millisecond)) + }() + } + defer func() { + // addErrorNonBlockingClose will close the response channel + // - maybe async, so we shouldn't do it here. + if m.respErr.Load() == nil { + xioutil.SafeClose(respHandler) + } + }() + var pingTimer <-chan time.Time + if m.deadline == 0 || m.deadline > m.clientPingInterval { + ticker := time.NewTicker(m.clientPingInterval) + defer ticker.Stop() + pingTimer = ticker.C + atomic.StoreInt64(&m.LastPong, time.Now().UnixNano()) + } + defer m.parent.deleteMux(false, m.MuxID) + for { + select { + case <-m.ctx.Done(): + if debugPrint { + fmt.Println("Client sending disconnect to mux", m.MuxID) + } + err := context.Cause(m.ctx) + if !errors.Is(err, errStreamEOF) { + m.addErrorNonBlockingClose(respHandler, err) + } + return + case resp, ok := <-respServer: + if !ok { + return + } + sendResp: + select { + case respHandler <- resp: + m.respMu.Lock() + if !m.closed { + gridLogIf(m.ctx, m.sendLocked(message{Op: OpUnblockSrvMux, MuxID: m.MuxID})) + } + m.respMu.Unlock() + case <-m.ctx.Done(): + // Client canceled. Don't block. + // Next loop will catch it. + case <-pingTimer: + if !m.doPing(respHandler) { + return + } + goto sendResp + } + case <-pingTimer: + if !m.doPing(respHandler) { + return + } + } + } +} + +// doPing checks last ping time and sends another ping. +func (m *muxClient) doPing(respHandler chan<- Response) (ok bool) { + m.respMu.Lock() + if m.closed { + m.respMu.Unlock() + // Already closed. This is not an error state; + // we may just be delivering the last responses. + return true + } + + // Only check ping when not closed. + if got := time.Since(time.Unix(0, atomic.LoadInt64(&m.LastPong))); got > m.clientPingInterval*2 { + m.respMu.Unlock() + if debugPrint { + fmt.Printf("Mux %d: last pong %v ago, disconnecting\n", m.MuxID, got) + } + m.addErrorNonBlockingClose(respHandler, ErrDisconnected) + return false + } + + // Send new ping + err := m.sendLocked(message{Op: OpPing, MuxID: m.MuxID}) + m.respMu.Unlock() + if err != nil { + m.addErrorNonBlockingClose(respHandler, err) + } + return err == nil +} + +// responseCh is the channel to that goes to the requester. +// internalResp is the channel that comes from the server. +func (m *muxClient) handleTwowayResponses(responseCh chan<- Response, internalResp <-chan Response) { + defer func() { + m.parent.deleteMux(false, m.MuxID) + // addErrorNonBlockingClose will close the response channel. + xioutil.SafeClose(responseCh) + }() + + // Cancelation and errors are handled by handleTwowayRequests below. + for resp := range internalResp { + m.send(message{Op: OpUnblockSrvMux, MuxID: m.MuxID}) + responseCh <- resp + } +} + +func (m *muxClient) handleTwowayRequests(errResp chan<- Response, requests <-chan []byte) { + var errState bool + if debugPrint { + start := time.Now() + defer func() { + fmt.Println("Mux", m.MuxID, "Request took", time.Since(start).Round(time.Millisecond)) + }() + } + + var pingTimer <-chan time.Time + if m.deadline == 0 || m.deadline > m.clientPingInterval { + ticker := time.NewTicker(m.clientPingInterval) + defer ticker.Stop() + pingTimer = ticker.C + atomic.StoreInt64(&m.LastPong, time.Now().UnixNano()) + } + + // Listen for client messages. +reqLoop: + for !errState { + select { + case <-m.ctx.Done(): + if debugPrint { + fmt.Println("Client sending disconnect to mux", m.MuxID) + } + m.addErrorNonBlockingClose(errResp, context.Cause(m.ctx)) + errState = true + continue + case <-pingTimer: + if !m.doPing(errResp) { + errState = true + continue + } + case req, ok := <-requests: + if !ok { + // Done send EOF + if debugPrint { + fmt.Println("Client done, sending EOF to mux", m.MuxID) + } + msg := message{ + Op: OpMuxClientMsg, + MuxID: m.MuxID, + Flags: FlagEOF, + } + msg.setZeroPayloadFlag() + err := m.send(msg) + if err != nil { + m.addErrorNonBlockingClose(errResp, err) + } + break reqLoop + } + // Grab a send token. + sendReq: + select { + case <-m.ctx.Done(): + m.addErrorNonBlockingClose(errResp, context.Cause(m.ctx)) + errState = true + continue + case <-pingTimer: + if !m.doPing(errResp) { + errState = true + continue + } + goto sendReq + case <-m.outBlock: + } + msg := message{ + Op: OpMuxClientMsg, + MuxID: m.MuxID, + Seq: 1, + Payload: req, + } + msg.setZeroPayloadFlag() + err := m.send(msg) + PutByteBuffer(req) + if err != nil { + m.addErrorNonBlockingClose(errResp, err) + errState = true + continue + } + msg.Seq++ + } + } + + if errState { + // Drain requests. + for { + select { + case r, ok := <-requests: + if !ok { + return + } + PutByteBuffer(r) + default: + return + } + } + } + + for !errState { + select { + case <-m.ctx.Done(): + if debugPrint { + fmt.Println("Client sending disconnect to mux", m.MuxID) + } + m.addErrorNonBlockingClose(errResp, context.Cause(m.ctx)) + return + case <-pingTimer: + errState = !m.doPing(errResp) + } + } +} + +// checkSeq will check if sequence number is correct and increment it by 1. +func (m *muxClient) checkSeq(seq uint32) (ok bool) { + if seq != m.RecvSeq { + if debugPrint { + fmt.Printf("MuxID: %d client, expected sequence %d, got %d\n", m.MuxID, m.RecvSeq, seq) + } + m.addResponse(Response{Err: ErrIncorrectSequence}) + return false + } + m.RecvSeq++ + return true +} + +// response will send handleIncoming response to client. +// may never block. +// Should return whether the next call would block. +func (m *muxClient) response(seq uint32, r Response) { + if debugReqs { + fmt.Println(m.MuxID, m.parent.String(), "RESP") + } + if debugPrint { + fmt.Printf("mux %d: got msg seqid %d, payload length: %d, err:%v\n", m.MuxID, seq, len(r.Msg), r.Err) + } + if !m.checkSeq(seq) { + if debugReqs { + fmt.Println(m.MuxID, m.parent.String(), "CHECKSEQ FAIL", m.RecvSeq, seq) + } + PutByteBuffer(r.Msg) + r.Msg = nil + r.Err = ErrIncorrectSequence + m.addResponse(r) + return + } + atomic.StoreInt64(&m.LastPong, time.Now().UnixNano()) + ok := m.addResponse(r) + if !ok { + PutByteBuffer(r.Msg) + } +} + +var errStreamEOF = errors.New("stream EOF") + +// error is a message from the server to disconnect. +func (m *muxClient) error(err RemoteErr) { + if debugPrint { + fmt.Printf("mux %d: got remote err:%v\n", m.MuxID, string(err)) + } + m.addResponse(Response{Err: &err}) +} + +func (m *muxClient) ack(seq uint32) { + if !m.checkSeq(seq) { + return + } + if m.acked || m.outBlock == nil { + return + } + available := cap(m.outBlock) + for range available { + m.outBlock <- struct{}{} + } + m.acked = true +} + +func (m *muxClient) unblockSend(seq uint32) { + if !m.checkSeq(seq) { + return + } + select { + case m.outBlock <- struct{}{}: + default: + gridLogIf(m.ctx, errors.New("output unblocked overflow")) + } +} + +func (m *muxClient) pong(msg pongMsg) { + if msg.NotFound || msg.Err != nil { + err := errors.New("remote terminated call") + if msg.Err != nil { + err = fmt.Errorf("remove pong failed: %v", &msg.Err) + } + m.addResponse(Response{Err: err}) + return + } + atomic.StoreInt64(&m.LastPong, time.Now().UnixNano()) +} + +// addResponse will add a response to the response channel. +// This function will never block +func (m *muxClient) addResponse(r Response) (ok bool) { + m.respMu.Lock() + defer m.respMu.Unlock() + if m.closed { + return false + } + select { + case m.respWait <- r: + if r.Err != nil { + if debugPrint { + fmt.Println("Closing mux", m.MuxID, "due to error:", r.Err) + } + m.closeLocked() + } + return true + default: + if m.stateless { + // Drop message if not stateful. + return ok + } + err := errors.New("INTERNAL ERROR: Response was blocked") + gridLogIf(m.ctx, err) + m.closeLocked() + return false + } +} + +func (m *muxClient) close() { + if debugPrint { + fmt.Println("closing outgoing mux", m.MuxID) + } + if !m.respMu.TryLock() { + // Cancel before locking - will unblock any pending sends. + if m.cancelFn != nil { + m.cancelFn(context.Canceled) + } + // Wait for senders to release. + m.respMu.Lock() + } + + defer m.respMu.Unlock() + m.closeLocked() +} + +func (m *muxClient) closeLocked() { + if m.closed { + return + } + // We hold the lock, so nobody can modify m.respWait while we're closing. + if m.respWait != nil { + xioutil.SafeClose(m.respWait) + m.respWait = nil + } + m.closed = true +} diff --git a/internal/grid/muxserver.go b/internal/grid/muxserver.go new file mode 100644 index 00000000..61707d5e --- /dev/null +++ b/internal/grid/muxserver.go @@ -0,0 +1,392 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package grid + +import ( + "context" + "errors" + "fmt" + "sync" + "sync/atomic" + "time" + + xioutil "github.com/minio/minio/internal/ioutil" +) + +type muxServer struct { + ID uint64 + LastPing int64 + SendSeq, RecvSeq uint32 + Resp chan []byte + BaseFlags Flags + ctx context.Context + cancel context.CancelFunc + inbound chan []byte + parent *Connection + sendMu sync.Mutex + recvMu sync.Mutex + outBlock chan struct{} + clientPingInterval time.Duration +} + +func newMuxStateless(ctx context.Context, msg message, c *Connection, handler StatelessHandler) *muxServer { + var cancel context.CancelFunc + ctx = setCaller(ctx, c.remote) + if msg.DeadlineMS > 0 { + ctx, cancel = context.WithTimeout(ctx, time.Duration(msg.DeadlineMS)*time.Millisecond) + } else { + ctx, cancel = context.WithCancel(ctx) + } + m := muxServer{ + ID: msg.MuxID, + RecvSeq: msg.Seq + 1, + SendSeq: msg.Seq, + ctx: ctx, + cancel: cancel, + parent: c, + LastPing: time.Now().Unix(), + BaseFlags: c.baseFlags, + } + go func() { + // TODO: Handle + }() + + return &m +} + +func newMuxStream(ctx context.Context, msg message, c *Connection, handler StreamHandler) *muxServer { + var cancel context.CancelFunc + ctx = setCaller(ctx, c.remote) + if len(handler.Subroute) > 0 { + ctx = setSubroute(ctx, handler.Subroute) + } + if msg.DeadlineMS > 0 { + ctx, cancel = context.WithTimeout(ctx, time.Duration(msg.DeadlineMS)*time.Millisecond+c.addDeadline) + } else { + ctx, cancel = context.WithCancel(ctx) + } + + send := make(chan []byte) + inboundCap, outboundCap := handler.InCapacity, handler.OutCapacity + if outboundCap <= 0 { + outboundCap = 1 + } + + m := muxServer{ + ID: msg.MuxID, + RecvSeq: msg.Seq + 1, + SendSeq: msg.Seq, + ctx: ctx, + cancel: cancel, + parent: c, + inbound: nil, + outBlock: make(chan struct{}, outboundCap), + LastPing: time.Now().Unix(), + BaseFlags: c.baseFlags, + clientPingInterval: c.clientPingInterval, + } + // Acknowledge Mux created. + // Send async. + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + var ack message + ack.Op = OpAckMux + ack.Flags = m.BaseFlags + ack.MuxID = m.ID + m.send(ack) + if debugPrint { + fmt.Println("connected stream mux:", ack.MuxID) + } + }() + + // Data inbound to the handler + var handlerIn chan []byte + if inboundCap > 0 { + m.inbound = make(chan []byte, inboundCap) + handlerIn = make(chan []byte, 1) + go func(inbound chan []byte) { + wg.Wait() + defer xioutil.SafeClose(handlerIn) + m.handleInbound(c, inbound, handlerIn) + }(m.inbound) + } + // Fill outbound block. + // Each token represents a message that can be sent to the client without blocking. + // The client will refill the tokens as they confirm delivery of the messages. + for range outboundCap { + m.outBlock <- struct{}{} + } + + // Handler goroutine. + var handlerErr atomic.Pointer[RemoteErr] + go func() { + wg.Wait() + defer xioutil.SafeClose(send) + err := m.handleRequests(ctx, msg, send, handler, handlerIn) + if err != nil { + handlerErr.Store(err) + } + }() + + // Response sender goroutine... + go func(outBlock <-chan struct{}) { + wg.Wait() + defer m.parent.deleteMux(true, m.ID) + m.sendResponses(ctx, send, c, &handlerErr, outBlock) + }(m.outBlock) + + // Remote aliveness check if needed. + if msg.DeadlineMS == 0 || msg.DeadlineMS > uint32(4*c.clientPingInterval/time.Millisecond) { + go func() { + wg.Wait() + m.checkRemoteAlive() + }() + } + return &m +} + +// handleInbound sends unblocks when we have delivered the message to the handler. +func (m *muxServer) handleInbound(c *Connection, inbound <-chan []byte, handlerIn chan<- []byte) { + for { + select { + case <-m.ctx.Done(): + return + case in, ok := <-inbound: + if !ok { + return + } + select { + case <-m.ctx.Done(): + return + case handlerIn <- in: + m.send(message{Op: OpUnblockClMux, MuxID: m.ID, Flags: c.baseFlags}) + } + } + } +} + +// sendResponses will send responses to the client. +func (m *muxServer) sendResponses(ctx context.Context, toSend <-chan []byte, c *Connection, handlerErr *atomic.Pointer[RemoteErr], outBlock <-chan struct{}) { + for { + // Process outgoing message. + var payload []byte + var ok bool + select { + case payload, ok = <-toSend: + case <-ctx.Done(): + return + } + select { + case <-ctx.Done(): + return + case <-outBlock: + } + msg := message{ + MuxID: m.ID, + Op: OpMuxServerMsg, + Flags: c.baseFlags, + } + if !ok { + hErr := handlerErr.Load() + if debugPrint { + fmt.Println("muxServer: Mux", m.ID, "send EOF", hErr) + } + msg.Flags |= FlagEOF + if hErr != nil { + msg.Flags |= FlagPayloadIsErr + msg.Payload = []byte(*hErr) + } + msg.setZeroPayloadFlag() + m.send(msg) + return + } + msg.Payload = payload + msg.setZeroPayloadFlag() + m.send(msg) + } +} + +// handleRequests will handle the requests from the client and call the handler function. +func (m *muxServer) handleRequests(ctx context.Context, msg message, send chan<- []byte, handler StreamHandler, handlerIn <-chan []byte) (handlerErr *RemoteErr) { + start := time.Now() + defer func() { + if debugPrint { + fmt.Println("Mux", m.ID, "Handler took", time.Since(start).Round(time.Millisecond)) + } + if r := recover(); r != nil { + gridLogIf(ctx, fmt.Errorf("grid handler (%v) panic: %v", msg.Handler, r)) + err := RemoteErr(fmt.Sprintf("handler panic: %v", r)) + handlerErr = &err + } + if debugPrint { + fmt.Println("muxServer: Mux", m.ID, "Returned with", handlerErr) + } + }() + // handlerErr is guarded by 'send' channel. + handlerErr = handler.Handle(ctx, msg.Payload, handlerIn, send) + return handlerErr +} + +// checkRemoteAlive will check if the remote is alive. +func (m *muxServer) checkRemoteAlive() { + t := time.NewTicker(m.clientPingInterval) + defer t.Stop() + for { + select { + case <-m.ctx.Done(): + return + case <-t.C: + last := time.Since(time.Unix(atomic.LoadInt64(&m.LastPing), 0)) + if last > 4*m.clientPingInterval { + gridLogIf(m.ctx, fmt.Errorf("canceling remote connection %s not seen for %v", m.parent, last)) + m.close() + return + } + } + } +} + +// checkSeq will check if sequence number is correct and increment it by 1. +func (m *muxServer) checkSeq(seq uint32) (ok bool) { + if seq != m.RecvSeq { + if debugPrint { + fmt.Printf("expected sequence %d, got %d\n", m.RecvSeq, seq) + } + m.disconnect(fmt.Sprintf("receive sequence number mismatch. want %d, got %d", m.RecvSeq, seq), false) + return false + } + m.RecvSeq++ + return true +} + +func (m *muxServer) message(msg message) { + if debugPrint { + fmt.Printf("muxServer: received message %d, length %d\n", msg.Seq, len(msg.Payload)) + } + if !m.checkSeq(msg.Seq) { + return + } + m.recvMu.Lock() + defer m.recvMu.Unlock() + if cap(m.inbound) == 0 { + m.disconnect("did not expect inbound message", true) + return + } + // Note, on EOF no value can be sent. + if msg.Flags&FlagEOF != 0 { + if len(msg.Payload) > 0 { + gridLogIf(m.ctx, fmt.Errorf("muxServer: EOF message with payload")) + } + if m.inbound != nil { + xioutil.SafeClose(m.inbound) + m.inbound = nil + } + return + } + + select { + case <-m.ctx.Done(): + case m.inbound <- msg.Payload: + if debugPrint { + fmt.Printf("muxServer: Sent seq %d to handler\n", msg.Seq) + } + default: + m.disconnect("handler blocked", true) + } +} + +func (m *muxServer) unblockSend(seq uint32) { + if !m.checkSeq(seq) { + return + } + m.recvMu.Lock() + defer m.recvMu.Unlock() + if m.outBlock == nil { + // Closed + return + } + select { + case m.outBlock <- struct{}{}: + default: + gridLogIf(m.ctx, errors.New("output unblocked overflow")) + } +} + +func (m *muxServer) ping(seq uint32) pongMsg { + if !m.checkSeq(seq) { + msg := fmt.Sprintf("receive sequence number mismatch. want %d, got %d", m.RecvSeq, seq) + return pongMsg{Err: &msg} + } + select { + case <-m.ctx.Done(): + err := context.Cause(m.ctx).Error() + return pongMsg{Err: &err} + default: + atomic.StoreInt64(&m.LastPing, time.Now().Unix()) + return pongMsg{} + } +} + +// disconnect will disconnect the mux. +// m.recvMu must be locked when calling this function. +func (m *muxServer) disconnect(msg string, locked bool) { + if debugPrint { + fmt.Println("Mux", m.ID, "disconnecting. Reason:", msg) + } + if msg != "" { + m.send(message{Op: OpMuxServerMsg, MuxID: m.ID, Flags: FlagPayloadIsErr | FlagEOF, Payload: []byte(msg)}) + } else { + m.send(message{Op: OpDisconnectClientMux, MuxID: m.ID}) + } + // Unlock, since we are calling deleteMux, which will call close - which will lock recvMu. + if locked { + m.recvMu.Unlock() + defer m.recvMu.Lock() + } + m.parent.deleteMux(true, m.ID) +} + +func (m *muxServer) send(msg message) { + m.sendMu.Lock() + defer m.sendMu.Unlock() + msg.MuxID = m.ID + msg.Seq = m.SendSeq + m.SendSeq++ + if debugPrint { + fmt.Printf("Mux %d, Sending %+v\n", m.ID, msg) + } + gridLogIf(m.ctx, m.parent.queueMsg(msg, nil)) +} + +func (m *muxServer) close() { + m.cancel() + m.recvMu.Lock() + defer m.recvMu.Unlock() + + if m.inbound != nil { + xioutil.SafeClose(m.inbound) + m.inbound = nil + } + + if m.outBlock != nil { + xioutil.SafeClose(m.outBlock) + m.outBlock = nil + } +} diff --git a/internal/grid/stream.go b/internal/grid/stream.go new file mode 100644 index 00000000..cbc69c1b --- /dev/null +++ b/internal/grid/stream.go @@ -0,0 +1,114 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package grid + +import ( + "context" + "errors" +) + +// A Stream is a two-way stream. +// All responses *must* be read by the caller. +// If the call is canceled through the context, +// the appropriate error will be returned. +type Stream struct { + // responses from the remote server. + // Channel will be closed after error or when remote closes. + // All responses *must* be read by the caller until either an error is returned or the channel is closed. + // Canceling the context will cause the context cancellation error to be returned. + responses <-chan Response + cancel context.CancelCauseFunc + + // Requests sent to the server. + // If the handler is defined with 0 incoming capacity this will be nil. + // Channel *must* be closed to signal the end of the stream. + // If the request context is canceled, the stream will no longer process requests. + // Requests sent cannot be used any further by the called. + Requests chan<- []byte + + muxID uint64 + ctx context.Context +} + +// Send a payload to the remote server. +func (s *Stream) Send(b []byte) error { + if s.Requests == nil { + return errors.New("stream does not accept requests") + } + select { + case s.Requests <- b: + return nil + case <-s.ctx.Done(): + return context.Cause(s.ctx) + } +} + +// Results returns the results from the remote server one by one. +// If any error is returned by the callback, the stream will be canceled. +// If the context is canceled, the stream will be canceled. +func (s *Stream) Results(next func(b []byte) error) (err error) { + done := false + defer func() { + if s.cancel != nil { + s.cancel(err) + } + + if !done { + // Drain channel. + for range s.responses { + } + } + }() + doneCh := s.ctx.Done() + for { + select { + case <-doneCh: + if err := context.Cause(s.ctx); !errors.Is(err, errStreamEOF) { + return err + } + // Fall through to be sure we have returned all responses. + doneCh = nil + case resp, ok := <-s.responses: + if !ok { + done = true + return nil + } + if resp.Err != nil { + s.cancel(resp.Err) + return resp.Err + } + err = next(resp.Msg) + if err != nil { + s.cancel(err) + return err + } + } + } +} + +// Done will return a channel that will be closed when the stream is done. +// This mirrors context.Done(). +func (s *Stream) Done() <-chan struct{} { + return s.ctx.Done() +} + +// Err will return the error that caused the stream to end. +// This mirrors context.Err(). +func (s *Stream) Err() error { + return s.ctx.Err() +} diff --git a/internal/grid/trace.go b/internal/grid/trace.go new file mode 100644 index 00000000..f8989390 --- /dev/null +++ b/internal/grid/trace.go @@ -0,0 +1,153 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package grid + +import ( + "context" + "fmt" + "net/http" + "strings" + "time" + + "github.com/minio/madmin-go/v3" + "github.com/minio/minio/internal/pubsub" +) + +// TraceParamsKey allows to pass trace parameters to the request via context. +// This is only needed when un-typed requests are used. +// MSS, map[string]string types are preferred, but any struct with exported fields will work. +type TraceParamsKey struct{} + +// traceRequests adds request tracing to the connection. +func (c *Connection) traceRequests(p *pubsub.PubSub[madmin.TraceInfo, madmin.TraceType]) { + c.trace = &tracer{ + Publisher: p, + TraceType: madmin.TraceInternal, + Prefix: "grid", + Local: c.Local, + Remote: c.Remote, + Subroute: "", + } +} + +// subroute adds a specific subroute to the request. +func (c *tracer) subroute(subroute string) *tracer { + if c == nil { + return nil + } + c2 := *c + c2.Subroute = subroute + return &c2 +} + +type tracer struct { + Publisher *pubsub.PubSub[madmin.TraceInfo, madmin.TraceType] + TraceType madmin.TraceType + Prefix string + Local string + Remote string + Subroute string +} + +const ( + httpScheme = "http://" + httpsScheme = "https://" +) + +func (c *muxClient) traceRoundtrip(ctx context.Context, t *tracer, h HandlerID, req []byte) ([]byte, error) { + if t == nil || t.Publisher.NumSubscribers(t.TraceType) == 0 { + return c.roundtrip(h, req) + } + + // Following trimming is needed for consistency between outputs with other internode traces. + local := strings.TrimPrefix(strings.TrimPrefix(t.Local, httpsScheme), httpScheme) + remote := strings.TrimPrefix(strings.TrimPrefix(t.Remote, httpsScheme), httpScheme) + + start := time.Now() + body := bytesOrLength(req) + resp, err := c.roundtrip(h, req) + end := time.Now() + status := http.StatusOK + errString := "" + if err != nil { + errString = err.Error() + if IsRemoteErr(err) == nil { + status = http.StatusInternalServerError + } else { + status = http.StatusBadRequest + } + } + + prefix := t.Prefix + if p := handlerPrefixes[h]; p != "" { + prefix = p + } + trace := madmin.TraceInfo{ + TraceType: t.TraceType, + FuncName: prefix + "." + h.String(), + NodeName: remote, + Time: start, + Duration: end.Sub(start), + Path: t.Subroute, + Error: errString, + Bytes: int64(len(req) + len(resp)), + HTTP: &madmin.TraceHTTPStats{ + ReqInfo: madmin.TraceRequestInfo{ + Time: start, + Proto: "grid", + Method: "REQ", + Client: local, + Headers: nil, + Path: t.Subroute, + Body: []byte(body), + }, + RespInfo: madmin.TraceResponseInfo{ + Time: end, + Headers: nil, + StatusCode: status, + Body: []byte(bytesOrLength(resp)), + }, + CallStats: madmin.TraceCallStats{ + InputBytes: len(req), + OutputBytes: len(resp), + TimeToFirstByte: end.Sub(start), + }, + }, + } + // If the context contains a TraceParamsKey, add it to the trace path. + v := ctx.Value(TraceParamsKey{}) + // Should match SingleHandler.Call checks. + switch typed := v.(type) { + case *MSS: + trace.Path += typed.ToQuery() + case map[string]string: + m := MSS(typed) + trace.Path += m.ToQuery() + case *URLValues: + trace.Path += typed.Values().Encode() + case *NoPayload, *Bytes: + trace.Path = fmt.Sprintf("%s?payload=%T", trace.Path, typed) + case string: + trace.Path = fmt.Sprintf("%s?%s", trace.Path, typed) + default: + } + trace.HTTP.ReqInfo.Path = trace.Path + + t.Publisher.Publish(trace) + return resp, err +} diff --git a/internal/grid/types.go b/internal/grid/types.go new file mode 100644 index 00000000..a16f4756 --- /dev/null +++ b/internal/grid/types.go @@ -0,0 +1,712 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package grid + +import ( + "bytes" + "encoding/json" + "errors" + "math" + "net/url" + "sort" + "strings" + "sync" + + "github.com/minio/minio/internal/bpool" + "github.com/tinylib/msgp/msgp" +) + +// Recycler will override the internal reuse in typed handlers. +// When this is supported, the handler will not do internal pooling of objects, +// call Recycle() when the object is no longer needed. +// The recycler should handle nil pointers. +type Recycler interface { + Recycle() +} + +// MSS is a map[string]string that can be serialized. +// It is not very efficient, but it is only used for easy parameter passing. +type MSS map[string]string + +// Get returns the value for the given key. +func (m *MSS) Get(key string) string { + if m == nil { + return "" + } + return (*m)[key] +} + +// Set a key, value pair. +func (m *MSS) Set(key, value string) { + if m == nil { + *m = mssPool.Get() + } + (*m)[key] = value +} + +// UnmarshalMsg deserializes m from the provided byte slice and returns the +// remainder of bytes. +func (m *MSS) UnmarshalMsg(bts []byte) (o []byte, err error) { + if m == nil { + return bts, errors.New("MSS: UnmarshalMsg on nil pointer") + } + if msgp.IsNil(bts) { + bts = bts[1:] + *m = nil + return bts, nil + } + var zb0002 uint32 + zb0002, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Values") + return o, err + } + dst := *m + if dst == nil { + dst = make(map[string]string, zb0002) + } else if len(dst) > 0 { + for key := range dst { + delete(dst, key) + } + } + for zb0002 > 0 { + var za0001 string + var za0002 string + zb0002-- + za0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Values") + return o, err + } + za0002, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, "Values", za0001) + return o, err + } + dst[za0001] = za0002 + } + *m = dst + return bts, nil +} + +// MarshalMsg appends the bytes representation of b to the provided byte slice. +func (m *MSS) MarshalMsg(bytes []byte) (o []byte, err error) { + if m == nil || *m == nil { + return msgp.AppendNil(bytes), nil + } + o = msgp.AppendMapHeader(bytes, uint32(len(*m))) + for za0001, za0002 := range *m { + o = msgp.AppendString(o, za0001) + o = msgp.AppendString(o, za0002) + } + return o, nil +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message. +func (m *MSS) Msgsize() int { + if m == nil || *m == nil { + return msgp.NilSize + } + s := msgp.MapHeaderSize + for za0001, za0002 := range *m { + s += msgp.StringPrefixSize + len(za0001) + msgp.StringPrefixSize + len(za0002) + } + return s +} + +// NewMSS returns a new MSS. +func NewMSS() *MSS { + m := MSS(mssPool.Get()) + clear(m) + return &m +} + +// NewMSSWith returns a new MSS with the given map. +func NewMSSWith(m map[string]string) *MSS { + m2 := MSS(m) + return &m2 +} + +var mssPool = bpool.Pool[map[string]string]{ + New: func() map[string]string { + return make(map[string]string, 5) + }, +} + +// Recycle the underlying map. +func (m *MSS) Recycle() { + if m != nil && *m != nil { + mssPool.Put(*m) + *m = nil + } +} + +// ToQuery constructs a URL query string from the MSS, including "?" if there are any keys. +func (m MSS) ToQuery() string { + if len(m) == 0 { + return "" + } + var buf strings.Builder + buf.WriteByte('?') + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + v := m[k] + keyEscaped := url.QueryEscape(k) + if buf.Len() > 1 { + buf.WriteByte('&') + } + buf.WriteString(keyEscaped) + buf.WriteByte('=') + buf.WriteString(url.QueryEscape(v)) + } + return buf.String() +} + +// NewBytes returns a new Bytes. +// A slice is preallocated. +func NewBytes() *Bytes { + b := Bytes(GetByteBuffer()[:0]) + return &b +} + +// NewBytesCap returns an empty Bytes with the given capacity. +func NewBytesCap(size int) *Bytes { + b := Bytes(GetByteBufferCap(size)) + return &b +} + +// NewBytesWith returns a new Bytes with the provided content. +// When sent as a parameter, the caller gives up ownership of the byte slice. +// When returned as response, the handler also gives up ownership of the byte slice. +func NewBytesWith(b []byte) *Bytes { + bb := Bytes(b) + return &bb +} + +// NewBytesWithCopyOf returns a new byte slice with a copy of the provided content. +func NewBytesWithCopyOf(b []byte) *Bytes { + if b == nil { + bb := Bytes(nil) + return &bb + } + bb := NewBytesCap(len(b)) + *bb = append(*bb, b...) + return bb +} + +// Bytes provides a byte slice that can be serialized. +type Bytes []byte + +// UnmarshalMsg deserializes b from the provided byte slice and returns the +// remainder of bytes. +func (b *Bytes) UnmarshalMsg(bytes []byte) ([]byte, error) { + if b == nil { + return bytes, errors.New("Bytes: UnmarshalMsg on nil pointer") + } + if bytes, err := msgp.ReadNilBytes(bytes); err == nil { + if *b != nil { + PutByteBuffer(*b) + } + *b = nil + return bytes, nil + } + val, bytes, err := msgp.ReadBytesZC(bytes) + if err != nil { + return bytes, err + } + if cap(*b) >= len(val) { + *b = (*b)[:len(val)] + copy(*b, val) + } else { + if cap(*b) == 0 && len(val) <= maxBufferSize { + *b = GetByteBufferCap(len(val)) + } else { + PutByteBuffer(*b) + *b = make([]byte, 0, len(val)) + } + in := *b + in = append(in[:0], val...) + *b = in + } + return bytes, nil +} + +// MarshalMsg appends the bytes representation of b to the provided byte slice. +func (b *Bytes) MarshalMsg(bytes []byte) ([]byte, error) { + if b == nil || *b == nil { + return msgp.AppendNil(bytes), nil + } + return msgp.AppendBytes(bytes, *b), nil +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message. +func (b *Bytes) Msgsize() int { + if b == nil || *b == nil { + return msgp.NilSize + } + return msgp.ArrayHeaderSize + len(*b) +} + +// Recycle puts the Bytes back into the pool. +func (b *Bytes) Recycle() { + if b != nil && *b != nil { + *b = (*b)[:0] + PutByteBuffer(*b) + *b = nil + } +} + +// URLValues can be used for url.Values. +type URLValues map[string][]string + +var urlValuesPool = bpool.Pool[map[string][]string]{ + New: func() map[string][]string { + return make(map[string][]string, 10) + }, +} + +// NewURLValues returns a new URLValues. +func NewURLValues() *URLValues { + u := URLValues(urlValuesPool.Get()) + return &u +} + +// NewURLValuesWith returns a new URLValues with the provided content. +func NewURLValuesWith(values map[string][]string) *URLValues { + u := URLValues(values) + return &u +} + +// Values returns the url.Values. +// If u is nil, an empty url.Values is returned. +// The values are a shallow copy of the underlying map. +func (u *URLValues) Values() url.Values { + if u == nil { + return url.Values{} + } + return url.Values(*u) +} + +// Recycle the underlying map. +func (u *URLValues) Recycle() { + if *u != nil { + for key := range *u { + delete(*u, key) + } + val := map[string][]string(*u) + urlValuesPool.Put(val) + *u = nil + } +} + +// MarshalMsg implements msgp.Marshaler +func (u URLValues) MarshalMsg(b []byte) (o []byte, err error) { + o = msgp.Require(b, u.Msgsize()) + o = msgp.AppendMapHeader(o, uint32(len(u))) + for zb0006, zb0007 := range u { + o = msgp.AppendString(o, zb0006) + o = msgp.AppendArrayHeader(o, uint32(len(zb0007))) + for zb0008 := range zb0007 { + o = msgp.AppendString(o, zb0007[zb0008]) + } + } + return o, err +} + +// UnmarshalMsg implements msgp.Unmarshaler +func (u *URLValues) UnmarshalMsg(bts []byte) (o []byte, err error) { + var zb0004 uint32 + zb0004, bts, err = msgp.ReadMapHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return o, err + } + if *u == nil { + *u = urlValuesPool.Get() + } + if len(*u) > 0 { + for key := range *u { + delete(*u, key) + } + } + + for zb0004 > 0 { + var zb0001 string + var zb0002 []string + zb0004-- + zb0001, bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err) + return o, err + } + var zb0005 uint32 + zb0005, bts, err = msgp.ReadArrayHeaderBytes(bts) + if err != nil { + err = msgp.WrapError(err, zb0001) + return o, err + } + if cap(zb0002) >= int(zb0005) { + zb0002 = zb0002[:zb0005] + } else { + zb0002 = make([]string, zb0005) + } + for zb0003 := range zb0002 { + zb0002[zb0003], bts, err = msgp.ReadStringBytes(bts) + if err != nil { + err = msgp.WrapError(err, zb0001, zb0003) + return o, err + } + } + (*u)[zb0001] = zb0002 + } + o = bts + return o, err +} + +// Msgsize returns an upper bound estimate of the number of bytes occupied by the serialized message +func (u URLValues) Msgsize() (s int) { + s = msgp.MapHeaderSize + for zb0006, zb0007 := range u { + _ = zb0007 + s += msgp.StringPrefixSize + len(zb0006) + msgp.ArrayHeaderSize + for zb0008 := range zb0007 { + s += msgp.StringPrefixSize + len(zb0007[zb0008]) + } + } + + return s +} + +// JSONPool is a pool for JSON objects that unmarshal into T. +type JSONPool[T any] struct { + pool sync.Pool + emptySz int +} + +// NewJSONPool returns a new JSONPool. +func NewJSONPool[T any]() *JSONPool[T] { + var t T + sz := 128 + if b, err := json.Marshal(t); err != nil { + sz = len(b) + } + return &JSONPool[T]{ + pool: sync.Pool{ + New: func() any { + var t T + return &t + }, + }, + emptySz: sz, + } +} + +func (p *JSONPool[T]) new() *T { + var zero T + if t, ok := p.pool.Get().(*T); ok { + *t = zero + return t + } + return &zero +} + +// JSON is a wrapper around a T object that can be serialized. +// There is an internal value +type JSON[T any] struct { + p *JSONPool[T] + val *T +} + +// NewJSON returns a new JSONPool. +// No initial value is set. +func (p *JSONPool[T]) NewJSON() *JSON[T] { + var j JSON[T] + j.p = p + return &j +} + +// NewJSONWith returns a new JSON with the provided value. +func (p *JSONPool[T]) NewJSONWith(val *T) *JSON[T] { + var j JSON[T] + j.p = p + j.val = val + return &j +} + +// Value returns the underlying value. +// If not set yet, a new value is created. +func (j *JSON[T]) Value() *T { + if j.val == nil { + j.val = j.p.new() + } + return j.val +} + +// ValueOrZero returns the underlying value. +// If the underlying value is nil, a zero value is returned. +func (j *JSON[T]) ValueOrZero() T { + if j == nil || j.val == nil { + var t T + return t + } + return *j.val +} + +// Set the underlying value. +func (j *JSON[T]) Set(v *T) { + j.val = v +} + +// Recycle the underlying value. +func (j *JSON[T]) Recycle() { + if j.val != nil { + j.p.pool.Put(j.val) + j.val = nil + } +} + +// MarshalMsg implements msgp.Marshaler +func (j *JSON[T]) MarshalMsg(b []byte) (o []byte, err error) { + if j.val == nil { + return msgp.AppendNil(b), nil + } + buf := bytes.NewBuffer(GetByteBuffer()[:0]) + defer func() { + PutByteBuffer(buf.Bytes()) + }() + enc := json.NewEncoder(buf) + err = enc.Encode(j.val) + if err != nil { + return b, err + } + return msgp.AppendBytes(b, buf.Bytes()), nil +} + +// UnmarshalMsg will JSON marshal the value and wrap as a msgp byte array. +// Nil values are supported. +func (j *JSON[T]) UnmarshalMsg(bytes []byte) ([]byte, error) { + if bytes, err := msgp.ReadNilBytes(bytes); err == nil { + if j.val != nil { + j.p.pool.Put(j.val) + } + j.val = nil + return bytes, nil + } + val, bytes, err := msgp.ReadBytesZC(bytes) + if err != nil { + return bytes, err + } + if j.val == nil { + j.val = j.p.new() + } else { + var t T + *j.val = t + } + return bytes, json.Unmarshal(val, j.val) +} + +// Msgsize returns the size of an empty JSON object. +func (j *JSON[T]) Msgsize() int { + return j.p.emptySz +} + +// NoPayload is a type that can be used for handlers that do not use a payload. +type NoPayload struct{} + +// Msgsize returns 0. +func (p NoPayload) Msgsize() int { + return 0 +} + +// UnmarshalMsg satisfies the interface, but is a no-op. +func (NoPayload) UnmarshalMsg(bytes []byte) ([]byte, error) { + return bytes, nil +} + +// MarshalMsg satisfies the interface, but is a no-op. +func (NoPayload) MarshalMsg(bytes []byte) ([]byte, error) { + return bytes, nil +} + +// NewNoPayload returns an empty NoPayload struct. +func NewNoPayload() NoPayload { + return NoPayload{} +} + +// Recycle is a no-op. +func (NoPayload) Recycle() {} + +// ArrayOf wraps an array of Messagepack compatible objects. +type ArrayOf[T RoundTripper] struct { + aPool sync.Pool // Arrays + ePool bpool.Pool[T] // Elements +} + +// NewArrayOf returns a new ArrayOf. +// You must provide a function that returns a new instance of T. +func NewArrayOf[T RoundTripper](newFn func() T) *ArrayOf[T] { + return &ArrayOf[T]{ + ePool: bpool.Pool[T]{New: func() T { + return newFn() + }}, + } +} + +// New returns a new empty Array. +func (p *ArrayOf[T]) New() *Array[T] { + return &Array[T]{ + p: p, + } +} + +// NewWith returns a new Array with the provided value (not copied). +func (p *ArrayOf[T]) NewWith(val []T) *Array[T] { + return &Array[T]{ + p: p, + val: val, + } +} + +func (p *ArrayOf[T]) newA(sz uint32) []T { + t, ok := p.aPool.Get().(*[]T) + if !ok || t == nil { + return make([]T, 0, sz) + } + t2 := *t + return t2[:0] +} + +func (p *ArrayOf[T]) putA(v []T) { + var zero T // nil + for i, t := range v { + //nolint:staticcheck // SA6002 IT IS A GENERIC VALUE! + p.ePool.Put(t) + v[i] = zero + } + if v != nil { + v = v[:0] + p.aPool.Put(&v) + } +} + +func (p *ArrayOf[T]) newE() T { + return p.ePool.Get() +} + +// Array provides a wrapper for an underlying array of serializable objects. +type Array[T RoundTripper] struct { + p *ArrayOf[T] + val []T +} + +// Msgsize returns the size of the array in bytes. +func (j *Array[T]) Msgsize() int { + if j.val == nil { + return msgp.NilSize + } + sz := msgp.ArrayHeaderSize + for _, v := range j.val { + sz += v.Msgsize() + } + return sz +} + +// Value returns the underlying value. +// Regular append mechanics should be observed. +// If no value has been set yet, a new array is created. +func (j *Array[T]) Value() []T { + if j.val == nil { + j.val = j.p.newA(10) + } + return j.val +} + +// Append a value to the underlying array. +// The returned Array is always the same as the one called. +func (j *Array[T]) Append(v ...T) *Array[T] { + if j.val == nil { + j.val = j.p.newA(uint32(len(v))) + } + j.val = append(j.val, v...) + return j +} + +// Set the underlying value. +func (j *Array[T]) Set(val []T) { + j.val = val +} + +// Recycle the underlying value. +func (j *Array[T]) Recycle() { + if j.val != nil { + j.p.putA(j.val) + j.val = nil + } +} + +// MarshalMsg implements msgp.Marshaler +func (j *Array[T]) MarshalMsg(b []byte) (o []byte, err error) { + if j.val == nil { + return msgp.AppendNil(b), nil + } + if uint64(len(j.val)) > math.MaxUint32 { + return b, errors.New("array: length of array exceeds math.MaxUint32") + } + b = msgp.AppendArrayHeader(b, uint32(len(j.val))) + for _, v := range j.val { + b, err = v.MarshalMsg(b) + if err != nil { + return b, err + } + } + return b, err +} + +// UnmarshalMsg will JSON marshal the value and wrap as a msgp byte array. +// Nil values are supported. +func (j *Array[T]) UnmarshalMsg(bytes []byte) ([]byte, error) { + if bytes, err := msgp.ReadNilBytes(bytes); err == nil { + if j.val != nil { + j.p.putA(j.val) + } + j.val = nil + return bytes, nil + } + l, bytes, err := msgp.ReadArrayHeaderBytes(bytes) + if err != nil { + return bytes, err + } + if j.val == nil { + j.val = j.p.newA(l) + } else { + j.val = j.val[:0] + } + for range l { + v := j.p.newE() + bytes, err = v.UnmarshalMsg(bytes) + if err != nil { + return bytes, err + } + j.val = append(j.val, v) + } + return bytes, nil +} diff --git a/internal/grid/types_test.go b/internal/grid/types_test.go new file mode 100644 index 00000000..f60fecd1 --- /dev/null +++ b/internal/grid/types_test.go @@ -0,0 +1,168 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package grid + +import ( + "reflect" + "testing" + + "github.com/tinylib/msgp/msgp" +) + +func TestMarshalUnmarshalMSS(t *testing.T) { + v := MSS{"abc": "def", "ghi": "jkl"} + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + var v2 MSS + left, err := v2.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) != 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } + if !reflect.DeepEqual(v, v2) { + t.Errorf("MSS: %v != %v", v, v2) + } +} + +func TestMarshalUnmarshalMSSNil(t *testing.T) { + v := MSS(nil) + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + v2 := MSS(make(map[string]string, 1)) + left, err := v2.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) != 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } + if !reflect.DeepEqual(v, v2) { + t.Errorf("MSS: %v != %v", v, v2) + } +} + +func BenchmarkMarshalMsgMSS(b *testing.B) { + v := MSS{"abc": "def", "ghi": "jkl"} + b.ReportAllocs() + + for b.Loop() { + v.MarshalMsg(nil) + } +} + +func BenchmarkAppendMsgMSS(b *testing.B) { + v := MSS{"abc": "def", "ghi": "jkl"} + bts := make([]byte, 0, v.Msgsize()) + bts, _ = v.MarshalMsg(bts[0:0]) + b.SetBytes(int64(len(bts))) + b.ReportAllocs() + + for b.Loop() { + bts, _ = v.MarshalMsg(bts[0:0]) + } +} + +func BenchmarkUnmarshalMSS(b *testing.B) { + v := MSS{"abc": "def", "ghi": "jkl"} + bts, _ := v.MarshalMsg(nil) + b.ReportAllocs() + b.SetBytes(int64(len(bts))) + + for b.Loop() { + _, err := v.UnmarshalMsg(bts) + if err != nil { + b.Fatal(err) + } + } +} + +func TestMarshalUnmarshalBytes(t *testing.T) { + v := Bytes([]byte("abc123123123")) + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + var v2 Bytes + left, err := v2.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) != 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } + if !reflect.DeepEqual(v, v2) { + t.Errorf("MSS: %v != %v", v, v2) + } +} + +func TestMarshalUnmarshalBytesNil(t *testing.T) { + v := Bytes(nil) + bts, err := v.MarshalMsg(nil) + if err != nil { + t.Fatal(err) + } + v2 := Bytes(make([]byte, 1)) + left, err := v2.UnmarshalMsg(bts) + if err != nil { + t.Fatal(err) + } + if len(left) != 0 { + t.Errorf("%d bytes left over after UnmarshalMsg(): %q", len(left), left) + } + + left, err = msgp.Skip(bts) + if err != nil { + t.Fatal(err) + } + if len(left) > 0 { + t.Errorf("%d bytes left over after Skip(): %q", len(left), left) + } + if !reflect.DeepEqual(v, v2) { + t.Errorf("MSS: %v != %v", v, v2) + } +} diff --git a/internal/handlers/forwarder.go b/internal/handlers/forwarder.go new file mode 100644 index 00000000..fd36a6be --- /dev/null +++ b/internal/handlers/forwarder.go @@ -0,0 +1,223 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package handlers + +import ( + "context" + "net" + "net/http" + "net/http/httputil" + "net/url" + "strings" + "time" + + "github.com/minio/minio/internal/bpool" +) + +const defaultFlushInterval = time.Duration(100) * time.Millisecond + +// Forwarder forwards all incoming HTTP requests to configured transport. +type Forwarder struct { + RoundTripper http.RoundTripper + PassHost bool + Logger func(error) + ErrorHandler func(http.ResponseWriter, *http.Request, error) + + // internal variables + rewriter *headerRewriter +} + +// NewForwarder creates an instance of Forwarder based on the provided list of configuration options +func NewForwarder(f *Forwarder) *Forwarder { + f.rewriter = &headerRewriter{} + if f.RoundTripper == nil { + f.RoundTripper = http.DefaultTransport + } + + return f +} + +type bufPool struct { + sz int + pool bpool.Pool[*[]byte] +} + +func (b *bufPool) Put(buf []byte) { + if cap(buf) < b.sz || cap(buf) > b.sz*2 { + // Buffer too small or will likely leak memory after being expanded. + // Drop it. + return + } + b.pool.Put(&buf) +} + +func (b *bufPool) Get() []byte { + bufp := b.pool.Get() + if bufp == nil || cap(*bufp) < b.sz { + return make([]byte, 0, b.sz) + } + return (*bufp)[:b.sz] +} + +func newBufPool(sz int) httputil.BufferPool { + return &bufPool{sz: sz, pool: bpool.Pool[*[]byte]{ + New: func() *[]byte { + buf := make([]byte, sz) + return &buf + }, + }} +} + +// ServeHTTP forwards HTTP traffic using the configured transport +func (f *Forwarder) ServeHTTP(w http.ResponseWriter, inReq *http.Request) { + outReq := new(http.Request) + *outReq = *inReq // includes shallow copies of maps, but we handle this in Director + + revproxy := httputil.ReverseProxy{ + Director: func(req *http.Request) { + f.modifyRequest(req, inReq.URL) + }, + BufferPool: newBufPool(128 << 10), + Transport: f.RoundTripper, + FlushInterval: defaultFlushInterval, + ErrorHandler: f.customErrHandler, + } + + if f.ErrorHandler != nil { + revproxy.ErrorHandler = f.ErrorHandler + } + + revproxy.ServeHTTP(w, outReq) +} + +// customErrHandler is originally implemented to avoid having the following error +// +// `http: proxy error: context canceled` printed by Golang +func (f *Forwarder) customErrHandler(w http.ResponseWriter, r *http.Request, err error) { + if f.Logger != nil && err != context.Canceled { + f.Logger(err) + } + w.WriteHeader(http.StatusBadGateway) +} + +func (f *Forwarder) getURLFromRequest(req *http.Request) *url.URL { + // If the Request was created by Go via a real HTTP request, RequestURI will + // contain the original query string. If the Request was created in code, RequestURI + // will be empty, and we will use the URL object instead + u := req.URL + if req.RequestURI != "" { + parsedURL, err := url.ParseRequestURI(req.RequestURI) + if err == nil { + u = parsedURL + } + } + return u +} + +// copyURL provides update safe copy by avoiding shallow copying User field +func copyURL(i *url.URL) *url.URL { + out := *i + if i.User != nil { + u := *i.User + out.User = &u + } + return &out +} + +// Modify the request to handle the target URL +func (f *Forwarder) modifyRequest(outReq *http.Request, target *url.URL) { + outReq.URL = copyURL(outReq.URL) + outReq.URL.Scheme = target.Scheme + outReq.URL.Host = target.Host + + u := f.getURLFromRequest(outReq) + + outReq.URL.Path = u.Path + outReq.URL.RawPath = u.RawPath + outReq.URL.RawQuery = u.RawQuery + outReq.RequestURI = "" // Outgoing request should not have RequestURI + + // Do not pass client Host header unless requested. + if !f.PassHost { + outReq.Host = target.Host + } + + // TODO: only supports HTTP 1.1 for now. + outReq.Proto = "HTTP/1.1" + outReq.ProtoMajor = 1 + outReq.ProtoMinor = 1 + + f.rewriter.Rewrite(outReq) + + // Disable closeNotify when method GET for http pipelining + if outReq.Method == http.MethodGet { + quietReq := outReq.WithContext(context.Background()) + *outReq = *quietReq + } +} + +// headerRewriter is responsible for removing hop-by-hop headers and setting forwarding headers +type headerRewriter struct{} + +// Clean up IP in case if it is ipv6 address and it has {zone} information in it, like +// "[fe80::d806:a55d:eb1b:49cc%vEthernet (vmxnet3 Ethernet Adapter - Virtual Switch)]:64692" +func ipv6fix(clientIP string) string { + return strings.Split(clientIP, "%")[0] +} + +func (rw *headerRewriter) Rewrite(req *http.Request) { + if clientIP, _, err := net.SplitHostPort(req.RemoteAddr); err == nil { + clientIP = ipv6fix(clientIP) + if req.Header.Get(xRealIP) == "" { + req.Header.Set(xRealIP, clientIP) + } + } + + xfProto := req.Header.Get(xForwardedProto) + if xfProto == "" { + if req.TLS != nil { + req.Header.Set(xForwardedProto, "https") + } else { + req.Header.Set(xForwardedProto, "http") + } + } + + if xfPort := req.Header.Get(xForwardedPort); xfPort == "" { + req.Header.Set(xForwardedPort, forwardedPort(req)) + } + + if xfHost := req.Header.Get(xForwardedHost); xfHost == "" && req.Host != "" { + req.Header.Set(xForwardedHost, req.Host) + } +} + +func forwardedPort(req *http.Request) string { + if req == nil { + return "" + } + + if _, port, err := net.SplitHostPort(req.Host); err == nil && port != "" { + return port + } + + if req.TLS != nil { + return "443" + } + + return "80" +} diff --git a/internal/handlers/proxy.go b/internal/handlers/proxy.go new file mode 100644 index 00000000..f095b1fd --- /dev/null +++ b/internal/handlers/proxy.go @@ -0,0 +1,153 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Originally from https://github.com/gorilla/handlers with following license +// https://raw.githubusercontent.com/gorilla/handlers/master/LICENSE, forked +// and heavily modified for MinIO's internal needs. + +package handlers + +import ( + "net" + "net/http" + "regexp" + "strings" + + "github.com/minio/minio/internal/config" + "github.com/minio/pkg/v3/env" +) + +var ( + // De-facto standard header keys. + xForwardedFor = http.CanonicalHeaderKey("X-Forwarded-For") + xForwardedHost = http.CanonicalHeaderKey("X-Forwarded-Host") + xForwardedPort = http.CanonicalHeaderKey("X-Forwarded-Port") + xForwardedProto = http.CanonicalHeaderKey("X-Forwarded-Proto") + xForwardedScheme = http.CanonicalHeaderKey("X-Forwarded-Scheme") + xRealIP = http.CanonicalHeaderKey("X-Real-IP") +) + +var ( + // RFC7239 defines a new "Forwarded: " header designed to replace the + // existing use of X-Forwarded-* headers. + // e.g. Forwarded: for=192.0.2.60;proto=https;by=203.0.113.43 + forwarded = http.CanonicalHeaderKey("Forwarded") + // Allows for a sub-match of the first value after 'for=' to the next + // comma, semi-colon or space. The match is case-insensitive. + forRegex = regexp.MustCompile(`(?i)(?:for=)([^(;|,| )]+)(.*)`) + // Allows for a sub-match for the first instance of scheme (http|https) + // prefixed by 'proto='. The match is case-insensitive. + protoRegex = regexp.MustCompile(`(?i)^(;|,| )+(?:proto=)(https|http)`) +) + +// Used to disable all processing of the X-Forwarded-For header in source IP discovery. +var enableXFFHeader = env.Get("_MINIO_API_XFF_HEADER", config.EnableOn) == config.EnableOn + +// GetSourceScheme retrieves the scheme from the X-Forwarded-Proto and RFC7239 +// Forwarded headers (in that order). +func GetSourceScheme(r *http.Request) string { + var scheme string + + // Retrieve the scheme from X-Forwarded-Proto. + if proto := r.Header.Get(xForwardedProto); proto != "" { + scheme = strings.ToLower(proto) + } else if proto = r.Header.Get(xForwardedScheme); proto != "" { + scheme = strings.ToLower(proto) + } else if proto := r.Header.Get(forwarded); proto != "" { + // match should contain at least two elements if the protocol was + // specified in the Forwarded header. The first element will always be + // the 'for=', which we ignore, subsequently we proceed to look for + // 'proto=' which should precede right after `for=` if not + // we simply ignore the values and return empty. This is in line + // with the approach we took for returning first ip from multiple + // params. + if match := forRegex.FindStringSubmatch(proto); len(match) > 1 { + if match = protoRegex.FindStringSubmatch(match[2]); len(match) > 1 { + scheme = strings.ToLower(match[2]) + } + } + } + + return scheme +} + +// GetSourceIPFromHeaders retrieves the IP from the X-Forwarded-For, X-Real-IP +// and RFC7239 Forwarded headers (in that order) +func GetSourceIPFromHeaders(r *http.Request) string { + var addr string + + if enableXFFHeader { + if fwd := r.Header.Get(xForwardedFor); fwd != "" { + // Only grab the first (client) address. Note that '192.168.0.1, + // 10.1.1.1' is a valid key for X-Forwarded-For where addresses after + // the first may represent forwarding proxies earlier in the chain. + s := strings.Index(fwd, ", ") + if s == -1 { + s = len(fwd) + } + addr = fwd[:s] + } + } + + if addr == "" { + if fwd := r.Header.Get(xRealIP); fwd != "" { + // X-Real-IP should only contain one IP address (the client making the + // request). + addr = fwd + } else if fwd := r.Header.Get(forwarded); fwd != "" { + // match should contain at least two elements if the protocol was + // specified in the Forwarded header. The first element will always be + // the 'for=' capture, which we ignore. In the case of multiple IP + // addresses (for=8.8.8.8, 8.8.4.4, 172.16.1.20 is valid) we only + // extract the first, which should be the client IP. + if match := forRegex.FindStringSubmatch(fwd); len(match) > 1 { + // IPv6 addresses in Forwarded headers are quoted-strings. We strip + // these quotes. + addr = strings.Trim(match[1], `"`) + } + } + } + + return addr +} + +// GetSourceIPRaw retrieves the IP from the request headers +// and falls back to r.RemoteAddr when necessary. +// however returns without bracketing. +func GetSourceIPRaw(r *http.Request) string { + addr := GetSourceIPFromHeaders(r) + if addr == "" { + addr = r.RemoteAddr + } + + // Default to remote address if headers not set. + raddr, _, _ := net.SplitHostPort(addr) + if raddr == "" { + return addr + } + return raddr +} + +// GetSourceIP retrieves the IP from the request headers +// and falls back to r.RemoteAddr when necessary. +func GetSourceIP(r *http.Request) string { + addr := GetSourceIPRaw(r) + if strings.ContainsRune(addr, ':') { + return "[" + addr + "]" + } + return addr +} diff --git a/internal/handlers/proxy_test.go b/internal/handlers/proxy_test.go new file mode 100644 index 00000000..952e7574 --- /dev/null +++ b/internal/handlers/proxy_test.go @@ -0,0 +1,110 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package handlers + +import ( + "net/http" + "testing" +) + +type headerTest struct { + key string // header key + val string // header val + expected string // expected result +} + +func TestGetScheme(t *testing.T) { + headers := []headerTest{ + {xForwardedProto, "https", "https"}, + {xForwardedProto, "http", "http"}, + {xForwardedProto, "HTTP", "http"}, + {xForwardedScheme, "https", "https"}, + {xForwardedScheme, "http", "http"}, + {xForwardedScheme, "HTTP", "http"}, + {forwarded, `For="[2001:db8:cafe::17]:4711`, ""}, // No proto + {forwarded, `for=192.0.2.43, for=198.51.100.17;proto=https`, ""}, // Multiple params, will be empty. + {forwarded, `for=172.32.10.15; proto=https;by=127.0.0.1;`, "https"}, // Space before proto + {forwarded, `for=192.0.2.60;proto=http;by=203.0.113.43`, "http"}, // Multiple params + } + for _, v := range headers { + req := &http.Request{ + Header: http.Header{ + v.key: []string{v.val}, + }, + } + res := GetSourceScheme(req) + if res != v.expected { + t.Errorf("wrong header for %s: got %s want %s", v.key, res, + v.expected) + } + } +} + +// TestGetSourceIP - check the source ip of a request is parsed correctly. +func TestGetSourceIP(t *testing.T) { + headers := []headerTest{ + {xForwardedFor, "8.8.8.8", "8.8.8.8"}, // Single address + {xForwardedFor, "8.8.8.8, 8.8.4.4", "8.8.8.8"}, // Multiple + {xForwardedFor, "", ""}, // None + {xRealIP, "8.8.8.8", "8.8.8.8"}, // Single address + {xRealIP, "[2001:db8:cafe::17]:4711", "[2001:db8:cafe::17]"}, // IPv6 address + {xRealIP, "", ""}, // None + {forwarded, `for="_gazonk"`, "_gazonk"}, // Hostname + {forwarded, `For="[2001:db8:cafe::17]:4711`, `[2001:db8:cafe::17]`}, // IPv6 address + {forwarded, `for=192.0.2.60;proto=http;by=203.0.113.43`, `192.0.2.60`}, // Multiple params + {forwarded, `for=192.0.2.43, for=198.51.100.17`, "192.0.2.43"}, // Multiple params + {forwarded, `for="workstation.local",for=198.51.100.17`, "workstation.local"}, // Hostname + } + + for _, v := range headers { + req := &http.Request{ + Header: http.Header{ + v.key: []string{v.val}, + }, + } + res := GetSourceIP(req) + if res != v.expected { + t.Errorf("wrong header for %s: got %s want %s", v.key, res, + v.expected) + } + } +} + +func TestXFFDisabled(t *testing.T) { + req := &http.Request{ + Header: http.Header{ + xForwardedFor: []string{"8.8.8.8"}, + xRealIP: []string{"1.1.1.1"}, + }, + } + // When X-Forwarded-For and X-Real-IP headers are both present, X-Forwarded-For takes precedence. + res := GetSourceIP(req) + if res != "8.8.8.8" { + t.Errorf("wrong header, xff takes precedence: got %s, want: 8.8.8.8", res) + } + + // When explicitly disabled, the XFF header is ignored and X-Real-IP is used. + enableXFFHeader = false + defer func() { + enableXFFHeader = true + }() + res = GetSourceIP(req) + if res != "1.1.1.1" { + t.Errorf("wrong header, xff is disabled: got %s, want: 1.1.1.1", res) + } +} diff --git a/internal/hash/checker.go b/internal/hash/checker.go new file mode 100644 index 00000000..df7a8918 --- /dev/null +++ b/internal/hash/checker.go @@ -0,0 +1,70 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package hash + +import ( + "bytes" + "errors" + "hash" + "io" + + "github.com/minio/minio/internal/ioutil" +) + +// Checker allows to verify the checksum of a reader. +type Checker struct { + c io.Closer + r io.Reader + h hash.Hash + + want []byte +} + +// NewChecker ensures that content with the specified length is read from rc. +// Calling Close on this will close upstream. +func NewChecker(rc io.ReadCloser, h hash.Hash, wantSum []byte, length int64) *Checker { + return &Checker{c: rc, r: ioutil.HardLimitReader(rc, length), h: h, want: wantSum} +} + +// Read satisfies io.Reader +func (c Checker) Read(p []byte) (n int, err error) { + n, err = c.r.Read(p) + if n > 0 { + c.h.Write(p[:n]) + } + if errors.Is(err, io.EOF) { + got := c.h.Sum(nil) + if !bytes.Equal(got, c.want) { + return n, ErrInvalidChecksum + } + return n, err + } + return n, err +} + +// Close satisfies io.Closer +func (c Checker) Close() error { + err := c.c.Close() + if err == nil { + got := c.h.Sum(nil) + if !bytes.Equal(got, c.want) { + return ErrInvalidChecksum + } + } + return err +} diff --git a/internal/hash/checksum.go b/internal/hash/checksum.go new file mode 100644 index 00000000..fbf62fd8 --- /dev/null +++ b/internal/hash/checksum.go @@ -0,0 +1,752 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package hash + +import ( + "bytes" + "context" + "crypto/sha1" + "encoding/base64" + "encoding/binary" + "fmt" + "hash" + "hash/crc32" + "hash/crc64" + "net/http" + "strconv" + "strings" + + "github.com/minio/minio/internal/hash/sha256" + xhttp "github.com/minio/minio/internal/http" + "github.com/minio/minio/internal/logger" +) + +func hashLogIf(ctx context.Context, err error) { + logger.LogIf(ctx, "hash", err) +} + +// MinIOMultipartChecksum is as metadata on multipart uploads to indicate checksum type. +const MinIOMultipartChecksum = "x-minio-multipart-checksum" + +// MinIOMultipartChecksumType is as metadata on multipart uploads to indicate checksum type. +const MinIOMultipartChecksumType = "x-minio-multipart-checksum-type" + +// ChecksumType contains information about the checksum type. +type ChecksumType uint32 + +const ( + + // ChecksumTrailing indicates the checksum will be sent in the trailing header. + // Another checksum type will be set. + ChecksumTrailing ChecksumType = 1 << iota + + // ChecksumSHA256 indicates a SHA256 checksum. + ChecksumSHA256 + // ChecksumSHA1 indicates a SHA-1 checksum. + ChecksumSHA1 + // ChecksumCRC32 indicates a CRC32 checksum with IEEE table. + ChecksumCRC32 + // ChecksumCRC32C indicates a CRC32 checksum with Castagnoli table. + ChecksumCRC32C + // ChecksumInvalid indicates an invalid checksum. + ChecksumInvalid + // ChecksumMultipart indicates the checksum is from a multipart upload. + ChecksumMultipart + // ChecksumIncludesMultipart indicates the checksum also contains part checksums. + ChecksumIncludesMultipart + // ChecksumCRC64NVME indicates CRC64 with 0xad93d23594c93659 polynomial. + ChecksumCRC64NVME + // ChecksumFullObject indicates the checksum is of the full object, + // not checksum of checksums. Should only be set on ChecksumMultipart + ChecksumFullObject + + // ChecksumNone indicates no checksum. + ChecksumNone ChecksumType = 0 + + baseTypeMask = ChecksumSHA256 | ChecksumSHA1 | ChecksumCRC32 | ChecksumCRC32C | ChecksumCRC64NVME +) + +// BaseChecksumTypes is a list of all the base checksum types. +var BaseChecksumTypes = []ChecksumType{ChecksumSHA256, ChecksumSHA1, ChecksumCRC32, ChecksumCRC64NVME, ChecksumCRC32C} + +// Checksum is a type and base 64 encoded value. +type Checksum struct { + Type ChecksumType + Encoded string + Raw []byte + WantParts int +} + +// Is returns if c is all of t. +func (c ChecksumType) Is(t ChecksumType) bool { + if t == ChecksumNone { + return c == ChecksumNone + } + return c&t == t +} + +// Base returns the base checksum (if any) +func (c ChecksumType) Base() ChecksumType { + return c & baseTypeMask +} + +// Key returns the header key. +// returns empty string if invalid or none. +func (c ChecksumType) Key() string { + switch { + case c.Is(ChecksumCRC32): + return xhttp.AmzChecksumCRC32 + case c.Is(ChecksumCRC32C): + return xhttp.AmzChecksumCRC32C + case c.Is(ChecksumSHA1): + return xhttp.AmzChecksumSHA1 + case c.Is(ChecksumSHA256): + return xhttp.AmzChecksumSHA256 + case c.Is(ChecksumCRC64NVME): + return xhttp.AmzChecksumCRC64NVME + } + return "" +} + +// RawByteLen returns the size of the un-encoded checksum. +func (c ChecksumType) RawByteLen() int { + switch { + case c.Is(ChecksumCRC32): + return 4 + case c.Is(ChecksumCRC32C): + return 4 + case c.Is(ChecksumSHA1): + return sha1.Size + case c.Is(ChecksumSHA256): + return sha256.Size + case c.Is(ChecksumCRC64NVME): + return crc64.Size + } + return 0 +} + +// IsSet returns whether the type is valid and known. +func (c ChecksumType) IsSet() bool { + return !c.Is(ChecksumInvalid) && !c.Base().Is(ChecksumNone) +} + +// ChecksumStringToType is like NewChecksumType but without the `mode` +func ChecksumStringToType(alg string) ChecksumType { + switch strings.ToUpper(alg) { + case "CRC32": + return ChecksumCRC32 + case "CRC32C": + return ChecksumCRC32C + case "SHA1": + return ChecksumSHA1 + case "SHA256": + return ChecksumSHA256 + case "CRC64NVME": + // AWS seems to ignore full value, and just assume it. + return ChecksumCRC64NVME + case "": + return ChecksumNone + } + return ChecksumInvalid +} + +// NewChecksumType returns a checksum type based on the algorithm string and obj type. +func NewChecksumType(alg, objType string) ChecksumType { + full := ChecksumFullObject + switch objType { + case xhttp.AmzChecksumTypeFullObject: + case xhttp.AmzChecksumTypeComposite, "": + full = 0 + default: + return ChecksumInvalid + } + + switch strings.ToUpper(alg) { + case "CRC32": + return ChecksumCRC32 | full + case "CRC32C": + return ChecksumCRC32C | full + case "SHA1": + if full != 0 { + return ChecksumInvalid + } + return ChecksumSHA1 + case "SHA256": + if full != 0 { + return ChecksumInvalid + } + return ChecksumSHA256 + case "CRC64NVME": + // AWS seems to ignore full value, and just assume it. + return ChecksumCRC64NVME + case "": + if full != 0 { + return ChecksumInvalid + } + return ChecksumNone + } + return ChecksumInvalid +} + +// NewChecksumHeader returns a checksum type based on the algorithm string. +func NewChecksumHeader(h http.Header) ChecksumType { + return NewChecksumType(h.Get(xhttp.AmzChecksumAlgo), h.Get(xhttp.AmzChecksumType)) +} + +// String returns the type as a string. +func (c ChecksumType) String() string { + switch { + case c.Is(ChecksumCRC32): + return "CRC32" + case c.Is(ChecksumCRC32C): + return "CRC32C" + case c.Is(ChecksumSHA1): + return "SHA1" + case c.Is(ChecksumSHA256): + return "SHA256" + case c.Is(ChecksumCRC64NVME): + return "CRC64NVME" + case c.Is(ChecksumNone): + return "" + } + return "invalid" +} + +// StringFull returns the type and all flags as a string. +func (c ChecksumType) StringFull() string { + out := []string{c.String()} + if c.Is(ChecksumMultipart) { + out = append(out, "MULTIPART") + } + if c.Is(ChecksumIncludesMultipart) { + out = append(out, "INCLUDESMP") + } + if c.Is(ChecksumTrailing) { + out = append(out, "TRAILING") + } + if c.Is(ChecksumFullObject) { + out = append(out, "FULLOBJ") + } + return strings.Join(out, "|") +} + +// FullObjectRequested will return if the checksum type indicates full object checksum was requested. +func (c ChecksumType) FullObjectRequested() bool { + return c&(ChecksumFullObject) == ChecksumFullObject || c.Is(ChecksumCRC64NVME) +} + +// IsMultipartComposite returns true if the checksum is multipart and full object was not requested. +func (c ChecksumType) IsMultipartComposite() bool { + return c.Is(ChecksumMultipart) && !c.FullObjectRequested() +} + +// ObjType returns a string to return as x-amz-checksum-type. +func (c ChecksumType) ObjType() string { + if c.FullObjectRequested() { + return xhttp.AmzChecksumTypeFullObject + } + if c.IsMultipartComposite() { + return xhttp.AmzChecksumTypeComposite + } + if !c.Is(ChecksumMultipart) { + return xhttp.AmzChecksumTypeFullObject + } + if c.IsSet() { + return xhttp.AmzChecksumTypeComposite + } + return "" +} + +// CanMerge will return if the checksum type indicates that checksums can be merged. +func (c ChecksumType) CanMerge() bool { + return c.Is(ChecksumCRC64NVME) || c.Is(ChecksumCRC32C) || c.Is(ChecksumCRC32) +} + +// Hasher returns a hasher corresponding to the checksum type. +// Returns nil if no checksum. +func (c ChecksumType) Hasher() hash.Hash { + switch { + case c.Is(ChecksumCRC32): + return crc32.NewIEEE() + case c.Is(ChecksumCRC32C): + return crc32.New(crc32.MakeTable(crc32.Castagnoli)) + case c.Is(ChecksumSHA1): + return sha1.New() + case c.Is(ChecksumSHA256): + return sha256.New() + case c.Is(ChecksumCRC64NVME): + return crc64.New(crc64Table) + } + return nil +} + +// Trailing return whether the checksum is trailing. +func (c ChecksumType) Trailing() bool { + return c.Is(ChecksumTrailing) +} + +// NewChecksumFromData returns a new Checksum, using specified algorithm type on data. +func NewChecksumFromData(t ChecksumType, data []byte) *Checksum { + if !t.IsSet() { + return nil + } + h := t.Hasher() + h.Write(data) + raw := h.Sum(nil) + c := Checksum{Type: t, Encoded: base64.StdEncoding.EncodeToString(raw), Raw: raw} + if !c.Valid() { + return nil + } + return &c +} + +// ReadCheckSums will read checksums from b and return them. +// Returns whether this is (part of) a multipart checksum. +func ReadCheckSums(b []byte, part int) (cs map[string]string, isMP bool) { + res := make(map[string]string, 1) + for len(b) > 0 { + t, n := binary.Uvarint(b) + if n < 0 { + break + } + b = b[n:] + + typ := ChecksumType(t) + length := typ.RawByteLen() + if length == 0 || len(b) < length { + break + } + + cs := base64.StdEncoding.EncodeToString(b[:length]) + b = b[length:] + if typ.Is(ChecksumMultipart) { + isMP = true + t, n := binary.Uvarint(b) + if n < 0 { + break + } + if !typ.FullObjectRequested() { + cs = fmt.Sprintf("%s-%d", cs, t) + } + b = b[n:] + if part > 0 { + cs = "" + } + if typ.Is(ChecksumIncludesMultipart) { + wantLen := int(t) * length + if len(b) < wantLen { + break + } + // Read part checksum + if part > 0 && uint64(part) <= t { + offset := (part - 1) * length + partCs := b[offset:] + cs = base64.StdEncoding.EncodeToString(partCs[:length]) + } + b = b[wantLen:] + } + } else if part > 1 { + // For non-multipart, checksum is part 1. + cs = "" + } + if cs != "" { + res[typ.String()] = cs + if ckType := typ.ObjType(); ckType != "" { + res[xhttp.AmzChecksumType] = ckType + } + } + } + if len(res) == 0 { + res = nil + } + return res, isMP +} + +// ReadPartCheckSums will read all part checksums from b and return them. +func ReadPartCheckSums(b []byte) (res []map[string]string) { + for len(b) > 0 { + t, n := binary.Uvarint(b) + if n <= 0 { + break + } + b = b[n:] + + typ := ChecksumType(t) + length := typ.RawByteLen() + if length == 0 || len(b) < length { + break + } + // Skip main checksum + b = b[length:] + parts, n := binary.Uvarint(b) + if n <= 0 { + break + } + if !typ.Is(ChecksumIncludesMultipart) { + continue + } + + if len(res) == 0 { + res = make([]map[string]string, parts) + } + b = b[n:] + for part := 0; part < int(parts); part++ { + if len(b) < length { + break + } + // Read part checksum + cs := base64.StdEncoding.EncodeToString(b[:length]) + b = b[length:] + if res[part] == nil { + res[part] = make(map[string]string, 1) + } + res[part][typ.String()] = cs + } + } + return res +} + +// NewChecksumWithType is similar to NewChecksumString but expects input algo of ChecksumType. +func NewChecksumWithType(alg ChecksumType, value string) *Checksum { + if !alg.IsSet() { + return nil + } + wantParts := 0 + if strings.ContainsRune(value, '-') { + valSplit := strings.Split(value, "-") + if len(valSplit) != 2 { + return nil + } + value = valSplit[0] + nParts, err := strconv.Atoi(valSplit[1]) + if err != nil { + return nil + } + alg |= ChecksumMultipart + wantParts = nParts + } + bvalue, err := base64.StdEncoding.DecodeString(value) + if err != nil { + return nil + } + c := Checksum{Type: alg, Encoded: value, Raw: bvalue, WantParts: wantParts} + if !c.Valid() { + return nil + } + return &c +} + +// NewChecksumString returns a new checksum from specified algorithm and base64 encoded value. +func NewChecksumString(alg, value string) *Checksum { + return NewChecksumWithType(NewChecksumType(alg, ""), value) +} + +// AppendTo will append the checksum to b. +// 'parts' is used when checksum has ChecksumMultipart set. +// ReadCheckSums reads the values back. +func (c *Checksum) AppendTo(b []byte, parts []byte) []byte { + if c == nil { + return nil + } + var tmp [binary.MaxVarintLen32]byte + n := binary.PutUvarint(tmp[:], uint64(c.Type)) + crc := c.Raw + if c.Type.Trailing() { + // When we serialize we don't care if it was trailing. + c.Type ^= ChecksumTrailing + } + if len(crc) != c.Type.RawByteLen() { + return b + } + b = append(b, tmp[:n]...) + b = append(b, crc...) + if c.Type.Is(ChecksumMultipart) { + var checksums int + if c.WantParts > 0 && !c.Type.Is(ChecksumIncludesMultipart) { + checksums = c.WantParts + } + // Ensure we don't divide by 0: + if c.Type.RawByteLen() == 0 || len(parts)%c.Type.RawByteLen() != 0 { + hashLogIf(context.Background(), fmt.Errorf("internal error: Unexpected checksum length: %d, each checksum %d", len(parts), c.Type.RawByteLen())) + checksums = 0 + parts = nil + } else if len(parts) > 0 { + checksums = len(parts) / c.Type.RawByteLen() + } + if !c.Type.Is(ChecksumIncludesMultipart) { + parts = nil + } + n := binary.PutUvarint(tmp[:], uint64(checksums)) + b = append(b, tmp[:n]...) + if len(parts) > 0 { + b = append(b, parts...) + } + } + return b +} + +// ChecksumFromBytes reconstructs a Checksum struct from the serialized bytes created in AppendTo() +// Returns nil if the bytes are invalid or empty. +// AppendTo() can append a serialized Checksum to another already-serialized Checksum, +// however, in practice, we only use one at a time. +// ChecksumFromBytes only returns the first one and no part checksums. +func ChecksumFromBytes(b []byte) *Checksum { + if len(b) == 0 { + return nil + } + + // Read checksum type + t, n := binary.Uvarint(b) + if n <= 0 { + return nil + } + b = b[n:] + + typ := ChecksumType(t) + length := typ.RawByteLen() + if length == 0 || len(b) < length { + return nil + } + + // Read raw checksum bytes + raw := make([]byte, length) + copy(raw, b[:length]) + b = b[length:] + + c := &Checksum{ + Type: typ, + Raw: raw, + Encoded: base64.StdEncoding.EncodeToString(raw), + } + + // Handle multipart checksums + if typ.Is(ChecksumMultipart) { + parts, n := binary.Uvarint(b) + if n <= 0 { + return nil + } + b = b[n:] + + c.WantParts = int(parts) + + if typ.Is(ChecksumIncludesMultipart) { + wantLen := int(parts) * length + if len(b) < wantLen { + return nil + } + } + } + + if !c.Valid() { + return nil + } + + return c +} + +// Valid returns whether checksum is valid. +func (c Checksum) Valid() bool { + if c.Type == ChecksumInvalid { + return false + } + if len(c.Encoded) == 0 || c.Type.Trailing() { + return c.Type.Is(ChecksumNone) || c.Type.Trailing() + } + return c.Type.RawByteLen() == len(c.Raw) +} + +// Matches returns whether given content matches c. +func (c Checksum) Matches(content []byte, parts int) error { + if len(c.Encoded) == 0 { + return nil + } + hasher := c.Type.Hasher() + _, err := hasher.Write(content) + if err != nil { + return err + } + sum := hasher.Sum(nil) + if c.WantParts > 0 && c.WantParts != parts { + return ChecksumMismatch{ + Want: fmt.Sprintf("%s-%d", c.Encoded, c.WantParts), + Got: fmt.Sprintf("%s-%d", base64.StdEncoding.EncodeToString(sum), parts), + } + } + + if !bytes.Equal(sum, c.Raw) { + return ChecksumMismatch{ + Want: c.Encoded, + Got: base64.StdEncoding.EncodeToString(sum), + } + } + return nil +} + +// AsMap returns the checksum as a map[string]string. +func (c *Checksum) AsMap() map[string]string { + if c == nil || !c.Valid() { + return nil + } + return map[string]string{ + c.Type.String(): c.Encoded, + xhttp.AmzChecksumType: c.Type.ObjType(), + } +} + +// Equal returns whether two checksum structs are equal in all their fields. +func (c *Checksum) Equal(s *Checksum) bool { + if c == nil || s == nil { + return c == s + } + return c.Type == s.Type && + c.Encoded == s.Encoded && + bytes.Equal(c.Raw, s.Raw) && + c.WantParts == s.WantParts +} + +// TransferChecksumHeader will transfer any checksum value that has been checked. +// If checksum was trailing, they must have been added to r.Trailer. +func TransferChecksumHeader(w http.ResponseWriter, r *http.Request) { + c, err := GetContentChecksum(r.Header) + if err != nil || c == nil { + return + } + t, s := c.Type, c.Encoded + if !c.Type.IsSet() { + return + } + if c.Type.Is(ChecksumTrailing) { + val := r.Trailer.Get(t.Key()) + if val != "" { + w.Header().Set(t.Key(), val) + } + return + } + w.Header().Set(t.Key(), s) +} + +// AddChecksumHeader will transfer any checksum value that has been checked. +func AddChecksumHeader(w http.ResponseWriter, c map[string]string) { + for k, v := range c { + if k == xhttp.AmzChecksumType { + w.Header().Set(xhttp.AmzChecksumType, v) + continue + } + cksum := NewChecksumString(k, v) + if cksum == nil { + continue + } + if cksum.Valid() { + w.Header().Set(cksum.Type.Key(), v) + } + } +} + +// GetContentChecksum returns content checksum. +// Returns ErrInvalidChecksum if so. +// Returns nil, nil if no checksum. +func GetContentChecksum(h http.Header) (*Checksum, error) { + if trailing := h.Values(xhttp.AmzTrailer); len(trailing) > 0 { + var res *Checksum + for _, header := range trailing { + var duplicates bool + for _, t := range BaseChecksumTypes { + if strings.EqualFold(t.Key(), header) { + duplicates = res != nil + res = NewChecksumWithType(t|ChecksumTrailing, "") + } + } + if duplicates { + return nil, ErrInvalidChecksum + } + } + if res != nil { + switch h.Get(xhttp.AmzChecksumType) { + case xhttp.AmzChecksumTypeFullObject: + if !res.Type.CanMerge() { + return nil, ErrInvalidChecksum + } + res.Type |= ChecksumFullObject + case xhttp.AmzChecksumTypeComposite, "": + default: + return nil, ErrInvalidChecksum + } + return res, nil + } + } + t, s := getContentChecksum(h) + if t == ChecksumNone { + if s == "" { + return nil, nil + } + return nil, ErrInvalidChecksum + } + cksum := NewChecksumWithType(t, s) + if cksum == nil { + return nil, ErrInvalidChecksum + } + return cksum, nil +} + +// getContentChecksum returns content checksum type and value. +// Returns ChecksumInvalid if so. +func getContentChecksum(h http.Header) (t ChecksumType, s string) { + t = ChecksumNone + alg := h.Get(xhttp.AmzChecksumAlgo) + if alg != "" { + t |= NewChecksumHeader(h) + if h.Get(xhttp.AmzChecksumType) == xhttp.AmzChecksumTypeFullObject { + if !t.CanMerge() { + return ChecksumInvalid, "" + } + t |= ChecksumFullObject + } + if t.IsSet() { + hdr := t.Key() + if s = h.Get(hdr); s == "" { + return ChecksumNone, "" + } + } + return t, s + } + checkType := func(c ChecksumType) { + if got := h.Get(c.Key()); got != "" { + // If already set, invalid + if t != ChecksumNone { + t = ChecksumInvalid + s = "" + } else { + t = c + s = got + } + if h.Get(xhttp.AmzChecksumType) == xhttp.AmzChecksumTypeFullObject { + if !t.CanMerge() { + t = ChecksumInvalid + s = "" + return + } + t |= ChecksumFullObject + } + return + } + } + for _, t := range BaseChecksumTypes { + checkType(t) + } + return t, s +} diff --git a/internal/hash/checksum_test.go b/internal/hash/checksum_test.go new file mode 100644 index 00000000..50480379 --- /dev/null +++ b/internal/hash/checksum_test.go @@ -0,0 +1,205 @@ +// Copyright (c) 2015-2025 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package hash + +import ( + "net/http/httptest" + "testing" + + xhttp "github.com/minio/minio/internal/http" +) + +// TestChecksumAddToHeader tests that adding and retrieving a checksum on a header works +func TestChecksumAddToHeader(t *testing.T) { + tests := []struct { + name string + checksum ChecksumType + fullobj bool + wantErr bool + }{ + {"CRC32-composite", ChecksumCRC32, false, false}, + {"CRC32-full-object", ChecksumCRC32, true, false}, + {"CRC32C-composite", ChecksumCRC32C, false, false}, + {"CRC32C-full-object", ChecksumCRC32C, true, false}, + {"CRC64NVME-full-object", ChecksumCRC64NVME, false, false}, // CRC64NVME is always full object + {"ChecksumSHA1-composite", ChecksumSHA1, false, false}, + {"ChecksumSHA256-composite", ChecksumSHA256, false, false}, + {"ChecksumSHA1-full-object", ChecksumSHA1, true, true}, // SHA1 does not support full object + {"ChecksumSHA256-full-object", ChecksumSHA256, true, true}, // SHA256 does not support full object + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Skip invalid cases where SHA1 or SHA256 is used with full object + if (tt.checksum.Is(ChecksumSHA1) || tt.checksum.Is(ChecksumSHA256)) && tt.fullobj { + // Validate that NewChecksumType correctly marks these as invalid + alg := tt.checksum.String() + typ := NewChecksumType(alg, xhttp.AmzChecksumTypeFullObject) + if !typ.Is(ChecksumInvalid) { + t.Fatalf("Expected ChecksumInvalid for %s with full object, got %s", tt.name, typ.StringFull()) + } + return + } + myData := []byte("this-is-a-checksum-data-test") + chksm := NewChecksumFromData(tt.checksum, myData) + if chksm == nil { + t.Fatalf("NewChecksumFromData failed for %s", tt.name) + } + if tt.fullobj { + chksm.Type |= ChecksumFullObject + } + + // CRC64NVME is always full object + if chksm.Type.Base().Is(ChecksumCRC64NVME) { + chksm.Type |= ChecksumFullObject + } + + // Prepare the checksum map with appropriate headers + m := chksm.AsMap() + m[xhttp.AmzChecksumAlgo] = chksm.Type.String() // Set the algorithm explicitly + if chksm.Type.FullObjectRequested() { + m[xhttp.AmzChecksumType] = xhttp.AmzChecksumTypeFullObject + } else { + m[xhttp.AmzChecksumType] = xhttp.AmzChecksumTypeComposite + } + + w := httptest.NewRecorder() + AddChecksumHeader(w, m) + gotChksm, err := GetContentChecksum(w.Result().Header) + if tt.wantErr { + if err == nil { + t.Fatalf("Expected error for %s, got none", tt.name) + } + return + } + if err != nil { + t.Fatalf("GetContentChecksum failed for %s: %v", tt.name, err) + } + + if gotChksm == nil { + t.Fatalf("Got nil checksum for %s", tt.name) + } + // Compare the full checksum structs + if !chksm.Equal(gotChksm) { + t.Errorf("Checksum mismatch for %s: expected %+v, got %+v", tt.name, chksm, gotChksm) + } + // Verify the checksum type + expectedType := chksm.Type + if gotChksm.Type != expectedType { + t.Errorf("Type mismatch for %s: expected %s, got %s", tt.name, expectedType.StringFull(), gotChksm.Type.StringFull()) + } + }) + } +} + +// TestChecksumSerializeDeserialize checks AppendTo can be reversed by ChecksumFromBytes +func TestChecksumSerializeDeserialize(t *testing.T) { + myData := []byte("this-is-a-checksum-data-test") + chksm := NewChecksumFromData(ChecksumCRC32, myData) + if chksm == nil { + t.Fatal("NewChecksumFromData returned nil") + } + // Serialize the checksum to bytes + b := chksm.AppendTo(nil, nil) + if b == nil { + t.Fatal("AppendTo returned nil") + } + + // Deserialize the checksum from bytes + chksmOut := ChecksumFromBytes(b) + if chksmOut == nil { + t.Fatal("ChecksumFromBytes returned nil") + } + + // Assert new checksum matches the content + matchError := chksmOut.Matches(myData, 0) + if matchError != nil { + t.Fatalf("Checksum mismatch on chksmOut: %v", matchError) + } + + // Assert they are exactly equal + if !chksmOut.Equal(chksm) { + t.Fatalf("Checksum mismatch: expected %+v, got %+v", chksm, chksmOut) + } +} + +// TestChecksumSerializeDeserializeMultiPart checks AppendTo can be reversed by ChecksumFromBytes +// for multipart checksum +func TestChecksumSerializeDeserializeMultiPart(t *testing.T) { + // Create dummy data that we'll split into 3 parts + dummyData := []byte("The quick brown fox jumps over the lazy dog. " + + "Pack my box with five dozen brown eggs. " + + "Have another go it will all make sense in the end!") + + // Split data into 3 parts + partSize := len(dummyData) / 3 + part1Data := dummyData[0:partSize] + part2Data := dummyData[partSize : 2*partSize] + part3Data := dummyData[2*partSize:] + + // Calculate CRC32C checksum for each part using NewChecksumFromData + checksumType := ChecksumCRC32C + + part1Checksum := NewChecksumFromData(checksumType, part1Data) + part2Checksum := NewChecksumFromData(checksumType, part2Data) + part3Checksum := NewChecksumFromData(checksumType, part3Data) + + // Combine the raw checksums (this is what happens in CompleteMultipartUpload) + var checksumCombined []byte + checksumCombined = append(checksumCombined, part1Checksum.Raw...) + checksumCombined = append(checksumCombined, part2Checksum.Raw...) + checksumCombined = append(checksumCombined, part3Checksum.Raw...) + + // Create the final checksum (checksum of the combined checksums) + // Add BOTH the multipart flag AND the includes-multipart flag + finalChecksumType := checksumType | ChecksumMultipart | ChecksumIncludesMultipart + finalChecksum := NewChecksumFromData(finalChecksumType, checksumCombined) + + // Set WantParts to indicate 3 parts + finalChecksum.WantParts = 3 + + // Test AppendTo serialization + var serialized []byte + serialized = finalChecksum.AppendTo(serialized, checksumCombined) + + // Use ChecksumFromBytes to deserialize the final checksum + chksmOut := ChecksumFromBytes(serialized) + if chksmOut == nil { + t.Fatal("ChecksumFromBytes returned nil") + } + + // Assert they are exactly equal + if !chksmOut.Equal(finalChecksum) { + t.Fatalf("Checksum mismatch: expected %+v, got %+v", finalChecksum, chksmOut) + } + + // Serialize what we got from ChecksumFromBytes + serializedOut := chksmOut.AppendTo(nil, checksumCombined) + + // Read part checksums from serializedOut + readParts := ReadPartCheckSums(serializedOut) + expectedChecksums := []string{ + part1Checksum.Encoded, + part2Checksum.Encoded, + part3Checksum.Encoded, + } + for i, expected := range expectedChecksums { + if got := readParts[i][ChecksumCRC32C.String()]; got != expected { + t.Fatalf("want part%dChecksum.Encoded %s, got %s", i+1, expected, got) + } + } +} diff --git a/internal/hash/crc.go b/internal/hash/crc.go new file mode 100644 index 00000000..6fc1d3ba --- /dev/null +++ b/internal/hash/crc.go @@ -0,0 +1,219 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package hash + +import ( + "encoding/base64" + "encoding/binary" + "fmt" + "hash/crc32" + "hash/crc64" + "math/bits" +) + +// AddPart will merge a part checksum into the current, +// as if the content of each was appended. +// The size of the content that produced the second checksum must be provided. +// Not all checksum types can be merged, use the CanMerge method to check. +// Checksum types must match. +func (c *Checksum) AddPart(other Checksum, size int64) error { + if !other.Type.CanMerge() { + return fmt.Errorf("checksum type cannot be merged") + } + if size == 0 { + return nil + } + if !c.Type.Is(other.Type.Base()) { + return fmt.Errorf("checksum type does not match got %s and %s", c.Type.String(), other.Type.String()) + } + // If never set, just add first checksum. + if len(c.Raw) == 0 { + c.Raw = other.Raw + c.Encoded = other.Encoded + return nil + } + if !c.Valid() { + return fmt.Errorf("invalid base checksum") + } + if !other.Valid() { + return fmt.Errorf("invalid part checksum") + } + + switch c.Type.Base() { + case ChecksumCRC32: + v := crc32Combine(crc32.IEEE, binary.BigEndian.Uint32(c.Raw), binary.BigEndian.Uint32(other.Raw), size) + binary.BigEndian.PutUint32(c.Raw, v) + case ChecksumCRC32C: + v := crc32Combine(crc32.Castagnoli, binary.BigEndian.Uint32(c.Raw), binary.BigEndian.Uint32(other.Raw), size) + binary.BigEndian.PutUint32(c.Raw, v) + case ChecksumCRC64NVME: + v := crc64Combine(bits.Reverse64(crc64NVMEPolynomial), binary.BigEndian.Uint64(c.Raw), binary.BigEndian.Uint64(other.Raw), size) + binary.BigEndian.PutUint64(c.Raw, v) + default: + return fmt.Errorf("unknown checksum type: %s", c.Type.String()) + } + c.Encoded = base64.StdEncoding.EncodeToString(c.Raw) + return nil +} + +const crc64NVMEPolynomial = 0xad93d23594c93659 + +var crc64Table = crc64.MakeTable(bits.Reverse64(crc64NVMEPolynomial)) + +// Following is ported from C to Go in 2016 by Justin Ruggles, with minimal alteration. +// Used uint for unsigned long. Used uint32 for input arguments in order to match +// the Go hash/crc32 package. zlib CRC32 combine (https://github.com/madler/zlib) +// Modified for hash/crc64 by Klaus Post, 2024. +func gf2MatrixTimes(mat []uint64, vec uint64) uint64 { + var sum uint64 + + for vec != 0 { + if vec&1 != 0 { + sum ^= mat[0] + } + vec >>= 1 + mat = mat[1:] + } + return sum +} + +func gf2MatrixSquare(square, mat []uint64) { + if len(square) != len(mat) { + panic("square matrix size mismatch") + } + for n := range mat { + square[n] = gf2MatrixTimes(mat, mat[n]) + } +} + +// crc32Combine returns the combined CRC-32 hash value of the two passed CRC-32 +// hash values crc1 and crc2. poly represents the generator polynomial +// and len2 specifies the byte length that the crc2 hash covers. +func crc32Combine(poly uint32, crc1, crc2 uint32, len2 int64) uint32 { + // degenerate case (also disallow negative lengths) + if len2 <= 0 { + return crc1 + } + + even := make([]uint64, 32) // even-power-of-two zeros operator + odd := make([]uint64, 32) // odd-power-of-two zeros operator + + // put operator for one zero bit in odd + odd[0] = uint64(poly) // CRC-32 polynomial + row := uint64(1) + for n := 1; n < 32; n++ { + odd[n] = row + row <<= 1 + } + + // put operator for two zero bits in even + gf2MatrixSquare(even, odd) + + // put operator for four zero bits in odd + gf2MatrixSquare(odd, even) + + // apply len2 zeros to crc1 (first square will put the operator for one + // zero byte, eight zero bits, in even) + crc1n := uint64(crc1) + for { + // apply zeros operator for this bit of len2 + gf2MatrixSquare(even, odd) + if len2&1 != 0 { + crc1n = gf2MatrixTimes(even, crc1n) + } + len2 >>= 1 + + // if no more bits set, then done + if len2 == 0 { + break + } + + // another iteration of the loop with odd and even swapped + gf2MatrixSquare(odd, even) + if len2&1 != 0 { + crc1n = gf2MatrixTimes(odd, crc1n) + } + len2 >>= 1 + + // if no more bits set, then done + if len2 == 0 { + break + } + } + + // return combined crc + crc1n ^= uint64(crc2) + return uint32(crc1n) +} + +func crc64Combine(poly uint64, crc1, crc2 uint64, len2 int64) uint64 { + // degenerate case (also disallow negative lengths) + if len2 <= 0 { + return crc1 + } + + even := make([]uint64, 64) // even-power-of-two zeros operator + odd := make([]uint64, 64) // odd-power-of-two zeros operator + + // put operator for one zero bit in odd + odd[0] = poly // CRC-64 polynomial + row := uint64(1) + for n := 1; n < 64; n++ { + odd[n] = row + row <<= 1 + } + + // put operator for two zero bits in even + gf2MatrixSquare(even, odd) + + // put operator for four zero bits in odd + gf2MatrixSquare(odd, even) + + // apply len2 zeros to crc1 (first square will put the operator for one + // zero byte, eight zero bits, in even) + crc1n := crc1 + for { + // apply zeros operator for this bit of len2 + gf2MatrixSquare(even, odd) + if len2&1 != 0 { + crc1n = gf2MatrixTimes(even, crc1n) + } + len2 >>= 1 + + // if no more bits set, then done + if len2 == 0 { + break + } + + // another iteration of the loop with odd and even swapped + gf2MatrixSquare(odd, even) + if len2&1 != 0 { + crc1n = gf2MatrixTimes(odd, crc1n) + } + len2 >>= 1 + + // if no more bits set, then done + if len2 == 0 { + break + } + } + + // return combined crc + crc1n ^= crc2 + return crc1n +} diff --git a/internal/hash/errors.go b/internal/hash/errors.go new file mode 100644 index 00000000..0d2f7649 --- /dev/null +++ b/internal/hash/errors.go @@ -0,0 +1,89 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package hash + +import ( + "errors" + "fmt" +) + +// SHA256Mismatch - when content sha256 does not match with what was sent from client. +type SHA256Mismatch struct { + ExpectedSHA256 string + CalculatedSHA256 string +} + +func (e SHA256Mismatch) Error() string { + return "Bad sha256: Expected " + e.ExpectedSHA256 + " does not match calculated " + e.CalculatedSHA256 +} + +// BadDigest - Content-MD5 you specified did not match what we received. +type BadDigest struct { + ExpectedMD5 string + CalculatedMD5 string +} + +func (e BadDigest) Error() string { + return "Bad digest: Expected " + e.ExpectedMD5 + " does not match calculated " + e.CalculatedMD5 +} + +// SizeTooSmall reader size too small +type SizeTooSmall struct { + Want int64 + Got int64 +} + +func (e SizeTooSmall) Error() string { + return fmt.Sprintf("Size small: got %d, want %d", e.Got, e.Want) +} + +// SizeTooLarge reader size too large +type SizeTooLarge struct { + Want int64 + Got int64 +} + +func (e SizeTooLarge) Error() string { + return fmt.Sprintf("Size large: got %d, want %d", e.Got, e.Want) +} + +// SizeMismatch error size mismatch +type SizeMismatch struct { + Want int64 + Got int64 +} + +func (e SizeMismatch) Error() string { + return fmt.Sprintf("Size mismatch: got %d, want %d", e.Got, e.Want) +} + +// ChecksumMismatch - when content checksum does not match with what was sent from client. +type ChecksumMismatch struct { + Want string + Got string +} + +func (e ChecksumMismatch) Error() string { + return "Bad checksum: Want " + e.Want + " does not match calculated " + e.Got +} + +// IsChecksumMismatch matches if 'err' is hash.ChecksumMismatch +func IsChecksumMismatch(err error) bool { + var herr ChecksumMismatch + return errors.As(err, &herr) +} diff --git a/internal/hash/reader.go b/internal/hash/reader.go new file mode 100644 index 00000000..ddae850a --- /dev/null +++ b/internal/hash/reader.go @@ -0,0 +1,402 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package hash + +import ( + "bytes" + "context" + "encoding/base64" + "encoding/hex" + "errors" + "hash" + "io" + "net/http" + + "github.com/minio/minio/internal/etag" + "github.com/minio/minio/internal/hash/sha256" + "github.com/minio/minio/internal/ioutil" +) + +// A Reader wraps an io.Reader and computes the MD5 checksum +// of the read content as ETag. Optionally, it also computes +// the SHA256 checksum of the content. +// +// If the reference values for the ETag and content SHA26 +// are not empty then it will check whether the computed +// match the reference values. +type Reader struct { + src io.Reader + bytesRead int64 + expectedMin int64 + expectedMax int64 + + size int64 + actualSize int64 + + checksum etag.ETag + contentSHA256 []byte + + // Client-provided content checksum + contentHash Checksum + contentHasher hash.Hash + disableMD5 bool + + // Server side computed checksum. In some cases, like CopyObject, a new checksum + // needs to be computed and saved on the destination object, but the client + // does not provide it. Not calculated if client-side contentHash is set. + ServerSideChecksumType ChecksumType + ServerSideHasher hash.Hash + ServerSideChecksumResult *Checksum + + trailer http.Header + + sha256 hash.Hash +} + +// Options are optional arguments to NewReaderWithOpts, Options +// simply converts positional arguments to NewReader() into a +// more flexible way to provide optional inputs. This is currently +// used by the FanOut API call mostly to disable expensive md5sum +// calculation repeatedly under hash.Reader. +type Options struct { + MD5Hex string + SHA256Hex string + Size int64 + ActualSize int64 + DisableMD5 bool + ForceMD5 []byte +} + +// NewReaderWithOpts is like NewReader but takes `Options` as argument, allowing +// callers to indicate if they want to disable md5sum checksum. +func NewReaderWithOpts(ctx context.Context, src io.Reader, opts Options) (*Reader, error) { + // return hard limited reader + return newReader(ctx, src, opts.Size, opts.MD5Hex, opts.SHA256Hex, opts.ActualSize, opts.DisableMD5, opts.ForceMD5) +} + +// NewReader returns a new Reader that wraps src and computes +// MD5 checksum of everything it reads as ETag. +// +// It also computes the SHA256 checksum of everything it reads +// if sha256Hex is not the empty string. +// +// If size resp. actualSize is unknown at the time of calling +// NewReader then it should be set to -1. +// When size is >=0 it *must* match the amount of data provided by r. +// +// NewReader may try merge the given size, MD5 and SHA256 values +// into src - if src is a Reader - to avoid computing the same +// checksums multiple times. +// NewReader enforces S3 compatibility strictly by ensuring caller +// does not send more content than specified size. +func NewReader(ctx context.Context, src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64) (*Reader, error) { + return newReader(ctx, src, size, md5Hex, sha256Hex, actualSize, false, nil) +} + +func newReader(ctx context.Context, src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64, disableMD5 bool, forceMD5 []byte) (*Reader, error) { + MD5, err := hex.DecodeString(md5Hex) + if err != nil { + return nil, BadDigest{ // TODO(aead): Return an error that indicates that an invalid ETag has been specified + ExpectedMD5: md5Hex, + CalculatedMD5: "", + } + } + SHA256, err := hex.DecodeString(sha256Hex) + if err != nil { + return nil, SHA256Mismatch{ // TODO(aead): Return an error that indicates that an invalid Content-SHA256 has been specified + ExpectedSHA256: sha256Hex, + CalculatedSHA256: "", + } + } + + // Merge the size, MD5 and SHA256 values if src is a Reader. + // The size may be set to -1 by callers if unknown. + if r, ok := src.(*Reader); ok { + if r.bytesRead > 0 { + return nil, errors.New("hash: already read from hash reader") + } + if len(r.checksum) != 0 && len(MD5) != 0 && !etag.Equal(r.checksum, MD5) { + return nil, BadDigest{ + ExpectedMD5: r.checksum.String(), + CalculatedMD5: md5Hex, + } + } + if len(r.contentSHA256) != 0 && len(SHA256) != 0 && !bytes.Equal(r.contentSHA256, SHA256) { + return nil, SHA256Mismatch{ + ExpectedSHA256: hex.EncodeToString(r.contentSHA256), + CalculatedSHA256: sha256Hex, + } + } + if r.size >= 0 && size >= 0 && r.size != size { + return nil, SizeMismatch{Want: r.size, Got: size} + } + + r.checksum = MD5 + r.contentSHA256 = SHA256 + if r.size < 0 && size >= 0 { + r.src = etag.Wrap(ioutil.HardLimitReader(r.src, size), r.src) + r.size = size + } + if r.actualSize <= 0 && actualSize >= 0 { + r.actualSize = actualSize + } + return r, nil + } + + if size >= 0 { + r := ioutil.HardLimitReader(src, size) + if !disableMD5 { + if _, ok := src.(etag.Tagger); !ok { + src = etag.NewReader(ctx, r, MD5, forceMD5) + } else { + src = etag.Wrap(r, src) + } + } else { + src = r + } + } else if _, ok := src.(etag.Tagger); !ok { + if !disableMD5 { + src = etag.NewReader(ctx, src, MD5, forceMD5) + } + } + var h hash.Hash + if len(SHA256) != 0 { + h = sha256.New() + } + return &Reader{ + src: src, + size: size, + actualSize: actualSize, + checksum: MD5, + contentSHA256: SHA256, + sha256: h, + disableMD5: disableMD5, + }, nil +} + +// ErrInvalidChecksum is returned when an invalid checksum is provided in headers. +var ErrInvalidChecksum = errors.New("invalid checksum") + +// SetExpectedMin set expected minimum data expected from reader +func (r *Reader) SetExpectedMin(expectedMin int64) { + r.expectedMin = expectedMin +} + +// SetExpectedMax set expected max data expected from reader +func (r *Reader) SetExpectedMax(expectedMax int64) { + r.expectedMax = expectedMax +} + +// AddChecksum will add checksum checks as specified in +// https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html +// Returns ErrInvalidChecksum if a problem with the checksum is found. +func (r *Reader) AddChecksum(req *http.Request, ignoreValue bool) error { + cs, err := GetContentChecksum(req.Header) + if err != nil { + return ErrInvalidChecksum + } + if cs == nil { + return nil + } + r.contentHash = *cs + if cs.Type.Trailing() { + r.trailer = req.Trailer + } + return r.AddNonTrailingChecksum(cs, ignoreValue) +} + +// AddChecksumNoTrailer will add checksum checks as specified in +// https://docs.aws.amazon.com/AmazonS3/latest/userguide/checking-object-integrity.html +// Returns ErrInvalidChecksum if a problem with the checksum is found. +func (r *Reader) AddChecksumNoTrailer(headers http.Header, ignoreValue bool) error { + cs, err := GetContentChecksum(headers) + if err != nil { + return ErrInvalidChecksum + } + if cs == nil { + return nil + } + r.contentHash = *cs + return r.AddNonTrailingChecksum(cs, ignoreValue) +} + +// AddNonTrailingChecksum will add a checksum to the reader. +// The checksum cannot be trailing. +func (r *Reader) AddNonTrailingChecksum(cs *Checksum, ignoreValue bool) error { + if cs == nil { + return nil + } + r.contentHash = *cs + if ignoreValue { + // Do not validate, but allow for transfer + return nil + } + + r.contentHasher = cs.Type.Hasher() + if r.contentHasher == nil { + return ErrInvalidChecksum + } + return nil +} + +// AddServerSideChecksumHasher adds a new hasher for computing the server-side checksum. +func (r *Reader) AddServerSideChecksumHasher(t ChecksumType) { + h := t.Hasher() + if h == nil { + return + } + r.ServerSideHasher = h + r.ServerSideChecksumType = t +} + +func (r *Reader) Read(p []byte) (int, error) { + n, err := r.src.Read(p) + r.bytesRead += int64(n) + if r.sha256 != nil { + r.sha256.Write(p[:n]) + } + if r.contentHasher != nil { + r.contentHasher.Write(p[:n]) + } else if r.ServerSideHasher != nil { + r.ServerSideHasher.Write(p[:n]) + } + + if err == io.EOF { // Verify content SHA256, if set. + if r.expectedMin > 0 { + if r.bytesRead < r.expectedMin { + return 0, SizeTooSmall{Want: r.expectedMin, Got: r.bytesRead} + } + } + if r.expectedMax > 0 { + if r.bytesRead > r.expectedMax { + return 0, SizeTooLarge{Want: r.expectedMax, Got: r.bytesRead} + } + } + + if r.sha256 != nil { + if sum := r.sha256.Sum(nil); !bytes.Equal(r.contentSHA256, sum) { + return n, SHA256Mismatch{ + ExpectedSHA256: hex.EncodeToString(r.contentSHA256), + CalculatedSHA256: hex.EncodeToString(sum), + } + } + } + if r.contentHasher != nil { + if r.contentHash.Type.Trailing() { + var err error + r.contentHash.Encoded = r.trailer.Get(r.contentHash.Type.Key()) + r.contentHash.Raw, err = base64.StdEncoding.DecodeString(r.contentHash.Encoded) + if err != nil || len(r.contentHash.Raw) == 0 { + return 0, ChecksumMismatch{Got: r.contentHash.Encoded} + } + } + if sum := r.contentHasher.Sum(nil); !bytes.Equal(r.contentHash.Raw, sum) { + err := ChecksumMismatch{ + Want: r.contentHash.Encoded, + Got: base64.StdEncoding.EncodeToString(sum), + } + return n, err + } + } else if r.ServerSideHasher != nil { + sum := r.ServerSideHasher.Sum(nil) + r.ServerSideChecksumResult = NewChecksumWithType(r.ServerSideChecksumType, base64.StdEncoding.EncodeToString(sum)) + } + } + if err != nil && err != io.EOF { + if v, ok := err.(etag.VerifyError); ok { + return n, BadDigest{ + ExpectedMD5: v.Expected.String(), + CalculatedMD5: v.Computed.String(), + } + } + } + return n, err +} + +// Size returns the absolute number of bytes the Reader +// will return during reading. It returns -1 for unlimited +// data. +func (r *Reader) Size() int64 { return r.size } + +// ActualSize returns the pre-modified size of the object. +// DecompressedSize - For compressed objects. +func (r *Reader) ActualSize() int64 { return r.actualSize } + +// ETag returns the ETag computed by an underlying etag.Tagger. +// If the underlying io.Reader does not implement etag.Tagger +// it returns nil. +func (r *Reader) ETag() etag.ETag { + if t, ok := r.src.(etag.Tagger); ok { + return t.ETag() + } + return nil +} + +// MD5Current returns the MD5 checksum of the content +// that has been read so far. +// +// Calling MD5Current again after reading more data may +// result in a different checksum. +func (r *Reader) MD5Current() []byte { + if r.disableMD5 { + return r.checksum + } + return r.ETag()[:] +} + +// SHA256 returns the SHA256 checksum set as reference value. +// +// It corresponds to the checksum that is expected and +// not the actual SHA256 checksum of the content. +func (r *Reader) SHA256() []byte { + return r.contentSHA256 +} + +// SHA256HexString returns a hex representation of the SHA256. +func (r *Reader) SHA256HexString() string { + return hex.EncodeToString(r.contentSHA256) +} + +// ContentCRCType returns the content checksum type. +func (r *Reader) ContentCRCType() ChecksumType { + return r.contentHash.Type +} + +// ContentCRC returns the content crc if set. +func (r *Reader) ContentCRC() map[string]string { + if r.contentHash.Type == ChecksumNone || !r.contentHash.Valid() { + return nil + } + if r.contentHash.Type.Trailing() { + return map[string]string{r.contentHash.Type.String(): r.trailer.Get(r.contentHash.Type.Key())} + } + return map[string]string{r.contentHash.Type.String(): r.contentHash.Encoded} +} + +// Checksum returns the content checksum if set. +func (r *Reader) Checksum() *Checksum { + if !r.contentHash.Type.IsSet() || !r.contentHash.Valid() { + return nil + } + return &r.contentHash +} + +var _ io.Closer = (*Reader)(nil) // compiler check + +// Close and release resources. +func (r *Reader) Close() error { return nil } diff --git a/internal/hash/reader_test.go b/internal/hash/reader_test.go new file mode 100644 index 00000000..a28ade5c --- /dev/null +++ b/internal/hash/reader_test.go @@ -0,0 +1,315 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package hash + +import ( + "bytes" + "encoding/base64" + "encoding/hex" + "fmt" + "io" + "testing" + + "github.com/minio/minio/internal/ioutil" +) + +// Tests functions like Size(), MD5*(), SHA256*() +func TestHashReaderHelperMethods(t *testing.T) { + r, err := NewReader(t.Context(), bytes.NewReader([]byte("abcd")), 4, "e2fc714c4727ee9395f324cd2e7f331f", "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", 4) + if err != nil { + t.Fatal(err) + } + _, err = io.Copy(io.Discard, r) + if err != nil { + t.Fatal(err) + } + md5sum := r.MD5Current() + if hex.EncodeToString(md5sum) != "e2fc714c4727ee9395f324cd2e7f331f" { + t.Errorf("Expected md5hex \"e2fc714c4727ee9395f324cd2e7f331f\", got %s", hex.EncodeToString(md5sum)) + } + if r.SHA256HexString() != "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589" { + t.Errorf("Expected sha256hex \"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589\", got %s", r.SHA256HexString()) + } + if base64.StdEncoding.EncodeToString(md5sum) != "4vxxTEcn7pOV8yTNLn8zHw==" { + t.Errorf("Expected md5base64 \"4vxxTEcn7pOV8yTNLn8zHw==\", got \"%s\"", base64.StdEncoding.EncodeToString(md5sum)) + } + if r.Size() != 4 { + t.Errorf("Expected size 4, got %d", r.Size()) + } + if r.ActualSize() != 4 { + t.Errorf("Expected size 4, got %d", r.ActualSize()) + } + expectedMD5, err := hex.DecodeString("e2fc714c4727ee9395f324cd2e7f331f") + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(r.MD5Current(), expectedMD5) { + t.Errorf("Expected md5hex \"e2fc714c4727ee9395f324cd2e7f331f\", got %s", hex.EncodeToString(r.MD5Current())) + } + expectedSHA256, err := hex.DecodeString("88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589") + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(r.SHA256(), expectedSHA256) { + t.Errorf("Expected md5hex \"88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589\", got %s", r.SHA256HexString()) + } +} + +// Tests hash reader checksum verification. +func TestHashReaderVerification(t *testing.T) { + testCases := []struct { + desc string + src io.Reader + size int64 + actualSize int64 + md5hex, sha256hex string + err error + }{ + 0: { + desc: "Success, no checksum verification provided.", + src: bytes.NewReader([]byte("abcd")), + size: 4, + actualSize: 4, + }, + { + desc: "Failure md5 mismatch.", + src: bytes.NewReader([]byte("abcd")), + size: 4, + actualSize: 4, + md5hex: "d41d8cd98f00b204e9800998ecf8427f", + err: BadDigest{ + "d41d8cd98f00b204e9800998ecf8427f", + "e2fc714c4727ee9395f324cd2e7f331f", + }, + }, + { + desc: "Failure sha256 mismatch.", + src: bytes.NewReader([]byte("abcd")), + size: 4, + actualSize: 4, + sha256hex: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031580", + err: SHA256Mismatch{ + "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031580", + "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", + }, + }, + { + desc: "Nested hash reader NewReader() should merge.", + src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "", 4), + size: 4, + actualSize: 4, + }, + { + desc: "Incorrect sha256, nested", + src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "", 4), + size: 4, + actualSize: 4, + sha256hex: "50d858e0985ecc7f60418aaf0cc5ab587f42c2570a884095a9e8ccacd0f6545c", + err: SHA256Mismatch{ + ExpectedSHA256: "50d858e0985ecc7f60418aaf0cc5ab587f42c2570a884095a9e8ccacd0f6545c", + CalculatedSHA256: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", + }, + }, + 5: { + desc: "Correct sha256, nested", + src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "", 4), + size: 4, + actualSize: 4, + sha256hex: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", + }, + { + desc: "Correct sha256, nested, truncated", + src: mustReader(t, bytes.NewReader([]byte("abcd-more-stuff-to-be ignored")), 4, "", "", 4), + size: 4, + actualSize: -1, + sha256hex: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", + err: ioutil.ErrOverread, + }, + 7: { + desc: "Correct sha256, nested, truncated, swapped", + src: mustReader(t, bytes.NewReader([]byte("abcd-more-stuff-to-be ignored")), 4, "", "", -1), + size: 4, + actualSize: -1, + sha256hex: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", + err: ioutil.ErrOverread, + }, + { + desc: "Incorrect MD5, nested", + src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "", 4), + size: 4, + actualSize: 4, + md5hex: "0773da587b322af3a8718cb418a715ce", + err: BadDigest{ + ExpectedMD5: "0773da587b322af3a8718cb418a715ce", + CalculatedMD5: "e2fc714c4727ee9395f324cd2e7f331f", + }, + }, + { + desc: "Correct sha256, truncated", + src: bytes.NewReader([]byte("abcd-morethan-4-bytes")), + size: 4, + actualSize: 4, + sha256hex: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", + err: ioutil.ErrOverread, + }, + { + desc: "Correct MD5, nested", + src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "", 4), + size: 4, + actualSize: 4, + md5hex: "e2fc714c4727ee9395f324cd2e7f331f", + }, + { + desc: "Correct MD5, truncated", + src: bytes.NewReader([]byte("abcd-morethan-4-bytes")), + size: 4, + actualSize: 4, + sha256hex: "", + md5hex: "e2fc714c4727ee9395f324cd2e7f331f", + err: ioutil.ErrOverread, + }, + { + desc: "Correct MD5, nested, truncated", + src: mustReader(t, bytes.NewReader([]byte("abcd-morestuff")), -1, "", "", -1), + size: 4, + actualSize: 4, + md5hex: "e2fc714c4727ee9395f324cd2e7f331f", + err: ioutil.ErrOverread, + }, + } + for i, testCase := range testCases { + t.Run(fmt.Sprintf("case-%d", i+1), func(t *testing.T) { + r, err := NewReader(t.Context(), testCase.src, testCase.size, testCase.md5hex, testCase.sha256hex, testCase.actualSize) + if err != nil { + t.Fatalf("Test %q: Initializing reader failed %s", testCase.desc, err) + } + _, err = io.Copy(io.Discard, r) + if err != nil { + if testCase.err == nil { + t.Errorf("Test %q; got unexpected error: %v", testCase.desc, err) + return + } + if err.Error() != testCase.err.Error() { + t.Errorf("Test %q: Expected error %s, got error %s", testCase.desc, testCase.err, err) + } + } + }) + } +} + +func mustReader(t *testing.T, src io.Reader, size int64, md5Hex, sha256Hex string, actualSize int64) *Reader { + r, err := NewReader(t.Context(), src, size, md5Hex, sha256Hex, actualSize) + if err != nil { + t.Fatal(err) + } + return r +} + +// Tests NewReader() constructor with invalid arguments. +func TestHashReaderInvalidArguments(t *testing.T) { + testCases := []struct { + desc string + src io.Reader + size int64 + actualSize int64 + md5hex, sha256hex string + success bool + }{ + { + desc: "Invalid md5sum NewReader() will fail.", + src: bytes.NewReader([]byte("abcd")), + size: 4, + actualSize: 4, + md5hex: "invalid-md5", + success: false, + }, + { + desc: "Invalid sha256 NewReader() will fail.", + src: bytes.NewReader([]byte("abcd")), + size: 4, + actualSize: 4, + sha256hex: "invalid-sha256", + success: false, + }, + { + desc: "Nested hash reader NewReader() should merge.", + src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "", 4), + size: 4, + actualSize: 4, + success: true, + }, + { + desc: "Mismatching sha256", + src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", 4), + size: 4, + actualSize: 4, + sha256hex: "50d858e0985ecc7f60418aaf0cc5ab587f42c2570a884095a9e8ccacd0f6545c", + success: false, + }, + { + desc: "Correct sha256", + src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "", "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", 4), + size: 4, + actualSize: 4, + sha256hex: "88d4266fd4e6338d13b845fcf289579d209c897823b9217da3e161936f031589", + success: true, + }, + { + desc: "Mismatching MD5", + src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "e2fc714c4727ee9395f324cd2e7f331f", "", 4), + size: 4, + actualSize: 4, + md5hex: "0773da587b322af3a8718cb418a715ce", + success: false, + }, + { + desc: "Correct MD5", + src: mustReader(t, bytes.NewReader([]byte("abcd")), 4, "e2fc714c4727ee9395f324cd2e7f331f", "", 4), + size: 4, + actualSize: 4, + md5hex: "e2fc714c4727ee9395f324cd2e7f331f", + success: true, + }, + { + desc: "Nothing, all ok", + src: bytes.NewReader([]byte("abcd")), + size: 4, + actualSize: 4, + success: true, + }, + { + desc: "Nested, size mismatch", + src: mustReader(t, bytes.NewReader([]byte("abcd-morestuff")), 4, "", "", -1), + size: 2, + actualSize: -1, + success: false, + }, + } + + for i, testCase := range testCases { + t.Run(fmt.Sprintf("case-%d", i+1), func(t *testing.T) { + _, err := NewReader(t.Context(), testCase.src, testCase.size, testCase.md5hex, testCase.sha256hex, testCase.actualSize) + if err != nil && testCase.success { + t.Errorf("Test %q: Expected success, but got error %s instead", testCase.desc, err) + } + if err == nil && !testCase.success { + t.Errorf("Test %q: Expected error, but got success", testCase.desc) + } + }) + } +} diff --git a/internal/hash/sha256/sh256.go b/internal/hash/sha256/sh256.go new file mode 100644 index 00000000..b25b1584 --- /dev/null +++ b/internal/hash/sha256/sh256.go @@ -0,0 +1,32 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package sha256 + +import ( + "crypto/sha256" + "hash" +) + +// New initializes a new sha256.New() +func New() hash.Hash { return sha256.New() } + +// Sum256 returns the SHA256 checksum of the data. +func Sum256(data []byte) [sha256.Size]byte { return sha256.Sum256(data) } + +// Size is the size of a SHA256 checksum in bytes. +const Size = sha256.Size diff --git a/internal/http/check_port_linux.go b/internal/http/check_port_linux.go new file mode 100644 index 00000000..eb92893e --- /dev/null +++ b/internal/http/check_port_linux.go @@ -0,0 +1,58 @@ +//go:build linux +// +build linux + +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package http + +import ( + "context" + "net" + "syscall" + "time" +) + +// CheckPortAvailability - check if given host and port is already in use. +// Note: The check method tries to listen on given port and closes it. +// It is possible to have a disconnected client in this tiny window of time. +func CheckPortAvailability(host, port string, opts TCPOptions) (err error) { + lc := &net.ListenConfig{ + Control: func(network, address string, c syscall.RawConn) error { + c.Control(func(fdPtr uintptr) { + if opts.Interface != "" { + // When interface is specified look for specifically port availability on + // the specified interface if any. + _ = syscall.SetsockoptString(int(fdPtr), syscall.SOL_SOCKET, syscall.SO_BINDTODEVICE, opts.Interface) + } + }) + return nil + }, + } + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + l, err := lc.Listen(ctx, "tcp", net.JoinHostPort(host, port)) + if err != nil { + return err + } + + // As we are able to listen on this network, the port is not in use. + // Close the listener and continue check other networks. + return l.Close() +} diff --git a/internal/http/check_port_others.go b/internal/http/check_port_others.go new file mode 100644 index 00000000..0905c0e5 --- /dev/null +++ b/internal/http/check_port_others.go @@ -0,0 +1,46 @@ +//go:build !linux +// +build !linux + +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package http + +import ( + "context" + "net" + "time" +) + +// CheckPortAvailability - check if given host and port is already in use. +// Note: The check method tries to listen on given port and closes it. +// It is possible to have a disconnected client in this tiny window of time. +func CheckPortAvailability(host, port string, opts TCPOptions) (err error) { + lc := &net.ListenConfig{} + + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + l, err := lc.Listen(ctx, "tcp", net.JoinHostPort(host, port)) + if err != nil { + return err + } + + // As we are able to listen on this network, the port is not in use. + // Close the listener and continue check other networks. + return l.Close() +} diff --git a/internal/http/check_port_test.go b/internal/http/check_port_test.go new file mode 100644 index 00000000..ab1b1feb --- /dev/null +++ b/internal/http/check_port_test.go @@ -0,0 +1,64 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package http + +import ( + "fmt" + "net" + "runtime" + "strconv" + "testing" +) + +// Tests for port availability logic written for server startup sequence. +func TestCheckPortAvailability(t *testing.T) { + if runtime.GOOS != "linux" { + t.Skip() + } + + l, err := net.Listen("tcp", "localhost:0") // ask kernel for a free port. + if err != nil { + t.Fatal(err) + } + defer l.Close() + + port := l.Addr().(*net.TCPAddr).Port + + testCases := []struct { + host string + port int + expectedErr error + }{ + {"", port, fmt.Errorf("listen tcp :%v: bind: address already in use", port)}, + {"127.0.0.1", port, fmt.Errorf("listen tcp 127.0.0.1:%v: bind: address already in use", port)}, + } + + for _, testCase := range testCases { + err := CheckPortAvailability(testCase.host, strconv.Itoa(testCase.port), TCPOptions{}) + switch { + case testCase.expectedErr == nil: + if err != nil { + t.Fatalf("error: expected = , got = %v", err) + } + case err == nil: + t.Fatalf("error: expected = %v, got = ", testCase.expectedErr) + case testCase.expectedErr.Error() != err.Error(): + t.Fatalf("error: expected = %v, got = %v", testCase.expectedErr, err) + } + } +} diff --git a/internal/http/close.go b/internal/http/close.go new file mode 100644 index 00000000..799ac76d --- /dev/null +++ b/internal/http/close.go @@ -0,0 +1,44 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package http + +import ( + "io" + + xioutil "github.com/minio/minio/internal/ioutil" +) + +// DrainBody close non nil response with any response Body. +// convenient wrapper to drain any remaining data on response body. +// +// Subsequently this allows golang http RoundTripper +// to reuse the same connection for future requests. +func DrainBody(respBody io.ReadCloser) { + // Callers should close resp.Body when done reading from it. + // If resp.Body is not closed, the Client's underlying RoundTripper + // (typically Transport) may not be able to reuse a persistent TCP + // connection to the server for a subsequent "keep-alive" request. + if respBody != nil { + // Drain any remaining Body and then close the connection. + // Without this closing connection would disallow re-using + // the same connection for future uses. + // - http://stackoverflow.com/a/17961593/4465767 + defer respBody.Close() + xioutil.DiscardReader(respBody) + } +} diff --git a/internal/http/dial_dnscache.go b/internal/http/dial_dnscache.go new file mode 100644 index 00000000..c4851202 --- /dev/null +++ b/internal/http/dial_dnscache.go @@ -0,0 +1,80 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package http + +import ( + "context" + "net" + "time" +) + +// LookupHost is a function to make custom lookupHost for optional cached DNS requests +type LookupHost func(ctx context.Context, host string) (addrs []string, err error) + +// DialContextWithLookupHost is a helper function which returns `net.DialContext` function. +// It randomly fetches an IP via custom LookupHost function and dials it by the given dial +// function. LookupHost may implement an internal DNS caching implementation, lookupHost +// input if nil then net.DefaultResolver.LookupHost is used. +// +// It dials one by one and returns first connected `net.Conn`. +// If it fails to dial all IPs from cache it returns first error. If no baseDialFunc +// is given, it sets default dial function. +// +// You can use returned dial function for `http.Transport.DialContext`. +// +// In this function, it uses functions from `rand` package. To make it really random, +// you MUST call `rand.Seed` and change the value from the default in your application +func DialContextWithLookupHost(lookupHost LookupHost, baseDialCtx DialContext) DialContext { + if lookupHost == nil { + lookupHost = net.DefaultResolver.LookupHost + } + + if baseDialCtx == nil { + // This is same as which `http.DefaultTransport` uses. + baseDialCtx = (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext + } + + return func(ctx context.Context, network, addr string) (conn net.Conn, err error) { + host, port, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + + if net.ParseIP(host) != nil { + // For IP only setups there is no need for DNS lookups. + return baseDialCtx(ctx, "tcp", addr) + } + + ips, err := lookupHost(ctx, host) + if err != nil { + return nil, err + } + + for _, ip := range ips { + conn, err = baseDialCtx(ctx, "tcp", net.JoinHostPort(ip, port)) + if err == nil { + break + } + } + + return conn, err + } +} diff --git a/internal/http/dial_linux.go b/internal/http/dial_linux.go new file mode 100644 index 00000000..fd279aab --- /dev/null +++ b/internal/http/dial_linux.go @@ -0,0 +1,134 @@ +//go:build linux +// +build linux + +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package http + +import ( + "context" + "net" + "syscall" + "time" + + "github.com/minio/minio/internal/deadlineconn" + "golang.org/x/sys/unix" +) + +func setTCPParametersFn(opts TCPOptions) func(network, address string, c syscall.RawConn) error { + return func(network, address string, c syscall.RawConn) error { + c.Control(func(fdPtr uintptr) { + // got socket file descriptor to set parameters. + fd := int(fdPtr) + + _ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEADDR, 1) + + _ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_REUSEPORT, 1) + + // Enable custom socket send/recv buffers. + if opts.SendBufSize > 0 { + _ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_SNDBUF, opts.SendBufSize) + } + + if opts.RecvBufSize > 0 { + _ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_RCVBUF, opts.RecvBufSize) + } + + if opts.NoDelay { + _ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, unix.TCP_NODELAY, 1) + _ = syscall.SetsockoptInt(fd, syscall.SOL_TCP, unix.TCP_CORK, 0) + } + + // Enable TCP open + // https://lwn.net/Articles/508865/ - 32k queue size. + _ = syscall.SetsockoptInt(fd, syscall.SOL_TCP, unix.TCP_FASTOPEN, 32*1024) + + // Enable TCP fast connect + // TCPFastOpenConnect sets the underlying socket to use + // the TCP fast open connect. This feature is supported + // since Linux 4.11. + _ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, unix.TCP_FASTOPEN_CONNECT, 1) + + // Enable TCP quick ACK, John Nagle says + // "Set TCP_QUICKACK. If you find a case where that makes things worse, let me know." + _ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, unix.TCP_QUICKACK, 1) + + /// Enable keep-alive + { + _ = unix.SetsockoptInt(fd, unix.SOL_SOCKET, unix.SO_KEEPALIVE, 1) + + // The time (in seconds) the connection needs to remain idle before + // TCP starts sending keepalive probes + _ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPIDLE, 15) + + // Number of probes. + // ~ cat /proc/sys/net/ipv4/tcp_keepalive_probes (defaults to 9, we reduce it to 5) + _ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPCNT, 5) + + // Wait time after successful probe in seconds. + // ~ cat /proc/sys/net/ipv4/tcp_keepalive_intvl (defaults to 75 secs, we reduce it to 15 secs) + _ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, syscall.TCP_KEEPINTVL, 15) + } + + // Set tcp user timeout in addition to the keep-alive - tcp-keepalive is not enough to close a socket + // with dead end because tcp-keepalive is not fired when there is data in the socket buffer. + // https://blog.cloudflare.com/when-tcp-sockets-refuse-to-die/ + // This is a sensitive configuration, it is better to set it to high values, > 60 secs since it can + // affect clients reading data with a very slow pace (disappropriate with socket buffer sizes) + if opts.UserTimeout > 0 { + _ = syscall.SetsockoptInt(fd, syscall.IPPROTO_TCP, unix.TCP_USER_TIMEOUT, opts.UserTimeout) + } + + if opts.Interface != "" { + if h, _, err := net.SplitHostPort(address); err == nil { + address = h + } + // Create socket on specific vrf device. + // To catch all kinds of special cases this filters specifically for loopback networks. + if ip := net.ParseIP(address); ip != nil && !ip.IsLoopback() { + _ = syscall.SetsockoptString(fd, syscall.SOL_SOCKET, syscall.SO_BINDTODEVICE, opts.Interface) + } + } + }) + return nil + } +} + +// DialContext is a function to make custom Dial for internode communications +type DialContext func(ctx context.Context, network, address string) (net.Conn, error) + +// NewInternodeDialContext setups a custom dialer for internode communication +func NewInternodeDialContext(dialTimeout time.Duration, opts TCPOptions) DialContext { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + dialer := &net.Dialer{ + Timeout: dialTimeout, + Control: setTCPParametersFn(opts), + } + conn, err := dialer.DialContext(ctx, network, addr) + if err != nil { + return nil, err + } + if opts.DriveOPTimeout != nil { + // Read deadlines are sufficient for now as per various + // scenarios of hung node detection, we may add Write deadlines + // if needed later on. + return deadlineconn.New(conn).WithReadDeadline(opts.DriveOPTimeout()), nil + } + return conn, nil + } +} diff --git a/internal/http/dial_others.go b/internal/http/dial_others.go new file mode 100644 index 00000000..fe548867 --- /dev/null +++ b/internal/http/dial_others.go @@ -0,0 +1,50 @@ +//go:build !linux +// +build !linux + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package http + +import ( + "context" + "net" + "syscall" + "time" +) + +// TODO: if possible implement for non-linux platforms, not a priority at the moment +// +//nolint:unused +func setTCPParametersFn(opts TCPOptions) func(network, address string, c syscall.RawConn) error { + return func(network, address string, c syscall.RawConn) error { + return nil + } +} + +// DialContext is a function to make custom Dial for internode communications +type DialContext func(ctx context.Context, network, address string) (net.Conn, error) + +// NewInternodeDialContext configures a custom dialer for internode communications +func NewInternodeDialContext(dialTimeout time.Duration, _ TCPOptions) DialContext { + return func(ctx context.Context, network, addr string) (net.Conn, error) { + dialer := &net.Dialer{ + Timeout: dialTimeout, + } + return dialer.DialContext(ctx, network, addr) + } +} diff --git a/internal/http/flush.go b/internal/http/flush.go new file mode 100644 index 00000000..e17cf7d5 --- /dev/null +++ b/internal/http/flush.go @@ -0,0 +1,27 @@ +// Copyright (c) 2015-2025 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package http + +import "net/http" + +// Flush the ResponseWriter. +func Flush(w http.ResponseWriter) { + if f, ok := w.(http.Flusher); ok { + f.Flush() + } +} diff --git a/internal/http/headers.go b/internal/http/headers.go new file mode 100644 index 00000000..9195e122 --- /dev/null +++ b/internal/http/headers.go @@ -0,0 +1,275 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package http + +// Standard S3 HTTP response constants +const ( + LastModified = "Last-Modified" + Date = "Date" + ETag = "ETag" + ContentType = "Content-Type" + ContentMD5 = "Content-Md5" + ContentEncoding = "Content-Encoding" + Expires = "Expires" + ContentLength = "Content-Length" + ContentLanguage = "Content-Language" + ContentRange = "Content-Range" + Connection = "Connection" + AcceptRanges = "Accept-Ranges" + AmzBucketRegion = "X-Amz-Bucket-Region" + ServerInfo = "Server" + RetryAfter = "Retry-After" + Location = "Location" + CacheControl = "Cache-Control" + ContentDisposition = "Content-Disposition" + Authorization = "Authorization" + Action = "Action" + Range = "Range" +) + +// Non standard S3 HTTP response constants +const ( + XCache = "X-Cache" + XCacheLookup = "X-Cache-Lookup" +) + +// Standard S3 HTTP request constants +const ( + IfModifiedSince = "If-Modified-Since" + IfUnmodifiedSince = "If-Unmodified-Since" + IfMatch = "If-Match" + IfNoneMatch = "If-None-Match" + + // Request tags used in GetObjectAttributes + Checksum = "Checksum" + StorageClass = "StorageClass" + ObjectSize = "ObjectSize" + ObjectParts = "ObjectParts" + + // S3 storage class + AmzStorageClass = "x-amz-storage-class" + + // S3 object version ID + AmzVersionID = "x-amz-version-id" + AmzDeleteMarker = "x-amz-delete-marker" + + // S3 object tagging + AmzObjectTagging = "X-Amz-Tagging" + AmzTagCount = "x-amz-tagging-count" + AmzTagDirective = "X-Amz-Tagging-Directive" + + // S3 transition restore + AmzRestore = "x-amz-restore" + AmzRestoreExpiryDays = "X-Amz-Restore-Expiry-Days" + AmzRestoreRequestDate = "X-Amz-Restore-Request-Date" + AmzRestoreOutputPath = "x-amz-restore-output-path" + + // S3 extensions + AmzCopySourceIfModifiedSince = "x-amz-copy-source-if-modified-since" + AmzCopySourceIfUnmodifiedSince = "x-amz-copy-source-if-unmodified-since" + + AmzCopySourceIfNoneMatch = "x-amz-copy-source-if-none-match" + AmzCopySourceIfMatch = "x-amz-copy-source-if-match" + + AmzCopySource = "X-Amz-Copy-Source" + AmzCopySourceVersionID = "X-Amz-Copy-Source-Version-Id" + AmzCopySourceRange = "X-Amz-Copy-Source-Range" + AmzMetadataDirective = "X-Amz-Metadata-Directive" + AmzObjectLockMode = "X-Amz-Object-Lock-Mode" + AmzObjectLockRetainUntilDate = "X-Amz-Object-Lock-Retain-Until-Date" + AmzObjectLockLegalHold = "X-Amz-Object-Lock-Legal-Hold" + AmzObjectLockBypassGovernance = "X-Amz-Bypass-Governance-Retention" + AmzBucketReplicationStatus = "X-Amz-Replication-Status" + + // AmzSnowballExtract will trigger unpacking of an archive content + AmzSnowballExtract = "X-Amz-Meta-Snowball-Auto-Extract" + // MinIOSnowballIgnoreDirs will skip creating empty directory objects. + MinIOSnowballIgnoreDirs = "X-Amz-Meta-Minio-Snowball-Ignore-Dirs" + // MinIOSnowballIgnoreErrors will ignore recoverable errors, typically single files failing to upload. + // An error will be printed to console instead. + MinIOSnowballIgnoreErrors = "X-Amz-Meta-Minio-Snowball-Ignore-Errors" + // MinIOSnowballPrefix will apply this prefix (plus / at end) to all extracted objects + MinIOSnowballPrefix = "X-Amz-Meta-Minio-Snowball-Prefix" + + // Object lock enabled + AmzObjectLockEnabled = "x-amz-bucket-object-lock-enabled" + + // Multipart parts count + AmzMpPartsCount = "x-amz-mp-parts-count" + + // Object date/time of expiration + AmzExpiration = "x-amz-expiration" + + // Dummy putBucketACL + AmzACL = "x-amz-acl" + + // Signature V4 related constants. + AmzContentSha256 = "X-Amz-Content-Sha256" + AmzDate = "X-Amz-Date" + AmzAlgorithm = "X-Amz-Algorithm" + AmzExpires = "X-Amz-Expires" + AmzSignedHeaders = "X-Amz-SignedHeaders" + AmzSignature = "X-Amz-Signature" + AmzCredential = "X-Amz-Credential" + AmzSecurityToken = "X-Amz-Security-Token" + AmzDecodedContentLength = "X-Amz-Decoded-Content-Length" + AmzTrailer = "X-Amz-Trailer" + AmzMaxParts = "X-Amz-Max-Parts" + AmzPartNumberMarker = "X-Amz-Part-Number-Marker" + + // Constants used for GetObjectAttributes and GetObjectVersionAttributes + AmzObjectAttributes = "X-Amz-Object-Attributes" + + AmzMetaUnencryptedContentLength = "X-Amz-Meta-X-Amz-Unencrypted-Content-Length" + AmzMetaUnencryptedContentMD5 = "X-Amz-Meta-X-Amz-Unencrypted-Content-Md5" + + // AWS server-side encryption headers for SSE-S3, SSE-KMS and SSE-C. + AmzServerSideEncryption = "X-Amz-Server-Side-Encryption" + AmzServerSideEncryptionKmsID = AmzServerSideEncryption + "-Aws-Kms-Key-Id" + AmzServerSideEncryptionKmsContext = AmzServerSideEncryption + "-Context" + AmzServerSideEncryptionCustomerAlgorithm = AmzServerSideEncryption + "-Customer-Algorithm" + AmzServerSideEncryptionCustomerKey = AmzServerSideEncryption + "-Customer-Key" + AmzServerSideEncryptionCustomerKeyMD5 = AmzServerSideEncryption + "-Customer-Key-Md5" + AmzServerSideEncryptionCopyCustomerAlgorithm = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Algorithm" + AmzServerSideEncryptionCopyCustomerKey = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key" + AmzServerSideEncryptionCopyCustomerKeyMD5 = "X-Amz-Copy-Source-Server-Side-Encryption-Customer-Key-Md5" + + AmzEncryptionAES = "AES256" + AmzEncryptionKMS = "aws:kms" + + // Signature v2 related constants + AmzSignatureV2 = "Signature" + AmzAccessKeyID = "AWSAccessKeyId" + + // Response request id. + AmzRequestID = "x-amz-request-id" + AmzRequestHostID = "x-amz-id-2" + + // Deployment id. + MinioDeploymentID = "x-minio-deployment-id" + + // Peer call + MinIOPeerCall = "x-minio-from-peer" + + // Server-Status + MinIOServerStatus = "x-minio-server-status" + + // Content Checksums + AmzChecksumAlgo = "x-amz-checksum-algorithm" + AmzChecksumCRC32 = "x-amz-checksum-crc32" + AmzChecksumCRC32C = "x-amz-checksum-crc32c" + AmzChecksumSHA1 = "x-amz-checksum-sha1" + AmzChecksumSHA256 = "x-amz-checksum-sha256" + AmzChecksumCRC64NVME = "x-amz-checksum-crc64nvme" + AmzChecksumMode = "x-amz-checksum-mode" + AmzChecksumType = "x-amz-checksum-type" + AmzChecksumTypeFullObject = "FULL_OBJECT" + AmzChecksumTypeComposite = "COMPOSITE" + + // S3 Express API related constant reject it. + AmzWriteOffsetBytes = "x-amz-write-offset-bytes" + + // Post Policy related + AmzMetaUUID = "X-Amz-Meta-Uuid" + AmzMetaName = "X-Amz-Meta-Name" + + // Delete special flag to force delete a bucket or a prefix + MinIOForceDelete = "x-minio-force-delete" + + // Create special flag to force create a bucket + MinIOForceCreate = "x-minio-force-create" + + // Header indicates if the mtime should be preserved by client + MinIOSourceMTime = "x-minio-source-mtime" + + // Header indicates if the etag should be preserved by client + MinIOSourceETag = "x-minio-source-etag" + + // Writes expected write quorum + MinIOWriteQuorum = "x-minio-write-quorum" + + // Reads expected read quorum + MinIOReadQuorum = "x-minio-read-quorum" + + // Indicates if we are using default storage class and there was problem loading config + // if this header is set to "true" + MinIOStorageClassDefaults = "x-minio-storage-class-defaults" + + // Reports number of drives currently healing + MinIOHealingDrives = "x-minio-healing-drives" + + // Header indicates if the delete marker should be preserved by client + MinIOSourceDeleteMarker = "x-minio-source-deletemarker" + + // Header indicates if the delete marker version needs to be purged. + MinIOSourceDeleteMarkerDelete = "x-minio-source-deletemarker-delete" + + // Header indicates permanent delete replication status. + MinIODeleteReplicationStatus = "X-Minio-Replication-Delete-Status" + // Header indicates delete-marker replication status. + MinIODeleteMarkerReplicationStatus = "X-Minio-Replication-DeleteMarker-Status" + // Header indicates if its a GET/HEAD proxy request for active-active replication + MinIOSourceProxyRequest = "X-Minio-Source-Proxy-Request" + // Header indicates that this request is a replication request to create a REPLICA + MinIOSourceReplicationRequest = "X-Minio-Source-Replication-Request" + // Header checks replication permissions without actually completing replication + MinIOSourceReplicationCheck = "X-Minio-Source-Replication-Check" + // Header indicates replication reset status. + MinIOReplicationResetStatus = "X-Minio-Replication-Reset-Status" + // Header indicating target cluster can receive delete marker replication requests because object has been replicated + MinIOTargetReplicationReady = "X-Minio-Replication-Ready" + // Header asking if cluster can receive delete marker replication request now. + MinIOCheckDMReplicationReady = "X-Minio-Check-Replication-Ready" + // Header indiicates last tag update time on source + MinIOSourceTaggingTimestamp = "X-Minio-Source-Replication-Tagging-Timestamp" + // Header indiicates last rtention update time on source + MinIOSourceObjectRetentionTimestamp = "X-Minio-Source-Replication-Retention-Timestamp" + // Header indiicates last rtention update time on source + MinIOSourceObjectLegalHoldTimestamp = "X-Minio-Source-Replication-LegalHold-Timestamp" + // Header indicates a Tag operation was performed on one/more peers successfully, though the + // current cluster does not have the object yet. This is in a site/bucket replication scenario. + MinIOTaggingProxied = "X-Minio-Tagging-Proxied" + // Header indicates the actual replicated object size + // In case of SSEC objects getting replicated (multipart) actual size would be needed at target + MinIOReplicationActualObjectSize = "X-Minio-Replication-Actual-Object-Size" + + // predicted date/time of transition + MinIOTransition = "X-Minio-Transition" + MinIOLifecycleCfgUpdatedAt = "X-Minio-LifecycleConfig-UpdatedAt" + // MinIOCompressed is returned when object is compressed + MinIOCompressed = "X-Minio-Compressed" + + // SUBNET related + SubnetAPIKey = "x-subnet-api-key" +) + +// Common http query params S3 API +const ( + VersionID = "versionId" + + PartNumber = "partNumber" + + UploadID = "uploadId" +) + +// http headers sent to webhook targets +const ( + // Reports the version of MinIO server + MinIOVersion = "x-minio-version" + WebhookEventPayloadCount = "x-minio-webhook-payload-count" +) diff --git a/internal/http/lambda-headers.go b/internal/http/lambda-headers.go new file mode 100644 index 00000000..593a473a --- /dev/null +++ b/internal/http/lambda-headers.go @@ -0,0 +1,57 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package http + +// Object Lambda headers +const ( + AmzRequestRoute = "x-amz-request-route" + AmzRequestToken = "x-amz-request-token" + + AmzFwdStatus = "x-amz-fwd-status" + AmzFwdErrorCode = "x-amz-fwd-error-code" + AmzFwdErrorMessage = "x-amz-fwd-error-message" + AmzFwdHeaderAcceptRanges = "x-amz-fwd-header-accept-ranges" + AmzFwdHeaderCacheControl = "x-amz-fwd-header-Cache-Control" + AmzFwdHeaderContentDisposition = "x-amz-fwd-header-Content-Disposition" + AmzFwdHeaderContentEncoding = "x-amz-fwd-header-Content-Encoding" + AmzFwdHeaderContentLanguage = "x-amz-fwd-header-Content-Language" + AmzFwdHeaderContentRange = "x-amz-fwd-header-Content-Range" + AmzFwdHeaderContentType = "x-amz-fwd-header-Content-Type" + AmzFwdHeaderChecksumCrc32 = "x-amz-fwd-header-x-amz-checksum-crc32" + AmzFwdHeaderChecksumCrc32c = "x-amz-fwd-header-x-amz-checksum-crc32c" + AmzFwdHeaderChecksumSha1 = "x-amz-fwd-header-x-amz-checksum-sha1" + AmzFwdHeaderChecksumSha256 = "x-amz-fwd-header-x-amz-checksum-sha256" + AmzFwdHeaderDeleteMarker = "x-amz-fwd-header-x-amz-delete-marker" + AmzFwdHeaderETag = "x-amz-fwd-header-ETag" + AmzFwdHeaderExpires = "x-amz-fwd-header-Expires" + AmzFwdHeaderExpiration = "x-amz-fwd-header-x-amz-expiration" + AmzFwdHeaderLastModified = "x-amz-fwd-header-Last-Modified" + + AmzFwdHeaderObjectLockMode = "x-amz-fwd-header-x-amz-object-lock-mode" + AmzFwdHeaderObjectLockLegalHold = "x-amz-fwd-header-x-amz-object-lock-legal-hold" + AmzFwdHeaderObjectLockRetainUntil = "x-amz-fwd-header-x-amz-object-lock-retain-until-date" + AmzFwdHeaderMPPartsCount = "x-amz-fwd-header-x-amz-mp-parts-count" + AmzFwdHeaderReplicationStatus = "x-amz-fwd-header-x-amz-replication-status" + AmzFwdHeaderSSE = "x-amz-fwd-header-x-amz-server-side-encryption" + AmzFwdHeaderSSEC = "x-amz-fwd-header-x-amz-server-side-encryption-customer-algorithm" + AmzFwdHeaderSSEKMSID = "x-amz-fwd-header-x-amz-server-side-encryption-aws-kms-key-id" + AmzFwdHeaderSSECMD5 = "x-amz-fwd-header-x-amz-server-side-encryption-customer-key-MD5" + AmzFwdHeaderStorageClass = "x-amz-fwd-header-x-amz-storage-class" + AmzFwdHeaderTaggingCount = "x-amz-fwd-header-x-amz-tagging-count" + AmzFwdHeaderVersionID = "x-amz-fwd-header-x-amz-version-id" +) diff --git a/internal/http/listener.go b/internal/http/listener.go new file mode 100644 index 00000000..bc6de3af --- /dev/null +++ b/internal/http/listener.go @@ -0,0 +1,191 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package http + +import ( + "context" + "fmt" + "net" + "slices" + "syscall" + "time" + + "github.com/minio/minio/internal/deadlineconn" +) + +type acceptResult struct { + conn net.Conn + err error + lidx int +} + +// httpListener - HTTP listener capable of handling multiple server addresses. +type httpListener struct { + opts TCPOptions + listeners []net.Listener // underlying TCP listeners. + acceptCh chan acceptResult // channel where all TCP listeners write accepted connection. + ctxDoneCh <-chan struct{} + ctxCanceler context.CancelFunc +} + +// start - starts separate goroutine for each TCP listener. A valid new connection is passed to httpListener.acceptCh. +func (listener *httpListener) start() { + // Closure to handle listener until httpListener.ctxDoneCh channel is closed. + handleListener := func(idx int, ln net.Listener) { + for { + conn, err := ln.Accept() + select { + case listener.acceptCh <- acceptResult{conn, err, idx}: + case <-listener.ctxDoneCh: + return + } + } + } + + // Start separate goroutine for each listener to handle connection. + for idx, ln := range listener.listeners { + go handleListener(idx, ln) + } +} + +// Accept - reads from httpListener.acceptCh for one of previously accepted TCP connection and returns the same. +func (listener *httpListener) Accept() (conn net.Conn, err error) { + select { + case result := <-listener.acceptCh: + if result.err != nil { + return nil, result.err + } + return deadlineconn.New(result.conn).WithReadDeadline(listener.opts.IdleTimeout).WithWriteDeadline(listener.opts.IdleTimeout), result.err + case <-listener.ctxDoneCh: + } + return nil, syscall.EINVAL +} + +// Close - closes underneath all TCP listeners. +func (listener *httpListener) Close() (err error) { + listener.ctxCanceler() + + for i := range listener.listeners { + listener.listeners[i].Close() + } + + return nil +} + +// Addr - net.Listener interface compatible method returns net.Addr. In case of multiple TCP listeners, it returns '0.0.0.0' as IP address. +func (listener *httpListener) Addr() (addr net.Addr) { + addr = listener.listeners[0].Addr() + if len(listener.listeners) == 1 { + return addr + } + + if tcpAddr, ok := addr.(*net.TCPAddr); ok { + return &net.TCPAddr{ + IP: net.IPv4zero, + Port: tcpAddr.Port, + Zone: tcpAddr.Zone, + } + } + panic("unknown address type on listener") +} + +// Addrs - returns all address information of TCP listeners. +func (listener *httpListener) Addrs() (addrs []net.Addr) { + addrs = make([]net.Addr, 0, len(listener.listeners)) + for i := range listener.listeners { + addrs = append(addrs, listener.listeners[i].Addr()) + } + + return addrs +} + +// TCPOptions specify customizable TCP optimizations on raw socket +type TCPOptions struct { + UserTimeout int // this value is expected to be in milliseconds + + // When the net.Conn is a remote drive this value is honored, we close the connection to remote peer proactively. + DriveOPTimeout func() time.Duration + + SendBufSize int // SO_SNDBUF size for the socket connection, NOTE: this sets server and client connection + RecvBufSize int // SO_RECVBUF size for the socket connection, NOTE: this sets server and client connection + NoDelay bool // Indicates callers to enable TCP_NODELAY on the net.Conn + Interface string // This is a VRF device passed via `--interface` flag + Trace func(msg string) // Trace when starting. + IdleTimeout time.Duration // Incoming TCP read/write timeout +} + +// ForWebsocket returns TCPOptions valid for websocket net.Conn +func (t TCPOptions) ForWebsocket() TCPOptions { + return TCPOptions{ + UserTimeout: t.UserTimeout, + Interface: t.Interface, + SendBufSize: t.SendBufSize, + RecvBufSize: t.RecvBufSize, + NoDelay: true, + } +} + +// newHTTPListener - creates new httpListener object which is interface compatible to net.Listener. +// httpListener is capable to +// * listen to multiple addresses +// * controls incoming connections only doing HTTP protocol +func newHTTPListener(ctx context.Context, serverAddrs []string, opts TCPOptions) (listener *httpListener, listenErrs []error) { + listeners := make([]net.Listener, 0, len(serverAddrs)) + listenErrs = make([]error, len(serverAddrs)) + + if opts.Trace == nil { + opts.Trace = func(msg string) {} // Noop if not defined. + } + + // Unix listener with special TCP options. + listenCfg := net.ListenConfig{ + Control: setTCPParametersFn(opts), + } + + for i, serverAddr := range serverAddrs { + l, e := listenCfg.Listen(ctx, "tcp", serverAddr) + if e != nil { + opts.Trace("listenCfg.Listen: " + e.Error()) + + listenErrs[i] = e + continue + } + opts.Trace("adding listener to " + l.Addr().String()) + + listeners = append(listeners, l) + } + + if len(listeners) == 0 { + // No listeners initialized, no need to continue + return listener, listenErrs + } + listeners = slices.Clip(listeners) + + ctx, cancel := context.WithCancel(ctx) + listener = &httpListener{ + listeners: listeners, + acceptCh: make(chan acceptResult, len(listeners)), + opts: opts, + ctxDoneCh: ctx.Done(), + ctxCanceler: cancel, + } + opts.Trace(fmt.Sprintf("opening %d listeners", len(listener.listeners))) + listener.start() + + return listener, listenErrs +} diff --git a/internal/http/listener_test.go b/internal/http/listener_test.go new file mode 100644 index 00000000..b1ffdb7a --- /dev/null +++ b/internal/http/listener_test.go @@ -0,0 +1,328 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package http + +import ( + "crypto/tls" + "net" + "runtime" + "strconv" + "strings" + "sync/atomic" + "testing" + "time" + + "github.com/minio/minio-go/v7/pkg/set" +) + +var serverPort uint32 = 60000 + +var getCert = func(*tls.ClientHelloInfo) (*tls.Certificate, error) { + certificate, err := getTLSCert() + if err != nil { + return nil, err + } + return &certificate, nil +} + +func getTLSCert() (tls.Certificate, error) { + keyPEMBlock := []byte(`-----BEGIN RSA PRIVATE KEY----- +MIIEpAIBAAKCAQEApEkbPrT6wzcWK1W5atQiGptvuBsRdf8MCg4u6SN10QbslA5k +6BYRdZfFeRpwAwYyzkumug6+eBJatDZEd7+0FF86yxB7eMTSiHKRZ5Mi5ZyCFsez +dndknGBeK6I80s1jd5ZsLLuMKErvbNwSbfX+X6d2mBeYW8Scv9N+qYnNrHHHohvX +oxy1gZ18EhhogQhrD22zaqg/jtmOT8ImUiXzB1mKInt2LlSkoRYuBzepkDJrsE1L +/cyYZbtcO/ASDj+/qQAuQ66v9pNyJkIQ7bDOUyxaT5Hx9XvbqI1OqUVAdGLLi+eZ +IFguFyYd0lemwdN/IDvxftzegTO3cO0D28d1UQIDAQABAoIBAB42x8j3lerTNcOQ +h4JLM157WcedSs/NsVQkGaKM//0KbfYo04wPivR6jjngj9suh6eDKE2tqoAAuCfO +lzcCzca1YOW5yUuDv0iS8YT//XoHF7HC1pGiEaHk40zZEKCgX3u98XUkpPlAFtqJ +euY4SKkk7l24cS/ncACjj/b0PhxJoT/CncuaaJKqmCc+vdL4wj1UcrSNPZqRjDR/ +sh5DO0LblB0XrqVjpNxqxM60/IkbftB8YTnyGgtO2tbTPr8KdQ8DhHQniOp+WEPV +u/iXt0LLM7u62LzadkGab2NDWS3agnmdvw2ADtv5Tt8fZ7WnPqiOpNyD5Bv1a3/h +YBw5HsUCgYEA0Sfv6BiSAFEby2KusRoq5UeUjp/SfL7vwpO1KvXeiYkPBh2XYVq2 +azMnOw7Rz5ixFhtUtto2XhYdyvvr3dZu1fNHtxWo9ITBivqTGGRNwfiaQa58Bugo +gy7vCdIE/f6xE5LYIovBnES2vs/ZayMyhTX84SCWd0pTY0kdDA8ePGsCgYEAyRSA +OTzX43KUR1G/trpuM6VBc0W6YUNYzGRa1TcUxBP4K7DfKMpPGg6ulqypfoHmu8QD +L+z+iQmG9ySSuvScIW6u8LgkrTwZga8y2eb/A2FAVYY/bnelef1aMkis+bBX2OQ4 +QAg2uq+pkhpW1k5NSS9lVCPkj4e5Ur9RCm9fRDMCgYAf3CSIR03eLHy+Y37WzXSh +TmELxL6sb+1Xx2Y+cAuBCda3CMTpeIb3F2ivb1d4dvrqsikaXW0Qse/B3tQUC7kA +cDmJYwxEiwBsajUD7yuFE5hzzt9nse+R5BFXfp1yD1zr7V9tC7rnUfRAZqrozgjB +D/NAW9VvwGupYRbCon7plwKBgQCRPfeoYGRoa9ji8w+Rg3QaZeGyy8jmfGjlqg9a +NyEOyIXXuThYFFmyrqw5NZpwQJBTTDApK/xnK7SLS6WY2Rr1oydFxRzo7KJX5B7M ++md1H4gCvqeOuWmThgbij1AyQsgRaDehOM2fZ0cKu2/B+Gkm1c9RSWPMsPKR7JMz +AGNFtQKBgQCRCFIdGJHnvz35vJfLoihifCejBWtZbAnZoBHpF3xMCtV755J96tUf +k1Tv9hz6WfSkOSlwLq6eGZY2dCENJRW1ft1UelpFvCjbfrfLvoFFLs3gu0lfqXHi +CS6fjhn9Ahvz10yD6fd4ixRUjoJvULzI0Sxc1O95SYVF1lIAuVr9Hw== +-----END RSA PRIVATE KEY-----`) + certPEMBlock := []byte(`-----BEGIN CERTIFICATE----- +MIIDXTCCAkWgAwIBAgIJAKlqK5HKlo9MMA0GCSqGSIb3DQEBCwUAMEUxCzAJBgNV +BAYTAkFVMRMwEQYDVQQIDApTb21lLVN0YXRlMSEwHwYDVQQKDBhJbnRlcm5ldCBX +aWRnaXRzIFB0eSBMdGQwHhcNMTcwNjE5MTA0MzEyWhcNMjcwNjE3MTA0MzEyWjBF +MQswCQYDVQQGEwJBVTETMBEGA1UECAwKU29tZS1TdGF0ZTEhMB8GA1UECgwYSW50 +ZXJuZXQgV2lkZ2l0cyBQdHkgTHRkMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIB +CgKCAQEApEkbPrT6wzcWK1W5atQiGptvuBsRdf8MCg4u6SN10QbslA5k6BYRdZfF +eRpwAwYyzkumug6+eBJatDZEd7+0FF86yxB7eMTSiHKRZ5Mi5ZyCFsezdndknGBe +K6I80s1jd5ZsLLuMKErvbNwSbfX+X6d2mBeYW8Scv9N+qYnNrHHHohvXoxy1gZ18 +EhhogQhrD22zaqg/jtmOT8ImUiXzB1mKInt2LlSkoRYuBzepkDJrsE1L/cyYZbtc +O/ASDj+/qQAuQ66v9pNyJkIQ7bDOUyxaT5Hx9XvbqI1OqUVAdGLLi+eZIFguFyYd +0lemwdN/IDvxftzegTO3cO0D28d1UQIDAQABo1AwTjAdBgNVHQ4EFgQUqMVdMIA1 +68Dv+iwGugAaEGUSd0IwHwYDVR0jBBgwFoAUqMVdMIA168Dv+iwGugAaEGUSd0Iw +DAYDVR0TBAUwAwEB/zANBgkqhkiG9w0BAQsFAAOCAQEAjQVoqRv2HlE5PJIX/qk5 +oMOKZlHTyJP+s2HzOOVt+eCE/jNdfC7+8R/HcPldQs7p9GqH2F6hQ9aOtDhJVEaU +pjxCi4qKeZ1kWwqv8UMBXW92eHGysBvE2Gmm/B1JFl8S2GR5fBmheZVnYW893MoI +gp+bOoCcIuMJRqCra4vJgrOsQjgRElQvd2OlP8qQzInf/fRqO/AnZPwMkGr3+KZ0 +BKEOXtmSZaPs3xEsnvJd8wrTgA0NQK7v48E+gHSXzQtaHmOLqisRXlUOu2r1gNCJ +rr3DRiUP6V/10CZ/ImeSJ72k69VuTw9vq2HzB4x6pqxF2X7JQSLUCS2wfNN13N0d +9A== +-----END CERTIFICATE-----`) + + return tls.X509KeyPair(certPEMBlock, keyPEMBlock) +} + +func getNextPort() string { + return strconv.Itoa(int(atomic.AddUint32(&serverPort, 1))) +} + +func getNonLoopBackIP(t *testing.T) string { + localIP4 := set.NewStringSet() + addrs, err := net.InterfaceAddrs() + if err != nil { + t.Fatalf("%s. Unable to get IP addresses of this host.", err) + } + + for _, addr := range addrs { + var ip net.IP + switch v := addr.(type) { + case *net.IPNet: + ip = v.IP + case *net.IPAddr: + ip = v.IP + } + + if ip.To4() != nil { + localIP4.Add(ip.String()) + } + } + + // Filter ipList by IPs those do not start with '127.'. + nonLoopBackIPs := localIP4.FuncMatch(func(ip string, matchString string) bool { + return !strings.HasPrefix(ip, "127.") + }, "") + if len(nonLoopBackIPs) == 0 { + t.Fatalf("No non-loop back IP address found for this host") + } + nonLoopBackIP := nonLoopBackIPs.ToSlice()[0] + return nonLoopBackIP +} + +func TestNewHTTPListener(t *testing.T) { + testCases := []struct { + serverAddrs []string + tcpKeepAliveTimeout time.Duration + readTimeout time.Duration + writeTimeout time.Duration + expectedListenErrs []bool + }{ + {[]string{"93.184.216.34:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{true}}, // 1 + {[]string{"example.org:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{true}}, // 2 + {[]string{"unknown-host"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{true}}, // 3 + {[]string{"unknown-host:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{true}}, // 4 + {[]string{"localhost:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{false}}, // 5 + {[]string{"localhost:65432", "93.184.216.34:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{false, true}}, // 6 + {[]string{"localhost:65432", "unknown-host:65432"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{false, true}}, // 7 + {[]string{"[::1:65432", "unknown-host:-1"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{true, true}}, // 7 + {[]string{"localhost:0"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{false}}, // 8 + {[]string{"localhost:0"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{false}}, // 9 + {[]string{"[::1]:3737", "127.0.0.1:90900"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{false, true}}, // 10 + {[]string{"[::1]:3737", "localhost:0"}, time.Duration(0), time.Duration(0), time.Duration(0), []bool{false, false}}, // 10 + } + + for testIdx, testCase := range testCases { + listener, listenErrs := newHTTPListener(t.Context(), + testCase.serverAddrs, + TCPOptions{}, + ) + for i, expectedListenErr := range testCase.expectedListenErrs { + if !expectedListenErr { + if listenErrs[i] != nil { + t.Fatalf("Test %d:, listenErrs[%d] error: expected = , got = %v", testIdx+1, i, listenErrs[i]) + } + } else if listenErrs[i] == nil { + t.Fatalf("Test %d: listenErrs[%d]: expected = %v, got = ", testIdx+1, i, expectedListenErr) + } + } + if listener != nil { + listener.Close() + } + } +} + +func TestHTTPListenerStartClose(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + + nonLoopBackIP := getNonLoopBackIP(t) + + testCases := []struct { + serverAddrs []string + }{ + {[]string{"localhost:0"}}, + {[]string{nonLoopBackIP + ":0"}}, + {[]string{"127.0.0.1:0", nonLoopBackIP + ":0"}}, + {[]string{"localhost:0"}}, + {[]string{nonLoopBackIP + ":0"}}, + {[]string{"127.0.0.1:0", nonLoopBackIP + ":0"}}, + } + +nextTest: + for i, testCase := range testCases { + listener, errs := newHTTPListener(t.Context(), + testCase.serverAddrs, + TCPOptions{}, + ) + for _, err := range errs { + if err != nil { + if strings.Contains(err.Error(), "The requested address is not valid in its context") { + // Ignore if IP is unbindable. + continue nextTest + } + if strings.Contains(err.Error(), "bind: address already in use") { + continue nextTest + } + t.Fatalf("Test %d: error: expected = , got = %v", i+1, err) + } + } + + for _, serverAddr := range listener.Addrs() { + conn, err := net.Dial("tcp", serverAddr.String()) + if err != nil { + t.Fatalf("Test %d: error: expected = , got = %v", i+1, err) + } + conn.Close() + } + + listener.Close() + } +} + +func TestHTTPListenerAddr(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + + nonLoopBackIP := getNonLoopBackIP(t) + var casePorts []string + for range 6 { + casePorts = append(casePorts, getNextPort()) + } + + testCases := []struct { + serverAddrs []string + expectedAddr string + }{ + {[]string{"localhost:" + casePorts[0]}, "127.0.0.1:" + casePorts[0]}, + {[]string{nonLoopBackIP + ":" + casePorts[1]}, nonLoopBackIP + ":" + casePorts[1]}, + {[]string{"127.0.0.1:" + casePorts[2], nonLoopBackIP + ":" + casePorts[2]}, "0.0.0.0:" + casePorts[2]}, + {[]string{"localhost:" + casePorts[3]}, "127.0.0.1:" + casePorts[3]}, + {[]string{nonLoopBackIP + ":" + casePorts[4]}, nonLoopBackIP + ":" + casePorts[4]}, + {[]string{"127.0.0.1:" + casePorts[5], nonLoopBackIP + ":" + casePorts[5]}, "0.0.0.0:" + casePorts[5]}, + } + +nextTest: + for i, testCase := range testCases { + listener, errs := newHTTPListener(t.Context(), + testCase.serverAddrs, + TCPOptions{}, + ) + for _, err := range errs { + if err != nil { + if strings.Contains(err.Error(), "The requested address is not valid in its context") { + // Ignore if IP is unbindable. + continue nextTest + } + if strings.Contains(err.Error(), "bind: address already in use") { + continue nextTest + } + t.Fatalf("Test %d: error: expected = , got = %v", i+1, err) + } + } + + addr := listener.Addr() + if addr.String() != testCase.expectedAddr { + t.Fatalf("Test %d: addr: expected = %v, got = %v", i+1, testCase.expectedAddr, addr) + } + + listener.Close() + } +} + +func TestHTTPListenerAddrs(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip() + } + + nonLoopBackIP := getNonLoopBackIP(t) + var casePorts []string + for range 6 { + casePorts = append(casePorts, getNextPort()) + } + + testCases := []struct { + serverAddrs []string + expectedAddrs set.StringSet + }{ + {[]string{"localhost:" + casePorts[0]}, set.CreateStringSet("127.0.0.1:" + casePorts[0])}, + {[]string{nonLoopBackIP + ":" + casePorts[1]}, set.CreateStringSet(nonLoopBackIP + ":" + casePorts[1])}, + {[]string{"127.0.0.1:" + casePorts[2], nonLoopBackIP + ":" + casePorts[2]}, set.CreateStringSet("127.0.0.1:"+casePorts[2], nonLoopBackIP+":"+casePorts[2])}, + {[]string{"localhost:" + casePorts[3]}, set.CreateStringSet("127.0.0.1:" + casePorts[3])}, + {[]string{nonLoopBackIP + ":" + casePorts[4]}, set.CreateStringSet(nonLoopBackIP + ":" + casePorts[4])}, + {[]string{"127.0.0.1:" + casePorts[5], nonLoopBackIP + ":" + casePorts[5]}, set.CreateStringSet("127.0.0.1:"+casePorts[5], nonLoopBackIP+":"+casePorts[5])}, + } + +nextTest: + for i, testCase := range testCases { + listener, errs := newHTTPListener(t.Context(), + testCase.serverAddrs, + TCPOptions{}, + ) + for _, err := range errs { + if err != nil { + if strings.Contains(err.Error(), "The requested address is not valid in its context") { + // Ignore if IP is unbindable. + continue nextTest + } + if strings.Contains(err.Error(), "bind: address already in use") { + continue nextTest + } + t.Fatalf("Test %d: error: expected = , got = %v", i+1, err) + } + } + + addrs := listener.Addrs() + addrSet := set.NewStringSet() + for _, addr := range addrs { + addrSet.Add(addr.String()) + } + + if !addrSet.Equals(testCase.expectedAddrs) { + t.Fatalf("Test %d: addr: expected = %v, got = %v", i+1, testCase.expectedAddrs, addrs) + } + + listener.Close() + } +} diff --git a/internal/http/request-recorder.go b/internal/http/request-recorder.go new file mode 100644 index 00000000..0e3df647 --- /dev/null +++ b/internal/http/request-recorder.go @@ -0,0 +1,72 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package http + +import ( + "bytes" + "io" +) + +// RequestRecorder - records the +// of a given io.Reader +type RequestRecorder struct { + // Data source to record + io.Reader + // Response body should be logged + LogBody bool + + // internal recording buffer + buf bytes.Buffer + // total bytes read including header size + bytesRead int +} + +// Close is a no operation closer +func (r *RequestRecorder) Close() error { + // no-op + return nil +} + +// Read reads from the internal reader and counts/save the body in the memory +func (r *RequestRecorder) Read(p []byte) (n int, err error) { + n, err = r.Reader.Read(p) + r.bytesRead += n + + if r.LogBody { + r.buf.Write(p[:n]) + } + if err != nil { + return n, err + } + return n, err +} + +// Size returns the body size if the currently read object +func (r *RequestRecorder) Size() int { + return r.bytesRead +} + +// Data returns the bytes that were recorded. +func (r *RequestRecorder) Data() []byte { + // If body logging is enabled then we return the actual body + if r.LogBody { + return r.buf.Bytes() + } + // ... otherwise we return placeholder + return blobBody +} diff --git a/internal/http/response-recorder.go b/internal/http/response-recorder.go new file mode 100644 index 00000000..d6a397dc --- /dev/null +++ b/internal/http/response-recorder.go @@ -0,0 +1,196 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package http + +import ( + "bufio" + "bytes" + "errors" + "fmt" + "io" + "net" + "net/http" + "time" + + "github.com/klauspost/compress/gzip" +) + +// ResponseRecorder - is a wrapper to trap the http response +// status code and to record the response body +type ResponseRecorder struct { + http.ResponseWriter + io.ReaderFrom + StatusCode int + // Log body of 4xx or 5xx responses + LogErrBody bool + // Log body of all responses + LogAllBody bool + + ttfbHeader time.Duration + ttfbBody time.Duration + + StartTime time.Time + // number of bytes written + bytesWritten int + // number of bytes of response headers written + headerBytesWritten int + // Internal recording buffer + headers bytes.Buffer + body bytes.Buffer + // Indicate if headers are written in the log + headersLogged bool +} + +// Hijack - hijacks the underlying connection +func (lrw *ResponseRecorder) Hijack() (net.Conn, *bufio.ReadWriter, error) { + hj, ok := lrw.ResponseWriter.(http.Hijacker) + if !ok { + return nil, nil, fmt.Errorf("response writer does not support hijacking. Type is %T", lrw.ResponseWriter) + } + return hj.Hijack() +} + +// TTFB of the request - this function needs to be called +// when the request is finished to provide accurate data +func (lrw *ResponseRecorder) TTFB() time.Duration { + if lrw.ttfbBody != 0 { + return lrw.ttfbBody + } + return lrw.ttfbHeader +} + +// NewResponseRecorder - returns a wrapped response writer to trap +// http status codes for auditing purposes. +func NewResponseRecorder(w http.ResponseWriter) *ResponseRecorder { + rf, _ := w.(io.ReaderFrom) + return &ResponseRecorder{ + ResponseWriter: w, + ReaderFrom: rf, + StatusCode: http.StatusOK, + StartTime: time.Now().UTC(), + } +} + +// ErrNotImplemented when a functionality is not implemented +var ErrNotImplemented = errors.New("not implemented") + +// ReadFrom implements support for calling internal io.ReaderFrom implementations +// returns an error if the underlying ResponseWriter does not implement io.ReaderFrom +func (lrw *ResponseRecorder) ReadFrom(r io.Reader) (int64, error) { + if lrw.ReaderFrom != nil { + n, err := lrw.ReaderFrom.ReadFrom(r) + lrw.bytesWritten += int(n) + return n, err + } + return 0, ErrNotImplemented +} + +func (lrw *ResponseRecorder) Write(p []byte) (int, error) { + if !lrw.headersLogged { + // We assume the response code to be '200 OK' when WriteHeader() is not called, + // that way following Golang HTTP response behavior. + lrw.WriteHeader(http.StatusOK) + } + n, err := lrw.ResponseWriter.Write(p) + lrw.bytesWritten += n + if lrw.ttfbBody == 0 { + lrw.ttfbBody = time.Now().UTC().Sub(lrw.StartTime) + } + + if (lrw.LogErrBody && lrw.StatusCode >= http.StatusBadRequest) || lrw.LogAllBody { + // If body is > 10MB, drop it. + if lrw.bytesWritten+len(p) > 10<<20 { + lrw.LogAllBody = false + lrw.body = bytes.Buffer{} + } else { + // Always logging error responses. + lrw.body.Write(p) + } + } + if err != nil { + return n, err + } + return n, err +} + +// Write the headers into the given buffer +func (lrw *ResponseRecorder) writeHeaders(w io.Writer, statusCode int, headers http.Header) { + n, _ := fmt.Fprintf(w, "%d %s\n", statusCode, http.StatusText(statusCode)) + lrw.headerBytesWritten += n + for k, v := range headers { + n, _ := fmt.Fprintf(w, "%s: %s\n", k, v[0]) + lrw.headerBytesWritten += n + } +} + +// blobBody returns a dummy body placeholder for blob (binary stream) +var blobBody = []byte("") + +// gzippedBody returns a dummy body placeholder for gzipped content +var gzippedBody = []byte("") + +// Body - Return response body. +func (lrw *ResponseRecorder) Body() []byte { + if lrw.Header().Get("Content-Encoding") == "gzip" { + if lrw.body.Len() > 1<<20 { + return gzippedBody + } + r, err := gzip.NewReader(&lrw.body) + if err != nil { + return gzippedBody + } + defer r.Close() + b, _ := io.ReadAll(io.LimitReader(r, 10<<20)) + return b + } + // If there was an error response or body logging is enabled + // then we return the body contents + if (lrw.LogErrBody && lrw.StatusCode >= http.StatusBadRequest) || lrw.LogAllBody { + return lrw.body.Bytes() + } + // ... otherwise we return the place holder + return blobBody +} + +// WriteHeader - writes http status code +func (lrw *ResponseRecorder) WriteHeader(code int) { + if !lrw.headersLogged { + lrw.ttfbHeader = time.Now().UTC().Sub(lrw.StartTime) + lrw.StatusCode = code + lrw.writeHeaders(&lrw.headers, code, lrw.Header()) + lrw.headersLogged = true + lrw.ResponseWriter.WriteHeader(code) + } +} + +// Flush - Calls the underlying Flush. +func (lrw *ResponseRecorder) Flush() { + if flusher, ok := lrw.ResponseWriter.(http.Flusher); ok { + flusher.Flush() + } +} + +// Size - returns the number of bytes written +func (lrw *ResponseRecorder) Size() int { + return lrw.bytesWritten +} + +// HeaderSize - returns the number of bytes of response headers written +func (lrw *ResponseRecorder) HeaderSize() int { + return lrw.headerBytesWritten +} diff --git a/internal/http/server.go b/internal/http/server.go new file mode 100644 index 00000000..2934fda6 --- /dev/null +++ b/internal/http/server.go @@ -0,0 +1,238 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package http + +import ( + "context" + "crypto/tls" + "errors" + "log" + "net" + "net/http" + "sync" + "sync/atomic" + "time" + + "github.com/dustin/go-humanize" +) + +var ( + // GlobalMinIOVersion - is sent in the header to all http targets + GlobalMinIOVersion string + + // GlobalDeploymentID - is sent in the header to all http targets + GlobalDeploymentID string +) + +const ( + // DefaultIdleTimeout for idle inactive connections + DefaultIdleTimeout = 30 * time.Second + + // DefaultReadHeaderTimeout for very slow inactive connections + DefaultReadHeaderTimeout = 30 * time.Second + + // DefaultMaxHeaderBytes - default maximum HTTP header size in bytes. + DefaultMaxHeaderBytes = 1 * humanize.MiByte +) + +// Server - extended http.Server supports multiple addresses to serve and enhanced connection handling. +type Server struct { + http.Server + Addrs []string // addresses on which the server listens for new connection. + TCPOptions TCPOptions // all the configurable TCP conn specific configurable options. + listenerMutex sync.Mutex // to guard 'listener' field. + listener *httpListener // HTTP listener for all 'Addrs' field. + inShutdown uint32 // indicates whether the server is in shutdown or not + requestCount int32 // counter holds no. of request in progress. +} + +// GetRequestCount - returns number of request in progress. +func (srv *Server) GetRequestCount() int { + return int(atomic.LoadInt32(&srv.requestCount)) +} + +// Init - init HTTP server +func (srv *Server) Init(listenCtx context.Context, listenErrCallback func(listenAddr string, err error)) (serve func() error, err error) { + // Take a copy of server fields. + var tlsConfig *tls.Config + if srv.TLSConfig != nil { + tlsConfig = srv.TLSConfig.Clone() + } + handler := srv.Handler // if srv.Handler holds non-synced state -> possible data race + + // Create new HTTP listener. + var listener *httpListener + listener, listenErrs := newHTTPListener( + listenCtx, + srv.Addrs, + srv.TCPOptions, + ) + + var interfaceFound bool + for i := range listenErrs { + if listenErrs[i] != nil { + listenErrCallback(srv.Addrs[i], listenErrs[i]) + } else { + interfaceFound = true + } + } + if !interfaceFound { + return nil, errors.New("no available interface found") + } + + // Wrap given handler to do additional + // * return 503 (service unavailable) if the server in shutdown. + wrappedHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // If server is in shutdown. + if atomic.LoadUint32(&srv.inShutdown) != 0 { + // To indicate disable keep-alive, server is shutting down. + w.Header().Set("Connection", "close") + + // Add 1 minute retry header, incase-client wants to honor it + w.Header().Set(RetryAfter, "60") + + w.WriteHeader(http.StatusServiceUnavailable) + w.Write([]byte(http.ErrServerClosed.Error())) + return + } + + atomic.AddInt32(&srv.requestCount, 1) + defer atomic.AddInt32(&srv.requestCount, -1) + + // Handle request using passed handler. + handler.ServeHTTP(w, r) + }) + + srv.listenerMutex.Lock() + srv.Handler = wrappedHandler + srv.listener = listener + srv.listenerMutex.Unlock() + + var l net.Listener = listener + if tlsConfig != nil { + l = tls.NewListener(listener, tlsConfig) + } + + serve = func() error { + return srv.Serve(l) + } + + return serve, err +} + +// Shutdown - shuts down HTTP server. +func (srv *Server) Shutdown() error { + srv.listenerMutex.Lock() + if srv.listener == nil { + srv.listenerMutex.Unlock() + return http.ErrServerClosed + } + srv.listenerMutex.Unlock() + + if atomic.AddUint32(&srv.inShutdown, 1) > 1 { + // shutdown in progress + return http.ErrServerClosed + } + + // Close underneath HTTP listener. + srv.listenerMutex.Lock() + err := srv.listener.Close() + srv.listenerMutex.Unlock() + if err != nil { + return err + } + + // Wait for opened connection to be closed up to Shutdown timeout. + return nil +} + +// UseIdleTimeout configure idle connection timeout +func (srv *Server) UseIdleTimeout(d time.Duration) *Server { + srv.IdleTimeout = d + return srv +} + +// UseReadTimeout configure connection request read timeout. +func (srv *Server) UseReadTimeout(d time.Duration) *Server { + srv.ReadTimeout = d + return srv +} + +// UseReadHeaderTimeout configure read header timeout +func (srv *Server) UseReadHeaderTimeout(d time.Duration) *Server { + srv.ReadHeaderTimeout = d + return srv +} + +// UseWriteTimeout configure connection response write timeout. +func (srv *Server) UseWriteTimeout(d time.Duration) *Server { + srv.WriteTimeout = d + return srv +} + +// UseHandler configure final handler for this HTTP *Server +func (srv *Server) UseHandler(h http.Handler) *Server { + srv.Handler = h + return srv +} + +// UseTLSConfig pass configured TLSConfig for this HTTP *Server +func (srv *Server) UseTLSConfig(cfg *tls.Config) *Server { + srv.TLSConfig = cfg + return srv +} + +// UseBaseContext use custom base context for this HTTP *Server +func (srv *Server) UseBaseContext(ctx context.Context) *Server { + srv.BaseContext = func(listener net.Listener) context.Context { + return ctx + } + return srv +} + +// UseCustomLogger use customized logger for this HTTP *Server +func (srv *Server) UseCustomLogger(l *log.Logger) *Server { + srv.ErrorLog = l + return srv +} + +// UseTCPOptions use custom TCP options on raw socket +func (srv *Server) UseTCPOptions(opts TCPOptions) *Server { + srv.TCPOptions = opts + return srv +} + +// NewServer - creates new HTTP server using given arguments. +func NewServer(addrs []string) *Server { + httpServer := &Server{ + Addrs: addrs, + } + // This is not configurable for now. + httpServer.MaxHeaderBytes = DefaultMaxHeaderBytes + return httpServer +} + +// SetMinIOVersion -- MinIO version from the main package is set here +func SetMinIOVersion(version string) { + GlobalMinIOVersion = version +} + +// SetDeploymentID -- Deployment Id from the main package is set here +func SetDeploymentID(deploymentID string) { + GlobalDeploymentID = deploymentID +} diff --git a/internal/http/server_test.go b/internal/http/server_test.go new file mode 100644 index 00000000..27f260ae --- /dev/null +++ b/internal/http/server_test.go @@ -0,0 +1,78 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package http + +import ( + "crypto/tls" + "fmt" + "net/http" + "reflect" + "testing" + + "github.com/minio/pkg/v3/certs" +) + +func TestNewServer(t *testing.T) { + nonLoopBackIP := getNonLoopBackIP(t) + handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + fmt.Fprintf(w, "Hello, world") + }) + + testCases := []struct { + addrs []string + handler http.Handler + certFn certs.GetCertificateFunc + }{ + {[]string{"127.0.0.1:9000"}, handler, nil}, + {[]string{nonLoopBackIP + ":9000"}, handler, nil}, + {[]string{"127.0.0.1:9000", nonLoopBackIP + ":9000"}, handler, nil}, + {[]string{"127.0.0.1:9000"}, handler, getCert}, + {[]string{nonLoopBackIP + ":9000"}, handler, getCert}, + {[]string{"127.0.0.1:9000", nonLoopBackIP + ":9000"}, handler, getCert}, + } + + for i, testCase := range testCases { + server := NewServer(testCase.addrs). + UseHandler(testCase.handler) + if testCase.certFn != nil { + server = server.UseTLSConfig(&tls.Config{ + PreferServerCipherSuites: true, + GetCertificate: testCase.certFn, + }) + } + if server == nil { + t.Fatalf("Case %v: server: expected: , got: ", (i + 1)) + } else if !reflect.DeepEqual(server.Addrs, testCase.addrs) { + t.Fatalf("Case %v: server.Addrs: expected: %v, got: %v", (i + 1), testCase.addrs, server.Addrs) + } + + if testCase.certFn == nil { + if server.TLSConfig != nil { + t.Fatalf("Case %v: server.TLSConfig: expected: , got: %v", (i + 1), server.TLSConfig) + } + } else { + if server.TLSConfig == nil { + t.Fatalf("Case %v: server.TLSConfig: expected: , got: ", (i + 1)) + } + } + + if server.MaxHeaderBytes != DefaultMaxHeaderBytes { + t.Fatalf("Case %v: server.MaxHeaderBytes: expected: %v, got: %v", (i + 1), DefaultMaxHeaderBytes, server.MaxHeaderBytes) + } + } +} diff --git a/internal/http/transports.go b/internal/http/transports.go new file mode 100644 index 00000000..fba86bd3 --- /dev/null +++ b/internal/http/transports.go @@ -0,0 +1,205 @@ +// Copyright (c) 2015-2022 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package http + +import ( + "context" + "crypto/tls" + "crypto/x509" + "net/http" + "syscall" + "time" + + "github.com/minio/pkg/v3/certs" +) + +// tlsClientSessionCacheSize is the cache size for client sessions. +var tlsClientSessionCacheSize = 100 + +const ( + // WriteBufferSize 64KiB moving up from 4KiB default + WriteBufferSize = 64 << 10 + + // ReadBufferSize 64KiB moving up from 4KiB default + ReadBufferSize = 64 << 10 +) + +// ConnSettings - contains connection settings. +type ConnSettings struct { + DialContext DialContext // Custom dialContext, DialTimeout is ignored if this is already setup. + LookupHost LookupHost // Custom lookupHost, is nil on containerized deployments. + DialTimeout time.Duration + + // TLS Settings + RootCAs *x509.CertPool + CipherSuites []uint16 + CurvePreferences []tls.CurveID + + // HTTP2 + EnableHTTP2 bool + + // TCP Options + TCPOptions TCPOptions +} + +func (s ConnSettings) getDefaultTransport(maxIdleConnsPerHost int) *http.Transport { + if maxIdleConnsPerHost <= 0 { + maxIdleConnsPerHost = 1024 + } + + dialContext := s.DialContext + if dialContext == nil { + dialContext = DialContextWithLookupHost(s.LookupHost, NewInternodeDialContext(s.DialTimeout, s.TCPOptions)) + } + + tlsClientConfig := tls.Config{ + RootCAs: s.RootCAs, + CipherSuites: s.CipherSuites, + CurvePreferences: s.CurvePreferences, + ClientSessionCache: tls.NewLRUClientSessionCache(tlsClientSessionCacheSize), + } + + // For more details about various values used here refer + // https://golang.org/pkg/net/http/#Transport documentation + tr := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + DialContext: dialContext, + MaxIdleConnsPerHost: maxIdleConnsPerHost, + WriteBufferSize: WriteBufferSize, + ReadBufferSize: ReadBufferSize, + IdleConnTimeout: 15 * time.Second, + ResponseHeaderTimeout: 15 * time.Minute, // Conservative timeout is the default (for MinIO internode) + TLSHandshakeTimeout: 10 * time.Second, + TLSClientConfig: &tlsClientConfig, + ForceAttemptHTTP2: s.EnableHTTP2, + // Go net/http automatically unzip if content-type is + // gzip disable this feature, as we are always interested + // in raw stream. + DisableCompression: true, + } + + // https://github.com/golang/go/issues/23559 + // https://github.com/golang/go/issues/42534 + // https://github.com/golang/go/issues/43989 + // https://github.com/golang/go/issues/33425 + // https://github.com/golang/go/issues/29246 + // if tlsConfig != nil { + // trhttp2, _ := http2.ConfigureTransports(tr) + // if trhttp2 != nil { + // // ReadIdleTimeout is the timeout after which a health check using ping + // // frame will be carried out if no frame is received on the + // // connection. 5 minutes is sufficient time for any idle connection. + // trhttp2.ReadIdleTimeout = 5 * time.Minute + // // PingTimeout is the timeout after which the connection will be closed + // // if a response to Ping is not received. + // trhttp2.PingTimeout = dialTimeout + // // DisableCompression, if true, prevents the Transport from + // // requesting compression with an "Accept-Encoding: gzip" + // trhttp2.DisableCompression = true + // } + // } + + return tr +} + +// NewInternodeHTTPTransport returns transport for internode MinIO connections. +func (s ConnSettings) NewInternodeHTTPTransport(maxIdleConnsPerHost int) func() http.RoundTripper { + tr := s.getDefaultTransport(maxIdleConnsPerHost) + + // Settings specific to internode requests. + tr.TLSHandshakeTimeout = 15 * time.Second + + return func() http.RoundTripper { + return tr + } +} + +// NewCustomHTTPProxyTransport is used only for proxied requests, specifically +// only supports HTTP/1.1 +func (s ConnSettings) NewCustomHTTPProxyTransport() func() *http.Transport { + s.EnableHTTP2 = false + tr := s.getDefaultTransport(0) + + // Settings specific to proxied requests. + tr.ResponseHeaderTimeout = 30 * time.Minute + + return func() *http.Transport { + return tr + } +} + +// NewHTTPTransportWithTimeout allows setting a timeout for response headers +func (s ConnSettings) NewHTTPTransportWithTimeout(timeout time.Duration) *http.Transport { + tr := s.getDefaultTransport(0) + + // Settings specific to this transport. + tr.ResponseHeaderTimeout = timeout + return tr +} + +// NewHTTPTransportWithClientCerts returns a new http configuration used for +// communicating with client cert authentication. +func (s ConnSettings) NewHTTPTransportWithClientCerts(ctx context.Context, clientCert, clientKey string) (*http.Transport, error) { + transport := s.NewHTTPTransportWithTimeout(1 * time.Minute) + if clientCert != "" && clientKey != "" { + c, err := certs.NewManager(ctx, clientCert, clientKey, tls.LoadX509KeyPair) + if err != nil { + return nil, err + } + if c != nil { + c.UpdateReloadDuration(10 * time.Second) + c.ReloadOnSignal(syscall.SIGHUP) // allow reloads upon SIGHUP + transport.TLSClientConfig.GetClientCertificate = c.GetClientCertificate + } + } + return transport, nil +} + +// NewRemoteTargetHTTPTransport returns a new http configuration +// used while communicating with the remote replication targets. +func (s ConnSettings) NewRemoteTargetHTTPTransport(insecure bool) func() *http.Transport { + tr := s.getDefaultTransport(0) + + tr.TLSHandshakeTimeout = 10 * time.Second + tr.ResponseHeaderTimeout = 0 + tr.TLSClientConfig.InsecureSkipVerify = insecure + + return func() *http.Transport { + return tr + } +} + +// uaTransport - User-Agent transport +type uaTransport struct { + ua string + rt http.RoundTripper +} + +func (u *uaTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req2 := req.Clone(req.Context()) + req2.Header.Set("User-Agent", u.ua) + return u.rt.RoundTrip(req2) +} + +// WithUserAgent wraps an existing transport with custom User-Agent +func WithUserAgent(rt http.RoundTripper, getUA func() string) http.RoundTripper { + return &uaTransport{ + ua: getUA(), + rt: rt, + } +} diff --git a/internal/init/init.go b/internal/init/init.go new file mode 100644 index 00000000..36b9fe93 --- /dev/null +++ b/internal/init/init.go @@ -0,0 +1,25 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package init + +import "os" + +func init() { + // All MinIO operations must be under UTC. + os.Setenv("TZ", "UTC") +} diff --git a/internal/init/init_darwin_amd64.go b/internal/init/init_darwin_amd64.go new file mode 100644 index 00000000..b72ba2a0 --- /dev/null +++ b/internal/init/init_darwin_amd64.go @@ -0,0 +1,36 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package init + +import ( + "os" + + "github.com/klauspost/cpuid/v2" +) + +func init() { + // All MinIO operations must be under UTC. + os.Setenv("TZ", "UTC") + + // Temporary workaround for + // https://github.com/golang/go/issues/49233 + // Keep until upstream has been fixed. + cpuid.CPU.Disable(cpuid.AVX512F, cpuid.AVX512BW, cpuid.AVX512CD, cpuid.AVX512DQ, + cpuid.AVX512ER, cpuid.AVX512FP16, cpuid.AVX512IFMA, cpuid.AVX512PF, cpuid.AVX512VBMI, + cpuid.AVX512VBMI2, cpuid.AVX512VL, cpuid.AVX512VNNI, cpuid.AVX512VP2INTERSECT, cpuid.AVX512VPOPCNTDQ) +} diff --git a/internal/ioutil/append-file_nix.go b/internal/ioutil/append-file_nix.go new file mode 100644 index 00000000..3f53fb1b --- /dev/null +++ b/internal/ioutil/append-file_nix.go @@ -0,0 +1,47 @@ +//go:build !windows +// +build !windows + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ioutil + +import ( + "io" + "os" +) + +// AppendFile - appends the file "src" to the file "dst" +func AppendFile(dst string, src string, osync bool) error { + flags := os.O_WRONLY | os.O_APPEND | os.O_CREATE + if osync { + flags |= os.O_SYNC + } + appendFile, err := os.OpenFile(dst, flags, 0o666) + if err != nil { + return err + } + defer appendFile.Close() + + srcFile, err := os.Open(src) + if err != nil { + return err + } + defer srcFile.Close() + _, err = io.Copy(appendFile, srcFile) + return err +} diff --git a/internal/ioutil/append-file_windows.go b/internal/ioutil/append-file_windows.go new file mode 100644 index 00000000..9a27eb93 --- /dev/null +++ b/internal/ioutil/append-file_windows.go @@ -0,0 +1,42 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ioutil + +import ( + "io" + "os" + + "github.com/minio/minio/internal/lock" +) + +// AppendFile - appends the file "src" to the file "dst" +func AppendFile(dst string, src string, osync bool) error { + appendFile, err := lock.Open(dst, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0o666) + if err != nil { + return err + } + defer appendFile.Close() + + srcFile, err := lock.Open(src, os.O_RDONLY, 0o666) + if err != nil { + return err + } + defer srcFile.Close() + _, err = io.Copy(appendFile, srcFile) + return err +} diff --git a/internal/ioutil/discard.go b/internal/ioutil/discard.go new file mode 100644 index 00000000..cfd5b8c6 --- /dev/null +++ b/internal/ioutil/discard.go @@ -0,0 +1,40 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ioutil + +import ( + "io" +) + +// Discard is just like io.Discard without the io.ReaderFrom compatible +// implementation which is buggy on NUMA systems, we have to use a simpler +// io.Writer implementation alone avoids also unnecessary buffer copies, +// and as such incurred latencies. +var Discard io.Writer = discard{} + +// discard is /dev/null for Golang. +type discard struct{} + +func (discard) Write(p []byte) (int, error) { + return len(p), nil +} + +// DiscardReader discarded reader +func DiscardReader(r io.Reader) { + Copy(Discard, r) +} diff --git a/internal/ioutil/hardlimitreader.go b/internal/ioutil/hardlimitreader.go new file mode 100644 index 00000000..3f37a294 --- /dev/null +++ b/internal/ioutil/hardlimitreader.go @@ -0,0 +1,56 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Package ioutil implements some I/O utility functions which are not covered +// by the standard library. +package ioutil + +import ( + "errors" + "io" +) + +// ErrOverread is returned to the reader when the hard limit of HardLimitReader is exceeded. +var ErrOverread = errors.New("input provided more bytes than specified") + +// HardLimitReader returns a Reader that reads from r +// but returns an error if the source provides more data than allowed. +// This means the source *will* be overread unless EOF is returned prior. +// The underlying implementation is a *HardLimitedReader. +// This will ensure that at most n bytes are returned and EOF is reached. +func HardLimitReader(r io.Reader, n int64) io.Reader { return &HardLimitedReader{r, n} } + +// A HardLimitedReader reads from R but limits the amount of +// data returned to just N bytes. Each call to Read +// updates N to reflect the new amount remaining. +// Read returns EOF when N <= 0 or when the underlying R returns EOF. +type HardLimitedReader struct { + R io.Reader // underlying reader + N int64 // max bytes remaining +} + +func (l *HardLimitedReader) Read(p []byte) (n int, err error) { + if l.N < 0 { + return 0, ErrOverread + } + n, err = l.R.Read(p) + l.N -= int64(n) + if l.N < 0 { + return 0, ErrOverread + } + return n, err +} diff --git a/internal/ioutil/ioutil.go b/internal/ioutil/ioutil.go new file mode 100644 index 00000000..2f214b77 --- /dev/null +++ b/internal/ioutil/ioutil.go @@ -0,0 +1,440 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +// Package ioutil implements some I/O utility functions which are not covered +// by the standard library. +package ioutil + +import ( + "context" + "errors" + "io" + "os" + "runtime/debug" + "time" + + "github.com/dustin/go-humanize" + "github.com/minio/minio/internal/bpool" + "github.com/minio/minio/internal/disk" +) + +// Block sizes constant. +const ( + SmallBlock = 32 * humanize.KiByte // Default r/w block size for smaller objects. + MediumBlock = 128 * humanize.KiByte // Default r/w block size for medium sized objects. + LargeBlock = 1 * humanize.MiByte // Default r/w block size for normal objects. +) + +// AlignedBytePool is a pool of fixed size aligned blocks +type AlignedBytePool struct { + size int + p bpool.Pool[*[]byte] +} + +// NewAlignedBytePool creates a new pool with the specified size. +func NewAlignedBytePool(sz int) *AlignedBytePool { + return &AlignedBytePool{size: sz, p: bpool.Pool[*[]byte]{New: func() *[]byte { + b := disk.AlignedBlock(sz) + return &b + }}} +} + +// aligned sync.Pool's +var ( + ODirectPoolLarge = NewAlignedBytePool(LargeBlock) + ODirectPoolMedium = NewAlignedBytePool(MediumBlock) + ODirectPoolSmall = NewAlignedBytePool(SmallBlock) +) + +// Get a block. +func (p *AlignedBytePool) Get() *[]byte { + return p.p.Get() +} + +// Put a block. +func (p *AlignedBytePool) Put(pb *[]byte) { + if pb != nil && len(*pb) == p.size { + p.p.Put(pb) + } +} + +// WriteOnCloser implements io.WriteCloser and always +// executes at least one write operation if it is closed. +// +// This can be useful within the context of HTTP. At least +// one write operation must happen to send the HTTP headers +// to the peer. +type WriteOnCloser struct { + io.Writer + hasWritten bool +} + +func (w *WriteOnCloser) Write(p []byte) (int, error) { + w.hasWritten = true + return w.Writer.Write(p) +} + +// Close closes the WriteOnCloser. It behaves like io.Closer. +func (w *WriteOnCloser) Close() error { + if !w.hasWritten { + _, err := w.Write(nil) + if err != nil { + return err + } + } + if closer, ok := w.Writer.(io.Closer); ok { + return closer.Close() + } + return nil +} + +// HasWritten returns true if at least one write operation was performed. +func (w *WriteOnCloser) HasWritten() bool { return w.hasWritten } + +// WriteOnClose takes an io.Writer and returns an ioutil.WriteOnCloser. +func WriteOnClose(w io.Writer) *WriteOnCloser { + return &WriteOnCloser{w, false} +} + +type ioret[V any] struct { + val V + err error +} + +// WithDeadline will execute a function with a deadline and return a value of a given type. +// If the deadline/context passes before the function finishes executing, +// the zero value and the context error is returned. +func WithDeadline[V any](ctx context.Context, timeout time.Duration, work func(ctx context.Context) (result V, err error)) (result V, err error) { + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + + c := make(chan ioret[V], 1) + go func() { + v, err := work(ctx) + c <- ioret[V]{val: v, err: err} + }() + + select { + case v := <-c: + return v.val, v.err + case <-ctx.Done(): + var zero V + return zero, ctx.Err() + } +} + +// DeadlineWorker implements the deadline/timeout resiliency pattern. +type DeadlineWorker struct { + timeout time.Duration +} + +// NewDeadlineWorker constructs a new DeadlineWorker with the given timeout. +// To return values, use the WithDeadline helper instead. +func NewDeadlineWorker(timeout time.Duration) *DeadlineWorker { + dw := &DeadlineWorker{ + timeout: timeout, + } + return dw +} + +// Run runs the given function, passing it a stopper channel. If the deadline passes before +// the function finishes executing, Run returns context.DeadlineExceeded to the caller. +// channel so that the work function can attempt to exit gracefully. +// Multiple calls to Run will run independently of each other. +func (d *DeadlineWorker) Run(work func() error) error { + _, err := WithDeadline[struct{}](context.Background(), d.timeout, func(ctx context.Context) (struct{}, error) { + return struct{}{}, work() + }) + return err +} + +// DeadlineWriter deadline writer with timeout +type DeadlineWriter struct { + io.WriteCloser + timeout time.Duration + err error +} + +// NewDeadlineWriter wraps a writer to make it respect given deadline +// value per Write(). If there is a blocking write, the returned Writer +// will return whenever the timer hits (the return values are n=0 +// and err=context.DeadlineExceeded.) +func NewDeadlineWriter(w io.WriteCloser, timeout time.Duration) io.WriteCloser { + return &DeadlineWriter{WriteCloser: w, timeout: timeout} +} + +func (w *DeadlineWriter) Write(buf []byte) (int, error) { + if w.err != nil { + return 0, w.err + } + + n, err := WithDeadline[int](context.Background(), w.timeout, func(ctx context.Context) (int, error) { + return w.WriteCloser.Write(buf) + }) + w.err = err + return n, err +} + +// Close closer interface to close the underlying closer +func (w *DeadlineWriter) Close() error { + err := w.WriteCloser.Close() + w.err = err + if err == nil { + w.err = errors.New("we are closed") // Avoids any reuse on the Write() side. + } + return err +} + +// LimitWriter implements io.WriteCloser. +// +// This is implemented such that we want to restrict +// an enscapsulated writer upto a certain length +// and skip a certain number of bytes. +type LimitWriter struct { + io.Writer + skipBytes int64 + wLimit int64 +} + +// Write implements the io.Writer interface limiting upto +// configured length, also skips the first N bytes. +func (w *LimitWriter) Write(p []byte) (n int, err error) { + n = len(p) + var n1 int + if w.skipBytes > 0 { + if w.skipBytes >= int64(len(p)) { + w.skipBytes -= int64(len(p)) + return n, nil + } + p = p[w.skipBytes:] + w.skipBytes = 0 + } + if w.wLimit == 0 { + return n, nil + } + if w.wLimit < int64(len(p)) { + n1, err = w.Writer.Write(p[:w.wLimit]) + w.wLimit -= int64(n1) + return n, err + } + n1, err = w.Writer.Write(p) + w.wLimit -= int64(n1) + return n, err +} + +// Close closes the LimitWriter. It behaves like io.Closer. +func (w *LimitWriter) Close() error { + if closer, ok := w.Writer.(io.Closer); ok { + return closer.Close() + } + return nil +} + +// LimitedWriter takes an io.Writer and returns an ioutil.LimitWriter. +func LimitedWriter(w io.Writer, skipBytes int64, limit int64) *LimitWriter { + return &LimitWriter{w, skipBytes, limit} +} + +type nopCloser struct { + io.Writer +} + +func (nopCloser) Close() error { return nil } + +// NopCloser returns a WriteCloser with a no-op Close method wrapping +// the provided Writer w. +func NopCloser(w io.Writer) io.WriteCloser { + return nopCloser{w} +} + +// SkipReader skips a given number of bytes and then returns all +// remaining data. +type SkipReader struct { + io.Reader + + skipCount int64 +} + +func (s *SkipReader) Read(p []byte) (int, error) { + l := int64(len(p)) + if l == 0 { + return 0, nil + } + if s.skipCount > 0 { + tmp := p + if s.skipCount > l && l < SmallBlock { + // We may get a very small buffer, so we grab a temporary buffer. + bufp := ODirectPoolSmall.Get() + tmp = *bufp + defer ODirectPoolSmall.Put(bufp) + l = int64(len(tmp)) + } + for s.skipCount > 0 { + if l > s.skipCount { + l = s.skipCount + } + n, err := s.Reader.Read(tmp[:l]) + if err != nil { + return 0, err + } + s.skipCount -= int64(n) + } + } + return s.Reader.Read(p) +} + +// NewSkipReader - creates a SkipReader +func NewSkipReader(r io.Reader, n int64) io.Reader { + return &SkipReader{r, n} +} + +// writerOnly hides an io.Writer value's optional ReadFrom method +// from io.Copy. +type writerOnly struct { + io.Writer +} + +// Copy is exactly like io.Copy but with reusable buffers. +func Copy(dst io.Writer, src io.Reader) (written int64, err error) { + bufp := ODirectPoolMedium.Get() + defer ODirectPoolMedium.Put(bufp) + buf := *bufp + + return io.CopyBuffer(writerOnly{dst}, src, buf) +} + +// SameFile returns if the files are same. +func SameFile(fi1, fi2 os.FileInfo) bool { + if !os.SameFile(fi1, fi2) { + return false + } + if !fi1.ModTime().Equal(fi2.ModTime()) { + return false + } + if fi1.Mode() != fi2.Mode() { + return false + } + return fi1.Size() == fi2.Size() +} + +// DirectioAlignSize - DirectIO alignment needs to be 4K. Defined here as +// directio.AlignSize is defined as 0 in MacOS causing divide by 0 error. +const DirectioAlignSize = 4096 + +// CopyAligned - copies from reader to writer using the aligned input +// buffer, it is expected that input buffer is page aligned to +// 4K page boundaries. Without passing aligned buffer may cause +// this function to return error. +// +// This code is similar in spirit to io.Copy but it is only to be +// used with DIRECT I/O based file descriptor and it is expected that +// input writer *os.File not a generic io.Writer. Make sure to have +// the file opened for writes with syscall.O_DIRECT flag. +func CopyAligned(w io.Writer, r io.Reader, alignedBuf []byte, totalSize int64, file *os.File) (int64, error) { + if totalSize == 0 { + return 0, nil + } + + var written int64 + for { + buf := alignedBuf + if totalSize > 0 { + remaining := totalSize - written + if remaining < int64(len(buf)) { + buf = buf[:remaining] + } + } + + nr, err := io.ReadFull(r, buf) + eof := errors.Is(err, io.EOF) || errors.Is(err, io.ErrUnexpectedEOF) + if err != nil && !eof { + return written, err + } + + buf = buf[:nr] + var ( + n int + un int + nw int64 + ) + + remain := len(buf) % DirectioAlignSize + if remain == 0 { + // buf is aligned for directio write() + n, err = w.Write(buf) + nw = int64(n) + } else { + if remain < len(buf) { + n, err = w.Write(buf[:len(buf)-remain]) + if err != nil { + return written, err + } + nw = int64(n) + } + + // Disable O_DIRECT on fd's on unaligned buffer + // perform an amortized Fdatasync(fd) on the fd at + // the end, this is performed by the caller before + // closing 'w'. + if err = disk.DisableDirectIO(file); err != nil { + return written, err + } + + // buf is not aligned, hence use writeUnaligned() + // for the remainder + un, err = w.Write(buf[len(buf)-remain:]) + nw += int64(un) + } + + if nw > 0 { + written += nw + } + + if err != nil { + return written, err + } + + if nw != int64(len(buf)) { + return written, io.ErrShortWrite + } + + if totalSize > 0 && written == totalSize { + // we have written the entire stream, return right here. + return written, nil + } + + if eof { + // We reached EOF prematurely but we did not write everything + // that we promised that we would write. + if totalSize > 0 && written != totalSize { + return written, io.ErrUnexpectedEOF + } + return written, nil + } + } +} + +// SafeClose safely closes any channel of any type +func SafeClose[T any](c chan<- T) { + if c != nil { + close(c) + return + } + // Print stack to check who is sending `c` as `nil` + // without crashing the server. + debug.PrintStack() +} diff --git a/internal/ioutil/ioutil_test.go b/internal/ioutil/ioutil_test.go new file mode 100644 index 00000000..b71c2f2e --- /dev/null +++ b/internal/ioutil/ioutil_test.go @@ -0,0 +1,226 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ioutil + +import ( + "bytes" + "context" + "errors" + "io" + "os" + "strings" + "testing" + "time" +) + +type sleepWriter struct { + timeout time.Duration +} + +func (w *sleepWriter) Write(p []byte) (n int, err error) { + time.Sleep(w.timeout) + return len(p), nil +} + +func (w *sleepWriter) Close() error { + return nil +} + +func TestDeadlineWorker(t *testing.T) { + work := NewDeadlineWorker(500 * time.Millisecond) + + err := work.Run(func() error { + time.Sleep(600 * time.Millisecond) + return nil + }) + if err != context.DeadlineExceeded { + t.Error("DeadlineWorker shouldn't be successful - should return context.DeadlineExceeded") + } + + err = work.Run(func() error { + time.Sleep(450 * time.Millisecond) + return nil + }) + if err != nil { + t.Error("DeadlineWorker should succeed") + } +} + +func TestDeadlineWriter(t *testing.T) { + w := NewDeadlineWriter(&sleepWriter{timeout: 500 * time.Millisecond}, 450*time.Millisecond) + _, err := w.Write([]byte("1")) + if err != context.DeadlineExceeded { + t.Error("DeadlineWriter shouldn't be successful - should return context.DeadlineExceeded") + } + _, err = w.Write([]byte("1")) + if err != context.DeadlineExceeded { + t.Error("DeadlineWriter shouldn't be successful - should return context.DeadlineExceeded") + } + w.Close() + w = NewDeadlineWriter(&sleepWriter{timeout: 100 * time.Millisecond}, 600*time.Millisecond) + n, err := w.Write([]byte("abcd")) + w.Close() + if err != nil { + t.Errorf("DeadlineWriter should succeed but failed with %s", err) + } + if n != 4 { + t.Errorf("DeadlineWriter should succeed but should have only written 4 bytes, but returned %d instead", n) + } +} + +func TestCloseOnWriter(t *testing.T) { + writer := WriteOnClose(io.Discard) + if writer.HasWritten() { + t.Error("WriteOnCloser must not be marked as HasWritten") + } + writer.Write(nil) + if !writer.HasWritten() { + t.Error("WriteOnCloser must be marked as HasWritten") + } + + writer = WriteOnClose(io.Discard) + writer.Close() + if !writer.HasWritten() { + t.Error("WriteOnCloser must be marked as HasWritten") + } +} + +// Test for AppendFile. +func TestAppendFile(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + name1 := f.Name() + defer os.Remove(name1) + f.WriteString("aaaaaaaaaa") + f.Close() + + f, err = os.CreateTemp(t.TempDir(), "") + if err != nil { + t.Fatal(err) + } + name2 := f.Name() + defer os.Remove(name2) + f.WriteString("bbbbbbbbbb") + f.Close() + + if err = AppendFile(name1, name2, false); err != nil { + t.Error(err) + } + + b, err := os.ReadFile(name1) + if err != nil { + t.Error(err) + } + + expected := "aaaaaaaaaabbbbbbbbbb" + if string(b) != expected { + t.Errorf("AppendFile() failed, expected: %s, got %s", expected, string(b)) + } +} + +func TestSkipReader(t *testing.T) { + testCases := []struct { + src io.Reader + skipLen int64 + expected string + }{ + {bytes.NewBuffer([]byte("")), 0, ""}, + {bytes.NewBuffer([]byte("")), 1, ""}, + {bytes.NewBuffer([]byte("abc")), 0, "abc"}, + {bytes.NewBuffer([]byte("abc")), 1, "bc"}, + {bytes.NewBuffer([]byte("abc")), 2, "c"}, + {bytes.NewBuffer([]byte("abc")), 3, ""}, + {bytes.NewBuffer([]byte("abc")), 4, ""}, + } + for i, testCase := range testCases { + r := NewSkipReader(testCase.src, testCase.skipLen) + b, err := io.ReadAll(r) + if err != nil { + t.Errorf("Case %d: Unexpected err %v", i, err) + } + if string(b) != testCase.expected { + t.Errorf("Case %d: Got wrong result: %v", i, string(b)) + } + } +} + +func TestSameFile(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "") + if err != nil { + t.Errorf("Error creating tmp file: %v", err) + } + tmpFile := f.Name() + f.Close() + defer os.Remove(f.Name()) + fi1, err := os.Stat(tmpFile) + if err != nil { + t.Fatalf("Error Stat(): %v", err) + } + fi2, err := os.Stat(tmpFile) + if err != nil { + t.Fatalf("Error Stat(): %v", err) + } + if !SameFile(fi1, fi2) { + t.Fatal("Expected the files to be same") + } + if err = os.WriteFile(tmpFile, []byte("aaa"), 0o644); err != nil { + t.Fatal(err) + } + fi2, err = os.Stat(tmpFile) + if err != nil { + t.Fatalf("Error Stat(): %v", err) + } + if SameFile(fi1, fi2) { + t.Fatal("Expected the files not to be same") + } +} + +func TestCopyAligned(t *testing.T) { + f, err := os.CreateTemp(t.TempDir(), "") + if err != nil { + t.Errorf("Error creating tmp file: %v", err) + } + defer f.Close() + defer os.Remove(f.Name()) + + r := strings.NewReader("hello world") + + bufp := ODirectPoolSmall.Get() + defer ODirectPoolSmall.Put(bufp) + + written, err := CopyAligned(f, io.LimitReader(r, 5), *bufp, r.Size(), f) + if !errors.Is(err, io.ErrUnexpectedEOF) { + t.Errorf("Expected io.ErrUnexpectedEOF, but got %v", err) + } + if written != 5 { + t.Errorf("Expected written to be '5', but got %v", written) + } + + f.Seek(0, io.SeekStart) + r.Seek(0, io.SeekStart) + + written, err = CopyAligned(f, r, *bufp, r.Size(), f) + if !errors.Is(err, nil) { + t.Errorf("Expected nil, but got %v", err) + } + if written != r.Size() { + t.Errorf("Expected written to be '%v', but got %v", r.Size(), written) + } +} diff --git a/internal/ioutil/read_file.go b/internal/ioutil/read_file.go new file mode 100644 index 00000000..29d023b2 --- /dev/null +++ b/internal/ioutil/read_file.go @@ -0,0 +1,79 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ioutil + +import ( + "io" + "io/fs" + "os" + + "github.com/minio/minio/internal/disk" +) + +var ( + // OpenFileDirectIO allows overriding default function. + OpenFileDirectIO = disk.OpenFileDirectIO + // OsOpen allows overriding default function. + OsOpen = os.Open + // OsOpenFile allows overriding default function. + OsOpenFile = os.OpenFile +) + +// ReadFileWithFileInfo reads the named file and returns the contents. +// A successful call returns err == nil, not err == EOF. +// Because ReadFile reads the whole file, it does not treat an EOF from Read +// as an error to be reported. +func ReadFileWithFileInfo(name string) ([]byte, fs.FileInfo, error) { + f, err := OsOpenFile(name, readMode, 0o666) + if err != nil { + return nil, nil, err + } + defer f.Close() + + st, err := f.Stat() + if err != nil { + return nil, nil, err + } + + dst := make([]byte, st.Size()) + _, err = io.ReadFull(f, dst) + return dst, st, err +} + +// ReadFile reads the named file and returns the contents. +// A successful call returns err == nil, not err == EOF. +// Because ReadFile reads the whole file, it does not treat an EOF from Read +// as an error to be reported. +// +// passes NOATIME flag for reads on Unix systems to avoid atime updates. +func ReadFile(name string) ([]byte, error) { + // Don't wrap with un-needed buffer. + // Don't use os.ReadFile, since it doesn't pass NO_ATIME when present. + f, err := OsOpenFile(name, readMode, 0o666) + if err != nil { + return nil, err + } + defer f.Close() + st, err := f.Stat() + if err != nil { + return io.ReadAll(f) + } + dst := make([]byte, st.Size()) + _, err = io.ReadFull(f, dst) + return dst, err +} diff --git a/internal/ioutil/read_file_noatime_notsupported.go b/internal/ioutil/read_file_noatime_notsupported.go new file mode 100644 index 00000000..9438371e --- /dev/null +++ b/internal/ioutil/read_file_noatime_notsupported.go @@ -0,0 +1,25 @@ +//go:build windows || darwin || freebsd +// +build windows darwin freebsd + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ioutil + +import "os" + +var readMode = os.O_RDONLY diff --git a/internal/ioutil/read_file_noatime_supported.go b/internal/ioutil/read_file_noatime_supported.go new file mode 100644 index 00000000..3726e44b --- /dev/null +++ b/internal/ioutil/read_file_noatime_supported.go @@ -0,0 +1,27 @@ +//go:build !windows && !darwin && !freebsd +// +build !windows,!darwin,!freebsd + +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ioutil + +import ( + "os" +) + +var readMode = os.O_RDONLY | 0x40000 // read with O_NOATIME diff --git a/internal/ioutil/wait_pipe.go b/internal/ioutil/wait_pipe.go new file mode 100644 index 00000000..67f490ba --- /dev/null +++ b/internal/ioutil/wait_pipe.go @@ -0,0 +1,67 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ioutil + +import ( + "io" + "sync" +) + +// PipeWriter is similar to io.PipeWriter with wait group +type PipeWriter struct { + *io.PipeWriter + once sync.Once + done func() +} + +// CloseWithError close with supplied error the writer end. +func (w *PipeWriter) CloseWithError(err error) error { + err = w.PipeWriter.CloseWithError(err) + w.once.Do(func() { + w.done() + }) + return err +} + +// PipeReader is similar to io.PipeReader with wait group +type PipeReader struct { + *io.PipeReader + wait func() +} + +// CloseWithError close with supplied error the reader end +func (r *PipeReader) CloseWithError(err error) error { + err = r.PipeReader.CloseWithError(err) + r.wait() + return err +} + +// WaitPipe implements wait-group backend io.Pipe to provide +// synchronization between read() end with write() end. +func WaitPipe() (*PipeReader, *PipeWriter) { + r, w := io.Pipe() + var wg sync.WaitGroup + wg.Add(1) + return &PipeReader{ + PipeReader: r, + wait: wg.Wait, + }, &PipeWriter{ + PipeWriter: w, + done: wg.Done, + } +} diff --git a/internal/jwt/parser.go b/internal/jwt/parser.go new file mode 100644 index 00000000..1a4ddb44 --- /dev/null +++ b/internal/jwt/parser.go @@ -0,0 +1,526 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package jwt + +// This file is a re-implementation of the original code here with some +// additional allocation tweaks reproduced using GODEBUG=allocfreetrace=1 +// original file https://github.com/golang-jwt/jwt/blob/main/parser.go +// borrowed under MIT License https://github.com/golang-jwt/jwt/blob/main/LICENSE + +import ( + "bytes" + "crypto" + "crypto/hmac" + "encoding/base64" + "errors" + "fmt" + "hash" + "time" + + "github.com/buger/jsonparser" + "github.com/dustin/go-humanize" + jwtgo "github.com/golang-jwt/jwt/v4" + jsoniter "github.com/json-iterator/go" + "github.com/minio/minio/internal/bpool" +) + +// SigningMethodHMAC - Implements the HMAC-SHA family of signing methods signing methods +// Expects key type of []byte for both signing and validation +type SigningMethodHMAC struct { + Name string + Hash crypto.Hash + HasherPool bpool.Pool[hash.Hash] +} + +// Specific instances for HS256, HS384, HS512 +var ( + SigningMethodHS256 *SigningMethodHMAC + SigningMethodHS384 *SigningMethodHMAC + SigningMethodHS512 *SigningMethodHMAC +) + +const base64BufferSize = 64 * humanize.KiByte + +var ( + base64BufPool bpool.Pool[*[]byte] + hmacSigners []*SigningMethodHMAC +) + +func init() { + base64BufPool = bpool.Pool[*[]byte]{ + New: func() *[]byte { + buf := make([]byte, base64BufferSize) + return &buf + }, + } + + hmacSigners = []*SigningMethodHMAC{ + {Name: "HS256", Hash: crypto.SHA256}, + {Name: "HS384", Hash: crypto.SHA384}, + {Name: "HS512", Hash: crypto.SHA512}, + } + for i := range hmacSigners { + h := hmacSigners[i].Hash + hmacSigners[i].HasherPool.New = func() hash.Hash { + return h.New() + } + } +} + +// HashBorrower allows borrowing hashes and will keep track of them. +func (s *SigningMethodHMAC) HashBorrower() HashBorrower { + return HashBorrower{pool: &s.HasherPool, borrowed: make([]hash.Hash, 0, 2)} +} + +// HashBorrower keeps track of borrowed hashers and allows to return them all. +type HashBorrower struct { + pool *bpool.Pool[hash.Hash] + borrowed []hash.Hash +} + +// Borrow a single hasher. +func (h *HashBorrower) Borrow() hash.Hash { + hasher := h.pool.Get() + h.borrowed = append(h.borrowed, hasher) + hasher.Reset() + return hasher +} + +// ReturnAll will return all borrowed hashes. +func (h *HashBorrower) ReturnAll() { + for _, hasher := range h.borrowed { + h.pool.Put(hasher) + } + h.borrowed = nil +} + +// StandardClaims are basically standard claims with "accessKey" +type StandardClaims struct { + AccessKey string `json:"accessKey,omitempty"` + jwtgo.StandardClaims +} + +// UnmarshalJSON provides custom JSON unmarshal. +// This is mainly implemented for speed. +func (c *StandardClaims) UnmarshalJSON(b []byte) (err error) { + return jsonparser.ObjectEach(b, func(key []byte, value []byte, dataType jsonparser.ValueType, _ int) error { + if len(key) == 0 { + return nil + } + switch key[0] { + case 'a': + if string(key) == "accessKey" { + if dataType != jsonparser.String { + return errors.New("accessKey: Expected string") + } + c.AccessKey, err = jsonparser.ParseString(value) + return err + } + if string(key) == "aud" { + if dataType != jsonparser.String { + return errors.New("aud: Expected string") + } + c.Audience, err = jsonparser.ParseString(value) + return err + } + case 'e': + if string(key) == "exp" { + if dataType != jsonparser.Number { + return errors.New("exp: Expected number") + } + c.ExpiresAt, err = jsonparser.ParseInt(value) + return err + } + case 'i': + if string(key) == "iat" { + if dataType != jsonparser.Number { + return errors.New("exp: Expected number") + } + c.IssuedAt, err = jsonparser.ParseInt(value) + return err + } + if string(key) == "iss" { + if dataType != jsonparser.String { + return errors.New("iss: Expected string") + } + c.Issuer, err = jsonparser.ParseString(value) + return err + } + case 'n': + if string(key) == "nbf" { + if dataType != jsonparser.Number { + return errors.New("nbf: Expected number") + } + c.NotBefore, err = jsonparser.ParseInt(value) + return err + } + case 's': + if string(key) == "sub" { + if dataType != jsonparser.String { + return errors.New("sub: Expected string") + } + c.Subject, err = jsonparser.ParseString(value) + return err + } + } + // Ignore unknown fields + return nil + }) +} + +// MapClaims - implements custom unmarshaller +type MapClaims struct { + AccessKey string `json:"accessKey,omitempty"` + jwtgo.MapClaims +} + +// GetAccessKey will return the access key. +// If nil an empty string will be returned. +func (c *MapClaims) GetAccessKey() string { + if c == nil { + return "" + } + return c.AccessKey +} + +// NewStandardClaims - initializes standard claims +func NewStandardClaims() *StandardClaims { + return &StandardClaims{} +} + +// SetIssuer sets issuer for these claims +func (c *StandardClaims) SetIssuer(issuer string) { + c.Issuer = issuer +} + +// SetAudience sets audience for these claims +func (c *StandardClaims) SetAudience(aud string) { + c.Audience = aud +} + +// SetExpiry sets expiry in unix epoch secs +func (c *StandardClaims) SetExpiry(t time.Time) { + c.ExpiresAt = t.Unix() +} + +// SetAccessKey sets access key as jwt subject and custom +// "accessKey" field. +func (c *StandardClaims) SetAccessKey(accessKey string) { + c.Subject = accessKey + c.AccessKey = accessKey +} + +// Valid - implements https://godoc.org/github.com/golang-jwt/jwt#Claims compatible +// claims interface, additionally validates "accessKey" fields. +func (c *StandardClaims) Valid() error { + if err := c.StandardClaims.Valid(); err != nil { + return err + } + + if c.AccessKey == "" && c.Subject == "" { + return jwtgo.NewValidationError("accessKey/sub missing", + jwtgo.ValidationErrorClaimsInvalid) + } + + return nil +} + +// NewMapClaims - Initializes a new map claims +func NewMapClaims() *MapClaims { + return &MapClaims{MapClaims: jwtgo.MapClaims{}} +} + +// Set Adds new arbitrary claim keys and values. +func (c *MapClaims) Set(key string, val any) { + if c == nil { + return + } + c.MapClaims[key] = val +} + +// Delete deletes a key named key. +func (c *MapClaims) Delete(key string) { + if c == nil { + return + } + delete(c.MapClaims, key) +} + +// Lookup returns the value and if the key is found. +func (c *MapClaims) Lookup(key string) (value string, ok bool) { + if c == nil { + return "", false + } + var vinterface any + vinterface, ok = c.MapClaims[key] + if ok { + value, ok = vinterface.(string) + } + return value, ok +} + +// SetExpiry sets expiry in unix epoch secs +func (c *MapClaims) SetExpiry(t time.Time) { + c.MapClaims["exp"] = t.Unix() +} + +// SetAccessKey sets access key as jwt subject and custom +// "accessKey" field. +func (c *MapClaims) SetAccessKey(accessKey string) { + c.MapClaims["sub"] = accessKey + c.MapClaims["accessKey"] = accessKey +} + +// Valid - implements https://godoc.org/github.com/golang-jwt/jwt#Claims compatible +// claims interface, additionally validates "accessKey" fields. +func (c *MapClaims) Valid() error { + if err := c.MapClaims.Valid(); err != nil { + return err + } + + if c.AccessKey == "" { + return jwtgo.NewValidationError("accessKey/sub missing", + jwtgo.ValidationErrorClaimsInvalid) + } + + return nil +} + +// Map returns underlying low-level map claims. +func (c *MapClaims) Map() map[string]any { + if c == nil { + return nil + } + return c.MapClaims +} + +// MarshalJSON marshals the MapClaims struct +func (c *MapClaims) MarshalJSON() ([]byte, error) { + json := jsoniter.ConfigCompatibleWithStandardLibrary + return json.Marshal(c.MapClaims) +} + +// ParseWithStandardClaims - parse the token string, valid methods. +func ParseWithStandardClaims(tokenStr string, claims *StandardClaims, key []byte) error { + // Key is not provided. + if key == nil { + // keyFunc was not provided, return error. + return jwtgo.NewValidationError("no key was provided.", jwtgo.ValidationErrorUnverifiable) + } + + bufp := base64BufPool.Get() + defer base64BufPool.Put(bufp) + + tokenBuf := base64BufPool.Get() + defer base64BufPool.Put(tokenBuf) + + token := *tokenBuf + // Copy token to buffer, truncate to length. + token = token[:copy(token[:base64BufferSize], tokenStr)] + + signer, err := ParseUnverifiedStandardClaims(token, claims, *bufp) + if err != nil { + return err + } + + i := bytes.LastIndexByte(token, '.') + if i < 0 { + return jwtgo.ErrSignatureInvalid + } + + n, err := base64DecodeBytes(token[i+1:], *bufp) + if err != nil { + return err + } + borrow := signer.HashBorrower() + hasher := hmac.New(borrow.Borrow, key) + hasher.Write(token[:i]) + if !hmac.Equal((*bufp)[:n], hasher.Sum(nil)) { + borrow.ReturnAll() + return jwtgo.ErrSignatureInvalid + } + borrow.ReturnAll() + + if claims.AccessKey == "" && claims.Subject == "" { + return jwtgo.NewValidationError("accessKey/sub missing", + jwtgo.ValidationErrorClaimsInvalid) + } + + // Signature is valid, lets validate the claims for + // other fields such as expiry etc. + return claims.Valid() +} + +// ParseUnverifiedStandardClaims - WARNING: Don't use this method unless you know what you're doing +// +// This method parses the token but doesn't validate the signature. It's only +// ever useful in cases where you know the signature is valid (because it has +// been checked previously in the stack) and you want to extract values from +// it. +func ParseUnverifiedStandardClaims(token []byte, claims *StandardClaims, buf []byte) (*SigningMethodHMAC, error) { + if bytes.Count(token, []byte(".")) != 2 { + return nil, jwtgo.ErrSignatureInvalid + } + + i := bytes.IndexByte(token, '.') + j := bytes.LastIndexByte(token, '.') + + n, err := base64DecodeBytes(token[:i], buf) + if err != nil { + return nil, &jwtgo.ValidationError{Inner: err, Errors: jwtgo.ValidationErrorMalformed} + } + headerDec := buf[:n] + buf = buf[n:] + + alg, _, _, err := jsonparser.Get(headerDec, "alg") + if err != nil { + return nil, &jwtgo.ValidationError{Inner: err, Errors: jwtgo.ValidationErrorMalformed} + } + + n, err = base64DecodeBytes(token[i+1:j], buf) + if err != nil { + return nil, &jwtgo.ValidationError{Inner: err, Errors: jwtgo.ValidationErrorMalformed} + } + + if err = claims.UnmarshalJSON(buf[:n]); err != nil { + return nil, &jwtgo.ValidationError{Inner: err, Errors: jwtgo.ValidationErrorMalformed} + } + + for _, signer := range hmacSigners { + if string(alg) == signer.Name { + return signer, nil + } + } + + return nil, jwtgo.NewValidationError(fmt.Sprintf("signing method (%s) is unavailable.", string(alg)), + jwtgo.ValidationErrorUnverifiable) +} + +// ParseWithClaims - parse the token string, valid methods. +func ParseWithClaims(tokenStr string, claims *MapClaims, fn func(*MapClaims) ([]byte, error)) error { + // Key lookup function has to be provided. + if fn == nil { + // keyFunc was not provided, return error. + return jwtgo.NewValidationError("no Keyfunc was provided.", jwtgo.ValidationErrorUnverifiable) + } + + bufp := base64BufPool.Get() + defer base64BufPool.Put(bufp) + + tokenBuf := base64BufPool.Get() + defer base64BufPool.Put(tokenBuf) + + token := *tokenBuf + // Copy token to buffer, truncate to length. + token = token[:copy(token[:base64BufferSize], tokenStr)] + + signer, err := ParseUnverifiedMapClaims(token, claims, *bufp) + if err != nil { + return err + } + + i := bytes.LastIndexByte(token, '.') + if i < 0 { + return jwtgo.ErrSignatureInvalid + } + + n, err := base64DecodeBytes(token[i+1:], *bufp) + if err != nil { + return err + } + + var ok bool + claims.AccessKey, ok = claims.Lookup("accessKey") + if !ok { + claims.AccessKey, ok = claims.Lookup("sub") + if !ok { + return jwtgo.NewValidationError("accessKey/sub missing", + jwtgo.ValidationErrorClaimsInvalid) + } + } + + // Lookup key from claims, claims may not be valid and may return + // invalid key which is okay as the signature verification will fail. + key, err := fn(claims) + if err != nil { + return err + } + borrow := signer.HashBorrower() + hasher := hmac.New(borrow.Borrow, key) + hasher.Write([]byte(tokenStr[:i])) + if !hmac.Equal((*bufp)[:n], hasher.Sum(nil)) { + borrow.ReturnAll() + return jwtgo.ErrSignatureInvalid + } + borrow.ReturnAll() + + // Signature is valid, lets validate the claims for + // other fields such as expiry etc. + return claims.Valid() +} + +// base64DecodeBytes returns the bytes represented by the base64 string s. +func base64DecodeBytes(b []byte, buf []byte) (int, error) { + return base64.RawURLEncoding.Decode(buf, b) +} + +// ParseUnverifiedMapClaims - WARNING: Don't use this method unless you know what you're doing +// +// This method parses the token but doesn't validate the signature. It's only +// ever useful in cases where you know the signature is valid (because it has +// been checked previously in the stack) and you want to extract values from +// it. +func ParseUnverifiedMapClaims(token []byte, claims *MapClaims, buf []byte) (*SigningMethodHMAC, error) { + if bytes.Count(token, []byte(".")) != 2 { + return nil, jwtgo.ErrSignatureInvalid + } + + i := bytes.IndexByte(token, '.') + j := bytes.LastIndexByte(token, '.') + + n, err := base64DecodeBytes(token[:i], buf) + if err != nil { + return nil, &jwtgo.ValidationError{Inner: err, Errors: jwtgo.ValidationErrorMalformed} + } + + headerDec := buf[:n] + buf = buf[n:] + alg, _, _, err := jsonparser.Get(headerDec, "alg") + if err != nil { + return nil, &jwtgo.ValidationError{Inner: err, Errors: jwtgo.ValidationErrorMalformed} + } + + n, err = base64DecodeBytes(token[i+1:j], buf) + if err != nil { + return nil, &jwtgo.ValidationError{Inner: err, Errors: jwtgo.ValidationErrorMalformed} + } + + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err = json.Unmarshal(buf[:n], &claims.MapClaims); err != nil { + return nil, &jwtgo.ValidationError{Inner: err, Errors: jwtgo.ValidationErrorMalformed} + } + + for _, signer := range hmacSigners { + if string(alg) == signer.Name { + return signer, nil + } + } + + return nil, jwtgo.NewValidationError(fmt.Sprintf("signing method (%s) is unavailable.", string(alg)), + jwtgo.ValidationErrorUnverifiable) +} diff --git a/internal/jwt/parser_test.go b/internal/jwt/parser_test.go new file mode 100644 index 00000000..a521bfcf --- /dev/null +++ b/internal/jwt/parser_test.go @@ -0,0 +1,213 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package jwt + +// This file is a re-implementation of the original code here with some +// additional allocation tweaks reproduced using GODEBUG=allocfreetrace=1 +// original file https://github.com/golang-jwt/jwt/blob/main/parser.go +// borrowed under MIT License https://github.com/golang-jwt/jwt/blob/main/LICENSE + +import ( + "fmt" + "testing" + "time" + + "github.com/golang-jwt/jwt/v4" +) + +var ( + defaultKeyFunc = func(claim *MapClaims) ([]byte, error) { return []byte("HelloSecret"), nil } + emptyKeyFunc = func(claim *MapClaims) ([]byte, error) { return nil, nil } + errorKeyFunc = func(claim *MapClaims) ([]byte, error) { return nil, fmt.Errorf("error loading key") } +) + +var jwtTestData = []struct { + name string + tokenString string + keyfunc func(*MapClaims) ([]byte, error) + claims jwt.Claims + valid bool + errors int32 +}{ + { + "basic", + "", + defaultKeyFunc, + &MapClaims{ + MapClaims: jwt.MapClaims{ + "foo": "bar", + }, + }, + true, + 0, + }, + { + "basic expired", + "", // autogen + defaultKeyFunc, + &MapClaims{ + MapClaims: jwt.MapClaims{ + "foo": "bar", + "exp": float64(time.Now().Unix() - 100), + }, + }, + false, + -1, + }, + { + "basic nbf", + "", // autogen + defaultKeyFunc, + &MapClaims{ + MapClaims: jwt.MapClaims{ + "foo": "bar", + "nbf": float64(time.Now().Unix() + 100), + }, + }, + false, + -1, + }, + { + "expired and nbf", + "", // autogen + defaultKeyFunc, + &MapClaims{ + MapClaims: jwt.MapClaims{ + "foo": "bar", + "nbf": float64(time.Now().Unix() + 100), + "exp": float64(time.Now().Unix() - 100), + }, + }, + false, + -1, + }, + { + "basic invalid", + "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJmb28iOiJiYXIifQ.EhkiHkoESI_cG3NPigFrxEk9Z60_oXrOT2vGm9Pn6RDgYNovYORQmmA0zs1AoAOf09ly2Nx2YAg6ABqAYga1AcMFkJljwxTT5fYphTuqpWdy4BELeSYJx5Ty2gmr8e7RonuUztrdD5WfPqLKMm1Ozp_T6zALpRmwTIW0QPnaBXaQD90FplAg46Iy1UlDKr-Eupy0i5SLch5Q-p2ZpaL_5fnTIUDlxC3pWhJTyx_71qDI-mAA_5lE_VdroOeflG56sSmDxopPEG3bFlSu1eowyBfxtu0_CuVd-M42RU75Zc4Gsj6uV77MBtbMrf4_7M_NUTSgoIF3fRqxrj0NzihIBg", + defaultKeyFunc, + &MapClaims{ + MapClaims: jwt.MapClaims{ + "foo": "bar", + }, + }, + false, + -1, + }, + { + "basic nokeyfunc", + "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJmb28iOiJiYXIifQ.FhkiHkoESI_cG3NPigFrxEk9Z60_oXrOT2vGm9Pn6RDgYNovYORQmmA0zs1AoAOf09ly2Nx2YAg6ABqAYga1AcMFkJljwxTT5fYphTuqpWdy4BELeSYJx5Ty2gmr8e7RonuUztrdD5WfPqLKMm1Ozp_T6zALpRmwTIW0QPnaBXaQD90FplAg46Iy1UlDKr-Eupy0i5SLch5Q-p2ZpaL_5fnTIUDlxC3pWhJTyx_71qDI-mAA_5lE_VdroOeflG56sSmDxopPEG3bFlSu1eowyBfxtu0_CuVd-M42RU75Zc4Gsj6uV77MBtbMrf4_7M_NUTSgoIF3fRqxrj0NzihIBg", + nil, + &MapClaims{ + MapClaims: jwt.MapClaims{ + "foo": "bar", + }, + }, + false, + -1, + }, + { + "basic nokey", + "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJmb28iOiJiYXIifQ.FhkiHkoESI_cG3NPigFrxEk9Z60_oXrOT2vGm9Pn6RDgYNovYORQmmA0zs1AoAOf09ly2Nx2YAg6ABqAYga1AcMFkJljwxTT5fYphTuqpWdy4BELeSYJx5Ty2gmr8e7RonuUztrdD5WfPqLKMm1Ozp_T6zALpRmwTIW0QPnaBXaQD90FplAg46Iy1UlDKr-Eupy0i5SLch5Q-p2ZpaL_5fnTIUDlxC3pWhJTyx_71qDI-mAA_5lE_VdroOeflG56sSmDxopPEG3bFlSu1eowyBfxtu0_CuVd-M42RU75Zc4Gsj6uV77MBtbMrf4_7M_NUTSgoIF3fRqxrj0NzihIBg", + emptyKeyFunc, + &MapClaims{ + MapClaims: jwt.MapClaims{ + "foo": "bar", + }, + }, + false, + -1, + }, + { + "basic errorkey", + "eyJ0eXAiOiJKV1QiLCJhbGciOiJSUzI1NiJ9.eyJmb28iOiJiYXIifQ.FhkiHkoESI_cG3NPigFrxEk9Z60_oXrOT2vGm9Pn6RDgYNovYORQmmA0zs1AoAOf09ly2Nx2YAg6ABqAYga1AcMFkJljwxTT5fYphTuqpWdy4BELeSYJx5Ty2gmr8e7RonuUztrdD5WfPqLKMm1Ozp_T6zALpRmwTIW0QPnaBXaQD90FplAg46Iy1UlDKr-Eupy0i5SLch5Q-p2ZpaL_5fnTIUDlxC3pWhJTyx_71qDI-mAA_5lE_VdroOeflG56sSmDxopPEG3bFlSu1eowyBfxtu0_CuVd-M42RU75Zc4Gsj6uV77MBtbMrf4_7M_NUTSgoIF3fRqxrj0NzihIBg", + errorKeyFunc, + &MapClaims{ + MapClaims: jwt.MapClaims{ + "foo": "bar", + }, + }, + false, + -1, + }, + { + "Standard Claims", + "", + defaultKeyFunc, + &StandardClaims{ + StandardClaims: jwt.StandardClaims{ + ExpiresAt: time.Now().Add(time.Second * 10).Unix(), + }, + }, + true, + 0, + }, +} + +func mapClaimsToken(claims *MapClaims) string { + claims.SetAccessKey("test") + j := jwt.NewWithClaims(jwt.SigningMethodHS512, claims) + tk, _ := j.SignedString([]byte("HelloSecret")) + return tk +} + +func standardClaimsToken(claims *StandardClaims) string { + claims.AccessKey = "test" + claims.Subject = "test" + j := jwt.NewWithClaims(jwt.SigningMethodHS512, claims) + tk, _ := j.SignedString([]byte("HelloSecret")) + return tk +} + +func TestParserParse(t *testing.T) { + // Iterate over test data set and run tests + for _, data := range jwtTestData { + t.Run(data.name, func(t *testing.T) { + // Parse the token + var err error + + // Figure out correct claims type + switch claims := data.claims.(type) { + case *MapClaims: + if data.tokenString == "" { + data.tokenString = mapClaimsToken(claims) + } + err = ParseWithClaims(data.tokenString, &MapClaims{}, data.keyfunc) + case *StandardClaims: + if data.tokenString == "" { + data.tokenString = standardClaimsToken(claims) + } + err = ParseWithStandardClaims(data.tokenString, &StandardClaims{}, []byte("HelloSecret")) + } + + if data.valid && err != nil { + t.Errorf("Error while verifying token: %T:%v", err, err) + } + + if !data.valid && err == nil { + t.Errorf("Invalid token passed validation") + } + + if data.errors != 0 { + _, ok := err.(*jwt.ValidationError) + if !ok { + t.Errorf("Expected *jwt.ValidationError, but got %#v instead", err) + } + } + }) + } +} diff --git a/internal/kms/config.go b/internal/kms/config.go new file mode 100644 index 00000000..158bd0d6 --- /dev/null +++ b/internal/kms/config.go @@ -0,0 +1,441 @@ +// Copyright (c) 2015-2023 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package kms + +import ( + "bytes" + "context" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "sync/atomic" + "syscall" + "time" + + "aead.dev/mtls" + "github.com/minio/kms-go/kes" + "github.com/minio/kms-go/kms" + "github.com/minio/pkg/v3/certs" + "github.com/minio/pkg/v3/ellipses" + "github.com/minio/pkg/v3/env" +) + +// Environment variables for MinIO KMS. +const ( + EnvKMSEndpoint = "MINIO_KMS_SERVER" // List of MinIO KMS endpoints, separated by ',' + EnvKMSEnclave = "MINIO_KMS_ENCLAVE" // MinIO KMS enclave in which the key and identity exists + EnvKMSDefaultKey = "MINIO_KMS_SSE_KEY" // Default key used for SSE-S3 or when no SSE-KMS key ID is specified + EnvKMSAPIKey = "MINIO_KMS_API_KEY" // Credential to access the MinIO KMS. +) + +// Environment variables for MinIO KES. +const ( + EnvKESEndpoint = "MINIO_KMS_KES_ENDPOINT" // One or multiple KES endpoints, separated by ',' + EnvKESDefaultKey = "MINIO_KMS_KES_KEY_NAME" // The default key name used for IAM data and when no key ID is specified on a bucket + EnvKESAPIKey = "MINIO_KMS_KES_API_KEY" // Access credential for KES - API keys and private key / certificate are mutually exclusive + EnvKESClientKey = "MINIO_KMS_KES_KEY_FILE" // Path to TLS private key for authenticating to KES with mTLS - usually prefer API keys + EnvKESClientCert = "MINIO_KMS_KES_CERT_FILE" // Path to TLS certificate for authenticating to KES with mTLS - usually prefer API keys + EnvKESServerCA = "MINIO_KMS_KES_CAPATH" // Path to file/directory containing CA certificates to verify the KES server certificate + EnvKESClientPassword = "MINIO_KMS_KES_KEY_PASSWORD" // Optional password to decrypt an encrypt TLS private key +) + +// Environment variables for static KMS key. +const ( + EnvKMSSecretKey = "MINIO_KMS_SECRET_KEY" // Static KMS key in the form ":". Implements a subset of KMS/KES APIs + EnvKMSSecretKeyFile = "MINIO_KMS_SECRET_KEY_FILE" // Path to a file to read the static KMS key from +) + +// EnvKMSReplicateKeyID is an env. variable that controls whether MinIO +// replicates the KMS key ID. By default, KMS key ID replication is enabled +// but can be turned off. +const EnvKMSReplicateKeyID = "MINIO_KMS_REPLICATE_KEYID" + +const ( + tlsClientSessionCacheSize = 100 +) + +var replicateKeyID = sync.OnceValue(func() bool { + if v, ok := os.LookupEnv(EnvKMSReplicateKeyID); ok && strings.ToLower(v) == "off" { + return false + } + return true // by default, replicating KMS key IDs is enabled +}) + +// ReplicateKeyID reports whether KMS key IDs should be included when +// replicating objects. It's enabled by default. To disable it, set: +// +// MINIO_KMS_REPLICATE_KEYID=off +// +// Some deployments use different KMS clusters with destinct keys on +// each site. Trying to replicate the KMS key ID can cause requests +// to fail in such setups. +func ReplicateKeyID() bool { return replicateKeyID() } + +// ConnectionOptions is a structure containing options for connecting +// to a KMS. +type ConnectionOptions struct { + CADir string // Path to directory (or file) containing CA certificates +} + +// Connect returns a new Conn to a KMS. It uses configuration from the +// environment and returns a: +// +// - connection to MinIO KMS if the "MINIO_KMS_SERVER" variable is present. +// - connection to MinIO KES if the "MINIO_KMS_KES_ENDPOINT" is present. +// - connection to a "local" KMS implementation using a static key if the +// "MINIO_KMS_SECRET_KEY" or "MINIO_KMS_SECRET_KEY_FILE" is present. +// +// It returns an error if connecting to the KMS implementation fails, +// e.g. due to incomplete config, or when configurations for multiple +// KMS implementations are present. +func Connect(ctx context.Context, opts *ConnectionOptions) (*KMS, error) { + if present, err := IsPresent(); !present || err != nil { + if err != nil { + return nil, err + } + return nil, errors.New("kms: no KMS configuration specified") + } + + lookup := func(key string) bool { + _, ok := os.LookupEnv(key) + return ok + } + switch { + case lookup(EnvKMSEndpoint): + rawEndpoint := env.Get(EnvKMSEndpoint, "") + if rawEndpoint == "" { + return nil, errors.New("kms: no KMS server endpoint provided") + } + endpoints, err := expandEndpoints(rawEndpoint) + if err != nil { + return nil, err + } + + key, err := mtls.ParsePrivateKey(env.Get(EnvKMSAPIKey, "")) + if err != nil { + return nil, err + } + + var rootCAs *x509.CertPool + if opts != nil && opts.CADir != "" { + rootCAs, err = certs.GetRootCAs(opts.CADir) + if err != nil { + return nil, err + } + } + + client, err := kms.NewClient(&kms.Config{ + Endpoints: endpoints, + APIKey: key, + TLS: &tls.Config{ + MinVersion: tls.VersionTLS12, + ClientSessionCache: tls.NewLRUClientSessionCache(tlsClientSessionCacheSize), + RootCAs: rootCAs, + }, + }) + if err != nil { + return nil, err + } + + return &KMS{ + Type: MinKMS, + DefaultKey: env.Get(EnvKMSDefaultKey, ""), + conn: &kmsConn{ + enclave: env.Get(EnvKMSEnclave, ""), + defaultKey: env.Get(EnvKMSDefaultKey, ""), + client: client, + }, + latencyBuckets: defaultLatencyBuckets, + latency: make([]atomic.Uint64, len(defaultLatencyBuckets)), + }, nil + case lookup(EnvKESEndpoint): + rawEndpoint := env.Get(EnvKESEndpoint, "") + if rawEndpoint == "" { + return nil, errors.New("kms: no KES server endpoint provided") + } + endpoints, err := expandEndpoints(rawEndpoint) + if err != nil { + return nil, err + } + + conf := &tls.Config{ + MinVersion: tls.VersionTLS12, + ClientSessionCache: tls.NewLRUClientSessionCache(tlsClientSessionCacheSize), + } + if s := env.Get(EnvKESAPIKey, ""); s != "" { + key, err := kes.ParseAPIKey(s) + if err != nil { + return nil, err + } + + cert, err := kes.GenerateCertificate(key) + if err != nil { + return nil, err + } + conf.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { return &cert, nil } + } else { + loadX509KeyPair := func(certFile, keyFile string) (tls.Certificate, error) { + // Manually load the certificate and private key into memory. + // We need to check whether the private key is encrypted, and + // if so, decrypt it using the user-provided password. + certBytes, err := os.ReadFile(certFile) + if err != nil { + return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err) + } + keyBytes, err := os.ReadFile(keyFile) + if err != nil { + return tls.Certificate{}, fmt.Errorf("Unable to load KES client private key as specified by the shell environment: %v", err) + } + privateKeyPEM, rest := pem.Decode(bytes.TrimSpace(keyBytes)) + if len(rest) != 0 { + return tls.Certificate{}, errors.New("Unable to load KES client private key as specified by the shell environment: private key contains additional data") + } + if x509.IsEncryptedPEMBlock(privateKeyPEM) { + keyBytes, err = x509.DecryptPEMBlock(privateKeyPEM, []byte(env.Get(EnvKESClientPassword, ""))) + if err != nil { + return tls.Certificate{}, fmt.Errorf("Unable to decrypt KES client private key as specified by the shell environment: %v", err) + } + keyBytes = pem.EncodeToMemory(&pem.Block{Type: privateKeyPEM.Type, Bytes: keyBytes}) + } + certificate, err := tls.X509KeyPair(certBytes, keyBytes) + if err != nil { + return tls.Certificate{}, fmt.Errorf("Unable to load KES client certificate as specified by the shell environment: %v", err) + } + return certificate, nil + } + + certificate, err := certs.NewCertificate(env.Get(EnvKESClientCert, ""), env.Get(EnvKESClientKey, ""), loadX509KeyPair) + if err != nil { + return nil, err + } + certificate.Watch(ctx, 15*time.Minute, syscall.SIGHUP) + + conf.GetClientCertificate = func(*tls.CertificateRequestInfo) (*tls.Certificate, error) { + cert := certificate.Get() + return &cert, nil + } + } + + var caDir string + if opts != nil { + caDir = opts.CADir + } + conf.RootCAs, err = certs.GetRootCAs(env.Get(EnvKESServerCA, caDir)) + if err != nil { + return nil, err + } + + client := kes.NewClientWithConfig("", conf) + client.Endpoints = endpoints + + // Keep the default key in the KES cache to prevent availability issues + // when MinIO restarts + go func() { + timer := time.NewTicker(10 * time.Second) + defer timer.Stop() + defaultKey := env.Get(EnvKESDefaultKey, "") + for { + select { + case <-ctx.Done(): + return + case <-timer.C: + client.DescribeKey(ctx, defaultKey) + } + } + }() + + return &KMS{ + Type: MinKES, + DefaultKey: env.Get(EnvKESDefaultKey, ""), + conn: &kesConn{ + defaultKeyID: env.Get(EnvKESDefaultKey, ""), + client: client, + }, + latencyBuckets: defaultLatencyBuckets, + latency: make([]atomic.Uint64, len(defaultLatencyBuckets)), + }, nil + default: + var s string + if lookup(EnvKMSSecretKeyFile) { + b, err := os.ReadFile(env.Get(EnvKMSSecretKeyFile, "")) + if err != nil && !os.IsNotExist(err) { + return nil, err + } + if os.IsNotExist(err) { + // Relative path where "/run/secrets" is the default docker path for secrets + b, err = os.ReadFile(filepath.Join("/run/secrets", env.Get(EnvKMSSecretKeyFile, ""))) + } + if err != nil { + return nil, err + } + s = string(b) + } else { + s = env.Get(EnvKMSSecretKey, "") + } + return ParseSecretKey(s) + } +} + +// IsPresent reports whether a KMS configuration is present. +// It returns an error if multiple KMS configurations are +// present or if one configuration is incomplete. +func IsPresent() (bool, error) { + // isPresent reports whether at least one of the + // given env. variables is present. + isPresent := func(vars ...string) bool { + for _, v := range vars { + if _, ok := os.LookupEnv(v); ok { + return ok + } + } + return false + } + + // First, check which KMS/KES env. variables are present. + // Only one set, either KMS, KES or static key must be + // present. + kmsPresent := isPresent( + EnvKMSEndpoint, + EnvKMSEnclave, + EnvKMSAPIKey, + EnvKMSDefaultKey, + ) + kesPresent := isPresent( + EnvKESEndpoint, + EnvKESDefaultKey, + EnvKESAPIKey, + EnvKESClientKey, + EnvKESClientCert, + EnvKESClientPassword, + EnvKESServerCA, + ) + // We have to handle a special case for MINIO_KMS_SECRET_KEY and + // MINIO_KMS_SECRET_KEY_FILE. The docker image always sets the + // MINIO_KMS_SECRET_KEY_FILE - either to the argument passed to + // the container or to a default string (e.g. "minio_master_key"). + // + // We have to distinguish a explicit config from an implicit. Hence, + // we unset the env. vars if they are set but empty or contain a path + // which does not exist. The downside of this check is that if + // MINIO_KMS_SECRET_KEY_FILE is set to a path that does not exist, + // the server does not complain and start without a KMS config. + // + // Until the container image changes, this behavior has to be preserved. + if isPresent(EnvKMSSecretKey) && os.Getenv(EnvKMSSecretKey) == "" { + os.Unsetenv(EnvKMSSecretKey) + } + if isPresent(EnvKMSSecretKeyFile) { + if filename := os.Getenv(EnvKMSSecretKeyFile); filename == "" { + os.Unsetenv(EnvKMSSecretKeyFile) + } else if _, err := os.Stat(filename); errors.Is(err, os.ErrNotExist) { + os.Unsetenv(EnvKMSSecretKeyFile) + } + } + // Now, the static key env. vars are only present if they contain explicit + // values. + staticKeyPresent := isPresent(EnvKMSSecretKey, EnvKMSSecretKeyFile) + + switch { + case kmsPresent && kesPresent: + return false, errors.New("kms: configuration for MinIO KMS and MinIO KES is present") + case kmsPresent && staticKeyPresent: + return false, errors.New("kms: configuration for MinIO KMS and static KMS key is present") + case kesPresent && staticKeyPresent: + return false, errors.New("kms: configuration for MinIO KES and static KMS key is present") + } + + // Next, we check that all required configuration for the concrete + // KMS is present. + // For example, the MinIO KMS requires an endpoint or a list of + // endpoints and authentication credentials. However, a path to + // CA certificates is optional. + switch { + default: + return false, nil // No KMS config present + case kmsPresent: + if !isPresent(EnvKMSEndpoint) { + return false, fmt.Errorf("kms: incomplete configuration for MinIO KMS: missing '%s'", EnvKMSEndpoint) + } + if !isPresent(EnvKMSEnclave) { + return false, fmt.Errorf("kms: incomplete configuration for MinIO KMS: missing '%s'", EnvKMSEnclave) + } + if !isPresent(EnvKMSDefaultKey) { + return false, fmt.Errorf("kms: incomplete configuration for MinIO KMS: missing '%s'", EnvKMSDefaultKey) + } + if !isPresent(EnvKMSAPIKey) { + return false, fmt.Errorf("kms: incomplete configuration for MinIO KMS: missing '%s'", EnvKMSAPIKey) + } + return true, nil + case staticKeyPresent: + if isPresent(EnvKMSSecretKey) && isPresent(EnvKMSSecretKeyFile) { + return false, fmt.Errorf("kms: invalid configuration for static KMS key: '%s' and '%s' are present", EnvKMSSecretKey, EnvKMSSecretKeyFile) + } + return true, nil + case kesPresent: + if !isPresent(EnvKESEndpoint) { + return false, fmt.Errorf("kms: incomplete configuration for MinIO KES: missing '%s'", EnvKESEndpoint) + } + if !isPresent(EnvKESDefaultKey) { + return false, fmt.Errorf("kms: incomplete configuration for MinIO KES: missing '%s'", EnvKESDefaultKey) + } + + if isPresent(EnvKESClientKey, EnvKESClientCert, EnvKESClientPassword) { + if isPresent(EnvKESAPIKey) { + return false, fmt.Errorf("kms: invalid configuration for MinIO KES: '%s' and client certificate is present", EnvKESAPIKey) + } + if !isPresent(EnvKESClientCert) { + return false, fmt.Errorf("kms: incomplete configuration for MinIO KES: missing '%s'", EnvKESClientCert) + } + if !isPresent(EnvKESClientKey) { + return false, fmt.Errorf("kms: incomplete configuration for MinIO KES: missing '%s'", EnvKESClientKey) + } + } else if !isPresent(EnvKESAPIKey) { + return false, errors.New("kms: incomplete configuration for MinIO KES: missing authentication method") + } + return true, nil + } +} + +func expandEndpoints(s string) ([]string, error) { + var endpoints []string + for endpoint := range strings.SplitSeq(s, ",") { + endpoint = strings.TrimSpace(endpoint) + if endpoint == "" { + continue + } + if !ellipses.HasEllipses(endpoint) { + endpoints = append(endpoints, endpoint) + continue + } + + pattern, err := ellipses.FindEllipsesPatterns(endpoint) + if err != nil { + return nil, fmt.Errorf("kms: invalid endpoint '%s': %v", endpoint, err) + } + for _, p := range pattern.Expand() { + endpoints = append(endpoints, strings.Join(p, "")) + } + } + return endpoints, nil +} diff --git a/internal/kms/config_test.go b/internal/kms/config_test.go new file mode 100644 index 00000000..d63d0652 --- /dev/null +++ b/internal/kms/config_test.go @@ -0,0 +1,105 @@ +// Copyright (c) 2015-2024 MinIO, Inc. +// +// # This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package kms + +import ( + "os" + "testing" +) + +func TestIsPresent(t *testing.T) { + for i, test := range isPresentTests { + os.Clearenv() + for k, v := range test.Env { + t.Setenv(k, v) + } + + ok, err := IsPresent() + if err != nil && !test.ShouldFail { + t.Fatalf("Test %d: %v", i, err) + } + if err == nil && test.ShouldFail { + t.Fatalf("Test %d: should have failed but succeeded", i) + } + + if !test.ShouldFail && ok != test.IsPresent { + t.Fatalf("Test %d: reported that KMS present=%v - want present=%v", i, ok, test.IsPresent) + } + } +} + +var isPresentTests = []struct { + Env map[string]string + IsPresent bool + ShouldFail bool +}{ + {Env: map[string]string{}}, // 0 + { // 1 + Env: map[string]string{ + EnvKMSSecretKey: "minioy-default-key:6jEQjjMh8iPq8/gqgb4eMDIZFOtPACIsr9kO+vx8JFs=", + }, + IsPresent: true, + }, + { // 2 + Env: map[string]string{ + EnvKMSEndpoint: "https://127.0.0.1:7373", + EnvKMSDefaultKey: "minio-key", + EnvKMSEnclave: "demo", + EnvKMSAPIKey: "k1:MBDtmC9ZAf3Wi4-oGglgKx_6T1jwJfct1IC15HOxetg", + }, + IsPresent: true, + }, + { // 3 + Env: map[string]string{ + EnvKESEndpoint: "https://127.0.0.1:7373", + EnvKESDefaultKey: "minio-key", + EnvKESAPIKey: "kes:v1:AGtR4PvKXNjz+/MlBX2Djg0qxwS3C4OjoDzsuFSQr82e", + }, + IsPresent: true, + }, + { // 4 + Env: map[string]string{ + EnvKESEndpoint: "https://127.0.0.1:7373", + EnvKESDefaultKey: "minio-key", + EnvKESClientKey: "/tmp/client.key", + EnvKESClientCert: "/tmp/client.crt", + }, + IsPresent: true, + }, + { // 5 + Env: map[string]string{ + EnvKMSEndpoint: "https://127.0.0.1:7373", + EnvKESEndpoint: "https://127.0.0.1:7373", + }, + ShouldFail: true, + }, + { // 6 + Env: map[string]string{ + EnvKMSEndpoint: "https://127.0.0.1:7373", + EnvKMSSecretKey: "minioy-default-key:6jEQjjMh8iPq8/gqgb4eMDIZFOtPACIsr9kO+vx8JFs=", + }, + ShouldFail: true, + }, + { // 7 + Env: map[string]string{ + EnvKMSEnclave: "foo", + EnvKESServerCA: "/etc/minio/certs", + }, + ShouldFail: true, + }, +} diff --git a/internal/kms/conn.go b/internal/kms/conn.go new file mode 100644 index 00000000..c63fcb6c --- /dev/null +++ b/internal/kms/conn.go @@ -0,0 +1,167 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package kms + +import ( + "context" + "encoding" + "encoding/json" + "strconv" + + jsoniter "github.com/json-iterator/go" + "github.com/minio/madmin-go/v3" +) + +// conn represents a connection to a KMS implementation. +// It's implemented by the MinKMS and KES client wrappers +// and the static / single key KMS. +type conn interface { + // Version returns version information about the KMS. + // + // TODO(aead): refactor this API call. It does not account + // for multiple endpoints. + Version(context.Context) (string, error) + + // APIs returns a list of APIs supported by the KMS server. + // + // TODO(aead): remove this API call. It's hardly useful. + APIs(context.Context) ([]madmin.KMSAPI, error) + + // Stat returns the current KMS status. + Status(context.Context) (map[string]madmin.ItemState, error) + + // CreateKey creates a new key at the KMS with the given key ID. + CreateKey(context.Context, *CreateKeyRequest) error + + ListKeys(context.Context, *ListRequest) ([]madmin.KMSKeyInfo, string, error) + + // GenerateKey generates a new data encryption key using the + // key referenced by the key ID. + // + // The KMS may use a default key if the key ID is empty. + // GenerateKey returns an error if the referenced key does + // not exist. + // + // The context is associated and tied to the generated DEK. + // The same context must be provided when the generated key + // should be decrypted. Therefore, it is the callers + // responsibility to remember the corresponding context for + // a particular DEK. The context may be nil. + GenerateKey(context.Context, *GenerateKeyRequest) (DEK, error) + + // DecryptKey decrypts the ciphertext with the key referenced + // by the key ID. The context must match the context value + // used to generate the ciphertext. + Decrypt(context.Context, *DecryptRequest) ([]byte, error) + + // MAC generates the checksum of the given req.Message using the key + // with the req.Name at the KMS. + MAC(context.Context, *MACRequest) ([]byte, error) +} + +var ( // compiler checks + _ conn = (*kmsConn)(nil) + _ conn = (*kesConn)(nil) + _ conn = secretKey{} +) + +// Supported KMS types +const ( + MinKMS Type = iota + 1 // MinIO KMS + MinKES // MinIO MinKES + Builtin // Builtin single key KMS implementation +) + +// Type identifies the KMS type. +type Type uint + +// String returns the Type's string representation +func (t Type) String() string { + switch t { + case MinKMS: + return "MinIO KMS" + case MinKES: + return "MinIO KES" + case Builtin: + return "MinIO builtin" + default: + return "!INVALID:" + strconv.Itoa(int(t)) + } +} + +// Status describes the current state of a KMS. +type Status struct { + Online map[string]struct{} + Offline map[string]Error +} + +// DEK is a data encryption key. It consists of a +// plaintext-ciphertext pair and the ID of the key +// used to generate the ciphertext. +// +// The plaintext can be used for cryptographic +// operations - like encrypting some data. The +// ciphertext is the encrypted version of the +// plaintext data and can be stored on untrusted +// storage. +type DEK struct { + KeyID string // Name of the master key + Version int // Version of the master key (MinKMS only) + Plaintext []byte // Paintext of the data encryption key + Ciphertext []byte // Ciphertext of the data encryption key +} + +var ( + _ encoding.TextMarshaler = (*DEK)(nil) + _ encoding.TextUnmarshaler = (*DEK)(nil) +) + +// MarshalText encodes the DEK's key ID and ciphertext +// as JSON. +func (d DEK) MarshalText() ([]byte, error) { + type JSON struct { + KeyID string `json:"keyid"` + Version uint32 `json:"version,omitempty"` + Ciphertext []byte `json:"ciphertext"` + } + return json.Marshal(JSON{ + KeyID: d.KeyID, + Version: uint32(d.Version), + Ciphertext: d.Ciphertext, + }) +} + +// UnmarshalText tries to decode text as JSON representation +// of a DEK and sets DEK's key ID and ciphertext to the +// decoded values. +// +// It sets DEK's plaintext to nil. +func (d *DEK) UnmarshalText(text []byte) error { + type JSON struct { + KeyID string `json:"keyid"` + Version uint32 `json:"version"` + Ciphertext []byte `json:"ciphertext"` + } + var v JSON + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err := json.Unmarshal(text, &v); err != nil { + return err + } + d.KeyID, d.Version, d.Plaintext, d.Ciphertext = v.KeyID, int(v.Version), nil, v.Ciphertext + return nil +} diff --git a/internal/kms/context.go b/internal/kms/context.go new file mode 100644 index 00000000..fd9c87e7 --- /dev/null +++ b/internal/kms/context.go @@ -0,0 +1,259 @@ +// Copyright (c) 2015-2021 MinIO, Inc. +// +// This file is part of MinIO Object Storage stack +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package kms + +import ( + "bytes" + "sort" + "unicode/utf8" +) + +// Context is a set of key-value pairs that +// are associated with a generate data encryption +// key (DEK). +// +// A KMS implementation may bind the context to the +// generated DEK such that the same context must be +// provided when decrypting an encrypted DEK. +type Context map[string]string + +// MarshalText returns a canonical text representation of +// the Context. + +// MarshalText sorts the context keys and writes the sorted +// key-value pairs as canonical JSON object. The sort order +// is based on the un-escaped keys. It never returns an error. +func (c Context) MarshalText() ([]byte, error) { + if len(c) == 0 { + return []byte{'{', '}'}, nil + } + + // Pre-allocate a buffer - 128 bytes is an arbitrary + // heuristic value that seems like a good starting size. + b := bytes.NewBuffer(make([]byte, 0, 128)) + if len(c) == 1 { + for k, v := range c { + b.WriteString(`{"`) + escapeStringJSON(b, k) + b.WriteString(`":"`) + escapeStringJSON(b, v) + b.WriteString(`"}`) + } + return b.Bytes(), nil + } + + sortedKeys := make([]string, 0, len(c)) + for k := range c { + sortedKeys = append(sortedKeys, k) + } + sort.Strings(sortedKeys) + + b.WriteByte('{') + for i, k := range sortedKeys { + b.WriteByte('"') + escapeStringJSON(b, k) + b.WriteString(`":"`) + escapeStringJSON(b, c[k]) + b.WriteByte('"') + if i < len(sortedKeys)-1 { + b.WriteByte(',') + } + } + b.WriteByte('}') + return b.Bytes(), nil +} + +// Adapted from Go stdlib. + +var hexTable = "0123456789abcdef" + +// escapeStringJSON will escape a string for JSON and write it to dst. +func escapeStringJSON(dst *bytes.Buffer, s string) { + start := 0 + for i := 0; i < len(s); { + if b := s[i]; b < utf8.RuneSelf { + if htmlSafeSet[b] { + i++ + continue + } + if start < i { + dst.WriteString(s[start:i]) + } + dst.WriteByte('\\') + switch b { + case '\\', '"': + dst.WriteByte(b) + case '\n': + dst.WriteByte('n') + case '\r': + dst.WriteByte('r') + case '\t': + dst.WriteByte('t') + default: + // This encodes bytes < 0x20 except for \t, \n and \r. + // If escapeHTML is set, it also escapes <, >, and & + // because they can lead to security holes when + // user-controlled strings are rendered into JSON + // and served to some browsers. + dst.WriteString(`u00`) + dst.WriteByte(hexTable[b>>4]) + dst.WriteByte(hexTable[b&0xF]) + } + i++ + start = i + continue + } + c, size := utf8.DecodeRuneInString(s[i:]) + if c == utf8.RuneError && size == 1 { + if start < i { + dst.WriteString(s[start:i]) + } + dst.WriteString(`\ufffd`) + i += size + start = i + continue + } + // U+2028 is LINE SEPARATOR. + // U+2029 is PARAGRAPH SEPARATOR. + // They are both technically valid characters in JSON strings, + // but don't work in JSONP, which has to be evaluated as JavaScript, + // and can lead to security holes there. It is valid JSON to + // escape them, so we do so unconditionally. + // See http://timelessrepo.com/json-isnt-a-javascript-subset for discussion. + if c == '\u2028' || c == '\u2029' { + if start < i { + dst.WriteString(s[start:i]) + } + dst.WriteString(`\u202`) + dst.WriteByte(hexTable[c&0xF]) + i += size + start = i + continue + } + i += size + } + if start < len(s) { + dst.WriteString(s[start:]) + } +} + +// htmlSafeSet holds the value true if the ASCII character with the given +// array position can be safely represented inside a JSON string, embedded +// inside of HTML